\documentclass[11pt,english]{article}
\usepackage{mathpazo}
\usepackage[a4paper]{geometry}
\geometry{verbose,tmargin=3cm,bmargin=3cm,lmargin=3cm,rmargin=3cm}
\usepackage{array}
\usepackage{booktabs}
\usepackage{multirow}
\usepackage{fancyvrb}

\DefineVerbatimEnvironment%
{code}{Verbatim}
{fontsize=\small, xleftmargin=1em}

\newcommand{\noun}[1]{\textsc{#1}}
%% Because html converters don't know tabularnewline
\providecommand{\tabularnewline}{\\}

\usepackage{babel}

\include{pkgdoc}

\newcommand{\ArgRet}[2]{%
  {\it Arguments}: {#1}%
  \ifx&#2&%
  \else
  \par\smallskip\noindent {\it Return type}: \texttt{#2}
  \fi%
  \par\medskip\par%
  }

\begin{document}

\title{The extra package\\
(a collection of various convenience functions for hansl programming) }

\date{January 2020, release 0.6, still growing\ldots }

\author{The \noun{gretl} team\thanks{Currently co-ordinated by Sven Schreiber.}}

\maketitle
\tableofcontents{}

\section{Usage}

This package is intended for hansl scripting, not for gretl's GUI.
(But of course other contributed function packages that make use of
functions in extra.gfn can provide GUI access for themselves.)

The usual one-time requirement is to do \cmd{pkg install extra.zip}
to get a copy on the local system (or install it via gretl's graphical
mechanism), and then in the respective hansl script have a line \cmd{include
extra.gfn}.

Note that functions that are exact lookalikes of \app{Matlab/Octave}
functions do not live here, but would go into the
\texttt{matlab\_utilities} package.

%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%

\section{Matrix-related functions}

\subsection{commute}

\ArgRet{\texttt{matrix A, int m, int n} (optional), \texttt{bool
post} (optional)}{matrix}

Returns $A$ premultiplied by $K_{mn}$ (the commutation matrix; more
efficient than explicit multiplication). In particular, \cmd{commute(vec(B),
rows(B), cols(B))} gives $vec(B')$. The optional argument $n$ defaults to
$m$ (giving $K_{mm}=K_{m}$). If the optional arg \texttt{post} is
non-zero, then does post-multiplication ($A\times K_{mn}$). 

\subsection{eliminate}

\ArgRet{\texttt{matrix vecA}}{matrix}

Each column of the input vecA is assumed to come from the operation vec(A) 
on a square matrix, thus rows(vecA) must be a square number.
Returns vech(A), which is the result of pre-multiplying vec(A) with the 
"elimination" matrix $L_m$.
If vecA has several columns, each column is 
treated separately as described above (and the results stacked side-by-side).

\subsection{duplicate}

\ArgRet{\texttt{matrix vechA}}{matrix}

The input is a vector assumed to come from an operation like vech(A).
Returns vec(A), which is the result of pre-multiplying vech(A) with the 
"duplication" matrix $D_m$. If vechA has several columns, each column is 
treated separately as described above (and the results stacked side-by-side).

\subsection{nearPSD}

\ArgRet{\texttt{matrix} pointer \texttt{{*}m}, \texttt{scalar epsilon}
(optional)}{scalar}

Forces the matrix $m$ into the positive semi-definite region. Algorithm
ported from ``DomPazz'' in Stackoverflow,
apparently mimicking the nearPD() function in R. Because of re-scaling
(to correlation matrix), the \texttt{epsilon} criterion value should
implicitly apply to the correlation-based eigenvalues. The return
value 0 or 1 indicates whether \texttt{m} was altered or not. 

\subsection{zeroifclose}

\ArgRet{\texttt{matrix} pointer \texttt{{*}m}, \texttt{scalar thresh}
(optional)}{scalar}

Sets elements of \texttt{m} to zero if they are really close. The
return value 0 or 1 indicates whether \texttt{m} was altered or not.

\subsection{drill}

\ArgRet{\texttt{matrices} array, \texttt{matrix rowspec} (optional), 
  \texttt{matrix colspec} (optional)}{matrix}

This function "drills through" a matrix array and returns a matrix;
for example, \cmd{drill(x, 2, 3)} returns a vector with the
\texttt{[2,3]} elements of all matrices in the \texttt{x}
array. Omitting one of rowspec, colspec or entering "0" means to
select all rows or columns respectively.  (Of course at least one of
rowspec, colspec must be specified.)

\emph{Nota bene}: all matrices in the array must be of the same dimensions.

%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%

\section{Other functions working without a dataset in place}

\subsection{scores2x2}

\ArgRet{\texttt{matrix in, bool verbose} (optional)}{matrix}

Computes some standard score measures for a $2\times 2$ contingency
table of the form:

\begin{center}
\begin{tabular}{cccc}
\toprule 
 &  & \multicolumn{2}{c}{Observed}\tabularnewline
 &  & 1 & 0\tabularnewline
\midrule
\multirow{2}{*}{Predicted} & 1 & h(its) & f(alse)\tabularnewline
 & 0 & m(iss) & z(eros)\tabularnewline
\bottomrule
\end{tabular}
\end{center}

\noindent and $n=h+f+m+z$ (total observations). Returns a column vector with the
elements listed in Table \ref{tab:scores2x2}. The input is always
sanitized by taking the upper 2x2 part, using absolute values, and
integer-ization. Warnings are issued if \texttt{verbose} is 1.


\begin{table}[htbp]
\begin{tabular}{rlp{0.35\textwidth}c}
  \hline
  \textbf{Number} &\textbf{Acronym} &  \textbf{Description} &\textbf{Formula} \\
  \hline
  1 & POD & prob of detection & $\frac{h}{h+m}$ \\
2 & POFD & prob of false detection & $\frac{f}{f+z}$ \\
3 & HR & hit rate & $\frac{h+z}{n}$ \\
4 & FAR & false alarm rate & $\frac{f}{h+f}$ \\
5 & CSI & critical success index & $\frac{h}{h+f+m}$\\
6 & OR & odds ratio & $\frac{h \cdot z}{f \cdot m}$\\
7 & BIAS & bias score & $\frac{h+f}{h+m}$\\ 
8 & TSS & true skill stat ($POD-POFD$); also known as the
          Hanssen-Kuipers score & $\frac{h}{h+m} -\frac{f}{f+z}$. \\
9 & HSS & Heidke skill score & $2 \frac{h \cdot z - f \cdot m}{(h+m) \cdot (m+z)+(h+f) \cdot (f+z)}$ \\
10 & ETS & equitable threat score & $\frac{h \cdot z-f \cdot m}{(f+m) \cdot n+(h \cdot z-f \cdot m)}$ \\
11 & PRC & precision & $\frac{h}{h+f}$ \\
12 & FSC & $F$-Score & $2 \frac{PRC \cdot POD}{PRC+POD} = 2 \frac{h}{1+h+m}$.\\
  \hline
\end{tabular}
\caption{Elements returned by the \cmd{scores2x2} function}
\label{tab:scores2x2}
\end{table}

\subsection{truncnorm}

\ArgRet{\texttt{int n, scalar m, scalar sigma, scalar below, scalar
above}}{matrix}

Generates $n$ truncated normal random values. Specify mean \texttt{m}
and std.\,dev. \texttt{sigma}, and the left/right truncation values
\texttt{below} and \texttt{above}. (Pass NA for any one of them to
skip the respective truncation.) Returns a col vector of values.

\subsection{WSRcritical}

\ArgRet{\texttt{int n, scalar prob }(optional)\texttt{, bool forcenorm}
(optional)}{matrix}


Concerns the distribution of Wilcoxon's signed rank test statistic for
\texttt{n} trials (at least 4). Tries to find the critical values
(low/hi) where the two-sided area to the outside is as close as
possible to the given \texttt{prob} (default: 0.05). (Note that
``outside'' means including the critical values themselves in the
exact/discrete case.) If we end up in the interior region not covered
by the exact table (for \texttt{prob} far away from 0 and also from
1), we fall back to the normal approximation. The function returns a
column vector \verb|{lo; hi; epv}|, where \texttt{epv} is the actual
probability mass (close to \texttt{prob} but not equal in general for
small samples). \texttt{lo} and \texttt{hi} can be non-integers in the normal
approximation case. The normal approximation instead of the exact
table values can be enforced with the \texttt{forcenorm} argument
(default: zero, do not enforce).

See also the sister function \cmd{WSRpvalue}.

\subsection{WSRpvalue}

\ArgRet{\texttt{int n, scalar W, bool forcenorm} (optional)}{scalar}

Concerns the distribution of Wilcoxon's signed rank test statistic for
\texttt{n} trials (at least 4), returns $P(X\geq W)$. In the interior
region not covered by the exact table, the true value is $\geq$ 12.5\%
(and $\leq$87.5\%) according to the table used,\footnote{Source of the
  table: Wilfrid J Dixon and Frank J. Massey, Jr., Introduction to
  Statistical Analysis, 2nd ed. (New York: McGraw-Hill, 1957), pp.
  443-444.} so typically based on such values H0 would not be
rejected. We fall back to the normal approximation in this region. In
the extreme outer regions not explicitly covered by the table, the
deviation from 0 or 1 will be smaller than 0.5\% = 0.005. We return
values 0.001 or 0.999 as an approximation here. The test statistic
\texttt{W} should usually be an integer, but in case of bindings it
could be fractional as well; in this case we also fall back to the
normal approximation.

The normal approximation instead of the exact table values can be
enforced with the \texttt{forcenorm} argument (default: zero, do not
enforce).

See also the sister function \cmd{WSRcritical}.

\subsection{powerset}

\ArgRet{\texttt{strings S}}{}

Computes the powerset of the input S, i.e. all possible combinations 
of the string elements in S. (Including the empty set / 
empty string "".) Each combination yields one string in the output 
array. Being a set, the ordering is not defined and arbitrary. 


\subsection{onemode}

\ArgRet{\texttt{matrix v}}{}

Finds the mode of the empirical distribution of the input data. 
When that is multi-modal the first mode is found (following the native
imaxc function). Returns a 2-element column vector with the modal 
value and its absolute frequency.

\subsection{bwritejson}

\ArgRet{\texttt{bundle b, string fname, bool export} (optional), 
    \texttt{string *jsonresult} (pointer form, optional), 
    \texttt{string misscode} (optional)}{scalar}

Similar to the native function \texttt{bwrite}: Instead of an XML representation 
of the input bundle this function produces a JSON serialized representation.
So far supported bundle member types are string, scalar, series (will be saved as a
vector), matrix, string arrays.

The standard usage is to pass a valid file name in \texttt{fname} and have the result 
written to that file. However, if the pointer string argument \texttt{jsonresult} is used, 
then the file name input is ignored (could be omitted) and the JSON string is just 
copied to the \texttt{jsonresult} string variable. 

Missing values are supported, but the JSON specification does not define them. The 
default is to print "NaN" for any missing value (in a numerical object), but this can 
be overridden with the string in the optional \texttt{misscode} argument.\footnote{Note 
that round-tripping the result with gretl's native \cmd{jsonget} and \cmd{jsongetb} 
functions does not fully work, apparently due to a limitation of the underlying third-party
library. However, this function is not meant for internal gretl usage, but for exporting data.
For just storing gretl bundles use the native XML-based format produced by gretl's
\cmd{bwrite}.}

Returns 0 after successful completion, or the non-zero gretl error code if writing the 
output file failed for some reason (in which case also a warning message is printed).

\subsection{splitfname}

\ArgRet{\texttt{string fn}}{strings (array)}

The idea is to take a file name or full path and extract three components:
\begin{enumerate}
\item The path prefix (may be empty; without the trailing / or \textbackslash)
\item The "base" component of the file name, without the extension and without the path prefix 
\item The file extension (without the dot; may be empty)
\end{enumerate}	
In principle should work with forward as well as backslashes and also with double forward slashes.

Example:

Input string: "/what/on/earth/isthisfile.gdt"

Output equivalent to: 

\cmd{defarray("/what/on/earth", "isthisfile", "gdt")}

%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%

\section{Functions requiring a dataset}

\subsection{gap\_filler}

\ArgRet{\texttt{series x, int method }(optional)}{series}

Simple convenience function to crudely get rid of missing values
interspersed between valid observations. Apart from the first argument
(series) accepts an integer parameter as second argument, whose
meaning is: 0: do nothing, leave the gaps; 1: NAs are replaced with
previous observations; 2: NAs are replaced with a linear
interpolation. Returns the filled series.

The very existence of the "0" method for interpolation may look
bizarre at first sight, but it may make sense to have in the context
of batch processing, like in the following example (hopefully,
self-explanatory):
\begin{code}
k = 1
loop foreach i X
   series z_$i = gap_filler($i, action[k++])
endloop
\end{code}

Note that the function only replaces NAs between valid observations;
therefore, if the origin series has missing values at the beginning or
the end of the sample, they will be in the returned series too.

\subsection{winsor}

\ArgRet{\texttt{series x, scalar p} (optional), \texttt{scalar
phi} (optional)}{series}

Returns a trimmed (``winsorized'') version
of the series, where outliers are replaced with implicit threshold
values. Truncation quantiles are determined according to relative
tail frequencies \texttt{p} and \texttt{phi}. Default lower and upper
frequencies are 0.05, but re-settable with \texttt{p}. Pass \texttt{phi}
in addition to \texttt{p} for an asymmetric trimming, then \texttt{p}
determines only the lower frequency and \texttt{phi} the upper. 

\section{Authors}
\begin{itemize}
\item gap\_filler, commute, eliminate, duplicate, truncnorm, powerset, drill:
 Jack Lucchetti

\item nearPSD, zeroifclose, scores2x2, WSRcritical, WSRpvalue, onemode,
splitfname:
Sven Schreiber 

\item winsor: Sven Schreiber, original code JoshuaHe
\item bwritejson: Artur Tarassow and Sven Schreiber

\end{itemize}

\section{Changelog }
\begin{itemize}
\item January 2020: 0.6, add drill, bwritejson, onemode, splitfname;
  finally remove the retired sepstr2arr (use native strsplit instead);
  slightly revise gap\_filler; rearrange the documentation a little
\item October 2018: 0.5, fix small commute bug; retire sepstr2arr; add powerset,
  eliminate, duplicate
\item February 2018: 0.41, allow non-integer input in WSRpvalue
\item January 2018: 0.4, add WSRcritical, WSRpvalue
\item December 2017: 0.3, add scores2x2; switch to pdf help document
\item September 2017: 0.2, add winsor 
\item July 2017: initial release
\end{itemize}

\end{document}

%%% Local Variables:
%%% mode: latex
%%% TeX-master: t
%%% End:
