%\VignetteDepends{}
%\VignetteIndexEntry{Native Interfaces}
%\VignettePackage{AdvancedR}

%% \documentclass[notesonly]{beamer}
%% \documentclass[handout,xcolor=pdftex,dvipsnames,table]{beamer}
\documentclass{beamer}

\usepackage{Bioconductor}
\SweaveOpts{keep.source=TRUE}

\title{Native Interfaces for \R{}}
\author{Seth Falcon}
\institute[FHCRC]{Fred Hutchinson Cancer Research Center}
\date{20-21 May, 2010}

\begin{document}

<<setup, echo=FALSE>>=
options(width = 40)
@

\frame{\titlepage}

\section{The .Call Native Interface}

\begin{frame}[fragile]{\code{which} as implemented in R prior to R-2.11}


\begin{verbatim}
which <- function(x) {
    seq_along(x)[x & !is.na(x)]
}
\end{verbatim}
\only<2>{

\begin{enumerate}
\item \code{is.na}
\item \code{!}
\item \code{\&}
\item \code{seq\_along}
\item \code{[}
\end{enumerate}
}

\note{
  How many passes are made through the input vector?

  Thanks to Patrick for the example

  Each operation scans through length(x) items and makes a copy

  It is the copies that can hurt performance
}

\end{frame}

\begin{frame}[fragile]{\code{which} in C}
\begin{verbatim}
 1 SEXP nid_which(SEXP v) {
 2     SEXP ans;
 3     int i, j = 0, len = Rf_length(v), *buf, *tf;
 4     buf = (int *) R_alloc(len, sizeof(int));
 5     tf = LOGICAL(v);
 6     for (i = 0; i < len; i++) {
 7         if (tf[i] == TRUE) buf[j] = i + 1; j++;
 8     }
 9     ans = Rf_allocVector(INTSXP, j);
10     memcpy(INTEGER(ans), buf, sizeof(int) * j);
11     return ans;
}
\end{verbatim}
\note{Make a single pass, but allocate twice}
\end{frame}

\begin{frame}{\code{which} is now 10x faster}
  \code{system.time(which(v))}
  \vskip 1cm

  \begin{tabular}{lll}
    R Version & System time & Elapsed time \\ \hline
    R-2.10  & 0.018  & 0.485 \\
    R-2.11  & 0.001  & 0.052
  \end{tabular}

  \vskip 1cm
  v is a logical vector with 10 million elements

  \note{10x is a good goal to bother with C}
\end{frame}


\begin{frame}{Making use of existing code}
  \begin{columns}
    \column{0.5\textwidth}
    Access algorithms
    \begin{itemize}
    \item RBGL
    \item RCurl
    \item Rsamtools
    \end{itemize}

    \column{0.5\textwidth}
    Interface to other systems
      \begin{itemize}
      \item RSQLite
      \item netcdf
      \item SJava
      \end{itemize}
  \end{columns}

  \note{
    LAZY in a good way

    Interfaces use R external pointers and finalizers
  }
\end{frame}

\begin{frame}[fragile]{Review of C}

\begin{semiverbatim}
#include <stdio.h>
double \alert{_nid_avg}(int *data, int len) \{
    int i;
    double ans = 0.0;
    for (i = 0; i < len; i++) \{
      ans += data[i];
    \}
    return ans / len;
\}
main() \{
    int ex_data[5] = \{1, 2, 3, 4, 5\};
    double m = _nid_avg(ex_data, 5);
    printf("%f\\n", m);
\}
\end{semiverbatim}
\note{What does this function do?}
\end{frame}

\begin{frame}{The .Call interface}
  \begin{figure}
    \includegraphics[width=4in]{nativeI/dot-call-summary}
  \end{figure}

  \note{
    Maximum control of conversion and copying

    Requires R headers, dealing with R objects in C

    Seamless intregration into R

    Safer and more powerful than .C
  }
\end{frame}

\begin{frame}{Symbolic Expressions (SEXPs)}
  \begin{itemize}
  \item The SEXP is R's fundamental data type at the C-level
  \item SEXP is short for \textit{symbolic expression} and is borrowed
    from Lisp
  \item History at \url{http://en.wikipedia.org/wiki/S-expression}
  \end{itemize}
  \note{These details help understand how to call R functions from C}
\end{frame}

\begin{frame}{.Call Start to Finish Demo}
  Demo
\end{frame}

\begin{frame}[fragile]{.Call Disection}

  {\LARGE R}

\begin{verbatim}
# R/somefile.R
avg <- function(x) .Call(.nid_avg, x)
# NAMESPACE
useDynLib("yourPackage", .nid_avg = nid_avg)
\end{verbatim}

\medskip

  {\LARGE C}

\begin{semiverbatim}
/* src/some.c */
#include <Rinternals.h>
\alert{SEXP} nid_avg(\alert{SEXP} data) \{ ... INTEGER(data) ... \}
\end{semiverbatim}

\note{The useDynLib call is one form of function registration}
\end{frame}

\begin{frame}[fragile]{.Call C function}
\begin{semiverbatim}
\alert<1>{#include <Rinternals.h>}

\alert<2>{SEXP nid_avg(SEXP data)}
\{
    SEXP ans;
    \alert<3>{double v = _nid_avg(INTEGER(data), Rf_length(data));}
    \alert<4,6>{PROTECT(ans = Rf_allocVector(REALSXP, 1));}
    \alert<5>{REAL(ans)[0] = v;}
    \alert<6>{UNPROTECT(1);}
    return ans;
    \alert<7>{/* shortcut: return Rf_ScalarReal(v); */}
\}
\end{semiverbatim}

\note{
  Call same average function shown before as helper

  Matched PROTECT/UNPROTECT
}
\end{frame}

\begin{frame}{Function Registration}
  \begin{itemize}
  \item In NAMESPACE
  \item In C via \code{R\_CallMethodDef} (See
    \code{ShortRead/src/R\_init\_ShortRead.c} for a nice example.
  \end{itemize}
  \note{
    Benefit of R\_CallMethodDef is arity checking

    NAMESPACE approach a good compromise to get started

    Can avoid reg entirely, but .Call not as nice
  }
\end{frame}

\section{R types at the C-level}

\begin{frame}{Just one thing to learn}

  \begin{itemize}
  \item {\LARGE Everything is a SEXP}
  \item But there are many different types of SEXPs
  \item INTSXP, REALSXP, STRSXP, LGLSXP, VECSXP, and more.
  \end{itemize}
\end{frame}

\begin{frame}{What's a SEXP?}
  \begin{figure}
    \includegraphics{nativeI/sexp-diagram}
  \end{figure}
\note{
  SEXPs know their type and length

  have data pointer, e.g. integer array
}
\end{frame}

\begin{frame}{Common SEXP subtypes}

  \begin{tabular}{lll}
  R function   & SEXP subtype & Data accessor \\ \hline
  integer()    & INTSXP    & int *INTEGER(x)  \\
  numeric()    & REALSXP   & double *REAL(x)  \\
  logical()    & LGLSXP    & int *LOGICAL(x)           \\
  character()  & STRSXP    & CHARSXP STRING\_ELT(x, i) \\
  list()       & VECSXP    & SEXP VECTOR\_ELT(x, i)    \\
  NULL         & NILSXP    & R\_NilValue               \\
  externalptr  & EXTPTRSXP & SEXP (accessor funcs)
\end{tabular}
  \note{
    Will look at STRSXP/CHARSXP in detail next

    Exercise handout has EXTPTRSXP example

    All SEXP types in Rinternals.h
    }
\end{frame}

\begin{frame}[fragile]{STRSXP/CHARSXP Exercise}
  Try this:
  \begin{verbatim}

    .Internal(inspect(c("abc", "abc", "xyz")))

  \end{verbatim}

  \note{
    Observe pointer addresses

    Observe structure (STRSXP containing CHARSXP)
  }
\end{frame}

\begin{frame}[fragile]{STRSXP/CHARSXP Exercise}
\begin{verbatim}
> .Internal(inspect(c("abc", "abc", "xyz")))

 @101d3f280 16 STRSXP g0c3 [] (len=3, tl=1)
   @101cc6278 09 CHARSXP g0c1 [gp=0x20] "abc"
   @101cc6278 09 CHARSXP g0c1 [gp=0x20] "abc"
   @101cc6308 09 CHARSXP g0c1 [gp=0x20] "xyz"
\end{verbatim}
\end{frame}

\begin{frame}{character vectors == STRSXPs + CHARSXPs}

  \note{
    character() in R maps to STRSXPs which contain CHARSXPs

    CHARSXPs contain plain, nul-terminated C-strings
  }

  \begin{figure}[ht]
    \includegraphics[width=2.75in]{nativeI/strsxp1}
  \end{figure}

\end{frame}

\begin{frame}[fragile]{CHARSXPs are special}
  \begin{figure}[ht]
    \includegraphics[width=2.25in]{nativeI/strsxp2}
  \end{figure}

  \note{
    Create CHARSXPs with mkChar, mkCharLen

    No CHARSXPs at R-level

    There is a global cache of CHARSXPs

    never allocate a CHARSXP
  }
\end{frame}

\begin{frame}[fragile]{STRSXP/CHARSXP API Example}
\begin{semiverbatim}
\alert<1>{SEXP s = Rf_allocVector(STRSXP, 5);}
PROTECT(s);
\alert<2>{SET_STRING_ELT(s, 0, mkChar("hello"));}
\alert<3>{SET_STRING_ELT(s, 1, mkChar("goodbye"));}
\alert<4>{SEXP c = STRING_ELT(s, 0);}
\alert<5>{const char *v = CHAR(c);}
UNPROTECT(1);
\end{semiverbatim}
\end{frame}

\section{Memory management in R}

\begin{frame}{R's memory model (dramatization)}
    \begin{figure}[ht]
    \includegraphics[width=4.5in]{nativeI/DramatizationofRsMemoryModel}
  \end{figure}

    \note{
      R allocates, tracks, and garbage collects SEXPs

      gc is triggered by R functions

      SEXPs that are not \textit{in-use} are recycled.

      A SEXP is \textit{in-use} if: PROTECTed, Precious, reachable
    }
\end{frame}

\begin{frame}{R's memory model}
  \begin{itemize}
   \item R allocates, tracks, and garbage collects SEXPs
   \item gc is triggered by R functions
   \item SEXPs that are not \textit{in-use} are recycled.
   \item A SEXP is \textit{in-use} if:
    \begin{itemize}
    \item it is on the protection stack (PROTECT/UNPROTECT)
    \item it is in the precious list (R\_PreserveObject/R\_ReleaseObject)
    \item it is \textit{reachable} from a SEXP in the in-use list.
    \end{itemize}
  \end{itemize}
  \begin{figure}[hb]
    \includegraphics[width=3.5in]{nativeI/DramatizationofRsMemoryModel}
  \end{figure}

  \note{
    There is a variable reference to it at the R level

    The garbage collector traverses all SEXPs reachable from
    an in-use list.  Any SEXPs that were not reached from the
    traversal of the in-use list are eligible for collection.
  }

\end{frame}

\begin{frame}{Preventing gc of SEXPs with the protection stack}
\begin{tabular}{ll}
  \code{PROTECT(s)}   & Push \code{s} onto the protection stack \\
  \code{UNPROTECT(n)} & Pop top \code{n} items off of the protection stack
\end{tabular}

\note{
  If you PROTECT something, you must UNPROTECT it.

  Putting an item inside a protected container protects the item.

  PROTECT/UNPROTECT are cheap; when in doubt, PROTECT

  Arguments passed via .Call are already protected.
}

\end{frame}

\begin{frame}{Generic memory: When everything \textit{isn't} a SEXP}

  \begin{tabular}{|l|l|} \hline
    Allocation Function & Lifecycle \\ \hline
    \code{R\_alloc}           & Memory is freed when returning from .Call \\
    \code{Calloc}/\code{Free} & Memory persists until Free is called \\ \hline
  \end{tabular}

  \note{
    Built-in OOM error checking

    Generally, prefer over malloc, but memory leaks if OOM error
    }
\end{frame}

\begin{frame}[fragile]{Case Study: C implementation of \code{which}}
  Code review of \code{nidemo/src/which.c}
\end{frame}

\section{Self-Study Exercises}
%%
%% library("nidemo")
%% library("lattice")
%% ## English
%% english <- alpha_freq("/usr/share/dict/words")
%% ## Yeast Gene (YDL143W  Chr 4)
%% dna <- alpha_freq(system.file("extdata/Sc.fa", package="Biostrings"))
%% english1 <- english / sum(english)
%% dna1 <- dna / sum(dna)
%% dna2 <- dna1[order(dna1, decreasing=TRUE)][1:5]
%% eng <- english1[order(english1, decreasing = TRUE)][1:5]
%% pdf("nativeI/english.pdf")
%% barchart(eng[order(eng)], xlab="frequency", scales=list(cex=3.0))
%% dev.off()
%% pdf("nativeI/yeastgene.pdf")
%% barchart(dna2[order(dna2)], xlab="frequency", scales=list(cex=3.0))
%% dev.off()

\begin{frame}{Alphabet frequency of a text file}
\begin{columns}
  \column{0.5\textwidth}
  Yeast Gene YDL143W
  \begin{figure}
    \includegraphics[width=2in]{nativeI/yeastgene}
  \end{figure}
  \column{0.5\textwidth}
    English Dictionary
    \begin{figure}
    \includegraphics[width=2in]{nativeI/english}
  \end{figure}

\end{columns}
\end{frame}

\begin{frame}{Self-Study in nidemo package}
  \begin{enumerate}
  \item Alphabet Frequency
    \begin{itemize}
    \item Explore R and C implementations
    \item Make C implementation robust
    \item Attach names attribute in C
    \item multiple file, matrix return enhancement
    \end{itemize}
  \item External pointer example
  \item Calling R from C example and enhancement
  \item Debugging: a video how-to
  \end{enumerate}
\end{frame}


\section{Odds and Ends}

\begin{frame}{Odds and Ends}
  \begin{enumerate}
  \item Debugging
  \item Public API
  \item Calling R from C
  \item Making sense of the remapped function names
  \item Finding C implementations of R functions
  \item Using a TAGS file
  \end{enumerate}
\end{frame}

\begin{frame}[fragile]{Debugging Native Code}
\begin{verbatim}
Rprintf("DEBUG: v => %d, x => %s\n", v, x);
\end{verbatim}

\begin{verbatim}
$ R -d gdb
(gdb) run
> # now R is running
\end{verbatim}

\begin{quote}
  There are two ways to write error-free programs; only the third one works.

  -- Alan Perlis, \textit{Epigrams on Programming}
\end{quote}

\end{frame}

\begin{frame}[fragile]{R's Public API}
\begin{verbatim}
mkdir R-devel-build; cd R-devel-build
~/src/R-devel-src/configure && make
## hint: configure --help

ls include

  R.h     Rdefines.h   Rinterface.h  S.h
  R_ext/  Rembedded.h  Rinternals.h
\end{verbatim}

Details in WRE
\end{frame}

\begin{frame}{Calling R functions from C}
  \begin{enumerate}
    \item Build a pairlist
    \item call \code{Rf\_eval(s, rho)}
  \end{enumerate}

  \note{
    Example in hand out and nidemo package

    Creating environments not documented, but public
  }
\end{frame}

\begin{frame}{Remapped functions}
  \begin{itemize}
  \item Source is unadorned
  \item Preprocessor adds \code{Rf\_}
  \item Object code contains \code{Rf\_}
  \end{itemize}
  \note{
    Purpose is to avoid conflict with other libs for common names
    like length, error

    Package code can use either form, but consistency is good
  }
\end{frame}

\begin{frame}[fragile]{Finding C implementations}

\begin{verbatim}
cd R-devel/src/main

grep '"any"' names.c
{"any", do_logic3, 2, ...}

grep -l do_logic3 *.c
logic.c
\end{verbatim}

\note{names.c is the mapping table, R's internal function registration}
\end{frame}

\begin{frame}[fragile]{R CMD rtags}
\begin{verbatim}
cd R-devel/src
R CMD rtags --no-Rd
# creates index file TAGS
\end{verbatim}

\end{frame}

\section{Resources}

\begin{frame}{Writing R Extensions}

  \code{RShowDoc("R-exts")}

\end{frame}

\begin{frame}{Book: The C Programming Language (K \& R)}

  \begin{tabular}{ll}
    Java in a Nutshell           & 1264 pages \\
    The C++ Programming Language & 1030 pages \\
    The C Programming Language   & \alert{274} pages
  \end{tabular}
\end{frame}

\begin{frame}{Resources}
  \begin{itemize}
  \item \textbf{Read WRE and ``K \& R''}
  \item Use the sources for R and packages
  \item R-devel, bioc-devel mailing lists
  \item Patience
  \end{itemize}

  \begin{figure}
    \includegraphics[width=2in]{nativeI/chickenpatience.jpg}
  \end{figure}

\end{frame}


%% If debugging is the process of removing bugs, then programming must be
%% the process of putting them in.
%% Edsger W. Dijkstra

%% Debugging is twice as hard as writing the code in the first
%% place. Therefore, if you write the code as cleverly as possible, you
%% are, by definition, not smart enough to debug it.

%% Brian W. Kernighan and P. J. Plauger in The Elements of Programming Style.


%% If you lie to the computer, it will get you.
%% Perry Farrar

%% Computer programming is tremendous fun. Like music, it is a skill that
%% derives from an unknown blend of innate talent and constant
%% practice. Like drawing, it can be shaped to a variety of ends –
%% commercial, artistic, and pure entertainment. Programmers have a
%% well-deserved reputation for working long hours, but are rarely
%% credited with being driven by creative fevers. Programmers talk about
%% software development on weekends, vacations, and over meals not
%% because they lack imagination, but because their imagination reveals
%% worlds that others cannot see.
%% Larry O'Brien and Bruce Eckel in Thinking in C#

\end{document}

