%\VignetteDepends{foreach, doMC, RSQLite, ncdf}
%\VignetteSuggests{Rmpi}
%\VignetteIndexEntry{High Performance Computing}
%\VignettePackage{AdvancedR}

\documentclass{beamer}

\usepackage{Bioconductor}
\SweaveOpts{keep.source=TRUE}

\title{High Performance Computing}
\author{Martin Morgan, Seth Falcon}
\institute[FHCRC]{Fred Hutchinson Cancer Research Center}
\date{20-21 May, 2010}

\begin{document}

<<setup, echo=FALSE>>=
options(width = 40)
@ 

\frame{\titlepage}

\begin{frame}{Motivation}
  Challenges
  \begin{itemize}
  \item Long calculations: bootstrap, MCMC, \ldots.
  \item Big data: genome-wide association studies, re-sequencing,
    \ldots.
  \item Long $\times$ big: \ldots
  \end{itemize}
  Solutions
  \begin{itemize}
  \item Avoid \R{} programming pitfalls -- \emph{very} significant
    benefits.
  \item Parallel evaluation, especially `embarrassingly parallel'
  \item Large data management
  \end{itemize}
\end{frame}

\frame{\tableofcontents}

\section{Programming pitfalls}

\subsection{Pitfalls and solutions}

\begin{frame}[fragile]{Programming pitfalls: easy solutions}
  \begin{itemize}
  \item Input only required data
<<io-sketch, eval=FALSE, keep.source=TRUE>>=
colClasses <-
  c("NULL", "integer", "numeric", "NULL")
df <- read.table("myfile", colClasses=colClasses)
@ 
%% 
  \item Preallocate-and-fill, not copy-and-append
<<preallocate-and-fill-sketch, eval=FALSE, keep.source=TRUE>>=
result <- numeric(nrow(df))
for (i in seq_len(nrow(df)))
  result[[i]] <- some_calc(df[i,])
@ 

  \item Vectorized calculations, not iteration
<<iterative, eval=TRUE, keep.source=TRUE>>=
x <- runif(100000); x2 <- x^2
m <- matrix(x2, nrow=1000); y <- rowSums(m)
@ 

  \item Avoid unnecessary character creation operations, e.g.,
    \Rfunarg{USE.NAMES=FALSE} in \Rfunction{sapply}, \Rfunarg{use.names=FALSE}
    in \Rfunction{unlist}.
\end{itemize}
\end{frame}

\begin{frame}[fragile]{Programming pitfalls: moderate solutions}
  \begin{itemize}
  \item Use appropriate functions, often from specialized packages.
<<inappropriate-functions-2, eval=FALSE, keep.source=TRUE>>=
library(limma) # microarray linear models
fit <- lmFit(eSet, design)
@ 
  \item Identify appropriate algorithms, e.g., \Rfunction{\%in\%} is
    $O(N)$, whereas naive might be $O(N^2)$
<<algo-poly, keep.source=TRUE>>=
x <- 1:100; s <- sample(x, 10)
inS <- x %in% s
@ 
  \item Use C or Fortran code. Requires knowledge of other programming
    languages, and how to integrate these into \R{}
  \end{itemize}
\end{frame}

\subsection{Measuring performance}

\begin{frame}[fragile]{Measuring performance: timing}
  \begin{itemize}
  \item Use \Rfunction{system.time} to measure total evaluation time
    \begin{itemize}
    \item \Rfunarg{gcFirst=TRUE} for `garbage collection'
    \end{itemize}
  \item Use \Rfunction{replicate} to average over invocations
  \end{itemize}
<<system.time>>=
m <- matrix(runif(200000), 20000)
replicate(5, system.time(apply(m, 1, sum))[[1]])
replicate(5, system.time(rowSums(m))[[1]])
@ 
\begin{itemize}
\item Cautionary tale: \url{http://tinyurl.com/29bd6xv}
\end{itemize}
\end{frame}

\begin{frame}[fragile]{Measuring performance: comparison}
  \begin{itemize}
  \item \Rfunction{identical} and \Rfunction{all.equal} ensure that
    `optimizations' produce correct results!
  \end{itemize}
<<identical>>=
res1 <- apply(m, 1, sum)
res2 <- rowSums(m)
identical(res1, res2)
identical(c(1, -1), c(x=1, y=-1))
all.equal(c(1, -1), c(x=1, y=-1), check.attributes=FALSE)
@ 
\end{frame}

\begin{frame}[fragile]{Measuring execution time: \Rfunction{Rprof}}

<<Rprof, eval=FALSE, keep.source=TRUE>>=
tmpf = tempfile()
Rprof(tmpf)
res1 <- apply(m,  1, sum)
Rprof(NULL); summaryRprof(tmpf)
@ 
\begin{verbatim}
$by.self
         self.time self.pct total.time total.pct
"apply"       0.16       80       0.20       100
"FUN"         0.02       10       0.02        10
"lapply"      0.02       10       0.02        10
"unlist"      0.00        0       0.02        10

$by.total
         total.time total.pct self.time self.pct
"apply"        0.20       100      0.16       80
"FUN"          0.02        10      0.02       10
"lapply"       0.02        10      0.02       10
"unlist"       0.02        10      0.00        0
\end{verbatim}
\end{frame}

\begin{frame}[fragile]{Measuring memory use: \Rfunction{tracemem}}
  \begin{itemize}
  \item Enable memory profiling
  \end{itemize}
\begin{verbatim}
> ~/src/R-devel/configure --help
> ~/src/R-devel/configure --enable-memory-profiling
> make -j
\end{verbatim}
\begin{itemize}
\item Copy-on-change semantics
\end{itemize}
\begin{verbatim}
> x <- 1:10; tracemem(x)
[1] "<0x1b1a8f8>"
> y <- x       # no change, so no copy
> x[1] <- 2L   # x, y now differ, so copy
tracemem[0x1b1a8f8 -> 0x1b1a8a0]: 
\end{verbatim}
\end{frame}

\begin{frame}[fragile]{Measuring memory use: \Rfunction{tracemem}}
  \begin{itemize}
  \item Copying in \R{} functions
  \end{itemize}
\begin{verbatim}
> l <- list(a=1:10, b=1:10); tracemem(l$a)
[1] "<0x1131ce0>"
> df0 <- as.data.frame(l)
tracemem[0x1131ce0 -> 0x1131bd8]: eval as.data.frame.list as.data.frame 
tracemem[0x1131bd8 -> 0x1131a20]: data.frame eval eval as.data.frame.list as.data.frame 
tracemem[0x1131a20 -> 0x11318c0]: as.data.frame.integer as.data.frame data.frame eval eval as.data.frame.list as.data.frame 
> df1 <- data.frame(a=l$a, b=l$b)
tracemem[0x1131ce0 -> 0x11332c0]: data.frame 
tracemem[0x11332c0 -> 0x1133160]: as.data.frame.integer as.data.frame data.frame 
> identical(df0, df1)
[1] TRUE
\end{verbatim}
\end{frame}

\subsection{Case Study: GWAS}

\begin{frame}[fragile]{Case study: GWAS}
  \begin{itemize}
  \item Subset of genome-wide association study data
  \end{itemize}
<<gwas-input>>=
fname1 <- system.file("extdata", "gwas_2.rda", 
                      package="AdvancedR")
load(fname1)
gwas[1:2, 1:8]
@ 
\end{frame}

\begin{frame}[fragile]{GWAS and glm}
  \begin{itemize}
  \item Interested in fitting generalized linear model to each SNP
  \end{itemize}
<<glm-1>>=
snp0 <- function(i, gwas) {
    snp <- gwas[[i+3L]]
    glm(CaseControl ~ Age + Sex + snp,
        family=binomial, data=gwas)$coef
}
system.time(sapply(1:10, snp0, gwas))
@ 
\end{frame}

\begin{frame}{GWAS case study: further directions}
  \Rfunction{glm} can be optimized for SNPs
  \begin{itemize}
  \item Build the design matrix for \code{CaseControl \~{} Age + Sex}
    once, rather than once per SNP
  \item Use the estimate without the SNP as a starting point
  \item \Rpackage{snpMatrix} fits GLMs very efficiently
  \end{itemize}
  Outcome
  \begin{itemize}
  \item $\sim1000$ SNPs per second
  \end{itemize}
  Important lessons
  \begin{itemize}
  \item Careful optimization can often greatly reduce evaluation time
  \item Others may likely have done the work for you!
  \end{itemize}
\end{frame}

\section{Parallel evaluation}

\subsection{Embarrassingly parallel problems}

\begin{frame}{`Embarrassingly parallel' problems} 
  Problems that are:
  \begin{itemize}
  \item  Easily divisible into different, more-or-less
    identical, independent \emph{tasks}
  \item Tasks distributed across distinct computational \emph{nodes}.
  \item Examples: bootstrap; MCMC; row- or column-wise matrix
    operations; `batch' processing of multiple files, \ldots
  \end{itemize}
  
  What to expect: ideal performance
  \begin{itemize}
  \item Execution time inversely proportional to number of available
    nodes: $10\times$ speed-up requires $10$ nodes, $100\times$
    speedup requires 100 nodes
  \item Communication (data transfer between nodes) is expensive
  \item `Coarse-grained' tasks work best
  \end{itemize}
\end{frame}

\begin{frame}{Packages and other solutions}
  \begin{tabular}{llp{.5\textwidth}}
    Package & Hardware & Challenges \\\hline
    \Rpackage{multicore} & Computer & Not Windows (\Rpackage{doSMP} soon)\\
    \Rpackage{Rmpi} & Cluster & 
      Additional job management software (e.g., slurm) \\
    \Rpackage{snow} & Cluster & 
      Light-weight; convenient if MPI not available \\
    BLAS, \Rpackage{pnmath} & Computer &
      Customize \R{} build; benefits math routines only
  \end{tabular}
  Parallel interfaces
  \begin{itemize}
  \item Package-specific, e.g., \Rfunction{mpi.parLapply}
  \item \Rpackage{foreach}, \Rpackage{iterators}, \Rpackage{doMC},
    \ldots: common interface; fault tolerance; alternative programming
    model
  \end{itemize}
\end{frame}

\begin{frame}{General guidelines for parallel computing}
  \begin{itemize}
  \item Maximize computation per job 
  \item Distribute data implicitly, e.g., using shared file systems
  \item Nodes transform large data to small summary
    \begin{itemize}
    \item     E.g.: \Rpackage{ShortRead} quality assessment.
    \end{itemize}
  \item Construct self-contained functions that avoid global
    variables.
  \item Random numbers need special care!
  \end{itemize}
\end{frame}

\subsection{Packages and evaluation models}

\begin{frame}[fragile]{\Rpackage{multicore}}
  \begin{itemize}
  \item Shared memory, i.e., one computer with several cores
  \end{itemize}
\begin{verbatim}
> system.time(lapply(1:10, snp0, gwas))
   user  system elapsed 
  1.672   0.016   1.687 
> library(multicore)
> system.time(mclapply(1:10, snp0, gwas))
   user  system elapsed 
  1.864   0.348   1.119 
\end{verbatim}
\end{frame}

\begin{frame}[fragile]{\Rpackage{multicore}: under the hood}
  \begin{itemize}
  \item Operating system \code{fork}: new process, initially identical
    to current, OS-level copy-on-change.
  \item \Rfunction{parallel}: spawns new process, returns process id,
    starts expression evaluation.
  \item \Rfunction{collect}: queries process id to retrieve result,
    terminates process.
  \item \code{mclapply}: orchestrates \Rfunction{parallel} /
    \Rfunction{collect}
  \end{itemize}
\end{frame}
  
\begin{frame}[fragile]{\Rpackage{foreach}}
  \begin{itemize}
  \item \Rfunction{foreach}: establishes a \code{for}-like iterator
  \item \Rfunction{\%dopar\%}: infix binary function; left-hand-side:
    \Rfunction{foreach}; right-hand-side: expression for evaluation
  \item Variety of parallel back-ends, e.g., \Rpackage{doMC} for
    \Rpackage{multicore}; register with \Rfunction{registerDoMC}
  \end{itemize}
<<foreach>>=
library(foreach)
if ("windows" != .Platform$OS.type) {
    library(doMC); registerDoMC()
    res <- foreach(i=1:10) %dopar% snp0(i, gwas)
}
@   
\end{frame}

\begin{frame}[fragile]{iterators and foreach}
  \Rpackage{iterators} package
  \begin{itemize}
  \item \Rfunction{iter}: create an iterator on an object
  \item \Rfunction{nextElem}: return the next element of the object
  \item Built-in (e.g, \Rfunction{iapply}, \Rfunction{isplit}) and
    customizable
  \end{itemize}
<<iterators>>=
snp1 <- function(snp, gwas) {
    glm(CaseControl ~ Age + Sex + snp,
        family=binomial, data=gwas)$coef
}
snps <- gwas[,11:20]
res <- foreach(it=iter(snps, "column")) %dopar% 
           snp1(it, gwas)
@ 
\end{frame}

\begin{frame}[fragile]{Rmpi on a cluster}
  \begin{itemize}
  \item `Message passing' interface (MPI)
  \end{itemize}
  Players
  \begin{itemize}
  \item \texttt{slurm}: allocate resources, e.g., \texttt{salloc -N 4}
    allocates 4 nodes for computation
  \item \texttt{mpi}: e.g., \texttt{mpirun -n 1} starts a program on
    one node
  \item \texttt{R} and the \Rpackage{Rmpi} package
  \end{itemize}
\end{frame}

\begin{frame}[fragile]{Interactive \Rpackage{Rmpi}: manager / worker}
\begin{verbatim}
hyrax1:~> salloc -N 4 mpirun -n 1 R --interactive --quiet
salloc: Granted job allocation 239631
> library(Rmpi)
> mpi.spawn.Rslaves()
[...SNIP...]
> mpi.parSapply(1:10,
                function(i) c(i=i, rank=mpi.comm.rank()))
     [,1] [,2] [,3] [,4] [,5] [,6] [,7] [,8] [,9] [,10]
i       1    2    3    4    5    6    7    8    9    10
rank    1    1    1    2    2    3    3    4    4     4
> mpi.quit()
salloc: Relinquishing job allocation 239631
\end{verbatim}
\end{frame}

\begin{frame}[fragile]{Manage / worker}
  \begin{itemize}
  \item `Manager' script that spawns workers, tells workers what to
    do, collates results
  \item Submit as `batch' job on a single \R{} node
  \item View example script with \code{readScript("spawn.R")}
  \end{itemize}
\begin{verbatim}
hyrax1:~> salloc -N 4 mpirun -n 1 \
    R CMD BATCH /path/to/spawn.R
\end{verbatim}
\end{frame}

\begin{frame}[fragile]{Single instruction, multiple data (SIMD)}
\begin{itemize}
\item Single script, evaluated on each node, \code{readScript("simd.R")}.
\item Script specializes data for specific node
\item After evaluation, script specializes so that one node collates
  results from others
\end{itemize}
\begin{verbatim}
hyrax1:~> salloc -N 4 mpirun -n 4 \
    R CMD BATCH --slave /path/to/simd.R
\end{verbatim}
\end{frame}

\subsection{Case Study: GWAS (continued)}

\begin{frame}{Case study: GWAS (continued)}
  \begin{itemize}
  \item Readily parallelized -- glm for each SNP fit independently
  \item Divide SNPs into equal sized groups, one group per node
  \item SIMD evaluation model
  \item Need to manage data -- appropriate SNPs and metadata to each
    node
  \end{itemize}
  Important lessons
  \begin{itemize}
  \item Parallel evaluation for real problems can be difficult
  \item Parallelization after optimization
  \item Optimize / parallelize only after confirming that no one else
    has already done the work!
  \end{itemize}
\end{frame}

\section{Large data management}

\begin{frame}{Large data management}
  Putting appropriate data in memory
  \begin{itemize}
  \item An R analysis can make multiple copies of each data set
  \item Limits performance (I/O, but also calculations)
  \item Wastes system resources (e.g., decreasing the number of
    parallel tasks that can be executed)
  \end{itemize}
  Solutions
  \begin{itemize}
  \item Text versus R binary files
  \item `Stream' processing
  \item Data base use
  \item High-performance numeric storage
  \end{itemize}
\end{frame}

\subsection{Text, binary, and streaming I/O}

\begin{frame}[fragile]{Text versus R binary files}
  \begin{itemize}
  \item Text is slower than compressed binary
  \item Compressed binary is slower than binary
  \end{itemize}
<<text-io>>=
ftmp <- tempfile()
write.csv(gwas, ftmp)
system.time(read.csv(ftmp, row.names=1))[[3]]
save(gwas, file=ftmp)
replicate(5, system.time(load(ftmp, new.env()))[[3]])
save(gwas, file=ftmp, compress=FALSE)
replicate(5, system.time(load(ftmp, new.env()))[[3]])
unlink(ftmp)
@ 
\end{frame}

\begin{frame}[fragile]{`Stream' processing}
  \begin{itemize}
  \item Read in a chunk, process, read in next chunk
  \item Use `connections' to keep file open between chunks
  \item Good for very large data sets (if necessary)
  \item A few packages, e.g., \Rpackage{biglm}, exploit this model
  \item See \code{readScript("fapply.R")}
  \end{itemize}
\end{frame}

\subsection{Data bases and netCDF}

\begin{frame}[fragile]{Data bases}
  SQL
  \begin{itemize}
  \item Represent data in a SQL data base
  \item Best for \emph{relational} (structured) data of moderate
    (e.g., millions of rows) size
  \item Not the best solution for, e.g., array-like numerical data
  \end{itemize}
  Use
  \begin{itemize}
  \item \Rpackage{DBI} package provides abstract interface
  \item\Rpackage{RSQLite} (built-in to R), \Rpackage{RMySQL},
    \Rpackage{RPostgreSQL}, ... provide implementations
  \end{itemize}
  Example: \Rpackage{RSQLite} set-up
<<sql-setup>>=
db0 <- tempfile()
library(RSQLite)
drv <- dbDriver("SQLite")
conn <- dbConnect(drv, dbname=db0)
@ 
\end{frame}

\begin{frame}[fragile]{GWAS metadata}
Create
<<sql-create, keep.source=TRUE>>=
gwasPhenotypes <- gwas[,1:3]
dbWriteTable(conn, "gwasPhenotypes", gwasPhenotypes)
@ 
Retrieve
<<db-fetch, keep.source=TRUE>>=
q <- dbSendQuery(conn, "SELECT * FROM gwasPhenotypes")
fetch(q, n = 2) # first 2; n = -1 for all
invisible(dbClearResult(q)) # close out query
@ 
Clean-up
<<db-clean>>=
invisible(dbDisconnect(conn))
@ 
\end{frame}

\begin{frame}[fragile]{NetCDF and the \Rpackage{ncdf} package}
  NetCDF and \Rpackage{ncdf}
  \begin{itemize}
  \item Network Common Data Form: array-oriented scientific data
  \item \Rpackage{ncdf}: R package for NetCDF access
    \begin{itemize}
    \item Warning: character arrays very inefficient in \Rpackage{ncdf}
    \end{itemize}
  \item \Rpackage{ncdf4}: recent; NetCDF 4 format; not yet avaiable
    for Windows
  \end{itemize}
  Data and library
<<ncdf-setup>>=
ngwas <- local({
    x0 <- lapply(gwas[,-(1:3)], as.integer)
    matrix(unlist(x0, use.names=FALSE), ncol=length(x0))
})
ncdf0 <- tempfile()
library(ncdf)
@ 
\end{frame}

\begin{frame}[fragile]{\Rpackage{ncdf}, continued}
  \begin{itemize}
  \item Define dimensions and variable
  \end{itemize}
<<ncdf-define-dims>>=
sampd <- dim.def.ncdf("Sample", "id", seq_len(nrow(ngwas)))
snpd <- dim.def.ncdf("SNP", "id", seq_len(ncol(ngwas)))
snpv <- var.def.ncdf("Genotype", 
                     units="1: AA, 2: AB; 3: BB",
                     dim=list(sampd, snpd), 
                     missval=-1L, prec="integer")
@ 
\begin{itemize}
\item Create file
\end{itemize}
<<ncdf-create-file>>=
nc <- create.ncdf(ncdf0, snpv)
put.var.ncdf(nc, snpv, ngwas)
invisible(close(nc))
@ 
\end{frame}

\begin{frame}[fragile]{\Rpackage{ncdf}, continued}
  \begin{itemize}
  \item Very favorable file I/O performance
  \end{itemize}
<<ncdf-read,keep.source=TRUE>>=
nc <- open.ncdf(ncdf0)
system.time({
    nc_gwas <- get.var.ncdf(nc, "Genotype")
})[[1]]
@ 
  \begin{itemize}
  \item Easy to obtains data slices
  \end{itemize}
<<ncdf-read-II, source=TRUE>>=
g <- get.var.ncdf(nc, "Genotype", start=c(30, 100),
       count=c(10, 20)) # samples 30:40, snps 100:120
g <- get.var.ncdf(nc, "Genotype", start=c(1,1000),
       count=c(-1, 100)) # all samples, snps 1000:1100
invisible(close(nc))
@   
  
\end{frame}

\subsection{Case Study: GWAS (concluded)}

\begin{frame}{Case study: GWAS (concluded)}
  Overall solution
  \begin{itemize}
  \item Optimize \Rfunction{glm} for SNPs
  \item Store SNP data as netCDF, metadata as SQL
  \item Use SIMD model to parallelize calculations
  \end{itemize}
  Outcome
  \begin{itemize}
  \item Initially: $<10$ SNPs per second
  \item Optimized: $\sim1000$ SNP per second
  \item 100 node cluster: $\sim100,000$ SNP per second
  \end{itemize}
\end{frame}

\section{Resources}

\begin{frame}{Resources}
  \begin{itemize}
  \item News group:
    \url{https://stat.ethz.cb/mailman/listinfo/r-sig-hpc}
  \item CRAN Task View:
    \url{http://cran.fhcrc.org/web/views/HighPerformanceComputing.html}
  \item Key packages: \Rpackage{multicore}, \Rpackage{Rmpi},
    \Rpackage{snow}, \Rpackage{foreach} (and friends);
    \Rpackage{RSQlite}, \Rpackage{ncdf}
  \end{itemize}
\end{frame}

\end{document}
