%\VignetteIndexEntry{1.4 Reads and Strings}

\chapter{Strings and Reads}
\label{chpt:SequencesAndReads}

\begin{table}
  \centering
  \caption{Selected \Bioconductor{} packages for representing strings
    and reads.}
  \label{tab:strings-pkgs}
  \begin{tabular}{lp{.7\textwidth}}
    Package & Description\\
    \hline
    \Biocpkg{Biostrings} & Classes (e.g., \Rclass{DNAStringSet}) and
    methods (e.g., \Rfunction{alphabetFrequency},
    \Rfunction{pairwiseAlignment}) for representing and manipulating
    DNA and other biological sequences.\\

    \Biocpkg{BSgenome} & Representation and manipulation of large
    (e.g., whole-genome) sequences.\\
    
    \Biocpkg{ShortRead} & I/O and manipulation of FASTQ files.\\
    \hline
  \end{tabular}
\end{table}

\section{DNA (and other) Strings with the \Biocpkg{Biostrings} package}

\begin{table}
  \centering
  \caption{Operations on strings in the \Biocpkg{Biostrings} package.}
  \label{tab:ops:biostrings}
  \begin{tabular}{lll}
    & Function & Description \\
    \hline

    Access & \Rfunction{length}, \Rfunction{names} & Number and
        names of sequences\\ 
    & \Rfunction{[}, \Rfunction{head}, \Rfunction{tail}, \Rfunction{rev}
    & Subset, first, last, or reverse sequences\\
    & \Rfunction{c} & Concatenate two or more objects\\
    & \Rfunction{width}, \Rfunction{nchar} & Number of letters of each
        sequence \\
    & \Rfunction{Views} & Light-weight sub-sequences of a sequence\\

    Compare & \Rfunction{==}, \Rfunction{!=}, \Rfunction{match}, 
        \Rfunction{\%in\%} & Element-wise comparison\\ 
    & \Rfunction{duplicated}, \Rfunction{unique} &
        Analog to \Rfunction{duplicated} and \Rfunction{unique} on character
        vectors \\
    & \Rfunction{sort}, \Rfunction{order} & Locale-independent sort, order\\
    & \Rfunction{split}, \Rfunction{relist} & Split or relist objects to,
        e.g., \Rclass{DNAStringSetList} \\

    Edit & \Rfunction{subseq}, \Rfunction{subseq<-} &
        Extract or replace sub-sequences in a set of sequences \\
    & \Rfunction{reverse}, \Rfunction{complement} & 
        Reverse, complement, or reverse-complement DNA \\
    & \Rfunction{reverseComplement} \\
    & \Rfunction{translate} & Translate DNA to Amino Acid sequences \\
    & \Rfunction{chartr} & Translate between letters \\
    & \Rfunction{replaceLetterAt} & Replace letters at a set of
        positions by new letters \\
    & \Rfunction{trimLRPatterns} & Trim or find flanking patterns \\

    Count & \Rfunction{alphabetFrequency} & Tabulate letter occurrence \\
    & \Rfunction{letterFrequency} \\
    & \multicolumn{2}{l}{\Rfunction{letterFrequencyInSlidingView}}\\
    & \Rfunction{consensusMatrix} & 
        Nucleotide $\times$ position summary of letter counts \\
    & \Rfunction{dinucleotideFrequency} & 2-mer, 3-mer, and k-mer counting \\
    & \Rfunction{trinucleotideFrequency} \\
    & \Rfunction{oligonucleotideFrequency} \\
    & \Rfunction{nucleotideFrequencyAt} & 
        Nucleotide counts at fixed sequence positions\\

    Match & \Rfunction{matchPattern}, \Rfunction{countPattern} & 
        Short patterns in one or many (\texttt{v*}) sequences\\
    & \multicolumn{2}{l}{\Rfunction{vmatchPattern}, \Rfunction{vcountPattern}}\\
    & \Rfunction{matchPDict}, \Rfunction{countPDict} & 
        Short patterns in one or many (\texttt{v*}) sequences (mismatch only)\\
    & \Rfunction{whichPDict}, \Rfunction{vcountPDict} \\
    & \Rfunction{vwhichPDict} \\
    & \Rfunction{pairwiseAlignment} & Needleman-Wunsch,
        Smith-Waterman, etc. pairwise alignment\\
    & \Rfunction{matchPWM}, \Rfunction{countPWM} & 
        Occurrences of a position weight matrix\\
    & \Rfunction{matchProbePair} & Find left or right flanking patterns \\
    & \Rfunction{findPalindromes} & Palindromic regions in a sequence. Also \\
    & & \Rfunction{findComplementedPalindromes} \\
    & \Rfunction{stringDist} & 
        Levenshtein, Hamming, or pairwise alignment scores\\

    I/0 & \Rfunction{readDNAStringSet} & 
        FASTA (or sequence only from FASTQ). Also\\
    & & \Rfunction{readBStringSet}, \Rfunction{readRNAStringSet},
        \Rfunction{readAAStringSet}\\
    & \Rfunction{writeXStringSet}\\
    & \Rfunction{writePairwiseAlignments} &
        Write \Rfunction{pairwiseAlignment} as ``pair'' format\\
    & \Rfunction{readDNAMultipleAlignment} &
        Multiple alignments (FASTA, ``stockholm'', or ``clustal''). Also\\
    & & \Rfunction{readRNAMultipleAlignment}, 
        \Rfunction{readAAMultipleAlignment} \\
    & \Rfunction{write.phylip}\\\hline
  \end{tabular}
\end{table}

Table~\ref{tab:strings-pkgs} lists select \Bioconductor{} packages for
representing strings and reads.  The \Biocpkg{Biostrings} package
provides tools for working with sequences.  The essential data
structures are \Rclass{DNAString} and \Rclass{DNAStringSet}, for
working with one or multiple DNA sequences. The \Biocpkg{Biostrings}
package contains additional classes for representing amino acid and
general biological strings.  The \Biocpkg{BSgenome} and related
packages (e.g., \Biocannopkg{BSgenome.Dmelanogaster.UCSC.dm3}) are
used to represent whole-genome
sequences. Table~\ref{tab:ops:biostrings} summarizes common
operations; The following exercise explores these packages.

\begin{Exercise}
  The objective of this exercise is to calculate the GC content of the
  exons of a single gene. We jump into the middle of some of the data
  structures common in \Bioconductor{}; these are introduced more
  thoroughly in later exercises..

  Load the \Biocannopkg{BSgenome.Dmelanogaster.UCSC.dm3} data package,
  containing the UCSC representation of \Dmel{} genome assembly
  dm3. Discover the content of the package by evaluating
  \Robject{Dmelanogaster}.
  
  Load the \Rpackage{Morgan2013} package, and evaluate the
  command \Rcode{data(ex)} to load an example of a
  \Rclass{GRangesList} object. the \Rclass{GRangesList} represents
  coordinates of exons in the \Dmel{} genome, grouped by gene.

  Look at \Rcode{ex[1]}. These are the genomic coordinates of the
  first gene in the \Rcode{ex} object. Load the \Dmel{} chromosome
  that this gene is on by subsetting the \Robject{Dmelanogaster}
  object.

  Use \Rfunction{Views} to create views on to the chromosome that span
  the start and end coordinates of all exons in the first gene; the
  start and end coordinates are accessed with \Rcode{start(ex[[1]])}
  and similar.

  Develop a function \Rfunction{gcFunction} to calculate GC
  content. Use this to calculate the GC content in each of the exons.
\end{Exercise}

\begin{Solution}
  Here we load the \Dmel{} genome, select a single chromosome, and
  create \Rfunction{Views} that reflect the ranges of the
  \texttt{FBgn0002183}.
<<gc-genome>>=
library(BSgenome.Dmelanogaster.UCSC.dm3)
Dmelanogaster
library(Morgan2013)
data(ex)
ex[1]
nm <- "chr3L"
chr <- Dmelanogaster[[nm]]
v <- Views(chr, start=start(ex[[1]]), end=end(ex[[1]]))
@ 
%% 
Here is the \Rfunction{gcFunction} helper function to calculate GC
content:
<<gcFunction-definition>>=
gcFunction
@ 
%
The \Rfunction{gcFunction} is really straight-forward: it invokes the
function \Rfunction{alphabetFrequency} from the \Biocpkg{Biostrings}
package. This returns a simple matrix of exon $\times$ nucleotide
probabilities. The row sums of the \texttt{G} and \texttt{C} columns
of this matrix are the GC contents of each exon.

The subject GC content is
<<gcFunction-genome, eval=FALSE>>=
subjectGC <- gcFunction(v)
@ 
\end{Solution}

\section{Reads and the \Biocpkg{ShortRead} package}
\label{sec:ShortRead}

\paragraph{Short read formats}

Illumina technologies generate sequences by measuring incorporation of
florescent nucleotides over successive PCR cycles. These sequencers
produce output in a variety of formats, but \emph{FASTQ} is
ubiquitous. Each read is represented by a record of four components:
<<fastq-format, echo=FALSE>>=
bigdata <- "~/BigData"
fls <- dir(file.path(bigdata, "fastq"), full=TRUE)
cat(noquote(readLines(fls[[1]], 4)), sep="\n")
@ 
The first and third lines (beginning with \verb|@| and \verb|+|
respectively) are unique identifiers. The identifier produced by the
sequencer typically includes a machine id followed by colon-separated
information on the lane, tile, x, and y coordinate of the read. The
example illustrated here also includes the SRA accession number, added
when the data was submitted to the archive. The machine identifier
could potentially be used to extract information about batch
effects. The spatial coordinates (lane, tile, x, y) are often used to
identify optical duplicates; spatial coordinates can also be used
during quality assessment to identify artifacts of sequencing, e.g.,
uneven amplification across the flow cell, though these spatial
effects are rarely pursued.

The second and fourth lines of the FASTQ record are the nucleotides
and qualities of each cycle in the read. This information is given in
5' to 3' orientation as seen by the sequencer. A letter \texttt{N} in
the sequence is used to signify bases that the sequencer was not able
to call.  The fourth line of the FASTQ record encodes the quality
(confidence) of the corresponding base call. The quality score is
encoded following one of several conventions, with the general notion
being that letters later in the visible ASCII alphabet
<<ascii, echo=FALSE>>=
cat(rawToChar(as.raw(32+1:75)),
    rawToChar(as.raw(32+76:94)), sep="\n")
@ 
are of higher quality; this is developed further below. Both the
sequence and quality scores may span multiple lines.

Technologies other than Illumina use different formats to represent
sequences. Roche 454 sequence data is generated by `flowing' labeled
nucleotides over samples, with greater intensity corresponding to
longer runs of A, C, G, or T. This data is represented as a series of
`flow grams' (a kind of run-length encoding of the read) in Standard
Flowgram Format (SFF). The \Bioconductor{} package
\Biocpkg{R453Plus1Toolbox} has facilities for parsing SFF files, but
after quality control steps the data are frequently represented (with
some loss of information) as FASTQ. 

\paragraph{Short reads in \R{}}

FASTQ files can be read in to \R{} using the \Rfunction{readFastq}
function from the \Biocpkg{ShortRead} package. Use this function by
providing the path to a FASTQ file. There are sample data files
available in the \texttt{~/BigData} directory, each consisting
of 1 million reads from a lane of the Pasilla data set.
<<readFastq>>=
library(ShortRead)
bigdata <- "~/BigData"  ## e.g., "C:/Users/mtmorgan/"
fastqDir <- file.path(bigdata, "fastq")
fastqFiles <- dir(fastqDir, full=TRUE)
fq <- readFastq(fastqFiles[1])
fq
@
The data are represented as an object of class \Rclass{ShortReadQ}.
<<sread>>=
head(sread(fq), 3)
head(quality(fq), 3)
head(id(fq), 3)
@ 
The \Rclass{ShortReadQ} class illustrates \emph{class
  inheritance}. It extends the \Rclass{ShortRead} class
<<getClasse>>=
getClass("ShortReadQ")
@ 
Methods defined on \Rclass{ShortRead} are available for
\Rclass{ShortReadQ}.
<<showMethods-ShortRead, eval=FALSE>>=
showMethods(class="ShortRead", where=search())
@ 
For instance, the \Rfunction{width} can be used to demonstrate that
all reads consist of \Sexpr{unique(width(fq))} nucleotides.
<<width-ShortRead>>=
table(width(fq))
@ 
The \Rfunction{alphabetByCycle} function summarizes use
of nucleotides at each cycle in a (equal width) \Rclass{ShortReadQ} or
\Rclass{DNAStringSet} instance.
<<width-ShortReadQ>>=
abc <- alphabetByCycle(sread(fq))
abc[1:4, 1:8]
@ 

FASTQ files are getting larger. A very common reason for looking at
data at this early stage in the processing pipeline is to explore
sequence quality. In these circumstances it is often not necessary to
parse the entire FASTQ file. Instead create a representative sample
<<FastqSampler>>=
sampler <- FastqSampler(fastqFiles[1], 1000000)
yield(sampler) # sample of 1000000 reads
@ 
A second common scenario is to pre-process reads, e.g., trimming
low-quality tails, adapter sequences, or artifacts of sample
preparation. The \Rclass{FastqStreamer} class can be used to `stream'
over the fastq files in chunks, processing each chunk independently.

\Biocpkg{ShortRead} contains facilities for quality assessment of
FASTQ files. Here we generate a report from a sample of 1 million
reads from each of our files and display it in a web browser
<<qa, eval=FALSE>>=
qas0 <- Map(function(fl, nm) {
    fq <- FastqSampler(fl)
    qa(yield(fq), nm)
}, fastqFiles,
   sub("_subset.fastq", "", basename(fastqFiles)))
qas <- do.call(rbind, qas0)
rpt <- report(qas, dest=tempfile())
browseURL(rpt)
@ 
A report from a larger subset of the experiment is available
<<report,eval=FALSE >>=
rpt <- system.file("GSM461176_81_qa_report", "index.html",
                   package="Morgan2013")
browseURL(rpt)
@ 

\begin{Exercise}
  Use the file path \Rfunction{bigdata} and the \Rfunction{file.path}
  and \Rfunction{dir} functions to locate the fastq file from
  \cite{pasilla} (the file was obtained as described in the
  \Rpackage{pasilla} experiment data package).

  Input the fastq files using \Rfunction{readFastq} from the
  \Biocpkg{ShortRead} package.
  
  Use \Rfunction{alphabetFrequency} to summarize the GC content of all
  reads (hint: use the \Rfunction{sread} accessor to extract the
  reads, and the \Rfunarg{collapse=TRUE} argument to the
  \Rfunction{alphabetFrequency} function). Using the helper function
  \Rfunction{gcFunction} defined in the \Rpackage{Morgan2013} package,
  draw a histogram of the distribution of GC frequencies across reads.
  
  Use \Rfunction{alphabetByCycle} to summarize the frequency of each
  nucleotide, at each cycle. Plot the results using
  \Rfunction{matplot}, from the \Rpackage{graphics} package.

  As an advanced exercise, and if on Mac or Linux, use the
  \Rpackage{parallel} package and \Rfunction{mclapply} to read and
  summarize the GC content of reads in two files in parallel.
\end{Exercise}

\begin{Solution}
Discovery:
<<fastq-discovery>>=
dir(bigdata)
fls <- dir(file.path(bigdata, "fastq"), full=TRUE)
@ 
Input:
<<fastq-input-gc, echo=FALSE>>=
rm(fq); invisible(gc())
@ 
<<fastq-input>>=
fq <- readFastq(fls[1])
@ 
GC content:
<<gcC>>=
alf0 <- alphabetFrequency(sread(fq), as.prob=TRUE, collapse=TRUE)
sum(alf0[c("G", "C")])
@ 
A histogram of the GC content of individual reads is obtained with
<<gc-reads>>=
gc <- gcFunction(sread(fq))
hist(gc)
@ 
Alphabet by cycle:
<<abc>>=
abc <- alphabetByCycle(sread(fq))
matplot(t(abc[c("A", "C", "G", "T"),]), type="l")
@ 
Advanced (Mac, Linux only): processing on multiple cores.
<<abc-mclapply, eval=FALSE>>=
library(parallel)
gc0 <- mclapply(fls, function(fl) {
  fq <- readFastq(fl)
  gc <- gcFunction(sread(fq))
  table(cut(gc, seq(0, 1, .05)))
})
## simplify list of length 2 to 2-D array
gc <- simplify2array(gc0)
matplot(gc, type="s")
@ 
\end{Solution}

\begin{Exercise}
  Use \Rfunction{quality} to extract the quality scores of the short
  reads. Interpret the encoding qualitatively.
  
  Convert the quality scores to a numeric matrix, using
  \Rfunction{as()}. Inspect the numeric matrix (e.g., using
  \Rfunction{dim}) and understand what it represents.
  
  Use \Rfunction{colMeans} to summarize the average quality score by
  cycle. Use \Rfunction{plot} to visualize this.
\end{Exercise}

\begin{Solution}
<<quality>>=
head(quality(fq))
qual <- as(quality(fq), "matrix")
dim(qual)
plot(colMeans(qual), type="b")
@ 
\end{Solution}

\begin{Exercise}
  As an independent exercise when an internet connection is available,
  visit the \Biocpkg{qrqc} landing page and explore the package
  vignette. Use the \Biocpkg{qrqc} package (you may need to install
  this) to generate base and average quality plots for the data, like
  those in the report generated by \Biocpkg{ShortRead}.
\end{Exercise}

%% \FIXME{Paired-end reads}
