%\VignetteDepends{GenomicRanges}
%\VignetteIndexEntry{Databases}
%\VignettePackage{AdvancedR}

\documentclass{beamer}

\usepackage{Bioconductor}
\SweaveOpts{keep.source=TRUE}

\title{Using Databases in R}
\author{Marc Carlson}
\institute[FHCRC]{Fred Hutchinson Cancer Research Center}
\date{May 20, 2010}

\begin{document}

<<setup, echo=FALSE>>=
options(width = 62)
@ 

%% use [fragile] in all frames with Sweave code chunks

\frame{\titlepage}

\section[Outline]{}
\frame{\tableofcontents}




%-----------------------------------------------
%-----------------------------------------------
\section{Introduction}



%-----------------------------------------------
\begin{frame}{Relational Databases}
\begin{block}{Relational database basics}
  \begin{itemize}
    \item Data stored in \textit{tables}
    \item Tables related through \textit{keys}
    \item Relational model called a \textit{schema}
    \item Tables designed to avoid redundancy
  \end{itemize}    
\end{block}
\begin{block}{Beneficial uses by R packages}
  \begin{itemize}
    \item Out-of-memory data storage
    \item Fast access to data subsets
    \item Databases accessible by other software
  \end{itemize}
\end{block}    
\end{frame}



%-----------------------------------------------
\begin{frame}{Uses of Relational Databases in Bioconductor}
\begin{block}{Annotation packages}
  \begin{itemize}
    \item Organism, genome (e.g. \Rpackage{org.Hs.eg.db})
    \item Microarray platforms (e.g. \Rpackage{hgu95av2.db})
    \item Homology (e.g. \Rpackage{hom.Hs.inp.db})
  \end{itemize}    
\end{block}
\begin{block}{Software packages}
  \begin{itemize}
    \item Transcript annotations (e.g. \Rpackage{GenomicFeatures})
    \item NGS experiments (e.g. \Rpackage{Genominator})
    \item Annotation infrastructure (e.g. \Rpackage{AnnotationDbi})
  \end{itemize}
\end{block}    
\end{frame}



%-----------------------------------------------
%-----------------------------------------------

%% Slides to introduce GenomicFeatures and discuss what it does. (and
%% why we store DBs on the disk) (10-15 slides)

\section{Example Databases: The GenomicFeatures Package}
%% Slides to introduce GenomicFeatures and discuss what it does. (and

\begin{frame}{The \Rpackage{GenomicFeatures} package}
  What is does
  \begin{itemize}
    \item Builds databases on the fly as needed.
    \item Supported sources: UCSC tracks, biomaRt, or custom.
    \item Has accessor methods to make it easy to use this data.
  \end{itemize}
  \vskip12pt
  Why it was built
  \begin{itemize}
    \item Needed a generic way to support access to multiple resources
    \item Users needed reproducible data for research
    \item Data needed to be formatted in a convenient way
  \end{itemize}
  \vskip12pt
  Why it uses a database
  \begin{itemize}
    \item Annotation data is highly relational
    \item Other annnotation data is also stored in SQLite databases
    \item Allows access from disc (which can free up memory)
  \end{itemize}
\end{frame}



%-----------------------------------------------
\begin{frame}{What do I mean by relational?}
  \begin{figure}
    \includegraphics[width=4.25in]{database/transcriptschema.png}
  \end{figure}
\end{frame}



%-----------------------------------------------
\begin{frame}[fragile]{TranscriptDb Basics}
\begin{block}{Making a TranscriptDb object}
<<makeUCSC, eval=FALSE>>=
library(GenomicFeatures)
mm9KG <-
  makeTranscriptDbFromUCSC(genome = "mm9",
                           tablename = "knownGene")
@
\end{block}
<<preload, echo=FALSE>>=
library(GenomicFeatures)
if(!exists("mm9KG",mode="TranscriptDb")) {
    mm9KG <-
      loadFeatures(system.file("extdata","mm9KG.sqlite",
                                package = "AdvancedR"))
}
@   
\begin{block}{Saving and Loading}
<<saveAndLoad, eval=FALSE>>=
saveFeatures(mm9KG, file="mm9KG.sqlite")
mm9KG <-
  loadFeatures(system.file("extdata", "mm9KG.sqlite",
                           package = "AdvancedR"))
@
\end{block}
\end{frame}


%-----------------------------------------------
\begin{frame}[fragile, shrink=15]{Accessing the TranscriptDb}
<<transcripts>>=
head(transcripts(mm9KG), 3)
@ 
\end{frame}


%-----------------------------------------------
\begin{frame}[fragile]{What actually happens when we do this?}
<<transcriptsDetails,eval=FALSE>>=
options(verbose=TRUE)
txs <- transcripts(mm9KG)
@ 
\begin{verbatim}
SQL QUERY: SELECT tx_chrom, tx_start, tx_end, tx_strand,
transcript._tx_id AS tx_id , tx_name FROM transcript
ORDER BY tx_chrom, tx_strand, tx_start, tx_end
\end{verbatim}

\begin{block}{}
  Notice how the database query is pretty simple?
  \begin{itemize}
    \item DB joins promote flexible access
    \item BUT: there is a cost if using A LOT of joins
    \item Therefore (in this case) a hybrid approach:
      Retrieve relevant records and subset in R
  \end{itemize}      
\end{block}
\end{frame}


%-----------------------------------------------
\begin{frame}{Our database schema}
  \begin{figure}
    \includegraphics[width=4.25in]{database/transcriptschema.png}
  \end{figure}
\end{frame}



%-----------------------------------------------
%-----------------------------------------------
%% I should probably show people how to see the schemas before I get
%% to this next part...  OR I can show them this next.

\section{Basic SQL}
%% Slides to teach some basic SQL so that people can follow along (10 slides)

\begin{frame}{SQL in 3 slides}
  Structured Query Language (SQL) is the most common language
  for interacting with relational databases.
\end{frame}

%-----------------------------------------------
\begin{frame}[fragile, shrink=15]{Database Retrieval}
\begin{block}{Single table selections}
  \begin{verbatim}
    SELECT * FROM gene;
    SELECT gene_id, gene._tx_id FROM gene;

    SELECT * FROM gene WHERE _tx_id=49245;
    SELECT * FROM transcript WHERE tx_name LIKE '%oap.1'; 
  \end{verbatim}
\end{block}
\begin{block}{Inner joins}
  \begin{verbatim}
    SELECT gene.gene_id,transcript._tx_id
      FROM gene, transcript
      WHERE gene._tx_id=transcript._tx_id;

    SELECT g.gene_id,t._tx_id 
      FROM gene AS g, transcript AS t
      WHERE g._tx_id=t._tx_id
      AND t._tx_id > 10;
  \end{verbatim}
\end{block}
\end{frame}

%-----------------------------------------------
\begin{frame}[fragile, shrink=15]{Database Modifications}
\begin{block}{CREATE TABLE}
  \begin{verbatim}
    CREATE TABLE foo (
      id INTEGER,
      string TEXT
    );
  \end{verbatim}
\end{block}
\begin{block}{INSERT}
  \begin{verbatim}
    INSERT INTO foo (id, string) VALUES (1,"bar");
  \end{verbatim}
\end{block}
\begin{block}{CREATE INDEX}
  \begin{verbatim}
    CREATE INDEX fooInd1 ON foo(id);
  \end{verbatim}
\end{block}
\end{frame}



% DEMO: demonstrate the SQL statements on the test DB.





%-----------------------------------------------
%-----------------------------------------------
\section{Using SQL from within R}
%% Slides to introduce DBI, RSQLite and how these are used to: 1)
%% build DBs, and 2) access the details of this database.(10-15 slides)


%-----------------------------------------------
\begin{frame}[fragile]{The \Rpackage{DBI} package}
  \begin{itemize}
    \item Provides a nice generic access to databases in R
    \item Many of the functions are convenient and simple to use
  \end{itemize}  
\end{frame}


%-----------------------------------------------
\begin{frame}[fragile]{Some popular DBI functions}
<<driverUsageAndListTables>>=
library(RSQLite) #loads DBI too, (but we need both)
drv <- dbDriver("SQLite")
con <- dbConnect(drv, dbname=system.file("extdata", 
                 "mm9KG.sqlite", package="AdvancedR"))
dbListTables(con)
dbListFields(con,"transcript")
@ 
\end{frame}


%-----------------------------------------------
\begin{frame}[fragile]{The dbGetQuery approach}
<<dbGetQuery>>=
dbGetQuery(con, "SELECT * FROM transcript LIMIT 3")
@ 
\end{frame}


%-----------------------------------------------
\begin{frame}[fragile]{The dbSendQuery approach}
If you use result sets, you also need to put them away
<<dbSendQuery>>=
res <- dbSendQuery(con, "SELECT * FROM transcript")
fetch(res, n= 3)
dbClearResult(res)
@ 
Calling fetch again will get the next three results.
This allows for simple iteration.
\end{frame}


%-----------------------------------------------
\begin{frame}[fragile]{Your turn}
Select the exons from the minus strand of chromosome 9.
<<exercise1,echo=FALSE>>=
sql <- "SELECT * FROM exon WHERE exon.exon_strand='-' 
        AND exon.exon_chrom='chr9'"
res <- dbGetQuery(con, sql)
@ 
\end{frame}


%-----------------------------------------------
\begin{frame}[fragile,shrink=15]{Setting up a new DB}
  First, lets close the connection to our other DB:
<<dbDisconnect>>=
   dbDisconnect(con)
@ 
  Then lets make a new database.
  Notice that we specify the database name with "dbname" 
  This allows it to be written to disc instead of just memory.
<<makeNewDb>>=
drv <- dbDriver("SQLite")
con <- dbConnect(drv, dbname="myNewDb.sqlite")
@ 
Once you have this, you may want to make a new table
<<CreateNewTable>>=
dbGetQuery(con, "CREATE Table foo (id INTEGER, string TEXT)")
@ 
\end{frame}


% -----------------------------------------------
\begin{frame}[fragile]{Your turn again}
  Create a database and then put a table in it called genePheno to store the genes mutated and a phenotypes associated with each.  Plan for genePheno to hold the following gene IDs and phenotypes (as a toy example):
\begin{verbatim}
data = data.frame(id=c(69773,20586,258822,18315),
                  string=c("Dead",
                           "Alive",
                           "Dead",
                           "Alive"),
                  stringsAsFactors=FALSE)
\end{verbatim}
<<exercise2,echo=FALSE>>=
drv <- dbDriver("SQLite")
dbcon <- dbConnect(drv, dbname="myExDb.sqlite")
cval <- dbGetQuery(dbcon, 
                   "CREATE Table genePheno 
                    (id INTEGER, string TEXT)")
@ 
\end{frame}


%-----------------------------------------------
\begin{frame}[fragile]{The \Rpackage{RSQLite} package}
  \begin{itemize}
    \item Provides SQLite access for R
    \item Much better support for complex inserts
  \end{itemize}  
\end{frame}




%-----------------------------------------------
\begin{frame}[fragile]{Prepared queries}
<<LabelledPreparedQueries>>=
data <- data.frame(c(226089,66745),
                  c("C030046E11Rik","Trpd52l3"), 
                  stringsAsFactors=FALSE)
names(data) <- c("id","string")
sql <- "INSERT INTO foo VALUES ($id, $string)"
dbBeginTransaction(con)
dbGetPreparedQuery(con, sql, bind.data = data)
dbCommit(con)
@ 
Notice that we want strings instead of factors in our data.frame
\end{frame}




% -----------------------------------------------
\begin{frame}[fragile]{Your turn part 3}
  Now take a moment to insert that data into your database.
<<exercise3,echo=FALSE>>=
data <- data.frame(id=c(69773,20586,258822,18315),
                   string=c("Dead",
                            "Alive",
                            "Dead",
                            "Alive"),
                   stringsAsFactors=FALSE)
sql <- "INSERT INTO genePheno VALUES ($id,$string)"
bval <- dbBeginTransaction(dbcon)
gval <- dbGetPreparedQuery(dbcon, sql, bind.data = data)
cval <- dbCommit(dbcon)
@ 
\end{frame}





%-----------------------------------------------
\begin{frame}[fragile]{in SQLite, you can ATTACH Dbs}
  The SQL what we want looks quite simple:
  \begin{verbatim}
    ATTACH "mm9KG.sqlite" AS db;
  \end{verbatim}  
  So we just need to do something like this:
<<ATTACH>>=
db <- system.file("extdata", "mm9KG.sqlite", 
                  package="AdvancedR")
dbGetQuery(con, sprintf("ATTACH '%s' AS db",db))
@ 
\end{frame}



%-----------------------------------------------
\begin{frame}[fragile, shrink=15]{You can join across attached Dbs}
  The SQL this looks like:
  \begin{verbatim}
    SELECT * FROM db.gene AS dbg, foo AS f 
    WHERE dbg.gene_id=f.id;
  \end{verbatim}  
  Then in R:
<<ATTACHJoin>>=
  sql <- "SELECT * FROM db.gene AS dbg, 
          foo AS f WHERE dbg.gene_id=f.id"
  res <- dbGetQuery(con, sql)
  res
@ 
\end{frame}




% -----------------------------------------------
\begin{frame}[fragile, shrink=15]{Your turn part 4}
  Now create a cross join to your database and extract the \_tx\_id 's from the gene table there using your gene IDs as a foreign key.
<<exercise4,echo=FALSE>>=
  #1st attach
  db <- system.file("extdata", "mm9KG.sqlite", 
                    package="AdvancedR")
  att <- dbGetQuery(dbcon, sprintf("ATTACH '%s' AS db",db))
  #then select
  sql <- "SELECT * FROM db.gene AS dbg, 
          genePheno AS gp WHERE dbg.gene_id=gp.id"
  res <- dbGetQuery(dbcon, sql)
@ 
\end{frame}





%-----------------------------------------------
\begin{frame}[fragile, shrink=15]{Your turn part 5}
 Now conect your cross join to the transcript table in the database and extract the fields from that table while still using your gene IDs as a foreign key.
<<exercise5,echo=FALSE>>=
  sql <- "SELECT * FROM db.gene AS dbg, 
          genePheno AS gp, transcript AS t 
          WHERE dbg.gene_id=gp.id
          AND dbg._tx_id=t._tx_id"
  res <- dbGetQuery(dbcon, sql)
@ 
\end{frame}





%TODO: some things are the wrong size on the laptop

%TODO: AFTER PRINTING: add extra copies of exercise slides (where things are verbatim so that the content is not repeated in the Stangle)
















 
 













%%%% Island of lost slides below:

%% TODO discuss dbname=:memory: (default)
%% TODO discuss RSQLites SQLite() and SQLITE_RO() modes etc.

%% TODO: discuss blobs???
% \begin{frame}[fragile]{handling blobs}
% <<code-block>>=
% @ 
% \end{frame}


%% parallel R on database access- should work now, but benefits for annotation packages are held back (I think) due to caching mechanism.  So instead use direct SQL queries.  So make some slides to discuss that:


% \begin{frame}[fragile]{An overview of the GenomicFeatures package}
% <<code-block>>=
% ## This example ONLY shows improvement for multicore the FIRST time its run...
% ## suspect there is some caching going on...
% library(multicore)
% library(org.Hs.eg.db)
% x <- list(org.Hs.egALIAS2EG, org.Hs.egCHRLOCEND)
% system.time(lapply(x, toTable))
% system.time(mclapply(x, toTable))
% @ 
% \end{frame}


%% mcapply seems to STILL be slower than lapply()...
% \begin{frame}[fragile]{TEST}
% <<code-block>>=
% library(DBI)
% library(RSQLite)
% x = list("genes", "kegg", "pfam", "chromosomes", "accessions", "go_cc_all")
% fun <- function(table){
%   con <- dbConnect(SQLite(), dbname = system.file("extdata","org.Hs.eg.sqlite",package="org.Hs.eg.db") ) 
%   res = sqliteExecStatement(con, paste("SELECT * FROM ",table,sep=""))
%   ans = fetch(res)
%   sqliteCloseResult(res)
% }
% res <- lapply(x, fun)
% head(res[[1]])
% head(res[[2]])

% system.time(lapply(x, fun))
% system.time(mclapply(x, fun))

% @ 
% \end{frame}



% \section{Resources}

% \begin{frame}[fragile]{Resources}
%   %% Include key resources -- urls, books, ...
% \end{frame}







<<cleanup,echo=FALSE>>=
unlink("myNewDb.sqlite")
unlink("myExDb.sqlite")
@ 


\end{document}
