2.TCGA-process.ipynb

{
 "cells": [
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# Convert downloaded TCGA datasets into sample × gene matrices"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 1,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "import os\n",
    "\n",
    "import pandas"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## Read sample information\n",
    "\n",
    "This file contains sample information. See the [online documentation](https://genome-cancer.soe.ucsc.edu/proj/site/xena/datapages/?dataset=TCGA.PANCAN.sampleMap/PANCAN_clinicalMatrix&host=https://tcga.xenahubs.net) for `PANCAN_clinicalMatrix`."
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 2,
   "metadata": {
    "collapsed": false
   },
   "outputs": [
    {
     "data": {
      "text/plain": [
       "(12088, 36)"
      ]
     },
     "execution_count": 2,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "path = os.path.join('download', 'PANCAN_clinicalMatrix.tsv.bz2')\n",
    "clinmat_df = (\n",
    "    pandas.read_table(path)\n",
    "    .rename(columns={'sampleID': 'sample_id'})\n",
    ")\n",
    "# Check that no sample_ids are duplicated\n",
    "assert not clinmat_df.sample_id.duplicated().any()\n",
    "clinmat_df.shape"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 3,
   "metadata": {
    "collapsed": false
   },
   "outputs": [
    {
     "data": {
      "text/plain": [
       "Primary Tumor                                      10518\n",
       "Solid Tissue Normal                                  782\n",
       "Metastatic                                           396\n",
       "Primary Blood Derived Cancer - Peripheral Blood      200\n",
       "Recurrent Tumor                                       46\n",
       "Additional - New Primary                              11\n",
       "Additional Metastatic                                  1\n",
       "Name: sample_type, dtype: int64"
      ]
     },
     "execution_count": 3,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "# Types of samples\n",
    "clinmat_df.sample_type.value_counts()"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## Read mutation data\n",
    "\n",
    "This file contains mutation data (which mutations each sample contains) See the [online documentation](https://genome-cancer.soe.ucsc.edu/proj/site/xena/datapages/?dataset=TCGA.PANCAN.sampleMap/PANCAN_mutation&host=https://tcga.xenahubs.net) for `PANCAN_mutation`."
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 4,
   "metadata": {
    "collapsed": false
   },
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>sample_id</th>\n",
       "      <th>chr</th>\n",
       "      <th>start</th>\n",
       "      <th>end</th>\n",
       "      <th>reference</th>\n",
       "      <th>alt</th>\n",
       "      <th>gene</th>\n",
       "      <th>effect</th>\n",
       "      <th>DNA_VAF</th>\n",
       "      <th>RNA_VAF</th>\n",
       "      <th>Amino_Acid_Change</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>TCGA-D8-A1J8-01</td>\n",
       "      <td>chr10</td>\n",
       "      <td>52587953</td>\n",
       "      <td>52587953</td>\n",
       "      <td>C</td>\n",
       "      <td>A</td>\n",
       "      <td>A1CF</td>\n",
       "      <td>Missense_Mutation</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>p.R236I</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>TCGA-BH-A0HP-01</td>\n",
       "      <td>chr10</td>\n",
       "      <td>52595854</td>\n",
       "      <td>52595854</td>\n",
       "      <td>G</td>\n",
       "      <td>A</td>\n",
       "      <td>A1CF</td>\n",
       "      <td>Missense_Mutation</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>p.A195V</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "         sample_id    chr     start       end reference alt  gene  \\\n",
       "0  TCGA-D8-A1J8-01  chr10  52587953  52587953         C   A  A1CF   \n",
       "1  TCGA-BH-A0HP-01  chr10  52595854  52595854         G   A  A1CF   \n",
       "\n",
       "              effect  DNA_VAF  RNA_VAF Amino_Acid_Change  \n",
       "0  Missense_Mutation      NaN      NaN           p.R236I  \n",
       "1  Missense_Mutation      NaN      NaN           p.A195V  "
      ]
     },
     "execution_count": 4,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "path = os.path.join('download', 'PANCAN_mutation.tsv.bz2')\n",
    "snp_mutation_df = (\n",
    "    pandas.read_table(path)\n",
    "    .rename(columns={'sample': 'sample_id'})\n",
    ")\n",
    "snp_mutation_df.head(2)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 5,
   "metadata": {
    "collapsed": false
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Removing 4.91% of the mutations, which are duplicates\n"
     ]
    }
   ],
   "source": [
    "# The mutation data contains duplicated rows. Disturbing issues like this are\n",
    "# common in bioinformatics. The following\n",
    "print('Removing {:.2%} of the mutations, which are duplicates'.format(snp_mutation_df.duplicated().sum() / len(snp_mutation_df)))\n",
    "snp_mutation_df.drop_duplicates(inplace=True)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 6,
   "metadata": {
    "collapsed": false
   },
   "outputs": [
    {
     "data": {
      "text/plain": [
       "8024"
      ]
     },
     "execution_count": 6,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "# Number of samples with at least one mutation\n",
    "snp_mutation_df.sample_id.nunique()"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "##### Mutations counts by type\n",
    "\n",
    "Notice that mutation type codes are not standardized. For example, `5_prime_UTR_variant`, `5'UTR`, `UTR_5_PRIME` all refer to the same type of mutation."
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 7,
   "metadata": {
    "collapsed": false,
    "scrolled": false
   },
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>index</th>\n",
       "      <th>effect</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>Missense_Mutation</td>\n",
       "      <td>1044846</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>Silent</td>\n",
       "      <td>432995</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2</th>\n",
       "      <td>Nonsense_Mutation</td>\n",
       "      <td>81092</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3</th>\n",
       "      <td>RNA</td>\n",
       "      <td>71493</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4</th>\n",
       "      <td>Frame_Shift_Del</td>\n",
       "      <td>46941</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>5</th>\n",
       "      <td>Splice_Site</td>\n",
       "      <td>43262</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>6</th>\n",
       "      <td>Frame_Shift_Ins</td>\n",
       "      <td>22546</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>7</th>\n",
       "      <td>missense_variant</td>\n",
       "      <td>20241</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>8</th>\n",
       "      <td>In_Frame_Del</td>\n",
       "      <td>11455</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>9</th>\n",
       "      <td>synonymous_variant</td>\n",
       "      <td>7907</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>10</th>\n",
       "      <td>Translation_Start_Site</td>\n",
       "      <td>3258</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>11</th>\n",
       "      <td>In_Frame_Ins</td>\n",
       "      <td>3052</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>12</th>\n",
       "      <td>stop_gained</td>\n",
       "      <td>1573</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>13</th>\n",
       "      <td>3_prime_UTR_variant</td>\n",
       "      <td>1420</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>14</th>\n",
       "      <td>Nonstop_Mutation</td>\n",
       "      <td>1318</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>15</th>\n",
       "      <td>exon_variant</td>\n",
       "      <td>945</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>16</th>\n",
       "      <td>EXON</td>\n",
       "      <td>420</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>17</th>\n",
       "      <td>5_prime_UTR_variant</td>\n",
       "      <td>395</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>18</th>\n",
       "      <td>splice_acceptor_variant</td>\n",
       "      <td>294</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>19</th>\n",
       "      <td>splice_region_variant</td>\n",
       "      <td>255</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>20</th>\n",
       "      <td>3'UTR</td>\n",
       "      <td>211</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>21</th>\n",
       "      <td>splice_donor_variant</td>\n",
       "      <td>203</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>22</th>\n",
       "      <td>Intron</td>\n",
       "      <td>148</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>23</th>\n",
       "      <td>5_prime_UTR_premature_start_codon_gain_variant</td>\n",
       "      <td>110</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>24</th>\n",
       "      <td>NON_SYNONYMOUS_CODING</td>\n",
       "      <td>95</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>25</th>\n",
       "      <td>INTRAGENIC</td>\n",
       "      <td>57</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>26</th>\n",
       "      <td>UTR_3_PRIME</td>\n",
       "      <td>38</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>27</th>\n",
       "      <td>SYNONYMOUS_CODING</td>\n",
       "      <td>36</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>28</th>\n",
       "      <td>start_lost</td>\n",
       "      <td>32</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>29</th>\n",
       "      <td>5'UTR</td>\n",
       "      <td>28</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>30</th>\n",
       "      <td>UTR_5_PRIME</td>\n",
       "      <td>22</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>31</th>\n",
       "      <td>stop_lost</td>\n",
       "      <td>19</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>32</th>\n",
       "      <td>IGR</td>\n",
       "      <td>16</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>33</th>\n",
       "      <td>stop_retained_variant</td>\n",
       "      <td>7</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>34</th>\n",
       "      <td>STOP_GAINED</td>\n",
       "      <td>6</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>35</th>\n",
       "      <td>initiator_codon_variant</td>\n",
       "      <td>2</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>36</th>\n",
       "      <td>SPLICE_SITE_ACCEPTOR</td>\n",
       "      <td>2</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>37</th>\n",
       "      <td>SYNONYMOUS_STOP</td>\n",
       "      <td>1</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>38</th>\n",
       "      <td>5'Flank</td>\n",
       "      <td>1</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "                                             index   effect\n",
       "0                                Missense_Mutation  1044846\n",
       "1                                           Silent   432995\n",
       "2                                Nonsense_Mutation    81092\n",
       "3                                              RNA    71493\n",
       "4                                  Frame_Shift_Del    46941\n",
       "5                                      Splice_Site    43262\n",
       "6                                  Frame_Shift_Ins    22546\n",
       "7                                 missense_variant    20241\n",
       "8                                     In_Frame_Del    11455\n",
       "9                               synonymous_variant     7907\n",
       "10                          Translation_Start_Site     3258\n",
       "11                                    In_Frame_Ins     3052\n",
       "12                                     stop_gained     1573\n",
       "13                             3_prime_UTR_variant     1420\n",
       "14                                Nonstop_Mutation     1318\n",
       "15                                    exon_variant      945\n",
       "16                                            EXON      420\n",
       "17                             5_prime_UTR_variant      395\n",
       "18                         splice_acceptor_variant      294\n",
       "19                           splice_region_variant      255\n",
       "20                                           3'UTR      211\n",
       "21                            splice_donor_variant      203\n",
       "22                                          Intron      148\n",
       "23  5_prime_UTR_premature_start_codon_gain_variant      110\n",
       "24                           NON_SYNONYMOUS_CODING       95\n",
       "25                                      INTRAGENIC       57\n",
       "26                                     UTR_3_PRIME       38\n",
       "27                               SYNONYMOUS_CODING       36\n",
       "28                                      start_lost       32\n",
       "29                                           5'UTR       28\n",
       "30                                     UTR_5_PRIME       22\n",
       "31                                       stop_lost       19\n",
       "32                                             IGR       16\n",
       "33                           stop_retained_variant        7\n",
       "34                                     STOP_GAINED        6\n",
       "35                         initiator_codon_variant        2\n",
       "36                            SPLICE_SITE_ACCEPTOR        2\n",
       "37                                 SYNONYMOUS_STOP        1\n",
       "38                                         5'Flank        1"
      ]
     },
     "execution_count": 7,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "snp_mutation_df.effect.value_counts().reset_index()"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### Convert SNP mutations to gene mutations"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 8,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "mutations = {\n",
    "    'Missense_Mutation',\n",
    "    'Nonsense_Mutation',\n",
    "}"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 9,
   "metadata": {
    "collapsed": false
   },
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>sample_id</th>\n",
       "      <th>gene</th>\n",
       "      <th>count</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>TCGA-02-0003-01</td>\n",
       "      <td>AKAP6</td>\n",
       "      <td>1</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>TCGA-02-0003-01</td>\n",
       "      <td>ANAPC4</td>\n",
       "      <td>1</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "         sample_id    gene  count\n",
       "0  TCGA-02-0003-01   AKAP6      1\n",
       "1  TCGA-02-0003-01  ANAPC4      1"
      ]
     },
     "execution_count": 9,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "gene_mutation_df = (snp_mutation_df\n",
    "    .query(\"effect in @mutations\")\n",
    "    .groupby(['sample_id', 'gene'])\n",
    "    .apply(len)\n",
    "    .reset_index()\n",
    "    .rename(columns={0: 'count'})\n",
    ")\n",
    "gene_mutation_df.head(2)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 10,
   "metadata": {
    "collapsed": false
   },
   "outputs": [
    {
     "data": {
      "text/plain": [
       "(7497, 21791)"
      ]
     },
     "execution_count": 10,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "# Create a sample (rows) by gene (columns) matrix of mutation status\n",
    "gene_mutation_mat_df = (gene_mutation_df\n",
    "    .pivot_table(index='sample_id', columns='gene', values='count', fill_value=0)\n",
    "    .astype(bool).astype(int)\n",
    ")\n",
    "gene_mutation_mat_df.shape"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 11,
   "metadata": {
    "collapsed": false
   },
   "outputs": [
    {
     "data": {
      "text/plain": [
       "'0.62% sample-gene pairs are mutated'"
      ]
     },
     "execution_count": 11,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "'{:.2%} sample-gene pairs are mutated'.format(\n",
    "    gene_mutation_mat_df.stack().mean())"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 12,
   "metadata": {
    "collapsed": false
   },
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>index</th>\n",
       "      <th>gene</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>TTN</td>\n",
       "      <td>2210</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>TP53</td>\n",
       "      <td>2045</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2</th>\n",
       "      <td>MUC16</td>\n",
       "      <td>1362</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3</th>\n",
       "      <td>PIK3CA</td>\n",
       "      <td>969</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4</th>\n",
       "      <td>CSMD3</td>\n",
       "      <td>859</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "    index  gene\n",
       "0     TTN  2210\n",
       "1    TP53  2045\n",
       "2   MUC16  1362\n",
       "3  PIK3CA   969\n",
       "4   CSMD3   859"
      ]
     },
     "execution_count": 12,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "# Top mutated genes\n",
    "gene_mutation_df.gene.value_counts().reset_index().head(5)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 13,
   "metadata": {
    "collapsed": false
   },
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>index</th>\n",
       "      <th>sample_id</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>TCGA-IB-7651-01</td>\n",
       "      <td>8704</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>TCGA-FW-A3R5-06</td>\n",
       "      <td>7244</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2</th>\n",
       "      <td>TCGA-AP-A0LM-01</td>\n",
       "      <td>7024</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3</th>\n",
       "      <td>TCGA-AP-A059-01</td>\n",
       "      <td>6426</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4</th>\n",
       "      <td>TCGA-B5-A0JY-01</td>\n",
       "      <td>6113</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "             index  sample_id\n",
       "0  TCGA-IB-7651-01       8704\n",
       "1  TCGA-FW-A3R5-06       7244\n",
       "2  TCGA-AP-A0LM-01       7024\n",
       "3  TCGA-AP-A059-01       6426\n",
       "4  TCGA-B5-A0JY-01       6113"
      ]
     },
     "execution_count": 13,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "# Top mutated samples\n",
    "gene_mutation_df.sample_id.value_counts().reset_index().head(5)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## Read gene expression data\n",
    "\n",
    "This file contains gene expression data from RNA-Sequencing. See the [online documentation](https://genome-cancer.soe.ucsc.edu/proj/site/xena/datapages/?dataset=TCGA.PANCAN.sampleMap/HiSeqV2&host=https://tcga.xenahubs.net) for `HiSeqV2`."
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 14,
   "metadata": {
    "collapsed": false
   },
   "outputs": [
    {
     "data": {
      "text/plain": [
       "(10459, 20501)"
      ]
     },
     "execution_count": 14,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "# Read the gene × sample dataset\n",
    "path = os.path.join('download', 'HiSeqV2.tsv.bz2')\n",
    "expr_df = pandas.read_table(path, index_col=0)\n",
    "\n",
    "# Process the dataset\n",
    "expr_df = (expr_df\n",
    "    # Remove genes containing a `?`\n",
    "    [~expr_df.index.str.contains('?', regex=False)]\n",
    "    # Sort samples\n",
    "    .sort_index()\n",
    "    # Transpose so the data is sample × gene\n",
    "    .transpose()\n",
    "    # Sort genes\n",
    "    .sort_index()\n",
    ")\n",
    "\n",
    "expr_df.index.rename('sample_id', inplace=True)\n",
    "\n",
    "expr_df.shape"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 15,
   "metadata": {
    "collapsed": false
   },
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th>Sample</th>\n",
       "      <th>A1BG</th>\n",
       "      <th>A1CF</th>\n",
       "      <th>A2BP1</th>\n",
       "      <th>A2LD1</th>\n",
       "      <th>A2M</th>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>sample_id</th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>TCGA-02-0047-01</th>\n",
       "      <td>6.9774</td>\n",
       "      <td>0.0</td>\n",
       "      <td>7.9403</td>\n",
       "      <td>5.8092</td>\n",
       "      <td>15.0538</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>TCGA-02-0055-01</th>\n",
       "      <td>8.6177</td>\n",
       "      <td>0.0</td>\n",
       "      <td>7.1122</td>\n",
       "      <td>6.4096</td>\n",
       "      <td>15.3879</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>TCGA-02-2483-01</th>\n",
       "      <td>8.0920</td>\n",
       "      <td>0.0</td>\n",
       "      <td>6.8077</td>\n",
       "      <td>5.1513</td>\n",
       "      <td>14.3622</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>TCGA-02-2485-01</th>\n",
       "      <td>6.4084</td>\n",
       "      <td>0.0</td>\n",
       "      <td>8.0120</td>\n",
       "      <td>6.9919</td>\n",
       "      <td>12.9292</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>TCGA-02-2486-01</th>\n",
       "      <td>6.7716</td>\n",
       "      <td>0.0</td>\n",
       "      <td>2.3973</td>\n",
       "      <td>7.5814</td>\n",
       "      <td>15.3224</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "Sample             A1BG  A1CF   A2BP1   A2LD1      A2M\n",
       "sample_id                                             \n",
       "TCGA-02-0047-01  6.9774   0.0  7.9403  5.8092  15.0538\n",
       "TCGA-02-0055-01  8.6177   0.0  7.1122  6.4096  15.3879\n",
       "TCGA-02-2483-01  8.0920   0.0  6.8077  5.1513  14.3622\n",
       "TCGA-02-2485-01  6.4084   0.0  8.0120  6.9919  12.9292\n",
       "TCGA-02-2486-01  6.7716   0.0  2.3973  7.5814  15.3224"
      ]
     },
     "execution_count": 15,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "# Peak at the data matrix\n",
    "expr_df.iloc[:5, :5]"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## Integrate expression and mutation data\n",
    "\n",
    "Find samples with both mutation and expression data. We assume that if a sample was not in `PANCAN_mutation`, it was not assayed for mutation. Hence, zero-mutation cancers are excluded even if they have mutation data."
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 16,
   "metadata": {
    "collapsed": false
   },
   "outputs": [
    {
     "data": {
      "text/plain": [
       "6825"
      ]
     },
     "execution_count": 16,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "sample_ids = list(gene_mutation_mat_df.index & expr_df.index)\n",
    "len(sample_ids)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 17,
   "metadata": {
    "collapsed": false
   },
   "outputs": [],
   "source": [
    "# Filter expression (x) and mutation (y) matrices for common samples\n",
    "x_df = expr_df.loc[sample_ids, :]\n",
    "y_df = gene_mutation_mat_df.loc[sample_ids, :]"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### Export matrices to TSVs\n",
    "\n",
    "Matrices are saved as sample × gene TSVs. Subsetted matrices are also exported to allow users to quickly explore small portions of the dataset."
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 18,
   "metadata": {
    "collapsed": false
   },
   "outputs": [],
   "source": [
    "def sample_df(df, nrows=None, ncols=None, row_seed=0, col_seed=0):\n",
    "    \"\"\"Randomly subset a dataframe, preserving row and column order.\"\"\"\n",
    "    if nrows is None:\n",
    "        nrows = len(df)\n",
    "    if ncols is None:\n",
    "        ncols = len(df.columns)\n",
    "    return (df\n",
    "        .sample(n=nrows, random_state=row_seed, axis='rows')\n",
    "        .sample(n=ncols, random_state=col_seed, axis='columns')\n",
    "        .sort_index(axis='rows')\n",
    "        .sort_index(axis='columns')\n",
    "    )"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 19,
   "metadata": {
    "collapsed": false
   },
   "outputs": [],
   "source": [
    "tsv_args = {'sep': '\\t', 'float_format': '%.3g'}\n",
    "\n",
    "for df, name in (x_df, 'expression-matrix'), (y_df, 'mutation-matrix'):\n",
    "\n",
    "    # Save full dataset\n",
    "    path = os.path.join('data', name + '.tsv.bz2')\n",
    "    df.to_csv(path, **tsv_args, compression='bz2')\n",
    "    \n",
    "    # Save subsetted datasets\n",
    "    for sample, nrows, ncols in ('small', 50, 15), ('all-samples', None, 15), ('all-genes', 50, None):\n",
    "        path = os.path.join('data', 'subset', '{}-{}.tsv'.format(name, sample))\n",
    "        sample_df(df, nrows=nrows, ncols=ncols).to_csv(path, **tsv_args)"
   ]
  }
 ],
 "metadata": {
  "anaconda-cloud": {},
  "kernelspec": {
   "display_name": "Python 3",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.5.2"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 0
}