13_recount3_near_ss_with_better_score_feature.ipynb

{
 "cells": [
  {
   "cell_type": "code",
   "execution_count": 1,
   "metadata": {
    "collapsed": true,
    "is_executing": true
   },
   "outputs": [],
   "source": [
    "import pandas as pd\n",
    "from pandarallel import pandarallel"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 2,
   "outputs": [
    {
     "data": {
      "text/plain": "         chr     start       end strand  recount3_score\n0       chr1     12227     12612      +           59151\n1       chr1     12721     13220      +           61021\n2       chr1     12057     12178      +               0\n3       chr1     12697     12974      +              21\n4       chr1     13052     13220      +             526\n...      ...       ...       ...    ...             ...\n519029  chrY  25464577  25465486      +               0\n519030  chrY  25513173  25513588      -           31374\n519031  chrY  25513745  25516715      -              67\n519032  chrY  25525288  25527646      -           18958\n519033  chrY  57213357  57213855      -               0\n\n[519034 rows x 5 columns]",
      "text/html": "<div>\n<style scoped>\n    .dataframe tbody tr th:only-of-type {\n        vertical-align: middle;\n    }\n\n    .dataframe tbody tr th {\n        vertical-align: top;\n    }\n\n    .dataframe thead th {\n        text-align: right;\n    }\n</style>\n<table border=\"1\" class=\"dataframe\">\n  <thead>\n    <tr style=\"text-align: right;\">\n      <th></th>\n      <th>chr</th>\n      <th>start</th>\n      <th>end</th>\n      <th>strand</th>\n      <th>recount3_score</th>\n    </tr>\n  </thead>\n  <tbody>\n    <tr>\n      <th>0</th>\n      <td>chr1</td>\n      <td>12227</td>\n      <td>12612</td>\n      <td>+</td>\n      <td>59151</td>\n    </tr>\n    <tr>\n      <th>1</th>\n      <td>chr1</td>\n      <td>12721</td>\n      <td>13220</td>\n      <td>+</td>\n      <td>61021</td>\n    </tr>\n    <tr>\n      <th>2</th>\n      <td>chr1</td>\n      <td>12057</td>\n      <td>12178</td>\n      <td>+</td>\n      <td>0</td>\n    </tr>\n    <tr>\n      <th>3</th>\n      <td>chr1</td>\n      <td>12697</td>\n      <td>12974</td>\n      <td>+</td>\n      <td>21</td>\n    </tr>\n    <tr>\n      <th>4</th>\n      <td>chr1</td>\n      <td>13052</td>\n      <td>13220</td>\n      <td>+</td>\n      <td>526</td>\n    </tr>\n    <tr>\n      <th>...</th>\n      <td>...</td>\n      <td>...</td>\n      <td>...</td>\n      <td>...</td>\n      <td>...</td>\n    </tr>\n    <tr>\n      <th>519029</th>\n      <td>chrY</td>\n      <td>25464577</td>\n      <td>25465486</td>\n      <td>+</td>\n      <td>0</td>\n    </tr>\n    <tr>\n      <th>519030</th>\n      <td>chrY</td>\n      <td>25513173</td>\n      <td>25513588</td>\n      <td>-</td>\n      <td>31374</td>\n    </tr>\n    <tr>\n      <th>519031</th>\n      <td>chrY</td>\n      <td>25513745</td>\n      <td>25516715</td>\n      <td>-</td>\n      <td>67</td>\n    </tr>\n    <tr>\n      <th>519032</th>\n      <td>chrY</td>\n      <td>25525288</td>\n      <td>25527646</td>\n      <td>-</td>\n      <td>18958</td>\n    </tr>\n    <tr>\n      <th>519033</th>\n      <td>chrY</td>\n      <td>57213357</td>\n      <td>57213855</td>\n      <td>-</td>\n      <td>0</td>\n    </tr>\n  </tbody>\n</table>\n<p>519034 rows × 5 columns</p>\n</div>"
     },
     "execution_count": 2,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "df = pd.read_csv(\"data/05_trainset_recount3_feature.tsv\", sep=\"\\t\", usecols=[\"chr\",\"start\",\"end\",\"strand\",\"recount3_score\"])\n",
    "df"
   ],
   "metadata": {
    "collapsed": false
   }
  },
  {
   "cell_type": "code",
   "execution_count": 3,
   "outputs": [],
   "source": [
    "gap = 5"
   ],
   "metadata": {
    "collapsed": false
   }
  },
  {
   "cell_type": "markdown",
   "source": [
    "# Start splice site (5\")"
   ],
   "metadata": {
    "collapsed": false
   }
  },
  {
   "cell_type": "code",
   "execution_count": 4,
   "outputs": [],
   "source": [
    "def better_rc3_match_start_ss(row):\n",
    "    potential_matches = !tabix data/resources/recount3.bed.gz {row.chr}:{row.start-gap}-{row.start+gap} | cut -f 2,5\n",
    "\n",
    "    potential_matches = [\n",
    "        (int(entry.split('\\t')[0]), int(entry.split('\\t')[1]))\n",
    "        for entry in potential_matches\n",
    "        if row['start'] - 5 <= int(entry.split('\\t')[0]) <= row['start'] + 5 and int(entry.split('\\t')[1]) > row[\"recount3_score\"]\n",
    "    ]\n",
    "    if potential_matches:\n",
    "        return True\n",
    "    return False"
   ],
   "metadata": {
    "collapsed": false
   }
  },
  {
   "cell_type": "code",
   "execution_count": 5,
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "INFO: Pandarallel will run on 12 workers.\n",
      "INFO: Pandarallel will use Memory file system to transfer data between the main process and workers.\n"
     ]
    }
   ],
   "source": [
    "pandarallel.initialize(nb_workers=12)"
   ],
   "metadata": {
    "collapsed": false
   }
  },
  {
   "cell_type": "code",
   "execution_count": 6,
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "CPU times: user 0 ns, sys: 58 ms, total: 58 ms\n",
      "Wall time: 6.67 s\n"
     ]
    },
    {
     "data": {
      "text/plain": "       chr     start       end strand  recount3_score  better_match_5\n0     chr1     12227     12612      +           59151           False\n1     chr1     12721     13220      +           61021            True\n2     chr1     12057     12178      +               0           False\n3     chr1     12697     12974      +              21            True\n4     chr1     13052     13220      +             526           False\n...    ...       ...       ...    ...             ...             ...\n9995  chr1  41848445  41848870      -          881344           False\n9996  chr1  41849089  41918412      -          188405           False\n9997  chr1  41918524  42035806      -           96929           False\n9998  chr1  41849089  41864577      -           61385            True\n9999  chr1  41864648  41865547      -           17770           False\n\n[10000 rows x 6 columns]",
      "text/html": "<div>\n<style scoped>\n    .dataframe tbody tr th:only-of-type {\n        vertical-align: middle;\n    }\n\n    .dataframe tbody tr th {\n        vertical-align: top;\n    }\n\n    .dataframe thead th {\n        text-align: right;\n    }\n</style>\n<table border=\"1\" class=\"dataframe\">\n  <thead>\n    <tr style=\"text-align: right;\">\n      <th></th>\n      <th>chr</th>\n      <th>start</th>\n      <th>end</th>\n      <th>strand</th>\n      <th>recount3_score</th>\n      <th>better_match_5</th>\n    </tr>\n  </thead>\n  <tbody>\n    <tr>\n      <th>0</th>\n      <td>chr1</td>\n      <td>12227</td>\n      <td>12612</td>\n      <td>+</td>\n      <td>59151</td>\n      <td>False</td>\n    </tr>\n    <tr>\n      <th>1</th>\n      <td>chr1</td>\n      <td>12721</td>\n      <td>13220</td>\n      <td>+</td>\n      <td>61021</td>\n      <td>True</td>\n    </tr>\n    <tr>\n      <th>2</th>\n      <td>chr1</td>\n      <td>12057</td>\n      <td>12178</td>\n      <td>+</td>\n      <td>0</td>\n      <td>False</td>\n    </tr>\n    <tr>\n      <th>3</th>\n      <td>chr1</td>\n      <td>12697</td>\n      <td>12974</td>\n      <td>+</td>\n      <td>21</td>\n      <td>True</td>\n    </tr>\n    <tr>\n      <th>4</th>\n      <td>chr1</td>\n      <td>13052</td>\n      <td>13220</td>\n      <td>+</td>\n      <td>526</td>\n      <td>False</td>\n    </tr>\n    <tr>\n      <th>...</th>\n      <td>...</td>\n      <td>...</td>\n      <td>...</td>\n      <td>...</td>\n      <td>...</td>\n      <td>...</td>\n    </tr>\n    <tr>\n      <th>9995</th>\n      <td>chr1</td>\n      <td>41848445</td>\n      <td>41848870</td>\n      <td>-</td>\n      <td>881344</td>\n      <td>False</td>\n    </tr>\n    <tr>\n      <th>9996</th>\n      <td>chr1</td>\n      <td>41849089</td>\n      <td>41918412</td>\n      <td>-</td>\n      <td>188405</td>\n      <td>False</td>\n    </tr>\n    <tr>\n      <th>9997</th>\n      <td>chr1</td>\n      <td>41918524</td>\n      <td>42035806</td>\n      <td>-</td>\n      <td>96929</td>\n      <td>False</td>\n    </tr>\n    <tr>\n      <th>9998</th>\n      <td>chr1</td>\n      <td>41849089</td>\n      <td>41864577</td>\n      <td>-</td>\n      <td>61385</td>\n      <td>True</td>\n    </tr>\n    <tr>\n      <th>9999</th>\n      <td>chr1</td>\n      <td>41864648</td>\n      <td>41865547</td>\n      <td>-</td>\n      <td>17770</td>\n      <td>False</td>\n    </tr>\n  </tbody>\n</table>\n<p>10000 rows × 6 columns</p>\n</div>"
     },
     "execution_count": 6,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "%%time\n",
    "df_test = df.copy().head(10000)\n",
    "df_test[\"better_match_5\"] = df_test.parallel_apply(better_rc3_match_start_ss, axis=1)\n",
    "df_test"
   ],
   "metadata": {
    "collapsed": false
   }
  },
  {
   "cell_type": "code",
   "execution_count": 7,
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "CPU times: user 61.3 ms, sys: 42.8 ms, total: 104 ms\n",
      "Wall time: 5min 14s\n"
     ]
    },
    {
     "data": {
      "text/plain": "         chr     start       end strand  recount3_score  \\\n0       chr1     12227     12612      +           59151   \n1       chr1     12721     13220      +           61021   \n2       chr1     12057     12178      +               0   \n3       chr1     12697     12974      +              21   \n4       chr1     13052     13220      +             526   \n...      ...       ...       ...    ...             ...   \n519029  chrY  25464577  25465486      +               0   \n519030  chrY  25513173  25513588      -           31374   \n519031  chrY  25513745  25516715      -              67   \n519032  chrY  25525288  25527646      -           18958   \n519033  chrY  57213357  57213855      -               0   \n\n        recount3_near_start_ss_with_better_score  \n0                                          False  \n1                                           True  \n2                                          False  \n3                                           True  \n4                                          False  \n...                                          ...  \n519029                                     False  \n519030                                     False  \n519031                                      True  \n519032                                     False  \n519033                                     False  \n\n[519034 rows x 6 columns]",
      "text/html": "<div>\n<style scoped>\n    .dataframe tbody tr th:only-of-type {\n        vertical-align: middle;\n    }\n\n    .dataframe tbody tr th {\n        vertical-align: top;\n    }\n\n    .dataframe thead th {\n        text-align: right;\n    }\n</style>\n<table border=\"1\" class=\"dataframe\">\n  <thead>\n    <tr style=\"text-align: right;\">\n      <th></th>\n      <th>chr</th>\n      <th>start</th>\n      <th>end</th>\n      <th>strand</th>\n      <th>recount3_score</th>\n      <th>recount3_near_start_ss_with_better_score</th>\n    </tr>\n  </thead>\n  <tbody>\n    <tr>\n      <th>0</th>\n      <td>chr1</td>\n      <td>12227</td>\n      <td>12612</td>\n      <td>+</td>\n      <td>59151</td>\n      <td>False</td>\n    </tr>\n    <tr>\n      <th>1</th>\n      <td>chr1</td>\n      <td>12721</td>\n      <td>13220</td>\n      <td>+</td>\n      <td>61021</td>\n      <td>True</td>\n    </tr>\n    <tr>\n      <th>2</th>\n      <td>chr1</td>\n      <td>12057</td>\n      <td>12178</td>\n      <td>+</td>\n      <td>0</td>\n      <td>False</td>\n    </tr>\n    <tr>\n      <th>3</th>\n      <td>chr1</td>\n      <td>12697</td>\n      <td>12974</td>\n      <td>+</td>\n      <td>21</td>\n      <td>True</td>\n    </tr>\n    <tr>\n      <th>4</th>\n      <td>chr1</td>\n      <td>13052</td>\n      <td>13220</td>\n      <td>+</td>\n      <td>526</td>\n      <td>False</td>\n    </tr>\n    <tr>\n      <th>...</th>\n      <td>...</td>\n      <td>...</td>\n      <td>...</td>\n      <td>...</td>\n      <td>...</td>\n      <td>...</td>\n    </tr>\n    <tr>\n      <th>519029</th>\n      <td>chrY</td>\n      <td>25464577</td>\n      <td>25465486</td>\n      <td>+</td>\n      <td>0</td>\n      <td>False</td>\n    </tr>\n    <tr>\n      <th>519030</th>\n      <td>chrY</td>\n      <td>25513173</td>\n      <td>25513588</td>\n      <td>-</td>\n      <td>31374</td>\n      <td>False</td>\n    </tr>\n    <tr>\n      <th>519031</th>\n      <td>chrY</td>\n      <td>25513745</td>\n      <td>25516715</td>\n      <td>-</td>\n      <td>67</td>\n      <td>True</td>\n    </tr>\n    <tr>\n      <th>519032</th>\n      <td>chrY</td>\n      <td>25525288</td>\n      <td>25527646</td>\n      <td>-</td>\n      <td>18958</td>\n      <td>False</td>\n    </tr>\n    <tr>\n      <th>519033</th>\n      <td>chrY</td>\n      <td>57213357</td>\n      <td>57213855</td>\n      <td>-</td>\n      <td>0</td>\n      <td>False</td>\n    </tr>\n  </tbody>\n</table>\n<p>519034 rows × 6 columns</p>\n</div>"
     },
     "execution_count": 7,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "%%time\n",
    "df[\"recount3_near_start_ss_with_better_score\"] = df.parallel_apply(better_rc3_match_start_ss, axis=1)\n",
    "df"
   ],
   "metadata": {
    "collapsed": false
   }
  },
  {
   "cell_type": "markdown",
   "source": [
    "# End splice site (3\")"
   ],
   "metadata": {
    "collapsed": false
   }
  },
  {
   "cell_type": "code",
   "execution_count": 8,
   "outputs": [],
   "source": [
    "def better_rc3_match_end_ss(row):\n",
    "    potential_matches = !tabix data/resources/recount3.bed.gz {row.chr}:{row.end-gap}-{row.end+gap} | cut -f 2,5\n",
    "\n",
    "\n",
    "    potential_matches = [\n",
    "        (int(entry.split('\\t')[0]), int(entry.split('\\t')[1]))\n",
    "        for entry in potential_matches\n",
    "        if row['end'] - 5 <= int(entry.split('\\t')[0]) <= row['end'] + 5 and int(entry.split('\\t')[1]) > row[\"recount3_score\"]\n",
    "    ]\n",
    "    if potential_matches:\n",
    "        return True\n",
    "    return False"
   ],
   "metadata": {
    "collapsed": false
   }
  },
  {
   "cell_type": "code",
   "execution_count": 9,
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "CPU times: user 20 ms, sys: 49.4 ms, total: 69.4 ms\n",
      "Wall time: 5min 11s\n"
     ]
    },
    {
     "data": {
      "text/plain": "         chr     start       end strand  recount3_score  \\\n0       chr1     12227     12612      +           59151   \n1       chr1     12721     13220      +           61021   \n2       chr1     12057     12178      +               0   \n3       chr1     12697     12974      +              21   \n4       chr1     13052     13220      +             526   \n...      ...       ...       ...    ...             ...   \n519029  chrY  25464577  25465486      +               0   \n519030  chrY  25513173  25513588      -           31374   \n519031  chrY  25513745  25516715      -              67   \n519032  chrY  25525288  25527646      -           18958   \n519033  chrY  57213357  57213855      -               0   \n\n        recount3_near_start_ss_with_better_score  \\\n0                                          False   \n1                                           True   \n2                                          False   \n3                                           True   \n4                                          False   \n...                                          ...   \n519029                                     False   \n519030                                     False   \n519031                                      True   \n519032                                     False   \n519033                                     False   \n\n        recount3_near_end_ss_with_better_score  \n0                                        False  \n1                                        False  \n2                                        False  \n3                                        False  \n4                                        False  \n...                                        ...  \n519029                                   False  \n519030                                   False  \n519031                                   False  \n519032                                   False  \n519033                                   False  \n\n[519034 rows x 7 columns]",
      "text/html": "<div>\n<style scoped>\n    .dataframe tbody tr th:only-of-type {\n        vertical-align: middle;\n    }\n\n    .dataframe tbody tr th {\n        vertical-align: top;\n    }\n\n    .dataframe thead th {\n        text-align: right;\n    }\n</style>\n<table border=\"1\" class=\"dataframe\">\n  <thead>\n    <tr style=\"text-align: right;\">\n      <th></th>\n      <th>chr</th>\n      <th>start</th>\n      <th>end</th>\n      <th>strand</th>\n      <th>recount3_score</th>\n      <th>recount3_near_start_ss_with_better_score</th>\n      <th>recount3_near_end_ss_with_better_score</th>\n    </tr>\n  </thead>\n  <tbody>\n    <tr>\n      <th>0</th>\n      <td>chr1</td>\n      <td>12227</td>\n      <td>12612</td>\n      <td>+</td>\n      <td>59151</td>\n      <td>False</td>\n      <td>False</td>\n    </tr>\n    <tr>\n      <th>1</th>\n      <td>chr1</td>\n      <td>12721</td>\n      <td>13220</td>\n      <td>+</td>\n      <td>61021</td>\n      <td>True</td>\n      <td>False</td>\n    </tr>\n    <tr>\n      <th>2</th>\n      <td>chr1</td>\n      <td>12057</td>\n      <td>12178</td>\n      <td>+</td>\n      <td>0</td>\n      <td>False</td>\n      <td>False</td>\n    </tr>\n    <tr>\n      <th>3</th>\n      <td>chr1</td>\n      <td>12697</td>\n      <td>12974</td>\n      <td>+</td>\n      <td>21</td>\n      <td>True</td>\n      <td>False</td>\n    </tr>\n    <tr>\n      <th>4</th>\n      <td>chr1</td>\n      <td>13052</td>\n      <td>13220</td>\n      <td>+</td>\n      <td>526</td>\n      <td>False</td>\n      <td>False</td>\n    </tr>\n    <tr>\n      <th>...</th>\n      <td>...</td>\n      <td>...</td>\n      <td>...</td>\n      <td>...</td>\n      <td>...</td>\n      <td>...</td>\n      <td>...</td>\n    </tr>\n    <tr>\n      <th>519029</th>\n      <td>chrY</td>\n      <td>25464577</td>\n      <td>25465486</td>\n      <td>+</td>\n      <td>0</td>\n      <td>False</td>\n      <td>False</td>\n    </tr>\n    <tr>\n      <th>519030</th>\n      <td>chrY</td>\n      <td>25513173</td>\n      <td>25513588</td>\n      <td>-</td>\n      <td>31374</td>\n      <td>False</td>\n      <td>False</td>\n    </tr>\n    <tr>\n      <th>519031</th>\n      <td>chrY</td>\n      <td>25513745</td>\n      <td>25516715</td>\n      <td>-</td>\n      <td>67</td>\n      <td>True</td>\n      <td>False</td>\n    </tr>\n    <tr>\n      <th>519032</th>\n      <td>chrY</td>\n      <td>25525288</td>\n      <td>25527646</td>\n      <td>-</td>\n      <td>18958</td>\n      <td>False</td>\n      <td>False</td>\n    </tr>\n    <tr>\n      <th>519033</th>\n      <td>chrY</td>\n      <td>57213357</td>\n      <td>57213855</td>\n      <td>-</td>\n      <td>0</td>\n      <td>False</td>\n      <td>False</td>\n    </tr>\n  </tbody>\n</table>\n<p>519034 rows × 7 columns</p>\n</div>"
     },
     "execution_count": 9,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "%%time\n",
    "df[\"recount3_near_end_ss_with_better_score\"] = df.parallel_apply(better_rc3_match_end_ss, axis=1)\n",
    "df"
   ],
   "metadata": {
    "collapsed": false
   }
  },
  {
   "cell_type": "code",
   "execution_count": 11,
   "outputs": [],
   "source": [
    "# output\n",
    "df.to_csv(\"data/13_recount3_near_ss_with_better_score_feature.tsv\", sep=\"\\t\", index=False)"
   ],
   "metadata": {
    "collapsed": false
   }
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "outputs": [],
   "source": [],
   "metadata": {
    "collapsed": false
   }
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 3",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 2
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython2",
   "version": "2.7.6"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 0
}