001_dude_vertical.ipynb

{
 "cells": [
  {
   "cell_type": "code",
   "execution_count": 1,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "import numpy as np"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 2,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "import pandas as pd"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 3,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "from sklearn.cross_validation import KFold, StratifiedKFold"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 4,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "# from scipy.stats import pearsonr"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 5,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "import oddt\n",
    "# from oddt.scoring.models.regressors import randomforest\n",
    "# from oddt.metrics import enrichment_factor,roc_auc, roc_log_auc\n",
    "# from sklearn.metrics import accuracy_score, precision_score, mean_squared_error, recall_score"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 6,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "from cluster_helper.cluster import cluster_view"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 7,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "dude_ids = ['aa2ar', 'abl1', 'ace', 'aces', 'ada', 'ada17', 'adrb1', 'adrb2', 'akt1', 'akt2', 'aldr', 'ampc', 'andr', 'aofb', 'bace1', 'braf', 'cah2', 'casp3', 'cdk2', 'comt', 'cp2c9', 'cp3a4', 'csf1r', 'cxcr4', 'def', 'dhi1', 'dpp4', 'drd3', 'dyr', 'egfr', 'esr1', 'esr2', 'fa10', 'fa7', 'fabp4', 'fak1', 'fgfr1', 'fkb1a', 'fnta', 'fpps', 'gcr', 'glcm', 'gria2', 'grik1', 'hdac2', 'hdac8', 'hivint', 'hivpr', 'hivrt', 'hmdh', 'hs90a', 'hxk4', 'igf1r', 'inha', 'ital', 'jak2', 'kif11', 'kit', 'kith', 'kpcb', 'lck', 'lkha4', 'mapk2', 'mcr', 'met', 'mk01', 'mk10', 'mk14', 'mmp13', 'mp2k1', 'nos1', 'nram', 'pa2ga', 'parp1', 'pde5a', 'pgh1', 'pgh2', 'plk1', 'pnph', 'ppara', 'ppard', 'pparg', 'prgr', 'ptn1', 'pur2', 'pygm', 'pyrd', 'reni', 'rock1', 'rxra', 'sahh', 'src', 'tgfr1', 'thb', 'thrb', 'try1', 'tryb1', 'tysy', 'urok', 'vgfr2', 'wee1', 'xiap']\n",
    "#dude_ids = ['lck', 'src', 'ada17', 'hivpr', 'mk14', 'mmp13', 'aa2ar', 'bace1', 'pparg', 'parp1', 'ace', 'thrb', 'cdk2', 'esr1', 'esr2', 'vgfr2', 'fnta', 'drd3', 'csf1r', 'dhi1', 'casp3', 'gria2', 'kit', 'dyr', 'braf', 'tryb1', 'hdac8', 'aldr', 'akt1', 'ital', 'kpcb', 'tysy', 'ppard', 'hivint', 'ppara', 'urok', 'wee1', 'reni', 'grik1', 'aces', 'fa10', 'dpp4', 'adrb2', 'jak2', 'hivrt', 'fkb1a', 'cah2', 'kif11', 'try1', 'adrb1', 'akt2', 'rock1', 'pa2ga', 'pygm', 'mapk2', 'fa7', 'tgfr1', 'mk10', 'fak1', 'gcr', 'hdac2', 'prgr', 'ptn1', 'nram', 'abl1', 'hs90a', 'egfr', 'hxk4', 'mk01', 'cxcr4', 'lkha4', 'ada', 'pur2', 'pnph', 'andr', 'rxra', 'fpps', 'cp3a4', 'met', 'ampc', 'mp2k1', 'pyrd', 'pgh1', 'kith', 'thb', 'comt', 'cp2c9', 'aofb', 'fabp4', 'mcr', 'inha', 'pgh2', 'def', 'xiap', 'glcm', 'pde5a', 'nos1', 'sahh', 'hmdh', 'igf1r', 'plk1', 'fgfr1']"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 16,
   "metadata": {
    "collapsed": false,
    "scrolled": false
   },
   "outputs": [],
   "source": [
    "def dude_vertical(cv_train, cv_test, v = 1, engine = 'dude'):\n",
    "\n",
    "    import numpy as np\n",
    "    import pandas as pd\n",
    "    \n",
    "    import oddt\n",
    "    from oddt.scoring.models.regressors import randomforest\n",
    "    from oddt.metrics import enrichment_factor,roc_auc, roc_log_auc\n",
    "    from sklearn.metrics import accuracy_score, precision_score, mean_squared_error, recall_score\n",
    "    from sklearn.utils import compute_sample_weight\n",
    "    from scipy.stats import pearsonr, spearmanr, kendalltau\n",
    "    \n",
    "    dude_ids = ['aa2ar', 'abl1', 'ace', 'aces', 'ada', 'ada17', 'adrb1', 'adrb2', 'akt1', 'akt2', 'aldr', 'ampc', 'andr', 'aofb', 'bace1', 'braf', 'cah2', 'casp3', 'cdk2', 'comt', 'cp2c9', 'cp3a4', 'csf1r', 'cxcr4', 'def', 'dhi1', 'dpp4', 'drd3', 'dyr', 'egfr', 'esr1', 'esr2', 'fa10', 'fa7', 'fabp4', 'fak1', 'fgfr1', 'fkb1a', 'fnta', 'fpps', 'gcr', 'glcm', 'gria2', 'grik1', 'hdac2', 'hdac8', 'hivint', 'hivpr', 'hivrt', 'hmdh', 'hs90a', 'hxk4', 'igf1r', 'inha', 'ital', 'jak2', 'kif11', 'kit', 'kith', 'kpcb', 'lck', 'lkha4', 'mapk2', 'mcr', 'met', 'mk01', 'mk10', 'mk14', 'mmp13', 'mp2k1', 'nos1', 'nram', 'pa2ga', 'parp1', 'pde5a', 'pgh1', 'pgh2', 'plk1', 'pnph', 'ppara', 'ppard', 'pparg', 'prgr', 'ptn1', 'pur2', 'pygm', 'pyrd', 'reni', 'rock1', 'rxra', 'sahh', 'src', 'tgfr1', 'thb', 'thrb', 'try1', 'tryb1', 'tysy', 'urok', 'vgfr2', 'wee1', 'xiap']\n",
    "    np_dude_ids = np.array(dude_ids)\n",
    "    \n",
    "    mtry=None\n",
    "    if v == 2:\n",
    "        mtry = 50\n",
    "    else:\n",
    "        mtry = 14\n",
    "    \n",
    "    oddt.random_seed(0)\n",
    "    # Random Forest\n",
    "    rf = randomforest(n_estimators=500, n_jobs=-1, verbose=1,\n",
    "                        max_features=mtry, \n",
    "                        oob_score=True,\n",
    "                        bootstrap = True,\n",
    "                        random_state = 0,\n",
    "                       )\n",
    "    \n",
    "    if v == 1:\n",
    "        col_range = range(1,37)\n",
    "        np_type = np.uint16\n",
    "    elif v == 2:\n",
    "        col_range = range(1,217)\n",
    "        np_type = np.uint16\n",
    "    elif v == 3:\n",
    "        col_range = range(1,43)   \n",
    "        np_type = np.float16\n",
    "    \n",
    "    act_cutoff = 6.\n",
    "    inactive_cutoff = 5.95\n",
    "    \n",
    "    # TRAIN\n",
    "    train = []\n",
    "    for dude_id in np_dude_ids[cv_train]:\n",
    "        # trap errors on reading\n",
    "        try:\n",
    "            actives_full = pd.read_csv('head1_full/%s/%s/%s_desc_v%i.csv.gz' % (dude_id, engine, 'actives', v), dtype={i: np_type for i in col_range})\n",
    "            decoys_full = pd.read_csv('head1_full/%s/%s/%s_desc_v%i.csv.gz' % (dude_id, engine, 'decoys', v), dtype={i: np_type for i in col_range})\n",
    "        except:\n",
    "            continue\n",
    "\n",
    "        #trap empty ones\n",
    "        if len(actives_full) == 0 or len(decoys_full) == 0:\n",
    "            continue\n",
    "\n",
    "        decoys_full['act'] = inactive_cutoff if act_cutoff > 10 else 10**(9-inactive_cutoff)\n",
    "\n",
    "        # generate one big table for dude_id\n",
    "        data = pd.concat((actives_full, decoys_full))\n",
    "\n",
    "        # normalize\n",
    "        if act_cutoff >= 10:\n",
    "            data['act'] = np.clip(data['act'], 1e-9, inactive_cutoff)\n",
    "        else:\n",
    "            data['act'] = np.clip(-np.log10(np.clip(data['act'], 1e-9, 1e9) * 1e-9), 0 , 15)\n",
    "        # binary\n",
    "        data['act_bin'] = data['act'] < act_cutoff if act_cutoff > 10 else data['act'] > act_cutoff# 1e-6 # close to zero\n",
    "        \n",
    "        if len(train) == 0:\n",
    "            train = data\n",
    "        else:\n",
    "            train = pd.concat((train, data))\n",
    "    \n",
    "    oddt.random_seed(0)\n",
    "    rf.fit(train[col_range], train['act'], sample_weight=compute_sample_weight('balanced', train['act_bin']))\n",
    "    \n",
    "    test = []\n",
    "    for dude_id in np_dude_ids[cv_test]:\n",
    "        # trap errors on reading\n",
    "        try:\n",
    "            actives_full = pd.read_csv('head1_full/%s/%s/%s_desc_v%i.csv.gz' % (dude_id, engine, 'actives', v), dtype={i: np_type for i in col_range})\n",
    "            decoys_full = pd.read_csv('head1_full/%s/%s/%s_desc_v%i.csv.gz' % (dude_id, engine, 'decoys', v), dtype={i: np_type for i in col_range})\n",
    "        except:\n",
    "            continue\n",
    "\n",
    "        #trap empty ones\n",
    "        if len(actives_full) == 0 or len(decoys_full) == 0:\n",
    "            continue\n",
    "\n",
    "        decoys_full['act'] = inactive_cutoff if act_cutoff > 10 else 10**(9-inactive_cutoff)\n",
    "\n",
    "        # generate one big table for dude_id\n",
    "        data = pd.concat((actives_full, decoys_full))\n",
    "\n",
    "        # normalize\n",
    "        if act_cutoff >= 10:\n",
    "            data['act'] = np.clip(data['act'], 1e-9, inactive_cutoff)\n",
    "        else:\n",
    "            data['act'] = np.clip(-np.log10(np.clip(data['act'], 1e-9, 1e9) * 1e-9), 0 , 15)\n",
    "        # binary\n",
    "        data['act_bin'] = data['act'] < act_cutoff if act_cutoff > 10 else data['act'] > act_cutoff# 1e-6 # close to zero\n",
    "        \n",
    "        if len(test) == 0:\n",
    "            test = data\n",
    "        else:\n",
    "            test = pd.concat((test, data))\n",
    "            \n",
    "    test['pred'] = rf.predict(test[col_range])\n",
    "    test['pred_bin'] = test['pred'] < act_cutoff if act_cutoff > 10 else test['pred'] > act_cutoff# 1e-6 # close to zero\n",
    "    \n",
    "    test_df = test.sort_values('pred', ascending=(act_cutoff >= 10))\n",
    "    \n",
    "    test_df['v'] = v\n",
    "    test_df['engine'] = engine\n",
    "    \n",
    "    return {\n",
    "            'v': v,\n",
    "            'engine': engine,\n",
    "            'roc_auc': roc_auc(test_df['act_bin'], test_df['pred_bin'], ascending_score=False),# Binary is descending\n",
    "            'ef1_perc': enrichment_factor(test_df['act_bin'], test_df['pred_bin'], kind='percentage'),\n",
    "            'ef0.1': enrichment_factor(test_df['act_bin'], test_df['pred_bin'], percentage=0.1),\n",
    "            'ef1': enrichment_factor(test_df['act_bin'], test_df['pred_bin']),\n",
    "            'ef2': enrichment_factor(test_df['act_bin'], test_df['pred_bin'], percentage=2),\n",
    "            'ef5': enrichment_factor(test_df['act_bin'], test_df['pred_bin'], percentage=5),\n",
    "            'ef10': enrichment_factor(test_df['act_bin'], test_df['pred_bin'], percentage=10),\n",
    "        \n",
    "        \n",
    "            'rp': pearsonr(test_df['act'], test_df['pred'])[0],\n",
    "            'rs': spearmanr(test_df['act'], test_df['pred'])[0],\n",
    "            'rk': kendalltau(test_df['act'], test_df['pred'])[0],\n",
    "        \n",
    "            'rp_active': pearsonr(test_df['act'][test_df['act_bin']], test_df['pred'][test_df['act_bin']])[0],\n",
    "            'rs_active': spearmanr(test_df['act'][test_df['act_bin']], test_df['pred'][test_df['act_bin']])[0],\n",
    "            'rk_active': kendalltau(test_df['act'][test_df['act_bin']], test_df['pred'][test_df['act_bin']])[0],\n",
    "        \n",
    "            'rp_inactive': pearsonr(test_df['act'][~test_df['act_bin']], test_df['pred'][~test_df['act_bin']])[0],\n",
    "            'rs_inactive': spearmanr(test_df['act'][~test_df['act_bin']], test_df['pred'][~test_df['act_bin']])[0],\n",
    "            'rk_inactive': kendalltau(test_df['act'][~test_df['act_bin']], test_df['pred'][~test_df['act_bin']])[0],\n",
    "        \n",
    "            'mse': mean_squared_error(test_df['act'], test_df['pred']),\n",
    "            'roc_log_auc': roc_log_auc(test_df['act_bin'], test_df['pred_bin'], ascending_score=False),# Binary is descending\n",
    "            'precision': precision_score(test_df['act_bin'], test_df['pred_bin']),\n",
    "            'accuracy': accuracy_score(test_df['act_bin'], test_df['pred_bin']),\n",
    "            'recall': recall_score(test_df['act_bin'], test_df['pred_bin']),\n",
    "            }, test_df"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "code",
   "execution_count": 17,
   "metadata": {
    "collapsed": false
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "1 Engines running\n",
      "Sending a shutdown signal to the controller and engines.\n",
      "CPU times: user 11.8 s, sys: 6.43 s, total: 18.2 s\n",
      "Wall time: 40min 13s\n"
     ]
    }
   ],
   "source": [
    "%%time\n",
    "a = []\n",
    "for n_folds in [5]: # n_folds\n",
    "    oddt.random_seed(0)\n",
    "    cv_split = KFold(len(dude_ids), n_folds=n_folds, shuffle=True)\n",
    "    for v in [2,3,1]:\n",
    "        for engine in ['dude', 'vina', 'dock']:\n",
    "            a.extend([{'test': test, 'train': train, 'v': v, 'engine': engine} for train, test in cv_split])\n",
    "\n",
    "args = pd.DataFrame(a)\n",
    "# #with cluster_view(scheduler=\"slurm\", queue='long', num_jobs=n if n<50 else 50, cores_per_job=12) as view:\n",
    "nodes = 20\n",
    "with cluster_view(scheduler=\"slurm\",\n",
    "                  queue='accel' if nodes <=5 else 'long',\n",
    "                  num_jobs=nodes if nodes<50 else 50,\n",
    "                  cores_per_job=16 if nodes <=5 else 12\n",
    "                 ) as view:\n",
    "    tmp = view.map(dude_vertical, args['train'], args['test'], args['v'], args['engine'])\n",
    "out = [t[0] for t in tmp]\n",
    "out2 = [t[1] for t in tmp]\n",
    "out_df = pd.DataFrame(out)\n",
    "out_df['n_folds'] = n_folds\n",
    "o = out_df\n",
    "\n",
    "o.to_csv('dude_vertical.csv')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 18,
   "metadata": {
    "collapsed": false
   },
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "      <th>accuracy</th>\n",
       "      <th>ef0.1</th>\n",
       "      <th>ef1</th>\n",
       "      <th>ef10</th>\n",
       "      <th>ef1_perc</th>\n",
       "      <th>ef2</th>\n",
       "      <th>ef5</th>\n",
       "      <th>mse</th>\n",
       "      <th>precision</th>\n",
       "      <th>recall</th>\n",
       "      <th>...</th>\n",
       "      <th>rk_inactive</th>\n",
       "      <th>roc_auc</th>\n",
       "      <th>roc_log_auc</th>\n",
       "      <th>rp</th>\n",
       "      <th>rp_active</th>\n",
       "      <th>rp_inactive</th>\n",
       "      <th>rs</th>\n",
       "      <th>rs_active</th>\n",
       "      <th>rs_inactive</th>\n",
       "      <th>n_folds</th>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>engine</th>\n",
       "      <th>v</th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th rowspan=\"3\" valign=\"top\">dock</th>\n",
       "      <th>1</th>\n",
       "      <td>0.866263</td>\n",
       "      <td>14.828669</td>\n",
       "      <td>7.417264</td>\n",
       "      <td>3.186220</td>\n",
       "      <td>0.078361</td>\n",
       "      <td>5.770809</td>\n",
       "      <td>4.221178</td>\n",
       "      <td>0.046916</td>\n",
       "      <td>0.030763</td>\n",
       "      <td>0.368608</td>\n",
       "      <td>...</td>\n",
       "      <td>0.002513</td>\n",
       "      <td>0.620168</td>\n",
       "      <td>0.204445</td>\n",
       "      <td>0.092907</td>\n",
       "      <td>0.037693</td>\n",
       "      <td>0.000494</td>\n",
       "      <td>0.069224</td>\n",
       "      <td>0.072098</td>\n",
       "      <td>0.003078</td>\n",
       "      <td>5</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2</th>\n",
       "      <td>0.860067</td>\n",
       "      <td>21.883359</td>\n",
       "      <td>10.127896</td>\n",
       "      <td>3.874574</td>\n",
       "      <td>0.107787</td>\n",
       "      <td>7.588668</td>\n",
       "      <td>5.228261</td>\n",
       "      <td>0.046000</td>\n",
       "      <td>0.035872</td>\n",
       "      <td>0.454233</td>\n",
       "      <td>...</td>\n",
       "      <td>0.005817</td>\n",
       "      <td>0.659439</td>\n",
       "      <td>0.211869</td>\n",
       "      <td>0.124270</td>\n",
       "      <td>0.053364</td>\n",
       "      <td>0.001115</td>\n",
       "      <td>0.086879</td>\n",
       "      <td>0.074812</td>\n",
       "      <td>0.007124</td>\n",
       "      <td>5</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3</th>\n",
       "      <td>0.871075</td>\n",
       "      <td>24.086582</td>\n",
       "      <td>11.185124</td>\n",
       "      <td>3.903067</td>\n",
       "      <td>0.120021</td>\n",
       "      <td>8.379892</td>\n",
       "      <td>5.525988</td>\n",
       "      <td>0.046116</td>\n",
       "      <td>0.037824</td>\n",
       "      <td>0.438826</td>\n",
       "      <td>...</td>\n",
       "      <td>0.002893</td>\n",
       "      <td>0.657364</td>\n",
       "      <td>0.218480</td>\n",
       "      <td>0.135188</td>\n",
       "      <td>0.043584</td>\n",
       "      <td>0.000543</td>\n",
       "      <td>0.084997</td>\n",
       "      <td>0.074095</td>\n",
       "      <td>0.003543</td>\n",
       "      <td>5</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th rowspan=\"3\" valign=\"top\">dude</th>\n",
       "      <th>1</th>\n",
       "      <td>0.802113</td>\n",
       "      <td>14.651980</td>\n",
       "      <td>7.124117</td>\n",
       "      <td>3.339116</td>\n",
       "      <td>0.113486</td>\n",
       "      <td>5.907921</td>\n",
       "      <td>4.380568</td>\n",
       "      <td>0.066191</td>\n",
       "      <td>0.040189</td>\n",
       "      <td>0.485864</td>\n",
       "      <td>...</td>\n",
       "      <td>0.003545</td>\n",
       "      <td>0.646544</td>\n",
       "      <td>0.178496</td>\n",
       "      <td>0.120794</td>\n",
       "      <td>0.062963</td>\n",
       "      <td>0.000888</td>\n",
       "      <td>0.092127</td>\n",
       "      <td>0.084798</td>\n",
       "      <td>0.004342</td>\n",
       "      <td>5</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2</th>\n",
       "      <td>0.797645</td>\n",
       "      <td>31.933480</td>\n",
       "      <td>13.726584</td>\n",
       "      <td>4.381992</td>\n",
       "      <td>0.219002</td>\n",
       "      <td>9.927954</td>\n",
       "      <td>6.252138</td>\n",
       "      <td>0.063604</td>\n",
       "      <td>0.048179</td>\n",
       "      <td>0.606001</td>\n",
       "      <td>...</td>\n",
       "      <td>0.001834</td>\n",
       "      <td>0.703385</td>\n",
       "      <td>0.189321</td>\n",
       "      <td>0.191884</td>\n",
       "      <td>0.053209</td>\n",
       "      <td>0.000969</td>\n",
       "      <td>0.119613</td>\n",
       "      <td>0.074958</td>\n",
       "      <td>0.002246</td>\n",
       "      <td>5</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3</th>\n",
       "      <td>0.814622</td>\n",
       "      <td>40.995739</td>\n",
       "      <td>15.513939</td>\n",
       "      <td>4.668232</td>\n",
       "      <td>0.247533</td>\n",
       "      <td>10.907265</td>\n",
       "      <td>6.830561</td>\n",
       "      <td>0.062752</td>\n",
       "      <td>0.051921</td>\n",
       "      <td>0.606555</td>\n",
       "      <td>...</td>\n",
       "      <td>-0.003884</td>\n",
       "      <td>0.712284</td>\n",
       "      <td>0.198929</td>\n",
       "      <td>0.222620</td>\n",
       "      <td>0.058549</td>\n",
       "      <td>0.000129</td>\n",
       "      <td>0.122990</td>\n",
       "      <td>0.079090</td>\n",
       "      <td>-0.004758</td>\n",
       "      <td>5</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th rowspan=\"3\" valign=\"top\">vina</th>\n",
       "      <th>1</th>\n",
       "      <td>0.786331</td>\n",
       "      <td>16.481953</td>\n",
       "      <td>7.336011</td>\n",
       "      <td>3.263182</td>\n",
       "      <td>0.123678</td>\n",
       "      <td>5.841842</td>\n",
       "      <td>4.300913</td>\n",
       "      <td>0.068279</td>\n",
       "      <td>0.040902</td>\n",
       "      <td>0.502840</td>\n",
       "      <td>...</td>\n",
       "      <td>-0.000642</td>\n",
       "      <td>0.647056</td>\n",
       "      <td>0.170797</td>\n",
       "      <td>0.128234</td>\n",
       "      <td>0.095805</td>\n",
       "      <td>0.000424</td>\n",
       "      <td>0.094419</td>\n",
       "      <td>0.103736</td>\n",
       "      <td>-0.000787</td>\n",
       "      <td>5</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2</th>\n",
       "      <td>0.770231</td>\n",
       "      <td>24.271567</td>\n",
       "      <td>10.986414</td>\n",
       "      <td>4.200237</td>\n",
       "      <td>0.183874</td>\n",
       "      <td>8.602261</td>\n",
       "      <td>5.834357</td>\n",
       "      <td>0.066257</td>\n",
       "      <td>0.046390</td>\n",
       "      <td>0.624323</td>\n",
       "      <td>...</td>\n",
       "      <td>0.004674</td>\n",
       "      <td>0.698583</td>\n",
       "      <td>0.174940</td>\n",
       "      <td>0.175672</td>\n",
       "      <td>0.093659</td>\n",
       "      <td>0.001111</td>\n",
       "      <td>0.119340</td>\n",
       "      <td>0.107136</td>\n",
       "      <td>0.005725</td>\n",
       "      <td>5</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3</th>\n",
       "      <td>0.788741</td>\n",
       "      <td>28.022906</td>\n",
       "      <td>13.260194</td>\n",
       "      <td>4.430342</td>\n",
       "      <td>0.223030</td>\n",
       "      <td>9.877661</td>\n",
       "      <td>6.332642</td>\n",
       "      <td>0.065593</td>\n",
       "      <td>0.049355</td>\n",
       "      <td>0.615194</td>\n",
       "      <td>...</td>\n",
       "      <td>-0.002118</td>\n",
       "      <td>0.703495</td>\n",
       "      <td>0.183877</td>\n",
       "      <td>0.202444</td>\n",
       "      <td>0.097002</td>\n",
       "      <td>0.000369</td>\n",
       "      <td>0.122265</td>\n",
       "      <td>0.109245</td>\n",
       "      <td>-0.002594</td>\n",
       "      <td>5</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "<p>9 rows × 22 columns</p>\n",
       "</div>"
      ],
      "text/plain": [
       "          accuracy      ef0.1        ef1      ef10  ef1_perc        ef2  \\\n",
       "engine v                                                                  \n",
       "dock   1  0.866263  14.828669   7.417264  3.186220  0.078361   5.770809   \n",
       "       2  0.860067  21.883359  10.127896  3.874574  0.107787   7.588668   \n",
       "       3  0.871075  24.086582  11.185124  3.903067  0.120021   8.379892   \n",
       "dude   1  0.802113  14.651980   7.124117  3.339116  0.113486   5.907921   \n",
       "       2  0.797645  31.933480  13.726584  4.381992  0.219002   9.927954   \n",
       "       3  0.814622  40.995739  15.513939  4.668232  0.247533  10.907265   \n",
       "vina   1  0.786331  16.481953   7.336011  3.263182  0.123678   5.841842   \n",
       "       2  0.770231  24.271567  10.986414  4.200237  0.183874   8.602261   \n",
       "       3  0.788741  28.022906  13.260194  4.430342  0.223030   9.877661   \n",
       "\n",
       "               ef5       mse  precision    recall   ...     rk_inactive  \\\n",
       "engine v                                            ...                   \n",
       "dock   1  4.221178  0.046916   0.030763  0.368608   ...        0.002513   \n",
       "       2  5.228261  0.046000   0.035872  0.454233   ...        0.005817   \n",
       "       3  5.525988  0.046116   0.037824  0.438826   ...        0.002893   \n",
       "dude   1  4.380568  0.066191   0.040189  0.485864   ...        0.003545   \n",
       "       2  6.252138  0.063604   0.048179  0.606001   ...        0.001834   \n",
       "       3  6.830561  0.062752   0.051921  0.606555   ...       -0.003884   \n",
       "vina   1  4.300913  0.068279   0.040902  0.502840   ...       -0.000642   \n",
       "       2  5.834357  0.066257   0.046390  0.624323   ...        0.004674   \n",
       "       3  6.332642  0.065593   0.049355  0.615194   ...       -0.002118   \n",
       "\n",
       "           roc_auc  roc_log_auc        rp  rp_active  rp_inactive        rs  \\\n",
       "engine v                                                                      \n",
       "dock   1  0.620168     0.204445  0.092907   0.037693     0.000494  0.069224   \n",
       "       2  0.659439     0.211869  0.124270   0.053364     0.001115  0.086879   \n",
       "       3  0.657364     0.218480  0.135188   0.043584     0.000543  0.084997   \n",
       "dude   1  0.646544     0.178496  0.120794   0.062963     0.000888  0.092127   \n",
       "       2  0.703385     0.189321  0.191884   0.053209     0.000969  0.119613   \n",
       "       3  0.712284     0.198929  0.222620   0.058549     0.000129  0.122990   \n",
       "vina   1  0.647056     0.170797  0.128234   0.095805     0.000424  0.094419   \n",
       "       2  0.698583     0.174940  0.175672   0.093659     0.001111  0.119340   \n",
       "       3  0.703495     0.183877  0.202444   0.097002     0.000369  0.122265   \n",
       "\n",
       "          rs_active  rs_inactive  n_folds  \n",
       "engine v                                   \n",
       "dock   1   0.072098     0.003078        5  \n",
       "       2   0.074812     0.007124        5  \n",
       "       3   0.074095     0.003543        5  \n",
       "dude   1   0.084798     0.004342        5  \n",
       "       2   0.074958     0.002246        5  \n",
       "       3   0.079090    -0.004758        5  \n",
       "vina   1   0.103736    -0.000787        5  \n",
       "       2   0.107136     0.005725        5  \n",
       "       3   0.109245    -0.002594        5  \n",
       "\n",
       "[9 rows x 22 columns]"
      ]
     },
     "execution_count": 18,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "o.groupby(['engine', 'v']).mean()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 19,
   "metadata": {
    "collapsed": false
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "111\n"
     ]
    }
   ],
   "source": [
    "print 111"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 23,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "test_df = pd.concat([df[['engine', 'v','act', 'act_bin', 'dude_id', 'name', 'pred', 'pred_bin']] for df in out2])"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 26,
   "metadata": {
    "collapsed": false
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "('dock', 1)\n",
      "('dock', 2)\n",
      "('dock', 3)\n",
      "('dude', 1)\n",
      "('dude', 2)\n",
      "('dude', 3)\n",
      "('vina', 1)\n",
      "('vina', 2)\n",
      "('vina', 3)\n",
      "CPU times: user 50.3 s, sys: 1.19 s, total: 51.5 s\n",
      "Wall time: 51.6 s\n"
     ]
    }
   ],
   "source": [
    "%%time\n",
    "for index, group in test_df.groupby(['engine', 'v']):\n",
    "    print index\n",
    "    group.to_csv('dude_pred_vertical_v%i_e%s.csv' % (v,engine), index=False)"
   ]
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 2",
   "language": "python",
   "name": "python2"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 2
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython2",
   "version": "2.7.12"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 0
}