eda + data augmentation.ipynb

{
 "cells": [
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## Load libraries"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 1,
   "metadata": {
    "_cell_guid": "b1076dfc-b9ad-4769-8c92-a6c4dae69d19",
    "_uuid": "8f2839f25d086af736a60e9eeb907d3b93b6e0e5"
   },
   "outputs": [],
   "source": [
    "import sys\n",
    "import os\n",
    "import numpy as np\n",
    "import pandas as pd\n",
    "import seaborn as sns\n",
    "\n",
    "from PIL import Image\n",
    "\n",
    "import torch\n",
    "import torch.nn as nn\n",
    "import torch.utils.data as D\n",
    "from torch.optim.lr_scheduler import ExponentialLR\n",
    "import torch.nn.functional as F\n",
    "from torch.autograd import Variable\n",
    "from multiprocessing import Pool\n",
    "\n",
    "from torchvision import transforms\n",
    "\n",
    "from ignite.engine import Events\n",
    "from scripts.ignite import create_supervised_evaluator, create_supervised_trainer\n",
    "from ignite.metrics import Loss, Accuracy\n",
    "from ignite.contrib.handlers.tqdm_logger import ProgressBar\n",
    "from ignite.handlers import  EarlyStopping, ModelCheckpoint\n",
    "from ignite.contrib.handlers import LinearCyclicalScheduler, CosineAnnealingScheduler\n",
    "\n",
    "import random\n",
    "\n",
    "from tqdm import tqdm_notebook\n",
    "\n",
    "from sklearn.model_selection import train_test_split\n",
    "\n",
    "from efficientnet_pytorch import EfficientNet, utils as enet_utils\n",
    "\n",
    "from scripts.evaluate import eval_model\n",
    "from scripts.transforms import gen_transform_train_wo_norm\n",
    "from scripts.plates_leak import apply_plates_leak\n",
    "\n",
    "import matplotlib.pyplot as plt\n",
    "\n",
    "import warnings\n",
    "warnings.filterwarnings('ignore')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 2,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Model_efficientnet-b1_7.pth    diabetic-retinopathy-resized  rxrxai\r\n",
      "aptos2019-blindness-detection  lost+found\t\t     rxrxkey.txt\r\n",
      "aptosplus\t\t       retinopathy-train-2015\t     rxrxmodels\r\n"
     ]
    }
   ],
   "source": [
    "!ls /storage"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 3,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "pixel_stats.csv\t\t       test.csv\t\t  train.zip\r\n",
      "pixel_stats.csv.zip\t       test.zip\t\t  train_controls.csv\r\n",
      "pixel_stats_agg.csv\t       test_controls.csv  training_aug.csv\r\n",
      "recursion_dataset_license.pdf  train\t\t  validation.csv\r\n",
      "sample_submission.csv\t       train.csv\r\n",
      "test\t\t\t       train.csv.zip\r\n"
     ]
    }
   ],
   "source": [
    "!ls /storage/rxrxai"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 4,
   "metadata": {
    "_cell_guid": "79c7e3d0-c299-4dcb-8224-4455121ee9b0",
    "_uuid": "d629ff2d2480ee46fbb7e2d37f6b5fab8052498a"
   },
   "outputs": [],
   "source": [
    "path_data = '/storage/rxrxai'\n",
    "device = 'cuda'\n",
    "batch_size = 4\n",
    "torch.manual_seed(0)\n",
    "model_name = 'efficientnet-b4'\n",
    "init_lr = 3e-4\n",
    "end_lr = 1e-7"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 5,
   "metadata": {},
   "outputs": [],
   "source": [
    "# dataframes for training, cross-validation, and testing\n",
    "df = pd.read_csv(path_data+'/train.csv')\n",
    "df_test = pd.read_csv(path_data+'/test.csv')\n",
    "df_all = pd.concat([df, df_test])\n",
    "df_pixels = pd.read_csv(path_data+'/pixel_stats.csv')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 6,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>experiment</th>\n",
       "      <th>id_code</th>\n",
       "      <th>plate</th>\n",
       "      <th>sirna</th>\n",
       "      <th>well</th>\n",
       "      <th>ds</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>HEPG2-01</td>\n",
       "      <td>HEPG2-01_1_B03</td>\n",
       "      <td>1</td>\n",
       "      <td>513.0</td>\n",
       "      <td>B03</td>\n",
       "      <td>train</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>HEPG2-01</td>\n",
       "      <td>HEPG2-01_1_B04</td>\n",
       "      <td>1</td>\n",
       "      <td>840.0</td>\n",
       "      <td>B04</td>\n",
       "      <td>train</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2</th>\n",
       "      <td>HEPG2-01</td>\n",
       "      <td>HEPG2-01_1_B05</td>\n",
       "      <td>1</td>\n",
       "      <td>1020.0</td>\n",
       "      <td>B05</td>\n",
       "      <td>train</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3</th>\n",
       "      <td>HEPG2-01</td>\n",
       "      <td>HEPG2-01_1_B06</td>\n",
       "      <td>1</td>\n",
       "      <td>254.0</td>\n",
       "      <td>B06</td>\n",
       "      <td>train</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4</th>\n",
       "      <td>HEPG2-01</td>\n",
       "      <td>HEPG2-01_1_B07</td>\n",
       "      <td>1</td>\n",
       "      <td>144.0</td>\n",
       "      <td>B07</td>\n",
       "      <td>train</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "  experiment         id_code  plate   sirna well     ds\n",
       "0   HEPG2-01  HEPG2-01_1_B03      1   513.0  B03  train\n",
       "1   HEPG2-01  HEPG2-01_1_B04      1   840.0  B04  train\n",
       "2   HEPG2-01  HEPG2-01_1_B05      1  1020.0  B05  train\n",
       "3   HEPG2-01  HEPG2-01_1_B06      1   254.0  B06  train\n",
       "4   HEPG2-01  HEPG2-01_1_B07      1   144.0  B07  train"
      ]
     },
     "execution_count": 6,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "def train_or_test(row):\n",
    "    return 'train' if not np.isnan(row['sirna']) else 'test'\n",
    "\n",
    "df_all['ds'] = df_all.apply(train_or_test, axis=1)\n",
    "df_all.head()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 7,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>experiment</th>\n",
       "      <th>id_code</th>\n",
       "      <th>plate</th>\n",
       "      <th>sirna</th>\n",
       "      <th>well</th>\n",
       "      <th>ds</th>\n",
       "      <th>cell</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>HEPG2-01</td>\n",
       "      <td>HEPG2-01_1_B03</td>\n",
       "      <td>1</td>\n",
       "      <td>513.0</td>\n",
       "      <td>B03</td>\n",
       "      <td>train</td>\n",
       "      <td>HEPG2</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>HEPG2-01</td>\n",
       "      <td>HEPG2-01_1_B04</td>\n",
       "      <td>1</td>\n",
       "      <td>840.0</td>\n",
       "      <td>B04</td>\n",
       "      <td>train</td>\n",
       "      <td>HEPG2</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2</th>\n",
       "      <td>HEPG2-01</td>\n",
       "      <td>HEPG2-01_1_B05</td>\n",
       "      <td>1</td>\n",
       "      <td>1020.0</td>\n",
       "      <td>B05</td>\n",
       "      <td>train</td>\n",
       "      <td>HEPG2</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3</th>\n",
       "      <td>HEPG2-01</td>\n",
       "      <td>HEPG2-01_1_B06</td>\n",
       "      <td>1</td>\n",
       "      <td>254.0</td>\n",
       "      <td>B06</td>\n",
       "      <td>train</td>\n",
       "      <td>HEPG2</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4</th>\n",
       "      <td>HEPG2-01</td>\n",
       "      <td>HEPG2-01_1_B07</td>\n",
       "      <td>1</td>\n",
       "      <td>144.0</td>\n",
       "      <td>B07</td>\n",
       "      <td>train</td>\n",
       "      <td>HEPG2</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>5</th>\n",
       "      <td>HEPG2-01</td>\n",
       "      <td>HEPG2-01_1_B08</td>\n",
       "      <td>1</td>\n",
       "      <td>503.0</td>\n",
       "      <td>B08</td>\n",
       "      <td>train</td>\n",
       "      <td>HEPG2</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>6</th>\n",
       "      <td>HEPG2-01</td>\n",
       "      <td>HEPG2-01_1_B09</td>\n",
       "      <td>1</td>\n",
       "      <td>188.0</td>\n",
       "      <td>B09</td>\n",
       "      <td>train</td>\n",
       "      <td>HEPG2</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>7</th>\n",
       "      <td>HEPG2-01</td>\n",
       "      <td>HEPG2-01_1_B10</td>\n",
       "      <td>1</td>\n",
       "      <td>700.0</td>\n",
       "      <td>B10</td>\n",
       "      <td>train</td>\n",
       "      <td>HEPG2</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>8</th>\n",
       "      <td>HEPG2-01</td>\n",
       "      <td>HEPG2-01_1_B11</td>\n",
       "      <td>1</td>\n",
       "      <td>1100.0</td>\n",
       "      <td>B11</td>\n",
       "      <td>train</td>\n",
       "      <td>HEPG2</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>9</th>\n",
       "      <td>HEPG2-01</td>\n",
       "      <td>HEPG2-01_1_B12</td>\n",
       "      <td>1</td>\n",
       "      <td>611.0</td>\n",
       "      <td>B12</td>\n",
       "      <td>train</td>\n",
       "      <td>HEPG2</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>10</th>\n",
       "      <td>HEPG2-01</td>\n",
       "      <td>HEPG2-01_1_B13</td>\n",
       "      <td>1</td>\n",
       "      <td>276.0</td>\n",
       "      <td>B13</td>\n",
       "      <td>train</td>\n",
       "      <td>HEPG2</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>11</th>\n",
       "      <td>HEPG2-01</td>\n",
       "      <td>HEPG2-01_1_B14</td>\n",
       "      <td>1</td>\n",
       "      <td>397.0</td>\n",
       "      <td>B14</td>\n",
       "      <td>train</td>\n",
       "      <td>HEPG2</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>12</th>\n",
       "      <td>HEPG2-01</td>\n",
       "      <td>HEPG2-01_1_B15</td>\n",
       "      <td>1</td>\n",
       "      <td>757.0</td>\n",
       "      <td>B15</td>\n",
       "      <td>train</td>\n",
       "      <td>HEPG2</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>13</th>\n",
       "      <td>HEPG2-01</td>\n",
       "      <td>HEPG2-01_1_B16</td>\n",
       "      <td>1</td>\n",
       "      <td>828.0</td>\n",
       "      <td>B16</td>\n",
       "      <td>train</td>\n",
       "      <td>HEPG2</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>14</th>\n",
       "      <td>HEPG2-01</td>\n",
       "      <td>HEPG2-01_1_B17</td>\n",
       "      <td>1</td>\n",
       "      <td>91.0</td>\n",
       "      <td>B17</td>\n",
       "      <td>train</td>\n",
       "      <td>HEPG2</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>15</th>\n",
       "      <td>HEPG2-01</td>\n",
       "      <td>HEPG2-01_1_B18</td>\n",
       "      <td>1</td>\n",
       "      <td>567.0</td>\n",
       "      <td>B18</td>\n",
       "      <td>train</td>\n",
       "      <td>HEPG2</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>16</th>\n",
       "      <td>HEPG2-01</td>\n",
       "      <td>HEPG2-01_1_B19</td>\n",
       "      <td>1</td>\n",
       "      <td>6.0</td>\n",
       "      <td>B19</td>\n",
       "      <td>train</td>\n",
       "      <td>HEPG2</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>17</th>\n",
       "      <td>HEPG2-01</td>\n",
       "      <td>HEPG2-01_1_B20</td>\n",
       "      <td>1</td>\n",
       "      <td>460.0</td>\n",
       "      <td>B20</td>\n",
       "      <td>train</td>\n",
       "      <td>HEPG2</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>18</th>\n",
       "      <td>HEPG2-01</td>\n",
       "      <td>HEPG2-01_1_B21</td>\n",
       "      <td>1</td>\n",
       "      <td>413.0</td>\n",
       "      <td>B21</td>\n",
       "      <td>train</td>\n",
       "      <td>HEPG2</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>19</th>\n",
       "      <td>HEPG2-01</td>\n",
       "      <td>HEPG2-01_1_B22</td>\n",
       "      <td>1</td>\n",
       "      <td>555.0</td>\n",
       "      <td>B22</td>\n",
       "      <td>train</td>\n",
       "      <td>HEPG2</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>20</th>\n",
       "      <td>HEPG2-01</td>\n",
       "      <td>HEPG2-01_1_B23</td>\n",
       "      <td>1</td>\n",
       "      <td>74.0</td>\n",
       "      <td>B23</td>\n",
       "      <td>train</td>\n",
       "      <td>HEPG2</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>21</th>\n",
       "      <td>HEPG2-01</td>\n",
       "      <td>HEPG2-01_1_C02</td>\n",
       "      <td>1</td>\n",
       "      <td>950.0</td>\n",
       "      <td>C02</td>\n",
       "      <td>train</td>\n",
       "      <td>HEPG2</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>22</th>\n",
       "      <td>HEPG2-01</td>\n",
       "      <td>HEPG2-01_1_C04</td>\n",
       "      <td>1</td>\n",
       "      <td>253.0</td>\n",
       "      <td>C04</td>\n",
       "      <td>train</td>\n",
       "      <td>HEPG2</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>23</th>\n",
       "      <td>HEPG2-01</td>\n",
       "      <td>HEPG2-01_1_C05</td>\n",
       "      <td>1</td>\n",
       "      <td>272.0</td>\n",
       "      <td>C05</td>\n",
       "      <td>train</td>\n",
       "      <td>HEPG2</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>24</th>\n",
       "      <td>HEPG2-01</td>\n",
       "      <td>HEPG2-01_1_C06</td>\n",
       "      <td>1</td>\n",
       "      <td>162.0</td>\n",
       "      <td>C06</td>\n",
       "      <td>train</td>\n",
       "      <td>HEPG2</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>25</th>\n",
       "      <td>HEPG2-01</td>\n",
       "      <td>HEPG2-01_1_C08</td>\n",
       "      <td>1</td>\n",
       "      <td>177.0</td>\n",
       "      <td>C08</td>\n",
       "      <td>train</td>\n",
       "      <td>HEPG2</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>26</th>\n",
       "      <td>HEPG2-01</td>\n",
       "      <td>HEPG2-01_1_C09</td>\n",
       "      <td>1</td>\n",
       "      <td>1023.0</td>\n",
       "      <td>C09</td>\n",
       "      <td>train</td>\n",
       "      <td>HEPG2</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>27</th>\n",
       "      <td>HEPG2-01</td>\n",
       "      <td>HEPG2-01_1_C10</td>\n",
       "      <td>1</td>\n",
       "      <td>282.0</td>\n",
       "      <td>C10</td>\n",
       "      <td>train</td>\n",
       "      <td>HEPG2</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>28</th>\n",
       "      <td>HEPG2-01</td>\n",
       "      <td>HEPG2-01_1_C12</td>\n",
       "      <td>1</td>\n",
       "      <td>762.0</td>\n",
       "      <td>C12</td>\n",
       "      <td>train</td>\n",
       "      <td>HEPG2</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>29</th>\n",
       "      <td>HEPG2-01</td>\n",
       "      <td>HEPG2-01_1_C13</td>\n",
       "      <td>1</td>\n",
       "      <td>547.0</td>\n",
       "      <td>C13</td>\n",
       "      <td>train</td>\n",
       "      <td>HEPG2</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>...</th>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1632</th>\n",
       "      <td>HEPG2-02</td>\n",
       "      <td>HEPG2-02_2_N19</td>\n",
       "      <td>2</td>\n",
       "      <td>747.0</td>\n",
       "      <td>N19</td>\n",
       "      <td>train</td>\n",
       "      <td>HEPG2</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1633</th>\n",
       "      <td>HEPG2-02</td>\n",
       "      <td>HEPG2-02_2_N20</td>\n",
       "      <td>2</td>\n",
       "      <td>1003.0</td>\n",
       "      <td>N20</td>\n",
       "      <td>train</td>\n",
       "      <td>HEPG2</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1634</th>\n",
       "      <td>HEPG2-02</td>\n",
       "      <td>HEPG2-02_2_N21</td>\n",
       "      <td>2</td>\n",
       "      <td>909.0</td>\n",
       "      <td>N21</td>\n",
       "      <td>train</td>\n",
       "      <td>HEPG2</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1635</th>\n",
       "      <td>HEPG2-02</td>\n",
       "      <td>HEPG2-02_2_N22</td>\n",
       "      <td>2</td>\n",
       "      <td>669.0</td>\n",
       "      <td>N22</td>\n",
       "      <td>train</td>\n",
       "      <td>HEPG2</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1636</th>\n",
       "      <td>HEPG2-02</td>\n",
       "      <td>HEPG2-02_2_N23</td>\n",
       "      <td>2</td>\n",
       "      <td>447.0</td>\n",
       "      <td>N23</td>\n",
       "      <td>train</td>\n",
       "      <td>HEPG2</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1637</th>\n",
       "      <td>HEPG2-02</td>\n",
       "      <td>HEPG2-02_2_O02</td>\n",
       "      <td>2</td>\n",
       "      <td>423.0</td>\n",
       "      <td>O02</td>\n",
       "      <td>train</td>\n",
       "      <td>HEPG2</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1638</th>\n",
       "      <td>HEPG2-02</td>\n",
       "      <td>HEPG2-02_2_O03</td>\n",
       "      <td>2</td>\n",
       "      <td>851.0</td>\n",
       "      <td>O03</td>\n",
       "      <td>train</td>\n",
       "      <td>HEPG2</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1639</th>\n",
       "      <td>HEPG2-02</td>\n",
       "      <td>HEPG2-02_2_O04</td>\n",
       "      <td>2</td>\n",
       "      <td>563.0</td>\n",
       "      <td>O04</td>\n",
       "      <td>train</td>\n",
       "      <td>HEPG2</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1640</th>\n",
       "      <td>HEPG2-02</td>\n",
       "      <td>HEPG2-02_2_O05</td>\n",
       "      <td>2</td>\n",
       "      <td>1017.0</td>\n",
       "      <td>O05</td>\n",
       "      <td>train</td>\n",
       "      <td>HEPG2</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1641</th>\n",
       "      <td>HEPG2-02</td>\n",
       "      <td>HEPG2-02_2_O06</td>\n",
       "      <td>2</td>\n",
       "      <td>1043.0</td>\n",
       "      <td>O06</td>\n",
       "      <td>train</td>\n",
       "      <td>HEPG2</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1642</th>\n",
       "      <td>HEPG2-02</td>\n",
       "      <td>HEPG2-02_2_O07</td>\n",
       "      <td>2</td>\n",
       "      <td>932.0</td>\n",
       "      <td>O07</td>\n",
       "      <td>train</td>\n",
       "      <td>HEPG2</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1643</th>\n",
       "      <td>HEPG2-02</td>\n",
       "      <td>HEPG2-02_2_O08</td>\n",
       "      <td>2</td>\n",
       "      <td>165.0</td>\n",
       "      <td>O08</td>\n",
       "      <td>train</td>\n",
       "      <td>HEPG2</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1644</th>\n",
       "      <td>HEPG2-02</td>\n",
       "      <td>HEPG2-02_2_O09</td>\n",
       "      <td>2</td>\n",
       "      <td>753.0</td>\n",
       "      <td>O09</td>\n",
       "      <td>train</td>\n",
       "      <td>HEPG2</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1645</th>\n",
       "      <td>HEPG2-02</td>\n",
       "      <td>HEPG2-02_2_O10</td>\n",
       "      <td>2</td>\n",
       "      <td>305.0</td>\n",
       "      <td>O10</td>\n",
       "      <td>train</td>\n",
       "      <td>HEPG2</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1646</th>\n",
       "      <td>HEPG2-02</td>\n",
       "      <td>HEPG2-02_2_O11</td>\n",
       "      <td>2</td>\n",
       "      <td>871.0</td>\n",
       "      <td>O11</td>\n",
       "      <td>train</td>\n",
       "      <td>HEPG2</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1647</th>\n",
       "      <td>HEPG2-02</td>\n",
       "      <td>HEPG2-02_2_O12</td>\n",
       "      <td>2</td>\n",
       "      <td>667.0</td>\n",
       "      <td>O12</td>\n",
       "      <td>train</td>\n",
       "      <td>HEPG2</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1648</th>\n",
       "      <td>HEPG2-02</td>\n",
       "      <td>HEPG2-02_2_O13</td>\n",
       "      <td>2</td>\n",
       "      <td>996.0</td>\n",
       "      <td>O13</td>\n",
       "      <td>train</td>\n",
       "      <td>HEPG2</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1649</th>\n",
       "      <td>HEPG2-02</td>\n",
       "      <td>HEPG2-02_2_O14</td>\n",
       "      <td>2</td>\n",
       "      <td>751.0</td>\n",
       "      <td>O14</td>\n",
       "      <td>train</td>\n",
       "      <td>HEPG2</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1650</th>\n",
       "      <td>HEPG2-02</td>\n",
       "      <td>HEPG2-02_2_O15</td>\n",
       "      <td>2</td>\n",
       "      <td>915.0</td>\n",
       "      <td>O15</td>\n",
       "      <td>train</td>\n",
       "      <td>HEPG2</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1651</th>\n",
       "      <td>HEPG2-02</td>\n",
       "      <td>HEPG2-02_2_O16</td>\n",
       "      <td>2</td>\n",
       "      <td>479.0</td>\n",
       "      <td>O16</td>\n",
       "      <td>train</td>\n",
       "      <td>HEPG2</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1652</th>\n",
       "      <td>HEPG2-02</td>\n",
       "      <td>HEPG2-02_2_O17</td>\n",
       "      <td>2</td>\n",
       "      <td>1063.0</td>\n",
       "      <td>O17</td>\n",
       "      <td>train</td>\n",
       "      <td>HEPG2</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1653</th>\n",
       "      <td>HEPG2-02</td>\n",
       "      <td>HEPG2-02_2_O18</td>\n",
       "      <td>2</td>\n",
       "      <td>219.0</td>\n",
       "      <td>O18</td>\n",
       "      <td>train</td>\n",
       "      <td>HEPG2</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1654</th>\n",
       "      <td>HEPG2-02</td>\n",
       "      <td>HEPG2-02_2_O19</td>\n",
       "      <td>2</td>\n",
       "      <td>216.0</td>\n",
       "      <td>O19</td>\n",
       "      <td>train</td>\n",
       "      <td>HEPG2</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1655</th>\n",
       "      <td>HEPG2-02</td>\n",
       "      <td>HEPG2-02_2_O20</td>\n",
       "      <td>2</td>\n",
       "      <td>475.0</td>\n",
       "      <td>O20</td>\n",
       "      <td>train</td>\n",
       "      <td>HEPG2</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1656</th>\n",
       "      <td>HEPG2-02</td>\n",
       "      <td>HEPG2-02_2_O21</td>\n",
       "      <td>2</td>\n",
       "      <td>694.0</td>\n",
       "      <td>O21</td>\n",
       "      <td>train</td>\n",
       "      <td>HEPG2</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1657</th>\n",
       "      <td>HEPG2-02</td>\n",
       "      <td>HEPG2-02_2_O22</td>\n",
       "      <td>2</td>\n",
       "      <td>131.0</td>\n",
       "      <td>O22</td>\n",
       "      <td>train</td>\n",
       "      <td>HEPG2</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1658</th>\n",
       "      <td>HEPG2-02</td>\n",
       "      <td>HEPG2-02_2_O23</td>\n",
       "      <td>2</td>\n",
       "      <td>166.0</td>\n",
       "      <td>O23</td>\n",
       "      <td>train</td>\n",
       "      <td>HEPG2</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1659</th>\n",
       "      <td>HEPG2-02</td>\n",
       "      <td>HEPG2-02_3_B03</td>\n",
       "      <td>3</td>\n",
       "      <td>437.0</td>\n",
       "      <td>B03</td>\n",
       "      <td>train</td>\n",
       "      <td>HEPG2</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1660</th>\n",
       "      <td>HEPG2-02</td>\n",
       "      <td>HEPG2-02_3_B04</td>\n",
       "      <td>3</td>\n",
       "      <td>271.0</td>\n",
       "      <td>B04</td>\n",
       "      <td>train</td>\n",
       "      <td>HEPG2</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1661</th>\n",
       "      <td>HEPG2-02</td>\n",
       "      <td>HEPG2-02_3_B05</td>\n",
       "      <td>3</td>\n",
       "      <td>252.0</td>\n",
       "      <td>B05</td>\n",
       "      <td>train</td>\n",
       "      <td>HEPG2</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "<p>1662 rows × 7 columns</p>\n",
       "</div>"
      ],
      "text/plain": [
       "     experiment         id_code  plate   sirna well     ds   cell\n",
       "0      HEPG2-01  HEPG2-01_1_B03      1   513.0  B03  train  HEPG2\n",
       "1      HEPG2-01  HEPG2-01_1_B04      1   840.0  B04  train  HEPG2\n",
       "2      HEPG2-01  HEPG2-01_1_B05      1  1020.0  B05  train  HEPG2\n",
       "3      HEPG2-01  HEPG2-01_1_B06      1   254.0  B06  train  HEPG2\n",
       "4      HEPG2-01  HEPG2-01_1_B07      1   144.0  B07  train  HEPG2\n",
       "5      HEPG2-01  HEPG2-01_1_B08      1   503.0  B08  train  HEPG2\n",
       "6      HEPG2-01  HEPG2-01_1_B09      1   188.0  B09  train  HEPG2\n",
       "7      HEPG2-01  HEPG2-01_1_B10      1   700.0  B10  train  HEPG2\n",
       "8      HEPG2-01  HEPG2-01_1_B11      1  1100.0  B11  train  HEPG2\n",
       "9      HEPG2-01  HEPG2-01_1_B12      1   611.0  B12  train  HEPG2\n",
       "10     HEPG2-01  HEPG2-01_1_B13      1   276.0  B13  train  HEPG2\n",
       "11     HEPG2-01  HEPG2-01_1_B14      1   397.0  B14  train  HEPG2\n",
       "12     HEPG2-01  HEPG2-01_1_B15      1   757.0  B15  train  HEPG2\n",
       "13     HEPG2-01  HEPG2-01_1_B16      1   828.0  B16  train  HEPG2\n",
       "14     HEPG2-01  HEPG2-01_1_B17      1    91.0  B17  train  HEPG2\n",
       "15     HEPG2-01  HEPG2-01_1_B18      1   567.0  B18  train  HEPG2\n",
       "16     HEPG2-01  HEPG2-01_1_B19      1     6.0  B19  train  HEPG2\n",
       "17     HEPG2-01  HEPG2-01_1_B20      1   460.0  B20  train  HEPG2\n",
       "18     HEPG2-01  HEPG2-01_1_B21      1   413.0  B21  train  HEPG2\n",
       "19     HEPG2-01  HEPG2-01_1_B22      1   555.0  B22  train  HEPG2\n",
       "20     HEPG2-01  HEPG2-01_1_B23      1    74.0  B23  train  HEPG2\n",
       "21     HEPG2-01  HEPG2-01_1_C02      1   950.0  C02  train  HEPG2\n",
       "22     HEPG2-01  HEPG2-01_1_C04      1   253.0  C04  train  HEPG2\n",
       "23     HEPG2-01  HEPG2-01_1_C05      1   272.0  C05  train  HEPG2\n",
       "24     HEPG2-01  HEPG2-01_1_C06      1   162.0  C06  train  HEPG2\n",
       "25     HEPG2-01  HEPG2-01_1_C08      1   177.0  C08  train  HEPG2\n",
       "26     HEPG2-01  HEPG2-01_1_C09      1  1023.0  C09  train  HEPG2\n",
       "27     HEPG2-01  HEPG2-01_1_C10      1   282.0  C10  train  HEPG2\n",
       "28     HEPG2-01  HEPG2-01_1_C12      1   762.0  C12  train  HEPG2\n",
       "29     HEPG2-01  HEPG2-01_1_C13      1   547.0  C13  train  HEPG2\n",
       "...         ...             ...    ...     ...  ...    ...    ...\n",
       "1632   HEPG2-02  HEPG2-02_2_N19      2   747.0  N19  train  HEPG2\n",
       "1633   HEPG2-02  HEPG2-02_2_N20      2  1003.0  N20  train  HEPG2\n",
       "1634   HEPG2-02  HEPG2-02_2_N21      2   909.0  N21  train  HEPG2\n",
       "1635   HEPG2-02  HEPG2-02_2_N22      2   669.0  N22  train  HEPG2\n",
       "1636   HEPG2-02  HEPG2-02_2_N23      2   447.0  N23  train  HEPG2\n",
       "1637   HEPG2-02  HEPG2-02_2_O02      2   423.0  O02  train  HEPG2\n",
       "1638   HEPG2-02  HEPG2-02_2_O03      2   851.0  O03  train  HEPG2\n",
       "1639   HEPG2-02  HEPG2-02_2_O04      2   563.0  O04  train  HEPG2\n",
       "1640   HEPG2-02  HEPG2-02_2_O05      2  1017.0  O05  train  HEPG2\n",
       "1641   HEPG2-02  HEPG2-02_2_O06      2  1043.0  O06  train  HEPG2\n",
       "1642   HEPG2-02  HEPG2-02_2_O07      2   932.0  O07  train  HEPG2\n",
       "1643   HEPG2-02  HEPG2-02_2_O08      2   165.0  O08  train  HEPG2\n",
       "1644   HEPG2-02  HEPG2-02_2_O09      2   753.0  O09  train  HEPG2\n",
       "1645   HEPG2-02  HEPG2-02_2_O10      2   305.0  O10  train  HEPG2\n",
       "1646   HEPG2-02  HEPG2-02_2_O11      2   871.0  O11  train  HEPG2\n",
       "1647   HEPG2-02  HEPG2-02_2_O12      2   667.0  O12  train  HEPG2\n",
       "1648   HEPG2-02  HEPG2-02_2_O13      2   996.0  O13  train  HEPG2\n",
       "1649   HEPG2-02  HEPG2-02_2_O14      2   751.0  O14  train  HEPG2\n",
       "1650   HEPG2-02  HEPG2-02_2_O15      2   915.0  O15  train  HEPG2\n",
       "1651   HEPG2-02  HEPG2-02_2_O16      2   479.0  O16  train  HEPG2\n",
       "1652   HEPG2-02  HEPG2-02_2_O17      2  1063.0  O17  train  HEPG2\n",
       "1653   HEPG2-02  HEPG2-02_2_O18      2   219.0  O18  train  HEPG2\n",
       "1654   HEPG2-02  HEPG2-02_2_O19      2   216.0  O19  train  HEPG2\n",
       "1655   HEPG2-02  HEPG2-02_2_O20      2   475.0  O20  train  HEPG2\n",
       "1656   HEPG2-02  HEPG2-02_2_O21      2   694.0  O21  train  HEPG2\n",
       "1657   HEPG2-02  HEPG2-02_2_O22      2   131.0  O22  train  HEPG2\n",
       "1658   HEPG2-02  HEPG2-02_2_O23      2   166.0  O23  train  HEPG2\n",
       "1659   HEPG2-02  HEPG2-02_3_B03      3   437.0  B03  train  HEPG2\n",
       "1660   HEPG2-02  HEPG2-02_3_B04      3   271.0  B04  train  HEPG2\n",
       "1661   HEPG2-02  HEPG2-02_3_B05      3   252.0  B05  train  HEPG2\n",
       "\n",
       "[1662 rows x 7 columns]"
      ]
     },
     "execution_count": 7,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "df_all['cell'] = df_all.apply(lambda row: row['experiment'].split(\"-\")[0], axis=1)\n",
    "df_all.head(277*6)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 8,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>experiment</th>\n",
       "      <th>id_code</th>\n",
       "      <th>plate</th>\n",
       "      <th>sirna</th>\n",
       "      <th>well</th>\n",
       "      <th>ds</th>\n",
       "      <th>cell</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>19892</th>\n",
       "      <td>U2OS-05</td>\n",
       "      <td>U2OS-05_4_O19</td>\n",
       "      <td>4</td>\n",
       "      <td>NaN</td>\n",
       "      <td>O19</td>\n",
       "      <td>test</td>\n",
       "      <td>U2OS</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>19893</th>\n",
       "      <td>U2OS-05</td>\n",
       "      <td>U2OS-05_4_O20</td>\n",
       "      <td>4</td>\n",
       "      <td>NaN</td>\n",
       "      <td>O20</td>\n",
       "      <td>test</td>\n",
       "      <td>U2OS</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>19894</th>\n",
       "      <td>U2OS-05</td>\n",
       "      <td>U2OS-05_4_O21</td>\n",
       "      <td>4</td>\n",
       "      <td>NaN</td>\n",
       "      <td>O21</td>\n",
       "      <td>test</td>\n",
       "      <td>U2OS</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>19895</th>\n",
       "      <td>U2OS-05</td>\n",
       "      <td>U2OS-05_4_O22</td>\n",
       "      <td>4</td>\n",
       "      <td>NaN</td>\n",
       "      <td>O22</td>\n",
       "      <td>test</td>\n",
       "      <td>U2OS</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>19896</th>\n",
       "      <td>U2OS-05</td>\n",
       "      <td>U2OS-05_4_O23</td>\n",
       "      <td>4</td>\n",
       "      <td>NaN</td>\n",
       "      <td>O23</td>\n",
       "      <td>test</td>\n",
       "      <td>U2OS</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "      experiment        id_code  plate  sirna well    ds  cell\n",
       "19892    U2OS-05  U2OS-05_4_O19      4    NaN  O19  test  U2OS\n",
       "19893    U2OS-05  U2OS-05_4_O20      4    NaN  O20  test  U2OS\n",
       "19894    U2OS-05  U2OS-05_4_O21      4    NaN  O21  test  U2OS\n",
       "19895    U2OS-05  U2OS-05_4_O22      4    NaN  O22  test  U2OS\n",
       "19896    U2OS-05  U2OS-05_4_O23      4    NaN  O23  test  U2OS"
      ]
     },
     "execution_count": 8,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "df_all.tail()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 9,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "HEPG2 0.21224154457072436\n",
      "HUVEC 0.48440366972477067\n",
      "RPE 0.21232370258797753\n",
      "U2OS 0.09103108311652745\n"
     ]
    }
   ],
   "source": [
    "train_length = len(df_all[df_all['ds'] == 'train'])\n",
    "\n",
    "for cell in ['HEPG2', 'HUVEC', 'RPE', 'U2OS']:\n",
    "    print(f\"{cell} {df_all[df_all['ds'] == 'train'].groupby('cell').agg(['count'])[['experiment']].loc[cell][0] / train_length}\")"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 10,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "365.15000000000003"
      ]
     },
     "execution_count": 10,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    ".01 * train_length"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 11,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "HEPG2 0.22259637131225812\n",
      "HUVEC 0.4445896366286375\n",
      "RPE 0.22199326531637936\n",
      "U2OS 0.11082072674272503\n"
     ]
    }
   ],
   "source": [
    "test_length = len(df_all[df_all['ds'] == 'test'])\n",
    "\n",
    "for cell in ['HEPG2', 'HUVEC', 'RPE', 'U2OS']:\n",
    "    print(f\"{cell} {df_all[df_all['ds'] == 'test'].groupby('cell').agg(['count'])[['experiment']].loc[cell][0] / test_length}\")"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 12,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "<matplotlib.axes._subplots.AxesSubplot at 0x7fa791b3edd8>"
      ]
     },
     "execution_count": 12,
     "metadata": {},
     "output_type": "execute_result"
    },
    {
     "data": {
      "image/png": "iVBORw0KGgoAAAANSUhEUgAAAZUAAAEKCAYAAADaa8itAAAABHNCSVQICAgIfAhkiAAAAAlwSFlzAAALEgAACxIB0t1+/AAAADl0RVh0U29mdHdhcmUAbWF0cGxvdGxpYiB2ZXJzaW9uIDMuMC4yLCBodHRwOi8vbWF0cGxvdGxpYi5vcmcvOIA7rQAAGAFJREFUeJzt3X20XXV95/H3BxDGVhlAMkgTaCiNtkA1SgZZKpaKD9FxAF1WwxpLUJaBJdhqn4Q6MzC0zGDVoiiDg2MEupSIIhI7YTCigraiJEKBoJSAUJIJJBKr9Yk2+J0/zu/CyeXe5Ej2uede8n6ttdfd+7sfzm/vdU4+2b+9zz6pKiRJ6sIuo26AJOnJw1CRJHXGUJEkdcZQkSR1xlCRJHXGUJEkdcZQkSR1xlCRJHXGUJEkdWa3UTdgqu277741d+7cUTdDkmaU1atXf6+qZm1vuZ0uVObOncuqVatG3QxJmlGS3DfIcnZ/SZI6Y6hIkjpjqEiSOmOoSJI6Y6hIkjpjqEiSOmOoSJI6M7RQSbI0ycYkt/fVPpXkljbcm+SWVp+b5Kd98z7St87hSW5LsjbJBUnS6vskWZnkrvZ372HtiyRpMMM8U7kEWNhfqKo3VtX8qpoPXAl8tm/23WPzqurUvvpFwFuBeW0Y2+YZwHVVNQ+4rk1LkkZoaN+or6obksydaF4723gD8NJtbSPJ/sCeVXVjm74MOB64BjgOOLoteinwFeBdO95ydekfz/mtUTdh2jjwv9426iZIQzeqaypHAQ9W1V19tYOS3Jzk+iRHtdpsYF3fMutaDWC/qtrQxh8A9htqiyVJ2zWqZ3+dAFzeN70BOLCqHkpyOPC5JIcOurGqqiQ12fwkS4AlAAceeOATbLIkaXum/EwlyW7A64BPjdWq6uGqeqiNrwbuBp4FrAfm9K0+p9UAHmzdY2PdZBsne82quriqFlTVglmztvuQTUnSEzSK7q+XAd+pqke7tZLMSrJrG/81ehfk72ndWz9McmS7DnMicHVbbTmwuI0v7qtLkkZkmLcUXw58HXh2knVJTm6zFrF11xfAS4Bb2y3GnwFOrarNbd7bgP8NrKV3BnNNq58HvDzJXfSC6rxh7YskaTDDvPvrhEnqJ01Qu5LeLcYTLb8KOGyC+kPAMTvWSklSl/xGvSSpM4aKJKkzhookqTOGiiSpM4aKJKkzhookqTOGiiSpM4aKJKkzhookqTOGiiSpM4aKJKkzhookqTOGiiSpM4aKJKkzhookqTOGiiSpM4aKJKkzhookqTOGiiSpM4aKJKkzhookqTNDC5UkS5NsTHJ7X+3sJOuT3NKGV/fNOzPJ2iR3JnllX31hq61NckZf/aAk32j1TyXZfVj7IkkazDDPVC4BFk5QP7+q5rdhBUCSQ4BFwKFtnf+ZZNckuwIXAq8CDgFOaMsCvKdt69eB7wMnD3FfJEkDGFqoVNUNwOYBFz8OWFZVD1fVd4G1wBFtWFtV91TVvwDLgOOSBHgp8Jm2/qXA8Z3ugCTpFzaKayqnJ7m1dY/t3Wqzgfv7llnXapPVnwH8U1VtGVefUJIlSVYlWbVp06au9kOSNM5Uh8pFwMHAfGAD8P6peNGquriqFlTVglmzZk3FS0rSTmm3qXyxqnpwbDzJR4G/aZPrgQP6Fp3TakxSfwjYK8lu7Wylf3lJ0ohM6ZlKkv37Jl8LjN0ZthxYlGSPJAcB84BvAjcB89qdXrvTu5i/vKoK+DLw+rb+YuDqqdgHSdLkhnamkuRy4Ghg3yTrgLOAo5PMBwq4FzgFoKrWJLkCuAPYApxWVY+07ZwOXAvsCiytqjXtJd4FLEvyF8DNwMeGtS+SpMEMLVSq6oQJypP+w19V5wLnTlBfAayYoH4PvbvDJEnThN+olyR1xlCRJHXGUJEkdcZQkSR1xlCRJHXGUJEkdcZQkSR1xlCRJHXGUJEkdcZQkSR1xlCRJHXGUJEkdcZQkSR1xlCRJHXGUJEkdcZQkSR1xlCRJHXGUJEkdcZQkSR1xlCRJHVmaKGSZGmSjUlu76u9N8l3ktya5Koke7X63CQ/TXJLGz7St87hSW5LsjbJBUnS6vskWZnkrvZ372HtiyRpMMM8U7kEWDiuthI4rKqeA/wDcGbfvLuran4bTu2rXwS8FZjXhrFtngFcV1XzgOvatCRphIYWKlV1A7B5XO0LVbWlTd4IzNnWNpLsD+xZVTdWVQGXAce32ccBl7bxS/vqkqQRGeU1lbcA1/RNH5Tk5iTXJzmq1WYD6/qWWddqAPtV1YY2/gCw32QvlGRJklVJVm3atKmj5kuSxhtJqCR5N7AF+EQrbQAOrKrnAX8IfDLJnoNur53F1DbmX1xVC6pqwaxZs3ag5ZKkbdltql8wyUnAa4BjWhhQVQ8DD7fx1UnuBp4FrGfrLrI5rQbwYJL9q2pD6ybbOEW7IEmaxJSeqSRZCPwpcGxV/aSvPivJrm381+hdkL+ndW/9MMmR7a6vE4Gr22rLgcVtfHFfXZI0IkM7U0lyOXA0sG+SdcBZ9O722gNY2e4MvrHd6fUS4Jwk/wr8HDi1qsYu8r+N3p1kT6V3DWbsOsx5wBVJTgbuA94wrH2RJA1maKFSVSdMUP7YJMteCVw5ybxVwGET1B8CjtmRNkqSuuU36iVJnTFUJEmdMVQkSZ0xVCRJnTFUJEmdMVQkSZ0xVCRJnTFUJEmdMVQkSZ0xVCRJnTFUJEmdMVQkSZ0xVCRJnTFUJEmdMVQkSZ0xVCRJnTFUJEmdMVQkSZ0xVCRJnTFUJEmdGShUklw3SG2CZZYm2Zjk9r7aPklWJrmr/d271ZPkgiRrk9ya5Pl96yxuy9+VZHFf/fAkt7V1LkiSQfZHkjQc2wyVJP8myT7Avkn2boGwT5K5wOwBtn8JsHBc7QzguqqaB1zXpgFeBcxrwxLgotaGfYCzgBcARwBnjQVRW+atfeuNfy1J0hTa3pnKKcBq4Dfa37HhauDD29t4Vd0AbB5XPg64tI1fChzfV7+sem4E9kqyP/BKYGVVba6q7wMrgYVt3p5VdWNVFXBZ37YkSSOw27ZmVtUHgQ8meXtVfaij19yvqja08QeA/dr4bOD+vuXWtdq26usmqEuSRmSboTKmqj6U5IXA3P51quqyHXnxqqoktSPbGESSJfS61DjwwAOH/XKStNMa9EL9XwPvA14M/Ps2LHiCr/lg67qi/d3Y6uuBA/qWm9Nq26rPmaD+OFV1cVUtqKoFs2bNeoLNliRtz0BnKvQC5JB27WJHLQcWA+e1v1f31U9PsozeRfkfVNWGJNcC/73v4vwrgDOranOSHyY5EvgGcCLQVRedJOkJGDRUbgeeCWzY3oL9klwOHE3v7rF19O7iOg+4IsnJwH3AG9riK4BXA2uBnwBvBmjh8efATW25c6pq7OL/2+jdYfZU4Jo2SJJGZNBQ2Re4I8k3gYfHilV17LZWqqoTJpl1zATLFnDaJNtZCiydoL4KOGxbbZAkTZ1BQ+XsYTZCkvTkMOjdX9cPuyGSpJlvoFBJ8s/A2EX63YGnAD+uqj2H1TBJ0swz6JnK08fG2/O1jgOOHFajJEkz0y/8lOL2GJXP0Xt8iiRJjxq0++t1fZO70Pveys+G0iJJ0ow16N1f/7FvfAtwL70uMEmSHjXoNZU3D7shkqSZb9Bnf81JclX7wa2NSa5MMmf7a0qSdiaDXqj/OL1nc/1KGz7fapIkPWrQUJlVVR+vqi1tuATwcb+SpK0MGioPJXlTkl3b8CbgoWE2TJI08wwaKm+h9zThB+g9qfj1wElDapMkaYYa9Jbic4DF7TfiSbIPvR/tesuwGiZJmnkGPVN5zligQO83ToDnDadJkqSZatBQ2aXvlxfHzlQGPcuRJO0kBg2G9wNfT/LpNv27wLnDaZIkaaYa9Bv1lyVZBby0lV5XVXcMr1mSpJlo4C6sFiIGiSRpUr/wo+8lSZrMlIdKkmcnuaVv+GGSdyQ5O8n6vvqr+9Y5M8naJHcmeWVffWGrrU1yxlTviyRpa1N+B1dV3QnMB0iyK7AeuAp4M3B+Vb2vf/kkhwCLgEPpPXfsi0me1WZfCLwcWAfclGT5jl7rOfxPLtuR1Z9UVr/3xFE3QeO86EMvGnUTpo2/ffvfjroJmsCobws+Bri7qu7r/UrxhI4DllXVw8B3k6wFjmjz1lbVPQBJlrVlve4jSSMy6msqi4DL+6ZPT3JrkqV934uZDdzft8y6VpusLkkakZGFSpLdgWOBse++XAQcTK9rbAO978Z09VpLkqxKsmrTpk1dbVaSNM4oz1ReBXyrqh4EqKoHq+qRqvo58FEe6+JaDxzQt96cVpus/jhVdXFVLaiqBbNm+cR+SRqWUYbKCfR1fSXZv2/ea4Hb2/hyYFGSPZIcBMwDvgncBMxLclA761nUlpUkjchILtQn+WV6d22d0lf+yyTzgQLuHZtXVWuSXEHvAvwW4LSqeqRt53TgWmBXYGlVrZmynZAkPc5IQqWqfgw8Y1zt97ax/LlM8KyxqloBrOi8gZKkJ2TUd39Jkp5EDBVJUmcMFUlSZwwVSVJnDBVJUmcMFUlSZwwVSVJnDBVJUmcMFUlSZwwVSVJnDBVJUmcMFUlSZwwVSVJnDBVJUmcMFUlSZwwVSVJnDBVJUmcMFUlSZwwVSVJnDBVJUmcMFUlSZ0YWKknuTXJbkluSrGq1fZKsTHJX+7t3qyfJBUnWJrk1yfP7trO4LX9XksWj2h9J0ujPVH6nquZX1YI2fQZwXVXNA65r0wCvAua1YQlwEfRCCDgLeAFwBHDWWBBJkqbeqENlvOOAS9v4pcDxffXLqudGYK8k+wOvBFZW1eaq+j6wElg41Y2WJPWMMlQK+EKS1UmWtNp+VbWhjT8A7NfGZwP39627rtUmq28lyZIkq5Ks2rRpU5f7IEnqs9sIX/vFVbU+yb8DVib5Tv/Mqqok1cULVdXFwMUACxYs6GSbkqTHG9mZSlWtb383AlfRuybyYOvWov3d2BZfDxzQt/qcVpusLkkagZGESpJfTvL0sXHgFcDtwHJg7A6uxcDVbXw5cGK7C+xI4Aetm+xa4BVJ9m4X6F/RapKkERhV99d+wFVJxtrwyar6v0luAq5IcjJwH/CGtvwK4NXAWuAnwJsBqmpzkj8HbmrLnVNVm6duNyRJ/UYSKlV1D/DcCeoPAcdMUC/gtEm2tRRY2nUbJUm/uOl2S7EkaQYzVCRJnTFUJEmdMVQkSZ0xVCRJnTFUJEmdMVQkSZ0xVCRJnTFUJEmdMVQkSZ0xVCRJnTFUJEmdMVQkSZ0xVCRJnRnlzwlL0shc/5LfHnUTpo3fvuH6zrblmYokqTOGiiSpM4aKJKkzhookqTOGiiSpM1MeKkkOSPLlJHckWZPkD1r97CTrk9zShlf3rXNmkrVJ7kzyyr76wlZbm+SMqd4XSdLWRnFL8Rbgj6rqW0meDqxOsrLNO7+q3te/cJJDgEXAocCvAF9M8qw2+0Lg5cA64KYky6vqjinZC0nS40x5qFTVBmBDG//nJN8GZm9jleOAZVX1MPDdJGuBI9q8tVV1D0CSZW1ZQ0WSRmSk11SSzAWeB3yjlU5PcmuSpUn2brXZwP19q61rtcnqkqQRGVmoJHkacCXwjqr6IXARcDAwn96ZzPs7fK0lSVYlWbVp06auNitJGmckoZLkKfQC5RNV9VmAqnqwqh6pqp8DH+WxLq71wAF9q89ptcnqj1NVF1fVgqpaMGvWrG53RpL0qFHc/RXgY8C3q+qv+ur79y32WuD2Nr4cWJRkjyQHAfOAbwI3AfOSHJRkd3oX85dPxT5IkiY2iru/XgT8HnBbklta7c+AE5LMBwq4FzgFoKrWJLmC3gX4LcBpVfUIQJLTgWuBXYGlVbVmKndEkrS1Udz99TUgE8xasY11zgXOnaC+YlvrSZKmlt+olyR1xlCRJHXGUJEkdcZQkSR1xlCRJHXGUJEkdcZQkSR1xlCRJHXGUJEkdcZQkSR1xlCRJHXGUJEkdcZQkSR1xlCRJHXGUJEkdcZQkSR1xlCRJHXGUJEkdcZQkSR1xlCRJHVmxodKkoVJ7kyyNskZo26PJO3MZnSoJNkVuBB4FXAIcEKSQ0bbKknaec3oUAGOANZW1T1V9S/AMuC4EbdJknZaMz1UZgP3902vazVJ0gjsNuoGTIUkS4AlbfJHSe4cZXsGtC/wvVE2IO9bPMqX79LIjyUAZ2XULejKtDie+X2PZ2cy0LH81UEWmumhsh44oG96TqttpaouBi6eqkZ1Icmqqlow6nY8GXgsu+Xx7NaT7XjO9O6vm4B5SQ5KsjuwCFg+4jZJ0k5rRp+pVNWWJKcD1wK7Akuras2ImyVJO60ZHSoAVbUCWDHqdgzBjOqum+Y8lt3yeHbrSXU8U1WjboMk6Ulipl9TkSRNI4bKkCT50bjpk5J8uI2fnWR9klv6hr2SHJ3kB23620nO6lv/iCRfSXJXkm8l+T9JfqvN+8MkdyS5Ncl1SQa69W+m2M6xvCTJ6ydaPsk9SZ49bt4Hkrxr3LEeG17WlnlmkmVJ7k6yOsmKJM8a7l5OH0keacfj9iSfT7JXq89N8tM2744kH0myy7j62HDiqPdjFNqxuH1c7ewkf5zkvUm+0z6nV40d17bMi5N8s83/TvsaxNi8Z7fP/ti/C9O6u8xQGZ3zq2p+3/BPrf7VqpoPLADelOT5SfYDrgD+rKrmVdXzgf8BHNzWuRlYUFXPAT4D/OUU78t0tYzeHYEAJNkFeH2rQzvWfcMXkwS4CvhKVR1cVYcDZwL7TXXjR+in7XgcBmwGTuubd3d7fz6H3qORju+v9w2XTXGbZ4KVwGHtc/oP9N5XJHkm8Eng1Kr6DeDFwClJ/kNb7wIe+/fiN4EPTX3TB2eoTFNV9WNgNfDrwOnApVX1d33zv1ZVn2vjX66qn7RZN9L7vo7gcuCNfdMvAe6rqvu2sc7vAP9aVR8ZK1TV31fVV4fUxunu60zwlIqq2gL8Hb33pwZQVV9oxw22/pyeBlxSVd9qy30P+FNg7AG5+9N7WsjYdm6bmhY/MYbK8Dy1vzsAOGfc/Hf2zf/y+JWTPAM4ElgDHAp8a8DXPRm4ZkcaPg1t71hOqH34fp7kua20iF7QjDlqXJfNwcBh9MJ8p9ce2HoME3z3K8kvtXlj/8AdPO5YHjWFTZ2J3sJjn9NDefx7blWrA5wPfCnJNUne2d9tNh3N+FuKp7Gftm4CoHcdgF6X1pjzq+p9E6x3VJKbgZ8D51XVmox7hEKSbwB7Al+oqj/oq7+pvcZvd7YX08O2juVEty/21y4HFiVZQ6+r5qy+eV+tqtf0rzj+WO+kntrCezbwbXrdNmMObvMKuLqqrkkyl8e6xXZ2k91O+2g9ybuBLcAnBtpg1ceTXAsspPfA3FOSPLeqHt7Rxg6DZyrTz1er6nlVdXhfF8wa4PljC1TVC4D/AvzbsVq7yPxu4Njp+mYbkoeAvccmkuzD1s9RWga8AXgZcGtVPbid7a0BDu+6kTPMWIj/KhAmuKbS3qNnj6R109tW78fm0fdk+w/Ra4D/VI99n+MOHv+eO5zeexGAqvp/VbW0qo6jF0iHdd/0bhgqM8OFwElJXthX+6WxkSTPA/4XvUDZONWNG7GvAG9M7zE9ACcBj3YnVtXd9D7Q57F119dkvgTsMe7um+fsjN057Trd7wN/lMRejQFU1Y+ADUleCo/+J2ch8LUkC+ldKzm27xooPPb5nt/WeQbwHtoNN+n9EOFT2vgzgWcwwTMOpwvfKKPzztZdNeb4yRasqgeSvBF4T5LZwEZ6/1COXVt4L/A04NOt++Yfq+rY4TR7eqmqv0lyOLA6ySPA3cCp4xa7nF6ofHZc/ajWlTPmL6rqM0leC3wgybuAnwH3Au8Yyg5Mc1V1c5JbgROAbd2scPC4Y7m0qi4YbuumrROBC5P8VZv+b1V1d+vC2gNY2T6nN1bVqVW1of1b8NEkT6d3dviBqvp8W/8VwAeT/KxN/0lVPTB1u/OL8Rv1kqTO2P0lSeqMoSJJ6oyhIknqjKEiSeqMoSJJ6oyhIk1DefxTrf941G2SBmGoSJI6Y6hIUyjJie33NP4+yV8nmZXkyiQ3teFFo26jtCP8Rr00RZIcCvxn4IVV9b32CI8P03u46NeSHAhcC/zmKNsp7QhDRZo6LwU+3X4vg6ra3B4Eekjf05H3TPK0UTVQ2lGGijRauwBHVtXP+os+gl8zlddUpKnzJeB321Nox55g+wXg7WMLjD2pVpqpPFORpkj7wbVzgevbE5Vvpvdo+Qvbk4B3A27g8U9ZlmYMn1IsSeqM3V+SpM4YKpKkzhgqkqTOGCqSpM4YKpKkzhgqkqTOGCqSpM4YKpKkzvx/hK8t46+aZlgAAAAASUVORK5CYII=\n",
      "text/plain": [
       "<Figure size 432x288 with 1 Axes>"
      ]
     },
     "metadata": {
      "needs_background": "light"
     },
     "output_type": "display_data"
    }
   ],
   "source": [
    "sns.countplot(x='cell',data=df_all[df_all['ds'] == 'train'])"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 13,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead tr th {\n",
       "        text-align: left;\n",
       "    }\n",
       "\n",
       "    .dataframe thead tr:last-of-type th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr>\n",
       "      <th></th>\n",
       "      <th>experiment</th>\n",
       "      <th>id_code</th>\n",
       "      <th>plate</th>\n",
       "      <th>sirna</th>\n",
       "      <th>well</th>\n",
       "      <th>ds</th>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th></th>\n",
       "      <th>count</th>\n",
       "      <th>count</th>\n",
       "      <th>count</th>\n",
       "      <th>count</th>\n",
       "      <th>count</th>\n",
       "      <th>count</th>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>cell</th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>HEPG2</th>\n",
       "      <td>7750</td>\n",
       "      <td>7750</td>\n",
       "      <td>7750</td>\n",
       "      <td>7750</td>\n",
       "      <td>7750</td>\n",
       "      <td>7750</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>HUVEC</th>\n",
       "      <td>17688</td>\n",
       "      <td>17688</td>\n",
       "      <td>17688</td>\n",
       "      <td>17688</td>\n",
       "      <td>17688</td>\n",
       "      <td>17688</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>RPE</th>\n",
       "      <td>7753</td>\n",
       "      <td>7753</td>\n",
       "      <td>7753</td>\n",
       "      <td>7753</td>\n",
       "      <td>7753</td>\n",
       "      <td>7753</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>U2OS</th>\n",
       "      <td>3324</td>\n",
       "      <td>3324</td>\n",
       "      <td>3324</td>\n",
       "      <td>3324</td>\n",
       "      <td>3324</td>\n",
       "      <td>3324</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "      experiment id_code  plate  sirna   well     ds\n",
       "           count   count  count  count  count  count\n",
       "cell                                                \n",
       "HEPG2       7750    7750   7750   7750   7750   7750\n",
       "HUVEC      17688   17688  17688  17688  17688  17688\n",
       "RPE         7753    7753   7753   7753   7753   7753\n",
       "U2OS        3324    3324   3324   3324   3324   3324"
      ]
     },
     "execution_count": 13,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "df_all[df_all['ds'] == 'train'].groupby('cell').agg(['count'])"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 14,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "<matplotlib.axes._subplots.AxesSubplot at 0x7fa791b32748>"
      ]
     },
     "execution_count": 14,
     "metadata": {},
     "output_type": "execute_result"
    },
    {
     "data": {
      "image/png": "iVBORw0KGgoAAAANSUhEUgAAAY4AAAEKCAYAAAAFJbKyAAAABHNCSVQICAgIfAhkiAAAAAlwSFlzAAALEgAACxIB0t1+/AAAADl0RVh0U29mdHdhcmUAbWF0cGxvdGxpYiB2ZXJzaW9uIDMuMC4yLCBodHRwOi8vbWF0cGxvdGxpYi5vcmcvOIA7rQAAE0xJREFUeJzt3XuwZWV95vHvAy1GYpRbF2oDaQoZEyAq0IWMRM2IhZg4wKS8kIpDa6hqrSLeJhc1mZl2UGo0miCiMcMEFCzHDiEmYIKDDBeDMaDdYNAGHRoUoQektZHECyZNfvPHfk+zaU937xfOOvsc+vup2tVrvetda//2qr3P0+vda62dqkKSpEntNu0CJEmLi8EhSepicEiSuhgckqQuBockqYvBIUnqYnBIkroYHJKkLgaHJKnLkmkXMIT99tuvli9fPu0yJGlRWbdu3XeqaunO+j0ug2P58uWsXbt22mVI0qKS5M5J+jlUJUnqYnBIkroYHJKkLgaHJKmLwSFJ6mJwSJK6GBySpC4GhySpi8EhSeryuLxyXPPrW2f+wrRLWDAO+q9fmXYJ0uA84pAkdTE4JEldDA5JUheDQ5LUxeCQJHUxOCRJXQwOSVIXg0OS1MXgkCR1MTgkSV0MDklSF4NDktTF4JAkdTE4JEldDA5JUpdBgyPJW5OsT/LVJJ9M8lNJDk5yQ5INSf4syR6t7xPb/Ia2fPnYdt7R2r+e5KVD1ixJ2rHBgiPJMuBNwIqqOgLYHTgVeC9wdlU9E7gfOL2tcjpwf2s/u/UjyWFtvcOBE4E/TrL7UHVLknZs6KGqJcCTkiwB9gTuAV4MXNKWXwic0qZPbvO05ccnSWtfU1U/rqpvABuAYwauW5K0HYMFR1VtBN4PfItRYDwArAO+V1VbWre7gWVtehlwV1t3S+u/73j7LOtIkubZkENVezM6WjgYeAbw04yGmoZ6vlVJ1iZZu2nTpqGeRpJ2eUMOVb0E+EZVbaqqfwE+BRwH7NWGrgAOADa26Y3AgQBt+VOB7463z7LOVlV1XlWtqKoVS5cuHeL1SJIYNji+BRybZM/2XcXxwC3ANcArWp+VwKVt+rI2T1t+dVVVaz+1nXV1MHAo8MUB65Yk7cCSnXd5dKrqhiSXADcCW4CbgPOAvwHWJHl3azu/rXI+8PEkG4DNjM6koqrWJ7mYUehsAc6oqoeGqluStGODBQdAVa0GVm/TfAeznBVVVQ8Cr9zOds4CzprzAiVJ3bxyXJLUxeCQJHUxOCRJXQwOSVIXg0OS1MXgkCR1MTgkSV0MDklSF4NDktTF4JAkdTE4JEldDA5JUheDQ5LUxeCQJHUxOCRJXQwOSVIXg0OS1MXgkCR1MTgkSV0MDklSF4NDktTF4JAkdTE4JEldDA5JUheDQ5LUxeCQJHUxOCRJXQwOSVIXg0OS1MXgkCR1MTgkSV0MDklSF4NDktTF4JAkdTE4JEldDA5JUheDQ5LUxeCQJHUZNDiS7JXkkiRfS3Jrkn+bZJ8kVya5rf27d+ubJB9MsiHJzUmOGtvOytb/tiQrh6xZkrRjQx9xnAP876r6OeA5wK3A24GrqupQ4Ko2D/Ay4ND2WAV8BCDJPsBq4HnAMcDqmbCRJM2/wYIjyVOBFwLnA1TVP1fV94CTgQtbtwuBU9r0ycBFNXI9sFeSpwMvBa6sqs1VdT9wJXDiUHVLknZsyCOOg4FNwEeT3JTkT5P8NLB/Vd3T+twL7N+mlwF3ja1/d2vbXvsjJFmVZG2StZs2bZrjlyJJmjFkcCwBjgI+UlVHAj/g4WEpAKqqgJqLJ6uq86pqRVWtWLp06VxsUpI0iyGD427g7qq6oc1fwihIvt2GoGj/3teWbwQOHFv/gNa2vXZJ0hQMFhxVdS9wV5JntabjgVuAy4CZM6NWApe26cuA09rZVccCD7QhrSuAE5Ls3b4UP6G1SZKmYMnA238j8IkkewB3AK9jFFYXJzkduBN4Vet7OfDLwAbgh60vVbU5ybuAL7V+Z1bV5oHrliRtx6DBUVVfBlbMsuj4WfoWcMZ2tnMBcMHcVidJejS8clyS1MXgkCR1MTgkSV0MDklSF4NDktTF4JAkdTE4JEldDA5JUheDQ5LUxeCQJHUxOCRJXQwOSVIXg0OS1MXgkCR1mSg4klw1SZsk6fFvh7/HkeSngD2B/dqv76UtegqwbODaJEkL0M5+yOn1wFuAZwDreDg4/hH40IB1SZIWqB0GR1WdA5yT5I1Vde481SRJWsAm+unYqjo3yfOB5ePrVNVFA9UlSVqgJgqOJB8HDgG+DDzUmgswOCRpFzNRcAArgMOqqoYsRpK08E16HcdXgacNWYgkaXGY9IhjP+CWJF8EfjzTWFUnDVKVJGnBmjQ43jlkEZKkxWPSs6o+N3QhkqTFYdKzqv6J0VlUAHsATwB+UFVPGaowSdLCNOkRx8/MTCcJcDJw7FBFSZIWru6749bIXwEvHaAeSdICN+lQ1a+Oze7G6LqOBwepSJK0oE16VtW/H5veAnyT0XCVJGkXM+l3HK8buhBJ0uIw6VDVAcC5wHGt6TrgzVV191CFDe3o3/E2WzPWve+0aZegMcede9zOO+0i/u6NfzftEjSLSb8c/yhwGaPf5XgG8OnWJknaxUwaHEur6qNVtaU9PgYsHbAuSdICNWlwfDfJa5Ls3h6vAb47ZGGSpIVp0uD4DeBVwL3APcArgNcOVJMkaQGb9HTcM4GVVXU/QJJ9gPczChRJ0i5k0iOOZ8+EBkBVbQaOnGTFNrR1U5K/bvMHJ7khyYYkf5Zkj9b+xDa/oS1fPraNd7T2ryfxinVJmqJJg2O3JHvPzLQjjkmPVt4M3Do2/17g7Kp6JnA/cHprPx24v7Wf3fqR5DDgVOBw4ETgj5PsPuFzS5Lm2KTB8YfA3yd5V5J3AV8A/mBnK7XrP34F+NM2H+DFwCWty4XAKW365DZPW3782A0V11TVj6vqG8AG4JgJ65YkzbFJrxy/KMlaRn/0AX61qm6ZYNUPAL8LzNxdd1/ge1W1pc3fDSxr08uAu9rzbUnyQOu/DLh+bJvj60iS5tmkw020oJgkLABI8nLgvqpal+SXHkVtXZKsAlYBHHTQQUM/nSTtsrpvq97hOOCkJN8E1jA6WjkH2CvJTGAdAGxs0xuBAwHa8qcyulZka/ss62xVVedV1YqqWrF0qdcmStJQBguOqnpHVR1QVcsZfbl9dVX9OnANo+tAAFYCl7bpy9o8bfnVVVWt/dR21tXBwKHAF4eqW5K0YxMPVc2htwFrkrwbuAk4v7WfD3w8yQZgM6OwoarWJ7mY0TDZFuCMqnpo/suWJME8BUdVXQtc26bvYJazoqrqQeCV21n/LOCs4SqUJE1qyO84JEmPQwaHJKmLwSFJ6mJwSJK6GBySpC4GhySpi8EhSepicEiSuhgckqQuBockqYvBIUnqYnBIkroYHJKkLgaHJKmLwSFJ6mJwSJK6GBySpC4GhySpi8EhSepicEiSuhgckqQuBockqYvBIUnqYnBIkroYHJKkLgaHJKmLwSFJ6mJwSJK6GBySpC4GhySpi8EhSepicEiSuhgckqQuBockqYvBIUnqYnBIkroYHJKkLkumXYAkDelzL3zRtEtYMF70t5+bk+0MdsSR5MAk1yS5Jcn6JG9u7fskuTLJbe3fvVt7knwwyYYkNyc5amxbK1v/25KsHKpmSdLODTlUtQX4rao6DDgWOCPJYcDbgauq6lDgqjYP8DLg0PZYBXwERkEDrAaeBxwDrJ4JG0nS/BssOKrqnqq6sU3/E3ArsAw4GbiwdbsQOKVNnwxcVCPXA3sleTrwUuDKqtpcVfcDVwInDlW3JGnH5uXL8STLgSOBG4D9q+qetuheYP82vQy4a2y1u1vb9tolSVMweHAkeTLwF8Bbquofx5dVVQE1R8+zKsnaJGs3bdo0F5uUJM1i0OBI8gRGofGJqvpUa/52G4Ki/Xtfa98IHDi2+gGtbXvtj1BV51XViqpasXTp0rl9IZKkrYY8qyrA+cCtVfVHY4suA2bOjFoJXDrWflo7u+pY4IE2pHUFcEKSvduX4ie0NknSFAx5HcdxwH8EvpLky63t94D3ABcnOR24E3hVW3Y58MvABuCHwOsAqmpzkncBX2r9zqyqzQPWLUnagcGCo6o+D2Q7i4+fpX8BZ2xnWxcAF8xddZKkR8tbjkiSuhgckqQuBockqYvBIUnqYnBIkroYHJKkLgaHJKmLwSFJ6mJwSJK6GBySpC4GhySpi8EhSepicEiSuhgckqQuBockqYvBIUnqYnBIkroYHJKkLgaHJKmLwSFJ6mJwSJK6GBySpC4GhySpi8EhSepicEiSuhgckqQuBockqYvBIUnqYnBIkroYHJKkLgaHJKmLwSFJ6mJwSJK6GBySpC4GhySpi8EhSepicEiSuhgckqQuiyY4kpyY5OtJNiR5+7TrkaRd1aIIjiS7Ax8GXgYcBvxaksOmW5Uk7ZoWRXAAxwAbquqOqvpnYA1w8pRrkqRd0mIJjmXAXWPzd7c2SdI8WzLtAuZKklXAqjb7/SRfn2Y9E9oP+M60i8j7V067hLky/f25OlN9+jk29f2ZNz1u9ufU9yUA2en+/NlJNrNYgmMjcODY/AGtbauqOg84bz6LeqySrK2qFdOu4/HC/Tm33J9z5/G2LxfLUNWXgEOTHJxkD+BU4LIp1yRJu6RFccRRVVuS/CZwBbA7cEFVrZ9yWZK0S1oUwQFQVZcDl0+7jjm2qIbWFgH359xyf86dx9W+TFVNuwZJ0iKyWL7jkCQtEAbHY5Dk+9vMvzbJh9r0O5NsTPLlscdeSX4pyQNt/tYkq8fWPybJtUluS3Jjkr9J8gtt2X9KckuSm5NclWSi0+YWi53sy48lecVs/ZPckeRZ2yz7QJK3bbOvZx4vaX2elmRNktuTrEtyeZJ/M+yrXDiSPNT2x1eTfDrJXq19eZIftWW3JPmTJLtt0z7zOG3ar2Ma2r746jZt70zy20nel+Rr7XP6lzP7tfX5xSRfbMu/1i4hmFn2rPbZn/m7sKCHtgyOYZ1dVc8de3yvtV9XVc8FVgCvSXJUkv2Bi4Hfq6pDq+oo4L8Dh7R1bgJWVNWzgUuAP5jn17JQrWF0lh0ASXYDXtHaoe3rscf/SRLgL4Frq+qQqjoaeAew/3wXP0U/avvjCGAzcMbYstvb+/PZjG7xc8p4+9jjonmueTG4EjiifU7/L6P3FUmeBvwv4A1V9XPALwKvT/Irbb0P8vDfi58Hzp3/0idncExRVf0AWAc8E/hN4MKq+sLY8s9X1V+16Wuq6odt0fWMrmURfBJ49dj8C4E7q+rOHazz74B/qao/mWmoqn+oqusGqnGh+3tmuRNDVW0BvsDo/akJVNVn236DR35OzwA+VlU3tn7fAX4XmLlh69MZ3RFjZjtfmZ+KHx2D47F50vihO3DmNsvfOrb8mm1XTrIvcCywHjgcuHHC5z0d+MxjKXwB2tm+nFX7gP1rkue0plMZhcmMF2wzvHIIcASjwN7ltRuIHs8s10Ul2bMtm/kjdsg2+/IF81jqYvQbPPw5PZyffM+tbe0AZwNXJ/lMkreOD3EtRIvmdNwF6kftkB4YjcszGn6acXZVvX+W9V6Q5CbgX4H3VNX6bHMrgCQ3AE8BPltVbx5rf017jhfN2atYGHa0L2c79W+87ZPAqUnWMxpWWT227Lqqevn4itvu613Uk1pALwNuZTTEMuOQtqyAS6vqM0mW8/AQ1q5ue6eibm1P8vvAFuATE22w6qNJrgBOZHQD19cneU5V/fixFjsEjzim47qqOrKqjh4bLlkPHDXToaqeB/wX4Kkzbe2L3d8HTlqob6iBfBfYe2YmyT488r4/a4BXAS8Bbq6qb+9ke+uBo+e6yEVmJqh/FgizfMfR3qPvnEp1C9sj3o/N1vdk+0/Py4Ffr4evd7iFn3zPHc3ovQhAVf2/qrqgqk5mFDpHzH3pc8PgWDg+DLw2yfPH2vacmUhyJPA/GIXGffNd3JRdC7w6o9vNALwW2Dr0V1W3M/rQvodHDlNtz9XAE7c5q+XZu+LQS/ve7E3AbyVxBGICVfV94J4kL4at/5E5Efh8khMZfXdx0th3kvDw5/u5bZ19gffSTnLJ6IfqntCmnwbsyzb341tIfKMM661taGnGKdvrWFX3Jnk18N4ky4D7GP0xnBnrfx/wZODP21DLt6rqpGHKXliq6q+THA2sS/IQcDvwhm26fZJRcHxqm/YXtGGXGe+uqkuS/AfgA0neBjwIfBN4yyAvYIGrqpuS3Az8GrCjEwQO2WZfXlBVHxy2ugXrNODDSf6ozf+3qrq9DTc9EbiyfU6vr6o3VNU97W/B/0zyM4yO8j5QVZ9u658AnJPkwTb/O1V17/y9nD5eOS5J6uJQlSSpi8EhSepicEiSuhgckqQuBockqYvBIU1RfvKOyr897ZqknTE4JEldDA5pAElOa7/J8A9JPp5kaZK/SPKl9jhu2jVKj5ZXjktzLMnhwH8Gnl9V32m3pPgQo5tefj7JQcAVwM9Ps07p0TI4pLn3YuDP228uUFWb2w0qDxu7M+9Tkjx5WgVKj4XBIc2P3YBjq+rB8UZv8a7FyO84pLl3NfDKdgfUmbunfhZ440yHmbukSouRRxzSHGs/zHUW8Ll2N9+bGN26/MPtLrRLgL/lJ+/wKy0K3h1XktTFoSpJUheDQ5LUxeCQJHUxOCRJXQwOSVIXg0OS1MXgkCR1MTgkSV3+PwHdRRYjh3gEAAAAAElFTkSuQmCC\n",
      "text/plain": [
       "<Figure size 432x288 with 1 Axes>"
      ]
     },
     "metadata": {
      "needs_background": "light"
     },
     "output_type": "display_data"
    }
   ],
   "source": [
    "sns.countplot(x='cell',data=df_all[df_all['ds'] == 'test'])"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 15,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead tr th {\n",
       "        text-align: left;\n",
       "    }\n",
       "\n",
       "    .dataframe thead tr:last-of-type th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr>\n",
       "      <th></th>\n",
       "      <th>experiment</th>\n",
       "      <th>id_code</th>\n",
       "      <th>plate</th>\n",
       "      <th>sirna</th>\n",
       "      <th>well</th>\n",
       "      <th>ds</th>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th></th>\n",
       "      <th>count</th>\n",
       "      <th>count</th>\n",
       "      <th>count</th>\n",
       "      <th>count</th>\n",
       "      <th>count</th>\n",
       "      <th>count</th>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>cell</th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>HEPG2</th>\n",
       "      <td>4429</td>\n",
       "      <td>4429</td>\n",
       "      <td>4429</td>\n",
       "      <td>0</td>\n",
       "      <td>4429</td>\n",
       "      <td>4429</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>HUVEC</th>\n",
       "      <td>8846</td>\n",
       "      <td>8846</td>\n",
       "      <td>8846</td>\n",
       "      <td>0</td>\n",
       "      <td>8846</td>\n",
       "      <td>8846</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>RPE</th>\n",
       "      <td>4417</td>\n",
       "      <td>4417</td>\n",
       "      <td>4417</td>\n",
       "      <td>0</td>\n",
       "      <td>4417</td>\n",
       "      <td>4417</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>U2OS</th>\n",
       "      <td>2205</td>\n",
       "      <td>2205</td>\n",
       "      <td>2205</td>\n",
       "      <td>0</td>\n",
       "      <td>2205</td>\n",
       "      <td>2205</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "      experiment id_code plate sirna  well    ds\n",
       "           count   count count count count count\n",
       "cell                                            \n",
       "HEPG2       4429    4429  4429     0  4429  4429\n",
       "HUVEC       8846    8846  8846     0  8846  8846\n",
       "RPE         4417    4417  4417     0  4417  4417\n",
       "U2OS        2205    2205  2205     0  2205  2205"
      ]
     },
     "execution_count": 15,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "df_all[df_all['ds'] == 'test'].groupby('cell').agg(['count'])"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 16,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>id_code</th>\n",
       "      <th>experiment</th>\n",
       "      <th>plate</th>\n",
       "      <th>well</th>\n",
       "      <th>site</th>\n",
       "      <th>channel</th>\n",
       "      <th>mean</th>\n",
       "      <th>std</th>\n",
       "      <th>median</th>\n",
       "      <th>min</th>\n",
       "      <th>max</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>HEPG2-01_1_B02</td>\n",
       "      <td>HEPG2-01</td>\n",
       "      <td>1</td>\n",
       "      <td>B02</td>\n",
       "      <td>1</td>\n",
       "      <td>1</td>\n",
       "      <td>71.063782</td>\n",
       "      <td>43.146240</td>\n",
       "      <td>67.0</td>\n",
       "      <td>7</td>\n",
       "      <td>255</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>HEPG2-01_1_B02</td>\n",
       "      <td>HEPG2-01</td>\n",
       "      <td>1</td>\n",
       "      <td>B02</td>\n",
       "      <td>1</td>\n",
       "      <td>2</td>\n",
       "      <td>32.174431</td>\n",
       "      <td>9.384594</td>\n",
       "      <td>31.0</td>\n",
       "      <td>6</td>\n",
       "      <td>98</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2</th>\n",
       "      <td>HEPG2-01_1_B02</td>\n",
       "      <td>HEPG2-01</td>\n",
       "      <td>1</td>\n",
       "      <td>B02</td>\n",
       "      <td>1</td>\n",
       "      <td>3</td>\n",
       "      <td>61.836025</td>\n",
       "      <td>23.377997</td>\n",
       "      <td>59.0</td>\n",
       "      <td>11</td>\n",
       "      <td>255</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3</th>\n",
       "      <td>HEPG2-01_1_B02</td>\n",
       "      <td>HEPG2-01</td>\n",
       "      <td>1</td>\n",
       "      <td>B02</td>\n",
       "      <td>1</td>\n",
       "      <td>4</td>\n",
       "      <td>56.983257</td>\n",
       "      <td>16.011435</td>\n",
       "      <td>56.0</td>\n",
       "      <td>11</td>\n",
       "      <td>156</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4</th>\n",
       "      <td>HEPG2-01_1_B02</td>\n",
       "      <td>HEPG2-01</td>\n",
       "      <td>1</td>\n",
       "      <td>B02</td>\n",
       "      <td>1</td>\n",
       "      <td>5</td>\n",
       "      <td>91.671993</td>\n",
       "      <td>39.221836</td>\n",
       "      <td>85.0</td>\n",
       "      <td>13</td>\n",
       "      <td>255</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "          id_code experiment  plate well  site  channel       mean        std  \\\n",
       "0  HEPG2-01_1_B02   HEPG2-01      1  B02     1        1  71.063782  43.146240   \n",
       "1  HEPG2-01_1_B02   HEPG2-01      1  B02     1        2  32.174431   9.384594   \n",
       "2  HEPG2-01_1_B02   HEPG2-01      1  B02     1        3  61.836025  23.377997   \n",
       "3  HEPG2-01_1_B02   HEPG2-01      1  B02     1        4  56.983257  16.011435   \n",
       "4  HEPG2-01_1_B02   HEPG2-01      1  B02     1        5  91.671993  39.221836   \n",
       "\n",
       "   median  min  max  \n",
       "0    67.0    7  255  \n",
       "1    31.0    6   98  \n",
       "2    59.0   11  255  \n",
       "3    56.0   11  156  \n",
       "4    85.0   13  255  "
      ]
     },
     "execution_count": 16,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "df_pixels.head()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 17,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "min    0\n",
       "dtype: int64"
      ]
     },
     "execution_count": 17,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "df_pixels[['min']].min()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 18,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>id_code</th>\n",
       "      <th>experiment</th>\n",
       "      <th>plate</th>\n",
       "      <th>well</th>\n",
       "      <th>site</th>\n",
       "      <th>channel</th>\n",
       "      <th>mean</th>\n",
       "      <th>std</th>\n",
       "      <th>median</th>\n",
       "      <th>min</th>\n",
       "      <th>max</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>289009</th>\n",
       "      <td>HUVEC-09_3_E02</td>\n",
       "      <td>HUVEC-09</td>\n",
       "      <td>3</td>\n",
       "      <td>E02</td>\n",
       "      <td>1</td>\n",
       "      <td>2</td>\n",
       "      <td>77.361301</td>\n",
       "      <td>18.421216</td>\n",
       "      <td>71.0</td>\n",
       "      <td>53</td>\n",
       "      <td>255</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "               id_code experiment  plate well  site  channel       mean  \\\n",
       "289009  HUVEC-09_3_E02   HUVEC-09      3  E02     1        2  77.361301   \n",
       "\n",
       "              std  median  min  max  \n",
       "289009  18.421216    71.0   53  255  "
      ]
     },
     "execution_count": 18,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "df_pixels[df_pixels['min'] == df_pixels[['min']].max()[0]].head()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 19,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>id_code</th>\n",
       "      <th>experiment</th>\n",
       "      <th>plate</th>\n",
       "      <th>well</th>\n",
       "      <th>site</th>\n",
       "      <th>channel</th>\n",
       "      <th>mean</th>\n",
       "      <th>std</th>\n",
       "      <th>median</th>\n",
       "      <th>min</th>\n",
       "      <th>max</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>44380</th>\n",
       "      <td>HEPG2-04_1_B04</td>\n",
       "      <td>HEPG2-04</td>\n",
       "      <td>1</td>\n",
       "      <td>B04</td>\n",
       "      <td>1</td>\n",
       "      <td>5</td>\n",
       "      <td>2.099361</td>\n",
       "      <td>2.016894</td>\n",
       "      <td>1.0</td>\n",
       "      <td>0</td>\n",
       "      <td>32</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>44386</th>\n",
       "      <td>HEPG2-04_1_B04</td>\n",
       "      <td>HEPG2-04</td>\n",
       "      <td>1</td>\n",
       "      <td>B04</td>\n",
       "      <td>2</td>\n",
       "      <td>5</td>\n",
       "      <td>2.907871</td>\n",
       "      <td>2.342452</td>\n",
       "      <td>2.0</td>\n",
       "      <td>0</td>\n",
       "      <td>33</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>44392</th>\n",
       "      <td>HEPG2-04_1_B05</td>\n",
       "      <td>HEPG2-04</td>\n",
       "      <td>1</td>\n",
       "      <td>B05</td>\n",
       "      <td>1</td>\n",
       "      <td>5</td>\n",
       "      <td>2.140354</td>\n",
       "      <td>2.482237</td>\n",
       "      <td>1.0</td>\n",
       "      <td>0</td>\n",
       "      <td>225</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>44398</th>\n",
       "      <td>HEPG2-04_1_B05</td>\n",
       "      <td>HEPG2-04</td>\n",
       "      <td>1</td>\n",
       "      <td>B05</td>\n",
       "      <td>2</td>\n",
       "      <td>5</td>\n",
       "      <td>3.846188</td>\n",
       "      <td>2.685085</td>\n",
       "      <td>4.0</td>\n",
       "      <td>0</td>\n",
       "      <td>28</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>44404</th>\n",
       "      <td>HEPG2-04_1_B06</td>\n",
       "      <td>HEPG2-04</td>\n",
       "      <td>1</td>\n",
       "      <td>B06</td>\n",
       "      <td>1</td>\n",
       "      <td>5</td>\n",
       "      <td>2.967934</td>\n",
       "      <td>2.221915</td>\n",
       "      <td>3.0</td>\n",
       "      <td>0</td>\n",
       "      <td>33</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "              id_code experiment  plate well  site  channel      mean  \\\n",
       "44380  HEPG2-04_1_B04   HEPG2-04      1  B04     1        5  2.099361   \n",
       "44386  HEPG2-04_1_B04   HEPG2-04      1  B04     2        5  2.907871   \n",
       "44392  HEPG2-04_1_B05   HEPG2-04      1  B05     1        5  2.140354   \n",
       "44398  HEPG2-04_1_B05   HEPG2-04      1  B05     2        5  3.846188   \n",
       "44404  HEPG2-04_1_B06   HEPG2-04      1  B06     1        5  2.967934   \n",
       "\n",
       "            std  median  min  max  \n",
       "44380  2.016894     1.0    0   32  \n",
       "44386  2.342452     2.0    0   33  \n",
       "44392  2.482237     1.0    0  225  \n",
       "44398  2.685085     4.0    0   28  \n",
       "44404  2.221915     3.0    0   33  "
      ]
     },
     "execution_count": 19,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "df_pixels[df_pixels['min'] == df_pixels[['min']].min()[0]].head()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 20,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "36515\n"
     ]
    }
   ],
   "source": [
    "print(len(df))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 21,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead tr th {\n",
       "        text-align: left;\n",
       "    }\n",
       "\n",
       "    .dataframe thead tr:last-of-type th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr>\n",
       "      <th></th>\n",
       "      <th>id_code</th>\n",
       "      <th>experiment</th>\n",
       "      <th>plate</th>\n",
       "      <th>well</th>\n",
       "      <th>site</th>\n",
       "      <th>mean</th>\n",
       "      <th>std</th>\n",
       "      <th>median</th>\n",
       "      <th>min</th>\n",
       "      <th>max</th>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th></th>\n",
       "      <th>count</th>\n",
       "      <th>count</th>\n",
       "      <th>count</th>\n",
       "      <th>count</th>\n",
       "      <th>count</th>\n",
       "      <th>count</th>\n",
       "      <th>count</th>\n",
       "      <th>count</th>\n",
       "      <th>count</th>\n",
       "      <th>count</th>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>channel</th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>125510</td>\n",
       "      <td>125510</td>\n",
       "      <td>125510</td>\n",
       "      <td>125510</td>\n",
       "      <td>125510</td>\n",
       "      <td>125510</td>\n",
       "      <td>125510</td>\n",
       "      <td>125510</td>\n",
       "      <td>125510</td>\n",
       "      <td>125510</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2</th>\n",
       "      <td>125510</td>\n",
       "      <td>125510</td>\n",
       "      <td>125510</td>\n",
       "      <td>125510</td>\n",
       "      <td>125510</td>\n",
       "      <td>125510</td>\n",
       "      <td>125510</td>\n",
       "      <td>125510</td>\n",
       "      <td>125510</td>\n",
       "      <td>125510</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3</th>\n",
       "      <td>125510</td>\n",
       "      <td>125510</td>\n",
       "      <td>125510</td>\n",
       "      <td>125510</td>\n",
       "      <td>125510</td>\n",
       "      <td>125510</td>\n",
       "      <td>125510</td>\n",
       "      <td>125510</td>\n",
       "      <td>125510</td>\n",
       "      <td>125510</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4</th>\n",
       "      <td>125510</td>\n",
       "      <td>125510</td>\n",
       "      <td>125510</td>\n",
       "      <td>125510</td>\n",
       "      <td>125510</td>\n",
       "      <td>125510</td>\n",
       "      <td>125510</td>\n",
       "      <td>125510</td>\n",
       "      <td>125510</td>\n",
       "      <td>125510</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>5</th>\n",
       "      <td>125510</td>\n",
       "      <td>125510</td>\n",
       "      <td>125510</td>\n",
       "      <td>125510</td>\n",
       "      <td>125510</td>\n",
       "      <td>125510</td>\n",
       "      <td>125510</td>\n",
       "      <td>125510</td>\n",
       "      <td>125510</td>\n",
       "      <td>125510</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>6</th>\n",
       "      <td>125510</td>\n",
       "      <td>125510</td>\n",
       "      <td>125510</td>\n",
       "      <td>125510</td>\n",
       "      <td>125510</td>\n",
       "      <td>125510</td>\n",
       "      <td>125510</td>\n",
       "      <td>125510</td>\n",
       "      <td>125510</td>\n",
       "      <td>125510</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "        id_code experiment   plate    well    site    mean     std  median  \\\n",
       "          count      count   count   count   count   count   count   count   \n",
       "channel                                                                      \n",
       "1        125510     125510  125510  125510  125510  125510  125510  125510   \n",
       "2        125510     125510  125510  125510  125510  125510  125510  125510   \n",
       "3        125510     125510  125510  125510  125510  125510  125510  125510   \n",
       "4        125510     125510  125510  125510  125510  125510  125510  125510   \n",
       "5        125510     125510  125510  125510  125510  125510  125510  125510   \n",
       "6        125510     125510  125510  125510  125510  125510  125510  125510   \n",
       "\n",
       "            min     max  \n",
       "          count   count  \n",
       "channel                  \n",
       "1        125510  125510  \n",
       "2        125510  125510  \n",
       "3        125510  125510  \n",
       "4        125510  125510  \n",
       "5        125510  125510  \n",
       "6        125510  125510  "
      ]
     },
     "execution_count": 21,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "df_pixels.groupby('channel').agg(['count'])    "
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 22,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>id_code</th>\n",
       "      <th>experiment</th>\n",
       "      <th>plate</th>\n",
       "      <th>well</th>\n",
       "      <th>site</th>\n",
       "      <th>channel</th>\n",
       "      <th>mean</th>\n",
       "      <th>std</th>\n",
       "      <th>median</th>\n",
       "      <th>min</th>\n",
       "      <th>max</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>162535</th>\n",
       "      <td>HEPG2-11_4_O21</td>\n",
       "      <td>HEPG2-11</td>\n",
       "      <td>4</td>\n",
       "      <td>O21</td>\n",
       "      <td>2</td>\n",
       "      <td>2</td>\n",
       "      <td>23.061954</td>\n",
       "      <td>12.409125</td>\n",
       "      <td>21.0</td>\n",
       "      <td>2</td>\n",
       "      <td>112</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>162541</th>\n",
       "      <td>HEPG2-11_4_O22</td>\n",
       "      <td>HEPG2-11</td>\n",
       "      <td>4</td>\n",
       "      <td>O22</td>\n",
       "      <td>1</td>\n",
       "      <td>2</td>\n",
       "      <td>9.546902</td>\n",
       "      <td>9.937065</td>\n",
       "      <td>6.0</td>\n",
       "      <td>2</td>\n",
       "      <td>145</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>162547</th>\n",
       "      <td>HEPG2-11_4_O22</td>\n",
       "      <td>HEPG2-11</td>\n",
       "      <td>4</td>\n",
       "      <td>O22</td>\n",
       "      <td>2</td>\n",
       "      <td>2</td>\n",
       "      <td>9.051575</td>\n",
       "      <td>10.988752</td>\n",
       "      <td>4.0</td>\n",
       "      <td>2</td>\n",
       "      <td>220</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>162553</th>\n",
       "      <td>HEPG2-11_4_O23</td>\n",
       "      <td>HEPG2-11</td>\n",
       "      <td>4</td>\n",
       "      <td>O23</td>\n",
       "      <td>1</td>\n",
       "      <td>2</td>\n",
       "      <td>11.207123</td>\n",
       "      <td>12.421270</td>\n",
       "      <td>6.0</td>\n",
       "      <td>2</td>\n",
       "      <td>141</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>162559</th>\n",
       "      <td>HEPG2-11_4_O23</td>\n",
       "      <td>HEPG2-11</td>\n",
       "      <td>4</td>\n",
       "      <td>O23</td>\n",
       "      <td>2</td>\n",
       "      <td>2</td>\n",
       "      <td>11.352909</td>\n",
       "      <td>13.563623</td>\n",
       "      <td>6.0</td>\n",
       "      <td>1</td>\n",
       "      <td>168</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "               id_code experiment  plate well  site  channel       mean  \\\n",
       "162535  HEPG2-11_4_O21   HEPG2-11      4  O21     2        2  23.061954   \n",
       "162541  HEPG2-11_4_O22   HEPG2-11      4  O22     1        2   9.546902   \n",
       "162547  HEPG2-11_4_O22   HEPG2-11      4  O22     2        2   9.051575   \n",
       "162553  HEPG2-11_4_O23   HEPG2-11      4  O23     1        2  11.207123   \n",
       "162559  HEPG2-11_4_O23   HEPG2-11      4  O23     2        2  11.352909   \n",
       "\n",
       "              std  median  min  max  \n",
       "162535  12.409125    21.0    2  112  \n",
       "162541   9.937065     6.0    2  145  \n",
       "162547  10.988752     4.0    2  220  \n",
       "162553  12.421270     6.0    2  141  \n",
       "162559  13.563623     6.0    1  168  "
      ]
     },
     "execution_count": 22,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "m = df_pixels[df_pixels['channel'] == 2]\n",
    "m = m[m['experiment'].str.contains('HEPG2')]\n",
    "m.tail()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 23,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "channel 1 - mean: 5.845691592341334 | std: 6.9056823714669875\n",
      "\n",
      "channel 2 - mean: 15.567965862276298 | std: 12.556696025163996\n",
      "\n",
      "channel 3 - mean: 10.10558293628062 | std: 5.584809318973713\n",
      "\n",
      "channel 4 - mean: 9.964395871351234 | std: 7.445013704896161\n",
      "\n",
      "channel 5 - mean: 5.57672050803717 | std: 4.668733754481159\n",
      "\n",
      "channel 6 - mean: 9.06773161153738 | std: 4.91070227013248\n",
      "\n",
      "  cell  channel        std       mean\n",
      "0  ALL      1.0   6.905682   5.845692\n",
      "1  ALL      2.0  12.556696  15.567966\n",
      "2  ALL      3.0   5.584809  10.105583\n",
      "3  ALL      4.0   7.445014   9.964396\n",
      "4  ALL      5.0   4.668734   5.576721\n",
      "5  ALL      6.0   4.910702   9.067732\n"
     ]
    }
   ],
   "source": [
    "stats_data = {'cell': [], 'channel': [], 'std': [], 'mean': []}\n",
    "\n",
    "stats_df = pd.DataFrame(data=stats_data)\n",
    "for channel in range(1,7):\n",
    "    temp = df_pixels[df_pixels['channel'] == channel]\n",
    "    stats_df = stats_df.append({'cell': 'ALL', 'channel': int(channel), 'std': temp[[\"std\"]].mean()[0], 'mean': temp[[\"mean\"]].mean()[0]}, ignore_index=True)\n",
    "    print(f'channel {channel} - mean: {temp[[\"mean\"]].mean()[0]} | std: {temp[[\"std\"]].mean()[0]}\\n')\n",
    "    \n",
    "print(stats_df)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 24,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "\n",
      "\n",
      "\n",
      "channel 1 HEPG2 - mean: 11.031356732537494 | std: 12.383423263164476\n",
      "\n",
      "channel 2 HEPG2 - mean: 15.786221102312357 | std: 11.940651802029715\n",
      "\n",
      "channel 3 HEPG2 - mean: 12.883933155952173 | std: 7.404890056698376\n",
      "\n",
      "channel 4 HEPG2 - mean: 15.145480148271501 | std: 10.083671129382434\n",
      "\n",
      "channel 5 HEPG2 - mean: 11.517225424319587 | std: 7.959842438990438\n",
      "\n",
      "channel 6 HEPG2 - mean: 11.841935962489181 | std: 6.836244188216102\n",
      "\n",
      "\n",
      "\n",
      "\n",
      "channel 1 HUVEC - mean: 4.333631709629353 | std: 5.168027755364031\n",
      "\n",
      "channel 2 HUVEC - mean: 15.80156420876473 | std: 13.100209123833368\n",
      "\n",
      "channel 3 HUVEC - mean: 7.1570779968701 | std: 3.7150705725246076\n",
      "\n",
      "channel 4 HUVEC - mean: 8.393391375691078 | std: 6.873972929357964\n",
      "\n",
      "channel 5 HUVEC - mean: 2.821444750663893 | std: 3.022117393519594\n",
      "\n",
      "channel 6 HUVEC - mean: 6.563911710411285 | std: 3.3992237249709327\n",
      "\n",
      "\n",
      "\n",
      "\n",
      "channel 1 RPE - mean: 2.6980435886108265 | std: 3.515845198630972\n",
      "\n",
      "channel 2 RPE - mean: 12.683445297258121 | std: 11.777622340405468\n",
      "\n",
      "channel 3 RPE - mean: 12.579947023423431 | std: 7.804733725110398\n",
      "\n",
      "channel 4 RPE - mean: 7.074881386871394 | std: 6.014346908827438\n",
      "\n",
      "channel 5 RPE - mean: 4.344344068146034 | std: 4.3233574782077095\n",
      "\n",
      "channel 6 RPE - mean: 8.94631434293704 | std: 4.979595424792221\n",
      "\n",
      "\n",
      "\n",
      "\n",
      "channel 1 U2OS - mean: 8.610294408432136 | std: 10.643278795499091\n",
      "\n",
      "channel 2 U2OS - mean: 20.315573365640237 | std: 13.019137335474026\n",
      "\n",
      "channel 3 U2OS - mean: 12.695983064895957 | std: 5.666254650446652\n",
      "\n",
      "channel 4 U2OS - mean: 12.45387823673216 | std: 7.522788892443989\n",
      "\n",
      "channel 5 U2OS - mean: 8.431911098390875 | std: 6.084863651635668\n",
      "\n",
      "channel 6 U2OS - mean: 15.246410689108854 | std: 7.774823745945344\n",
      "\n",
      "     cell  channel        std       mean\n",
      "0     ALL      1.0   6.905682   5.845692\n",
      "1     ALL      2.0  12.556696  15.567966\n",
      "2     ALL      3.0   5.584809  10.105583\n",
      "3     ALL      4.0   7.445014   9.964396\n",
      "4     ALL      5.0   4.668734   5.576721\n",
      "5     ALL      6.0   4.910702   9.067732\n",
      "6   HEPG2      1.0  12.383423  11.031357\n",
      "7   HEPG2      2.0  11.940652  15.786221\n",
      "8   HEPG2      3.0   7.404890  12.883933\n",
      "9   HEPG2      4.0  10.083671  15.145480\n",
      "10  HEPG2      5.0   7.959842  11.517225\n",
      "11  HEPG2      6.0   6.836244  11.841936\n",
      "12  HUVEC      1.0   5.168028   4.333632\n",
      "13  HUVEC      2.0  13.100209  15.801564\n",
      "14  HUVEC      3.0   3.715071   7.157078\n",
      "15  HUVEC      4.0   6.873973   8.393391\n",
      "16  HUVEC      5.0   3.022117   2.821445\n",
      "17  HUVEC      6.0   3.399224   6.563912\n",
      "18    RPE      1.0   3.515845   2.698044\n",
      "19    RPE      2.0  11.777622  12.683445\n",
      "20    RPE      3.0   7.804734  12.579947\n",
      "21    RPE      4.0   6.014347   7.074881\n",
      "22    RPE      5.0   4.323357   4.344344\n",
      "23    RPE      6.0   4.979595   8.946314\n",
      "24   U2OS      1.0  10.643279   8.610294\n",
      "25   U2OS      2.0  13.019137  20.315573\n",
      "26   U2OS      3.0   5.666255  12.695983\n",
      "27   U2OS      4.0   7.522789  12.453878\n",
      "28   U2OS      5.0   6.084864   8.431911\n",
      "29   U2OS      6.0   7.774824  15.246411\n"
     ]
    }
   ],
   "source": [
    "for cell in ['HEPG2', 'HUVEC', 'RPE', 'U2OS']:\n",
    "    print('\\n\\n')\n",
    "    for channel in range(1,7):\n",
    "        temp = df_pixels[df_pixels['channel'] == channel]\n",
    "        temp = temp[temp['experiment'].str.contains(cell)]\n",
    "        stats_df = stats_df.append({'cell': cell, 'channel': int(channel), 'std': temp[[\"std\"]].mean()[0], 'mean': temp[[\"mean\"]].mean()[0]}, ignore_index=True)\n",
    "        print(f'channel {channel} {cell} - mean: {temp[[\"mean\"]].mean()[0]} | std: {temp[[\"std\"]].mean()[0]}\\n')\n",
    "        \n",
    "print(stats_df)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 25,
   "metadata": {},
   "outputs": [],
   "source": [
    "stats_df.to_csv('/storage/rxrxai/pixel_stats_agg.csv', index=False)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 26,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead tr th {\n",
       "        text-align: left;\n",
       "    }\n",
       "\n",
       "    .dataframe thead tr:last-of-type th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr>\n",
       "      <th></th>\n",
       "      <th>experiment</th>\n",
       "      <th>id_code</th>\n",
       "      <th>plate</th>\n",
       "      <th>well</th>\n",
       "      <th>ds</th>\n",
       "      <th>cell</th>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th></th>\n",
       "      <th>count</th>\n",
       "      <th>count</th>\n",
       "      <th>count</th>\n",
       "      <th>count</th>\n",
       "      <th>count</th>\n",
       "      <th>count</th>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>sirna</th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0.0</th>\n",
       "      <td>33</td>\n",
       "      <td>33</td>\n",
       "      <td>33</td>\n",
       "      <td>33</td>\n",
       "      <td>33</td>\n",
       "      <td>33</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1.0</th>\n",
       "      <td>33</td>\n",
       "      <td>33</td>\n",
       "      <td>33</td>\n",
       "      <td>33</td>\n",
       "      <td>33</td>\n",
       "      <td>33</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2.0</th>\n",
       "      <td>33</td>\n",
       "      <td>33</td>\n",
       "      <td>33</td>\n",
       "      <td>33</td>\n",
       "      <td>33</td>\n",
       "      <td>33</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3.0</th>\n",
       "      <td>33</td>\n",
       "      <td>33</td>\n",
       "      <td>33</td>\n",
       "      <td>33</td>\n",
       "      <td>33</td>\n",
       "      <td>33</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4.0</th>\n",
       "      <td>33</td>\n",
       "      <td>33</td>\n",
       "      <td>33</td>\n",
       "      <td>33</td>\n",
       "      <td>33</td>\n",
       "      <td>33</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>5.0</th>\n",
       "      <td>33</td>\n",
       "      <td>33</td>\n",
       "      <td>33</td>\n",
       "      <td>33</td>\n",
       "      <td>33</td>\n",
       "      <td>33</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>6.0</th>\n",
       "      <td>33</td>\n",
       "      <td>33</td>\n",
       "      <td>33</td>\n",
       "      <td>33</td>\n",
       "      <td>33</td>\n",
       "      <td>33</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>7.0</th>\n",
       "      <td>33</td>\n",
       "      <td>33</td>\n",
       "      <td>33</td>\n",
       "      <td>33</td>\n",
       "      <td>33</td>\n",
       "      <td>33</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>8.0</th>\n",
       "      <td>33</td>\n",
       "      <td>33</td>\n",
       "      <td>33</td>\n",
       "      <td>33</td>\n",
       "      <td>33</td>\n",
       "      <td>33</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>9.0</th>\n",
       "      <td>33</td>\n",
       "      <td>33</td>\n",
       "      <td>33</td>\n",
       "      <td>33</td>\n",
       "      <td>33</td>\n",
       "      <td>33</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>10.0</th>\n",
       "      <td>33</td>\n",
       "      <td>33</td>\n",
       "      <td>33</td>\n",
       "      <td>33</td>\n",
       "      <td>33</td>\n",
       "      <td>33</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>11.0</th>\n",
       "      <td>33</td>\n",
       "      <td>33</td>\n",
       "      <td>33</td>\n",
       "      <td>33</td>\n",
       "      <td>33</td>\n",
       "      <td>33</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>12.0</th>\n",
       "      <td>33</td>\n",
       "      <td>33</td>\n",
       "      <td>33</td>\n",
       "      <td>33</td>\n",
       "      <td>33</td>\n",
       "      <td>33</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>13.0</th>\n",
       "      <td>33</td>\n",
       "      <td>33</td>\n",
       "      <td>33</td>\n",
       "      <td>33</td>\n",
       "      <td>33</td>\n",
       "      <td>33</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>14.0</th>\n",
       "      <td>33</td>\n",
       "      <td>33</td>\n",
       "      <td>33</td>\n",
       "      <td>33</td>\n",
       "      <td>33</td>\n",
       "      <td>33</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>15.0</th>\n",
       "      <td>33</td>\n",
       "      <td>33</td>\n",
       "      <td>33</td>\n",
       "      <td>33</td>\n",
       "      <td>33</td>\n",
       "      <td>33</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>16.0</th>\n",
       "      <td>33</td>\n",
       "      <td>33</td>\n",
       "      <td>33</td>\n",
       "      <td>33</td>\n",
       "      <td>33</td>\n",
       "      <td>33</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>17.0</th>\n",
       "      <td>33</td>\n",
       "      <td>33</td>\n",
       "      <td>33</td>\n",
       "      <td>33</td>\n",
       "      <td>33</td>\n",
       "      <td>33</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>18.0</th>\n",
       "      <td>33</td>\n",
       "      <td>33</td>\n",
       "      <td>33</td>\n",
       "      <td>33</td>\n",
       "      <td>33</td>\n",
       "      <td>33</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>19.0</th>\n",
       "      <td>33</td>\n",
       "      <td>33</td>\n",
       "      <td>33</td>\n",
       "      <td>33</td>\n",
       "      <td>33</td>\n",
       "      <td>33</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>20.0</th>\n",
       "      <td>33</td>\n",
       "      <td>33</td>\n",
       "      <td>33</td>\n",
       "      <td>33</td>\n",
       "      <td>33</td>\n",
       "      <td>33</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>21.0</th>\n",
       "      <td>33</td>\n",
       "      <td>33</td>\n",
       "      <td>33</td>\n",
       "      <td>33</td>\n",
       "      <td>33</td>\n",
       "      <td>33</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>22.0</th>\n",
       "      <td>33</td>\n",
       "      <td>33</td>\n",
       "      <td>33</td>\n",
       "      <td>33</td>\n",
       "      <td>33</td>\n",
       "      <td>33</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>23.0</th>\n",
       "      <td>33</td>\n",
       "      <td>33</td>\n",
       "      <td>33</td>\n",
       "      <td>33</td>\n",
       "      <td>33</td>\n",
       "      <td>33</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>24.0</th>\n",
       "      <td>33</td>\n",
       "      <td>33</td>\n",
       "      <td>33</td>\n",
       "      <td>33</td>\n",
       "      <td>33</td>\n",
       "      <td>33</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>25.0</th>\n",
       "      <td>32</td>\n",
       "      <td>32</td>\n",
       "      <td>32</td>\n",
       "      <td>32</td>\n",
       "      <td>32</td>\n",
       "      <td>32</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>26.0</th>\n",
       "      <td>33</td>\n",
       "      <td>33</td>\n",
       "      <td>33</td>\n",
       "      <td>33</td>\n",
       "      <td>33</td>\n",
       "      <td>33</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>27.0</th>\n",
       "      <td>33</td>\n",
       "      <td>33</td>\n",
       "      <td>33</td>\n",
       "      <td>33</td>\n",
       "      <td>33</td>\n",
       "      <td>33</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>28.0</th>\n",
       "      <td>33</td>\n",
       "      <td>33</td>\n",
       "      <td>33</td>\n",
       "      <td>33</td>\n",
       "      <td>33</td>\n",
       "      <td>33</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>29.0</th>\n",
       "      <td>33</td>\n",
       "      <td>33</td>\n",
       "      <td>33</td>\n",
       "      <td>33</td>\n",
       "      <td>33</td>\n",
       "      <td>33</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>...</th>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1078.0</th>\n",
       "      <td>33</td>\n",
       "      <td>33</td>\n",
       "      <td>33</td>\n",
       "      <td>33</td>\n",
       "      <td>33</td>\n",
       "      <td>33</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1079.0</th>\n",
       "      <td>33</td>\n",
       "      <td>33</td>\n",
       "      <td>33</td>\n",
       "      <td>33</td>\n",
       "      <td>33</td>\n",
       "      <td>33</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1080.0</th>\n",
       "      <td>33</td>\n",
       "      <td>33</td>\n",
       "      <td>33</td>\n",
       "      <td>33</td>\n",
       "      <td>33</td>\n",
       "      <td>33</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1081.0</th>\n",
       "      <td>33</td>\n",
       "      <td>33</td>\n",
       "      <td>33</td>\n",
       "      <td>33</td>\n",
       "      <td>33</td>\n",
       "      <td>33</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1082.0</th>\n",
       "      <td>33</td>\n",
       "      <td>33</td>\n",
       "      <td>33</td>\n",
       "      <td>33</td>\n",
       "      <td>33</td>\n",
       "      <td>33</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1083.0</th>\n",
       "      <td>33</td>\n",
       "      <td>33</td>\n",
       "      <td>33</td>\n",
       "      <td>33</td>\n",
       "      <td>33</td>\n",
       "      <td>33</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1084.0</th>\n",
       "      <td>33</td>\n",
       "      <td>33</td>\n",
       "      <td>33</td>\n",
       "      <td>33</td>\n",
       "      <td>33</td>\n",
       "      <td>33</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1085.0</th>\n",
       "      <td>33</td>\n",
       "      <td>33</td>\n",
       "      <td>33</td>\n",
       "      <td>33</td>\n",
       "      <td>33</td>\n",
       "      <td>33</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1086.0</th>\n",
       "      <td>33</td>\n",
       "      <td>33</td>\n",
       "      <td>33</td>\n",
       "      <td>33</td>\n",
       "      <td>33</td>\n",
       "      <td>33</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1087.0</th>\n",
       "      <td>32</td>\n",
       "      <td>32</td>\n",
       "      <td>32</td>\n",
       "      <td>32</td>\n",
       "      <td>32</td>\n",
       "      <td>32</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1088.0</th>\n",
       "      <td>33</td>\n",
       "      <td>33</td>\n",
       "      <td>33</td>\n",
       "      <td>33</td>\n",
       "      <td>33</td>\n",
       "      <td>33</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1089.0</th>\n",
       "      <td>33</td>\n",
       "      <td>33</td>\n",
       "      <td>33</td>\n",
       "      <td>33</td>\n",
       "      <td>33</td>\n",
       "      <td>33</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1090.0</th>\n",
       "      <td>33</td>\n",
       "      <td>33</td>\n",
       "      <td>33</td>\n",
       "      <td>33</td>\n",
       "      <td>33</td>\n",
       "      <td>33</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1091.0</th>\n",
       "      <td>33</td>\n",
       "      <td>33</td>\n",
       "      <td>33</td>\n",
       "      <td>33</td>\n",
       "      <td>33</td>\n",
       "      <td>33</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1092.0</th>\n",
       "      <td>33</td>\n",
       "      <td>33</td>\n",
       "      <td>33</td>\n",
       "      <td>33</td>\n",
       "      <td>33</td>\n",
       "      <td>33</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1093.0</th>\n",
       "      <td>33</td>\n",
       "      <td>33</td>\n",
       "      <td>33</td>\n",
       "      <td>33</td>\n",
       "      <td>33</td>\n",
       "      <td>33</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1094.0</th>\n",
       "      <td>33</td>\n",
       "      <td>33</td>\n",
       "      <td>33</td>\n",
       "      <td>33</td>\n",
       "      <td>33</td>\n",
       "      <td>33</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1095.0</th>\n",
       "      <td>33</td>\n",
       "      <td>33</td>\n",
       "      <td>33</td>\n",
       "      <td>33</td>\n",
       "      <td>33</td>\n",
       "      <td>33</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1096.0</th>\n",
       "      <td>33</td>\n",
       "      <td>33</td>\n",
       "      <td>33</td>\n",
       "      <td>33</td>\n",
       "      <td>33</td>\n",
       "      <td>33</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1097.0</th>\n",
       "      <td>33</td>\n",
       "      <td>33</td>\n",
       "      <td>33</td>\n",
       "      <td>33</td>\n",
       "      <td>33</td>\n",
       "      <td>33</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1098.0</th>\n",
       "      <td>33</td>\n",
       "      <td>33</td>\n",
       "      <td>33</td>\n",
       "      <td>33</td>\n",
       "      <td>33</td>\n",
       "      <td>33</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1099.0</th>\n",
       "      <td>33</td>\n",
       "      <td>33</td>\n",
       "      <td>33</td>\n",
       "      <td>33</td>\n",
       "      <td>33</td>\n",
       "      <td>33</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1100.0</th>\n",
       "      <td>33</td>\n",
       "      <td>33</td>\n",
       "      <td>33</td>\n",
       "      <td>33</td>\n",
       "      <td>33</td>\n",
       "      <td>33</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1101.0</th>\n",
       "      <td>32</td>\n",
       "      <td>32</td>\n",
       "      <td>32</td>\n",
       "      <td>32</td>\n",
       "      <td>32</td>\n",
       "      <td>32</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1102.0</th>\n",
       "      <td>33</td>\n",
       "      <td>33</td>\n",
       "      <td>33</td>\n",
       "      <td>33</td>\n",
       "      <td>33</td>\n",
       "      <td>33</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1103.0</th>\n",
       "      <td>33</td>\n",
       "      <td>33</td>\n",
       "      <td>33</td>\n",
       "      <td>33</td>\n",
       "      <td>33</td>\n",
       "      <td>33</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1104.0</th>\n",
       "      <td>33</td>\n",
       "      <td>33</td>\n",
       "      <td>33</td>\n",
       "      <td>33</td>\n",
       "      <td>33</td>\n",
       "      <td>33</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1105.0</th>\n",
       "      <td>33</td>\n",
       "      <td>33</td>\n",
       "      <td>33</td>\n",
       "      <td>33</td>\n",
       "      <td>33</td>\n",
       "      <td>33</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1106.0</th>\n",
       "      <td>33</td>\n",
       "      <td>33</td>\n",
       "      <td>33</td>\n",
       "      <td>33</td>\n",
       "      <td>33</td>\n",
       "      <td>33</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1107.0</th>\n",
       "      <td>29</td>\n",
       "      <td>29</td>\n",
       "      <td>29</td>\n",
       "      <td>29</td>\n",
       "      <td>29</td>\n",
       "      <td>29</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "<p>1108 rows × 6 columns</p>\n",
       "</div>"
      ],
      "text/plain": [
       "       experiment id_code plate  well    ds  cell\n",
       "            count   count count count count count\n",
       "sirna                                            \n",
       "0.0            33      33    33    33    33    33\n",
       "1.0            33      33    33    33    33    33\n",
       "2.0            33      33    33    33    33    33\n",
       "3.0            33      33    33    33    33    33\n",
       "4.0            33      33    33    33    33    33\n",
       "5.0            33      33    33    33    33    33\n",
       "6.0            33      33    33    33    33    33\n",
       "7.0            33      33    33    33    33    33\n",
       "8.0            33      33    33    33    33    33\n",
       "9.0            33      33    33    33    33    33\n",
       "10.0           33      33    33    33    33    33\n",
       "11.0           33      33    33    33    33    33\n",
       "12.0           33      33    33    33    33    33\n",
       "13.0           33      33    33    33    33    33\n",
       "14.0           33      33    33    33    33    33\n",
       "15.0           33      33    33    33    33    33\n",
       "16.0           33      33    33    33    33    33\n",
       "17.0           33      33    33    33    33    33\n",
       "18.0           33      33    33    33    33    33\n",
       "19.0           33      33    33    33    33    33\n",
       "20.0           33      33    33    33    33    33\n",
       "21.0           33      33    33    33    33    33\n",
       "22.0           33      33    33    33    33    33\n",
       "23.0           33      33    33    33    33    33\n",
       "24.0           33      33    33    33    33    33\n",
       "25.0           32      32    32    32    32    32\n",
       "26.0           33      33    33    33    33    33\n",
       "27.0           33      33    33    33    33    33\n",
       "28.0           33      33    33    33    33    33\n",
       "29.0           33      33    33    33    33    33\n",
       "...           ...     ...   ...   ...   ...   ...\n",
       "1078.0         33      33    33    33    33    33\n",
       "1079.0         33      33    33    33    33    33\n",
       "1080.0         33      33    33    33    33    33\n",
       "1081.0         33      33    33    33    33    33\n",
       "1082.0         33      33    33    33    33    33\n",
       "1083.0         33      33    33    33    33    33\n",
       "1084.0         33      33    33    33    33    33\n",
       "1085.0         33      33    33    33    33    33\n",
       "1086.0         33      33    33    33    33    33\n",
       "1087.0         32      32    32    32    32    32\n",
       "1088.0         33      33    33    33    33    33\n",
       "1089.0         33      33    33    33    33    33\n",
       "1090.0         33      33    33    33    33    33\n",
       "1091.0         33      33    33    33    33    33\n",
       "1092.0         33      33    33    33    33    33\n",
       "1093.0         33      33    33    33    33    33\n",
       "1094.0         33      33    33    33    33    33\n",
       "1095.0         33      33    33    33    33    33\n",
       "1096.0         33      33    33    33    33    33\n",
       "1097.0         33      33    33    33    33    33\n",
       "1098.0         33      33    33    33    33    33\n",
       "1099.0         33      33    33    33    33    33\n",
       "1100.0         33      33    33    33    33    33\n",
       "1101.0         32      32    32    32    32    32\n",
       "1102.0         33      33    33    33    33    33\n",
       "1103.0         33      33    33    33    33    33\n",
       "1104.0         33      33    33    33    33    33\n",
       "1105.0         33      33    33    33    33    33\n",
       "1106.0         33      33    33    33    33    33\n",
       "1107.0         29      29    29    29    29    29\n",
       "\n",
       "[1108 rows x 6 columns]"
      ]
     },
     "execution_count": 26,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "df_train_sirna = df_all[df_all['ds'] == 'train']\n",
    "df_train_sirna.groupby('sirna').agg(['count'])"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 27,
   "metadata": {},
   "outputs": [],
   "source": [
    "# !cat /storage/rxrxai/sample_submission.csv`"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# data augmentation"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## data augmentation"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 28,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead tr th {\n",
       "        text-align: left;\n",
       "    }\n",
       "\n",
       "    .dataframe thead tr:last-of-type th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr>\n",
       "      <th></th>\n",
       "      <th>experiment</th>\n",
       "      <th>id_code</th>\n",
       "      <th>plate</th>\n",
       "      <th>sirna</th>\n",
       "      <th>well</th>\n",
       "      <th>ds</th>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th></th>\n",
       "      <th>count</th>\n",
       "      <th>count</th>\n",
       "      <th>count</th>\n",
       "      <th>count</th>\n",
       "      <th>count</th>\n",
       "      <th>count</th>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>cell</th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>HEPG2</th>\n",
       "      <td>7750</td>\n",
       "      <td>7750</td>\n",
       "      <td>7750</td>\n",
       "      <td>7750</td>\n",
       "      <td>7750</td>\n",
       "      <td>7750</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>HUVEC</th>\n",
       "      <td>17688</td>\n",
       "      <td>17688</td>\n",
       "      <td>17688</td>\n",
       "      <td>17688</td>\n",
       "      <td>17688</td>\n",
       "      <td>17688</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>RPE</th>\n",
       "      <td>7753</td>\n",
       "      <td>7753</td>\n",
       "      <td>7753</td>\n",
       "      <td>7753</td>\n",
       "      <td>7753</td>\n",
       "      <td>7753</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>U2OS</th>\n",
       "      <td>3324</td>\n",
       "      <td>3324</td>\n",
       "      <td>3324</td>\n",
       "      <td>3324</td>\n",
       "      <td>3324</td>\n",
       "      <td>3324</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "      experiment id_code  plate  sirna   well     ds\n",
       "           count   count  count  count  count  count\n",
       "cell                                                \n",
       "HEPG2       7750    7750   7750   7750   7750   7750\n",
       "HUVEC      17688   17688  17688  17688  17688  17688\n",
       "RPE         7753    7753   7753   7753   7753   7753\n",
       "U2OS        3324    3324   3324   3324   3324   3324"
      ]
     },
     "execution_count": 28,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "df_all[df_all['ds'] == 'train'].groupby('cell').agg(['count'])"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 29,
   "metadata": {},
   "outputs": [],
   "source": [
    "n_augmentations_dict = {\n",
    "    \"HEPG2\": 4,\n",
    "    \"HUVEC\": 1,\n",
    "    \"RPE\": 4,\n",
    "    \"U2OS\": 10,\n",
    "}"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 30,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "36515\n"
     ]
    }
   ],
   "source": [
    "df_train_cp = df_all[df_all['ds'] == 'train'].copy()\n",
    "df_train_cp['aug'] = False\n",
    "df_train_cp.head()\n",
    "print(len(df_train_cp))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 31,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "36564\n",
      "36564\n"
     ]
    }
   ],
   "source": [
    "experiments = list(df_train_cp[['experiment']].T.squeeze().unique())\n",
    "experiments\n",
    "print(len(experiments)*1108)\n",
    "print(len(experiments)*1108)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 32,
   "metadata": {},
   "outputs": [],
   "source": [
    "sub_valid = []\n",
    "five_percent = 55\n",
    "for exp in experiments:\n",
    "    sub_valid += [df_train_cp.loc[df_train_cp['experiment'] == exp].sample(n=five_percent, random_state=100)]\n",
    "\n",
    "df_valid = pd.concat(sub_valid)\n",
    "df_valid.head()\n",
    "df_valid.to_csv('/storage/rxrxai/validation.csv')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 33,
   "metadata": {},
   "outputs": [],
   "source": [
    "def add_aug_rows(packed_args):\n",
    "    i, row = packed_args\n",
    "\n",
    "    df = pd.DataFrame(data={'experiment': [], 'id_code': [], 'plate': [], 'sirna': [], 'well': [], 'ds': [], 'cell': [], 'aug': []})\n",
    "\n",
    "    for nth_aug in range(n_augmentations_dict[row['cell']]):\n",
    "        df = df.append({'experiment': f'{row[\"experiment\"]}_a{nth_aug}', 'id_code': f'{row[\"id_code\"]}_a{nth_aug}', 'plate': row['plate'], 'sirna': row['sirna'], 'well': row['well'], 'ds': row['ds'], 'cell': row['cell'], 'aug': True}, ignore_index=True)\n",
    "    \n",
    "    return df"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 34,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "CPU times: user 45.6 s, sys: 771 ms, total: 46.4 s\n",
      "Wall time: 2min 50s\n"
     ]
    }
   ],
   "source": [
    "%%time\n",
    "pool = Pool()                   \n",
    "augs = pool.map(add_aug_rows, df_train_cp.iterrows())"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 35,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "True\n",
      "36515\n",
      "[    experiment            id_code  plate  sirna well     ds   cell  aug\n",
      "0  HEPG2-01_a0  HEPG2-01_1_B03_a0    1.0  513.0  B03  train  HEPG2  1.0\n",
      "1  HEPG2-01_a1  HEPG2-01_1_B03_a1    1.0  513.0  B03  train  HEPG2  1.0\n",
      "2  HEPG2-01_a2  HEPG2-01_1_B03_a2    1.0  513.0  B03  train  HEPG2  1.0\n",
      "3  HEPG2-01_a3  HEPG2-01_1_B03_a3    1.0  513.0  B03  train  HEPG2  1.0,     experiment            id_code  plate  sirna well     ds   cell  aug\n",
      "0  HEPG2-01_a0  HEPG2-01_1_B04_a0    1.0  840.0  B04  train  HEPG2  1.0\n",
      "1  HEPG2-01_a1  HEPG2-01_1_B04_a1    1.0  840.0  B04  train  HEPG2  1.0\n",
      "2  HEPG2-01_a2  HEPG2-01_1_B04_a2    1.0  840.0  B04  train  HEPG2  1.0\n",
      "3  HEPG2-01_a3  HEPG2-01_1_B04_a3    1.0  840.0  B04  train  HEPG2  1.0,     experiment            id_code  plate   sirna well     ds   cell  aug\n",
      "0  HEPG2-01_a0  HEPG2-01_1_B05_a0    1.0  1020.0  B05  train  HEPG2  1.0\n",
      "1  HEPG2-01_a1  HEPG2-01_1_B05_a1    1.0  1020.0  B05  train  HEPG2  1.0\n",
      "2  HEPG2-01_a2  HEPG2-01_1_B05_a2    1.0  1020.0  B05  train  HEPG2  1.0\n",
      "3  HEPG2-01_a3  HEPG2-01_1_B05_a3    1.0  1020.0  B05  train  HEPG2  1.0,     experiment            id_code  plate  sirna well     ds   cell  aug\n",
      "0  HEPG2-01_a0  HEPG2-01_1_B06_a0    1.0  254.0  B06  train  HEPG2  1.0\n",
      "1  HEPG2-01_a1  HEPG2-01_1_B06_a1    1.0  254.0  B06  train  HEPG2  1.0\n",
      "2  HEPG2-01_a2  HEPG2-01_1_B06_a2    1.0  254.0  B06  train  HEPG2  1.0\n",
      "3  HEPG2-01_a3  HEPG2-01_1_B06_a3    1.0  254.0  B06  train  HEPG2  1.0,     experiment            id_code  plate  sirna well     ds   cell  aug\n",
      "0  HEPG2-01_a0  HEPG2-01_1_B07_a0    1.0  144.0  B07  train  HEPG2  1.0\n",
      "1  HEPG2-01_a1  HEPG2-01_1_B07_a1    1.0  144.0  B07  train  HEPG2  1.0\n",
      "2  HEPG2-01_a2  HEPG2-01_1_B07_a2    1.0  144.0  B07  train  HEPG2  1.0\n",
      "3  HEPG2-01_a3  HEPG2-01_1_B07_a3    1.0  144.0  B07  train  HEPG2  1.0,     experiment            id_code  plate  sirna well     ds   cell  aug\n",
      "0  HEPG2-01_a0  HEPG2-01_1_B08_a0    1.0  503.0  B08  train  HEPG2  1.0\n",
      "1  HEPG2-01_a1  HEPG2-01_1_B08_a1    1.0  503.0  B08  train  HEPG2  1.0\n",
      "2  HEPG2-01_a2  HEPG2-01_1_B08_a2    1.0  503.0  B08  train  HEPG2  1.0\n",
      "3  HEPG2-01_a3  HEPG2-01_1_B08_a3    1.0  503.0  B08  train  HEPG2  1.0,     experiment            id_code  plate  sirna well     ds   cell  aug\n",
      "0  HEPG2-01_a0  HEPG2-01_1_B09_a0    1.0  188.0  B09  train  HEPG2  1.0\n",
      "1  HEPG2-01_a1  HEPG2-01_1_B09_a1    1.0  188.0  B09  train  HEPG2  1.0\n",
      "2  HEPG2-01_a2  HEPG2-01_1_B09_a2    1.0  188.0  B09  train  HEPG2  1.0\n",
      "3  HEPG2-01_a3  HEPG2-01_1_B09_a3    1.0  188.0  B09  train  HEPG2  1.0,     experiment            id_code  plate  sirna well     ds   cell  aug\n",
      "0  HEPG2-01_a0  HEPG2-01_1_B10_a0    1.0  700.0  B10  train  HEPG2  1.0\n",
      "1  HEPG2-01_a1  HEPG2-01_1_B10_a1    1.0  700.0  B10  train  HEPG2  1.0\n",
      "2  HEPG2-01_a2  HEPG2-01_1_B10_a2    1.0  700.0  B10  train  HEPG2  1.0\n",
      "3  HEPG2-01_a3  HEPG2-01_1_B10_a3    1.0  700.0  B10  train  HEPG2  1.0,     experiment            id_code  plate   sirna well     ds   cell  aug\n",
      "0  HEPG2-01_a0  HEPG2-01_1_B11_a0    1.0  1100.0  B11  train  HEPG2  1.0\n",
      "1  HEPG2-01_a1  HEPG2-01_1_B11_a1    1.0  1100.0  B11  train  HEPG2  1.0\n",
      "2  HEPG2-01_a2  HEPG2-01_1_B11_a2    1.0  1100.0  B11  train  HEPG2  1.0\n",
      "3  HEPG2-01_a3  HEPG2-01_1_B11_a3    1.0  1100.0  B11  train  HEPG2  1.0,     experiment            id_code  plate  sirna well     ds   cell  aug\n",
      "0  HEPG2-01_a0  HEPG2-01_1_B12_a0    1.0  611.0  B12  train  HEPG2  1.0\n",
      "1  HEPG2-01_a1  HEPG2-01_1_B12_a1    1.0  611.0  B12  train  HEPG2  1.0\n",
      "2  HEPG2-01_a2  HEPG2-01_1_B12_a2    1.0  611.0  B12  train  HEPG2  1.0\n",
      "3  HEPG2-01_a3  HEPG2-01_1_B12_a3    1.0  611.0  B12  train  HEPG2  1.0]\n"
     ]
    }
   ],
   "source": [
    "print(isinstance(augs, list))\n",
    "print(len(augs))\n",
    "print(augs[:10])"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 36,
   "metadata": {},
   "outputs": [],
   "source": [
    "df_train_final = df_train_cp.copy()\n",
    "train_and_augs = [df_train_final] + augs\n",
    "df_train_final = pd.concat(train_and_augs, ignore_index=True)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 37,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>experiment</th>\n",
       "      <th>id_code</th>\n",
       "      <th>plate</th>\n",
       "      <th>sirna</th>\n",
       "      <th>well</th>\n",
       "      <th>ds</th>\n",
       "      <th>cell</th>\n",
       "      <th>aug</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>HEPG2-01</td>\n",
       "      <td>HEPG2-01_1_B03</td>\n",
       "      <td>1.0</td>\n",
       "      <td>513.0</td>\n",
       "      <td>B03</td>\n",
       "      <td>train</td>\n",
       "      <td>HEPG2</td>\n",
       "      <td>0.0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>HEPG2-01</td>\n",
       "      <td>HEPG2-01_1_B04</td>\n",
       "      <td>1.0</td>\n",
       "      <td>840.0</td>\n",
       "      <td>B04</td>\n",
       "      <td>train</td>\n",
       "      <td>HEPG2</td>\n",
       "      <td>0.0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2</th>\n",
       "      <td>HEPG2-01</td>\n",
       "      <td>HEPG2-01_1_B05</td>\n",
       "      <td>1.0</td>\n",
       "      <td>1020.0</td>\n",
       "      <td>B05</td>\n",
       "      <td>train</td>\n",
       "      <td>HEPG2</td>\n",
       "      <td>0.0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3</th>\n",
       "      <td>HEPG2-01</td>\n",
       "      <td>HEPG2-01_1_B06</td>\n",
       "      <td>1.0</td>\n",
       "      <td>254.0</td>\n",
       "      <td>B06</td>\n",
       "      <td>train</td>\n",
       "      <td>HEPG2</td>\n",
       "      <td>0.0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4</th>\n",
       "      <td>HEPG2-01</td>\n",
       "      <td>HEPG2-01_1_B07</td>\n",
       "      <td>1.0</td>\n",
       "      <td>144.0</td>\n",
       "      <td>B07</td>\n",
       "      <td>train</td>\n",
       "      <td>HEPG2</td>\n",
       "      <td>0.0</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "  experiment         id_code  plate   sirna well     ds   cell  aug\n",
       "0   HEPG2-01  HEPG2-01_1_B03    1.0   513.0  B03  train  HEPG2  0.0\n",
       "1   HEPG2-01  HEPG2-01_1_B04    1.0   840.0  B04  train  HEPG2  0.0\n",
       "2   HEPG2-01  HEPG2-01_1_B05    1.0  1020.0  B05  train  HEPG2  0.0\n",
       "3   HEPG2-01  HEPG2-01_1_B06    1.0   254.0  B06  train  HEPG2  0.0\n",
       "4   HEPG2-01  HEPG2-01_1_B07    1.0   144.0  B07  train  HEPG2  0.0"
      ]
     },
     "execution_count": 37,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "df_train_final.head()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 38,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>experiment</th>\n",
       "      <th>id_code</th>\n",
       "      <th>plate</th>\n",
       "      <th>sirna</th>\n",
       "      <th>well</th>\n",
       "      <th>ds</th>\n",
       "      <th>cell</th>\n",
       "      <th>aug</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>149450</th>\n",
       "      <td>U2OS-03_a5</td>\n",
       "      <td>U2OS-03_4_O23_a5</td>\n",
       "      <td>4.0</td>\n",
       "      <td>509.0</td>\n",
       "      <td>O23</td>\n",
       "      <td>train</td>\n",
       "      <td>U2OS</td>\n",
       "      <td>1.0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>149451</th>\n",
       "      <td>U2OS-03_a6</td>\n",
       "      <td>U2OS-03_4_O23_a6</td>\n",
       "      <td>4.0</td>\n",
       "      <td>509.0</td>\n",
       "      <td>O23</td>\n",
       "      <td>train</td>\n",
       "      <td>U2OS</td>\n",
       "      <td>1.0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>149452</th>\n",
       "      <td>U2OS-03_a7</td>\n",
       "      <td>U2OS-03_4_O23_a7</td>\n",
       "      <td>4.0</td>\n",
       "      <td>509.0</td>\n",
       "      <td>O23</td>\n",
       "      <td>train</td>\n",
       "      <td>U2OS</td>\n",
       "      <td>1.0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>149453</th>\n",
       "      <td>U2OS-03_a8</td>\n",
       "      <td>U2OS-03_4_O23_a8</td>\n",
       "      <td>4.0</td>\n",
       "      <td>509.0</td>\n",
       "      <td>O23</td>\n",
       "      <td>train</td>\n",
       "      <td>U2OS</td>\n",
       "      <td>1.0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>149454</th>\n",
       "      <td>U2OS-03_a9</td>\n",
       "      <td>U2OS-03_4_O23_a9</td>\n",
       "      <td>4.0</td>\n",
       "      <td>509.0</td>\n",
       "      <td>O23</td>\n",
       "      <td>train</td>\n",
       "      <td>U2OS</td>\n",
       "      <td>1.0</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "        experiment           id_code  plate  sirna well     ds  cell  aug\n",
       "149450  U2OS-03_a5  U2OS-03_4_O23_a5    4.0  509.0  O23  train  U2OS  1.0\n",
       "149451  U2OS-03_a6  U2OS-03_4_O23_a6    4.0  509.0  O23  train  U2OS  1.0\n",
       "149452  U2OS-03_a7  U2OS-03_4_O23_a7    4.0  509.0  O23  train  U2OS  1.0\n",
       "149453  U2OS-03_a8  U2OS-03_4_O23_a8    4.0  509.0  O23  train  U2OS  1.0\n",
       "149454  U2OS-03_a9  U2OS-03_4_O23_a9    4.0  509.0  O23  train  U2OS  1.0"
      ]
     },
     "execution_count": 38,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "df_train_final.tail()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 39,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "149455\n"
     ]
    }
   ],
   "source": [
    "print(len(df_train_final))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 40,
   "metadata": {},
   "outputs": [],
   "source": [
    "df_train_final_wo_valid = df_train_final[~df_train_final.isin(df_valid)].dropna()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 41,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "147640\n"
     ]
    }
   ],
   "source": [
    "print(len(df_train_final_wo_valid))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 42,
   "metadata": {},
   "outputs": [],
   "source": [
    "df_train_final_wo_valid.to_csv('/storage/rxrxai/training_aug.csv', index=False)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 43,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>experiment</th>\n",
       "      <th>id_code</th>\n",
       "      <th>plate</th>\n",
       "      <th>sirna</th>\n",
       "      <th>well</th>\n",
       "      <th>ds</th>\n",
       "      <th>cell</th>\n",
       "      <th>aug</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>149450</th>\n",
       "      <td>U2OS-03_a5</td>\n",
       "      <td>U2OS-03_4_O23_a5</td>\n",
       "      <td>4.0</td>\n",
       "      <td>509.0</td>\n",
       "      <td>O23</td>\n",
       "      <td>train</td>\n",
       "      <td>U2OS</td>\n",
       "      <td>1.0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>149451</th>\n",
       "      <td>U2OS-03_a6</td>\n",
       "      <td>U2OS-03_4_O23_a6</td>\n",
       "      <td>4.0</td>\n",
       "      <td>509.0</td>\n",
       "      <td>O23</td>\n",
       "      <td>train</td>\n",
       "      <td>U2OS</td>\n",
       "      <td>1.0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>149452</th>\n",
       "      <td>U2OS-03_a7</td>\n",
       "      <td>U2OS-03_4_O23_a7</td>\n",
       "      <td>4.0</td>\n",
       "      <td>509.0</td>\n",
       "      <td>O23</td>\n",
       "      <td>train</td>\n",
       "      <td>U2OS</td>\n",
       "      <td>1.0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>149453</th>\n",
       "      <td>U2OS-03_a8</td>\n",
       "      <td>U2OS-03_4_O23_a8</td>\n",
       "      <td>4.0</td>\n",
       "      <td>509.0</td>\n",
       "      <td>O23</td>\n",
       "      <td>train</td>\n",
       "      <td>U2OS</td>\n",
       "      <td>1.0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>149454</th>\n",
       "      <td>U2OS-03_a9</td>\n",
       "      <td>U2OS-03_4_O23_a9</td>\n",
       "      <td>4.0</td>\n",
       "      <td>509.0</td>\n",
       "      <td>O23</td>\n",
       "      <td>train</td>\n",
       "      <td>U2OS</td>\n",
       "      <td>1.0</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "        experiment           id_code  plate  sirna well     ds  cell  aug\n",
       "149450  U2OS-03_a5  U2OS-03_4_O23_a5    4.0  509.0  O23  train  U2OS  1.0\n",
       "149451  U2OS-03_a6  U2OS-03_4_O23_a6    4.0  509.0  O23  train  U2OS  1.0\n",
       "149452  U2OS-03_a7  U2OS-03_4_O23_a7    4.0  509.0  O23  train  U2OS  1.0\n",
       "149453  U2OS-03_a8  U2OS-03_4_O23_a8    4.0  509.0  O23  train  U2OS  1.0\n",
       "149454  U2OS-03_a9  U2OS-03_4_O23_a9    4.0  509.0  O23  train  U2OS  1.0"
      ]
     },
     "execution_count": 43,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "df_train_final_wo_valid.tail()"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# Images"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 44,
   "metadata": {},
   "outputs": [],
   "source": [
    "df_train_cp_imgs = df_train_cp.copy().to_records(index=False)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 45,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "'HEPG2'"
      ]
     },
     "execution_count": 45,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "df_train_cp_imgs[0].cell"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 46,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "487739\r\n"
     ]
    }
   ],
   "source": [
    "!ls -l1 /storage/rxrxai/train/**/* | wc -l"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 59,
   "metadata": {},
   "outputs": [],
   "source": [
    "def get_img_path(index, channel, site, aug=\"\"):\n",
    "    experiment, well, plate = df_train_cp_imgs[index].experiment, df_train_cp_imgs[index].well, df_train_cp_imgs[index].plate\n",
    "    return '/'.join(['/storage/rxrxai', 'train', f'{experiment}{aug}', f'Plate{plate}', f'{well}_s{site}_w{channel}.png'])\n",
    "\n",
    "def add_aug_images(packed_args):\n",
    "    i, row = packed_args\n",
    "\n",
    "    for nth_aug in range(n_augmentations_dict[row.cell]):\n",
    "        transform = gen_transform_train_wo_norm()\n",
    "\n",
    "        for channel in range(1,7):\n",
    "            for site in [1,2]:\n",
    "                experiment = df_train_cp_imgs[i].experiment\n",
    "                plate = df_train_cp_imgs[i].plate\n",
    "                aug_suffix = f'_a{nth_aug}'\n",
    "                aug_dir = f'/storage/rxrxai/train/{experiment}{aug_suffix}/Plate{plate}'\n",
    "                \n",
    "                if not os.path.exists(aug_dir):\n",
    "                    os.makedirs(aug_dir)\n",
    "\n",
    "                with Image.open(get_img_path(i, channel, site)) as img:\n",
    "                    image = transform(img)\n",
    "                    image.save(get_img_path(i, channel, site, aug=aug_suffix))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "pool = Pool()                   \n",
    "pool.map(add_aug_images, enumerate(df_train_cp_imgs))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "!ls -l1 /storage/rxrxai/train/**/* | wc -l"
   ]
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 3",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.6.5"
  },
  "widgets": {
   "application/vnd.jupyter.widget-state+json": {
    "state": {
     "2b62ae829edc4d60acf1d9a9e1d598d8": {
      "model_module": "@jupyter-widgets/controls",
      "model_module_version": "1.4.0",
      "model_name": "DescriptionStyleModel",
      "state": {
       "_model_module": "@jupyter-widgets/controls",
       "_model_module_version": "1.4.0",
       "_model_name": "DescriptionStyleModel",
       "_view_count": null,
       "_view_module": "@jupyter-widgets/base",
       "_view_module_version": "1.1.0",
       "_view_name": "StyleView",
       "description_width": ""
      }
     },
     "7740dfb227e54da8b1510dac2d094406": {
      "model_module": "@jupyter-widgets/base",
      "model_module_version": "1.1.0",
      "model_name": "LayoutModel",
      "state": {
       "_model_module": "@jupyter-widgets/base",
       "_model_module_version": "1.1.0",
       "_model_name": "LayoutModel",
       "_view_count": null,
       "_view_module": "@jupyter-widgets/base",
       "_view_module_version": "1.1.0",
       "_view_name": "LayoutView",
       "align_content": null,
       "align_items": null,
       "align_self": null,
       "border": null,
       "bottom": null,
       "display": null,
       "flex": null,
       "flex_flow": null,
       "grid_area": null,
       "grid_auto_columns": null,
       "grid_auto_flow": null,
       "grid_auto_rows": null,
       "grid_column": null,
       "grid_gap": null,
       "grid_row": null,
       "grid_template_areas": null,
       "grid_template_columns": null,
       "grid_template_rows": null,
       "height": null,
       "justify_content": null,
       "left": null,
       "margin": null,
       "max_height": null,
       "max_width": null,
       "min_height": null,
       "min_width": null,
       "order": null,
       "overflow": null,
       "overflow_x": null,
       "overflow_y": null,
       "padding": null,
       "right": null,
       "top": null,
       "visibility": null,
       "width": null
      }
     },
     "921a9c670b6e4a2db86c75a7ff5d9ee6": {
      "model_module": "@jupyter-widgets/base",
      "model_module_version": "1.1.0",
      "model_name": "LayoutModel",
      "state": {
       "_model_module": "@jupyter-widgets/base",
       "_model_module_version": "1.1.0",
       "_model_name": "LayoutModel",
       "_view_count": null,
       "_view_module": "@jupyter-widgets/base",
       "_view_module_version": "1.1.0",
       "_view_name": "LayoutView",
       "align_content": null,
       "align_items": null,
       "align_self": null,
       "border": null,
       "bottom": null,
       "display": null,
       "flex": null,
       "flex_flow": null,
       "grid_area": null,
       "grid_auto_columns": null,
       "grid_auto_flow": null,
       "grid_auto_rows": null,
       "grid_column": null,
       "grid_gap": null,
       "grid_row": null,
       "grid_template_areas": null,
       "grid_template_columns": null,
       "grid_template_rows": null,
       "height": null,
       "justify_content": null,
       "left": null,
       "margin": null,
       "max_height": null,
       "max_width": null,
       "min_height": null,
       "min_width": null,
       "order": null,
       "overflow": null,
       "overflow_x": null,
       "overflow_y": null,
       "padding": null,
       "right": null,
       "top": null,
       "visibility": null,
       "width": null
      }
     },
     "9dfcb7497f8842af817750eec565b8b9": {
      "model_module": "@jupyter-widgets/controls",
      "model_module_version": "1.4.0",
      "model_name": "HTMLModel",
      "state": {
       "_dom_classes": [],
       "_model_module": "@jupyter-widgets/controls",
       "_model_module_version": "1.4.0",
       "_model_name": "HTMLModel",
       "_view_count": null,
       "_view_module": "@jupyter-widgets/controls",
       "_view_module_version": "1.4.0",
       "_view_name": "HTMLView",
       "description": "",
       "description_tooltip": null,
       "layout": "IPY_MODEL_921a9c670b6e4a2db86c75a7ff5d9ee6",
       "placeholder": "​",
       "style": "IPY_MODEL_2b62ae829edc4d60acf1d9a9e1d598d8",
       "value": " 94% 2151/2283 [22:45&lt;01:23,  1.58it/s]"
      }
     },
     "d2df0eb5abab4e3895ec792681cfa8d2": {
      "model_module": "@jupyter-widgets/controls",
      "model_module_version": "1.4.0",
      "model_name": "ProgressStyleModel",
      "state": {
       "_model_module": "@jupyter-widgets/controls",
       "_model_module_version": "1.4.0",
       "_model_name": "ProgressStyleModel",
       "_view_count": null,
       "_view_module": "@jupyter-widgets/base",
       "_view_module_version": "1.1.0",
       "_view_name": "StyleView",
       "bar_color": null,
       "description_width": "initial"
      }
     },
     "e3ff3ae302394523bb5b28ee009842d5": {
      "model_module": "@jupyter-widgets/controls",
      "model_module_version": "1.4.0",
      "model_name": "HBoxModel",
      "state": {
       "_dom_classes": [],
       "_model_module": "@jupyter-widgets/controls",
       "_model_module_version": "1.4.0",
       "_model_name": "HBoxModel",
       "_view_count": null,
       "_view_module": "@jupyter-widgets/controls",
       "_view_module_version": "1.4.0",
       "_view_name": "HBoxView",
       "box_style": "",
       "children": [
        "IPY_MODEL_ff74a4321a59419cb24e116db9dd1e3e",
        "IPY_MODEL_9dfcb7497f8842af817750eec565b8b9"
       ],
       "layout": "IPY_MODEL_7740dfb227e54da8b1510dac2d094406"
      }
     },
     "fad7703039454db7af5d7fb4bce65003": {
      "model_module": "@jupyter-widgets/base",
      "model_module_version": "1.1.0",
      "model_name": "LayoutModel",
      "state": {
       "_model_module": "@jupyter-widgets/base",
       "_model_module_version": "1.1.0",
       "_model_name": "LayoutModel",
       "_view_count": null,
       "_view_module": "@jupyter-widgets/base",
       "_view_module_version": "1.1.0",
       "_view_name": "LayoutView",
       "align_content": null,
       "align_items": null,
       "align_self": null,
       "border": null,
       "bottom": null,
       "display": null,
       "flex": null,
       "flex_flow": null,
       "grid_area": null,
       "grid_auto_columns": null,
       "grid_auto_flow": null,
       "grid_auto_rows": null,
       "grid_column": null,
       "grid_gap": null,
       "grid_row": null,
       "grid_template_areas": null,
       "grid_template_columns": null,
       "grid_template_rows": null,
       "height": null,
       "justify_content": null,
       "left": null,
       "margin": null,
       "max_height": null,
       "max_width": null,
       "min_height": null,
       "min_width": null,
       "order": null,
       "overflow": null,
       "overflow_x": null,
       "overflow_y": null,
       "padding": null,
       "right": null,
       "top": null,
       "visibility": null,
       "width": null
      }
     },
     "ff74a4321a59419cb24e116db9dd1e3e": {
      "model_module": "@jupyter-widgets/controls",
      "model_module_version": "1.4.0",
      "model_name": "IntProgressModel",
      "state": {
       "_dom_classes": [],
       "_model_module": "@jupyter-widgets/controls",
       "_model_module_version": "1.4.0",
       "_model_name": "IntProgressModel",
       "_view_count": null,
       "_view_module": "@jupyter-widgets/controls",
       "_view_module_version": "1.4.0",
       "_view_name": "ProgressView",
       "bar_style": "",
       "description": "Loss: 128.54232788085938",
       "description_tooltip": null,
       "layout": "IPY_MODEL_fad7703039454db7af5d7fb4bce65003",
       "max": 2283,
       "min": 0,
       "orientation": "horizontal",
       "style": "IPY_MODEL_d2df0eb5abab4e3895ec792681cfa8d2",
       "value": 2151
      }
     }
    },
    "version_major": 2,
    "version_minor": 0
   }
  }
 },
 "nbformat": 4,
 "nbformat_minor": 2
}