regression_models_3_bins_330.ipynb

{
 "cells": [
  {
   "cell_type": "code",
   "execution_count": 1,
   "outputs": [],
   "source": [
    "%load_ext autoreload"
   ],
   "metadata": {
    "collapsed": false,
    "ExecuteTime": {
     "end_time": "2023-10-09T20:56:09.869213500Z",
     "start_time": "2023-10-09T20:56:09.806714300Z"
    }
   }
  },
  {
   "cell_type": "code",
   "execution_count": 2,
   "metadata": {
    "collapsed": true,
    "ExecuteTime": {
     "end_time": "2023-07-06T10:47:18.167223200Z",
     "start_time": "2023-07-06T10:47:15.667861600Z"
    }
   },
   "outputs": [],
   "source": [
    "from collections import defaultdict\n",
    "import joblib\n",
    "from tqdm.notebook import tqdm\n",
    "from ML.data_preprocessing import Dataset\n",
    "from ML.machine_learning_models import *\n",
    "# ML utils\n",
    "from ML.machine_learning_models import Model_Evaluation as ml_evaluation\n",
    "from ML.ml_utils_reg import create_directory, ECFP4, set_global_determinism, potency_classes\n",
    "from ML.utils_dataset import dataset_train_test\n",
    "%autoreload 2"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 3,
   "outputs": [],
   "source": [
    "# parameters\n",
    "# Models\n",
    "model_list =  ['1-NN','kNN','SVR','RFR','MR'] #\n",
    "# Number of trials (datasplits)\n",
    "trial_splits = 10\n",
    "#Molecular Fingerprint\n",
    "fingerprint = 'ECFP4'\n",
    "# Approach\n",
    "approach = 'Balanced'\n",
    "# Training set sizes\n",
    "set_sizes = [6, 12, 18, 30, 48, 78, 126, 204, 330]"
   ],
   "metadata": {
    "collapsed": false,
    "ExecuteTime": {
     "end_time": "2023-07-06T10:47:20.266484200Z",
     "start_time": "2023-07-06T10:47:18.991571400Z"
    }
   }
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "outputs": [],
   "source": [
    "# Dataset path\n",
    "db_path = \"./dataset/Datasets Chembl/\"\n",
    "\n",
    "# Load actives dB\n",
    "regression_db_all = pd.read_csv(db_path + f'chembl_30_IC50_500_CPDs.csv')\n",
    "regression_db_all['potency_class'] = potency_classes(regression_db_all.pPot.values, [5, 7, 9, 11])\n",
    "regression_db_all"
   ],
   "metadata": {
    "collapsed": false
   }
  },
  {
   "cell_type": "markdown",
   "source": [
    "# Select Activity classes"
   ],
   "metadata": {
    "collapsed": false
   }
  },
  {
   "cell_type": "code",
   "execution_count": 9,
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "['CHEMBL280', 'CHEMBL203', 'CHEMBL2409']\n"
     ]
    }
   ],
   "source": [
    "tid_list = []\n",
    "n = 110\n",
    "for tid in regression_db_all.chembl_tid.unique()[:]:\n",
    "    df_reg_tid = regression_db_all.loc[regression_db_all.chembl_tid == tid]\n",
    "    if df_reg_tid['potency_class'].unique().tolist() == [5, 7, 9]:\n",
    "        if len(df_reg_tid.loc[df_reg_tid['potency_class'] == 5]) > n and len(df_reg_tid.loc[df_reg_tid['potency_class'] == 7]) > n and len(df_reg_tid.loc[df_reg_tid['potency_class'] == 9]) > n:\n",
    "            tid_list.append(tid)\n",
    "\n",
    "print(tid_list)"
   ],
   "metadata": {
    "collapsed": false,
    "ExecuteTime": {
     "end_time": "2023-07-06T10:48:28.118044200Z",
     "start_time": "2023-07-06T10:48:27.879852500Z"
    }
   }
  },
  {
   "cell_type": "markdown",
   "source": [
    "# Selected dataset"
   ],
   "metadata": {
    "collapsed": false
   }
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "outputs": [],
   "source": [
    "regression_tids = tid_list\n",
    "regression_db = regression_db_all.loc[regression_db_all.chembl_tid.isin(regression_tids)]\n",
    "regression_db"
   ],
   "metadata": {
    "collapsed": false
   }
  },
  {
   "cell_type": "markdown",
   "source": [
    "# Results path"
   ],
   "metadata": {
    "collapsed": false
   }
  },
  {
   "cell_type": "code",
   "execution_count": 11,
   "outputs": [],
   "source": [
    "# Results path\n",
    "main_folder = 'regression_models_increase_tr_sizes_3_bins_330'\n",
    "result_path = f'./{main_folder}/{fingerprint}/{approach}/'"
   ],
   "metadata": {
    "collapsed": false,
    "ExecuteTime": {
     "end_time": "2023-07-06T10:48:39.254281900Z",
     "start_time": "2023-07-06T10:48:39.138323800Z"
    }
   }
  },
  {
   "cell_type": "markdown",
   "source": [
    "# Targets IDs"
   ],
   "metadata": {
    "collapsed": false
   }
  },
  {
   "cell_type": "code",
   "execution_count": 13,
   "outputs": [
    {
     "data": {
      "text/plain": "array(['CHEMBL280', 'CHEMBL203', 'CHEMBL2409'], dtype=object)"
     },
     "execution_count": 13,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "regression_tids = regression_db.chembl_tid.unique()\n",
    "regression_tids"
   ],
   "metadata": {
    "collapsed": false,
    "ExecuteTime": {
     "end_time": "2023-07-06T10:48:40.004865200Z",
     "start_time": "2023-07-06T10:48:39.870823900Z"
    }
   }
  },
  {
   "cell_type": "code",
   "execution_count": 14,
   "outputs": [
    {
     "data": {
      "text/plain": "  0%|          | 0/3 [00:00<?, ?it/s]",
      "application/vnd.jupyter.widget-view+json": {
       "version_major": 2,
       "version_minor": 0,
       "model_id": "6667320295154c70970cf17adc2d841a"
      }
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Created new directory './pred_models_increase_tr_sizes_3_bins_330/ECFP4/Balanced/CHEMBL280/'\n",
      "Training on CHEMBL280\n",
      "Starting Trial 0\n",
      "6\n",
      "Training 1-NN\n",
      "Training kNN\n",
      "Training SVR\n",
      "Training RFR\n",
      "Training MR\n",
      "12\n",
      "Training 1-NN\n",
      "Training kNN\n",
      "Training SVR\n",
      "Training RFR\n",
      "Training MR\n",
      "18\n",
      "Training 1-NN\n",
      "Training kNN\n",
      "Training SVR\n",
      "Training RFR\n",
      "Training MR\n",
      "30\n",
      "Training 1-NN\n",
      "Training kNN\n",
      "Training SVR\n",
      "Training RFR\n",
      "Training MR\n",
      "48\n",
      "Training 1-NN\n",
      "Training kNN\n",
      "Training SVR\n",
      "Training RFR\n",
      "Training MR\n",
      "78\n",
      "Training 1-NN\n",
      "Training kNN\n",
      "Training SVR\n",
      "Training RFR\n",
      "Training MR\n",
      "126\n",
      "Training 1-NN\n",
      "Training kNN\n",
      "Training SVR\n",
      "Training RFR\n",
      "Training MR\n",
      "204\n",
      "Training 1-NN\n",
      "Training kNN\n",
      "Training SVR\n",
      "Training RFR\n",
      "Training MR\n",
      "330\n",
      "Training 1-NN\n",
      "Training kNN\n",
      "Training SVR\n",
      "Training RFR\n",
      "Training MR\n",
      "Starting Trial 1\n",
      "6\n",
      "Training 1-NN\n",
      "Training kNN\n",
      "Training SVR\n",
      "Training RFR\n",
      "Training MR\n",
      "12\n",
      "Training 1-NN\n",
      "Training kNN\n",
      "Training SVR\n",
      "Training RFR\n",
      "Training MR\n",
      "18\n",
      "Training 1-NN\n",
      "Training kNN\n",
      "Training SVR\n",
      "Training RFR\n",
      "Training MR\n",
      "30\n",
      "Training 1-NN\n",
      "Training kNN\n",
      "Training SVR\n",
      "Training RFR\n",
      "Training MR\n",
      "48\n",
      "Training 1-NN\n",
      "Training kNN\n",
      "Training SVR\n",
      "Training RFR\n",
      "Training MR\n",
      "78\n",
      "Training 1-NN\n",
      "Training kNN\n",
      "Training SVR\n",
      "Training RFR\n",
      "Training MR\n",
      "126\n",
      "Training 1-NN\n",
      "Training kNN\n",
      "Training SVR\n",
      "Training RFR\n",
      "Training MR\n",
      "204\n",
      "Training 1-NN\n",
      "Training kNN\n",
      "Training SVR\n",
      "Training RFR\n",
      "Training MR\n",
      "330\n",
      "Training 1-NN\n",
      "Training kNN\n",
      "Training SVR\n",
      "Training RFR\n",
      "Training MR\n",
      "Starting Trial 2\n",
      "6\n",
      "Training 1-NN\n",
      "Training kNN\n",
      "Training SVR\n",
      "Training RFR\n",
      "Training MR\n",
      "12\n",
      "Training 1-NN\n",
      "Training kNN\n",
      "Training SVR\n",
      "Training RFR\n",
      "Training MR\n",
      "18\n",
      "Training 1-NN\n",
      "Training kNN\n",
      "Training SVR\n",
      "Training RFR\n",
      "Training MR\n",
      "30\n",
      "Training 1-NN\n",
      "Training kNN\n",
      "Training SVR\n",
      "Training RFR\n",
      "Training MR\n",
      "48\n",
      "Training 1-NN\n",
      "Training kNN\n",
      "Training SVR\n",
      "Training RFR\n",
      "Training MR\n",
      "78\n",
      "Training 1-NN\n",
      "Training kNN\n",
      "Training SVR\n",
      "Training RFR\n",
      "Training MR\n",
      "126\n",
      "Training 1-NN\n",
      "Training kNN\n",
      "Training SVR\n",
      "Training RFR\n",
      "Training MR\n",
      "204\n",
      "Training 1-NN\n",
      "Training kNN\n",
      "Training SVR\n",
      "Training RFR\n",
      "Training MR\n",
      "330\n",
      "Training 1-NN\n",
      "Training kNN\n",
      "Training SVR\n",
      "Training RFR\n",
      "Training MR\n",
      "Starting Trial 3\n",
      "6\n",
      "Training 1-NN\n",
      "Training kNN\n",
      "Training SVR\n",
      "Training RFR\n",
      "Training MR\n",
      "12\n",
      "Training 1-NN\n",
      "Training kNN\n",
      "Training SVR\n",
      "Training RFR\n",
      "Training MR\n",
      "18\n",
      "Training 1-NN\n",
      "Training kNN\n",
      "Training SVR\n",
      "Training RFR\n",
      "Training MR\n",
      "30\n",
      "Training 1-NN\n",
      "Training kNN\n",
      "Training SVR\n",
      "Training RFR\n",
      "Training MR\n",
      "48\n",
      "Training 1-NN\n",
      "Training kNN\n",
      "Training SVR\n",
      "Training RFR\n",
      "Training MR\n",
      "78\n",
      "Training 1-NN\n",
      "Training kNN\n",
      "Training SVR\n",
      "Training RFR\n",
      "Training MR\n",
      "126\n",
      "Training 1-NN\n",
      "Training kNN\n",
      "Training SVR\n",
      "Training RFR\n",
      "Training MR\n",
      "204\n",
      "Training 1-NN\n",
      "Training kNN\n",
      "Training SVR\n",
      "Training RFR\n",
      "Training MR\n",
      "330\n",
      "Training 1-NN\n",
      "Training kNN\n",
      "Training SVR\n",
      "Training RFR\n",
      "Training MR\n",
      "Starting Trial 4\n",
      "6\n",
      "Training 1-NN\n",
      "Training kNN\n",
      "Training SVR\n",
      "Training RFR\n",
      "Training MR\n",
      "12\n",
      "Training 1-NN\n",
      "Training kNN\n",
      "Training SVR\n",
      "Training RFR\n",
      "Training MR\n",
      "18\n",
      "Training 1-NN\n",
      "Training kNN\n",
      "Training SVR\n",
      "Training RFR\n",
      "Training MR\n",
      "30\n",
      "Training 1-NN\n",
      "Training kNN\n",
      "Training SVR\n",
      "Training RFR\n",
      "Training MR\n",
      "48\n",
      "Training 1-NN\n",
      "Training kNN\n",
      "Training SVR\n",
      "Training RFR\n",
      "Training MR\n",
      "78\n",
      "Training 1-NN\n",
      "Training kNN\n",
      "Training SVR\n",
      "Training RFR\n",
      "Training MR\n",
      "126\n",
      "Training 1-NN\n",
      "Training kNN\n",
      "Training SVR\n",
      "Training RFR\n",
      "Training MR\n",
      "204\n",
      "Training 1-NN\n",
      "Training kNN\n",
      "Training SVR\n",
      "Training RFR\n",
      "Training MR\n",
      "330\n",
      "Training 1-NN\n",
      "Training kNN\n",
      "Training SVR\n",
      "Training RFR\n",
      "Training MR\n",
      "Starting Trial 5\n",
      "6\n",
      "Training 1-NN\n",
      "Training kNN\n",
      "Training SVR\n",
      "Training RFR\n",
      "Training MR\n",
      "12\n",
      "Training 1-NN\n",
      "Training kNN\n",
      "Training SVR\n",
      "Training RFR\n",
      "Training MR\n",
      "18\n",
      "Training 1-NN\n",
      "Training kNN\n",
      "Training SVR\n",
      "Training RFR\n",
      "Training MR\n",
      "30\n",
      "Training 1-NN\n",
      "Training kNN\n",
      "Training SVR\n",
      "Training RFR\n",
      "Training MR\n",
      "48\n",
      "Training 1-NN\n",
      "Training kNN\n",
      "Training SVR\n",
      "Training RFR\n",
      "Training MR\n",
      "78\n",
      "Training 1-NN\n",
      "Training kNN\n",
      "Training SVR\n",
      "Training RFR\n",
      "Training MR\n",
      "126\n",
      "Training 1-NN\n",
      "Training kNN\n",
      "Training SVR\n",
      "Training RFR\n",
      "Training MR\n",
      "204\n",
      "Training 1-NN\n",
      "Training kNN\n",
      "Training SVR\n",
      "Training RFR\n",
      "Training MR\n",
      "330\n",
      "Training 1-NN\n",
      "Training kNN\n",
      "Training SVR\n",
      "Training RFR\n",
      "Training MR\n",
      "Starting Trial 6\n",
      "6\n",
      "Training 1-NN\n",
      "Training kNN\n",
      "Training SVR\n",
      "Training RFR\n",
      "Training MR\n",
      "12\n",
      "Training 1-NN\n",
      "Training kNN\n",
      "Training SVR\n",
      "Training RFR\n",
      "Training MR\n",
      "18\n",
      "Training 1-NN\n",
      "Training kNN\n",
      "Training SVR\n",
      "Training RFR\n",
      "Training MR\n",
      "30\n",
      "Training 1-NN\n",
      "Training kNN\n",
      "Training SVR\n",
      "Training RFR\n",
      "Training MR\n",
      "48\n",
      "Training 1-NN\n",
      "Training kNN\n",
      "Training SVR\n",
      "Training RFR\n",
      "Training MR\n",
      "78\n",
      "Training 1-NN\n",
      "Training kNN\n",
      "Training SVR\n",
      "Training RFR\n",
      "Training MR\n",
      "126\n",
      "Training 1-NN\n",
      "Training kNN\n",
      "Training SVR\n",
      "Training RFR\n",
      "Training MR\n",
      "204\n",
      "Training 1-NN\n",
      "Training kNN\n",
      "Training SVR\n",
      "Training RFR\n",
      "Training MR\n",
      "330\n",
      "Training 1-NN\n",
      "Training kNN\n",
      "Training SVR\n",
      "Training RFR\n",
      "Training MR\n",
      "Starting Trial 7\n",
      "6\n",
      "Training 1-NN\n",
      "Training kNN\n",
      "Training SVR\n",
      "Training RFR\n",
      "Training MR\n",
      "12\n",
      "Training 1-NN\n",
      "Training kNN\n",
      "Training SVR\n",
      "Training RFR\n",
      "Training MR\n",
      "18\n",
      "Training 1-NN\n",
      "Training kNN\n",
      "Training SVR\n",
      "Training RFR\n",
      "Training MR\n",
      "30\n",
      "Training 1-NN\n",
      "Training kNN\n",
      "Training SVR\n",
      "Training RFR\n",
      "Training MR\n",
      "48\n",
      "Training 1-NN\n",
      "Training kNN\n",
      "Training SVR\n",
      "Training RFR\n",
      "Training MR\n",
      "78\n",
      "Training 1-NN\n",
      "Training kNN\n",
      "Training SVR\n",
      "Training RFR\n",
      "Training MR\n",
      "126\n",
      "Training 1-NN\n",
      "Training kNN\n",
      "Training SVR\n",
      "Training RFR\n",
      "Training MR\n",
      "204\n",
      "Training 1-NN\n",
      "Training kNN\n",
      "Training SVR\n",
      "Training RFR\n",
      "Training MR\n",
      "330\n",
      "Training 1-NN\n",
      "Training kNN\n",
      "Training SVR\n",
      "Training RFR\n",
      "Training MR\n",
      "Starting Trial 8\n",
      "6\n",
      "Training 1-NN\n",
      "Training kNN\n",
      "Training SVR\n",
      "Training RFR\n",
      "Training MR\n",
      "12\n",
      "Training 1-NN\n",
      "Training kNN\n",
      "Training SVR\n",
      "Training RFR\n",
      "Training MR\n",
      "18\n",
      "Training 1-NN\n",
      "Training kNN\n",
      "Training SVR\n",
      "Training RFR\n",
      "Training MR\n",
      "30\n",
      "Training 1-NN\n",
      "Training kNN\n",
      "Training SVR\n",
      "Training RFR\n",
      "Training MR\n",
      "48\n",
      "Training 1-NN\n",
      "Training kNN\n",
      "Training SVR\n",
      "Training RFR\n",
      "Training MR\n",
      "78\n",
      "Training 1-NN\n",
      "Training kNN\n",
      "Training SVR\n",
      "Training RFR\n",
      "Training MR\n",
      "126\n",
      "Training 1-NN\n",
      "Training kNN\n",
      "Training SVR\n",
      "Training RFR\n",
      "Training MR\n",
      "204\n",
      "Training 1-NN\n",
      "Training kNN\n",
      "Training SVR\n",
      "Training RFR\n",
      "Training MR\n",
      "330\n",
      "Training 1-NN\n",
      "Training kNN\n",
      "Training SVR\n",
      "Training RFR\n",
      "Training MR\n",
      "Starting Trial 9\n",
      "6\n",
      "Training 1-NN\n",
      "Training kNN\n",
      "Training SVR\n",
      "Training RFR\n",
      "Training MR\n",
      "12\n",
      "Training 1-NN\n",
      "Training kNN\n",
      "Training SVR\n",
      "Training RFR\n",
      "Training MR\n",
      "18\n",
      "Training 1-NN\n",
      "Training kNN\n",
      "Training SVR\n",
      "Training RFR\n",
      "Training MR\n",
      "30\n",
      "Training 1-NN\n",
      "Training kNN\n",
      "Training SVR\n",
      "Training RFR\n",
      "Training MR\n",
      "48\n",
      "Training 1-NN\n",
      "Training kNN\n",
      "Training SVR\n",
      "Training RFR\n",
      "Training MR\n",
      "78\n",
      "Training 1-NN\n",
      "Training kNN\n",
      "Training SVR\n",
      "Training RFR\n",
      "Training MR\n",
      "126\n",
      "Training 1-NN\n",
      "Training kNN\n",
      "Training SVR\n",
      "Training RFR\n",
      "Training MR\n",
      "204\n",
      "Training 1-NN\n",
      "Training kNN\n",
      "Training SVR\n",
      "Training RFR\n",
      "Training MR\n",
      "330\n",
      "Training 1-NN\n",
      "Training kNN\n",
      "Training SVR\n",
      "Training RFR\n",
      "Training MR\n",
      "Created new directory './pred_models_increase_tr_sizes_3_bins_330/ECFP4/Balanced/CHEMBL203/'\n",
      "Training on CHEMBL203\n",
      "Starting Trial 0\n",
      "6\n",
      "Training 1-NN\n",
      "Training kNN\n",
      "Training SVR\n",
      "Training RFR\n",
      "Training MR\n",
      "12\n",
      "Training 1-NN\n",
      "Training kNN\n",
      "Training SVR\n",
      "Training RFR\n",
      "Training MR\n",
      "18\n",
      "Training 1-NN\n",
      "Training kNN\n",
      "Training SVR\n",
      "Training RFR\n",
      "Training MR\n",
      "30\n",
      "Training 1-NN\n",
      "Training kNN\n",
      "Training SVR\n",
      "Training RFR\n",
      "Training MR\n",
      "48\n",
      "Training 1-NN\n",
      "Training kNN\n",
      "Training SVR\n",
      "Training RFR\n",
      "Training MR\n",
      "78\n",
      "Training 1-NN\n",
      "Training kNN\n",
      "Training SVR\n",
      "Training RFR\n",
      "Training MR\n",
      "126\n",
      "Training 1-NN\n",
      "Training kNN\n",
      "Training SVR\n",
      "Training RFR\n",
      "Training MR\n",
      "204\n",
      "Training 1-NN\n",
      "Training kNN\n",
      "Training SVR\n",
      "Training RFR\n",
      "Training MR\n",
      "330\n",
      "Training 1-NN\n",
      "Training kNN\n",
      "Training SVR\n",
      "Training RFR\n",
      "Training MR\n",
      "Starting Trial 1\n",
      "6\n",
      "Training 1-NN\n",
      "Training kNN\n",
      "Training SVR\n",
      "Training RFR\n",
      "Training MR\n",
      "12\n",
      "Training 1-NN\n",
      "Training kNN\n",
      "Training SVR\n",
      "Training RFR\n",
      "Training MR\n",
      "18\n",
      "Training 1-NN\n",
      "Training kNN\n",
      "Training SVR\n",
      "Training RFR\n",
      "Training MR\n",
      "30\n",
      "Training 1-NN\n",
      "Training kNN\n",
      "Training SVR\n",
      "Training RFR\n",
      "Training MR\n",
      "48\n",
      "Training 1-NN\n",
      "Training kNN\n",
      "Training SVR\n",
      "Training RFR\n",
      "Training MR\n",
      "78\n",
      "Training 1-NN\n",
      "Training kNN\n",
      "Training SVR\n",
      "Training RFR\n",
      "Training MR\n",
      "126\n",
      "Training 1-NN\n",
      "Training kNN\n",
      "Training SVR\n",
      "Training RFR\n",
      "Training MR\n",
      "204\n",
      "Training 1-NN\n",
      "Training kNN\n",
      "Training SVR\n",
      "Training RFR\n",
      "Training MR\n",
      "330\n",
      "Training 1-NN\n",
      "Training kNN\n",
      "Training SVR\n",
      "Training RFR\n",
      "Training MR\n",
      "Starting Trial 2\n",
      "6\n",
      "Training 1-NN\n",
      "Training kNN\n",
      "Training SVR\n",
      "Training RFR\n",
      "Training MR\n",
      "12\n",
      "Training 1-NN\n",
      "Training kNN\n",
      "Training SVR\n",
      "Training RFR\n",
      "Training MR\n",
      "18\n",
      "Training 1-NN\n",
      "Training kNN\n",
      "Training SVR\n",
      "Training RFR\n",
      "Training MR\n",
      "30\n",
      "Training 1-NN\n",
      "Training kNN\n",
      "Training SVR\n",
      "Training RFR\n",
      "Training MR\n",
      "48\n",
      "Training 1-NN\n",
      "Training kNN\n",
      "Training SVR\n",
      "Training RFR\n",
      "Training MR\n",
      "78\n",
      "Training 1-NN\n",
      "Training kNN\n",
      "Training SVR\n",
      "Training RFR\n",
      "Training MR\n",
      "126\n",
      "Training 1-NN\n",
      "Training kNN\n",
      "Training SVR\n",
      "Training RFR\n",
      "Training MR\n",
      "204\n",
      "Training 1-NN\n",
      "Training kNN\n",
      "Training SVR\n",
      "Training RFR\n",
      "Training MR\n",
      "330\n",
      "Training 1-NN\n",
      "Training kNN\n",
      "Training SVR\n",
      "Training RFR\n",
      "Training MR\n",
      "Starting Trial 3\n",
      "6\n",
      "Training 1-NN\n",
      "Training kNN\n",
      "Training SVR\n",
      "Training RFR\n",
      "Training MR\n",
      "12\n",
      "Training 1-NN\n",
      "Training kNN\n",
      "Training SVR\n",
      "Training RFR\n",
      "Training MR\n",
      "18\n",
      "Training 1-NN\n",
      "Training kNN\n",
      "Training SVR\n",
      "Training RFR\n",
      "Training MR\n",
      "30\n",
      "Training 1-NN\n",
      "Training kNN\n",
      "Training SVR\n",
      "Training RFR\n",
      "Training MR\n",
      "48\n",
      "Training 1-NN\n",
      "Training kNN\n",
      "Training SVR\n",
      "Training RFR\n",
      "Training MR\n",
      "78\n",
      "Training 1-NN\n",
      "Training kNN\n",
      "Training SVR\n",
      "Training RFR\n",
      "Training MR\n",
      "126\n",
      "Training 1-NN\n",
      "Training kNN\n",
      "Training SVR\n",
      "Training RFR\n",
      "Training MR\n",
      "204\n",
      "Training 1-NN\n",
      "Training kNN\n",
      "Training SVR\n",
      "Training RFR\n",
      "Training MR\n",
      "330\n",
      "Training 1-NN\n",
      "Training kNN\n",
      "Training SVR\n",
      "Training RFR\n",
      "Training MR\n",
      "Starting Trial 4\n",
      "6\n",
      "Training 1-NN\n",
      "Training kNN\n",
      "Training SVR\n",
      "Training RFR\n",
      "Training MR\n",
      "12\n",
      "Training 1-NN\n",
      "Training kNN\n",
      "Training SVR\n",
      "Training RFR\n",
      "Training MR\n",
      "18\n",
      "Training 1-NN\n",
      "Training kNN\n",
      "Training SVR\n",
      "Training RFR\n",
      "Training MR\n",
      "30\n",
      "Training 1-NN\n",
      "Training kNN\n",
      "Training SVR\n",
      "Training RFR\n",
      "Training MR\n",
      "48\n",
      "Training 1-NN\n",
      "Training kNN\n",
      "Training SVR\n",
      "Training RFR\n",
      "Training MR\n",
      "78\n",
      "Training 1-NN\n",
      "Training kNN\n",
      "Training SVR\n",
      "Training RFR\n",
      "Training MR\n",
      "126\n",
      "Training 1-NN\n",
      "Training kNN\n",
      "Training SVR\n",
      "Training RFR\n",
      "Training MR\n",
      "204\n",
      "Training 1-NN\n",
      "Training kNN\n",
      "Training SVR\n",
      "Training RFR\n",
      "Training MR\n",
      "330\n",
      "Training 1-NN\n",
      "Training kNN\n",
      "Training SVR\n",
      "Training RFR\n",
      "Training MR\n",
      "Starting Trial 5\n",
      "6\n",
      "Training 1-NN\n",
      "Training kNN\n",
      "Training SVR\n",
      "Training RFR\n",
      "Training MR\n",
      "12\n",
      "Training 1-NN\n",
      "Training kNN\n",
      "Training SVR\n",
      "Training RFR\n",
      "Training MR\n",
      "18\n",
      "Training 1-NN\n",
      "Training kNN\n",
      "Training SVR\n",
      "Training RFR\n",
      "Training MR\n",
      "30\n",
      "Training 1-NN\n",
      "Training kNN\n",
      "Training SVR\n",
      "Training RFR\n",
      "Training MR\n",
      "48\n",
      "Training 1-NN\n",
      "Training kNN\n",
      "Training SVR\n",
      "Training RFR\n",
      "Training MR\n",
      "78\n",
      "Training 1-NN\n",
      "Training kNN\n",
      "Training SVR\n",
      "Training RFR\n",
      "Training MR\n",
      "126\n",
      "Training 1-NN\n",
      "Training kNN\n",
      "Training SVR\n",
      "Training RFR\n",
      "Training MR\n",
      "204\n",
      "Training 1-NN\n",
      "Training kNN\n",
      "Training SVR\n",
      "Training RFR\n",
      "Training MR\n",
      "330\n",
      "Training 1-NN\n",
      "Training kNN\n",
      "Training SVR\n",
      "Training RFR\n",
      "Training MR\n",
      "Starting Trial 6\n",
      "6\n",
      "Training 1-NN\n",
      "Training kNN\n",
      "Training SVR\n",
      "Training RFR\n",
      "Training MR\n",
      "12\n",
      "Training 1-NN\n",
      "Training kNN\n",
      "Training SVR\n",
      "Training RFR\n",
      "Training MR\n",
      "18\n",
      "Training 1-NN\n",
      "Training kNN\n",
      "Training SVR\n",
      "Training RFR\n",
      "Training MR\n",
      "30\n",
      "Training 1-NN\n",
      "Training kNN\n",
      "Training SVR\n",
      "Training RFR\n",
      "Training MR\n",
      "48\n",
      "Training 1-NN\n",
      "Training kNN\n",
      "Training SVR\n",
      "Training RFR\n",
      "Training MR\n",
      "78\n",
      "Training 1-NN\n",
      "Training kNN\n",
      "Training SVR\n",
      "Training RFR\n",
      "Training MR\n",
      "126\n",
      "Training 1-NN\n",
      "Training kNN\n",
      "Training SVR\n",
      "Training RFR\n",
      "Training MR\n",
      "204\n",
      "Training 1-NN\n",
      "Training kNN\n",
      "Training SVR\n",
      "Training RFR\n",
      "Training MR\n",
      "330\n",
      "Training 1-NN\n",
      "Training kNN\n",
      "Training SVR\n",
      "Training RFR\n",
      "Training MR\n",
      "Starting Trial 7\n",
      "6\n",
      "Training 1-NN\n",
      "Training kNN\n",
      "Training SVR\n",
      "Training RFR\n",
      "Training MR\n",
      "12\n",
      "Training 1-NN\n",
      "Training kNN\n",
      "Training SVR\n",
      "Training RFR\n",
      "Training MR\n",
      "18\n",
      "Training 1-NN\n",
      "Training kNN\n",
      "Training SVR\n",
      "Training RFR\n",
      "Training MR\n",
      "30\n",
      "Training 1-NN\n",
      "Training kNN\n",
      "Training SVR\n",
      "Training RFR\n",
      "Training MR\n",
      "48\n",
      "Training 1-NN\n",
      "Training kNN\n",
      "Training SVR\n",
      "Training RFR\n",
      "Training MR\n",
      "78\n",
      "Training 1-NN\n",
      "Training kNN\n",
      "Training SVR\n",
      "Training RFR\n",
      "Training MR\n",
      "126\n",
      "Training 1-NN\n",
      "Training kNN\n",
      "Training SVR\n",
      "Training RFR\n",
      "Training MR\n",
      "204\n",
      "Training 1-NN\n",
      "Training kNN\n",
      "Training SVR\n",
      "Training RFR\n",
      "Training MR\n",
      "330\n",
      "Training 1-NN\n",
      "Training kNN\n",
      "Training SVR\n",
      "Training RFR\n",
      "Training MR\n",
      "Starting Trial 8\n",
      "6\n",
      "Training 1-NN\n",
      "Training kNN\n",
      "Training SVR\n",
      "Training RFR\n",
      "Training MR\n",
      "12\n",
      "Training 1-NN\n",
      "Training kNN\n",
      "Training SVR\n",
      "Training RFR\n",
      "Training MR\n",
      "18\n",
      "Training 1-NN\n",
      "Training kNN\n",
      "Training SVR\n",
      "Training RFR\n",
      "Training MR\n",
      "30\n",
      "Training 1-NN\n",
      "Training kNN\n",
      "Training SVR\n",
      "Training RFR\n",
      "Training MR\n",
      "48\n",
      "Training 1-NN\n",
      "Training kNN\n",
      "Training SVR\n",
      "Training RFR\n",
      "Training MR\n",
      "78\n",
      "Training 1-NN\n",
      "Training kNN\n",
      "Training SVR\n",
      "Training RFR\n",
      "Training MR\n",
      "126\n",
      "Training 1-NN\n",
      "Training kNN\n",
      "Training SVR\n",
      "Training RFR\n",
      "Training MR\n",
      "204\n",
      "Training 1-NN\n",
      "Training kNN\n",
      "Training SVR\n",
      "Training RFR\n",
      "Training MR\n",
      "330\n",
      "Training 1-NN\n",
      "Training kNN\n",
      "Training SVR\n",
      "Training RFR\n",
      "Training MR\n",
      "Starting Trial 9\n",
      "6\n",
      "Training 1-NN\n",
      "Training kNN\n",
      "Training SVR\n",
      "Training RFR\n",
      "Training MR\n",
      "12\n",
      "Training 1-NN\n",
      "Training kNN\n",
      "Training SVR\n",
      "Training RFR\n",
      "Training MR\n",
      "18\n",
      "Training 1-NN\n",
      "Training kNN\n",
      "Training SVR\n",
      "Training RFR\n",
      "Training MR\n",
      "30\n",
      "Training 1-NN\n",
      "Training kNN\n",
      "Training SVR\n",
      "Training RFR\n",
      "Training MR\n",
      "48\n",
      "Training 1-NN\n",
      "Training kNN\n",
      "Training SVR\n",
      "Training RFR\n",
      "Training MR\n",
      "78\n",
      "Training 1-NN\n",
      "Training kNN\n",
      "Training SVR\n",
      "Training RFR\n",
      "Training MR\n",
      "126\n",
      "Training 1-NN\n",
      "Training kNN\n",
      "Training SVR\n",
      "Training RFR\n",
      "Training MR\n",
      "204\n",
      "Training 1-NN\n",
      "Training kNN\n",
      "Training SVR\n",
      "Training RFR\n",
      "Training MR\n",
      "330\n",
      "Training 1-NN\n",
      "Training kNN\n",
      "Training SVR\n",
      "Training RFR\n",
      "Training MR\n",
      "Created new directory './pred_models_increase_tr_sizes_3_bins_330/ECFP4/Balanced/CHEMBL2409/'\n",
      "Training on CHEMBL2409\n",
      "Starting Trial 0\n",
      "6\n",
      "Training 1-NN\n",
      "Training kNN\n",
      "Training SVR\n",
      "Training RFR\n",
      "Training MR\n",
      "12\n",
      "Training 1-NN\n",
      "Training kNN\n",
      "Training SVR\n",
      "Training RFR\n",
      "Training MR\n",
      "18\n",
      "Training 1-NN\n",
      "Training kNN\n",
      "Training SVR\n",
      "Training RFR\n",
      "Training MR\n",
      "30\n",
      "Training 1-NN\n",
      "Training kNN\n",
      "Training SVR\n",
      "Training RFR\n",
      "Training MR\n",
      "48\n",
      "Training 1-NN\n",
      "Training kNN\n",
      "Training SVR\n",
      "Training RFR\n",
      "Training MR\n",
      "78\n",
      "Training 1-NN\n",
      "Training kNN\n",
      "Training SVR\n",
      "Training RFR\n",
      "Training MR\n",
      "126\n",
      "Training 1-NN\n",
      "Training kNN\n",
      "Training SVR\n",
      "Training RFR\n",
      "Training MR\n",
      "204\n",
      "Training 1-NN\n",
      "Training kNN\n",
      "Training SVR\n",
      "Training RFR\n",
      "Training MR\n",
      "330\n",
      "Training 1-NN\n",
      "Training kNN\n",
      "Training SVR\n",
      "Training RFR\n",
      "Training MR\n",
      "Starting Trial 1\n",
      "6\n",
      "Training 1-NN\n",
      "Training kNN\n",
      "Training SVR\n",
      "Training RFR\n",
      "Training MR\n",
      "12\n",
      "Training 1-NN\n",
      "Training kNN\n",
      "Training SVR\n",
      "Training RFR\n",
      "Training MR\n",
      "18\n",
      "Training 1-NN\n",
      "Training kNN\n",
      "Training SVR\n",
      "Training RFR\n",
      "Training MR\n",
      "30\n",
      "Training 1-NN\n",
      "Training kNN\n",
      "Training SVR\n",
      "Training RFR\n",
      "Training MR\n",
      "48\n",
      "Training 1-NN\n",
      "Training kNN\n",
      "Training SVR\n",
      "Training RFR\n",
      "Training MR\n",
      "78\n",
      "Training 1-NN\n",
      "Training kNN\n",
      "Training SVR\n",
      "Training RFR\n",
      "Training MR\n",
      "126\n",
      "Training 1-NN\n",
      "Training kNN\n",
      "Training SVR\n",
      "Training RFR\n",
      "Training MR\n",
      "204\n",
      "Training 1-NN\n",
      "Training kNN\n",
      "Training SVR\n",
      "Training RFR\n",
      "Training MR\n",
      "330\n",
      "Training 1-NN\n",
      "Training kNN\n",
      "Training SVR\n",
      "Training RFR\n",
      "Training MR\n",
      "Starting Trial 2\n",
      "6\n",
      "Training 1-NN\n",
      "Training kNN\n",
      "Training SVR\n",
      "Training RFR\n",
      "Training MR\n",
      "12\n",
      "Training 1-NN\n",
      "Training kNN\n",
      "Training SVR\n",
      "Training RFR\n",
      "Training MR\n",
      "18\n",
      "Training 1-NN\n",
      "Training kNN\n",
      "Training SVR\n",
      "Training RFR\n",
      "Training MR\n",
      "30\n",
      "Training 1-NN\n",
      "Training kNN\n",
      "Training SVR\n",
      "Training RFR\n",
      "Training MR\n",
      "48\n",
      "Training 1-NN\n",
      "Training kNN\n",
      "Training SVR\n",
      "Training RFR\n",
      "Training MR\n",
      "78\n",
      "Training 1-NN\n",
      "Training kNN\n",
      "Training SVR\n",
      "Training RFR\n",
      "Training MR\n",
      "126\n",
      "Training 1-NN\n",
      "Training kNN\n",
      "Training SVR\n",
      "Training RFR\n",
      "Training MR\n",
      "204\n",
      "Training 1-NN\n",
      "Training kNN\n",
      "Training SVR\n",
      "Training RFR\n",
      "Training MR\n",
      "330\n",
      "Training 1-NN\n",
      "Training kNN\n",
      "Training SVR\n",
      "Training RFR\n",
      "Training MR\n",
      "Starting Trial 3\n",
      "6\n",
      "Training 1-NN\n",
      "Training kNN\n",
      "Training SVR\n",
      "Training RFR\n",
      "Training MR\n",
      "12\n",
      "Training 1-NN\n",
      "Training kNN\n",
      "Training SVR\n",
      "Training RFR\n",
      "Training MR\n",
      "18\n",
      "Training 1-NN\n",
      "Training kNN\n",
      "Training SVR\n",
      "Training RFR\n",
      "Training MR\n",
      "30\n",
      "Training 1-NN\n",
      "Training kNN\n",
      "Training SVR\n",
      "Training RFR\n",
      "Training MR\n",
      "48\n",
      "Training 1-NN\n",
      "Training kNN\n",
      "Training SVR\n",
      "Training RFR\n",
      "Training MR\n",
      "78\n",
      "Training 1-NN\n",
      "Training kNN\n",
      "Training SVR\n",
      "Training RFR\n",
      "Training MR\n",
      "126\n",
      "Training 1-NN\n",
      "Training kNN\n",
      "Training SVR\n",
      "Training RFR\n",
      "Training MR\n",
      "204\n",
      "Training 1-NN\n",
      "Training kNN\n",
      "Training SVR\n",
      "Training RFR\n",
      "Training MR\n",
      "330\n",
      "Training 1-NN\n",
      "Training kNN\n",
      "Training SVR\n",
      "Training RFR\n",
      "Training MR\n",
      "Starting Trial 4\n",
      "6\n",
      "Training 1-NN\n",
      "Training kNN\n",
      "Training SVR\n",
      "Training RFR\n",
      "Training MR\n",
      "12\n",
      "Training 1-NN\n",
      "Training kNN\n",
      "Training SVR\n",
      "Training RFR\n",
      "Training MR\n",
      "18\n",
      "Training 1-NN\n",
      "Training kNN\n",
      "Training SVR\n",
      "Training RFR\n",
      "Training MR\n",
      "30\n",
      "Training 1-NN\n",
      "Training kNN\n",
      "Training SVR\n",
      "Training RFR\n",
      "Training MR\n",
      "48\n",
      "Training 1-NN\n",
      "Training kNN\n",
      "Training SVR\n",
      "Training RFR\n",
      "Training MR\n",
      "78\n",
      "Training 1-NN\n",
      "Training kNN\n",
      "Training SVR\n",
      "Training RFR\n",
      "Training MR\n",
      "126\n",
      "Training 1-NN\n",
      "Training kNN\n",
      "Training SVR\n",
      "Training RFR\n",
      "Training MR\n",
      "204\n",
      "Training 1-NN\n",
      "Training kNN\n",
      "Training SVR\n",
      "Training RFR\n",
      "Training MR\n",
      "330\n",
      "Training 1-NN\n",
      "Training kNN\n",
      "Training SVR\n",
      "Training RFR\n",
      "Training MR\n",
      "Starting Trial 5\n",
      "6\n",
      "Training 1-NN\n",
      "Training kNN\n",
      "Training SVR\n",
      "Training RFR\n",
      "Training MR\n",
      "12\n",
      "Training 1-NN\n",
      "Training kNN\n",
      "Training SVR\n",
      "Training RFR\n",
      "Training MR\n",
      "18\n",
      "Training 1-NN\n",
      "Training kNN\n",
      "Training SVR\n",
      "Training RFR\n",
      "Training MR\n",
      "30\n",
      "Training 1-NN\n",
      "Training kNN\n",
      "Training SVR\n",
      "Training RFR\n",
      "Training MR\n",
      "48\n",
      "Training 1-NN\n",
      "Training kNN\n",
      "Training SVR\n",
      "Training RFR\n",
      "Training MR\n",
      "78\n",
      "Training 1-NN\n",
      "Training kNN\n",
      "Training SVR\n",
      "Training RFR\n",
      "Training MR\n",
      "126\n",
      "Training 1-NN\n",
      "Training kNN\n",
      "Training SVR\n",
      "Training RFR\n",
      "Training MR\n",
      "204\n",
      "Training 1-NN\n",
      "Training kNN\n",
      "Training SVR\n",
      "Training RFR\n",
      "Training MR\n",
      "330\n",
      "Training 1-NN\n",
      "Training kNN\n",
      "Training SVR\n",
      "Training RFR\n",
      "Training MR\n",
      "Starting Trial 6\n",
      "6\n",
      "Training 1-NN\n",
      "Training kNN\n",
      "Training SVR\n",
      "Training RFR\n",
      "Training MR\n",
      "12\n",
      "Training 1-NN\n",
      "Training kNN\n",
      "Training SVR\n",
      "Training RFR\n",
      "Training MR\n",
      "18\n",
      "Training 1-NN\n",
      "Training kNN\n",
      "Training SVR\n",
      "Training RFR\n",
      "Training MR\n",
      "30\n",
      "Training 1-NN\n",
      "Training kNN\n",
      "Training SVR\n",
      "Training RFR\n",
      "Training MR\n",
      "48\n",
      "Training 1-NN\n",
      "Training kNN\n",
      "Training SVR\n",
      "Training RFR\n",
      "Training MR\n",
      "78\n",
      "Training 1-NN\n",
      "Training kNN\n",
      "Training SVR\n",
      "Training RFR\n",
      "Training MR\n",
      "126\n",
      "Training 1-NN\n",
      "Training kNN\n",
      "Training SVR\n",
      "Training RFR\n",
      "Training MR\n",
      "204\n",
      "Training 1-NN\n",
      "Training kNN\n",
      "Training SVR\n",
      "Training RFR\n",
      "Training MR\n",
      "330\n",
      "Training 1-NN\n",
      "Training kNN\n",
      "Training SVR\n",
      "Training RFR\n",
      "Training MR\n",
      "Starting Trial 7\n",
      "6\n",
      "Training 1-NN\n",
      "Training kNN\n",
      "Training SVR\n",
      "Training RFR\n",
      "Training MR\n",
      "12\n",
      "Training 1-NN\n",
      "Training kNN\n",
      "Training SVR\n",
      "Training RFR\n",
      "Training MR\n",
      "18\n",
      "Training 1-NN\n",
      "Training kNN\n",
      "Training SVR\n",
      "Training RFR\n",
      "Training MR\n",
      "30\n",
      "Training 1-NN\n",
      "Training kNN\n",
      "Training SVR\n",
      "Training RFR\n",
      "Training MR\n",
      "48\n",
      "Training 1-NN\n",
      "Training kNN\n",
      "Training SVR\n",
      "Training RFR\n",
      "Training MR\n",
      "78\n",
      "Training 1-NN\n",
      "Training kNN\n",
      "Training SVR\n",
      "Training RFR\n",
      "Training MR\n",
      "126\n",
      "Training 1-NN\n",
      "Training kNN\n",
      "Training SVR\n",
      "Training RFR\n",
      "Training MR\n",
      "204\n",
      "Training 1-NN\n",
      "Training kNN\n",
      "Training SVR\n",
      "Training RFR\n",
      "Training MR\n",
      "330\n",
      "Training 1-NN\n",
      "Training kNN\n",
      "Training SVR\n",
      "Training RFR\n",
      "Training MR\n",
      "Starting Trial 8\n",
      "6\n",
      "Training 1-NN\n",
      "Training kNN\n",
      "Training SVR\n",
      "Training RFR\n",
      "Training MR\n",
      "12\n",
      "Training 1-NN\n",
      "Training kNN\n",
      "Training SVR\n",
      "Training RFR\n",
      "Training MR\n",
      "18\n",
      "Training 1-NN\n",
      "Training kNN\n",
      "Training SVR\n",
      "Training RFR\n",
      "Training MR\n",
      "30\n",
      "Training 1-NN\n",
      "Training kNN\n",
      "Training SVR\n",
      "Training RFR\n",
      "Training MR\n",
      "48\n",
      "Training 1-NN\n",
      "Training kNN\n",
      "Training SVR\n",
      "Training RFR\n",
      "Training MR\n",
      "78\n",
      "Training 1-NN\n",
      "Training kNN\n",
      "Training SVR\n",
      "Training RFR\n",
      "Training MR\n",
      "126\n",
      "Training 1-NN\n",
      "Training kNN\n",
      "Training SVR\n",
      "Training RFR\n",
      "Training MR\n",
      "204\n",
      "Training 1-NN\n",
      "Training kNN\n",
      "Training SVR\n",
      "Training RFR\n",
      "Training MR\n",
      "330\n",
      "Training 1-NN\n",
      "Training kNN\n",
      "Training SVR\n",
      "Training RFR\n",
      "Training MR\n",
      "Starting Trial 9\n",
      "6\n",
      "Training 1-NN\n",
      "Training kNN\n",
      "Training SVR\n",
      "Training RFR\n",
      "Training MR\n",
      "12\n",
      "Training 1-NN\n",
      "Training kNN\n",
      "Training SVR\n",
      "Training RFR\n",
      "Training MR\n",
      "18\n",
      "Training 1-NN\n",
      "Training kNN\n",
      "Training SVR\n",
      "Training RFR\n",
      "Training MR\n",
      "30\n",
      "Training 1-NN\n",
      "Training kNN\n",
      "Training SVR\n",
      "Training RFR\n",
      "Training MR\n",
      "48\n",
      "Training 1-NN\n",
      "Training kNN\n",
      "Training SVR\n",
      "Training RFR\n",
      "Training MR\n",
      "78\n",
      "Training 1-NN\n",
      "Training kNN\n",
      "Training SVR\n",
      "Training RFR\n",
      "Training MR\n",
      "126\n",
      "Training 1-NN\n",
      "Training kNN\n",
      "Training SVR\n",
      "Training RFR\n",
      "Training MR\n",
      "204\n",
      "Training 1-NN\n",
      "Training kNN\n",
      "Training SVR\n",
      "Training RFR\n",
      "Training MR\n",
      "330\n",
      "Training 1-NN\n",
      "Training kNN\n",
      "Training SVR\n",
      "Training RFR\n",
      "Training MR\n"
     ]
    }
   ],
   "source": [
    "# Performance/prediction datasets\n",
    "dict_storage = defaultdict(pd.DataFrame)\n",
    "parameter_resume = []\n",
    "\n",
    "performance_train_df = pd.DataFrame()\n",
    "predictions_train_df = pd.DataFrame()\n",
    "performance_test_df = pd.DataFrame()\n",
    "predictions_test_df = pd.DataFrame()\n",
    "\n",
    "for target in tqdm(regression_tids):\n",
    "\n",
    "    target_path = create_directory(f'./{main_folder}/{fingerprint}/{approach}/{target}/')\n",
    "\n",
    "    print(f'Training on {target}')\n",
    "\n",
    "    # Select Target Database\n",
    "    regression_db_tid = regression_db.loc[regression_db.chembl_tid == target]\n",
    "    #display(regression_db_tid)\n",
    "    df_regression = dataset_train_test(regression_db_tid, pot_bins=[5,7,9], tr_set_sizes=set_sizes, balance_test_set=True, n_trials=trial_splits, plot=False)\n",
    "\n",
    "    #display(df_regression)\n",
    "    for trial in range(trial_splits):\n",
    "        print(f'Starting Trial {trial}')\n",
    "\n",
    "        df_regression_trial = df_regression.loc[df_regression.trial == trial]\n",
    "\n",
    "        #Set seed\n",
    "        set_global_determinism(seed=trial)\n",
    "\n",
    "        # TEST set\n",
    "        df_regression_test = df_regression_trial.loc[(df_regression_trial.dataset == \"test\")]\n",
    "        test_set = Dataset(np.array(ECFP4(df_regression_test.nonstereo_aromatic_smiles.values)), np.array(df_regression_test.pPot.values))\n",
    "        test_set.add_instance(\"target\", df_regression_test.chembl_tid.values)\n",
    "        test_set.add_instance(\"smiles\", df_regression_test.nonstereo_aromatic_smiles.values)\n",
    "        test_set.add_instance(\"cid\", df_regression_test.chembl_cid.values)\n",
    "\n",
    "        for size in set_sizes:\n",
    "            print(size)\n",
    "\n",
    "            # TRAIN set\n",
    "            df_regression_train = df_regression_trial.loc[(df_regression_trial.dataset == \"train\") & (df_regression_trial.set_size == size)]\n",
    "            train_set = Dataset(np.array(ECFP4(df_regression_train.nonstereo_aromatic_smiles.values)), np.array(df_regression_train.pPot.values))\n",
    "            train_set.add_instance(\"target\", df_regression_train.chembl_tid.values)\n",
    "            train_set.add_instance(\"smiles\", df_regression_train.nonstereo_aromatic_smiles.values)\n",
    "            train_set.add_instance(\"cid\", df_regression_train.chembl_cid.values)\n",
    "            train_set.add_instance(\"potency_classes\", df_regression_train.potency_class.values)\n",
    "\n",
    "            for model in model_list:\n",
    "\n",
    "                print(f'Training {model}')\n",
    "\n",
    "                # Create saving directory\n",
    "                model_fpath = create_directory(f\"./{main_folder}/{fingerprint}/{approach}/{target}/{model}/{size}\", verbose=False)\n",
    "\n",
    "                ml_model = MLModel(train_set, model, data_type='balanced')\n",
    "                joblib.dump(ml_model.model, os.path.join(model_fpath, f\"{model}_{trial}.pkl\"))\n",
    "\n",
    "                #Best model parameters\n",
    "                opt_parameters_dict = {**{'model': model,\n",
    "                                            'trial': trial,\n",
    "                                            'Target ID': target,\n",
    "                                            'Training size':size}, **ml_model.best_params}\n",
    "                parameter_resume.append(opt_parameters_dict)\n",
    "\n",
    "                # TEST\n",
    "                #Model Evaluation\n",
    "                model_eval_train = ml_evaluation(ml_model, train_set, train_set, model_id=model)\n",
    "                model_eval_test = ml_evaluation(ml_model, test_set, train_set, model_id=model)\n",
    "\n",
    "                #Performance df\n",
    "                performance_train = model_eval_train.pred_performance\n",
    "                performance_train[\"trial\"] = trial\n",
    "                performance_train[\"Approach\"] = approach\n",
    "                performance_train[\"Training size\"] = size\n",
    "                performance_train_df = pd.concat([performance_train_df, performance_train])\n",
    "\n",
    "                # Prediction df\n",
    "                predictions_train = model_eval_train.predictions\n",
    "                predictions_train[\"trial\"] = trial\n",
    "                predictions_train[\"Approach\"] = approach\n",
    "                predictions_train[\"Training size\"] = size\n",
    "                predictions_train_df = pd.concat([predictions_train_df, predictions_train])\n",
    "\n",
    "                #Performance df\n",
    "                performance_test = model_eval_test.pred_performance\n",
    "                performance_test[\"trial\"] = trial\n",
    "                performance_test[\"Approach\"] = approach\n",
    "                performance_test[\"Training size\"] = size\n",
    "                performance_test_df = pd.concat([performance_test_df, performance_test])\n",
    "\n",
    "                # Prediction df\n",
    "                predictions_test = model_eval_test.predictions\n",
    "                predictions_test[\"trial\"] = trial\n",
    "                predictions_test[\"Approach\"] = approach\n",
    "                predictions_test[\"Training size\"] = size\n",
    "                predictions_test_df = pd.concat([predictions_test_df, predictions_test])\n",
    "\n",
    "parameter_df = pd.DataFrame(parameter_resume)\n",
    "\n",
    "# Save results\n",
    "performance_train_df.to_csv(os.path.join(result_path, f'performance_train.csv'))\n",
    "predictions_train_df.to_csv(os.path.join(result_path, f'predictions_train.csv'))\n",
    "performance_test_df.to_csv(os.path.join(result_path, f'performance_test.csv'))\n",
    "predictions_test_df.to_csv(os.path.join(result_path, f'predictions_test.csv'))\n",
    "parameter_df.to_csv(os.path.join(result_path, f'model_best_parameters.csv'))"
   ],
   "metadata": {
    "collapsed": false,
    "ExecuteTime": {
     "end_time": "2023-07-06T12:44:22.727204700Z",
     "start_time": "2023-07-06T10:48:52.861046900Z"
    }
   }
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 3",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 2
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython2",
   "version": "2.7.6"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 0
}