approach 1.5 - deep learning multimodel.ipynb

{
 "cells": [
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# Intro\n",
    "After seeing 'simple' neural networks weren't performing that well. I decided to look for the best publicly available neural network on the competition and learn for it. it was : https://www.kaggle.com/code/dngngy/icr-tensorflow-multimodel\n",
    "\n",
    "\n",
    "I simplified, cleaned and adapted its code, and analyzed it. The key takeaway is that for neural networks to be closer to competitive for tabular data ensembles were of great help."
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 1,
   "id": "b999c207",
   "metadata": {
    "execution": {
     "iopub.execute_input": "2024-02-16T12:19:56.049183Z",
     "iopub.status.busy": "2024-02-16T12:19:56.048658Z",
     "iopub.status.idle": "2024-02-16T12:19:56.095875Z",
     "shell.execute_reply": "2024-02-16T12:19:56.094452Z"
    },
    "papermill": {
     "duration": 0.056045,
     "end_time": "2024-02-16T12:19:56.099119",
     "exception": false,
     "start_time": "2024-02-16T12:19:56.043074",
     "status": "completed"
    },
    "tags": []
   },
   "outputs": [],
   "source": [
    "def create_tf_ffn_preds():\n",
    "    \"\"\"Process the dataframe, create the stratified kfolds, create the model, and use it to make the submissions\"\"\"\n",
    "    import pandas as pd\n",
    "    import seaborn as sns\n",
    "    import matplotlib.pyplot as plt\n",
    "\n",
    "    import tensorflow as tf\n",
    "    from tensorflow.keras import Sequential, layers, Input, Model\n",
    "    from tensorflow.experimental import numpy as np\n",
    "    from tensorflow.keras.losses import Loss\n",
    "    from tensorflow.keras import backend as K\n",
    "    from tensorflow.keras.callbacks import EarlyStopping\n",
    "\n",
    "    from sklearn.model_selection import train_test_split,StratifiedKFold\n",
    "    from sklearn.impute import KNNImputer\n",
    "    from sklearn.preprocessing import StandardScaler,OneHotEncoder\n",
    "    from sklearn.compose import ColumnTransformer\n",
    "    from sklearn.pipeline import Pipeline\n",
    "    seed = 1\n",
    "    np.random.seed(seed)\n",
    "    tf.random.set_seed(seed)\n",
    "\n",
    "    class Path:\n",
    "        sample_submission: str = \"/kaggle/input/icr-identify-age-related-conditions/sample_submission.csv\"\n",
    "        greeks: str = \"/kaggle/input/icr-identify-age-related-conditions/greeks.csv\"\n",
    "        train: str = \"/kaggle/input/icr-identify-age-related-conditions/train.csv\"\n",
    "        to_predict: str = \"/kaggle/input/icr-identify-age-related-conditions/test.csv\" #the data that needs to be predicted if it has disease or not\n",
    "                   \n",
    "\n",
    "    def get_corr_cols(data:pd.DataFrame,label:str, corr:float) -> list:\n",
    "        \"\"\"\n",
    "            Get columns with high correlations to label\n",
    "        \"\"\"\n",
    "        assert corr <=1, \"corr shoud be in [0,1]\"\n",
    "        correlations = data.corrwith(data[label],numeric_only=True).to_frame()\n",
    "        correlations[\"abs\"] = correlations[0].abs()\n",
    "        correlations = correlations.sort_values(\"abs\", ascending=False)\n",
    "        correlations.drop(label,inplace=True)\n",
    "        train_cols = correlations[correlations[\"abs\"] >= corr].index.to_list() \n",
    "        return train_cols\n",
    "    \n",
    "\n",
    "    def create_processing_pipeline(numeric_cols, cat_cols) ->Pipeline:\n",
    "        preprocessor = ColumnTransformer(\n",
    "                transformers=[\n",
    "                    ('numeric_scaler',  StandardScaler(), numeric_cols),\n",
    "                    ('categorical', OneHotEncoder(), cat_cols)\n",
    "                ])\n",
    "        preprocess_pipeline = Pipeline([\n",
    "            (\"preprocessor\", preprocessor),\n",
    "            (\"imputer\",KNNImputer())\n",
    "        ])\n",
    "\n",
    "        return preprocess_pipeline\n",
    "    \n",
    "    def make_datasets(train_x, train_y, valid_x, valid_y):\n",
    "        ds_train= tf.data.Dataset.from_tensor_slices(\n",
    "            (train_x,train_y.astype(\"float64\"))).batch(16).shuffle(1000, seed=seed)\n",
    "        ds_valid= tf.data.Dataset.from_tensor_slices(\n",
    "            (valid_x,valid_y.astype(\"float64\"))).batch(16).shuffle(1000, seed=seed)\n",
    "        return ds_train, ds_valid\n",
    "\n",
    "    class BalancedLogLoss(Loss):\n",
    "        def call(self, y_true, y_pred):\n",
    "            # y_true: correct labels 0, 1\n",
    "            # y_pred: predicted probabilities of class=1\n",
    "            # Implements the Evaluation equation with w_0 = w_1 = 1.\n",
    "            # Calculate the number of observations for each class\n",
    "            zero = tf.constant(1e-15)\n",
    "            N_0 = tf.clip_by_value(tf.math.reduce_sum(1 - y_true), zero,1e10)\n",
    "            N_1 = tf.clip_by_value(tf.math.reduce_sum(y_true), zero,1e10)\n",
    "            # Calculate the predicted probabilities for each class\n",
    "            p_1 = tf.clip_by_value(y_pred, zero, 1-zero)\n",
    "            p_0 = tf.clip_by_value(1-p_1, zero, 1-zero)\n",
    "            # Calculate the average log loss for each class\n",
    "            log_loss_0 = -tf.math.reduce_sum((1 - y_true) * tf.math.log(p_0)) / N_0 \n",
    "            log_loss_1 = -tf.math.reduce_sum(y_true * tf.math.log(p_1)) / N_1 \n",
    "            # return the (not further weighted) average of the averages\n",
    "            return (log_loss_0 + log_loss_1)/2 \n",
    "\n",
    "    class MultiMoldel():\n",
    "        def __init__(self, models:list) -> None:\n",
    "            self.models = models\n",
    "            self.average = layers.Average()\n",
    "        def __call__(self,x):\n",
    "            outputs = []\n",
    "            for model in model_dicts:\n",
    "                submodel_x = model[\"preprocess_pipeline\"].transform(x) #Use the processing pipeline the model was fit with.\n",
    "                output = model[\"model\"](submodel_x)\n",
    "                outputs.append(output)\n",
    "            return self.average(outputs)\n",
    "    \n",
    "    train_df, greeks_df, df_to_predict, sample_submission_df = pd.read_csv(Path.train,index_col=\"Id\"), pd.read_csv(Path.greeks,index_col=\"Id\"), pd.read_csv(Path.to_predict,index_col=\"Id\"), pd.read_csv(Path.sample_submission,index_col=\"Id\")\n",
    "    \n",
    "    \n",
    "    cat_cols = [col for col in train_df if train_df[col].dtype == \"O\"]\n",
    "    numeric_cols = get_corr_cols(train_df,\"Class\",0.16)\n",
    "    input_shape = (len(numeric_cols) + 2, ) # magic number\n",
    "    \n",
    "    #Separate target column\n",
    "    train_x = train_df.iloc[:,:-1]; train_y = train_df.iloc[:,-1]\n",
    "    \n",
    "    skf = StratifiedKFold(n_splits=5)\n",
    "    splits = enumerate(skf.split(train_x, train_y))\n",
    "    model_dicts = []\n",
    "    lrs = [5.0119e-04,0.0056,1.0000e-04,5.6234e-05,7.0795e-05]  #The lrs were chosen with a lr callback, and thening the lr. The epochs by ploting learning with those rates. See the original for more details.\n",
    "    epochs = [108,50,15,15,15]\n",
    "    for i, (train_index, valid_index) in splits: #The train_ids and valid ids of each split (aka the ids for the fold)\n",
    "        fold_train_x, fold_train_y, fold_valx, fold_valy = train_x.iloc[train_index], train_y.iloc[train_index], train_x.iloc[valid_index], train_y.iloc[valid_index]\n",
    "        fold_x_processing_pipeline = create_processing_pipeline(numeric_cols, cat_cols).fit(fold_train_x)\n",
    "        \n",
    "        fold_train_x ,fold_valx = fold_x_processing_pipeline.transform(fold_train_x), fold_x_processing_pipeline.transform(fold_valx)\n",
    "        \n",
    "        ds_train, ds_valid, = make_datasets(fold_train_x, fold_train_y, fold_valx, fold_valy)\n",
    "        \n",
    "\n",
    "        model = Sequential([\n",
    "            Input(shape=input_shape),\n",
    "            layers.Lambda(lambda x: tf.expand_dims(x, axis=-1)),\n",
    "\n",
    "            layers.Conv1D(16,5,padding=\"same\",\n",
    "                      kernel_initializer='lecun_normal',\n",
    "                      activation='selu'),\n",
    "            layers.Conv1D(32,2,padding=\"same\",\n",
    "                      kernel_initializer='lecun_normal',\n",
    "                      activation='selu'),\n",
    "            layers.MaxPool1D(),\n",
    "\n",
    "            layers.Flatten(),\n",
    "            layers.AlphaDropout(0.5),\n",
    "            layers.Dense(128,\n",
    "                     kernel_initializer='lecun_normal',\n",
    "                     activation=\"selu\"),\n",
    "            layers.Dense(1,activation=\"sigmoid\")\n",
    "        ])\n",
    "\n",
    "\n",
    "        model.compile(\n",
    "            optimizer=tf.keras.optimizers.Adam(lrs[i]),\n",
    "            loss= BalancedLogLoss(),\n",
    "            metrics=\"accuracy\"\n",
    "        )\n",
    "\n",
    "        history = model.fit(\n",
    "            ds_train,\n",
    "            validation_data=ds_valid,\n",
    "            epochs=epochs[i],\n",
    "            verbose=0\n",
    "        )\n",
    "        model_dicts.append(\n",
    "        {\n",
    "            \"model\":model,\n",
    "            \"preprocess_pipeline\": fold_x_processing_pipeline,\n",
    "            \"history\":history\n",
    "        })\n",
    "\n",
    "\n",
    "    multi_model = MultiMoldel([model_dict['model'] for model_dict in model_dicts])\n",
    "    pred = multi_model(df_to_predict)\n",
    "    submission = pd.read_csv(Path.sample_submission)\n",
    "    submission[\"class_1\"] = pred.numpy()\n",
    "    submission[\"class_0\"] = 1 - pred.numpy()\n",
    "    submission.to_csv(\"submission.csv\",index=False)\n",
    "    \n",
    "    return model_dicts, multi_model, pred, submission\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 2,
   "id": "3a1bbffd",
   "metadata": {
    "execution": {
     "iopub.execute_input": "2024-02-16T12:19:56.107386Z",
     "iopub.status.busy": "2024-02-16T12:19:56.106865Z",
     "iopub.status.idle": "2024-02-16T12:20:40.456823Z",
     "shell.execute_reply": "2024-02-16T12:20:40.455934Z"
    },
    "papermill": {
     "duration": 44.357325,
     "end_time": "2024-02-16T12:20:40.459647",
     "exception": false,
     "start_time": "2024-02-16T12:19:56.102322",
     "status": "completed"
    },
    "tags": []
   },
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "/opt/conda/lib/python3.10/site-packages/scipy/__init__.py:146: UserWarning: A NumPy version >=1.16.5 and <1.23.0 is required for this version of SciPy (detected version 1.23.5\n",
      "  warnings.warn(f\"A NumPy version >={np_minversion} and <{np_maxversion}\"\n"
     ]
    }
   ],
   "source": [
    "model_dicts, multi_model, pred, submission = create_tf_ffn_preds()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 3,
   "id": "eccc7265",
   "metadata": {
    "execution": {
     "iopub.execute_input": "2024-02-16T12:20:40.467368Z",
     "iopub.status.busy": "2024-02-16T12:20:40.466934Z",
     "iopub.status.idle": "2024-02-16T12:20:40.490832Z",
     "shell.execute_reply": "2024-02-16T12:20:40.489740Z"
    },
    "papermill": {
     "duration": 0.030385,
     "end_time": "2024-02-16T12:20:40.493190",
     "exception": false,
     "start_time": "2024-02-16T12:20:40.462805",
     "status": "completed"
    },
    "tags": []
   },
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>Id</th>\n",
       "      <th>class_0</th>\n",
       "      <th>class_1</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>00eed32682bb</td>\n",
       "      <td>0.811411</td>\n",
       "      <td>0.188589</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>010ebe33f668</td>\n",
       "      <td>0.811411</td>\n",
       "      <td>0.188589</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2</th>\n",
       "      <td>02fa521e1838</td>\n",
       "      <td>0.811411</td>\n",
       "      <td>0.188589</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3</th>\n",
       "      <td>040e15f562a2</td>\n",
       "      <td>0.811411</td>\n",
       "      <td>0.188589</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4</th>\n",
       "      <td>046e85c7cc7f</td>\n",
       "      <td>0.811411</td>\n",
       "      <td>0.188589</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "             Id   class_0   class_1\n",
       "0  00eed32682bb  0.811411  0.188589\n",
       "1  010ebe33f668  0.811411  0.188589\n",
       "2  02fa521e1838  0.811411  0.188589\n",
       "3  040e15f562a2  0.811411  0.188589\n",
       "4  046e85c7cc7f  0.811411  0.188589"
      ]
     },
     "execution_count": 3,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "submission"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "0b19b883",
   "metadata": {
    "papermill": {
     "duration": 0.002717,
     "end_time": "2024-02-16T12:20:40.498989",
     "exception": false,
     "start_time": "2024-02-16T12:20:40.496272",
     "status": "completed"
    },
    "tags": []
   },
   "outputs": [],
   "source": []
  }
 ],
 "metadata": {
  "kaggle": {
   "accelerator": "none",
   "dataSources": [
    {
     "databundleVersionId": 5687476,
     "sourceId": 52784,
     "sourceType": "competition"
    }
   ],
   "dockerImageVersionId": 30474,
   "isGpuEnabled": false,
   "isInternetEnabled": false,
   "language": "python",
   "sourceType": "notebook"
  },
  "kernelspec": {
   "display_name": "Python 3",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.10.10"
  },
  "papermill": {
   "default_parameters": {},
   "duration": 60.216568,
   "end_time": "2024-02-16T12:20:43.279424",
   "environment_variables": {},
   "exception": null,
   "input_path": "__notebook__.ipynb",
   "output_path": "__notebook__.ipynb",
   "parameters": {},
   "start_time": "2024-02-16T12:19:43.062856",
   "version": "2.4.0"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 5
}