auto-ml-and-kaggle.ipynb

{
 "cells": [
  {
   "cell_type": "markdown",
   "metadata": {
    "slideshow": {
     "slide_type": "-"
    }
   },
   "source": [
    "# Getting Ahead in Kaggle Competitions With Automated Machine Learning"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## Vlad Iliescu\n",
    "\n",
    "- Head of AI @ strongbytes.ai\n",
    "- Organizer @ ndrconf.ai\n",
    "- Microsoft MVP on AI"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "Reaching new heights the Titanic: Machine Learning from Disaster competition with as little effort as possible\n",
    "\n",
    "https://www.kaggle.com/c/titanic"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "import pandas as pd\n",
    "\n",
    "train = pd.read_csv('./data/train.csv')\n",
    "\n",
    "train.head()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "test = pd.read_csv('./data/test.csv')\n",
    "test.head()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "submission = pd.read_csv('./data/gender_submission.csv')\n",
    "submission.head()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "from sklearn.tree import DecisionTreeClassifier\n",
    "\n",
    "y = train['Survived']\n",
    "X = train[['PassengerId', 'Pclass', 'Name', 'Sex', 'Age', 'SibSp', 'Parch', 'Ticket', 'Fare', 'Cabin', 'Embarked']]\n",
    "\n",
    "model = DecisionTreeClassifier()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "model.fit(X, y)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "train.info()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "columns = ['PassengerId', 'Pclass', 'Age', 'SibSp', 'Parch', 'Fare']\n",
    "X = train[columns]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "model.fit(X, y)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "X.info()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "X['Age'].fillna(X['Age'].mean(), inplace=True)\n",
    "X.info()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "model.fit(X, y)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "model.predict(test[columns])"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "test['Age'].fillna(test['Age'].mean(), inplace=True)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "model.predict(test[columns])"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "test[columns].info()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "test['Fare'].fillna(test['Fare'].mean(), inplace=True)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "model.predict(test[columns])"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "y_predicted = model.predict(test[columns])\n",
    "\n",
    "output = pd.DataFrame({'PassengerId': test['PassengerId'], 'Survived': y_predicted})\n",
    "output"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "output.to_csv('./submissions/scikit.csv', index=False)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "![a](./img/here_to_help_2x.png)\n",
    "\n",
    "Source: https://xkcd.com/1831/"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# Enter Automated ML"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### Requirements:\n",
    "- An Azure account\n",
    "- A _Resource Group_ with a _Machine Learning Service Workspace_\n",
    "- https://docs.microsoft.com/en-us/azure/machine-learning/tutorial-1st-experiment-sdk-setup"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "import azureml.core\n",
    "\n",
    "print(\"Azure ML SDK Version: \", azureml.core.VERSION)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "# Authenticate if we haven't already\n",
    "\n",
    "from azureml.core.authentication import InteractiveLoginAuthentication\n",
    "\n",
    "credentials = InteractiveLoginAuthentication()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "# Load the workspace info\n",
    "\n",
    "from azureml.core import Workspace\n",
    "\n",
    "ws = Workspace.from_config(auth=credentials)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "# Create a new experiment if we haven't already\n",
    "\n",
    "from azureml.core import Experiment\n",
    "exp = Experiment(workspace=ws, name='Titanic')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "import logging\n",
    "from azureml.train.automl import AutoMLConfig\n",
    "\n",
    "# See possible options here - https://docs.microsoft.com/en-us/azure/machine-learning/service/how-to-configure-auto-train#primary-metric\n",
    "config = AutoMLConfig(\n",
    "    task='classification',\n",
    "    primary_metric='AUC_weighted',\n",
    "    iterations=3,\n",
    "    training_data=train,\n",
    "    label_column_name='Survived',\n",
    "    n_cross_validations=5,\n",
    "    iteration_timeout_minutes=1,\n",
    "#    preprocess=True,  # old\n",
    "#    featurization='auto',  # new\n",
    "    verbosity=logging.INFO,\n",
    "    model_explainability=False\n",
    ")\n",
    "\n",
    "# ~1.5 minutes to run\n",
    "local_run = exp.submit(config, show_output=True)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "from azureml.widgets import RunDetails\n",
    "RunDetails(local_run).show()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "best_run, fitted_model = local_run.get_output()\n",
    "print('Best run:', best_run)\n",
    "print('-----')\n",
    "print('Best model:', fitted_model)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "def generate_submission_file(model, X_test):    \n",
    "    output = pd.DataFrame({\n",
    "        'PassengerId': X_test['PassengerId'], \n",
    "        'Survived': model.predict(X_test)\n",
    "    })\n",
    "    output.to_csv('./submissions/automl.csv', index=False)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "generate_submission_file(fitted_model, pd.read_csv('./data/test.csv'))    "
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## Analyze the trained model"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "fitted_model.named_steps"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "transformer = fitted_model.named_steps['datatransformer']\n",
    "scaler = fitted_model.named_steps['MaxAbsScaler']\n",
    "classifier = fitted_model.named_steps['LightGBMClassifier']    "
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "help(transformer)\n",
    "# type\n",
    "# help\n",
    "# dir"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "type(scaler)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "classifier.model\n",
    "\n",
    "#Examine ~/anaconda/envs/<env_name>/lib/python3.6/site-packages/azureml/automl/runtime/shared/model_wrappers.py"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "transformer.get_engineered_feature_names()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "len(transformer.get_engineered_feature_names())"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "transformer.get_featurization_summary()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "test_preprocessed = transformer.transform(test)\n",
    "\n",
    "test_preprocessed"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "pd.DataFrame(test_preprocessed.toarray(), columns=transformer.get_engineered_feature_names())"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "test = pd.read_csv('./data/test.csv')\n",
    "test.tail(10)[['Age']]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "test_scaled = scaler.transform(test_preprocessed)\n",
    "\n",
    "test_scaled"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "pd.DataFrame(test_scaled.toarray(), columns=transformer.get_engineered_feature_names())"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "results = classifier.predict(test_scaled)\n",
    "\n",
    "results"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## More iterations"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "# ~7 minutes to run\n",
    "\n",
    "config = AutoMLConfig(\n",
    "    task='classification',\n",
    "    primary_metric='AUC_weighted',\n",
    "    iterations=15,\n",
    "    training_data=train,\n",
    "    label_column_name='Survived',\n",
    "    n_cross_validations=5,\n",
    "#     preprocess=True,\n",
    "    verbosity=logging.INFO,\n",
    "    model_explainability=False\n",
    ")\n",
    "\n",
    "local_run = exp.submit(config, show_output=True)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "best_run, fitted_model_15 = local_run.get_output()\n",
    "\n",
    "fitted_model_15"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "fitted_model_15.named_steps"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "classifier = fitted_model_15.named_steps['prefittedsoftvotingclassifier']\n",
    "classifier"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "classifier.__module__"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "classifier.estimators"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "classifier.weights"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "X_test = pd.read_csv('./data/test.csv')\n",
    "generate_submission_file(fitted_model_15, X_test)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## Using explainability"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "config = AutoMLConfig(\n",
    "    task='classification',\n",
    "    primary_metric='AUC_weighted',\n",
    "    iterations=5,\n",
    "    training_data=train,\n",
    "    label_column_name='Survived',\n",
    "    n_cross_validations=5,\n",
    "    iteration_timeout_minutes=2,\n",
    "#     preprocess=True,\n",
    "    model_explainability=True\n",
    ")\n",
    "\n",
    "run_exp = exp.submit(config=config, show_output=True)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "from azureml.contrib.interpret.explanation.explanation_client import ExplanationClient\n",
    "\n",
    "best_run, fitted_model_expl = run_exp.get_output()\n",
    "\n",
    "client = ExplanationClient.from_run(best_run)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "dir(client)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "explanations = client.download_model_explanation(raw=False)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "dir(explanations)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "explanations.get_feature_importance_dict()"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## Lessons learned"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "# Light pre-processing - extract titles from names\n",
    "\n",
    "train['Name'].str.extract(' ([A-Za-z]+)\\.', expand=True)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "def process(df: pd.DataFrame):\n",
    "    frame = df.copy()\n",
    "    frame['HasNoSibSp'] = frame['SibSp'] == 0\n",
    "    frame['HasNoParch'] = frame['Parch'] == 0\n",
    "    \n",
    "    frame['SibSp'] = frame['SibSp'].astype('float64')\n",
    "    frame['Parch'] = frame['Parch'].astype('float64')\n",
    "    frame['Pclass'] = frame['Pclass'].astype('category')\n",
    "\n",
    "    frame['Title'] = frame['Name'].str.extract(' ([A-Za-z]+)\\.', expand=True)\n",
    "    frame['Title'] = frame['Title'].replace('Mlle', 'Miss')\n",
    "    frame['Title'] = frame['Title'].replace('Ms', 'Miss')\n",
    "    frame['Title'] = frame['Title'].replace('Mme', 'Mrs')\n",
    "    \n",
    "    frame.drop(columns=['PassengerId'], inplace=True)\n",
    "    \n",
    "    return frame"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "train_df = process(train)\n",
    "\n",
    "from azureml.train.automl import AutoMLConfig\n",
    "\n",
    "config = AutoMLConfig(\n",
    "    task='classification',\n",
    "    primary_metric='accuracy',\n",
    "    iterations=15,\n",
    "    training_data=train_df,\n",
    "    label_column_name='Survived',\n",
    "    n_cross_validations=5,\n",
    "    iteration_timeout_minutes=2,\n",
    "#     preprocess=True,\n",
    "    model_explainability=False,\n",
    ")\n",
    "\n",
    "run_simplified = exp.submit(config=config, show_output=True)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "best_run, fitted_model_simplified = run_simplified.get_output()\n",
    "\n",
    "test = pd.read_csv('./data/test.csv')\n",
    "test_df = process(test)\n",
    "\n",
    "output = pd.DataFrame({\n",
    "    'PassengerId': test['PassengerId'],\n",
    "    'Survived': fitted_model_simplified.predict(test_df)\n",
    "})\n",
    "\n",
    "output.to_csv('./submissions/automl_simplified.csv', index=False)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# Tweak some hyper parameters"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "train_df = process(train)\n",
    "\n",
    "from azureml.train.automl import AutoMLConfig\n",
    "\n",
    "config = AutoMLConfig(\n",
    "    task='classification',\n",
    "    primary_metric='accuracy',\n",
    "    iterations=7,\n",
    "    training_data=train_df,\n",
    "    label_column_name='Survived',\n",
    "    n_cross_validations=5,\n",
    "#     preprocess=True,\n",
    "    model_explainability=True,\n",
    ")\n",
    "\n",
    "run_simplified = exp.submit(config=config, show_output=True)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "best_run, fitted_model_tweaked = run_simplified.get_output()\n",
    "\n",
    "test = pd.read_csv('./data/test.csv')\n",
    "test_df = process(test)\n",
    "\n",
    "output = pd.DataFrame({\n",
    "    'PassengerId': test['PassengerId'],\n",
    "    'Survived': fitted_model_tweaked.predict(test_df)\n",
    "})\n",
    "\n",
    "output.to_csv('./submissions/automl_tweakable.csv', index=False)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "slideshow": {
     "slide_type": "skip"
    }
   },
   "outputs": [],
   "source": [
    "# train_df = process(train)\n",
    "\n",
    "\n",
    "fitted_model_tweaked.fit(train_df.drop(columns=['Survived']), train_df['Survived'])\n",
    "\n",
    "output = pd.DataFrame({\n",
    "    'PassengerId': test['PassengerId'],\n",
    "    'Survived': fitted_model_tweaked.predict(test_df)\n",
    "})\n",
    "\n",
    "output.to_csv('./submissions/automl_tweakable.csv', index=False)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "# Tweak params (but make sure you use the right params for the model)\n",
    "\n",
    "# params={\n",
    "#     'alpha': 0.0001,\n",
    "#     'l1_ratio': 1,\n",
    "# }\n",
    "\n",
    "params={\n",
    "    'reg_alpha': 0.5\n",
    "}\n",
    "\n",
    "fitted_model_tweaked.steps[-1][-1].estimators[0][-1][-1].set_params(**params)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "fitted_model_tweaked.steps[-1][-1].estimators[0][-1][-1].get_params()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "# train_df = process(train)\n",
    "\n",
    "\n",
    "fitted_model_tweaked.fit(train_df.drop(columns=['Survived']), train_df['Survived'])\n",
    "\n",
    "output = pd.DataFrame({\n",
    "    'PassengerId': test['PassengerId'],\n",
    "    'Survived': fitted_model_tweaked.predict(test_df)\n",
    "})\n",
    "\n",
    "output.to_csv('./automl_simplified_tweaked_2.csv', index=False)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# The End"
   ]
  }
 ],
 "metadata": {
  "celltoolbar": "Slideshow",
  "kernelspec": {
   "display_name": "ainights",
   "language": "python",
   "name": "ainights"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.6.10"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 4
}