mlflow-example-real-or-not-disaster-tweets-modeling-SVC.ipynb

{
 "cells": [
  {
   "source": [
    "# Author: Mihir Gajjar, Data Science Division, Statistics Canada."
   ],
   "cell_type": "markdown",
   "metadata": {}
  },
  {
   "source": [
    "# Goal: Demonstrate Mlflow features."
   ],
   "cell_type": "markdown",
   "metadata": {}
  },
  {
   "cell_type": "markdown",
   "metadata": {
    "papermill": {
     "duration": 0.027186,
     "end_time": "2021-04-23T18:51:01.353346",
     "exception": false,
     "start_time": "2021-04-23T18:51:01.326160",
     "status": "completed"
    },
    "tags": []
   },
   "source": [
    "## Imports"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 242,
   "metadata": {
    "papermill": {
     "duration": 1.945868,
     "end_time": "2021-04-23T18:51:03.329864",
     "exception": false,
     "start_time": "2021-04-23T18:51:01.383996",
     "status": "completed"
    },
    "tags": []
   },
   "outputs": [],
   "source": [
    "import time\n",
    "\n",
    "import pandas as pd\n",
    "\n",
    "from nltk import WordNetLemmatizer\n",
    "from nltk.tokenize import word_tokenize\n",
    "from nltk.corpus import stopwords\n",
    "\n",
    "from sklearn.svm import SVC\n",
    "from sklearn.model_selection import cross_validate, StratifiedKFold\n",
    "from sklearn.feature_extraction.text import TfidfVectorizer\n",
    "from sklearn.metrics import f1_score, make_scorer\n",
    "\n",
    "import mlflow"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 243,
   "metadata": {},
   "outputs": [
    {
     "output_type": "stream",
     "name": "stdout",
     "text": [
      "Run name:  1626200071\n"
     ]
    }
   ],
   "source": [
    "run_name = str(int(time.time()))\n",
    "print('Run name: ', run_name)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {
    "papermill": {
     "duration": 0.026237,
     "end_time": "2021-04-23T18:51:03.382853",
     "exception": false,
     "start_time": "2021-04-23T18:51:03.356616",
     "status": "completed"
    },
    "tags": []
   },
   "source": [
    "## Reading the data"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 244,
   "metadata": {},
   "outputs": [],
   "source": [
    "# Kaggle competition data download link: https://www.kaggle.com/c/nlp-getting-started/data\n",
    "train_data = pd.read_csv(\"./data/train.csv\")\n",
    "test_data = pd.read_csv(\"./data/test.csv\")"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 245,
   "metadata": {
    "papermill": {
     "duration": 0.062637,
     "end_time": "2021-04-23T18:51:03.620245",
     "exception": false,
     "start_time": "2021-04-23T18:51:03.557608",
     "status": "completed"
    },
    "tags": []
   },
   "outputs": [
    {
     "output_type": "execute_result",
     "data": {
      "text/plain": [
       "         id keyword location  \\\n",
       "0         1     NaN      NaN   \n",
       "1         4     NaN      NaN   \n",
       "2         5     NaN      NaN   \n",
       "3         6     NaN      NaN   \n",
       "4         7     NaN      NaN   \n",
       "...     ...     ...      ...   \n",
       "7608  10869     NaN      NaN   \n",
       "7609  10870     NaN      NaN   \n",
       "7610  10871     NaN      NaN   \n",
       "7611  10872     NaN      NaN   \n",
       "7612  10873     NaN      NaN   \n",
       "\n",
       "                                                   text  target  \n",
       "0     Our Deeds are the Reason of this #earthquake M...       1  \n",
       "1                Forest fire near La Ronge Sask. Canada       1  \n",
       "2     All residents asked to 'shelter in place' are ...       1  \n",
       "3     13,000 people receive #wildfires evacuation or...       1  \n",
       "4     Just got sent this photo from Ruby #Alaska as ...       1  \n",
       "...                                                 ...     ...  \n",
       "7608  Two giant cranes holding a bridge collapse int...       1  \n",
       "7609  @aria_ahrary @TheTawniest The out of control w...       1  \n",
       "7610  M1.94 [01:04 UTC]?5km S of Volcano Hawaii. htt...       1  \n",
       "7611  Police investigating after an e-bike collided ...       1  \n",
       "7612  The Latest: More Homes Razed by Northern Calif...       1  \n",
       "\n",
       "[7613 rows x 5 columns]"
      ],
      "text/html": "<div>\n<style scoped>\n    .dataframe tbody tr th:only-of-type {\n        vertical-align: middle;\n    }\n\n    .dataframe tbody tr th {\n        vertical-align: top;\n    }\n\n    .dataframe thead th {\n        text-align: right;\n    }\n</style>\n<table border=\"1\" class=\"dataframe\">\n  <thead>\n    <tr style=\"text-align: right;\">\n      <th></th>\n      <th>id</th>\n      <th>keyword</th>\n      <th>location</th>\n      <th>text</th>\n      <th>target</th>\n    </tr>\n  </thead>\n  <tbody>\n    <tr>\n      <th>0</th>\n      <td>1</td>\n      <td>NaN</td>\n      <td>NaN</td>\n      <td>Our Deeds are the Reason of this #earthquake M...</td>\n      <td>1</td>\n    </tr>\n    <tr>\n      <th>1</th>\n      <td>4</td>\n      <td>NaN</td>\n      <td>NaN</td>\n      <td>Forest fire near La Ronge Sask. Canada</td>\n      <td>1</td>\n    </tr>\n    <tr>\n      <th>2</th>\n      <td>5</td>\n      <td>NaN</td>\n      <td>NaN</td>\n      <td>All residents asked to 'shelter in place' are ...</td>\n      <td>1</td>\n    </tr>\n    <tr>\n      <th>3</th>\n      <td>6</td>\n      <td>NaN</td>\n      <td>NaN</td>\n      <td>13,000 people receive #wildfires evacuation or...</td>\n      <td>1</td>\n    </tr>\n    <tr>\n      <th>4</th>\n      <td>7</td>\n      <td>NaN</td>\n      <td>NaN</td>\n      <td>Just got sent this photo from Ruby #Alaska as ...</td>\n      <td>1</td>\n    </tr>\n    <tr>\n      <th>...</th>\n      <td>...</td>\n      <td>...</td>\n      <td>...</td>\n      <td>...</td>\n      <td>...</td>\n    </tr>\n    <tr>\n      <th>7608</th>\n      <td>10869</td>\n      <td>NaN</td>\n      <td>NaN</td>\n      <td>Two giant cranes holding a bridge collapse int...</td>\n      <td>1</td>\n    </tr>\n    <tr>\n      <th>7609</th>\n      <td>10870</td>\n      <td>NaN</td>\n      <td>NaN</td>\n      <td>@aria_ahrary @TheTawniest The out of control w...</td>\n      <td>1</td>\n    </tr>\n    <tr>\n      <th>7610</th>\n      <td>10871</td>\n      <td>NaN</td>\n      <td>NaN</td>\n      <td>M1.94 [01:04 UTC]?5km S of Volcano Hawaii. htt...</td>\n      <td>1</td>\n    </tr>\n    <tr>\n      <th>7611</th>\n      <td>10872</td>\n      <td>NaN</td>\n      <td>NaN</td>\n      <td>Police investigating after an e-bike collided ...</td>\n      <td>1</td>\n    </tr>\n    <tr>\n      <th>7612</th>\n      <td>10873</td>\n      <td>NaN</td>\n      <td>NaN</td>\n      <td>The Latest: More Homes Razed by Northern Calif...</td>\n      <td>1</td>\n    </tr>\n  </tbody>\n</table>\n<p>7613 rows × 5 columns</p>\n</div>"
     },
     "metadata": {},
     "execution_count": 245
    }
   ],
   "source": [
    "train_data"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 246,
   "metadata": {
    "papermill": {
     "duration": 0.039109,
     "end_time": "2021-04-23T18:51:03.689717",
     "exception": false,
     "start_time": "2021-04-23T18:51:03.650608",
     "status": "completed"
    },
    "tags": []
   },
   "outputs": [
    {
     "output_type": "stream",
     "name": "stdout",
     "text": [
      "The length of the training data is 7613\nThe length of the test data is 3263\n"
     ]
    }
   ],
   "source": [
    "print('The length of the training data is %d' % len(train_data))\n",
    "print('The length of the test data is %d' % len(test_data))"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {
    "papermill": {
     "duration": 0.030966,
     "end_time": "2021-04-23T18:51:04.510410",
     "exception": false,
     "start_time": "2021-04-23T18:51:04.479444",
     "status": "completed"
    },
    "tags": []
   },
   "source": [
    "## Text preprocessing"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 247,
   "metadata": {
    "papermill": {
     "duration": 0.043537,
     "end_time": "2021-04-23T18:51:04.585510",
     "exception": false,
     "start_time": "2021-04-23T18:51:04.541973",
     "status": "completed"
    },
    "tags": []
   },
   "outputs": [],
   "source": [
    "def clean_text(text):\n",
    "    # split into words\n",
    "    tokens = word_tokenize(text)\n",
    "    # remove all tokens that are not alphanumeric. Can also use .isalpha() here if do not want to keep numbers.\n",
    "    words = [word for word in tokens if word.isalnum()]\n",
    "    # remove stopwords\n",
    "    stop_words = stopwords.words('english')\n",
    "    words = [word for word in words if word not in stop_words]\n",
    "    # performing lemmatization\n",
    "    wordnet_lemmatizer = WordNetLemmatizer()\n",
    "    words = [wordnet_lemmatizer.lemmatize(word) for word in words]\n",
    "    # Converting list of words to string\n",
    "    words = ' '.join(words)\n",
    "    return words"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 248,
   "metadata": {
    "papermill": {
     "duration": 7.004522,
     "end_time": "2021-04-23T18:51:11.683552",
     "exception": false,
     "start_time": "2021-04-23T18:51:04.679030",
     "status": "completed"
    },
    "tags": []
   },
   "outputs": [],
   "source": [
    "train_data['cleaned_text'] = train_data['text'].apply(clean_text)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 249,
   "metadata": {
    "papermill": {
     "duration": 0.040577,
     "end_time": "2021-04-23T18:51:11.756310",
     "exception": false,
     "start_time": "2021-04-23T18:51:11.715733",
     "status": "completed"
    },
    "tags": []
   },
   "outputs": [
    {
     "output_type": "execute_result",
     "data": {
      "text/plain": [
       "'.@NorwayMFA #Bahrain police had previously died in a road accident they were not killed by explosion https://t.co/gFJfgTodad'"
      ]
     },
     "metadata": {},
     "execution_count": 249
    }
   ],
   "source": [
    "train_data['text'].iloc[100]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 250,
   "metadata": {
    "papermill": {
     "duration": 0.042325,
     "end_time": "2021-04-23T18:51:11.831363",
     "exception": false,
     "start_time": "2021-04-23T18:51:11.789038",
     "status": "completed"
    },
    "tags": []
   },
   "outputs": [
    {
     "output_type": "execute_result",
     "data": {
      "text/plain": [
       "'NorwayMFA Bahrain police previously died road accident killed explosion http'"
      ]
     },
     "metadata": {},
     "execution_count": 250
    }
   ],
   "source": [
    "train_data['cleaned_text'].iloc[100]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 251,
   "metadata": {
    "papermill": {
     "duration": 0.044042,
     "end_time": "2021-04-23T18:51:11.986684",
     "exception": false,
     "start_time": "2021-04-23T18:51:11.942642",
     "status": "completed"
    },
    "tags": []
   },
   "outputs": [
    {
     "output_type": "execute_result",
     "data": {
      "text/plain": [
       "1"
      ]
     },
     "metadata": {},
     "execution_count": 251
    }
   ],
   "source": [
    "train_data['target'].iloc[100]"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {
    "papermill": {
     "duration": 0.035719,
     "end_time": "2021-04-23T18:51:12.625794",
     "exception": false,
     "start_time": "2021-04-23T18:51:12.590075",
     "status": "completed"
    },
    "tags": []
   },
   "source": [
    "## Tf-idf features"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 252,
   "metadata": {},
   "outputs": [],
   "source": [
    "ngram_range=(1,1)\n",
    "max_features=100\n",
    "norm='l2'"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 253,
   "metadata": {
    "papermill": {
     "duration": 0.250149,
     "end_time": "2021-04-23T18:51:12.911770",
     "exception": false,
     "start_time": "2021-04-23T18:51:12.661621",
     "status": "completed"
    },
    "tags": []
   },
   "outputs": [],
   "source": [
    "tfidf_vectorizer = TfidfVectorizer(ngram_range=ngram_range, max_features=max_features, norm=norm)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 254,
   "metadata": {},
   "outputs": [
    {
     "output_type": "execute_result",
     "data": {
      "text/plain": [
       "<7613x100 sparse matrix of type '<class 'numpy.float64'>'\n",
       "\twith 15838 stored elements in Compressed Sparse Row format>"
      ]
     },
     "metadata": {},
     "execution_count": 254
    }
   ],
   "source": [
    "train_data_tfidf = tfidf_vectorizer.fit_transform(train_data['cleaned_text'])\n",
    "train_data_tfidf"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 255,
   "metadata": {
    "papermill": {
     "duration": 0.046372,
     "end_time": "2021-04-23T18:51:12.994876",
     "exception": false,
     "start_time": "2021-04-23T18:51:12.948504",
     "status": "completed"
    },
    "tags": []
   },
   "outputs": [
    {
     "output_type": "execute_result",
     "data": {
      "text/plain": [
       "['accident',\n",
       " 'amp',\n",
       " 'and',\n",
       " 'as',\n",
       " 'attack',\n",
       " 'back',\n",
       " 'best',\n",
       " 'body',\n",
       " 'bomb',\n",
       " 'building']"
      ]
     },
     "metadata": {},
     "execution_count": 255
    }
   ],
   "source": [
    "tfidf_vectorizer.get_feature_names()[:10]"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {
    "papermill": {
     "duration": 0.036486,
     "end_time": "2021-04-23T18:51:13.068304",
     "exception": false,
     "start_time": "2021-04-23T18:51:13.031818",
     "status": "completed"
    },
    "tags": []
   },
   "source": [
    "## Baseline model"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 256,
   "metadata": {},
   "outputs": [],
   "source": [
    "strat_k_fold = StratifiedKFold(n_splits=5, shuffle=True, random_state=42)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 257,
   "metadata": {},
   "outputs": [],
   "source": [
    "scoring_function_f1 = make_scorer(f1_score, pos_label=1, average='binary')"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {
    "papermill": {
     "duration": 0.036477,
     "end_time": "2021-04-23T18:51:13.141749",
     "exception": false,
     "start_time": "2021-04-23T18:51:13.105272",
     "status": "completed"
    },
    "tags": []
   },
   "source": [
    "## SVC cross validation."
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 258,
   "metadata": {},
   "outputs": [],
   "source": [
    "C = 1.0\n",
    "kernel='poly'\n",
    "max_iter=-1\n",
    "random_state=42"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 259,
   "metadata": {
    "papermill": {
     "duration": 0.045878,
     "end_time": "2021-04-23T18:51:13.224558",
     "exception": false,
     "start_time": "2021-04-23T18:51:13.178680",
     "status": "completed"
    },
    "tags": []
   },
   "outputs": [],
   "source": [
    "svc = SVC(C=C, kernel=kernel, max_iter=max_iter, random_state=random_state)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 260,
   "metadata": {
    "papermill": {
     "duration": 2.269566,
     "end_time": "2021-04-23T18:51:15.531951",
     "exception": false,
     "start_time": "2021-04-23T18:51:13.262385",
     "status": "completed"
    },
    "tags": []
   },
   "outputs": [
    {
     "output_type": "execute_result",
     "data": {
      "text/plain": [
       "{'fit_time': array([0.99043322, 0.99829006, 0.94024873, 0.97373009, 0.96771407]),\n",
       " 'score_time': array([0.13656974, 0.1343472 , 0.13345313, 0.13198996, 0.13271189]),\n",
       " 'test_score': array([0.60486891, 0.65035517, 0.5557656 , 0.5426945 , 0.63071895]),\n",
       " 'train_score': array([0.71281362, 0.76168757, 0.71334394, 0.7291713 , 0.75554698])}"
      ]
     },
     "metadata": {},
     "execution_count": 260
    }
   ],
   "source": [
    "cv_results = cross_validate(estimator=svc, X=train_data_tfidf, y=train_data['target'], scoring=scoring_function_f1, cv=strat_k_fold, n_jobs=-1, return_train_score=True)\n",
    "cv_results"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 261,
   "metadata": {},
   "outputs": [],
   "source": [
    "def mean_sd_cv_results(cv_results, metric='F1'):\n",
    "    print(f\"{metric} Train CV results: {cv_results['train_score'].mean().round(3)} +- {cv_results['train_score'].std().round(3)}\")\n",
    "    print(f\"{metric} Val CV results: {cv_results['test_score'].mean().round(3)} +- {cv_results['test_score'].std().round(3)}\")"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 262,
   "metadata": {},
   "outputs": [
    {
     "output_type": "stream",
     "name": "stdout",
     "text": [
      "F1 Train CV results: 0.735 +- 0.021\nF1 Val CV results: 0.597 +- 0.042\n"
     ]
    }
   ],
   "source": [
    "mean_sd_cv_results(cv_results)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 263,
   "metadata": {},
   "outputs": [
    {
     "output_type": "stream",
     "name": "stdout",
     "text": [
      "[NbConvertApp] Converting notebook mlflow-example-real-or-not-disaster-tweets-modeling-SVC.ipynb to html\n",
      "[NbConvertApp] Writing 610630 bytes to mlflow-example-real-or-not-disaster-tweets-modeling-SVC.html\n"
     ]
    }
   ],
   "source": [
    "! jupyter nbconvert --to html mlflow-example-real-or-not-disaster-tweets-modeling-SVC.ipynb"
   ]
  },
  {
   "source": [
    "## Mlflow logging"
   ],
   "cell_type": "markdown",
   "metadata": {}
  },
  {
   "cell_type": "code",
   "execution_count": 264,
   "metadata": {},
   "outputs": [],
   "source": [
    "server_uri = 'http://127.0.0.1:5000'\n",
    "mlflow.set_tracking_uri(server_uri)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 265,
   "metadata": {},
   "outputs": [],
   "source": [
    "mlflow.set_experiment('nlp_with_disaster_tweets')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 266,
   "metadata": {},
   "outputs": [],
   "source": [
    "# MLflow logging.\n",
    "with mlflow.start_run(run_name=run_name) as run:\n",
    "\n",
    "    # Logging tags\n",
    "    # run_name.\n",
    "    mlflow.set_tag(key='Run name', value=run_name)\n",
    "    # Goal.\n",
    "    mlflow.set_tag(key='Goal', value='Check model performance and decide whether we require further pre-processing/hyper-parameter tuning.')\n",
    "    # Modeling exp.\n",
    "    mlflow.set_tag(key='Modeling technique', value='SVC')\n",
    "\n",
    "    # Logging parameters\n",
    "    mlflow.log_param(key='ngram_range', value=ngram_range)\n",
    "    mlflow.log_param(key='max_features', value=max_features)\n",
    "    mlflow.log_param(key='norm', value=norm)\n",
    "    mlflow.log_param(key='C', value=C)\n",
    "    mlflow.log_param(key='kernel', value=kernel)\n",
    "    mlflow.log_param(key='max_iter', value=max_iter)\n",
    "    mlflow.log_param(key='random_state', value=random_state)\n",
    "\n",
    "    # Logging the SVC model.\n",
    "    mlflow.sklearn.log_model(sk_model=svc, artifact_path='svc_model')\n",
    "   \n",
    "    # Logging metrics.\n",
    "    # mean F1-score - train.\n",
    "    mlflow.log_metric(key='mean F1-score - train', value=cv_results['train_score'].mean().round(3))\n",
    "    # mean F1-score - val.\n",
    "    mlflow.log_metric(key='mean F1-score - val', value=cv_results['test_score'].mean().round(3))\n",
    "    # std F1-score - train.\n",
    "    mlflow.log_metric(key='std F1-score - train', value=cv_results['train_score'].std().round(3))\n",
    "    # std F1-score - val.\n",
    "    mlflow.log_metric(key='std F1-score - val', value=cv_results['test_score'].std().round(3))\n",
    "   \n",
    "    # Logging the notebook.\n",
    "    # Nb.\n",
    "    mlflow.log_artifact(local_path='real-or-not-disaster-tweets-modeling-SVC.ipynb', artifact_path='Notebook')\n",
    "    # Nb in HTML.\n",
    "    mlflow.log_artifact(local_path='real-or-not-disaster-tweets-modeling-SVC.html', artifact_path='Notebook')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  }
 ],
 "metadata": {
  "kernelspec": {
   "name": "python3",
   "display_name": "Python 3.9.1 64-bit ('nlp_disaster_tweets': conda)"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.9.1"
  },
  "metadata": {
   "interpreter": {
    "hash": "eeaa6f7f9d55900aa89bc3c04a1751ad3ee5030fffb5fb27314dbd7b21d279ee"
   }
  },
  "papermill": {
   "duration": 20.205122,
   "end_time": "2021-04-23T18:51:16.435914",
   "environment_variables": {},
   "exception": null,
   "input_path": "__notebook__.ipynb",
   "output_path": "__notebook__.ipynb",
   "parameters": {},
   "start_time": "2021-04-23T18:50:56.230792",
   "version": "2.1.0"
  },
  "interpreter": {
   "hash": "eeaa6f7f9d55900aa89bc3c04a1751ad3ee5030fffb5fb27314dbd7b21d279ee"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 4
}