examples/ieee_fraud_simulated_concept_drift_card6.ipynb

{
 "cells": [
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "208fdfb6",
   "metadata": {},
   "outputs": [],
   "source": [
    "import pandas as pd\n",
    "import numpy as np\n",
    "from sklearn.model_selection import train_test_split\n",
    "from sklearn.metrics import log_loss, roc_auc_score, accuracy_score\n",
    "from xgboost import XGBClassifier\n",
    "\n",
    "import cinnamon\n",
    "from cinnamon.drift import ModelDriftExplainer, AdversarialDriftExplainer\n",
    "\n",
    "# pandas config\n",
    "pd.set_option('display.max_columns', 500)\n",
    "pd.set_option('display.max_rows', 500)\n",
    "\n",
    "seed = 2021"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "52e44e05",
   "metadata": {},
   "source": [
    "# IEEE fraud data"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "89e02e80",
   "metadata": {},
   "source": [
    "Download data with kaggle CLI if it is setup on your computer:"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "dd8d0059",
   "metadata": {},
   "outputs": [],
   "source": [
    "#!kaggle competitions download -c ieee-fraud-detection"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "45b35398",
   "metadata": {},
   "source": [
    "Else you can download the data here: https://www.kaggle.com/c/ieee-fraud-detection/data, and you will have to accept the competition rules"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "b491e5b8",
   "metadata": {},
   "outputs": [],
   "source": [
    "df = pd.read_csv('data/train_transaction.csv')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "aab1755d",
   "metadata": {},
   "outputs": [],
   "source": [
    "print(df.shape)"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "4cb0eed6",
   "metadata": {},
   "source": [
    "# Preprocessing"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "3f619582",
   "metadata": {},
   "outputs": [],
   "source": [
    "# count missing values per column\n",
    "missing_values = df.isnull().sum(axis=0)\n",
    "missing_values"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "d9cf9f5b",
   "metadata": {},
   "outputs": [],
   "source": [
    "# keep only columns with less than 10000 values\n",
    "selected_columns = [col for col in df.columns if missing_values[col] < 10000]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "39e947fe",
   "metadata": {},
   "outputs": [],
   "source": [
    "# in the resulting columns, drop rows with any missing value\n",
    "df = df[selected_columns].dropna(axis=0, how='any')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "bc354ce2",
   "metadata": {},
   "outputs": [],
   "source": [
    "# for the variable 'card6', keep only rows corresponding to 'debit' and 'credit'modalities\n",
    "df = df.loc[df['card6'].isin(['debit', 'credit']), :].copy()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "de334813",
   "metadata": {},
   "outputs": [],
   "source": [
    "df.head()"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "1bf70cfa",
   "metadata": {},
   "source": [
    "# Sampling\n",
    "\n",
    "We replicate a typical production situation where we would have:\n",
    "- training data\n",
    "- test data\n",
    "- production data\n",
    "\n",
    "Also, we introduce so data drift on the variable `card6` by keeping only transactions which correspond to credit card. In a real application, this would correspond to the case where we are not able to identify fraud (the target label) for debit card transactions.\n",
    "\n",
    "This data drift corresponds to a case of censoring. Generally it would correspond to concept drift."
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "d39c8a54",
   "metadata": {},
   "outputs": [],
   "source": [
    "# select features by keeping only numerical features\n",
    "features = [col for col in df.columns if col not in ['TransactionID', 'isFraud', 'TransactionDT',\n",
    "                                                     'ProductCD', 'card4', 'card6']]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "66601a4e",
   "metadata": {},
   "outputs": [],
   "source": [
    "# we do a time split (shuffle=False) to seperate between df_temp (training-test data)\n",
    "# and df_prod (production data)\n",
    "df_temp, df_prod = train_test_split(df.copy(), test_size=0.25, shuffle=False)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "9bcde369",
   "metadata": {},
   "outputs": [],
   "source": [
    "# the majority of transactions are made with debit cards\n",
    "df_temp['card6'].value_counts()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "8d97717e",
   "metadata": {},
   "outputs": [],
   "source": [
    "# drop all debit card transactions in train-test data\n",
    "# we do a time split (shuffle=False) to seperate between train data and test data\n",
    "X_train, X_test, y_train, y_test = train_test_split(df_temp.loc[df_temp['card6'].values == 'credit', features].copy(),\n",
    "                                                    df_temp.loc[df_temp['card6'].values == 'credit', 'isFraud'].values,\n",
    "                                                    test_size=1/3,\n",
    "                                                    shuffle=False,\n",
    "                                                    random_state=seed)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "30dd532d",
   "metadata": {},
   "outputs": [],
   "source": [
    "X_prod, y_prod = df_prod[features], df_prod['isFraud'].values"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "2a957aac",
   "metadata": {},
   "source": [
    "# Build model"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "79c9db1f",
   "metadata": {},
   "outputs": [],
   "source": [
    "clf = XGBClassifier(n_estimators=1000,\n",
    "                    booster=\"gbtree\",\n",
    "                    objective=\"binary:logistic\",\n",
    "                    learning_rate=0.1,\n",
    "                    max_depth=6,\n",
    "                    use_label_encoder=False,\n",
    "                    seed=seed)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "bffd4f89",
   "metadata": {},
   "outputs": [],
   "source": [
    "clf.fit(X=X_train, y=y_train, eval_set=[(X_test, y_test)], early_stopping_rounds=20,\n",
    "        verbose=10, eval_metric=['auc', 'logloss'])"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "07f36e6d",
   "metadata": {},
   "source": [
    "# Detection of data drift\n",
    "\n",
    "We do detect a data drift in this case. Our three indicators:\n",
    "\n",
    "- distribution of predictions\n",
    "- distribution of target labels\n",
    "- performance metrics\n",
    "\n",
    "show a data drift"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "772cdb4a",
   "metadata": {},
   "outputs": [],
   "source": [
    "drift_explainer = ModelDriftExplainer(clf)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "981f458a",
   "metadata": {},
   "outputs": [],
   "source": [
    "drift_explainer.fit(X1=X_test, X2=X_prod, y1=y_test, y2=y_prod)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "b68b066c",
   "metadata": {
    "scrolled": false
   },
   "outputs": [],
   "source": [
    "cinnamon.plot_prediction_drift(drift_explainer, figsize=(7, 5), bins=100)\n",
    "drift_explainer.get_prediction_drift()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "728b3734",
   "metadata": {},
   "outputs": [],
   "source": [
    "cinnamon.plot_target_drift(drift_explainer)\n",
    "drift_explainer.get_target_drift()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "85f1a942",
   "metadata": {},
   "outputs": [],
   "source": [
    "print(f'log_loss test: {log_loss(y_test, clf.predict_proba(X_test))}')\n",
    "print(f'log_loss prod: {log_loss(y_prod, clf.predict_proba(X_prod))}')\n",
    "\n",
    "print(f'AUC test: {roc_auc_score(y_test, clf.predict_proba(X_test)[:, 1])}')\n",
    "print(f'AUC prod: {roc_auc_score(y_prod, clf.predict_proba(X_prod)[:, 1])}')"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "ac562dc1",
   "metadata": {},
   "source": [
    "# Explain data drift"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "713ca0ca",
   "metadata": {},
   "outputs": [],
   "source": [
    "# plot drift importances in order to identify features that have the higher impacts on data drift\n",
    "cinnamon.plot_tree_based_drift_importances(drift_explainer, type='node_size')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "300bd069",
   "metadata": {},
   "outputs": [],
   "source": [
    "# first drift value feature : 'D1'\n",
    "cinnamon.plot_feature_drift(drift_explainer, 'D1', bins=100)\n",
    "drift_explainer.get_feature_drift('D1')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "0e01bd76",
   "metadata": {},
   "outputs": [],
   "source": [
    "cinnamon.plot_feature_drift(drift_explainer, 'C13', bins=100)\n",
    "drift_explainer.get_feature_drift('C13')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "1c7d2691",
   "metadata": {},
   "outputs": [],
   "source": [
    "cinnamon.plot_feature_drift(drift_explainer, 'C2', bins=100)\n",
    "drift_explainer.get_feature_drift('C2')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "3f165f2e",
   "metadata": {},
   "outputs": [],
   "source": [
    "cinnamon.plot_feature_drift(drift_explainer, 'TransactionAmt', bins=100)\n",
    "drift_explainer.get_feature_drift('TransactionAmt')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "31b60510",
   "metadata": {},
   "outputs": [],
   "source": [
    "# feature importance of the model\n",
    "pd.DataFrame(clf.feature_importances_, X_train.columns).sort_values(0, ascending=False)"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "0dc33a09",
   "metadata": {},
   "source": [
    "# Correction of data drift"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "f8d69a54",
   "metadata": {},
   "source": [
    "## Correction on test dataset\n",
    "\n",
    "We apply our methodology which uses adversarial learning to correct data drift between test and prod data.\n",
    "\n",
    "We then check our three indicators of data drift in order to see if we get improvement."
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "40bfdda6",
   "metadata": {
    "scrolled": true
   },
   "outputs": [],
   "source": [
    "# weights computed with the adversarial method\n",
    "# feature_subset=['D1', 'C13', 'C2', 'TransactionAmt']: only the first fourth features in terms of\n",
    "# drift value are selected\n",
    "sample_weights_test_adversarial = (AdversarialDriftExplainer(feature_subset=['D1', 'C13'], #, 'C2', 'TransactionAmt'\n",
    "                                                              seed=2021)\n",
    "                                    .fit(X_test, X_prod)\n",
    "                                    .get_adversarial_correction_weights(max_ratio=10))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "1a23a22f",
   "metadata": {},
   "outputs": [],
   "source": [
    "drift_explainer2 = ModelDriftExplainer(clf).fit(X1=X_test, X2=X_prod, y1=y_test, y2=y_prod,\n",
    "                                                sample_weights1=sample_weights_test_adversarial)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "a24f94ac",
   "metadata": {},
   "outputs": [],
   "source": [
    "# the drift on distribution of predictions is lowered thaks to our technique \n",
    "cinnamon.plot_prediction_drift(drift_explainer2, bins=100)\n",
    "drift_explainer2.get_prediction_drift()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "f04e7fb7",
   "metadata": {},
   "outputs": [],
   "source": [
    "# the target algo re-equilibrated in the good direction\n",
    "cinnamon.plot_target_drift(drift_explainer2)\n",
    "drift_explainer2.get_target_drift()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "fc7904cc",
   "metadata": {},
   "outputs": [],
   "source": [
    "# valid loss is closer to prod loss, but there is still a difference\n",
    "print(f'log_loss valid: {log_loss(y_test, clf.predict_proba(X_test), sample_weight=sample_weights_test_adversarial)}')\n",
    "print(f'log_loss prod: {log_loss(y_prod, clf.predict_proba(X_prod))}')\n",
    "\n",
    "print(f'AUC valid: {roc_auc_score(y_test, clf.predict_proba(X_test)[:, 1], sample_weight=sample_weights_test_adversarial)}')\n",
    "print(f'AUC prod: {roc_auc_score(y_prod, clf.predict_proba(X_prod)[:, 1])}')"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "dbdb694b",
   "metadata": {},
   "source": [
    "## Correction on test dataset and train dataset (in order to retrain the model)\n",
    "\n",
    "We apply the same adversarial strategy on training data.\n",
    "\n",
    "With the model retrain on re-weighted samples, new weights, we observe there is no obvious improvement in model performance on production data. This needs to be further investigated."
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "12adb363",
   "metadata": {
    "scrolled": true
   },
   "outputs": [],
   "source": [
    "# weights computed with the adversarial method on training data\n",
    "sample_weights_train_adversarial = (AdversarialDriftExplainer(feature_subset=['D1', 'C13', ], #'C2', 'TransactionAmt'\n",
    "                                                              seed=2021)\n",
    "                                    .fit(X_train, X_prod)\n",
    "                                    .get_adversarial_correction_weights(max_ratio=10))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "6a028d43",
   "metadata": {},
   "outputs": [],
   "source": [
    "clf2 = XGBClassifier(n_estimators=1000,\n",
    "                    booster=\"gbtree\",\n",
    "                    objective=\"binary:logistic\",\n",
    "                    learning_rate=0.1,\n",
    "                    max_depth=6,\n",
    "                    use_label_encoder=False,\n",
    "                    seed=seed)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "5f51d194",
   "metadata": {},
   "outputs": [],
   "source": [
    "# train a new classifier with the reweighted samples\n",
    "# we use a power factor 0.3 on sample_weights_train_adversarial weights to smooth them\n",
    "clf2.fit(X=X_train, y=y_train, eval_set=[(X_test, y_test)], sample_weight=sample_weights_train_adversarial**0.3,\n",
    "         early_stopping_rounds=20, verbose=10, eval_metric=['auc', 'logloss'],\n",
    "         sample_weight_eval_set=[sample_weights_test_adversarial])"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "60445e1e",
   "metadata": {},
   "outputs": [],
   "source": [
    "# with the reweighting, we see a small improvement for performance on production data, but is it significative ?\n",
    "print(f'log_loss test: {log_loss(y_test, clf2.predict_proba(X_test), sample_weight=sample_weights_test_adversarial)}')\n",
    "print(f'log_loss prod: {log_loss(y_prod, clf2.predict_proba(X_prod))}')\n",
    "\n",
    "print(f'AUC test: {roc_auc_score(y_test, clf2.predict_proba(X_test)[:, 1], sample_weight=sample_weights_test_adversarial)}')\n",
    "print(f'AUC prod: {roc_auc_score(y_prod, clf2.predict_proba(X_prod)[:, 1])}')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "a5a2f50e",
   "metadata": {},
   "outputs": [],
   "source": []
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 3 (ipykernel)",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.9.12"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 5
}