kalapa.ipynb

{
 "cells": [
  {
   "cell_type": "code",
   "execution_count": 1,
   "metadata": {},
   "outputs": [],
   "source": [
    "import os, sys, re\n",
    "import numpy as np\n",
    "import pandas as pd\n",
    "import matplotlib\n",
    "import pickle\n",
    "import tarfile\n",
    "from datetime import datetime\n",
    "from subprocess import call, Popen\n",
    "from scipy import interp\n",
    "from sklearn.model_selection import StratifiedKFold\n",
    "from sklearn.ensemble import RandomForestClassifier\n",
    "from sklearn.metrics import auc\n",
    "from sklearn.metrics import roc_curve\n",
    "from sklearn.metrics import roc_auc_score\n",
    "from sklearn.metrics import plot_roc_curve\n",
    "import matplotlib.pyplot as plt\n",
    "from utils import *"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 2,
   "metadata": {},
   "outputs": [],
   "source": [
    "matplotlib.use(\"pdf\")"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 3,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Train:  (30000, 64)  | Test:  (20000, 63)\n",
      "------------------------------------------------------------\n"
     ]
    }
   ],
   "source": [
    "INPUT_DIR = 'data'\n",
    "\n",
    "train_df = pd.read_csv(os.path.join(INPUT_DIR, \"train.csv\"), dtype=str)\n",
    "test_df = pd.read_csv(os.path.join(INPUT_DIR, \"test.csv\"), dtype=str)\n",
    "\n",
    "print(\"Train: \", train_df.shape, \" | Test: \", test_df.shape)\n",
    "\n",
    "print(\"-\"*60)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# Pre-processing and Feature Engineering"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 4,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "\n",
      "@Data Pre-Processing\n",
      "\n",
      "+ Feature Engineering\n",
      "train_df.shape =  (30000, 66)  | test_df.shape =  (20000, 65)\n"
     ]
    }
   ],
   "source": [
    "print(\"\\n@Data Pre-Processing\")\n",
    "\n",
    "print(\"\\n+ Feature Engineering\")\n",
    "\n",
    "train_df = age_group(train_df)\n",
    "test_df = age_group(test_df)\n",
    "\n",
    "train_df = field7_count(train_df)\n",
    "test_df = field7_count(test_df)\n",
    "\n",
    "print(\"train_df.shape = \", train_df.shape, \" | test_df.shape = \", test_df.shape)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 5,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Binning: FIELD_1 bin_num_limit=8 stop_limit=0.05 count_distr_limit=0.05\n",
      "Binning: FIELD_10 bin_num_limit=8 stop_limit=0.05 count_distr_limit=0.05\n",
      "Binning: FIELD_11 bin_num_limit=8 stop_limit=0.05 count_distr_limit=0.05\n",
      "Binning: FIELD_12 bin_num_limit=8 stop_limit=0.05 count_distr_limit=0.05\n",
      "Binning: FIELD_13 bin_num_limit=8 stop_limit=0.05 count_distr_limit=0.05\n",
      "Binning: FIELD_14 bin_num_limit=8 stop_limit=0.05 count_distr_limit=0.05\n",
      "Binning: FIELD_15 bin_num_limit=8 stop_limit=0.05 count_distr_limit=0.05\n",
      "Binning: FIELD_16 bin_num_limit=8 stop_limit=0.05 count_distr_limit=0.05\n",
      "Binning: FIELD_17 bin_num_limit=8 stop_limit=0.05 count_distr_limit=0.05\n",
      "Binning: FIELD_18 bin_num_limit=8 stop_limit=0.05 count_distr_limit=0.05\n",
      "Binning: FIELD_19 bin_num_limit=8 stop_limit=0.05 count_distr_limit=0.05\n",
      "Binning: FIELD_2 bin_num_limit=8 stop_limit=0.05 count_distr_limit=0.05\n",
      "Binning: FIELD_20 bin_num_limit=8 stop_limit=0.05 count_distr_limit=0.05\n",
      "Binning: FIELD_21 bin_num_limit=8 stop_limit=0.05 count_distr_limit=0.05\n",
      "Binning: FIELD_22 bin_num_limit=8 stop_limit=0.05 count_distr_limit=0.05\n",
      "Binning: FIELD_23 bin_num_limit=8 stop_limit=0.05 count_distr_limit=0.05\n",
      "Binning: FIELD_24 bin_num_limit=8 stop_limit=0.05 count_distr_limit=0.05\n",
      "Binning: FIELD_25 bin_num_limit=8 stop_limit=0.05 count_distr_limit=0.05\n",
      "Binning: FIELD_26 bin_num_limit=8 stop_limit=0.05 count_distr_limit=0.05\n",
      "Binning: FIELD_27 bin_num_limit=8 stop_limit=0.05 count_distr_limit=0.05\n",
      "Binning: FIELD_28 bin_num_limit=8 stop_limit=0.05 count_distr_limit=0.05\n",
      "Binning: FIELD_29 bin_num_limit=8 stop_limit=0.05 count_distr_limit=0.05\n",
      "Binning: FIELD_3 bin_num_limit=8 stop_limit=0.05 count_distr_limit=0.05\n",
      "Binning: FIELD_30 bin_num_limit=8 stop_limit=0.05 count_distr_limit=0.05\n",
      "Binning: FIELD_31 bin_num_limit=8 stop_limit=0.05 count_distr_limit=0.05\n",
      "Binning: FIELD_33 bin_num_limit=8 stop_limit=0.05 count_distr_limit=0.05\n",
      "Binning: FIELD_34 bin_num_limit=8 stop_limit=0.05 count_distr_limit=0.05\n",
      "Binning: FIELD_35 bin_num_limit=8 stop_limit=0.05 count_distr_limit=0.05\n",
      "Binning: FIELD_36 bin_num_limit=8 stop_limit=0.05 count_distr_limit=0.05\n",
      "Binning: FIELD_37 bin_num_limit=8 stop_limit=0.05 count_distr_limit=0.05\n",
      "Binning: FIELD_38 bin_num_limit=8 stop_limit=0.05 count_distr_limit=0.05\n",
      "Binning: FIELD_39 bin_num_limit=8 stop_limit=0.05 count_distr_limit=0.05\n",
      "Binning: FIELD_4 bin_num_limit=8 stop_limit=0.05 count_distr_limit=0.05\n",
      "Binning: FIELD_40 bin_num_limit=8 stop_limit=0.05 count_distr_limit=0.05\n",
      "Binning: FIELD_41 bin_num_limit=8 stop_limit=0.05 count_distr_limit=0.05\n",
      "Binning: FIELD_42 bin_num_limit=8 stop_limit=0.05 count_distr_limit=0.05\n",
      "Binning: FIELD_43 bin_num_limit=8 stop_limit=0.05 count_distr_limit=0.05\n",
      "Binning: FIELD_44 bin_num_limit=8 stop_limit=0.05 count_distr_limit=0.05\n",
      "Binning: FIELD_45 bin_num_limit=8 stop_limit=0.05 count_distr_limit=0.05\n",
      "Binning: FIELD_46 bin_num_limit=8 stop_limit=0.05 count_distr_limit=0.05\n",
      "Binning: FIELD_47 bin_num_limit=8 stop_limit=0.05 count_distr_limit=0.05\n",
      "Binning: FIELD_48 bin_num_limit=8 stop_limit=0.05 count_distr_limit=0.05\n",
      "Binning: FIELD_49 bin_num_limit=8 stop_limit=0.05 count_distr_limit=0.05\n",
      "Binning: FIELD_5 bin_num_limit=8 stop_limit=0.05 count_distr_limit=0.05\n",
      "Binning: FIELD_50 bin_num_limit=8 stop_limit=0.05 count_distr_limit=0.05\n",
      "Binning: FIELD_51 bin_num_limit=8 stop_limit=0.05 count_distr_limit=0.05\n",
      "Binning: FIELD_52 bin_num_limit=8 stop_limit=0.05 count_distr_limit=0.05\n",
      "Binning: FIELD_53 bin_num_limit=8 stop_limit=0.05 count_distr_limit=0.05\n",
      "Binning: FIELD_54 bin_num_limit=8 stop_limit=0.05 count_distr_limit=0.05\n",
      "Binning: FIELD_55 bin_num_limit=8 stop_limit=0.05 count_distr_limit=0.05\n",
      "Binning: FIELD_56 bin_num_limit=8 stop_limit=0.05 count_distr_limit=0.05\n",
      "Binning: FIELD_57 bin_num_limit=8 stop_limit=0.05 count_distr_limit=0.05\n",
      "Binning: FIELD_6 bin_num_limit=8 stop_limit=0.05 count_distr_limit=0.05\n",
      "Binning: FIELD_7_COUNT bin_num_limit=8 stop_limit=0.05 count_distr_limit=0.05\n",
      "Binning: FIELD_8 bin_num_limit=8 stop_limit=0.05 count_distr_limit=0.05\n",
      "Binning: FIELD_9 bin_num_limit=8 stop_limit=0.05 count_distr_limit=0.05\n",
      "Binning: age_group bin_num_limit=8 stop_limit=0.05 count_distr_limit=0.05\n",
      "Binning: age_source1 bin_num_limit=8 stop_limit=0.05 count_distr_limit=0.05\n",
      "Binning: age_source2 bin_num_limit=8 stop_limit=0.05 count_distr_limit=0.05\n",
      "     After dropping:  (30000, 61) (20000, 60)\n",
      "----------------------------------------\n",
      "Done!\n",
      "------------------------------------------------------------\n"
     ]
    }
   ],
   "source": [
    "# WOE Binning\n",
    "bin_num_limit = 8\n",
    "stop_limit = 0.05\n",
    "count_distr_limit = 0.05\n",
    "\n",
    "woe_cols = [\n",
    "    'FIELD_1', 'FIELD_10', 'FIELD_11', 'FIELD_12', 'FIELD_13', 'FIELD_14',\n",
    "    'FIELD_15', 'FIELD_16', 'FIELD_17', 'FIELD_18', 'FIELD_19', 'FIELD_2',\n",
    "    'FIELD_20', 'FIELD_21', 'FIELD_22', 'FIELD_23', 'FIELD_24', 'FIELD_25',\n",
    "    'FIELD_26', 'FIELD_27', 'FIELD_28', 'FIELD_29', 'FIELD_3', 'FIELD_30',\n",
    "    'FIELD_31', 'FIELD_33', 'FIELD_34', 'FIELD_35', 'FIELD_36', 'FIELD_37', \n",
    "    'FIELD_38', 'FIELD_39', 'FIELD_4', 'FIELD_40', 'FIELD_41', 'FIELD_42', \n",
    "    'FIELD_43', 'FIELD_44', 'FIELD_45', 'FIELD_46', 'FIELD_47', 'FIELD_48', \n",
    "    'FIELD_49', 'FIELD_5', 'FIELD_50', 'FIELD_51', 'FIELD_52', 'FIELD_53', \n",
    "    'FIELD_54', 'FIELD_55', 'FIELD_56', 'FIELD_57', 'FIELD_6', 'FIELD_7_COUNT', \n",
    "    'FIELD_8', 'FIELD_9', 'age_group', 'age_source1', 'age_source2'\n",
    "]\n",
    "commands = []\n",
    "for col_name in woe_cols:\n",
    "    os.makedirs(os.path.join(INPUT_DIR, \"woe/%s\"%col_name), exist_ok=True)\n",
    "    df1 = train_df[[\"id\", \"label\", col_name]]\n",
    "    df2 = test_df[[\"id\", col_name]]\n",
    "    df1.to_csv(os.path.join(INPUT_DIR, \"woe/%s/train.csv\"%col_name), index=False, encoding=\"utf-8\")\n",
    "    df2.to_csv(os.path.join(INPUT_DIR, \"woe/%s/test.csv\"%col_name), index=False, encoding=\"utf-8\")\n",
    "\n",
    "    print(f\"Binning: {col_name}\", f\"bin_num_limit={bin_num_limit}\", f\"stop_limit={stop_limit}\", f\"count_distr_limit={count_distr_limit}\")\n",
    "    commands.append(\n",
    "        f\"Rscript woe.r {INPUT_DIR}/woe {col_name} {bin_num_limit} {stop_limit} {count_distr_limit}\"\n",
    "    )\n",
    "\n",
    "procs = [Popen(c.strip().split()) for c in commands]\n",
    "for p in procs:\n",
    "    p.wait()\n",
    "    \n",
    "train_woe_df = {}\n",
    "test_woe_df = {}\n",
    "for col_name in woe_cols:\n",
    "    df1 = pd.read_csv(os.path.join(INPUT_DIR, 'woe/%s/train_woe.csv'%col_name))\n",
    "    df2 = pd.read_csv(os.path.join(INPUT_DIR, 'woe/%s/test_woe.csv'%col_name))\n",
    "    for c in df1.columns:\n",
    "        train_woe_df[c] = df1[c]\n",
    "    for c in df2.columns:\n",
    "        test_woe_df[c] = df2[c]\n",
    "\n",
    "train_fe_df = pd.DataFrame.from_dict(train_woe_df)\n",
    "test_fe_df = pd.DataFrame.from_dict(test_woe_df)\n",
    "\n",
    "print(\" \"*4, \"After dropping: \", train_fe_df.shape, test_fe_df.shape)\n",
    "print(\"-\"*40)\n",
    "\n",
    "print(\"Done!\")\n",
    "print(\"-\"*60)\n",
    "\n",
    "train_fe_df.to_csv(os.path.join(INPUT_DIR, \"train_fe.csv\"), index=False, encoding=\"utf-8\")\n",
    "test_fe_df.to_csv(os.path.join(INPUT_DIR, \"test_fe.csv\"), index=False, encoding=\"utf-8\")"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 6,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "\n",
      "@Learning\n",
      "+ Data Splitting\n",
      "Stratified 5-fold, seed=2020\n",
      "FOLD 1\n",
      "FOLD 2\n",
      "FOLD 3\n",
      "FOLD 4\n",
      "FOLD 5\n",
      "--------------------------------------------------\n"
     ]
    }
   ],
   "source": [
    "# 4. Data Spliting\n",
    "print(\"\\n@Learning\")\n",
    "n_folds = 5\n",
    "seed = 2020\n",
    "    \n",
    "print(\"+ Data Splitting\")\n",
    "\n",
    "train_fe_df.label.replace(\"Good\", 0, inplace=True)\n",
    "train_fe_df.label.replace(\"Bad\", 1, inplace=True)\n",
    "print(f\"Stratified {n_folds}-fold, seed={seed}\")\n",
    "y = train_fe_df[\"label\"].values\n",
    "cv = StratifiedKFold(n_splits=n_folds, shuffle=True, random_state=seed)\n",
    "for i, (train, val) in enumerate(cv.split(np.zeros(len(y)), y)):\n",
    "    print(\"FOLD %d\" % (i + 1))\n",
    "    os.makedirs(os.path.join(INPUT_DIR, \"fold%d\" % i), exist_ok=True)\n",
    "    train_df, val_df = train_fe_df.loc[train], train_fe_df.loc[val]\n",
    "    # use all positive examples for training and evaluation\n",
    "    train_df = pd.concat([train_df, val_df[val_df.label == 1]])\n",
    "    val_df = pd.concat([val_df, train_df[train_df.label == 1]])\n",
    "    train_df.to_csv(os.path.join(INPUT_DIR, \"fold%d/train.csv\" % i), index=False)\n",
    "    val_df.to_csv(os.path.join(INPUT_DIR, \"fold%d/val.csv\" % i), index=False)\n",
    "\n",
    "print(\"-\"*50)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# Training"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 7,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "(20000, 67)\n",
      "FOLD 1\n",
      "(24097, 67) (6486, 67)\n",
      "val AUC = 0.9104\n",
      "FOLD 2\n",
      "(24097, 67) (6486, 67)\n",
      "val AUC = 0.9120\n",
      "FOLD 3\n",
      "(24097, 67) (6486, 67)\n",
      "val AUC = 0.9154\n",
      "FOLD 4\n",
      "(24097, 67) (6486, 67)\n",
      "val AUC = 0.9170\n",
      "FOLD 5\n",
      "(24098, 67) (6486, 67)\n",
      "val AUC = 0.9152\n",
      "Mean AUC = 0.9107, GINI 0.8213\n",
      "0.9104\t0.9120\t0.9154\t0.9170\t0.9152\n"
     ]
    }
   ],
   "source": [
    "seed = 2020\n",
    "n_trees = 767\n",
    "max_depth = 17\n",
    "min_samples_split = 2\n",
    "min_samples_leaf = 1 \n",
    "max_features = 'auto'\n",
    "class_weight = None\n",
    "bootstrap = True\n",
    "n_folds = 5\n",
    "\n",
    "embeddings = pd.read_pickle(\"./data/embeddings.pkl\").to_numpy(dtype=np.float32)\n",
    "\n",
    "# submission input\n",
    "X_submit = pd.read_csv(os.path.join(INPUT_DIR, \"test_fe.csv\"))\n",
    "submit_id = X_submit.id.to_numpy(int)\n",
    "submit_dict = {\"id\": submit_id}\n",
    "X_submit.drop(columns=[\"id\"], inplace=True)\n",
    "X_submit = X_submit.to_numpy(dtype=np.float32)\n",
    "X_submit = np.concatenate([X_submit, embeddings[submit_id]], axis=1)\n",
    "print(X_submit.shape)\n",
    "\n",
    "# training and evaluation\n",
    "tprs = []\n",
    "aucs = []\n",
    "mean_fpr = np.linspace(0, 1, 100)\n",
    "\n",
    "fold_aucs = []\n",
    "\n",
    "fig, ax = plt.subplots()\n",
    "for i in range(n_folds):\n",
    "    print(\"FOLD %d\" % (i + 1))\n",
    "\n",
    "    train_df = pd.read_csv(os.path.join(INPUT_DIR, \"fold%d/train.csv\" % i))\n",
    "    val_df = pd.read_csv(os.path.join(INPUT_DIR, \"fold%d/val.csv\" % i))\n",
    "    train_id = train_df.id.to_numpy(int)\n",
    "    val_id = val_df.id.to_numpy(int)\n",
    "    train_df.drop(columns=[\"id\"], inplace=True)\n",
    "    val_df.drop(columns=[\"id\"], inplace=True)\n",
    "    \n",
    "    y_train = train_df[\"label\"].to_numpy(dtype=np.float32)\n",
    "    X_train = train_df.drop(columns=[\"label\"]).to_numpy(dtype=np.float32)\n",
    "    X_train = np.concatenate([X_train, embeddings[train_id]], axis=1)\n",
    "    y_val = val_df[\"label\"].to_numpy(dtype=np.float32)\n",
    "    X_val = val_df.drop(columns=[\"label\"]).to_numpy(dtype=np.float32)\n",
    "    X_val = np.concatenate([X_val, embeddings[val_id]], axis=1)\n",
    "    print(X_train.shape, X_val.shape)\n",
    "\n",
    "  \n",
    "    clf = RandomForestClassifier(\n",
    "        n_estimators=n_trees,\n",
    "        max_depth=max_depth,\n",
    "        min_samples_split=min_samples_split,\n",
    "        min_samples_leaf=min_samples_leaf,\n",
    "        max_features=max_features,\n",
    "        random_state=seed,\n",
    "        class_weight=class_weight,\n",
    "        bootstrap=True,\n",
    "        n_jobs=1\n",
    "    )\n",
    "\n",
    "    clf.fit(X_train, y_train)\n",
    "\n",
    "    auc_ = roc_auc_score(y_val, clf.predict_proba(X_val)[:, 1])\n",
    "    fold_aucs.append(auc_)\n",
    "    print(f\"val AUC = {auc_:.4f}\")\n",
    "\n",
    "    y_submit = clf.predict_proba(X_submit)[:, 1]\n",
    "    submit_dict[\"fold%d\" % i] = y_submit\n",
    "\n",
    "    viz = plot_roc_curve(\n",
    "        clf, X_val, y_val, name=f\"ROC Fold {i}\", alpha=0.3, lw=1, ax=ax\n",
    "    )\n",
    "    interp_tpr = interp(mean_fpr, viz.fpr, viz.tpr)\n",
    "    interp_tpr[0] = 0.0\n",
    "    tprs.append(interp_tpr)\n",
    "    aucs.append(viz.roc_auc)\n",
    "\n",
    "ax.plot([0, 1], [0, 1], linestyle=\"--\", lw=2, color=\"r\", label=\"Random\", alpha=0.8)\n",
    "\n",
    "mean_tpr = np.mean(tprs, axis=0)\n",
    "mean_tpr[-1] = 1.0\n",
    "mean_auc = auc(mean_fpr, mean_tpr)\n",
    "std_auc = np.std(aucs)\n",
    "ax.plot(\n",
    "    mean_fpr,\n",
    "    mean_tpr,\n",
    "    color=\"b\",\n",
    "    label=r\"Mean ROC (AUC = %0.3f $\\pm$ %0.3f)\" % (mean_auc, std_auc),\n",
    "    lw=2,\n",
    "    alpha=0.8,\n",
    ")\n",
    "\n",
    "std_tpr = np.std(tprs, axis=0)\n",
    "tprs_upper = np.minimum(mean_tpr + std_tpr, 1)\n",
    "tprs_lower = np.maximum(mean_tpr - std_tpr, 0)\n",
    "ax.fill_between(\n",
    "    mean_fpr,\n",
    "    tprs_lower,\n",
    "    tprs_upper,\n",
    "    color=\"grey\",\n",
    "    alpha=0.2,\n",
    "    label=r\"$\\pm$ 1 std. dev.\",\n",
    ")\n",
    "\n",
    "ax.set(xlim=[-0.05, 1.05], ylim=[-0.05, 1.05], title=\"ROC Curves\")\n",
    "ax.legend(loc=\"lower right\")\n",
    "plt.savefig(\"roc.png\")\n",
    "\n",
    "print(\"Mean AUC = %0.4f, GINI %0.4f\" % (mean_auc, 2 * mean_auc - 1.0))\n",
    "\n",
    "print(\"\\t\".join(f\"{x:.4f}\" for x in fold_aucs))\n",
    "\n",
    "\n",
    "# averaging for submission\n",
    "res_df = pd.DataFrame(submit_dict)\n",
    "res_df[\"label\"] = res_df[[\"fold%d\" % i for i in range(n_folds)]].mean(axis=1)\n",
    "res_df[[\"id\", \"label\"]].to_csv(\"submission.csv\", index=False)\n",
    "\n",
    "for i in range(n_folds):\n",
    "    res_df[\"label\"] = res_df[f\"fold{i}\"]\n",
    "    res_df[[\"id\", \"label\"]].to_csv(\"submission_fold%d.csv\" % i, index=False)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# Post-processing"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "print(\"\\n@Rules\")\n",
    "\n",
    "sub_df = pd.read_csv(\"submission_fold1.csv\")\n",
    "\n",
    "# Smoothing\n",
    "y = sub_df.label.to_numpy()\n",
    "rank = np.argsort(y)\n",
    "y_smooth = np.arange(len(rank)) * (1.0 / (len(rank) - 1))\n",
    "y[rank] = y_smooth\n",
    "sub_df.label = y\n",
    "\n",
    "rule_df = pd.read_csv(\"./rules.csv\", dtype=str, encoding=\"utf-8\")\n",
    "test_df = pd.read_csv(os.path.join(INPUT_DIR, \"test.csv\"), dtype=str, encoding=\"utf-8\")\n",
    "\n",
    "mask = np.ones(sub_df.shape[0])\n",
    "\n",
    "for col in rule_df.columns:\n",
    "    patterns = set(str_normalize(v) for v in rule_df[col].unique())\n",
    "    patterns -= set(['nan'])\n",
    "    if len(patterns) == 0:\n",
    "        continue\n",
    "    \n",
    "    col_mask = test_df[col].apply(lambda x: 0. if str_normalize(x) in patterns else 1.)    \n",
    "    mask *= col_mask.to_numpy()\n",
    "    \n",
    "# Smoothing\n",
    "y = sub_df.label.to_numpy()\n",
    "org_idx = np.argwhere(mask).ravel()\n",
    "y_masked = y[org_idx]\n",
    "rank = np.argsort(y_masked)\n",
    "y_smooth = np.arange(1, len(rank) + 1) * (1.0 / (len(rank) + 1))\n",
    "y_masked[rank] = y_smooth\n",
    "y[org_idx] = y_masked\n",
    "\n",
    "sub_df.label = y * mask\n",
    "sub_df.to_csv(\"final_submission.csv\", index=False)"
   ]
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 3",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.7.4"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 4
}