08_ms_interpret.ipynb

{
 "cells": [
  {
   "cell_type": "code",
   "execution_count": 1,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "(132029, 105)\n",
      "(23561, 104)\n",
      "(132029, 105)\n",
      "(23561, 105)\n",
      "(155590, 105)\n",
      "X shape: (132029, 98)\n",
      "y shape: (132029,)\n",
      "test shape (23561, 98)\n",
      "98\n"
     ]
    }
   ],
   "source": [
    "import numpy as np\n",
    "import pandas as pd\n",
    "from pandas import DataFrame\n",
    "from pandas import Series\n",
    "import matplotlib\n",
    "import matplotlib.pyplot as plt\n",
    "import lightgbm as lgb\n",
    "import operator\n",
    "import time\n",
    "\n",
    "# %%\n",
    "\n",
    "# 1.读取文件\n",
    "train = pd.read_csv(\"new_data/train.csv\")\n",
    "train_target = pd.read_csv('new_data/train_target.csv')\n",
    "train = train.merge(train_target, on='id')\n",
    "test = pd.read_csv(\"new_data/test.csv\")\n",
    "print(train.shape)\n",
    "print(test.shape)\n",
    "# 2.合并数据\n",
    "test['target'] = -1\n",
    "data = pd.concat([train, test], sort=False, axis=0)\n",
    "print(train.shape)\n",
    "print(test.shape)\n",
    "print(data.shape)\n",
    "\n",
    "# %%\n",
    "\n",
    "# 简单数据描述\n",
    "stats = []\n",
    "for col in train.columns:\n",
    "    stats.append((col, train[col].nunique(), train[col].isnull().sum() * 100 / train.shape[0],\n",
    "                  train[col].value_counts(normalize=True, dropna=False).values[0] * 100, train[col].dtype))\n",
    "\n",
    "stats_df = pd.DataFrame(stats, columns=['Feature', 'Unique_values', 'Percentage of missing values',\n",
    "                                        'Percentage of values in the biggest category', 'type'])\n",
    "stats_df.sort_values('Unique_values', ascending=False)[:30]\n",
    "\n",
    "# %%\n",
    "\n",
    "stats = []\n",
    "for col in test.columns:\n",
    "    stats.append((col, test[col].nunique(), test[col].isnull().sum() * 100 / test.shape[0],\n",
    "                  test[col].value_counts(normalize=True, dropna=False).values[0] * 100, test[col].dtype))\n",
    "\n",
    "stats_df = pd.DataFrame(stats, columns=['Feature', 'Unique_values', 'Percentage of missing values',\n",
    "                                        'Percentage of values in the biggest category', 'type'])\n",
    "stats_df.sort_values('Unique_values', ascending=False)[:30]\n",
    "\n",
    "# %%\n",
    "\n",
    "# 特征工程\n",
    "# 根据 unique values确定\n",
    "\n",
    "no_feas = ['id', 'target'] + ['certId', 'bankCard', 'dist', 'residentAddr','certValidStop', 'certValidBegin']\n",
    "data['certPeriod'] = data['certValidStop'] - data['certValidBegin']\n",
    "numerical_features = ['certValidStop', 'certValidBegin', 'lmt', 'age', 'certPeriod']\n",
    "# data['certBalidStop_certValidBegin_ratio']=data ['certBalidStop']/data['certValidBegin']\n",
    "# data['lmt_age_ratio']=data ['lmt']/data['age']\n",
    "# data['certPeriod_age_ratio']=data ['certPeriod']/data['age']\n",
    "#\n",
    "# data['lmt_age_mul']=data ['lmt']*data['age']\n",
    "# data['certPeriod_age_mul']=data ['certPeriod']*data['age']\n",
    "\n",
    "categorical_features = [fea for fea in data.columns if fea not in numerical_features + no_feas]\n",
    "# cols = [col for col in (set(numerical_features))]\n",
    "# for col in cols:\n",
    "#     data[col + '_Rank'] = data[col].rank()\n",
    "\n",
    "# from tqdm import tqdm\n",
    "# for cate in tqdm(['certId', 'bankCard', 'dist', 'residentAddr']):\n",
    "#     for fea in numerical_features:\n",
    "#         grouped_df = data.groupby(cate).agg({fea: ['mean','skew',pd.DataFrame.kurt]})\n",
    "#         grouped_df.columns = [cate+'_' + '_'.join(col).strip() for col in grouped_df.columns.values]\n",
    "#         grouped_df = grouped_df.reset_index()\n",
    "#         data = pd.merge(data, grouped_df, on=cate, how='left')\n",
    "# %%\n",
    "\n",
    "features = [fea for fea in data.columns if fea not in no_feas]\n",
    "\n",
    "# %%\n",
    "\n",
    "train = data.loc[data['target'] != -1, :]  # train set\n",
    "test = data.loc[data['target'] == -1, :]  # test set\n",
    "y = train['target'].values.astype(int)\n",
    "X = train[features].values\n",
    "print(\"X shape:\", X.shape)\n",
    "print(\"y shape:\", y.shape)\n",
    "test_data = test[features].values\n",
    "print(\"test shape\", test_data.shape)\n",
    "\n",
    "print(len(features))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 4,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "start：********************************\n",
      "................Start predict .........................\n",
      "valid auc: 0.696286852127362\n",
      "................Start predict .........................\n",
      "valid auc: 0.7219555867093919\n",
      "................Start predict .........................\n",
      "valid auc: 0.6936038074438595\n",
      "................Start predict .........................\n",
      "valid auc: 0.7051756297525494\n",
      "................Start predict .........................\n",
      "valid auc: 0.7425175868216376\n",
      "the cv information:\n",
      "[0.696286852127362, 0.7219555867093919, 0.6936038074438595, 0.7051756297525494, 0.7425175868216376]\n",
      "cv mean score 0.7119078925709601\n",
      "......................run with time:  22.011788368225098\n",
      "over:*********************************\n",
      "mean auc: 0.7119078925709601\n",
      "总的结果： (5, 23561)\n",
      "result shape: (23561,)\n"
     ]
    }
   ],
   "source": [
    "import numpy as np\n",
    "import time\n",
    "import pandas as pd\n",
    "from interpret.glassbox import ExplainableBoostingClassifier\n",
    "from sklearn.model_selection import KFold\n",
    "from sklearn.metrics import r2_score,roc_auc_score\n",
    "from sklearn.model_selection import StratifiedKFold\n",
    "from sklearn.metrics import roc_auc_score\n",
    "\n",
    "print(\"start：********************************\")\n",
    "start = time.time()\n",
    "\n",
    "N = 5\n",
    "skf = StratifiedKFold(n_splits=N, shuffle=True, random_state=2019)\n",
    "\n",
    "auc_cv = []\n",
    "pred_cv = []\n",
    "for k, (train_in, test_in) in enumerate(skf.split(X, y)):\n",
    "    X_train, X_test, y_train, y_test = X[train_in], X[test_in], \\\n",
    "                                       y[train_in], y[test_in]\n",
    "\n",
    "\n",
    "    ebm = ExplainableBoostingClassifier(n_estimators=50,\n",
    "                                       random_state=2019)\n",
    "    ebm.fit(X_train, y_train)  # Works on dataframes and numpy arrays\n",
    "    \n",
    "    print('................Start predict .........................')\n",
    "    # 预测\n",
    "    y_pred = ebm.predict_proba(X_test)[:, 1]\n",
    "    # 评估\n",
    "    tmp_auc = roc_auc_score(y_test, y_pred)\n",
    "    auc_cv.append(tmp_auc)\n",
    "    print(\"valid auc:\", tmp_auc)\n",
    "    # test\n",
    "    pred = ebm.predict_proba(test_data)[:, 1]\n",
    "    pred_cv.append(pred)\n",
    "# K交叉验证的平均分数\n",
    "print('the cv information:')\n",
    "print(auc_cv)\n",
    "print('cv mean score', np.mean(auc_cv))\n",
    "\n",
    "end = time.time()\n",
    "print(\"......................run with time: \", (end - start) / 60.0)\n",
    "print(\"over:*********************************\")\n",
    "\n",
    "# 10.5折交叉验证结果均值融合，保存文件\n",
    "mean_auc = np.mean(auc_cv)\n",
    "print(\"mean auc:\", mean_auc)\n",
    "filepath = 'result/inter_' + str(mean_auc) + '.csv'  # 线下平均分数\n",
    "\n",
    "# 转为array\n",
    "res = np.array(pred_cv)\n",
    "print(\"总的结果：\", res.shape)\n",
    "# 最后结果平均，mean\n",
    "r = res.mean(axis=0)\n",
    "print('result shape:', r.shape)\n",
    "result = DataFrame()\n",
    "result['id'] = test['id']\n",
    "result['target'] = r\n",
    "result.to_csv(filepath, index=False, sep=\",\")"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 3",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.7.3"
  },
  "pycharm": {
   "stem_cell": {
    "cell_type": "raw",
    "metadata": {
     "collapsed": false
    },
    "source": []
   }
  }
 },
 "nbformat": 4,
 "nbformat_minor": 1
}