model/GBDT_LR.ipynb

{
 "cells": [
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# LR、GBDT、GBDT+LR\n",
    "\n",
    "训练三个模型对数据进行预测， 分别是LR模型、GBDT模型和两者的组合模型（GBDT负责对各个特征进行交叉， 把原始特征向量转换为新的离散型特征向量， 然后再使用逻辑回归模型）， 然后分别观察它们的预测效果。\n",
    "\n",
    "对于不同的模型， 特征会有不同的处理方式，具体如下：\n",
    "- 逻辑回归模型：连续特征归一化， 离散特征one-hot编码\n",
    "- GBDT模型：离散特征one-hot编码（树模型连续特征不需要归一化）\n",
    "- GBDT+LR模型：离散特征one-hot编码（由于LR使用的特征是GBDT的输出， 原数据依然是GBDT进行处理， 所以只需要离散特征one-hot编码）\n",
    "\n",
    "**任务：**  \n",
    "开发预测广告点击率(CTR)的模型。即给定一个用户和他正在访问的页面，预测他点击给定广告的概率。\n",
    "\n",
    "**数据集：**\n",
    "- Label：目标变量，0表示未点击， 1表示点击\n",
    "- l1-l13: 13列的数值特征，大部分是计数特征\n",
    "- C1-C26: 26列的分类特征，这些特征的值离散成了32位的数据"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 1,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "   Label   I1   I2   I3   I4      I5     I6     I7    I8    I9  ...       C17  \\\n",
      "0      1  1.0    0  1.0 -1.0   227.0    1.0  173.0  18.0  50.0  ...  3486227d   \n",
      "1      1  4.0    1  1.0  2.0    27.0    2.0    4.0   2.0   2.0  ...  07c540c4   \n",
      "2      1  0.0  806 -1.0 -1.0  1752.0  142.0    2.0   0.0  50.0  ...  07c540c4   \n",
      "\n",
      "        C18       C19       C20       C21 C22       C23       C24       C25  \\\n",
      "0  e88ffc9d  c393dc22  b1252a9d  57c90cd9  -1  bcdee96c  4d19a3eb  cb079c2d   \n",
      "1  92555263        -1        -1  242bb710  -1  3a171ecb  72c78f11        -1   \n",
      "2  25c88e42  21ddcdc9  b1252a9d  a0136dd2  -1  32c7478e  8fc66e78  001f3601   \n",
      "\n",
      "        C26  \n",
      "0  456c12a0  \n",
      "1        -1  \n",
      "2  f37f3967  \n",
      "\n",
      "[3 rows x 40 columns]\n"
     ]
    }
   ],
   "source": [
    "\"\"\"导包\"\"\"\n",
    "\n",
    "import numpy as np\n",
    "import pandas as pd\n",
    "\n",
    "from sklearn.linear_model import LogisticRegression\n",
    "from sklearn.model_selection import train_test_split\n",
    "import lightgbm as lgb\n",
    "from sklearn.preprocessing import MinMaxScaler, OneHotEncoder, LabelEncoder\n",
    "from sklearn.metrics import log_loss\n",
    "import gc\n",
    "from scipy import sparse\n",
    "import warnings\n",
    "warnings.filterwarnings('ignore')\n",
    "\n",
    "\"\"\"数据读取\"\"\"\n",
    "\n",
    "path = '../data/'\n",
    "df_train = pd.read_csv(path + 'train.csv')\n",
    "df_test = pd.read_csv(path + 'test.csv')\n",
    "\n",
    "\"\"\"数据预处理\"\"\"\n",
    "\n",
    "# 去掉 Id 列\n",
    "df_train.drop(['Id'], axis=1, inplace=True)\n",
    "df_test.drop(['Id'], axis=1, inplace=True)\n",
    "\n",
    "# 测试集新增 Label 列\n",
    "df_test['Label'] = -1\n",
    "\n",
    "# 合并测试集和训练集\n",
    "data = pd.concat([df_train, df_test])\n",
    "\n",
    "# 填充缺失值\n",
    "data.fillna(-1, inplace=True)\n",
    "\n",
    "\"\"\"划分连续和离散特征\"\"\"\n",
    "\n",
    "continuous_fea = ['I'+str(i+1) for i in range(13)]\n",
    "category_fea = ['C'+str(i+1) for i in range(26)]\n",
    "\n",
    "print(data[: 3])"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 5,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "tr_logloss:  0.12423395164787936\n",
      "val_logloss:  0.44407245698774495\n",
      "predict:  [0.44783059 0.80628705 0.1756691  0.02070154 0.13984202 0.46490042\n",
      " 0.43386417 0.07089967 0.07121148 0.27896238]\n"
     ]
    }
   ],
   "source": [
    "\"\"\"逻辑回归建模\"\"\"\n",
    "def lr_model(data, category_fea, continuous_fea):\n",
    "    \"\"\"\n",
    "        逻辑回归建模\n",
    "        \n",
    "        :param data: df, 数据表\n",
    "        :param category_fea: list, 离散特征列\n",
    "        :param continuous_fea: list, 连续特征列\n",
    "    \"\"\"\n",
    "    \n",
    "    # 连续特征归一化\n",
    "    scaler = MinMaxScaler()\n",
    "    for col in continuous_fea:\n",
    "        data[col] = scaler.fit_transform(data[col].values.reshape(-1, 1))\n",
    "    \n",
    "    # 离散特征one-hot编码\n",
    "    for col in category_fea:\n",
    "        onehot_feats = pd.get_dummies(data[col], prefix=col)\n",
    "        data.drop([col], axis=1, inplace=True)\n",
    "        data = pd.concat([data, onehot_feats], axis=1)\n",
    "    \n",
    "    # 拆分训练集和测试集\n",
    "    train = data[data['Label'] != -1]\n",
    "    target = train.pop('Label')\n",
    "    test = data[data['Label'] == -1]\n",
    "    test.drop(['Label'], axis=1, inplace=True)\n",
    "    \n",
    "    # 划分训练集\n",
    "    x_train, x_val, y_train, y_val = train_test_split(train, target, test_size=0.2, random_state=2020)\n",
    "\n",
    "    # 建立模型\n",
    "    lr = LogisticRegression()\n",
    "    lr.fit(x_train, y_train)\n",
    "    tr_logloss = log_loss(y_train, lr.predict_proba(x_train)[:, 1])  # −(ylog(p)+(1−y)log(1−p)) log_loss\n",
    "    val_logloss = log_loss(y_val, lr.predict_proba(x_val)[:, 1])\n",
    "    print('tr_logloss: ', tr_logloss)\n",
    "    print('val_logloss: ', val_logloss)\n",
    "    \n",
    "    # 模型预测\n",
    "    y_pred = lr.predict_proba(test)[:, 1]  # predict_proba 返回n行k列的矩阵，第i行第j列上的数值是模型预测第i个预测样本为某个标签的概率，这里的1表示点击的概率\n",
    "    print('predict: ', y_pred[:10]) # 这里看前10个，预测为点击的概率\n",
    "    \n",
    "\"\"\"训练和预测逻辑回归模型\"\"\"\n",
    "lr_model(data.copy(), category_fea, continuous_fea)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 60,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "[1]\ttrain's binary_logloss: 0.523857\tval's binary_logloss: 0.457806\n",
      "Training until validation scores don't improve for 100 rounds\n",
      "[2]\ttrain's binary_logloss: 0.521371\tval's binary_logloss: 0.457213\n",
      "[3]\ttrain's binary_logloss: 0.519084\tval's binary_logloss: 0.456616\n",
      "[4]\ttrain's binary_logloss: 0.516882\tval's binary_logloss: 0.456046\n",
      "[5]\ttrain's binary_logloss: 0.514449\tval's binary_logloss: 0.455649\n",
      "[6]\ttrain's binary_logloss: 0.512277\tval's binary_logloss: 0.455319\n",
      "[7]\ttrain's binary_logloss: 0.509973\tval's binary_logloss: 0.455039\n",
      "[8]\ttrain's binary_logloss: 0.507717\tval's binary_logloss: 0.454523\n",
      "[9]\ttrain's binary_logloss: 0.505668\tval's binary_logloss: 0.454546\n",
      "[10]\ttrain's binary_logloss: 0.503491\tval's binary_logloss: 0.454134\n",
      "[11]\ttrain's binary_logloss: 0.501469\tval's binary_logloss: 0.453151\n",
      "[12]\ttrain's binary_logloss: 0.499463\tval's binary_logloss: 0.452609\n",
      "[13]\ttrain's binary_logloss: 0.497257\tval's binary_logloss: 0.452419\n",
      "[14]\ttrain's binary_logloss: 0.495206\tval's binary_logloss: 0.451944\n",
      "[15]\ttrain's binary_logloss: 0.493127\tval's binary_logloss: 0.451356\n",
      "[16]\ttrain's binary_logloss: 0.491215\tval's binary_logloss: 0.451181\n",
      "[17]\ttrain's binary_logloss: 0.489258\tval's binary_logloss: 0.450615\n",
      "[18]\ttrain's binary_logloss: 0.487424\tval's binary_logloss: 0.450754\n",
      "[19]\ttrain's binary_logloss: 0.485589\tval's binary_logloss: 0.450428\n",
      "[20]\ttrain's binary_logloss: 0.483753\tval's binary_logloss: 0.450043\n",
      "[21]\ttrain's binary_logloss: 0.481871\tval's binary_logloss: 0.449924\n",
      "[22]\ttrain's binary_logloss: 0.480233\tval's binary_logloss: 0.449518\n",
      "[23]\ttrain's binary_logloss: 0.478629\tval's binary_logloss: 0.449086\n",
      "[24]\ttrain's binary_logloss: 0.476933\tval's binary_logloss: 0.448878\n",
      "[25]\ttrain's binary_logloss: 0.47504\tval's binary_logloss: 0.448413\n",
      "[26]\ttrain's binary_logloss: 0.473416\tval's binary_logloss: 0.44809\n",
      "[27]\ttrain's binary_logloss: 0.471608\tval's binary_logloss: 0.448072\n",
      "[28]\ttrain's binary_logloss: 0.469969\tval's binary_logloss: 0.447721\n",
      "[29]\ttrain's binary_logloss: 0.46833\tval's binary_logloss: 0.447358\n",
      "[30]\ttrain's binary_logloss: 0.466517\tval's binary_logloss: 0.447445\n",
      "[31]\ttrain's binary_logloss: 0.464722\tval's binary_logloss: 0.447317\n",
      "[32]\ttrain's binary_logloss: 0.463079\tval's binary_logloss: 0.447152\n",
      "[33]\ttrain's binary_logloss: 0.461319\tval's binary_logloss: 0.446674\n",
      "[34]\ttrain's binary_logloss: 0.459722\tval's binary_logloss: 0.446455\n",
      "[35]\ttrain's binary_logloss: 0.457986\tval's binary_logloss: 0.446345\n",
      "[36]\ttrain's binary_logloss: 0.456293\tval's binary_logloss: 0.446229\n",
      "[37]\ttrain's binary_logloss: 0.454638\tval's binary_logloss: 0.446135\n",
      "[38]\ttrain's binary_logloss: 0.452929\tval's binary_logloss: 0.445476\n",
      "[39]\ttrain's binary_logloss: 0.451349\tval's binary_logloss: 0.445208\n",
      "[40]\ttrain's binary_logloss: 0.449759\tval's binary_logloss: 0.444538\n",
      "[41]\ttrain's binary_logloss: 0.448181\tval's binary_logloss: 0.444151\n",
      "[42]\ttrain's binary_logloss: 0.446656\tval's binary_logloss: 0.443786\n",
      "[43]\ttrain's binary_logloss: 0.445243\tval's binary_logloss: 0.443472\n",
      "[44]\ttrain's binary_logloss: 0.443697\tval's binary_logloss: 0.443249\n",
      "[45]\ttrain's binary_logloss: 0.442193\tval's binary_logloss: 0.443062\n",
      "[46]\ttrain's binary_logloss: 0.440697\tval's binary_logloss: 0.442784\n",
      "[47]\ttrain's binary_logloss: 0.439353\tval's binary_logloss: 0.442828\n",
      "[48]\ttrain's binary_logloss: 0.437963\tval's binary_logloss: 0.442524\n",
      "[49]\ttrain's binary_logloss: 0.436672\tval's binary_logloss: 0.442423\n",
      "[50]\ttrain's binary_logloss: 0.435301\tval's binary_logloss: 0.442193\n",
      "[51]\ttrain's binary_logloss: 0.433889\tval's binary_logloss: 0.441743\n",
      "[52]\ttrain's binary_logloss: 0.432548\tval's binary_logloss: 0.441336\n",
      "[53]\ttrain's binary_logloss: 0.431017\tval's binary_logloss: 0.441085\n",
      "[54]\ttrain's binary_logloss: 0.429677\tval's binary_logloss: 0.440932\n",
      "[55]\ttrain's binary_logloss: 0.428271\tval's binary_logloss: 0.440676\n",
      "[56]\ttrain's binary_logloss: 0.42698\tval's binary_logloss: 0.44047\n",
      "[57]\ttrain's binary_logloss: 0.425539\tval's binary_logloss: 0.440397\n",
      "[58]\ttrain's binary_logloss: 0.424123\tval's binary_logloss: 0.440136\n",
      "[59]\ttrain's binary_logloss: 0.422809\tval's binary_logloss: 0.439885\n",
      "[60]\ttrain's binary_logloss: 0.42146\tval's binary_logloss: 0.439716\n",
      "[61]\ttrain's binary_logloss: 0.420068\tval's binary_logloss: 0.439454\n",
      "[62]\ttrain's binary_logloss: 0.418745\tval's binary_logloss: 0.439493\n",
      "[63]\ttrain's binary_logloss: 0.41755\tval's binary_logloss: 0.439334\n",
      "[64]\ttrain's binary_logloss: 0.416342\tval's binary_logloss: 0.439451\n",
      "[65]\ttrain's binary_logloss: 0.415058\tval's binary_logloss: 0.439173\n",
      "[66]\ttrain's binary_logloss: 0.413658\tval's binary_logloss: 0.43881\n",
      "[67]\ttrain's binary_logloss: 0.4123\tval's binary_logloss: 0.438656\n",
      "[68]\ttrain's binary_logloss: 0.410981\tval's binary_logloss: 0.438352\n",
      "[69]\ttrain's binary_logloss: 0.409717\tval's binary_logloss: 0.438017\n",
      "[70]\ttrain's binary_logloss: 0.408442\tval's binary_logloss: 0.437799\n",
      "[71]\ttrain's binary_logloss: 0.407242\tval's binary_logloss: 0.437543\n",
      "[72]\ttrain's binary_logloss: 0.406051\tval's binary_logloss: 0.437359\n",
      "[73]\ttrain's binary_logloss: 0.404727\tval's binary_logloss: 0.437274\n",
      "[74]\ttrain's binary_logloss: 0.403419\tval's binary_logloss: 0.437135\n",
      "[75]\ttrain's binary_logloss: 0.402132\tval's binary_logloss: 0.437162\n",
      "[76]\ttrain's binary_logloss: 0.401026\tval's binary_logloss: 0.437043\n",
      "[77]\ttrain's binary_logloss: 0.399681\tval's binary_logloss: 0.436681\n",
      "[78]\ttrain's binary_logloss: 0.398398\tval's binary_logloss: 0.436374\n",
      "[79]\ttrain's binary_logloss: 0.397186\tval's binary_logloss: 0.436306\n",
      "[80]\ttrain's binary_logloss: 0.395998\tval's binary_logloss: 0.436238\n",
      "[81]\ttrain's binary_logloss: 0.394678\tval's binary_logloss: 0.436146\n",
      "[82]\ttrain's binary_logloss: 0.393508\tval's binary_logloss: 0.436026\n",
      "[83]\ttrain's binary_logloss: 0.392385\tval's binary_logloss: 0.435897\n",
      "[84]\ttrain's binary_logloss: 0.391328\tval's binary_logloss: 0.435948\n",
      "[85]\ttrain's binary_logloss: 0.390191\tval's binary_logloss: 0.435803\n",
      "[86]\ttrain's binary_logloss: 0.389034\tval's binary_logloss: 0.435726\n",
      "[87]\ttrain's binary_logloss: 0.388039\tval's binary_logloss: 0.435854\n",
      "[88]\ttrain's binary_logloss: 0.3868\tval's binary_logloss: 0.435683\n",
      "[89]\ttrain's binary_logloss: 0.38566\tval's binary_logloss: 0.435685\n",
      "[90]\ttrain's binary_logloss: 0.384527\tval's binary_logloss: 0.435758\n",
      "[91]\ttrain's binary_logloss: 0.383388\tval's binary_logloss: 0.435756\n",
      "[92]\ttrain's binary_logloss: 0.38231\tval's binary_logloss: 0.435815\n",
      "[93]\ttrain's binary_logloss: 0.381094\tval's binary_logloss: 0.435682\n",
      "[94]\ttrain's binary_logloss: 0.379807\tval's binary_logloss: 0.435561\n",
      "[95]\ttrain's binary_logloss: 0.37872\tval's binary_logloss: 0.435383\n",
      "[96]\ttrain's binary_logloss: 0.377682\tval's binary_logloss: 0.435011\n",
      "[97]\ttrain's binary_logloss: 0.376613\tval's binary_logloss: 0.434783\n",
      "[98]\ttrain's binary_logloss: 0.375463\tval's binary_logloss: 0.434865\n",
      "[99]\ttrain's binary_logloss: 0.374318\tval's binary_logloss: 0.434926\n",
      "[100]\ttrain's binary_logloss: 0.373331\tval's binary_logloss: 0.435077\n",
      "[101]\ttrain's binary_logloss: 0.372209\tval's binary_logloss: 0.43519\n",
      "[102]\ttrain's binary_logloss: 0.371232\tval's binary_logloss: 0.434997\n",
      "[103]\ttrain's binary_logloss: 0.370158\tval's binary_logloss: 0.43491\n",
      "[104]\ttrain's binary_logloss: 0.369108\tval's binary_logloss: 0.434854\n",
      "[105]\ttrain's binary_logloss: 0.368046\tval's binary_logloss: 0.435009\n",
      "[106]\ttrain's binary_logloss: 0.366972\tval's binary_logloss: 0.434982\n",
      "[107]\ttrain's binary_logloss: 0.365952\tval's binary_logloss: 0.435174\n",
      "[108]\ttrain's binary_logloss: 0.365084\tval's binary_logloss: 0.435282\n",
      "[109]\ttrain's binary_logloss: 0.364023\tval's binary_logloss: 0.435066\n",
      "[110]\ttrain's binary_logloss: 0.363132\tval's binary_logloss: 0.43526\n",
      "[111]\ttrain's binary_logloss: 0.362204\tval's binary_logloss: 0.435537\n",
      "[112]\ttrain's binary_logloss: 0.361217\tval's binary_logloss: 0.435766\n",
      "[113]\ttrain's binary_logloss: 0.360214\tval's binary_logloss: 0.43583\n",
      "[114]\ttrain's binary_logloss: 0.35936\tval's binary_logloss: 0.435792\n",
      "[115]\ttrain's binary_logloss: 0.358383\tval's binary_logloss: 0.435522\n",
      "[116]\ttrain's binary_logloss: 0.357288\tval's binary_logloss: 0.435637\n",
      "[117]\ttrain's binary_logloss: 0.356216\tval's binary_logloss: 0.435376\n",
      "[118]\ttrain's binary_logloss: 0.355096\tval's binary_logloss: 0.435324\n",
      "[119]\ttrain's binary_logloss: 0.354101\tval's binary_logloss: 0.43536\n",
      "[120]\ttrain's binary_logloss: 0.353266\tval's binary_logloss: 0.435261\n",
      "[121]\ttrain's binary_logloss: 0.352268\tval's binary_logloss: 0.435374\n",
      "[122]\ttrain's binary_logloss: 0.351148\tval's binary_logloss: 0.435321\n",
      "[123]\ttrain's binary_logloss: 0.350471\tval's binary_logloss: 0.435319\n",
      "[124]\ttrain's binary_logloss: 0.349503\tval's binary_logloss: 0.435183\n",
      "[125]\ttrain's binary_logloss: 0.348856\tval's binary_logloss: 0.435206\n",
      "[126]\ttrain's binary_logloss: 0.348053\tval's binary_logloss: 0.435206\n",
      "[127]\ttrain's binary_logloss: 0.347443\tval's binary_logloss: 0.434997\n",
      "[128]\ttrain's binary_logloss: 0.346512\tval's binary_logloss: 0.434941\n",
      "[129]\ttrain's binary_logloss: 0.345885\tval's binary_logloss: 0.434922\n",
      "[130]\ttrain's binary_logloss: 0.345112\tval's binary_logloss: 0.43477\n",
      "[131]\ttrain's binary_logloss: 0.344716\tval's binary_logloss: 0.434628\n",
      "[132]\ttrain's binary_logloss: 0.343697\tval's binary_logloss: 0.434359\n",
      "[133]\ttrain's binary_logloss: 0.342596\tval's binary_logloss: 0.434238\n",
      "[134]\ttrain's binary_logloss: 0.341666\tval's binary_logloss: 0.434321\n",
      "[135]\ttrain's binary_logloss: 0.340742\tval's binary_logloss: 0.434473\n",
      "[136]\ttrain's binary_logloss: 0.340167\tval's binary_logloss: 0.43421\n",
      "[137]\ttrain's binary_logloss: 0.339296\tval's binary_logloss: 0.434303\n",
      "[138]\ttrain's binary_logloss: 0.338391\tval's binary_logloss: 0.434198\n",
      "[139]\ttrain's binary_logloss: 0.337404\tval's binary_logloss: 0.434061\n",
      "[140]\ttrain's binary_logloss: 0.336569\tval's binary_logloss: 0.434292\n",
      "[141]\ttrain's binary_logloss: 0.335612\tval's binary_logloss: 0.43427\n",
      "[142]\ttrain's binary_logloss: 0.334555\tval's binary_logloss: 0.434261\n",
      "[143]\ttrain's binary_logloss: 0.333721\tval's binary_logloss: 0.434249\n",
      "[144]\ttrain's binary_logloss: 0.332994\tval's binary_logloss: 0.434215\n",
      "[145]\ttrain's binary_logloss: 0.332136\tval's binary_logloss: 0.434258\n",
      "[146]\ttrain's binary_logloss: 0.331524\tval's binary_logloss: 0.434092\n",
      "[147]\ttrain's binary_logloss: 0.330627\tval's binary_logloss: 0.434013\n",
      "[148]\ttrain's binary_logloss: 0.330059\tval's binary_logloss: 0.434193\n",
      "[149]\ttrain's binary_logloss: 0.329078\tval's binary_logloss: 0.434141\n",
      "[150]\ttrain's binary_logloss: 0.328427\tval's binary_logloss: 0.434367\n",
      "[151]\ttrain's binary_logloss: 0.327484\tval's binary_logloss: 0.434264\n",
      "[152]\ttrain's binary_logloss: 0.326809\tval's binary_logloss: 0.434526\n",
      "[153]\ttrain's binary_logloss: 0.326118\tval's binary_logloss: 0.434624\n",
      "[154]\ttrain's binary_logloss: 0.325297\tval's binary_logloss: 0.434752\n",
      "[155]\ttrain's binary_logloss: 0.324526\tval's binary_logloss: 0.434965\n",
      "[156]\ttrain's binary_logloss: 0.323737\tval's binary_logloss: 0.435022\n",
      "[157]\ttrain's binary_logloss: 0.323309\tval's binary_logloss: 0.43498\n",
      "[158]\ttrain's binary_logloss: 0.322412\tval's binary_logloss: 0.435164\n",
      "[159]\ttrain's binary_logloss: 0.321956\tval's binary_logloss: 0.435101\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "[160]\ttrain's binary_logloss: 0.321577\tval's binary_logloss: 0.435013\n",
      "[161]\ttrain's binary_logloss: 0.321154\tval's binary_logloss: 0.435011\n",
      "[162]\ttrain's binary_logloss: 0.32053\tval's binary_logloss: 0.435055\n",
      "[163]\ttrain's binary_logloss: 0.320052\tval's binary_logloss: 0.435117\n",
      "[164]\ttrain's binary_logloss: 0.319495\tval's binary_logloss: 0.435364\n",
      "[165]\ttrain's binary_logloss: 0.318879\tval's binary_logloss: 0.435365\n",
      "[166]\ttrain's binary_logloss: 0.318222\tval's binary_logloss: 0.435419\n",
      "[167]\ttrain's binary_logloss: 0.317357\tval's binary_logloss: 0.435392\n",
      "[168]\ttrain's binary_logloss: 0.316691\tval's binary_logloss: 0.435438\n",
      "[169]\ttrain's binary_logloss: 0.316256\tval's binary_logloss: 0.435394\n",
      "[170]\ttrain's binary_logloss: 0.315882\tval's binary_logloss: 0.435375\n",
      "[171]\ttrain's binary_logloss: 0.315545\tval's binary_logloss: 0.435288\n",
      "[172]\ttrain's binary_logloss: 0.314797\tval's binary_logloss: 0.435324\n",
      "[173]\ttrain's binary_logloss: 0.313944\tval's binary_logloss: 0.435237\n",
      "[174]\ttrain's binary_logloss: 0.313389\tval's binary_logloss: 0.435224\n",
      "[175]\ttrain's binary_logloss: 0.313052\tval's binary_logloss: 0.4353\n",
      "[176]\ttrain's binary_logloss: 0.312292\tval's binary_logloss: 0.435154\n",
      "[177]\ttrain's binary_logloss: 0.311895\tval's binary_logloss: 0.435003\n",
      "[178]\ttrain's binary_logloss: 0.311528\tval's binary_logloss: 0.434973\n",
      "[179]\ttrain's binary_logloss: 0.310734\tval's binary_logloss: 0.434731\n",
      "[180]\ttrain's binary_logloss: 0.31011\tval's binary_logloss: 0.434715\n",
      "[181]\ttrain's binary_logloss: 0.309429\tval's binary_logloss: 0.434621\n",
      "[182]\ttrain's binary_logloss: 0.308658\tval's binary_logloss: 0.434776\n",
      "[183]\ttrain's binary_logloss: 0.308014\tval's binary_logloss: 0.434838\n",
      "[184]\ttrain's binary_logloss: 0.307468\tval's binary_logloss: 0.434988\n",
      "[185]\ttrain's binary_logloss: 0.307104\tval's binary_logloss: 0.434971\n",
      "[186]\ttrain's binary_logloss: 0.306441\tval's binary_logloss: 0.434988\n",
      "[187]\ttrain's binary_logloss: 0.305569\tval's binary_logloss: 0.435141\n",
      "[188]\ttrain's binary_logloss: 0.305271\tval's binary_logloss: 0.435215\n",
      "[189]\ttrain's binary_logloss: 0.304607\tval's binary_logloss: 0.435181\n",
      "[190]\ttrain's binary_logloss: 0.303981\tval's binary_logloss: 0.435055\n",
      "[191]\ttrain's binary_logloss: 0.303315\tval's binary_logloss: 0.435251\n",
      "[192]\ttrain's binary_logloss: 0.30274\tval's binary_logloss: 0.435405\n",
      "[193]\ttrain's binary_logloss: 0.301864\tval's binary_logloss: 0.435372\n",
      "[194]\ttrain's binary_logloss: 0.301484\tval's binary_logloss: 0.435452\n",
      "[195]\ttrain's binary_logloss: 0.301014\tval's binary_logloss: 0.435727\n",
      "[196]\ttrain's binary_logloss: 0.300083\tval's binary_logloss: 0.435756\n",
      "[197]\ttrain's binary_logloss: 0.299544\tval's binary_logloss: 0.435986\n",
      "[198]\ttrain's binary_logloss: 0.299242\tval's binary_logloss: 0.436053\n",
      "[199]\ttrain's binary_logloss: 0.298515\tval's binary_logloss: 0.43649\n",
      "[200]\ttrain's binary_logloss: 0.298193\tval's binary_logloss: 0.436511\n",
      "[201]\ttrain's binary_logloss: 0.297521\tval's binary_logloss: 0.436551\n",
      "[202]\ttrain's binary_logloss: 0.296716\tval's binary_logloss: 0.436666\n",
      "[203]\ttrain's binary_logloss: 0.295858\tval's binary_logloss: 0.436609\n",
      "[204]\ttrain's binary_logloss: 0.295111\tval's binary_logloss: 0.43681\n",
      "[205]\ttrain's binary_logloss: 0.294636\tval's binary_logloss: 0.436886\n",
      "[206]\ttrain's binary_logloss: 0.293799\tval's binary_logloss: 0.437118\n",
      "[207]\ttrain's binary_logloss: 0.293252\tval's binary_logloss: 0.437264\n",
      "[208]\ttrain's binary_logloss: 0.29258\tval's binary_logloss: 0.437418\n",
      "[209]\ttrain's binary_logloss: 0.292261\tval's binary_logloss: 0.437434\n",
      "[210]\ttrain's binary_logloss: 0.291606\tval's binary_logloss: 0.437469\n",
      "[211]\ttrain's binary_logloss: 0.291045\tval's binary_logloss: 0.437558\n",
      "[212]\ttrain's binary_logloss: 0.290564\tval's binary_logloss: 0.43754\n",
      "[213]\ttrain's binary_logloss: 0.290205\tval's binary_logloss: 0.43755\n",
      "[214]\ttrain's binary_logloss: 0.289993\tval's binary_logloss: 0.437641\n",
      "[215]\ttrain's binary_logloss: 0.289417\tval's binary_logloss: 0.437635\n",
      "[216]\ttrain's binary_logloss: 0.288719\tval's binary_logloss: 0.43774\n",
      "[217]\ttrain's binary_logloss: 0.288504\tval's binary_logloss: 0.437777\n",
      "[218]\ttrain's binary_logloss: 0.287934\tval's binary_logloss: 0.43793\n",
      "[219]\ttrain's binary_logloss: 0.287508\tval's binary_logloss: 0.438196\n",
      "[220]\ttrain's binary_logloss: 0.286953\tval's binary_logloss: 0.438378\n",
      "[221]\ttrain's binary_logloss: 0.286314\tval's binary_logloss: 0.438319\n",
      "[222]\ttrain's binary_logloss: 0.285475\tval's binary_logloss: 0.438295\n",
      "[223]\ttrain's binary_logloss: 0.284867\tval's binary_logloss: 0.43833\n",
      "[224]\ttrain's binary_logloss: 0.284166\tval's binary_logloss: 0.438426\n",
      "[225]\ttrain's binary_logloss: 0.28362\tval's binary_logloss: 0.438208\n",
      "[226]\ttrain's binary_logloss: 0.282989\tval's binary_logloss: 0.43842\n",
      "[227]\ttrain's binary_logloss: 0.282301\tval's binary_logloss: 0.438607\n",
      "[228]\ttrain's binary_logloss: 0.281832\tval's binary_logloss: 0.438457\n",
      "[229]\ttrain's binary_logloss: 0.28132\tval's binary_logloss: 0.438614\n",
      "[230]\ttrain's binary_logloss: 0.28054\tval's binary_logloss: 0.438824\n",
      "[231]\ttrain's binary_logloss: 0.280026\tval's binary_logloss: 0.438863\n",
      "[232]\ttrain's binary_logloss: 0.27956\tval's binary_logloss: 0.438812\n",
      "[233]\ttrain's binary_logloss: 0.279157\tval's binary_logloss: 0.438906\n",
      "[234]\ttrain's binary_logloss: 0.27876\tval's binary_logloss: 0.439077\n",
      "[235]\ttrain's binary_logloss: 0.278232\tval's binary_logloss: 0.439069\n",
      "[236]\ttrain's binary_logloss: 0.277598\tval's binary_logloss: 0.439115\n",
      "[237]\ttrain's binary_logloss: 0.277383\tval's binary_logloss: 0.439168\n",
      "[238]\ttrain's binary_logloss: 0.276814\tval's binary_logloss: 0.439099\n",
      "[239]\ttrain's binary_logloss: 0.276113\tval's binary_logloss: 0.439041\n",
      "[240]\ttrain's binary_logloss: 0.275878\tval's binary_logloss: 0.439028\n",
      "[241]\ttrain's binary_logloss: 0.275332\tval's binary_logloss: 0.438959\n",
      "[242]\ttrain's binary_logloss: 0.275045\tval's binary_logloss: 0.438895\n",
      "[243]\ttrain's binary_logloss: 0.274489\tval's binary_logloss: 0.438907\n",
      "[244]\ttrain's binary_logloss: 0.274296\tval's binary_logloss: 0.438981\n",
      "[245]\ttrain's binary_logloss: 0.273753\tval's binary_logloss: 0.439018\n",
      "[246]\ttrain's binary_logloss: 0.273214\tval's binary_logloss: 0.439103\n",
      "[247]\ttrain's binary_logloss: 0.272726\tval's binary_logloss: 0.439223\n",
      "Early stopping, best iteration is:\n",
      "[147]\ttrain's binary_logloss: 0.330627\tval's binary_logloss: 0.434013\n",
      "tr_logloss:  0.3306267544890511\n",
      "val_logloss:  0.4340127102761729\n",
      "predict:  [0.39638553 0.34273839 0.23514265 0.14554212 0.16483957 0.36111202\n",
      " 0.16796998 0.10459984 0.0929376  0.30083677]\n"
     ]
    }
   ],
   "source": [
    "\"\"\"GBDT建模\"\"\"\n",
    "def gbdt_model(data, category_fea, continuous_fea):\n",
    "    \"\"\"\n",
    "        GBDT建模\n",
    "        \n",
    "        :param data: df, 数据表\n",
    "        :param category_fea: list, 离散特征列\n",
    "        :param continuous_fea: list, 连续特征列\n",
    "    \"\"\"\n",
    "    \n",
    "    # 离散特征one-hot编码\n",
    "    for col in category_fea:\n",
    "        onehot_feats = pd.get_dummies(data[col], prefix=col)\n",
    "        data.drop([col], axis=1, inplace=True)\n",
    "        data = pd.concat([data, onehot_feats], axis=1)\n",
    "\n",
    "    # 拆分训练集和测试集\n",
    "    train = data[data['Label'] != -1]\n",
    "    target = train.pop('Label')\n",
    "    test = data[data['Label'] == -1]\n",
    "    test.drop(['Label'], axis=1, inplace=True)\n",
    "    \n",
    "    # 划分数据集\n",
    "    x_train, x_val, y_train, y_val = train_test_split(train, target, test_size=0.2, random_state=2020)\n",
    "    \n",
    "    # 建立模型\n",
    "    gbm = lgb.LGBMClassifier(boosting_type='gbdt',  # 这里用gbdt\n",
    "                             objective='binary', \n",
    "                             subsample=0.8,\n",
    "                             min_child_weight=0.5, \n",
    "                             colsample_bytree=0.7,\n",
    "                             num_leaves=100,\n",
    "                             max_depth=12,\n",
    "                             learning_rate=0.01,\n",
    "                             n_estimators=10000\n",
    "                            )\n",
    "    gbm.fit(x_train, y_train, \n",
    "            eval_set=[(x_train, y_train), (x_val, y_val)], \n",
    "            eval_names=['train', 'val'],\n",
    "            eval_metric='binary_logloss',\n",
    "            early_stopping_rounds=100,\n",
    "           )\n",
    "    \n",
    "    tr_logloss = log_loss(y_train, gbm.predict_proba(x_train)[:, 1])   # −(ylog(p)+(1−y)log(1−p)) log_loss\n",
    "    val_logloss = log_loss(y_val, gbm.predict_proba(x_val)[:, 1])\n",
    "    print('tr_logloss: ', tr_logloss)\n",
    "    print('val_logloss: ', val_logloss)\n",
    "    \n",
    "    # 模型预测\n",
    "    y_pred = gbm.predict_proba(test)[:, 1]  # predict_proba 返回n行k列的矩阵，第i行第j列上的数值是模型预测第i个预测样本为某个标签的概率，这里的1表示点击的概率\n",
    "    print('predict: ', y_pred[:10]) # 这里看前10个，预测为点击的概率\n",
    "    \n",
    "\"\"\"训练和预测GBDT模型\"\"\"\n",
    "gbdt_model(data.copy(), category_fea, continuous_fea)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 74,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "[1]\ttrain's binary_logloss: 0.523857\tval's binary_logloss: 0.457806\n",
      "Training until validation scores don't improve for 100 rounds\n",
      "[2]\ttrain's binary_logloss: 0.521371\tval's binary_logloss: 0.457213\n",
      "[3]\ttrain's binary_logloss: 0.519084\tval's binary_logloss: 0.456616\n",
      "[4]\ttrain's binary_logloss: 0.516882\tval's binary_logloss: 0.456046\n",
      "[5]\ttrain's binary_logloss: 0.514449\tval's binary_logloss: 0.455649\n",
      "[6]\ttrain's binary_logloss: 0.512277\tval's binary_logloss: 0.455319\n",
      "[7]\ttrain's binary_logloss: 0.509973\tval's binary_logloss: 0.455039\n",
      "[8]\ttrain's binary_logloss: 0.507717\tval's binary_logloss: 0.454523\n",
      "[9]\ttrain's binary_logloss: 0.505668\tval's binary_logloss: 0.454546\n",
      "[10]\ttrain's binary_logloss: 0.503491\tval's binary_logloss: 0.454134\n",
      "[11]\ttrain's binary_logloss: 0.501469\tval's binary_logloss: 0.453151\n",
      "[12]\ttrain's binary_logloss: 0.499463\tval's binary_logloss: 0.452609\n",
      "[13]\ttrain's binary_logloss: 0.497257\tval's binary_logloss: 0.452419\n",
      "[14]\ttrain's binary_logloss: 0.495206\tval's binary_logloss: 0.451944\n",
      "[15]\ttrain's binary_logloss: 0.493127\tval's binary_logloss: 0.451356\n",
      "[16]\ttrain's binary_logloss: 0.491215\tval's binary_logloss: 0.451181\n",
      "[17]\ttrain's binary_logloss: 0.489258\tval's binary_logloss: 0.450615\n",
      "[18]\ttrain's binary_logloss: 0.487424\tval's binary_logloss: 0.450754\n",
      "[19]\ttrain's binary_logloss: 0.485589\tval's binary_logloss: 0.450428\n",
      "[20]\ttrain's binary_logloss: 0.483753\tval's binary_logloss: 0.450043\n",
      "[21]\ttrain's binary_logloss: 0.481871\tval's binary_logloss: 0.449924\n",
      "[22]\ttrain's binary_logloss: 0.480233\tval's binary_logloss: 0.449518\n",
      "[23]\ttrain's binary_logloss: 0.478629\tval's binary_logloss: 0.449086\n",
      "[24]\ttrain's binary_logloss: 0.476933\tval's binary_logloss: 0.448878\n",
      "[25]\ttrain's binary_logloss: 0.47504\tval's binary_logloss: 0.448413\n",
      "[26]\ttrain's binary_logloss: 0.473416\tval's binary_logloss: 0.44809\n",
      "[27]\ttrain's binary_logloss: 0.471608\tval's binary_logloss: 0.448072\n",
      "[28]\ttrain's binary_logloss: 0.469969\tval's binary_logloss: 0.447721\n",
      "[29]\ttrain's binary_logloss: 0.46833\tval's binary_logloss: 0.447358\n",
      "[30]\ttrain's binary_logloss: 0.466517\tval's binary_logloss: 0.447445\n",
      "[31]\ttrain's binary_logloss: 0.464722\tval's binary_logloss: 0.447317\n",
      "[32]\ttrain's binary_logloss: 0.463079\tval's binary_logloss: 0.447152\n",
      "[33]\ttrain's binary_logloss: 0.461319\tval's binary_logloss: 0.446674\n",
      "[34]\ttrain's binary_logloss: 0.459722\tval's binary_logloss: 0.446455\n",
      "[35]\ttrain's binary_logloss: 0.457986\tval's binary_logloss: 0.446345\n",
      "[36]\ttrain's binary_logloss: 0.456293\tval's binary_logloss: 0.446229\n",
      "[37]\ttrain's binary_logloss: 0.454638\tval's binary_logloss: 0.446135\n",
      "[38]\ttrain's binary_logloss: 0.452929\tval's binary_logloss: 0.445476\n",
      "[39]\ttrain's binary_logloss: 0.451349\tval's binary_logloss: 0.445208\n",
      "[40]\ttrain's binary_logloss: 0.449759\tval's binary_logloss: 0.444538\n",
      "[41]\ttrain's binary_logloss: 0.448181\tval's binary_logloss: 0.444151\n",
      "[42]\ttrain's binary_logloss: 0.446656\tval's binary_logloss: 0.443786\n",
      "[43]\ttrain's binary_logloss: 0.445243\tval's binary_logloss: 0.443472\n",
      "[44]\ttrain's binary_logloss: 0.443697\tval's binary_logloss: 0.443249\n",
      "[45]\ttrain's binary_logloss: 0.442193\tval's binary_logloss: 0.443062\n",
      "[46]\ttrain's binary_logloss: 0.440697\tval's binary_logloss: 0.442784\n",
      "[47]\ttrain's binary_logloss: 0.439353\tval's binary_logloss: 0.442828\n",
      "[48]\ttrain's binary_logloss: 0.437963\tval's binary_logloss: 0.442524\n",
      "[49]\ttrain's binary_logloss: 0.436672\tval's binary_logloss: 0.442423\n",
      "[50]\ttrain's binary_logloss: 0.435301\tval's binary_logloss: 0.442193\n",
      "[51]\ttrain's binary_logloss: 0.433889\tval's binary_logloss: 0.441743\n",
      "[52]\ttrain's binary_logloss: 0.432548\tval's binary_logloss: 0.441336\n",
      "[53]\ttrain's binary_logloss: 0.431017\tval's binary_logloss: 0.441085\n",
      "[54]\ttrain's binary_logloss: 0.429677\tval's binary_logloss: 0.440932\n",
      "[55]\ttrain's binary_logloss: 0.428271\tval's binary_logloss: 0.440676\n",
      "[56]\ttrain's binary_logloss: 0.42698\tval's binary_logloss: 0.44047\n",
      "[57]\ttrain's binary_logloss: 0.425539\tval's binary_logloss: 0.440397\n",
      "[58]\ttrain's binary_logloss: 0.424123\tval's binary_logloss: 0.440136\n",
      "[59]\ttrain's binary_logloss: 0.422809\tval's binary_logloss: 0.439885\n",
      "[60]\ttrain's binary_logloss: 0.42146\tval's binary_logloss: 0.439716\n",
      "[61]\ttrain's binary_logloss: 0.420068\tval's binary_logloss: 0.439454\n",
      "[62]\ttrain's binary_logloss: 0.418745\tval's binary_logloss: 0.439493\n",
      "[63]\ttrain's binary_logloss: 0.41755\tval's binary_logloss: 0.439334\n",
      "[64]\ttrain's binary_logloss: 0.416342\tval's binary_logloss: 0.439451\n",
      "[65]\ttrain's binary_logloss: 0.415058\tval's binary_logloss: 0.439173\n",
      "[66]\ttrain's binary_logloss: 0.413658\tval's binary_logloss: 0.43881\n",
      "[67]\ttrain's binary_logloss: 0.4123\tval's binary_logloss: 0.438656\n",
      "[68]\ttrain's binary_logloss: 0.410981\tval's binary_logloss: 0.438352\n",
      "[69]\ttrain's binary_logloss: 0.409717\tval's binary_logloss: 0.438017\n",
      "[70]\ttrain's binary_logloss: 0.408442\tval's binary_logloss: 0.437799\n",
      "[71]\ttrain's binary_logloss: 0.407242\tval's binary_logloss: 0.437543\n",
      "[72]\ttrain's binary_logloss: 0.406051\tval's binary_logloss: 0.437359\n",
      "[73]\ttrain's binary_logloss: 0.404727\tval's binary_logloss: 0.437274\n",
      "[74]\ttrain's binary_logloss: 0.403419\tval's binary_logloss: 0.437135\n",
      "[75]\ttrain's binary_logloss: 0.402132\tval's binary_logloss: 0.437162\n",
      "[76]\ttrain's binary_logloss: 0.401026\tval's binary_logloss: 0.437043\n",
      "[77]\ttrain's binary_logloss: 0.399681\tval's binary_logloss: 0.436681\n",
      "[78]\ttrain's binary_logloss: 0.398398\tval's binary_logloss: 0.436374\n",
      "[79]\ttrain's binary_logloss: 0.397186\tval's binary_logloss: 0.436306\n",
      "[80]\ttrain's binary_logloss: 0.395998\tval's binary_logloss: 0.436238\n",
      "[81]\ttrain's binary_logloss: 0.394678\tval's binary_logloss: 0.436146\n",
      "[82]\ttrain's binary_logloss: 0.393508\tval's binary_logloss: 0.436026\n",
      "[83]\ttrain's binary_logloss: 0.392385\tval's binary_logloss: 0.435897\n",
      "[84]\ttrain's binary_logloss: 0.391328\tval's binary_logloss: 0.435948\n",
      "[85]\ttrain's binary_logloss: 0.390191\tval's binary_logloss: 0.435803\n",
      "[86]\ttrain's binary_logloss: 0.389034\tval's binary_logloss: 0.435726\n",
      "[87]\ttrain's binary_logloss: 0.388039\tval's binary_logloss: 0.435854\n",
      "[88]\ttrain's binary_logloss: 0.3868\tval's binary_logloss: 0.435683\n",
      "[89]\ttrain's binary_logloss: 0.38566\tval's binary_logloss: 0.435685\n",
      "[90]\ttrain's binary_logloss: 0.384527\tval's binary_logloss: 0.435758\n",
      "[91]\ttrain's binary_logloss: 0.383388\tval's binary_logloss: 0.435756\n",
      "[92]\ttrain's binary_logloss: 0.38231\tval's binary_logloss: 0.435815\n",
      "[93]\ttrain's binary_logloss: 0.381094\tval's binary_logloss: 0.435682\n",
      "[94]\ttrain's binary_logloss: 0.379807\tval's binary_logloss: 0.435561\n",
      "[95]\ttrain's binary_logloss: 0.37872\tval's binary_logloss: 0.435383\n",
      "[96]\ttrain's binary_logloss: 0.377682\tval's binary_logloss: 0.435011\n",
      "[97]\ttrain's binary_logloss: 0.376613\tval's binary_logloss: 0.434783\n",
      "[98]\ttrain's binary_logloss: 0.375463\tval's binary_logloss: 0.434865\n",
      "[99]\ttrain's binary_logloss: 0.374318\tval's binary_logloss: 0.434926\n",
      "[100]\ttrain's binary_logloss: 0.373331\tval's binary_logloss: 0.435077\n",
      "[101]\ttrain's binary_logloss: 0.372209\tval's binary_logloss: 0.43519\n",
      "[102]\ttrain's binary_logloss: 0.371232\tval's binary_logloss: 0.434997\n",
      "[103]\ttrain's binary_logloss: 0.370158\tval's binary_logloss: 0.43491\n",
      "[104]\ttrain's binary_logloss: 0.369108\tval's binary_logloss: 0.434854\n",
      "[105]\ttrain's binary_logloss: 0.368046\tval's binary_logloss: 0.435009\n",
      "[106]\ttrain's binary_logloss: 0.366972\tval's binary_logloss: 0.434982\n",
      "[107]\ttrain's binary_logloss: 0.365952\tval's binary_logloss: 0.435174\n",
      "[108]\ttrain's binary_logloss: 0.365084\tval's binary_logloss: 0.435282\n",
      "[109]\ttrain's binary_logloss: 0.364023\tval's binary_logloss: 0.435066\n",
      "[110]\ttrain's binary_logloss: 0.363132\tval's binary_logloss: 0.43526\n",
      "[111]\ttrain's binary_logloss: 0.362204\tval's binary_logloss: 0.435537\n",
      "[112]\ttrain's binary_logloss: 0.361217\tval's binary_logloss: 0.435766\n",
      "[113]\ttrain's binary_logloss: 0.360214\tval's binary_logloss: 0.43583\n",
      "[114]\ttrain's binary_logloss: 0.35936\tval's binary_logloss: 0.435792\n",
      "[115]\ttrain's binary_logloss: 0.358383\tval's binary_logloss: 0.435522\n",
      "[116]\ttrain's binary_logloss: 0.357288\tval's binary_logloss: 0.435637\n",
      "[117]\ttrain's binary_logloss: 0.356216\tval's binary_logloss: 0.435376\n",
      "[118]\ttrain's binary_logloss: 0.355096\tval's binary_logloss: 0.435324\n",
      "[119]\ttrain's binary_logloss: 0.354101\tval's binary_logloss: 0.43536\n",
      "[120]\ttrain's binary_logloss: 0.353266\tval's binary_logloss: 0.435261\n",
      "[121]\ttrain's binary_logloss: 0.352268\tval's binary_logloss: 0.435374\n",
      "[122]\ttrain's binary_logloss: 0.351148\tval's binary_logloss: 0.435321\n",
      "[123]\ttrain's binary_logloss: 0.350471\tval's binary_logloss: 0.435319\n",
      "[124]\ttrain's binary_logloss: 0.349503\tval's binary_logloss: 0.435183\n",
      "[125]\ttrain's binary_logloss: 0.348856\tval's binary_logloss: 0.435206\n",
      "[126]\ttrain's binary_logloss: 0.348053\tval's binary_logloss: 0.435206\n",
      "[127]\ttrain's binary_logloss: 0.347443\tval's binary_logloss: 0.434997\n",
      "[128]\ttrain's binary_logloss: 0.346512\tval's binary_logloss: 0.434941\n",
      "[129]\ttrain's binary_logloss: 0.345885\tval's binary_logloss: 0.434922\n",
      "[130]\ttrain's binary_logloss: 0.345112\tval's binary_logloss: 0.43477\n",
      "[131]\ttrain's binary_logloss: 0.344716\tval's binary_logloss: 0.434628\n",
      "[132]\ttrain's binary_logloss: 0.343697\tval's binary_logloss: 0.434359\n",
      "[133]\ttrain's binary_logloss: 0.342596\tval's binary_logloss: 0.434238\n",
      "[134]\ttrain's binary_logloss: 0.341666\tval's binary_logloss: 0.434321\n",
      "[135]\ttrain's binary_logloss: 0.340742\tval's binary_logloss: 0.434473\n",
      "[136]\ttrain's binary_logloss: 0.340167\tval's binary_logloss: 0.43421\n",
      "[137]\ttrain's binary_logloss: 0.339296\tval's binary_logloss: 0.434303\n",
      "[138]\ttrain's binary_logloss: 0.338391\tval's binary_logloss: 0.434198\n",
      "[139]\ttrain's binary_logloss: 0.337404\tval's binary_logloss: 0.434061\n",
      "[140]\ttrain's binary_logloss: 0.336569\tval's binary_logloss: 0.434292\n",
      "[141]\ttrain's binary_logloss: 0.335612\tval's binary_logloss: 0.43427\n",
      "[142]\ttrain's binary_logloss: 0.334555\tval's binary_logloss: 0.434261\n",
      "[143]\ttrain's binary_logloss: 0.333721\tval's binary_logloss: 0.434249\n",
      "[144]\ttrain's binary_logloss: 0.332994\tval's binary_logloss: 0.434215\n",
      "[145]\ttrain's binary_logloss: 0.332136\tval's binary_logloss: 0.434258\n",
      "[146]\ttrain's binary_logloss: 0.331524\tval's binary_logloss: 0.434092\n",
      "[147]\ttrain's binary_logloss: 0.330627\tval's binary_logloss: 0.434013\n",
      "[148]\ttrain's binary_logloss: 0.330059\tval's binary_logloss: 0.434193\n",
      "[149]\ttrain's binary_logloss: 0.329078\tval's binary_logloss: 0.434141\n",
      "[150]\ttrain's binary_logloss: 0.328427\tval's binary_logloss: 0.434367\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "[151]\ttrain's binary_logloss: 0.327484\tval's binary_logloss: 0.434264\n",
      "[152]\ttrain's binary_logloss: 0.326809\tval's binary_logloss: 0.434526\n",
      "[153]\ttrain's binary_logloss: 0.326118\tval's binary_logloss: 0.434624\n",
      "[154]\ttrain's binary_logloss: 0.325297\tval's binary_logloss: 0.434752\n",
      "[155]\ttrain's binary_logloss: 0.324526\tval's binary_logloss: 0.434965\n",
      "[156]\ttrain's binary_logloss: 0.323737\tval's binary_logloss: 0.435022\n",
      "[157]\ttrain's binary_logloss: 0.323309\tval's binary_logloss: 0.43498\n",
      "[158]\ttrain's binary_logloss: 0.322412\tval's binary_logloss: 0.435164\n",
      "[159]\ttrain's binary_logloss: 0.321956\tval's binary_logloss: 0.435101\n",
      "[160]\ttrain's binary_logloss: 0.321577\tval's binary_logloss: 0.435013\n",
      "[161]\ttrain's binary_logloss: 0.321154\tval's binary_logloss: 0.435011\n",
      "[162]\ttrain's binary_logloss: 0.32053\tval's binary_logloss: 0.435055\n",
      "[163]\ttrain's binary_logloss: 0.320052\tval's binary_logloss: 0.435117\n",
      "[164]\ttrain's binary_logloss: 0.319495\tval's binary_logloss: 0.435364\n",
      "[165]\ttrain's binary_logloss: 0.318879\tval's binary_logloss: 0.435365\n",
      "[166]\ttrain's binary_logloss: 0.318222\tval's binary_logloss: 0.435419\n",
      "[167]\ttrain's binary_logloss: 0.317357\tval's binary_logloss: 0.435392\n",
      "[168]\ttrain's binary_logloss: 0.316691\tval's binary_logloss: 0.435438\n",
      "[169]\ttrain's binary_logloss: 0.316256\tval's binary_logloss: 0.435394\n",
      "[170]\ttrain's binary_logloss: 0.315882\tval's binary_logloss: 0.435375\n",
      "[171]\ttrain's binary_logloss: 0.315545\tval's binary_logloss: 0.435288\n",
      "[172]\ttrain's binary_logloss: 0.314797\tval's binary_logloss: 0.435324\n",
      "[173]\ttrain's binary_logloss: 0.313944\tval's binary_logloss: 0.435237\n",
      "[174]\ttrain's binary_logloss: 0.313389\tval's binary_logloss: 0.435224\n",
      "[175]\ttrain's binary_logloss: 0.313052\tval's binary_logloss: 0.4353\n",
      "[176]\ttrain's binary_logloss: 0.312292\tval's binary_logloss: 0.435154\n",
      "[177]\ttrain's binary_logloss: 0.311895\tval's binary_logloss: 0.435003\n",
      "[178]\ttrain's binary_logloss: 0.311528\tval's binary_logloss: 0.434973\n",
      "[179]\ttrain's binary_logloss: 0.310734\tval's binary_logloss: 0.434731\n",
      "[180]\ttrain's binary_logloss: 0.31011\tval's binary_logloss: 0.434715\n",
      "[181]\ttrain's binary_logloss: 0.309429\tval's binary_logloss: 0.434621\n",
      "[182]\ttrain's binary_logloss: 0.308658\tval's binary_logloss: 0.434776\n",
      "[183]\ttrain's binary_logloss: 0.308014\tval's binary_logloss: 0.434838\n",
      "[184]\ttrain's binary_logloss: 0.307468\tval's binary_logloss: 0.434988\n",
      "[185]\ttrain's binary_logloss: 0.307104\tval's binary_logloss: 0.434971\n",
      "[186]\ttrain's binary_logloss: 0.306441\tval's binary_logloss: 0.434988\n",
      "[187]\ttrain's binary_logloss: 0.305569\tval's binary_logloss: 0.435141\n",
      "[188]\ttrain's binary_logloss: 0.305271\tval's binary_logloss: 0.435215\n",
      "[189]\ttrain's binary_logloss: 0.304607\tval's binary_logloss: 0.435181\n",
      "[190]\ttrain's binary_logloss: 0.303981\tval's binary_logloss: 0.435055\n",
      "[191]\ttrain's binary_logloss: 0.303315\tval's binary_logloss: 0.435251\n",
      "[192]\ttrain's binary_logloss: 0.30274\tval's binary_logloss: 0.435405\n",
      "[193]\ttrain's binary_logloss: 0.301864\tval's binary_logloss: 0.435372\n",
      "[194]\ttrain's binary_logloss: 0.301484\tval's binary_logloss: 0.435452\n",
      "[195]\ttrain's binary_logloss: 0.301014\tval's binary_logloss: 0.435727\n",
      "[196]\ttrain's binary_logloss: 0.300083\tval's binary_logloss: 0.435756\n",
      "[197]\ttrain's binary_logloss: 0.299544\tval's binary_logloss: 0.435986\n",
      "[198]\ttrain's binary_logloss: 0.299242\tval's binary_logloss: 0.436053\n",
      "[199]\ttrain's binary_logloss: 0.298515\tval's binary_logloss: 0.43649\n",
      "[200]\ttrain's binary_logloss: 0.298193\tval's binary_logloss: 0.436511\n",
      "[201]\ttrain's binary_logloss: 0.297521\tval's binary_logloss: 0.436551\n",
      "[202]\ttrain's binary_logloss: 0.296716\tval's binary_logloss: 0.436666\n",
      "[203]\ttrain's binary_logloss: 0.295858\tval's binary_logloss: 0.436609\n",
      "[204]\ttrain's binary_logloss: 0.295111\tval's binary_logloss: 0.43681\n",
      "[205]\ttrain's binary_logloss: 0.294636\tval's binary_logloss: 0.436886\n",
      "[206]\ttrain's binary_logloss: 0.293799\tval's binary_logloss: 0.437118\n",
      "[207]\ttrain's binary_logloss: 0.293252\tval's binary_logloss: 0.437264\n",
      "[208]\ttrain's binary_logloss: 0.29258\tval's binary_logloss: 0.437418\n",
      "[209]\ttrain's binary_logloss: 0.292261\tval's binary_logloss: 0.437434\n",
      "[210]\ttrain's binary_logloss: 0.291606\tval's binary_logloss: 0.437469\n",
      "[211]\ttrain's binary_logloss: 0.291045\tval's binary_logloss: 0.437558\n",
      "[212]\ttrain's binary_logloss: 0.290564\tval's binary_logloss: 0.43754\n",
      "[213]\ttrain's binary_logloss: 0.290205\tval's binary_logloss: 0.43755\n",
      "[214]\ttrain's binary_logloss: 0.289993\tval's binary_logloss: 0.437641\n",
      "[215]\ttrain's binary_logloss: 0.289417\tval's binary_logloss: 0.437635\n",
      "[216]\ttrain's binary_logloss: 0.288719\tval's binary_logloss: 0.43774\n",
      "[217]\ttrain's binary_logloss: 0.288504\tval's binary_logloss: 0.437777\n",
      "[218]\ttrain's binary_logloss: 0.287934\tval's binary_logloss: 0.43793\n",
      "[219]\ttrain's binary_logloss: 0.287508\tval's binary_logloss: 0.438196\n",
      "[220]\ttrain's binary_logloss: 0.286953\tval's binary_logloss: 0.438378\n",
      "[221]\ttrain's binary_logloss: 0.286314\tval's binary_logloss: 0.438319\n",
      "[222]\ttrain's binary_logloss: 0.285475\tval's binary_logloss: 0.438295\n",
      "[223]\ttrain's binary_logloss: 0.284867\tval's binary_logloss: 0.43833\n",
      "[224]\ttrain's binary_logloss: 0.284166\tval's binary_logloss: 0.438426\n",
      "[225]\ttrain's binary_logloss: 0.28362\tval's binary_logloss: 0.438208\n",
      "[226]\ttrain's binary_logloss: 0.282989\tval's binary_logloss: 0.43842\n",
      "[227]\ttrain's binary_logloss: 0.282301\tval's binary_logloss: 0.438607\n",
      "[228]\ttrain's binary_logloss: 0.281832\tval's binary_logloss: 0.438457\n",
      "[229]\ttrain's binary_logloss: 0.28132\tval's binary_logloss: 0.438614\n",
      "[230]\ttrain's binary_logloss: 0.28054\tval's binary_logloss: 0.438824\n",
      "[231]\ttrain's binary_logloss: 0.280026\tval's binary_logloss: 0.438863\n",
      "[232]\ttrain's binary_logloss: 0.27956\tval's binary_logloss: 0.438812\n",
      "[233]\ttrain's binary_logloss: 0.279157\tval's binary_logloss: 0.438906\n",
      "[234]\ttrain's binary_logloss: 0.27876\tval's binary_logloss: 0.439077\n",
      "[235]\ttrain's binary_logloss: 0.278232\tval's binary_logloss: 0.439069\n",
      "[236]\ttrain's binary_logloss: 0.277598\tval's binary_logloss: 0.439115\n",
      "[237]\ttrain's binary_logloss: 0.277383\tval's binary_logloss: 0.439168\n",
      "[238]\ttrain's binary_logloss: 0.276814\tval's binary_logloss: 0.439099\n",
      "[239]\ttrain's binary_logloss: 0.276113\tval's binary_logloss: 0.439041\n",
      "[240]\ttrain's binary_logloss: 0.275878\tval's binary_logloss: 0.439028\n",
      "[241]\ttrain's binary_logloss: 0.275332\tval's binary_logloss: 0.438959\n",
      "[242]\ttrain's binary_logloss: 0.275045\tval's binary_logloss: 0.438895\n",
      "[243]\ttrain's binary_logloss: 0.274489\tval's binary_logloss: 0.438907\n",
      "[244]\ttrain's binary_logloss: 0.274296\tval's binary_logloss: 0.438981\n",
      "[245]\ttrain's binary_logloss: 0.273753\tval's binary_logloss: 0.439018\n",
      "[246]\ttrain's binary_logloss: 0.273214\tval's binary_logloss: 0.439103\n",
      "[247]\ttrain's binary_logloss: 0.272726\tval's binary_logloss: 0.439223\n",
      "Early stopping, best iteration is:\n",
      "[147]\ttrain's binary_logloss: 0.330627\tval's binary_logloss: 0.434013\n",
      "tr-logloss:  0.012043875601047957\n",
      "val-logloss:  0.30527705372305225\n",
      "[9.42504639e-01 2.76597656e-01 2.27699953e-02 1.86025096e-02\n",
      " 4.45934326e-03 6.48474724e-01 6.80003741e-03 3.57831915e-03\n",
      " 4.16062509e-04 5.44962925e-02]\n"
     ]
    }
   ],
   "source": [
    "\"\"\"GBDT+LR建模\"\"\"\n",
    "def gbdt_lr_model(data, category_feature, continuous_feature):  # 0.43616\n",
    "    \"\"\"\n",
    "        GBDT+LR建模\n",
    "        \n",
    "        :param data: df, 数据表\n",
    "        :param category_fea: list, 离散特征列\n",
    "        :param continuous_fea: list, 连续特征列\n",
    "    \"\"\"\n",
    "    \n",
    "    # 离散特征one-hot编码\n",
    "    for col in category_feature:\n",
    "        onehot_feats = pd.get_dummies(data[col], prefix = col)\n",
    "        data.drop([col], axis = 1, inplace = True)\n",
    "        data = pd.concat([data, onehot_feats], axis = 1)\n",
    "\n",
    "    # 拆分训练集和测试集\n",
    "    train = data[data['Label'] != -1]\n",
    "    target = train.pop('Label')\n",
    "    test = data[data['Label'] == -1]\n",
    "    test.drop(['Label'], axis = 1, inplace = True)\n",
    "\n",
    "    # 划分数据集\n",
    "    x_train, x_val, y_train, y_val = train_test_split(train, target, test_size = 0.2, random_state = 2020)\n",
    "\n",
    "    # 建立GBDT模型\n",
    "    gbm = lgb.LGBMClassifier(objective='binary',\n",
    "                            subsample= 0.8,\n",
    "                            min_child_weight= 0.5,\n",
    "                            colsample_bytree= 0.7,\n",
    "                            num_leaves=100,\n",
    "                            max_depth = 12,\n",
    "                            learning_rate=0.01,\n",
    "                            n_estimators=1000,\n",
    "                            )\n",
    "\n",
    "    gbm.fit(x_train, y_train,\n",
    "            eval_set = [(x_train, y_train), (x_val, y_val)],\n",
    "            eval_names = ['train', 'val'],\n",
    "            eval_metric = 'binary_logloss',\n",
    "            early_stopping_rounds = 100,\n",
    "            )\n",
    "    \n",
    "    # GBDT负责对各个特征进行交叉， 把原始特征向量转换为新的离散型特征向量\n",
    "    model = gbm.booster_\n",
    "\n",
    "    gbdt_feats_train = model.predict(train, pred_leaf = True)\n",
    "    gbdt_feats_test = model.predict(test, pred_leaf = True)\n",
    "    gbdt_feats_name = ['gbdt_leaf_' + str(i) for i in range(gbdt_feats_train.shape[1])]\n",
    "    df_train_gbdt_feats = pd.DataFrame(gbdt_feats_train, columns = gbdt_feats_name) \n",
    "    df_test_gbdt_feats = pd.DataFrame(gbdt_feats_test, columns = gbdt_feats_name)\n",
    "\n",
    "    # 新特征（连续特征 + 离散特征one-hot编码 + GBDT生成的离散特征）\n",
    "    train = pd.concat([train, df_train_gbdt_feats], axis = 1)\n",
    "    test = pd.concat([test, df_test_gbdt_feats], axis = 1)\n",
    "    train_len = train.shape[0]\n",
    "    data = pd.concat([train, test])    \n",
    "    del train\n",
    "    del test\n",
    "    gc.collect()  # 清理内存\n",
    "\n",
    "    # 连续特征归一化\n",
    "    scaler = MinMaxScaler()\n",
    "    for col in continuous_feature:\n",
    "        data[col] = scaler.fit_transform(data[col].values.reshape(-1, 1))\n",
    "\n",
    "    # GBDT生成的离散特征one-hot编码\n",
    "    for col in gbdt_feats_name:\n",
    "        onehot_feats = pd.get_dummies(data[col], prefix = col)\n",
    "        data.drop([col], axis = 1, inplace = True)\n",
    "        data = pd.concat([data, onehot_feats], axis = 1)\n",
    "\n",
    "    train = data[: train_len]\n",
    "    test = data[train_len:]\n",
    "    del data\n",
    "    gc.collect()\n",
    "\n",
    "    # 划分数据集\n",
    "    x_train, x_val, y_train, y_val = train_test_split(train, target, test_size = 0.3, random_state = 2018)\n",
    "\n",
    "    # 建立逻辑回归模型\n",
    "    lr = LogisticRegression()\n",
    "    lr.fit(x_train, y_train)\n",
    "    tr_logloss = log_loss(y_train, lr.predict_proba(x_train)[:, 1])\n",
    "    val_logloss = log_loss(y_val, lr.predict_proba(x_val)[:, 1])\n",
    "    print('tr-logloss: ', tr_logloss)\n",
    "    print('val-logloss: ', val_logloss)\n",
    "    \n",
    "    # 模型预测\n",
    "    y_pred = lr.predict_proba(test)[:, 1]\n",
    "    print(y_pred[:10])\n",
    "\n",
    "\"\"\"训练和预测GBDT+LR模型\"\"\"\n",
    "gbdt_lr_model(data.copy(), category_fea, continuous_fea)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 3",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.8.8"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 4
}