ch02/ch02_07.ipynb

{
 "cells": [
  {
   "cell_type": "markdown",
   "metadata": {
    "_cell_guid": "e12020f7-4f94-4ecc-9007-9b7a6e7458a6",
    "_uuid": "1fecb0980d8d422ec0f005c4bfd6225385c2c60f"
   },
   "source": [
    "This notebook is a sample code with Japanese comments.\n",
    "\n",
    "# 2.7 submitのその前に！　「Cross Validation」の大切さを知ろう"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 1,
   "metadata": {},
   "outputs": [],
   "source": [
    "import numpy as np\n",
    "import pandas as pd\n",
    "\n",
    "\n",
    "train = pd.read_csv('../input/titanic/train.csv')\n",
    "test = pd.read_csv('../input/titanic/test.csv')\n",
    "gender_submission = pd.read_csv('../input/titanic/gender_submission.csv')\n",
    "\n",
    "data = pd.concat([train, test], sort=False)\n",
    "\n",
    "data['Sex'].replace(['male', 'female'], [0, 1], inplace=True)\n",
    "data['Embarked'].fillna(('S'), inplace=True)\n",
    "data['Embarked'] = data['Embarked'].map({'S': 0, 'C': 1, 'Q': 2}).astype(int)\n",
    "data['Fare'].fillna(np.mean(data['Fare']), inplace=True)\n",
    "data['Age'].fillna(data['Age'].median(), inplace=True)\n",
    "data['FamilySize'] = data['Parch'] + data['SibSp'] + 1\n",
    "data['IsAlone'] = 0\n",
    "data.loc[data['FamilySize'] == 1, 'IsAlone'] = 1"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 2,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>PassengerId</th>\n",
       "      <th>Survived</th>\n",
       "      <th>Pclass</th>\n",
       "      <th>Name</th>\n",
       "      <th>Sex</th>\n",
       "      <th>Age</th>\n",
       "      <th>SibSp</th>\n",
       "      <th>Parch</th>\n",
       "      <th>Ticket</th>\n",
       "      <th>Fare</th>\n",
       "      <th>Cabin</th>\n",
       "      <th>Embarked</th>\n",
       "      <th>FamilySize</th>\n",
       "      <th>IsAlone</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>1</td>\n",
       "      <td>0.0</td>\n",
       "      <td>3</td>\n",
       "      <td>Braund, Mr. Owen Harris</td>\n",
       "      <td>0</td>\n",
       "      <td>22.0</td>\n",
       "      <td>1</td>\n",
       "      <td>0</td>\n",
       "      <td>A/5 21171</td>\n",
       "      <td>7.2500</td>\n",
       "      <td>NaN</td>\n",
       "      <td>0</td>\n",
       "      <td>2</td>\n",
       "      <td>0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>2</td>\n",
       "      <td>1.0</td>\n",
       "      <td>1</td>\n",
       "      <td>Cumings, Mrs. John Bradley (Florence Briggs Th...</td>\n",
       "      <td>1</td>\n",
       "      <td>38.0</td>\n",
       "      <td>1</td>\n",
       "      <td>0</td>\n",
       "      <td>PC 17599</td>\n",
       "      <td>71.2833</td>\n",
       "      <td>C85</td>\n",
       "      <td>1</td>\n",
       "      <td>2</td>\n",
       "      <td>0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2</th>\n",
       "      <td>3</td>\n",
       "      <td>1.0</td>\n",
       "      <td>3</td>\n",
       "      <td>Heikkinen, Miss. Laina</td>\n",
       "      <td>1</td>\n",
       "      <td>26.0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>STON/O2. 3101282</td>\n",
       "      <td>7.9250</td>\n",
       "      <td>NaN</td>\n",
       "      <td>0</td>\n",
       "      <td>1</td>\n",
       "      <td>1</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3</th>\n",
       "      <td>4</td>\n",
       "      <td>1.0</td>\n",
       "      <td>1</td>\n",
       "      <td>Futrelle, Mrs. Jacques Heath (Lily May Peel)</td>\n",
       "      <td>1</td>\n",
       "      <td>35.0</td>\n",
       "      <td>1</td>\n",
       "      <td>0</td>\n",
       "      <td>113803</td>\n",
       "      <td>53.1000</td>\n",
       "      <td>C123</td>\n",
       "      <td>0</td>\n",
       "      <td>2</td>\n",
       "      <td>0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4</th>\n",
       "      <td>5</td>\n",
       "      <td>0.0</td>\n",
       "      <td>3</td>\n",
       "      <td>Allen, Mr. William Henry</td>\n",
       "      <td>0</td>\n",
       "      <td>35.0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>373450</td>\n",
       "      <td>8.0500</td>\n",
       "      <td>NaN</td>\n",
       "      <td>0</td>\n",
       "      <td>1</td>\n",
       "      <td>1</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "   PassengerId  Survived  Pclass  \\\n",
       "0            1       0.0       3   \n",
       "1            2       1.0       1   \n",
       "2            3       1.0       3   \n",
       "3            4       1.0       1   \n",
       "4            5       0.0       3   \n",
       "\n",
       "                                                Name  Sex   Age  SibSp  Parch  \\\n",
       "0                            Braund, Mr. Owen Harris    0  22.0      1      0   \n",
       "1  Cumings, Mrs. John Bradley (Florence Briggs Th...    1  38.0      1      0   \n",
       "2                             Heikkinen, Miss. Laina    1  26.0      0      0   \n",
       "3       Futrelle, Mrs. Jacques Heath (Lily May Peel)    1  35.0      1      0   \n",
       "4                           Allen, Mr. William Henry    0  35.0      0      0   \n",
       "\n",
       "             Ticket     Fare Cabin  Embarked  FamilySize  IsAlone  \n",
       "0         A/5 21171   7.2500   NaN         0           2        0  \n",
       "1          PC 17599  71.2833   C85         1           2        0  \n",
       "2  STON/O2. 3101282   7.9250   NaN         0           1        1  \n",
       "3            113803  53.1000  C123         0           2        0  \n",
       "4            373450   8.0500   NaN         0           1        1  "
      ]
     },
     "execution_count": 2,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "data.head()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 3,
   "metadata": {},
   "outputs": [],
   "source": [
    "delete_columns = ['Name', 'PassengerId', 'Ticket', 'Cabin']\n",
    "data.drop(delete_columns, axis=1, inplace=True)\n",
    "\n",
    "train = data[:len(train)]\n",
    "test = data[len(train):]\n",
    "\n",
    "y_train = train['Survived']\n",
    "X_train = train.drop('Survived', axis=1)\n",
    "X_test = test.drop('Survived', axis=1)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 4,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>Pclass</th>\n",
       "      <th>Sex</th>\n",
       "      <th>Age</th>\n",
       "      <th>SibSp</th>\n",
       "      <th>Parch</th>\n",
       "      <th>Fare</th>\n",
       "      <th>Embarked</th>\n",
       "      <th>FamilySize</th>\n",
       "      <th>IsAlone</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>3</td>\n",
       "      <td>0</td>\n",
       "      <td>22.0</td>\n",
       "      <td>1</td>\n",
       "      <td>0</td>\n",
       "      <td>7.2500</td>\n",
       "      <td>0</td>\n",
       "      <td>2</td>\n",
       "      <td>0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>1</td>\n",
       "      <td>1</td>\n",
       "      <td>38.0</td>\n",
       "      <td>1</td>\n",
       "      <td>0</td>\n",
       "      <td>71.2833</td>\n",
       "      <td>1</td>\n",
       "      <td>2</td>\n",
       "      <td>0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2</th>\n",
       "      <td>3</td>\n",
       "      <td>1</td>\n",
       "      <td>26.0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>7.9250</td>\n",
       "      <td>0</td>\n",
       "      <td>1</td>\n",
       "      <td>1</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3</th>\n",
       "      <td>1</td>\n",
       "      <td>1</td>\n",
       "      <td>35.0</td>\n",
       "      <td>1</td>\n",
       "      <td>0</td>\n",
       "      <td>53.1000</td>\n",
       "      <td>0</td>\n",
       "      <td>2</td>\n",
       "      <td>0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4</th>\n",
       "      <td>3</td>\n",
       "      <td>0</td>\n",
       "      <td>35.0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>8.0500</td>\n",
       "      <td>0</td>\n",
       "      <td>1</td>\n",
       "      <td>1</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "   Pclass  Sex   Age  SibSp  Parch     Fare  Embarked  FamilySize  IsAlone\n",
       "0       3    0  22.0      1      0   7.2500         0           2        0\n",
       "1       1    1  38.0      1      0  71.2833         1           2        0\n",
       "2       3    1  26.0      0      0   7.9250         0           1        1\n",
       "3       1    1  35.0      1      0  53.1000         0           2        0\n",
       "4       3    0  35.0      0      0   8.0500         0           1        1"
      ]
     },
     "execution_count": 4,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "X_train.head()"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# ホールドアウト検証"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 5,
   "metadata": {},
   "outputs": [],
   "source": [
    "from sklearn.model_selection import train_test_split\n",
    "\n",
    "\n",
    "X_train, X_valid, y_train, y_valid = train_test_split(X_train, y_train, test_size=0.3, random_state=0, stratify=y_train)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 6,
   "metadata": {},
   "outputs": [],
   "source": [
    "categorical_features = ['Embarked', 'Pclass', 'Sex']"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 7,
   "metadata": {},
   "outputs": [],
   "source": [
    "params = {\n",
    "    'objective': 'binary',\n",
    "    'max_bin': 300,\n",
    "    'learning_rate': 0.05,\n",
    "    'num_leaves': 40\n",
    "}"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 8,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Training until validation scores don't improve for 10 rounds\n",
      "[10]\ttraining's binary_logloss: 0.505699\tvalid_1's binary_logloss: 0.532106\n",
      "[20]\ttraining's binary_logloss: 0.427825\tvalid_1's binary_logloss: 0.482279\n",
      "[30]\ttraining's binary_logloss: 0.377242\tvalid_1's binary_logloss: 0.456641\n",
      "[40]\ttraining's binary_logloss: 0.345424\tvalid_1's binary_logloss: 0.447083\n",
      "[50]\ttraining's binary_logloss: 0.323113\tvalid_1's binary_logloss: 0.440407\n",
      "[60]\ttraining's binary_logloss: 0.302727\tvalid_1's binary_logloss: 0.434527\n",
      "[70]\ttraining's binary_logloss: 0.285597\tvalid_1's binary_logloss: 0.434932\n",
      "Early stopping, best iteration is:\n",
      "[66]\ttraining's binary_logloss: 0.293072\tvalid_1's binary_logloss: 0.433251\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "/opt/conda/lib/python3.6/site-packages/lightgbm/basic.py:1243: UserWarning: Using categorical_feature in Dataset.\n",
      "  warnings.warn('Using categorical_feature in Dataset.')\n"
     ]
    }
   ],
   "source": [
    "import lightgbm as lgb\n",
    "\n",
    "\n",
    "lgb_train = lgb.Dataset(X_train, y_train,\n",
    "                                         categorical_feature=categorical_features)\n",
    "lgb_eval = lgb.Dataset(X_valid, y_valid, reference=lgb_train,\n",
    "                                        categorical_feature=categorical_features)\n",
    "\n",
    "model = lgb.train(params, lgb_train,\n",
    "                               valid_sets=[lgb_train, lgb_eval],\n",
    "                               verbose_eval=10,\n",
    "                               num_boost_round=1000,\n",
    "                               early_stopping_rounds=10)\n",
    "\n",
    "y_pred = model.predict(X_test, num_iteration=model.best_iteration)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 9,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "array([0.03605598, 0.40306884, 0.10732166, 0.0802399 , 0.46011271,\n",
       "       0.20222002, 0.64929492, 0.11896033, 0.7452973 , 0.01917651])"
      ]
     },
     "execution_count": 9,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "y_pred[:10]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 10,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "array([0, 0, 0, 0, 0, 0, 1, 0, 1, 0])"
      ]
     },
     "execution_count": 10,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "y_pred = (y_pred > 0.5).astype(int)\n",
    "y_pred[:10]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 11,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>PassengerId</th>\n",
       "      <th>Survived</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>892</td>\n",
       "      <td>0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>893</td>\n",
       "      <td>0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2</th>\n",
       "      <td>894</td>\n",
       "      <td>0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3</th>\n",
       "      <td>895</td>\n",
       "      <td>0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4</th>\n",
       "      <td>896</td>\n",
       "      <td>0</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "   PassengerId  Survived\n",
       "0          892         0\n",
       "1          893         0\n",
       "2          894         0\n",
       "3          895         0\n",
       "4          896         0"
      ]
     },
     "execution_count": 11,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "sub = pd.read_csv('../input/titanic/gender_submission.csv')\n",
    "\n",
    "sub['Survived'] = y_pred\n",
    "sub.to_csv('submission_lightgbm_holdout.csv', index=False)\n",
    "\n",
    "sub.head()"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# 交差検証（Cross Validation）"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 12,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>Pclass</th>\n",
       "      <th>Sex</th>\n",
       "      <th>Age</th>\n",
       "      <th>SibSp</th>\n",
       "      <th>Parch</th>\n",
       "      <th>Fare</th>\n",
       "      <th>Embarked</th>\n",
       "      <th>FamilySize</th>\n",
       "      <th>IsAlone</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>3</td>\n",
       "      <td>0</td>\n",
       "      <td>22.0</td>\n",
       "      <td>1</td>\n",
       "      <td>0</td>\n",
       "      <td>7.2500</td>\n",
       "      <td>0</td>\n",
       "      <td>2</td>\n",
       "      <td>0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>1</td>\n",
       "      <td>1</td>\n",
       "      <td>38.0</td>\n",
       "      <td>1</td>\n",
       "      <td>0</td>\n",
       "      <td>71.2833</td>\n",
       "      <td>1</td>\n",
       "      <td>2</td>\n",
       "      <td>0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2</th>\n",
       "      <td>3</td>\n",
       "      <td>1</td>\n",
       "      <td>26.0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>7.9250</td>\n",
       "      <td>0</td>\n",
       "      <td>1</td>\n",
       "      <td>1</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3</th>\n",
       "      <td>1</td>\n",
       "      <td>1</td>\n",
       "      <td>35.0</td>\n",
       "      <td>1</td>\n",
       "      <td>0</td>\n",
       "      <td>53.1000</td>\n",
       "      <td>0</td>\n",
       "      <td>2</td>\n",
       "      <td>0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4</th>\n",
       "      <td>3</td>\n",
       "      <td>0</td>\n",
       "      <td>35.0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>8.0500</td>\n",
       "      <td>0</td>\n",
       "      <td>1</td>\n",
       "      <td>1</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "   Pclass  Sex   Age  SibSp  Parch     Fare  Embarked  FamilySize  IsAlone\n",
       "0       3    0  22.0      1      0   7.2500         0           2        0\n",
       "1       1    1  38.0      1      0  71.2833         1           2        0\n",
       "2       3    1  26.0      0      0   7.9250         0           1        1\n",
       "3       1    1  35.0      1      0  53.1000         0           2        0\n",
       "4       3    0  35.0      0      0   8.0500         0           1        1"
      ]
     },
     "execution_count": 12,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "train = pd.read_csv('../input/titanic/train.csv')\n",
    "test = pd.read_csv('../input/titanic/test.csv')\n",
    "gender_submission = pd.read_csv('../input/titanic/gender_submission.csv')\n",
    "\n",
    "data = pd.concat([train, test], sort=False)\n",
    "\n",
    "data['Sex'].replace(['male', 'female'], [0, 1], inplace=True)\n",
    "data['Embarked'].fillna(('S'), inplace=True)\n",
    "data['Embarked'] = data['Embarked'].map({'S': 0, 'C': 1, 'Q': 2}).astype(int)\n",
    "data['Fare'].fillna(np.mean(data['Fare']), inplace=True)\n",
    "data['Age'].fillna(data['Age'].median(), inplace=True)\n",
    "data['FamilySize'] = data['Parch'] + data['SibSp'] + 1\n",
    "data['IsAlone'] = 0\n",
    "data.loc[data['FamilySize'] == 1, 'IsAlone'] = 1\n",
    "\n",
    "delete_columns = ['Name', 'PassengerId', 'Ticket', 'Cabin']\n",
    "data.drop(delete_columns, axis=1, inplace=True)\n",
    "\n",
    "train = data[:len(train)]\n",
    "test = data[len(train):]\n",
    "\n",
    "y_train = train['Survived']\n",
    "X_train = train.drop('Survived', axis=1)\n",
    "X_test = test.drop('Survived', axis=1)\n",
    "\n",
    "X_train.head()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 13,
   "metadata": {},
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "/opt/conda/lib/python3.6/site-packages/lightgbm/basic.py:1243: UserWarning: Using categorical_feature in Dataset.\n",
      "  warnings.warn('Using categorical_feature in Dataset.')\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Training until validation scores don't improve for 10 rounds\n",
      "[10]\ttraining's binary_logloss: 0.506339\tvalid_1's binary_logloss: 0.516198\n",
      "[20]\ttraining's binary_logloss: 0.428668\tvalid_1's binary_logloss: 0.446596\n",
      "[30]\ttraining's binary_logloss: 0.383412\tvalid_1's binary_logloss: 0.411961\n",
      "[40]\ttraining's binary_logloss: 0.35367\tvalid_1's binary_logloss: 0.397122\n",
      "[50]\ttraining's binary_logloss: 0.329451\tvalid_1's binary_logloss: 0.391041\n",
      "[60]\ttraining's binary_logloss: 0.307092\tvalid_1's binary_logloss: 0.38325\n",
      "[70]\ttraining's binary_logloss: 0.290771\tvalid_1's binary_logloss: 0.377067\n",
      "[80]\ttraining's binary_logloss: 0.274935\tvalid_1's binary_logloss: 0.373918\n",
      "[90]\ttraining's binary_logloss: 0.260335\tvalid_1's binary_logloss: 0.370602\n",
      "Early stopping, best iteration is:\n",
      "[85]\ttraining's binary_logloss: 0.267203\tvalid_1's binary_logloss: 0.369116\n",
      "Training until validation scores don't improve for 10 rounds\n",
      "[10]\ttraining's binary_logloss: 0.504211\tvalid_1's binary_logloss: 0.534594\n",
      "[20]\ttraining's binary_logloss: 0.422236\tvalid_1's binary_logloss: 0.481243\n",
      "[30]\ttraining's binary_logloss: 0.37173\tvalid_1's binary_logloss: 0.459512\n",
      "[40]\ttraining's binary_logloss: 0.341914\tvalid_1's binary_logloss: 0.453871\n",
      "[50]\ttraining's binary_logloss: 0.318699\tvalid_1's binary_logloss: 0.451806\n",
      "[60]\ttraining's binary_logloss: 0.296894\tvalid_1's binary_logloss: 0.449685\n",
      "Early stopping, best iteration is:\n",
      "[58]\ttraining's binary_logloss: 0.301388\tvalid_1's binary_logloss: 0.449112\n",
      "Training until validation scores don't improve for 10 rounds\n",
      "[10]\ttraining's binary_logloss: 0.506747\tvalid_1's binary_logloss: 0.507713\n",
      "[20]\ttraining's binary_logloss: 0.42895\tvalid_1's binary_logloss: 0.449278\n",
      "[30]\ttraining's binary_logloss: 0.384356\tvalid_1's binary_logloss: 0.41254\n",
      "[40]\ttraining's binary_logloss: 0.355218\tvalid_1's binary_logloss: 0.396228\n",
      "[50]\ttraining's binary_logloss: 0.333925\tvalid_1's binary_logloss: 0.387799\n",
      "[60]\ttraining's binary_logloss: 0.313799\tvalid_1's binary_logloss: 0.385787\n",
      "[70]\ttraining's binary_logloss: 0.298536\tvalid_1's binary_logloss: 0.38534\n",
      "[80]\ttraining's binary_logloss: 0.283249\tvalid_1's binary_logloss: 0.383338\n",
      "[90]\ttraining's binary_logloss: 0.270355\tvalid_1's binary_logloss: 0.385343\n",
      "Early stopping, best iteration is:\n",
      "[80]\ttraining's binary_logloss: 0.283249\tvalid_1's binary_logloss: 0.383338\n",
      "Training until validation scores don't improve for 10 rounds\n",
      "[10]\ttraining's binary_logloss: 0.497977\tvalid_1's binary_logloss: 0.525947\n",
      "[20]\ttraining's binary_logloss: 0.416873\tvalid_1's binary_logloss: 0.471473\n",
      "[30]\ttraining's binary_logloss: 0.368347\tvalid_1's binary_logloss: 0.446558\n",
      "[40]\ttraining's binary_logloss: 0.338844\tvalid_1's binary_logloss: 0.437886\n",
      "[50]\ttraining's binary_logloss: 0.318398\tvalid_1's binary_logloss: 0.437554\n",
      "Early stopping, best iteration is:\n",
      "[41]\ttraining's binary_logloss: 0.336795\tvalid_1's binary_logloss: 0.437121\n",
      "Training until validation scores don't improve for 10 rounds\n",
      "[10]\ttraining's binary_logloss: 0.494316\tvalid_1's binary_logloss: 0.552225\n",
      "[20]\ttraining's binary_logloss: 0.417327\tvalid_1's binary_logloss: 0.487758\n",
      "[30]\ttraining's binary_logloss: 0.370965\tvalid_1's binary_logloss: 0.453038\n",
      "[40]\ttraining's binary_logloss: 0.341695\tvalid_1's binary_logloss: 0.439283\n",
      "[50]\ttraining's binary_logloss: 0.319843\tvalid_1's binary_logloss: 0.4351\n",
      "[60]\ttraining's binary_logloss: 0.301123\tvalid_1's binary_logloss: 0.435584\n",
      "Early stopping, best iteration is:\n",
      "[51]\ttraining's binary_logloss: 0.31821\tvalid_1's binary_logloss: 0.4347\n"
     ]
    }
   ],
   "source": [
    "from sklearn.model_selection import KFold\n",
    "\n",
    "\n",
    "y_preds = []\n",
    "models = []\n",
    "oof_train = np.zeros((len(X_train),))\n",
    "cv = KFold(n_splits=5, shuffle=True, random_state=0)\n",
    "\n",
    "categorical_features = ['Embarked', 'Pclass', 'Sex']\n",
    "\n",
    "params = {\n",
    "    'objective': 'binary',\n",
    "    'max_bin': 300,\n",
    "    'learning_rate': 0.05,\n",
    "    'num_leaves': 40\n",
    "}\n",
    "\n",
    "for fold_id, (train_index, valid_index) in enumerate(cv.split(X_train)):\n",
    "    X_tr = X_train.loc[train_index, :]\n",
    "    X_val = X_train.loc[valid_index, :]\n",
    "    y_tr = y_train[train_index]\n",
    "    y_val = y_train[valid_index]\n",
    "\n",
    "    lgb_train = lgb.Dataset(X_tr, y_tr,\n",
    "                                             categorical_feature=categorical_features)\n",
    "    lgb_eval = lgb.Dataset(X_val, y_val, reference=lgb_train,\n",
    "                                            categorical_feature=categorical_features)\n",
    "\n",
    "    model = lgb.train(params, lgb_train,\n",
    "                                   valid_sets=[lgb_train, lgb_eval],\n",
    "                                   verbose_eval=10,\n",
    "                                   num_boost_round=1000,\n",
    "                                   early_stopping_rounds=10)\n",
    "\n",
    "\n",
    "    oof_train[valid_index] = model.predict(X_val, num_iteration=model.best_iteration)\n",
    "    y_pred = model.predict(X_test, num_iteration=model.best_iteration)\n",
    "\n",
    "    y_preds.append(y_pred)\n",
    "    models.append(model)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 14,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "===CV scores===\n",
      "[0.36911611932674965, 0.4491122965802196, 0.3833384988458873, 0.43712149656630844, 0.43469994547894103]\n",
      "0.41467767135962114\n"
     ]
    }
   ],
   "source": [
    "pd.DataFrame(oof_train).to_csv('oof_train_kfold.csv', index=False)\n",
    "\n",
    "scores = [\n",
    "    m.best_score['valid_1']['binary_logloss'] for m in models\n",
    "]\n",
    "score = sum(scores) / len(scores)\n",
    "print('===CV scores===')\n",
    "print(scores)\n",
    "print(score)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 15,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "0.8226711560044894"
      ]
     },
     "execution_count": 15,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "from sklearn.metrics import accuracy_score\n",
    "\n",
    "\n",
    "y_pred_oof = (oof_train > 0.5).astype(int)\n",
    "accuracy_score(y_train, y_pred_oof)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 16,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "5"
      ]
     },
     "execution_count": 16,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "len(y_preds)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 17,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "array([0.07120088, 0.18601086, 0.02400096, 0.40979176, 0.40890152,\n",
       "       0.4906518 , 0.580799  , 0.21157361, 0.82465467, 0.01063327])"
      ]
     },
     "execution_count": 17,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "y_preds[0][:10]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 18,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "array([0, 0, 0, 0, 0, 0, 1, 0, 1, 0])"
      ]
     },
     "execution_count": 18,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "y_sub = sum(y_preds) / len(y_preds)\n",
    "y_sub = (y_sub > 0.5).astype(int)\n",
    "y_sub[:10]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 19,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>PassengerId</th>\n",
       "      <th>Survived</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>892</td>\n",
       "      <td>0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>893</td>\n",
       "      <td>0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2</th>\n",
       "      <td>894</td>\n",
       "      <td>0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3</th>\n",
       "      <td>895</td>\n",
       "      <td>0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4</th>\n",
       "      <td>896</td>\n",
       "      <td>0</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "   PassengerId  Survived\n",
       "0          892         0\n",
       "1          893         0\n",
       "2          894         0\n",
       "3          895         0\n",
       "4          896         0"
      ]
     },
     "execution_count": 19,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "sub['Survived'] = y_sub\n",
    "sub.to_csv('submission_lightgbm_kfold.csv', index=False)\n",
    "\n",
    "sub.head()"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# データセットの分割方法"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 20,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "fold_id: 0\n",
      "y_tr y==1 rate: 0.38342696629213485\n",
      "y_val y==1 rate: 0.3854748603351955\n",
      "fold_id: 1\n",
      "y_tr y==1 rate: 0.3856942496493689\n",
      "y_val y==1 rate: 0.37640449438202245\n",
      "fold_id: 2\n",
      "y_tr y==1 rate: 0.39831697054698456\n",
      "y_val y==1 rate: 0.3258426966292135\n",
      "fold_id: 3\n",
      "y_tr y==1 rate: 0.3856942496493689\n",
      "y_val y==1 rate: 0.37640449438202245\n",
      "fold_id: 4\n",
      "y_tr y==1 rate: 0.36605890603085556\n",
      "y_val y==1 rate: 0.4550561797752809\n"
     ]
    }
   ],
   "source": [
    "from sklearn.model_selection import KFold\n",
    "\n",
    "\n",
    "cv = KFold(n_splits=5, shuffle=True, random_state=0)\n",
    "for fold_id, (train_index, valid_index) in enumerate(cv.split(X_train)):\n",
    "    X_tr = X_train.loc[train_index, :]\n",
    "    X_val = X_train.loc[valid_index, :]\n",
    "    y_tr = y_train[train_index]\n",
    "    y_val = y_train[valid_index]\n",
    "\n",
    "    print(f'fold_id: {fold_id}')\n",
    "    print(f'y_tr y==1 rate: {sum(y_tr)/len(y_tr)}')\n",
    "    print(f'y_val y==1 rate: {sum(y_val)/len(y_val)}')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 21,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "fold_id: 0\n",
      "y_tr y==1 rate: 0.38342696629213485\n",
      "y_val y==1 rate: 0.3854748603351955\n",
      "fold_id: 1\n",
      "y_tr y==1 rate: 0.38342696629213485\n",
      "y_val y==1 rate: 0.3854748603351955\n",
      "fold_id: 2\n",
      "y_tr y==1 rate: 0.38429172510518933\n",
      "y_val y==1 rate: 0.38202247191011235\n",
      "fold_id: 3\n",
      "y_tr y==1 rate: 0.38429172510518933\n",
      "y_val y==1 rate: 0.38202247191011235\n",
      "fold_id: 4\n",
      "y_tr y==1 rate: 0.38375350140056025\n",
      "y_val y==1 rate: 0.384180790960452\n"
     ]
    }
   ],
   "source": [
    "from sklearn.model_selection import StratifiedKFold\n",
    "\n",
    "\n",
    "cv = StratifiedKFold(n_splits=5, shuffle=True, random_state=0)\n",
    "for fold_id, (train_index, valid_index) in enumerate(cv.split(X_train, y_train)):\n",
    "    X_tr = X_train.loc[train_index, :]\n",
    "    X_val = X_train.loc[valid_index, :]\n",
    "    y_tr = y_train[train_index]\n",
    "    y_val = y_train[valid_index]\n",
    "\n",
    "    print(f'fold_id: {fold_id}')\n",
    "    print(f'y_tr y==1 rate: {sum(y_tr)/len(y_tr)}')\n",
    "    print(f'y_val y==1 rate: {sum(y_val)/len(y_val)}')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 22,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Training until validation scores don't improve for 10 rounds\n",
      "[10]\ttraining's binary_logloss: 0.497173\tvalid_1's binary_logloss: 0.548361\n",
      "[20]\ttraining's binary_logloss: 0.415497\tvalid_1's binary_logloss: 0.495272\n",
      "[30]\ttraining's binary_logloss: 0.368839\tvalid_1's binary_logloss: 0.469941\n",
      "[40]\ttraining's binary_logloss: 0.337704\tvalid_1's binary_logloss: 0.455887\n",
      "[50]\ttraining's binary_logloss: 0.31511\tvalid_1's binary_logloss: 0.448996\n",
      "[60]\ttraining's binary_logloss: 0.294249\tvalid_1's binary_logloss: 0.454425\n",
      "Early stopping, best iteration is:\n",
      "[50]\ttraining's binary_logloss: 0.31511\tvalid_1's binary_logloss: 0.448996\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "/opt/conda/lib/python3.6/site-packages/lightgbm/basic.py:1243: UserWarning: Using categorical_feature in Dataset.\n",
      "  warnings.warn('Using categorical_feature in Dataset.')\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Training until validation scores don't improve for 10 rounds\n",
      "[10]\ttraining's binary_logloss: 0.503723\tvalid_1's binary_logloss: 0.512133\n",
      "[20]\ttraining's binary_logloss: 0.427557\tvalid_1's binary_logloss: 0.451233\n",
      "[30]\ttraining's binary_logloss: 0.382201\tvalid_1's binary_logloss: 0.413801\n",
      "[40]\ttraining's binary_logloss: 0.352641\tvalid_1's binary_logloss: 0.400884\n",
      "[50]\ttraining's binary_logloss: 0.330781\tvalid_1's binary_logloss: 0.397443\n",
      "[60]\ttraining's binary_logloss: 0.311659\tvalid_1's binary_logloss: 0.396539\n",
      "Early stopping, best iteration is:\n",
      "[55]\ttraining's binary_logloss: 0.320785\tvalid_1's binary_logloss: 0.394951\n",
      "Training until validation scores don't improve for 10 rounds\n",
      "[10]\ttraining's binary_logloss: 0.505627\tvalid_1's binary_logloss: 0.512659\n",
      "[20]\ttraining's binary_logloss: 0.427425\tvalid_1's binary_logloss: 0.445498\n",
      "[30]\ttraining's binary_logloss: 0.379715\tvalid_1's binary_logloss: 0.411298\n",
      "[40]\ttraining's binary_logloss: 0.348551\tvalid_1's binary_logloss: 0.395933\n",
      "[50]\ttraining's binary_logloss: 0.324856\tvalid_1's binary_logloss: 0.38959\n",
      "[60]\ttraining's binary_logloss: 0.304612\tvalid_1's binary_logloss: 0.383849\n",
      "[70]\ttraining's binary_logloss: 0.28703\tvalid_1's binary_logloss: 0.384518\n",
      "Early stopping, best iteration is:\n",
      "[64]\ttraining's binary_logloss: 0.298425\tvalid_1's binary_logloss: 0.382113\n",
      "Training until validation scores don't improve for 10 rounds\n",
      "[10]\ttraining's binary_logloss: 0.501523\tvalid_1's binary_logloss: 0.536382\n",
      "[20]\ttraining's binary_logloss: 0.420578\tvalid_1's binary_logloss: 0.482229\n",
      "[30]\ttraining's binary_logloss: 0.377598\tvalid_1's binary_logloss: 0.461077\n",
      "[40]\ttraining's binary_logloss: 0.349149\tvalid_1's binary_logloss: 0.448919\n",
      "[50]\ttraining's binary_logloss: 0.32552\tvalid_1's binary_logloss: 0.440548\n",
      "[60]\ttraining's binary_logloss: 0.305819\tvalid_1's binary_logloss: 0.435093\n",
      "[70]\ttraining's binary_logloss: 0.288929\tvalid_1's binary_logloss: 0.435729\n",
      "Early stopping, best iteration is:\n",
      "[65]\ttraining's binary_logloss: 0.297083\tvalid_1's binary_logloss: 0.433996\n",
      "Training until validation scores don't improve for 10 rounds\n",
      "[10]\ttraining's binary_logloss: 0.497914\tvalid_1's binary_logloss: 0.534421\n",
      "[20]\ttraining's binary_logloss: 0.417822\tvalid_1's binary_logloss: 0.489135\n",
      "[30]\ttraining's binary_logloss: 0.368492\tvalid_1's binary_logloss: 0.466391\n",
      "[40]\ttraining's binary_logloss: 0.337723\tvalid_1's binary_logloss: 0.458283\n",
      "[50]\ttraining's binary_logloss: 0.316279\tvalid_1's binary_logloss: 0.461188\n",
      "Early stopping, best iteration is:\n",
      "[40]\ttraining's binary_logloss: 0.337723\tvalid_1's binary_logloss: 0.458283\n"
     ]
    }
   ],
   "source": [
    "from sklearn.model_selection import StratifiedKFold\n",
    "\n",
    "\n",
    "y_preds = []\n",
    "models = []\n",
    "oof_train = np.zeros((len(X_train),))\n",
    "cv = StratifiedKFold(n_splits=5, shuffle=True, random_state=0)\n",
    "\n",
    "categorical_features = ['Embarked', 'Pclass', 'Sex']\n",
    "\n",
    "params = {\n",
    "    'objective': 'binary',\n",
    "    'max_bin': 300,\n",
    "    'learning_rate': 0.05,\n",
    "    'num_leaves': 40\n",
    "}\n",
    "\n",
    "for fold_id, (train_index, valid_index) in enumerate(cv.split(X_train, y_train)):\n",
    "    X_tr = X_train.loc[train_index, :]\n",
    "    X_val = X_train.loc[valid_index, :]\n",
    "    y_tr = y_train[train_index]\n",
    "    y_val = y_train[valid_index]\n",
    "\n",
    "    lgb_train = lgb.Dataset(X_tr, y_tr,\n",
    "                                             categorical_feature=categorical_features)\n",
    "    lgb_eval = lgb.Dataset(X_val, y_val, reference=lgb_train,\n",
    "                                            categorical_feature=categorical_features)\n",
    "\n",
    "    model = lgb.train(params, lgb_train,\n",
    "                                   valid_sets=[lgb_train, lgb_eval],\n",
    "                                   verbose_eval=10,\n",
    "                                   num_boost_round=1000,\n",
    "                                   early_stopping_rounds=10)\n",
    "\n",
    "    oof_train[valid_index] = model.predict(X_val, num_iteration=model.best_iteration)\n",
    "    y_pred = model.predict(X_test, num_iteration=model.best_iteration)\n",
    "\n",
    "    y_preds.append(y_pred)\n",
    "    models.append(model)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 23,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "[0.12960226 0.97051442 0.21027611 0.97354071 0.14069274 0.14504993\n",
      " 0.12983479 0.35898845 0.48175438 0.94465218]\n",
      "===CV scores===\n",
      "[0.4489959713813429, 0.39495136241745693, 0.38211327231368947, 0.43399642500840246, 0.4582826654642754]\n",
      "0.4236679393170334\n"
     ]
    }
   ],
   "source": [
    "pd.DataFrame(oof_train).to_csv('oof_train_skfold.csv', index=False)\n",
    "print(oof_train[:10])\n",
    "\n",
    "scores = [\n",
    "    m.best_score['valid_1']['binary_logloss'] for m in models\n",
    "]\n",
    "score = sum(scores) / len(scores)\n",
    "print('===CV scores===')\n",
    "print(scores)\n",
    "print(score)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 24,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "0.8092031425364759"
      ]
     },
     "execution_count": 24,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "from sklearn.metrics import accuracy_score\n",
    "\n",
    "\n",
    "y_pred_oof = (oof_train > 0.5).astype(int)\n",
    "accuracy_score(y_train, y_pred_oof)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 25,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "array([0, 0, 0, 0, 0, 0, 1, 0, 1, 0])"
      ]
     },
     "execution_count": 25,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "y_sub = sum(y_preds) / len(y_preds)\n",
    "y_sub = (y_sub > 0.5).astype(int)\n",
    "y_sub[:10]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 26,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>PassengerId</th>\n",
       "      <th>Survived</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>892</td>\n",
       "      <td>0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>893</td>\n",
       "      <td>0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2</th>\n",
       "      <td>894</td>\n",
       "      <td>0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3</th>\n",
       "      <td>895</td>\n",
       "      <td>0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4</th>\n",
       "      <td>896</td>\n",
       "      <td>0</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "   PassengerId  Survived\n",
       "0          892         0\n",
       "1          893         0\n",
       "2          894         0\n",
       "3          895         0\n",
       "4          896         0"
      ]
     },
     "execution_count": 26,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "sub['Survived'] = y_sub\n",
    "sub.to_csv('submission_lightgbm_skfold.csv', index=False)\n",
    "\n",
    "sub.head()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  }
 ],
 "metadata": {
  "file_extension": ".py",
  "kernelspec": {
   "display_name": "Python 3",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.6.6"
  },
  "mimetype": "text/x-python",
  "name": "python",
  "npconvert_exporter": "python",
  "pygments_lexer": "ipython3",
  "version": 3
 },
 "nbformat": 4,
 "nbformat_minor": 2
}