Learning.ipynb

{
 "cells": [
  {
   "cell_type": "code",
   "execution_count": 1,
   "metadata": {
    "collapsed": false,
    "scrolled": true
   },
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "C:\\Anaconda3\\lib\\site-packages\\sklearn\\cross_validation.py:44: DeprecationWarning: This module was deprecated in version 0.18 in favor of the model_selection module into which all the refactored classes and functions are moved. Also note that the interface of the new CV iterators are different from that of this module. This module will be removed in 0.20.\n",
      "  \"This module will be removed in 0.20.\", DeprecationWarning)\n"
     ]
    }
   ],
   "source": [
    "import numpy as np\n",
    "import pandas as pd\n",
    "import string\n",
    "from sklearn.feature_extraction.text import TfidfVectorizer, CountVectorizer\n",
    "from scipy.stats import boxcox\n",
    "from scipy import sparse\n",
    "from sklearn.model_selection import StratifiedKFold\n",
    "from itertools import product\n",
    "import xgboost as xgb\n",
    "from sklearn import preprocessing\n",
    "\n",
    "from sklearn import cross_validation\n",
    "from sklearn.cross_validation import KFold\n",
    "from sklearn.metrics import log_loss\n",
    "import lightgbm as lgbm"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# Kaggle 제출 용"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "#### 일반 데이터 셋"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 10,
   "metadata": {
    "collapsed": false,
    "scrolled": true
   },
   "outputs": [],
   "source": [
    "x_train = pd.read_csv('x_train.csv')\n",
    "x_test = pd.read_csv('x_test.csv')\n",
    "y_train = pd.read_csv('y_train.csv')\n",
    "y_train.interest_level=y_train.interest_level.astype(int)\n",
    "listing_id_df = pd.read_csv('test_listing_id.csv')\n",
    "listing_id = listing_id_df['listing_id'].values"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "#### 프라이스 데이터 셋"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 4,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "x_train = pd.read_csv('x_train_price.csv')\n",
    "x_test = pd.read_csv('x_test_price.csv')\n",
    "y_train = pd.read_csv('y_train_price.csv')\n",
    "y_train.interest_level=y_train.interest_level.astype(int)\n",
    "listing_id_df = pd.read_csv('test_listing_id_price.csv')\n",
    "listing_id = listing_id_df['listing_id'].values"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "#### v0.1 DataSet"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 26,
   "metadata": {
    "collapsed": false,
    "scrolled": true
   },
   "outputs": [],
   "source": [
    "x_train = pd.read_csv('x_train_v0.1.csv')\n",
    "x_test = pd.read_csv('x_test_v0.1.csv')\n",
    "y_train = pd.read_csv('y_train_v0.1.csv')\n",
    "y_train.interest_level=y_train.interest_level.astype(int)\n",
    "listing_id_df = pd.read_csv('test_listing_id_v0.1.csv')\n",
    "listing_id = listing_id_df['listing_id'].values"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "#### 7번째 data set에 building ID만 채운 것"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 9,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "x_train = pd.read_csv('x_train_new_seventh.csv')\n",
    "x_test = pd.read_csv('x_test_new_seventh.csv')\n",
    "y_train = pd.read_csv('y_train_new_seventh.csv')\n",
    "y_train.interest_level=y_train.interest_level.astype(int)\n",
    "listing_id_df = pd.read_csv('test_listing_new_seventh.csv')\n",
    "listing_id = listing_id_df['listing_id'].values"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "#### 7번째 data set"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 12,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "x_train = pd.read_csv('x_train_seventh.csv')\n",
    "x_test = pd.read_csv('x_test_seventh.csv')\n",
    "y_train = pd.read_csv('y_train_seventh.csv')\n",
    "y_train.interest_level=y_train.interest_level.astype(int)\n",
    "listing_id_df = pd.read_csv('test_listing_seventh.csv')\n",
    "listing_id = listing_id_df['listing_id'].values"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 24,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "x_train = pd.read_csv('x_train_seventh_fillna.csv')\n",
    "x_test = pd.read_csv('x_test_seventh_fillna.csv')\n",
    "y_train = pd.read_csv('y_train_seventh_fillna.csv')\n",
    "y_train.interest_level=y_train.interest_level.astype(int)\n",
    "listing_id_df = pd.read_csv('test_listing_seventh_fillna.csv')\n",
    "listing_id = listing_id_df['listing_id'].values"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "#### 8번째 데이터셋"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 4,
   "metadata": {
    "collapsed": false
   },
   "outputs": [],
   "source": [
    "x_train = pd.read_csv('x_train_eight.csv')\n",
    "x_test = pd.read_csv('x_test_eight.csv')\n",
    "y_train = pd.read_csv('y_train_eight.csv')\n",
    "y_train.interest_level=y_train.interest_level.astype(int)\n",
    "listing_id_df = pd.read_csv('test_listing_eight.csv')\n",
    "listing_id = listing_id_df['listing_id'].values"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "#### 9번째 데이터셋"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 79,
   "metadata": {
    "collapsed": false
   },
   "outputs": [],
   "source": [
    "x_train = pd.read_csv('x_train_nineth.csv')\n",
    "x_test = pd.read_csv('x_test_nineth.csv')\n",
    "y_train = pd.read_csv('y_train_nineth.csv')\n",
    "y_train.interest_level=y_train.interest_level.astype(int)\n",
    "listing_id_df = pd.read_csv('test_listing_nineth.csv')\n",
    "listing_id = listing_id_df['listing_id'].values"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "#### 10번째 데이터셋"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 202,
   "metadata": {
    "collapsed": false
   },
   "outputs": [],
   "source": [
    "x_train = pd.read_csv('x_train_tenth.csv')\n",
    "x_test = pd.read_csv('x_test_tenth.csv')\n",
    "y_train = pd.read_csv('y_train_tenth.csv')\n",
    "y_train.interest_level=y_train.interest_level.astype(int)\n",
    "listing_id_df = pd.read_csv('test_listing_tenth.csv')\n",
    "listing_id = listing_id_df['listing_id'].values"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "#### 12번째 데이터셋"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 6,
   "metadata": {
    "collapsed": false
   },
   "outputs": [],
   "source": [
    "x_train = pd.read_csv('x_train_12th.csv')\n",
    "x_test = pd.read_csv('x_test_12th.csv')\n",
    "y_train = pd.read_csv('y_train_12th.csv')\n",
    "y_train.interest_level=y_train.interest_level.astype(int)\n",
    "listing_id_df = pd.read_csv('test_listing_12th.csv')\n",
    "listing_id = listing_id_df['listing_id'].values"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "#### 13번째 데이터셋"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 6,
   "metadata": {
    "collapsed": false
   },
   "outputs": [],
   "source": [
    "x_train = pd.read_csv('x_train_16th.csv')\n",
    "x_test = pd.read_csv('x_test_16th.csv')\n",
    "y_train = pd.read_csv('y_train_16th.csv')\n",
    "y_train.interest_level=y_train.interest_level.astype(int)\n",
    "listing_id_df = pd.read_csv('test_listing_16th.csv')\n",
    "listing_id = listing_id_df['listing_id'].values"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 7,
   "metadata": {
    "collapsed": false
   },
   "outputs": [
    {
     "data": {
      "text/plain": [
       "(49352, 295)"
      ]
     },
     "execution_count": 7,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "x_train.shape"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 8,
   "metadata": {
    "collapsed": false
   },
   "outputs": [
    {
     "data": {
      "text/plain": [
       "(74659, 295)"
      ]
     },
     "execution_count": 8,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "x_test.shape"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# LightGBM for StackNet"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 43,
   "metadata": {
    "collapsed": false,
    "scrolled": true
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "\n",
      " Fold 1\n",
      "Train until valid scores didn't improve in 100 rounds.\n",
      "[100]\tvalid_0's multi_logloss: 0.597333\n",
      "[200]\tvalid_0's multi_logloss: 0.557206\n",
      "[300]\tvalid_0's multi_logloss: 0.543779\n",
      "[400]\tvalid_0's multi_logloss: 0.537129\n",
      "[500]\tvalid_0's multi_logloss: 0.533212\n",
      "[600]\tvalid_0's multi_logloss: 0.531199\n",
      "[700]\tvalid_0's multi_logloss: 0.529957\n",
      "[800]\tvalid_0's multi_logloss: 0.529073\n",
      "[900]\tvalid_0's multi_logloss: 0.528685\n",
      "[1000]\tvalid_0's multi_logloss: 0.528216\n",
      "[1100]\tvalid_0's multi_logloss: 0.528038\n",
      "[1200]\tvalid_0's multi_logloss: 0.527944\n",
      "Early stopping, best iteration is:\n",
      "[1195]\tvalid_0's multi_logloss: 0.527907\n",
      "eval-MAE: 0.529092\n",
      "\n",
      " Fold 2\n",
      "Train until valid scores didn't improve in 100 rounds.\n",
      "[100]\tvalid_0's multi_logloss: 0.565221\n",
      "[200]\tvalid_0's multi_logloss: 0.521801\n",
      "[300]\tvalid_0's multi_logloss: 0.507708\n",
      "[400]\tvalid_0's multi_logloss: 0.50095\n",
      "[500]\tvalid_0's multi_logloss: 0.496913\n",
      "[600]\tvalid_0's multi_logloss: 0.494563\n",
      "[700]\tvalid_0's multi_logloss: 0.49301\n",
      "[800]\tvalid_0's multi_logloss: 0.491921\n",
      "[900]\tvalid_0's multi_logloss: 0.491553\n",
      "[1000]\tvalid_0's multi_logloss: 0.491317\n",
      "[1100]\tvalid_0's multi_logloss: 0.491084\n",
      "[1200]\tvalid_0's multi_logloss: 0.491108\n",
      "Early stopping, best iteration is:\n",
      "[1123]\tvalid_0's multi_logloss: 0.491047\n",
      "eval-MAE: 0.491169\n",
      "\n",
      " Fold 3\n",
      "Train until valid scores didn't improve in 100 rounds.\n",
      "[100]\tvalid_0's multi_logloss: 0.567897\n",
      "[200]\tvalid_0's multi_logloss: 0.523672\n",
      "[300]\tvalid_0's multi_logloss: 0.509484\n",
      "[400]\tvalid_0's multi_logloss: 0.503063\n",
      "[500]\tvalid_0's multi_logloss: 0.499354\n",
      "[600]\tvalid_0's multi_logloss: 0.497268\n",
      "[700]\tvalid_0's multi_logloss: 0.495671\n",
      "[800]\tvalid_0's multi_logloss: 0.494645\n",
      "[900]\tvalid_0's multi_logloss: 0.494108\n",
      "[1000]\tvalid_0's multi_logloss: 0.49391\n",
      "[1100]\tvalid_0's multi_logloss: 0.493573\n",
      "[1200]\tvalid_0's multi_logloss: 0.493527\n",
      "Early stopping, best iteration is:\n",
      "[1170]\tvalid_0's multi_logloss: 0.493453\n",
      "eval-MAE: 0.494095\n",
      "\n",
      " Fold 4\n",
      "Train until valid scores didn't improve in 100 rounds.\n",
      "[100]\tvalid_0's multi_logloss: 0.577523\n",
      "[200]\tvalid_0's multi_logloss: 0.53469\n",
      "[300]\tvalid_0's multi_logloss: 0.522111\n",
      "[400]\tvalid_0's multi_logloss: 0.516331\n",
      "[500]\tvalid_0's multi_logloss: 0.514004\n",
      "[600]\tvalid_0's multi_logloss: 0.51251\n",
      "[700]\tvalid_0's multi_logloss: 0.511477\n",
      "[800]\tvalid_0's multi_logloss: 0.510798\n",
      "[900]\tvalid_0's multi_logloss: 0.510609\n",
      "[1000]\tvalid_0's multi_logloss: 0.510445\n",
      "Early stopping, best iteration is:\n",
      "[972]\tvalid_0's multi_logloss: 0.510321\n",
      "eval-MAE: 0.511870\n",
      "\n",
      " Fold 5\n",
      "Train until valid scores didn't improve in 100 rounds.\n",
      "[100]\tvalid_0's multi_logloss: 0.597762\n",
      "[200]\tvalid_0's multi_logloss: 0.556729\n",
      "[300]\tvalid_0's multi_logloss: 0.543686\n",
      "[400]\tvalid_0's multi_logloss: 0.537845\n",
      "[500]\tvalid_0's multi_logloss: 0.534669\n",
      "[600]\tvalid_0's multi_logloss: 0.532558\n",
      "[700]\tvalid_0's multi_logloss: 0.531583\n",
      "[800]\tvalid_0's multi_logloss: 0.530823\n",
      "[900]\tvalid_0's multi_logloss: 0.530347\n",
      "[1000]\tvalid_0's multi_logloss: 0.530011\n",
      "[1100]\tvalid_0's multi_logloss: 0.529816\n",
      "Early stopping, best iteration is:\n",
      "[1077]\tvalid_0's multi_logloss: 0.529704\n",
      "eval-MAE: 0.530795\n",
      "Average Lolikelihood:  0.511404180401\n",
      "1 0.529091933714\n",
      "2 0.491168761925\n",
      "3 0.494094611916\n",
      "4 0.511870308759\n",
      "5 0.530795285692\n",
      "test prediction\n",
      "best round: 1195\n",
      "merging columns\n",
      "exporting files\n",
      "Write results...\n",
      "Writing submission to output/submission_0.511404180401_ex120.csv\n",
      "fscore result\n",
      "Done.\n"
     ]
    }
   ],
   "source": [
    "ex_no = 120\n",
    "\n",
    "train_file=\"stacked/train_stacknet_\"+str(ex_no)+\".csv\"\n",
    "test_file=\"stacked/test_stacknet_\"+str(ex_no)+\".csv\"\n",
    "\n",
    " #Create Arrays for meta\n",
    "train_stacker=[ [0.0 for s in range(3)]  for k in range (0,(x_train.shape[0])) ]\n",
    "test_stacker=[[0.0 for s in range(3)]   for k in range (0,(x_test.shape[0]))]\n",
    "        \n",
    "cv_sum = 0\n",
    "fpred = []\n",
    "lgbm_rounds = []\n",
    "NFOLDS = 5\n",
    "cv_score_list = []\n",
    "lgbm_params = {\n",
    "    'boosting_type': 'gbdt', 'objective': 'multiclass', 'nthread': 4, 'silent': True,\n",
    "    'num_leaves': 32, 'learning_rate': 0.03, 'max_depth': -1,\n",
    "    'max_bin': 255, 'subsample_for_bin': 50000, 'num_class':3,'metric': 'multi_logloss',\n",
    "    'subsample': 0.8, 'subsample_freq': 1, 'colsample_bytree': 0.7, 'reg_alpha': 1, 'reg_lambda': 1.2,\n",
    "    'min_split_gain': 0.6, 'min_child_weight': 4, 'min_child_samples': 10, 'scale_pos_weight': 1}\n",
    "\n",
    "kf = KFold(x_train.shape[0], n_folds=NFOLDS)\n",
    "for i, (train_index, cross_index) in enumerate(kf):\n",
    "    print('\\n Fold %d' % (i+1))\n",
    "    \n",
    "    X_train, X_cross = x_train.iloc[train_index], x_train.iloc[cross_index]\n",
    "    Y_train, Y_cross = y_train.iloc[train_index], y_train.iloc[cross_index]\n",
    "    \n",
    "    #X_train_temp,X_cross_temp = add_manager_skill(X_train,X_cross,Y_train)\n",
    "    #W_train, W_cross = hcc_encode_wrapping(X_train,X_cross,Y_train,categorical)\n",
    "    \n",
    "    d_train = lgbm.Dataset(X_train, label=Y_train['interest_level'].values,silent=True)\n",
    "    d_valid = lgbm.Dataset(X_cross, label=Y_cross['interest_level'].values,silent=True)\n",
    "    #watchlist = [(d_train, 'train'), (d_valid, 'eval')]\n",
    "    \n",
    "    model = lgbm.train(lgbm_params, train_set=d_train, num_boost_round=100000, valid_sets=d_valid,\n",
    "                        early_stopping_rounds=100, evals_result=None, verbose_eval=100, learning_rates=None, \n",
    "                        callbacks=None)\n",
    "    lgbm_rounds.append(model.best_iteration)\n",
    "    scores_val = model.predict(X_cross, num_iteration=model.best_iteration)\n",
    "    cv_score = log_loss(Y_cross, scores_val)\n",
    "    print('eval-MAE: %.6f' % cv_score)\n",
    "    cv_score_list.append(cv_score)\n",
    "    \n",
    "    cv_sum = cv_sum + cv_score    \n",
    "    \n",
    "    reshaped_preds =scores_val.reshape( X_cross.shape[0], 3)\n",
    "    \n",
    "    no=0\n",
    "    for real_index in cross_index:\n",
    "        for d in range (0,3):\n",
    "            train_stacker[real_index][d]=(reshaped_preds[no][d])\n",
    "        no+=1\n",
    "    \n",
    "score = cv_sum / NFOLDS\n",
    "print(\"Average Lolikelihood: \", score)\n",
    "cv_index = 1\n",
    "for cv_s in cv_score_list:\n",
    "    print(cv_index,cv_s)\n",
    "    cv_index+=1\n",
    "\n",
    "print(\"test prediction\")\n",
    "bst_round = max(lgbm_rounds)\n",
    "print(\"best round:\", bst_round )\n",
    "\n",
    "#x_train_temp,x_test_temp = add_manager_skill(x_train,x_test,y_train)\n",
    "#W,W_test = hcc_encode_wrapping(x_train,x_test,y_train,categorical)\n",
    "W = x_train.copy()\n",
    "W_test = x_test.copy()\n",
    "d_train_all = lgbm.Dataset(W, label=y_train['interest_level'].values,silent=True)\n",
    "d_test = lgbm.Dataset(W_test)\n",
    "\n",
    "model = lgbm.train(lgbm_params,d_train_all,bst_round)\n",
    "predictions = model.predict(W_test)\n",
    "preds=predictions.reshape( x_test.shape[0], 3)\n",
    "\n",
    "for pr in range (0,len(preds)):  \n",
    "    for d in range (0,3):            \n",
    "        test_stacker[pr][d]=(preds[pr][d])\n",
    "\n",
    "print(\"merging columns\")\n",
    "X=np.column_stack((W,train_stacker))\n",
    "X_test=np.column_stack((W_test,test_stacker))   \n",
    "\n",
    "# stack target to train\n",
    "X=np.column_stack((y_train,X))\n",
    "   \n",
    "X_test=np.column_stack((listing_id,X_test))\n",
    "\n",
    "print(\"exporting files\")\n",
    "np.savetxt(train_file, X, delimiter=\",\", fmt='%.5f')\n",
    "np.savetxt(test_file, X_test, delimiter=\",\", fmt='%.5f')\n",
    "                             \n",
    "print(\"Write results...\")\n",
    "output_file = \"output/submission_\" + str( (score ))+\"_ex\" + str(ex_no) + \".csv\"\n",
    "print(\"Writing submission to %s\" % output_file)\n",
    "f = open(output_file, \"w\")   \n",
    "f.write(\"listing_id,low,medium,high\\n\")# the header   \n",
    "for g in range(0, len(test_stacker)):\n",
    "    f.write(\"%s\" % (listing_id[g]))\n",
    "    for prediction in test_stacker[g]:\n",
    "        f.write(\",%f\" % (prediction))    \n",
    "    f.write(\"\\n\")\n",
    "f.close()\n",
    "\n",
    "print(\"fscore result\")\n",
    "fscore_df = pd.DataFrame(model.feature_importance())\n",
    "fscore_df.rename(columns={0:'fscore'},inplace=True)\n",
    "fscore_df['feature_name'] = model.feature_name()\n",
    "fscore_df['fscore'] = fscore_df['fscore'].astype(int)\n",
    "fscore_df.sort_values(by='fscore',ascending=False,inplace=True)\n",
    "fscoe_output = 'fscore/ex_'+str(ex_no)+'_fscore.csv'\n",
    "fscore_df.to_csv(fscoe_output)\n",
    "\n",
    "#del x_train_temp,x_test_temp,W,W_test,X,X_test\n",
    "print(\"Done.\")"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# XGBOOST for StackNet"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 139,
   "metadata": {
    "collapsed": true
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "\n",
      " Fold 1\n",
      "[0]\ttrain-mlogloss:1.07742\teval-mlogloss:1.07812\n",
      "Multiple eval metrics have been passed: 'eval-mlogloss' will be used for early stopping.\n",
      "\n",
      "Will train until eval-mlogloss hasn't improved in 100 rounds.\n",
      "[25]\ttrain-mlogloss:0.765209\teval-mlogloss:0.78453\n",
      "[50]\ttrain-mlogloss:0.640209\teval-mlogloss:0.673823\n",
      "[75]\ttrain-mlogloss:0.580436\teval-mlogloss:0.625023\n",
      "[100]\ttrain-mlogloss:0.547145\teval-mlogloss:0.599865\n",
      "[125]\ttrain-mlogloss:0.525777\teval-mlogloss:0.58467\n",
      "[150]\ttrain-mlogloss:0.510554\teval-mlogloss:0.574706\n",
      "[175]\ttrain-mlogloss:0.497881\teval-mlogloss:0.567403\n",
      "[200]\ttrain-mlogloss:0.486802\teval-mlogloss:0.561703\n",
      "[225]\ttrain-mlogloss:0.47711\teval-mlogloss:0.557117\n",
      "[250]\ttrain-mlogloss:0.468171\teval-mlogloss:0.553237\n",
      "[275]\ttrain-mlogloss:0.460033\teval-mlogloss:0.550065\n",
      "[300]\ttrain-mlogloss:0.452643\teval-mlogloss:0.547433\n",
      "[325]\ttrain-mlogloss:0.446549\teval-mlogloss:0.545101\n",
      "[350]\ttrain-mlogloss:0.439684\teval-mlogloss:0.542904\n",
      "[375]\ttrain-mlogloss:0.433674\teval-mlogloss:0.541178\n",
      "[400]\ttrain-mlogloss:0.428015\teval-mlogloss:0.539644\n",
      "[425]\ttrain-mlogloss:0.422088\teval-mlogloss:0.538218\n",
      "[450]\ttrain-mlogloss:0.416814\teval-mlogloss:0.537006\n",
      "[475]\ttrain-mlogloss:0.411323\teval-mlogloss:0.53582\n",
      "[500]\ttrain-mlogloss:0.405987\teval-mlogloss:0.534643\n",
      "[525]\ttrain-mlogloss:0.401496\teval-mlogloss:0.53377\n",
      "[550]\ttrain-mlogloss:0.396521\teval-mlogloss:0.533007\n",
      "[575]\ttrain-mlogloss:0.392107\teval-mlogloss:0.532631\n",
      "[600]\ttrain-mlogloss:0.387771\teval-mlogloss:0.531871\n",
      "[625]\ttrain-mlogloss:0.38327\teval-mlogloss:0.531515\n",
      "[650]\ttrain-mlogloss:0.378704\teval-mlogloss:0.531092\n",
      "[675]\ttrain-mlogloss:0.374555\teval-mlogloss:0.530547\n",
      "[700]\ttrain-mlogloss:0.370252\teval-mlogloss:0.530149\n",
      "[725]\ttrain-mlogloss:0.366192\teval-mlogloss:0.529791\n",
      "[750]\ttrain-mlogloss:0.362267\teval-mlogloss:0.529386\n",
      "[775]\ttrain-mlogloss:0.358149\teval-mlogloss:0.529179\n",
      "[800]\ttrain-mlogloss:0.354293\teval-mlogloss:0.528852\n",
      "[825]\ttrain-mlogloss:0.350718\teval-mlogloss:0.528808\n",
      "[850]\ttrain-mlogloss:0.347024\teval-mlogloss:0.528737\n",
      "[875]\ttrain-mlogloss:0.343425\teval-mlogloss:0.528523\n",
      "[900]\ttrain-mlogloss:0.339866\teval-mlogloss:0.528314\n",
      "[925]\ttrain-mlogloss:0.336709\teval-mlogloss:0.528311\n",
      "[950]\ttrain-mlogloss:0.333157\teval-mlogloss:0.528281\n",
      "[975]\ttrain-mlogloss:0.32983\teval-mlogloss:0.528134\n",
      "[1000]\ttrain-mlogloss:0.326261\teval-mlogloss:0.527972\n",
      "[1025]\ttrain-mlogloss:0.322739\teval-mlogloss:0.528097\n",
      "[1050]\ttrain-mlogloss:0.319465\teval-mlogloss:0.52804\n",
      "[1075]\ttrain-mlogloss:0.316066\teval-mlogloss:0.528072\n",
      "Stopping. Best iteration:\n",
      "[997]\ttrain-mlogloss:0.326726\teval-mlogloss:0.527952\n",
      "\n",
      "eval-MAE: 0.527952\n",
      "\n",
      " Fold 2\n",
      "[0]\ttrain-mlogloss:1.07782\teval-mlogloss:1.07768\n",
      "Multiple eval metrics have been passed: 'eval-mlogloss' will be used for early stopping.\n",
      "\n",
      "Will train until eval-mlogloss hasn't improved in 100 rounds.\n",
      "[25]\ttrain-mlogloss:0.771192\teval-mlogloss:0.770384\n",
      "[50]\ttrain-mlogloss:0.648068\teval-mlogloss:0.648663\n",
      "[75]\ttrain-mlogloss:0.589142\teval-mlogloss:0.593354\n",
      "[100]\ttrain-mlogloss:0.556302\teval-mlogloss:0.564707\n",
      "[125]\ttrain-mlogloss:0.534582\teval-mlogloss:0.547663\n",
      "[150]\ttrain-mlogloss:0.519333\teval-mlogloss:0.536813\n",
      "[175]\ttrain-mlogloss:0.507107\teval-mlogloss:0.529056\n",
      "[200]\ttrain-mlogloss:0.495809\teval-mlogloss:0.523091\n",
      "[225]\ttrain-mlogloss:0.48643\teval-mlogloss:0.518993\n",
      "[250]\ttrain-mlogloss:0.477755\teval-mlogloss:0.515234\n",
      "[275]\ttrain-mlogloss:0.46997\teval-mlogloss:0.512012\n",
      "[300]\ttrain-mlogloss:0.462683\teval-mlogloss:0.509723\n",
      "[325]\ttrain-mlogloss:0.455434\teval-mlogloss:0.507456\n",
      "[350]\ttrain-mlogloss:0.448724\teval-mlogloss:0.505465\n",
      "[375]\ttrain-mlogloss:0.442579\teval-mlogloss:0.503779\n",
      "[400]\ttrain-mlogloss:0.437005\teval-mlogloss:0.502416\n",
      "[425]\ttrain-mlogloss:0.431054\teval-mlogloss:0.501032\n",
      "[450]\ttrain-mlogloss:0.425748\teval-mlogloss:0.500161\n",
      "[475]\ttrain-mlogloss:0.420411\teval-mlogloss:0.499279\n",
      "[500]\ttrain-mlogloss:0.415427\teval-mlogloss:0.498513\n",
      "[525]\ttrain-mlogloss:0.410878\teval-mlogloss:0.497878\n",
      "[550]\ttrain-mlogloss:0.406145\teval-mlogloss:0.497153\n",
      "[575]\ttrain-mlogloss:0.401603\teval-mlogloss:0.496566\n",
      "[600]\ttrain-mlogloss:0.397143\teval-mlogloss:0.496014\n",
      "[625]\ttrain-mlogloss:0.392487\teval-mlogloss:0.495469\n",
      "[650]\ttrain-mlogloss:0.38798\teval-mlogloss:0.495168\n",
      "[675]\ttrain-mlogloss:0.383964\teval-mlogloss:0.494622\n",
      "[700]\ttrain-mlogloss:0.379734\teval-mlogloss:0.494309\n",
      "[725]\ttrain-mlogloss:0.376023\teval-mlogloss:0.494113\n",
      "[750]\ttrain-mlogloss:0.371799\teval-mlogloss:0.493736\n",
      "[775]\ttrain-mlogloss:0.368058\teval-mlogloss:0.493465\n",
      "[800]\ttrain-mlogloss:0.364017\teval-mlogloss:0.493272\n",
      "[825]\ttrain-mlogloss:0.360127\teval-mlogloss:0.492974\n",
      "[850]\ttrain-mlogloss:0.356207\teval-mlogloss:0.492772\n",
      "[875]\ttrain-mlogloss:0.352666\teval-mlogloss:0.492644\n",
      "[900]\ttrain-mlogloss:0.349191\teval-mlogloss:0.492613\n",
      "[925]\ttrain-mlogloss:0.345763\teval-mlogloss:0.492678\n",
      "[950]\ttrain-mlogloss:0.342255\teval-mlogloss:0.492606\n",
      "[975]\ttrain-mlogloss:0.338922\teval-mlogloss:0.492654\n",
      "[1000]\ttrain-mlogloss:0.335559\teval-mlogloss:0.492625\n",
      "[1025]\ttrain-mlogloss:0.332373\teval-mlogloss:0.492677\n",
      "[1050]\ttrain-mlogloss:0.329067\teval-mlogloss:0.492743\n",
      "Stopping. Best iteration:\n",
      "[962]\ttrain-mlogloss:0.340582\teval-mlogloss:0.492511\n",
      "\n",
      "eval-MAE: 0.492511\n",
      "\n",
      " Fold 3\n",
      "[0]\ttrain-mlogloss:1.07777\teval-mlogloss:1.07789\n",
      "Multiple eval metrics have been passed: 'eval-mlogloss' will be used for early stopping.\n",
      "\n",
      "Will train until eval-mlogloss hasn't improved in 100 rounds.\n",
      "[25]\ttrain-mlogloss:0.770336\teval-mlogloss:0.77206\n",
      "[50]\ttrain-mlogloss:0.6475\teval-mlogloss:0.651472\n",
      "[75]\ttrain-mlogloss:0.588917\teval-mlogloss:0.59659\n",
      "[100]\ttrain-mlogloss:0.556271\teval-mlogloss:0.568092\n",
      "[125]\ttrain-mlogloss:0.534943\teval-mlogloss:0.551203\n",
      "[150]\ttrain-mlogloss:0.518659\teval-mlogloss:0.53994\n",
      "[175]\ttrain-mlogloss:0.506067\teval-mlogloss:0.532136\n",
      "[200]\ttrain-mlogloss:0.494768\teval-mlogloss:0.52621\n",
      "[225]\ttrain-mlogloss:0.485298\teval-mlogloss:0.52167\n",
      "[250]\ttrain-mlogloss:0.475873\teval-mlogloss:0.517732\n",
      "[275]\ttrain-mlogloss:0.467635\teval-mlogloss:0.514523\n",
      "[300]\ttrain-mlogloss:0.459784\teval-mlogloss:0.511778\n",
      "[325]\ttrain-mlogloss:0.452896\teval-mlogloss:0.509652\n",
      "[350]\ttrain-mlogloss:0.446357\teval-mlogloss:0.507755\n",
      "[375]\ttrain-mlogloss:0.440605\teval-mlogloss:0.506295\n",
      "[400]\ttrain-mlogloss:0.43467\teval-mlogloss:0.504901\n",
      "[425]\ttrain-mlogloss:0.428994\teval-mlogloss:0.503462\n",
      "[450]\ttrain-mlogloss:0.423493\teval-mlogloss:0.50226\n",
      "[475]\ttrain-mlogloss:0.418269\teval-mlogloss:0.501171\n",
      "[500]\ttrain-mlogloss:0.413165\teval-mlogloss:0.500338\n",
      "[525]\ttrain-mlogloss:0.408265\teval-mlogloss:0.499423\n",
      "[550]\ttrain-mlogloss:0.403571\teval-mlogloss:0.498591\n",
      "[575]\ttrain-mlogloss:0.399148\teval-mlogloss:0.498026\n",
      "[600]\ttrain-mlogloss:0.394339\teval-mlogloss:0.497297\n",
      "[625]\ttrain-mlogloss:0.390238\teval-mlogloss:0.496634\n",
      "[650]\ttrain-mlogloss:0.386196\teval-mlogloss:0.496165\n",
      "[675]\ttrain-mlogloss:0.381733\teval-mlogloss:0.495667\n",
      "[700]\ttrain-mlogloss:0.377543\teval-mlogloss:0.495347\n",
      "[725]\ttrain-mlogloss:0.373778\teval-mlogloss:0.495007\n",
      "[750]\ttrain-mlogloss:0.369652\teval-mlogloss:0.494695\n",
      "[775]\ttrain-mlogloss:0.365705\teval-mlogloss:0.494445\n",
      "[800]\ttrain-mlogloss:0.361722\teval-mlogloss:0.494244\n",
      "[825]\ttrain-mlogloss:0.357942\teval-mlogloss:0.493901\n",
      "[850]\ttrain-mlogloss:0.354035\teval-mlogloss:0.493629\n",
      "[875]\ttrain-mlogloss:0.350259\teval-mlogloss:0.493418\n",
      "[900]\ttrain-mlogloss:0.346557\teval-mlogloss:0.493287\n",
      "[925]\ttrain-mlogloss:0.343005\teval-mlogloss:0.493153\n",
      "[950]\ttrain-mlogloss:0.339294\teval-mlogloss:0.493218\n",
      "[975]\ttrain-mlogloss:0.335826\teval-mlogloss:0.493013\n",
      "[1000]\ttrain-mlogloss:0.33249\teval-mlogloss:0.49295\n",
      "[1025]\ttrain-mlogloss:0.328827\teval-mlogloss:0.492778\n",
      "[1050]\ttrain-mlogloss:0.325477\teval-mlogloss:0.492777\n",
      "[1075]\ttrain-mlogloss:0.32218\teval-mlogloss:0.492906\n",
      "[1100]\ttrain-mlogloss:0.318872\teval-mlogloss:0.492848\n",
      "[1125]\ttrain-mlogloss:0.315613\teval-mlogloss:0.49278\n",
      "[1150]\ttrain-mlogloss:0.312365\teval-mlogloss:0.492842\n",
      "[1175]\ttrain-mlogloss:0.309221\teval-mlogloss:0.492906\n",
      "[1200]\ttrain-mlogloss:0.306438\teval-mlogloss:0.493031\n",
      "Stopping. Best iteration:\n",
      "[1121]\ttrain-mlogloss:0.316188\teval-mlogloss:0.492735\n",
      "\n",
      "eval-MAE: 0.492735\n",
      "\n",
      " Fold 4\n",
      "[0]\ttrain-mlogloss:1.07764\teval-mlogloss:1.07797\n",
      "Multiple eval metrics have been passed: 'eval-mlogloss' will be used for early stopping.\n",
      "\n",
      "Will train until eval-mlogloss hasn't improved in 100 rounds.\n",
      "[25]\ttrain-mlogloss:0.768453\teval-mlogloss:0.776096\n",
      "[50]\ttrain-mlogloss:0.645111\teval-mlogloss:0.659269\n",
      "[75]\ttrain-mlogloss:0.585977\teval-mlogloss:0.606005\n",
      "[100]\ttrain-mlogloss:0.553045\teval-mlogloss:0.578552\n",
      "[125]\ttrain-mlogloss:0.531661\teval-mlogloss:0.562127\n",
      "[150]\ttrain-mlogloss:0.515667\teval-mlogloss:0.55142\n",
      "[175]\ttrain-mlogloss:0.503219\teval-mlogloss:0.543884\n",
      "[200]\ttrain-mlogloss:0.492069\teval-mlogloss:0.537856\n",
      "[225]\ttrain-mlogloss:0.481794\teval-mlogloss:0.533199\n",
      "[250]\ttrain-mlogloss:0.472752\teval-mlogloss:0.529815\n",
      "[275]\ttrain-mlogloss:0.464722\teval-mlogloss:0.527063\n",
      "[300]\ttrain-mlogloss:0.45718\teval-mlogloss:0.524632\n",
      "[325]\ttrain-mlogloss:0.450207\teval-mlogloss:0.522569\n",
      "[350]\ttrain-mlogloss:0.443383\teval-mlogloss:0.52075\n",
      "[375]\ttrain-mlogloss:0.437167\teval-mlogloss:0.519414\n",
      "[400]\ttrain-mlogloss:0.431618\teval-mlogloss:0.51839\n",
      "[425]\ttrain-mlogloss:0.425929\teval-mlogloss:0.517232\n",
      "[450]\ttrain-mlogloss:0.420637\teval-mlogloss:0.516246\n",
      "[475]\ttrain-mlogloss:0.415531\teval-mlogloss:0.515357\n",
      "[500]\ttrain-mlogloss:0.410673\teval-mlogloss:0.514742\n",
      "[525]\ttrain-mlogloss:0.406016\teval-mlogloss:0.514164\n",
      "[550]\ttrain-mlogloss:0.401076\teval-mlogloss:0.513561\n",
      "[575]\ttrain-mlogloss:0.396472\teval-mlogloss:0.513088\n",
      "[600]\ttrain-mlogloss:0.391731\teval-mlogloss:0.512764\n",
      "[625]\ttrain-mlogloss:0.387574\teval-mlogloss:0.51245\n",
      "[650]\ttrain-mlogloss:0.383149\teval-mlogloss:0.512068\n",
      "[675]\ttrain-mlogloss:0.378647\teval-mlogloss:0.511643\n",
      "[700]\ttrain-mlogloss:0.374456\teval-mlogloss:0.511457\n",
      "[725]\ttrain-mlogloss:0.370281\teval-mlogloss:0.511118\n",
      "[750]\ttrain-mlogloss:0.365875\teval-mlogloss:0.511021\n",
      "[775]\ttrain-mlogloss:0.361944\teval-mlogloss:0.510756\n",
      "[800]\ttrain-mlogloss:0.35795\teval-mlogloss:0.510554\n",
      "[825]\ttrain-mlogloss:0.353996\teval-mlogloss:0.510443\n",
      "[850]\ttrain-mlogloss:0.35002\teval-mlogloss:0.510347\n",
      "[875]\ttrain-mlogloss:0.346293\teval-mlogloss:0.510217\n",
      "[900]\ttrain-mlogloss:0.342748\teval-mlogloss:0.51012\n",
      "[925]\ttrain-mlogloss:0.33915\teval-mlogloss:0.510282\n",
      "[950]\ttrain-mlogloss:0.335823\teval-mlogloss:0.510276\n",
      "[975]\ttrain-mlogloss:0.332599\teval-mlogloss:0.510428\n",
      "[1000]\ttrain-mlogloss:0.329175\teval-mlogloss:0.510212\n",
      "Stopping. Best iteration:\n",
      "[913]\ttrain-mlogloss:0.340881\teval-mlogloss:0.510111\n",
      "\n",
      "eval-MAE: 0.510111\n",
      "\n",
      " Fold 5\n",
      "[0]\ttrain-mlogloss:1.07743\teval-mlogloss:1.07831\n",
      "Multiple eval metrics have been passed: 'eval-mlogloss' will be used for early stopping.\n",
      "\n",
      "Will train until eval-mlogloss hasn't improved in 100 rounds.\n",
      "[25]\ttrain-mlogloss:0.764965\teval-mlogloss:0.78626\n",
      "[50]\ttrain-mlogloss:0.639999\teval-mlogloss:0.675026\n",
      "[75]\ttrain-mlogloss:0.580182\teval-mlogloss:0.624978\n",
      "[100]\ttrain-mlogloss:0.546636\teval-mlogloss:0.599155\n",
      "[125]\ttrain-mlogloss:0.525072\teval-mlogloss:0.584063\n",
      "[150]\ttrain-mlogloss:0.509043\teval-mlogloss:0.573805\n",
      "[175]\ttrain-mlogloss:0.496689\teval-mlogloss:0.566607\n",
      "[200]\ttrain-mlogloss:0.486296\teval-mlogloss:0.561123\n",
      "[225]\ttrain-mlogloss:0.476487\teval-mlogloss:0.556483\n",
      "[250]\ttrain-mlogloss:0.468057\teval-mlogloss:0.552824\n",
      "[275]\ttrain-mlogloss:0.460693\teval-mlogloss:0.549909\n",
      "[300]\ttrain-mlogloss:0.453333\teval-mlogloss:0.547416\n",
      "[325]\ttrain-mlogloss:0.446559\teval-mlogloss:0.545453\n",
      "[350]\ttrain-mlogloss:0.440049\teval-mlogloss:0.543771\n",
      "[375]\ttrain-mlogloss:0.433738\teval-mlogloss:0.542099\n",
      "[400]\ttrain-mlogloss:0.428115\teval-mlogloss:0.540671\n",
      "[425]\ttrain-mlogloss:0.422577\teval-mlogloss:0.539506\n",
      "[450]\ttrain-mlogloss:0.417219\teval-mlogloss:0.538447\n",
      "[475]\ttrain-mlogloss:0.411978\teval-mlogloss:0.537497\n",
      "[500]\ttrain-mlogloss:0.406893\teval-mlogloss:0.536798\n",
      "[525]\ttrain-mlogloss:0.401903\teval-mlogloss:0.535981\n",
      "[550]\ttrain-mlogloss:0.396977\teval-mlogloss:0.535447\n",
      "[575]\ttrain-mlogloss:0.392607\teval-mlogloss:0.534903\n",
      "[600]\ttrain-mlogloss:0.388593\teval-mlogloss:0.534397\n",
      "[625]\ttrain-mlogloss:0.384584\teval-mlogloss:0.533877\n",
      "[650]\ttrain-mlogloss:0.380043\teval-mlogloss:0.533423\n",
      "[675]\ttrain-mlogloss:0.375509\teval-mlogloss:0.532911\n",
      "[700]\ttrain-mlogloss:0.371484\teval-mlogloss:0.532596\n",
      "[725]\ttrain-mlogloss:0.367407\teval-mlogloss:0.532227\n",
      "[750]\ttrain-mlogloss:0.363353\teval-mlogloss:0.53203\n",
      "[775]\ttrain-mlogloss:0.359402\teval-mlogloss:0.531674\n",
      "[800]\ttrain-mlogloss:0.355642\teval-mlogloss:0.531577\n",
      "[825]\ttrain-mlogloss:0.351967\teval-mlogloss:0.531467\n",
      "[850]\ttrain-mlogloss:0.348322\teval-mlogloss:0.531408\n",
      "[875]\ttrain-mlogloss:0.344731\teval-mlogloss:0.53115\n",
      "[900]\ttrain-mlogloss:0.34113\teval-mlogloss:0.530862\n",
      "[925]\ttrain-mlogloss:0.337398\teval-mlogloss:0.530793\n",
      "[950]\ttrain-mlogloss:0.3339\teval-mlogloss:0.530795\n",
      "[975]\ttrain-mlogloss:0.33053\teval-mlogloss:0.530712\n",
      "[1000]\ttrain-mlogloss:0.327167\teval-mlogloss:0.530796\n",
      "[1025]\ttrain-mlogloss:0.323761\teval-mlogloss:0.530685\n",
      "[1050]\ttrain-mlogloss:0.320335\teval-mlogloss:0.530698\n",
      "[1075]\ttrain-mlogloss:0.317066\teval-mlogloss:0.530689\n",
      "[1100]\ttrain-mlogloss:0.313979\teval-mlogloss:0.530657\n",
      "[1125]\ttrain-mlogloss:0.310742\teval-mlogloss:0.530691\n",
      "[1150]\ttrain-mlogloss:0.307645\teval-mlogloss:0.530667\n",
      "[1175]\ttrain-mlogloss:0.304491\teval-mlogloss:0.530564\n",
      "[1200]\ttrain-mlogloss:0.30158\teval-mlogloss:0.530572\n",
      "[1225]\ttrain-mlogloss:0.298487\teval-mlogloss:0.530516\n",
      "[1250]\ttrain-mlogloss:0.295564\teval-mlogloss:0.530684\n",
      "[1275]\ttrain-mlogloss:0.292702\teval-mlogloss:0.530883\n",
      "[1300]\ttrain-mlogloss:0.290001\teval-mlogloss:0.530988\n",
      "[1325]\ttrain-mlogloss:0.287011\teval-mlogloss:0.531064\n",
      "Stopping. Best iteration:\n",
      "[1227]\ttrain-mlogloss:0.298218\teval-mlogloss:0.530487\n",
      "\n",
      "eval-MAE: 0.530487\n",
      "Average Lolikelihood:  0.510759077085\n",
      "1 0.527952130986\n",
      "1 0.492510737197\n",
      "1 0.49273476636\n",
      "1 0.510110864739\n",
      "1 0.530486886143\n",
      "test prediction\n",
      "best round: 1227\n",
      "merging columns\n",
      "exporting files\n",
      "Write results...\n",
      "Writing submission to output/submission_0.510759077085_ex117.csv\n",
      "fscore result\n",
      "Done.\n"
     ]
    }
   ],
   "source": [
    "ex_no = 117\n",
    "\n",
    "train_file=\"stacked/train_stacknet_\"+str(ex_no)+\".csv\"\n",
    "test_file=\"stacked/test_stacknet_\"+str(ex_no)+\".csv\"\n",
    "\n",
    " #Create Arrays for meta\n",
    "train_stacker=[ [0.0 for s in range(3)]  for k in range (0,(x_train.shape[0])) ]\n",
    "test_stacker=[[0.0 for s in range(3)]   for k in range (0,(x_test.shape[0]))]\n",
    "        \n",
    "cv_sum = 0\n",
    "fpred = []\n",
    "xgb_rounds = []\n",
    "NFOLDS = 5\n",
    "cv_sum_list=[]\n",
    "params = {\n",
    "    'eta':0.03,\n",
    "    'colsample_bytree':0.7,\n",
    "    'subsample':0.8,\n",
    "    'seed':0,\n",
    "    'nthread':16,\n",
    "    'objective':'multi:softprob',\n",
    "    'eval_metric':'mlogloss',\n",
    "    'num_class':3,\n",
    "    'silent':1,\n",
    "    'min_child_weight':12,\n",
    "    'max_depth':6,\n",
    "    'gamma':0,\n",
    "    'lambda':1,\n",
    "    'alpha':1,\n",
    "    'max_delta_step':0\n",
    "}\n",
    "\n",
    "kf = KFold(x_train.shape[0], n_folds=NFOLDS)\n",
    "for i, (train_index, cross_index) in enumerate(kf):\n",
    "    print('\\n Fold %d' % (i+1))\n",
    "    \n",
    "    X_train, X_cross = x_train.iloc[train_index], x_train.iloc[cross_index]\n",
    "    Y_train, Y_cross = y_train.iloc[train_index], y_train.iloc[cross_index]\n",
    "    \n",
    "    #X_train_temp,X_cross_temp = add_manager_skill(X_train,X_cross,Y_train)\n",
    "    #W_train, W_cross = hcc_encode_wrapping(X_train_temp,X_cross_temp,Y_train,categorical)\n",
    "    \n",
    "    d_train = xgb.DMatrix(X_train, label=Y_train)\n",
    "    d_valid = xgb.DMatrix(X_cross, label=Y_cross)\n",
    "    watchlist = [(d_train, 'train'), (d_valid, 'eval')]\n",
    "    \n",
    "    bst = xgb.train(params,\n",
    "                    d_train,\n",
    "                    100000,\n",
    "                    watchlist,\n",
    "                    early_stopping_rounds=100,  \n",
    "                    verbose_eval=25)\n",
    "    \n",
    "    xgb_rounds.append(bst.best_iteration)\n",
    "    scores_val = bst.predict(d_valid, ntree_limit=bst.best_ntree_limit)\n",
    "    cv_score = log_loss(Y_cross, scores_val)\n",
    "    print('eval-MAE: %.6f' % cv_score)\n",
    "    cv_sum_list.append(cv_score)\n",
    "    cv_sum = cv_sum + cv_score    \n",
    "    \n",
    "    reshaped_preds =scores_val.reshape( X_cross.shape[0], 3)\n",
    "    \n",
    "    no=0\n",
    "    for real_index in cross_index:\n",
    "        for d in range (0,3):\n",
    "            train_stacker[real_index][d]=(reshaped_preds[no][d])\n",
    "        no+=1\n",
    "    \n",
    "score = cv_sum / NFOLDS\n",
    "print(\"Average Lolikelihood: \", score)\n",
    "cv_index = 1\n",
    "for cv_s in cv_sum_list:\n",
    "    print(cv_index,cv_s)\n",
    "    \n",
    "print(\"test prediction\")\n",
    "bst_round = max(xgb_rounds)\n",
    "print(\"best round:\", bst_round )\n",
    "\n",
    "#x_train_temp,x_test_temp = add_manager_skill(x_train,x_test,y_train)\n",
    "#W,W_test = hcc_encode_wrapping(x_train_temp,x_test_temp,y_train,categorical)\n",
    "\n",
    "d_train_all = xgb.DMatrix(x_train, label=y_train)\n",
    "d_test = xgb.DMatrix(x_test)\n",
    "\n",
    "bst = xgb.train(params,d_train_all,bst_round)\n",
    "predictions = bst.predict(d_test)\n",
    "preds=predictions.reshape( x_test.shape[0], 3)\n",
    "\n",
    "for pr in range (0,len(preds)):  \n",
    "    for d in range (0,3):            \n",
    "        test_stacker[pr][d]=(preds[pr][d])\n",
    "\n",
    "print(\"merging columns\")\n",
    "X=np.column_stack((x_train,train_stacker))\n",
    "X_test=np.column_stack((x_test,test_stacker))   \n",
    "\n",
    "# stack target to train\n",
    "X=np.column_stack((y_train,X))\n",
    "   \n",
    "X_test=np.column_stack((listing_id,X_test))\n",
    "\n",
    "print(\"exporting files\")\n",
    "np.savetxt(train_file, X, delimiter=\",\", fmt='%.5f')\n",
    "np.savetxt(test_file, X_test, delimiter=\",\", fmt='%.5f')\n",
    "                             \n",
    "print(\"Write results...\")\n",
    "output_file = \"output/submission_\" + str( (score ))+\"_ex\" + str(ex_no) + \".csv\"\n",
    "print(\"Writing submission to %s\" % output_file)\n",
    "f = open(output_file, \"w\")   \n",
    "f.write(\"listing_id,low,medium,high\\n\")# the header   \n",
    "for g in range(0, len(test_stacker)):\n",
    "    f.write(\"%s\" % (listing_id[g]))\n",
    "    for prediction in test_stacker[g]:\n",
    "        f.write(\",%f\" % (prediction))    \n",
    "    f.write(\"\\n\")\n",
    "f.close()\n",
    "\n",
    "print(\"fscore result\")\n",
    "fscore_df = pd.DataFrame.from_dict(bst.get_fscore(),orient='index')\n",
    "fscore_df.rename(columns={0:'fscore'},inplace=True)\n",
    "fscore_df['fscore'] = fscore_df['fscore'].astype(int)\n",
    "fscore_df.sort_values(by='fscore',ascending=False,inplace=True)\n",
    "fscoe_output = 'fscore/ex_'+str(ex_no)+'_fscore.csv'\n",
    "fscore_df.to_csv(fscoe_output)\n",
    "\n",
    "del X,X_test\n",
    "print(\"Done.\")"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## 사용하지 않는 Code"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 2,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "categorical4 = ['manager_id',\n",
    " 'building_id','longi_lati']\n",
    "\n",
    "categorical = ['manager_id',\n",
    " 'building_id','display_address']\n",
    "\n",
    "categorical3 = ['display_address',\n",
    " 'manager_id',\n",
    " 'building_id',\n",
    " 'street_address','manager_id_building_id']\n",
    "\n",
    "categorical2 = ['display_address',\n",
    " 'manager_id',\n",
    " 'building_id',\n",
    " 'street_address',\n",
    " 'display_address_manager_id',\n",
    " 'display_address_building_id',\n",
    " 'manager_id_building_id',\n",
    " 'manager_id_street_address',\n",
    " 'building_id_street_address']\n",
    "\n",
    "def hcc_encode(train_df, test_df, variable, target, prior_prob, k, f=1, g=1, r_k=None, update_df=None):\n",
    "    \"\"\"\n",
    "    See \"A Preprocessing Scheme for High-Cardinality Categorical Attributes in\n",
    "    Classification and Prediction Problems\" by Daniele Micci-Barreca\n",
    "    \"\"\"\n",
    "    hcc_name = \"_\".join([\"hcc\", variable, target])\n",
    "\n",
    "    grouped = train_df.groupby(variable)[target].agg({\"size\": \"size\", \"mean\": \"mean\"})\n",
    "    grouped[\"lambda\"] = 1 / (g + np.exp((k - grouped[\"size\"]) / f))\n",
    "    grouped[hcc_name] = grouped[\"lambda\"] * grouped[\"mean\"] + (1 - grouped[\"lambda\"]) * prior_prob\n",
    "\n",
    "    df = test_df[[variable]].join(grouped, on=variable, how=\"left\")[hcc_name].fillna(prior_prob)\n",
    "    if r_k: df *= np.random.uniform(1 - r_k, 1 + r_k, len(test_df))     # Add uniform noise. Not mentioned in original paper\n",
    "\n",
    "    if update_df is None: update_df = test_df\n",
    "    if hcc_name not in update_df.columns: update_df[hcc_name] = np.nan\n",
    "    update_df.update(df)\n",
    "    return\n",
    "\n",
    "def hcc_encode_wrapping(train_df,test_df,y_df,col_list):\n",
    "    test_temp_df = test_df.copy()\n",
    "    merged_df = pd.concat([train_df,y_df],axis=1)\n",
    "    \n",
    "    merged_df = merged_df.join(pd.get_dummies(merged_df[\"interest_level\"], \n",
    "                                      prefix=\"pred\").astype(int))\n",
    "    \n",
    "    prior_0, prior_1, prior_2 = merged_df[[\"pred_0\", \"pred_1\", \"pred_2\"]].mean()\n",
    "    \n",
    "    skf = StratifiedKFold(5)\n",
    "    attributes = product((col_list), zip((\"pred_0\",\"pred_1\", \"pred_2\"), (prior_0, prior_1, prior_2)))\n",
    "    \n",
    "    for variable, (target, prior) in attributes:\n",
    "        hcc_encode(merged_df, test_temp_df, variable, target, prior, k=5, r_k=None)\n",
    "        for train, test in skf.split(np.zeros(len(merged_df)), merged_df['interest_level']):\n",
    "            hcc_encode(merged_df.iloc[train], merged_df.iloc[test], variable, target, prior, k=5, r_k=0.01, update_df=merged_df)\n",
    "            \n",
    "    del merged_df['pred_0'], merged_df['pred_1'], merged_df['pred_2'], merged_df['interest_level']\n",
    "    \n",
    "    return merged_df,test_temp_df"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 3,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "def add_manager_skill(train,test,y_df):\n",
    "    train_df = train.copy()\n",
    "    test_df = test.copy()\n",
    "    temp = pd.concat([train_df.manager_id,pd.get_dummies(y_df[\"interest_level\"])], axis = 1).groupby('manager_id').mean()\n",
    "    temp.columns = ['high_frac','low_frac', 'medium_frac']\n",
    "    temp['count_frac'] = train_df.groupby('manager_id').count().iloc[:,1]\n",
    "    temp['manager_skill'] = temp['high_frac']*2 + temp['medium_frac']\n",
    "\n",
    "    # get ixes for unranked managers...\n",
    "    unranked_managers_ixes = temp['count_frac']<20\n",
    "    # ... and ranked ones\n",
    "    ranked_managers_ixes = ~unranked_managers_ixes\n",
    "\n",
    "    # compute mean values from ranked managers and assign them to unranked ones\n",
    "    mean_values = temp.loc[ranked_managers_ixes, ['high_frac','low_frac', 'medium_frac','manager_skill']].mean()\n",
    "    temp.loc[unranked_managers_ixes,['high_frac','low_frac', 'medium_frac','manager_skill']] = mean_values.values\n",
    "    \n",
    "    train_df_index = train_df.index\n",
    "    train_df = train_df.merge(temp.reset_index(),how='left', left_on='manager_id', right_on='manager_id')\n",
    "    train_df.index = train_df_index\n",
    "    \n",
    "    test_df_index = test_df.index\n",
    "    test_df = test_df.merge(temp.reset_index(),how='left', left_on='manager_id', right_on='manager_id')\n",
    "    test_df.loc[test_df['high_frac'].isnull(),['high_frac','low_frac', 'medium_frac','manager_skill']] = mean_values.values\n",
    "    test_df.index = test_df_index\n",
    "    \n",
    "    not_use_col = ['high_frac', 'low_frac', 'medium_frac', 'count_frac']\n",
    "    for col in not_use_col:\n",
    "        del train_df[col]\n",
    "        del test_df[col]\n",
    "    \n",
    "    return train_df,test_df"
   ]
  }
 ],
 "metadata": {
  "anaconda-cloud": {},
  "kernelspec": {
   "display_name": "Python [Root]",
   "language": "python",
   "name": "Python [Root]"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.5.2"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 0
}