raifhack-dspyt-com-final-solution.ipynb

{"metadata":{"kernelspec":{"language":"python","display_name":"Python 3","name":"python3"},"language_info":{"name":"python","version":"3.7.10","mimetype":"text/x-python","codemirror_mode":{"name":"ipython","version":3},"pygments_lexer":"ipython3","nbconvert_exporter":"python","file_extension":".py"}},"nbformat_minor":4,"nbformat":4,"cells":[{"cell_type":"code","source":"!pip install scikit-learn==0.24.0","metadata":{"_uuid":"b8803cdf-c377-435c-9c0a-d82d9287efb0","_cell_guid":"7502d2f9-11d3-4481-b054-dc60ce2f4860","collapsed":false,"jupyter":{"outputs_hidden":false},"execution":{"iopub.status.busy":"2021-09-25T17:32:10.903478Z","iopub.execute_input":"2021-09-25T17:32:10.904167Z","iopub.status.idle":"2021-09-25T17:32:24.217958Z","shell.execute_reply.started":"2021-09-25T17:32:10.904128Z","shell.execute_reply":"2021-09-25T17:32:24.216907Z"},"trusted":true},"execution_count":6,"outputs":[]},{"cell_type":"code","source":"# нужные библиотеки\nimport numpy as np\nimport pandas as pd\nimport gc\nimport random\nimport re\nimport typing\nimport numpy as np\n\nfrom sklearn.model_selection import KFold\nfrom sklearn.model_selection import cross_val_score\nfrom sklearn.model_selection import train_test_split\nfrom sklearn.preprocessing import PolynomialFeatures\nfrom sklearn.metrics import mean_absolute_percentage_error, r2_score, mean_squared_error\n\nimport lightgbm as lgbm\n\npd.set_option(\"display.max_rows\", 100)\npd.set_option(\"display.max_columns\", 100)","metadata":{"_uuid":"f336503b-5c0e-41f3-83a1-bbe510f894e9","_cell_guid":"98a4fd91-c750-4b01-b2c5-dbe111a924cc","collapsed":false,"jupyter":{"outputs_hidden":false},"execution":{"iopub.status.busy":"2021-09-25T17:32:24.221437Z","iopub.execute_input":"2021-09-25T17:32:24.225305Z","iopub.status.idle":"2021-09-25T17:32:27.100122Z","shell.execute_reply.started":"2021-09-25T17:32:24.225260Z","shell.execute_reply":"2021-09-25T17:32:27.099326Z"},"trusted":true},"execution_count":7,"outputs":[]},{"cell_type":"code","source":"# реализация метрики от организаторов\nTHRESHOLD = 0.15\nNEGATIVE_WEIGHT = 1.1\n\ndef deviation_metric_one_sample(y_true: typing.Union[float, int], y_pred: typing.Union[float, int]) -> float:\n    \"\"\"\n    Реализация кастомной метрики для хакатона.\n\n    :param y_true: float, реальная цена\n    :param y_pred: float, предсказанная цена\n    :return: float, значение метрики\n    \"\"\"\n    deviation = (y_pred - y_true) / np.maximum(1e-8, y_true)\n    if np.abs(deviation) <= THRESHOLD:\n        return 0\n    elif deviation <= - 4 * THRESHOLD:\n        return 9 * NEGATIVE_WEIGHT\n    elif deviation < -THRESHOLD:\n        return NEGATIVE_WEIGHT * ((deviation / THRESHOLD) + 1) ** 2\n    elif deviation < 4 * THRESHOLD:\n        return ((deviation / THRESHOLD) - 1) ** 2\n    else:\n        return 9\n\n\ndef deviation_metric(y_true: np.array, y_pred: np.array) -> float:\n    return np.array([deviation_metric_one_sample(y_true[n], y_pred[n]) for n in range(len(y_true))]).mean()\n\ndef median_absolute_percentage_error(y_true: np.array, y_pred: np.array) -> float:\n    return np.median(np.abs(y_pred-y_true)/y_true)\n\ndef metrics_stat(y_true: np.array, y_pred: np.array) -> typing.Dict[str,float]:\n    mape = mean_absolute_percentage_error(y_true, y_pred)\n    mdape = median_absolute_percentage_error(y_true, y_pred)\n    rmse = mean_squared_error(y_true, y_pred, squared=False)\n    r2 = r2_score(y_true, y_pred)\n    raif_metric = deviation_metric(y_true, y_pred)\n    return {'mape':mape, 'mdape':mdape, 'rmse': rmse, 'r2': r2, 'raif_metric':raif_metric}","metadata":{"_uuid":"7f0c4f90-7e55-4869-a560-3d06f4655b59","_cell_guid":"395e842d-c7c5-4168-bab5-4c14e24d6847","collapsed":false,"jupyter":{"outputs_hidden":false},"execution":{"iopub.status.busy":"2021-09-25T17:32:27.103193Z","iopub.execute_input":"2021-09-25T17:32:27.103433Z","iopub.status.idle":"2021-09-25T17:32:27.116176Z","shell.execute_reply.started":"2021-09-25T17:32:27.103403Z","shell.execute_reply":"2021-09-25T17:32:27.115305Z"},"trusted":true},"execution_count":8,"outputs":[]},{"cell_type":"code","source":"# загрузка данных\ntrain = pd.read_csv('../input/raif21/train.csv', low_memory=False)\ntest = pd.read_csv('../input/raif21/test.csv', low_memory=False)\ntest_submission = pd.read_csv('../input/raif21/test_submission.csv', low_memory=False)\n\n# заменим иероглифы, чтобы lgbm в будущем не ругался\ntrain.loc[train['osm_city_nearest_name'] == '饶河县', 'osm_city_nearest_name'] = 'неизвестно'","metadata":{"_uuid":"662f71b1-120f-476e-b055-33344f7395bd","_cell_guid":"d874f280-fc04-4bf1-83ea-e202167fd6c4","collapsed":false,"jupyter":{"outputs_hidden":false},"execution":{"iopub.status.busy":"2021-09-25T17:32:27.118339Z","iopub.execute_input":"2021-09-25T17:32:27.118617Z","iopub.status.idle":"2021-09-25T17:32:32.594017Z","shell.execute_reply.started":"2021-09-25T17:32:27.118584Z","shell.execute_reply":"2021-09-25T17:32:32.593270Z"},"trusted":true},"execution_count":9,"outputs":[]},{"cell_type":"code","source":"# оставим в трейне только города, которые есть в тесте\ntrain = train[train['city'].isin(test['city'])]","metadata":{"_uuid":"00157100-85c3-4fda-a638-ddfed7c1cc3f","_cell_guid":"438e8105-d57e-4a6c-9159-a1c33f6137e6","collapsed":false,"jupyter":{"outputs_hidden":false},"execution":{"iopub.status.busy":"2021-09-25T17:32:32.595382Z","iopub.execute_input":"2021-09-25T17:32:32.595660Z","iopub.status.idle":"2021-09-25T17:32:32.691856Z","shell.execute_reply.started":"2021-09-25T17:32:32.595626Z","shell.execute_reply":"2021-09-25T17:32:32.690927Z"},"trusted":true},"execution_count":10,"outputs":[]},{"cell_type":"code","source":"# оставим только цены по типу 1\ntrain = train[train['price_type'] == 1].reset_index(drop=True)\n\n# в репозитории уже файл обрезанный по этому пункту","metadata":{"_uuid":"ab3f4a6c-4079-465d-b62c-3caf67fe378d","_cell_guid":"73076138-3f8e-4195-a8bb-f5c58dc11b95","collapsed":false,"jupyter":{"outputs_hidden":false},"execution":{"iopub.status.busy":"2021-09-25T17:32:32.693345Z","iopub.execute_input":"2021-09-25T17:32:32.693629Z","iopub.status.idle":"2021-09-25T17:32:32.713889Z","shell.execute_reply.started":"2021-09-25T17:32:32.693597Z","shell.execute_reply":"2021-09-25T17:32:32.713126Z"},"trusted":true},"execution_count":11,"outputs":[]},{"cell_type":"code","source":"# преобразуем некоторые признаки, чтобы можно было использовать для обучения\ntrain['date'] = train['date'].apply(lambda x: x.replace('-', '')).astype(int)\ntrain['id'] = train['id'].apply(lambda x: x.replace('COL_', '')).astype(int)\n\ntest['date'] = test['date'].apply(lambda x: x.replace('-', '')).astype(int)\ntest['id'] = test['id'].apply(lambda x: x.replace('COL_', '')).astype(int)","metadata":{"_uuid":"73bdc51f-1dca-4f90-a8c1-49c6bb1c47ab","_cell_guid":"08f4b1aa-4c05-4411-b86d-e91ba9e6afa3","collapsed":false,"jupyter":{"outputs_hidden":false},"execution":{"iopub.status.busy":"2021-09-25T17:32:32.716394Z","iopub.execute_input":"2021-09-25T17:32:32.716749Z","iopub.status.idle":"2021-09-25T17:32:32.737692Z","shell.execute_reply.started":"2021-09-25T17:32:32.716709Z","shell.execute_reply":"2021-09-25T17:32:32.737054Z"},"trusted":true},"execution_count":12,"outputs":[]},{"cell_type":"code","source":"# выделим таргет и перегруппируем в единный формат данные\ntarget = train['per_square_meter_price'].values\ntrain = train.drop('per_square_meter_price', axis=1)\n\ndata = pd.concat([train, test], axis=0).reset_index(drop=True)","metadata":{"_uuid":"6278fe17-894e-4505-86cf-c862086746bc","_cell_guid":"22373a69-4fc2-465f-a1a7-3f3aa6d4732a","collapsed":false,"jupyter":{"outputs_hidden":false},"execution":{"iopub.status.busy":"2021-09-25T17:32:32.740087Z","iopub.execute_input":"2021-09-25T17:32:32.740571Z","iopub.status.idle":"2021-09-25T17:32:32.758500Z","shell.execute_reply.started":"2021-09-25T17:32:32.740547Z","shell.execute_reply":"2021-09-25T17:32:32.757866Z"},"trusted":true},"execution_count":13,"outputs":[]},{"cell_type":"code","source":"# почистим признак floor\ndata['floor'] = data['floor'].mask(data['floor'] == '-1.0', -1) \\\n              .mask(data['floor'] == '-2.0', -2) \\\n              .mask(data['floor'] == '-3.0', -3) \\\n              .mask(data['floor'] == 'подвал, 1', 1) \\\n              .mask(data['floor'] == 'подвал', -1) \\\n              .mask(data['floor'] == 'цоколь, 1', 1) \\\n              .mask(data['floor'] == '1,2,антресоль', 1) \\\n              .mask(data['floor'] == 'цоколь', 0) \\\n              .mask(data['floor'] == 'тех.этаж (6)', 6) \\\n              .mask(data['floor'] == 'Подвал', -1) \\\n              .mask(data['floor'] == 'Цоколь', 0) \\\n              .mask(data['floor'] == 'фактически на уровне 1 этажа', 1) \\\n              .mask(data['floor'] == '1,2,3', 1) \\\n              .mask(data['floor'] == '1, подвал', 1) \\\n              .mask(data['floor'] == '1,2,3,4', 1) \\\n              .mask(data['floor'] == '1,2', 1) \\\n              .mask(data['floor'] == '1,2,3,4,5', 1) \\\n              .mask(data['floor'] == '5, мансарда', 5) \\\n              .mask(data['floor'] == '1-й, подвал', 1) \\\n              .mask(data['floor'] == '1, подвал, антресоль', 1) \\\n              .mask(data['floor'] == 'мезонин', 2) \\\n              .mask(data['floor'] == 'подвал, 1-3', 1) \\\n              .mask(data['floor'] == '1 (Цокольный этаж)', 0) \\\n              .mask(data['floor'] == '3, Мансарда (4 эт)', 3) \\\n              .mask(data['floor'] == 'подвал,1', 1) \\\n              .mask(data['floor'] == '1, антресоль', 1) \\\n              .mask(data['floor'] == '1-3', 1) \\\n              .mask(data['floor'] == 'мансарда (4эт)', 4) \\\n              .mask(data['floor'] == '1, 2.', 1) \\\n              .mask(data['floor'] == 'подвал , 1 ', 1) \\\n              .mask(data['floor'] == '1, 2', 1) \\\n              .mask(data['floor'] == 'подвал, 1,2,3', 1) \\\n              .mask(data['floor'] == '1 + подвал (без отделки)', 1) \\\n              .mask(data['floor'] == 'мансарда', 3) \\\n              .mask(data['floor'] == '2,3', 2) \\\n              .mask(data['floor'] == '4, 5', 4) \\\n              .mask(data['floor'] == '1-й, 2-й', 1) \\\n              .mask(data['floor'] == '1 этаж, подвал', 1) \\\n              .mask(data['floor'] == '1, цоколь', 1) \\\n              .mask(data['floor'] == 'подвал, 1-7, техэтаж', 1) \\\n              .mask(data['floor'] == '3 (антресоль)', 3) \\\n              .mask(data['floor'] == '1, 2, 3', 1) \\\n              .mask(data['floor'] == 'Цоколь, 1,2(мансарда)', 1) \\\n              .mask(data['floor'] == 'подвал, 3. 4 этаж', 3) \\\n              .mask(data['floor'] == 'подвал, 1-4 этаж', 1) \\\n              .mask(data['floor'] == 'подва, 1.2 этаж', 1) \\\n              .mask(data['floor'] == '2, 3', 2) \\\n              .mask(data['floor'] == '7,8', 7) \\\n              .mask(data['floor'] == '1 этаж', 1) \\\n              .mask(data['floor'] == '1-й', 1) \\\n              .mask(data['floor'] == '3 этаж', 3) \\\n              .mask(data['floor'] == '4 этаж', 4) \\\n              .mask(data['floor'] == '5 этаж', 5) \\\n              .mask(data['floor'] == 'подвал,1,2,3,4,5', 1) \\\n              .mask(data['floor'] == 'подвал, цоколь, 1 этаж', 1) \\\n              .mask(data['floor'] == '3, мансарда', 3) \\\n              .mask(data['floor'] == 'цоколь, 1, 2,3,4,5,6', 1) \\\n              .mask(data['floor'] == ' 1, 2, Антресоль', 1) \\\n              .mask(data['floor'] == '3 этаж, мансарда (4 этаж)', 3) \\\n              .mask(data['floor'] == 'цокольный', 0) \\\n              .mask(data['floor'] == '1,2 ', 1) \\\n              .mask(data['floor'] == '3,4', 3) \\\n              .mask(data['floor'] == 'подвал, 1 и 4 этаж', 1) \\\n              .mask(data['floor'] == '5(мансарда)', 5) \\\n              .mask(data['floor'] == 'технический этаж,5,6', 5) \\\n              .mask(data['floor'] == ' 1-2, подвальный', 1) \\\n              .mask(data['floor'] == '1, 2, 3, мансардный', 1) \\\n              .mask(data['floor'] == 'подвал, 1, 2, 3', 1) \\\n              .mask(data['floor'] == '1,2,3, антресоль, технический этаж', 1) \\\n              .mask(data['floor'] == '3, 4', 3) \\\n              .mask(data['floor'] == '1-3 этажи, цоколь (188,4 кв.м), подвал (104 кв.м)', 1) \\\n              .mask(data['floor'] == '1,2,3,4, подвал', 1) \\\n              .mask(data['floor'] == '2-й', 2) \\\n              .mask(data['floor'] == '1, 2 этаж', 1) \\\n              .mask(data['floor'] == 'подвал, 1, 2', 1) \\\n              .mask(data['floor'] == '1-7', 1) \\\n              .mask(data['floor'] == '1 (по док-м цоколь)', 1) \\\n              .mask(data['floor'] == '1,2,подвал ', 1) \\\n              .mask(data['floor'] == 'подвал, 2', 2) \\\n              .mask(data['floor'] == 'подвал,1,2,3', 1) \\\n              .mask(data['floor'] == '1,2,3 этаж, подвал ', 1) \\\n              .mask(data['floor'] == '1,2,3 этаж, подвал', 1) \\\n              .mask(data['floor'] == '2, 3, 4, тех.этаж', 2) \\\n              .mask(data['floor'] == 'цокольный, 1,2', 1) \\\n              .mask(data['floor'] == 'Техническое подполье', -1) \\\n              .mask(data['floor'] == '1.2', 1) \\\n              .astype(float)","metadata":{"_uuid":"95c1e810-9da6-4644-be28-7eb25dc870fb","_cell_guid":"db1b5c6b-b075-4aa6-9dcb-5fa91cac8740","collapsed":false,"jupyter":{"outputs_hidden":false},"execution":{"iopub.status.busy":"2021-09-25T17:32:32.759688Z","iopub.execute_input":"2021-09-25T17:32:32.760091Z","iopub.status.idle":"2021-09-25T17:32:32.887599Z","shell.execute_reply.started":"2021-09-25T17:32:32.760055Z","shell.execute_reply":"2021-09-25T17:32:32.886885Z"},"trusted":true},"execution_count":14,"outputs":[]},{"cell_type":"code","source":"# так как улица не привяза к городу, нужно делать связку город+улица, но это слишком огромное количество фичей получится\n# поэтому просто дропнем\ndata = data.drop(['street'], axis=1)","metadata":{"_uuid":"886dfab1-31d6-42b4-a610-02b9e25d3978","_cell_guid":"63f535c2-519e-4012-8745-b69e58177573","collapsed":false,"jupyter":{"outputs_hidden":false},"execution":{"iopub.status.busy":"2021-09-25T17:32:32.890637Z","iopub.execute_input":"2021-09-25T17:32:32.890905Z","iopub.status.idle":"2021-09-25T17:32:32.902091Z","shell.execute_reply.started":"2021-09-25T17:32:32.890873Z","shell.execute_reply":"2021-09-25T17:32:32.901306Z"},"trusted":true},"execution_count":15,"outputs":[]},{"cell_type":"code","source":"# выделим категориальные фичи\ncategor_features = ['city', 'osm_city_nearest_name', 'region']\n\n# избавимся от символов, так как значение признака пойдет в его название\ndata['city'] = data['city'].apply(lambda x: re.sub('[^A-ZА-Яа-яa-z0-9_]+', '', x))\ndata['osm_city_nearest_name'] = data['osm_city_nearest_name'].apply(lambda x: re.sub('[^A-ZА-Яа-яa-z0-9_]+', '', x))\ndata['region'] = data['region'].apply(lambda x: re.sub('[^A-ZА-Яа-яa-z0-9_]+', '', x))","metadata":{"_uuid":"f8fa5287-5df2-4e43-9880-302473aee97a","_cell_guid":"3305fd93-4524-404b-998e-416bd186e005","collapsed":false,"jupyter":{"outputs_hidden":false},"execution":{"iopub.status.busy":"2021-09-25T17:32:32.903206Z","iopub.execute_input":"2021-09-25T17:32:32.903656Z","iopub.status.idle":"2021-09-25T17:32:32.956695Z","shell.execute_reply.started":"2021-09-25T17:32:32.903620Z","shell.execute_reply":"2021-09-25T17:32:32.955785Z"},"trusted":true},"execution_count":16,"outputs":[]},{"cell_type":"code","source":"# one hot encoding категориальных признаков\nfor feat in categor_features:\n    data_temp = pd.get_dummies(data[feat], drop_first=True)\n    data.drop(feat, axis=1, inplace=True)\n    data_temp.columns = [feat + '_' + str(col) for col in list(data_temp)]\n    data = pd.concat([data, data_temp], axis=1)","metadata":{"_uuid":"71c7eb98-3fdc-4d7a-a11b-d64bceab799f","_cell_guid":"c8d49e97-5363-47a1-aff4-bc8cdadd57b8","collapsed":false,"jupyter":{"outputs_hidden":false},"execution":{"iopub.status.busy":"2021-09-25T17:32:32.957896Z","iopub.execute_input":"2021-09-25T17:32:32.958159Z","iopub.status.idle":"2021-09-25T17:32:32.997625Z","shell.execute_reply.started":"2021-09-25T17:32:32.958124Z","shell.execute_reply":"2021-09-25T17:32:32.996884Z"},"trusted":true},"execution_count":17,"outputs":[]},{"cell_type":"code","source":"# разделим на итоговые датафреймы\ntrain = data[data['id'].isin(train['id'])]\ntest = data[data['id'].isin(test['id'])]\ntarget = target.copy()","metadata":{"_uuid":"295a684f-e183-4845-8031-3624b934cb85","_cell_guid":"4a967219-f671-418e-a594-8cb48e1e5037","collapsed":false,"jupyter":{"outputs_hidden":false},"execution":{"iopub.status.busy":"2021-09-25T17:32:32.998835Z","iopub.execute_input":"2021-09-25T17:32:32.999179Z","iopub.status.idle":"2021-09-25T17:32:33.018000Z","shell.execute_reply.started":"2021-09-25T17:32:32.999140Z","shell.execute_reply":"2021-09-25T17:32:33.017285Z"},"trusted":true},"execution_count":18,"outputs":[]},{"cell_type":"code","source":"# подбор параметров для LightGBM\n\nimport optuna\nfrom optuna.samplers import TPESampler\nsampler = TPESampler(seed=13)\n\ndef create_model(trial):\n    num_leaves = trial.suggest_int(\"num_leaves\", 2, 1500)\n    n_estimators = trial.suggest_int(\"n_estimators\", 10, 500)\n    max_depth = trial.suggest_int('max_depth', 2, 25)\n    min_child_samples = trial.suggest_int('min_child_samples', 2, 3000)\n    learning_rate = trial.suggest_uniform('learning_rate', 0.00001, 0.99)\n    min_data_in_leaf = trial.suggest_int('min_data_in_leaf', 2, 300)\n    feature_fraction = trial.suggest_uniform('feature_fraction', 0.00001, 1.0)\n    \n    model = lgbm.LGBMRegressor(\n        num_leaves=num_leaves,\n        n_estimators=n_estimators, \n        max_depth=max_depth, \n        min_child_samples=min_child_samples, \n        min_data_in_leaf=min_data_in_leaf,\n        learning_rate=learning_rate,\n        feature_fraction=feature_fraction,\n        random_state=13,\n        n_jobs=-1\n)\n    return model\n\ndef objective(trial):\n    model = create_model(trial)\n    X_train, X_test, y_train, y_test = train_test_split(train, target, random_state=random.randint(1, 10000))\n    model.fit(X_train, y_train)\n    result = model.predict(X_test)\n    score = deviation_metric(y_test.values, result)\n    return score\n\n# раскоментить надо последние 3 строчки для запуска оптимизатора, результат у меня вышел, показанный ниже\n\n# study = optuna.create_study(direction=\"minimize\", sampler=sampler)\n# study.optimize(objective, n_trials=1000)\n# params_lgbm = study.best_params","metadata":{"_uuid":"74290f99-1072-4baf-b9f8-6b49e529e393","_cell_guid":"8de8019d-033c-4f48-8c48-bdfc17da4402","collapsed":false,"jupyter":{"outputs_hidden":false},"execution":{"iopub.status.busy":"2021-09-25T17:32:33.019218Z","iopub.execute_input":"2021-09-25T17:32:33.019575Z","iopub.status.idle":"2021-09-25T17:32:33.030333Z","shell.execute_reply.started":"2021-09-25T17:32:33.019539Z","shell.execute_reply":"2021-09-25T17:32:33.029620Z"},"trusted":true},"execution_count":19,"outputs":[]},{"cell_type":"code","source":"# полученные парметры\nparams_lgbm = {'num_leaves': 887,\n               'n_estimators': 480,\n               'max_depth': 11,\n               'min_child_samples': 1073,\n               'learning_rate': 0.05348257149091985,\n               'min_data_in_leaf': 2,\n               'feature_fraction': 0.9529134909800754\n              }","metadata":{"_uuid":"4d50aa5a-89ae-4975-8365-ea2160936f35","_cell_guid":"1325e800-1684-42e0-9c78-9b44507eb938","collapsed":false,"jupyter":{"outputs_hidden":false},"execution":{"iopub.status.busy":"2021-09-25T17:32:33.031774Z","iopub.execute_input":"2021-09-25T17:32:33.032310Z","iopub.status.idle":"2021-09-25T17:32:33.041263Z","shell.execute_reply.started":"2021-09-25T17:32:33.032273Z","shell.execute_reply":"2021-09-25T17:32:33.040489Z"},"trusted":true},"execution_count":20,"outputs":[]},{"cell_type":"code","source":"# заведем модель\nmodel = lgbm.LGBMRegressor(**params_lgbm)","metadata":{"_uuid":"8a14fdb8-422f-45a1-b5db-d4c1ed57195f","_cell_guid":"3bef4703-32f5-413b-9557-dba30bee689c","collapsed":false,"jupyter":{"outputs_hidden":false},"execution":{"iopub.status.busy":"2021-09-25T17:32:33.044173Z","iopub.execute_input":"2021-09-25T17:32:33.044740Z","iopub.status.idle":"2021-09-25T17:32:33.050181Z","shell.execute_reply.started":"2021-09-25T17:32:33.044708Z","shell.execute_reply":"2021-09-25T17:32:33.049520Z"},"trusted":true},"execution_count":21,"outputs":[]},{"cell_type":"code","source":"%%time\n\n# параметры валидации, обучение будет идти на 5х10 фолдах\nkFold_random_state = [13, 666, 228, 777, 42]\nn_splits = 10\n\n# финальный показатель метрики\nfinal_loss = list()\n\n# файл для записи финального результата\nsubmission = test_submission.copy()\nsubmission.iloc[:, 1] = 0\n\n\nfor ind_k, random_state in enumerate(kFold_random_state):\n    kFold = KFold(n_splits=n_splits, shuffle=True, random_state=random_state)\n    total_loss = list()\n\n    for iteration, (train_index, valid_index) in enumerate(kFold.split(train, target)):\n\n        X_train, X_valid = train.iloc[train_index].reset_index(drop=True), train.iloc[valid_index].reset_index(drop=True)\n        y_train, y_valid = target[train_index], target[valid_index]\n\n        model.fit(X_train, y_train)\n        valid_pred = model.predict(X_valid)\n        loss = deviation_metric(y_valid, valid_pred)\n\n        predict = model.predict(test)\n        submission['per_square_meter_price'] = submission['per_square_meter_price'] + predict / 50\n\n        total_loss.append(np.mean(loss))\n\n    final_loss.append(np.mean(total_loss))\n    print(f'Fold({[\"1-10\", \"11-20\", \"21-30\", \"31-40\", \"41-50\"][ind_k]}) deviation_metric: {np.mean(total_loss)}')\nprint(f'Final deviation_metric: {np.mean(final_loss)}')","metadata":{"_uuid":"f18dfea3-f2c8-44f7-b8f9-4db708aeeb2f","_cell_guid":"2af128cf-d333-4617-9711-9a45b2c17f2d","collapsed":false,"scrolled":true,"jupyter":{"outputs_hidden":false},"execution":{"iopub.status.busy":"2021-09-25T16:37:17.665989Z","iopub.execute_input":"2021-09-25T16:37:17.666563Z","iopub.status.idle":"2021-09-25T16:44:07.932417Z","shell.execute_reply.started":"2021-09-25T16:37:17.666531Z","shell.execute_reply":"2021-09-25T16:44:07.931725Z"},"trusted":true},"execution_count":20,"outputs":[]},{"cell_type":"code","source":"submission_raw = submission.copy()","metadata":{"_uuid":"b8733d8a-9e7b-4304-b46d-845b39317ff9","_cell_guid":"2efbc711-df7c-4839-b803-1c7fbd03fa78","collapsed":false,"jupyter":{"outputs_hidden":false},"execution":{"iopub.status.busy":"2021-09-25T16:44:07.936754Z","iopub.execute_input":"2021-09-25T16:44:07.940181Z","iopub.status.idle":"2021-09-25T16:44:07.944887Z","shell.execute_reply.started":"2021-09-25T16:44:07.940139Z","shell.execute_reply":"2021-09-25T16:44:07.944237Z"},"trusted":true},"execution_count":21,"outputs":[]},{"cell_type":"code","source":"# немного подрежем выбросы сверху\nsubmission_raw['per_square_meter_price'] = submission_raw['per_square_meter_price'] * 0.9\n\nsubmission_raw.loc[submission_raw['per_square_meter_price'] >= 200000, 'per_square_meter_price'] \\\n    = submission_raw.loc[submission_raw['per_square_meter_price'] >= 200000, 'per_square_meter_price'] * 0.9","metadata":{"_uuid":"9156c99f-3b19-48b3-9078-4bdca67503f1","_cell_guid":"d9e41794-80c8-4ce9-959a-aff110a8e3a2","collapsed":false,"jupyter":{"outputs_hidden":false},"execution":{"iopub.status.busy":"2021-09-25T16:44:07.945864Z","iopub.execute_input":"2021-09-25T16:44:07.946083Z","iopub.status.idle":"2021-09-25T16:44:07.960536Z","shell.execute_reply.started":"2021-09-25T16:44:07.946058Z","shell.execute_reply":"2021-09-25T16:44:07.959847Z"},"trusted":true},"execution_count":22,"outputs":[]},{"cell_type":"code","source":"submission_raw.to_csv('submission_lgb.csv', index=False)","metadata":{"_uuid":"9c8f180f-76ee-4bab-8398-82038f1473e5","_cell_guid":"2306a3a7-5a76-4d51-8607-7d9f1a821b40","collapsed":false,"jupyter":{"outputs_hidden":false},"execution":{"iopub.status.busy":"2021-09-25T16:44:07.962243Z","iopub.execute_input":"2021-09-25T16:44:07.962480Z","iopub.status.idle":"2021-09-25T16:44:07.984215Z","shell.execute_reply.started":"2021-09-25T16:44:07.962454Z","shell.execute_reply":"2021-09-25T16:44:07.983376Z"},"trusted":true},"execution_count":23,"outputs":[]},{"cell_type":"code","source":"import xgboost as xgb","metadata":{"_uuid":"decb31ee-8e8c-4062-9704-9325cacd3da0","_cell_guid":"5aaf23ae-22b8-401a-a101-2852aadd2153","collapsed":false,"jupyter":{"outputs_hidden":false},"execution":{"iopub.status.busy":"2021-09-25T16:44:07.985183Z","iopub.execute_input":"2021-09-25T16:44:07.985421Z","iopub.status.idle":"2021-09-25T16:44:08.070773Z","shell.execute_reply.started":"2021-09-25T16:44:07.985395Z","shell.execute_reply":"2021-09-25T16:44:08.069816Z"},"trusted":true},"execution_count":24,"outputs":[]},{"cell_type":"code","source":"xgb_params = {\n    'eta': 0.05,\n    'max_depth': 5,\n    'subsample': 0.7,\n    'colsample_bytree': 0.7,\n    'objective': 'reg:linear',\n    'eval_metric': 'rmse',\n}","metadata":{"execution":{"iopub.status.busy":"2021-09-25T16:44:08.074721Z","iopub.execute_input":"2021-09-25T16:44:08.074990Z","iopub.status.idle":"2021-09-25T16:44:08.080617Z","shell.execute_reply.started":"2021-09-25T16:44:08.074962Z","shell.execute_reply":"2021-09-25T16:44:08.079462Z"},"trusted":true},"execution_count":25,"outputs":[]},{"cell_type":"code","source":"# заведем модель\nmodel = xgb.XGBRegressor(**xgb_params)","metadata":{"_uuid":"2b7a5780-819a-4b55-8505-1d058a7e46a6","_cell_guid":"1875f924-fcfd-4ad5-af97-f604af82f347","collapsed":false,"jupyter":{"outputs_hidden":false},"execution":{"iopub.status.busy":"2021-09-25T16:44:08.082107Z","iopub.execute_input":"2021-09-25T16:44:08.082412Z","iopub.status.idle":"2021-09-25T16:44:08.095587Z","shell.execute_reply.started":"2021-09-25T16:44:08.082371Z","shell.execute_reply":"2021-09-25T16:44:08.094836Z"},"trusted":true},"execution_count":26,"outputs":[]},{"cell_type":"code","source":"%%time\n\n# параметры валидации, обучение будет идти на 5х10 фолдах\nkFold_random_state = [13, 666, 228, 777, 42]\nn_splits = 10\n\n# финальный показатель метрики\nfinal_loss = list()\n\n# файл для записи финального результата\nsubmission = test_submission.copy()\nsubmission.iloc[:, 1] = 0\n\n\nfor ind_k, random_state in enumerate(kFold_random_state):\n    kFold = KFold(n_splits=n_splits, shuffle=True, random_state=random_state)\n    total_loss = list()\n\n    for iteration, (train_index, valid_index) in enumerate(kFold.split(train, target)):\n\n        X_train, X_valid = train.iloc[train_index].reset_index(drop=True), train.iloc[valid_index].reset_index(drop=True)\n        y_train, y_valid = target[train_index], target[valid_index]\n\n        model.fit(X_train, y_train)\n        valid_pred = model.predict(X_valid)\n        loss = deviation_metric(y_valid, valid_pred)\n\n        predict = model.predict(test)\n        submission['per_square_meter_price'] = submission['per_square_meter_price'] + predict / 50\n\n        total_loss.append(np.mean(loss))\n\n    final_loss.append(np.mean(total_loss))\n    print(f'Fold({[\"1-10\", \"11-20\", \"21-30\", \"31-40\", \"41-50\"][ind_k]}) deviation_metric: {np.mean(total_loss)}')\nprint(f'Final deviation_metric: {np.mean(final_loss)}')","metadata":{"_uuid":"06b1f838-7cf9-4dea-bb98-e095da7692ac","_cell_guid":"185c68b2-0531-4070-8c61-417193a94a4f","collapsed":false,"jupyter":{"outputs_hidden":false},"execution":{"iopub.status.busy":"2021-09-25T16:44:08.097958Z","iopub.execute_input":"2021-09-25T16:44:08.098180Z","iopub.status.idle":"2021-09-25T16:46:22.465634Z","shell.execute_reply.started":"2021-09-25T16:44:08.098157Z","shell.execute_reply":"2021-09-25T16:46:22.464921Z"},"trusted":true},"execution_count":27,"outputs":[]},{"cell_type":"code","source":"submission_xgb = submission.copy()\n\n# немного подрежем выбросы сверху\nsubmission_xgb['per_square_meter_price'] = submission_xgb['per_square_meter_price'] * 0.9\n\nsubmission_xgb.loc[submission_xgb['per_square_meter_price'] >= 200000, 'per_square_meter_price'] \\\n    = submission_xgb.loc[submission_xgb['per_square_meter_price'] >= 200000, 'per_square_meter_price'] * 0.9","metadata":{"_uuid":"69154567-b9e6-4287-a10f-297ec1bebbac","_cell_guid":"bd0606c5-6fe1-45cd-a51d-6d3ed806e8fe","collapsed":false,"jupyter":{"outputs_hidden":false},"execution":{"iopub.status.busy":"2021-09-25T16:46:22.469664Z","iopub.execute_input":"2021-09-25T16:46:22.470136Z","iopub.status.idle":"2021-09-25T16:46:22.482578Z","shell.execute_reply.started":"2021-09-25T16:46:22.470094Z","shell.execute_reply":"2021-09-25T16:46:22.481856Z"},"trusted":true},"execution_count":28,"outputs":[]},{"cell_type":"code","source":"import catboost","metadata":{"execution":{"iopub.status.busy":"2021-09-25T16:46:22.486677Z","iopub.execute_input":"2021-09-25T16:46:22.487027Z","iopub.status.idle":"2021-09-25T16:46:22.695665Z","shell.execute_reply.started":"2021-09-25T16:46:22.486970Z","shell.execute_reply":"2021-09-25T16:46:22.695054Z"},"trusted":true},"execution_count":29,"outputs":[]},{"cell_type":"code","source":"# заведем модель\nmodel = catboost.CatBoostRegressor()","metadata":{"execution":{"iopub.status.busy":"2021-09-25T16:46:22.698983Z","iopub.execute_input":"2021-09-25T16:46:22.699265Z","iopub.status.idle":"2021-09-25T16:46:22.705499Z","shell.execute_reply.started":"2021-09-25T16:46:22.699239Z","shell.execute_reply":"2021-09-25T16:46:22.704968Z"},"trusted":true},"execution_count":30,"outputs":[]},{"cell_type":"code","source":"%%time\n\n# параметры валидации, обучение будет идти на 5х10 фолдах\nkFold_random_state = [13, 666, 228, 777, 42]\nn_splits = 10\n\n# финальный показатель метрики\nfinal_loss = list()\n\n# файл для записи финального результата\nsubmission = test_submission.copy()\nsubmission.iloc[:, 1] = 0\n\n\nfor ind_k, random_state in enumerate(kFold_random_state):\n    kFold = KFold(n_splits=n_splits, shuffle=True, random_state=random_state)\n    total_loss = list()\n\n    for iteration, (train_index, valid_index) in enumerate(kFold.split(train, target)):\n\n        X_train, X_valid = train.iloc[train_index].reset_index(drop=True), train.iloc[valid_index].reset_index(drop=True)\n        y_train, y_valid = target[train_index], target[valid_index]\n\n        model.fit(X_train, y_train, verbose=0)\n        valid_pred = model.predict(X_valid)\n        loss = deviation_metric(y_valid, valid_pred)\n\n        predict = model.predict(test)\n        submission['per_square_meter_price'] = submission['per_square_meter_price'] + predict / 50\n\n        total_loss.append(np.mean(loss))\n\n    final_loss.append(np.mean(total_loss))\n    print(f'Fold({[\"1-10\", \"11-20\", \"21-30\", \"31-40\", \"41-50\"][ind_k]}) deviation_metric: {np.mean(total_loss)}')\nprint(f'Final deviation_metric: {np.mean(final_loss)}')","metadata":{"execution":{"iopub.status.busy":"2021-09-25T16:46:55.235439Z","iopub.execute_input":"2021-09-25T16:46:55.236353Z","iopub.status.idle":"2021-09-25T16:52:23.463887Z","shell.execute_reply.started":"2021-09-25T16:46:55.236320Z","shell.execute_reply":"2021-09-25T16:52:23.462950Z"},"trusted":true},"execution_count":35,"outputs":[]},{"cell_type":"code","source":"submission_cat = submission.copy()\n\n# немного подрежем выбросы сверху\nsubmission_cat['per_square_meter_price'] = submission_cat['per_square_meter_price'] * 0.9\n\nsubmission_cat.loc[submission_cat['per_square_meter_price'] >= 200000, 'per_square_meter_price'] \\\n    = submission_cat.loc[submission_cat['per_square_meter_price'] >= 200000, 'per_square_meter_price'] * 0.9","metadata":{"execution":{"iopub.status.busy":"2021-09-25T16:52:23.465825Z","iopub.execute_input":"2021-09-25T16:52:23.466068Z","iopub.status.idle":"2021-09-25T16:52:23.473976Z","shell.execute_reply.started":"2021-09-25T16:52:23.466041Z","shell.execute_reply":"2021-09-25T16:52:23.473182Z"},"trusted":true},"execution_count":36,"outputs":[]},{"cell_type":"code","source":"# submission = submission.copy()\n\nsubmission['per_square_meter_price'] = submission_cat['per_square_meter_price'].values*0.2 + submission_xgb['per_square_meter_price']*0.05 + \\\n    submission_raw['per_square_meter_price'].values*0.75\nsubmission.loc[submission['per_square_meter_price'] >= 200000, 'per_square_meter_price'] \\\n    = submission.loc[submission['per_square_meter_price'] >= 200000, 'per_square_meter_price'] * 0.94","metadata":{"execution":{"iopub.status.busy":"2021-09-25T18:02:59.984447Z","iopub.execute_input":"2021-09-25T18:02:59.985184Z","iopub.status.idle":"2021-09-25T18:02:59.993426Z","shell.execute_reply.started":"2021-09-25T18:02:59.985144Z","shell.execute_reply":"2021-09-25T18:02:59.992548Z"},"trusted":true},"execution_count":19,"outputs":[]},{"cell_type":"code","source":"submission.to_csv('submission.csv', index=False)","metadata":{"execution":{"iopub.status.busy":"2021-09-25T18:03:00.677624Z","iopub.execute_input":"2021-09-25T18:03:00.677929Z","iopub.status.idle":"2021-09-25T18:03:00.697906Z","shell.execute_reply.started":"2021-09-25T18:03:00.677900Z","shell.execute_reply":"2021-09-25T18:03:00.696804Z"},"trusted":true},"execution_count":20,"outputs":[]},{"cell_type":"code","source":"submission","metadata":{"execution":{"iopub.status.busy":"2021-09-25T18:03:05.817915Z","iopub.execute_input":"2021-09-25T18:03:05.818206Z","iopub.status.idle":"2021-09-25T18:03:05.833207Z","shell.execute_reply.started":"2021-09-25T18:03:05.818177Z","shell.execute_reply":"2021-09-25T18:03:05.832025Z"},"trusted":true},"execution_count":21,"outputs":[]},{"cell_type":"code","source":"","metadata":{},"execution_count":null,"outputs":[]}]}