Mutual_info.ipynb

{"nbformat":4,"nbformat_minor":0,"metadata":{"kernelspec":{"display_name":"Python 3","language":"python","name":"python3"},"language_info":{"codemirror_mode":{"name":"ipython","version":3},"file_extension":".py","mimetype":"text/x-python","name":"python","nbconvert_exporter":"python","pygments_lexer":"ipython3","version":"3.7.3"},"colab":{"name":"Mutual_info.ipynb","provenance":[{"file_id":"0B8Qi3IatzTvddWQ4R2ZaVU1zTU5ieEFIX2hwdVBtQXBQRmw0","timestamp":1576097189184}]}},"cells":[{"cell_type":"code","metadata":{"id":"5ZxgRmA6tiBO","colab_type":"code","colab":{}},"source":["from sklearn.feature_selection import mutual_info_classif\n","import pandas as pd\n","import numpy as np\n","from sklearn.feature_selection import SelectKBest\n","from sklearn.feature_selection import chi2\n","from scipy.stats import chisquare"],"execution_count":0,"outputs":[]},{"cell_type":"code","metadata":{"id":"j8YsV_RstiBa","colab_type":"code","colab":{}},"source":["#MAPPING OF COLUMNS to be read as objects\n","\n","map_object_cols={}\n","\n","map_object_cols['ProductCD']='object'\n","map_object_cols['DeviceType']='object'\n","map_object_cols['addr1']='object'\n","map_object_cols['addr2']='object'\n","map_object_cols['device_name']='object'\n","map_object_cols['had_id']='object'\n","\n","map_object_cols['P_emaildomain_bin']='object'\n","map_object_cols['P_emaildomain_suffix']='object'\n","map_object_cols['R_emaildomain_bin']='object'\n","map_object_cols['R_emaildomain_suffix']='object'\n","\n","map_object_cols['_Month']='object'\n","map_object_cols['_Weekdays']='object'\n","map_object_cols['_Days']='object'\n","map_object_cols['_Hours']='object'\n","\n","for i in range(12,39):\n","    col_name= 'id_'+str(i)\n","    map_object_cols[col_name]='object'\n","\n","for i in range(1,10):\n","    col_name= 'M'+str(i)\n","    map_object_cols[col_name]='object'\n","\n","for i in range(1,7):\n","    col_name= 'card'+str(i)\n","    map_object_cols[col_name]='object'"],"execution_count":0,"outputs":[]},{"cell_type":"code","metadata":{"id":"F4Z2tydGtiBi","colab_type":"code","colab":{"base_uri":"https://localhost:8080/","height":380},"outputId":"e51a0938-69bc-4c40-c51d-1dc004c40de8","executionInfo":{"status":"error","timestamp":1577604383316,"user_tz":-120,"elapsed":1456,"user":{"displayName":"din bav","photoUrl":"","userId":"06942881549123238413"}}},"source":["#df=pd.read_csv('train-fraud_data_remove_nulls_fill_mean-mode.csv',compression='gzip', dtype=map_object_cols)#1\n","df=pd.read_csv('train-fraud_data_remove_nulls_fill_random_distribution.csv',compression='gzip', dtype=map_object_cols)#2"],"execution_count":3,"outputs":[{"output_type":"error","ename":"FileNotFoundError","evalue":"ignored","traceback":["\u001b[0;31m---------------------------------------------------------------------------\u001b[0m","\u001b[0;31mFileNotFoundError\u001b[0m                         Traceback (most recent call last)","\u001b[0;32m<ipython-input-3-2157f6c11151>\u001b[0m in \u001b[0;36m<module>\u001b[0;34m()\u001b[0m\n\u001b[0;32m----> 1\u001b[0;31m \u001b[0mdf\u001b[0m\u001b[0;34m=\u001b[0m\u001b[0mpd\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mread_csv\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0;34m'train-fraud_data_remove_nulls_fill_random_distribution.csv'\u001b[0m\u001b[0;34m,\u001b[0m\u001b[0mcompression\u001b[0m\u001b[0;34m=\u001b[0m\u001b[0;34m'gzip'\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mdtype\u001b[0m\u001b[0;34m=\u001b[0m\u001b[0mmap_object_cols\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;31m#2\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0m","\u001b[0;32m/usr/local/lib/python3.6/dist-packages/pandas/io/parsers.py\u001b[0m in \u001b[0;36mparser_f\u001b[0;34m(filepath_or_buffer, sep, delimiter, header, names, index_col, usecols, squeeze, prefix, mangle_dupe_cols, dtype, engine, converters, true_values, false_values, skipinitialspace, skiprows, skipfooter, nrows, na_values, keep_default_na, na_filter, verbose, skip_blank_lines, parse_dates, infer_datetime_format, keep_date_col, date_parser, dayfirst, cache_dates, iterator, chunksize, compression, thousands, decimal, lineterminator, quotechar, quoting, doublequote, escapechar, comment, encoding, dialect, error_bad_lines, warn_bad_lines, delim_whitespace, low_memory, memory_map, float_precision)\u001b[0m\n\u001b[1;32m    683\u001b[0m         )\n\u001b[1;32m    684\u001b[0m \u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0;32m--> 685\u001b[0;31m         \u001b[0;32mreturn\u001b[0m \u001b[0m_read\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mfilepath_or_buffer\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mkwds\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0m\u001b[1;32m    686\u001b[0m \u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m    687\u001b[0m     \u001b[0mparser_f\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0m__name__\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0mname\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n","\u001b[0;32m/usr/local/lib/python3.6/dist-packages/pandas/io/parsers.py\u001b[0m in \u001b[0;36m_read\u001b[0;34m(filepath_or_buffer, kwds)\u001b[0m\n\u001b[1;32m    455\u001b[0m \u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m    456\u001b[0m     \u001b[0;31m# Create the parser.\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0;32m--> 457\u001b[0;31m     \u001b[0mparser\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0mTextFileReader\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mfp_or_buf\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0;34m**\u001b[0m\u001b[0mkwds\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0m\u001b[1;32m    458\u001b[0m \u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m    459\u001b[0m     \u001b[0;32mif\u001b[0m \u001b[0mchunksize\u001b[0m \u001b[0;32mor\u001b[0m \u001b[0miterator\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n","\u001b[0;32m/usr/local/lib/python3.6/dist-packages/pandas/io/parsers.py\u001b[0m in \u001b[0;36m__init__\u001b[0;34m(self, f, engine, **kwds)\u001b[0m\n\u001b[1;32m    893\u001b[0m             \u001b[0mself\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0moptions\u001b[0m\u001b[0;34m[\u001b[0m\u001b[0;34m\"has_index_names\"\u001b[0m\u001b[0;34m]\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0mkwds\u001b[0m\u001b[0;34m[\u001b[0m\u001b[0;34m\"has_index_names\"\u001b[0m\u001b[0;34m]\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m    894\u001b[0m \u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0;32m--> 895\u001b[0;31m         \u001b[0mself\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0m_make_engine\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mself\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mengine\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0m\u001b[1;32m    896\u001b[0m \u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m    897\u001b[0m     \u001b[0;32mdef\u001b[0m \u001b[0mclose\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mself\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n","\u001b[0;32m/usr/local/lib/python3.6/dist-packages/pandas/io/parsers.py\u001b[0m in \u001b[0;36m_make_engine\u001b[0;34m(self, engine)\u001b[0m\n\u001b[1;32m   1133\u001b[0m     \u001b[0;32mdef\u001b[0m \u001b[0m_make_engine\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mself\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mengine\u001b[0m\u001b[0;34m=\u001b[0m\u001b[0;34m\"c\"\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m   1134\u001b[0m         \u001b[0;32mif\u001b[0m \u001b[0mengine\u001b[0m \u001b[0;34m==\u001b[0m \u001b[0;34m\"c\"\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0;32m-> 1135\u001b[0;31m             \u001b[0mself\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0m_engine\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0mCParserWrapper\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mself\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mf\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0;34m**\u001b[0m\u001b[0mself\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0moptions\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0m\u001b[1;32m   1136\u001b[0m         \u001b[0;32melse\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m   1137\u001b[0m             \u001b[0;32mif\u001b[0m \u001b[0mengine\u001b[0m \u001b[0;34m==\u001b[0m \u001b[0;34m\"python\"\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n","\u001b[0;32m/usr/local/lib/python3.6/dist-packages/pandas/io/parsers.py\u001b[0m in \u001b[0;36m__init__\u001b[0;34m(self, src, **kwds)\u001b[0m\n\u001b[1;32m   1915\u001b[0m         \u001b[0mkwds\u001b[0m\u001b[0;34m[\u001b[0m\u001b[0;34m\"usecols\"\u001b[0m\u001b[0;34m]\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0mself\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0musecols\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m   1916\u001b[0m \u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0;32m-> 1917\u001b[0;31m         \u001b[0mself\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0m_reader\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0mparsers\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mTextReader\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0msrc\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0;34m**\u001b[0m\u001b[0mkwds\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0m\u001b[1;32m   1918\u001b[0m         \u001b[0mself\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0munnamed_cols\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0mself\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0m_reader\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0munnamed_cols\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m   1919\u001b[0m \u001b[0;34m\u001b[0m\u001b[0m\n","\u001b[0;32mpandas/_libs/parsers.pyx\u001b[0m in \u001b[0;36mpandas._libs.parsers.TextReader.__cinit__\u001b[0;34m()\u001b[0m\n","\u001b[0;32mpandas/_libs/parsers.pyx\u001b[0m in \u001b[0;36mpandas._libs.parsers.TextReader._setup_parser_source\u001b[0;34m()\u001b[0m\n","\u001b[0;32m/usr/lib/python3.6/gzip.py\u001b[0m in \u001b[0;36m__init__\u001b[0;34m(self, filename, mode, compresslevel, fileobj, mtime)\u001b[0m\n\u001b[1;32m    161\u001b[0m             \u001b[0mmode\u001b[0m \u001b[0;34m+=\u001b[0m \u001b[0;34m'b'\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m    162\u001b[0m         \u001b[0;32mif\u001b[0m \u001b[0mfileobj\u001b[0m \u001b[0;32mis\u001b[0m \u001b[0;32mNone\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0;32m--> 163\u001b[0;31m             \u001b[0mfileobj\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0mself\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mmyfileobj\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0mbuiltins\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mopen\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mfilename\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mmode\u001b[0m \u001b[0;32mor\u001b[0m \u001b[0;34m'rb'\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0m\u001b[1;32m    164\u001b[0m         \u001b[0;32mif\u001b[0m \u001b[0mfilename\u001b[0m \u001b[0;32mis\u001b[0m \u001b[0;32mNone\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m    165\u001b[0m             \u001b[0mfilename\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0mgetattr\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mfileobj\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0;34m'name'\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0;34m''\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n","\u001b[0;31mFileNotFoundError\u001b[0m: [Errno 2] No such file or directory: 'train-fraud_data_remove_nulls_fill_random_distribution.csv'"]}]},{"cell_type":"code","metadata":{"id":"7tO_af7AtiBp","colab_type":"code","colab":{}},"source":["df.drop('had_id', axis=1,inplace=True)"],"execution_count":0,"outputs":[]},{"cell_type":"code","metadata":{"id":"8CBza15MtiBx","colab_type":"code","colab":{}},"source":["# TO DROP?\n","\n","\n","y=df['isFraud']\n","df_numeric=df._get_numeric_data()\n","df_numeric.drop('isFraud', axis=1,inplace=True)\n","numeric_cols=list(df_numeric.columns)\n","len(numeric_cols)"],"execution_count":0,"outputs":[]},{"cell_type":"code","metadata":{"id":"FZgiyqNUtiB3","colab_type":"code","outputId":"1f2ce948-0d38-4401-b966-4c2b6cfbaef3","colab":{}},"source":["y=df['isFraud']\n","df_category=df.select_dtypes(include=['object'])\n","df_category.shape"],"execution_count":0,"outputs":[{"output_type":"execute_result","data":{"text/plain":["(413377, 42)"]},"metadata":{"tags":[]},"execution_count":7}]},{"cell_type":"code","metadata":{"id":"E6fa4z5AtiCD","colab_type":"code","outputId":"b54444ae-ae2e-4dec-ef64-e29f868a5f85","colab":{}},"source":["object_cols=df_category.columns\n","len(object_cols)\n","#object_cols"],"execution_count":0,"outputs":[{"output_type":"execute_result","data":{"text/plain":["42"]},"metadata":{"tags":[]},"execution_count":8}]},{"cell_type":"code","metadata":{"id":"klvGDaiAtiCJ","colab_type":"code","outputId":"f6c8b100-2e67-4629-a9b2-36f171bd5b1e","colab":{}},"source":["df_category.isnull().sum().sum()"],"execution_count":0,"outputs":[{"output_type":"execute_result","data":{"text/plain":["0"]},"metadata":{"tags":[]},"execution_count":9}]},{"cell_type":"code","metadata":{"id":"JNSJ_9_btiCR","colab_type":"code","outputId":"a804a8f3-36a4-4ed1-e24d-0f3675a082f8","colab":{}},"source":["from pandas.api.types import CategoricalDtype\n","\n","# M1-M9: T/F/Blank\n","df_category['M1']=df_category['M1'].astype(CategoricalDtype(categories=['F','T']))\n","df_category['M1'] = df_category.M1.cat.codes\n","df_category['M2']=df_category['M2'].astype(CategoricalDtype(categories=['F','T']))\n","df_category['M2'] = df_category.M2.cat.codes\n","df_category['M3']=df_category['M3'].astype(CategoricalDtype(categories=['F','T']))\n","df_category['M3'] = df_category.M3.cat.codes\n","\n","df_category['M4']=df_category['M4'].astype(CategoricalDtype(categories=['M0','M1','M2']))\n","df_category['M4'] = df_category.M4.cat.codes\n","\n","df_category['M5']=df_category['M5'].astype(CategoricalDtype(categories=['F','T']))\n","df_category['M5'] = df_category.M5.cat.codes\n","df_category['M6']=df_category['M6'].astype(CategoricalDtype(categories=['F','T']))\n","df_category['M6'] = df_category.M6.cat.codes\n","df_category['M7']=df_category['M7'].astype(CategoricalDtype(categories=['F','T']))\n","df_category['M7'] = df_category.M7.cat.codes\n","df_category['M8']=df_category['M8'].astype(CategoricalDtype(categories=['F','T']))\n","df_category['M8'] = df_category.M8.cat.codes\n","df_category['M9']=df_category['M9'].astype(CategoricalDtype(categories=['F','T']))\n","df_category['M9'] = df_category.M9.cat.codes\n","\n","df_category['ProductCD']=df_category['ProductCD'].astype(CategoricalDtype())\n","df_category['ProductCD'] = df_category.ProductCD.cat.codes\n","\n","df_category['device_name']=df_category['device_name'].astype(CategoricalDtype())\n","df_category['device_name'] = df_category.device_name.cat.codes\n","\n","df_category['DeviceType']=df_category['DeviceType'].astype(CategoricalDtype())\n","df_category['DeviceType'] = df_category.DeviceType.cat.codes\n","\n","df_category['addr1']=df_category['addr1'].astype(CategoricalDtype())\n","df_category['addr1'] = df_category.addr1.cat.codes\n","df_category['addr2']=df_category['addr2'].astype(CategoricalDtype())\n","df_category['addr2'] = df_category.addr2.cat.codes\n","\n","df_category['card1']=df_category['card1'].astype(CategoricalDtype())\n","df_category['card1'] = df_category.card1.cat.codes\n","df_category['card2']=df_category['card2'].astype(CategoricalDtype())\n","df_category['card2'] = df_category.card2.cat.codes\n","df_category['card3']=df_category['card3'].astype(CategoricalDtype())\n","df_category['card3'] = df_category.card3.cat.codes\n","df_category['card4']=df_category['card4'].astype(CategoricalDtype())\n","df_category['card4'] = df_category.card4.cat.codes\n","df_category['card5']=df_category['card5'].astype(CategoricalDtype())\n","df_category['card5'] = df_category.card5.cat.codes\n","df_category['card6']=df_category['card6'].astype(CategoricalDtype())\n","df_category['card6'] = df_category.card6.cat.codes\n","\n","df_category['P_emaildomain_bin']=df_category['P_emaildomain_bin'].astype(CategoricalDtype())\n","df_category['P_emaildomain_bin'] = df_category.P_emaildomain_bin.cat.codes\n","df_category['R_emaildomain_bin']=df_category['R_emaildomain_bin'].astype(CategoricalDtype())\n","df_category['R_emaildomain_bin'] = df_category.R_emaildomain_bin.cat.codes\n","df_category['P_emaildomain_suffix']=df_category['P_emaildomain_suffix'].astype(CategoricalDtype())\n","df_category['P_emaildomain_suffix'] = df_category.P_emaildomain_suffix.cat.codes\n","df_category['R_emaildomain_suffix']=df_category['R_emaildomain_suffix'].astype(CategoricalDtype())\n","df_category['R_emaildomain_suffix'] = df_category.R_emaildomain_suffix.cat.codes\n","\n","df_category['_Month']=df_category['_Month'].astype(CategoricalDtype())\n","df_category['_Month'] = df_category._Month.cat.codes\n","df_category['_Weekdays']=df_category['_Weekdays'].astype(CategoricalDtype())\n","df_category['_Weekdays'] = df_category._Weekdays.cat.codes\n","df_category['_Hours']=df_category['_Hours'].astype(CategoricalDtype())\n","df_category['_Hours'] = df_category._Hours.cat.codes\n","df_category['_Days']=df_category['_Days'].astype(CategoricalDtype())\n","df_category['_Days'] = df_category._Days.cat.codes\n","\n","df_category['id_12']=df_category['id_12'].astype(CategoricalDtype())\n","df_category['id_12'] = df_category.id_12.cat.codes \n","df_category['id_13']=df_category['id_13'].astype(CategoricalDtype())\n","df_category['id_13'] = df_category.id_13.cat.codes \n","df_category['id_15']=df_category['id_15'].astype(CategoricalDtype())\n","df_category['id_15'] = df_category.id_15.cat.codes \n","df_category['id_16']=df_category['id_16'].astype(CategoricalDtype())\n","df_category['id_16'] = df_category.id_16.cat.codes \n","df_category['id_17']=df_category['id_17'].astype(CategoricalDtype())\n","df_category['id_17'] = df_category.id_17.cat.codes \n","df_category['id_19']=df_category['id_19'].astype(CategoricalDtype())\n","df_category['id_19'] = df_category.id_19.cat.codes \n","df_category['id_20']=df_category['id_20'].astype(CategoricalDtype())\n","df_category['id_20'] = df_category.id_20.cat.codes\n","df_category['id_28']=df_category['id_28'].astype(CategoricalDtype())\n","df_category['id_28'] = df_category.id_28.cat.codes\n","df_category['id_29']=df_category['id_29'].astype(CategoricalDtype())\n","df_category['id_29'] = df_category.id_29.cat.codes\n","df_category['id_31']=df_category['id_31'].astype(CategoricalDtype())\n","df_category['id_31'] = df_category.id_31.cat.codes\n","df_category['id_35']=df_category['id_35'].astype(CategoricalDtype())\n","df_category['id_35'] = df_category.id_35.cat.codes\n","df_category['id_36']=df_category['id_36'].astype(CategoricalDtype())\n","df_category['id_36'] = df_category.id_36.cat.codes\n","df_category['id_37']=df_category['id_37'].astype(CategoricalDtype())\n","df_category['id_37'] = df_category.id_37.cat.codes\n","df_category['id_38']=df_category['id_38'].astype(CategoricalDtype())\n","df_category['id_38'] = df_category.id_38.cat.codes"],"execution_count":0,"outputs":[{"output_type":"stream","text":["C:\\Anaconda\\lib\\site-packages\\ipykernel_launcher.py:4: SettingWithCopyWarning: \n","A value is trying to be set on a copy of a slice from a DataFrame.\n","Try using .loc[row_indexer,col_indexer] = value instead\n","\n","See the caveats in the documentation: http://pandas.pydata.org/pandas-docs/stable/user_guide/indexing.html#returning-a-view-versus-a-copy\n","  after removing the cwd from sys.path.\n","C:\\Anaconda\\lib\\site-packages\\ipykernel_launcher.py:5: SettingWithCopyWarning: \n","A value is trying to be set on a copy of a slice from a DataFrame.\n","Try using .loc[row_indexer,col_indexer] = value instead\n","\n","See the caveats in the documentation: http://pandas.pydata.org/pandas-docs/stable/user_guide/indexing.html#returning-a-view-versus-a-copy\n","  \"\"\"\n","C:\\Anaconda\\lib\\site-packages\\ipykernel_launcher.py:6: SettingWithCopyWarning: \n","A value is trying to be set on a copy of a slice from a DataFrame.\n","Try using .loc[row_indexer,col_indexer] = value instead\n","\n","See the caveats in the documentation: http://pandas.pydata.org/pandas-docs/stable/user_guide/indexing.html#returning-a-view-versus-a-copy\n","  \n","C:\\Anaconda\\lib\\site-packages\\ipykernel_launcher.py:7: SettingWithCopyWarning: \n","A value is trying to be set on a copy of a slice from a DataFrame.\n","Try using .loc[row_indexer,col_indexer] = value instead\n","\n","See the caveats in the documentation: http://pandas.pydata.org/pandas-docs/stable/user_guide/indexing.html#returning-a-view-versus-a-copy\n","  import sys\n","C:\\Anaconda\\lib\\site-packages\\ipykernel_launcher.py:8: SettingWithCopyWarning: \n","A value is trying to be set on a copy of a slice from a DataFrame.\n","Try using .loc[row_indexer,col_indexer] = value instead\n","\n","See the caveats in the documentation: http://pandas.pydata.org/pandas-docs/stable/user_guide/indexing.html#returning-a-view-versus-a-copy\n","  \n","C:\\Anaconda\\lib\\site-packages\\ipykernel_launcher.py:9: SettingWithCopyWarning: \n","A value is trying to be set on a copy of a slice from a DataFrame.\n","Try using .loc[row_indexer,col_indexer] = value instead\n","\n","See the caveats in the documentation: http://pandas.pydata.org/pandas-docs/stable/user_guide/indexing.html#returning-a-view-versus-a-copy\n","  if __name__ == '__main__':\n","C:\\Anaconda\\lib\\site-packages\\ipykernel_launcher.py:11: SettingWithCopyWarning: \n","A value is trying to be set on a copy of a slice from a DataFrame.\n","Try using .loc[row_indexer,col_indexer] = value instead\n","\n","See the caveats in the documentation: http://pandas.pydata.org/pandas-docs/stable/user_guide/indexing.html#returning-a-view-versus-a-copy\n","  # This is added back by InteractiveShellApp.init_path()\n","C:\\Anaconda\\lib\\site-packages\\ipykernel_launcher.py:12: SettingWithCopyWarning: \n","A value is trying to be set on a copy of a slice from a DataFrame.\n","Try using .loc[row_indexer,col_indexer] = value instead\n","\n","See the caveats in the documentation: http://pandas.pydata.org/pandas-docs/stable/user_guide/indexing.html#returning-a-view-versus-a-copy\n","  if sys.path[0] == '':\n","C:\\Anaconda\\lib\\site-packages\\ipykernel_launcher.py:14: SettingWithCopyWarning: \n","A value is trying to be set on a copy of a slice from a DataFrame.\n","Try using .loc[row_indexer,col_indexer] = value instead\n","\n","See the caveats in the documentation: http://pandas.pydata.org/pandas-docs/stable/user_guide/indexing.html#returning-a-view-versus-a-copy\n","  \n","C:\\Anaconda\\lib\\site-packages\\ipykernel_launcher.py:15: SettingWithCopyWarning: \n","A value is trying to be set on a copy of a slice from a DataFrame.\n","Try using .loc[row_indexer,col_indexer] = value instead\n","\n","See the caveats in the documentation: http://pandas.pydata.org/pandas-docs/stable/user_guide/indexing.html#returning-a-view-versus-a-copy\n","  from ipykernel import kernelapp as app\n","C:\\Anaconda\\lib\\site-packages\\ipykernel_launcher.py:16: SettingWithCopyWarning: \n","A value is trying to be set on a copy of a slice from a DataFrame.\n","Try using .loc[row_indexer,col_indexer] = value instead\n","\n","See the caveats in the documentation: http://pandas.pydata.org/pandas-docs/stable/user_guide/indexing.html#returning-a-view-versus-a-copy\n","  app.launch_new_instance()\n","C:\\Anaconda\\lib\\site-packages\\ipykernel_launcher.py:17: SettingWithCopyWarning: \n","A value is trying to be set on a copy of a slice from a DataFrame.\n","Try using .loc[row_indexer,col_indexer] = value instead\n","\n","See the caveats in the documentation: http://pandas.pydata.org/pandas-docs/stable/user_guide/indexing.html#returning-a-view-versus-a-copy\n","C:\\Anaconda\\lib\\site-packages\\ipykernel_launcher.py:18: SettingWithCopyWarning: \n","A value is trying to be set on a copy of a slice from a DataFrame.\n","Try using .loc[row_indexer,col_indexer] = value instead\n","\n","See the caveats in the documentation: http://pandas.pydata.org/pandas-docs/stable/user_guide/indexing.html#returning-a-view-versus-a-copy\n","C:\\Anaconda\\lib\\site-packages\\ipykernel_launcher.py:19: SettingWithCopyWarning: \n","A value is trying to be set on a copy of a slice from a DataFrame.\n","Try using .loc[row_indexer,col_indexer] = value instead\n","\n","See the caveats in the documentation: http://pandas.pydata.org/pandas-docs/stable/user_guide/indexing.html#returning-a-view-versus-a-copy\n","C:\\Anaconda\\lib\\site-packages\\ipykernel_launcher.py:20: SettingWithCopyWarning: \n","A value is trying to be set on a copy of a slice from a DataFrame.\n","Try using .loc[row_indexer,col_indexer] = value instead\n","\n","See the caveats in the documentation: http://pandas.pydata.org/pandas-docs/stable/user_guide/indexing.html#returning-a-view-versus-a-copy\n","C:\\Anaconda\\lib\\site-packages\\ipykernel_launcher.py:21: SettingWithCopyWarning: \n","A value is trying to be set on a copy of a slice from a DataFrame.\n","Try using .loc[row_indexer,col_indexer] = value instead\n","\n","See the caveats in the documentation: http://pandas.pydata.org/pandas-docs/stable/user_guide/indexing.html#returning-a-view-versus-a-copy\n","C:\\Anaconda\\lib\\site-packages\\ipykernel_launcher.py:22: SettingWithCopyWarning: \n","A value is trying to be set on a copy of a slice from a DataFrame.\n","Try using .loc[row_indexer,col_indexer] = value instead\n","\n","See the caveats in the documentation: http://pandas.pydata.org/pandas-docs/stable/user_guide/indexing.html#returning-a-view-versus-a-copy\n","C:\\Anaconda\\lib\\site-packages\\ipykernel_launcher.py:23: SettingWithCopyWarning: \n","A value is trying to be set on a copy of a slice from a DataFrame.\n","Try using .loc[row_indexer,col_indexer] = value instead\n","\n","See the caveats in the documentation: http://pandas.pydata.org/pandas-docs/stable/user_guide/indexing.html#returning-a-view-versus-a-copy\n","C:\\Anaconda\\lib\\site-packages\\ipykernel_launcher.py:25: SettingWithCopyWarning: \n","A value is trying to be set on a copy of a slice from a DataFrame.\n","Try using .loc[row_indexer,col_indexer] = value instead\n","\n","See the caveats in the documentation: http://pandas.pydata.org/pandas-docs/stable/user_guide/indexing.html#returning-a-view-versus-a-copy\n","C:\\Anaconda\\lib\\site-packages\\ipykernel_launcher.py:26: SettingWithCopyWarning: \n","A value is trying to be set on a copy of a slice from a DataFrame.\n","Try using .loc[row_indexer,col_indexer] = value instead\n","\n","See the caveats in the documentation: http://pandas.pydata.org/pandas-docs/stable/user_guide/indexing.html#returning-a-view-versus-a-copy\n","C:\\Anaconda\\lib\\site-packages\\ipykernel_launcher.py:28: SettingWithCopyWarning: \n","A value is trying to be set on a copy of a slice from a DataFrame.\n","Try using .loc[row_indexer,col_indexer] = value instead\n","\n","See the caveats in the documentation: http://pandas.pydata.org/pandas-docs/stable/user_guide/indexing.html#returning-a-view-versus-a-copy\n","C:\\Anaconda\\lib\\site-packages\\ipykernel_launcher.py:29: SettingWithCopyWarning: \n","A value is trying to be set on a copy of a slice from a DataFrame.\n","Try using .loc[row_indexer,col_indexer] = value instead\n","\n","See the caveats in the documentation: http://pandas.pydata.org/pandas-docs/stable/user_guide/indexing.html#returning-a-view-versus-a-copy\n","C:\\Anaconda\\lib\\site-packages\\ipykernel_launcher.py:31: SettingWithCopyWarning: \n","A value is trying to be set on a copy of a slice from a DataFrame.\n","Try using .loc[row_indexer,col_indexer] = value instead\n","\n","See the caveats in the documentation: http://pandas.pydata.org/pandas-docs/stable/user_guide/indexing.html#returning-a-view-versus-a-copy\n","C:\\Anaconda\\lib\\site-packages\\ipykernel_launcher.py:32: SettingWithCopyWarning: \n","A value is trying to be set on a copy of a slice from a DataFrame.\n","Try using .loc[row_indexer,col_indexer] = value instead\n","\n","See the caveats in the documentation: http://pandas.pydata.org/pandas-docs/stable/user_guide/indexing.html#returning-a-view-versus-a-copy\n"],"name":"stderr"},{"output_type":"stream","text":["C:\\Anaconda\\lib\\site-packages\\ipykernel_launcher.py:34: SettingWithCopyWarning: \n","A value is trying to be set on a copy of a slice from a DataFrame.\n","Try using .loc[row_indexer,col_indexer] = value instead\n","\n","See the caveats in the documentation: http://pandas.pydata.org/pandas-docs/stable/user_guide/indexing.html#returning-a-view-versus-a-copy\n","C:\\Anaconda\\lib\\site-packages\\ipykernel_launcher.py:35: SettingWithCopyWarning: \n","A value is trying to be set on a copy of a slice from a DataFrame.\n","Try using .loc[row_indexer,col_indexer] = value instead\n","\n","See the caveats in the documentation: http://pandas.pydata.org/pandas-docs/stable/user_guide/indexing.html#returning-a-view-versus-a-copy\n","C:\\Anaconda\\lib\\site-packages\\ipykernel_launcher.py:36: SettingWithCopyWarning: \n","A value is trying to be set on a copy of a slice from a DataFrame.\n","Try using .loc[row_indexer,col_indexer] = value instead\n","\n","See the caveats in the documentation: http://pandas.pydata.org/pandas-docs/stable/user_guide/indexing.html#returning-a-view-versus-a-copy\n","C:\\Anaconda\\lib\\site-packages\\ipykernel_launcher.py:37: SettingWithCopyWarning: \n","A value is trying to be set on a copy of a slice from a DataFrame.\n","Try using .loc[row_indexer,col_indexer] = value instead\n","\n","See the caveats in the documentation: http://pandas.pydata.org/pandas-docs/stable/user_guide/indexing.html#returning-a-view-versus-a-copy\n","C:\\Anaconda\\lib\\site-packages\\ipykernel_launcher.py:39: SettingWithCopyWarning: \n","A value is trying to be set on a copy of a slice from a DataFrame.\n","Try using .loc[row_indexer,col_indexer] = value instead\n","\n","See the caveats in the documentation: http://pandas.pydata.org/pandas-docs/stable/user_guide/indexing.html#returning-a-view-versus-a-copy\n","C:\\Anaconda\\lib\\site-packages\\ipykernel_launcher.py:40: SettingWithCopyWarning: \n","A value is trying to be set on a copy of a slice from a DataFrame.\n","Try using .loc[row_indexer,col_indexer] = value instead\n","\n","See the caveats in the documentation: http://pandas.pydata.org/pandas-docs/stable/user_guide/indexing.html#returning-a-view-versus-a-copy\n","C:\\Anaconda\\lib\\site-packages\\ipykernel_launcher.py:41: SettingWithCopyWarning: \n","A value is trying to be set on a copy of a slice from a DataFrame.\n","Try using .loc[row_indexer,col_indexer] = value instead\n","\n","See the caveats in the documentation: http://pandas.pydata.org/pandas-docs/stable/user_guide/indexing.html#returning-a-view-versus-a-copy\n","C:\\Anaconda\\lib\\site-packages\\ipykernel_launcher.py:42: SettingWithCopyWarning: \n","A value is trying to be set on a copy of a slice from a DataFrame.\n","Try using .loc[row_indexer,col_indexer] = value instead\n","\n","See the caveats in the documentation: http://pandas.pydata.org/pandas-docs/stable/user_guide/indexing.html#returning-a-view-versus-a-copy\n","C:\\Anaconda\\lib\\site-packages\\ipykernel_launcher.py:43: SettingWithCopyWarning: \n","A value is trying to be set on a copy of a slice from a DataFrame.\n","Try using .loc[row_indexer,col_indexer] = value instead\n","\n","See the caveats in the documentation: http://pandas.pydata.org/pandas-docs/stable/user_guide/indexing.html#returning-a-view-versus-a-copy\n","C:\\Anaconda\\lib\\site-packages\\ipykernel_launcher.py:44: SettingWithCopyWarning: \n","A value is trying to be set on a copy of a slice from a DataFrame.\n","Try using .loc[row_indexer,col_indexer] = value instead\n","\n","See the caveats in the documentation: http://pandas.pydata.org/pandas-docs/stable/user_guide/indexing.html#returning-a-view-versus-a-copy\n","C:\\Anaconda\\lib\\site-packages\\ipykernel_launcher.py:45: SettingWithCopyWarning: \n","A value is trying to be set on a copy of a slice from a DataFrame.\n","Try using .loc[row_indexer,col_indexer] = value instead\n","\n","See the caveats in the documentation: http://pandas.pydata.org/pandas-docs/stable/user_guide/indexing.html#returning-a-view-versus-a-copy\n","C:\\Anaconda\\lib\\site-packages\\ipykernel_launcher.py:46: SettingWithCopyWarning: \n","A value is trying to be set on a copy of a slice from a DataFrame.\n","Try using .loc[row_indexer,col_indexer] = value instead\n","\n","See the caveats in the documentation: http://pandas.pydata.org/pandas-docs/stable/user_guide/indexing.html#returning-a-view-versus-a-copy\n","C:\\Anaconda\\lib\\site-packages\\ipykernel_launcher.py:47: SettingWithCopyWarning: \n","A value is trying to be set on a copy of a slice from a DataFrame.\n","Try using .loc[row_indexer,col_indexer] = value instead\n","\n","See the caveats in the documentation: http://pandas.pydata.org/pandas-docs/stable/user_guide/indexing.html#returning-a-view-versus-a-copy\n","C:\\Anaconda\\lib\\site-packages\\ipykernel_launcher.py:48: SettingWithCopyWarning: \n","A value is trying to be set on a copy of a slice from a DataFrame.\n","Try using .loc[row_indexer,col_indexer] = value instead\n","\n","See the caveats in the documentation: http://pandas.pydata.org/pandas-docs/stable/user_guide/indexing.html#returning-a-view-versus-a-copy\n","C:\\Anaconda\\lib\\site-packages\\ipykernel_launcher.py:49: SettingWithCopyWarning: \n","A value is trying to be set on a copy of a slice from a DataFrame.\n","Try using .loc[row_indexer,col_indexer] = value instead\n","\n","See the caveats in the documentation: http://pandas.pydata.org/pandas-docs/stable/user_guide/indexing.html#returning-a-view-versus-a-copy\n","C:\\Anaconda\\lib\\site-packages\\ipykernel_launcher.py:50: SettingWithCopyWarning: \n","A value is trying to be set on a copy of a slice from a DataFrame.\n","Try using .loc[row_indexer,col_indexer] = value instead\n","\n","See the caveats in the documentation: http://pandas.pydata.org/pandas-docs/stable/user_guide/indexing.html#returning-a-view-versus-a-copy\n","C:\\Anaconda\\lib\\site-packages\\ipykernel_launcher.py:52: SettingWithCopyWarning: \n","A value is trying to be set on a copy of a slice from a DataFrame.\n","Try using .loc[row_indexer,col_indexer] = value instead\n","\n","See the caveats in the documentation: http://pandas.pydata.org/pandas-docs/stable/user_guide/indexing.html#returning-a-view-versus-a-copy\n","C:\\Anaconda\\lib\\site-packages\\ipykernel_launcher.py:53: SettingWithCopyWarning: \n","A value is trying to be set on a copy of a slice from a DataFrame.\n","Try using .loc[row_indexer,col_indexer] = value instead\n","\n","See the caveats in the documentation: http://pandas.pydata.org/pandas-docs/stable/user_guide/indexing.html#returning-a-view-versus-a-copy\n","C:\\Anaconda\\lib\\site-packages\\ipykernel_launcher.py:54: SettingWithCopyWarning: \n","A value is trying to be set on a copy of a slice from a DataFrame.\n","Try using .loc[row_indexer,col_indexer] = value instead\n","\n","See the caveats in the documentation: http://pandas.pydata.org/pandas-docs/stable/user_guide/indexing.html#returning-a-view-versus-a-copy\n","C:\\Anaconda\\lib\\site-packages\\ipykernel_launcher.py:55: SettingWithCopyWarning: \n","A value is trying to be set on a copy of a slice from a DataFrame.\n","Try using .loc[row_indexer,col_indexer] = value instead\n","\n","See the caveats in the documentation: http://pandas.pydata.org/pandas-docs/stable/user_guide/indexing.html#returning-a-view-versus-a-copy\n","C:\\Anaconda\\lib\\site-packages\\ipykernel_launcher.py:56: SettingWithCopyWarning: \n","A value is trying to be set on a copy of a slice from a DataFrame.\n","Try using .loc[row_indexer,col_indexer] = value instead\n","\n","See the caveats in the documentation: http://pandas.pydata.org/pandas-docs/stable/user_guide/indexing.html#returning-a-view-versus-a-copy\n","C:\\Anaconda\\lib\\site-packages\\ipykernel_launcher.py:57: SettingWithCopyWarning: \n","A value is trying to be set on a copy of a slice from a DataFrame.\n","Try using .loc[row_indexer,col_indexer] = value instead\n","\n","See the caveats in the documentation: http://pandas.pydata.org/pandas-docs/stable/user_guide/indexing.html#returning-a-view-versus-a-copy\n","C:\\Anaconda\\lib\\site-packages\\ipykernel_launcher.py:58: SettingWithCopyWarning: \n","A value is trying to be set on a copy of a slice from a DataFrame.\n","Try using .loc[row_indexer,col_indexer] = value instead\n","\n","See the caveats in the documentation: http://pandas.pydata.org/pandas-docs/stable/user_guide/indexing.html#returning-a-view-versus-a-copy\n","C:\\Anaconda\\lib\\site-packages\\ipykernel_launcher.py:59: SettingWithCopyWarning: \n","A value is trying to be set on a copy of a slice from a DataFrame.\n","Try using .loc[row_indexer,col_indexer] = value instead\n","\n","See the caveats in the documentation: http://pandas.pydata.org/pandas-docs/stable/user_guide/indexing.html#returning-a-view-versus-a-copy\n","C:\\Anaconda\\lib\\site-packages\\ipykernel_launcher.py:61: SettingWithCopyWarning: \n","A value is trying to be set on a copy of a slice from a DataFrame.\n","Try using .loc[row_indexer,col_indexer] = value instead\n","\n","See the caveats in the documentation: http://pandas.pydata.org/pandas-docs/stable/user_guide/indexing.html#returning-a-view-versus-a-copy\n","C:\\Anaconda\\lib\\site-packages\\ipykernel_launcher.py:62: SettingWithCopyWarning: \n","A value is trying to be set on a copy of a slice from a DataFrame.\n","Try using .loc[row_indexer,col_indexer] = value instead\n","\n","See the caveats in the documentation: http://pandas.pydata.org/pandas-docs/stable/user_guide/indexing.html#returning-a-view-versus-a-copy\n"],"name":"stderr"},{"output_type":"stream","text":["C:\\Anaconda\\lib\\site-packages\\ipykernel_launcher.py:63: SettingWithCopyWarning: \n","A value is trying to be set on a copy of a slice from a DataFrame.\n","Try using .loc[row_indexer,col_indexer] = value instead\n","\n","See the caveats in the documentation: http://pandas.pydata.org/pandas-docs/stable/user_guide/indexing.html#returning-a-view-versus-a-copy\n","C:\\Anaconda\\lib\\site-packages\\ipykernel_launcher.py:64: SettingWithCopyWarning: \n","A value is trying to be set on a copy of a slice from a DataFrame.\n","Try using .loc[row_indexer,col_indexer] = value instead\n","\n","See the caveats in the documentation: http://pandas.pydata.org/pandas-docs/stable/user_guide/indexing.html#returning-a-view-versus-a-copy\n","C:\\Anaconda\\lib\\site-packages\\ipykernel_launcher.py:65: SettingWithCopyWarning: \n","A value is trying to be set on a copy of a slice from a DataFrame.\n","Try using .loc[row_indexer,col_indexer] = value instead\n","\n","See the caveats in the documentation: http://pandas.pydata.org/pandas-docs/stable/user_guide/indexing.html#returning-a-view-versus-a-copy\n","C:\\Anaconda\\lib\\site-packages\\ipykernel_launcher.py:66: SettingWithCopyWarning: \n","A value is trying to be set on a copy of a slice from a DataFrame.\n","Try using .loc[row_indexer,col_indexer] = value instead\n","\n","See the caveats in the documentation: http://pandas.pydata.org/pandas-docs/stable/user_guide/indexing.html#returning-a-view-versus-a-copy\n","C:\\Anaconda\\lib\\site-packages\\ipykernel_launcher.py:67: SettingWithCopyWarning: \n","A value is trying to be set on a copy of a slice from a DataFrame.\n","Try using .loc[row_indexer,col_indexer] = value instead\n","\n","See the caveats in the documentation: http://pandas.pydata.org/pandas-docs/stable/user_guide/indexing.html#returning-a-view-versus-a-copy\n","C:\\Anaconda\\lib\\site-packages\\ipykernel_launcher.py:68: SettingWithCopyWarning: \n","A value is trying to be set on a copy of a slice from a DataFrame.\n","Try using .loc[row_indexer,col_indexer] = value instead\n","\n","See the caveats in the documentation: http://pandas.pydata.org/pandas-docs/stable/user_guide/indexing.html#returning-a-view-versus-a-copy\n","C:\\Anaconda\\lib\\site-packages\\ipykernel_launcher.py:70: SettingWithCopyWarning: \n","A value is trying to be set on a copy of a slice from a DataFrame.\n","Try using .loc[row_indexer,col_indexer] = value instead\n","\n","See the caveats in the documentation: http://pandas.pydata.org/pandas-docs/stable/user_guide/indexing.html#returning-a-view-versus-a-copy\n","C:\\Anaconda\\lib\\site-packages\\ipykernel_launcher.py:71: SettingWithCopyWarning: \n","A value is trying to be set on a copy of a slice from a DataFrame.\n","Try using .loc[row_indexer,col_indexer] = value instead\n","\n","See the caveats in the documentation: http://pandas.pydata.org/pandas-docs/stable/user_guide/indexing.html#returning-a-view-versus-a-copy\n","C:\\Anaconda\\lib\\site-packages\\ipykernel_launcher.py:72: SettingWithCopyWarning: \n","A value is trying to be set on a copy of a slice from a DataFrame.\n","Try using .loc[row_indexer,col_indexer] = value instead\n","\n","See the caveats in the documentation: http://pandas.pydata.org/pandas-docs/stable/user_guide/indexing.html#returning-a-view-versus-a-copy\n","C:\\Anaconda\\lib\\site-packages\\ipykernel_launcher.py:73: SettingWithCopyWarning: \n","A value is trying to be set on a copy of a slice from a DataFrame.\n","Try using .loc[row_indexer,col_indexer] = value instead\n","\n","See the caveats in the documentation: http://pandas.pydata.org/pandas-docs/stable/user_guide/indexing.html#returning-a-view-versus-a-copy\n","C:\\Anaconda\\lib\\site-packages\\ipykernel_launcher.py:74: SettingWithCopyWarning: \n","A value is trying to be set on a copy of a slice from a DataFrame.\n","Try using .loc[row_indexer,col_indexer] = value instead\n","\n","See the caveats in the documentation: http://pandas.pydata.org/pandas-docs/stable/user_guide/indexing.html#returning-a-view-versus-a-copy\n","C:\\Anaconda\\lib\\site-packages\\ipykernel_launcher.py:75: SettingWithCopyWarning: \n","A value is trying to be set on a copy of a slice from a DataFrame.\n","Try using .loc[row_indexer,col_indexer] = value instead\n","\n","See the caveats in the documentation: http://pandas.pydata.org/pandas-docs/stable/user_guide/indexing.html#returning-a-view-versus-a-copy\n","C:\\Anaconda\\lib\\site-packages\\ipykernel_launcher.py:76: SettingWithCopyWarning: \n","A value is trying to be set on a copy of a slice from a DataFrame.\n","Try using .loc[row_indexer,col_indexer] = value instead\n","\n","See the caveats in the documentation: http://pandas.pydata.org/pandas-docs/stable/user_guide/indexing.html#returning-a-view-versus-a-copy\n","C:\\Anaconda\\lib\\site-packages\\ipykernel_launcher.py:77: SettingWithCopyWarning: \n","A value is trying to be set on a copy of a slice from a DataFrame.\n","Try using .loc[row_indexer,col_indexer] = value instead\n","\n","See the caveats in the documentation: http://pandas.pydata.org/pandas-docs/stable/user_guide/indexing.html#returning-a-view-versus-a-copy\n","C:\\Anaconda\\lib\\site-packages\\ipykernel_launcher.py:78: SettingWithCopyWarning: \n","A value is trying to be set on a copy of a slice from a DataFrame.\n","Try using .loc[row_indexer,col_indexer] = value instead\n","\n","See the caveats in the documentation: http://pandas.pydata.org/pandas-docs/stable/user_guide/indexing.html#returning-a-view-versus-a-copy\n","C:\\Anaconda\\lib\\site-packages\\ipykernel_launcher.py:79: SettingWithCopyWarning: \n","A value is trying to be set on a copy of a slice from a DataFrame.\n","Try using .loc[row_indexer,col_indexer] = value instead\n","\n","See the caveats in the documentation: http://pandas.pydata.org/pandas-docs/stable/user_guide/indexing.html#returning-a-view-versus-a-copy\n","C:\\Anaconda\\lib\\site-packages\\ipykernel_launcher.py:80: SettingWithCopyWarning: \n","A value is trying to be set on a copy of a slice from a DataFrame.\n","Try using .loc[row_indexer,col_indexer] = value instead\n","\n","See the caveats in the documentation: http://pandas.pydata.org/pandas-docs/stable/user_guide/indexing.html#returning-a-view-versus-a-copy\n","C:\\Anaconda\\lib\\site-packages\\ipykernel_launcher.py:81: SettingWithCopyWarning: \n","A value is trying to be set on a copy of a slice from a DataFrame.\n","Try using .loc[row_indexer,col_indexer] = value instead\n","\n","See the caveats in the documentation: http://pandas.pydata.org/pandas-docs/stable/user_guide/indexing.html#returning-a-view-versus-a-copy\n","C:\\Anaconda\\lib\\site-packages\\ipykernel_launcher.py:82: SettingWithCopyWarning: \n","A value is trying to be set on a copy of a slice from a DataFrame.\n","Try using .loc[row_indexer,col_indexer] = value instead\n","\n","See the caveats in the documentation: http://pandas.pydata.org/pandas-docs/stable/user_guide/indexing.html#returning-a-view-versus-a-copy\n","C:\\Anaconda\\lib\\site-packages\\ipykernel_launcher.py:83: SettingWithCopyWarning: \n","A value is trying to be set on a copy of a slice from a DataFrame.\n","Try using .loc[row_indexer,col_indexer] = value instead\n","\n","See the caveats in the documentation: http://pandas.pydata.org/pandas-docs/stable/user_guide/indexing.html#returning-a-view-versus-a-copy\n","C:\\Anaconda\\lib\\site-packages\\ipykernel_launcher.py:84: SettingWithCopyWarning: \n","A value is trying to be set on a copy of a slice from a DataFrame.\n","Try using .loc[row_indexer,col_indexer] = value instead\n","\n","See the caveats in the documentation: http://pandas.pydata.org/pandas-docs/stable/user_guide/indexing.html#returning-a-view-versus-a-copy\n","C:\\Anaconda\\lib\\site-packages\\ipykernel_launcher.py:85: SettingWithCopyWarning: \n","A value is trying to be set on a copy of a slice from a DataFrame.\n","Try using .loc[row_indexer,col_indexer] = value instead\n","\n","See the caveats in the documentation: http://pandas.pydata.org/pandas-docs/stable/user_guide/indexing.html#returning-a-view-versus-a-copy\n","C:\\Anaconda\\lib\\site-packages\\ipykernel_launcher.py:86: SettingWithCopyWarning: \n","A value is trying to be set on a copy of a slice from a DataFrame.\n","Try using .loc[row_indexer,col_indexer] = value instead\n","\n","See the caveats in the documentation: http://pandas.pydata.org/pandas-docs/stable/user_guide/indexing.html#returning-a-view-versus-a-copy\n","C:\\Anaconda\\lib\\site-packages\\ipykernel_launcher.py:87: SettingWithCopyWarning: \n","A value is trying to be set on a copy of a slice from a DataFrame.\n","Try using .loc[row_indexer,col_indexer] = value instead\n","\n","See the caveats in the documentation: http://pandas.pydata.org/pandas-docs/stable/user_guide/indexing.html#returning-a-view-versus-a-copy\n","C:\\Anaconda\\lib\\site-packages\\ipykernel_launcher.py:88: SettingWithCopyWarning: \n","A value is trying to be set on a copy of a slice from a DataFrame.\n","Try using .loc[row_indexer,col_indexer] = value instead\n","\n","See the caveats in the documentation: http://pandas.pydata.org/pandas-docs/stable/user_guide/indexing.html#returning-a-view-versus-a-copy\n","C:\\Anaconda\\lib\\site-packages\\ipykernel_launcher.py:89: SettingWithCopyWarning: \n","A value is trying to be set on a copy of a slice from a DataFrame.\n","Try using .loc[row_indexer,col_indexer] = value instead\n","\n","See the caveats in the documentation: http://pandas.pydata.org/pandas-docs/stable/user_guide/indexing.html#returning-a-view-versus-a-copy\n","C:\\Anaconda\\lib\\site-packages\\ipykernel_launcher.py:90: SettingWithCopyWarning: \n","A value is trying to be set on a copy of a slice from a DataFrame.\n","Try using .loc[row_indexer,col_indexer] = value instead\n","\n","See the caveats in the documentation: http://pandas.pydata.org/pandas-docs/stable/user_guide/indexing.html#returning-a-view-versus-a-copy\n","C:\\Anaconda\\lib\\site-packages\\ipykernel_launcher.py:91: SettingWithCopyWarning: \n","A value is trying to be set on a copy of a slice from a DataFrame.\n","Try using .loc[row_indexer,col_indexer] = value instead\n","\n","See the caveats in the documentation: http://pandas.pydata.org/pandas-docs/stable/user_guide/indexing.html#returning-a-view-versus-a-copy\n","C:\\Anaconda\\lib\\site-packages\\ipykernel_launcher.py:92: SettingWithCopyWarning: \n","A value is trying to be set on a copy of a slice from a DataFrame.\n","Try using .loc[row_indexer,col_indexer] = value instead\n","\n","See the caveats in the documentation: http://pandas.pydata.org/pandas-docs/stable/user_guide/indexing.html#returning-a-view-versus-a-copy\n","C:\\Anaconda\\lib\\site-packages\\ipykernel_launcher.py:93: SettingWithCopyWarning: \n","A value is trying to be set on a copy of a slice from a DataFrame.\n","Try using .loc[row_indexer,col_indexer] = value instead\n","\n","See the caveats in the documentation: http://pandas.pydata.org/pandas-docs/stable/user_guide/indexing.html#returning-a-view-versus-a-copy\n","C:\\Anaconda\\lib\\site-packages\\ipykernel_launcher.py:94: SettingWithCopyWarning: \n","A value is trying to be set on a copy of a slice from a DataFrame.\n","Try using .loc[row_indexer,col_indexer] = value instead\n","\n","See the caveats in the documentation: http://pandas.pydata.org/pandas-docs/stable/user_guide/indexing.html#returning-a-view-versus-a-copy\n","C:\\Anaconda\\lib\\site-packages\\ipykernel_launcher.py:95: SettingWithCopyWarning: \n","A value is trying to be set on a copy of a slice from a DataFrame.\n","Try using .loc[row_indexer,col_indexer] = value instead\n","\n","See the caveats in the documentation: http://pandas.pydata.org/pandas-docs/stable/user_guide/indexing.html#returning-a-view-versus-a-copy\n","C:\\Anaconda\\lib\\site-packages\\ipykernel_launcher.py:96: SettingWithCopyWarning: \n","A value is trying to be set on a copy of a slice from a DataFrame.\n","Try using .loc[row_indexer,col_indexer] = value instead\n","\n","See the caveats in the documentation: http://pandas.pydata.org/pandas-docs/stable/user_guide/indexing.html#returning-a-view-versus-a-copy\n","C:\\Anaconda\\lib\\site-packages\\ipykernel_launcher.py:97: SettingWithCopyWarning: \n","A value is trying to be set on a copy of a slice from a DataFrame.\n","Try using .loc[row_indexer,col_indexer] = value instead\n","\n","See the caveats in the documentation: http://pandas.pydata.org/pandas-docs/stable/user_guide/indexing.html#returning-a-view-versus-a-copy\n"],"name":"stderr"}]},{"cell_type":"code","metadata":{"id":"55D6pxsOtiCc","colab_type":"code","colab":{}},"source":["import os\n","def start_timer():\n","    (utime, stime) = os.times()[0:2]\n","    start_time= utime+stime\n","    return start_time\n","\n","def end_timer(start_time):\n","    (utime, stime) = os.times()[0:2]\n","    end_time= utime+stime\n","    time_to_string= \"%05.5f seconds\" % (end_time-start_time)\n","    return time_to_string"],"execution_count":0,"outputs":[]},{"cell_type":"code","metadata":{"id":"3YuMwBF5tiCi","colab_type":"code","colab":{}},"source":[""],"execution_count":0,"outputs":[]},{"cell_type":"code","metadata":{"id":"a55SCkCttiCm","colab_type":"code","colab":{}},"source":["#Work with categorical columns\n","def calculate_mi_category(x,y):  \n","    mi_dict ={}  \n","    print('Start calculating mutual info for columns:' + str(list(x.columns)))  \n","    start_time=start_timer() \n","    list_ind=list(range(len(x.columns)))\n","    feature_scores = mutual_info_classif(x, y, discrete_features=list_ind)\n","    print('Mutual info calculation took '+end_timer(start_time))  \n","    for i in range(len(x.columns)):  \n","        print(x.columns[i]+','+str(feature_scores[i]))  \n","        mi_dict[x.columns[i]] = feature_scores[i]  \n","    print(mi_dict)  \n","    return mi_dict"],"execution_count":0,"outputs":[]},{"cell_type":"code","metadata":{"id":"cklTmbJetiCs","colab_type":"code","outputId":"26563c78-1e78-468e-9283-2751f03f1048","colab":{}},"source":["list_ind=list(range(len(df_category.columns)))\n","print(list_ind)"],"execution_count":0,"outputs":[{"output_type":"stream","text":["[0, 1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12, 13, 14, 15, 16, 17, 18, 19, 20, 21, 22, 23, 24, 25, 26, 27, 28, 29, 30, 31, 32, 33, 34, 35, 36, 37, 38, 39, 40, 41]\n"],"name":"stdout"}]},{"cell_type":"code","metadata":{"id":"4WM2J_NhtiCx","colab_type":"code","outputId":"3b6c5fdd-82bc-4868-f938-39bd6a86c09d","colab":{}},"source":["#MI calculation of categorical columns\n","mi_per_column = {}  \n","mi_dict=calculate_mi_category(df_category,y)  \n","mi_per_column.update(mi_dict)"],"execution_count":0,"outputs":[{"output_type":"stream","text":["Start calculating mutual info for columns:['ProductCD', 'card1', 'card2', 'card3', 'card4', 'card5', 'card6', 'addr1', 'addr2', 'M1', 'M2', 'M3', 'M4', 'M5', 'M6', 'M7', 'M8', 'M9', 'id_12', 'id_13', 'id_15', 'id_16', 'id_17', 'id_19', 'id_20', 'id_28', 'id_29', 'id_31', 'id_35', 'id_36', 'id_37', 'id_38', 'DeviceType', '_Weekdays', '_Hours', '_Days', 'P_emaildomain_bin', 'P_emaildomain_suffix', 'R_emaildomain_bin', 'R_emaildomain_suffix', 'device_name', '_Month']\n","Mutual info calculation took 1.43750 seconds\n","ProductCD,0.010213577538186792\n","card1,0.03408187124266054\n","card2,0.01674442389468834\n","card3,0.009780680760046059\n","card4,0.00023746022758847941\n","card5,0.0061377863528340685\n","card6,0.004492155662813475\n","addr1,0.010450301861583747\n","addr2,0.009449784667285388\n","M1,0.004005214670817929\n","M2,0.004093382610964681\n","M3,0.003820260880026804\n","M4,9.228590527184978e-05\n","M5,2.3901373379910107e-05\n","M6,0.0012208701911344752\n","M7,0.00015431716290916636\n","M8,0.0011284138065375017\n","M9,3.3800802739514246e-05\n","id_12,0.006946129797153658\n","id_13,0.009190431825167964\n","id_15,0.0010484864514078499\n","id_16,0.00028353554106155116\n","id_17,0.010930619476640497\n","id_19,0.014125189527554032\n","id_20,0.013265809389697493\n","id_28,0.0072639918934557166\n","id_29,0.0004383165133765715\n","id_31,0.0018236933962204297\n","id_35,0.00019494770835713424\n","id_36,2.781267893986819e-08\n","id_37,0.006317725067608415\n","id_38,0.006058598076185993\n","DeviceType,0.00487910340575147\n","_Weekdays,6.600348990129512e-05\n","_Hours,0.0015133036043322584\n","_Days,0.00014819131782922105\n","P_emaildomain_bin,0.0018657351637627953\n","P_emaildomain_suffix,0.0003151873890022234\n","R_emaildomain_bin,0.010273468718128557\n","R_emaildomain_suffix,0.00024056445606271285\n","device_name,0.009052937942013853\n","_Month,0.0005063466005384575\n","{'ProductCD': 0.010213577538186792, 'card1': 0.03408187124266054, 'card2': 0.01674442389468834, 'card3': 0.009780680760046059, 'card4': 0.00023746022758847941, 'card5': 0.0061377863528340685, 'card6': 0.004492155662813475, 'addr1': 0.010450301861583747, 'addr2': 0.009449784667285388, 'M1': 0.004005214670817929, 'M2': 0.004093382610964681, 'M3': 0.003820260880026804, 'M4': 9.228590527184978e-05, 'M5': 2.3901373379910107e-05, 'M6': 0.0012208701911344752, 'M7': 0.00015431716290916636, 'M8': 0.0011284138065375017, 'M9': 3.3800802739514246e-05, 'id_12': 0.006946129797153658, 'id_13': 0.009190431825167964, 'id_15': 0.0010484864514078499, 'id_16': 0.00028353554106155116, 'id_17': 0.010930619476640497, 'id_19': 0.014125189527554032, 'id_20': 0.013265809389697493, 'id_28': 0.0072639918934557166, 'id_29': 0.0004383165133765715, 'id_31': 0.0018236933962204297, 'id_35': 0.00019494770835713424, 'id_36': 2.781267893986819e-08, 'id_37': 0.006317725067608415, 'id_38': 0.006058598076185993, 'DeviceType': 0.00487910340575147, '_Weekdays': 6.600348990129512e-05, '_Hours': 0.0015133036043322584, '_Days': 0.00014819131782922105, 'P_emaildomain_bin': 0.0018657351637627953, 'P_emaildomain_suffix': 0.0003151873890022234, 'R_emaildomain_bin': 0.010273468718128557, 'R_emaildomain_suffix': 0.00024056445606271285, 'device_name': 0.009052937942013853, '_Month': 0.0005063466005384575}\n"],"name":"stdout"}]},{"cell_type":"code","metadata":{"id":"EzG4-Q4dtiC4","colab_type":"code","colab":{}},"source":["#Write to csv results of categorical columns\n","\n","import csv  \n","with open('features_MI_fill_random_distribution.csv','a', newline='') as f:  \n","    w = csv.writer(f)  \n","    w.writerows(mi_per_column.items())"],"execution_count":0,"outputs":[]},{"cell_type":"code","metadata":{"id":"LCE-5VLntiC-","colab_type":"code","outputId":"ef042dca-4f35-4b88-d3f2-972ac264c783","colab":{}},"source":["#List of columns with float numbers\n","\n","float_cols = df.select_dtypes(include=['floating']).columns\n","len(float_cols)"],"execution_count":0,"outputs":[{"output_type":"execute_result","data":{"text/plain":["321"]},"metadata":{"tags":[]},"execution_count":35}]},{"cell_type":"code","metadata":{"id":"oN3Ml4LatiDE","colab_type":"code","colab":{}},"source":["#Colums with integer numbers only\n","\n","float_to_int_cols=set()\n","\n","for col in float_cols:\n","    col_df = df[col].dropna()\n","    col_should_be_int = col_df.map(float.is_integer).all() \n","    if col_should_be_int:\n","        float_to_int_cols.add(col)  "],"execution_count":0,"outputs":[]},{"cell_type":"code","metadata":{"id":"6hs-lzfbtiDK","colab_type":"code","outputId":"058479f8-e12e-49ed-a3df-a0fa0689be13","colab":{}},"source":["len(float_to_int_cols)"],"execution_count":0,"outputs":[{"output_type":"execute_result","data":{"text/plain":["260"]},"metadata":{"tags":[]},"execution_count":37}]},{"cell_type":"code","metadata":{"id":"PbWvEsXgtiDR","colab_type":"code","outputId":"57f2b8e0-61d5-4634-c9ca-11fca074acbb","colab":{}},"source":["#Creating dictionary to distinguish cardinality >10 or <10\n","i=0\n","is_discrete={}\n","discrete_cols=[]\n","nondiscrete_cols=[]\n","\n","for col in float_to_int_cols:\n","    count=df[col].value_counts()\n","    #print(col, count)\n","    if df[col].nunique()<10:\n","        #print(col, count)\n","        i=i+1\n","        discrete_cols.append(col)\n","    else:\n","        nondiscrete_cols.append(col)\n","print (i)"],"execution_count":0,"outputs":[{"output_type":"stream","text":["87\n"],"name":"stdout"}]},{"cell_type":"code","metadata":{"id":"r6_sEJcXtiDX","colab_type":"code","outputId":"3b450794-1f58-4121-df07-4c76fac0b965","colab":{}},"source":["print(is_discrete)"],"execution_count":0,"outputs":[{"output_type":"stream","text":["{'V117': True, 'D4': False, 'D2': False, 'V112': True, 'V243': False, 'V281': False, 'V83': True, 'V257': False, 'D1': False, 'D11': False, 'V32': False, 'V98': False, 'V223': False, 'V262': False, 'V10': True, 'V90': True, 'V285': False, 'V42': True, 'V34': False, 'V288': False, 'V20': False, 'V78': False, 'V2': True, 'V114': True, 'V284': False, 'V41': True, 'V287': False, 'V167': False, 'C9': False, 'V298': False, 'V192': False, 'V261': False, 'V19': True, 'V43': True, 'V45': False, 'V191': False, 'V168': False, 'V249': False, 'id_02': False, 'V30': True, 'V241': True, 'C2': False, 'V176': False, 'V253': False, 'V71': True, 'V299': False, 'V217': False, 'V194': True, 'V15': True, 'V245': False, 'V82': True, 'V24': False, 'V178': False, 'V304': False, 'V26': False, 'V111': True, 'V21': True, 'V247': False, 'V48': True, 'V72': False, 'V38': False, 'V68': True, 'V220': False, 'V197': False, 'id_06': False, 'V59': False, 'V226': False, 'V50': True, 'V16': False, 'V303': False, 'V6': False, 'V35': True, 'V75': True, 'V22': True, 'V56': False, 'V5': True, 'V116': True, 'V104': False, 'V302': False, 'V120': True, 'V121': True, 'V234': False, 'D15': False, 'V99': False, 'V14': True, 'V25': True, 'V100': False, 'V101': False, 'V242': False, 'C5': False, 'V39': False, 'V286': True, 'V33': True, 'V93': True, 'V73': True, 'D3': False, 'V289': False, 'V293': False, 'V238': False, 'C3': False, 'V63': True, 'V105': False, 'V237': False, 'C14': False, 'V173': True, 'V8': True, 'V87': False, 'C7': False, 'C11': False, 'V54': True, 'V79': True, 'V115': True, 'C4': False, 'C10': False, 'V44': False, 'V12': True, 'V225': False, 'V301': False, 'id_01': False, 'C1': False, 'V180': False, 'V46': True, 'V11': True, 'V49': True, 'id_05': False, 'V3': False, 'C6': False, 'V69': True, 'V118': True, 'V182': False, 'V169': False, 'V4': True, 'V280': False, 'V196': False, 'V18': False, 'C13': False, 'V84': True, 'V239': False, 'V40': False, 'V228': False, 'V250': False, 'V13': True, 'V184': False, 'V295': False, 'V235': False, 'V256': False, 'V227': False, 'V183': False, 'V60': False, 'V219': False, 'V97': False, 'V222': False, 'V70': True, 'V108': True, 'V37': False, 'V232': False, 'V294': False, 'V110': True, 'V186': False, 'V252': False, 'V88': True, 'V95': False, 'V61': True, 'V290': False, 'V31': True, 'V17': False, 'V244': False, 'V96': False, 'V74': True, 'V109': True, 'V189': False, 'V36': True, 'V106': False, 'V258': False, 'V57': True, 'V221': False, 'V92': True, 'V23': False, 'V53': True, 'D10': False, 'V255': False, 'V282': False, 'V107': True, 'V201': False, 'V179': False, 'C8': False, 'V103': False, 'V174': True, 'V185': False, 'V187': False, 'V94': True, 'V236': False, 'V229': False, 'V231': False, 'V52': True, 'dist1': False, 'V55': False, 'V170': False, 'V76': True, 'V125': False, 'V86': False, 'V195': False, 'V102': False, 'V89': True, 'V28': True, 'V119': True, 'V300': False, 'V181': False, 'V248': False, 'V123': False, 'V91': True, 'V230': False, 'V66': True, 'V198': False, 'V292': False, 'V193': False, 'V113': True, 'V188': False, 'V305': True, 'V81': False, 'V175': False, 'V246': False, 'V122': True, 'V291': False, 'V85': True, 'V29': True, 'V65': True, 'V190': False, 'V254': False, 'V172': False, 'D5': False, 'V259': False, 'V218': False, 'V51': True, 'V296': False, 'V77': False, 'V224': False, 'V27': True, 'V251': False, 'V283': False, 'V62': False, 'V7': False, 'V233': False, 'C12': False, 'V199': False, 'V297': False, 'V124': False, 'V279': False, 'V80': False, 'V200': False, 'V64': True, 'V171': False, 'V47': True, 'V177': False, 'V260': True, 'V240': True, 'V1': True, 'V9': True, 'V67': True, 'V58': False}\n"],"name":"stdout"}]},{"cell_type":"code","metadata":{"id":"8iHqrZ6gtiDg","colab_type":"code","outputId":"7aab93f7-bdcc-4dd1-cfe2-5bede65da246","colab":{}},"source":["#List of float columns without integers\n","\n","float_cols = set(float_cols)-set(float_to_int_cols) \n","len(float_cols)"],"execution_count":0,"outputs":[{"output_type":"execute_result","data":{"text/plain":["61"]},"metadata":{"tags":[]},"execution_count":41}]},{"cell_type":"code","metadata":{"id":"9isrrryDtiDt","colab_type":"code","outputId":"80835d61-8118-460c-be80-2926a0878c0d","colab":{}},"source":["print(len(discrete_cols))\n","#print(nondiscrete_cols)"],"execution_count":0,"outputs":[{"output_type":"stream","text":["87\n"],"name":"stdout"}]},{"cell_type":"code","metadata":{"id":"URzOVPIktiD5","colab_type":"code","outputId":"73ce6dde-2c5a-436b-85b4-171bba446691","colab":{}},"source":["#List of columns that should be treated with MI integral (float+integers with cardinality > 10)\n","\n","float_cols=set(float_cols).union(nondiscrete_cols)\n","print(len(float_cols))"],"execution_count":0,"outputs":[{"output_type":"stream","text":["234\n"],"name":"stdout"}]},{"cell_type":"code","metadata":{"id":"b7Wo6ziitiD_","colab_type":"code","outputId":"daa14f40-ea4e-4e8d-fa6a-4213d790bcad","colab":{}},"source":["#MI calculation of integers that behave like categorical columns (cardinality <10)\n","mi_per_column = {}  \n","mi_dict=calculate_mi_category(df[discrete_cols],y)  \n","mi_per_column.update(mi_dict)"],"execution_count":0,"outputs":[{"output_type":"stream","text":["Start calculating mutual info for columns:['V117', 'V112', 'V83', 'V10', 'V90', 'V42', 'V2', 'V114', 'V41', 'V19', 'V43', 'V30', 'V241', 'V71', 'V194', 'V15', 'V82', 'V111', 'V21', 'V48', 'V68', 'V50', 'V35', 'V75', 'V22', 'V5', 'V116', 'V120', 'V121', 'V14', 'V25', 'V286', 'V33', 'V93', 'V73', 'V63', 'V173', 'V8', 'V54', 'V79', 'V115', 'V12', 'V46', 'V11', 'V49', 'V69', 'V118', 'V4', 'V84', 'V13', 'V70', 'V108', 'V110', 'V88', 'V61', 'V31', 'V74', 'V109', 'V36', 'V57', 'V92', 'V53', 'V107', 'V174', 'V94', 'V52', 'V76', 'V89', 'V28', 'V119', 'V91', 'V66', 'V113', 'V305', 'V122', 'V85', 'V29', 'V65', 'V51', 'V27', 'V64', 'V47', 'V260', 'V240', 'V1', 'V9', 'V67']\n","Mutual info calculation took 3.29688 seconds\n","V117,0.0004620507567662364\n","V112,0.001975940244088467\n","V83,0.0020352106094551916\n","V10,0.000430069909450993\n","V90,0.005084946145698049\n","V42,0.006734521626659095\n","V2,0.003918343455823243\n","V114,0.0019174620750561453\n","V41,6.4691384514953265e-06\n","V19,0.0003364745651482496\n","V43,0.006941791244995152\n","V30,0.005014937337487004\n","V241,0.0064439219010543145\n","V71,0.006132605516898261\n","V194,0.008199192841987672\n","V15,0.00800067672761103\n","V82,0.0013660026988056957\n","V111,0.002369253831914983\n","V21,0.007077016103953165\n","V48,0.003799831506460132\n","V68,2.6614867686692434e-06\n","V50,0.006428120567492348\n","V35,0.001661865496440327\n","V75,0.0018472305495337627\n","V22,0.007197805655014323\n","V5,0.002755908905251458\n","V116,0.0016283248722629585\n","V120,0.00010885820185796577\n","V121,5.6777162501335523e-05\n","V14,0.0004909605921569332\n","V25,0.0006298593431823217\n","V286,1.685215158097006e-05\n","V33,0.008563897861330674\n","V93,0.008115041226568816\n","V73,0.007019550702713286\n","V63,0.005868683732225188\n","V173,4.073031399197017e-05\n","V8,0.003924055039749165\n","V54,0.002684543136333375\n","V79,0.00900132887232844\n","V115,0.0011323636925722625\n","V12,0.002148186246263943\n","V46,0.001558176710970787\n","V11,0.0005114366927721736\n","V49,0.004068163731981746\n","V69,0.004630425812862473\n","V118,0.0003321837820634932\n","V4,0.0026473232152070935\n","V84,0.007194158528898078\n","V13,0.0024740137220546723\n","V70,0.0046689088934856945\n","V108,0.0017181388327408298\n","V110,0.0014517694500256197\n","V88,0.00038997464270149745\n","V61,0.0013850457569511809\n","V31,0.007128883457209319\n","V74,0.007699965664694257\n","V109,0.0010708488056869103\n","V36,0.0018684878675520114\n","V57,0.0069042828506336845\n","V92,0.007592929542188926\n","V53,0.002323296410323823\n","V107,1.6102584475248083e-06\n","V174,0.00088283913793132\n","V94,0.008570178112682191\n","V52,0.00847565300473882\n","V76,0.0020808702589479935\n","V89,1.5211025045935819e-05\n","V28,1.7911927511655552e-05\n","V119,0.00043352138860704506\n","V91,0.00524474509190061\n","V66,0.002082311325149027\n","V113,0.00219081870902971\n","V305,8.616034452753069e-08\n","V122,0.00010953593585738277\n","V85,0.007221822780696864\n","V29,0.0051856853640545346\n","V65,0.0010373896036161775\n","V51,0.008293572525820926\n","V27,2.473673178005249e-05\n","V64,0.006102323653501719\n","V47,0.0018868843534344587\n","V260,0.004291663918276772\n","V240,0.006431164230958043\n","V1,0.003934534173843943\n","V9,0.004236372015377066\n","V67,0.003373518791647918\n","{'V117': 0.0004620507567662364, 'V112': 0.001975940244088467, 'V83': 0.0020352106094551916, 'V10': 0.000430069909450993, 'V90': 0.005084946145698049, 'V42': 0.006734521626659095, 'V2': 0.003918343455823243, 'V114': 0.0019174620750561453, 'V41': 6.4691384514953265e-06, 'V19': 0.0003364745651482496, 'V43': 0.006941791244995152, 'V30': 0.005014937337487004, 'V241': 0.0064439219010543145, 'V71': 0.006132605516898261, 'V194': 0.008199192841987672, 'V15': 0.00800067672761103, 'V82': 0.0013660026988056957, 'V111': 0.002369253831914983, 'V21': 0.007077016103953165, 'V48': 0.003799831506460132, 'V68': 2.6614867686692434e-06, 'V50': 0.006428120567492348, 'V35': 0.001661865496440327, 'V75': 0.0018472305495337627, 'V22': 0.007197805655014323, 'V5': 0.002755908905251458, 'V116': 0.0016283248722629585, 'V120': 0.00010885820185796577, 'V121': 5.6777162501335523e-05, 'V14': 0.0004909605921569332, 'V25': 0.0006298593431823217, 'V286': 1.685215158097006e-05, 'V33': 0.008563897861330674, 'V93': 0.008115041226568816, 'V73': 0.007019550702713286, 'V63': 0.005868683732225188, 'V173': 4.073031399197017e-05, 'V8': 0.003924055039749165, 'V54': 0.002684543136333375, 'V79': 0.00900132887232844, 'V115': 0.0011323636925722625, 'V12': 0.002148186246263943, 'V46': 0.001558176710970787, 'V11': 0.0005114366927721736, 'V49': 0.004068163731981746, 'V69': 0.004630425812862473, 'V118': 0.0003321837820634932, 'V4': 0.0026473232152070935, 'V84': 0.007194158528898078, 'V13': 0.0024740137220546723, 'V70': 0.0046689088934856945, 'V108': 0.0017181388327408298, 'V110': 0.0014517694500256197, 'V88': 0.00038997464270149745, 'V61': 0.0013850457569511809, 'V31': 0.007128883457209319, 'V74': 0.007699965664694257, 'V109': 0.0010708488056869103, 'V36': 0.0018684878675520114, 'V57': 0.0069042828506336845, 'V92': 0.007592929542188926, 'V53': 0.002323296410323823, 'V107': 1.6102584475248083e-06, 'V174': 0.00088283913793132, 'V94': 0.008570178112682191, 'V52': 0.00847565300473882, 'V76': 0.0020808702589479935, 'V89': 1.5211025045935819e-05, 'V28': 1.7911927511655552e-05, 'V119': 0.00043352138860704506, 'V91': 0.00524474509190061, 'V66': 0.002082311325149027, 'V113': 0.00219081870902971, 'V305': 8.616034452753069e-08, 'V122': 0.00010953593585738277, 'V85': 0.007221822780696864, 'V29': 0.0051856853640545346, 'V65': 0.0010373896036161775, 'V51': 0.008293572525820926, 'V27': 2.473673178005249e-05, 'V64': 0.006102323653501719, 'V47': 0.0018868843534344587, 'V260': 0.004291663918276772, 'V240': 0.006431164230958043, 'V1': 0.003934534173843943, 'V9': 0.004236372015377066, 'V67': 0.003373518791647918}\n"],"name":"stdout"}]},{"cell_type":"code","metadata":{"id":"SNpGyv3PtiEJ","colab_type":"code","colab":{}},"source":["with open('features_MI_fill_random_distribution.csv','a', newline='') as f:  \n","    w = csv.writer(f)  \n","    w.writerows(mi_per_column.items())"],"execution_count":0,"outputs":[]},{"cell_type":"code","metadata":{"id":"4eS8_gZ0tiEV","colab_type":"code","colab":{}},"source":["def batch(iterable, n=1):\n","    l = len(iterable)\n","    for ndx in range(0, l, n):\n","        yield iterable[ndx:min(ndx + n, l)]"],"execution_count":0,"outputs":[]},{"cell_type":"code","metadata":{"id":"qkhWhrfGtiEc","colab_type":"code","colab":{}},"source":["# MI for float numbers (integrals)\n","\n","def calculate_mi_cont(x,y):  \n","    mi_dict ={}  \n","    print('Start calculating mutual info for columns:' + str(list(x.columns)))  \n","    start_time=start_timer()  \n","    feature_scores = mutual_info_classif(x, y)  \n","    print('Mutual info calculation took '+end_timer(start_time))  \n","    for i in range(len(x.columns)):  \n","        print(x.columns[i]+','+str(feature_scores[i]))  \n","        mi_dict[x.columns[i]] = feature_scores[i]  \n","    print(mi_dict)  \n","    return mi_dict\n"],"execution_count":0,"outputs":[]},{"cell_type":"code","metadata":{"id":"q7h2HTCTtiEg","colab_type":"code","outputId":"ea734f41-8d7b-48c0-84ce-926e77c0f946","colab":{}},"source":["mi_per_column = {}  \n","for x in batch(list(float_cols), 5):  \n","    mi_dict=calculate_mi_cont(df[x],y)  \n","    mi_per_column.update(mi_dict)"],"execution_count":0,"outputs":[{"output_type":"stream","text":["Start calculating mutual info for columns:['D4', 'V266', 'D2', 'V243', 'V281']\n","Mutual info calculation took 24.60938 seconds\n","D4,0.005680147123761747\n","V266,0.008470253765797686\n","D2,0.002283721502656544\n","V243,0.03579016834080351\n","V281,0.002346933776525839\n","{'D4': 0.005680147123761747, 'V266': 0.008470253765797686, 'D2': 0.002283721502656544, 'V243': 0.03579016834080351, 'V281': 0.002346933776525839}\n","Start calculating mutual info for columns:['V257', 'D1', 'D11', 'V207', 'V32']\n","Mutual info calculation took 24.51562 seconds\n","V257,0.05461392254982633\n","D1,0.005488326987590053\n","D11,0.004221977494940576\n","V207,0.010718157633529124\n","V32,0.008610606405535792\n","{'V257': 0.05461392254982633, 'D1': 0.005488326987590053, 'D11': 0.004221977494940576, 'V207': 0.010718157633529124, 'V32': 0.008610606405535792}\n","Start calculating mutual info for columns:['V98', 'V320', 'V223', 'V262', 'V285']\n","Mutual info calculation took 23.71875 seconds\n","V98,0.0004615885762360161\n","V320,0.003339217811255235\n","V223,0.0009058708005165172\n","V262,0.014580515223079793\n","V285,0.0048802562384764325\n","{'V98': 0.0004615885762360161, 'V320': 0.003339217811255235, 'V223': 0.0009058708005165172, 'V262': 0.014580515223079793, 'V285': 0.0048802562384764325}\n","Start calculating mutual info for columns:['V34', 'V288', 'V20', 'V78', 'V284']\n","Mutual info calculation took 25.29688 seconds\n","V34,0.010584375914218125\n","V288,0.0023732573910648647\n","V20,0.05219868595841293\n","V78,0.05086299841256281\n","V284,0.0009938331551946344\n","{'V34': 0.010584375914218125, 'V288': 0.0023732573910648647, 'V20': 0.05219868595841293, 'V78': 0.05086299841256281, 'V284': 0.0009938331551946344}\n","Start calculating mutual info for columns:['V319', 'V287', 'V167', 'V273', 'V321']\n","Mutual info calculation took 24.34375 seconds\n","V319,0.0008204091464247831\n","V287,0.0031379837353140427\n","V167,0.004790648279015208\n","V273,0.005260586116459853\n","V321,0.0018436745845766822\n","{'V319': 0.0008204091464247831, 'V287': 0.0031379837353140427, 'V167': 0.004790648279015208, 'V273': 0.005260586116459853, 'V321': 0.0018436745845766822}\n","Start calculating mutual info for columns:['C9', 'V298', 'V192', 'V215', 'V261']\n","Mutual info calculation took 24.06250 seconds\n","C9,0.010758781024086561\n","V298,0.0009880134585833478\n","V192,0.0710784435058518\n","V215,0.009830529176542369\n","V261,0.010548849813460048\n","{'C9': 0.010758781024086561, 'V298': 0.0009880134585833478, 'V192': 0.0710784435058518, 'V215': 0.009830529176542369, 'V261': 0.010548849813460048}\n","Start calculating mutual info for columns:['V45', 'V191', 'V277', 'V168', 'V203']\n","Mutual info calculation took 24.60938 seconds\n","V45,0.05176245711172167\n","V191,0.07622721750228223\n","V277,0.009734339580139917\n","V168,0.004810224363344329\n","V203,0.006206770294695518\n","{'V45': 0.05176245711172167, 'V191': 0.07622721750228223, 'V277': 0.009734339580139917, 'V168': 0.004810224363344329, 'V203': 0.006206770294695518}\n","Start calculating mutual info for columns:['V249', 'id_02', 'V315', 'V211', 'C2']\n","Mutual info calculation took 24.23438 seconds\n","V249,0.08371389462352763\n","id_02,0.0006416203931249864\n","V315,0.008856341470941276\n","V211,0.006250574714916857\n","C2,0.010206001702556922\n","{'V249': 0.08371389462352763, 'id_02': 0.0006416203931249864, 'V315': 0.008856341470941276, 'V211': 0.006250574714916857, 'C2': 0.010206001702556922}\n","Start calculating mutual info for columns:['V176', 'V253', 'V299', 'V268', 'V278']\n","Mutual info calculation took 25.12500 seconds\n","V176,0.0560723441339952\n","V253,0.030191043753483804\n","V299,0.0008402489772703081\n","V268,0.010250324147408119\n","V278,0.00883971983304388\n","{'V176': 0.0560723441339952, 'V253': 0.030191043753483804, 'V299': 0.0008402489772703081, 'V268': 0.010250324147408119, 'V278': 0.00883971983304388}\n","Start calculating mutual info for columns:['V217', 'V245', 'V267', 'V24', 'V178']\n","Mutual info calculation took 24.48438 seconds\n","V217,0.007228823320921229\n","V245,0.05197979592346047\n","V267,0.010685754426217331\n","V24,0.05639173502884609\n","V178,0.004768795570428086\n","{'V217': 0.007228823320921229, 'V245': 0.05197979592346047, 'V267': 0.010685754426217331, 'V24': 0.05639173502884609, 'V178': 0.004768795570428086}\n","Start calculating mutual info for columns:['V26', 'V247', 'V271', 'V72', 'V38']\n","Mutual info calculation took 23.78125 seconds\n","V26,0.06467776766808453\n","V247,0.07320300866169549\n","V271,0.010686287162630315\n","V72,0.007949923896268585\n","V38,0.03999147954466764\n","{'V26': 0.06467776766808453, 'V247': 0.07320300866169549, 'V271': 0.010686287162630315, 'V72': 0.007949923896268585, 'V38': 0.03999147954466764}\n","Start calculating mutual info for columns:['V265', 'V276', 'V220', 'V197', 'id_06']\n","Mutual info calculation took 23.51562 seconds\n","V265,0.004843652340578375\n","V276,0.008067524726061537\n","V220,0.0009732167589593388\n","V197,0.013510305810576106\n","id_06,0.04072564881794993\n","{'V265': 0.004843652340578375, 'V276': 0.008067524726061537, 'V220': 0.0009732167589593388, 'V197': 0.013510305810576106, 'id_06': 0.04072564881794993}\n","Start calculating mutual info for columns:['V59', 'V226', 'V314', 'V16', 'V303']\n","Mutual info calculation took 24.15625 seconds\n","V59,0.00746900078418733\n","V226,0.0005639016109164618\n","V314,0.008651201927613261\n","V16,0.008870935802734792\n","V303,0.012315751385224116\n","{'V59': 0.00746900078418733, 'V226': 0.0005639016109164618, 'V314': 0.008651201927613261, 'V16': 0.008870935802734792, 'V303': 0.012315751385224116}\n","Start calculating mutual info for columns:['V6', 'V208', 'V205', 'V317', 'V56']\n","Mutual info calculation took 24.09375 seconds\n","V6,0.046940541102762356\n","V208,0.010761426890554704\n","V205,0.009344527346566656\n","V317,0.009414089182242735\n","V56,0.04987225950713303\n","{'V6': 0.046940541102762356, 'V208': 0.010761426890554704, 'V205': 0.009344527346566656, 'V317': 0.009414089182242735, 'V56': 0.04987225950713303}\n","Start calculating mutual info for columns:['V104', 'V302', 'V307', 'V272', 'V234']\n"],"name":"stdout"}]},{"cell_type":"code","metadata":{"id":"mTcUgH3YtiEm","colab_type":"code","colab":{}},"source":["len(mi_per_column)"],"execution_count":0,"outputs":[]},{"cell_type":"code","metadata":{"id":"gknDh1HztiEs","colab_type":"code","colab":{}},"source":["with open('features_MI_fill_random_distribution.csv','a', newline='') as f:  \n","    w = csv.writer(f)  \n","    w.writerows(mi_per_column.items())"],"execution_count":0,"outputs":[]},{"cell_type":"code","metadata":{"id":"vJFBZKtItiEw","colab_type":"code","colab":{}},"source":["selector = SelectKBest(chi2, k=5).fit(df_category,y)\n","X_new=selector.transform(df_category)\n","\n","scores = selector.scores_"],"execution_count":0,"outputs":[]},{"cell_type":"code","metadata":{"id":"IQ-loni_tiE1","colab_type":"code","colab":{}},"source":["score_col={}\n","for i in range(len(df_category.columns)):\n","    print(df_category.columns[i]+', '+str(scores[i]))\n","    score_col[df_category.columns[i]]=scores[i]\n","              \n","print(score_col)"],"execution_count":0,"outputs":[]},{"cell_type":"code","metadata":{"id":"wBk5y9BktiE6","colab_type":"code","colab":{}},"source":["sorted_score_col = sorted(score_col.items(), key=lambda kv: kv[1])  \n","\n","print(sorted_score_col)"],"execution_count":0,"outputs":[]},{"cell_type":"code","metadata":{"id":"vwg0d1IbtiE8","colab_type":"code","colab":{}},"source":["#df_numeric"],"execution_count":0,"outputs":[]},{"cell_type":"code","metadata":{"id":"8hFUWT-ytiE9","colab_type":"code","colab":{}},"source":[""],"execution_count":0,"outputs":[]},{"cell_type":"code","metadata":{"id":"7-toLLKLtiE_","colab_type":"code","colab":{}},"source":[""],"execution_count":0,"outputs":[]},{"cell_type":"code","metadata":{"id":"dFEttcAKtiFA","colab_type":"code","colab":{}},"source":["def calculate_mi(x,y):  \n","    mi_dict ={}  \n","    print('Start calculating mutual info for columns:' + str(list(x.columns)))  \n","    start_time=start_timer()  \n","    feature_scores = mutual_info_classif(x, y)  \n","    print('Mutual info calculation took '+end_timer(start_time))  \n","    for i in range(len(x.columns)):  \n","        print(x.columns[i]+','+str(feature_scores[i]))  \n","        mi_dict[x.columns[i]] = feature_scores[i]  \n","    print(mi_dict)  \n","    return mi_dict"],"execution_count":0,"outputs":[]},{"cell_type":"code","metadata":{"id":"P5EGXaCbtiFC","colab_type":"code","colab":{}},"source":["mi_per_column = {}  \n","for x in batch(numeric_cols, 5):  \n","    mi_dict=calculate_mi(df_numeric[x],y)  \n","    mi_per_column.update(mi_dict)"],"execution_count":0,"outputs":[]},{"cell_type":"code","metadata":{"id":"Pr45XjTMtiFF","colab_type":"code","colab":{}},"source":["#Test two dicts  \n","mi_per_col={}  \n","mi_dict=calculate_mi(df_numeric[['C1','C2','C3']],y)  \n","mi_per_col.update(mi_dict)  \n","mi_dict=calculate_mi(df_numeric[['C4','C5']],y)  \n","mi_per_col.update(mi_dict)  \n","print(mi_per_col)"],"execution_count":0,"outputs":[]},{"cell_type":"code","metadata":{"id":"JMzBUHVKtiFI","colab_type":"code","colab":{}},"source":["#df1=pd.read_csv('mi_numeric-1.csv')#1\n","df1=pd.read_csv('mi_numeric-2.csv')#2\n"],"execution_count":0,"outputs":[]},{"cell_type":"code","metadata":{"id":"fcWrr5jUtiFL","colab_type":"code","colab":{}},"source":["df1=df1.sort_values(by=['MI'], ascending=False)"],"execution_count":0,"outputs":[]},{"cell_type":"code","metadata":{"id":"3S96jJA4tiFN","colab_type":"code","colab":{}},"source":["df1.head(30)"],"execution_count":0,"outputs":[]},{"cell_type":"code","metadata":{"id":"Ts0a3obVtiFR","colab_type":"code","colab":{}},"source":["features=df_numeric[list(df1['Col_Name'].head(30))]\n","target=df['isFraud']"],"execution_count":0,"outputs":[]},{"cell_type":"code","metadata":{"id":"f4NAB0extiFV","colab_type":"code","colab":{}},"source":["from sklearn.linear_model import LogisticRegression\n","\n","model = LogisticRegression()\n","X, y = features,target\n","model.fit(X,y)\n","# compute with formulas from the theory\n","yhat = model.predict(X)\n","SS_Residual = sum((y-yhat)**2)\n","SS_Total = sum((y-np.mean(y))**2)\n","r_squared = 1 - (float(SS_Residual))/SS_Total\n","adjusted_r_squared = 1 - (1-r_squared)*(len(y)-1)/(len(y)-X.shape[1]-1)\n","print('r_squared= ',r_squared)\n","print('adjusted_r_squared= ',adjusted_r_squared)"],"execution_count":0,"outputs":[]},{"cell_type":"code","metadata":{"id":"REaO1kdytiFa","colab_type":"code","colab":{}},"source":["from sklearn import datasets, linear_model\n","from sklearn.model_selection import cross_validate\n","from sklearn.metrics import make_scorer\n","from sklearn.metrics import confusion_matrix\n","from sklearn.svm import LinearSVC"],"execution_count":0,"outputs":[]},{"cell_type":"code","metadata":{"id":"hWs2FKXmtiFc","colab_type":"code","colab":{}},"source":["cv_results = cross_validate(model, X, y, cv=5)\n","sorted(cv_results.keys())"],"execution_count":0,"outputs":[]},{"cell_type":"code","metadata":{"id":"7I213ZRbtiFh","colab_type":"code","colab":{}},"source":["cv_results['test_score']"],"execution_count":0,"outputs":[]},{"cell_type":"code","metadata":{"id":"zJQmkN76tiFm","colab_type":"code","colab":{}},"source":["#df2=pd.read_csv('test-fraud_data_remove_nulls_fill_mean-mode.csv',compression='gzip', dtype=map_object_cols)#1\n","df2=pd.read_csv('test-fraud_data_remove_nulls_fill_random_distribution.csv',compression='gzip', dtype=map_object_cols)#2"],"execution_count":0,"outputs":[]},{"cell_type":"code","metadata":{"id":"7PhS-bsttiFo","colab_type":"code","colab":{}},"source":["features_test=df2[list(df1['Col_Name'].head(30))]\n","target=df2['isFraud']"],"execution_count":0,"outputs":[]},{"cell_type":"code","metadata":{"id":"3nltq2TrtiFp","colab_type":"code","colab":{}},"source":["predictions = model.predict(features_test)"],"execution_count":0,"outputs":[]},{"cell_type":"code","metadata":{"id":"xv8m6EkYtiFr","colab_type":"code","colab":{}},"source":["predictions"],"execution_count":0,"outputs":[]},{"cell_type":"code","metadata":{"id":"DJtAANA-tiFt","colab_type":"code","colab":{}},"source":["## The line / model\n","from matplotlib import pyplot as plt\n","\n","plt.scatter(target, predictions)\n","#plt.xlabel('True_Values')\n","#plt.ylabel('Predictions')"],"execution_count":0,"outputs":[]},{"cell_type":"code","metadata":{"id":"dg6rHDD0tiFx","colab_type":"code","colab":{}},"source":["#print the accuracy score\n","\n","print ('Score:', model.score(features_test, target))"],"execution_count":0,"outputs":[]},{"cell_type":"code","metadata":{"id":"1ZCKpkndtiF1","colab_type":"code","colab":{}},"source":["#CONFUSION MATRIX\n","\n","from sklearn.metrics import confusion_matrix \n","from sklearn.metrics import accuracy_score \n","from sklearn.metrics import classification_report \n","\n","results = confusion_matrix(target, predictions) \n","print('Confusion Matrix :')\n","print(results) \n","print ('Accuracy Score :',accuracy_score(target, predictions)) \n","print ('Report : ')\n","print (classification_report(target, predictions)) "],"execution_count":0,"outputs":[]},{"cell_type":"code","metadata":{"id":"Yfa5trQutiF4","colab_type":"code","colab":{}},"source":[""],"execution_count":0,"outputs":[]}]}