car_purchase_preprocessing.ipynb

{
 "cells": [
  {
   "cell_type": "code",
<<<<<<< HEAD
   "execution_count": 78,
=======
   "execution_count": 16,
>>>>>>> c5276f11f63080e0701ee240d67e42d0190ca261
   "metadata": {},
   "outputs": [],
   "source": [
    "import pandas as pd\n",
    "import numpy as np\n",
    "import glob\n",
    "import os\n",
    "import matplotlib.pyplot as plt\n",
    "from matplotlib.pyplot import figure\n",
    "from scipy import stats\n",
    "\n",
    "import statsmodels.api as sm\n",
<<<<<<< HEAD
    "from statsmodels.formula.api import ols\n",
    "\n",
    "from statsmodels.stats.multicomp import pairwise_tukeyhsd"
=======
    "from statsmodels.formula.api import ols"
>>>>>>> c5276f11f63080e0701ee240d67e42d0190ca261
   ]
  },
  {
   "cell_type": "code",
<<<<<<< HEAD
   "execution_count": 85,
=======
   "execution_count": 35,
>>>>>>> c5276f11f63080e0701ee240d67e42d0190ca261
   "metadata": {},
   "outputs": [],
   "source": [
    "class CarData:\n",
    "    \n",
    "    missing = -1\n",
    "    duplicates = -1\n",
    "    \n",
    "    price_outliers = -1\n",
    "    mileage_outliers = -1\n",
    "    year_outliers = -1\n",
    "    total_discard = -1\n",
    "    \n",
    "    def __init__(self, path, price_outlier_mt = 200000, mileage_outlier_mt = 400000,\n",
    "                 year_outlier_lt = 1995, engine_outlier_mt = 4000, engine_outlier_lt = 750,\n",
    "                brands = ['ford']):\n",
    "        \n",
    "        #define outliers values\n",
    "        self.price_outlier_mt = price_outlier_mt\n",
    "        self.mileage_outlier_mt = mileage_outlier_mt\n",
    "        self.year_outlier_lt =  year_outlier_lt\n",
    "        self.engine_outlier_mt = engine_outlier_mt\n",
    "        self.engine_outlier_lt = engine_outlier_lt\n",
    "        self.brands = brands\n",
    "        \n",
    "        #defince variable data types\n",
    "        self.numeric_variables = ['price', 'mileage_km', 'engine_cm3', 'year']\n",
    "        self.categorical_variables = ['engine_type', 'city', 'province']\n",
    "        \n",
    "        #read all .csv files from the directory\n",
    "        self.data = pd.concat(map(pd.read_csv, glob.glob(os.path.join(path, \"*.csv\"))), sort=False)\n",
    "        \n",
    "        #drop the duplicates and save the number of duplicates - many duplicates due to data gathering method\n",
    "        self.duplicates = len(self.data) - len(self.data.drop_duplicates())\n",
    "        self.data.drop_duplicates(inplace = True)\n",
    "        \n",
    "        #rename columns\n",
    "        self.data.columns = ['title', 'price', 'sub_title', 'mileage_km', 'year', 'engine_cm3',\n",
    "                'engine_type', 'city', 'province', 'negotiable']\n",
    "        \n",
    "        #drop NaNs and save the number of rows dropped to the missing varaible\n",
    "        self.missing = self.data['engine_type'].isna().sum()\n",
    "        self.data.dropna(subset = ['engine_type'], axis = 'index', inplace = True)\n",
    "        \n",
    "        self.missing = self.missing + self.data['city'].isna().sum()\n",
    "        self.data.dropna(subset = ['city'], axis = 'index', inplace = True)\n",
    "        \n",
    "        self.missing = self.missing + self.data['engine_cm3'].isna().sum()\n",
    "        self.data.dropna(subset = ['engine_cm3'], axis = 'index', inplace = True)\n",
    "        \n",
    "        #clean up the columns\n",
    "        self.data['price'] = self.data['price'].apply(lambda x: x.replace(\",\", \"\"))\n",
    "        self.data['price'] = self.data['price'].apply(lambda x: x.replace(\" \", \"\")).astype('int')\n",
    "        \n",
    "        self.data['mileage_km'] = self.data['mileage_km'].apply(lambda x: x.replace(\"km\", \"\"))\n",
    "        self.data['mileage_km'] = self.data['mileage_km'].apply(lambda x: x.replace(\" \", \"\")).astype('float')\n",
    "        \n",
    "        self.data['engine_cm3'] = self.data['engine_cm3'].astype('str')\n",
    "        self.data['engine_cm3'] = self.data['engine_cm3'].apply(lambda x: x.replace('cm3', ''))\n",
    "        self.data['engine_cm3'] = self.data['engine_cm3'].apply(lambda x: x.replace(' ','')).astype('int')\n",
    "        \n",
    "        self.data['province'] = self.data['province'].astype('str')\n",
    "        self.data['province'] = self.data['province'].apply(lambda x: x.replace('(',''))\n",
    "        self.data['province'] = self.data['province'].apply(lambda x: x.replace(')',''))\n",
    "        \n",
    "        self.data['sub_title'] = self.data['sub_title'].astype('str') #may change that in the future - possible info loss due to lowercase\n",
    "        self.data['sub_title'] = self.data['sub_title'].apply(lambda x: x.lower())\n",
    "        \n",
    "        self.data['title'] = self.data['title'].astype('str') #may change that in the future - possible info loss due to lowercase\n",
    "        self.data['title'] = self.data['title'].apply(lambda x: x.lower())\n",
    "    \n",
    "        #Add ID column\n",
    "        self.data.insert(loc = 0, column = 'ID', value = range(1, len(self.data)+1))\n",
    "\n",
    "        #discard outliers and calculate the numbers\n",
    "        self.total_discard = len(self.data) - len(self.data[(self.data['price'] <= self.price_outlier_mt) &\n",
    "                                                        (self.data['mileage_km'] <= self.mileage_outlier_mt) &\n",
    "                                                        (self.data['year'] >= self.year_outlier_lt) &\n",
    "                                                        (self.data['engine_cm3'] <= self.engine_outlier_mt) &\n",
    "                                                        (self.data['engine_cm3'] >= self.engine_outlier_lt)])        \n",
    "        \n",
    "        self.price_outliers = len(self.data[self.data['price'] > price_outlier_mt])\n",
    "        self.data = self.data[self.data['price'] <= price_outlier_mt]\n",
    "        \n",
    "        self.mileage_outliers = len(self.data[self.data['mileage_km'] > mileage_outlier_mt])\n",
    "        self.data = self.data[self.data['mileage_km'] <= mileage_outlier_mt]\n",
    "        \n",
    "        self.year_outliers = len(self.data[self.data['year'] < year_outlier_lt])\n",
    "        self.data = self.data[self.data['year'] >= year_outlier_lt]\n",
    "        \n",
    "        self.engine_outliers = len(self.data[(self.data['engine_cm3'] > engine_outlier_mt) |\n",
    "                                            (self.data['engine_cm3'] < engine_outlier_lt)])\n",
    "        self.data = self.data[(self.data['engine_cm3'] <= engine_outlier_mt) & \n",
    "                             (self.data['engine_cm3'] >= engine_outlier_lt)]\n",
    "        \n",
    "    def describe(self):\n",
    "        desc_stats = round(pd.DataFrame(\n",
    "                        data = self.data[['price', 'mileage_km', 'year', 'engine_cm3']].describe(),\n",
    "                        columns = self.data[['price', 'mileage_km', 'year', 'engine_cm3']].columns),2)\n",
    "        return desc_stats\n",
    "    \n",
    "    def outliers(self):\n",
    "        print('Offers with price greater than '+str(self.price_outlier_mt)+' have been discarded')\n",
    "        print('The number of such offers = '+str(self.price_outliers))\n",
    "        print('')\n",
    "        print('Offers with mileage greater than '+str(self.mileage_outlier_mt)+' have been discarded')\n",
    "        print('The number of such offers = '+str(self.mileage_outliers))\n",
    "        print('')\n",
    "        print('Offers with year lower than '+str(self.year_outlier_lt)+' have been discarded')\n",
    "        print('The number of such offers = '+str(self.year_outliers))\n",
    "        print('')\n",
    "        print('Offers with engine_cm3 greater than '+str(self.engine_outlier_lt)+\n",
    "              ' and lower than '+str(self.engine_outlier_mt)+' have been discarded')\n",
    "        print('The number of such offers = '+str(self.engine_outliers))\n",
    "        print('')\n",
    "        print('Total number of discarded offers = '+str(self.total_discard)\n",
    "              +'('+str(round(self.total_discard/len(self.data)*100,2))+'%)'\n",
    "              +' - may be different to the sum of above due to overlap')\n",
    "        \n",
    "    def scatter_nox(self, var = 'all', figsize_1 = 7, figsize_2 = 5):\n",
    "        if var != 'all' and var not in self.data.columns:\n",
    "            print('Variable not found in the dataset')\n",
    "        if var == 'all':\n",
    "            plt.rcParams[\"figure.figsize\"] = (figsize_1,figsize_2)\n",
    "            plt.scatter(y = self.data['mileage_km'], x = range(1, len(self.data)+1), s=1)\n",
    "            plt.title('mileage_km')\n",
    "            plt.show()\n",
    "\n",
    "            plt.scatter(y = self.data['price'], x = range(1, len(self.data)+1), s=1)\n",
    "            plt.title('price')\n",
    "            plt.show()\n",
    "\n",
    "            plt.scatter(y = self.data['year'], x = range(1, len(self.data)+1), s=1)\n",
    "            plt.title('year')\n",
    "            plt.show()\n",
    "\n",
    "            plt.scatter(y = self.data['engine_cm3'], x = range(1, len(self.data)+1), s=1)\n",
    "            plt.title('engine_cm3')\n",
    "            plt.show()\n",
    "        else:\n",
    "            plt.rcParams[\"figure.figsize\"] = (figsize_1,figsize_2)\n",
    "            plt.scatter(y = self.data[var], x = range(1, len(self.data)+1), s=1)\n",
    "            plt.title(var)\n",
    "            plt.show()\n",
    "            \n",
    "    def scatter(self, var = 'all'):\n",
    "        if var != 'all' and var not in self.data.columns:\n",
    "            print('Variable not found in the dataset')\n",
    "        list_comb = []\n",
    "        if var == 'all':\n",
    "            for variable1 in enumerate(self.numeric_variables):\n",
    "                for variable2 in enumerate(self.numeric_variables):\n",
    "                    if variable1 != variable2 and variable1[1]+variable2[1] not in list_comb and variable2[1]+variable1[1] not in list_comb:\n",
    "                        plt.scatter(y = self.data[variable1[1]], x = self.data[variable2[1]], s=1)\n",
    "                        plt.title(\"Correlation between \"+variable1[1]+' and '+variable2[1])\n",
    "                        plt.ylabel(variable1[1])\n",
    "                        plt.xlabel(variable2[1])\n",
    "                        plt.show()\n",
    "                        list_comb.append(variable1[1]+variable2[1])\n",
    "        #else: - TO DO\n",
    "    \n",
    "    def hist(self, var = 'all', bins = 50):\n",
    "        if var != 'all' and var not in self.data.columns:\n",
    "            print('Variable not found in the dataset')\n",
    "        #prints histograms for numeric variables\n",
    "        if var == 'all':\n",
    "            for variable in enumerate(self.numeric_variables):\n",
    "                plt.hist(x = self.data[variable[1]], bins = bins)\n",
    "                plt.title(variable[1])\n",
    "                plt.show()\n",
    "     \n",
    "    def price_cat_vars(self):\n",
    "        for variable in enumerate(self.categorical_variables):\n",
    "            # shows desrptive statistics of categorical variables\n",
    "            print(x.data.groupby(x.data[variable[1]])['price'].describe())\n",
    "            #the variables need further preprocessing\n",
    "            \n",
    "    def add_dummies(self, cat_list, columns):\n",
    "        #adds dummmies from cat_list, checks in every column of columns\n",
    "        for column in enumerate(columns):\n",
    "            for cat in enumerate(cat_list):\n",
    "                col_name = column[1] + '_' + cat[1]\n",
    "                self.data[col_name] = self.data[column[1]].str.contains(cat[1]).astype('int')\n",
    "                \n",
    "                #append newly craeted varaibles to categorical variables\n",
    "                if self.data[col_name].sum() > 0:\n",
    "                    self.categorical_variables.append(col_name)\n",
    "                else:\n",
    "                    self.data.drop(columns = [col_name], inplace = True)\n",
    "        return self\n",
    "        \n",
    "    def ind_test(self, var, alpha = 0.05):\n",
    "        if alpha > 1 or alpha < 0:\n",
    "            print('Incorrectalpha value. Select a value from <0;1>.')\n",
    "            \n",
    "        if var != 'all' and var not in self.data.columns:\n",
    "            print('Variable not found in the dataset')\n",
    "        pivot = round(self.data.pivot_table(values = 'price', index = var, aggfunc = ['count', 'mean']),2)\n",
    "        pivot.columns = ['count', 'mean']\n",
    "        \n",
    "        mean_price = self.data['price'].mean()\n",
    "        \n",
    "        pivot['sm'] = pivot['mean']/((pivot['count'])**(1/2))\n",
    "        \n",
    "        pivot['t'] = (pivot['mean']-mean_price)/pivot['sm']\n",
    "        pivot['df'] = pivot['count']-1\n",
    "\n",
    "        #calculate p-value\n",
    "        pivot['t_border'] = stats.t.ppf(1-alpha/2, pivot['df'])\n",
    "        \n",
    "        #implementation here is not 100% mathematically correct\n",
    "        return pivot\n",
    "    \n",
    "    def anova(self, var = 'all'):\n",
    "        if var == 'all':\n",
    "            for variable in enumerate(self.categorical_variables):\n",
<<<<<<< HEAD
    "                anova_data = self.data[[variable[1], 'price']].reset_index().copy()\n",
    "                anova_data.columns = ['index', variable[1], 'price']\n",
    "                equation_string = 'price ~ '+str(variable[1])\n",
    "                model = ols(equation_string, data=anova_data).fit()\n",
    "                anova_table = sm.stats.anova_lm(model, typ=2)\n",
    "                print(anova_table)\n",
    "                print()\n",
    "                \n",
    "                #pairwise comparisons\n",
    "                pairwise_comparison = pairwise_tukeyhsd(endog = anova_data['price'],\n",
    "                                                        groups = anova_data[variable[1]],\n",
    "                                                        alpha = 0.05)\n",
    "                print(pairwise_comparison)"
=======
    "\n",
    "                equation_string = 'value ~ C('+str(variable[1]+')')\n",
    "                model = ols(equation_string, data=d_melt).fit()\n",
    "                anova_table = sm.stats.anova_lm(model, typ=2)\n",
    "        return anova_table"
>>>>>>> c5276f11f63080e0701ee240d67e42d0190ca261
   ]
  },
  {
   "cell_type": "code",
<<<<<<< HEAD
   "execution_count": 86,
=======
   "execution_count": 36,
>>>>>>> c5276f11f63080e0701ee240d67e42d0190ca261
   "metadata": {},
   "outputs": [],
   "source": [
    "x = CarData('data/')"
   ]
  },
  {
   "cell_type": "code",
<<<<<<< HEAD
   "execution_count": 87,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "                   sum_sq       df           F         PR(>F)\n",
      "engine_type  1.155656e+12      4.0  297.763602  2.668046e-252\n",
      "Residual     3.399278e+13  35034.0         NaN            NaN\n",
      "\n",
      "          Multiple Comparison of Means - Tukey HSD, FWER=0.05           \n",
      "========================================================================\n",
      "   group1      group2     meandiff  p-adj    lower       upper    reject\n",
      "------------------------------------------------------------------------\n",
      "    Benzyna Benzyna+CNG -11682.8896 0.767  -40014.661  16648.8818  False\n",
      "    Benzyna Benzyna+LPG -15139.2284 0.001 -17051.6983 -13226.7585   True\n",
      "    Benzyna      Diesel   2121.9383 0.001   1178.2621   3065.6146   True\n",
      "    Benzyna     Hybryda  40483.7631 0.001  35893.6398  45073.8864   True\n",
      "Benzyna+CNG Benzyna+LPG  -3456.3388   0.9 -31837.0873  24924.4097  False\n",
      "Benzyna+CNG      Diesel  13804.8279 0.651 -14527.1305  42136.7864  False\n",
      "Benzyna+CNG     Hybryda  52166.6527 0.001  23480.7925  80852.5129   True\n",
      "Benzyna+LPG      Diesel  17261.1668 0.001  15345.9282  19176.4053   True\n",
      "Benzyna+LPG     Hybryda  55622.9915 0.001  50739.6683  60506.3147   True\n",
      "     Diesel     Hybryda  38361.8248 0.001  33770.5473  42953.1023   True\n",
      "------------------------------------------------------------------------\n"
=======
   "execution_count": 37,
   "metadata": {},
   "outputs": [
    {
     "ename": "KeyError",
     "evalue": "\"The following 'value_vars' are not present in the DataFrame: ['Benzyna', 'Benzyna+CNG', 'Benzyna+LPG', 'Diesel', 'Hybryda']\"",
     "output_type": "error",
     "traceback": [
      "\u001b[1;31m---------------------------------------------------------------------------\u001b[0m",
      "\u001b[1;31mKeyError\u001b[0m                                  Traceback (most recent call last)",
      "\u001b[1;32m<ipython-input-37-c237f808d7ae>\u001b[0m in \u001b[0;36m<module>\u001b[1;34m\u001b[0m\n\u001b[1;32m----> 1\u001b[1;33m \u001b[0mx\u001b[0m\u001b[1;33m.\u001b[0m\u001b[0manova\u001b[0m\u001b[1;33m(\u001b[0m\u001b[1;33m)\u001b[0m\u001b[1;33m\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n\u001b[0m",
      "\u001b[1;32m<ipython-input-35-b405fa77f3ad>\u001b[0m in \u001b[0;36manova\u001b[1;34m(self, var)\u001b[0m\n\u001b[0;32m    212\u001b[0m         \u001b[1;32mif\u001b[0m \u001b[0mvar\u001b[0m \u001b[1;33m==\u001b[0m \u001b[1;34m'all'\u001b[0m\u001b[1;33m:\u001b[0m\u001b[1;33m\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n\u001b[0;32m    213\u001b[0m             \u001b[1;32mfor\u001b[0m \u001b[0mvariable\u001b[0m \u001b[1;32min\u001b[0m \u001b[0menumerate\u001b[0m\u001b[1;33m(\u001b[0m\u001b[0mself\u001b[0m\u001b[1;33m.\u001b[0m\u001b[0mcategorical_variables\u001b[0m\u001b[1;33m)\u001b[0m\u001b[1;33m:\u001b[0m\u001b[1;33m\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n\u001b[1;32m--> 214\u001b[1;33m                 \u001b[0mdf_melt\u001b[0m \u001b[1;33m=\u001b[0m \u001b[0mpd\u001b[0m\u001b[1;33m.\u001b[0m\u001b[0mmelt\u001b[0m\u001b[1;33m(\u001b[0m\u001b[0mself\u001b[0m\u001b[1;33m.\u001b[0m\u001b[0mdata\u001b[0m\u001b[1;33m[\u001b[0m\u001b[0mvariable\u001b[0m\u001b[1;33m[\u001b[0m\u001b[1;36m1\u001b[0m\u001b[1;33m]\u001b[0m\u001b[1;33m]\u001b[0m\u001b[1;33m.\u001b[0m\u001b[0mreset_index\u001b[0m\u001b[1;33m(\u001b[0m\u001b[1;33m)\u001b[0m\u001b[1;33m,\u001b[0m \u001b[0mid_vars\u001b[0m\u001b[1;33m=\u001b[0m\u001b[1;33m[\u001b[0m\u001b[1;34m'index'\u001b[0m\u001b[1;33m]\u001b[0m\u001b[1;33m,\u001b[0m \u001b[0mvalue_vars\u001b[0m\u001b[1;33m=\u001b[0m\u001b[1;33m[\u001b[0m\u001b[0mself\u001b[0m\u001b[1;33m.\u001b[0m\u001b[0mdata\u001b[0m\u001b[1;33m[\u001b[0m\u001b[0mvariable\u001b[0m\u001b[1;33m[\u001b[0m\u001b[1;36m1\u001b[0m\u001b[1;33m]\u001b[0m\u001b[1;33m]\u001b[0m\u001b[1;33m.\u001b[0m\u001b[0munique\u001b[0m\u001b[1;33m(\u001b[0m\u001b[1;33m)\u001b[0m\u001b[1;33m]\u001b[0m\u001b[1;33m)\u001b[0m\u001b[1;33m\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n\u001b[0m\u001b[0;32m    215\u001b[0m                 \u001b[0mdf_melt\u001b[0m\u001b[1;33m.\u001b[0m\u001b[0mcolumns\u001b[0m \u001b[1;33m=\u001b[0m \u001b[1;33m[\u001b[0m\u001b[1;34m'index'\u001b[0m\u001b[1;33m,\u001b[0m \u001b[0mvariable\u001b[0m\u001b[1;33m[\u001b[0m\u001b[1;36m1\u001b[0m\u001b[1;33m]\u001b[0m \u001b[1;33m,\u001b[0m \u001b[1;34m'value'\u001b[0m\u001b[1;33m]\u001b[0m\u001b[1;33m\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n\u001b[0;32m    216\u001b[0m                 \u001b[0mequation_string\u001b[0m \u001b[1;33m=\u001b[0m \u001b[1;34m'value ~ C('\u001b[0m\u001b[1;33m+\u001b[0m\u001b[0mstr\u001b[0m\u001b[1;33m(\u001b[0m\u001b[0mvariable\u001b[0m\u001b[1;33m[\u001b[0m\u001b[1;36m1\u001b[0m\u001b[1;33m]\u001b[0m\u001b[1;33m+\u001b[0m\u001b[1;34m')'\u001b[0m\u001b[1;33m)\u001b[0m\u001b[1;33m\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n",
      "\u001b[1;32mC:\\Anaconda\\lib\\site-packages\\pandas\\core\\reshape\\melt.py\u001b[0m in \u001b[0;36mmelt\u001b[1;34m(frame, id_vars, value_vars, var_name, value_name, col_level)\u001b[0m\n\u001b[0;32m     72\u001b[0m                     \u001b[1;34m\"The following 'value_vars' are not present in\"\u001b[0m\u001b[1;33m\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n\u001b[0;32m     73\u001b[0m                     \u001b[1;34m\" the DataFrame: {missing}\"\u001b[0m\u001b[1;33m\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n\u001b[1;32m---> 74\u001b[1;33m                     \u001b[1;34m\"\"\u001b[0m\u001b[1;33m.\u001b[0m\u001b[0mformat\u001b[0m\u001b[1;33m(\u001b[0m\u001b[0mmissing\u001b[0m\u001b[1;33m=\u001b[0m\u001b[0mlist\u001b[0m\u001b[1;33m(\u001b[0m\u001b[0mmissing\u001b[0m\u001b[1;33m)\u001b[0m\u001b[1;33m)\u001b[0m\u001b[1;33m\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n\u001b[0m\u001b[0;32m     75\u001b[0m                 )\n\u001b[0;32m     76\u001b[0m         \u001b[0mframe\u001b[0m \u001b[1;33m=\u001b[0m \u001b[0mframe\u001b[0m\u001b[1;33m.\u001b[0m\u001b[0mloc\u001b[0m\u001b[1;33m[\u001b[0m\u001b[1;33m:\u001b[0m\u001b[1;33m,\u001b[0m \u001b[0mid_vars\u001b[0m \u001b[1;33m+\u001b[0m \u001b[0mvalue_vars\u001b[0m\u001b[1;33m]\u001b[0m\u001b[1;33m\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n",
      "\u001b[1;31mKeyError\u001b[0m: \"The following 'value_vars' are not present in the DataFrame: ['Benzyna', 'Benzyna+CNG', 'Benzyna+LPG', 'Diesel', 'Hybryda']\""
>>>>>>> c5276f11f63080e0701ee240d67e42d0190ca261
     ]
    }
   ],
   "source": [
<<<<<<< HEAD
    "x.categorical_variables = ['engine_type']\n",
=======
>>>>>>> c5276f11f63080e0701ee240d67e42d0190ca261
    "x.anova()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 6,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>count</th>\n",
       "      <th>mean</th>\n",
       "      <th>sm</th>\n",
       "      <th>t</th>\n",
       "      <th>df</th>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>engine_type</th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <td>Benzyna</td>\n",
       "      <td>16411</td>\n",
       "      <td>32548.33</td>\n",
       "      <td>254.074563</td>\n",
       "      <td>-1.583366</td>\n",
       "      <td>16410</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <td>Benzyna+CNG</td>\n",
       "      <td>9</td>\n",
       "      <td>20865.44</td>\n",
       "      <td>6955.146667</td>\n",
       "      <td>-1.737589</td>\n",
       "      <td>8</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <td>Benzyna+LPG</td>\n",
       "      <td>2244</td>\n",
       "      <td>17409.11</td>\n",
       "      <td>367.506600</td>\n",
       "      <td>-42.289072</td>\n",
       "      <td>2243</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <td>Diesel</td>\n",
       "      <td>16025</td>\n",
       "      <td>34670.27</td>\n",
       "      <td>273.878667</td>\n",
       "      <td>6.278864</td>\n",
       "      <td>16024</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <td>Hybryda</td>\n",
       "      <td>350</td>\n",
       "      <td>73032.10</td>\n",
       "      <td>3903.729949</td>\n",
       "      <td>10.267482</td>\n",
       "      <td>349</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "             count      mean           sm          t     df\n",
       "engine_type                                                \n",
       "Benzyna      16411  32548.33   254.074563  -1.583366  16410\n",
       "Benzyna+CNG      9  20865.44  6955.146667  -1.737589      8\n",
       "Benzyna+LPG   2244  17409.11   367.506600 -42.289072   2243\n",
       "Diesel       16025  34670.27   273.878667   6.278864  16024\n",
       "Hybryda        350  73032.10  3903.729949  10.267482    349"
      ]
     },
     "execution_count": 6,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "x.ind_test('engine_type')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "code",
   "execution_count": 18,
   "metadata": {},
   "outputs": [],
   "source": [
    "x = x.add_dummies(['ford', 'mercedes', 'suzuki'], ['title', 'sub_title'])"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 19,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>ID</th>\n",
       "      <th>title</th>\n",
       "      <th>price</th>\n",
       "      <th>sub_title</th>\n",
       "      <th>mileage_km</th>\n",
       "      <th>year</th>\n",
       "      <th>engine_cm3</th>\n",
       "      <th>engine_type</th>\n",
       "      <th>city</th>\n",
       "      <th>province</th>\n",
       "      <th>negotiable</th>\n",
       "      <th>title_ford</th>\n",
       "      <th>title_mercedes</th>\n",
       "      <th>title_suzuki</th>\n",
       "      <th>sub_title_ford</th>\n",
       "      <th>sub_title_mercedes</th>\n",
       "      <th>sub_title_suzuki</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <td>0</td>\n",
       "      <td>1</td>\n",
       "      <td>smart fortwo cabrio</td>\n",
       "      <td>18800</td>\n",
       "      <td>cabrio automat nawigacja kamera led szwajcaria</td>\n",
       "      <td>37000.0</td>\n",
       "      <td>2016</td>\n",
       "      <td>999</td>\n",
       "      <td>Benzyna</td>\n",
       "      <td>Bielsko-Biała</td>\n",
       "      <td>Śląskie</td>\n",
       "      <td>Do negocjacji, Faktura VAT</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <td>1</td>\n",
       "      <td>2</td>\n",
       "      <td>ford kuga</td>\n",
       "      <td>34700</td>\n",
       "      <td>nawigacja/kamera cofania/zadbany/</td>\n",
       "      <td>198000.0</td>\n",
       "      <td>2011</td>\n",
       "      <td>2000</td>\n",
       "      <td>Diesel</td>\n",
       "      <td>Dąbrowa</td>\n",
       "      <td>Pomorskie</td>\n",
       "      <td>Do negocjacji</td>\n",
       "      <td>1</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <td>2</td>\n",
       "      <td>3</td>\n",
       "      <td>chevrolet captiva 2.4</td>\n",
       "      <td>27900</td>\n",
       "      <td>2.4 136km klima alus navi dvd skóra 4x4 poleca...</td>\n",
       "      <td>246000.0</td>\n",
       "      <td>2009</td>\n",
       "      <td>2405</td>\n",
       "      <td>Benzyna</td>\n",
       "      <td>Płock</td>\n",
       "      <td>Mazowieckie</td>\n",
       "      <td>Do negocjacji</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <td>3</td>\n",
       "      <td>4</td>\n",
       "      <td>renault clio 1.2</td>\n",
       "      <td>37900</td>\n",
       "      <td>1.2 benz 73 km !! idealny !! full !! warszawa</td>\n",
       "      <td>15000.0</td>\n",
       "      <td>2017</td>\n",
       "      <td>1149</td>\n",
       "      <td>Benzyna</td>\n",
       "      <td>Warszawa</td>\n",
       "      <td>Mazowieckie</td>\n",
       "      <td>Do negocjacji</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <td>4</td>\n",
       "      <td>5</td>\n",
       "      <td>opel meriva</td>\n",
       "      <td>7900</td>\n",
       "      <td>mały przebieg 109 tyś/km bezwypadkowy</td>\n",
       "      <td>109000.0</td>\n",
       "      <td>2006</td>\n",
       "      <td>1400</td>\n",
       "      <td>Benzyna</td>\n",
       "      <td>Żyrardów</td>\n",
       "      <td>Mazowieckie</td>\n",
       "      <td>NaN</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "   ID                  title  price  \\\n",
       "0   1    smart fortwo cabrio  18800   \n",
       "1   2              ford kuga  34700   \n",
       "2   3  chevrolet captiva 2.4  27900   \n",
       "3   4       renault clio 1.2  37900   \n",
       "4   5            opel meriva   7900   \n",
       "\n",
       "                                           sub_title  mileage_km  year  \\\n",
       "0     cabrio automat nawigacja kamera led szwajcaria     37000.0  2016   \n",
       "1                  nawigacja/kamera cofania/zadbany/    198000.0  2011   \n",
       "2  2.4 136km klima alus navi dvd skóra 4x4 poleca...    246000.0  2009   \n",
       "3      1.2 benz 73 km !! idealny !! full !! warszawa     15000.0  2017   \n",
       "4              mały przebieg 109 tyś/km bezwypadkowy    109000.0  2006   \n",
       "\n",
       "   engine_cm3 engine_type           city     province  \\\n",
       "0         999     Benzyna  Bielsko-Biała      Śląskie   \n",
       "1        2000      Diesel        Dąbrowa    Pomorskie   \n",
       "2        2405     Benzyna          Płock  Mazowieckie   \n",
       "3        1149     Benzyna       Warszawa  Mazowieckie   \n",
       "4        1400     Benzyna       Żyrardów  Mazowieckie   \n",
       "\n",
       "                   negotiable  title_ford  title_mercedes  title_suzuki  \\\n",
       "0  Do negocjacji, Faktura VAT           0               0             0   \n",
       "1               Do negocjacji           1               0             0   \n",
       "2               Do negocjacji           0               0             0   \n",
       "3               Do negocjacji           0               0             0   \n",
       "4                         NaN           0               0             0   \n",
       "\n",
       "   sub_title_ford  sub_title_mercedes  sub_title_suzuki  \n",
       "0               0                   0                 0  \n",
       "1               0                   0                 0  \n",
       "2               0                   0                 0  \n",
       "3               0                   0                 0  \n",
       "4               0                   0                 0  "
      ]
     },
     "execution_count": 19,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "x.data.head()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 48,
   "metadata": {
    "scrolled": true
   },
   "outputs": [],
   "source": [
    "data = x.data"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 58,
   "metadata": {},
   "outputs": [],
   "source": [
    "data2 = AddDummy(data, 'title', 'ford')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 66,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "786919459"
      ]
     },
     "execution_count": 66,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "data['price'].sum()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 3,
   "metadata": {},
   "outputs": [],
   "source": [
    "x = CarData('data/')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 4,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Offers with price greater than 200000 have been discarded\n",
      "The number of such offers = 294\n",
      "\n",
      "Offers with mileage greater than 400000 have been discarded\n",
      "The number of such offers = 166\n",
      "\n",
      "Offers with year lower than 1995 have been discarded\n",
      "The number of such offers = 288\n",
      "\n",
      "Offers with engine_cm3 greater than 750 and lower than 4000 have been discarded\n",
      "The number of such offers = 378\n",
      "\n",
      "Total number of discarded offers = 1126(4.68%) - may be different to the sum of above due to overlap\n"
     ]
    }
   ],
   "source": [
    "x.outliers()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 5,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>ID</th>\n",
       "      <th>title</th>\n",
       "      <th>price</th>\n",
       "      <th>sub_title</th>\n",
       "      <th>mileage_km</th>\n",
       "      <th>year</th>\n",
       "      <th>engine_cm3</th>\n",
       "      <th>engine_type</th>\n",
       "      <th>city</th>\n",
       "      <th>province</th>\n",
       "      <th>negotiable</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <td>0</td>\n",
       "      <td>1</td>\n",
       "      <td>Smart Fortwo cabrio</td>\n",
       "      <td>18800</td>\n",
       "      <td>Cabrio Automat Nawigacja Kamera Led Szwajcaria</td>\n",
       "      <td>37000.0</td>\n",
       "      <td>2016</td>\n",
       "      <td>999</td>\n",
       "      <td>Benzyna</td>\n",
       "      <td>Bielsko-Biała</td>\n",
       "      <td>Śląskie</td>\n",
       "      <td>Do negocjacji, Faktura VAT</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <td>1</td>\n",
       "      <td>2</td>\n",
       "      <td>Ford Kuga</td>\n",
       "      <td>34700</td>\n",
       "      <td>Nawigacja/kamera cofania/Zadbany/</td>\n",
       "      <td>198000.0</td>\n",
       "      <td>2011</td>\n",
       "      <td>2000</td>\n",
       "      <td>Diesel</td>\n",
       "      <td>Dąbrowa</td>\n",
       "      <td>Pomorskie</td>\n",
       "      <td>Do negocjacji</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <td>2</td>\n",
       "      <td>3</td>\n",
       "      <td>Chevrolet Captiva 2.4</td>\n",
       "      <td>27900</td>\n",
       "      <td>2.4 136km klima alus navi DVD skóra 4x4 Poleca...</td>\n",
       "      <td>246000.0</td>\n",
       "      <td>2009</td>\n",
       "      <td>2405</td>\n",
       "      <td>Benzyna</td>\n",
       "      <td>Płock</td>\n",
       "      <td>Mazowieckie</td>\n",
       "      <td>Do negocjacji</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <td>3</td>\n",
       "      <td>4</td>\n",
       "      <td>Renault Clio 1.2</td>\n",
       "      <td>37900</td>\n",
       "      <td>1.2 BENZ 73 KM !! IDEALNY !! FULL !! Warszawa</td>\n",
       "      <td>15000.0</td>\n",
       "      <td>2017</td>\n",
       "      <td>1149</td>\n",
       "      <td>Benzyna</td>\n",
       "      <td>Warszawa</td>\n",
       "      <td>Mazowieckie</td>\n",
       "      <td>Do negocjacji</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <td>4</td>\n",
       "      <td>5</td>\n",
       "      <td>Opel Meriva</td>\n",
       "      <td>7900</td>\n",
       "      <td>mały przebieg 109 tyś/km bezwypadkowy</td>\n",
       "      <td>109000.0</td>\n",
       "      <td>2006</td>\n",
       "      <td>1400</td>\n",
       "      <td>Benzyna</td>\n",
       "      <td>Żyrardów</td>\n",
       "      <td>Mazowieckie</td>\n",
       "      <td>NaN</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "   ID                  title  price  \\\n",
       "0   1    Smart Fortwo cabrio  18800   \n",
       "1   2              Ford Kuga  34700   \n",
       "2   3  Chevrolet Captiva 2.4  27900   \n",
       "3   4       Renault Clio 1.2  37900   \n",
       "4   5            Opel Meriva   7900   \n",
       "\n",
       "                                           sub_title  mileage_km  year  \\\n",
       "0     Cabrio Automat Nawigacja Kamera Led Szwajcaria     37000.0  2016   \n",
       "1                  Nawigacja/kamera cofania/Zadbany/    198000.0  2011   \n",
       "2  2.4 136km klima alus navi DVD skóra 4x4 Poleca...    246000.0  2009   \n",
       "3      1.2 BENZ 73 KM !! IDEALNY !! FULL !! Warszawa     15000.0  2017   \n",
       "4              mały przebieg 109 tyś/km bezwypadkowy    109000.0  2006   \n",
       "\n",
       "   engine_cm3 engine_type           city     province  \\\n",
       "0         999     Benzyna  Bielsko-Biała      Śląskie   \n",
       "1        2000      Diesel        Dąbrowa    Pomorskie   \n",
       "2        2405     Benzyna          Płock  Mazowieckie   \n",
       "3        1149     Benzyna       Warszawa  Mazowieckie   \n",
       "4        1400     Benzyna       Żyrardów  Mazowieckie   \n",
       "\n",
       "                   negotiable  \n",
       "0  Do negocjacji, Faktura VAT  \n",
       "1               Do negocjacji  \n",
       "2               Do negocjacji  \n",
       "3               Do negocjacji  \n",
       "4                         NaN  "
      ]
     },
     "execution_count": 5,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "x.data.head()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 20,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "(737, 8782)"
      ]
     },
     "execution_count": 20,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "x.missing, x.duplicates"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 64,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>price</th>\n",
       "      <th>mileage_km</th>\n",
       "      <th>year</th>\n",
       "      <th>engine_cm3</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <td>count</td>\n",
       "      <td>24073.00</td>\n",
       "      <td>24073.00</td>\n",
       "      <td>24073.00</td>\n",
       "      <td>24073.00</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <td>mean</td>\n",
       "      <td>32688.88</td>\n",
       "      <td>159511.92</td>\n",
       "      <td>2009.88</td>\n",
       "      <td>1789.87</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <td>std</td>\n",
       "      <td>31457.86</td>\n",
       "      <td>83724.08</td>\n",
       "      <td>5.33</td>\n",
       "      <td>483.63</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <td>min</td>\n",
       "      <td>250.00</td>\n",
       "      <td>0.00</td>\n",
       "      <td>1995.00</td>\n",
       "      <td>796.00</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <td>25%</td>\n",
       "      <td>12200.00</td>\n",
       "      <td>99000.00</td>\n",
       "      <td>2006.00</td>\n",
       "      <td>1461.00</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <td>50%</td>\n",
       "      <td>22900.00</td>\n",
       "      <td>163000.00</td>\n",
       "      <td>2010.00</td>\n",
       "      <td>1794.00</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <td>75%</td>\n",
       "      <td>41800.00</td>\n",
       "      <td>217000.00</td>\n",
       "      <td>2014.00</td>\n",
       "      <td>1997.00</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <td>max</td>\n",
       "      <td>200000.00</td>\n",
       "      <td>400000.00</td>\n",
       "      <td>2020.00</td>\n",
       "      <td>4000.00</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "           price  mileage_km      year  engine_cm3\n",
       "count   24073.00    24073.00  24073.00    24073.00\n",
       "mean    32688.88   159511.92   2009.88     1789.87\n",
       "std     31457.86    83724.08      5.33      483.63\n",
       "min       250.00        0.00   1995.00      796.00\n",
       "25%     12200.00    99000.00   2006.00     1461.00\n",
       "50%     22900.00   163000.00   2010.00     1794.00\n",
       "75%     41800.00   217000.00   2014.00     1997.00\n",
       "max    200000.00   400000.00   2020.00     4000.00"
      ]
     },
     "execution_count": 64,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "x.describe()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "x.scatter_nox()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "plt.hist(x.data['mileage_km'], bins = 100)\n",
    "plt.title('abc')\n",
    "plt.show()\n",
    "\n",
    "plt.hist(x.data['price'],bins = 100)\n",
    "plt.show()\n",
    "\n",
    "plt.hist(x.data['year'], bins = 100)\n",
    "plt.show()\n",
    "\n",
    "plt.hist(x.data['engine_cm3'], bins = 100)\n",
    "plt.show()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "x.data[x.data['price']>1100000]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "x.data[x.data['mileage_km']>1500000]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "scrolled": true
   },
   "outputs": [],
   "source": [
    "x.data[x.data['year']<1990]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "round(pd.DataFrame(\n",
    "    data = x.data[['price', 'mileage_km', 'year', 'engine_cm3']].describe(),\n",
    "    columns = x.data[['price', 'mileage_km', 'year', 'engine_cm3']].columns),2)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "x.data.isnull().sum()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "data = pd.read_csv('run_results_21-04-2020.csv')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "data = ReadAndPreprocess(df)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "df = pd.read_csv('data/run_results_21-04-2020.csv')\n",
    "df.head()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "df2 = pd.read_csv('data/run_results_25-04_2020.csv')\n",
    "df2.head()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "df3 = pd.read_csv('data/run_results_26-04-2020.csv')\n",
    "df3.head()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "x.df_head()"
   ]
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 3",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.7.4"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 2
}