Transforming Validation Data to Match Training Data.ipynb

{
 "cells": [
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# Tranforming Validation Data to Match Training Data"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### Loading Data and Dependencies"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 1,
   "metadata": {},
   "outputs": [],
   "source": [
    "import pandas as pd\n",
    "import numpy as np\n",
    "\n",
    "#Final Train\n",
    "train_fnl = pd.read_csv('Processed_DataSets/Final_Train.csv',encoding='ISO-8859-1',low_memory=False,index_col=0)\n",
    "train_fnl.index = train_fnl['ticket_id']\n",
    "train_fnl.drop(['ticket_id'],axis=1,inplace=True)\n",
    "\n",
    "#Validation\n",
    "val = pd.read_csv('Orignal_DataSet/validation.csv',encoding='ISO-8859-1',low_memory=False)\n",
    "\n",
    "#Init. Train\n",
    "\n",
    "train = pd.read_csv('Orignal_DataSet/train.csv',encoding='ISO-8859-1',low_memory=False)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## Tranformation Funtions"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 2,
   "metadata": {},
   "outputs": [],
   "source": [
    "def get_violator_feature(df):\n",
    "    df['len_violator'] = df['violator_name'].apply(lambda x : len(str(x)) if 'Inc' not in str(x) and 'L.L.C' not in str(x) \n",
    "                                                 and 'INC' not in str(x) and 'LLC' not in str(x) else 0)\n",
    "    df['violator_type'] = df['len_violator'].apply(lambda x : 'Agency' if x==0 or x >20 else 'Person' )\n",
    "    return df\n",
    "\n",
    "def get_discription_feature(df):\n",
    "    #Lenght of violation    \n",
    "    df['len_description'] = df['violation_description'].str.len()\n",
    "    #Number of violation\n",
    "    df['count_violation'] = df['violation_description'].apply(lambda x : len(x.split(',')))\n",
    "    return df\n",
    "\n",
    "def get_disposition_features(df):\n",
    "    # Reason of Guilt\n",
    "    df['responsible_by'] = df['disposition'].apply(lambda x : 0 if 'Not responsible' in x else -1 if 'PENDING' in x else x.split()[-1])\n",
    "    #Correcting Data Errors\n",
    "    df.loc[df['responsible_by'] == 'Deter','responsible_by'] = 'Determination'\n",
    "    df.loc[df['responsible_by'] == 'Determi','responsible_by'] = 'Determination'\n",
    "    df.loc[df['responsible_by'] == 'Admis','responsible_by'] = 'Admission'\n",
    "    # If Fine was Waived or not\n",
    "    df['fine_waived'] = df['disposition'].apply(lambda x : 1 if 'Fine Waived' in x else 0)\n",
    "    return df\n",
    "\n",
    "def get_date_time_feature(df):\n",
    "    #time of ticket_issued\n",
    "    df['ticket_time'] = df['ticket_issued_date'].apply(lambda x : int(''.join(x[11:].split(':')[0:2])))\n",
    "    #Month_bin\n",
    "    df['month_bin'] = df['ticket_issued_date'].apply(lambda x : 1 if 1 <= int(x[5:7]) <= 3 else 2 if 4 <= int(x[5:7]) <= 6 \n",
    "    else 3 if 7 <= int(x[5:7]) <= 9 else 4)\n",
    "    return df"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 3,
   "metadata": {},
   "outputs": [],
   "source": [
    "#Transforming\n",
    "\n",
    "val = get_date_time_feature(val)\n",
    "val = get_discription_feature(val)\n",
    "val  = get_disposition_features(val)\n",
    "val = get_violator_feature(val);\n",
    "val = val[['compliance','ticket_id','count_violation','ticket_time','month_bin','fine_amount','late_fee','discount_amount','judgment_amount',\n",
    "          'len_violator','len_description','agency_name','country','violator_type','responsible_by','fine_waived']]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 4,
   "metadata": {},
   "outputs": [],
   "source": [
    "#taking useful Columns from init. Train data\n",
    "\n",
    "train = get_date_time_feature(train)\n",
    "train = get_discription_feature(train)\n",
    "train  = get_disposition_features(train)\n",
    "train = get_violator_feature(train);\n",
    "\n",
    "#features to be Normalze\n",
    "train = train[['fine_amount','late_fee','discount_amount','judgment_amount','len_violator','len_description']] \n"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## Normalization and Feature Selection"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 5,
   "metadata": {},
   "outputs": [],
   "source": [
    "from sklearn.preprocessing import MinMaxScaler\n",
    "\n",
    "scale = MinMaxScaler(feature_range=(0,5)).fit(train) # Using Training MinMax Scale to avoid Data Leakage\n",
    "\n",
    "dummy = scale.transform(val[['fine_amount','late_fee','discount_amount','judgment_amount',\n",
    "                             'len_violator','len_description']])"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 6,
   "metadata": {},
   "outputs": [],
   "source": [
    "dummy = pd.DataFrame(data=dummy)\n",
    "dummy.columns = ['fine_amount_N','late_fee_N','discount_amount_N','judgment_amount_N',\n",
    "                             'len_violator_N','len_description_N']\n",
    "val.drop(['fine_amount','late_fee','discount_amount','judgment_amount',\n",
    "                             'len_violator','len_description'],axis=1,inplace=True)\n",
    "val = pd.concat([val,dummy],axis=1)\n",
    "val.set_index('ticket_id',inplace=True)\n",
    "\n",
    "#handling inf and -inf values\n",
    "for cl in ['count_violation','ticket_time','month_bin','fine_amount_N','late_fee_N','discount_amount_N',\n",
    "          'judgment_amount_N','len_violator_N','len_description_N']:\n",
    "    val[cl] = val[cl].astype(np.float32)\n",
    "\n",
    "val.replace([np.inf, -np.inf], np.nan)\n",
    "val.dropna(inplace=True)\n",
    "\n",
    "#Getting dummies Variable\n",
    "\n",
    "val['fine_waived'] = val['fine_waived'].astype(object)\n",
    "val = pd.get_dummies(val)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## Saving DataSet for Validation of Selected Model"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 7,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "compliance                                                      int64\n",
       "count_violation                                               float32\n",
       "ticket_time                                                   float32\n",
       "month_bin                                                     float32\n",
       "fine_amount_N                                                 float32\n",
       "late_fee_N                                                    float32\n",
       "discount_amount_N                                             float32\n",
       "judgment_amount_N                                             float32\n",
       "len_violator_N                                                float32\n",
       "len_description_N                                             float32\n",
       "agency_name_Buildings, Safety Engineering & Env Department      uint8\n",
       "agency_name_Department of Public Works                          uint8\n",
       "agency_name_Detroit Police Department                           uint8\n",
       "country_USA                                                     uint8\n",
       "violator_type_Agency                                            uint8\n",
       "violator_type_Person                                            uint8\n",
       "responsible_by_Admission                                        uint8\n",
       "responsible_by_Default                                          uint8\n",
       "responsible_by_Determination                                    uint8\n",
       "responsible_by_Dismissal                                        uint8\n",
       "fine_waived_0                                                   uint8\n",
       "fine_waived_1                                                   uint8\n",
       "dtype: object"
      ]
     },
     "execution_count": 7,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "val.dtypes"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 8,
   "metadata": {},
   "outputs": [],
   "source": [
    "val.to_csv('Processed_DataSets/Final_Validation_DataSet.csv')"
   ]
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 3",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.7.9"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 4
}