trip_data_analysis.ipynb

{
  "nbformat": 4,
  "nbformat_minor": 0,
  "metadata": {
    "colab": {
      "provenance": [],
      "collapsed_sections": [],
      "include_colab_link": true
    },
    "kernelspec": {
      "name": "python3",
      "display_name": "Python 3"
    },
    "language_info": {
      "name": "python"
    }
  },
  "cells": [
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "view-in-github",
        "colab_type": "text"
      },
      "source": [
        "<a href=\"https://colab.research.google.com/github/AylinNaebzadeh/NYC-Yellow-Taxi-Trip-Data-Analysis/blob/main/trip_data_analysis.ipynb\" target=\"_parent\"><img src=\"https://colab.research.google.com/assets/colab-badge.svg\" alt=\"Open In Colab\"/></a>"
      ]
    },
    {
      "cell_type": "code",
      "source": [
        "#  pip install -U -q PyDrive --> for importing google drive in colab"
      ],
      "metadata": {
        "id": "m-FkoPX0Tr1N"
      },
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "markdown",
      "source": [
        "# New Section"
      ],
      "metadata": {
        "id": "kV4DmGsX2zK-"
      }
    },
    {
      "cell_type": "code",
      "execution_count": null,
      "metadata": {
        "id": "vlzI8I4a3zkV",
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "outputId": "794cb823-82f4-465d-9c57-7981d054becc"
      },
      "outputs": [
        {
          "output_type": "stream",
          "name": "stdout",
          "text": [
            "1.0.2\n"
          ]
        }
      ],
      "source": [
        "from pydrive.auth import GoogleAuth\n",
        "from pydrive.drive import GoogleDrive\n",
        "from google.colab import files\n",
        "import numpy as np\n",
        "from google.colab import auth\n",
        "from oauth2client.client import GoogleCredentials\n",
        "import pandas as pd\n",
        "import matplotlib.pyplot as plt\n",
        "import seaborn as sns\n",
        "import datetime\n",
        "import sklearn\n",
        "from sklearn.preprocessing import LabelEncoder\n",
        "from scipy.stats import zscore\n",
        "from sklearn.cluster import KMeans\n",
        "from sklearn.model_selection import train_test_split\n",
        "from sklearn.preprocessing import StandardScaler\n",
        "from sklearn.tree import DecisionTreeClassifier, export_graphviz, plot_tree\n",
        "from sklearn.metrics import accuracy_score, confusion_matrix, classification_report\n",
        "from IPython.display import Image  \n",
        "from sklearn import tree\n",
        "import gdown\n",
        "from graphviz import Source \n",
        "import graphviz\n",
        "print(sklearn.__version__)"
      ]
    },
    {
      "cell_type": "code",
      "source": [
        "def delete_columns(df, del_columns):\n",
        "  for col in del_columns:\n",
        "    del df[col]\n",
        "  return df"
      ],
      "metadata": {
        "id": "Ex3t7zp1fLPm"
      },
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "code",
      "source": [
        "def download_csv_file():\n",
        "  auth.authenticate_user()\n",
        "  gauth = GoogleAuth()\n",
        "  gauth.credentials = GoogleCredentials.get_application_default()\n",
        "  drive = GoogleDrive(gauth)\n",
        "  link = 'https://drive.google.com/open?id=1Gepus8IDhEiXBzk3U0aQGlwgw99SRdLx'\n",
        "  fluff, id = link.split('=')\n",
        "  downloaded = drive.CreateFile({'id':id}) \n",
        "  downloaded.GetContentFile('sample-nyc-data.csv')  \n"
      ],
      "metadata": {
        "id": "Xd7fQCkTJcPI"
      },
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "code",
      "source": [
        "def create_histogram(df):\n",
        "  dataframe = df\n",
        "  del_columns = [\"tpep_pickup_datetime\", \"tpep_dropoff_datetime\",\n",
        "               \"pickup_longitude\", \"pickup_latitude\",\n",
        "               \"RatecodeID\", \"store_and_fwd_flag\",\n",
        "               \"dropoff_longitude\", \"dropoff_latitude\",\n",
        "               \"improvement_surcharge\", \"tolls_amount\",\n",
        "               \"tip_amount\", \"mta_tax\", \"extra\", \"pickup_date\", \"dropoff_date\", \"drop_month\", \"pick_month\"]\n",
        "  for col in dataframe.columns:\n",
        "    plt.hist(dataframe[col], edgecolor='white')\n",
        "    plt.xlabel(col)\n",
        "    plt.yscale(\"log\")\n",
        "    plt.show()"
      ],
      "metadata": {
        "id": "TZesLtwPjZX6"
      },
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "code",
      "source": [
        "def calculate_min_max_var_avg(data):\n",
        "    stats = pd.DataFrame()\n",
        "    stats[\"min\"] = data.min()\n",
        "    stats[\"var\"] = data.var()\n",
        "    stats[\"avg\"] = data.mean()\n",
        "    stats[\"max\"] = data.max()\n",
        "    return stats"
      ],
      "metadata": {
        "id": "6tn6PGB2e6Z3"
      },
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "code",
      "source": [
        "def convert_df_to_csv(df):\n",
        "  df.to_csv('extended_data.csv')"
      ],
      "metadata": {
        "id": "9S7lIOfDE3lw"
      },
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "code",
      "source": [
        "def create_boxplot(data):\n",
        "  \"\"\"\n",
        "  find an outlier using IQR method and box plot in 1-dimensional data.\n",
        "  \"\"\"\n",
        "  new_data=data[['VendorID', 'passenger_count', 'trip_distance', 'payment_type',\n",
        "                 'fare_amount', 'trip_duration_min', 'speed', 'total_amount', 'tip_amount']]\n",
        "  for column in new_data:\n",
        "    plt.figure()\n",
        "    data.boxplot([column])"
      ],
      "metadata": {
        "id": "gpr0u0P0cQyA"
      },
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "code",
      "source": [
        "def normalize_min_max(df):\n",
        "  cols = []\n",
        "  del_columns = [\n",
        "                \"tpep_pickup_datetime\", \"tpep_dropoff_datetime\",\n",
        "                \"pickup_longitude\", \"pickup_latitude\",\n",
        "                \"RatecodeID\", \"store_and_fwd_flag\",\n",
        "                \"dropoff_longitude\", \"dropoff_latitude\",\n",
        "                \"improvement_surcharge\",\n",
        "                \"mta_tax\",  \"pickup_date\",\n",
        "                \"dropoff_date\", \"drop_month\",\n",
        "                \"pick_month\"]\n",
        "  df = delete_columns(df, del_columns)\n",
        "  for col in df.columns:\n",
        "    max = df[col].loc[df[col].idxmax()]     \n",
        "    min = df[col].loc[df[col].idxmin()]\n",
        "    normalized_col=(df[col] - min)/(max-min)\n",
        "    df[str(col) + \"_normalized\"] = normalized_col \n",
        "    cols.append(str(col) + \"_normalized\")\n",
        "  df2 = df[cols]\n",
        "  return df2"
      ],
      "metadata": {
        "id": "sP8Ed6azbl_e"
      },
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "code",
      "source": [
        "def clustering(df, df2):\n",
        "  # ok columns with clustering: 'speed', 'total_amount', 'trip_distance', 'tip_amount', 'fare_amount'\n",
        "  columns = ['speed_normalized', 'total_amount_normalized',\n",
        "             'trip_distance_normalized', 'tip_amount_normalized',\n",
        "             'fare_amount_normalized', 'passenger_count_normalized',\n",
        "             'trip_duration_min_normalized']\n",
        "  df2=df2[columns]\n",
        "  # elbow_k_means(df2)\n",
        "  kmeans = KMeans(n_clusters=4, init='k-means++', random_state=42).fit(df2)\n",
        "  centroids = kmeans.cluster_centers_\n",
        "  df = df.assign(cluster=kmeans.labels_)\n",
        "\n",
        "  #for 3d\n",
        "  # fig = plt.figure(figsize=(12, 12))\n",
        "  # ax = fig.add_subplot(projection='3d')\n",
        "  # ax.scatter(df['trip_distance'], df['tip_amount'], df['passenger_count'], marker=\"s\", c=df[\"cluster\"], s=40, cmap=\"RdBu\")\n",
        "\n",
        "  # print(\"x: trip_distance, y: tip_amount\")\n",
        "  # plt.scatter(df['trip_distance'], df['tip_amount'], c= kmeans.labels_.astype(float), s=50, alpha=0.5)\n",
        "  # plt.scatter(centroids[:, 0], centroids[:, 1], c='red', s=50)\n",
        "  # plt.show()\n",
        "\n",
        "  # print(\"x: speed, y: tip_amount\")\n",
        "  # plt.scatter(df['speed'], df['tip_amount'], c= kmeans.labels_.astype(float), s=50, alpha=0.5)\n",
        "  # plt.scatter(centroids[:, 0], centroids[:, 1], c='red', s=50)\n",
        "  # plt.show()\n",
        "\n",
        "  # print(\"x: total_amount, y: tip_amount\")\n",
        "  # plt.scatter(df['total_amount'], df['tip_amount'], c= kmeans.labels_.astype(float), s=50, alpha=0.5)\n",
        "  # plt.scatter(centroids[:, 0], centroids[:, 1], c='red', s=50)\n",
        "  # plt.show()\n",
        "\n",
        "  # print(\"x: speed, y: trip_distance\")\n",
        "  # plt.scatter(df['speed'], df['trip_distance'], c= kmeans.labels_.astype(float), s=50, alpha=0.5)\n",
        "  # plt.scatter(centroids[:, 0], centroids[:, 1], c='red', s=50)\n",
        "  # plt.show()\n",
        "\n",
        "  # print(\"x: speed, y: trip_duration_min\")\n",
        "  # plt.scatter(df['speed'], df['trip_duration_min'], c= kmeans.labels_.astype(float), s=50, alpha=0.5)\n",
        "  # plt.scatter(centroids[:, 0], centroids[:, 1], c='red', s=50)\n",
        "  # # plt.show()\n",
        "\n",
        "\n",
        "  # print(\"x: passenger_count, y: trip_distance\")\n",
        "  # plt.scatter(df['passenger_count'], df['trip_distance'], c= kmeans.labels_.astype(float), s=50, alpha=0.5)\n",
        "  # plt.scatter(centroids[:, 0], centroids[:, 1], c='red', s=50)\n",
        "  # plt.show()\n",
        "\n",
        "  return df"
      ],
      "metadata": {
        "id": "oY4s0RAOv_eT"
      },
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "code",
      "source": [
        "def elbow_k_means(df):\n",
        "  wcss = []\n",
        "  for i in range(1, 11):\n",
        "    clustering = KMeans(n_clusters=i, init='k-means++', random_state=42)\n",
        "    clustering.fit(df)\n",
        "    wcss.append(clustering.inertia_)\n",
        "  \n",
        "  ks = [1, 2, 3, 4, 5, 6, 7, 8, 9, 10]\n",
        "  sns.lineplot(x = ks, y = wcss);\n"
      ],
      "metadata": {
        "id": "8NLFGjQiqago"
      },
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "code",
      "source": [
        "def create_correlation_matrix(data_df):\n",
        "  new_data=data_df[['VendorID', 'passenger_count', 'RatecodeID', 'trip_distance', 'payment_type',\n",
        "                 'fare_amount', 'trip_duration_hour', 'speed', 'total_amount', 'tip_amount']]\n",
        "\n",
        "  label_encoder = LabelEncoder()\n",
        "  data = new_data\n",
        "  for i in range(len(data.columns)):\n",
        "      column = new_data.columns[i]\n",
        "      data[column] = label_encoder.fit_transform(new_data[column])\n",
        "      print(f\" dataframe {column} uniques: {len(data[column].unique())} \")\n",
        "\n",
        "  x = data\n",
        "  cor = x.corr().round(2)\n",
        "  plt.figure(figsize=(10,8),linewidth=10,edgecolor=\"#04253a\" )\n",
        "  sns.heatmap(cor, annot=True, cmap=\"Blues\")\n",
        "  plt.show()"
      ],
      "metadata": {
        "id": "87fbu1IOcXlP"
      },
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "code",
      "source": [
        "def calculate_Zscore(df):\n",
        "  columns = list(df.columns)\n",
        "  del_columns = [\"tpep_pickup_datetime\", \"tpep_dropoff_datetime\",\n",
        "                 \"pickup_longitude\", \"pickup_latitude\",\n",
        "                 \"RatecodeID\", \"store_and_fwd_flag\",\n",
        "                 \"dropoff_longitude\", \"dropoff_latitude\",\n",
        "                 \"improvement_surcharge\", \"tolls_amount\",\n",
        "                 \"tip_amount\", \"mta_tax\", \"extra\", \"pickup_date\", \"dropoff_date\", \"drop_month\", \"pick_month\"]\n",
        "  columns = [col for col in columns if col not in del_columns]\n",
        "  for col in columns:\n",
        "    col_zscore = col + '_zscore'\n",
        "    df[col_zscore] = (df[col] - df[col].mean())/df[col].std(ddof=0)\n",
        "  return df"
      ],
      "metadata": {
        "id": "68SablKTrP0d"
      },
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "code",
      "source": [
        "def calculate_BoxWhiskerScore(df):\n",
        "  columns = list(df.columns)\n",
        "  del_columns = [\"tpep_pickup_datetime\", \"tpep_dropoff_datetime\",\n",
        "                 \"pickup_longitude\", \"pickup_latitude\",\n",
        "                 \"RatecodeID\", \"store_and_fwd_flag\",\n",
        "                 \"dropoff_longitude\", \"dropoff_latitude\",\n",
        "                 \"improvement_surcharge\", \"tolls_amount\",\n",
        "                 \"tip_amount\", \"mta_tax\", \"extra\", \"pickup_date\", \"dropoff_date\", \"drop_month\", \"pick_month\",'payment_type']\n",
        "  columns = [col for col in columns if col not in del_columns]\n",
        "  for col in columns:\n",
        "    if \"_zscore\" not in col:\n",
        "      col_boxwhiskerscore = col + '_boxwhiskerscore'\n",
        "      Q1 = np.percentile(df[col], 25, interpolation = 'midpoint')\n",
        "      Q3 = np.percentile(df[col], 75, interpolation = 'midpoint')\n",
        "      IQR = Q3 - Q1\n",
        "      upper_bound = Q3+1.5*IQR\n",
        "      df[col_boxwhiskerscore] = df[col]/upper_bound\n",
        "  return df "
      ],
      "metadata": {
        "id": "WWTnxIZ66KPD"
      },
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "code",
      "source": [
        "def del_missing_value(df):\n",
        "    total = df.isnull().sum().sort_values(ascending=False)\n",
        "    percent = df.isnull().sum()/df.isnull().count().sort_values(ascending=False)\n",
        "    missing_data = pd.concat([total, percent], axis=1, keys=['Total', 'Percent'])\n",
        "    # f, ax = plt.subplots(figsize=(15, 6))\n",
        "    # plt.xticks(rotation='90')\n",
        "    # sns.barplot(x=missing_data.index, y=missing_data['Percent'])\n",
        "    # plt.xlabel('df', fontsize=15)\n",
        "    # plt.ylabel('Percent of missing values', fontsize=15)\n",
        "    # plt.title('Percent missing data by feature', fontsize=15)\n",
        "    # missing_data"
      ],
      "metadata": {
        "id": "4Z-0x7Py7mMC"
      },
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "code",
      "source": [
        "def pre_processing(df):\n",
        "  df.drop_duplicates(inplace=True)\n",
        "  del_missing_value(df)\n",
        "  # Deleting data manually\n",
        "  # df = df[df[\"speed\"] < 200]\n",
        "  df = df[df[\"speed\"] != 0]\n",
        "  # Calculate Z-Score and BoxWhisker-Score\n",
        "  df = calculate_Zscore(df)\n",
        "  df = calculate_BoxWhiskerScore(df)\n",
        "  # Deleting data accoriding to Z-Score\n",
        "  print(\"Before Z-Scoer: \", df.shape)\n",
        "  for col in df.columns:\n",
        "    try:\n",
        "      df = df[df[col + \"_zscore\"] < 4]\n",
        "      df = df[df[col + \"_zscore\"] > -4]\n",
        "    except:\n",
        "      pass\n",
        "  print(\"After Z-Scoer: \", df.shape)\n",
        "  df = df[df[\"trip_duration_min\"] < 200]\n",
        "  print(\"Deleting manually duration: \", df.shape)\n",
        "  df = df[df[\"speed\"] < 200]\n",
        "  print(\"Deleting manually speed: \", df.shape)\n",
        "  # Deleting data accoriding to BoxWhisker-Score\n",
        "  # for col in df.columns:\n",
        "  #   df = df[df[col + \"_boxwhiskerscore\"] > 4]\n",
        "  # print(\"After BoxWhisker-Score: \", df.shape)\n",
        "  # print(df.shape)\n",
        "\n",
        "  return df"
      ],
      "metadata": {
        "id": "3486XW0C4Sim"
      },
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "code",
      "source": [
        "def add_features(df):\n",
        "  # Adding new parameters\n",
        "  df[\"tpep_pickup_datetime\"] = pd.to_datetime(df[\"tpep_pickup_datetime\"])\n",
        "  df[\"tpep_dropoff_datetime\"] = pd.to_datetime(df[\"tpep_dropoff_datetime\"])\n",
        "  df[\"pickup_date\"] = df[\"tpep_pickup_datetime\"].dt.date \n",
        "  df[\"dropoff_date\"] = df[\"tpep_dropoff_datetime\"].dt.date\n",
        "  df[\"pick_month\"] = pd.DatetimeIndex(df['tpep_pickup_datetime']).month\n",
        "  df[\"drop_month\"] = pd.DatetimeIndex(df['tpep_dropoff_datetime']).month\n",
        "  df[\"trip_duration_hour\"] = (df[\"tpep_dropoff_datetime\"] - df[\"tpep_pickup_datetime\"]).dt.total_seconds() / 3600\n",
        "  df[\"trip_duration_min\"] = (df[\"tpep_dropoff_datetime\"] - df[\"tpep_pickup_datetime\"]).dt.total_seconds() / 60\n",
        "  # Deleting Durations\n",
        "  df2 = df[~df[\"trip_duration_min\"].isin([0])]\n",
        "  # df = df[df[\"trip_duration_min\"] < 180]\n",
        "  # df = df[df[\"trip_duration_min\"] != 0]\n",
        "  df2[\"speed\"] = df2[\"trip_distance\"] * 1.6093435 / df2[\"trip_duration_hour\"]\n",
        "  pd.set_option('display.max_rows', None)\n",
        "  # BoxWhiskerScore and Z-Score\n",
        "  calculate_Zscore(df)\n",
        "  calculate_BoxWhiskerScore(df)\n",
        "  return df2"
      ],
      "metadata": {
        "id": "sNPlGkAJEyXS"
      },
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "code",
      "source": [
        "def count_payment_types_per_group(data):\n",
        "  new_df =  data.groupby('payment_type').size().reset_index(name='count')\n",
        "  return new_df"
      ],
      "metadata": {
        "id": "mn0SOdwv-dZG"
      },
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "code",
      "source": [
        "def decision_tree(data):\n",
        "\n",
        "  X =data.drop(['VendorID', 'VendorID_zscore', 'VendorID_boxwhiskerscore', 'VendorID_normalized',\n",
        "                 'VendorID_zscore_normalized', 'VendorID_boxwhiskerscore_normalized', 'cluster'], axis=1)\n",
        "  \n",
        "  y = data['cluster']\n",
        "\n",
        "  X_train, X_test, y_train, y_test = train_test_split(X, y, test_size=0.20)\n",
        "\n",
        "  # fill NaN annd infinite\n",
        "  X_train = X_train.replace((np.inf, -np.inf, np.nan), 0).reset_index(drop=True)\n",
        "  X_test = X_test.replace((np.inf, -np.inf, np.nan), 0).reset_index(drop=True)\n",
        "\n",
        "  classifier = DecisionTreeClassifier(max_depth=5)\n",
        "  classifier.fit(X_train, y_train)\n",
        "\n",
        "  y_pred = classifier.predict(X_test)\n",
        "\n",
        "  # validation\n",
        "  print(confusion_matrix(y_test, y_pred))\n",
        "  print(classification_report(y_test, y_pred))\n",
        "\n",
        "  # visualization\n",
        "  export_graphviz(\n",
        "        classifier,\n",
        "        out_file=\"tree.dot\",\n",
        "        feature_names = classifier.feature_names_in_,\n",
        "        class_names=[\"0\", \"1\", \"2\", \"3\"],\n",
        "        rounded=True,\n",
        "        filled=True\n",
        "    )\n",
        "  \n",
        "  view = Source.from_file(\"tree.dot\")\n",
        "  view.render('tree', format='jpg',view=True)\n",
        "  view.view()\n"
      ],
      "metadata": {
        "id": "4SD20PnVa62-"
      },
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "code",
      "source": [
        "def main():\n",
        "  download_csv_file()\n",
        "  df = pd.read_csv('sample-nyc-data.csv', low_memory=False) # bad practice !!!!!!!!!!!!!!!!!!!\n",
        "  # new_df = count_payment_types_per_group(df)\n",
        "  # print(new_df)\n",
        "  \n",
        "  df = add_features(df)\n",
        "  # convert_df_to_csv(df)\n",
        "  df = pre_processing(df)\n",
        "  # df.to_csv('pre_proccessed.csv')\n",
        "  # files.download(\"pre_proccessed.csv\")\n",
        "\n",
        "  # create_correlation_matrix(df)\n",
        "\n",
        "\n",
        "  # clustering start\n",
        "  df2 = normalize_min_max(df)\n",
        "  df = clustering(df, df2)\n",
        "  # print(df['cluster'])\n",
        "  # for i in df['cluster']:\n",
        "  #   print(i)\n",
        "  # df.to_csv(\"clustering.csv\", encoding = 'utf-8-sig')\n",
        "  # files.download(\"clustering.csv\")\n",
        "  # if 'cluster' in df.columns:\n",
        "  #   print(\"Cluster column exists\")\n",
        "  # print(df.keys)\n",
        "  # clustering end\n",
        "\n",
        "  decision_tree(df)\n",
        "  "
      ],
      "metadata": {
        "id": "RKvICFznIo0f"
      },
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "code",
      "source": [
        "main()"
      ],
      "metadata": {
        "id": "xEheQLG-I3sf",
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "outputId": "f49abb2c-3627-4952-9954-14bf5e53e9da"
      },
      "execution_count": null,
      "outputs": [
        {
          "output_type": "stream",
          "name": "stderr",
          "text": [
            "/usr/local/lib/python3.7/dist-packages/ipykernel_launcher.py:15: SettingWithCopyWarning: \n",
            "A value is trying to be set on a copy of a slice from a DataFrame.\n",
            "Try using .loc[row_indexer,col_indexer] = value instead\n",
            "\n",
            "See the caveats in the documentation: https://pandas.pydata.org/pandas-docs/stable/user_guide/indexing.html#returning-a-view-versus-a-copy\n",
            "  from ipykernel import kernelapp as app\n",
            "/usr/local/lib/python3.7/dist-packages/ipykernel_launcher.py:12: SettingWithCopyWarning: \n",
            "A value is trying to be set on a copy of a slice from a DataFrame.\n",
            "Try using .loc[row_indexer,col_indexer] = value instead\n",
            "\n",
            "See the caveats in the documentation: https://pandas.pydata.org/pandas-docs/stable/user_guide/indexing.html#returning-a-view-versus-a-copy\n",
            "  if sys.path[0] == '':\n",
            "/usr/local/lib/python3.7/dist-packages/ipykernel_launcher.py:17: SettingWithCopyWarning: \n",
            "A value is trying to be set on a copy of a slice from a DataFrame.\n",
            "Try using .loc[row_indexer,col_indexer] = value instead\n",
            "\n",
            "See the caveats in the documentation: https://pandas.pydata.org/pandas-docs/stable/user_guide/indexing.html#returning-a-view-versus-a-copy\n"
          ]
        },
        {
          "output_type": "stream",
          "name": "stdout",
          "text": [
            "Before Z-Scoer:  (89418, 43)\n",
            "After Z-Scoer:  (87335, 43)\n",
            "Deleting manually duration:  (87335, 43)\n",
            "Deleting manually speed:  (87322, 43)\n",
            "[[ 3435     0    36    41]\n",
            " [    1  2373     0    15]\n",
            " [  130     0 10351     0]\n",
            " [   34     4     0  1045]]\n",
            "              precision    recall  f1-score   support\n",
            "\n",
            "           0       0.95      0.98      0.97      3512\n",
            "           1       1.00      0.99      1.00      2389\n",
            "           2       1.00      0.99      0.99     10481\n",
            "           3       0.95      0.96      0.96      1083\n",
            "\n",
            "    accuracy                           0.99     17465\n",
            "   macro avg       0.97      0.98      0.98     17465\n",
            "weighted avg       0.99      0.99      0.99     17465\n",
            "\n"
          ]
        }
      ]
    }
  ]
}