drive_recommendation.ipynb

{
 "cells": [
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "scrolled": false
   },
   "outputs": [],
   "source": [
    "import pandas as pd\n",
    "import numpy as np\n",
    "from sklearn.cluster import AgglomerativeClustering\n",
    "from sklearn.cluster import KMeans\n",
    "import matplotlib.pyplot as plt\n",
    "from src.models.drive_recommendation.temporary import populate_geo_coordinates\n",
    "from src.models.drive_recommendation.constants import OUTPUT_FILE\n",
    "import time\n",
    "import os\n",
    "BASEPATH = '/home/jasmeet16-jtg/projects/calyx_ds/src/models/drive_recommendation'\n",
    "\n",
    "cluster_on_distance_file_path = os.path.join(BASEPATH, 'pool_recommendation_data/demo_drive_recommendation_cluster_on_distance.csv')\n",
    "cluster_on_distance_norm_count_file_path = os.path.join(BASEPATH, 'pool_recommendation_data/demo_drive_recommendation_cluster_on_distance_norm_count.csv')\n",
    "pool_w_geoloc_file_path = os.path.join(BASEPATH, 'pool_recommendation_data/pool_df_w_geoloc.csv')\n",
    "\n",
    "def cluster_on_distance(pool_df):\n",
    "\n",
    "    #distance_threshold of 1.4846059080767864 equals to almost 150 KMs.\n",
    "    hc = AgglomerativeClustering(affinity = 'euclidean', linkage = 'ward', distance_threshold=1.4846059080767864)\n",
    "    y_hc = hc.fit_predict(np.array(pool_df[['lat', 'lng']]))\n",
    "    pool_df = pd.concat([pool_df, pd.DataFrame(y_hc)], axis=1).rename(columns={0:'cluster_number'})\n",
    "    pool_df = pool_df.sort_values(by='cluster_number')\n",
    "    centroid_lat, centroid_lng = [], []\n",
    "    for i in range(0, pool_df['cluster_number'].nunique()):\n",
    "        df1 = pool_df[pool_df['cluster_number']==i]\n",
    "        lt = [df1['lat'].min(), df1['lng'].min()]\n",
    "        rb = [df1['lat'].max(), df1['lng'].max()]\n",
    "        centroid_lat.extend([(lt[0] + rb[0])/2] * len(df1))\n",
    "        centroid_lng.extend([(lt[1] + rb[1])/2] * len(df1))\n",
    "    pool_df['centriod_lat'] = centroid_lat\n",
    "    pool_df['centriod_lng'] = centroid_lng\n",
    "    return pool_df\n",
    "\n",
    "def cluster_on_count(input_df):\n",
    "    input_df = input_df.dropna()\n",
    "    result_df = pd.DataFrame()\n",
    "\n",
    "    for i in range(input_df['cluster_number'].nunique()):\n",
    "        df1 = input_df[input_df['cluster_number']==i]\n",
    "        if not len(df1):\n",
    "            print(i)\n",
    "            continue\n",
    "        max_ = df1['ranking'].max()\n",
    "        max_count = df1['count'].max()\n",
    "        norm_count = []\n",
    "        for index, row in df1.iterrows():\n",
    "            norm_count.append((row['count']/max_count)*max_)\n",
    "        df1['norm_count'] = norm_count\n",
    "        n_clusters = 4 if len(df1) >= 4 else len(df1)\n",
    "        kmeans = KMeans(n_clusters=n_clusters).fit(df1[['ranking', 'norm_count']])\n",
    "        df1['internal_clusters'] = list(kmeans.labels_)\n",
    "        df1.sort_values(by='internal_clusters')\n",
    "        result_df = pd.concat([result_df, df1])\n",
    "    result_df = result_df.sort_values(by=['cluster_number', 'internal_clusters'])\n",
    "    return result_df\n",
    " \n",
    "def make_drives(input_df, company_lat, company_lng, req, no_of_drive_to_recommend=10):\n",
    "    input_df['lat_diff'] = input_df[['centriod_lat']].apply(lambda x: np.power(abs(x-company_lat), 2))\n",
    "    input_df['lng_diff'] = input_df[['centriod_lng']].apply(lambda x: np.power(abs(x-company_lng), 2))\n",
    "    input_df['distance'] = np.sqrt(input_df['lat_diff']+input_df['lng_diff'])\n",
    "    input_df = input_df.sort_values(by='distance', ascending=True)\n",
    "    \n",
    "    def getuniq(l):\n",
    "        a = []\n",
    "        i = 0\n",
    "        while i < len(l):\n",
    "            s = l[i]\n",
    "            a.append(s)\n",
    "            while i < len(l) and l[i] == s:\n",
    "                i += 1\n",
    "        return a\n",
    "            \n",
    "    x = getuniq(list(input_df['cluster_number']))\n",
    "    result_df = pd.DataFrame()\n",
    "    j,i=0,0\n",
    "    while j<len(x) and i<no_of_drive_to_recommend:\n",
    "        temp_df = input_df[input_df['cluster_number']==x[j]]\n",
    "        j += 1\n",
    "        group_obj = temp_df.groupby('internal_clusters')['ranking'].apply(np.mean).reset_index()\n",
    "        group_obj = group_obj.sort_values('ranking')\n",
    "        drive_df = pd.DataFrame()\n",
    "        sum_ = 0\n",
    "        for index, row in group_obj.iterrows():\n",
    "            cluster_df = temp_df[temp_df['internal_clusters']==row['internal_clusters']]\n",
    "            for index, row in cluster_df.iterrows():\n",
    "                drive_df = drive_df.append(row)\n",
    "                sum_ += row['count']\n",
    "                if sum_>req:\n",
    "                    drive_df['Rank'] = i\n",
    "                    result_df = pd.concat([result_df, drive_df])\n",
    "                    drive_df = pd.DataFrame()\n",
    "                    sum_ = 0\n",
    "                    i += 1\n",
    "            if sum_ >= int(0.6*req):\n",
    "                drive_df['Rank'] = i\n",
    "                result_df = pd.concat([result_df, drive_df])\n",
    "                drive_df = pd.DataFrame()\n",
    "                sum_ = 0\n",
    "                i += 1\n",
    "    return result_df\n",
    "\n",
    "if not os.path.exists(pool_w_geoloc_file_path):\n",
    "    pool_df_w_geoloc = populate_geo_coordinates()\n",
    "    pool_df_w_geoloc.to_csv(pool_w_geoloc_file_path)\n",
    "else:\n",
    "    pool_df_w_geoloc = pd.read_csv(pool_w_geoloc_file_path)\n",
    "s = time.time()\n",
    "college_cluster_wrt_distance = cluster_on_distance(pool_df_w_geoloc)\n",
    "college_cluster_wrt_distance.to_csv(cluster_on_distance_file_path)\n",
    "college_cluster_wrt_count_distance = cluster_on_count(college_cluster_wrt_distance)\n",
    "college_cluster_wrt_count_distance.to_csv(cluster_on_distance_norm_count_file_path)\n",
    "company_lat, company_lng, req = 28.503129, 77.083702, 100\n",
    "result = make_drives(college_cluster_wrt_count_distance, company_lat, company_lng, req)\n",
    "e = time.time()\n",
    "print(\"Time Taken\", e-s)\n",
    "result.to_csv(OUTPUT_FILE)\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "scrolled": false
   },
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "scrolled": false
   },
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "scrolled": false
   },
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "scrolled": false
   },
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "scrolled": false
   },
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "scrolled": false
   },
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "scrolled": false
   },
   "outputs": [],
   "source": []
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "calyx_ds3",
   "language": "python",
   "name": "calyx_ds3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.5.2"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 2
}