saf_data_indexing.ipynb

{
 "cells": [
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## Imports & Installs"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "#### Installs"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "!pip install geopandas\n",
    "!pip install geopy\n",
    "!pip install shapely\n",
    "!pip install pyproj"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### Imports"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 1,
   "metadata": {},
   "outputs": [],
   "source": [
    "import pandas as pd\n",
    "import numpy as np\n",
    "import math\n",
    "import random\n",
    "\n",
    "import matplotlib.pyplot as plt\n",
    "\n",
    "import geopandas as gpd\n",
    "import geopy\n",
    "from geopy.distance import distance, Distance, great_circle\n",
    "import shapely\n",
    "from shapely.geometry import Polygon\n",
    "import pyproj"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## Stop and Frisk Indexing"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "#### shapefile"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 3,
   "metadata": {},
   "outputs": [],
   "source": [
    "# shapefile used to determine indexing grid variables\n",
    "shp = gpd.read_file('/Users/ernestvmo/Downloads/Borough Boundaries/geo_export_c4a7f77b-811d-4998-b675-a83cf42208e0.shp')\n",
    "shp['geometry'] = shp['geometry']"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 4,
   "metadata": {},
   "outputs": [],
   "source": [
    "min_x = min(shp.bounds['minx'].values)\n",
    "max_x = max(shp.bounds['maxx'].values)\n",
    "\n",
    "min_y = min(shp.bounds['miny'].values)\n",
    "max_y = max(shp.bounds['maxy'].values)\n",
    "\n",
    "grid_00 = (min_x, max_y)\n",
    "grid_01 = (max_x, max_y)\n",
    "grid_10 = (min_x, min_y)\n",
    "grid_11 = (max_x, min_y)\n",
    "\n",
    "cell_x_diff = 6579.3\n",
    "cell_y_diff = -6574.5\n",
    "\n",
    "# calculated using great_circle distance\n",
    "height = 46.63508344453982 # height of NYC \n",
    "width = 46.68408524309399 # width of NYC"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 7,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "(40.496133987611806, 40.91553277650264)"
      ]
     },
     "execution_count": 7,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "min_y, max_y"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "#### Load data"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 37,
   "metadata": {},
   "outputs": [],
   "source": [
    "stop_frisk_file = \"/Users/ernestvmo/OneDrive - Universiteit Leiden/Q1/Urban Computing/Project/data/NYPDstopAndFrisk12-13.csv\"\n",
    "saf_df = pd.read_csv(stop_frisk_file)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### Assign Index using Modulo"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 38,
   "metadata": {},
   "outputs": [],
   "source": [
    "saf_df = saf_df[['datestop', 'xcoord', 'ycoord']]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "code",
   "execution_count": 39,
   "metadata": {},
   "outputs": [],
   "source": [
    "saf_df['nyc_index_x'], saf_df['nyc_index_y'] = np.floor((saf_df['xcoord'].values - min_x) / cell_x_diff), np.floor(((width // 2) - (saf_df['ycoord'].values - min_y) / abs(cell_y_diff)))\n",
    "saf_df = saf_df.astype({'nyc_index_x':'int64', 'nyc_index_y':'int64'})\n",
    "saf_df['coordinates'] = list(zip(saf_df.nyc_index_x, saf_df.nyc_index_y))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 40,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>datestop</th>\n",
       "      <th>xcoord</th>\n",
       "      <th>ycoord</th>\n",
       "      <th>nyc_index_x</th>\n",
       "      <th>nyc_index_y</th>\n",
       "      <th>coordinates</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>2012-07-02</td>\n",
       "      <td>1019585</td>\n",
       "      <td>184371</td>\n",
       "      <td>16</td>\n",
       "      <td>13</td>\n",
       "      <td>(16, 13)</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>2012-07-03</td>\n",
       "      <td>987078</td>\n",
       "      <td>215157</td>\n",
       "      <td>11</td>\n",
       "      <td>8</td>\n",
       "      <td>(11, 8)</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2</th>\n",
       "      <td>2012-07-05</td>\n",
       "      <td>1002416</td>\n",
       "      <td>231297</td>\n",
       "      <td>13</td>\n",
       "      <td>6</td>\n",
       "      <td>(13, 6)</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3</th>\n",
       "      <td>2012-07-06</td>\n",
       "      <td>988511</td>\n",
       "      <td>164316</td>\n",
       "      <td>11</td>\n",
       "      <td>16</td>\n",
       "      <td>(11, 16)</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4</th>\n",
       "      <td>2012-07-08</td>\n",
       "      <td>995824</td>\n",
       "      <td>230943</td>\n",
       "      <td>12</td>\n",
       "      <td>6</td>\n",
       "      <td>(12, 6)</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>...</th>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>343455</th>\n",
       "      <td>2013-06-30</td>\n",
       "      <td>966801</td>\n",
       "      <td>156143</td>\n",
       "      <td>8</td>\n",
       "      <td>17</td>\n",
       "      <td>(8, 17)</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>343456</th>\n",
       "      <td>2013-06-30</td>\n",
       "      <td>954749</td>\n",
       "      <td>149274</td>\n",
       "      <td>6</td>\n",
       "      <td>18</td>\n",
       "      <td>(6, 18)</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>343457</th>\n",
       "      <td>2013-06-30</td>\n",
       "      <td>954749</td>\n",
       "      <td>149274</td>\n",
       "      <td>6</td>\n",
       "      <td>18</td>\n",
       "      <td>(6, 18)</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>343458</th>\n",
       "      <td>2013-06-30</td>\n",
       "      <td>964020</td>\n",
       "      <td>154103</td>\n",
       "      <td>7</td>\n",
       "      <td>17</td>\n",
       "      <td>(7, 17)</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>343459</th>\n",
       "      <td>2013-06-30</td>\n",
       "      <td>964020</td>\n",
       "      <td>154103</td>\n",
       "      <td>7</td>\n",
       "      <td>17</td>\n",
       "      <td>(7, 17)</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "<p>343460 rows × 6 columns</p>\n",
       "</div>"
      ],
      "text/plain": [
       "          datestop   xcoord  ycoord  nyc_index_x  nyc_index_y coordinates\n",
       "0       2012-07-02  1019585  184371           16           13    (16, 13)\n",
       "1       2012-07-03   987078  215157           11            8     (11, 8)\n",
       "2       2012-07-05  1002416  231297           13            6     (13, 6)\n",
       "3       2012-07-06   988511  164316           11           16    (11, 16)\n",
       "4       2012-07-08   995824  230943           12            6     (12, 6)\n",
       "...            ...      ...     ...          ...          ...         ...\n",
       "343455  2013-06-30   966801  156143            8           17     (8, 17)\n",
       "343456  2013-06-30   954749  149274            6           18     (6, 18)\n",
       "343457  2013-06-30   954749  149274            6           18     (6, 18)\n",
       "343458  2013-06-30   964020  154103            7           17     (7, 17)\n",
       "343459  2013-06-30   964020  154103            7           17     (7, 17)\n",
       "\n",
       "[343460 rows x 6 columns]"
      ]
     },
     "execution_count": 40,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "saf_df"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### Grouping"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 128,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "datestop       object\n",
       "xcoord          int64\n",
       "ycoord          int64\n",
       "nyc_index_x     int64\n",
       "nyc_index_y     int64\n",
       "coordinates    object\n",
       "dtype: object"
      ]
     },
     "execution_count": 128,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "saf_df.dtypes"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 121,
   "metadata": {},
   "outputs": [],
   "source": [
    "grouped_saf_1 = saf_df.groupby(['datestop', 'coordinates']).size().reset_index()\n",
    "grouped_saf_2 = saf_df.groupby(['datestop', 'nyc_index_x', 'nyc_index_y']).size().reset_index()\n",
    "\n",
    "grouped_saf_1 = grouped_saf_1.rename(columns={0:'SAF_COUNT'})\n",
    "grouped_saf_2 = grouped_saf_2.rename(columns={0:'SAF_COUNT'})"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 125,
   "metadata": {},
   "outputs": [],
   "source": [
    "save_location = '/Users/ernestvmo/OneDrive - Universiteit Leiden/Q1/Urban Computing/Project/data/grouped_csv/'\n",
    "\n",
    "grouped_saf_1.to_csv(save_location + 'saf_grouped.csv')\n",
    "# grouped_nypd_complaints_2.to_csv('grouped_nypd_complaints.csv')"
   ]
  }
 ],
 "metadata": {
  "interpreter": {
   "hash": "aee8b7b246df8f9039afb4144a1f6fd8d2ca17a180786b69acc140d282b71a49"
  },
  "kernelspec": {
   "display_name": "Python 3.9.4 64-bit",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.9.4"
  },
  "orig_nbformat": 4
 },
 "nbformat": 4,
 "nbformat_minor": 2
}