nypd_complaints_indexing.ipynb

{
 "cells": [
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## Imports & Installs"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "#### Installs"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 1,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Requirement already satisfied: geopandas in /Library/Frameworks/Python.framework/Versions/3.9/lib/python3.9/site-packages (0.10.2)\n",
      "Requirement already satisfied: pandas>=0.25.0 in /Library/Frameworks/Python.framework/Versions/3.9/lib/python3.9/site-packages (from geopandas) (1.3.3)\n",
      "Requirement already satisfied: pyproj>=2.2.0 in /Library/Frameworks/Python.framework/Versions/3.9/lib/python3.9/site-packages (from geopandas) (3.3.0)\n",
      "Requirement already satisfied: fiona>=1.8 in /Library/Frameworks/Python.framework/Versions/3.9/lib/python3.9/site-packages (from geopandas) (1.8.20)\n",
      "Requirement already satisfied: shapely>=1.6 in /Library/Frameworks/Python.framework/Versions/3.9/lib/python3.9/site-packages (from geopandas) (1.8.0)\n",
      "Requirement already satisfied: cligj>=0.5 in /Library/Frameworks/Python.framework/Versions/3.9/lib/python3.9/site-packages (from fiona>=1.8->geopandas) (0.7.2)\n",
      "Requirement already satisfied: six>=1.7 in /Library/Frameworks/Python.framework/Versions/3.9/lib/python3.9/site-packages (from fiona>=1.8->geopandas) (1.15.0)\n",
      "Requirement already satisfied: click>=4.0 in /Library/Frameworks/Python.framework/Versions/3.9/lib/python3.9/site-packages (from fiona>=1.8->geopandas) (8.0.3)\n",
      "Requirement already satisfied: munch in /Library/Frameworks/Python.framework/Versions/3.9/lib/python3.9/site-packages (from fiona>=1.8->geopandas) (2.5.0)\n",
      "Requirement already satisfied: click-plugins>=1.0 in /Library/Frameworks/Python.framework/Versions/3.9/lib/python3.9/site-packages (from fiona>=1.8->geopandas) (1.1.1)\n",
      "Requirement already satisfied: attrs>=17 in /Library/Frameworks/Python.framework/Versions/3.9/lib/python3.9/site-packages (from fiona>=1.8->geopandas) (21.2.0)\n",
      "Requirement already satisfied: certifi in /Library/Frameworks/Python.framework/Versions/3.9/lib/python3.9/site-packages (from fiona>=1.8->geopandas) (2021.10.8)\n",
      "Requirement already satisfied: setuptools in /Library/Frameworks/Python.framework/Versions/3.9/lib/python3.9/site-packages (from fiona>=1.8->geopandas) (49.2.1)\n",
      "Requirement already satisfied: python-dateutil>=2.7.3 in /Library/Frameworks/Python.framework/Versions/3.9/lib/python3.9/site-packages (from pandas>=0.25.0->geopandas) (2.8.2)\n",
      "Requirement already satisfied: numpy>=1.17.3 in /Library/Frameworks/Python.framework/Versions/3.9/lib/python3.9/site-packages (from pandas>=0.25.0->geopandas) (1.19.5)\n",
      "Requirement already satisfied: pytz>=2017.3 in /Library/Frameworks/Python.framework/Versions/3.9/lib/python3.9/site-packages (from pandas>=0.25.0->geopandas) (2021.3)\n",
      "Requirement already satisfied: geopy in /Library/Frameworks/Python.framework/Versions/3.9/lib/python3.9/site-packages (2.2.0)\n",
      "Requirement already satisfied: geographiclib<2,>=1.49 in /Library/Frameworks/Python.framework/Versions/3.9/lib/python3.9/site-packages (from geopy) (1.52)\n",
      "Requirement already satisfied: shapely in /Library/Frameworks/Python.framework/Versions/3.9/lib/python3.9/site-packages (1.8.0)\n",
      "Requirement already satisfied: pyproj in /Library/Frameworks/Python.framework/Versions/3.9/lib/python3.9/site-packages (3.3.0)\n",
      "Requirement already satisfied: certifi in /Library/Frameworks/Python.framework/Versions/3.9/lib/python3.9/site-packages (from pyproj) (2021.10.8)\n"
     ]
    }
   ],
   "source": [
    "!pip install geopandas\n",
    "!pip install geopy\n",
    "!pip install shapely\n",
    "!pip install pyproj"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### Imports"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 2,
   "metadata": {},
   "outputs": [],
   "source": [
    "import pandas as pd\n",
    "import numpy as np\n",
    "import math\n",
    "import random\n",
    "\n",
    "import matplotlib.pyplot as plt\n",
    "\n",
    "import geopandas as gpd\n",
    "import geopy\n",
    "from geopy.distance import distance, Distance, great_circle\n",
    "import shapely\n",
    "from shapely.geometry import Polygon\n",
    "import pyproj"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## NYPD Complaints Indexing"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "#### shapefile"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 3,
   "metadata": {},
   "outputs": [],
   "source": [
    "# shapefile used to determine indexing grid variables\n",
    "shp = gpd.read_file('/Users/ernestvmo/Downloads/Borough Boundaries/geo_export_c4a7f77b-811d-4998-b675-a83cf42208e0.shp')\n",
    "shp['geometry'] = shp['geometry'].values.to_crs(2263)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 4,
   "metadata": {},
   "outputs": [],
   "source": [
    "min_x = min(shp.bounds['minx'].values)\n",
    "max_x = max(shp.bounds['maxx'].values)\n",
    "\n",
    "min_y = min(shp.bounds['miny'].values)\n",
    "max_y = max(shp.bounds['maxy'].values)\n",
    "\n",
    "grid_00 = (min_x, max_y)\n",
    "grid_01 = (max_x, max_y)\n",
    "grid_10 = (min_x, min_y)\n",
    "grid_11 = (max_x, min_y)\n",
    "\n",
    "cell_x_diff = 6579.3\n",
    "cell_y_diff = -6574.5\n",
    "\n",
    "# calculated using great_circle distance\n",
    "height = 46.63508344453982 # height of NYC in km\n",
    "width = 46.68408524309399 # width of NYC in km"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 8,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "120128.36999505223"
      ]
     },
     "execution_count": 8,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "min_y"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "#### Load data"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 38,
   "metadata": {},
   "outputs": [],
   "source": [
    "nypd_complaints_file = \"/Users/ernestvmo/OneDrive - Universiteit Leiden/Q1/Urban Computing/Project/data/NYPDcomplaintData.csv\"\n",
    "nypd_complaints_df = pd.read_csv(nypd_complaints_file)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### Assign Index using Modulo"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 39,
   "metadata": {},
   "outputs": [],
   "source": [
    "nypd_complaints_df['CMPLNT_FR_DT'] = pd.to_datetime(nypd_complaints_df['CMPLNT_FR_DT'])\n",
    "nypd_complaints_df = nypd_complaints_df[['CMPLNT_FR_DT', 'X_COORD_CD', 'Y_COORD_CD']]\n",
    "nypd_complaints_df = nypd_complaints_df.rename(columns={'X_COORD_CD':'xcoord', 'Y_COORD_CD':'ycoord'})\n",
    "nypd_complaints_df = nypd_complaints_df[~nypd_complaints_df['xcoord'].isna()]\n",
    "nypd_complaints_df = nypd_complaints_df[~nypd_complaints_df['ycoord'].isna()]\n",
    "\n",
    "nypd_complaints_df['xcoord'] = nypd_complaints_df['xcoord'].str.replace(',','').astype(float)\n",
    "nypd_complaints_df['ycoord'] = nypd_complaints_df['ycoord'].str.replace(',','').astype(float)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 40,
   "metadata": {},
   "outputs": [],
   "source": [
    "nypd_complaints_df['nyc_index_x'], nypd_complaints_df['nyc_index_y'] = np.floor((nypd_complaints_df['xcoord'].values - min_x) / cell_x_diff), np.floor(((width // 2) - (nypd_complaints_df['ycoord'].values - min_y) / abs(cell_y_diff)))\n",
    "nypd_complaints_df = nypd_complaints_df.astype({'nyc_index_x':'int64', 'nyc_index_y':'int64'})"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### Grouping"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 52,
   "metadata": {},
   "outputs": [],
   "source": [
    "nypd_complaints_df = nypd_complaints_df.rename(columns={'CMPLNT_FR_DT':'datestop'})\n",
    "nypd_complaints_df['coordinates'] = list(zip(nypd_complaints_df.nyc_index_x, nypd_complaints_df.nyc_index_y))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 69,
   "metadata": {},
   "outputs": [],
   "source": [
    "grouped_nypd_complaints_1 = nypd_complaints_df.groupby(['datestop', 'coordinates']).size().reset_index()\n",
    "grouped_nypd_complaints_2 = nypd_complaints_df.groupby(['datestop', 'nyc_index_x', 'nyc_index_y']).size().reset_index()\n",
    "\n",
    "grouped_nypd_complaints_1 = grouped_nypd_complaints_1.rename(columns={0:'nypd_compaints_COUNT'})\n",
    "grouped_nypd_complaints_2 = grouped_nypd_complaints_2.rename(columns={0:'nypd_compaints_COUNT'})"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 70,
   "metadata": {},
   "outputs": [],
   "source": [
    "save_location = '/Users/ernestvmo/OneDrive - Universiteit Leiden/Q1/Urban Computing/Project/data/grouped_csv/'\n",
    "\n",
    "grouped_nypd_complaints_1.to_csv(save_location + 'grouped_nypd_complaints.csv')\n",
    "# grouped_nypd_complaints_2.to_csv('grouped_nypd_complaints.csv')"
   ]
  }
 ],
 "metadata": {
  "interpreter": {
   "hash": "aee8b7b246df8f9039afb4144a1f6fd8d2ca17a180786b69acc140d282b71a49"
  },
  "kernelspec": {
   "display_name": "Python 3.9.4 64-bit",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.9.4"
  },
  "orig_nbformat": 4
 },
 "nbformat": 4,
 "nbformat_minor": 2
}