master_data_fetch.ipynb

{
 "cells": [
  {
   "cell_type": "code",
   "execution_count": 18,
   "metadata": {},
   "outputs": [],
   "source": [
    "# import packages\n",
    "import pandas as pd\n",
    "import json\n",
    "import osmnx as ox\n",
    "import requests\n",
    "import random\n",
    "import time"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 19,
   "metadata": {},
   "outputs": [],
   "source": [
    "# function to generate random lat lon coords within a closed range\n",
    "def random_coords(center_coords, count):\n",
    "    lat = []\n",
    "    lon = []\n",
    "    \n",
    "    for i in range(count):\n",
    "        #set random generator\n",
    "        lat.append(random.uniform(center_coords[0]-0.1, center_coords[0]+0.1))\n",
    "        lon.append(random.uniform(center_coords[1]-0.1, center_coords[1]+0.1))\n",
    "    \n",
    "    return lat, lon"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 20,
   "metadata": {},
   "outputs": [],
   "source": [
    "# determine center coords to be downtown Seattle\n",
    "center_coords = [47.608013, -122.335167]\n",
    "\n",
    "lat, lon = random_coords(center_coords, 1000)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "# set empty walk_score list\n",
    "walk_score = []\n",
    "\n",
    "apikey = 'APIKEY'\n",
    "\n",
    "\n",
    "\n",
    "for i in range(len(lat)):\n",
    "    url = 'http://api.walkscore.com/score?format=json&lat='+str(lat[i])+'&lon='+str(lon[i])+'&wsapikey='+apikey\n",
    "    r = requests.get(url)\n",
    "    data = r.json()\n",
    "    if 'walkscore' in data:\n",
    "        walk_score.append(data['walkscore'])\n",
    "    else:\n",
    "        walk_score.append('N/A')\n",
    "        \n",
    "    time.sleep(0.2)    "
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "# create dataframe\n",
    "new = pd.DataFrame(list(zip(lat, lon, walk_score)),  columns=['lat', 'lon', 'walk_score'])\n",
    "new.info()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "GEOID = []\n",
    "for i in range(len(new)):\n",
    "    url = 'https://geocoding.geo.census.gov/geocoder/geographies/coordinates?x='+str(new.lon[i])+'&y='+str(new.lat[i])+'&benchmark=Public_AR_ACS2018&vintage=Current_ACS2018&format=json'\n",
    "    r = requests.get(url)\n",
    "    data = r.json()\n",
    "    if 'GEOID' in data['result']['geographies']['Census Tracts'][0]:\n",
    "        tag = data['result']['geographies']['Census Tracts'][0]['GEOID']\n",
    "    else:\n",
    "        tag = 'N/A'\n",
    "        \n",
    "    GEOID.append(tag)\n",
    "    \n",
    "new['GEOID'] = GEOID"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "new = new[new['GEOID'] != 'N/A']\n",
    "new = new.dropna()\n",
    "\n",
    "new['GEOID'] = new['GEOID'].astype(int)\n",
    "\n",
    "new = new.reset_index(drop=True)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "G = ox.graph_from_place('Seattle, Washington', network_type='walk')\n",
    "ox.plot_graph(G)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "# number of intersections in graph, that is, nodes with >1 street emanating from them\n",
    "intersection_count = []\n",
    "\n",
    "#how many streets (edges in the undirected representation of the graph) emanate from each node (ie,intersection or dead-end) on average (mean)\n",
    "streets_per_node_avg = []\n",
    "\n",
    "# edge_length_total divided by the sum of the great circle distances between the nodes of each edge\n",
    "circuity_avg = []\n",
    "\n",
    "# mean edge length in the undirected representation of the graph, in meters\n",
    "street_length_avg = []\n",
    "\n",
    "\n",
    "## Street network within 0.25km of the lat-lon point\n",
    "for i in range(len(new['lat'])):\n",
    "    try:\n",
    "        G = ox.graph_from_point((new['lat'][i], new['lon'][i]), distance=250, network_type='walk')\n",
    "        stats = ox.basic_stats(G)\n",
    "        intersection_count.append(stats['intersection_count'])\n",
    "        streets_per_node_avg.append(stats['streets_per_node_avg'])\n",
    "        circuity_avg.append(stats['circuity_avg'])\n",
    "        street_length_avg.append(stats['street_length_avg'])\n",
    "    except:\n",
    "        intersection_count.append(0)\n",
    "        streets_per_node_avg.append(0)\n",
    "        circuity_avg.append(0)\n",
    "        street_length_avg.append(0)\n",
    "        \n",
    "new['intersection_count'] = intersection_count\n",
    "new['streets_per_node_avg'] = streets_per_node_avg\n",
    "new['circuity_avg'] = circuity_avg\n",
    "new['street_length_avg'] = street_length_avg  "
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "apikey = apikey\n",
    "\n",
    "def location_api(df, amenity, radius, apikey):\n",
    "    amenity_count = [] \n",
    "    for i in range(len(df)):\n",
    "        count = 0    \n",
    "        url = 'https://us1.locationiq.com/v1/nearby.php?key='+apikey+'&lat='+str(df['lat'][i])+'&lon='+str(df['lon'][i])+'&tag='+amenity+'&radius='+radius+'&format=json'\n",
    "        r = requests.get(url)\n",
    "        data = r.json()\n",
    "        if 'error' in data:\n",
    "            amenity_count.append(0)\n",
    "            time.sleep(0.62)\n",
    "        else:   \n",
    "            count = len(data)\n",
    "            amenity_count.append(count)\n",
    "            time.sleep(0.75)\n",
    "\n",
    "    df[amenity+'_count'] = amenity_count\n",
    "    #df[amenity+'wide_count'] = amenity_count\n",
    "    \n",
    "    return df"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "new.info()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "new = location_api(new, 'restaurant', '1000', private_token);\n",
    "new = location_api(new, 'school', '1000', private_token);\n",
    "new = location_api(new, 'park', '1000', private_token);\n",
    "new = location_api(new, 'bus_station', '10000', private_token);\n",
    "new = location_api(new, 'supermarket', '1000', private_token);\n",
    "new = location_api(new, 'pub', '1000', private_token);"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "apikey=apikey\n",
    "\n",
    "def location_api(df, amenity, radius, apikey):\n",
    "    amenity_count = [] \n",
    "    for i in range(len(df)):\n",
    "        count = 0    \n",
    "        url = 'https://us1.locationiq.com/v1/nearby.php?key='+apikey+'&lat='+str(df['lat'][i])+'&lon='+str(df['lon'][i])+'&tag='+amenity+'&radius='+radius+'&format=json'\n",
    "        r = requests.get(url)\n",
    "        data = r.json()\n",
    "        if 'error' in data:\n",
    "            amenity_count.append(0)\n",
    "            time.sleep(0.62)\n",
    "        else:   \n",
    "            count = len(data)\n",
    "            amenity_count.append(count)\n",
    "            time.sleep(0.75)\n",
    "\n",
    "    #df[amenity+'_count'] = amenity_count\n",
    "    df[amenity+'wide_count'] = amenity_count\n",
    "    \n",
    "    return df"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "new = location_api(new, 'park', '2500', private_token);\n",
    "new = location_api(new, 'restaurant', '1500', private_token);"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "new.info()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "new.to_csv('Data/master_walk_score.csv')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "new.shape"
   ]
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 3",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.7.0"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 2
}