PyConDe 2018 - Interactive Visualization of Traffic Data.ipynb

{
 "cells": [
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "**Basic Imports**"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "ExecuteTime": {
     "end_time": "2018-10-17T13:10:57.106000Z",
     "start_time": "2018-10-17T13:10:55.355000Z"
    }
   },
   "outputs": [],
   "source": [
    "import geopandas as gpd\n",
    "import pandas as pd\n",
    "import dask.dataframe as dd\n",
    "\n",
    "import numpy as np\n",
    "from shapely.geometry import Polygon, MultiPolygon\n",
    "from IPython.display import display, display_html\n",
    "%matplotlib inline\n",
    "\n",
    "def display_side_by_side(*args):\n",
    "    html_str=''\n",
    "    for df in args:\n",
    "        html_str+= \"&emsp;\" + df.to_html()\n",
    "    display_html(html_str.replace('table','table style=\"display:inline\"'),raw=True)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# Download Data\n",
    "\n",
    "To download all neccessary data for this notebook, please run the following cell:"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "ExecuteTime": {
     "end_time": "2018-10-17T13:01:30.497000Z",
     "start_time": "2018-10-17T12:41:03.166000Z"
    }
   },
   "outputs": [],
   "source": [
    "import os\n",
    "import requests\n",
    "import os\n",
    "from multiprocessing.dummy import Pool\n",
    "from functools import partial\n",
    "import zipfile\n",
    "\n",
    "def download_file(url, export_directory=\"\", filename=None):\n",
    "    print(\"Download file %s to directory %s\"%(url, export_directory))\n",
    "    if filename ==  None:\n",
    "        local_filename = os.path.join(export_directory,url.split('/')[-1])\n",
    "    else:\n",
    "        local_filename = os.path.join(export_directory,filename)\n",
    "    # NOTE the stream=True parameter\n",
    "    r = requests.get(url, stream=True)\n",
    "    with open(local_filename, 'wb') as f:\n",
    "        for chunk in r.iter_content(chunk_size=1024):\n",
    "            if chunk: # filter out keep-alive new chunks\n",
    "                f.write(chunk)\n",
    "                #f.flush() commented by recommendation from J.F.Sebastian\n",
    "    return local_filename\n",
    "\n",
    "#Create Folders for Data:\n",
    "if not os.path.exists(\"Data\"):\n",
    "    os.mkdir(\"Data\")\n",
    "\n",
    "if not os.path.exists(os.path.join(\"Data\",\"New York Taxi\")):\n",
    "    os.mkdir(os.path.join(\"Data\",\"New York Taxi\"))\n",
    "\n",
    "if not os.path.exists(os.path.join(\"Data\",\"OSM GPX\")):\n",
    "    os.mkdir(os.path.join(\"Data\",\"OSM GPX\"))\n",
    "\n",
    "#Download New York Taxi Data for Yellow Cabs in 2017:\n",
    "download_taxi_data = partial(download_file, export_directory=os.path.join(\"Data\",\"New York Taxi\"))\n",
    "request_strings = [r\"https://s3.amazonaws.com/nyc-tlc/trip+data/yellow_tripdata_2017-%02d.csv\"%i for i in range(1,13)]\n",
    "pool = Pool(processes=4)\n",
    "pool.map(download_taxi_data, request_strings)\n",
    "\n",
    "#Download New York Taxi Data for Green Cabs in 2017:\n",
    "download_taxi_data = partial(download_file, export_directory=os.path.join(\"Data\",\"New York Taxi\"))\n",
    "request_strings = [r\"https://s3.amazonaws.com/nyc-tlc/trip+data/green_tripdata_2017-%02d.csv\"%i for i in range(1,13)]\n",
    "pool = Pool(processes=4)\n",
    "pool.map(download_taxi_data, request_strings)\n",
    "\n",
    "#Download shapefile of New York Taxi Zones:\n",
    "print(\"Download Taxi Zones Shapefiles.\")\n",
    "zip_path = download_taxi_data(r\"https://s3.amazonaws.com/nyc-tlc/misc/taxi_zones.zip\")\n",
    "print(\"Unzip Taxi Zones Shapefiles.\")\n",
    "with zipfile.ZipFile(zip_path, \"r\") as z:\n",
    "    z.extractall(os.path.dirname(zip_path))\n",
    "os.remove(zip_path)\n",
    "\n",
    "#Download OSM path data:\n",
    "osm_path = download_file(r\"https://ptv2box.ptvgroup.com/index.php/s/9sTUmxdF80NU2nr/download\", export_directory=os.path.join(\"Data\",\"OSM GPX\"), filename=\"OSM_GPX.parquet\")"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# Interactive Maps for New York Taxi Data 2017"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## New York Taxi zones"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {
    "ExecuteTime": {
     "end_time": "2018-08-16T18:44:00.299000Z",
     "start_time": "2018-08-16T18:44:00.292000Z"
    }
   },
   "source": [
    "In this section, we will load an process the shapefile **\"taxi_zones.shp\"** containing the New York Taxis Zones, such that we can use them as a basis for the **Bokeh** plot. In the next cell, we use GeoPandas to load the shapefile of Taxi zones and transform coordinate system to Web Mercador (<a href=\"http://spatialreference.org/ref/sr-org/epsg3857-wgs84-web-mercator-auxiliary-sphere/\">EPSG-Code</a> of Web Mercador Projection = 3785 ):"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "ExecuteTime": {
     "end_time": "2018-10-17T13:03:10.140000Z",
     "start_time": "2018-10-17T13:03:09.309000Z"
    }
   },
   "outputs": [],
   "source": [
    "df_taxizones = gpd.read_file(r\"Data\\New York Taxi\\taxi_zones.shp\")\n",
    "df_taxizones.to_crs(epsg=3785, inplace=True)     #EPSG-Code of Web Mercador\n",
    "display(df_taxizones.head())\n",
    "print(\"Number of Polygons: %d\"%len(df_taxizones))\n",
    "df_taxizones.plot()"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "Simplify Shape of Zones (otherwise slow peformance of plot):"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "ExecuteTime": {
     "end_time": "2018-10-17T13:03:10.610000Z",
     "start_time": "2018-10-17T13:03:10.327000Z"
    }
   },
   "outputs": [],
   "source": [
    "df_taxizones[\"geometry\"] = df_taxizones[\"geometry\"].simplify(100)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "Convert WKT Polygons to X and Y arrays with corresponding coordinates. Take into account **multipolygons** and separate them into individual shapes:"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "ExecuteTime": {
     "end_time": "2018-10-17T13:03:11.224000Z",
     "start_time": "2018-10-17T13:03:11.149000Z"
    }
   },
   "outputs": [],
   "source": [
    "data = []\n",
    "for zonename, LocationID, borough, shape in df_taxizones[[\"zone\", \"LocationID\", \"borough\", \"geometry\"]].values:\n",
    "    #If shape is polygon, extract X and Y coordinates of boundary line:\n",
    "    if isinstance(shape, Polygon):\n",
    "        X, Y = shape.boundary.xy\n",
    "        X = [int(x) for x in X]\n",
    "        Y = [int(y) for y in Y]\n",
    "        data.append([LocationID, zonename, borough, X, Y])\n",
    "        \n",
    "    #If shape is Multipolygon, extract X and Y coordinates of each sub-Polygon:\n",
    "    if isinstance(shape, MultiPolygon):\n",
    "        for poly in shape:\n",
    "            X, Y = poly.boundary.xy\n",
    "            X = [int(x) for x in X]\n",
    "            Y = [int(y) for y in Y]\n",
    "            data.append([LocationID, zonename, borough, X, Y])\n",
    "\n",
    "#Create new DataFrame with X an Y coordinates separated:\n",
    "df_taxizones = pd.DataFrame(data, columns=[\"LocationID\", \"ZoneName\", \"Borough\", \"X\", \"Y\"])\n",
    "display(df_taxizones.head())\n",
    "print(\"Number of Polygons: %d\"%len(df_taxizones))"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## New York Taxi Data"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "Here, we read in the data from New York Taxis and aggregate them to show us information about how frequent each taxi zone is visited. We will use Dask.DataFrame, such that the whole dataset can be loaded at once without blowing up our memory and to get a nice speedup due to the parallelizm of Dask. Let us start a Dask Client and Local Cluster (after the execution of the cell, click on the **Dashboard Link** to view the Dask Dashboard, where you can see the resource consumption of our computation"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "ExecuteTime": {
     "end_time": "2018-10-17T13:03:18.902000Z",
     "start_time": "2018-10-17T13:03:12.732000Z"
    }
   },
   "outputs": [],
   "source": [
    "from dask.distributed import Client, LocalCluster\n",
    "import dask.dataframe as dd\n",
    "from dask import compute\n",
    "\n",
    "cluster = LocalCluster()\n",
    "print(cluster)\n",
    "client = Client(cluster)\n",
    "client"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "Using Dask, we now read in the New York Taxi Data for the yellow and green cabs into  distributed DataFrames. Note: A **Dask.DataFrame** is a *delayed object* and to calculate results, one has to trigger the computation via the **.compute()** method."
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "ExecuteTime": {
     "end_time": "2018-10-17T13:03:22.022000Z",
     "start_time": "2018-10-17T13:03:18.924000Z"
    },
    "code_folding": []
   },
   "outputs": [],
   "source": [
    "df_taxis_yellow = dd.read_csv(r\"Data\\New York Taxi\\yellow_tripdata_2017-*.csv\", \n",
    "                       usecols=[\"tpep_pickup_datetime\", \"tpep_dropoff_datetime\", \"passenger_count\",\n",
    "                               \"PULocationID\", \"DOLocationID\"],\n",
    "                       parse_dates=[\"tpep_pickup_datetime\", \"tpep_dropoff_datetime\"])\n",
    "df_taxis_green = dd.read_csv(r\"Data\\New York Taxi\\green_tripdata_2017-*.csv\", \n",
    "                       usecols=[\"lpep_pickup_datetime\", \"lpep_dropoff_datetime\", \"passenger_count\",\n",
    "                               \"PULocationID\", \"DOLocationID\"],\n",
    "                       parse_dates=[\"lpep_pickup_datetime\", \"lpep_dropoff_datetime\"]).rename(\n",
    "                        columns = {\"lpep_pickup_datetime\": \"tpep_pickup_datetime\", \n",
    "                                   \"lpep_dropoff_datetime\": \"tpep_dropoff_datetime\"})\n",
    "df_taxis = dd.concat([df_taxis_yellow, df_taxis_green])\n",
    "\n",
    "#Filter data for correct year :\n",
    "df_taxis = df_taxis[(df_taxis[\"tpep_pickup_datetime\"].dt.year == 2017)&(df_taxis[\"tpep_dropoff_datetime\"].dt.year == 2017)]\n",
    "\n",
    "df_taxis.head()"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "Now, we calculate the number of boarding and alighting passengers for each zone and every hour and every day using the **GroupBy** Method. For this, we first create two columns specifying the hour and daytype of Pickup and Dropoff. Then, we define the groupby operations for Pickups and Dropoffs: and finally we trigger the parallelized computation using **dask.compute**:"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "ExecuteTime": {
     "end_time": "2018-10-17T13:05:52.951000Z",
     "start_time": "2018-10-17T13:03:22.026000Z"
    }
   },
   "outputs": [],
   "source": [
    "df_taxis[\"Pickup_Hour\"] = df_taxis[\"tpep_pickup_datetime\"].dt.hour\n",
    "df_taxis[\"Dropoff_Hour\"] = df_taxis[\"tpep_dropoff_datetime\"].dt.hour\n",
    "df_taxis[\"weekday\"] = df_taxis[\"tpep_dropoff_datetime\"].dt.weekday\n",
    "pickups = df_taxis.groupby(by=[\"Pickup_Hour\", \"weekday\", \"PULocationID\"])[\"passenger_count\"].sum()\n",
    "dropoffs = df_taxis.groupby(by=[\"Dropoff_Hour\", \"weekday\", \"DOLocationID\"])[\"passenger_count\"].sum()\n",
    "pickups, dropoffs = compute(pickups, dropoffs)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "**Time Series of Pickups and Dropoffs**"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {
    "ExecuteTime": {
     "end_time": "2018-08-17T07:49:12.765000Z",
     "start_time": "2018-08-17T07:49:12.758000Z"
    }
   },
   "source": [
    "Aggregate Pickups and Dropoffs hourly:"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "ExecuteTime": {
     "end_time": "2018-10-17T13:05:53.019000Z",
     "start_time": "2018-10-17T13:05:52.959000Z"
    }
   },
   "outputs": [],
   "source": [
    "df_pudo = pd.DataFrame(pickups.groupby(level=0).sum())\n",
    "df_pudo[\"Dropoff\"] = dropoffs.groupby(level=0).sum()\n",
    "df_pudo.columns = [\"P\", \"D\"]\n",
    "df_pudo.index.rename(\"Hour\", inplace=True)\n",
    "df_pudo.head(3)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {
    "ExecuteTime": {
     "end_time": "2018-08-17T07:49:27.619000Z",
     "start_time": "2018-08-17T07:49:27.613000Z"
    }
   },
   "source": [
    "Plot with Holoviews (Backend: Bokeh)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "ExecuteTime": {
     "end_time": "2018-10-17T13:05:55.310000Z",
     "start_time": "2018-10-17T13:05:53.027000Z"
    }
   },
   "outputs": [],
   "source": [
    "import holoviews as hv\n",
    "hv.extension('bokeh')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "ExecuteTime": {
     "end_time": "2018-10-17T13:05:55.642000Z",
     "start_time": "2018-10-17T13:05:55.316000Z"
    }
   },
   "outputs": [],
   "source": [
    "%%opts Bars.Grouped [group_index='Group' toolbar='above' tools=['hover'] width=800]\n",
    "from itertools import product\n",
    "hours, groups = df_pudo.index.values, ['P', 'D']\n",
    "keys = product(hours, groups)\n",
    "bars = hv.Bars([(hour, pudo, df_pudo.loc[hour, pudo]) for hour, pudo in keys],\n",
    "               ['Hour', \"Group\"], \"Passengers\")\n",
    "bars.relabel(group='Grouped')"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {
    "ExecuteTime": {
     "end_time": "2018-08-17T03:26:27.375000Z",
     "start_time": "2018-08-17T03:26:27.329000Z"
    }
   },
   "source": [
    "**Finally, join the Taxi Zones DataFrame with the information about the Pickups and Dropoffs for every hour and weekday:**"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "ExecuteTime": {
     "end_time": "2018-10-17T13:05:55.680000Z",
     "start_time": "2018-10-17T13:05:55.646000Z"
    }
   },
   "outputs": [],
   "source": [
    "display_side_by_side(pd.DataFrame(pickups).head(), df_taxizones[[\"LocationID\", \"ZoneName\", \"X\"]].head())"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "ExecuteTime": {
     "end_time": "2018-10-17T13:06:00.090000Z",
     "start_time": "2018-10-17T13:05:55.685000Z"
    }
   },
   "outputs": [],
   "source": [
    "pickups = pd.DataFrame(pickups)\n",
    "dropoffs = pd.DataFrame(dropoffs)\n",
    "\n",
    "for hour in range(24):\n",
    "    \n",
    "    for weekday in range(7):\n",
    "    \n",
    "        #Get pickups and dropoff for this hour and weekday:\n",
    "        p = pd.DataFrame(pickups.loc[(hour, weekday)]).reset_index().rename(columns={\"PULocationID\" : \"LocationID\"})\n",
    "        d = pd.DataFrame(dropoffs.loc[(hour, weekday)]).reset_index().rename(columns={\"DOLocationID\" : \"LocationID\"})\n",
    "\n",
    "        #Add information of pickups and dropoff to the New York Taxi Zone DataFrame:\n",
    "        df_taxizones = pd.merge(df_taxizones, p, on=\"LocationID\", how=\"left\").fillna(0)\n",
    "        df_taxizones.rename(columns={\"passenger_count\" : \"PU_Passenger_%d_%d\"%(weekday, hour)}, inplace=True) \n",
    "        df_taxizones = pd.merge(df_taxizones, d, on=\"LocationID\", how=\"left\").fillna(0)\n",
    "        df_taxizones.rename(columns={\"passenger_count\" : \"DO_Passenger_%d_%d\"%(weekday, hour)}, inplace=True)\n",
    "        \n",
    "df_taxizones.head(2)      "
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {
    "ExecuteTime": {
     "end_time": "2018-08-17T03:42:13.741000Z",
     "start_time": "2018-08-17T03:42:13.094000Z"
    }
   },
   "source": [
    "## Plot Interactive Demand Map using Bokeh"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### Draw Taxi Zones on Map"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "Bokeh Imports"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "ExecuteTime": {
     "end_time": "2018-10-17T13:06:14.479000Z",
     "start_time": "2018-10-17T13:06:14.455000Z"
    }
   },
   "outputs": [],
   "source": [
    "from bokeh.io import output_notebook, output_file, show\n",
    "from bokeh.plotting import figure\n",
    "from bokeh.models import HoverTool, Select, ColumnDataSource, WheelZoomTool, LogColorMapper, LinearColorMapper\n",
    "from bokeh.palettes import OrRd9 as palette\n",
    "output_notebook()"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "Define Source for Plot. Bokeh, like its high-level API Holoviews, can convert dicts and DataFrames to a ColumnDataSource. Its columns can than be used to specify, what should be plotted."
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "ExecuteTime": {
     "end_time": "2018-10-17T13:06:15.514000Z",
     "start_time": "2018-10-17T13:06:15.373000Z"
    }
   },
   "outputs": [],
   "source": [
    "df_taxizones[\"Passengers\"] = df_taxizones[\"PU_Passenger_0_7\"]\n",
    "source = ColumnDataSource(df_taxizones)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {
    "ExecuteTime": {
     "end_time": "2018-08-17T05:40:44.239000Z",
     "start_time": "2018-08-17T05:40:44.235000Z"
    }
   },
   "source": [
    "Define Colormapper for zones"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "ExecuteTime": {
     "end_time": "2018-10-17T13:06:16.274000Z",
     "start_time": "2018-10-17T13:06:16.263000Z"
    }
   },
   "outputs": [],
   "source": [
    "max_passengers_per_hour = df_taxizones[filter(lambda x: \"Passenger_\" in x, df_taxizones.columns)].max().max()\n",
    "color_mapper = LinearColorMapper(palette=palette[::-1], high=max_passengers_per_hour, low=0)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "Define Figure"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "ExecuteTime": {
     "end_time": "2018-10-17T13:06:17.188000Z",
     "start_time": "2018-10-17T13:06:17.172000Z"
    }
   },
   "outputs": [],
   "source": [
    "p = figure(title=\"Titel\",\n",
    "           plot_width=900, plot_height=450,\n",
    "           toolbar_location=None,\n",
    "           tools=\"pan,wheel_zoom,box_zoom,reset,save\",\n",
    "           active_scroll=\"wheel_zoom\")\n",
    "p.xaxis.visible = False\n",
    "p.yaxis.visible = False\n",
    "\n",
    "#Get rid of zoom on axes:\n",
    "for t in p.tools:\n",
    "    if type(t) == WheelZoomTool:\n",
    "        t.zoom_on_axis = False"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "Add Background Map (Custom Tile-Maps: http://geo.holoviews.org/Working_with_Bokeh.html)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "ExecuteTime": {
     "end_time": "2018-10-17T13:06:18.125000Z",
     "start_time": "2018-10-17T13:06:18.116000Z"
    }
   },
   "outputs": [],
   "source": [
    "from bokeh.models import WMTSTileSource\n",
    "\n",
    "#Use OpenStreetMap Tiles:\n",
    "tiles = WMTSTileSource(url='http://c.tile.openstreetmap.org/{Z}/{X}/{Y}.png')\n",
    "\n",
    "#Add Tile Layer and set alpha-value:\n",
    "tile_layer = p.add_tile(tiles)\n",
    "tile_layer.alpha = 0.6"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "Draw Taxi Zone Polygons on the Map. Pass the ColumnDataSource as **source**, such you can use the column names to pass data to the renderer. We use the **Passengers** column to draw a <a href=\"https://en.wikipedia.org/wiki/Choropleth_map\">Choropleth map</a>. "
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "ExecuteTime": {
     "end_time": "2018-10-17T13:06:19.284000Z",
     "start_time": "2018-10-17T13:06:19.091000Z"
    }
   },
   "outputs": [],
   "source": [
    "patches = p.patches(xs=\"X\", ys=\"Y\", source=source,\n",
    "                  fill_color={'field': 'Passengers', 'transform': color_mapper},\n",
    "                  line_color=\"black\", alpha=0.5)\n",
    "show(p)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "Add the Hovertool to show data of each zone (the attributes of the selected zone can be accessed by the **@** key): "
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "ExecuteTime": {
     "end_time": "2018-10-17T13:06:20.274000Z",
     "start_time": "2018-10-17T13:06:20.076000Z"
    }
   },
   "outputs": [],
   "source": [
    "#Add Hover Tool:\n",
    "hovertool = HoverTool(tooltips=[(\"Passengers:\", \"@Passengers\")])\n",
    "p.add_tools(hovertool)\n",
    "\n",
    "show(p)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "Add more advanced Hover Tools via HTML:"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "ExecuteTime": {
     "end_time": "2018-10-17T13:06:21.543000Z",
     "start_time": "2018-10-17T13:06:21.303000Z"
    }
   },
   "outputs": [],
   "source": [
    "#Add Hovertool via HTML:\n",
    "hovertool = HoverTool(tooltips=\"\"\"\n",
    "<head>\n",
    "<style>\n",
    "#demo {\n",
    "background-color:  rgba(255,255,255, .9);  \n",
    "-moz-box-shadow: inset 0 0 15px 5px rgba(200,200,200, .9);\n",
    "-webkit-box-shadow: inset 0 0 15px 5px rgba(200,200,200, .9);\n",
    "box-shadow: inset 0 0 15px 5px rgba(200,200,200, .9);\n",
    "border-radius: 5px;\n",
    "-moz-border-radius: 5px;\n",
    "-webkit-border-radius: 5px;\n",
    "}\n",
    "</style>\n",
    "</head>\n",
    "\n",
    "\n",
    "<div id=\"demo\">\n",
    "<h2 style=\"text-align: center;\">@ZoneName</h2>\n",
    "<h3 style=\"text-align: center;\">@Borough</h3>\n",
    "<h4 style=\"text-align: center;\">@Passengers Passengers</h4>\n",
    "<p><img style=\"width: 100px; height: auto; display: block; margin-left: auto; margin-right: auto;\"src=\"https://pbs.twimg.com/profile_images/426051037577760768/2d7K9aJc_400x400.jpeg\" alt=\"Country Flag\" /></p>\n",
    "</div>\"\"\")\n",
    "p.add_tools(hovertool)\n",
    "\n",
    "show(p)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {
    "ExecuteTime": {
     "end_time": "2018-08-17T06:56:44.888000Z",
     "start_time": "2018-08-17T06:56:44.204000Z"
    }
   },
   "source": [
    "### Add Interactivity"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "Add Slider widget for selecting the hour of the day:"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "ExecuteTime": {
     "end_time": "2018-10-17T13:06:23.436000Z",
     "start_time": "2018-10-17T13:06:23.388000Z"
    }
   },
   "outputs": [],
   "source": [
    "from bokeh.models.widgets import Slider\n",
    "\n",
    "slider = Slider(start=0, end=23, value=7, step=1, title=\"Hour\", width=600)\n",
    "\n",
    "show(slider)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "Add RadioButton widgets for selecting (Pickups/Dropoffs) and the weekday:"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "ExecuteTime": {
     "end_time": "2018-10-17T13:06:24.291000Z",
     "start_time": "2018-10-17T13:06:24.261000Z"
    }
   },
   "outputs": [],
   "source": [
    "from bokeh.models.widgets import RadioButtonGroup, Div\n",
    "from bokeh.layouts import column, row\n",
    "\n",
    "radiobuttons_weekday = RadioButtonGroup(\n",
    "    labels=[\"Mon\", \"Tue\", \"Wed\", \"Thu\", \"Fri\", \"Sat\", \"Sun\"],\n",
    "    active=0,\n",
    "    width=400)\n",
    "\n",
    "radiobuttons_pudo = RadioButtonGroup(\n",
    "        labels=[\"Pickups\", \"Dropoff\"], active=0)\n",
    "\n",
    "layout_widgets = column(slider, row(radiobuttons_weekday, Div(width=80), radiobuttons_pudo))\n",
    "\n",
    "show(layout_widgets)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "Add Interaction via JavaScript Callback:"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "ExecuteTime": {
     "end_time": "2018-10-17T13:06:25.460000Z",
     "start_time": "2018-10-17T13:06:25.211000Z"
    }
   },
   "outputs": [],
   "source": [
    "from bokeh.models.callbacks import CustomJS\n",
    "\n",
    "#Define callback-function with JavaScript Code:\n",
    "callback = CustomJS(args=dict(p=p, source=source, slider=slider, \n",
    "                              radiobuttons_pudo=radiobuttons_pudo, \n",
    "                              radiobuttons_weekday=radiobuttons_weekday),\n",
    "                    code=\"\"\"\n",
    "                    \n",
    "//Get value of slider for hour:\n",
    "var hour = slider.value;\n",
    "\n",
    "//Get value of weekday:\n",
    "var weekday = radiobuttons_weekday.active;\n",
    "\n",
    "//Get value of Pickups/Dropoffs RadioButtons:\n",
    "if (radiobuttons_pudo.active == 0)\n",
    "    var pudo = \"PU\"\n",
    "else\n",
    "    var pudo = \"DO\"\n",
    "\n",
    "//Change data of \"Passengers\" column in data source to passenger data of the selected hour:\n",
    "source.data[\"Passengers\"] = source.data[pudo + \"_Passenger_\" + weekday + \"_\" + hour];\n",
    "source.change.emit();\n",
    "                    \n",
    "                    \"\"\")\n",
    "\n",
    "#Bind Callback to value change of slider and radiobuttons:\n",
    "slider.js_on_change(\"value\", callback)\n",
    "radiobuttons_pudo.js_on_change(\"active\", callback)\n",
    "radiobuttons_weekday.js_on_change(\"active\", callback)\n",
    "\n",
    "#Show all elements:\n",
    "layout = column( layout_widgets , p)\n",
    "show(layout)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "Change to logarithmic Colormapper (to see patterns in zones with low demand):"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "ExecuteTime": {
     "end_time": "2018-10-17T13:06:27.104000Z",
     "start_time": "2018-10-17T13:06:26.861000Z"
    }
   },
   "outputs": [],
   "source": [
    "color_mapper_log = LogColorMapper(palette=palette[::-1], high=max_passengers_per_hour, low=0)\n",
    "patches.glyph.fill_color[\"transform\"] = color_mapper_log\n",
    "show(layout)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### Dropoff and Pickup Zones"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "To clearly see the structure of the Taxi transport flow for each cell, let us look at the difference between Pickups and Dropoffs in a zone. If this is a positive number, more people leave the zone than enter it:"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "ExecuteTime": {
     "end_time": "2018-10-17T13:06:29.382000Z",
     "start_time": "2018-10-17T13:06:29.008000Z"
    }
   },
   "outputs": [],
   "source": [
    "df_pudo = df_taxizones.copy()\n",
    "\n",
    "for hour in range(24):\n",
    "    for weekday in range(7):\n",
    "        df_pudo[\"PUDO_%d_%d\"%(weekday, hour)] = df_pudo[\"PU_Passenger_%d_%d\"%(weekday, hour)] - df_pudo[\"DO_Passenger_%d_%d\"%(weekday, hour)]\n",
    "        df_pudo[\"PUDO_%d_%d\"%(weekday, hour)] = df_pudo[\"PUDO_%d_%d\"%(weekday, hour)].apply(lambda x: \"PU > DO\" if x>0 else \"PU < DO\")\n",
    "df_pudo.drop(columns=filter(lambda x: \"Passenger\" in x, df_pudo.columns), inplace=True)\n",
    "df_pudo[\"PUDO\"] = df_pudo[\"PUDO_0_7\"]\n",
    "df_pudo.head(3)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "ExecuteTime": {
     "end_time": "2018-10-17T13:06:30.464000Z",
     "start_time": "2018-10-17T13:06:29.941000Z"
    }
   },
   "outputs": [],
   "source": [
    "from bokeh.transform import factor_cmap\n",
    "\n",
    "#Define Categorical Color Map for Plot (red=\"PU > DO\", blue=\"PU < DO\"):\n",
    "categorical_cmap = factor_cmap(\"PUDO\", palette=[\"red\", \"blue\"], factors=[\"PU > DO\", \"PU < DO\"] )\n",
    "\n",
    "#Define Source for Plot:\n",
    "source = ColumnDataSource(df_pudo)\n",
    "\n",
    "#Define Figure for Plot:\n",
    "p = figure(title=\"Titel\",\n",
    "           plot_width=900, plot_height=450,\n",
    "           toolbar_location=None,\n",
    "           tools=\"pan,wheel_zoom,box_zoom,reset,save\",\n",
    "           active_scroll=\"wheel_zoom\")\n",
    "p.xaxis.visible = False\n",
    "p.yaxis.visible = False\n",
    "\n",
    "#Get rid of zoom on axes:\n",
    "for t in p.tools:\n",
    "    if type(t) == WheelZoomTool:\n",
    "        t.zoom_on_axis = False\n",
    "\n",
    "#Use OpenStreetMap Tiles:\n",
    "tiles = WMTSTileSource(url='http://c.tile.openstreetmap.org/{Z}/{X}/{Y}.png')\n",
    "\n",
    "#Add Tile Layer and set alpha-value:\n",
    "tile_layer = p.add_tile(tiles)\n",
    "tile_layer.alpha = 0.6\n",
    "\n",
    "patches = p.patches(xs=\"X\", ys=\"Y\", source=source,\n",
    "                  fill_color=categorical_cmap,\n",
    "                  line_color=\"black\", alpha=0.5,\n",
    "                  legend=\"PUDO\")\n",
    "\n",
    "\n",
    "slider = Slider(start=0, end=23, value=7, step=1, title=\"Hour\", width=350)\n",
    "\n",
    "radiobuttons_weekday = RadioButtonGroup(\n",
    "    labels=[\"Mon\", \"Tue\", \"Wed\", \"Thu\", \"Fri\", \"Sat\", \"Sun\"],\n",
    "    active=0,\n",
    "    width=350)\n",
    "\n",
    "#Define callback-function with JavaScript Code:\n",
    "callback = CustomJS(args=dict(p=p, source=source, slider=slider,\n",
    "                              radiobuttons_weekday=radiobuttons_weekday),\n",
    "                    code=\"\"\"\n",
    "                    \n",
    "//Get value of slider for hour:\n",
    "var hour = slider.value;\n",
    "\n",
    "//Get value of weekday:\n",
    "var weekday = radiobuttons_weekday.active;\n",
    "\n",
    "//Change data of \"PUDO\" column in data source to passenger data of the selected hour:\n",
    "source.data[\"PUDO\"] = source.data[\"PUDO_\" + weekday + \"_\" + hour];\n",
    "source.change.emit();\n",
    "\n",
    "                    \"\"\")\n",
    "\n",
    "#Bind Callback to value change of slider and radiobuttons:\n",
    "slider.js_on_change(\"value\", callback)\n",
    "radiobuttons_weekday.js_on_change(\"active\", callback)\n",
    "\n",
    "show(column(row(slider, Div(width=100), radiobuttons_weekday), p))"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {
    "ExecuteTime": {
     "end_time": "2018-08-20T12:07:54.448000Z",
     "start_time": "2018-08-20T12:07:54.400000Z"
    },
    "collapsed": true
   },
   "source": [
    "# OSM GPS Data and Datashader"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## Load and process data"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "Use GPX Dataset for Germany (only identifiable trips) from <a href=\"https://wiki.openstreetmap.org/wiki/Planet.gpx\">OpenStreetMap</a> (**© OpenStreetMap-Mitwirkende**). From the GPX files a final DataFrame (only containing Latitude and Longitude) was created. Let us load the data into the DataFrame: \n",
    "\n",
    "**Note**: The <a href=\"https://www.openstreetmap.org/copyright\">OSM Licence Conditions</a> also apply to the provided data. "
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "ExecuteTime": {
     "end_time": "2018-10-17T13:06:44.773000Z",
     "start_time": "2018-10-17T13:06:36.007000Z"
    }
   },
   "outputs": [],
   "source": [
    "df_osm = pd.read_parquet(\"Data\\OSM GPX\\OSM_GPX.parquet\")\n",
    "display_side_by_side(df_osm.query(\"TripId==1\").head(), df_osm.query(\"TripId==2\").head())"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "Calculate distances between the GPS points of a trip:"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "ExecuteTime": {
     "end_time": "2018-10-17T13:09:02.566000Z",
     "start_time": "2018-10-17T13:08:45.693000Z"
    }
   },
   "outputs": [],
   "source": [
    "def calc_distance_from_lonlat(x1, y1, x2, y2):\n",
    "    \"\"\"Calculates the distance between two points in WGS84 projection (Lat, Lon)\n",
    "       based on a linear approximation around the origin of the first point. Works \n",
    "       well for smaller distances < 100km.   \n",
    "       \"\"\"\n",
    "\n",
    "    return np.sqrt((np.cos(y1 * 2 * np.pi / 360) * (x2 - x1))**2 +\n",
    "                   (y2 - y1)**2) * 6371 * 2 * np.pi / 360\n",
    "\n",
    "#Calculate postlength of every GPS point:\n",
    "df_osm[\"Postlength\"] = calc_distance_from_lonlat(df_osm[\"longitude\"], df_osm[\"latitude\"], df_osm[\"longitude\"].shift(-1), df_osm[\"latitude\"].shift(-1))\n",
    "#Set Postlength of every last element of each trip to NaN:\n",
    "df_osm.loc[df_osm[\"TripId\"]-df_osm[\"TripId\"].shift(-1)!=0, \"Postlength\"] = np.NaN\n",
    "\n",
    "display_side_by_side(df_osm.query(\"TripId==1\").head(), df_osm.query(\"TripId==1\").tail())"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## Basic Plots"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "**Let us quickly create some easy plots with basic statistics about the GPS data**"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "Histogram with number of points per trip:"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "ExecuteTime": {
     "end_time": "2018-10-17T13:09:06.185000Z",
     "start_time": "2018-10-17T13:09:02.571000Z"
    }
   },
   "outputs": [],
   "source": [
    "%%output filename=\"Export/Trip_points\" fig=\"html\"\n",
    "%%opts Histogram [width=500 tools=['hover']]\n",
    "%%opts Histogram (fill_color='red')\n",
    "\n",
    "#Count Number of Points for every trip:\n",
    "trip_points = df_osm.groupby(\"TripId\")[\"latitude\"].count().values\n",
    "#Calculate 90 percent quantile for setting plotting range:\n",
    "quantile_90 = np.percentile(trip_points, 90)\n",
    "\n",
    "#Plot Histogram via Holoviews (output to Data/Trip_points due to the %%magic command in the top of the cell):\n",
    "frequencies, edges = np.histogram(trip_points, np.arange(0, trip_points.max()+100, 100))\n",
    "p_trip_points = hv.Histogram((edges, frequencies), label=\"Number of GPS Points per Trip\")\n",
    "p_trip_points = p_trip_points.redim.label(x=\"Number of Points\", Frequency=\"Trips\")\n",
    "p_trip_points = p_trip_points.redim.range(x=(0,quantile_90))\n",
    "p_trip_points"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "Trip length distribution "
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "ExecuteTime": {
     "end_time": "2018-10-17T13:09:09.302000Z",
     "start_time": "2018-10-17T13:09:06.192000Z"
    }
   },
   "outputs": [],
   "source": [
    "%%output filename=\"Export/Trip_lengths\" fig=\"html\"\n",
    "%%opts Histogram [width=500 tools=['hover']]\n",
    "%%opts Histogram (fill_color='red')\n",
    "\n",
    "#Count Number of Points for every trip:\n",
    "trip_length = df_osm.groupby(\"TripId\")[\"Postlength\"].sum().values\n",
    "#Calculate 90 percent quantile for setting plotting range:\n",
    "quantile_90 = np.percentile(trip_length, 90)\n",
    "\n",
    "#Plot Histogram via Holoviews (output to Data/Trip_points due to the %%magic command in the top of the cell):\n",
    "frequencies, edges = np.histogram(trip_length, np.arange(0, 1000, 1))\n",
    "p_trip_length = hv.Histogram((edges, frequencies), label=\"Trip Distances\")\n",
    "p_trip_length = p_trip_length.redim.label(x=\"Length [km]\", Frequency=\"Trips\")\n",
    "p_trip_length = p_trip_length.redim.range(x=(0,quantile_90))\n",
    "\n",
    "p_trip_length"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## Heatmap with DataShader"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "Add coordinates in Web Mercador:"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "ExecuteTime": {
     "end_time": "2018-10-17T13:09:31.824000Z",
     "start_time": "2018-10-17T13:09:09.314000Z"
    }
   },
   "outputs": [],
   "source": [
    "from pyproj import Proj, transform\n",
    "\n",
    "#Define initial and output Projection (WGS84 and Web-Mercador):\n",
    "inProj = Proj(init='epsg:4326')\n",
    "outProj = Proj(init='epsg:3857')\n",
    "\n",
    "#Add Web Mercador coordinates to columns X, Y\n",
    "df_osm[\"X\"], df_osm[\"Y\"] = transform(inProj, outProj, df_osm[\"longitude\"].values, df_osm[\"latitude\"].values)\n",
    "\n",
    "df_osm.head()"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "Filter data with Germany Bounding-Box:"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "ExecuteTime": {
     "end_time": "2018-10-17T13:09:45.602000Z",
     "start_time": "2018-10-17T13:09:31.828000Z"
    }
   },
   "outputs": [],
   "source": [
    "minlat, maxlat, minlon, maxlon =  47.3, 55, 5.9, 15.1            #Bounding Box Germany\n",
    "#maxlat, minlon, minlat, maxlon =  52.57, 13.25, 52.45, 13.5     #Bounding Box Berlin City\n",
    "df_osm_filtered = df_osm[(df_osm[\"latitude\"]>minlat)&(df_osm[\"latitude\"]<maxlat)&(df_osm[\"longitude\"]>minlon)&(df_osm[\"longitude\"]<maxlon)].copy()"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "Plot Heatmap of GPS Data using DataShader. Underlay heatmap with OSM map using GeoViews:"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "ExecuteTime": {
     "end_time": "2018-10-17T13:09:49.211000Z",
     "start_time": "2018-10-17T13:09:45.636000Z"
    }
   },
   "outputs": [],
   "source": [
    "import holoviews as hv\n",
    "import geoviews as gv\n",
    "from colorcet import fire\n",
    "from holoviews.operation.datashader import datashade\n",
    "hv.extension('bokeh')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "ExecuteTime": {
     "end_time": "2018-10-17T13:09:57.441000Z",
     "start_time": "2018-10-17T13:09:49.217000Z"
    }
   },
   "outputs": [],
   "source": [
    "%%output filename=\"Export//Heatmap\" fig=\"html\"\n",
    "\n",
    "#Define size of image:\n",
    "width = 1000\n",
    "height = 600\n",
    "\n",
    "#Define sampling size of DataShader aggregation:\n",
    "x_sampling = int((df_osm_filtered[\"X\"].max()-df_osm_filtered[\"X\"].min())/width)\n",
    "y_sampling = int((df_osm_filtered[\"Y\"].max()-df_osm_filtered[\"Y\"].min())/height)\n",
    "\n",
    "#Define DataShader Heatmap and Background Map Tiles:\n",
    "tile_opts  = dict(width=width,height=height,xaxis=None,yaxis=None,bgcolor='black',show_grid=False)\n",
    "map_tiles  = gv.WMTS('http://c.tile.openstreetmap.org/{Z}/{X}/{Y}.png').opts(style=dict(alpha=0.5), plot=tile_opts)\n",
    "points     = hv.Points(df_osm_filtered, ['X', 'Y'])\n",
    "trips      = datashade(points, x_sampling=1, y_sampling=1, cmap=fire, width=width, height=width)\n",
    "\n",
    "#Combine DataShader Heatmap and Background Map Tiles:\n",
    "p = map_tiles * trips\n",
    "\n",
    "#Set wheelzoom active\n",
    "def set_active_tool(plot, element):\n",
    "    plot.state.toolbar.active_scroll = plot.state.tools[3]\n",
    "p = p.options(finalize_hooks=[set_active_tool])\n",
    "\n",
    "p"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## Embed Interactive Plots in HTML Template"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "The generated analysis and plots can be either put together like seen in the section about the New York Taxis zones, or one might use an **HTML Template** to embed the plots and data. Here, we use the Jinja package to easily embed the analysis. Let us first create some basic stats for an **Overview Page**:"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "ExecuteTime": {
     "end_time": "2018-10-17T13:10:02.677000Z",
     "start_time": "2018-10-17T13:09:57.446000Z"
    }
   },
   "outputs": [],
   "source": [
    "overview_table = []\n",
    "overview_table.append(dict(key=\"Source\", value='<a href=\"https://wiki.openstreetmap.org/wiki/Planet.gpx\">Open Street Map (Planet GPX)</a>'))\n",
    "overview_table.append(dict(key=\"Minimum Latitude\", value=df_osm[\"latitude\"].min().round(5))) \n",
    "overview_table.append(dict(key=\"Maximum Latitude\", value=df_osm[\"latitude\"].max().round(5))) \n",
    "overview_table.append(dict(key=\"Minimum Longitude\", value=df_osm[\"longitude\"].min().round(5))) \n",
    "overview_table.append(dict(key=\"Maximum Longitude\", value=df_osm[\"longitude\"].max().round(5)))\n",
    "overview_table.append(dict(key=\"Number of Trips\", value=len(df_osm[\"TripId\"].unique()))) \n",
    "overview_table.append(dict(key=\"Number of GPS Points\", value=len(df_osm)))\n",
    "overview_table.append(dict(key=\"Average number of GPS points per trip\", value=len(df_osm)/len(df_osm[\"TripId\"].unique())))\n",
    "overview_table.append(dict(key=\"Average distance per trip\", value=\"%d km\"%trip_length.mean()))\n",
    "\n",
    "overview_table"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "Load Interactive Plots as HTML:"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "ExecuteTime": {
     "end_time": "2018-10-17T13:10:02.700000Z",
     "start_time": "2018-10-17T13:10:02.682000Z"
    }
   },
   "outputs": [],
   "source": [
    "with open(\"Export/Trip_points.html\", \"r\") as f:\n",
    "    p_trips = f.read()\n",
    "with open(\"Export/Trip_lengths.html\", \"r\") as f:\n",
    "    p_lengths = f.read()\n",
    "with open(\"Export/Heatmap.html\", \"r\") as f:\n",
    "    p_heatmap = f.read()"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {
    "ExecuteTime": {
     "end_time": "2018-08-22T09:30:21.266000Z",
     "start_time": "2018-08-22T09:30:21.261000Z"
    }
   },
   "source": [
    "Load HTML Template via Jinja and embed Basic Statistics as a Table and the Interactive Plots:"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "ExecuteTime": {
     "end_time": "2018-10-17T13:10:03.674000Z",
     "start_time": "2018-10-17T13:10:02.705000Z"
    }
   },
   "outputs": [],
   "source": [
    "#Load Template in 'Data/Template.html' via Jinja:\n",
    "from jinja2 import Template\n",
    "with open('Data/Template.html') as f:\n",
    "    template = Template(f.read())\n",
    "\n",
    "#Render template with our analysis and plots:\n",
    "final_html = template.render(overview_table=overview_table, p_trips=p_trips,\n",
    "                            p_lengths=p_lengths, p_heatmap=p_heatmap)\n",
    "\n",
    "#Output final HTML in \"Data/\"Overview.html\":\n",
    "import codecs\n",
    "with codecs.open(\"Export/Overview.html\", \"w\", encoding=\"UTF-8\") as f:\n",
    "    f.write(final_html)     \n",
    "\n",
    "#Open Resulting HTML:\n",
    "import webbrowser, os\n",
    "webbrowser.open(os.path.abspath(\"Export/Overview.html\"));"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# PTV Locations"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "Read in data from CSV"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "ExecuteTime": {
     "end_time": "2018-10-17T13:14:27.864000Z",
     "start_time": "2018-10-17T13:14:27.828000Z"
    }
   },
   "outputs": [],
   "source": [
    "df_locations = pd.read_csv(\"Data/PTV_Locations.csv\", sep=\",\")\n",
    "df_locations.fillna(\"\", inplace=True)\n",
    "df_locations.head()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "ExecuteTime": {
     "end_time": "2018-10-17T13:14:31.587000Z",
     "start_time": "2018-10-17T13:14:30.493000Z"
    }
   },
   "outputs": [],
   "source": [
    "from bokeh.io import output_notebook, show\n",
    "from bokeh.plotting import figure\n",
    "from bokeh.layouts import column\n",
    "from bokeh.models import HoverTool, Select, ColumnDataSource, WMTSTileSource\n",
    "from bokeh.models.callbacks import CustomJS\n",
    "from bokeh.tile_providers import STAMEN_TERRAIN\n",
    "from bokeh.models import WheelZoomTool\n",
    "output_notebook()\n",
    "\n",
    "#Define Figure:\n",
    "p = figure(title=\"PTV Locations\",\n",
    "           plot_width=700, plot_height=450,\n",
    "           x_axis_label='x_axis',\n",
    "           y_axis_label='y_axis',\n",
    "           toolbar_location=None,\n",
    "           tools=\"pan,wheel_zoom,box_zoom,reset,save\",\n",
    "           active_scroll=\"wheel_zoom\")\n",
    "p.xaxis.visible = False\n",
    "p.yaxis.visible = False\n",
    "\n",
    "#Get ridd of zoom on axes:\n",
    "for t in p.tools:\n",
    "    if type(t) == WheelZoomTool:\n",
    "        t.zoom_on_axis = False\n",
    "\n",
    "#Add Background Map:\n",
    "#p.add_tile(STAMEN_TERRAIN)\n",
    "tile = WMTSTileSource(url='https://server.arcgisonline.com/ArcGIS/rest/services/World_Topo_Map/MapServer/tile/{z}/{y}/{x}')\n",
    "p.add_tile(tile)\n",
    "\n",
    "#Define ColumnDataSource:\n",
    "source = ColumnDataSource(df_locations)\n",
    "\n",
    "\n",
    "#Plot PTV Locations on World Map:\n",
    "ax = p.scatter(x=\"X\", y=\"Y\", source=source,\n",
    "               size=10, fill_color=\"red\", line_color=\"black\")\n",
    "\n",
    "#Add Hover Tool:\n",
    "my_hover = HoverTool(attachment=\"below\")\n",
    "css_style = \"\"\"<style>\n",
    "#hover {\n",
    "    font-family: \"Trebuchet MS\", Arial, Helvetica, sans-serif;\n",
    "    border-collapse: collapse;\n",
    "    width = 250px;\n",
    "}\n",
    "\n",
    "#hover td, #hover th {\n",
    "    border: 1px solid #ddd;\n",
    "    text-align: center;\n",
    "    padding: 5px;\n",
    "    width:125px;\n",
    "}\n",
    "\n",
    "#hover tr:nth-child(even){background-color: #f2f2f2;}\n",
    "\n",
    "\n",
    "#hover th {\n",
    "    padding-top: 5px;\n",
    "    padding-bottom: 5px;\n",
    "    text-align: center;\n",
    "    background-color: #b50000;\n",
    "    color: white;\n",
    "    width:125px;\n",
    "}\n",
    "\n",
    "#hover ex {\n",
    "    padding-top: 5px;\n",
    "    padding-bottom: 5px;\n",
    "    text-align: center;\n",
    "    background-color: #b50000;\n",
    "    color: black;\n",
    "    width:125px;\n",
    "}\n",
    "</style>\"\"\"\n",
    "my_hover.tooltips =  css_style + \"\"\"\n",
    "<div style=\"width:500px\"><h2>@Name  <img style=\"margin-right: 35px;float:right;\" src=\"https://www.xing.com/img/custom/cp/assets/logo/b/4/7/11079/square_128px/PTV_Group_10cm_farbig20140304-11381-15uhov8.jpg\" alt=\"\" />  </h2> </div>\n",
    "<h3>@Country </h3>\n",
    "<h4>@City, @Address </h4>\n",
    "<h5>Telefone: @Telefone </h5>\n",
    "<h5>Email: <a href=@Email> @Email </a> </h5>\n",
    "\"\"\"\n",
    "p.add_tools(my_hover)\n",
    "\n",
    "\n",
    "show(p)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "Export components to create HTML content that can be embedded into another DIV:"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "ExecuteTime": {
     "end_time": "2018-10-17T13:14:38.359000Z",
     "start_time": "2018-10-17T13:14:38.323000Z"
    }
   },
   "outputs": [],
   "source": [
    "#Get components of plot:\n",
    "from bokeh.embed import components\n",
    "script, div = components(p)\n",
    "\n",
    "#Define source for Bokeh CSS and JS:\n",
    "import bokeh\n",
    "version = bokeh.__version__\n",
    "source = \"\"\"<link\n",
    "    href=\"https://cdn.pydata.org/bokeh/release/bokeh-<release>.min.css\"\n",
    "    rel=\"stylesheet\" type=\"text/css\">\n",
    "<link\n",
    "    href=\"https://cdn.pydata.org/bokeh/release/bokeh-widgets-<release>.min.css\"\n",
    "    rel=\"stylesheet\" type=\"text/css\">\n",
    "<link\n",
    "    href=\"https://cdn.pydata.org/bokeh/release/bokeh-tables-<release>.min.css\"\n",
    "    rel=\"stylesheet\" type=\"text/css\">\n",
    "\n",
    "<script src=\"https://cdn.pydata.org/bokeh/release/bokeh-<release>.min.js\"></script>\n",
    "<script src=\"https://cdn.pydata.org/bokeh/release/bokeh-widgets-<release>.min.js\"></script>\n",
    "<script src=\"https://cdn.pydata.org/bokeh/release/bokeh-tables-<release>.min.js\"></script>\"\"\".replace(\"<release>\", version)\n",
    "\n",
    "#Export final HTML content:\n",
    "html_content = source + script + div\n",
    "\n",
    "with open(\"Export//PTV_Locations.html\", \"w\") as f:\n",
    "    f.write(html_content)"
   ]
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 2",
   "language": "python",
   "name": "python2"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 2
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython2",
   "version": "2.7.15"
  },
  "toc": {
   "nav_menu": {},
   "number_sections": true,
   "sideBar": true,
   "skip_h1_title": false,
   "title_cell": "Table of Contents",
   "title_sidebar": "Contents",
   "toc_cell": false,
   "toc_position": {
    "height": "calc(100% - 180px)",
    "left": "10px",
    "top": "150px",
    "width": "246px"
   },
   "toc_section_display": true,
   "toc_window_display": true
  }
 },
 "nbformat": 4,
 "nbformat_minor": 2
}