python/ons_charts.ipynb

{
 "cells": [
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# ONS Charts\n",
    "\n",
    "Created by Michael George (AKA Logiqx)\n",
    "\n",
    "Website: https://logiqx.github.io/covid-stats/"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## Imports\n",
    "\n",
    "Standard python libraries plus determination of projdir, basic printable class, etc"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 1,
   "metadata": {},
   "outputs": [],
   "source": [
    "import os\n",
    "from datetime import date, datetime, timedelta\n",
    "import calendar\n",
    "\n",
    "import numpy as np\n",
    "import matplotlib.pyplot as plt\n",
    "import matplotlib.ticker as tck\n",
    "\n",
    "import common_core\n",
    "import ons_core"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## NumPy Helper Functions\n",
    "\n",
    "Useful functionality such as moving average or rolling sum"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 2,
   "metadata": {},
   "outputs": [],
   "source": [
    "def calculateAverages(cache, areaName, avgYears, verbose=common_core.verbose):\n",
    "    \"\"\"Calculate X year averages and ranges\"\"\"\n",
    "\n",
    "    tmpMin = np.array([], dtype=\"u4\")\n",
    "    tmpMax = np.array([], dtype=\"u4\")\n",
    "    tmpAvg = np.array([])\n",
    "\n",
    "    yearIdx = np.where(cache[areaName][ons_core.WEEK_NUMBER] == 1)[0]\n",
    "\n",
    "    for i in range(avgYears, len(yearIdx)):\n",
    "\n",
    "        if i == len(yearIdx) - 1:\n",
    "            numWeeks = len(cache[areaName]) - yearIdx[i]\n",
    "        else:\n",
    "            numWeeks = yearIdx[i + 1] - yearIdx[i]\n",
    "\n",
    "        grid = cache[areaName][yearIdx[i - 5]:yearIdx[i - 5] + numWeeks][ons_core.TOTAL_OCCURRENCES]\n",
    "        for j in range(avgYears - 1, 0, -1):\n",
    "            grid = np.vstack([\n",
    "                grid,\n",
    "                cache[areaName][yearIdx[i - j]:yearIdx[i - j] + numWeeks][ons_core.TOTAL_OCCURRENCES]\n",
    "            ])\n",
    "\n",
    "        tmpMin = np.append(tmpMin, np.min(grid, axis = 0))\n",
    "        tmpMax = np.append(tmpMax, np.max(grid, axis = 0))\n",
    "        tmpAvg = np.append(tmpAvg, np.average(grid, axis = 0))\n",
    "\n",
    "    minimums = np.concatenate((np.zeros(yearIdx[avgYears], dtype=\"u4\"), tmpMin))\n",
    "    maximums = np.concatenate((np.zeros(yearIdx[avgYears], dtype=\"u4\"), tmpMax))\n",
    "    averages = np.concatenate((np.zeros(yearIdx[avgYears]), tmpAvg))\n",
    "\n",
    "    return minimums, maximums, averages"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## Plot Data\n",
    "\n",
    "Simple plots of ONS data"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 3,
   "metadata": {},
   "outputs": [],
   "source": [
    "SOURCE_ONS = \"Source: Office for National Statistics\"\n",
    "\n",
    "GITHUB_URL = \"https://logiqx.github.io/covid-stats/weekly-deaths\""
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 4,
   "metadata": {},
   "outputs": [],
   "source": [
    "def updateAxisLabels(ax, xTickLabels, xMin, xMax, yMin, yMax):\n",
    "    '''Update axis limits and labels'''\n",
    "\n",
    "    # Determine x-axis tick interval\n",
    "    tickInterval = len(xTickLabels) // 52\n",
    "    if tickInterval == 0:\n",
    "        tickInterval = 1\n",
    "\n",
    "    # Switch to an interval of 13 or 26 weeks over long time periods \n",
    "    if tickInterval > 8:\n",
    "        tickInterval = 26\n",
    "    elif tickInterval > 4:\n",
    "        tickInterval = 13\n",
    "\n",
    "    # Update x-axis to use desired interval\n",
    "    ax.set_xlim(xmin=xMin - (xMin - ax.get_xlim()[0]) / 5, xmax=xMax + (ax.get_xlim()[1] - xMax) / 5)\n",
    "    ax.set_xticks(np.arange(0, len(xTickLabels), tickInterval))\n",
    "    \n",
    "    # Change the x-axis to shown the actual dates\n",
    "    ax.set_xticklabels(xTickLabels[::tickInterval], rotation=90)\n",
    "\n",
    "    # Determine y-axis tick interval\n",
    "    yMax *= 1.1\n",
    "    if yMax > 10000:\n",
    "        tickInterval = yMax // 10000 * 1000\n",
    "    elif yMax > 1000:\n",
    "        tickInterval = yMax // 1000 * 100\n",
    "    else:\n",
    "        tickInterval = yMax // 10\n",
    "\n",
    "    # Update y-axis to use desired interval\n",
    "    ax.set_ylim(ymin=0, ymax=yMax)\n",
    "    ax.set_yticks(np.arange(0, yMax, tickInterval))\n",
    "    \n",
    "    # Ensure thousands are shown using commas\n",
    "    ax.get_yaxis().set_major_formatter(tck.FuncFormatter(lambda x, p: format(int(x), ',')))\n",
    "\n",
    "    # Assume all charts show weekly deaths\n",
    "    ax.set_ylabel('Number of weekly deaths')\n",
    "\n",
    "    \n",
    "def addTextLegend(ax, legendLoc=\"upper left\"):\n",
    "    \"\"\"Add text boxes and legend to chart\"\"\"\n",
    "    \n",
    "    textStr = f'Created {datetime.now().strftime(\"%-d %b %y\")}\\n@Mike_aka_Logiqx'\n",
    "\n",
    "    if legendLoc == \"upper left\":\n",
    "        ax.text(0.01, 0.04, textStr, transform=ax.transAxes,\n",
    "                horizontalalignment='left', verticalalignment='bottom', fontsize=\"small\")\n",
    "    else:\n",
    "        ax.text(0.01, 0.98, textStr, transform=ax.transAxes,\n",
    "                horizontalalignment='left', verticalalignment='top', fontsize=\"small\")\n",
    "\n",
    "    ax.legend(loc=legendLoc, borderaxespad=1, fontsize=\"small\")"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 5,
   "metadata": {},
   "outputs": [],
   "source": [
    "def plotHistory(cache, minimums, maximums, averages, avgYears, areaName, ax, verbose=common_core.verbose):\n",
    "    '''Plot data for visual inspection'''\n",
    "    \n",
    "    areaData = cache[areaName]\n",
    "\n",
    "    startDate = \"2010-01-08\"\n",
    "    startIdx = np.where(areaData[ons_core.WEEK_ENDED] == startDate)[0][0]\n",
    "    stopIdx = np.where(areaData[ons_core.TOTAL_REGISTRATIONS] > 0)[0][-1]\n",
    "\n",
    "    patchDate = \"2020-08-07\"\n",
    "    patchIdx = np.where(areaData[ons_core.WEEK_ENDED] == patchDate)[0][0]\n",
    "\n",
    "    # Total occurrences\n",
    "    y_points = areaData[ons_core.TOTAL_OCCURRENCES][startIdx:stopIdx]\n",
    "    x_points = np.arange(len(y_points))       \n",
    "    xMin, xMax = min(x_points), max(x_points)\n",
    "    yMin, yMax = min(y_points), max(y_points)\n",
    "    ax.plot(x_points, y_points, label = \"Total deaths - all causes\", color=\"navy\", zorder=4)\n",
    "\n",
    "    # Average of past X years\n",
    "    y_points = averages[startIdx:stopIdx]\n",
    "    x_points = np.arange(len(y_points))       \n",
    "    ax.plot(x_points, y_points, label = f\"{avgYears} year average/mean\", color=\"navy\", linestyle=\"dotted\")\n",
    "\n",
    "    # Average of 2015-2019\n",
    "    y_points = averages[patchIdx - 53:stopIdx - 53]\n",
    "    x_points = np.arange(patchIdx - startIdx, patchIdx - startIdx + len(y_points))\n",
    "    ax.plot(x_points, y_points, label = f\"{2020 - avgYears}-2019 average/mean\", color=\"black\", linestyle=\"dotted\")\n",
    "\n",
    "    # Max + min of past X years\n",
    "    y1_points = minimums[startIdx:stopIdx]\n",
    "    y2_points = maximums[startIdx:stopIdx]\n",
    "    x_points = np.arange(len(y1_points))       \n",
    "    ax.fill_between(x_points, y1_points, y2_points, label = f\"{avgYears} year max/min/range\", color=\"lavender\")\n",
    "\n",
    "    # Max + min of 2015-2019\n",
    "    y1_points = minimums[patchIdx - 53:stopIdx - 53]\n",
    "    y2_points = maximums[patchIdx - 53:stopIdx - 53]\n",
    "    x_points = np.arange(patchIdx - startIdx, patchIdx - startIdx + len(y1_points))\n",
    "    ax.fill_between(x_points, y1_points, y2_points, label = f\"{2020 - avgYears}-2019 max/min/range\", color=\"lightgray\")\n",
    "\n",
    "    # Chart titles and labels\n",
    "    ax.set_title(f\"Weekly Deaths in {areaName} since January 2010\")\n",
    "\n",
    "    # Add note about date of occurrence\n",
    "    lastDate = datetime.strptime(areaData[ons_core.WEEK_ENDED][stopIdx - 1], \"%Y-%m-%d\").strftime(\"%A %-d %B %Y\")\n",
    "    textStr = f'All weekly figures are based on date of occurrence\\n\\n' + \\\n",
    "                f'Showing deaths occurring up to {lastDate}'\n",
    "    ax.text(0.5, 0.98, textStr, transform=ax.transAxes, horizontalalignment='center', verticalalignment='top')\n",
    "\n",
    "    # Add note about dates on axis\n",
    "    textStr = 'Note: Date labels on the x-axis are 26 weeks apart'\n",
    "    ax.text(0.5, 0.04, textStr, transform=ax.transAxes, horizontalalignment='center', verticalalignment='bottom')\n",
    "\n",
    "    # Source can go in a different place, depending on the chart\n",
    "    ax.text(0.99, 0.04, SOURCE_ONS, transform=ax.transAxes,\n",
    "            horizontalalignment='right', verticalalignment='bottom', fontsize=\"small\")\n",
    "\n",
    "    # Update the x-axis and y-axis\n",
    "    xTickLabels = []\n",
    "    for weekEnded in areaData[ons_core.WEEK_ENDED][startIdx:stopIdx]:\n",
    "        formattedDate = datetime.strptime(weekEnded, \"%Y-%m-%d\").strftime(\"%-d %b %y\")\n",
    "        xTickLabels.append(formattedDate)\n",
    "    updateAxisLabels(ax, xTickLabels, xMin, xMax, yMin, yMax)\n",
    "\n",
    "    # Add text and legend\n",
    "    addTextLegend(ax)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 6,
   "metadata": {},
   "outputs": [],
   "source": [
    "def plotLatest(cache, covid, minimums, maximums, averages, avgYears, areaName, ax, verbose=common_core.verbose):\n",
    "    '''Plot data for visual inspection'''\n",
    "    \n",
    "    areaData = cache[areaName]\n",
    "\n",
    "    startDate = \"2020-01-03\"\n",
    "    startIdx = np.where(areaData[ons_core.WEEK_ENDED] == startDate)[0][0]\n",
    "    stopIdx = np.where(areaData[ons_core.TOTAL_REGISTRATIONS] > 0)[0][-1]\n",
    "\n",
    "    patchDate = \"2020-08-07\"\n",
    "    patchIdx = np.where(areaData[ons_core.WEEK_ENDED] == patchDate)[0][0]\n",
    "\n",
    "    covidStartIdx = np.where(covid[ons_core.WEEK_ENDED] == startDate)[0][0]\n",
    "    covidStopIdx = np.where(covid[ons_core.COVID_REGISTRATIONS] > 0)[0][-1] + 1\n",
    "    \n",
    "    # Determine ratios where COVID-19 is the underlying cause\n",
    "    covidUnderlying = covid[covidStartIdx:covidStopIdx][ons_core.COVID_UNDERLYING].astype(np.float64)\n",
    "    covidRegistrations = covid[covidStartIdx:covidStopIdx][ons_core.COVID_REGISTRATIONS].astype(np.float64)\n",
    "    underlying = np.divide(covidUnderlying, covidRegistrations,\n",
    "                           out=np.zeros_like(covidUnderlying), where=covidRegistrations != 0)\n",
    "\n",
    "    xMin = yMin = stopIdx\n",
    "    xMax = yMax = 0\n",
    "\n",
    "    # Total occurrences\n",
    "    y_points = areaData[ons_core.TOTAL_OCCURRENCES][startIdx:stopIdx]\n",
    "    x_points = np.arange(len(y_points))       \n",
    "    xMin, xMax = min(x_points), max(x_points)\n",
    "    yMin, yMax = min(y_points), max(y_points)\n",
    "    if y_points[0] > yMax * 0.6:\n",
    "        legendLoc = \"lower left\"\n",
    "    else:\n",
    "        legendLoc = \"upper left\"\n",
    "    ax.plot(x_points, y_points, label = \"Total deaths - all causes\", color=\"navy\", zorder=4)\n",
    "\n",
    "    # COVID-19 underlying\n",
    "    y_points = areaData[ons_core.COVID_OCCURRENCES][startIdx:stopIdx] * underlying\n",
    "    x_points = np.arange(len(y_points))       \n",
    "    ax.plot(x_points, y_points, label = r\"COVID-19 underlying cause $^{1}$\", color=\"red\", zorder=3)\n",
    "\n",
    "    # COVID-19 mentioned\n",
    "    y_points = areaData[ons_core.COVID_OCCURRENCES][startIdx:stopIdx]\n",
    "    x_points = np.arange(len(y_points))       \n",
    "    ax.plot(x_points, y_points, label = r\"COVID-19 mentioned on cert $^{2}$\", color=\"darkorange\", zorder=2)\n",
    "\n",
    "    # COVID-19 not underlying\n",
    "    y_points = areaData[ons_core.TOTAL_OCCURRENCES][startIdx:stopIdx] - \\\n",
    "                areaData[ons_core.COVID_OCCURRENCES][startIdx:stopIdx] * underlying\n",
    "    x_points = np.arange(len(y_points))       \n",
    "    ax.plot(x_points, y_points, label = \"COVID-19 not underlying\", color=\"royalblue\", zorder=3)\n",
    "\n",
    "    # COVID-19 not mentioned\n",
    "    y_points = areaData[ons_core.TOTAL_OCCURRENCES][startIdx:stopIdx] - \\\n",
    "                areaData[ons_core.COVID_OCCURRENCES][startIdx:stopIdx]\n",
    "    x_points = np.arange(len(y_points))       \n",
    "    ax.plot(x_points, y_points, label = \"COVID-19 not mentioned\", color=\"deepskyblue\", zorder=2)\n",
    "\n",
    "    # Average of past X years\n",
    "    y_points = averages[startIdx:stopIdx]\n",
    "    x_points = np.arange(len(y_points))       \n",
    "    ax.plot(x_points, y_points, label = f\"{avgYears} year average/mean\", color=\"navy\", linestyle=\"dotted\")\n",
    "\n",
    "    # Average of 2015-2019\n",
    "    y_points = averages[patchIdx - 53:stopIdx - 53]\n",
    "    x_points = np.arange(patchIdx - startIdx, patchIdx - startIdx + len(y_points))\n",
    "    ax.plot(x_points, y_points, label = f\"{2020 - avgYears}-2019 average/mean\", color=\"black\", linestyle=\"dotted\")\n",
    "\n",
    "    # Max + min of past X years\n",
    "    y1_points = minimums[startIdx:stopIdx]\n",
    "    y2_points = maximums[startIdx:stopIdx]\n",
    "    x_points = np.arange(len(y1_points))       \n",
    "    ax.fill_between(x_points, y1_points, y2_points, label = f\"{avgYears} year max/min/range\", color=\"lavender\")\n",
    "\n",
    "    # Max + min of 2015-2019\n",
    "    y1_points = minimums[patchIdx - 53:stopIdx - 53]\n",
    "    y2_points = maximums[patchIdx - 53:stopIdx - 53]\n",
    "    x_points = np.arange(patchIdx - startIdx, patchIdx - startIdx + len(y1_points))\n",
    "    ax.fill_between(x_points, y1_points, y2_points, label = f\"{2020 - avgYears}-2019 max/min/range\", color=\"lightgray\")\n",
    "\n",
    "    # Chart titles and labels\n",
    "    ax.set_title(f\"Weekly Deaths in {areaName} since January 2020\")\n",
    "\n",
    "    # Add note about date of occurrence\n",
    "    lastDate = datetime.strptime(areaData[ons_core.WEEK_ENDED][stopIdx - 1], \"%Y-%m-%d\").strftime(\"%A %-d %B %Y\")\n",
    "    textStr = f'All weekly figures are based on date of occurrence\\n\\nShowing deaths occurring up to {lastDate}'\n",
    "    ax.text(0.5, 0.98, textStr, transform=ax.transAxes, horizontalalignment='center', verticalalignment='top')\n",
    "\n",
    "    # Source can go in a different place, depending on the chart\n",
    "    ax.text(0.99, 0.98, SOURCE_ONS, transform=ax.transAxes,\n",
    "            horizontalalignment='right', verticalalignment='top', fontsize=\"small\")\n",
    "\n",
    "    # Update the x-axis and y-axis\n",
    "    xTickLabels = []\n",
    "    for weekEnded in areaData[ons_core.WEEK_ENDED][startIdx:stopIdx]:\n",
    "        formattedDate = datetime.strptime(weekEnded, \"%Y-%m-%d\").strftime(\"%-d %b %y\")\n",
    "        xTickLabels.append(formattedDate)\n",
    "    updateAxisLabels(ax, xTickLabels, xMin, xMax, yMin, yMax)\n",
    "\n",
    "    # Add text and legend\n",
    "    addTextLegend(ax, legendLoc=legendLoc)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 7,
   "metadata": {},
   "outputs": [],
   "source": [
    "def plotDelays(cache, areaName, ax, verbose=common_core.verbose):\n",
    "    '''Plot data for visual inspection'''\n",
    "    \n",
    "    fields = [\n",
    "        {\n",
    "            \"name\": ons_core.TOTAL_OCCURRENCES,\n",
    "            \"label\": \"Total deaths (date of death)\",\n",
    "            \"color\": \"navy\",\n",
    "            \"linestyle\": \"solid\",\n",
    "            \"extra\" : 0\n",
    "        },\n",
    "        {\n",
    "            \"name\": ons_core.TOTAL_REGISTRATIONS,\n",
    "            \"label\": \"Total deaths (date of registration)\",\n",
    "            \"color\": \"cornflowerblue\",\n",
    "            \"linestyle\": \"dotted\",\n",
    "            \"extra\" : 1\n",
    "        },\n",
    "        {\n",
    "            \"name\": ons_core.COVID_OCCURRENCES,\n",
    "            \"label\": r\"COVID-19 mentioned (occurrences) $^{2}$\",\n",
    "            \"color\": \"red\",\n",
    "            \"linestyle\": \"solid\",\n",
    "            \"extra\" : 0\n",
    "        },\n",
    "        {\n",
    "            \"name\": ons_core.COVID_REGISTRATIONS,\n",
    "            \"label\": r\"COVID-19 mentioned (registrations) $^{2}$\",\n",
    "            \"color\": \"lightcoral\",\n",
    "            \"linestyle\": \"dotted\",\n",
    "            \"extra\" : 1\n",
    "        }\n",
    "    ]\n",
    "    \n",
    "    areaData = cache[areaName]\n",
    "\n",
    "    startDate = \"2020-01-03\"\n",
    "    startIdx = np.where(areaData[ons_core.WEEK_ENDED] == startDate)[0][0]\n",
    "    stopIdx = np.where(areaData[ons_core.TOTAL_REGISTRATIONS] > 0)[0][-1]\n",
    "\n",
    "    xMin = yMin = stopIdx\n",
    "    xMax = yMax = 0\n",
    "    \n",
    "    # Counts direct from the ONS\n",
    "    for field in fields:\n",
    "        y_points = areaData[field[\"name\"]][startIdx:stopIdx + field[\"extra\"]]\n",
    "        x_points = np.arange(len(y_points))       \n",
    "        ax.plot(x_points, y_points, label = field[\"label\"], color=field[\"color\"], linestyle=field[\"linestyle\"])\n",
    "        if len(x_points) > 0:\n",
    "            xMin = min(xMin, min(x_points))\n",
    "            yMin = min(yMin, min(y_points))\n",
    "            xMax = max(xMax, max(x_points))\n",
    "            yMax = max(yMax, max(y_points))\n",
    "\n",
    "    # Chart titles and labels\n",
    "    ax.set_title(f\"Comparison of Date of Death vs Date of Registration in {areaName}\")\n",
    "\n",
    "    # Add note about date of occurrence\n",
    "    lastDate = datetime.strptime(areaData[ons_core.WEEK_ENDED][stopIdx], \"%Y-%m-%d\").strftime(\"%A %-d %B %Y\")\n",
    "    textStr = f'This shows the difference between date of occurrence (date of death) and date of registration\\n\\n' + \\\n",
    "                f'Showing deaths registered up to {lastDate}'\n",
    "    ax.text(0.5, 0.98, textStr, transform=ax.transAxes, horizontalalignment='center', verticalalignment='top')\n",
    "\n",
    "    # Source can go in a different place, depending on the chart\n",
    "    ax.text(0.99, 0.98, SOURCE_ONS, transform=ax.transAxes,\n",
    "            horizontalalignment='right', verticalalignment='top', fontsize=\"small\")\n",
    "\n",
    "    # Update the x-axis and y-axis\n",
    "    xTickLabels = []\n",
    "    for weekEnded in areaData[ons_core.WEEK_ENDED][startIdx:stopIdx + 1]:\n",
    "        formattedDate = datetime.strptime(weekEnded, \"%Y-%m-%d\").strftime(\"%-d %b %y\")\n",
    "        xTickLabels.append(formattedDate)\n",
    "    updateAxisLabels(ax, xTickLabels, xMin, xMax, yMin, yMax)\n",
    "\n",
    "    # Add text and legend\n",
    "    addTextLegend(ax)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 8,
   "metadata": {},
   "outputs": [],
   "source": [
    "def plotPolar(cache, areaName, fig, verbose=common_core.verbose):\n",
    "    '''Plot data as polar plot - aka Florence Nightale diagram'''\n",
    "\n",
    "    # Purely for convenience\n",
    "    areaData = cache[areaName]\n",
    "\n",
    "    # Determine date range\n",
    "    startDate = \"2001-01-05\"\n",
    "    startIdx = np.where(areaData[ons_core.WEEK_ENDED] == startDate)[0][0]\n",
    "    stopIdx = np.where(areaData[ons_core.TOTAL_REGISTRATIONS] > 0)[0][-1]\n",
    "\n",
    "    # Data for the specific date range\n",
    "    occurrences = areaData[ons_core.TOTAL_OCCURRENCES][startIdx:stopIdx]\n",
    "    weekEndeds = areaData[ons_core.WEEK_ENDED][startIdx:stopIdx]\n",
    "    weekNumbers = areaData[ons_core.WEEK_NUMBER][startIdx:stopIdx]\n",
    "    weekOffsets = np.where(weekNumbers == 1)[0]\n",
    "\n",
    "    # Pre-claculate theta values for polar plot\n",
    "    thetas = []\n",
    "    currYear = None\n",
    "    yDays = 365\n",
    "    for weekEnded in weekEndeds:\n",
    "        if weekEnded[:4] != currYear:\n",
    "            currYear = int(weekEnded[:4])\n",
    "            yDays = 365\n",
    "            if calendar.isleap(currYear):\n",
    "                yDays += 1\n",
    "                \n",
    "        yDay = datetime.strptime(weekEnded, \"%Y-%m-%d\").timetuple().tm_yday - 1\n",
    "        thetas.append(2 * np.pi * yDay / yDays)\n",
    "\n",
    "    # Keep track of maximum y-value\n",
    "    yMax = 0\n",
    "    \n",
    "    # Create subplot and attach to figure\n",
    "    ax = plt.subplot(111, projection='polar')\n",
    "    fig.add_subplot(ax)\n",
    "\n",
    "    # Choose colour cycler\n",
    "    ax.set_prop_cycle(plt.cycler(\"color\", plt.cm.viridis(np.linspace(1, 0, len(weekOffsets)))))\n",
    "\n",
    "    # Plot the years individually but connecting them up\n",
    "    for i in range(len(weekOffsets)):\n",
    "        year = int(weekEndeds[weekOffsets[i]][:4])\n",
    "\n",
    "        if i == len(weekOffsets) - 1:\n",
    "            y_points = occurrences[weekOffsets[-1] - 1:]\n",
    "            theta = thetas[weekOffsets[-1] - 1:]\n",
    "        elif i > 0:\n",
    "            y_points = occurrences[weekOffsets[i] - 1:weekOffsets[i + 1]]\n",
    "            theta = thetas[weekOffsets[i] - 1:weekOffsets[i + 1]]\n",
    "        else:\n",
    "            y_points = occurrences[weekOffsets[i]:weekOffsets[i + 1] + 1]\n",
    "            theta = thetas[weekOffsets[i]:weekOffsets[i + 1] + 1]\n",
    "            \n",
    "        # Update y-max\n",
    "        yMax = max(yMax, max(y_points))\n",
    "\n",
    "        # Plot the data\n",
    "        ax.plot(theta, y_points, label=year)\n",
    "   \n",
    "    # Consider 1 Jan to be \"north\" and move around the plot clockwise\n",
    "    ax.set_theta_zero_location(\"N\")\n",
    "    ax.set_theta_direction(-1)\n",
    "\n",
    "    # Calculate desired interval of radial ticks\n",
    "    for i in [10000, 1000, 100]:\n",
    "        if yMax > i:\n",
    "            tickInterval = (yMax // i) * i / 10\n",
    "            if yMax % tickInterval < tickInterval / 2:\n",
    "                yMax = (yMax // tickInterval + 1) * tickInterval\n",
    "            else:\n",
    "                yMax = (yMax // tickInterval + 2) * tickInterval\n",
    "            break\n",
    "\n",
    "    # Update y-axis to use desired interval\n",
    "    ax.set_rmin(0)\n",
    "    ax.set_rmax(yMax)\n",
    "    ax.set_rticks(np.arange(0, yMax, tickInterval))\n",
    "    ax.get_yaxis().set_major_formatter(tck.FuncFormatter(lambda x, p: format(int(x), ',')))\n",
    "    \n",
    "    # Angle radial labels towards NWbN / 11 o'clock / 1 Nov\n",
    "    ax.set_rlabel_position(-30)  \n",
    "    #ax.grid(True)\n",
    "\n",
    "    # Set theta labels to the 1st of each month\n",
    "    angles = []\n",
    "    labels = []\n",
    "    for i in range(1, 13):\n",
    "        dt = date(date.today().year, i, 1)\n",
    "        angles.append((dt.timetuple().tm_yday - 1) / 365.4 * 360)\n",
    "        labels.append(dt.strftime(\"1 %b\"))\n",
    "    ax.set_thetagrids(angles, labels=labels)\n",
    "\n",
    "    # Add note about date of occurrence\n",
    "    textStr = 'All weekly figures are based on date of occurrence'\n",
    "    fig.text(0.5, 0.95, textStr, horizontalalignment='center', verticalalignment='top')\n",
    "    \n",
    "    lastDate = datetime.strptime(areaData[ons_core.WEEK_ENDED][stopIdx - 1], \"%Y-%m-%d\").strftime(\"%A %-d %B %Y\")\n",
    "    textStr = f'Showing deaths occurring up to {lastDate}'\n",
    "    fig.text(0.5, 0.93, textStr, horizontalalignment='center', verticalalignment='top')\n",
    "\n",
    "    # Source can go in a different place, depending on the chart\n",
    "    fig.text(0.96, 0.98, SOURCE_ONS, horizontalalignment='right', verticalalignment='top', fontsize=\"small\")\n",
    "\n",
    "    # Add text and legend\n",
    "    textStr = f'Created {datetime.now().strftime(\"%-d %b %y\")} by @Mike_aka_Logiqx'\n",
    "    fig.text(0.96, 0.10, textStr, horizontalalignment='right', verticalalignment='bottom', fontsize=\"small\")\n",
    "\n",
    "    ax.legend(loc='upper left', fontsize=\"small\", bbox_to_anchor=(1, 1.1))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 9,
   "metadata": {},
   "outputs": [],
   "source": [
    "def legacy(cache):\n",
    "    \"\"\"Date ranges for some legacy testing\"\"\"\n",
    "    \n",
    "    # Check heatwave of August 2003 - https://www.eurosurveillance.org/content/10.2807/esm.10.07.00558-en\n",
    "    startDate = '2003-07-01'\n",
    "    stopDate = '2003-09-01'\n",
    "\n",
    "    # Show reg delays\n",
    "    startDate = date(2010, 1, 1).strftime('%Y-%m-%d')\n",
    "    stopDate = date(2020, 1, 8)\n",
    "\n",
    "    # Same range as error check\n",
    "    startDate = '2014-01-01'\n",
    "    stopDate = '2018-12-31'\n",
    "\n",
    "\n",
    "def plotAreas(verbose=common_core.verbose):\n",
    "    '''Plot data for visual inspection'''\n",
    "\n",
    "    # Load all of the data from disk\n",
    "    cache = ons_core.loadCsvFiles(ons_core.ONS_DEATHS, \"weekly\", verbose=False)\n",
    "    covid = ons_core.loadCsvFile(ons_core.ONS_DEATHS, \"weekly\", \"nation\", \"england_wales_covid\", verbose=common_core.verbose)\n",
    "    \n",
    "    # Shift COVID-19 registrations left so they can be used for occurrences\n",
    "    covid[ons_core.COVID_REGISTRATIONS] = ons_core.shiftRegistrations(covid[ons_core.COVID_REGISTRATIONS])\n",
    "    covid[ons_core.COVID_UNDERLYING] = ons_core.shiftRegistrations(covid[ons_core.COVID_UNDERLYING])\n",
    "\n",
    "    # Figures for each area\n",
    "    for areaName in common_core.nationNames + common_core.regionNames:\n",
    "        if areaName in cache:\n",
    "            \n",
    "            fig, axs = plt.subplots(3, figsize=(16, 18), dpi=100)\n",
    "\n",
    "            avgYears=5\n",
    "            minimums, maximums, averages = calculateAverages(cache, areaName, avgYears, verbose=verbose)\n",
    "\n",
    "            plotHistory(cache, minimums, maximums, averages, avgYears, areaName, axs[0], verbose=verbose)\n",
    "            plotLatest(cache, covid, minimums, maximums, averages, avgYears, areaName, axs[1], verbose=verbose)\n",
    "            plotDelays(cache, areaName, axs[2], verbose=verbose)\n",
    "\n",
    "            ucodStr = r\"$^{1}$ The underlying cause of death is defined as the disease or injury which initiated \" + \\\n",
    "                       \"the chain of morbid events leading directly to death.\"\n",
    "            mentionStr = r\"$^{2}$ COVID-19 may be mentioned in part I (conditions leading to death) or part II \" + \\\n",
    "                          \"(significant conditions contributing to death) of the medical certificate cause of death (MCCD).\"\n",
    "\n",
    "            gitHubStr = \"\\nA detailed explanation along with a link to the code and data \" + \\\n",
    "                         f\"that were used to create these charts can be found at {GITHUB_URL}\"\n",
    "            \n",
    "            textStr = \"\\n\".join([ucodStr, mentionStr, gitHubStr])\n",
    "            fig.text(0.5, 0.02, textStr, horizontalalignment='center', verticalalignment='bottom')\n",
    "\n",
    "            plt.subplots_adjust(hspace=0.4)\n",
    "            plt.show()\n",
    "\n",
    "            partName = os.path.join(\"docs\", \"weekly-deaths\", common_core.getSafeName(areaName) + \".png\")\n",
    "            fileName = os.path.join(common_core.projdir, partName)\n",
    "\n",
    "            print(f\"Saving {partName}...\")\n",
    "            fig.savefig(fileName, bbox_inches='tight', facecolor='w')\n",
    "            plt.close(fig)\n",
    "            \n",
    "    # Figures for multiple areas (combined) - 5 and 10 year averages\n",
    "    areaGroups = \\\n",
    "    [\n",
    "        [\"London\", \"South East\", \"East of England\"],\n",
    "        [\"North West\", \"North East\", \"Yorkshire and The Humber\"],\n",
    "        [\"West Midlands\", \"East Midlands\", \"South West\"],\n",
    "        [\"England\", \"Wales\", \"England and Wales\"]\n",
    "    ]\n",
    "    \n",
    "    groupNo = 1\n",
    "    for areaNames in areaGroups:\n",
    "        for avgYears in (5, 10):\n",
    "\n",
    "            fig, axs = plt.subplots(len(areaNames), figsize=(16, 6 * len(areaNames)), dpi=100)\n",
    "\n",
    "            axsIdx = 0\n",
    "            for areaName in areaNames:\n",
    "                minimums, maximums, averages = calculateAverages(cache, areaName, avgYears, verbose=verbose)\n",
    "                plotLatest(cache, covid, minimums, maximums, averages, avgYears, areaName, axs[axsIdx], verbose=verbose)\n",
    "                axsIdx += 1\n",
    "\n",
    "            ucodStr = r\"$^{1}$ The underlying cause of death is defined as the disease or injury which initiated \" + \\\n",
    "                       \"the chain of morbid events leading directly to death.\"\n",
    "            mentionStr = r\"$^{2}$ COVID-19 may be mentioned in part I (conditions leading to death) or part II \" + \\\n",
    "                          \"(significant conditions contributing to death) of the medical certificate cause of death (MCCD).\"\n",
    "\n",
    "            gitHubStr = \"\\nA detailed explanation along with a link to the code and data \" + \\\n",
    "                         f\"that were used to create these charts can be found at {GITHUB_URL}\"\n",
    "\n",
    "            textStr = \"\\n\".join([ucodStr, mentionStr, gitHubStr])\n",
    "            fig.text(0.5, 0.02, textStr, horizontalalignment='center', verticalalignment='bottom')\n",
    "\n",
    "            plt.subplots_adjust(hspace=0.4)\n",
    "            plt.show()\n",
    "\n",
    "            partName = os.path.join(\"docs\", \"weekly-deaths\", f\"{avgYears}_years_{groupNo}\" + \".png\")\n",
    "            fileName = os.path.join(common_core.projdir, partName)\n",
    "\n",
    "            print(f\"Saving {partName}...\")\n",
    "            fig.savefig(fileName, bbox_inches='tight', facecolor='w')\n",
    "            plt.close(fig)\n",
    "\n",
    "        groupNo += 1\n",
    "            \n",
    "    # Florence Nightingale diagrams for each year\n",
    "    for areaName in common_core.nationNames + common_core.regionNames:\n",
    "        if areaName in cache:\n",
    "            \n",
    "            fig = plt.figure(figsize=(12, 12), dpi=100)\n",
    "           \n",
    "            plotPolar(cache, areaName, fig, verbose=verbose)\n",
    "\n",
    "            fig.suptitle(f\"Weekly Deaths in {areaName} since January 2001\", fontsize=\"x-large\", fontweight=\"bold\")\n",
    "\n",
    "            gitHubStr = f\"A detailed explanation along with a link to the code and data can be found at {GITHUB_URL}\"\n",
    "            \n",
    "            fig.text(0.5, 0.08, gitHubStr, horizontalalignment='center', verticalalignment='top')\n",
    "\n",
    "            plt.subplots_adjust(hspace=0.4)\n",
    "            fig.show()\n",
    "\n",
    "            partName = os.path.join(\"docs\", \"weekly-deaths\", common_core.getSafeName(areaName) + \"_polar.png\")\n",
    "            fileName = os.path.join(common_core.projdir, partName)\n",
    "\n",
    "            print(f\"Saving {partName}...\")\n",
    "            fig.savefig(fileName, bbox_inches='tight', facecolor='w')\n",
    "            plt.close(fig)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 10,
   "metadata": {},
   "outputs": [],
   "source": [
    "def saveIndexPage(h2_text, suffix=\"\"):\n",
    "    \"\"\"Save HTML page as index to images\"\"\"\n",
    "\n",
    "    indexPageDetails = \\\n",
    "    {\n",
    "        \"TITLE_TEXT\": \"Weekly Deaths\",\n",
    "        \"H1_TEXT\": \"Weekly Deaths in England and Wales\",\n",
    "        \"H2_TEXT\": h2_text,\n",
    "        \"CUSTOM_CSS\": \"css/custom.css\",\n",
    "        \"FURTHER_HREF\": \".\"\n",
    "    }\n",
    "    fileName = os.path.join(common_core.projdir, \"docs\", \"weekly-deaths\", \"regions\" + suffix + \".html\")\n",
    "    indexPage = common_core.IndexPage(fileName, indexPageDetails)\n",
    "\n",
    "    nationNames = [common_core.ENGLAND, common_core.WALES, common_core.ENGLAND_WALES]\n",
    "\n",
    "    galleryDetails = \\\n",
    "    {\n",
    "        \"P_TEXT\": \", \".join(nationNames)\n",
    "    }\n",
    "    gallery = indexPage.addGallery(galleryDetails[\"P_TEXT\"], galleryDetails)\n",
    "\n",
    "    for nationName in nationNames:\n",
    "        fileName = os.path.join(common_core.projdir, \"docs\", \"weekly-deaths\",\n",
    "                                common_core.getSafeName(nationName) + suffix + \".png\")\n",
    "        \n",
    "        figureDetails = \\\n",
    "        {\n",
    "            \"ALT_TEXT\": nationName,\n",
    "            \"CAPTION_TEXT\": nationName\n",
    "        }\n",
    "        figure = gallery.addFigure(nationName, \".\", fileName, figureDetails)\n",
    "        figure.createThumb(suffix=\"_thumb\")\n",
    "\n",
    "    regionGroups = \\\n",
    "    [\n",
    "        [\"London\", \"South East\", \"East of England\"],\n",
    "        [\"North West\", \"North East\", \"Yorkshire and The Humber\"],\n",
    "        [\"East Midlands\", \"West Midlands\", \"South West\"]\n",
    "    ]\n",
    "\n",
    "    for regionNames in regionGroups:\n",
    "        galleryDetails = \\\n",
    "        {\n",
    "            \"P_TEXT\": \", \".join(regionNames)\n",
    "        }\n",
    "        gallery = indexPage.addGallery(galleryDetails[\"P_TEXT\"], galleryDetails)\n",
    "\n",
    "        for regionName in regionNames:\n",
    "            fileName = os.path.join(common_core.projdir, \"docs\", \"weekly-deaths\",\n",
    "                                    common_core.getSafeName(regionName) + suffix + \".png\")\n",
    "\n",
    "            figureDetails = \\\n",
    "            {\n",
    "                \"ALT_TEXT\": regionName,\n",
    "                \"CAPTION_TEXT\": regionName\n",
    "            }\n",
    "            figure = gallery.addFigure(regionName, \".\", fileName, figureDetails)\n",
    "            figure.createThumb(suffix=\"_thumb\")\n",
    "\n",
    "    html = indexPage.saveHtml()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 11,
   "metadata": {},
   "outputs": [],
   "source": [
    "def saveAltIndexPage(h2_text, suffix=\"\"):\n",
    "    \"\"\"Save HTML page as index to images\"\"\"\n",
    "\n",
    "    indexPageDetails = \\\n",
    "    {\n",
    "        \"TITLE_TEXT\": \"Weekly Deaths\",\n",
    "        \"H1_TEXT\": \"Weekly Deaths in England and Wales\",\n",
    "        \"H2_TEXT\": h2_text,\n",
    "        \"CUSTOM_CSS\": \"css/alt.css\",\n",
    "        \"FURTHER_HREF\": \".\"\n",
    "    }\n",
    "    fileName = os.path.join(common_core.projdir, \"docs\", \"weekly-deaths\", \"regions\" + suffix + \".html\")\n",
    "    indexPage = common_core.IndexPage(fileName, indexPageDetails)\n",
    "\n",
    "    for numYears in [5, 10]:\n",
    "\n",
    "        baseName = f\"{numYears}_years\"\n",
    "        galleryDetails = \\\n",
    "        {\n",
    "            \"P_TEXT\": f\"Comparing to {numYears} Average\"\n",
    "        }\n",
    "        gallery = indexPage.addGallery(galleryDetails[\"P_TEXT\"], galleryDetails)\n",
    "\n",
    "        for imgNo in range(1, 5):\n",
    "            fileName = os.path.join(common_core.projdir, \"docs\", \"weekly-deaths\", f\"{numYears}_years_{imgNo}.png\")\n",
    "        \n",
    "            figureName = f\"Group {imgNo}\"\n",
    "            figureDetails = \\\n",
    "            {\n",
    "                \"ALT_TEXT\": figureName,\n",
    "                \"CAPTION_TEXT\": figureName\n",
    "            }\n",
    "            figure = gallery.addFigure(figureName, \".\", fileName, figureDetails)\n",
    "            figure.createThumb(suffix=\"_thumb\")\n",
    "\n",
    "    html = indexPage.saveHtml()"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## Interactive Testing"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 12,
   "metadata": {
    "scrolled": false
   },
   "outputs": [],
   "source": [
    "if __name__ == '__main__':\n",
    "\n",
    "    plotAreas(verbose=False)\n",
    "    \n",
    "    saveIndexPage(\"All-cause Mortality from 2010-2021\")\n",
    "    saveIndexPage(\"Radar / Polar Charts for 2001-2021\", suffix=\"_polar\")\n",
    "    saveAltIndexPage(\"All-cause mortality during 2020-2021\", suffix=\"_alt\")"
   ]
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 3",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.8.8"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 2
}