-
-
Notifications
You must be signed in to change notification settings - Fork 323
Commit
This commit does not belong to any branch on this repository, and may belong to a fork outside of the repository.
- Loading branch information
Showing
5 changed files
with
169 additions
and
5 deletions.
There are no files selected for viewing
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
| Original file line number | Diff line number | Diff line change |
|---|---|---|
|
|
@@ -8,3 +8,4 @@ class Sources(str, Enum): | |
|
|
||
| jhu = "jhu" | ||
| csbs = "csbs" | ||
| nyt = "nyt" | ||
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
| Original file line number | Diff line number | Diff line change |
|---|---|---|
| @@ -0,0 +1,32 @@ | ||
| """app.locations.nyt.py""" | ||
| from . import TimelinedLocation | ||
|
|
||
|
|
||
| class NYTLocation(TimelinedLocation): | ||
| """ | ||
| A NYT (county) Timelinedlocation. | ||
| """ | ||
|
|
||
| # pylint: disable=too-many-arguments,redefined-builtin | ||
| def __init__(self, id, state, county, coordinates, last_updated, timelines): | ||
| super().__init__(id, "US", state, coordinates, last_updated, timelines) | ||
|
|
||
| self.state = state | ||
| self.county = county | ||
|
|
||
| def serialize(self, timelines=False): # pylint: disable=arguments-differ,unused-argument | ||
| """ | ||
| Serializes the location into a dict. | ||
| :returns: The serialized location. | ||
| :rtype: dict | ||
| """ | ||
| serialized = super().serialize(timelines) | ||
|
|
||
| # Update with new fields. | ||
| serialized.update( | ||
| {"state": self.state, "county": self.county,} | ||
| ) | ||
|
|
||
| # Return the serialized location. | ||
| return serialized |
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
| Original file line number | Diff line number | Diff line change |
|---|---|---|
| @@ -0,0 +1,126 @@ | ||
| """app.services.location.nyt.py""" | ||
| import csv | ||
| from datetime import datetime | ||
|
|
||
| from asyncache import cached | ||
| from cachetools import TTLCache | ||
|
|
||
| from ...coordinates import Coordinates | ||
| from ...location.nyt import NYTLocation | ||
| from ...timeline import Timeline | ||
| from ...utils import httputils | ||
| from . import LocationService | ||
|
|
||
|
|
||
| class NYTLocationService(LocationService): | ||
| """ | ||
| Service for retrieving locations from New York Times (https://github.com/nytimes/covid-19-data). | ||
| """ | ||
|
|
||
| async def get_all(self): | ||
| # Get the locations. | ||
| locations = await get_locations() | ||
| return locations | ||
|
|
||
| async def get(self, loc_id): # pylint: disable=arguments-differ | ||
| # Get location at the index equal to provided id. | ||
| locations = await self.get_all() | ||
| return locations[loc_id] | ||
|
|
||
|
|
||
| # --------------------------------------------------------------- | ||
|
|
||
|
|
||
| # Base URL for fetching category. | ||
| BASE_URL = "https://raw.githubusercontent.com/nytimes/covid-19-data/master/us-counties.csv" | ||
|
|
||
|
|
||
| def get_grouped_locations_dict(data): | ||
| """ | ||
| Helper function to group history for locations into one dict. | ||
| :returns: The complete data for each unique US county | ||
| :rdata: dict | ||
| """ | ||
| grouped_locations = {} | ||
|
|
||
| # in increasing order of dates | ||
| for row in data: | ||
| county_state = (row["county"], row["state"]) | ||
| date = row["date"] | ||
| confirmed = row["cases"] | ||
| deaths = row["deaths"] | ||
|
|
||
| # initialize if not existing | ||
| if county_state not in grouped_locations: | ||
| grouped_locations[county_state] = {"confirmed": [], "deaths": []} | ||
|
|
||
| # append confirmed tuple to county_state (date, # confirmed) | ||
| grouped_locations[county_state]["confirmed"].append((date, confirmed)) | ||
| # append deaths tuple to county_state (date, # deaths) | ||
| grouped_locations[county_state]["deaths"].append((date, deaths)) | ||
|
|
||
| return grouped_locations | ||
|
|
||
|
|
||
| @cached(cache=TTLCache(maxsize=1024, ttl=3600)) | ||
| async def get_locations(): | ||
| """ | ||
| Returns a list containing parsed NYT data by US county. The data is cached for 1 hour. | ||
| :returns: The complete data for US Counties. | ||
| :rtype: dict | ||
| """ | ||
|
|
||
| # Request the data. | ||
| async with httputils.CLIENT_SESSION.get(BASE_URL) as response: | ||
| text = await response.text() | ||
|
|
||
| # Parse the CSV. | ||
| data = list(csv.DictReader(text.splitlines())) | ||
|
|
||
| # Group together locations (NYT data ordered by dates not location). | ||
| grouped_locations = get_grouped_locations_dict(data) | ||
|
|
||
| # The normalized locations. | ||
| locations = [] | ||
|
|
||
| idx = 0 | ||
| for county_state, histories in grouped_locations.items(): | ||
| # Make location history for confirmed and deaths from dates. | ||
| # List is tuples of (date, amount) in order of increasing dates. | ||
| confirmed_list = histories["confirmed"] | ||
| confirmed_history = {date: int(amount or 0) for date, amount in confirmed_list} | ||
|
|
||
| deaths_list = histories["deaths"] | ||
| deaths_history = {date: int(amount or 0) for date, amount in deaths_list} | ||
|
|
||
| # Normalize the item and append to locations. | ||
| locations.append( | ||
| NYTLocation( | ||
| id=idx, | ||
| state=county_state[1], | ||
| county=county_state[0], | ||
| coordinates=Coordinates(None, None), # NYT does not provide coordinates | ||
| last_updated=datetime.utcnow().isoformat() + "Z", # since last request | ||
| timelines={ | ||
| "confirmed": Timeline( | ||
| { | ||
| datetime.strptime(date, "%Y-%m-%d").isoformat() + "Z": amount | ||
| for date, amount in confirmed_history.items() | ||
| } | ||
| ), | ||
| "deaths": Timeline( | ||
| { | ||
| datetime.strptime(date, "%Y-%m-%d").isoformat() + "Z": amount | ||
| for date, amount in deaths_history.items() | ||
| } | ||
| ), | ||
| "recovered": Timeline({}), | ||
| }, | ||
| ) | ||
| ) | ||
|
|
||
| idx += 1 | ||
|
|
||
| return locations |