From 6e5bf9fa4cc5715d2ffe6810b1fd60e15ed19e12 Mon Sep 17 00:00:00 2001 From: Gabriel Date: Sat, 2 May 2020 11:10:34 -0400 Subject: [PATCH 01/15] reduce cache size (#21) --- app/services/location/jhu.py | 4 ++-- app/services/location/nyt.py | 2 +- 2 files changed, 3 insertions(+), 3 deletions(-) diff --git a/app/services/location/jhu.py b/app/services/location/jhu.py index 1a11e8ac..07919cfe 100644 --- a/app/services/location/jhu.py +++ b/app/services/location/jhu.py @@ -46,7 +46,7 @@ async def get(self, loc_id): # pylint: disable=arguments-differ ) -@cached(cache=TTLCache(maxsize=128, ttl=1800)) +@cached(cache=TTLCache(maxsize=32, ttl=1800)) async def get_category(category): """ Retrieves the data for the provided category. The data is cached for 30 minutes locally, 1 hour via shared Redis. @@ -129,7 +129,7 @@ async def get_category(category): return results -@cached(cache=TTLCache(maxsize=1024, ttl=1800)) +@cached(cache=TTLCache(maxsize=1, ttl=1800)) async def get_locations(): """ Retrieves the locations from the categories. The locations are cached for 1 hour. diff --git a/app/services/location/nyt.py b/app/services/location/nyt.py index 8b70c5cc..3c092500 100644 --- a/app/services/location/nyt.py +++ b/app/services/location/nyt.py @@ -66,7 +66,7 @@ def get_grouped_locations_dict(data): return grouped_locations -@cached(cache=TTLCache(maxsize=128, ttl=3600)) +@cached(cache=TTLCache(maxsize=1, ttl=3600)) async def get_locations(): """ Returns a list containing parsed NYT data by US county. The data is cached for 1 hour. From da822ba93ff099b5b6189cd700313ef77f972952 Mon Sep 17 00:00:00 2001 From: Gabriel Date: Sat, 2 May 2020 11:54:09 -0400 Subject: [PATCH 02/15] Update Procfile --- Procfile | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/Procfile b/Procfile index 094ebc0a..517e2a0c 100644 --- a/Procfile +++ b/Procfile @@ -1 +1 @@ -web: gunicorn app.main:APP -w 4 -k uvicorn.workers.UvicornWorker +web: gunicorn app.main:APP -w 3 -k uvicorn.workers.UvicornWorker From b08565210a0169c1e24d01fb1e60592b06711cad Mon Sep 17 00:00:00 2001 From: Gabriel Date: Sun, 3 May 2020 18:55:49 -0400 Subject: [PATCH 03/15] Update Procfile --- Procfile | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/Procfile b/Procfile index 517e2a0c..094ebc0a 100644 --- a/Procfile +++ b/Procfile @@ -1 +1 @@ -web: gunicorn app.main:APP -w 3 -k uvicorn.workers.UvicornWorker +web: gunicorn app.main:APP -w 4 -k uvicorn.workers.UvicornWorker From a1dcfb4d87cfddb2a67b364580af366455b485d1 Mon Sep 17 00:00:00 2001 From: Gabriel Date: Sun, 3 May 2020 18:56:33 -0400 Subject: [PATCH 04/15] Update jhu.py --- app/services/location/jhu.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/app/services/location/jhu.py b/app/services/location/jhu.py index 07919cfe..fdfebe7a 100644 --- a/app/services/location/jhu.py +++ b/app/services/location/jhu.py @@ -46,7 +46,7 @@ async def get(self, loc_id): # pylint: disable=arguments-differ ) -@cached(cache=TTLCache(maxsize=32, ttl=1800)) +@cached(cache=TTLCache(maxsize=4, ttl=1800)) async def get_category(category): """ Retrieves the data for the provided category. The data is cached for 30 minutes locally, 1 hour via shared Redis. From cc768a967b10d629594f8805f7109655c30eb716 Mon Sep 17 00:00:00 2001 From: Gabriel Date: Sun, 3 May 2020 20:47:02 -0400 Subject: [PATCH 05/15] update max requests and max jitters --- Procfile | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/Procfile b/Procfile index 094ebc0a..e62649d6 100644 --- a/Procfile +++ b/Procfile @@ -1 +1 @@ -web: gunicorn app.main:APP -w 4 -k uvicorn.workers.UvicornWorker +web: gunicorn app.main:APP -w 4 --max-requests 700 --max-requests-jitter 300 -k uvicorn.workers.UvicornWorker From ba056ce8ccd32cabdb724b7765d285035733e196 Mon Sep 17 00:00:00 2001 From: DeepSource Bot Date: Thu, 7 May 2020 19:34:53 +0000 Subject: [PATCH 06/15] Add .deepsource.toml --- .deepsource.toml | 10 ++++++++++ 1 file changed, 10 insertions(+) create mode 100644 .deepsource.toml diff --git a/.deepsource.toml b/.deepsource.toml new file mode 100644 index 00000000..f772ce8c --- /dev/null +++ b/.deepsource.toml @@ -0,0 +1,10 @@ +version = 1 + +test_patterns = ["tests/**"] + +[[analyzers]] +name = "python" +enabled = true + + [analyzers.meta] + runtime_version = "3.x.x" From e54724e05f480104b5d27b03680f0581d0ebb52b Mon Sep 17 00:00:00 2001 From: Gabriel Date: Thu, 7 May 2020 22:46:14 -0400 Subject: [PATCH 07/15] Update Procfile --- Procfile | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/Procfile b/Procfile index e62649d6..aa7dd193 100644 --- a/Procfile +++ b/Procfile @@ -1 +1 @@ -web: gunicorn app.main:APP -w 4 --max-requests 700 --max-requests-jitter 300 -k uvicorn.workers.UvicornWorker +web: gunicorn app.main:APP -w 2 --max-requests 1000 --max-requests-jitter 400 -k uvicorn.workers.UvicornWorker From 170eb121f0e403cf249a96632e3a451083f6d19a Mon Sep 17 00:00:00 2001 From: Gabriel Date: Sat, 9 May 2020 15:14:41 -0400 Subject: [PATCH 08/15] Cache mem optimize (#23) * formatting (120 length -> 100) * csbs to/from Redis * nyt to/from Redis * ignore locustfile * unused coordinates --- .gitignore | 1 + app/data/__init__.py | 6 +- app/io.py | 13 +++- app/location/__init__.py | 8 ++- app/main.py | 6 +- app/routers/v1.py | 6 +- app/routers/v2.py | 14 ++++- app/services/location/csbs.py | 91 +++++++++++++++------------- app/services/location/jhu.py | 4 +- app/services/location/nyt.py | 110 ++++++++++++++++++---------------- app/utils/populations.py | 4 +- pyproject.toml | 6 +- tasks.py | 13 +++- tests/test_io.py | 6 +- tests/test_location.py | 7 ++- tests/test_routes.py | 8 ++- tests/test_timeline.py | 7 ++- 17 files changed, 194 insertions(+), 116 deletions(-) diff --git a/.gitignore b/.gitignore index 9c41818c..ab6f17ff 100644 --- a/.gitignore +++ b/.gitignore @@ -51,6 +51,7 @@ htmlcov/ nosetests.xml coverage.xml *,cover +locustfile.py # Translations *.mo diff --git a/app/data/__init__.py b/app/data/__init__.py index 265bf3d3..60a75dac 100644 --- a/app/data/__init__.py +++ b/app/data/__init__.py @@ -4,7 +4,11 @@ from ..services.location.nyt import NYTLocationService # Mapping of services to data-sources. -DATA_SOURCES = {"jhu": JhuLocationService(), "csbs": CSBSLocationService(), "nyt": NYTLocationService()} +DATA_SOURCES = { + "jhu": JhuLocationService(), + "csbs": CSBSLocationService(), + "nyt": NYTLocationService(), +} def data_source(source): diff --git a/app/io.py b/app/io.py index 3bd443b6..2a563b15 100644 --- a/app/io.py +++ b/app/io.py @@ -10,7 +10,11 @@ def save( - name: str, content: Union[str, Dict, List], write_mode: str = "w", indent: int = 2, **json_dumps_kwargs + name: str, + content: Union[str, Dict, List], + write_mode: str = "w", + indent: int = 2, + **json_dumps_kwargs, ) -> pathlib.Path: """Save content to a file. If content is a dictionary, use json.dumps().""" path = DATA / name @@ -35,7 +39,12 @@ class AIO: @classmethod async def save( - cls, name: str, content: Union[str, Dict, List], write_mode: str = "w", indent: int = 2, **json_dumps_kwargs + cls, + name: str, + content: Union[str, Dict, List], + write_mode: str = "w", + indent: int = 2, + **json_dumps_kwargs, ): """Save content to a file. If content is a dictionary, use json.dumps().""" path = DATA / name diff --git a/app/location/__init__.py b/app/location/__init__.py index d12f28c3..1da5e9e5 100644 --- a/app/location/__init__.py +++ b/app/location/__init__.py @@ -11,7 +11,7 @@ class Location: # pylint: disable=too-many-instance-attributes """ def __init__( - self, id, country, province, coordinates, last_updated, confirmed, deaths, recovered + self, id, country, province, coordinates, last_updated, confirmed, deaths, recovered, ): # pylint: disable=too-many-arguments # General info. self.id = id @@ -66,7 +66,11 @@ def serialize(self): # Last updated. "last_updated": self.last_updated, # Latest data (statistics). - "latest": {"confirmed": self.confirmed, "deaths": self.deaths, "recovered": self.recovered}, + "latest": { + "confirmed": self.confirmed, + "deaths": self.deaths, + "recovered": self.recovered, + }, } diff --git a/app/main.py b/app/main.py index f8ed1b41..3e3c3317 100644 --- a/app/main.py +++ b/app/main.py @@ -49,7 +49,11 @@ # Enable CORS. APP.add_middleware( - CORSMiddleware, allow_credentials=True, allow_origins=["*"], allow_methods=["*"], allow_headers=["*"], + CORSMiddleware, + allow_credentials=True, + allow_origins=["*"], + allow_methods=["*"], + allow_headers=["*"], ) APP.add_middleware(GZipMiddleware, minimum_size=1000) diff --git a/app/routers/v1.py b/app/routers/v1.py index 662514a0..517bc625 100644 --- a/app/routers/v1.py +++ b/app/routers/v1.py @@ -19,7 +19,11 @@ async def all_categories(): "deaths": deaths, "recovered": recovered, # Latest. - "latest": {"confirmed": confirmed["latest"], "deaths": deaths["latest"], "recovered": recovered["latest"],}, + "latest": { + "confirmed": confirmed["latest"], + "deaths": deaths["latest"], + "recovered": recovered["latest"], + }, } diff --git a/app/routers/v2.py b/app/routers/v2.py index de5a5312..fe9d2475 100644 --- a/app/routers/v2.py +++ b/app/routers/v2.py @@ -65,11 +65,17 @@ async def get_locations( # Do filtering. try: - locations = [location for location in locations if str(getattr(location, key)).lower() == str(value)] + locations = [ + location + for location in locations + if str(getattr(location, key)).lower() == str(value) + ] except AttributeError: pass if not locations: - raise HTTPException(404, detail=f"Source `{source}` does not have the desired location data.") + raise HTTPException( + 404, detail=f"Source `{source}` does not have the desired location data.", + ) # Return final serialized data. return { @@ -84,7 +90,9 @@ async def get_locations( # pylint: disable=invalid-name @V2.get("/locations/{id}", response_model=LocationResponse) -async def get_location_by_id(request: Request, id: int, source: Sources = "jhu", timelines: bool = True): +async def get_location_by_id( + request: Request, id: int, source: Sources = "jhu", timelines: bool = True +): """ Getting specific location by id. """ diff --git a/app/services/location/csbs.py b/app/services/location/csbs.py index 68bdb01c..e8668280 100644 --- a/app/services/location/csbs.py +++ b/app/services/location/csbs.py @@ -6,6 +6,7 @@ from asyncache import cached from cachetools import TTLCache +from ...caches import check_cache, load_cache from ...coordinates import Coordinates from ...location.csbs import CSBSLocation from ...utils import httputils @@ -34,7 +35,7 @@ async def get(self, loc_id): # pylint: disable=arguments-differ BASE_URL = "https://facts.csbs.org/covid-19/covid19_county.csv" -@cached(cache=TTLCache(maxsize=1, ttl=3600)) +@cached(cache=TTLCache(maxsize=1, ttl=1800)) async def get_locations(): """ Retrieves county locations; locations are cached for 1 hour @@ -44,48 +45,54 @@ async def get_locations(): """ data_id = "csbs.locations" LOGGER.info(f"{data_id} Requesting data...") - async with httputils.CLIENT_SESSION.get(BASE_URL) as response: - text = await response.text() - - LOGGER.debug(f"{data_id} Data received") - - data = list(csv.DictReader(text.splitlines())) - LOGGER.debug(f"{data_id} CSV parsed") - - locations = [] - - for i, item in enumerate(data): - # General info. - state = item["State Name"] - county = item["County Name"] - - # Ensure country is specified. - if county in {"Unassigned", "Unknown"}: - continue - - # Coordinates. - coordinates = Coordinates(item["Latitude"], item["Longitude"]) # pylint: disable=unused-variable - - # Date string without "EDT" at end. - last_update = " ".join(item["Last Update"].split(" ")[0:2]) - - # Append to locations. - locations.append( - CSBSLocation( - # General info. - i, - state, - county, - # Coordinates. - Coordinates(item["Latitude"], item["Longitude"]), - # Last update (parse as ISO). - datetime.strptime(last_update, "%Y-%m-%d %H:%M").isoformat() + "Z", - # Statistics. - int(item["Confirmed"] or 0), - int(item["Death"] or 0), + # check shared cache + cache_results = await check_cache(data_id) + if cache_results: + LOGGER.info(f"{data_id} using shared cache results") + locations = cache_results + else: + LOGGER.info(f"{data_id} shared cache empty") + async with httputils.CLIENT_SESSION.get(BASE_URL) as response: + text = await response.text() + + LOGGER.debug(f"{data_id} Data received") + + data = list(csv.DictReader(text.splitlines())) + LOGGER.debug(f"{data_id} CSV parsed") + + locations = [] + + for i, item in enumerate(data): + # General info. + state = item["State Name"] + county = item["County Name"] + + # Ensure country is specified. + if county in {"Unassigned", "Unknown"}: + continue + + # Date string without "EDT" at end. + last_update = " ".join(item["Last Update"].split(" ")[0:2]) + + # Append to locations. + locations.append( + CSBSLocation( + # General info. + i, + state, + county, + # Coordinates. + Coordinates(item["Latitude"], item["Longitude"]), + # Last update (parse as ISO). + datetime.strptime(last_update, "%Y-%m-%d %H:%M").isoformat() + "Z", + # Statistics. + int(item["Confirmed"] or 0), + int(item["Death"] or 0), + ) ) - ) - LOGGER.info(f"{data_id} Data normalized") + LOGGER.info(f"{data_id} Data normalized") + # save the results to distributed cache + await load_cache(data_id, locations) # Return the locations. return locations diff --git a/app/services/location/jhu.py b/app/services/location/jhu.py index fdfebe7a..06fa3fe0 100644 --- a/app/services/location/jhu.py +++ b/app/services/location/jhu.py @@ -41,9 +41,7 @@ async def get(self, loc_id): # pylint: disable=arguments-differ # Base URL for fetching category. -BASE_URL = ( - "https://raw.githubusercontent.com/CSSEGISandData/2019-nCoV/master/csse_covid_19_data/csse_covid_19_time_series/" -) +BASE_URL = "https://raw.githubusercontent.com/CSSEGISandData/2019-nCoV/master/csse_covid_19_data/csse_covid_19_time_series/" @cached(cache=TTLCache(maxsize=4, ttl=1800)) diff --git a/app/services/location/nyt.py b/app/services/location/nyt.py index 3c092500..3ba12d02 100644 --- a/app/services/location/nyt.py +++ b/app/services/location/nyt.py @@ -6,6 +6,7 @@ from asyncache import cached from cachetools import TTLCache +from ...caches import check_cache, load_cache from ...coordinates import Coordinates from ...location.nyt import NYTLocation from ...timeline import Timeline @@ -66,7 +67,7 @@ def get_grouped_locations_dict(data): return grouped_locations -@cached(cache=TTLCache(maxsize=1, ttl=3600)) +@cached(cache=TTLCache(maxsize=1, ttl=1800)) async def get_locations(): """ Returns a list containing parsed NYT data by US county. The data is cached for 1 hour. @@ -77,55 +78,64 @@ async def get_locations(): data_id = "nyt.locations" # Request the data. LOGGER.info(f"{data_id} Requesting data...") - async with httputils.CLIENT_SESSION.get(BASE_URL) as response: - text = await response.text() - - LOGGER.debug(f"{data_id} Data received") - - # Parse the CSV. - data = list(csv.DictReader(text.splitlines())) - LOGGER.debug(f"{data_id} CSV parsed") - - # Group together locations (NYT data ordered by dates not location). - grouped_locations = get_grouped_locations_dict(data) - - # The normalized locations. - locations = [] - - for idx, (county_state, histories) in enumerate(grouped_locations.items()): - # Make location history for confirmed and deaths from dates. - # List is tuples of (date, amount) in order of increasing dates. - confirmed_list = histories["confirmed"] - confirmed_history = {date: int(amount or 0) for date, amount in confirmed_list} - - deaths_list = histories["deaths"] - deaths_history = {date: int(amount or 0) for date, amount in deaths_list} - - # Normalize the item and append to locations. - locations.append( - NYTLocation( - id=idx, - state=county_state[1], - county=county_state[0], - coordinates=Coordinates(None, None), # NYT does not provide coordinates - last_updated=datetime.utcnow().isoformat() + "Z", # since last request - timelines={ - "confirmed": Timeline( - { - datetime.strptime(date, "%Y-%m-%d").isoformat() + "Z": amount - for date, amount in confirmed_history.items() - } - ), - "deaths": Timeline( - { - datetime.strptime(date, "%Y-%m-%d").isoformat() + "Z": amount - for date, amount in deaths_history.items() - } - ), - "recovered": Timeline({}), - }, + # check shared cache + cache_results = await check_cache(data_id) + if cache_results: + LOGGER.info(f"{data_id} using shared cache results") + locations = cache_results + else: + LOGGER.info(f"{data_id} shared cache empty") + async with httputils.CLIENT_SESSION.get(BASE_URL) as response: + text = await response.text() + + LOGGER.debug(f"{data_id} Data received") + + # Parse the CSV. + data = list(csv.DictReader(text.splitlines())) + LOGGER.debug(f"{data_id} CSV parsed") + + # Group together locations (NYT data ordered by dates not location). + grouped_locations = get_grouped_locations_dict(data) + + # The normalized locations. + locations = [] + + for idx, (county_state, histories) in enumerate(grouped_locations.items()): + # Make location history for confirmed and deaths from dates. + # List is tuples of (date, amount) in order of increasing dates. + confirmed_list = histories["confirmed"] + confirmed_history = {date: int(amount or 0) for date, amount in confirmed_list} + + deaths_list = histories["deaths"] + deaths_history = {date: int(amount or 0) for date, amount in deaths_list} + + # Normalize the item and append to locations. + locations.append( + NYTLocation( + id=idx, + state=county_state[1], + county=county_state[0], + coordinates=Coordinates(None, None), # NYT does not provide coordinates + last_updated=datetime.utcnow().isoformat() + "Z", # since last request + timelines={ + "confirmed": Timeline( + { + datetime.strptime(date, "%Y-%m-%d").isoformat() + "Z": amount + for date, amount in confirmed_history.items() + } + ), + "deaths": Timeline( + { + datetime.strptime(date, "%Y-%m-%d").isoformat() + "Z": amount + for date, amount in deaths_history.items() + } + ), + "recovered": Timeline({}), + }, + ) ) - ) - LOGGER.info(f"{data_id} Data normalized") + LOGGER.info(f"{data_id} Data normalized") + # save the results to distributed cache + await load_cache(data_id, locations) return locations diff --git a/app/utils/populations.py b/app/utils/populations.py index 24f0fa4e..c02f15a9 100644 --- a/app/utils/populations.py +++ b/app/utils/populations.py @@ -28,7 +28,9 @@ def fetch_populations(save=False): # Fetch the countries. try: - countries = requests.get(GEONAMES_URL, params={"username": "dperic"}, timeout=1.25).json()["geonames"] + countries = requests.get(GEONAMES_URL, params={"username": "dperic"}, timeout=1.25).json()[ + "geonames" + ] # Go through all the countries and perform the mapping. for country in countries: mappings.update({country["countryCode"]: int(country["population"]) or None}) diff --git a/pyproject.toml b/pyproject.toml index b6bc6af6..df1ad168 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -1,5 +1,5 @@ [tool.black] -line-length = 120 +line-length = 100 target-version = ['py36', 'py37', 'py38'] include = '\.pyi?$' exclude = ''' @@ -23,7 +23,7 @@ multi_line_output = 3 include_trailing_comma = "True" force_grid_wrap = 0 use_parentheses = "True" -line_length = 120 +line_length = 100 [tool.pylint.master] extension-pkg-whitelist = "pydantic" @@ -42,7 +42,7 @@ logging-modules = "logging" allow-wildcard-with-all = "no" [tool.pylint.format] indent-after-paren = "4" -max-line-length = "120" # matches black setting +max-line-length = "100" # matches black setting max-module-lines = "800" no-space-check = ''' trailing-comma, diff --git a/tasks.py b/tasks.py index ae1f09cd..0f6d6995 100644 --- a/tasks.py +++ b/tasks.py @@ -72,12 +72,21 @@ def test(ctx): @invoke.task def generate_reqs(ctx): """Generate requirements.txt""" - reqs = ["pipenv lock -r > requirements.txt", "pipenv lock -r --dev > requirements-dev.txt"] + reqs = [ + "pipenv lock -r > requirements.txt", + "pipenv lock -r --dev > requirements-dev.txt", + ] [ctx.run(req) for req in reqs] @invoke.task -def docker(ctx, build=False, run=False, tag="covid-tracker-api:latest", name=f"covid-api-{random.randint(0,999)}"): +def docker( + ctx, + build=False, + run=False, + tag="covid-tracker-api:latest", + name=f"covid-api-{random.randint(0,999)}", +): """Build and run docker container.""" if not any([build, run]): raise invoke.Exit(message="Specify either --build or --run", code=1) diff --git a/tests/test_io.py b/tests/test_io.py index c5d16c3a..ba926011 100644 --- a/tests/test_io.py +++ b/tests/test_io.py @@ -10,7 +10,11 @@ [ ("test_file.txt", string.ascii_lowercase, {}), ("test_json_file.json", {"a": 0, "b": 1, "c": 2}, {}), - ("test_custom_json.json", {"z": -1, "b": 1, "y": -2, "a": 0}, {"indent": 4, "sort_keys": True}), + ( + "test_custom_json.json", + {"z": -1, "b": 1, "y": -2, "a": 0}, + {"indent": 4, "sort_keys": True}, + ), ], ) diff --git a/tests/test_location.py b/tests/test_location.py index 567eddcd..50129a52 100644 --- a/tests/test_location.py +++ b/tests/test_location.py @@ -48,7 +48,12 @@ def test_location_class( # Location. location_obj = location.TimelinedLocation( - test_id, country, province, coords, now, {"confirmed": confirmed, "deaths": deaths, "recovered": recovered,} + test_id, + country, + province, + coords, + now, + {"confirmed": confirmed, "deaths": deaths, "recovered": recovered,}, ) assert location_obj.country_code == country_code diff --git a/tests/test_routes.py b/tests/test_routes.py index 52d26843..eea153bc 100644 --- a/tests/test_routes.py +++ b/tests/test_routes.py @@ -126,7 +126,9 @@ async def test_v2_locations_id(self): return_data = response.json() - filepath = "tests/expected_output/v2_{state}_id_{test_id}.json".format(state=state, test_id=test_id) + filepath = "tests/expected_output/v2_{state}_id_{test_id}.json".format( + state=state, test_id=test_id + ) with open(filepath, "r") as file: expected_json_output = file.read() @@ -151,7 +153,9 @@ async def test_v2_locations_id(self): ({"source": "jhu", "country_code": "US"}, 404), ], ) -async def test_locations_status_code(async_api_client, query_params, expected_status, mock_client_session): +async def test_locations_status_code( + async_api_client, query_params, expected_status, mock_client_session +): response = await async_api_client.get("/v2/locations", query_string=query_params) print(f"GET {response.url}\n{response}") diff --git a/tests/test_timeline.py b/tests/test_timeline.py index 056286aa..79612f5a 100644 --- a/tests/test_timeline.py +++ b/tests/test_timeline.py @@ -21,7 +21,12 @@ def test_timeline_class(): assert history_data.latest == 7 # validate order - assert list(dict(history_data.timeline).keys()) == ["1/22/20", "1/23/20", "1/24/20", "1/25/20"] + assert list(dict(history_data.timeline).keys()) == [ + "1/22/20", + "1/23/20", + "1/24/20", + "1/25/20", + ] # validate serialize check_serialize = { From 7fe38b9b7304b25d7edee2a7d923d1b25db0c60d Mon Sep 17 00:00:00 2001 From: Gabriel Date: Sat, 9 May 2020 15:32:19 -0400 Subject: [PATCH 09/15] Sentry (#24) * add sentry-asgi * sentry middleware --- Pipfile | 1 + Pipfile.lock | 103 ++++++++++++++++++++++++------------------- app/config.py | 2 + app/main.py | 10 +++++ requirements-dev.txt | 16 +++---- requirements.txt | 2 + 6 files changed, 81 insertions(+), 53 deletions(-) diff --git a/Pipfile b/Pipfile index 584e4e10..f35399b4 100644 --- a/Pipfile +++ b/Pipfile @@ -33,6 +33,7 @@ pydantic = {extras = ["dotenv"],version = "*"} python-dateutil = "*" requests = "*" scout-apm = "*" +sentry-asgi = "*" uvicorn = "*" [requires] diff --git a/Pipfile.lock b/Pipfile.lock index b1551678..8968d338 100644 --- a/Pipfile.lock +++ b/Pipfile.lock @@ -1,7 +1,7 @@ { "_meta": { "hash": { - "sha256": "1a448c6a753787b0b71c702c6b3baa4063b468afef4847b413459801d6e56592" + "sha256": "b4129f7720f63d1dce07c8c1bf41a3aa96580e7cb3a53e7daa09db1cc1e0fa1f" }, "pipfile-spec": 6, "requires": { @@ -411,6 +411,20 @@ "index": "pypi", "version": "==2.14.1" }, + "sentry-asgi": { + "hashes": [ + "sha256:0fc35fc6da9c16c0353f087c29fcfcd694b767bd1a77a2e768f519618eb3defd" + ], + "index": "pypi", + "version": "==0.2.0" + }, + "sentry-sdk": { + "hashes": [ + "sha256:23808d571d2461a4ce3784ec12bbee5bdb8c026c143fe79d36cef8a6d653e71f", + "sha256:bb90a4e19c7233a580715fc986cc44be2c48fc10b31e71580a2037e1c94b6950" + ], + "version": "==0.14.3" + }, "six": { "hashes": [ "sha256:236bdbdce46e6e6a3d61a337c0f8b763ca1e8717c03b369e87a7ec7ce1319c0a", @@ -525,10 +539,10 @@ }, "astroid": { "hashes": [ - "sha256:29fa5d46a2404d01c834fcb802a3943685f1fc538eb2a02a161349f5505ac196", - "sha256:2fecea42b20abb1922ed65c7b5be27edfba97211b04b2b6abc6a43549a024ea6" + "sha256:4c17cea3e592c21b6e222f673868961bad77e1f985cb1694ed077475a89229c1", + "sha256:d8506842a3faf734b81599c8b98dcc423de863adcc1999248480b18bd31a0f38" ], - "version": "==2.4.0" + "version": "==2.4.1" }, "async-asgi-testclient": { "hashes": [ @@ -649,17 +663,17 @@ }, "gitdb": { "hashes": [ - "sha256:6f0ecd46f99bb4874e5678d628c3a198e2b4ef38daea2756a2bfd8df7dd5c1a5", - "sha256:ba1132c0912e8c917aa8aa990bee26315064c7b7f171ceaaac0afeb1dc656c6a" + "sha256:91f36bfb1ab7949b3b40e23736db18231bf7593edada2ba5c3a174a7b23657ac", + "sha256:c9e1f2d0db7ddb9a704c2a0217be31214e91a4fe1dea1efad19ae42ba0c285c9" ], - "version": "==4.0.4" + "version": "==4.0.5" }, "gitpython": { "hashes": [ - "sha256:6d4f10e2aaad1864bb0f17ec06a2c2831534140e5883c350d58b4e85189dab74", - "sha256:71b8dad7409efbdae4930f2b0b646aaeccce292484ffa0bc74f1195582578b3d" + "sha256:864a47472548f3ba716ca202e034c1900f197c0fb3a08f641c20c3cafd15ed94", + "sha256:da3b2cf819974789da34f95ac218ef99f515a928685db141327c09b73dd69c09" ], - "version": "==3.1.1" + "version": "==3.1.2" }, "idna": { "hashes": [ @@ -800,11 +814,11 @@ }, "pylint": { "hashes": [ - "sha256:588e114e3f9a1630428c35b7dd1c82c1c93e1b0e78ee312ae4724c5e1a1e0245", - "sha256:bd556ba95a4cf55a1fc0004c00cf4560b1e70598a54a74c6904d933c8f3bd5a8" + "sha256:b95e31850f3af163c2283ed40432f053acbc8fc6eba6a069cb518d9dbf71848c", + "sha256:dd506acce0427e9e08fb87274bcaa953d38b50a58207170dbf5b36cf3e16957b" ], "index": "pypi", - "version": "==2.5.0" + "version": "==2.5.2" }, "pyparsing": { "hashes": [ @@ -815,19 +829,18 @@ }, "pytest": { "hashes": [ - "sha256:0e5b30f5cb04e887b91b1ee519fa3d89049595f428c1db76e73bd7f17b09b172", - "sha256:84dde37075b8805f3d1f392cc47e38a0e59518fb46a431cfdaf7cf1ce805f970" + "sha256:95c710d0a72d91c13fae35dce195633c929c3792f54125919847fdcdf7caa0d3", + "sha256:eb2b5e935f6a019317e455b6da83dd8650ac9ffd2ee73a7b657a30873d67a698" ], "index": "pypi", - "version": "==5.4.1" + "version": "==5.4.2" }, "pytest-asyncio": { "hashes": [ - "sha256:6096d101a1ae350d971df05e25f4a8b4d3cd13ffb1b32e42d902ac49670d2bfa", - "sha256:c54866f3cf5dd2063992ba2c34784edae11d3ed19e006d220a3cf0bfc4191fcb" + "sha256:475bd2f3dc0bc11d2463656b3cbaafdbec5a47b47508ea0b329ee693040eebd2" ], "index": "pypi", - "version": "==0.11.0" + "version": "==0.12.0" }, "pytest-cov": { "hashes": [ @@ -855,29 +868,29 @@ }, "regex": { "hashes": [ - "sha256:08119f707f0ebf2da60d2f24c2f39ca616277bb67ef6c92b72cbf90cbe3a556b", - "sha256:0ce9537396d8f556bcfc317c65b6a0705320701e5ce511f05fc04421ba05b8a8", - "sha256:1cbe0fa0b7f673400eb29e9ef41d4f53638f65f9a2143854de6b1ce2899185c3", - "sha256:2294f8b70e058a2553cd009df003a20802ef75b3c629506be20687df0908177e", - "sha256:23069d9c07e115537f37270d1d5faea3e0bdded8279081c4d4d607a2ad393683", - "sha256:24f4f4062eb16c5bbfff6a22312e8eab92c2c99c51a02e39b4eae54ce8255cd1", - "sha256:295badf61a51add2d428a46b8580309c520d8b26e769868b922750cf3ce67142", - "sha256:2a3bf8b48f8e37c3a40bb3f854bf0121c194e69a650b209628d951190b862de3", - "sha256:4385f12aa289d79419fede43f979e372f527892ac44a541b5446617e4406c468", - "sha256:5635cd1ed0a12b4c42cce18a8d2fb53ff13ff537f09de5fd791e97de27b6400e", - "sha256:5bfed051dbff32fd8945eccca70f5e22b55e4148d2a8a45141a3b053d6455ae3", - "sha256:7e1037073b1b7053ee74c3c6c0ada80f3501ec29d5f46e42669378eae6d4405a", - "sha256:90742c6ff121a9c5b261b9b215cb476eea97df98ea82037ec8ac95d1be7a034f", - "sha256:a58dd45cb865be0ce1d5ecc4cfc85cd8c6867bea66733623e54bd95131f473b6", - "sha256:c087bff162158536387c53647411db09b6ee3f9603c334c90943e97b1052a156", - "sha256:c162a21e0da33eb3d31a3ac17a51db5e634fc347f650d271f0305d96601dc15b", - "sha256:c9423a150d3a4fc0f3f2aae897a59919acd293f4cb397429b120a5fcd96ea3db", - "sha256:ccccdd84912875e34c5ad2d06e1989d890d43af6c2242c6fcfa51556997af6cd", - "sha256:e91ba11da11cf770f389e47c3f5c30473e6d85e06d7fd9dcba0017d2867aab4a", - "sha256:ea4adf02d23b437684cd388d557bf76e3afa72f7fed5bbc013482cc00c816948", - "sha256:fb95debbd1a824b2c4376932f2216cc186912e389bdb0e27147778cf6acb3f89" - ], - "version": "==2020.4.4" + "sha256:021a0ae4d2baeeb60a3014805a2096cb329bd6d9f30669b7ad0da51a9cb73349", + "sha256:04d6e948ef34d3eac133bedc0098364a9e635a7914f050edb61272d2ddae3608", + "sha256:099568b372bda492be09c4f291b398475587d49937c659824f891182df728cdf", + "sha256:0ff50843535593ee93acab662663cb2f52af8e31c3f525f630f1dc6156247938", + "sha256:1b17bf37c2aefc4cac8436971fe6ee52542ae4225cfc7762017f7e97a63ca998", + "sha256:1e2255ae938a36e9bd7db3b93618796d90c07e5f64dd6a6750c55f51f8b76918", + "sha256:2bc6a17a7fa8afd33c02d51b6f417fc271538990297167f68a98cae1c9e5c945", + "sha256:3ab5e41c4ed7cd4fa426c50add2892eb0f04ae4e73162155cd668257d02259dd", + "sha256:3b059e2476b327b9794c792c855aa05531a3f3044737e455d283c7539bd7534d", + "sha256:4df91094ced6f53e71f695c909d9bad1cca8761d96fd9f23db12245b5521136e", + "sha256:5493a02c1882d2acaaf17be81a3b65408ff541c922bfd002535c5f148aa29f74", + "sha256:5b741ecc3ad3e463d2ba32dce512b412c319993c1bb3d999be49e6092a769fb2", + "sha256:652ab4836cd5531d64a34403c00ada4077bb91112e8bcdae933e2eae232cf4a8", + "sha256:669a8d46764a09f198f2e91fc0d5acdac8e6b620376757a04682846ae28879c4", + "sha256:73a10404867b835f1b8a64253e4621908f0d71150eb4e97ab2e7e441b53e9451", + "sha256:7ce4a213a96d6c25eeae2f7d60d4dad89ac2b8134ec3e69db9bc522e2c0f9388", + "sha256:8127ca2bf9539d6a64d03686fd9e789e8c194fc19af49b69b081f8c7e6ecb1bc", + "sha256:b5b5b2e95f761a88d4c93691716ce01dc55f288a153face1654f868a8034f494", + "sha256:b7c9f65524ff06bf70c945cd8d8d1fd90853e27ccf86026af2afb4d9a63d06b1", + "sha256:f7f2f4226db6acd1da228adf433c5c3792858474e49d80668ea82ac87cf74a03", + "sha256:fa09da4af4e5b15c0e8b4986a083f3fd159302ea115a6cc0649cd163435538b8" + ], + "version": "==2020.5.7" }, "requests": { "hashes": [ @@ -904,10 +917,10 @@ }, "smmap": { "hashes": [ - "sha256:52ea78b3e708d2c2b0cfe93b6fc3fbeec53db913345c26be6ed84c11ed8bebc1", - "sha256:b46d3fc69ba5f367df96d91f8271e8ad667a198d5a28e215a6c3d9acd133a911" + "sha256:54c44c197c819d5ef1991799a7e30b662d1e520f2ac75c9efbeb54a742214cf4", + "sha256:9c98bbd1f9786d22f14b3d4126894d56befb835ec90cef151af566c7e19b5d24" ], - "version": "==3.0.2" + "version": "==3.0.4" }, "stevedore": { "hashes": [ diff --git a/app/config.py b/app/config.py index ab60d42b..377ebc13 100644 --- a/app/config.py +++ b/app/config.py @@ -13,6 +13,8 @@ class _Settings(BaseSettings): local_redis_url: AnyUrl = None # Scout APM scout_name: str = None + # Sentry + sentry_dsn: str = None @functools.lru_cache() diff --git a/app/main.py b/app/main.py index 3e3c3317..50be93e8 100644 --- a/app/main.py +++ b/app/main.py @@ -4,6 +4,8 @@ import logging import pydantic +import sentry_asgi +import sentry_sdk import uvicorn from fastapi import FastAPI, Request, Response from fastapi.middleware.cors import CORSMiddleware @@ -23,6 +25,9 @@ SETTINGS = get_settings() +if SETTINGS.sentry_dsn: # pragma: no cover + sentry_sdk.init(dsn=SETTINGS.sentry_dsn) + APP = FastAPI( title="Coronavirus Tracker", description=( @@ -47,6 +52,11 @@ else: LOGGER.debug("No SCOUT_NAME config") +# Sentry Error Tracking +if SETTINGS.sentry_dsn: # pragma: no cover + LOGGER.info("Adding Sentry middleware") + APP.add_middleware(sentry_asgi.SentryMiddleware) + # Enable CORS. APP.add_middleware( CORSMiddleware, diff --git a/requirements-dev.txt b/requirements-dev.txt index 1d919ece..d95c199e 100644 --- a/requirements-dev.txt +++ b/requirements-dev.txt @@ -1,6 +1,6 @@ -i https://pypi.org/simple appdirs==1.4.3 -astroid==2.4.0 +astroid==2.4.1 async-asgi-testclient==1.4.4 async-generator==1.10 asyncmock==0.4.2 @@ -13,8 +13,8 @@ click==7.1.2 coverage==5.1 coveralls==2.0.0 docopt==0.6.2 -gitdb==4.0.4 -gitpython==3.1.1 +gitdb==4.0.5 +gitpython==3.1.2 idna==2.9 importlib-metadata==1.6.0 ; python_version < '3.8' invoke==1.4.1 @@ -29,17 +29,17 @@ pathspec==0.8.0 pbr==5.4.5 pluggy==0.13.1 py==1.8.1 -pylint==2.5.0 +pylint==2.5.2 pyparsing==2.4.7 -pytest-asyncio==0.11.0 +pytest-asyncio==0.12.0 pytest-cov==2.8.1 -pytest==5.4.1 +pytest==5.4.2 pyyaml==5.3.1 -regex==2020.4.4 +regex==2020.5.7 requests==2.23.0 responses==0.10.14 six==1.14.0 -smmap==3.0.2 +smmap==3.0.4 stevedore==1.32.0 toml==0.10.0 typed-ast==1.4.1 diff --git a/requirements.txt b/requirements.txt index dd2ece5a..cbee4d21 100644 --- a/requirements.txt +++ b/requirements.txt @@ -30,6 +30,8 @@ python-dateutil==2.8.1 python-dotenv==0.13.0 requests==2.23.0 scout-apm==2.14.1 +sentry-asgi==0.2.0 +sentry-sdk==0.14.3 six==1.14.0 starlette==0.13.2 urllib3[secure]==1.25.9 ; python_version >= '3.5' From 96efc9231754c06ca2fd5d524f0fae3b04feb7c8 Mon Sep 17 00:00:00 2001 From: Gabriel Date: Sat, 9 May 2020 16:04:43 -0400 Subject: [PATCH 10/15] use official middleware (#25) --- Pipfile | 2 +- Pipfile.lock | 10 ++-------- app/main.py | 4 ++-- requirements.txt | 1 - 4 files changed, 5 insertions(+), 12 deletions(-) diff --git a/Pipfile b/Pipfile index f35399b4..57ef4ae7 100644 --- a/Pipfile +++ b/Pipfile @@ -33,7 +33,7 @@ pydantic = {extras = ["dotenv"],version = "*"} python-dateutil = "*" requests = "*" scout-apm = "*" -sentry-asgi = "*" +sentry-sdk = "*" uvicorn = "*" [requires] diff --git a/Pipfile.lock b/Pipfile.lock index 8968d338..7cdfd8a1 100644 --- a/Pipfile.lock +++ b/Pipfile.lock @@ -1,7 +1,7 @@ { "_meta": { "hash": { - "sha256": "b4129f7720f63d1dce07c8c1bf41a3aa96580e7cb3a53e7daa09db1cc1e0fa1f" + "sha256": "dfa074e03982c046ee011817d151762138abfc1f13ae4e67700233599af18c3e" }, "pipfile-spec": 6, "requires": { @@ -411,18 +411,12 @@ "index": "pypi", "version": "==2.14.1" }, - "sentry-asgi": { - "hashes": [ - "sha256:0fc35fc6da9c16c0353f087c29fcfcd694b767bd1a77a2e768f519618eb3defd" - ], - "index": "pypi", - "version": "==0.2.0" - }, "sentry-sdk": { "hashes": [ "sha256:23808d571d2461a4ce3784ec12bbee5bdb8c026c143fe79d36cef8a6d653e71f", "sha256:bb90a4e19c7233a580715fc986cc44be2c48fc10b31e71580a2037e1c94b6950" ], + "index": "pypi", "version": "==0.14.3" }, "six": { diff --git a/app/main.py b/app/main.py index 50be93e8..b43b4aae 100644 --- a/app/main.py +++ b/app/main.py @@ -4,7 +4,6 @@ import logging import pydantic -import sentry_asgi import sentry_sdk import uvicorn from fastapi import FastAPI, Request, Response @@ -12,6 +11,7 @@ from fastapi.middleware.gzip import GZipMiddleware from fastapi.responses import JSONResponse from scout_apm.async_.starlette import ScoutMiddleware +from sentry_sdk.integrations.asgi import SentryAsgiMiddleware from .config import get_settings from .data import data_source @@ -55,7 +55,7 @@ # Sentry Error Tracking if SETTINGS.sentry_dsn: # pragma: no cover LOGGER.info("Adding Sentry middleware") - APP.add_middleware(sentry_asgi.SentryMiddleware) + APP.add_middleware(SentryAsgiMiddleware) # Enable CORS. APP.add_middleware( diff --git a/requirements.txt b/requirements.txt index cbee4d21..02ab222e 100644 --- a/requirements.txt +++ b/requirements.txt @@ -30,7 +30,6 @@ python-dateutil==2.8.1 python-dotenv==0.13.0 requests==2.23.0 scout-apm==2.14.1 -sentry-asgi==0.2.0 sentry-sdk==0.14.3 six==1.14.0 starlette==0.13.2 From ecb0a6867b03909fc76220817f7f4951b064c4f8 Mon Sep 17 00:00:00 2001 From: Gabriel Date: Sat, 9 May 2020 18:36:44 -0400 Subject: [PATCH 11/15] cache redis json serialization error --- app/services/location/csbs.py | 8 ++++++-- 1 file changed, 6 insertions(+), 2 deletions(-) diff --git a/app/services/location/csbs.py b/app/services/location/csbs.py index e8668280..ddfe48b7 100644 --- a/app/services/location/csbs.py +++ b/app/services/location/csbs.py @@ -92,7 +92,11 @@ async def get_locations(): ) LOGGER.info(f"{data_id} Data normalized") # save the results to distributed cache - await load_cache(data_id, locations) - + # TODO: fix json serialization + try: + await load_cache(data_id, locations) + except TypeError as type_err: + LOGGER.error(type_err) + # Return the locations. return locations From 464453db2f30d9a9e8042bbc27e75618aeffe9f9 Mon Sep 17 00:00:00 2001 From: Gabriel Date: Sat, 9 May 2020 19:20:02 -0400 Subject: [PATCH 12/15] fix nyt redis serialization error --- app/services/location/nyt.py | 6 +++++- 1 file changed, 5 insertions(+), 1 deletion(-) diff --git a/app/services/location/nyt.py b/app/services/location/nyt.py index 3ba12d02..52b565ae 100644 --- a/app/services/location/nyt.py +++ b/app/services/location/nyt.py @@ -136,6 +136,10 @@ async def get_locations(): ) LOGGER.info(f"{data_id} Data normalized") # save the results to distributed cache - await load_cache(data_id, locations) + # TODO: fix json serialization + try: + await load_cache(data_id, locations) + except TypeError as type_err: + LOGGER.error(type_err) return locations From 52ef1cd0709149679a6a3e4720ff5378a12524a4 Mon Sep 17 00:00:00 2001 From: codedawi Date: Tue, 19 May 2020 22:43:39 -0400 Subject: [PATCH 13/15] :thumbsup: fixing recovery data in api/v2 jhu --- app/services/location/jhu.py | 46 ++++++++++++++++++++++++++++++------ tests/test_jhu.py | 8 +++++++ tests/test_routes.py | 8 +++---- 3 files changed, 50 insertions(+), 12 deletions(-) diff --git a/app/services/location/jhu.py b/app/services/location/jhu.py index 1a11e8ac..7a622301 100644 --- a/app/services/location/jhu.py +++ b/app/services/location/jhu.py @@ -142,22 +142,31 @@ async def get_locations(): # Get all of the data categories locations. confirmed = await get_category("confirmed") deaths = await get_category("deaths") - # recovered = await get_category("recovered") + recovered = await get_category("recovered") locations_confirmed = confirmed["locations"] locations_deaths = deaths["locations"] - # locations_recovered = recovered["locations"] + locations_recovered = recovered["locations"] # Final locations to return. locations = [] - + # *************************************************************************** + # TODO: This iteration approach assumes the indexes remain the same + # and opens us to a CRITICAL ERROR. The removal of a column in the data source + # would break the API or SHIFT all the data confirmed, deaths, recovery producting + # incorrect data to consumers. + # *************************************************************************** # Go through locations. for index, location in enumerate(locations_confirmed): # Get the timelines. + + # TEMP: Fix for merging recovery data. See TODO above for more details. + key = (location['country'], location['province']) + timelines = { - "confirmed": locations_confirmed[index]["history"], - "deaths": locations_deaths[index]["history"], - # 'recovered' : locations_recovered[index]['history'], + "confirmed": location["history"], + "deaths": parse_history(key, locations_deaths, index), + "recovered": parse_history(key, locations_recovered, index), } # Grab coordinates. @@ -188,7 +197,12 @@ async def get_locations(): for date, amount in timelines["deaths"].items() } ), - "recovered": Timeline({}), + "recovered": Timeline( + { + datetime.strptime(date, "%m/%d/%y").isoformat() + "Z": amount + for date, amount in timelines["recovered"].items() + } + ), }, ) ) @@ -196,3 +210,21 @@ async def get_locations(): # Finally, return the locations. return locations + + +def parse_history(key: tuple, locations: list, index: int): + """ + Helper for validating and extracting history content from + locations data based on index. Validates with the current country/province + key to make sure no index/column issue. + + TEMP: solution because implement a more efficient and better approach in the refactor. + """ + location_history = {} + try: + if key == (locations[index]["country"], locations[index]["province"]): + location_history = locations[index]["history"] + except IndexError or KeyError as e: + LOGGER.warn(f"iteration data merge error: {index} {key}") + + return location_history \ No newline at end of file diff --git a/tests/test_jhu.py b/tests/test_jhu.py index 3790218d..d3af46bb 100644 --- a/tests/test_jhu.py +++ b/tests/test_jhu.py @@ -22,3 +22,11 @@ async def test_get_locations(mock_client_session): # `jhu.get_locations()` creates id based on confirmed list location_confirmed = await jhu.get_category("confirmed") assert len(output) == len(location_confirmed["locations"]) + + # `jhu.get_locations()` creates id based on deaths list + location_deaths = await jhu.get_category("deaths") + assert len(output) == len(location_deaths["locations"]) + + # `jhu.get_locations()` creates id based on recovered list + location_recovered = await jhu.get_category("recovered") + assert len(output) == len(location_recovered["locations"]) diff --git a/tests/test_routes.py b/tests/test_routes.py index 52d26843..f3730720 100644 --- a/tests/test_routes.py +++ b/tests/test_routes.py @@ -112,8 +112,7 @@ async def test_v2_locations(self): with open(filepath, "r") as file: expected_json_output = file.read() - # TODO: Why is this failing? - # assert return_data == json.loads(expected_json_output) + assert return_data == json.loads(expected_json_output) async def test_v2_locations_id(self): state = "locations" @@ -130,8 +129,7 @@ async def test_v2_locations_id(self): with open(filepath, "r") as file: expected_json_output = file.read() - # TODO: Why is this failing? - # assert return_data == expected_json_output + assert return_data == json.loads(expected_json_output) @pytest.mark.asyncio @@ -183,4 +181,4 @@ async def test_latest(async_api_client, query_params, mock_client_session): assert response.status_code == 200 assert response_json["latest"]["confirmed"] - assert response_json["latest"]["deaths"] + assert response_json["latest"]["deaths"] \ No newline at end of file From 4106b7c63e0ec5ff7f9c5d5521335968783b1cf9 Mon Sep 17 00:00:00 2001 From: codedawi Date: Tue, 19 May 2020 22:54:38 -0400 Subject: [PATCH 14/15] :rotating_light: fix linting issue --- app/services/location/jhu.py | 12 ++++++------ 1 file changed, 6 insertions(+), 6 deletions(-) diff --git a/app/services/location/jhu.py b/app/services/location/jhu.py index 7a622301..7d72557b 100644 --- a/app/services/location/jhu.py +++ b/app/services/location/jhu.py @@ -161,7 +161,7 @@ async def get_locations(): # Get the timelines. # TEMP: Fix for merging recovery data. See TODO above for more details. - key = (location['country'], location['province']) + key = (location["country"], location["province"]) timelines = { "confirmed": location["history"], @@ -216,7 +216,7 @@ def parse_history(key: tuple, locations: list, index: int): """ Helper for validating and extracting history content from locations data based on index. Validates with the current country/province - key to make sure no index/column issue. + key to make sure no index/column issue. TEMP: solution because implement a more efficient and better approach in the refactor. """ @@ -224,7 +224,7 @@ def parse_history(key: tuple, locations: list, index: int): try: if key == (locations[index]["country"], locations[index]["province"]): location_history = locations[index]["history"] - except IndexError or KeyError as e: - LOGGER.warn(f"iteration data merge error: {index} {key}") - - return location_history \ No newline at end of file + except (IndexError, KeyError): + LOGGER.debug(f"iteration data merge error: {index} {key}") + + return location_history From 761f62de7b31561861f146f6ab6bb5e11de45ee3 Mon Sep 17 00:00:00 2001 From: codedawi Date: Tue, 19 May 2020 23:37:09 -0400 Subject: [PATCH 15/15] :white_check_mark: unittest for parse val history --- tests/test_jhu.py | 16 ++++++++++++++++ 1 file changed, 16 insertions(+) diff --git a/tests/test_jhu.py b/tests/test_jhu.py index d3af46bb..f6af4b9e 100644 --- a/tests/test_jhu.py +++ b/tests/test_jhu.py @@ -30,3 +30,19 @@ async def test_get_locations(mock_client_session): # `jhu.get_locations()` creates id based on recovered list location_recovered = await jhu.get_category("recovered") assert len(output) == len(location_recovered["locations"]) + + +@pytest.mark.parametrize( + "key, locations, index, expected", + [ + (("Thailand", "TH"), [{"country": "Thailand", "province": "TH", "history": {"test": "yes"}}], 0, {"test": "yes"}), # Success + (("Deutschland", "DE"), [{"country": "Deutschland", "province": "DE", "history": {"test": "no"}}], 1, {}), # IndexError + (("US", "NJ"), [{"country": "Deutschland", "province": "DE", "history": {"test": "no"}}], 0, {}), # Invaid Key Merge + ], +) +def test_parse_history(key, locations, index, expected): + """ + Test validating and extracting history content from + locations data based on index. + """ + assert jhu.parse_history(key, locations, index) == expected