From 32d6df753bcaa7da4328d9323430801379696abe Mon Sep 17 00:00:00 2001 From: Amaris Sim Date: Wed, 25 Jun 2025 14:14:46 -0400 Subject: [PATCH 1/3] changed logic to throw error --- nhsn/delphi_nhsn/pull.py | 7 ++++--- nhsn/tests/test_pull.py | 15 +++++++++++++++ 2 files changed, 19 insertions(+), 3 deletions(-) diff --git a/nhsn/delphi_nhsn/pull.py b/nhsn/delphi_nhsn/pull.py index 855c0952f..1f0c4ed94 100644 --- a/nhsn/delphi_nhsn/pull.py +++ b/nhsn/delphi_nhsn/pull.py @@ -33,7 +33,6 @@ def check_last_updated(socrata_token, dataset_id, logger): ------- """ - recently_updated_source = True try: client = Socrata("data.cdc.gov", socrata_token) response = client.get_metadata(dataset_id) @@ -49,10 +48,12 @@ def check_last_updated(socrata_token, dataset_id, logger): ) else: logger.info(f"{prelim_prefix}NHSN data is stale; Skipping", updated_timestamp=updated_timestamp) + return recently_updated_source # pylint: disable=W0703 except Exception as e: - logger.info("error while processing socrata metadata; treating data as stale", error=str(e)) - return recently_updated_source + logger.info("error while processing socrata metadata", error=str(e)) + raise e + def pull_data(socrata_token: str, dataset_id: str, backup_dir: str, logger): diff --git a/nhsn/tests/test_pull.py b/nhsn/tests/test_pull.py index ec2de51ef..7b98588db 100644 --- a/nhsn/tests/test_pull.py +++ b/nhsn/tests/test_pull.py @@ -176,3 +176,18 @@ def test_check_last_updated(self, mock_socrata, dataset, updatedAt, caplog): stale_msg = f"{dataset['msg_prefix']}NHSN data is stale; Skipping" assert stale_msg in caplog.text + + + @patch("delphi_nhsn.pull.Socrata") + @pytest.mark.parametrize('dataset', DATASETS, ids=["data", "prelim_data"]) + def test_check_last_updated_error(self, mock_socrata, dataset, caplog): + mock_client = MagicMock() + mock_socrata.return_value = mock_client + updatedAt = time.time() + mock_client.get_metadata.return_value = {"rowsUpdatedAt": updatedAt } + logger = get_structured_logger() + + with pytest.raises(KeyError): + check_last_updated(mock_client, dataset["id"], logger) + assert "error while processing socrata metadata" in caplog.text + From f9502409c90ea3dbd624d961b884b203cb827083 Mon Sep 17 00:00:00 2001 From: aysim319 Date: Thu, 26 Jun 2025 08:48:31 -0400 Subject: [PATCH 2/3] Update nhsn/delphi_nhsn/pull.py Co-authored-by: george --- nhsn/delphi_nhsn/pull.py | 5 ++--- 1 file changed, 2 insertions(+), 3 deletions(-) diff --git a/nhsn/delphi_nhsn/pull.py b/nhsn/delphi_nhsn/pull.py index 1f0c4ed94..e05d42e9d 100644 --- a/nhsn/delphi_nhsn/pull.py +++ b/nhsn/delphi_nhsn/pull.py @@ -51,9 +51,8 @@ def check_last_updated(socrata_token, dataset_id, logger): return recently_updated_source # pylint: disable=W0703 except Exception as e: - logger.info("error while processing socrata metadata", error=str(e)) - raise e - + logger.error("error while processing socrata metadata", error=str(e)) + raise def pull_data(socrata_token: str, dataset_id: str, backup_dir: str, logger): From 8f7fed8fe027849bf3d186c58a1634bdc083b1cb Mon Sep 17 00:00:00 2001 From: aysim319 Date: Thu, 26 Jun 2025 08:49:57 -0400 Subject: [PATCH 3/3] Apply suggestions from code review Co-authored-by: george --- nhsn/tests/test_pull.py | 9 ++++----- 1 file changed, 4 insertions(+), 5 deletions(-) diff --git a/nhsn/tests/test_pull.py b/nhsn/tests/test_pull.py index 7b98588db..0c6da19e9 100644 --- a/nhsn/tests/test_pull.py +++ b/nhsn/tests/test_pull.py @@ -179,15 +179,14 @@ def test_check_last_updated(self, mock_socrata, dataset, updatedAt, caplog): @patch("delphi_nhsn.pull.Socrata") - @pytest.mark.parametrize('dataset', DATASETS, ids=["data", "prelim_data"]) - def test_check_last_updated_error(self, mock_socrata, dataset, caplog): + def test_check_last_updated_error(self, mock_socrata, caplog): mock_client = MagicMock() + # mock metadata missing important field, "viewLastModified": + mock_client.get_metadata.return_value = {"rowsUpdatedAt": time.time()} mock_socrata.return_value = mock_client - updatedAt = time.time() - mock_client.get_metadata.return_value = {"rowsUpdatedAt": updatedAt } logger = get_structured_logger() with pytest.raises(KeyError): - check_last_updated(mock_client, dataset["id"], logger) + check_last_updated("fakesocratatoken", MAIN_DATASET_ID, logger) assert "error while processing socrata metadata" in caplog.text