From aef95115e303170dc6a805e8352ca8dee361685e Mon Sep 17 00:00:00 2001 From: Kathryn M Mazaitis Date: Wed, 22 Jun 2022 16:32:50 -0400 Subject: [PATCH 1/2] Remove try/except from get_timestamp by converting to re --- claims_hosp/.pylintrc | 1 - .../download_claims_ftp_files.py | 17 ++++++----------- 2 files changed, 6 insertions(+), 12 deletions(-) diff --git a/claims_hosp/.pylintrc b/claims_hosp/.pylintrc index 8ba5e540a..7fc2f5c30 100644 --- a/claims_hosp/.pylintrc +++ b/claims_hosp/.pylintrc @@ -8,7 +8,6 @@ disable=logging-format-interpolation, no-self-use, # Allow pytest classes to have one test. too-few-public-methods, - broad-except [BASIC] diff --git a/claims_hosp/delphi_claims_hosp/download_claims_ftp_files.py b/claims_hosp/delphi_claims_hosp/download_claims_ftp_files.py index 7aa748d4f..7cb30c60e 100644 --- a/claims_hosp/delphi_claims_hosp/download_claims_ftp_files.py +++ b/claims_hosp/delphi_claims_hosp/download_claims_ftp_files.py @@ -5,6 +5,7 @@ import datetime import functools from os import path +import re # third party import paramiko @@ -24,19 +25,13 @@ def print_callback(filename, logger, bytes_so_far, bytes_total): if (rough_percent_transferred % 25) == 0: logger.info("Transfer in progress", filename=filename, percent=rough_percent_transferred) - +FILENAME_TIMESTAMP = re.compile(r".*EDI_AGG_INPATIENT_(?P[0-9]*)_(?P[0-9]*)[^0-9]*") def get_timestamp(name): """Get the reference date in datetime format.""" - try: - split_name = name.split("_") - yyyymmdd = split_name[3] - hhmm = ''.join(filter(str.isdigit, split_name[4])) - timestamp = datetime.datetime.strptime(''.join([yyyymmdd, hhmm]), - "%Y%m%d%H%M") - except Exception: - timestamp = datetime.datetime(1900, 1, 1) - - return timestamp + m = FILENAME_TIMESTAMP.match(name) + if not m: + return datetime.datetime(1900, 1, 1) + return datetime.datetime.strptime(''.join(m.groups()), "%Y%m%d%H%M") def change_date_format(name): """Flip date from YYYYMMDD to MMDDYYYY.""" From 170f2e349c91c4806f4bc583af44169a6de33d64 Mon Sep 17 00:00:00 2001 From: Kathryn M Mazaitis Date: Fri, 24 Jun 2022 17:26:05 -0400 Subject: [PATCH 2/2] Update expected file count for inpatient-only --- .../delphi_claims_hosp/download_claims_ftp_files.py | 7 ++++--- 1 file changed, 4 insertions(+), 3 deletions(-) diff --git a/claims_hosp/delphi_claims_hosp/download_claims_ftp_files.py b/claims_hosp/delphi_claims_hosp/download_claims_ftp_files.py index 7cb30c60e..6c2a3f184 100644 --- a/claims_hosp/delphi_claims_hosp/download_claims_ftp_files.py +++ b/claims_hosp/delphi_claims_hosp/download_claims_ftp_files.py @@ -70,9 +70,10 @@ def download(ftp_credentials, out_path, logger): files_to_download.append(fileattr.filename) logger.info("File to download", filename=fileattr.filename) - # make sure we don't download more that the 3 chunked drops (2x a day) for OP - # and the 1 chunk (2x a day) for IP - 01/07/21, *2 for multiple day drops - assert len(files_to_download) <= 2 * ((3 * 2) + 2), "more files dropped than expected" + # make sure we don't download more than the 1 chunk (2x a day) drops for IP - 01/07/21, + # *2 for multiple day drops + assert len(files_to_download) <= 2 * (2), \ + f"more files dropped ({len(files_to_download)}) than expected (4)" filepaths_to_download = {} for file in files_to_download: