diff --git a/claims_hosp/.pylintrc b/claims_hosp/.pylintrc index 8ba5e540a..7fc2f5c30 100644 --- a/claims_hosp/.pylintrc +++ b/claims_hosp/.pylintrc @@ -8,7 +8,6 @@ disable=logging-format-interpolation, no-self-use, # Allow pytest classes to have one test. too-few-public-methods, - broad-except [BASIC] diff --git a/claims_hosp/delphi_claims_hosp/download_claims_ftp_files.py b/claims_hosp/delphi_claims_hosp/download_claims_ftp_files.py index 7aa748d4f..6c2a3f184 100644 --- a/claims_hosp/delphi_claims_hosp/download_claims_ftp_files.py +++ b/claims_hosp/delphi_claims_hosp/download_claims_ftp_files.py @@ -5,6 +5,7 @@ import datetime import functools from os import path +import re # third party import paramiko @@ -24,19 +25,13 @@ def print_callback(filename, logger, bytes_so_far, bytes_total): if (rough_percent_transferred % 25) == 0: logger.info("Transfer in progress", filename=filename, percent=rough_percent_transferred) - +FILENAME_TIMESTAMP = re.compile(r".*EDI_AGG_INPATIENT_(?P[0-9]*)_(?P[0-9]*)[^0-9]*") def get_timestamp(name): """Get the reference date in datetime format.""" - try: - split_name = name.split("_") - yyyymmdd = split_name[3] - hhmm = ''.join(filter(str.isdigit, split_name[4])) - timestamp = datetime.datetime.strptime(''.join([yyyymmdd, hhmm]), - "%Y%m%d%H%M") - except Exception: - timestamp = datetime.datetime(1900, 1, 1) - - return timestamp + m = FILENAME_TIMESTAMP.match(name) + if not m: + return datetime.datetime(1900, 1, 1) + return datetime.datetime.strptime(''.join(m.groups()), "%Y%m%d%H%M") def change_date_format(name): """Flip date from YYYYMMDD to MMDDYYYY.""" @@ -75,9 +70,10 @@ def download(ftp_credentials, out_path, logger): files_to_download.append(fileattr.filename) logger.info("File to download", filename=fileattr.filename) - # make sure we don't download more that the 3 chunked drops (2x a day) for OP - # and the 1 chunk (2x a day) for IP - 01/07/21, *2 for multiple day drops - assert len(files_to_download) <= 2 * ((3 * 2) + 2), "more files dropped than expected" + # make sure we don't download more than the 1 chunk (2x a day) drops for IP - 01/07/21, + # *2 for multiple day drops + assert len(files_to_download) <= 2 * (2), \ + f"more files dropped ({len(files_to_download)}) than expected (4)" filepaths_to_download = {} for file in files_to_download: