From 76f278876ef92bcf936d59439000f87fd00c3c97 Mon Sep 17 00:00:00 2001 From: gunnarnewell Date: Wed, 1 Sep 2021 15:18:22 -0700 Subject: [PATCH] Added default dest_dir and internalized the 'feathering' of the inventory file --- pipeline/mrms_download.py | 15 +++++++++++---- 1 file changed, 11 insertions(+), 4 deletions(-) diff --git a/pipeline/mrms_download.py b/pipeline/mrms_download.py index be86897..11145f8 100644 --- a/pipeline/mrms_download.py +++ b/pipeline/mrms_download.py @@ -4,16 +4,23 @@ from urllib.request import urlopen -def download(inventory, dest_dir, max_download=4): - """Given an inventory file for a particular day, and a destination - directory, download all files in the inventory that are not +def download(inventory, dest_dir = os.path.join(os.getcwd(),'Data'), max_download=4): + """Given an inventory file (or dataframe as returned by inventory) + for a particular day, and a destination directory (default to current + directory\Data), download all files in the inventory that are not already in the destination directory with the same size. While at it, don't download more than `max_download` files in this call. """ + + try: + inv_df = feather.read_feather(inventory) + except: + feather.write_feather(inventory, 'temp.feather') + inv_df = feather.read_feather('temp.feather') + os.remove('temp.feather') - inv_df = feather.read_feather(inventory) downloads = 0 if not os.path.exists(dest_dir):