From 7269e3205cc57ec9769e2e6eab7e2b346daef1de Mon Sep 17 00:00:00 2001 From: Priya Kasimbeg Date: Wed, 15 Nov 2023 17:09:15 +0000 Subject: [PATCH 1/2] change mode of extract method for imagenet --- datasets/dataset_setup.py | 10 +++++----- 1 file changed, 5 insertions(+), 5 deletions(-) diff --git a/datasets/dataset_setup.py b/datasets/dataset_setup.py index 925d542cb..f5366e1e3 100644 --- a/datasets/dataset_setup.py +++ b/datasets/dataset_setup.py @@ -386,11 +386,11 @@ def download_fastmri(data_dir, return data_dir -def extract(source, dest): +def extract(source, dest, mode='r:xz'): if not os.path.exists(dest): os.path.makedirs(dest) logging.info(f'Extracting {source} to {dest}') - tar = tarfile.open(source, 'r:xz') + tar = tarfile.open(source, mode) logging.info('Opened tar') tar.extractall(dest) @@ -543,7 +543,7 @@ def setup_imagenet_pytorch(data_dir): logging.info('Extracting imagenet train data') extract( os.path.join(imagenet_pytorch_data_dir, IMAGENET_TRAIN_TAR_FILENAME), - os.path.join(imagenet_pytorch_data_dir, 'train')) + os.path.join(imagenet_pytorch_data_dir, 'train'), mode='r:') train_tar_filenames = os.listdir( os.path.join(imagenet_pytorch_data_dir, 'train')) @@ -552,13 +552,13 @@ def setup_imagenet_pytorch(data_dir): dir_name = tar_filename[:-4] extract( os.path.join(imagenet_pytorch_data_dir, IMAGENET_TRAIN_TAR_FILENAME), - os.path.join(imagenet_pytorch_data_dir, 'train', dir_name)) + os.path.join(imagenet_pytorch_data_dir, 'train', dir_name), mode='r:') # Extract val data logging.info('Extracting imagenet val data') extract( os.path.join(imagenet_pytorch_data_dir, IMAGENET_VAL_TAR_FILENAME), - os.path.join(imagenet_pytorch_data_dir, 'val')) + os.path.join(imagenet_pytorch_data_dir, 'val'), mode='r:') valprep_command = [ 'wget', From 38fd3524bccc07ff2604bfc91c9b64423d6efa80 Mon Sep 17 00:00:00 2001 From: Priya Kasimbeg Date: Wed, 15 Nov 2023 17:12:10 +0000 Subject: [PATCH 2/2] formatting --- datasets/dataset_setup.py | 9 ++++++--- 1 file changed, 6 insertions(+), 3 deletions(-) diff --git a/datasets/dataset_setup.py b/datasets/dataset_setup.py index f5366e1e3..2f808b64b 100644 --- a/datasets/dataset_setup.py +++ b/datasets/dataset_setup.py @@ -543,7 +543,8 @@ def setup_imagenet_pytorch(data_dir): logging.info('Extracting imagenet train data') extract( os.path.join(imagenet_pytorch_data_dir, IMAGENET_TRAIN_TAR_FILENAME), - os.path.join(imagenet_pytorch_data_dir, 'train'), mode='r:') + os.path.join(imagenet_pytorch_data_dir, 'train'), + mode='r:') train_tar_filenames = os.listdir( os.path.join(imagenet_pytorch_data_dir, 'train')) @@ -552,13 +553,15 @@ def setup_imagenet_pytorch(data_dir): dir_name = tar_filename[:-4] extract( os.path.join(imagenet_pytorch_data_dir, IMAGENET_TRAIN_TAR_FILENAME), - os.path.join(imagenet_pytorch_data_dir, 'train', dir_name), mode='r:') + os.path.join(imagenet_pytorch_data_dir, 'train', dir_name), + mode='r:') # Extract val data logging.info('Extracting imagenet val data') extract( os.path.join(imagenet_pytorch_data_dir, IMAGENET_VAL_TAR_FILENAME), - os.path.join(imagenet_pytorch_data_dir, 'val'), mode='r:') + os.path.join(imagenet_pytorch_data_dir, 'val'), + mode='r:') valprep_command = [ 'wget',