Skip to content

Commit

Permalink
Fixes issue (127) where append_date_dirs wasn't working properly. (#130)
Browse files Browse the repository at this point in the history
* Fixed bug where partition_keys was being modified in-place, and causing problems laters.

* Adjusted how target_path is constructed so that trailing forward slashes in target_path are all removed.

* Added typing.cast to comply with pytype.
  • Loading branch information
pbattaglia committed Mar 31, 2022
1 parent c7a6c4f commit ecb194c
Showing 1 changed file with 7 additions and 4 deletions.
11 changes: 7 additions & 4 deletions weather_dl/download_pipeline/parsers.py
Expand Up @@ -328,8 +328,9 @@ def require(condition: bool, message: str) -> None:
num_partition_keys = len(partition_keys)
if use_date_as_directory(config):
num_partition_keys -= 1
if str(params.get('target_path')).endswith('/'):
params['target_path'] = params.get('target_path')[:-1]
target_path = t.cast(str, params.get('target_path', ''))
if target_path != '':
params['target_path'] = target_path.rstrip('/')

require(num_template_replacements == num_partition_keys,
"""
Expand All @@ -352,10 +353,12 @@ def prepare_target_name(config: Config) -> str:

target_path = t.cast(str, parameters.get('target_path', ''))
target_filename = t.cast(str, parameters.get('target_filename', ''))
partition_keys = t.cast(t.List[str], parameters.get('partition_keys', list()))
partition_keys = t.cast(t.List[str],
cp.copy(parameters.get('partition_keys', list())))

if use_date_as_directory(config):
date_vals = config['selection']['date'][0].split('-')
date = t.cast(str, config['selection']['date'][0])
date_vals = date.split('-')
target_path = os.path.join(target_path, *date_vals)
partition_keys.remove('date')

Expand Down

0 comments on commit ecb194c

Please sign in to comment.