From 3e0084dd8647f5bd2548728796f82ca6d8d9f933 Mon Sep 17 00:00:00 2001 From: Andrey Sheshukov Date: Sat, 20 May 2023 23:50:53 +0300 Subject: [PATCH 1/5] Adding default option of downloading files from given URL, instead of fixed github one --- python/snewpy/_model_downloader.py | 35 ++++++++++++---------------- python/snewpy/models/model_files.yml | 6 ++++- 2 files changed, 20 insertions(+), 21 deletions(-) diff --git a/python/snewpy/_model_downloader.py b/python/snewpy/_model_downloader.py index 1a5982211..35788cfa1 100644 --- a/python/snewpy/_model_downloader.py +++ b/python/snewpy/_model_downloader.py @@ -101,7 +101,7 @@ def load(self) -> Path: return self.path -def from_zenodo(zenodo_id:str, model:str, filename:str, path:str=model_path): +def from_zenodo(zenodo_id:str, model:str, filename:str): """Access files on Zenodo. Parameters @@ -113,7 +113,7 @@ def from_zenodo(zenodo_id:str, model:str, filename:str, path:str=model_path): Returns ------- - file : FileHandle object. + file_url, md5sum """ zenodo_url = f'https://zenodo.org/api/records/{zenodo_id}' path = Path(path)/str(model) @@ -124,17 +124,13 @@ def from_zenodo(zenodo_id:str, model:str, filename:str, path:str=model_path): # Search for model file string in Zenodo request for this record. file = next((_file for _file in record['files'] if _file['key'] == filename), None) - # If matched, return a FileHandle which takes care of downloading and - # checksum (or loads a local data file). Otherwise raise an exception. + # If matched, return a tuple of URL and checksum.Otherwise raise an exception. if file is not None: - return FileHandle(path = path/file['key'], - remote= file['links']['self'], - md5 = file['checksum'].split(':')[1]) + return file['links']['self'], file['checksum'].split(':')[1] else: raise MissingFileError(filename) - -def from_github(release_version:str, model:str, filename:str, path:str=model_path): +def from_github(release_version:str, model:str, filename:str): """Access files on GitHub. Parameters @@ -146,15 +142,10 @@ def from_github(release_version:str, model:str, filename:str, path:str=model_pat Returns ------- - file : FileHandle object. + file_url, md5sum """ github_url = f'https://github.com/SNEWS2/snewpy/raw/v{release_version}/models/{model}/{filename}' - localpath = Path(path)/str(model) - localpath.mkdir(exist_ok=True, parents=True) - - return FileHandle(path = localpath/filename, - remote = github_url) - + return github_url, None def get_model_data(model: str, filename: str, path: str = model_path) -> Path: """Access model data. Configuration for each model is in a YAML file @@ -183,16 +174,20 @@ def get_model_data(model: str, filename: str, path: str = model_path) -> Path: if model in models.keys(): # Get data from GitHub or Zenodo. modconf = models[model] - repo = modconf['repository'] + repo = modconf.pop('repository') if repo == 'github': params['release_version'] = modconf['release_version'] - fh = from_github(**params) + url, md5 = from_github(**params) elif repo == 'zenodo': params['zenodo_id'] = modconf['zenodo_id'] - fh = from_zenodo(**params) + url, md5 = from_zenodo(**params) else: - raise ValueError(f'Repository {repo} not recognized') + #format the url directly + url, md5 = repo.format(**params, **modconf), None + localpath = Path(path)/str(model) + localpath.mkdir(exist_ok=True, parents=True) + fh = FileHandle(path = localpath/filename,remote = url, md5=md5) return fh.load() else: raise KeyError(f'No configuration for {model}') diff --git a/python/snewpy/models/model_files.yml b/python/snewpy/models/model_files.yml index 291661886..9c18785f8 100644 --- a/python/snewpy/models/model_files.yml +++ b/python/snewpy/models/model_files.yml @@ -1,10 +1,14 @@ # Define remote locations of simulation files for each model. # 1. Model data on GitHub must provide the SNEWPY release version. # 2. Model data on Zenodo must provide the Zenodo record ID. + +config: + - &github "https://github.com/SNEWS2/snewpy/raw/v{release_version}/models/{model}/{filename}" + models: Bollig_2016: - repository: github + repository: *github release_version: 1.2 Fornax_2019: From f63b214d1e497eb2d10c266db38a50aad98ea14d Mon Sep 17 00:00:00 2001 From: Andrey Sheshukov Date: Sat, 20 May 2023 23:58:05 +0300 Subject: [PATCH 2/5] Fixed the function call arguments --- python/snewpy/_model_downloader.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/python/snewpy/_model_downloader.py b/python/snewpy/_model_downloader.py index 35788cfa1..aace15b8a 100644 --- a/python/snewpy/_model_downloader.py +++ b/python/snewpy/_model_downloader.py @@ -164,7 +164,7 @@ def get_model_data(model: str, filename: str, path: str = model_path) -> Path: if os.path.isabs(filename): return Path(filename) - params = { 'model':model, 'filename':filename, 'path':path } + params = { 'model':model, 'filename':filename } # Parse YAML file with model repository configurations. with open_text('snewpy.models', 'model_files.yml') as f: From 5de7a5aa7c99a16be55ff5bfcfdc7e82fdc0c868 Mon Sep 17 00:00:00 2001 From: Andrey Sheshukov Date: Sat, 3 Jun 2023 14:09:11 +0300 Subject: [PATCH 3/5] Remove the 'github' repository option --- python/snewpy/_model_downloader.py | 28 +--------------------------- 1 file changed, 1 insertion(+), 27 deletions(-) diff --git a/python/snewpy/_model_downloader.py b/python/snewpy/_model_downloader.py index aace15b8a..b014b8116 100644 --- a/python/snewpy/_model_downloader.py +++ b/python/snewpy/_model_downloader.py @@ -109,18 +109,13 @@ def from_zenodo(zenodo_id:str, model:str, filename:str): zenodo_id : Zenodo record for model files. model : Name of the model class for this model file. filename : Expected filename storing simulation data. - path : Local installation path (defaults to astropy cache). Returns ------- file_url, md5sum """ zenodo_url = f'https://zenodo.org/api/records/{zenodo_id}' - path = Path(path)/str(model) - path.mkdir(exist_ok=True, parents=True) - record = requests.get(zenodo_url).json() - # Search for model file string in Zenodo request for this record. file = next((_file for _file in record['files'] if _file['key'] == filename), None) @@ -130,23 +125,6 @@ def from_zenodo(zenodo_id:str, model:str, filename:str): else: raise MissingFileError(filename) -def from_github(release_version:str, model:str, filename:str): - """Access files on GitHub. - - Parameters - ---------- - release_version: SNEWPY release with corresponding model data. - model : Name of the model class for this model file. - filename : Expected filename storing simulation data. - path : Local installation path (defaults to astropy cache). - - Returns - ------- - file_url, md5sum - """ - github_url = f'https://github.com/SNEWS2/snewpy/raw/v{release_version}/models/{model}/{filename}' - return github_url, None - def get_model_data(model: str, filename: str, path: str = model_path) -> Path: """Access model data. Configuration for each model is in a YAML file distributed with SNEWPY. @@ -175,11 +153,7 @@ def get_model_data(model: str, filename: str, path: str = model_path) -> Path: # Get data from GitHub or Zenodo. modconf = models[model] repo = modconf.pop('repository') - - if repo == 'github': - params['release_version'] = modconf['release_version'] - url, md5 = from_github(**params) - elif repo == 'zenodo': + if repo == 'zenodo': params['zenodo_id'] = modconf['zenodo_id'] url, md5 = from_zenodo(**params) else: From f25a98b2fcc91e17f2faceb7121beb2daae94555 Mon Sep 17 00:00:00 2001 From: Andrey Sheshukov Date: Sat, 3 Jun 2023 14:24:48 +0300 Subject: [PATCH 4/5] Provide {snewpy_version} default variable. Remove 'github' repository option --- python/snewpy/_model_downloader.py | 7 +++-- python/snewpy/models/model_files.yml | 41 ++++++++++------------------ 2 files changed, 19 insertions(+), 29 deletions(-) diff --git a/python/snewpy/_model_downloader.py b/python/snewpy/_model_downloader.py index b014b8116..97b6c8641 100644 --- a/python/snewpy/_model_downloader.py +++ b/python/snewpy/_model_downloader.py @@ -18,6 +18,7 @@ from typing import Optional from snewpy import model_path +from snewpy import __version__ as snewpy_version import logging logger = logging.getLogger('FileHandle') @@ -142,7 +143,7 @@ def get_model_data(model: str, filename: str, path: str = model_path) -> Path: if os.path.isabs(filename): return Path(filename) - params = { 'model':model, 'filename':filename } + params = { 'model':model, 'filename':filename, 'snewpy_version':snewpy_version} # Parse YAML file with model repository configurations. with open_text('snewpy.models', 'model_files.yml') as f: @@ -158,9 +159,11 @@ def get_model_data(model: str, filename: str, path: str = model_path) -> Path: url, md5 = from_zenodo(**params) else: #format the url directly - url, md5 = repo.format(**params, **modconf), None + params.update(modconf) #default parameters can be overriden in the model config + url, md5 = repo.format(**params), None localpath = Path(path)/str(model) localpath.mkdir(exist_ok=True, parents=True) + print(f'URL={url}') fh = FileHandle(path = localpath/filename,remote = url, md5=md5) return fh.load() else: diff --git a/python/snewpy/models/model_files.yml b/python/snewpy/models/model_files.yml index 9c18785f8..d03deb1b5 100644 --- a/python/snewpy/models/model_files.yml +++ b/python/snewpy/models/model_files.yml @@ -3,59 +3,46 @@ # 2. Model data on Zenodo must provide the Zenodo record ID. config: - - &github "https://github.com/SNEWS2/snewpy/raw/v{release_version}/models/{model}/{filename}" + - &snewpy "https://github.com/SNEWS2/snewpy/raw/v{snewpy_version}/models/{model}/{filename}" models: Bollig_2016: - repository: *github - release_version: 1.2 + repository: *snewpy Fornax_2019: - repository: github - release_version: 1.2 + repository: *snewpy Fornax_2021: - repository: github - release_version: 1.2 + repository: *snewpy Kuroda_2020: - repository: github - release_version: 1.2 + repository: *snewpy Nakazato_2013: - repository: github - release_version: 1.2 + repository: *snewpy OConnor_2013: - repository: github - release_version: 1.2 + repository: *snewpy OConnor_2015: - repository: github - release_version: 1.2 + repository: *snewpy Sukhbold_2015: - repository: github - release_version: 1.2 + repository: *snewpy Tamborra_2014: - repository: github - release_version: 1.2 + repository: *snewpy Walk_2018: - repository: github - release_version: 1.2 + repository: *snewpy Walk_2019: - repository: github - release_version: 1.2 + repository: *snewpy Warren_2020: - repository: github - release_version: 1.2 + repository: *snewpy Zha_2021: - repository: github - release_version: 1.2 + repository: *snewpy From 8891bc7194bbb60b3b09c6dbea39472e69edf4c9 Mon Sep 17 00:00:00 2001 From: Andrey Sheshukov Date: Sat, 3 Jun 2023 14:38:38 +0300 Subject: [PATCH 5/5] remove debug output --- python/snewpy/_model_downloader.py | 1 - 1 file changed, 1 deletion(-) diff --git a/python/snewpy/_model_downloader.py b/python/snewpy/_model_downloader.py index 97b6c8641..9b8bbfe06 100644 --- a/python/snewpy/_model_downloader.py +++ b/python/snewpy/_model_downloader.py @@ -163,7 +163,6 @@ def get_model_data(model: str, filename: str, path: str = model_path) -> Path: url, md5 = repo.format(**params), None localpath = Path(path)/str(model) localpath.mkdir(exist_ok=True, parents=True) - print(f'URL={url}') fh = FileHandle(path = localpath/filename,remote = url, md5=md5) return fh.load() else: