diff --git a/cosmotech/coal/cosmotech_api/dataset/download/file.py b/cosmotech/coal/cosmotech_api/dataset/download/file.py index 9bf4d53f..418459d0 100644 --- a/cosmotech/coal/cosmotech_api/dataset/download/file.py +++ b/cosmotech/coal/cosmotech_api/dataset/download/file.py @@ -118,7 +118,7 @@ def process_txt(target_file) -> Dict[str, Any]: LOGGER.debug(T("coal.services.dataset.processing_text").format(file_name=target_file)) with open(target_file, "r") as _file: current_filename = os.path.basename(target_file) - content[current_filename] = "\n".join(line for line in _file) + content[current_filename] = "".join(line for line in _file) line_count = content[current_filename].count("\n") + 1 LOGGER.debug( @@ -128,7 +128,7 @@ def process_txt(target_file) -> Dict[str, Any]: def read_file(file_name, file): - @timed(f"process{file_name}", debug=True) + @timed(f"process {file_name}", debug=True) def timed_read_file(file_name, file): content = {} if ".xls" in file_name: diff --git a/cosmotech/coal/cosmotech_api/runner/datasets.py b/cosmotech/coal/cosmotech_api/runner/datasets.py index c5198dad..91256705 100644 --- a/cosmotech/coal/cosmotech_api/runner/datasets.py +++ b/cosmotech/coal/cosmotech_api/runner/datasets.py @@ -63,9 +63,9 @@ def download_dataset( csm_version = semver_of("cosmotech_api") if csm_version.major >= 5: - download_dataset_v5(organization_id, workspace_id, dataset_id, read_files) + return download_dataset_v5(organization_id, workspace_id, dataset_id, read_files) else: - download_dataset_v4(organization_id, workspace_id, dataset_id, read_files) + return download_dataset_v4(organization_id, workspace_id, dataset_id, read_files) def download_dataset_v5( @@ -94,6 +94,7 @@ def download_dataset_v5( workspace_id=workspace_id, dataset_id=dataset_id) + content = dict() tmp_dataset_dir = tempfile.mkdtemp() tmp_dataset_dir_path = Path(tmp_dataset_dir) for part in dataset.parts: @@ -103,10 +104,10 @@ def download_dataset_v5( binary_file.write(data_part) if read_files: - content = file.read_file(part_file_path) + content.update(file.read_file(part.source_name, part_file_path)) return { - "type": "twincache", + "type": "csm_dataset", "content": content, "name": dataset.name, "folder_path": str(part_file_path), diff --git a/cosmotech/coal/utils/decorator.py b/cosmotech/coal/utils/decorator.py index faea44cc..b436d153 100644 --- a/cosmotech/coal/utils/decorator.py +++ b/cosmotech/coal/utils/decorator.py @@ -11,12 +11,13 @@ def decorator(func): @wraps(func) def wrapper(*args, **kwargs): process_start = time.time() - func(*args, **kwargs) + r = func(*args, **kwargs) process_time = time.time() - process_start msg = T("coal.common.timing.operation_completed").format(operation=operation, time=process_time) if debug: LOGGER.debug(msg) else: LOGGER.info(msg) + return r return wrapper return decorator diff --git a/cosmotech/coal/utils/semver.py b/cosmotech/coal/utils/semver.py new file mode 100644 index 00000000..6a23ed87 --- /dev/null +++ b/cosmotech/coal/utils/semver.py @@ -0,0 +1,6 @@ +from packaging.version import Version +import importlib.metadata + + +def semver_of(package: str) -> Version: + return Version(importlib.metadata.version(package))