diff --git a/CHANGELOG.md b/CHANGELOG.md index 7460b49b4b..b94c363ff5 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -25,6 +25,7 @@ and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0 - Added `custom_source_time` parameter to `ComponentModeler` classes (`ModalComponentModeler` and `TerminalComponentModeler`), allowing specification of custom source time dependence. - Validation for `run_only` field in component modelers to catch duplicate or invalid matrix indices early with clear error messages. - Introduced a profile-based configuration manager with TOML persistence and runtime overrides exposed via `tidy3d.config`. +- Added support of `os.PathLike` objects as paths like `pathlib.Path` alongside `str` paths in all path-related functions. ### Changed - Improved performance of antenna metrics calculation by utilizing cached wave amplitude calculations instead of recomputing wave amplitudes for each port excitation in the `TerminalComponentModelerData`. diff --git a/tests/test_web/test_tidy3d_stub.py b/tests/test_web/test_tidy3d_stub.py index 4e0d94007f..b51d5d519e 100644 --- a/tests/test_web/test_tidy3d_stub.py +++ b/tests/test_web/test_tidy3d_stub.py @@ -1,8 +1,10 @@ from __future__ import annotations import os +from pathlib import Path import numpy as np +import pytest import responses import tidy3d as td @@ -162,6 +164,40 @@ def test_stub_data_lazy_loading(tmp_path): _ = sim_data.monitor_data +@pytest.mark.parametrize( + "path_builder", + ( + lambda tmp_path, name: Path(tmp_path) / name, + lambda tmp_path, name: str(Path(tmp_path) / name), + ), +) +def test_stub_pathlike_roundtrip(tmp_path, path_builder): + """Ensure stub read/write helpers accept pathlib.Path and posixpath inputs.""" + + # Simulation stub roundtrip + sim = make_sim() + stub = Tidy3dStub(simulation=sim) + sim_path = path_builder(tmp_path, "pathlike_sim.json") + stub.to_file(sim_path) + assert os.path.exists(sim_path) + sim_loaded = Tidy3dStub.from_file(sim_path) + assert sim_loaded == sim + + # Simulation data stub roundtrip + sim_data = make_sim_data() + stub_data = Tidy3dStubData(data=sim_data) + data_path = path_builder(tmp_path, "pathlike_data.hdf5") + stub_data.to_file(data_path) + assert os.path.exists(data_path) + + data_loaded = Tidy3dStubData.from_file(data_path) + assert data_loaded.simulation == sim_data.simulation + + # Postprocess using the same PathLike ensures downstream helpers accept the type + processed = Tidy3dStubData.postprocess(data_path, lazy=True) + assert isinstance(processed, SimulationData) + + def test_default_task_name(): sim = make_sim() stub = Tidy3dStub(simulation=sim) diff --git a/tests/test_web/test_tidy3d_task.py b/tests/test_web/test_tidy3d_task.py index 2030e68a2e..5270964df8 100644 --- a/tests/test_web/test_tidy3d_task.py +++ b/tests/test_web/test_tidy3d_task.py @@ -376,3 +376,16 @@ def mock(*args, **kwargs): task.get_log(LOG_FNAME) with open(LOG_FNAME) as f: assert f.read() == "0.3,5.7" + + +@responses.activate +def test_get_running_tasks(set_api_key): + responses.add( + responses.GET, + f"{Env.current.web_api_endpoint}/tidy3d/py/tasks", + json={"data": [{"taskId": "1234", "status": "queued"}]}, + status=200, + ) + + tasks = SimulationTask.get_running_tasks() + assert len(tasks) == 1 diff --git a/tests/test_web/test_webapi.py b/tests/test_web/test_webapi.py index 3d4c7ce949..2b7c6e8a1f 100644 --- a/tests/test_web/test_webapi.py +++ b/tests/test_web/test_webapi.py @@ -2,7 +2,10 @@ from __future__ import annotations import os +import posixpath from concurrent.futures import Future +from os import PathLike +from pathlib import Path from types import SimpleNamespace import numpy as np @@ -739,8 +742,8 @@ def status(self): events.append((self.task_id, "status", status)) return status - def download(self, path: str): - events.append((self.task_id, "download", path)) + def download(self, path: PathLike): + events.append((self.task_id, "download", str(path))) monkeypatch.setattr("tidy3d.web.api.container.ThreadPoolExecutor", ImmediateExecutor) monkeypatch.setattr("tidy3d.web.api.container.time.sleep", lambda *_args, **_kwargs: None) @@ -766,7 +769,7 @@ def download(self, path: str): } for task_id, _, path in downloads: - assert path == expected_paths[task_id] + assert str(path) == expected_paths[task_id] job1_download_idx = next( i @@ -797,8 +800,8 @@ def status(self): events.append((self.task_id, "status", status)) return status - def download(self, path: str): - events.append((self.task_id, "download", path)) + def download(self, path: PathLike): + events.append((self.task_id, "download", str(path))) monkeypatch.setattr("tidy3d.web.api.container.ThreadPoolExecutor", ImmediateExecutor) monkeypatch.setattr("tidy3d.web.api.container.time.sleep", lambda *_args, **_kwargs: None) @@ -819,6 +822,7 @@ def download(self, path: str): batch.monitor(download_on_success=True, path_dir=str(tmp_path)) downloads = [event for event in events if event[1] == "download"] + assert downloads == [("task_b_id", "download", os.path.join(str(tmp_path), "task_b_id.hdf5"))] @@ -996,3 +1000,92 @@ def test_run_single_offline_eager(monkeypatch, tmp_path): assert isinstance(sim_data, SimulationData) assert sim_data.__class__.__name__ == "SimulationData" # no proxy + + +class FauxPath: + """Minimal PathLike to exercise __fspath__ support.""" + + def __init__(self, path: PathLike | str): + self._p = os.fspath(path) + + def __fspath__(self) -> str: + return self._p + + +def _pathlib_builder(tmp_path, name: str): + return Path(tmp_path) / name + + +def _posix_builder(tmp_path, name: str): + return posixpath.join(tmp_path.as_posix(), name) + + +def _str_builder(tmp_path, name: str): + return str(Path(tmp_path) / name) + + +def _fspath_builder(tmp_path, name: str): + return FauxPath(Path(tmp_path) / name) + + +@pytest.mark.parametrize( + "path_builder", + [_pathlib_builder, _posix_builder, _str_builder, _fspath_builder], + ids=["pathlib.Path", "posixpath_str", "str", "PathLike"], +) +def test_run_single_offline_eager_accepts_pathlikes(monkeypatch, tmp_path, path_builder): + """run(sim, path=...) accepts any PathLike.""" + sim = make_sim() + task_name = "pathlike_single" + out_file = path_builder(tmp_path, "sim.hdf5") + + # Patch webapi for offline run and to write to the provided path + apply_common_patches(monkeypatch, tmp_path, taskid_to_sim={task_name: sim}) + + sim_data = run(sim, task_name=task_name, path=out_file) + + # File existed (written via patched load) and types are correct + assert os.path.exists(os.fspath(out_file)) + assert isinstance(sim_data, SimulationData) + assert sim_data.simulation == sim + + +@pytest.mark.parametrize( + "path_builder", + [_pathlib_builder, _posix_builder, _str_builder, _fspath_builder], + ids=["pathlib.Path", "posixpath_str", "str", "PathLike"], +) +def test_job_run_accepts_pathlikes(monkeypatch, tmp_path, path_builder): + """Job.run(path=...) accepts any PathLike.""" + sim = make_sim() + task_name = "job_pathlike" + out_file = path_builder(tmp_path, "job_out.hdf5") + + apply_common_patches(monkeypatch, tmp_path, taskid_to_sim={task_name: sim}) + + j = Job(simulation=sim, task_name=task_name, folder_name=PROJECT_NAME) + _ = j.run(path=out_file) + + assert os.path.exists(os.fspath(out_file)) + + +@pytest.mark.parametrize( + "dir_builder", + [_pathlib_builder, _posix_builder, _str_builder, _fspath_builder], + ids=["pathlib.Path", "posixpath_str", "str", "PathLike"], +) +def test_batch_run_accepts_pathlike_dir(monkeypatch, tmp_path, dir_builder): + """Batch.run(path_dir=...) accepts any PathLike directory location.""" + sims = {"A": make_sim(), "B": make_sim()} + out_dir = dir_builder(tmp_path, "batch_out") + + # Map task_ids to sims: upload() is patched to return task_name, which for dict input + # corresponds to the dict keys ("A", "B"), so we map those. + apply_common_patches(monkeypatch, tmp_path, taskid_to_sim={"A": sims["A"], "B": sims["B"]}) + + b = Batch(simulations=sims, folder_name=PROJECT_NAME) + b.run(path_dir=out_dir) + + # Directory created and two .hdf5 outputs produced + out_dir_str = os.fspath(out_dir) + assert os.path.isdir(out_dir_str) diff --git a/tidy3d/components/base.py b/tidy3d/components/base.py index 5498cf1738..6b7f2be82b 100644 --- a/tidy3d/components/base.py +++ b/tidy3d/components/base.py @@ -7,10 +7,11 @@ import json import math import os -import pathlib import tempfile from functools import wraps from math import ceil +from os import PathLike +from pathlib import Path from typing import Any, Callable, Literal, Optional, Union import h5py @@ -112,12 +113,13 @@ def make_json_compatible(json_string: str) -> str: return json_string.replace(tmp_string, '"-Infinity"') -def _get_valid_extension(fname: str) -> str: +def _get_valid_extension(fname: PathLike) -> str: """Return the file extension from fname, validated to accepted ones.""" valid_extensions = [".json", ".yaml", ".hdf5", ".h5", ".hdf5.gz"] - extensions = [s.lower() for s in pathlib.Path(fname).suffixes[-2:]] + path = Path(fname) + extensions = [s.lower() for s in path.suffixes[-2:]] if len(extensions) == 0: - raise FileError(f"File '{fname}' missing extension.") + raise FileError(f"File '{path}' missing extension.") single_extension = extensions[-1] if single_extension in valid_extensions: return single_extension @@ -125,7 +127,7 @@ def _get_valid_extension(fname: str) -> str: if double_extension in valid_extensions: return double_extension raise FileError( - f"File extension must be one of {', '.join(valid_extensions)}; file '{fname}' does not " + f"File extension must be one of {', '.join(valid_extensions)}; file '{path}' does not " "match any of those." ) @@ -364,7 +366,7 @@ def help(self, methods: bool = False) -> None: @classmethod def from_file( cls, - fname: str, + fname: PathLike, group_path: Optional[str] = None, lazy: bool = False, on_load: Optional[Callable] = None, @@ -374,7 +376,7 @@ def from_file( Parameters ---------- - fname : str + fname : PathLike Full path to the file to load the :class:`Tidy3dBaseModel` from. group_path : str | None = None Path to a group inside the file to use as the base level. Only for hdf5 files. @@ -409,12 +411,12 @@ def from_file( return obj @classmethod - def dict_from_file(cls, fname: str, group_path: Optional[str] = None) -> dict: + def dict_from_file(cls, fname: PathLike, group_path: Optional[str] = None) -> dict: """Loads a dictionary containing the model from a .yaml, .json, .hdf5, or .hdf5.gz file. Parameters ---------- - fname : str + fname : PathLike Full path to the file to load the :class:`Tidy3dBaseModel` from. group_path : str, optional Path to a group inside the file to use as the base level. @@ -428,9 +430,9 @@ def dict_from_file(cls, fname: str, group_path: Optional[str] = None) -> dict: ------- >>> simulation = Simulation.from_file(fname='folder/sim.json') # doctest: +SKIP """ - - extension = _get_valid_extension(fname) - kwargs = {"fname": fname} + fname_path = Path(fname) + extension = _get_valid_extension(fname_path) + kwargs = {"fname": fname_path} if group_path is not None: if extension == ".hdf5" or extension == ".hdf5.gz": @@ -447,19 +449,18 @@ def dict_from_file(cls, fname: str, group_path: Optional[str] = None) -> dict: }[extension] return converter(**kwargs) - def to_file(self, fname: str) -> None: + def to_file(self, fname: PathLike) -> None: """Exports :class:`Tidy3dBaseModel` instance to .yaml, .json, or .hdf5 file Parameters ---------- - fname : str + fname : PathLike Full path to the .yaml or .json file to save the :class:`Tidy3dBaseModel` to. Example ------- >>> simulation.to_file(fname='folder/sim.json') # doctest: +SKIP """ - extension = _get_valid_extension(fname) converter = { ".json": self.to_json, @@ -470,12 +471,12 @@ def to_file(self, fname: str) -> None: return converter(fname=fname) @classmethod - def from_json(cls, fname: str, **parse_obj_kwargs) -> Self: + def from_json(cls, fname: PathLike, **parse_obj_kwargs) -> Self: """Load a :class:`Tidy3dBaseModel` from .json file. Parameters ---------- - fname : str + fname : PathLike Full path to the .json file to load the :class:`Tidy3dBaseModel` from. Returns @@ -493,12 +494,12 @@ def from_json(cls, fname: str, **parse_obj_kwargs) -> Self: return cls.parse_obj(model_dict, **parse_obj_kwargs) @classmethod - def dict_from_json(cls, fname: str) -> dict: + def dict_from_json(cls, fname: PathLike) -> dict: """Load dictionary of the model from a .json file. Parameters ---------- - fname : str + fname : PathLike Full path to the .json file to load the :class:`Tidy3dBaseModel` from. Returns @@ -514,12 +515,12 @@ def dict_from_json(cls, fname: str) -> dict: model_dict = json.load(json_fhandle) return model_dict - def to_json(self, fname: str) -> None: + def to_json(self, fname: PathLike) -> None: """Exports :class:`Tidy3dBaseModel` instance to .json file Parameters ---------- - fname : str + fname : PathLike Full path to the .json file to save the :class:`Tidy3dBaseModel` to. Example @@ -529,16 +530,18 @@ def to_json(self, fname: str) -> None: export_model = self.to_static() json_string = export_model._json(indent=INDENT_JSON_FILE) self._warn_if_contains_data(json_string) - with open(fname, "w", encoding="utf-8") as file_handle: + path = Path(fname) + path.parent.mkdir(parents=True, exist_ok=True) + with path.open("w", encoding="utf-8") as file_handle: file_handle.write(json_string) @classmethod - def from_yaml(cls, fname: str, **parse_obj_kwargs) -> Self: + def from_yaml(cls, fname: PathLike, **parse_obj_kwargs) -> Self: """Loads :class:`Tidy3dBaseModel` from .yaml file. Parameters ---------- - fname : str + fname : PathLike Full path to the .yaml file to load the :class:`Tidy3dBaseModel` from. **parse_obj_kwargs Keyword arguments passed to pydantic's ``parse_obj`` method. @@ -556,12 +559,12 @@ def from_yaml(cls, fname: str, **parse_obj_kwargs) -> Self: return cls.parse_obj(model_dict, **parse_obj_kwargs) @classmethod - def dict_from_yaml(cls, fname: str) -> dict: + def dict_from_yaml(cls, fname: PathLike) -> dict: """Load dictionary of the model from a .yaml file. Parameters ---------- - fname : str + fname : PathLike Full path to the .yaml file to load the :class:`Tidy3dBaseModel` from. Returns @@ -577,12 +580,12 @@ def dict_from_yaml(cls, fname: str) -> dict: model_dict = yaml.safe_load(yaml_in) return model_dict - def to_yaml(self, fname: str) -> None: + def to_yaml(self, fname: PathLike) -> None: """Exports :class:`Tidy3dBaseModel` instance to .yaml file. Parameters ---------- - fname : str + fname : PathLike Full path to the .yaml file to save the :class:`Tidy3dBaseModel` to. Example @@ -593,7 +596,9 @@ def to_yaml(self, fname: str) -> None: json_string = export_model._json() self._warn_if_contains_data(json_string) model_dict = json.loads(json_string) - with open(fname, "w+", encoding="utf-8") as file_handle: + path = Path(fname) + path.parent.mkdir(parents=True, exist_ok=True) + with path.open("w+", encoding="utf-8") as file_handle: yaml.dump(model_dict, file_handle, indent=INDENT_JSON_FILE) @staticmethod @@ -657,7 +662,7 @@ def _json_string_key(index: int) -> str: return JSON_TAG @classmethod - def _json_string_from_hdf5(cls, fname: str) -> str: + def _json_string_from_hdf5(cls, fname: PathLike) -> str: """Load the model json string from an hdf5 file.""" with h5py.File(fname, "r") as f_handle: num_string_parts = len([key for key in f_handle.keys() if JSON_TAG in key]) @@ -668,13 +673,16 @@ def _json_string_from_hdf5(cls, fname: str) -> str: @classmethod def dict_from_hdf5( - cls, fname: str, group_path: str = "", custom_decoders: Optional[list[Callable]] = None + cls, + fname: PathLike, + group_path: str = "", + custom_decoders: Optional[list[Callable]] = None, ) -> dict: """Loads a dictionary containing the model contents from a .hdf5 file. Parameters ---------- - fname : str + fname : PathLike Full path to the .hdf5 file to load the :class:`Tidy3dBaseModel` from. group_path : str, optional Path to a group inside the file to selectively load a sub-element of the model only. @@ -697,6 +705,8 @@ def is_data_array(value: Any) -> bool: """Whether a value is supposed to be a data array based on the contents.""" return isinstance(value, str) and value in DATA_ARRAY_MAP + fname_path = Path(fname) + def load_data_from_file(model_dict: dict, group_path: str = "") -> None: """For every DataArray item in dictionary, load path of hdf5 group as value.""" @@ -707,7 +717,7 @@ def load_data_from_file(model_dict: dict, group_path: str = "") -> None: if custom_decoders: for custom_decoder in custom_decoders: custom_decoder( - fname=fname, + fname=str(fname_path), group_path=subpath, model_dict=model_dict, key=key, @@ -717,7 +727,9 @@ def load_data_from_file(model_dict: dict, group_path: str = "") -> None: # write the path to the element of the json dict where the data_array should be if is_data_array(value): data_array_type = DATA_ARRAY_MAP[value] - model_dict[key] = data_array_type.from_hdf5(fname=fname, group_path=subpath) + model_dict[key] = data_array_type.from_hdf5( + fname=fname_path, group_path=subpath + ) continue # if a list, assign each element a unique key, recurse @@ -735,7 +747,7 @@ def load_data_from_file(model_dict: dict, group_path: str = "") -> None: elif isinstance(value, dict): load_data_from_file(model_dict=value, group_path=subpath) - model_dict = json.loads(cls._json_string_from_hdf5(fname=fname)) + model_dict = json.loads(cls._json_string_from_hdf5(fname=fname_path)) group_path = cls._construct_group_path(group_path) model_dict = cls.get_sub_model(group_path=group_path, model_dict=model_dict) load_data_from_file(model_dict=model_dict, group_path=group_path) @@ -744,7 +756,7 @@ def load_data_from_file(model_dict: dict, group_path: str = "") -> None: @classmethod def from_hdf5( cls, - fname: str, + fname: PathLike, group_path: str = "", custom_decoders: Optional[list[Callable]] = None, **parse_obj_kwargs, @@ -753,7 +765,7 @@ def from_hdf5( Parameters ---------- - fname : str + fname : PathLike Full path to the .hdf5 file to load the :class:`Tidy3dBaseModel` from. group_path : str, optional Path to a group inside the file to selectively load a sub-element of the model only. @@ -772,21 +784,23 @@ def from_hdf5( group_path = cls._construct_group_path(group_path) model_dict = cls.dict_from_hdf5( - fname=fname, group_path=group_path, custom_decoders=custom_decoders + fname=fname, + group_path=group_path, + custom_decoders=custom_decoders, ) return cls.parse_obj(model_dict, **parse_obj_kwargs) def to_hdf5( self, - fname: str, + fname: PathLike | io.BytesIO, custom_encoders: Optional[list[Callable]] = None, ) -> None: """Exports :class:`Tidy3dBaseModel` instance to .hdf5 file. Parameters ---------- - fname : str - Full path to the .hdf5 file to save the :class:`Tidy3dBaseModel` to. + fname : PathLike | BytesIO + Full path to the .hdf5 file or buffer to save the :class:`Tidy3dBaseModel` to. custom_encoders : List[Callable] List of functions accepting (fname: str, group_path: str, value: Any) that take the ``value`` supplied and write it to the hdf5 ``fname`` at ``group_path``. @@ -803,7 +817,8 @@ def to_hdf5( traced_keys_payload = self.attrs.get(TRACED_FIELD_KEYS_ATTR) if traced_keys_payload is None: traced_keys_payload = self._serialized_traced_field_keys() - with h5py.File(fname, "w") as f_handle: + path = Path(fname) if isinstance(fname, PathLike) else fname + with h5py.File(path, "w") as f_handle: json_str = export_model._json() for ind in range(ceil(len(json_str) / MAX_STRING_LENGTH)): ind_start = int(ind * MAX_STRING_LENGTH) @@ -840,13 +855,16 @@ def add_data_to_file(data_dict: dict, group_path: str = "") -> None: @classmethod def dict_from_hdf5_gz( - cls, fname: str, group_path: str = "", custom_decoders: Optional[list[Callable]] = None + cls, + fname: PathLike, + group_path: str = "", + custom_decoders: Optional[list[Callable]] = None, ) -> dict: """Loads a dictionary containing the model contents from a .hdf5.gz file. Parameters ---------- - fname : str + fname : PathLike Full path to the .hdf5.gz file to load the :class:`Tidy3dBaseModel` from. group_path : str, optional Path to a group inside the file to selectively load a sub-element of the model only. @@ -864,22 +882,25 @@ def dict_from_hdf5_gz( ------- >>> sim_dict = Simulation.dict_from_hdf5(fname='folder/sim.hdf5.gz') # doctest: +SKIP """ - file, extracted = tempfile.mkstemp(".hdf5") - os.close(file) + file_descriptor, extracted = tempfile.mkstemp(".hdf5") + os.close(file_descriptor) + extracted_path = Path(extracted) try: - extract_gzip_file(fname, extracted) + extract_gzip_file(fname, extracted_path) result = cls.dict_from_hdf5( - extracted, group_path=group_path, custom_decoders=custom_decoders + extracted_path, + group_path=group_path, + custom_decoders=custom_decoders, ) finally: - os.unlink(extracted) + extracted_path.unlink(missing_ok=True) return result @classmethod def from_hdf5_gz( cls, - fname: str, + fname: PathLike, group_path: str = "", custom_decoders: Optional[list[Callable]] = None, **parse_obj_kwargs, @@ -888,7 +909,7 @@ def from_hdf5_gz( Parameters ---------- - fname : str + fname : PathLike Full path to the .hdf5.gz file to load the :class:`Tidy3dBaseModel` from. group_path : str, optional Path to a group inside the file to selectively load a sub-element of the model only. @@ -907,17 +928,21 @@ def from_hdf5_gz( group_path = cls._construct_group_path(group_path) model_dict = cls.dict_from_hdf5_gz( - fname=fname, group_path=group_path, custom_decoders=custom_decoders + fname=fname, + group_path=group_path, + custom_decoders=custom_decoders, ) return cls.parse_obj(model_dict, **parse_obj_kwargs) - def to_hdf5_gz(self, fname: str, custom_encoders: Optional[list[Callable]] = None) -> None: + def to_hdf5_gz( + self, fname: PathLike | io.BytesIO, custom_encoders: Optional[list[Callable]] = None + ) -> None: """Exports :class:`Tidy3dBaseModel` instance to .hdf5.gz file. Parameters ---------- - fname : str - Full path to the .hdf5.gz file to save the :class:`Tidy3dBaseModel` to. + fname : PathLike | BytesIO + Full path to the .hdf5.gz file or buffer to save the :class:`Tidy3dBaseModel` to. custom_encoders : List[Callable] List of functions accepting (fname: str, group_path: str, value: Any) that take the ``value`` supplied and write it to the hdf5 ``fname`` at ``group_path``. @@ -926,7 +951,6 @@ def to_hdf5_gz(self, fname: str, custom_encoders: Optional[list[Callable]] = Non ------- >>> simulation.to_hdf5_gz(fname='folder/sim.hdf5.gz') # doctest: +SKIP """ - file, decompressed = tempfile.mkstemp(".hdf5") os.close(file) try: @@ -1322,9 +1346,12 @@ def _make_lazy_proxy( class _LazyProxy(target_cls): def __init__( - self, fname: str, group_path: Optional[str], parse_obj_kwargs: Optional[dict[str, Any]] + self, + fname: PathLike, + group_path: Optional[str], + parse_obj_kwargs: Optional[dict[str, Any]], ): - object.__setattr__(self, "_lazy_fname", fname) + object.__setattr__(self, "_lazy_fname", Path(fname)) object.__setattr__(self, "_lazy_group_path", group_path) object.__setattr__(self, "_lazy_parse_obj_kwargs", dict(parse_obj_kwargs or {})) diff --git a/tidy3d/components/data/data_array.py b/tidy3d/components/data/data_array.py index 74304a2283..08038b2a5c 100644 --- a/tidy3d/components/data/data_array.py +++ b/tidy3d/components/data/data_array.py @@ -2,8 +2,10 @@ from __future__ import annotations +import pathlib from abc import ABC from collections.abc import Mapping +from os import PathLike from typing import Any, Optional, Union import autograd.numpy as anp @@ -220,12 +222,14 @@ def is_uniform(self): raw_data = self.data.ravel() return np.allclose(raw_data, raw_data[0]) - def to_hdf5(self, fname: Union[str, h5py.File], group_path: str) -> None: + def to_hdf5(self, fname: Union[PathLike, h5py.File], group_path: str) -> None: """Save an xr.DataArray to the hdf5 file or file handle with a given path to the group.""" # file name passed - if isinstance(fname, str): - with h5py.File(fname, "w") as f_handle: + if isinstance(fname, (str, pathlib.Path)): + path = pathlib.Path(fname) + path.parent.mkdir(parents=True, exist_ok=True) + with h5py.File(path, "w") as f_handle: self.to_hdf5_handle(f_handle=f_handle, group_path=group_path) # file handle passed @@ -244,9 +248,10 @@ def to_hdf5_handle(self, f_handle: h5py.File, group_path: str) -> None: sub_group[key] = val @classmethod - def from_hdf5(cls, fname: str, group_path: str) -> Self: + def from_hdf5(cls, fname: PathLike, group_path: str) -> Self: """Load an DataArray from an hdf5 file with a given path to the group.""" - with h5py.File(fname, "r") as f: + path = pathlib.Path(fname) + with h5py.File(path, "r") as f: sub_group = f[group_path] values = np.array(sub_group[DATA_ARRAY_VALUE_NAME]) coords = {dim: np.array(sub_group[dim]) for dim in cls._dims if dim in sub_group} @@ -256,13 +261,14 @@ def from_hdf5(cls, fname: str, group_path: str) -> Self: return cls(values, coords=coords, dims=cls._dims) @classmethod - def from_file(cls, fname: str, group_path: str) -> Self: + def from_file(cls, fname: PathLike, group_path: str) -> Self: """Load an DataArray from an hdf5 file with a given path to the group.""" - if ".hdf5" not in fname: + path = pathlib.Path(fname) + if not any(suffix.lower() == ".hdf5" for suffix in path.suffixes): raise FileError( - f"'DataArray' objects must be written to '.hdf5' format. Given filename of {fname}." + f"'DataArray' objects must be written to '.hdf5' format. Given filename of {path}." ) - return cls.from_hdf5(fname=fname, group_path=group_path) + return cls.from_hdf5(fname=path, group_path=group_path) def __hash__(self) -> int: """Generate hash value for a :class:`.DataArray` instance, needed for custom components.""" diff --git a/tidy3d/components/data/monitor_data.py b/tidy3d/components/data/monitor_data.py index b8f310561a..b42b6fc454 100644 --- a/tidy3d/components/data/monitor_data.py +++ b/tidy3d/components/data/monitor_data.py @@ -6,6 +6,7 @@ import warnings from abc import ABC from math import isclose +from os import PathLike from typing import Any, Callable, Literal, Optional, Union, get_args import autograd.numpy as np @@ -1081,7 +1082,7 @@ def translated_copy(self, vector: Coordinate) -> ElectromagneticFieldData: def to_zbf( self, - fname: str, + fname: PathLike, units: UnitsZBF = "mm", background_refractive_index: float = 1, n_x: Optional[int] = None, @@ -1102,7 +1103,7 @@ def to_zbf( Parameters ---------- - fname : str + fname : PathLike Full path to the ``.zbf`` file to be written. units : UnitsZBF = "mm" Spatial units used for the ``.zbf`` file. Options are ``"mm"``, ``"cm"``, ``"in"``, or ``"m"``. diff --git a/tidy3d/components/data/sim_data.py b/tidy3d/components/data/sim_data.py index 9c81bdbfae..669e69c338 100644 --- a/tidy3d/components/data/sim_data.py +++ b/tidy3d/components/data/sim_data.py @@ -7,6 +7,7 @@ import re from abc import ABC from collections import defaultdict +from os import PathLike from typing import Callable, Optional, Union import h5py @@ -371,12 +372,14 @@ def get_intensity(self, field_monitor_name: str) -> xr.DataArray: ) @classmethod - def mnt_data_from_file(cls, fname: str, mnt_name: str, **parse_obj_kwargs) -> MonitorDataType: + def mnt_data_from_file( + cls, fname: PathLike, mnt_name: str, **parse_obj_kwargs + ) -> MonitorDataType: """Loads data for a specific monitor from a .hdf5 file with data for a ``SimulationData``. Parameters ---------- - fname : str + fname : PathLike Full path to an hdf5 file containing :class:`.SimulationData` data. mnt_name : str, optional ``.name`` of the monitor to load the data from. @@ -1317,12 +1320,12 @@ def _get_adjoint_data(self, structure_index: int, data_type: str) -> MonitorData monitor_name = Structure._get_monitor_name(index=structure_index, data_type=data_type) return self[monitor_name] - def to_mat_file(self, fname: str, **kwargs): + def to_mat_file(self, fname: PathLike, **kwargs): """Output the ``SimulationData`` object as ``.mat`` MATLAB file. Parameters ---------- - fname : str + fname : PathLike Full path to the output file. Should include ``.mat`` file extension. **kwargs : dict, optional Extra arguments to ``scipy.io.savemat``: see ``scipy`` documentation for more detail. diff --git a/tidy3d/components/data/unstructured/base.py b/tidy3d/components/data/unstructured/base.py index b5866fd2d6..5ab0581680 100644 --- a/tidy3d/components/data/unstructured/base.py +++ b/tidy3d/components/data/unstructured/base.py @@ -4,6 +4,7 @@ import numbers from abc import ABC, abstractmethod +from os import PathLike from typing import Literal, Optional, Union import numpy as np @@ -474,8 +475,9 @@ def _vtk_obj(self): @staticmethod @requires_vtk - def _read_vtkUnstructuredGrid(fname: str): + def _read_vtkUnstructuredGrid(fname: PathLike): """Load a :class:`vtkUnstructuredGrid` from a file.""" + fname = str(fname) reader = vtk["mod"].vtkXMLUnstructuredGridReader() reader.SetFileName(fname) reader.Update() @@ -485,8 +487,9 @@ def _read_vtkUnstructuredGrid(fname: str): @staticmethod @requires_vtk - def _read_vtkLegacyFile(fname: str): + def _read_vtkLegacyFile(fname: PathLike): """Load a grid from a legacy `.vtk` file.""" + fname = str(fname) reader = vtk["mod"].vtkGenericDataObjectReader() reader.SetFileName(fname) reader.Update() @@ -532,7 +535,7 @@ def _from_vtk_obj_internal( @requires_vtk def from_vtu( cls, - file: str, + file: PathLike, field: Optional[str] = None, remove_degenerate_cells: bool = False, remove_unused_points: bool = False, @@ -542,7 +545,7 @@ def from_vtu( Parameters ---------- - file : str + file : PathLike Full path to the .vtu file to load the unstructured data from. field : str = None Name of the field to load. @@ -571,7 +574,7 @@ def from_vtu( @requires_vtk def from_vtk( cls, - file: str, + file: PathLike, field: Optional[str] = None, remove_degenerate_cells: bool = False, remove_unused_points: bool = False, @@ -581,7 +584,7 @@ def from_vtk( Parameters ---------- - file : str + file : PathLike Full path to the .vtk file to load the unstructured data from. field : str = None Name of the field to load. @@ -607,15 +610,15 @@ def from_vtk( ) @requires_vtk - def to_vtu(self, fname: str): + def to_vtu(self, fname: PathLike): """Exports unstructured grid data into a .vtu file. Parameters ---------- - fname : str + fname : PathLike Full path to the .vtu file to save the unstructured data to. """ - + fname = str(fname) writer = vtk["mod"].vtkXMLUnstructuredGridWriter() writer.SetFileName(fname) writer.SetInputData(self._vtk_obj) diff --git a/tidy3d/components/file_util.py b/tidy3d/components/file_util.py index c57fb15322..bdfe7326a1 100644 --- a/tidy3d/components/file_util.py +++ b/tidy3d/components/file_util.py @@ -3,35 +3,47 @@ from __future__ import annotations import gzip +import pathlib import shutil +from io import BytesIO +from os import PathLike from typing import Any import numpy as np -def compress_file_to_gzip(input_file, output_gz_file): +def compress_file_to_gzip(input_file: PathLike, output_gz_file: PathLike | BytesIO) -> None: """ - Compresses a file using gzip. + Compress a file using gzip. - Args: - input_file (str): The path of the input file. - output_gz_file (str): The path of the output gzip file. + Parameters + ---------- + input_file : PathLike + The path to the input file. + output_gz_file : PathLike | BytesIO + The path to the output gzip file or an in-memory buffer. """ - with open(input_file, "rb") as file_in: + input_file = pathlib.Path(input_file) + with input_file.open("rb") as file_in: with gzip.open(output_gz_file, "wb") as file_out: shutil.copyfileobj(file_in, file_out) -def extract_gzip_file(input_gz_file, output_file): +def extract_gzip_file(input_gz_file: PathLike, output_file: PathLike) -> None: """ - Extract a gzip file. + Extract a gzip-compressed file. - Args: - input_gz_file (str): The path of the gzip input file. - output_file (str): The path of the output file. + Parameters + ---------- + input_gz_file : PathLike + The path to the gzip-compressed input file. + output_file : PathLike + The path to the extracted output file. """ - with gzip.open(input_gz_file, "rb") as file_in: - with open(output_file, "wb") as file_out: + input_path = pathlib.Path(input_gz_file) + output_path = pathlib.Path(output_file) + with gzip.open(input_path, "rb") as file_in: + with output_path.open("wb") as file_out: shutil.copyfileobj(file_in, file_out) diff --git a/tidy3d/components/geometry/base.py b/tidy3d/components/geometry/base.py index bc9912541b..59b1241e47 100644 --- a/tidy3d/components/geometry/base.py +++ b/tidy3d/components/geometry/base.py @@ -5,6 +5,7 @@ import functools import pathlib from abc import ABC, abstractmethod +from os import PathLike from typing import Any, Callable, Optional, Union import autograd.numpy as np @@ -1412,7 +1413,7 @@ def to_gds( @verify_packages_import(["gdstk"]) def to_gds_file( self, - fname: str, + fname: PathLike, x: Optional[float] = None, y: Optional[float] = None, z: Optional[float] = None, @@ -1424,7 +1425,7 @@ def to_gds_file( Parameters ---------- - fname : str + fname : PathLike Full path to the .gds file to save the :class:`Geometry` slice to. x : float = None Position of plane in x direction, only one of x,y,z can be specified to define plane. @@ -1450,7 +1451,8 @@ def to_gds_file( library = gdstk.Library() cell = library.new_cell(gds_cell_name) self.to_gds(cell, x=x, y=y, z=z, gds_layer=gds_layer, gds_dtype=gds_dtype) - pathlib.Path(fname).parent.mkdir(parents=True, exist_ok=True) + fname = pathlib.Path(fname) + fname.parent.mkdir(parents=True, exist_ok=True) library.write_gds(fname) def _compute_derivatives(self, derivative_info: DerivativeInfo) -> AutogradFieldMap: diff --git a/tidy3d/components/simulation.py b/tidy3d/components/simulation.py index b654b935aa..0b127d8bb5 100644 --- a/tidy3d/components/simulation.py +++ b/tidy3d/components/simulation.py @@ -6,6 +6,7 @@ import pathlib from abc import ABC, abstractmethod from collections import defaultdict +from os import PathLike from typing import Literal, Optional, Union, get_args import autograd.numpy as np @@ -5215,7 +5216,7 @@ def to_gds( def to_gds_file( self, - fname: str, + fname: PathLike, x: Optional[float] = None, y: Optional[float] = None, z: Optional[float] = None, @@ -5230,7 +5231,7 @@ def to_gds_file( Parameters ---------- - fname : str + fname : PathLike Full path to the .gds file to save the :class:`.Simulation` slice to. x : float = None Position of plane in x direction, only one of x,y,z can be specified to define plane. @@ -5281,7 +5282,8 @@ def to_gds_file( frequency=frequency, gds_layer_dtype_map=gds_layer_dtype_map, ) - pathlib.Path(fname).parent.mkdir(parents=True, exist_ok=True) + fname = pathlib.Path(fname) + fname.parent.mkdir(parents=True, exist_ok=True) library.write_gds(fname) """ Plotting """ diff --git a/tidy3d/components/structure.py b/tidy3d/components/structure.py index f091622eaf..d4d10aca5d 100644 --- a/tidy3d/components/structure.py +++ b/tidy3d/components/structure.py @@ -5,6 +5,7 @@ import pathlib from collections import defaultdict from functools import cmp_to_key +from os import PathLike from typing import Optional, Union import autograd.numpy as anp @@ -525,7 +526,7 @@ def to_gds( def to_gds_file( self, - fname: str, + fname: PathLike, x: Optional[float] = None, y: Optional[float] = None, z: Optional[float] = None, @@ -539,7 +540,7 @@ def to_gds_file( Parameters ---------- - fname : str + fname : PathLike Full path to the .gds file to save the :class:`.Structure` slice to. x : float = None Position of plane in x direction, only one of x,y,z can be specified to define plane. @@ -579,7 +580,8 @@ def to_gds_file( gds_layer=gds_layer, gds_dtype=gds_dtype, ) - pathlib.Path(fname).parent.mkdir(parents=True, exist_ok=True) + fname = pathlib.Path(fname) + fname.parent.mkdir(parents=True, exist_ok=True) library.write_gds(fname) @classmethod diff --git a/tidy3d/log.py b/tidy3d/log.py index 3cfa825773..36eeb957e5 100644 --- a/tidy3d/log.py +++ b/tidy3d/log.py @@ -5,6 +5,7 @@ import inspect from contextlib import contextmanager from datetime import datetime +from os import PathLike from typing import Callable, Optional, Union from rich.console import Console @@ -385,7 +386,7 @@ def set_logging_console(stderr: bool = False) -> None: def set_logging_file( - fname: str, + fname: PathLike, filemode: str = "w", level: LogValue = DEFAULT_LEVEL, log_path: bool = False, @@ -395,7 +396,7 @@ def set_logging_file( Parameters ---------- - fname : str + fname : PathLike Path to file to direct the output to. If empty string, a previously set logging file will be closed, if any, but nothing else happens. filemode : str @@ -418,7 +419,7 @@ def set_logging_file( finally: del log.handlers["file"] - if fname == "": + if str(fname) == "": # Empty string can be passed to just stop previously opened file handler return diff --git a/tidy3d/material_library/material_library.py b/tidy3d/material_library/material_library.py index 37bb932b27..fc911b9c47 100644 --- a/tidy3d/material_library/material_library.py +++ b/tidy3d/material_library/material_library.py @@ -3,6 +3,7 @@ from __future__ import annotations import json +from os import PathLike from typing import Union import pydantic.v1 as pd @@ -37,9 +38,8 @@ ) -def export_matlib_to_file(fname: str = "matlib.json") -> None: +def export_matlib_to_file(fname: PathLike = "matlib.json") -> None: """Write the material library to a .json file.""" - mat_lib_dict = { f'{mat.name} ("{mat_name}")': { var_name: json.loads(var.medium._json_string) for var_name, var in mat.variants.items() diff --git a/tidy3d/plugins/dispersion/fit.py b/tidy3d/plugins/dispersion/fit.py index b9977e08b2..e9c0051015 100644 --- a/tidy3d/plugins/dispersion/fit.py +++ b/tidy3d/plugins/dispersion/fit.py @@ -4,6 +4,7 @@ import codecs import csv +from os import PathLike from typing import Optional import numpy as np @@ -698,12 +699,12 @@ def from_url( return cls(wvl_um=n_lam[:, 0], n_data=n_lam[:, 1], **kwargs) @classmethod - def from_file(cls, fname: str, **loadtxt_kwargs) -> DispersionFitter: + def from_file(cls, fname: PathLike, **loadtxt_kwargs) -> DispersionFitter: """Loads :class:`DispersionFitter` from file containing wavelength, n, k data. Parameters ---------- - fname : str + fname : PathLike Path to file containing wavelength (um), n, k (optional) data in columns. **loadtxt_kwargs Kwargs passed to ``np.loadtxt``, such as ``skiprows``, ``delimiter``. diff --git a/tidy3d/plugins/smatrix/run.py b/tidy3d/plugins/smatrix/run.py index dc9d734fb8..8035e91229 100644 --- a/tidy3d/plugins/smatrix/run.py +++ b/tidy3d/plugins/smatrix/run.py @@ -1,6 +1,7 @@ from __future__ import annotations import json +from os import PathLike from tidy3d.components.base import Tidy3dBaseModel from tidy3d.components.data.index import SimulationDataMap @@ -17,7 +18,7 @@ def compose_modeler( - modeler_file: str, + modeler_file: PathLike, ) -> ComponentModelerType: """Load a component modeler from an HDF5 file. @@ -27,7 +28,7 @@ def compose_modeler( Parameters ---------- - modeler_file : str + modeler_file : PathLike Path to the HDF5 file containing the modeler definition. Returns diff --git a/tidy3d/updater.py b/tidy3d/updater.py index bc7f017fab..5d4a9c8752 100644 --- a/tidy3d/updater.py +++ b/tidy3d/updater.py @@ -4,6 +4,8 @@ import functools import json +from os import PathLike +from pathlib import Path from typing import Callable, Optional import pydantic.v1 as pd @@ -89,26 +91,20 @@ class Updater(pd.BaseModel): sim_dict: dict @classmethod - def from_file(cls, fname: str) -> Updater: + def from_file(cls, fname: PathLike) -> Updater: """Dictionary representing the simulation loaded from file.""" - + path = Path(fname) # TODO: fix this, it broke - if any(ext in fname for ext in (".hdf5", ".gz")): - sim_dict = Tidy3dBaseModel.from_file(fname=fname).dict() - + if path.suffix in {".hdf5", ".gz"}: + sim_dict = Tidy3dBaseModel.from_file(fname=str(path)).dict() else: - # try: - with open(fname, encoding="utf-8") as f: - if ".json" in fname: + with path.open(encoding="utf-8") as f: + if path.suffix == ".json": sim_dict = json.load(f) - elif ".yaml" in fname: + elif path.suffix == ".yaml": sim_dict = yaml.safe_load(f) else: raise FileError('file extension must be ".json", ".yaml", ".hdf5", or ".gz"') - - # except Exception as e: - # raise FileError(f"Could not load file {fname}") from e - return cls(sim_dict=sim_dict) @classmethod diff --git a/tidy3d/web/api/asynchronous.py b/tidy3d/web/api/asynchronous.py index c5a346abdd..3dc63f3491 100644 --- a/tidy3d/web/api/asynchronous.py +++ b/tidy3d/web/api/asynchronous.py @@ -2,6 +2,7 @@ from __future__ import annotations +from os import PathLike from typing import Literal, Optional, Union from tidy3d.components.types.workflow import WorkflowType @@ -14,7 +15,7 @@ def run_async( simulations: Union[dict[str, WorkflowType], tuple[WorkflowType], list[WorkflowType]], folder_name: str = "default", - path_dir: str = DEFAULT_DATA_DIR, + path_dir: PathLike = DEFAULT_DATA_DIR, callback_url: Optional[str] = None, num_workers: Optional[int] = None, verbose: bool = True, @@ -37,7 +38,7 @@ def run_async( Mapping of task name to simulation or list of simulations. folder_name : str = "default" Name of folder to store each task on web UI. - path_dir : str + path_dir : PathLike Base directory where data will be downloaded, by default current working directory. callback_url : str = None Http PUT url to receive simulation finish event. The body content is a json file with diff --git a/tidy3d/web/api/autograd/autograd.py b/tidy3d/web/api/autograd/autograd.py index 9a29d71af6..1b5592f27b 100644 --- a/tidy3d/web/api/autograd/autograd.py +++ b/tidy3d/web/api/autograd/autograd.py @@ -2,7 +2,7 @@ from __future__ import annotations import typing -from os.path import dirname +from os import PathLike from pathlib import Path from autograd.builtins import dict as dict_ag @@ -102,7 +102,7 @@ def run( simulation: WorkflowType, task_name: typing.Optional[str] = None, folder_name: str = "default", - path: str = "simulation_data.hdf5", + path: PathLike = "simulation_data.hdf5", callback_url: typing.Optional[str] = None, verbose: bool = True, progress_callback_upload: typing.Optional[typing.Callable[[float], None]] = None, @@ -130,7 +130,7 @@ def run( Name of task. If not provided, a default name will be generated. folder_name : str = "default" Name of folder to store task on web UI. - path : str = "simulation_data.hdf5" + path : PathLike = "simulation_data.hdf5" Path to download results file (.hdf5), including filename. callback_url : str = None Http PUT url to receive simulation finish event. The body content is a json file with @@ -225,11 +225,13 @@ def run( # component modeler path: route autograd-valid modelers to local run from tidy3d.plugins.smatrix.component_modelers.types import ComponentModelerType + path = Path(path) + if isinstance(simulation, typing.get_args(ComponentModelerType)): if any(is_valid_for_autograd(s) for s in simulation.sim_dict.values()): from tidy3d.plugins.smatrix import run as smatrix_run - path_dir = dirname(path) or "." + path_dir = path.parent return smatrix_run._run_local( simulation, path_dir=path_dir, @@ -287,7 +289,7 @@ def run( def run_async( simulations: typing.Union[dict[str, td.Simulation], tuple[td.Simulation], list[td.Simulation]], folder_name: str = "default", - path_dir: str = DEFAULT_DATA_DIR, + path_dir: PathLike = DEFAULT_DATA_DIR, callback_url: typing.Optional[str] = None, num_workers: typing.Optional[int] = None, verbose: bool = True, @@ -312,7 +314,7 @@ def run_async( Mapping of task name to simulation or list of simulations. folder_name : str = "default" Name of folder to store each task on web UI. - path_dir : str + path_dir : PathLike Base directory where data will be downloaded, by default current working directory. callback_url : str = None Http PUT url to receive simulation finish event. The body content is a json file with @@ -374,6 +376,8 @@ def run_async( sim_dict[task_name] = sim simulations = sim_dict + path_dir = Path(path_dir) + if is_valid_for_autograd_async(simulations): return _run_async( simulations=simulations, @@ -748,7 +752,7 @@ def vjp(data_fields_vjp: AutogradFieldMap) -> AutogradFieldMap: path_dir_adj.mkdir(parents=True, exist_ok=True) batch_data_adj, _ = _run_async_tidy3d( - sims_adj_dict, path_dir=str(path_dir_adj), **run_kwargs + sims_adj_dict, path_dir=path_dir_adj, **run_kwargs ) td.log.info("Completed local batch adjoint simulations") @@ -886,7 +890,7 @@ def vjp(data_fields_dict_vjp: dict[str, AutogradFieldMap]) -> dict[str, Autograd path_dir_adj.mkdir(parents=True, exist_ok=True) batch_data_adj, _ = _run_async_tidy3d( - all_sims_adj, path_dir=str(path_dir_adj), **run_async_kwargs + all_sims_adj, path_dir=path_dir_adj, **run_async_kwargs ) # Process results for each adjoint task diff --git a/tidy3d/web/api/autograd/engine.py b/tidy3d/web/api/autograd/engine.py index 5aa90efc3f..23e97ef2fc 100644 --- a/tidy3d/web/api/autograd/engine.py +++ b/tidy3d/web/api/autograd/engine.py @@ -1,6 +1,6 @@ from __future__ import annotations -from os.path import basename, dirname, join +from pathlib import Path import tidy3d as td from tidy3d.web.api.container import DEFAULT_DATA_PATH, Batch, Job @@ -26,11 +26,13 @@ def _run_tidy3d( if job.simulation_type == "autograd_fwd": verbose = run_kwargs.get("verbose", False) upload_sim_fields_keys(run_kwargs["sim_fields_keys"], task_id=job.task_id, verbose=verbose) - path = run_kwargs.get("path", DEFAULT_DATA_PATH) + path = Path(run_kwargs.get("path", DEFAULT_DATA_PATH)) priority = run_kwargs.get("priority") if task_name.endswith("_adjoint"): - path_parts = basename(path).split(".") - path = join(dirname(path), path_parts[0] + "_adjoint." + ".".join(path_parts[1:])) + suffixes = "".join(path.suffixes) + base_name = path.name + base_without_suffix = base_name[: -len(suffixes)] if suffixes else base_name + path = path.with_name(f"{base_without_suffix}_adjoint{suffixes}") data = job.run(path, priority=priority) return data, job.task_id @@ -61,7 +63,7 @@ def _run_async_tidy3d( task_id = task_ids[task_name] upload_sim_fields_keys(sim_fields_keys, task_id=task_id, verbose=verbose) - if path_dir: + if path_dir is not None: batch_data = batch.run(path_dir, priority=priority) else: batch_data = batch.run(priority=priority) diff --git a/tidy3d/web/api/container.py b/tidy3d/web/api/container.py index ed9cb775cc..ff37886761 100644 --- a/tidy3d/web/api/container.py +++ b/tidy3d/web/api/container.py @@ -3,11 +3,12 @@ from __future__ import annotations import concurrent -import os import time from abc import ABC from collections.abc import Mapping from concurrent.futures import ThreadPoolExecutor +from os import PathLike +from pathlib import Path from typing import Literal, Optional, Union import pydantic.v1 as pd @@ -60,7 +61,7 @@ class WebContainer(Tidy3dBaseModel, ABC): @staticmethod @abstractmethod - def _check_path_dir(path: str) -> None: + def _check_path_dir(path: PathLike) -> None: """Make sure local output directory exists and create it if not.""" @staticmethod @@ -240,12 +241,12 @@ class Job(WebContainer): "reduce_simulation", ) - def to_file(self, fname: str) -> None: + def to_file(self, fname: PathLike) -> None: """Exports :class:`Tidy3dBaseModel` instance to .yaml, .json, or .hdf5 file Parameters ---------- - fname : str + fname : PathLike Full path to the .yaml or .json file to save the :class:`Tidy3dBaseModel` to. Example @@ -257,13 +258,13 @@ def to_file(self, fname: str) -> None: super(Job, self).to_file(fname=fname) # noqa: UP008 def run( - self, path: str = DEFAULT_DATA_PATH, priority: Optional[int] = None + self, path: PathLike = DEFAULT_DATA_PATH, priority: Optional[int] = None ) -> WorkflowDataType: """Run :class:`Job` all the way through and return data. Parameters ---------- - path : str = "./simulation_data.hdf5" + path : PathLike = "./simulation_data.hdf5" Path to download results file (.hdf5), including filename. priority: int = None Priority of the simulation in the Virtual GPU (vGPU) queue (1 = lowest, 10 = highest). @@ -373,12 +374,12 @@ def monitor(self) -> None: """ web.monitor(self.task_id, verbose=self.verbose) - def download(self, path: str = DEFAULT_DATA_PATH) -> None: + def download(self, path: PathLike = DEFAULT_DATA_PATH) -> None: """Download results of simulation. Parameters ---------- - path : str = "./simulation_data.hdf5" + path : PathLike = "./simulation_data.hdf5" Path to download data as ``.hdf5`` file (including filename). Note @@ -388,12 +389,12 @@ def download(self, path: str = DEFAULT_DATA_PATH) -> None: self._check_path_dir(path=path) web.download(task_id=self.task_id, path=path, verbose=self.verbose) - def load(self, path: str = DEFAULT_DATA_PATH) -> WorkflowDataType: + def load(self, path: PathLike = DEFAULT_DATA_PATH) -> WorkflowDataType: """Download job results and load them into a data object. Parameters ---------- - path : str = "./simulation_data.hdf5" + path : PathLike = "./simulation_data.hdf5" Path to download data as ``.hdf5`` file (including filename). Returns @@ -402,7 +403,12 @@ def load(self, path: str = DEFAULT_DATA_PATH) -> WorkflowDataType: Object containing simulation results. """ self._check_path_dir(path=path) - data = web.load(task_id=self.task_id, path=path, verbose=self.verbose, lazy=self.lazy) + data = web.load( + task_id=self.task_id, + path=path, + verbose=self.verbose, + lazy=self.lazy, + ) if isinstance(self.simulation, ModeSolver): self.simulation._patch_data(data=data) return data @@ -478,17 +484,18 @@ def postprocess_start(self, worker_group: Optional[str] = None, verbose: bool = ) @staticmethod - def _check_path_dir(path: str) -> None: + def _check_path_dir(path: PathLike) -> None: """Make sure parent directory of ``path`` exists and create it if not. Parameters ---------- - path : str + path : PathLike Path to file to be created (including filename). """ - parent_dir = os.path.dirname(path) - if len(parent_dir) > 0 and not os.path.exists(parent_dir): - os.makedirs(parent_dir, exist_ok=True) + path = Path(path) + parent_dir = path.parent + if parent_dir != Path(".") and not parent_dir.exists(): + parent_dir.mkdir(parents=True, exist_ok=True) @pd.root_validator(pre=True) def set_task_name_if_none(cls, values): @@ -549,7 +556,7 @@ class BatchData(Tidy3dBaseModel, Mapping): def load_sim_data(self, task_name: str) -> WorkflowDataType: """Load a simulation data object from file by task name.""" - task_data_path = self.task_paths[task_name] + task_data_path = Path(self.task_paths[task_name]) task_id = self.task_ids[task_name] web.get_info(task_id) @@ -568,12 +575,14 @@ def __len__(self): return len(self.task_paths) @classmethod - def load(cls, path_dir: str = DEFAULT_DATA_DIR, replace_existing: bool = False) -> BatchData: + def load( + cls, path_dir: PathLike = DEFAULT_DATA_DIR, replace_existing: bool = False + ) -> BatchData: """Load :class:`Batch` from file, download results, and load them. Parameters ---------- - path_dir : str = './' + path_dir : PathLike = './' Base directory where data will be downloaded, by default current working directory. A `batch.hdf5` file must be present in the directory. replace_existing : bool = False @@ -585,10 +594,10 @@ def load(cls, path_dir: str = DEFAULT_DATA_DIR, replace_existing: bool = False) Contains Union[:class:`.SimulationData`, :class:`.HeatSimulationData`, :class:`.EMESimulationData`] for each Union[:class:`.Simulation`, :class:`.HeatSimulation`, :class:`.EMESimulation`] in :class:`Batch`. """ - - batch_file = Batch._batch_path(path_dir=path_dir) + base_dir = Path(path_dir) + batch_file = Batch._batch_path(path_dir=base_dir) batch = Batch.from_file(batch_file) - return batch.load(path_dir=path_dir, replace_existing=replace_existing) + return batch.load(path_dir=base_dir, replace_existing=replace_existing) class Batch(WebContainer): @@ -703,14 +712,14 @@ class Batch(WebContainer): def run( self, - path_dir: str = DEFAULT_DATA_DIR, + path_dir: PathLike = DEFAULT_DATA_DIR, priority: Optional[int] = None, ) -> BatchData: """Upload and run each simulation in :class:`Batch`. Parameters ---------- - path_dir : str + path_dir : PathLike Base directory where data will be downloaded, by default current working directory. priority: int = None Priority of the simulation in the Virtual GPU (vGPU) queue (1 = lowest, 10 = highest). @@ -790,12 +799,12 @@ def jobs(self) -> dict[TaskName, Job]: jobs[task_name] = job return jobs - def to_file(self, fname: str) -> None: + def to_file(self, fname: PathLike) -> None: """Exports :class:`Tidy3dBaseModel` instance to .yaml, .json, or .hdf5 file Parameters ---------- - fname : str + fname : PathLike Full path to the .yaml or .json file to save the :class:`Tidy3dBaseModel` to. Example @@ -916,7 +925,7 @@ def monitor( self, *, download_on_success: bool = False, - path_dir: str = DEFAULT_DATA_DIR, + path_dir: PathLike = DEFAULT_DATA_DIR, replace_existing: bool = False, postprocess_worker_group: Optional[str] = None, ) -> None: @@ -933,7 +942,7 @@ def monitor( download_on_success : bool = False If ``True``, automatically start downloading the results for a job as soon as it reaches ``success``. - path_dir : str = './' + path_dir : PathLike = './' Base directory where data will be downloaded, by default the current working directory. Only used when ``download_on_success`` is ``True``. replace_existing : bool = False @@ -964,19 +973,19 @@ def schedule_download(job) -> None: if task_id in downloads_started: return - job_path_str = self._job_data_path(task_id=task_id, path_dir=path_dir) - if os.path.exists(job_path_str): + job_path = self._job_data_path(task_id=task_id, path_dir=path_dir) + if job_path.exists(): if not replace_existing: downloads_started.add(task_id) log.info( - f"File '{job_path_str}' already exists. Skipping download " + f"File '{job_path}' already exists. Skipping download " "(set `replace_existing=True` to overwrite)." ) return - log.info(f"File '{job_path_str}' already exists. Overwriting.") + log.info(f"File '{job_path}' already exists. Overwriting.") downloads_started.add(task_id) - download_futures[task_id] = download_executor.submit(job.download, job_path_str) + download_futures[task_id] = download_executor.submit(job.download, job_path) # ----- continue condition & status formatting ------------------------------- def check_continue_condition(job) -> bool: @@ -1125,46 +1134,48 @@ def pbar_description( download_executor.shutdown(wait=True) @staticmethod - def _job_data_path(task_id: TaskId, path_dir: str = DEFAULT_DATA_DIR): + def _job_data_path(task_id: TaskId, path_dir: PathLike = DEFAULT_DATA_DIR) -> Path: """Default path to data of a single :class:`Job` in :class:`Batch`. Parameters ---------- task_id : str task_id corresponding to a :class:`Job`. - path_dir : str = './' + path_dir : PathLike = './' Base directory where data will be downloaded, by default, the current working directory. Returns ------- - str + Path Full path to the data file. """ - return os.path.join(path_dir, f"{task_id!s}.hdf5") + return Path(path_dir) / f"{task_id!s}.hdf5" @staticmethod - def _batch_path(path_dir: str = DEFAULT_DATA_DIR): + def _batch_path(path_dir: PathLike = DEFAULT_DATA_DIR) -> Path: """Default path to save :class:`Batch` hdf5 file. Parameters ---------- - path_dir : str = './' + path_dir : PathLike = './' Base directory where the batch.hdf5 will be downloaded, by default, the current working directory. Returns ------- - str + Path Full path to the batch file. """ - return os.path.join(path_dir, "batch.hdf5") + return Path(path_dir) / "batch.hdf5" - def download(self, path_dir: str = DEFAULT_DATA_DIR, replace_existing: bool = False) -> None: + def download( + self, path_dir: PathLike = DEFAULT_DATA_DIR, replace_existing: bool = False + ) -> None: """Download results of each task. Parameters ---------- - path_dir : str = './' + path_dir : PathLike = './' Base directory where data will be downloaded, by default the current working directory. replace_existing : bool = False Downloads the data even if path exists (overwriting the existing). @@ -1182,8 +1193,8 @@ def download(self, path_dir: str = DEFAULT_DATA_DIR, replace_existing: bool = Fa num_existing = 0 for _, job in self.jobs.items(): - job_path_str = self._job_data_path(task_id=job.task_id, path_dir=path_dir) - if os.path.exists(job_path_str): + job_path = self._job_data_path(task_id=job.task_id, path_dir=path_dir) + if job_path.exists(): num_existing += 1 if num_existing > 0: files_plural = "files have" if num_existing > 1 else "file has" @@ -1197,19 +1208,19 @@ def download(self, path_dir: str = DEFAULT_DATA_DIR, replace_existing: bool = Fa with ThreadPoolExecutor(max_workers=self.num_workers) as executor: fns = [] for task_name, job in self.jobs.items(): - job_path_str = self._job_data_path(task_id=job.task_id, path_dir=path_dir) - if os.path.exists(job_path_str): + job_path = self._job_data_path(task_id=job.task_id, path_dir=path_dir) + if job_path.exists(): if replace_existing: - log.info(f"File '{job_path_str}' already exists. Overwriting.") + log.info(f"File '{job_path}' already exists. Overwriting.") else: - log.info(f"File '{job_path_str}' already exists. Skipping.") + log.info(f"File '{job_path}' already exists. Skipping.") continue if "error" in job.status: log.warning(f"Not downloading '{task_name}' as the task errored.") continue - def fn(job=job, job_path_str=job_path_str) -> None: - return job.download(path=job_path_str) + def fn(job=job, job_path=job_path) -> None: + return job.download(path=job_path) fns.append(fn) @@ -1233,7 +1244,7 @@ def fn(job=job, job_path_str=job_path_str) -> None: def load( self, - path_dir: str = DEFAULT_DATA_DIR, + path_dir: PathLike = DEFAULT_DATA_DIR, replace_existing: bool = False, skip_download: bool = False, ) -> BatchData: @@ -1241,7 +1252,7 @@ def load( Parameters ---------- - path_dir : str = './' + path_dir : PathLike = './' Base directory where data will be downloaded, by default current working directory. replace_existing : bool = False Downloads the data even if path exists (overwriting the existing). @@ -1269,7 +1280,7 @@ def load( log.warning(f"Not loading '{task_name}' as the task errored.") continue - task_paths[task_name] = self._job_data_path(task_id=job.task_id, path_dir=path_dir) + task_paths[task_name] = str(self._job_data_path(task_id=job.task_id, path_dir=path_dir)) task_ids[task_name] = self.jobs[task_name].task_id data = BatchData( @@ -1350,13 +1361,14 @@ def estimate_cost(self, verbose: bool = True) -> float: return batch_cost @staticmethod - def _check_path_dir(path_dir: str) -> None: + def _check_path_dir(path_dir: PathLike) -> None: """Make sure ``path_dir`` exists and create it if not. Parameters ---------- - path_dir : str + path_dir : PathLike Directory path where files will be saved. """ - if len(path_dir) > 0 and not os.path.exists(path_dir): - os.makedirs(path_dir, exist_ok=True) + path_dir = Path(path_dir) + if path_dir != Path(".") and not path_dir.exists(): + path_dir.mkdir(parents=True, exist_ok=True) diff --git a/tidy3d/web/api/mode.py b/tidy3d/web/api/mode.py index f69be5cd26..b09f1bfd20 100644 --- a/tidy3d/web/api/mode.py +++ b/tidy3d/web/api/mode.py @@ -7,6 +7,7 @@ import tempfile import time from datetime import datetime +from os import PathLike from typing import Callable, Literal, Optional, Union import pydantic.v1 as pydantic @@ -50,7 +51,7 @@ def run( task_name: str = "Untitled", mode_solver_name: str = "mode_solver", folder_name: str = "Mode Solver", - results_file: str = "mode_solver.hdf5", + results_file: PathLike = "mode_solver.hdf5", verbose: bool = True, progress_callback_upload: Optional[Callable[[float], None]] = None, progress_callback_download: Optional[Callable[[float], None]] = None, @@ -70,7 +71,7 @@ def run( The name of the mode solver to create the in task. folder_name : str = "Mode Solver" Name of folder to store task on web UI. - results_file : str = "mode_solver.hdf5" + results_file : PathLike = "mode_solver.hdf5" Path to download results file (.hdf5). verbose : bool = True If ``True``, will print status, otherwise, will run silently. @@ -372,8 +373,8 @@ def get( cls, task_id: str, solver_id: str, - to_file: str = "mode_solver.hdf5", - sim_file: str = "simulation.hdf5", + to_file: PathLike = "mode_solver.hdf5", + sim_file: PathLike = "simulation.hdf5", verbose: bool = True, progress_callback: Optional[Callable[[float], None]] = None, ) -> ModeSolverTask: @@ -385,9 +386,9 @@ def get( Unique identifier of the task on server. solver_id: str Unique identifier of the mode solver in the task. - to_file: str = "mode_solver.hdf5" + to_file: PathLike = "mode_solver.hdf5" File to store the mode solver downloaded from the task. - sim_file: str = "simulation.hdf5" + sim_file: PathLike = "simulation.hdf5" File to store the simulation downloaded from the task. verbose: bool = True Whether to display progress bars. @@ -497,8 +498,8 @@ def abort(self): def get_modesolver( self, - to_file: str = "mode_solver.hdf5", - sim_file: str = "simulation.hdf5", + to_file: PathLike = "mode_solver.hdf5", + sim_file: PathLike = "simulation.hdf5", verbose: bool = True, progress_callback: Optional[Callable[[float], None]] = None, ) -> ModeSolver: @@ -506,9 +507,9 @@ def get_modesolver( Parameters ---------- - to_file: str = "mode_solver.hdf5" + to_file: PathLike = "mode_solver.hdf5" File to store the mode solver downloaded from the task. - sim_file: str = "simulation.hdf5" + sim_file: PathLike = "simulation.hdf5" File to store the simulation downloaded from the task, if any. verbose: bool = True Whether to display progress bars. @@ -582,7 +583,7 @@ def get_modesolver( def get_result( self, - to_file: str = "mode_solver_data.hdf5", + to_file: PathLike = "mode_solver_data.hdf5", verbose: bool = True, progress_callback: Optional[Callable[[float], None]] = None, ) -> ModeSolverData: @@ -590,7 +591,7 @@ def get_result( Parameters ---------- - to_file: str = "mode_solver_data.hdf5" + to_file: PathLike = "mode_solver_data.hdf5" File to store the mode solver downloaded from the task. verbose: bool = True Whether to display progress bars. @@ -645,7 +646,7 @@ def get_result( def get_log( self, - to_file: str = "mode_solver.log", + to_file: PathLike = "mode_solver.log", verbose: bool = True, progress_callback: Optional[Callable[[float], None]] = None, ) -> pathlib.Path: @@ -653,7 +654,7 @@ def get_log( Parameters ---------- - to_file: str = "mode_solver.log" + to_file: PathLike = "mode_solver.log" File to store the mode solver downloaded from the task. verbose: bool = True Whether to display progress bars. diff --git a/tidy3d/web/api/run.py b/tidy3d/web/api/run.py index c65f8e94a3..a4b3cb7666 100644 --- a/tidy3d/web/api/run.py +++ b/tidy3d/web/api/run.py @@ -1,6 +1,8 @@ from __future__ import annotations import typing +from os import PathLike +from pathlib import Path from tidy3d.components.types.workflow import WorkflowDataType, WorkflowType from tidy3d.config import config @@ -81,7 +83,7 @@ def run( simulation: RunInput, task_name: typing.Optional[str] = None, folder_name: str = "default", - path: typing.Optional[str] = None, + path: typing.Optional[PathLike] = None, callback_url: typing.Optional[str] = None, verbose: bool = True, progress_callback_upload: typing.Optional[typing.Callable[[float], None]] = None, @@ -127,7 +129,7 @@ def run( Optional name for a single run. Prefixed for multiple runs. folder_name : str = "default" Folder shown on the web UI. - path : Optional[str] = None + path : Optional[PathLike] = None Output path. Interpreted as a file path for single simulations and a directory for multiple simulations. Defaults are "simulation.hdf5" (single simulation) and the current directory (multiple simulations). callback_url : Optional[str] = None @@ -230,7 +232,7 @@ def run( key_prefix = "" if len(h2sim) == 1: - path = path if path is not None else DEFAULT_DATA_PATH + path = path if path is not None else Path(DEFAULT_DATA_PATH) h, sim = next(iter(h2sim.items())) data = { h: run_autograd( @@ -257,11 +259,11 @@ def run( else: key_prefix = f"{task_name}_" if task_name else "" sims = {f"{key_prefix}{h}": s for h, s in h2sim.items()} - path = path if path is not None else DEFAULT_DATA_DIR + path_dir = Path(path) if path is not None else Path(DEFAULT_DATA_DIR) data = run_async( simulations=sims, folder_name=folder_name, - path_dir=path, + path_dir=path_dir, callback_url=callback_url, num_workers=max_workers, verbose=verbose, diff --git a/tidy3d/web/api/tidy3d_stub.py b/tidy3d/web/api/tidy3d_stub.py index 22152e21ab..20fddb5bab 100644 --- a/tidy3d/web/api/tidy3d_stub.py +++ b/tidy3d/web/api/tidy3d_stub.py @@ -4,6 +4,8 @@ import json from datetime import datetime +from os import PathLike +from pathlib import Path from typing import Callable, Optional import pydantic.v1 as pd @@ -73,13 +75,13 @@ class Tidy3dStub(BaseModel, TaskStub): simulation: WorkflowType = pd.Field(discriminator="type") @classmethod - def from_file(cls, file_path: str) -> WorkflowType: + def from_file(cls, file_path: PathLike) -> WorkflowType: """Loads a Union[:class:`.Simulation`, :class:`.HeatSimulation`, :class:`.EMESimulation`] from .yaml, .json, or .hdf5 file. Parameters ---------- - file_path : str + file_path : PathLike Full path to the .yaml or .json or .hdf5 file to load the Union[:class:`.Simulation`, :class:`.HeatSimulation`, :class:`.EMESimulation`] from. @@ -92,13 +94,14 @@ def from_file(cls, file_path: str) -> WorkflowType: ------- >>> simulation = Simulation.from_file(fname='folder/sim.json') # doctest: +SKIP """ - extension = _get_valid_extension(file_path) + path = Path(file_path) + extension = _get_valid_extension(path) if extension == ".json": - json_str = read_simulation_from_json(file_path) + json_str = read_simulation_from_json(path) elif extension == ".hdf5": - json_str = read_simulation_from_hdf5(file_path) + json_str = read_simulation_from_hdf5(path) elif extension == ".hdf5.gz": - json_str = read_simulation_from_hdf5_gz(file_path) + json_str = read_simulation_from_hdf5_gz(path) data = json.loads(json_str) type_ = data["type"] @@ -123,20 +126,20 @@ def from_file(cls, file_path: str) -> WorkflowType: ) sim_class = class_map[type_] - sim = sim_class.from_file(file_path) + sim = sim_class.from_file(path) return sim def to_file( self, - file_path: str, + file_path: PathLike, ): """Exports Union[:class:`.Simulation`, :class:`.HeatSimulation`, :class:`.EMESimulation`] instance to .yaml, .json, or .hdf5 file Parameters ---------- - file_path : str + file_path : PathLike Full path to the .yaml or .json or .hdf5 file to save the :class:`Stub` to. Example @@ -145,16 +148,16 @@ def to_file( """ self.simulation.to_file(file_path) - def to_hdf5_gz(self, fname: str, custom_encoders: Optional[list[Callable]] = None) -> None: + def to_hdf5_gz(self, fname: PathLike, custom_encoders: Optional[list[Callable]] = None) -> None: """Exports Union[:class:`.Simulation`, :class:`.HeatSimulation`, :class:`.EMESimulation`] instance to .hdf5.gz file. Parameters ---------- - fname : str + fname : PathLike Full path to the .hdf5.gz file to save the Union[:class:`.Simulation`, :class:`.HeatSimulation`, :class:`.EMESimulation`] to. custom_encoders : List[Callable] - List of functions accepting (fname: str, group_path: str, value: Any) that take + List of functions accepting (fname: PathLike, group_path: str, value: Any) that take the ``value`` supplied and write it to the hdf5 ``fname`` at ``group_path``. Example @@ -210,14 +213,14 @@ class Tidy3dStubData(BaseModel, TaskStubData): @classmethod def from_file( - cls, file_path: str, lazy: bool = False, on_load: Optional[Callable] = None + cls, file_path: PathLike, lazy: bool = False, on_load: Optional[Callable] = None ) -> WorkflowDataType: """Loads a Union[:class:`.SimulationData`, :class:`.HeatSimulationData`, :class:`.EMESimulationData`] from .yaml, .json, or .hdf5 file. Parameters ---------- - file_path : str + file_path : PathLike Full path to the .yaml or .json or .hdf5 file to load the Union[:class:`.SimulationData`, :class:`.HeatSimulationData`, :class:`.EMESimulationData`] from. lazy : bool = False @@ -234,13 +237,14 @@ def from_file( Union[:class:`.SimulationData`, :class:`.HeatSimulationData`, :class:`.EMESimulationData`] An instance of the component class calling ``load``. """ - extension = _get_valid_extension(file_path) + path = Path(file_path) + extension = _get_valid_extension(path) if extension == ".json": - json_str = read_simulation_from_json(file_path) + json_str = read_simulation_from_json(path) elif extension == ".hdf5": - json_str = read_simulation_from_hdf5(file_path) + json_str = read_simulation_from_hdf5(path) elif extension == ".hdf5.gz": - json_str = read_simulation_from_hdf5_gz(file_path) + json_str = read_simulation_from_hdf5_gz(path) data = json.loads(json_str) type_ = data["type"] @@ -266,17 +270,17 @@ def from_file( ) data_class = data_class_map[type_] - sim_data = data_class.from_file(file_path, lazy=lazy, on_load=on_load) + sim_data = data_class.from_file(path, lazy=lazy, on_load=on_load) return sim_data - def to_file(self, file_path: str): + def to_file(self, file_path: PathLike): """Exports Union[:class:`.SimulationData`, :class:`.HeatSimulationData`, :class:`.EMESimulationData`] instance to .yaml, .json, or .hdf5 file Parameters ---------- - file_path : str + file_path : PathLike Full path to the .yaml or .json or .hdf5 file to save the Union[:class:`.SimulationData`, :class:`.HeatSimulationData`, :class:`.EMESimulationData`] to. @@ -287,13 +291,13 @@ def to_file(self, file_path: str): self.data.to_file(file_path) @classmethod - def postprocess(cls, file_path: str, lazy: bool = True) -> WorkflowDataType: + def postprocess(cls, file_path: PathLike, lazy: bool = True) -> WorkflowDataType: """Load .yaml, .json, or .hdf5 file to Union[:class:`.SimulationData`, :class:`.HeatSimulationData`, :class:`.EMESimulationData`] instance. Parameters ---------- - file_path : str + file_path : PathLike Full path to the .yaml or .json or .hdf5 file to save the Union[:class:`.SimulationData`, :class:`.HeatSimulationData`, :class:`.EMESimulationData`] to. lazy : bool = False diff --git a/tidy3d/web/api/webapi.py b/tidy3d/web/api/webapi.py index 7da3e9f7ca..64911251e8 100644 --- a/tidy3d/web/api/webapi.py +++ b/tidy3d/web/api/webapi.py @@ -3,9 +3,10 @@ from __future__ import annotations import json -import os import tempfile import time +from os import PathLike +from pathlib import Path from typing import Callable, Literal, Optional, Union from requests import HTTPError @@ -321,7 +322,7 @@ def run( simulation: WorkflowType, task_name: Optional[str] = None, folder_name: str = "default", - path: str = "simulation_data.hdf5", + path: PathLike = "simulation_data.hdf5", callback_url: Optional[str] = None, verbose: bool = True, progress_callback_upload: Optional[Callable[[float], None]] = None, @@ -347,7 +348,7 @@ def run( Name of task. If not provided, a default name will be generated. folder_name : str = "default" Name of folder to store task on web UI. - path : str = "simulation_data.hdf5" + path : PathLike = "simulation_data.hdf5" Path to download results file (.hdf5), including filename. callback_url : str = None Http PUT url to receive simulation finish event. The body content is a json file with @@ -1041,7 +1042,7 @@ def abort(task_id: TaskId): @wait_for_connection def download( task_id: TaskId, - path: str = "simulation_data.hdf5", + path: PathLike = "simulation_data.hdf5", verbose: bool = True, progress_callback: Optional[Callable[[float], None]] = None, ) -> None: @@ -1051,7 +1052,7 @@ def download( ---------- task_id : str Unique identifier of task on server. Returned by :meth:`upload`. - path : str = "simulation_data.hdf5" + path : PathLike = "simulation_data.hdf5" Download path to .hdf5 data file (including filename). verbose : bool = True If ``True``, will print progressbars and status, otherwise, will run silently. @@ -1059,13 +1060,13 @@ def download( Optional callback function called when downloading file with ``bytes_in_chunk`` as argument. """ - # Component modeler batch download path + path = Path(path) + if _is_modeler_batch(task_id): # Use a more descriptive default filename for component modeler downloads. # If the caller left the default as 'simulation_data.hdf5', prefer 'cm_data.hdf5'. - if os.path.basename(path) == "simulation_data.hdf5": - base_dir = os.path.dirname(path) or "." - path = os.path.join(base_dir, "cm_data.hdf5") + if path.name == "simulation_data.hdf5": + path = path.with_name("cm_data.hdf5") def _download_cm() -> bool: try: @@ -1117,20 +1118,19 @@ def _download_cm() -> bool: @wait_for_connection -def download_json(task_id: TaskId, path: str = SIM_FILE_JSON, verbose: bool = True) -> None: +def download_json(task_id: TaskId, path: PathLike = SIM_FILE_JSON, verbose: bool = True) -> None: """Download the ``.json`` file associated with the :class:`.Simulation` of a given task. Parameters ---------- task_id : str Unique identifier of task on server. Returned by :meth:`upload`. - path : str = "simulation.json" + path : PathLike = "simulation.json" Download path to .json file of simulation (including filename). verbose : bool = True If ``True``, will print progressbars and status, otherwise, will run silently. """ - task = SimulationTask(taskId=task_id) task.get_simulation_json(path, verbose=verbose) @@ -1144,7 +1144,7 @@ def delete_old(days_old: int, folder_name: str = "default") -> int: @wait_for_connection def load_simulation( - task_id: TaskId, path: str = SIM_FILE_JSON, verbose: bool = True + task_id: TaskId, path: PathLike = SIM_FILE_JSON, verbose: bool = True ) -> WorkflowType: """Download the ``.json`` file of a task and load the associated simulation. @@ -1152,7 +1152,7 @@ def load_simulation( ---------- task_id : str Unique identifier of task on server. Returned by :meth:`upload`. - path : str = "simulation.json" + path : PathLike = "simulation.json" Download path to .json file of simulation (including filename). verbose : bool = True If ``True``, will print progressbars and status, otherwise, will run silently. @@ -1162,7 +1162,6 @@ def load_simulation( Union[:class:`.Simulation`, :class:`.HeatSimulation`, :class:`.EMESimulation`] Simulation loaded from downloaded json file. """ - task = SimulationTask.get(task_id) task.get_simulation_json(path, verbose=verbose) return Tidy3dStub.from_file(path) @@ -1171,7 +1170,7 @@ def load_simulation( @wait_for_connection def download_log( task_id: TaskId, - path: str = "tidy3d.log", + path: PathLike = "tidy3d.log", verbose: bool = True, progress_callback: Optional[Callable[[float], None]] = None, ) -> None: @@ -1181,7 +1180,7 @@ def download_log( ---------- task_id : str Unique identifier of task on server. Returned by :meth:`upload`. - path : str = "tidy3d.log" + path : PathLike = "tidy3d.log" Download path to log file (including filename). verbose : bool = True If ``True``, will print progressbars and status, otherwise, will run silently. @@ -1199,7 +1198,7 @@ def download_log( @wait_for_connection def load( task_id: TaskId, - path: str = "simulation_data.hdf5", + path: PathLike = "simulation_data.hdf5", replace_existing: bool = True, verbose: bool = True, progress_callback: Optional[Callable[[float], None]] = None, @@ -1225,7 +1224,7 @@ def load( ---------- task_id : str Unique identifier of task on server. Returned by :meth:`upload`. - path : str + path : PathLike Download path to .hdf5 data file (including filename). replace_existing : bool = True Downloads the data even if path exists (overwriting the existing). @@ -1243,15 +1242,21 @@ def load( Object containing simulation data. """ # For component modeler batches, default to a clearer filename if the default was used. - if _is_modeler_batch(task_id): - base_dir = os.path.dirname(path) or "." - if os.path.basename(path) == "simulation_data.hdf5": - path = os.path.join(base_dir, "cm_data.hdf5") - elif os.path.basename(path) == "simulation_data.hdf5.gz": - path = os.path.join(base_dir, "cm_data.hdf5.gz") + path = Path(path) - if not os.path.exists(path) or replace_existing: - download(task_id=task_id, path=path, verbose=verbose, progress_callback=progress_callback) + if _is_modeler_batch(task_id): + if path.name == "simulation_data.hdf5": + path = path.with_name("cm_data.hdf5") + elif path.name == "simulation_data.hdf5.gz": + path = path.with_name("cm_data.hdf5.gz") + + if not path.exists() or replace_existing: + download( + task_id=task_id, + path=path, + verbose=verbose, + progress_callback=progress_callback, + ) if verbose: console = get_logging_console() @@ -1459,7 +1464,7 @@ def delete(task_id: TaskId, versions: bool = False) -> TaskInfo: @wait_for_connection def download_simulation( task_id: TaskId, - path: str = SIM_FILE_HDF5, + path: PathLike = SIM_FILE_HDF5, verbose: bool = True, progress_callback: Optional[Callable[[float], None]] = None, ) -> None: @@ -1469,7 +1474,7 @@ def download_simulation( ---------- task_id : str Unique identifier of task on server. Returned by :meth:`upload`. - path : str = "simulation.hdf5" + path : PathLike = "simulation.hdf5" Download path to .hdf5 file of simulation (including filename). verbose : bool = True If ``True``, will print progressbars and status, otherwise, will run silently. @@ -1486,7 +1491,10 @@ def download_simulation( task = SimulationTask(taskId=task_id) task.get_simulation_hdf5( - path, verbose=verbose, progress_callback=progress_callback, remote_sim_file=remote_sim_file + path, + verbose=verbose, + progress_callback=progress_callback, + remote_sim_file=remote_sim_file, ) diff --git a/tidy3d/web/core/file_util.py b/tidy3d/web/core/file_util.py index fe45122bb2..b3987f495c 100644 --- a/tidy3d/web/core/file_util.py +++ b/tidy3d/web/core/file_util.py @@ -12,33 +12,37 @@ from tidy3d.web.core.constants import JSON_TAG -def compress_file_to_gzip(input_file, output_gz_file): - """ - Compresses a file using gzip. - - Args: - input_file (str): The path of the input file. - output_gz_file (str): The path of the output gzip file. +def compress_file_to_gzip(input_file: os.PathLike, output_gz_file: os.PathLike) -> None: + """Compresses a file using gzip. + + Parameters + ---------- + input_file : PathLike + The path of the input file. + output_gz_file : PathLike + The path of the output gzip file. """ with open(input_file, "rb") as file_in: with gzip.open(output_gz_file, "wb") as file_out: shutil.copyfileobj(file_in, file_out) -def extract_gzip_file(input_gz_file, output_file): - """ - Extract a gzip file. +def extract_gzip_file(input_gz_file: os.PathLike, output_file: os.PathLike) -> None: + """Extract a gzip file. - Args: - input_gz_file (str): The path of the gzip input file. - output_file (str): The path of the output file. + Parameters + ---------- + input_gz_file : PathLike + The path of the gzip input file. + output_file : PathLike + The path of the output file. """ with gzip.open(input_gz_file, "rb") as file_in: with open(output_file, "wb") as file_out: shutil.copyfileobj(file_in, file_out) -def read_simulation_from_hdf5_gz(file_name: str) -> str: +def read_simulation_from_hdf5_gz(file_name: os.PathLike) -> str: """read simulation str from hdf5.gz""" hdf5_file, hdf5_file_path = tempfile.mkstemp(".hdf5") @@ -56,14 +60,14 @@ def read_simulation_from_hdf5_gz(file_name: str) -> str: as methods in Tidy3dBaseModel. For consistency it would be best if this duplication is avoided.""" -def _json_string_key(index): +def _json_string_key(index: int) -> str: """Get json string key for string chunk number ``index``.""" if index: return f"{JSON_TAG}_{index}" return JSON_TAG -def read_simulation_from_hdf5(file_name: str) -> str: +def read_simulation_from_hdf5(file_name: os.PathLike) -> bytes: """read simulation str from hdf5""" with h5py.File(file_name, "r") as f_handle: num_string_parts = len([key for key in f_handle.keys() if JSON_TAG in key]) @@ -76,7 +80,7 @@ def read_simulation_from_hdf5(file_name: str) -> str: """End TODO""" -def read_simulation_from_json(file_name: str) -> str: +def read_simulation_from_json(file_name: os.PathLike) -> str: """read simulation str from json""" with open(file_name) as json_file: json_data = json_file.read() diff --git a/tidy3d/web/core/s3utils.py b/tidy3d/web/core/s3utils.py index cb87a69a22..2965e8ca61 100644 --- a/tidy3d/web/core/s3utils.py +++ b/tidy3d/web/core/s3utils.py @@ -3,12 +3,13 @@ from __future__ import annotations import os -import pathlib import tempfile import urllib from collections.abc import Mapping from datetime import datetime from enum import Enum +from os import PathLike +from pathlib import Path from typing import Callable, Optional import boto3 @@ -184,7 +185,7 @@ def _get_progress(action: _S3Action): def get_s3_sts_token( - resource_id: str, file_name: str, extra_arguments: Optional[Mapping[str, str]] = None + resource_id: str, file_name: PathLike, extra_arguments: Optional[Mapping[str, str]] = None ) -> _S3STSToken: """Get s3 sts token for the given resource id and file name. @@ -192,7 +193,7 @@ def get_s3_sts_token( ---------- resource_id : str The resource id, e.g. task id. - file_name : str + file_name : PathLike The remote file name on S3. extra_arguments : Mapping[str, str] Additional arguments for the query url. @@ -202,6 +203,7 @@ def get_s3_sts_token( _S3STSToken The S3 STS token. """ + file_name = str(Path(file_name).as_posix()) cache_key = f"{resource_id}:{file_name}" if cache_key not in _s3_sts_tokens or _s3_sts_tokens[cache_key].is_expired(): method = f"tidy3d/py/tasks/{resource_id}/file?filename={file_name}" @@ -215,8 +217,8 @@ def get_s3_sts_token( def upload_file( resource_id: str, - path: str, - remote_filename: str, + path: PathLike, + remote_filename: PathLike, verbose: bool = True, progress_callback: Optional[Callable[[float], None]] = None, extra_arguments: Optional[Mapping[str, str]] = None, @@ -227,9 +229,9 @@ def upload_file( ---------- resource_id : str The resource id, e.g. task id. - path : str + path : PathLike Path to the file to upload. - remote_filename : str + remote_filename : PathLike The remote file name on S3 relative to the resource context root path. verbose : bool = True Whether to display a progressbar for the upload. @@ -239,6 +241,7 @@ def upload_file( Additional arguments used to specify the upload bucket. """ + path = Path(path) token = get_s3_sts_token(resource_id, remote_filename, extra_arguments) def _upload(_callback: Callable) -> None: @@ -250,7 +253,7 @@ def _upload(_callback: Callable) -> None: Callback function for upload, accepts ``bytes_in_chunk`` """ - with open(path, "rb") as data: + with path.open("rb") as data: token.get_client().upload_fileobj( data, Bucket=token.get_bucket(), @@ -267,8 +270,10 @@ def _upload(_callback: Callable) -> None: else: if verbose: with _get_progress(_S3Action.UPLOADING) as progress: - total_size = pathlib.Path(path).stat().st_size - task_id = progress.add_task("upload", filename=remote_filename, total=total_size) + total_size = path.stat().st_size + task_id = progress.add_task( + "upload", filename=str(remote_filename), total=total_size + ) def _callback(bytes_in_chunk): progress.update(task_id, advance=bytes_in_chunk) @@ -283,21 +288,22 @@ def _callback(bytes_in_chunk): def download_file( resource_id: str, - remote_filename: str, - to_file: Optional[str] = None, + remote_filename: PathLike, + to_file: Optional[PathLike] = None, verbose: bool = True, progress_callback: Optional[Callable[[float], None]] = None, -) -> pathlib.Path: +) -> Path: """Download file from S3. Parameters ---------- resource_id : str The resource id, e.g. task id. - remote_filename : str + remote_filename : PathLike Path to the remote file. - to_file : str = None - Local filename to save to, if not specified, use the remote_filename. + to_file : PathLike = None + Local filename to save to; if not specified, defaults to ``remote_filename`` in a + directory named after ``resource_id``. verbose : bool = True Whether to display a progressbar for the upload progress_callback : Callable[[float], None] = None @@ -309,14 +315,13 @@ def download_file( meta_data = client.head_object(Bucket=token.get_bucket(), Key=token.get_s3_key()) # Get only last part of the remote file name - remote_basename = pathlib.Path(remote_filename).name + remote_basename = Path(remote_filename).name # set to_file if None - if not to_file: - path = pathlib.Path(resource_id) - to_path = path / remote_basename + if to_file is None: + to_path = Path(resource_id) / remote_basename else: - to_path = pathlib.Path(to_file) + to_path = Path(to_file) # make the leading directories in the 'to_path', if any to_path.parent.mkdir(parents=True, exist_ok=True) @@ -336,15 +341,15 @@ def _download(_callback: Callable) -> None: try: fd, tmp_file_path_str = tempfile.mkstemp(suffix=IN_TRANSIT_SUFFIX, dir=to_path.parent) os.close(fd) # `tempfile.mkstemp()` creates and opens a randomly named file. close it. - to_path_tmp = pathlib.Path(tmp_file_path_str) + to_path_tmp = Path(tmp_file_path_str) client.download_file( Bucket=token.get_bucket(), - Filename=tmp_file_path_str, + Filename=str(to_path_tmp), Key=token.get_s3_key(), Callback=_callback, Config=_s3_config, ) - to_path_tmp.rename(to_file) + to_path_tmp.rename(to_path) except Exception as e: to_path_tmp.unlink(missing_ok=True) # Delete incompletely downloaded file. raise e @@ -373,11 +378,11 @@ def _callback(bytes_in_chunk): def download_gz_file( resource_id: str, - remote_filename: str, - to_file: Optional[str] = None, + remote_filename: PathLike, + to_file: Optional[PathLike] = None, verbose: bool = True, progress_callback: Optional[Callable[[float], None]] = None, -) -> pathlib.Path: +) -> Path: """Download a ``.gz`` file and unzip it into ``to_file``, unless ``to_file`` itself ends in .gz @@ -385,10 +390,11 @@ def download_gz_file( ---------- resource_id : str The resource id, e.g. task id. - remote_filename : str + remote_filename : PathLike Path to the remote file. - to_file : str = None - Local filename to save to, if not specified, use the remote_filename. + to_file : Optional[PathLike] = None + Local filename to save to; if not specified, defaults to ``remote_filename`` with the + ``.gz`` suffix removed in a directory named after ``resource_id``. verbose : bool = True Whether to display a progressbar for the upload progress_callback : Callable[[float], None] = None @@ -396,11 +402,20 @@ def download_gz_file( """ # If to_file is a gzip extension, just download - if to_file.lower().endswith(".gz"): + if to_file is None: + remote_basename = Path(remote_filename).name + if remote_basename.endswith(".gz"): + remote_basename = remote_basename[:-3] + to_path = Path(resource_id) / remote_basename + else: + to_path = Path(to_file) + + suffixes = "".join(to_path.suffixes).lower() + if suffixes.endswith(".gz"): return download_file( resource_id, remote_filename, - to_file=to_file, + to_file=to_path, verbose=verbose, progress_callback=progress_callback, ) @@ -411,18 +426,17 @@ def download_gz_file( os.close(tmp_file) # make the leading directories in the 'to_file', if any - to_path = pathlib.Path(to_file) to_path.parent.mkdir(parents=True, exist_ok=True) try: download_file( resource_id, remote_filename, - to_file=tmp_file_path_str, + to_file=Path(tmp_file_path_str), verbose=verbose, progress_callback=progress_callback, ) if os.path.exists(tmp_file_path_str): - extract_gzip_file(tmp_file_path_str, to_path) + extract_gzip_file(Path(tmp_file_path_str), to_path) else: raise WebError(f"Failed to download and extract '{remote_filename}'.") finally: diff --git a/tidy3d/web/core/stub.py b/tidy3d/web/core/stub.py index 153fd69f22..53ec961462 100644 --- a/tidy3d/web/core/stub.py +++ b/tidy3d/web/core/stub.py @@ -3,16 +3,17 @@ from __future__ import annotations from abc import ABC, abstractmethod +from os import PathLike class TaskStubData(ABC): @abstractmethod - def from_file(self, file_path) -> TaskStubData: + def from_file(self, file_path: PathLike) -> TaskStubData: """Loads a :class:`TaskStubData` from .yaml, .json, or .hdf5 file. Parameters ---------- - file_path : str + file_path : PathLike Full path to the .yaml or .json or .hdf5 file to load the :class:`Stub` from. Returns @@ -23,12 +24,12 @@ def from_file(self, file_path) -> TaskStubData: """ @abstractmethod - def to_file(self, file_path): + def to_file(self, file_path: PathLike): """Loads a :class:`Stub` from .yaml, .json, or .hdf5 file. Parameters ---------- - file_path : str + file_path : PathLike Full path to the .yaml or .json or .hdf5 file to load the :class:`Stub` from. Returns @@ -40,12 +41,12 @@ def to_file(self, file_path): class TaskStub(ABC): @abstractmethod - def from_file(self, file_path) -> TaskStub: + def from_file(self, file_path: PathLike) -> TaskStub: """Loads a :class:`TaskStubData` from .yaml, .json, or .hdf5 file. Parameters ---------- - file_path : str + file_path : PathLike Full path to the .yaml or .json or .hdf5 file to load the :class:`Stub` from. Returns @@ -55,12 +56,12 @@ def from_file(self, file_path) -> TaskStub: """ @abstractmethod - def to_file(self, file_path): + def to_file(self, file_path: PathLike): """Loads a :class:`TaskStub` from .yaml, .json, .hdf5 or .hdf5.gz file. Parameters ---------- - file_path : str + file_path : PathLike Full path to the .yaml or .json or .hdf5 file to load the :class:`TaskStub` from. Returns @@ -70,11 +71,11 @@ def to_file(self, file_path): """ @abstractmethod - def to_hdf5_gz(self, fname: str) -> None: + def to_hdf5_gz(self, fname: PathLike) -> None: """Exports :class:`TaskStub` instance to .hdf5.gz file. Parameters ---------- - fname : str + fname : PathLike Full path to the .hdf5.gz file to save the :class:`TaskStub` to. """ diff --git a/tidy3d/web/core/task_core.py b/tidy3d/web/core/task_core.py index 619ff4bc8d..696e7edab0 100644 --- a/tidy3d/web/core/task_core.py +++ b/tidy3d/web/core/task_core.py @@ -7,6 +7,7 @@ import tempfile import time from datetime import datetime +from os import PathLike from typing import Callable, Optional, Union from botocore.exceptions import ClientError @@ -296,6 +297,21 @@ def get(cls, task_id: str, verbose: bool = True) -> SimulationTask: task = SimulationTask(**resp) if resp else None return task + @classmethod + def get_running_tasks(cls) -> list[SimulationTask]: + """Get a list of running tasks from the server" + + Returns + ------- + List[:class:`.SimulationTask`] + :class:`.SimulationTask` object containing info about status, + size, credits of task and others. + """ + resp = http.get("tidy3d/py/tasks") + if not resp: + return [] + return parse_obj_as(list[SimulationTask], resp) + def delete(self, versions: bool = False): """Delete current task from server. @@ -319,12 +335,12 @@ def delete(self, versions: bool = False): else: # Fallback to old method if we can't get the groupId and version http.delete(f"tidy3d/tasks/{self.task_id}") - def get_simulation_json(self, to_file: str, verbose: bool = True) -> pathlib.Path: + def get_simulation_json(self, to_file: PathLike, verbose: bool = True): """Get json file for a :class:`.Simulation` from server. Parameters ---------- - to_file: str + to_file: PathLike Save file to path. verbose: bool = True Whether to display progress bars. @@ -337,13 +353,16 @@ def get_simulation_json(self, to_file: str, verbose: bool = True) -> pathlib.Pat if not self.task_id: raise WebError("Expected field 'task_id' is unset.") + to_file = pathlib.Path(to_file) + hdf5_file, hdf5_file_path = tempfile.mkstemp(".hdf5") os.close(hdf5_file) try: self.get_simulation_hdf5(hdf5_file_path) if os.path.exists(hdf5_file_path): json_string = read_simulation_from_hdf5(hdf5_file_path) - with open(to_file, "w") as file: + to_file.parent.mkdir(parents=True, exist_ok=True) + with to_file.open("w", encoding="utf-8") as file: # Write the string to the file file.write(json_string.decode("utf-8")) if verbose: @@ -359,7 +378,7 @@ def upload_simulation( stub: TaskStub, verbose: bool = True, progress_callback: Optional[Callable[[float], None]] = None, - remote_sim_file: str = SIM_FILE_HDF5_GZ, + remote_sim_file: PathLike = SIM_FILE_HDF5_GZ, ) -> None: """Upload :class:`.Simulation` object to Server. @@ -395,7 +414,7 @@ def upload_simulation( def upload_file( self, - local_file: str, + local_file: PathLike, remote_filename: str, verbose: bool = True, progress_callback: Optional[Callable[[float], None]] = None, @@ -405,8 +424,8 @@ def upload_file( as :class".simulation". Parameters ---------- - local_file: str - local file path. + local_file: PathLike + Local file path. remote_filename: str file name on the server verbose: bool = True @@ -502,16 +521,16 @@ def estimate_cost(self, solver_version=None) -> float: def get_sim_data_hdf5( self, - to_file: str, + to_file: PathLike, verbose: bool = True, progress_callback: Optional[Callable[[float], None]] = None, - remote_data_file: str = SIMULATION_DATA_HDF5_GZ, + remote_data_file: PathLike = SIMULATION_DATA_HDF5_GZ, ) -> pathlib.Path: """Get simulation data file from Server. Parameters ---------- - to_file: str + to_file: PathLike Save file to path. verbose: bool = True Whether to display progress bars. @@ -526,12 +545,14 @@ def get_sim_data_hdf5( if not self.task_id: raise WebError("Expected field 'task_id' is unset.") + target_path = pathlib.Path(to_file) + file = None try: file = download_gz_file( resource_id=self.task_id, remote_filename=remote_data_file, - to_file=to_file, + to_file=target_path, verbose=verbose, progress_callback=progress_callback, ) @@ -545,7 +566,7 @@ def get_sim_data_hdf5( file = download_file( resource_id=self.task_id, remote_filename=remote_data_file[:-3], - to_file=to_file, + to_file=target_path, verbose=verbose, progress_callback=progress_callback, ) @@ -559,16 +580,16 @@ def get_sim_data_hdf5( def get_simulation_hdf5( self, - to_file: str, + to_file: PathLike, verbose: bool = True, progress_callback: Optional[Callable[[float], None]] = None, - remote_sim_file: str = SIM_FILE_HDF5_GZ, + remote_sim_file: PathLike = SIM_FILE_HDF5_GZ, ) -> pathlib.Path: """Get simulation.hdf5 file from Server. Parameters ---------- - to_file: str + to_file: PathLike Save file to path. verbose: bool = True Whether to display progress bars. @@ -583,10 +604,12 @@ def get_simulation_hdf5( if not self.task_id: raise WebError("Expected field 'task_id' is unset.") + target_path = pathlib.Path(to_file) + return download_gz_file( resource_id=self.task_id, remote_filename=remote_sim_file, - to_file=to_file, + to_file=target_path, verbose=verbose, progress_callback=progress_callback, ) @@ -613,7 +636,7 @@ def get_running_info(self) -> tuple[float, float]: def get_log( self, - to_file: str, + to_file: PathLike, verbose: bool = True, progress_callback: Optional[Callable[[float], None]] = None, ) -> pathlib.Path: @@ -621,7 +644,7 @@ def get_log( Parameters ---------- - to_file: str + to_file: PathLike Save file to path. verbose: bool = True Whether to display progress bars. @@ -637,20 +660,22 @@ def get_log( if not self.task_id: raise WebError("Expected field 'task_id' is unset.") + target_path = pathlib.Path(to_file) + return download_file( self.task_id, SIM_LOG_FILE, - to_file=to_file, + to_file=target_path, verbose=verbose, progress_callback=progress_callback, ) - def get_error_json(self, to_file: str, verbose: bool = True) -> pathlib.Path: + def get_error_json(self, to_file: PathLike, verbose: bool = True) -> pathlib.Path: """Get error json file for a :class:`.Simulation` from server. Parameters ---------- - to_file: str + to_file: PathLike Save file to path. verbose: bool = True Whether to display progress bars. @@ -663,10 +688,12 @@ def get_error_json(self, to_file: str, verbose: bool = True) -> pathlib.Path: if not self.task_id: raise WebError("Expected field 'task_id' is unset.") + target_path = pathlib.Path(to_file) + return download_file( self.task_id, SIM_ERROR_FILE, - to_file=to_file, + to_file=target_path, verbose=verbose, ) @@ -961,7 +988,7 @@ def wait_for_run(self, timeout: Optional[float] = None, batch_type: str = "") -> def get_data_hdf5( self, remote_data_file_gz: str, - to_file: str, + to_file: PathLike, verbose: bool = True, progress_callback: Optional[Callable[[float], None]] = None, ) -> pathlib.Path: @@ -971,7 +998,7 @@ def get_data_hdf5( ---------- remote_data_file_gz : str Remote gzipped filename to download (e.g., 'output/cm_data.hdf5.gz'). - to_file : str + to_file : PathLike Local path where the downloaded file will be saved. verbose : bool, default=True If ``True``, shows progress logs and messages.