From 0b077fbd9507811a3b905c712da71325dba1c907 Mon Sep 17 00:00:00 2001 From: qingeng Date: Mon, 10 Jul 2023 14:46:05 +0800 Subject: [PATCH] Uploading tasks files as hdf5.gz instead of json or hdf5 --- CHANGELOG.md | 5 +- tests/sims/simulation_2_4_0rc2.json | 1881 ------------------------ tests/test_plugins/test_mode_solver.py | 38 +- tests/test_web/test_tidy3d_task.py | 30 +- tests/test_web/test_webapi.py | 20 +- tidy3d/plugins/mode/web.py | 105 +- tidy3d/web/__init__.py | 15 +- tidy3d/web/file_util.py | 41 + tidy3d/web/s3utils.py | 67 +- tidy3d/web/simulation_task.py | 147 +- tidy3d/web/webapi.py | 32 +- 11 files changed, 269 insertions(+), 2112 deletions(-) delete mode 100644 tests/sims/simulation_2_4_0rc2.json create mode 100644 tidy3d/web/file_util.py diff --git a/CHANGELOG.md b/CHANGELOG.md index 882788eb6..fc0ea1906 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -19,12 +19,15 @@ and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0 - Added option to allow DC component in `GaussianPulse` spectrum, by setting `remove_dc_component=False` in `GaussianPulse`. - Jax installation from `pip install "tidy3d[jax]"` handled same way on windows as other OS if python >= 3.9. - `colocate` introduced as an argument to `ModeSolver` and a Field in `ModeSolverMonitor`, set to True by default. -- `FieldTimeMonitor`-s and `FieldMonitor`-s that have `colocate=True` return fields colocated to the grid boundaries rather than centers. This matches better user expectations for example when the simulation has a symmetry (explicitly defined, or implicit) w.r.t. a given axis. When colocating to centers, fields would be ``dl / 2`` away from that symmetry plane, and components that are expected to go to zero do not (of course, they still do if interpolated to the symmetry plane). Another convenient use case is that it is easier to place a 2D monitor exactly on a grid boundary in the automatically generated grid, by simply passing an override structure with the monitor geometry. +- `FieldTimeMonitor`-s and `FieldMonitor`-s that have `colocate=True` return fields colocated to the grid boundaries rather than centers. This matches better user expectations for example when the simulation has a symmetry (explicitly defined, or implicit) w.r.t. a given axis. When colocating to centers, fields would be `dl / 2` away from that symmetry plane, and components that are expected to go to zero do not (of course, they still do if interpolated to the symmetry plane). Another convenient use case is that it is easier to place a 2D monitor exactly on a grid boundary in the automatically generated grid, by simply passing an override structure with the monitor geometry. - In these monitors, `colocate` is now set to `True` by default. This is to avoid a lot of potential confusion coming from returning non-colocated fields by default, when colocated fields need to be used when computing quantities that depend on more than one field component, like flux or field intensity. - Field colocation for computations like flux, Poynting, and modal overlap also happen to cell boundaries rather than centers. The effect on final results should be close to imperceptible as verified by a large number of backend tests and our online examples. Any difference can be at most on the scale of the difference that can be observed when slightly modifying the grid resolution. - `GeometryGroup` accepts other `GeometryGroup` instances as group elements. +- FDTD and mode solver tasks always upload `hdf5.gz` file instead of `hdf5` or `json`. +- `web.download_json()` will download `simulation.hdf5.gz` and unzip it, then load the json from the hdf5 file. +- `SimulationTask.get_simulation_hdf5()` will download simulation.hdf5.gz and unzip it to hdf5 file. ### Fixed - Bug in angled mode solver with negative `angle_theta`. diff --git a/tests/sims/simulation_2_4_0rc2.json b/tests/sims/simulation_2_4_0rc2.json deleted file mode 100644 index 82afb6f4f..000000000 --- a/tests/sims/simulation_2_4_0rc2.json +++ /dev/null @@ -1,1881 +0,0 @@ -{ - "type": "Simulation", - "center": [ - 0.0, - 0.0, - 0.0 - ], - "size": [ - 8.0, - 8.0, - 8.0 - ], - "run_time": 1e-12, - "medium": { - "name": null, - "frequency_range": null, - "allow_gain": false, - "nonlinear_spec": null, - "type": "Medium", - "permittivity": 1.0, - "conductivity": 0.0 - }, - "symmetry": [ - 0, - 0, - 0 - ], - "structures": [ - { - "geometry": { - "type": "Box", - "center": [ - -1.0, - 0.0, - 0.0 - ], - "size": [ - 1.0, - 1.0, - 1.0 - ] - }, - "name": null, - "type": "Structure", - "medium": { - "name": null, - "frequency_range": null, - "allow_gain": false, - "nonlinear_spec": null, - "type": "Medium", - "permittivity": 2.0, - "conductivity": 0.0 - } - }, - { - "geometry": { - "type": "Box", - "center": [ - -1.0, - 0.0, - 0.0 - ], - "size": [ - 1.0, - "Infinity", - 1.0 - ] - }, - "name": null, - "type": "Structure", - "medium": { - "name": null, - "frequency_range": null, - "allow_gain": false, - "nonlinear_spec": null, - "type": "Medium", - "permittivity": 1.0, - "conductivity": 3.0 - } - }, - { - "geometry": { - "type": "Sphere", - "radius": 1.0, - "center": [ - 1.0, - 0.0, - 1.0 - ] - }, - "name": null, - "type": "Structure", - "medium": { - "name": null, - "frequency_range": null, - "allow_gain": false, - "nonlinear_spec": null, - "type": "Sellmeier", - "coeffs": [ - [ - 1.03961212, - 0.00600069867 - ], - [ - 0.231792344, - 0.0200179144 - ] - ] - } - }, - { - "geometry": { - "type": "Box", - "center": [ - -1.0, - 0.0, - 0.0 - ], - "size": [ - 1.0, - 1.0, - 1.0 - ] - }, - "name": null, - "type": "Structure", - "medium": { - "name": null, - "frequency_range": null, - "allow_gain": false, - "nonlinear_spec": null, - "type": "Lorentz", - "eps_inf": 2.0, - "coeffs": [ - [ - 1.0, - 2.0, - 3.0 - ] - ] - } - }, - { - "geometry": { - "type": "Box", - "center": [ - -1.0, - 0.0, - 0.0 - ], - "size": [ - 1.0, - 1.0, - 1.0 - ] - }, - "name": null, - "type": "Structure", - "medium": { - "name": null, - "frequency_range": null, - "allow_gain": false, - "nonlinear_spec": null, - "type": "Debye", - "eps_inf": 2.0, - "coeffs": [ - [ - 1.0, - 3.0 - ] - ] - } - }, - { - "geometry": { - "type": "TriangleMesh", - "mesh_dataset": { - "type": "TriangleMeshDataset", - "surface_mesh": "TriangleMeshDataArray" - } - }, - "name": null, - "type": "Structure", - "medium": { - "name": null, - "frequency_range": null, - "allow_gain": false, - "nonlinear_spec": null, - "type": "Debye", - "eps_inf": 2.0, - "coeffs": [ - [ - 1.0, - 3.0 - ] - ] - } - }, - { - "geometry": { - "type": "Box", - "center": [ - -1.0, - 0.0, - 0.0 - ], - "size": [ - 1.0, - 1.0, - 1.0 - ] - }, - "name": null, - "type": "Structure", - "medium": { - "name": null, - "frequency_range": null, - "allow_gain": false, - "nonlinear_spec": null, - "type": "Drude", - "eps_inf": 2.0, - "coeffs": [ - [ - 1.0, - 3.0 - ] - ] - } - }, - { - "geometry": { - "type": "Box", - "center": [ - -1.0, - 0.0, - 0.0 - ], - "size": [ - 1.0, - 0.0, - 1.0 - ] - }, - "name": null, - "type": "Structure", - "medium": { - "name": null, - "frequency_range": null, - "allow_gain": false, - "nonlinear_spec": null, - "type": "Medium2D", - "ss": { - "name": null, - "frequency_range": null, - "allow_gain": false, - "nonlinear_spec": null, - "type": "PoleResidue", - "eps_inf": 1.0, - "poles": [ - [ - { - "real": 0.0, - "imag": 0.0 - }, - { - "real": 254117040158918.28, - "imag": 0.0 - } - ] - ] - }, - "tt": { - "name": null, - "frequency_range": null, - "allow_gain": false, - "nonlinear_spec": null, - "type": "PoleResidue", - "eps_inf": 1.0, - "poles": [ - [ - { - "real": 0.0, - "imag": 0.0 - }, - { - "real": 254117040158918.28, - "imag": 0.0 - } - ] - ] - } - } - }, - { - "geometry": { - "type": "GeometryGroup", - "geometries": [ - { - "type": "Box", - "center": [ - -1.0, - 0.0, - 0.0 - ], - "size": [ - 1.0, - 1.0, - 1.0 - ] - } - ] - }, - "name": null, - "type": "Structure", - "medium": { - "name": "PEC", - "frequency_range": null, - "allow_gain": false, - "nonlinear_spec": null, - "type": "PECMedium" - } - }, - { - "geometry": { - "type": "Cylinder", - "axis": 1, - "sidewall_angle": 0.0, - "reference_plane": "middle", - "radius": 1.0, - "center": [ - 1.0, - 0.0, - -1.0 - ], - "length": 2.0 - }, - "name": null, - "type": "Structure", - "medium": { - "name": null, - "frequency_range": null, - "allow_gain": null, - "nonlinear_spec": null, - "type": "AnisotropicMedium", - "xx": { - "name": null, - "frequency_range": null, - "allow_gain": false, - "nonlinear_spec": null, - "type": "Medium", - "permittivity": 1.0, - "conductivity": 0.0 - }, - "yy": { - "name": null, - "frequency_range": null, - "allow_gain": false, - "nonlinear_spec": null, - "type": "Medium", - "permittivity": 2.0, - "conductivity": 0.0 - }, - "zz": { - "name": null, - "frequency_range": null, - "allow_gain": false, - "nonlinear_spec": null, - "type": "Medium", - "permittivity": 3.0, - "conductivity": 0.0 - } - } - }, - { - "geometry": { - "type": "PolySlab", - "axis": 2, - "sidewall_angle": 0.0, - "reference_plane": "middle", - "slab_bounds": [ - -1.0, - 1.0 - ], - "dilation": 0.0, - "vertices": [ - [ - -1.5, - -1.5 - ], - [ - -0.5, - -1.5 - ], - [ - -0.5, - -0.5 - ] - ] - }, - "name": null, - "type": "Structure", - "medium": { - "name": null, - "frequency_range": null, - "allow_gain": false, - "nonlinear_spec": null, - "type": "PoleResidue", - "eps_inf": 1.0, - "poles": [ - [ - { - "real": 0.0, - "imag": 6206417594288582.0 - }, - { - "real": -0.0, - "imag": -3.311074436985222e+16 - } - ] - ] - } - }, - { - "geometry": { - "type": "Box", - "center": [ - -1.0, - 0.5, - 0.5 - ], - "size": [ - 1.0, - 1.0, - 1.0 - ] - }, - "name": null, - "type": "Structure", - "medium": { - "name": null, - "frequency_range": null, - "allow_gain": false, - "nonlinear_spec": null, - "type": "CustomMedium", - "interp_method": "nearest", - "subpixel": false, - "permittivity": "SpatialDataArray", - "conductivity": null, - "eps_dataset": null - } - }, - { - "geometry": { - "type": "Box", - "center": [ - -1.0, - 0.5, - 0.5 - ], - "size": [ - 1.0, - 1.0, - 1.0 - ] - }, - "name": null, - "type": "Structure", - "medium": { - "name": null, - "frequency_range": null, - "allow_gain": false, - "nonlinear_spec": null, - "type": "CustomDrude", - "eps_inf": "SpatialDataArray", - "coeffs": [ - [ - "SpatialDataArray", - "SpatialDataArray" - ] - ], - "interp_method": "nearest", - "subpixel": false - } - }, - { - "geometry": { - "type": "Box", - "center": [ - -1.0, - 0.5, - 0.5 - ], - "size": [ - 1.0, - 1.0, - 1.0 - ] - }, - "name": null, - "type": "Structure", - "medium": { - "name": null, - "frequency_range": null, - "allow_gain": false, - "nonlinear_spec": null, - "type": "CustomLorentz", - "eps_inf": "SpatialDataArray", - "coeffs": [ - [ - "SpatialDataArray", - "SpatialDataArray", - "SpatialDataArray" - ] - ], - "interp_method": "nearest", - "subpixel": false - } - }, - { - "geometry": { - "type": "Box", - "center": [ - -1.0, - 0.5, - 0.5 - ], - "size": [ - 1.0, - 1.0, - 1.0 - ] - }, - "name": null, - "type": "Structure", - "medium": { - "name": null, - "frequency_range": null, - "allow_gain": false, - "nonlinear_spec": null, - "type": "CustomDebye", - "eps_inf": "SpatialDataArray", - "coeffs": [ - [ - "SpatialDataArray", - "SpatialDataArray" - ] - ], - "interp_method": "nearest", - "subpixel": false - } - }, - { - "geometry": { - "type": "Box", - "center": [ - -1.0, - 0.5, - 0.5 - ], - "size": [ - 1.0, - 1.0, - 1.0 - ] - }, - "name": null, - "type": "Structure", - "medium": { - "name": null, - "frequency_range": null, - "allow_gain": false, - "nonlinear_spec": null, - "type": "CustomPoleResidue", - "eps_inf": "SpatialDataArray", - "poles": [ - [ - "SpatialDataArray", - "SpatialDataArray" - ] - ], - "interp_method": "nearest", - "subpixel": false - } - }, - { - "geometry": { - "type": "Box", - "center": [ - -1.0, - 0.5, - 0.5 - ], - "size": [ - 1.0, - 1.0, - 1.0 - ] - }, - "name": null, - "type": "Structure", - "medium": { - "name": null, - "frequency_range": null, - "allow_gain": false, - "nonlinear_spec": null, - "type": "CustomSellmeier", - "coeffs": [ - [ - "SpatialDataArray", - "SpatialDataArray" - ] - ], - "interp_method": "nearest", - "subpixel": false - } - }, - { - "geometry": { - "type": "Box", - "center": [ - -1.0, - 0.5, - 0.5 - ], - "size": [ - 1.0, - 1.0, - 1.0 - ] - }, - "name": null, - "type": "Structure", - "medium": { - "name": null, - "frequency_range": null, - "allow_gain": false, - "nonlinear_spec": { - "numiters": 20, - "type": "NonlinearSusceptibility", - "chi3": 0.1 - }, - "type": "Medium", - "permittivity": 1.0, - "conductivity": 0.0 - } - }, - { - "geometry": { - "type": "PolySlab", - "axis": 2, - "sidewall_angle": 0.0, - "reference_plane": "middle", - "slab_bounds": [ - -1.0, - 1.0 - ], - "dilation": 0.0, - "vertices": [ - [ - -1.5, - -1.5 - ], - [ - -0.5, - -1.5 - ], - [ - -0.5, - -0.5 - ] - ] - }, - "name": null, - "type": "Structure", - "medium": { - "name": null, - "frequency_range": null, - "allow_gain": false, - "nonlinear_spec": null, - "type": "PoleResidue", - "eps_inf": 1.0, - "poles": [ - [ - { - "real": 0.0, - "imag": 6206417594288582.0 - }, - { - "real": -0.0, - "imag": -3.311074436985222e+16 - } - ] - ] - } - }, - { - "geometry": { - "type": "TriangleMesh", - "mesh_dataset": { - "type": "TriangleMeshDataset", - "surface_mesh": "TriangleMeshDataArray" - } - }, - "name": null, - "type": "Structure", - "medium": { - "name": null, - "frequency_range": null, - "allow_gain": false, - "nonlinear_spec": null, - "type": "Medium", - "permittivity": 5.0, - "conductivity": 0.0 - } - }, - { - "geometry": { - "type": "GeometryGroup", - "geometries": [ - { - "type": "TriangleMesh", - "mesh_dataset": { - "type": "TriangleMeshDataset", - "surface_mesh": "TriangleMeshDataArray" - } - }, - { - "type": "TriangleMesh", - "mesh_dataset": { - "type": "TriangleMeshDataset", - "surface_mesh": "TriangleMeshDataArray" - } - } - ] - }, - "name": null, - "type": "Structure", - "medium": { - "name": null, - "frequency_range": null, - "allow_gain": false, - "nonlinear_spec": null, - "type": "Medium", - "permittivity": 5.0, - "conductivity": 0.0 - } - } - ], - "sources": [ - { - "type": "UniformCurrentSource", - "center": [ - 0.0, - 0.5, - 0.0 - ], - "size": [ - 0.0, - 0.0, - 0.0 - ], - "source_time": { - "amplitude": 1.0, - "phase": 0.0, - "type": "GaussianPulse", - "freq0": 200000000000000.0, - "fwidth": 40000000000000.0, - "offset": 5.0, - "remove_dc_component": true - }, - "name": null, - "interpolate": true, - "polarization": "Hx" - }, - { - "type": "PointDipole", - "center": [ - 0.0, - 0.5, - 0.0 - ], - "size": [ - 0, - 0, - 0 - ], - "source_time": { - "amplitude": 1.0, - "phase": 0.0, - "type": "GaussianPulse", - "freq0": 200000000000000.0, - "fwidth": 40000000000000.0, - "offset": 5.0, - "remove_dc_component": true - }, - "name": null, - "interpolate": true, - "polarization": "Ex" - }, - { - "type": "ModeSource", - "center": [ - 0.0, - 0.5, - 0.0 - ], - "size": [ - 2.0, - 0.0, - 2.0 - ], - "source_time": { - "amplitude": 1.0, - "phase": 0.0, - "type": "GaussianPulse", - "freq0": 200000000000000.0, - "fwidth": 40000000000000.0, - "offset": 5.0, - "remove_dc_component": true - }, - "name": null, - "num_freqs": 1, - "direction": "-", - "mode_spec": { - "num_modes": 1, - "target_neff": null, - "num_pml": [ - 0, - 0 - ], - "filter_pol": null, - "angle_theta": 0.0, - "angle_phi": 0.0, - "precision": "single", - "bend_radius": null, - "bend_axis": null, - "track_freq": "central", - "group_index_step": false, - "type": "ModeSpec" - }, - "mode_index": 0 - }, - { - "type": "PlaneWave", - "center": [ - 0.0, - 0.0, - 0.0 - ], - "size": [ - 0.0, - "Infinity", - "Infinity" - ], - "source_time": { - "amplitude": 1.0, - "phase": 0.0, - "type": "GaussianPulse", - "freq0": 200000000000000.0, - "fwidth": 40000000000000.0, - "offset": 5.0, - "remove_dc_component": true - }, - "name": null, - "direction": "+", - "angle_theta": 0.0, - "angle_phi": 0.0, - "pol_angle": 0.1 - }, - { - "type": "GaussianBeam", - "center": [ - 0.0, - 0.0, - 0.0 - ], - "size": [ - 0.0, - 3.0, - 3.0 - ], - "source_time": { - "amplitude": 1.0, - "phase": 0.0, - "type": "GaussianPulse", - "freq0": 200000000000000.0, - "fwidth": 40000000000000.0, - "offset": 5.0, - "remove_dc_component": true - }, - "name": null, - "num_freqs": 1, - "direction": "+", - "angle_theta": 0.0, - "angle_phi": 0.0, - "pol_angle": 1.5707963267948966, - "waist_radius": 1.0, - "waist_distance": 0.0 - }, - { - "type": "AstigmaticGaussianBeam", - "center": [ - 0.0, - 0.0, - 0.0 - ], - "size": [ - 0.0, - 3.0, - 3.0 - ], - "source_time": { - "amplitude": 1.0, - "phase": 0.0, - "type": "GaussianPulse", - "freq0": 200000000000000.0, - "fwidth": 40000000000000.0, - "offset": 5.0, - "remove_dc_component": true - }, - "name": null, - "num_freqs": 1, - "direction": "+", - "angle_theta": 0.0, - "angle_phi": 0.0, - "pol_angle": 1.5707963267948966, - "waist_sizes": [ - 1.0, - 2.0 - ], - "waist_distances": [ - 3.0, - 4.0 - ] - }, - { - "type": "CustomFieldSource", - "center": [ - 0.0, - 1.0, - 2.0 - ], - "size": [ - 2.0, - 2.0, - 0.0 - ], - "source_time": { - "amplitude": 1.0, - "phase": 0.0, - "type": "GaussianPulse", - "freq0": 200000000000000.0, - "fwidth": 40000000000000.0, - "offset": 5.0, - "remove_dc_component": true - }, - "name": null, - "field_dataset": { - "type": "FieldDataset", - "Ex": "ScalarFieldDataArray", - "Ey": null, - "Ez": null, - "Hx": null, - "Hy": null, - "Hz": null - } - }, - { - "type": "CustomCurrentSource", - "center": [ - 0.0, - 1.0, - 2.0 - ], - "size": [ - 2.0, - 2.0, - 0.0 - ], - "source_time": { - "amplitude": 1.0, - "phase": 0.0, - "type": "GaussianPulse", - "freq0": 200000000000000.0, - "fwidth": 40000000000000.0, - "offset": 5.0, - "remove_dc_component": true - }, - "name": null, - "interpolate": true, - "current_dataset": { - "type": "FieldDataset", - "Ex": "ScalarFieldDataArray", - "Ey": null, - "Ez": null, - "Hx": null, - "Hy": null, - "Hz": null - } - }, - { - "type": "TFSF", - "center": [ - 1.0, - 2.0, - -3.0 - ], - "size": [ - 2.5, - 2.5, - 0.5 - ], - "source_time": { - "amplitude": 1.0, - "phase": 0.0, - "type": "GaussianPulse", - "freq0": 200000000000000.0, - "fwidth": 40000000000000.0, - "offset": 5.0, - "remove_dc_component": true - }, - "name": null, - "direction": "+", - "angle_theta": 0.5235987755982988, - "angle_phi": 0.6283185307179586, - "pol_angle": 0.0, - "injection_axis": 2 - }, - { - "type": "UniformCurrentSource", - "center": [ - 0.0, - 0.5, - 0.0 - ], - "size": [ - 0.0, - 0.0, - 0.0 - ], - "source_time": { - "amplitude": 1.0, - "phase": 0.0, - "type": "CustomSourceTime", - "freq0": 200000000000000.0, - "fwidth": 40000000000000.0, - "offset": 0.0, - "source_time_dataset": { - "type": "TimeDataset", - "values": "TimeDataArray" - } - }, - "name": null, - "interpolate": true, - "polarization": "Hx" - } - ], - "boundary_spec": { - "x": { - "plus": { - "name": null, - "type": "PML", - "num_layers": 20, - "parameters": { - "sigma_order": 3, - "sigma_min": 0.0, - "sigma_max": 1.5, - "type": "PMLParams", - "kappa_order": 3, - "kappa_min": 1.0, - "kappa_max": 3.0, - "alpha_order": 1, - "alpha_min": 0.0, - "alpha_max": 0.0 - } - }, - "minus": { - "name": null, - "type": "Absorber", - "num_layers": 100, - "parameters": { - "sigma_order": 3, - "sigma_min": 0.0, - "sigma_max": 6.4, - "type": "AbsorberParams" - } - }, - "type": "Boundary" - }, - "y": { - "plus": { - "name": null, - "type": "BlochBoundary", - "bloch_vec": 1.0 - }, - "minus": { - "name": null, - "type": "BlochBoundary", - "bloch_vec": 1.0 - }, - "type": "Boundary" - }, - "z": { - "plus": { - "name": null, - "type": "Periodic" - }, - "minus": { - "name": null, - "type": "Periodic" - }, - "type": "Boundary" - }, - "type": "BoundarySpec" - }, - "monitors": [ - { - "type": "FieldMonitor", - "center": [ - 0.0, - 0.0, - 0.0 - ], - "size": [ - 0.0, - 0.0, - 0.0 - ], - "name": "field", - "freqs": [ - 150000000000000.0, - 200000000000000.0 - ], - "apodization": { - "start": null, - "end": null, - "width": null, - "type": "ApodizationSpec" - }, - "fields": [ - "Ex" - ], - "interval_space": [ - 1, - 1, - 1 - ], - "colocate": false - }, - { - "type": "FieldTimeMonitor", - "center": [ - 0.0, - 0.0, - 0.0 - ], - "size": [ - 0.0, - 0.0, - 0.0 - ], - "name": "field_time", - "start": 0.0, - "stop": null, - "interval": 100, - "fields": [ - "Ex", - "Ey", - "Ez", - "Hx", - "Hy", - "Hz" - ], - "interval_space": [ - 1, - 1, - 1 - ], - "colocate": false - }, - { - "type": "FluxMonitor", - "center": [ - 0.0, - 0.0, - 0.0 - ], - "size": [ - 1.0, - 1.0, - 0.0 - ], - "name": "flux", - "freqs": [ - 200000000000000.0, - 250000000000000.0 - ], - "apodization": { - "start": null, - "end": null, - "width": null, - "type": "ApodizationSpec" - }, - "normal_dir": "+", - "exclude_surfaces": null - }, - { - "type": "FluxTimeMonitor", - "center": [ - 0.0, - 0.0, - 0.0 - ], - "size": [ - 1.0, - 1.0, - 0.0 - ], - "name": "flux_time", - "start": 0.0, - "stop": null, - "interval": 1, - "normal_dir": "+", - "exclude_surfaces": null - }, - { - "type": "PermittivityMonitor", - "center": [ - 0.0, - 0.0, - 0.0 - ], - "size": [ - 1.0, - 1.0, - 0.1 - ], - "name": "eps", - "freqs": [ - 100000000000000.0 - ], - "apodization": { - "start": null, - "end": null, - "width": null, - "type": "ApodizationSpec" - } - }, - { - "type": "ModeMonitor", - "center": [ - 0.0, - 0.0, - 0.0 - ], - "size": [ - 1.0, - 1.0, - 0.0 - ], - "name": "mode", - "freqs": [ - 200000000000000.0, - 250000000000000.0 - ], - "apodization": { - "start": null, - "end": null, - "width": null, - "type": "ApodizationSpec" - }, - "mode_spec": { - "num_modes": 1, - "target_neff": null, - "num_pml": [ - 0, - 0 - ], - "filter_pol": null, - "angle_theta": 0.0, - "angle_phi": 0.0, - "precision": "single", - "bend_radius": null, - "bend_axis": null, - "track_freq": "central", - "group_index_step": false, - "type": "ModeSpec" - } - }, - { - "type": "ModeSolverMonitor", - "center": [ - 0.0, - 0.0, - 0.0 - ], - "size": [ - 1.0, - 1.0, - 0.0 - ], - "name": "mode_solver", - "freqs": [ - 200000000000000.0, - 250000000000000.0 - ], - "apodization": { - "start": null, - "end": null, - "width": null, - "type": "ApodizationSpec" - }, - "mode_spec": { - "num_modes": 1, - "target_neff": null, - "num_pml": [ - 0, - 0 - ], - "filter_pol": null, - "angle_theta": 0.0, - "angle_phi": 0.0, - "precision": "single", - "bend_radius": null, - "bend_axis": null, - "track_freq": "central", - "group_index_step": false, - "type": "ModeSpec" - }, - "direction": "+" - }, - { - "type": "FieldProjectionAngleMonitor", - "center": [ - 0.0, - 0.0, - 0.0 - ], - "size": [ - 0.0, - 2.0, - 2.0 - ], - "name": "proj_angle", - "freqs": [ - 250000000000000.0, - 300000000000000.0 - ], - "apodization": { - "start": null, - "end": null, - "width": null, - "type": "ApodizationSpec" - }, - "normal_dir": "+", - "exclude_surfaces": null, - "custom_origin": [ - 1.0, - 2.0, - 3.0 - ], - "far_field_approx": true, - "proj_distance": 1000000.0, - "theta": [ - -1.5707963267948966, - -1.5390630676677268, - -1.5073298085405573, - -1.4755965494133876, - -1.443863290286218, - -1.4121300311590483, - -1.3803967720318788, - -1.348663512904709, - -1.3169302537775396, - -1.2851969946503699, - -1.2534637355232003, - -1.2217304763960306, - -1.189997217268861, - -1.1582639581416914, - -1.1265306990145216, - -1.0947974398873521, - -1.0630641807601826, - -1.0313309216330129, - -0.9995976625058433, - -0.9678644033786736, - -0.936131144251504, - -0.9043978851243344, - -0.8726646259971648, - -0.8409313668699951, - -0.8091981077428254, - -0.7774648486156558, - -0.7457315894884862, - -0.7139983303613165, - -0.6822650712341469, - -0.6505318121069773, - -0.6187985529798077, - -0.5870652938526381, - -0.5553320347254684, - -0.5235987755982987, - -0.4918655164711292, - -0.46013225734395946, - -0.42839899821678995, - -0.3966657390896202, - -0.3649324799624507, - -0.333199220835281, - -0.30146596170811146, - -0.26973270258094173, - -0.23799944345377222, - -0.2062661843266025, - -0.17453292519943298, - -0.14279966607226324, - -0.11106640694509373, - -0.079333147817924, - -0.047599888690754266, - -0.015866629563584755, - 0.015866629563584977, - 0.04759988869075449, - 0.07933314781792422, - 0.11106640694509373, - 0.14279966607226346, - 0.17453292519943298, - 0.2062661843266027, - 0.23799944345377222, - 0.26973270258094195, - 0.30146596170811146, - 0.3331992208352812, - 0.3649324799624507, - 0.39666573908962044, - 0.42839899821678995, - 0.4601322573439597, - 0.4918655164711292, - 0.5235987755982991, - 0.5553320347254687, - 0.5870652938526382, - 0.6187985529798077, - 0.6505318121069776, - 0.6822650712341471, - 0.7139983303613167, - 0.7457315894884862, - 0.7774648486156561, - 0.8091981077428256, - 0.8409313668699951, - 0.8726646259971647, - 0.9043978851243346, - 0.9361311442515041, - 0.9678644033786736, - 0.9995976625058436, - 1.031330921633013, - 1.0630641807601826, - 1.0947974398873521, - 1.126530699014522, - 1.1582639581416916, - 1.189997217268861, - 1.2217304763960306, - 1.2534637355232006, - 1.28519699465037, - 1.3169302537775396, - 1.348663512904709, - 1.380396772031879, - 1.4121300311590486, - 1.443863290286218, - 1.475596549413388, - 1.5073298085405575, - 1.539063067667727, - 1.5707963267948966 - ], - "phi": [ - 0.0, - 1.5707963267948966 - ] - }, - { - "type": "FieldProjectionCartesianMonitor", - "center": [ - 0.0, - 0.0, - 0.0 - ], - "size": [ - 0.0, - 2.0, - 2.0 - ], - "name": "proj_cartesian", - "freqs": [ - 250000000000000.0, - 300000000000000.0 - ], - "apodization": { - "start": null, - "end": null, - "width": null, - "type": "ApodizationSpec" - }, - "normal_dir": "+", - "exclude_surfaces": null, - "custom_origin": [ - 1.0, - 2.0, - 3.0 - ], - "far_field_approx": true, - "proj_axis": 2, - "proj_distance": 5.0, - "x": [ - -1.0, - 0.0, - 1.0 - ], - "y": [ - -2.0, - -1.0, - 0.0, - 1.0, - 2.0 - ] - }, - { - "type": "FieldProjectionKSpaceMonitor", - "center": [ - 0.0, - 0.0, - 0.0 - ], - "size": [ - 0.0, - 2.0, - 2.0 - ], - "name": "proj_kspace", - "freqs": [ - 250000000000000.0, - 300000000000000.0 - ], - "apodization": { - "start": null, - "end": null, - "width": null, - "type": "ApodizationSpec" - }, - "normal_dir": "+", - "exclude_surfaces": null, - "custom_origin": [ - 1.0, - 2.0, - 3.0 - ], - "far_field_approx": true, - "proj_axis": 2, - "proj_distance": 1000000.0, - "ux": [ - 0.1, - 0.2 - ], - "uy": [ - 0.3, - 0.4, - 0.5 - ] - }, - { - "type": "FieldProjectionAngleMonitor", - "center": [ - 0.0, - 0.0, - 0.0 - ], - "size": [ - 0.0, - 2.0, - 2.0 - ], - "name": "proj_angle_exact", - "freqs": [ - 250000000000000.0, - 300000000000000.0 - ], - "apodization": { - "start": null, - "end": null, - "width": null, - "type": "ApodizationSpec" - }, - "normal_dir": "+", - "exclude_surfaces": null, - "custom_origin": [ - 1.0, - 2.0, - 3.0 - ], - "far_field_approx": true, - "proj_distance": 1000000.0, - "theta": [ - -1.5707963267948966, - -1.5390630676677268, - -1.5073298085405573, - -1.4755965494133876, - -1.443863290286218, - -1.4121300311590483, - -1.3803967720318788, - -1.348663512904709, - -1.3169302537775396, - -1.2851969946503699, - -1.2534637355232003, - -1.2217304763960306, - -1.189997217268861, - -1.1582639581416914, - -1.1265306990145216, - -1.0947974398873521, - -1.0630641807601826, - -1.0313309216330129, - -0.9995976625058433, - -0.9678644033786736, - -0.936131144251504, - -0.9043978851243344, - -0.8726646259971648, - -0.8409313668699951, - -0.8091981077428254, - -0.7774648486156558, - -0.7457315894884862, - -0.7139983303613165, - -0.6822650712341469, - -0.6505318121069773, - -0.6187985529798077, - -0.5870652938526381, - -0.5553320347254684, - -0.5235987755982987, - -0.4918655164711292, - -0.46013225734395946, - -0.42839899821678995, - -0.3966657390896202, - -0.3649324799624507, - -0.333199220835281, - -0.30146596170811146, - -0.26973270258094173, - -0.23799944345377222, - -0.2062661843266025, - -0.17453292519943298, - -0.14279966607226324, - -0.11106640694509373, - -0.079333147817924, - -0.047599888690754266, - -0.015866629563584755, - 0.015866629563584977, - 0.04759988869075449, - 0.07933314781792422, - 0.11106640694509373, - 0.14279966607226346, - 0.17453292519943298, - 0.2062661843266027, - 0.23799944345377222, - 0.26973270258094195, - 0.30146596170811146, - 0.3331992208352812, - 0.3649324799624507, - 0.39666573908962044, - 0.42839899821678995, - 0.4601322573439597, - 0.4918655164711292, - 0.5235987755982991, - 0.5553320347254687, - 0.5870652938526382, - 0.6187985529798077, - 0.6505318121069776, - 0.6822650712341471, - 0.7139983303613167, - 0.7457315894884862, - 0.7774648486156561, - 0.8091981077428256, - 0.8409313668699951, - 0.8726646259971647, - 0.9043978851243346, - 0.9361311442515041, - 0.9678644033786736, - 0.9995976625058436, - 1.031330921633013, - 1.0630641807601826, - 1.0947974398873521, - 1.126530699014522, - 1.1582639581416916, - 1.189997217268861, - 1.2217304763960306, - 1.2534637355232006, - 1.28519699465037, - 1.3169302537775396, - 1.348663512904709, - 1.380396772031879, - 1.4121300311590486, - 1.443863290286218, - 1.475596549413388, - 1.5073298085405575, - 1.539063067667727, - 1.5707963267948966 - ], - "phi": [ - 0.0, - 1.5707963267948966 - ] - }, - { - "type": "DiffractionMonitor", - "center": [ - 0.0, - 0.0, - 0.0 - ], - "size": [ - 0.0, - "Infinity", - "Infinity" - ], - "name": "diffraction", - "freqs": [ - 100000000000000.0, - 200000000000000.0 - ], - "apodization": { - "start": null, - "end": null, - "width": null, - "type": "ApodizationSpec" - }, - "normal_dir": "+" - } - ], - "grid_spec": { - "grid_x": { - "type": "AutoGrid", - "min_steps_per_wvl": 10.0, - "max_scale": 1.4, - "dl_min": 0.0, - "mesher": { - "type": "GradedMesher" - } - }, - "grid_y": { - "type": "CustomGrid", - "dl": [ - 0.04, - 0.04, - 0.04, - 0.04, - 0.04, - 0.04, - 0.04, - 0.04, - 0.04, - 0.04, - 0.04, - 0.04, - 0.04, - 0.04, - 0.04, - 0.04, - 0.04, - 0.04, - 0.04, - 0.04, - 0.04, - 0.04, - 0.04, - 0.04, - 0.04, - 0.04, - 0.04, - 0.04, - 0.04, - 0.04, - 0.04, - 0.04, - 0.04, - 0.04, - 0.04, - 0.04, - 0.04, - 0.04, - 0.04, - 0.04, - 0.04, - 0.04, - 0.04, - 0.04, - 0.04, - 0.04, - 0.04, - 0.04, - 0.04, - 0.04, - 0.04, - 0.04, - 0.04, - 0.04, - 0.04, - 0.04, - 0.04, - 0.04, - 0.04, - 0.04, - 0.04, - 0.04, - 0.04, - 0.04, - 0.04, - 0.04, - 0.04, - 0.04, - 0.04, - 0.04, - 0.04, - 0.04, - 0.04, - 0.04, - 0.04, - 0.04, - 0.04, - 0.04, - 0.04, - 0.04, - 0.04, - 0.04, - 0.04, - 0.04, - 0.04, - 0.04, - 0.04, - 0.04, - 0.04, - 0.04, - 0.04, - 0.04, - 0.04, - 0.04, - 0.04, - 0.04, - 0.04, - 0.04, - 0.04, - 0.04 - ], - "custom_offset": null - }, - "grid_z": { - "type": "UniformGrid", - "dl": 0.05 - }, - "wavelength": null, - "override_structures": [ - { - "geometry": { - "type": "Box", - "center": [ - -1.0, - 0.0, - 0.0 - ], - "size": [ - 1.0, - 1.0, - 1.0 - ] - }, - "name": null, - "type": "Structure", - "medium": { - "name": null, - "frequency_range": null, - "allow_gain": false, - "nonlinear_spec": null, - "type": "Medium", - "permittivity": 2.0, - "conductivity": 0.0 - } - } - ], - "type": "GridSpec" - }, - "shutoff": 0.0001, - "subpixel": false, - "normalize_index": 0, - "courant": 0.8, - "version": "2.4.0rc2" -} \ No newline at end of file diff --git a/tests/test_plugins/test_mode_solver.py b/tests/test_plugins/test_mode_solver.py index 06c609909..421c8e382 100644 --- a/tests/test_plugins/test_mode_solver.py +++ b/tests/test_plugins/test_mode_solver.py @@ -13,6 +13,7 @@ from tidy3d.plugins.mode.solver import compute_modes from tidy3d import ScalarFieldDataArray from tidy3d.web.environment import Env +from tidy3d.version import __version__ WAVEGUIDE = td.Structure(geometry=td.Box(size=(100, 0.5, 0.5)), medium=td.Medium(permittivity=4.0)) @@ -46,7 +47,7 @@ def mock_download(task_id, remote_path, to_file, *args, **kwargs): sources=[SRC], ) mode_spec = td.ModeSpec( - num_modes=1, + num_modes=3, target_neff=2.0, filter_pol="tm", precision="double", @@ -62,7 +63,6 @@ def mock_download(task_id, remote_path, to_file, *args, **kwargs): monkeypatch.setattr(td.web.http_management, "api_key", lambda: "api_key") monkeypatch.setattr("tidy3d.plugins.mode.web.upload_file", void) - monkeypatch.setattr("tidy3d.plugins.mode.web.upload_string", void) monkeypatch.setattr("tidy3d.plugins.mode.web.download_file", mock_download) responses.add( @@ -82,34 +82,8 @@ def mock_download(task_id, remote_path, to_file, *args, **kwargs): "projectId": PROJECT_ID, "taskName": TASK_NAME, "modeSolverName": MODESOLVER_NAME, - "fileType": "Json", - "protocolVersion": __version__, - } - ) - ], - json={ - "data": { - "refId": TASK_ID, - "id": SOLVER_ID, - "status": "draft", - "createdAt": "2023-05-19T16:47:57.190Z", - "charge": 0, - "fileType": "Json", - } - }, - status=200, - ) - - responses.add( - responses.POST, - f"{Env.current.web_api_endpoint}/tidy3d/modesolver/py", - match=[ - responses.matchers.json_params_matcher( - { - "projectId": PROJECT_ID, - "taskName": TASK_NAME, - "modeSolverName": MODESOLVER_NAME, - "fileType": "Hdf5", + "fileType": "Gz", + "source": "Python", "protocolVersion": __version__, } ) @@ -121,7 +95,7 @@ def mock_download(task_id, remote_path, to_file, *args, **kwargs): "status": "draft", "createdAt": "2023-05-19T16:47:57.190Z", "charge": 0, - "fileType": "Hdf5", + "fileType": "Gz", } }, status=200, @@ -153,7 +127,7 @@ def mock_download(task_id, remote_path, to_file, *args, **kwargs): "status": "queued", "createdAt": "2023-05-19T16:47:57.190Z", "charge": 0, - "fileType": "Json", + "fileType": "Gz", } }, status=200, diff --git a/tests/test_web/test_tidy3d_task.py b/tests/test_web/test_tidy3d_task.py index 6182e0e36..c4baf5f7e 100644 --- a/tests/test_web/test_tidy3d_task.py +++ b/tests/test_web/test_tidy3d_task.py @@ -5,9 +5,11 @@ import responses from responses import matchers +import tidy3d as td from tidy3d.web.environment import Env, EnvironmentConfig from tidy3d.web.simulation_task import Folder, SimulationTask from tidy3d.version import __version__ +from tidy3d.web.file_util import compress_file_to_gzip, read_simulation_from_hdf5 test_env = EnvironmentConfig( name="test", @@ -19,6 +21,18 @@ Env.set_current(test_env) +def make_sim(): + """Makes a simulation.""" + pulse = td.GaussianPulse(freq0=200e12, fwidth=20e12) + pt_dipole = td.PointDipole(source_time=pulse, polarization="Ex") + return td.Simulation( + size=(1, 1, 1), + grid_spec=td.GridSpec.auto(wavelength=1.0), + run_time=1e-12, + sources=[pt_dipole], + ) + + @pytest.fixture def set_api_key(monkeypatch): """Set the api key.""" @@ -82,10 +96,13 @@ def test_query_task(set_api_key): @responses.activate def test_get_simulation_json(monkeypatch, set_api_key, tmp_path): + sim = make_sim() + def mock_download(*args, **kwargs): - file_path = kwargs["to_file"] - with open(file_path, "w") as f: - f.write("test data") + to_file = kwargs["to_file"] + file_path = "simulation.hdf5" + sim.to_file(file_path) + compress_file_to_gzip(file_path, to_file) monkeypatch.setattr("tidy3d.web.simulation_task.download_file", mock_download) @@ -103,7 +120,7 @@ def mock_download(*args, **kwargs): task = SimulationTask.get("3eb06d16-208b-487b-864b-e9b1d3e010a7") JSON_NAME = str(tmp_path / "task.json") task.get_simulation_json(JSON_NAME) - assert os.path.getsize(JSON_NAME) > 0 + assert td.Simulation.from_file(JSON_NAME) == sim @responses.activate @@ -146,6 +163,7 @@ def test_create(set_api_key): { "taskName": "test task", "callbackUrl": None, + "fileType": "Gz", "simulationType": "tidy3d", "parentTasks": None, } @@ -184,6 +202,7 @@ def test_submit(set_api_key): { "taskName": task_name, "callbackUrl": None, + "fileType": "Gz", "simulationType": "tidy3d", "parentTasks": None, } @@ -296,7 +315,8 @@ def mock(*args, **kwargs): task = SimulationTask.get("3eb06d16-208b-487b-864b-e9b1d3e010a7") LOG_FNAME = str(tmp_path / "test.log") task.get_log(LOG_FNAME) - assert os.path.getsize(LOG_FNAME) > 0 + with open(LOG_FNAME, "r") as f: + assert f.read() == "0.3,5.7" @responses.activate diff --git a/tests/test_web/test_webapi.py b/tests/test_web/test_webapi.py index b229e620b..a5e831745 100644 --- a/tests/test_web/test_webapi.py +++ b/tests/test_web/test_webapi.py @@ -7,6 +7,7 @@ from responses import matchers +from tidy3d import Simulation from tidy3d.exceptions import SetupError from tidy3d.web.environment import Env from tidy3d.web.webapi import delete, delete_old, download, download_json, run, abort @@ -14,10 +15,10 @@ from tidy3d.web.webapi import load, load_simulation, start, upload, monitor, real_cost from tidy3d.web.container import Job, Batch from tidy3d.web.asynchronous import run_async +from tidy3d.web.file_util import compress_file_to_gzip from tidy3d.__main__ import main - TASK_NAME = "task_name_test" TASK_ID = "1234" CREATED_AT = "2022-01-01T00:00:00.000Z" @@ -68,6 +69,7 @@ def mock_upload(monkeypatch, set_api_key): "callbackUrl": None, "simulationType": "tidy3d", "parentTasks": None, + "fileType": "Gz", } ) ], @@ -84,7 +86,7 @@ def mock_upload(monkeypatch, set_api_key): def mock_download(*args, **kwargs): pass - monkeypatch.setattr("tidy3d.web.simulation_task.upload_string", mock_download) + monkeypatch.setattr("tidy3d.web.simulation_task.upload_file", mock_download) @pytest.fixture @@ -298,16 +300,18 @@ def test_estimate_cost(set_api_key, mock_get_info, mock_metadata): @responses.activate def test_download_json(monkeypatch, mock_get_info, tmp_path): + sim = make_sim() + def mock_download(*args, **kwargs): - file_path = kwargs["to_file"] - with open(file_path, "w") as f: - f.write("0.3,5.7") + file_path = "simulation.hdf5" + sim.to_file(file_path) + compress_file_to_gzip(file_path, "simulation.hdf5.gz") monkeypatch.setattr("tidy3d.web.simulation_task.download_file", mock_download) - download_json(TASK_ID, str(tmp_path / "web_test_tmp.json")) - with open(str(tmp_path / "web_test_tmp.json"), "r") as f: - assert f.read() == "0.3,5.7" + fname_tmp = str(tmp_path / "web_test_tmp.json") + download_json(TASK_ID, fname_tmp) + assert Simulation.from_file(fname_tmp) == sim @responses.activate diff --git a/tidy3d/plugins/mode/web.py b/tidy3d/plugins/mode/web.py index 9c84560bd..b308c9dfe 100644 --- a/tidy3d/plugins/mode/web.py +++ b/tidy3d/plugins/mode/web.py @@ -17,16 +17,20 @@ from ...components.data.monitor_data import ModeSolverData from ...exceptions import WebError from ...log import log +from ...web.file_util import compress_file_to_gzip, extract_gz_file from ...web.http_management import http -from ...web.s3utils import download_file, upload_file, upload_string -from ...web.simulation_task import Folder, SIMULATION_JSON +from ...web.s3utils import download_file, upload_file +from ...web.simulation_task import Folder, SIMULATION_JSON, SIM_FILE_HDF5_GZ from ...web.types import ResourceLifecycle, Submittable from .mode_solver import ModeSolver, MODE_MONITOR_NAME +from ...version import __version__ MODESOLVER_API = "tidy3d/modesolver/py" MODESOLVER_JSON = "mode_solver.json" MODESOLVER_HDF5 = "mode_solver.hdf5" +MODESOLVER_GZ = "mode_solver.hdf5.gz" + MODESOLVER_LOG = "output/result.log" MODESOLVER_RESULT = "output/result.hdf5" @@ -95,7 +99,7 @@ def run( status = task.get_info().status if status == "error": - raise WebError("Error runnig mode solver.") + raise WebError("Error running mode solver.") log.log(log_level, f"Mode solver status: {status}") if verbose: @@ -196,9 +200,10 @@ def create( { "projectId": folder.folder_id, "taskName": task_name, - "modeSolverName": mode_solver_name, - "fileType": "Hdf5" if len(mode_solver.simulation.custom_datasets) > 0 else "Json", "protocolVersion": __version__, + "modeSolverName": mode_solver_name, + "fileType": "Gz", + "source": "Python", }, ) log.info( @@ -256,6 +261,7 @@ def get_info(self) -> ModeSolverTask: resp = http.get(f"{MODESOLVER_API}/{self.task_id}/{self.solver_id}") return ModeSolverTask(**resp, mode_solver=self.mode_solver) + # pylint: disable=protected-access def upload( self, verbose: bool = True, progress_callback: Callable[[float], None] = None ) -> None: @@ -270,46 +276,51 @@ def upload( """ mode_solver = self.mode_solver.copy() - # Upload simulation as json for GUI display - upload_string( - self.task_id, - mode_solver.simulation._json_string, # pylint: disable=protected-access - SIMULATION_JSON, - verbose=verbose, - progress_callback=progress_callback, - ) + sim = mode_solver.simulation - if self.file_type == "Hdf5": - # Upload a single HDF5 file with the full data - file, file_name = tempfile.mkstemp() - os.close(file) - mode_solver.to_hdf5(file_name) - - try: - upload_file( - self.solver_id, - file_name, - MODESOLVER_HDF5, - verbose=verbose, - progress_callback=progress_callback, - ) - finally: - os.unlink(file_name) - else: - # Send only mode solver, without simulation - mode_solver_spec = mode_solver.dict() + file, file_name = tempfile.mkstemp() + gz_file, gz_file_name = tempfile.mkstemp() + os.close(file) + os.close(gz_file) + + sim.to_hdf5(file_name) + try: + # Upload simulation.hdf5.gz for GUI display + # compress .hdf5 to .hdf5.gz + compress_file_to_gzip(file_name, gz_file_name) - # Upload mode solver without simulation: 'construct' skips all validation - mode_solver_spec["simulation"] = None - mode_solver = ModeSolver.construct(**mode_solver_spec) - upload_string( + upload_file( + self.task_id, + gz_file_name, + SIM_FILE_HDF5_GZ, + verbose=verbose, + progress_callback=progress_callback, + ) + finally: + os.unlink(file_name) + os.unlink(gz_file_name) + + # Upload a single HDF5 file with the full data + file, file_name = tempfile.mkstemp() + gz_file, gz_file_name = tempfile.mkstemp() + os.close(file) + os.close(gz_file) + mode_solver.to_hdf5(file_name) + + try: + # compress .hdf5 to .hdf5.gz + compress_file_to_gzip(file_name, gz_file_name) + + upload_file( self.solver_id, - mode_solver._json_string, # pylint: disable=protected-access - MODESOLVER_JSON, + gz_file_name, + MODESOLVER_GZ, verbose=verbose, progress_callback=progress_callback, - extra_arguments={"type": "ms"}, ) + finally: + os.unlink(file_name) + os.unlink(gz_file_name) # pylint: disable=arguments-differ def submit(self): @@ -365,7 +376,21 @@ def get_modesolver( stored in the same path as 'to_file', but with '.hdf5' extension, and neither 'to_file' or 'sim_file' will be created. """ - if self.file_type == "Hdf5": + if self.file_type == "Gz": + to_gz = pathlib.Path(to_file).with_suffix(".hdf5.gz") + to_hdf5 = pathlib.Path(to_file).with_suffix(".hdf5") + download_file( + self.task_id, + MODESOLVER_GZ, + to_file=to_gz, + verbose=verbose, + progress_callback=progress_callback, + ) + extract_gz_file(to_gz, to_hdf5) + to_file = str(to_hdf5) + mode_solver = ModeSolver.from_hdf5(to_hdf5) + + elif self.file_type == "Hdf5": to_hdf5 = pathlib.Path(to_file).with_suffix(".hdf5") download_file( self.solver_id, diff --git a/tidy3d/web/__init__.py b/tidy3d/web/__init__.py index 99a9ef4e2..dc8f92700 100644 --- a/tidy3d/web/__init__.py +++ b/tidy3d/web/__init__.py @@ -2,19 +2,8 @@ import sys from .cli.migrate import migrate -from .webapi import ( - run, - upload, - get_info, - start, - monitor, - delete, - abort, - download, - load, - estimate_cost, -) -from .webapi import get_tasks, delete_old, download_json, download_log, load_simulation, real_cost +from .webapi import run, upload, get_info, start, monitor, delete, download, load, estimate_cost +from .webapi import get_tasks, delete_old, download_log, download_json, load_simulation, real_cost from .container import Job, Batch, BatchData from .cli import tidy3d_cli from .cli.app import configure_fn as configure diff --git a/tidy3d/web/file_util.py b/tidy3d/web/file_util.py new file mode 100644 index 000000000..a3acb836b --- /dev/null +++ b/tidy3d/web/file_util.py @@ -0,0 +1,41 @@ +"""compress and extract file""" + +import gzip + +import h5py + +from tidy3d.components.base import JSON_TAG + + +def compress_file_to_gzip(input_file, output_gz_file): + """ + Compresses a file using gzip. + + Args: + input_file (str): The path of the input file. + output_gz_file (str): The path of the output gzip file. + """ + with open(input_file, "rb") as file_in: + with gzip.open(output_gz_file, "wb") as file_out: + file_out.writelines(file_in) + + +def extract_gz_file(input_gz_file, output_file): + """ + Extract the GZ file + + Args: + input_gz_file (str): The path of the gzip input file. + output_file (str): The path of the output file. + """ + with gzip.open(input_gz_file, "rb") as f_in: + with open(output_file, "wb") as f_out: + f_out.write(f_in.read()) + + +def read_simulation_from_hdf5(file_name: str): + """read simulation str from hdf5""" + + with h5py.File(file_name, "r") as f_handle: + json_string = f_handle[JSON_TAG][()] + return json_string diff --git a/tidy3d/web/s3utils.py b/tidy3d/web/s3utils.py index b0b19a31b..a83f124fe 100644 --- a/tidy3d/web/s3utils.py +++ b/tidy3d/web/s3utils.py @@ -1,7 +1,6 @@ # pylint:disable=unused-argument """handles filesystem, storage """ -import io import pathlib import urllib from datetime import datetime @@ -203,69 +202,6 @@ def get_s3_sts_token( return _s3_sts_tokens[cache_key] -# pylint: disable=too-many-arguments -def upload_string( - resource_id: str, - content: str, - remote_filename: str, - verbose: bool = True, - progress_callback: Callable[[float], None] = None, - extra_arguments: Mapping[str, str] = None, -): - """Upload a string to a file on S3. - - Parameters - ---------- - resource_id : str - The resource id, e.g. task id. - content : str - The content of the file - remote_filename : str - The remote file name on S3 relative to the resource context root path. - verbose : bool = True - Whether to display a progressbar for the upload. - progress_callback : Callable[[float], None] = None - User-supplied callback function with ``bytes_in_chunk`` as argument. - extra_arguments : Mapping[str, str] - Additional arguments used to specify the upload bucket. - """ - - token = get_s3_sts_token(resource_id, remote_filename, extra_arguments) - - def _upload(_callback: Callable) -> None: - """Perform the upload with a callback fn - - Parameters - ---------- - _callback : Callable[[float], None] - Callback function for upload, accepts ``bytes_in_chunk`` - """ - token.get_client().upload_fileobj( - io.BytesIO(content.encode("utf-8")), - Bucket=token.get_bucket(), - Key=token.get_s3_key(), - Callback=_callback, - Config=_s3_config, - ) - - if progress_callback is not None: - _upload(progress_callback) - else: - if verbose: - with _get_progress(_S3Action.UPLOADING) as progress: - total_size = len(content) - task_id = progress.add_task("upload", filename=remote_filename, total=total_size) - - def _callback(bytes_in_chunk): - progress.update(task_id, advance=bytes_in_chunk) - - _upload(_callback) - progress.update(task_id, completed=total_size, refresh=True) - - elif progress_callback is None: - _upload(lambda bytes_in_chunk: None) - - # pylint: disable=too-many-arguments def upload_file( resource_id: str, @@ -311,6 +247,9 @@ def _upload(_callback: Callable) -> None: Key=token.get_s3_key(), Callback=_callback, Config=_s3_config, + ExtraArgs={"ContentEncoding": "gzip"} + if token.get_s3_key().endswith(".gz") + else None, ) if progress_callback is not None: diff --git a/tidy3d/web/simulation_task.py b/tidy3d/web/simulation_task.py index 4e97b178e..536dc428f 100644 --- a/tidy3d/web/simulation_task.py +++ b/tidy3d/web/simulation_task.py @@ -8,21 +8,25 @@ from typing import List, Optional, Callable, Tuple import pydantic as pd from pydantic import Extra, Field, parse_obj_as +from rich.console import Console from tidy3d import Simulation from tidy3d.version import __version__ +from tidy3d.exceptions import WebError from .cache import FOLDER_CACHE from .http_management import http -from .s3utils import download_file, upload_file, upload_string +from .s3utils import download_file, upload_file from .types import Queryable, ResourceLifecycle, Submittable from .types import Tidy3DResource +from .file_util import compress_file_to_gzip, extract_gz_file, read_simulation_from_hdf5 SIMULATION_JSON = "simulation.json" -SIMULATION_HDF5 = "output/monitor_data.hdf5" +SIMULATION_DATA_HDF5 = "output/monitor_data.hdf5" RUNNING_INFO = "output/solver_progress.csv" SIM_LOG_FILE = "output/tidy3d.log" SIM_FILE_HDF5 = "simulation.hdf5" +SIM_FILE_HDF5_GZ = "simulation.hdf5.gz" class Folder(Tidy3DResource, Queryable, extra=Extra.allow): @@ -197,6 +201,7 @@ def create( callback_url: str = None, simulation_type: str = "tidy3d", parent_tasks: List[str] = None, + file_type: str = "Gz", ) -> SimulationTask: """Create a new task on the server. @@ -217,6 +222,8 @@ def create( Type of simulation being uploaded. parent_tasks : List[str] List of related task ids. + file_type: str + the simulation file type Json, Hdf5, Gz Returns ------- @@ -232,6 +239,7 @@ def create( "callbackUrl": callback_url, "simulationType": simulation_type, "parentTasks": parent_tasks, + "fileType": file_type, }, ) @@ -296,9 +304,7 @@ def get_simulation(self) -> Optional[Simulation]: return self.simulation return None - def get_simulation_json( - self, to_file: str, verbose: bool = True, progress_callback: Callable[[float], None] = None - ) -> pathlib.Path: + def get_simulation_json(self, to_file: str, verbose: bool = True) -> pathlib.Path: """Get json file for a :class:`.Simulation` from server. Parameters @@ -307,8 +313,6 @@ def get_simulation_json( save file to path. verbose: bool = True Whether to display progress bars. - progress_callback : Callable[[float], None] = None - Optional callback function called while downloading the data. Returns ------- @@ -316,13 +320,18 @@ def get_simulation_json( Path to saved file. """ assert self.task_id - return download_file( - self.task_id, - SIMULATION_JSON, - to_file=to_file, - verbose=verbose, - progress_callback=progress_callback, - ) + self.get_simulation_hdf5(SIM_FILE_HDF5) + if os.path.exists(SIM_FILE_HDF5): + json_string = read_simulation_from_hdf5(SIM_FILE_HDF5) + os.remove(SIM_FILE_HDF5) + with open(to_file, "w") as file: + # Write the string to the file + file.write(json_string.decode("utf-8")) + if verbose: + console = Console() + console.log("Generate simulation.json successfully.") + else: + raise WebError("Failed to download simulation.json.") def upload_simulation( self, verbose: bool = True, progress_callback: Callable[[float], None] = None @@ -338,28 +347,26 @@ def upload_simulation( """ assert self.task_id assert self.simulation - upload_string( - self.task_id, - self.simulation._json_string, # pylint: disable=protected-access - SIMULATION_JSON, - verbose=verbose, - progress_callback=progress_callback, - ) - if len(self.simulation.custom_datasets) > 0: - # Also upload hdf5 containing all data. - file, file_name = tempfile.mkstemp() - os.close(file) - self.simulation.to_hdf5(file_name) - try: - upload_file( - self.task_id, - file_name, - SIM_FILE_HDF5, - verbose=verbose, - progress_callback=progress_callback, - ) - finally: - os.unlink(file_name) + + # Also upload hdf5.gz containing all data. + file, file_name = tempfile.mkstemp() + gz_file, gz_file_name = tempfile.mkstemp() + os.close(file) + os.close(gz_file) + self.simulation.to_hdf5(file_name) + try: + # compress .hdf5 to .hdf5.gz + compress_file_to_gzip(file_name, gz_file_name) + upload_file( + self.task_id, + gz_file_name, + SIM_FILE_HDF5_GZ, + verbose=verbose, + progress_callback=progress_callback, + ) + finally: + os.unlink(file_name) + os.unlink(gz_file_name) def upload_file( self, @@ -411,12 +418,26 @@ def submit( worker group """ if self.simulation: - upload_string( - self.task_id, - self.simulation._json_string, # pylint: disable=protected-access - SIMULATION_JSON, - verbose=False, - ) + # Also upload hdf5.gz containing all data. + file, file_name = tempfile.mkstemp() + gz_file, gz_file_name = tempfile.mkstemp() + os.close(file) + os.close(gz_file) + self.simulation.to_hdf5(file_name) + try: + # compress .hdf5 to .hdf5.gz + compress_file_to_gzip(file_name, gz_file_name) + + upload_file( + self.task_id, + gz_file_name, + SIM_FILE_HDF5_GZ, + verbose=False, + progress_callback=None, + ) + finally: + os.unlink(file_name) + os.unlink(gz_file_name) if solver_version: protocol_version = None @@ -462,10 +483,10 @@ def estimate_cost(self, solver_version=None) -> float: ) return resp - def get_simulation_hdf5( + def get_sim_data_hdf5( self, to_file: str, verbose: bool = True, progress_callback: Callable[[float], None] = None ) -> pathlib.Path: - """Get hdf5 file from Server. + """Get output/monitor_data.hdf5 file from Server. Parameters ---------- @@ -484,12 +505,48 @@ def get_simulation_hdf5( assert self.task_id return download_file( self.task_id, - SIMULATION_HDF5, + SIMULATION_DATA_HDF5, to_file=to_file, verbose=verbose, progress_callback=progress_callback, ) + def get_simulation_hdf5( + self, to_file: str, verbose: bool = True, progress_callback: Callable[[float], None] = None + ) -> pathlib.Path: + """Get simulation.hdf5 file from Server. + + Parameters + ---------- + to_file: str + save file to path. + verbose: bool = True + Whether to display progress bars. + progress_callback : Callable[[float], None] = None + Optional callback function called while downloading the data. + + Returns + ------- + path: pathlib.Path + Path to saved file. + """ + assert self.task_id + download_file( + self.task_id, + SIM_FILE_HDF5_GZ, + to_file=SIM_FILE_HDF5_GZ, + verbose=verbose, + progress_callback=progress_callback, + ) + if os.path.exists(SIM_FILE_HDF5_GZ): + extract_gz_file(SIM_FILE_HDF5_GZ, to_file) + os.remove(SIM_FILE_HDF5_GZ) + if verbose: + console = Console() + console.log(f"Extract {SIM_FILE_HDF5_GZ} to {to_file} successfully.") + else: + raise WebError("Failed to download simulation.hdf5") + def get_running_info(self) -> Tuple[float, float]: """Gets the % done and field_decay for a running task. diff --git a/tidy3d/web/webapi.py b/tidy3d/web/webapi.py index 5dcab21d8..108b7ca9d 100644 --- a/tidy3d/web/webapi.py +++ b/tidy3d/web/webapi.py @@ -182,7 +182,7 @@ def upload( # pylint:disable=too-many-locals,too-many-arguments log.debug("Creating task.") task = SimulationTask.create( - simulation, task_name, folder_name, callback_url, simulation_type, parent_tasks + simulation, task_name, folder_name, callback_url, simulation_type, parent_tasks, "Gz" ) if verbose: console = Console() @@ -388,8 +388,8 @@ def monitor_preprocess() -> None: console.log("running solver") console.log( "To cancel the simulation, use 'web.abort(task_id)' or 'web.delete(task_id)' " - "or abort/delete the task in the web" - " UI. Terminating the Python script will not stop the job running on the cloud." + "or abort/delete the task in the web " + "UI. Terminating the Python script will not stop the job running on the cloud." ) with Progress(console=console) as progress: pbar_pd = progress.add_task("% done", total=100) @@ -454,16 +454,11 @@ def download( """ task = SimulationTask(taskId=task_id) - task.get_simulation_hdf5(path, verbose=verbose, progress_callback=progress_callback) + task.get_sim_data_hdf5(path, verbose=verbose, progress_callback=progress_callback) @wait_for_connection -def download_json( - task_id: TaskId, - path: str = SIM_FILE_JSON, - verbose: bool = True, - progress_callback: Callable[[float], None] = None, -) -> None: +def download_json(task_id: TaskId, path: str = SIM_FILE_JSON, verbose: bool = True) -> None: """Download the `.json` file associated with the :class:`.Simulation` of a given task. Parameters @@ -474,13 +469,11 @@ def download_json( Download path to .json file of simulation (including filename). verbose : bool = True If `True`, will print progressbars and status, otherwise, will run silently. - progress_callback : Callable[[float], None] = None - Optional callback function called when downloading file with ``bytes_in_chunk`` as argument. """ task = SimulationTask(taskId=task_id) - task.get_simulation_json(path, verbose=verbose, progress_callback=progress_callback) + task.get_simulation_json(path, verbose=verbose) @wait_for_connection @@ -510,12 +503,7 @@ def download_hdf5( @wait_for_connection -def load_simulation( - task_id: TaskId, - path: str = SIM_FILE_JSON, - verbose: bool = True, - progress_callback: Callable[[float], None] = None, -) -> Simulation: +def load_simulation(task_id: TaskId, path: str = SIM_FILE_JSON, verbose: bool = True) -> Simulation: """Download the `.json` file of a task and load the associated :class:`.Simulation`. Parameters @@ -526,8 +514,6 @@ def load_simulation( Download path to .json file of simulation (including filename). verbose : bool = True If `True`, will print progressbars and status, otherwise, will run silently. - progress_callback : Callable[[float], None] = None - Optional callback function called when downloading file with ``bytes_in_chunk`` as argument. Returns ------- @@ -537,7 +523,7 @@ def load_simulation( # task = SimulationTask.get(task_id) task = SimulationTask(taskId=task_id) - task.get_simulation_json(path, verbose=verbose, progress_callback=progress_callback) + task.get_simulation_json(path, verbose=verbose) return Simulation.from_file(path) @@ -577,7 +563,7 @@ def load( verbose: bool = True, progress_callback: Callable[[float], None] = None, ) -> SimulationData: - """Download and Load simultion results into :class:`.SimulationData` object. + """Download and Load simulation results into :class:`.SimulationData` object. Parameters ----------