SCRF-1630: Refactor RF webapi for new workflow

.github/workflows/tidy3d-python-client-tests.yml

@@ -347,7 +347,7 @@ jobs:
           BRANCH_NAME="${STEPS_EXTRACT_BRANCH_NAME_OUTPUTS_BRANCH_NAME}"
           echo $BRANCH_NAME
           # Allow only Jira keys from known projects, even if the branch has an author prefix
-          ALLOWED_JIRA_PROJECTS=("FXC" "SCEM")
+          ALLOWED_JIRA_PROJECTS=("FXC" "SCEM" "SCRF")
           JIRA_PROJECT_PATTERN=$(IFS='|'; echo "${ALLOWED_JIRA_PROJECTS[*]}")
           JIRA_PATTERN="(${JIRA_PROJECT_PATTERN})-[0-9]+"
 

docs/api/submit_simulations.rst

@@ -94,7 +94,6 @@ Information Containers
    :template: module.rst
 
    tidy3d.web.core.task_info.TaskInfo
-   tidy3d.web.core.task_info.TaskStatus
 
 
 Mode Solver Web API

tests/test_plugins/smatrix/test_run_functions.py

@@ -1,6 +1,5 @@
 from __future__ import annotations
 
-import json
 from unittest.mock import MagicMock
 
 import pydantic.v1 as pd
@@ -14,38 +13,15 @@
     make_component_modeler as make_modal_component_modeler,
 )
 from tidy3d import SimulationDataMap
-from tidy3d.components.base import Tidy3dBaseModel
 from tidy3d.components.data.sim_data import SimulationData
 from tidy3d.plugins.smatrix.data.terminal import TerminalComponentModelerData
 from tidy3d.plugins.smatrix.run import (
     _run_local,
-    compose_modeler,
     compose_modeler_data,
     create_batch,
 )
 
 
-def test_compose_modeler_unsupported_type(tmp_path, monkeypatch):
-    # Create a dummy HDF5 file path
-    modeler_file = tmp_path / "dummy_modeler.hdf5"
-
-    # Prepare a dummy JSON string with an unsupported type
-    dummy_json = {"type": "UnsupportedComponentModeler", "some_key": "some_value"}
-    dummy_json_str = json.dumps(dummy_json)
-
-    # Mock Tidy3dBaseModel._json_string_from_hdf5 to return our dummy JSON string
-    def mock_json_string_from_hdf5(filepath):
-        if filepath == str(modeler_file):
-            return dummy_json_str
-        return ""
-
-    monkeypatch.setattr(Tidy3dBaseModel, "_json_string_from_hdf5", mock_json_string_from_hdf5)
-
-    # Expect a TypeError when calling compose_modeler with the unsupported type
-    with pytest.raises(TypeError, match="Unsupported modeler type: str"):
-        compose_modeler(modeler_file=str(modeler_file))
-
-
 def test_create_batch(monkeypatch, tmp_path):
     # Mock Batch and Batch.to_file
     mock_batch_instance = MagicMock()

tests/test_plugins/test_array_factor.py

@@ -363,16 +363,15 @@ def make_antenna_sim():
         remove_dc_component=False,  # Include DC component for more accuracy at low frequencies
     )
 
-    sim_unit = list(modeler.sim_dict.values())[0]
-
-    return sim_unit
+    return modeler
 
 
 def test_rectangular_array_calculator_array_make_antenna_array():
     """Test automatic antenna array creation."""
     freq0 = 10e9
     wavelength0 = td.C_0 / 10e9
-    sim_unit = make_antenna_sim()
+    modeler = make_antenna_sim()
+    sim_unit = list(modeler.sim_dict.values())[0]
     array_calculator = mw.RectangularAntennaArrayCalculator(
         array_size=(1, 2, 3),
         spacings=(0.5 * wavelength0, 0.6 * wavelength0, 0.4 * wavelength0),
@@ -437,8 +436,9 @@ def test_rectangular_array_calculator_array_make_antenna_array():
     assert len(sim_array.sources) == 6
 
     # check that override_structures are duplicated
-    assert len(sim_unit.grid_spec.override_structures) == 2
-    assert len(sim_array.grid_spec.override_structures) == 7
+    # assert len(sim_unit.grid_spec.override_structures) == 2
+    # assert len(sim_array.grid_spec.override_structures) == 7
+    assert sim_unit.grid.boundaries == modeler.base_sim.grid.boundaries
 
     # check that phase shifts are applied correctly
     phases_expected = array_calculator._antenna_phases
@@ -674,7 +674,8 @@ def test_rectangular_array_calculator_simulation_data_from_array_factor():
         phase_shifts=(np.pi / 3, np.pi / 4, np.pi / 5),
     )
 
-    sim_unit = make_antenna_sim()
+    modeler = make_antenna_sim()
+    sim_unit = list(modeler.sim_dict.values())[0]
 
     monitor = sim_unit.monitors[0]
     monitor_directivity = sim_unit.monitors[2]

tests/test_web/test_local_cache.py

@@ -43,7 +43,7 @@
     resolve_local_cache,
 )
 from tidy3d.web.cli.app import tidy3d_cli
-from tidy3d.web.core.task_core import BatchTask
+from tidy3d.web.core.task_core import BatchTask, SimulationTask
 
 common.CONNECTION_RETRY_TIME = 0.1
 
@@ -245,7 +245,9 @@ def _fake_field_map_from_file(*args, **kwargs):
     monkeypatch.setattr(
         io_utils, "load_simulation", lambda task_id, *args, **kwargs: TASK_TO_SIM[task_id]
     )
-    monkeypatch.setattr(BatchTask, "is_batch", lambda *args, **kwargs: "success")
+    monkeypatch.setattr(
+        SimulationTask, "get", lambda *args, **kwargs: SimpleNamespace(taskType="FDTD")
+    )
     monkeypatch.setattr(
         BatchTask, "detail", lambda *args, **kwargs: SimpleNamespace(status="success")
     )

tests/test_web/test_webapi.py

@@ -415,7 +415,7 @@ def test_run_with_invalid_priority(mock_webapi, priority):
 
 
 @responses.activate
-def test_get_run_info(mock_get_run_info):
+def test_get_run_info(mock_get_run_info, mock_get_info):
     assert get_run_info(TASK_ID) == (100, 0)
 
 

tests/test_web/test_webapi_eme.py

@@ -260,7 +260,7 @@ def test_get_info(mock_get_info):
 
 
 @responses.activate
-def test_get_run_info(mock_get_run_info):
+def test_get_run_info(mock_get_run_info, mock_get_info):
     assert get_run_info(TASK_ID) == (100, 0)
 
 

tests/test_web/test_webapi_heat.py

@@ -250,7 +250,7 @@ def test_get_info(mock_get_info):
 
 
 @responses.activate
-def test_get_run_info(mock_get_run_info):
+def test_get_run_info(mock_get_run_info, mock_get_info):
     assert get_run_info(TASK_ID) == (100, 0)
 
 

tests/test_web/test_webapi_mode.py

@@ -307,7 +307,7 @@ def test_get_info(mock_get_info):
 
 
 @responses.activate
-def test_get_run_info(mock_get_run_info):
+def test_get_run_info(mock_get_run_info, mock_get_info):
     assert get_run_info(TASK_ID) == (100, 0)
 
 

tests/test_web/test_webapi_mode_sim.py

@@ -303,7 +303,7 @@ def test_get_info(mock_get_info):
 
 
 @responses.activate
-def test_get_run_info(mock_get_run_info):
+def test_get_run_info(mock_get_run_info, mock_get_info):
     assert get_run_info(TASK_ID) == (100, 0)
 
 

tidy3d/__init__.py

@@ -2,6 +2,7 @@
 
 from __future__ import annotations
 
+from tidy3d.components.base import Tidy3dBaseModel
 from tidy3d.components.boundary import BroadbandModeABCFitterParam, BroadbandModeABCSpec
 from tidy3d.components.data.index import SimulationDataMap
 from tidy3d.components.frequency_extrapolation import LowFrequencySmoothingSpec
@@ -812,6 +813,7 @@ def set_logging_level(level: str) -> None:
     "TemperatureData",
     "TemperatureMonitor",
     "TetrahedralGridDataset",
+    "Tidy3dBaseModel",
     "Transformed",
     "TriangleMesh",
     "TriangularGridDataset",

tidy3d/components/mode/mode_solver.py

@@ -2714,7 +2714,7 @@ def _validate_modes_size(self) -> None:
                 "frequencies or modes."
             )
 
-    def validate_pre_upload(self, source_required: bool = True) -> None:
+    def validate_pre_upload(self) -> None:
         """Validate the fully initialized mode solver is ok for upload to our servers."""
         self._validate_modes_size()
 

tidy3d/components/mode/simulation.py

@@ -612,8 +612,8 @@ def plot_pml_mode_plane(
         """
         return self._mode_solver.plot_pml(ax=ax)
 
-    def validate_pre_upload(self, source_required: bool = False) -> None:
+    def validate_pre_upload(self) -> None:
         super().validate_pre_upload()
-        self._mode_solver.validate_pre_upload(source_required=source_required)
+        self._mode_solver.validate_pre_upload()
 
     _boundaries_for_zero_dims = validate_boundaries_for_zero_dims(warn_on_change=False)

tidy3d/components/tcad/mesher.py

@@ -24,3 +24,9 @@ class VolumeMesher(Tidy3dBaseModel):
 
     def _get_simulation_types(self) -> list[TCADAnalysisTypes]:
         return [TCADAnalysisTypes.MESH]
+
+    def validate_pre_upload(self):
+        """Validate the VolumeMesher before uploading to the cloud.
+        Currently no validation but method is required when calling ``web.upload``.
+        """
+        return

tidy3d/plugins/smatrix/component_modelers/base.py

@@ -343,5 +343,9 @@ def run(
         )
         return data.smatrix()
 
+    def validate_pre_upload(self):
+        """Validate the modeler before upload."""
+        self.base_sim.validate_pre_upload(source_required=False)
+
 
 AbstractComponentModeler.update_forward_refs()

tidy3d/plugins/smatrix/component_modelers/modal.py

@@ -59,6 +59,11 @@ class ModalComponentModeler(AbstractComponentModeler):
         "by ``element_mappings``, the simulation corresponding to this column is skipped automatically.",
     )
 
+    @property
+    def base_sim(self):
+        """The base simulation."""
+        return self.simulation
+
     @cached_property
     def sim_dict(self) -> SimulationMap:
         """Generates all :class:`.Simulation` objects for the S-matrix calculation.
@@ -368,3 +373,9 @@ def get_max_mode_indices(matrix_elements: tuple[str, int]) -> int:
         max_mode_index_in = get_max_mode_indices(self.matrix_indices_source)
 
         return max_mode_index_out, max_mode_index_in
+
+    def task_name_from_index(self, matrix_index: MatrixIndex) -> str:
+        """Compute task name for a given (port_name, mode_index) without constructing simulations."""
+        port_name, mode_index = matrix_index
+        port = self.get_port_by_name(port_name=port_name)
+        return self.get_task_name(port=port, mode_index=mode_index)

tidy3d/plugins/smatrix/component_modelers/terminal.py

@@ -223,15 +223,17 @@ def _warn_refactor_2_10(cls, values):
 
     @property
     def _sim_with_sources(self) -> Simulation:
-        """Instance of :class:`.Simulation` with all sources and absorbers added for each port, for troubleshooting."""
+        """Instance of :class:`.Simulation` with all sources and absorbers added for each port, for plotting."""
 
         sources = [port.to_source(self._source_time) for port in self.ports]
         absorbers = [
             port.to_absorber()
             for port in self.ports
             if isinstance(port, WavePort) and port.absorber
         ]
-        return self.simulation.updated_copy(sources=sources, internal_absorbers=absorbers)
+        return self.simulation.updated_copy(
+            sources=sources, internal_absorbers=absorbers, validate=False
+        )
 
     @equal_aspect
     @add_ax_if_none
@@ -382,18 +384,17 @@ def matrix_indices_run_sim(self) -> tuple[NetworkIndex, ...]:
     def sim_dict(self) -> SimulationMap:
         """Generate all the :class:`.Simulation` objects for the port parameter calculation."""
 
+        # Check base simulation for grid size at ports
+        TerminalComponentModeler._check_grid_size_at_ports(self.base_sim, self._lumped_ports)
+        TerminalComponentModeler._check_grid_size_at_wave_ports(self.base_sim, self._wave_ports)
+
         sim_dict = {}
         # Now, create simulations with wave port sources and mode solver monitors for computing port modes
         for network_index in self.matrix_indices_run_sim:
             task_name, sim_with_src = self._add_source_to_sim(network_index)
             # update simulation
             sim_dict[task_name] = sim_with_src
 
-        # Check final simulations for grid size at ports
-        for _, sim in sim_dict.items():
-            TerminalComponentModeler._check_grid_size_at_ports(sim, self._lumped_ports)
-            TerminalComponentModeler._check_grid_size_at_wave_ports(sim, self._wave_ports)
-
         return SimulationMap(keys=tuple(sim_dict.keys()), values=tuple(sim_dict.values()))
 
     @cached_property
@@ -414,7 +415,10 @@ def _base_sim_no_radiation_monitors(self) -> Simulation:
 
         # Make an initial simulation with new grid_spec to determine where LumpedPorts are snapped
         sim_wo_source = self.simulation.updated_copy(
-            grid_spec=grid_spec, lumped_elements=lumped_resistors
+            grid_spec=grid_spec,
+            lumped_elements=lumped_resistors,
+            validate=False,
+            deep=False,
         )
         snap_centers = {}
         for port in self._lumped_ports:
@@ -480,7 +484,11 @@ def _base_sim_no_radiation_monitors(self) -> Simulation:
             )
 
         # update base simulation with updated set of shared components
-        sim_wo_source = sim_wo_source.copy(update=update_dict)
+        sim_wo_source = sim_wo_source.updated_copy(
+            **update_dict,
+            validate=False,
+            deep=False,
+        )
 
         # extrude port structures
         sim_wo_source = self._extrude_port_structures(sim=sim_wo_source)
@@ -527,7 +535,10 @@ def base_sim(self) -> Simulation:
         """The base simulation with all components added, including radiation monitors."""
         base_sim_tmp = self._base_sim_no_radiation_monitors
         mnts_with_radiation = list(base_sim_tmp.monitors) + list(self._finalized_radiation_monitors)
-        return base_sim_tmp.updated_copy(monitors=mnts_with_radiation)
+        grid_spec = GridSpec.from_grid(base_sim_tmp.grid)
+        grid_spec.attrs["from_grid_spec"] = base_sim_tmp.grid_spec
+        # We skipped validations up to now, here we finally validate the base sim
+        return base_sim_tmp.updated_copy(monitors=mnts_with_radiation, grid_spec=grid_spec)
 
     def _generate_radiation_monitor(
         self, simulation: Simulation, auto_spec: DirectivityMonitorSpec
@@ -712,7 +723,10 @@ def _add_source_to_sim(self, source_index: NetworkIndex) -> tuple[str, Simulatio
             )
         task_name = self.get_task_name(port=port, mode_index=mode_index)
 
-        return (task_name, self.base_sim.updated_copy(sources=[port_source]))
+        return (
+            task_name,
+            self.base_sim.updated_copy(sources=[port_source], validate=False, deep=False),
+        )
 
     @cached_property
     def _source_time(self):
@@ -863,6 +877,11 @@ def get_radiation_monitor_by_name(self, monitor_name: str) -> DirectivityMonitor
                 return monitor
         raise Tidy3dKeyError(f"No radiation monitor named '{monitor_name}'.")
 
+    def task_name_from_index(self, source_index: NetworkIndex) -> str:
+        """Compute task name for a given network index without constructing simulations."""
+        port, mode_index = self.network_dict[source_index]
+        return self.get_task_name(port=port, mode_index=mode_index)
+
     def _extrude_port_structures(self, sim: Simulation) -> Simulation:
         """
         Extrude structures intersecting a port plane when a wave port lies on a structure boundary.
@@ -983,6 +1002,8 @@ def _extrude_port_structures(self, sim: Simulation) -> Simulation:
             sim = sim.updated_copy(
                 grid_spec=GridSpec.from_grid(sim.grid),
                 structures=[*sim.structures, *all_new_structures],
+                validate=False,
+                deep=False,
             )
 
         return sim