Enforcing limits on the internal memory needed by monitors

CHANGELOG.md

@@ -22,6 +22,8 @@ and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0
 - API for specifying one or more nonlinear models via `NonlinearSpec.models`.
 - `freqs` and `direction` are optional in `ModeSolver` methods converting to monitor and source, respectively. If not supplied, uses the values from the `ModeSolver` instance calling the method.
 - Removed spurious ``-1`` factor in field amplitudes injected by field sources in some cases. The injected ``E``-field should now exactly match the analytic, mode, or custom fields that the source is expected to inject, both in the forward and in the backward direction.
+- Restriction on the maximum memory that a monitor would need internally during the solver run, even if the final monitor data is smaller.
+- Restriction on the maximum size of mode solver data produced by a `ModeSolver` server call.
 
 ### Fixed
 - Fixed the duplication of log messages in Jupyter when `set_logging_file` is used.

tests/test_components/test_simulation.py

@@ -1606,7 +1606,7 @@ def test_warn_large_epsilon(log_capture, size, num_struct, log_level):
 
 @pytest.mark.parametrize("dl, log_level", [(0.1, None), (0.005, "WARNING")])
 def test_warn_large_mode_monitor(log_capture, dl, log_level):
-    """Make sure we get a warning if the epsilon grid is too large."""
+    """Make sure we get a warning if the mode monitor grid is too large."""
 
     sim = td.Simulation(
         size=(2.0, 2.0, 2.0),
@@ -1631,7 +1631,7 @@ def test_warn_large_mode_monitor(log_capture, dl, log_level):
 
 @pytest.mark.parametrize("dl, log_level", [(0.1, None), (0.005, "WARNING")])
 def test_warn_large_mode_source(log_capture, dl, log_level):
-    """Make sure we get a warning if the epsilon grid is too large."""
+    """Make sure we get a warning if the mode source grid is too large."""
 
     sim = td.Simulation(
         size=(2.0, 2.0, 2.0),
@@ -1649,6 +1649,31 @@ def test_warn_large_mode_source(log_capture, dl, log_level):
     assert_log_level(log_capture, log_level)
 
 
+def test_error_large_monitors():
+    """Test if various large monitors cause pre-upload validation to error."""
+
+    sim = td.Simulation(
+        size=(2.0, 2.0, 2.0),
+        grid_spec=td.GridSpec.uniform(dl=0.005),
+        run_time=1e-12,
+        boundary_spec=td.BoundarySpec.all_sides(boundary=td.Periodic()),
+    )
+    mnt_size = (td.inf, 0, td.inf)
+    mnt_test = [
+        td.ModeMonitor(size=mnt_size, freqs=[1], name="test", mode_spec=td.ModeSpec()),
+        td.ModeSolverMonitor(size=mnt_size, freqs=[1], name="test", mode_spec=td.ModeSpec()),
+        td.FluxMonitor(size=mnt_size, freqs=[1], name="test"),
+        td.FluxTimeMonitor(size=mnt_size, name="test"),
+        td.DiffractionMonitor(size=mnt_size, freqs=[1], name="test"),
+        td.FieldProjectionAngleMonitor(size=mnt_size, freqs=[1], name="test", theta=[0], phi=[0]),
+    ]
+
+    for monitor in mnt_test:
+        with pytest.raises(SetupError):
+            s = sim.updated_copy(monitors=[monitor])
+            s.validate_pre_upload()
+
+
 def test_dt():
     """make sure dt is reduced when there is a medium with eps_inf < 1."""
     sim = td.Simulation(

tests/test_plugins/test_mode_solver.py

@@ -11,6 +11,7 @@
 from tidy3d.plugins.mode.mode_solver import MODE_MONITOR_NAME
 from tidy3d.plugins.mode.derivatives import create_sfactor_b, create_sfactor_f
 from tidy3d.plugins.mode.solver import compute_modes
+from tidy3d.exceptions import SetupError
 from ..utils import assert_log_level, log_capture
 from tidy3d import ScalarFieldDataArray
 from tidy3d.web.core.environment import Env
@@ -243,6 +244,17 @@ def test_mode_solver_validation():
         direction="+",
     )
 
+    # mode data too large
+    simulation = td.Simulation(
+        size=SIM_SIZE,
+        grid_spec=td.GridSpec.uniform(dl=0.001),
+        run_time=1e-12,
+    )
+    ms = ms.updated_copy(simulation=simulation, freqs=np.linspace(1e12, 2e12, 50))
+
+    with pytest.raises(SetupError):
+        ms.validate_pre_upload()
+
 
 @pytest.mark.parametrize("group_index_step, log_level", ((1e-7, "WARNING"), (1e-5, "INFO")))
 def test_mode_solver_group_index_warning(group_index_step, log_level, log_capture):

tidy3d/components/monitor.py

@@ -1,5 +1,5 @@
 """Objects that define how data is recorded from simulation."""
-from abc import ABC
+from abc import ABC, abstractmethod
 from typing import Union, Tuple
 
 import pydantic.v1 as pydantic
@@ -45,6 +45,14 @@ class Monitor(AbstractMonitor):
         "and is hard-coded for other monitors depending on their specific function.",
     )
 
+    @abstractmethod
+    def storage_size(self, num_cells: int, tmesh: ArrayFloat1D) -> int:
+        """Size of monitor storage given the number of points after discretization."""
+
+    def _storage_size_solver(self, num_cells: int, tmesh: ArrayFloat1D) -> int:
+        """Size of intermediate data recorded by the monitor during a solver run."""
+        return self.storage_size(num_cells=num_cells, tmesh=tmesh)
+
 
 class FreqMonitor(Monitor, ABC):
     """:class:`Monitor` that records data in the frequency-domain."""
@@ -303,6 +311,11 @@ def _warn_num_modes(cls, val, values):
             )
         return val
 
+    def _storage_size_solver(self, num_cells: int, tmesh: ArrayFloat1D) -> int:
+        """Size of intermediate data recorded by the monitor during a solver run."""
+        # Need to store all fields on the mode surface
+        return BYTES_COMPLEX * num_cells * len(self.freqs) * self.mode_spec.num_modes * 6
+
 
 class FieldMonitor(AbstractFieldMonitor, FreqMonitor):
     """:class:`Monitor` that records electromagnetic fields in the frequency domain.
@@ -454,6 +467,13 @@ def check_excluded_surfaces(cls, values):
             )
         return values
 
+    def _storage_size_solver(self, num_cells: int, tmesh: ArrayFloat1D) -> int:
+        """Size of intermediate data recorded by the monitor during a solver run."""
+        # Need to store all fields on the integration surface. Frequency-domain monitors store at
+        # all frequencies, time domain at the current time step only.
+        num_sample = len(getattr(self, "freqs", [0]))
+        return BYTES_COMPLEX * num_cells * num_sample * 6
+
 
 class AbstractFluxMonitor(SurfaceIntegrationMonitor, ABC):
     """:class:`Monitor` that records flux during the solver run."""

tidy3d/components/simulation.py

@@ -72,6 +72,7 @@
 MAX_GRID_CELLS = 20e9
 MAX_CELLS_TIMES_STEPS = 1e16
 WARN_MONITOR_DATA_SIZE_GB = 10
+MAX_MONITOR_INTERNAL_DATA_SIZE_GB = 50
 MAX_SIMULATION_DATA_SIZE_GB = 50
 WARN_MODE_NUM_CELLS = 1e5
 
@@ -992,7 +993,7 @@ def _validate_monitor_size(self) -> None:
         with log as consolidated_logger:
             datas = self.monitors_data_size
             for monitor_ind, (monitor_name, monitor_size) in enumerate(datas.items()):
-                monitor_size_gb = monitor_size / 2**30
+                monitor_size_gb = monitor_size / 1e9
                 if monitor_size_gb > WARN_MONITOR_DATA_SIZE_GB:
                     consolidated_logger.warning(
                         f"Monitor '{monitor_name}' estimated storage is {monitor_size_gb:1.2f}GB. "
@@ -1009,6 +1010,21 @@ def _validate_monitor_size(self) -> None:
                 f"a maximum of {MAX_SIMULATION_DATA_SIZE_GB:.2f}GB are allowed."
             )
 
+        # Some monitors store much less data than what is needed internally. Make sure that the
+        # internal storage also does not exceed the limit.
+        for monitor in self.monitors:
+            num_cells = self._monitor_num_cells(monitor)
+            # intermediate storage needed, in GB
+            solver_data = monitor._storage_size_solver(num_cells=num_cells, tmesh=self.tmesh) / 1e9
+            if solver_data > MAX_MONITOR_INTERNAL_DATA_SIZE_GB:
+                raise SetupError(
+                    f"Estimated internal storage of monitor '{monitor.name}' is "
+                    f"{solver_data:1.2f}GB, which is larger than the maximum allowed "
+                    f"{MAX_MONITOR_INTERNAL_DATA_SIZE_GB:.2f}GB. Consider making it smaller, "
+                    "using fewer frequencies, or spatial or temporal downsampling using "
+                    "'interval_space' and 'interval', respectively."
+                )
+
     def _validate_modes_size(self) -> None:
         """Warn if mode sources or monitors have a large number of points."""
 
@@ -1049,19 +1065,20 @@ def warn_mode_size(monitor: AbstractModeMonitor, msg_header: str, custom_loc: Li
     @cached_property
     def monitors_data_size(self) -> Dict[str, float]:
         """Dictionary mapping monitor names to their estimated storage size in bytes."""
-        tmesh = self.tmesh
         data_size = {}
         for monitor in self.monitors:
-            name = monitor.name
-            num_cells = self.discretize_monitor(monitor).num_cells
-            # take monitor downsampling into account
-            num_cells = monitor.downsampled_num_cells(num_cells)
-            num_cells = np.prod(num_cells)
-            monitor_size = monitor.storage_size(num_cells=num_cells, tmesh=tmesh)
-            data_size[name] = float(monitor_size)
-
+            num_cells = self._monitor_num_cells(monitor)
+            storage_size = float(monitor.storage_size(num_cells=num_cells, tmesh=self.tmesh))
+            data_size[monitor.name] = storage_size
         return data_size
 
+    def _monitor_num_cells(self, monitor: Monitor) -> int:
+        """Total number of cells included by monitor based on simulation grid."""
+        num_cells = self.discretize_monitor(monitor).num_cells
+        # take monitor downsampling into account
+        num_cells = monitor.downsampled_num_cells(num_cells)
+        return np.prod(np.array(num_cells, dtype=np.int64))
+
     def _validate_datasets_not_none(self) -> None:
         """Ensures that all custom datasets are defined."""
         if any(dataset is None for dataset in self.custom_datasets):

tidy3d/plugins/mode/mode_solver.py

@@ -23,7 +23,7 @@
 from ...components.data.data_array import FreqModeDataArray
 from ...components.data.sim_data import SimulationData
 from ...components.data.monitor_data import ModeSolverData
-from ...exceptions import ValidationError
+from ...exceptions import ValidationError, SetupError
 from ...constants import C_0
 from .solver import compute_modes
 
@@ -36,6 +36,9 @@
 # Warning for field intensity at edges over total field intensity larger than this value
 FIELD_DECAY_CUTOFF = 1e-2
 
+# Maximum allowed size of the field data produced by the mode solver
+MAX_MODES_DATA_SIZE_GB = 20
+
 
 class ModeSolver(Tidy3dBaseModel):
     """Interface for solving electromagnetic eigenmodes in a 2D plane with translational
@@ -845,5 +848,19 @@ def plot_field(
             **sel_kwargs,
         )
 
+    def _validate_modes_size(self):
+        """Make sure that the total size of the modes fields is not too large."""
+        monitor = self.to_mode_solver_monitor(name=MODE_MONITOR_NAME)
+        num_cells = self.simulation._monitor_num_cells(monitor)
+        # size in GB
+        total_size = monitor._storage_size_solver(num_cells=num_cells, tmesh=[]) / 1e9
+        if total_size > MAX_MODES_DATA_SIZE_GB:
+            raise SetupError(
+                f"Mode solver has {total_size:.2f}GB of estimated storage, "
+                f"a maximum of {MAX_MODES_DATA_SIZE_GB:.2f}GB is allowed. Consider making the "
+                "mode plane smaller, or decreasing the resolution or number of requested "
+                "frequencies or modes."
+            )
+
     def validate_pre_upload(self, source_required: bool = True):
-        pass
+        self._validate_modes_size()

tidy3d/web/api/mode.py

@@ -183,6 +183,7 @@ def create(
         """
         folder = Folder.get(folder_name, create=True)
 
+        mode_solver.validate_pre_upload()
         mode_solver.simulation.validate_pre_upload(source_required=False)
         resp = http.post(
             MODESOLVER_API,