raise if nans in first or last value of data or lookup

docs/whats_new.rst

@@ -5,6 +5,7 @@ v3.11.0 (2023/07)
 New Features
 ~~~~~~~~~~~~
 - Add the possibility to run a model one or several steps at a time, updating model variables in the process. (`@rogersamso <https://github.com/rogersamso>`_)
+- An error is raised when the data of a LOOKUP or DATA types has nans either in the first or last value, so that nans are not extrapolated. (`@rogersamso <https://github.com/rogersamso>`_)
 
 Breaking changes
 ~~~~~~~~~~~~~~~~

pysd/py_backend/external.py

@@ -311,7 +311,7 @@ def _get_series_data(self, series_across, series_row_or_col, cell, size):
                   + "Data given in:\n"
                   + self._file_sheet
                   + "\tData name:\t'{}'\n".format(cell)
-                  + " has not the same size as the given coordinates"
+                  + " does not have the same size as the given coordinates"
                   )
 
         return series, data
@@ -472,6 +472,14 @@ def _initialize_data(self, element_type):
             if self.interp != "raw":
                 self._fill_missing(series, data)
 
+        # raise always if the first or last values of data are nan
+        if np.any(np.isnan(data[0])) or np.any(np.isnan(data[-1])):
+            raise ValueError(
+                self.py_name + "\n"
+                    + "the first or last values of the data are empty in:\n"
+                    + self._file_sheet
+                    + "this may produce unwanted results when extrapolating.")
+
         reshape_dims = tuple([len(series)] + utils.compute_shape(self.coords))
 
         if len(reshape_dims) > 1:

tests/pytest_pysd/pytest_pysd.py

@@ -55,8 +55,8 @@ def test_run_ignore_missing(self, _root):
         # ignore warnings for missing values
         model = pysd.read_vensim(model_mdl, missing_values="ignore")
 
+        # ignore warnings for missing values
         with pytest.warns(UserWarning, match='extrapolating data'):
-            # ignore warnings for missing values
             model.run()
 
         with pytest.warns(UserWarning, match='missing'):

tests/pytest_types/external/pytest_external.py

@@ -310,6 +310,31 @@ def test_read_empty_cells_pandas(self, _root):
         with pytest.raises(ValueError, match=error_message):
             ext._get_data_from_file((100, 140), (2, 20))
 
+    def test_initialize_data_with_first_or_last_empty(self, _root):
+        from pysd.py_backend.external import External
+
+        ext = External("external")
+        ext.missing = "ignore"
+        ext.interp = "raw"
+        ext.file = "data/input.xlsx"
+        ext.sheet = "Horizontal missing 2"
+        ext.x_row_or_col = "time_missing_first_or_last"
+        ext.cell = "val_missing_first"
+        ext.root = _root
+        ext.coords = {}
+
+        error_message = r"^external\nthe first or last values of the data "
+        with pytest.raises(ValueError, match=error_message):
+            ext._initialize_data("lookup")
+
+        ext.cell = "val_missing_last"
+        with pytest.raises(ValueError, match=error_message):
+            ext._initialize_data("lookup")
+
+        ext.coords = {"a": ["b", "c"]}
+        ext.cell = "val_missing_first_and_last"
+        with pytest.raises(ValueError, match=error_message):
+            ext._initialize_data("data")
 
 class TestData():
     """
@@ -1975,10 +2000,10 @@ def test_data_interp_h1dm_row(self, _root):
                                      final_coords=coords,
                                      py_name=py_name)
 
+        error_message = r"test_data_interp_h1dm_row\nthe first or last values"
         with pytest.warns(UserWarning, match="Not able to interpolate"):
-            data.initialize()
-
-        assert all(np.isnan(data.data.values))
+            with pytest.raises(ValueError, match=error_message):
+                data.initialize()
 
     def test_data_interp_h1dm_row2(self, _root):
         """
@@ -2006,13 +2031,10 @@ def test_data_interp_h1dm_row2(self, _root):
                                      interp=interp,
                                      final_coords=coords,
                                      py_name=py_name)
-
+        error_message = r"test_data_interp_h1dm_row2\nthe first or last values"
         with pytest.warns(UserWarning, match="Not able to interpolate"):
-            data.initialize()
-
-        assert not any(np.isnan(data.data.loc[:, "B"].values))
-        assert not any(np.isnan(data.data.loc[:, "C"].values))
-        assert all(np.isnan(data.data.loc[:, "D"].values))
+            with pytest.raises(ValueError, match=error_message):
+                data.initialize()
 
     def test_data_interp_h1dm(self, _root, _exp):
         """
@@ -2403,8 +2425,10 @@ def test_data_interp_hn3dmd_raw(self, _root):
                  interp=interp,
                  coords=coords_2)
 
+        error_message = r"test_data_interp_hn3dmd_raw\nthe first or last"
         with pytest.warns(UserWarning, match='missing'):
-            data.initialize()
+            with pytest.raises(ValueError, match=error_message):
+                data.initialize()
 
     def test_lookup_hn3dmd_raise(self, _root):
         """
@@ -2768,7 +2792,7 @@ def test_data_interp_hns(self, _root):
                                      final_coords=coords,
                                      py_name=py_name)
 
-        error_message = "has not the same size as the given coordinates"
+        error_message = "does not have the same size as the given coordinates"
         with pytest.raises(ValueError, match=error_message):
             data.initialize()