Update dependencies (#574)

Co-authored-by: Stefan Pfenninger <stefan@pfenninger.org>

pyproject.toml

@@ -14,13 +14,10 @@ markers = ["serial", "time_intensive"]
 filterwarnings = [
   # https://github.com/pytest-dev/pytest-xdist/issues/825
   "ignore:The --rsyncdir command line argument and rsyncdirs config variable are deprecated.:DeprecationWarning",
-  "ignore:.*`group_share` constraints will be removed in v0.7.0.*:FutureWarning:",
-  "ignore:.*Plotting will no longer be available as a method.*:FutureWarning:",
-  "ignore:.*There will be no default cost class for the objective function in v0.7.0.*:FutureWarning:",
-  "ignore:.*`charge_rate` is renamed to `energy_cap_per_storage_cap_max` and will be removed in v0.7.0.*:FutureWarning:",
-  "ignore:.*Deprecated, pass a TempConstr or use Model.addLConstr.*:DeprecationWarning:pyomo",
-  "ignore:.*distutils Version classes are deprecated.*:DeprecationWarning:",
-  "ignore:.*`np.float` is a deprecated alias.*:DeprecationWarning:",
+  "ignore:(?s).*datetime.datetime.utcfromtimestamp():",
+  "ignore:(?s).*Pyarrow will become a required dependency of pandas:DeprecationWarning",
+  "ignore:.*The return type of `Dataset.dims` will be changed to return a set of dimension names.*:FutureWarning",
+  "ignore:.*Mismatched null-like values None and nan found.*:FutureWarning"
 ]
 
 [tool.coverage.run]

requirements/base.txt

@@ -8,9 +8,9 @@ jsonschema >= 4, < 5
 natsort >= 8, < 9
 netcdf4 >= 1.2, < 1.7
 numpy >= 1, < 2
-pandas >= 2.1.3, < 2.2
+pandas >= 2.1.3, < 2.3  # Minimum bound is 2.1.3 because of a regression in v2.1.0/2.1.1 inflating time/memory consumption on groupby operations with MultiIndex
 pyomo >= 6.5, < 7
 pyparsing >= 3.0, < 3.1
-ruamel.yaml >= 0.17, < 0.18
+ruamel.yaml >= 0.18, < 0.19
 typing-extensions >= 4, < 5
-xarray >= 2023.10, < 2024.3
+xarray >= 2024.1, < 2024.4

requirements/dev.txt

@@ -9,7 +9,7 @@ mkdocstrings-python >= 1.7, < 2
 pandas-stubs
 plotly >= 5, < 6
 pre-commit < 4
-pytest < 8
+pytest >= 8, < 9
 pytest-cov < 5
 pytest-order < 2
 pytest-xdist < 4 # pytest distributed testing plugin

src/calliope/config/config_schema.yaml

@@ -41,7 +41,7 @@ properties:
           time_resample:
             type: ["null", string]
             default: null
-            description: setting to adjust time resolution, e.g. "2H" for 2-hourly
+            description: setting to adjust time resolution, e.g. "2h" for 2-hourly
             pattern: "^[0-9]+[a-zA-Z]"
           time_cluster:
             type: ["null", string]

src/calliope/example_models/national_scale/scenarios.yaml

@@ -20,7 +20,7 @@ overrides:
       init:
         name: "National-scale example model with time resampling"
         time_subset: ["2005-01", "2005-01"]
-        time_resample: "6H"
+        time_resample: "6h"
 
   time_clustering:
     config:
@@ -70,8 +70,8 @@ overrides:
       init.time_subset: ["2005-01-01", "2005-01-10"]
       build:
         mode: operate
-        operate_window: 12H
-        operate_horizon: 24H
+        operate_window: 12h
+        operate_horizon: 24h
     nodes:
       region1.techs.ccgt.flow_cap: 30000
 

src/calliope/example_models/urban_scale/scenarios.yaml

@@ -51,8 +51,8 @@ overrides:
       init.time_subset: ["2005-07-01", "2005-07-10"]
       build:
         mode: operate
-        operate_window: 24H
-        operate_horizon: 48H
+        operate_window: 2h
+        operate_horizon: 48h
 
     nodes:
       X1:

src/calliope/exceptions.py

@@ -10,18 +10,18 @@
 from typing import Optional, Union
 
 # Enable simple format when printing ModelWarnings
-formatwarning_orig = warnings.formatwarning
+formatwarning_orig = warnings.showwarning
 
 
-def _formatwarning(message, category, filename, lineno, line=None):
+def _formatwarning(message, category, filename, lineno, file=None, line=None):
     """Formats ModelWarnings as "Warning: message" without extra crud"""
     if category == ModelWarning:
         return "Warning: " + str(message) + "\n"
     else:
-        return formatwarning_orig(message, category, filename, lineno, line)
+        return formatwarning_orig(message, category, filename, lineno, file, line)
 
 
-warnings.formatwarning = _formatwarning
+warnings.showwarning = _formatwarning
 
 
 class ModelError(Exception):

src/calliope/preprocess/data_sources.py

@@ -279,7 +279,7 @@ def _df_to_ds(self, df: pd.DataFrame) -> xr.Dataset:
 
         tdf: pd.Series
         if isinstance(df, pd.DataFrame):
-            tdf = df.stack(df.columns.names)  # type: ignore
+            tdf = df.stack(df.columns.names, future_stack=True).dropna()
         else:
             tdf = df
 

src/calliope/preprocess/time.py

@@ -93,14 +93,14 @@ def subset_timeseries(ds: xr.Dataset, time_subset: list[str]) -> xr.Dataset:
 
 def resample(data: xr.Dataset, resolution: str) -> xr.Dataset:
     """
-    Function to resample timeseries data from the input resolution (e.g. 1H), to
-    the given resolution (e.g. 2H)
+    Function to resample timeseries data from the input resolution (e.g. 1h), to
+    the given resolution (e.g. 2h)
 
     Args:
         data (xarray.Dataset): Calliope model data, containing only timeseries data variables.
         resolution (str):
             time resolution of the output data, given in Pandas time frequency format.
-            E.g. 1H = 1 hour, 1W = 1 week, 1M = 1 month, 1T = 1 minute.
+            E.g. 1h = 1 hour, 1W = 1 week, 1M = 1 month, 1T = 1 minute.
             Multiples allowed.
 
     Returns:

tests/test_core_model.py

@@ -309,5 +309,5 @@ def test_rerun_operate(self, caplog, operate_model):
     def test_operate_timeseries(self, operate_model):
         assert all(
             operate_model.results.timesteps
-            == pd.date_range("2005-01", "2005-01-02 23:00:00", freq="H")
+            == pd.date_range("2005-01", "2005-01-02 23:00:00", freq="h")
         )

tests/test_core_preprocess.py

@@ -182,7 +182,7 @@ def override(param):
         )
         assert all(
             model.inputs.timesteps.to_index()
-            == pd.date_range("2005-01", "2005-01-01 23:00:00", freq="H")
+            == pd.date_range("2005-01", "2005-01-01 23:00:00", freq="h")
         )
 
         # should fail: must be a list, not a string

tests/test_core_util.py

@@ -328,7 +328,7 @@ def expected_model_def_defaults(self):
             {
                 "objective_cost_weights": 1,
                 "available_area": np.inf,
-                "color": None,
+                "color": np.nan,
                 "cap_method": "continuous",
                 "include_storage": False,
                 "flow_cap_per_storage_cap_min": 0,

tests/test_example_models.py

@@ -148,7 +148,7 @@ def example_tester(self):
 
         assert all(
             model.results.timesteps
-            == pd.date_range("2005-01", "2005-01-03 23:00:00", freq="H")
+            == pd.date_range("2005-01", "2005-01-03 23:00:00", freq="h")
         )
 
     def test_nationalscale_example_results_cbc(self):
@@ -489,5 +489,5 @@ def test_operate_example_results(self):
 
         assert all(
             model.results.timesteps
-            == pd.date_range("2005-07", "2005-07-04 23:00:00", freq="H")
+            == pd.date_range("2005-07", "2005-07-04 23:00:00", freq="h")
         )

tests/test_math.py

@@ -645,6 +645,9 @@ def test_annual_capacity_factor(self, build_and_compare):
     @pytest.mark.filterwarnings(
         "ignore:(?s).*This parameter will only take effect if you have already defined:calliope.exceptions.ModelWarning"
     )
+    @pytest.mark.filterwarnings(
+        "ignore:(?s).*Possibly missing data on the timesteps dimension:calliope.exceptions.ModelWarning"
+    )
     def test_downtime(self, build_and_compare):
         overrides = {
             "parameters": {

tests/test_preprocess_time.py

@@ -25,7 +25,7 @@ def test_change_date_format(self):
         model = build_test_model(override_dict=override, scenario="simple_conversion")
         assert all(
             model.inputs.timesteps.to_index()
-            == pd.date_range("2005-01", "2005-01-02 23:00:00", freq="H")
+            == pd.date_range("2005-01", "2005-01-02 23:00:00", freq="h")
         )
 
     def test_incorrect_date_format_one(self):
@@ -126,7 +126,7 @@ def test_resampling_to_6h_then_clustering(self):
         model = build_test_model(
             scenario="simple_supply",
             time_subset=["2005-01-01", "2005-01-04"],
-            time_resample="6H",
+            time_resample="6h",
             time_cluster="data_sources/cluster_days.csv",
         )
 
@@ -155,7 +155,7 @@ def test_15min_resampling_to_6h(self):
             "data_sources.demand_elec.source: data_sources/demand_elec_15mins.csv"
         )
 
-        model = build_test_model(override, scenario="simple_supply", time_resample="6H")
+        model = build_test_model(override, scenario="simple_supply", time_resample="6h")
         data = model._model_data
 
         dtindex = pd.DatetimeIndex(
@@ -183,7 +183,7 @@ def test_15min_to_2h_resampling_to_2h(self):
         )
 
         model = build_test_model(
-            override, scenario="simple_supply,one_day", time_resample="2H"
+            override, scenario="simple_supply,one_day", time_resample="2h"
         )
         data = model._model_data
 
@@ -230,7 +230,7 @@ def test_different_ts_resolutions_resampling_to_6h(self):
             """
         )
 
-        model = build_test_model(override, scenario="simple_supply", time_resample="6H")
+        model = build_test_model(override, scenario="simple_supply", time_resample="6h")
 
         dtindex = pd.DatetimeIndex(
             [