Update for pyam 07

CHANGELOG.rst

@@ -3,6 +3,7 @@ Changelog
 
 master
 ------
+- (`#116 <https://github.com/GranthamImperial/silicone/pull/116>`_) Updated to work with new version of pyam
 - (`#115 <https://github.com/GranthamImperial/silicone/pull/115>`_) Pinned black
 - (`#113 <https://github.com/GranthamImperial/silicone/pull/113>`_) Added a warning for using ratio-based crunchers with negative values. Fixed some unit conversion todos (not user-facing).
 - (`#112 <https://github.com/GranthamImperial/silicone/pull/112>`_) Enabled more general unit conversion, bug fix and improvement for infill_composite_values.

src/silicone/database_crunchers/constant_ratio.py

@@ -96,7 +96,7 @@ def filler(in_iamdf):
             ValueError
                 The key year for filling is not in ``in_iamdf``.
             """
-            output_ts = in_iamdf.filter(variable=variable_leaders)
+            output_ts = in_iamdf.filter(variable=variable_leaders).data
             if any(output_ts["value"] < 0):
                 warn_str = "Note that the lead variable {} goes negative.".format(
                     variable_leaders
@@ -110,6 +110,6 @@ def filler(in_iamdf):
             output_ts["variable"] = variable_follower
             output_ts["unit"] = units
 
-            return output_ts
+            return IamDataFrame(output_ts)
 
         return filler

src/silicone/multiple_infillers/decompose_collection_with_time_dep_ratio.py

@@ -72,6 +72,8 @@ def _construct_consistent_values(self, aggregate_name, components, db_to_generat
         # Units are sorted in alphabetical order so we choose the first to get -equiv
         use["unit"] = units.iloc[0]
         use["variable"] = aggregate_name
+        for col in relevant_db.extra_cols:
+            use[col] = ""
         return pyam.IamDataFrame(use)
 
     def _set_of_units_without_equiv(self, df):

src/silicone/multiple_infillers/infill_all_required_emissions_for_openscm.py

@@ -1,5 +1,4 @@
 import logging
-import re
 import warnings
 
 import pandas as pd
@@ -122,8 +121,14 @@ def infill_all_required_variables(
             != to_fill_old_prefix
         ):
             raise ValueError("Not all of the data begins with the expected prefix")
-        to_fill.data["variable"] = to_fill.data["variable"].str.replace(
-            re.escape(to_fill_old_prefix + "|"), ""
+        to_fill.rename(
+            {
+                "variable": {
+                    var: var.replace(to_fill_old_prefix + "|", "")
+                    for var in to_fill.variables()
+                }
+            },
+            inplace=True,
         )
     if infilled_data_prefix:
         if any(
@@ -209,7 +214,14 @@ def infill_all_required_variables(
             **kwargs,
         )
     if infilled_data_prefix:
-        to_fill.data["variable"] = infilled_data_prefix + "|" + to_fill.data["variable"]
+        to_fill.rename(
+            {
+                "variable": {
+                    var: infilled_data_prefix + "|" + var for var in to_fill.variables()
+                }
+            },
+            inplace=True,
+        )
     return to_fill
 
 

src/silicone/multiple_infillers/infill_composite_values.py

@@ -1,5 +1,7 @@
 import logging
 
+from pyam import IamDataFrame
+
 from silicone.utils import _construct_consistent_values
 
 logger = logging.getLogger(__name__)
@@ -69,14 +71,14 @@ def infill_composite_values(df, composite_dic=None):
     df = df.filter(variable=composite_dic.keys(), keep=False)
     for composite, composite_variables in composite_dic.items():
         if isinstance(composite_variables, dict):
-            temp_df = df.filter(variable=[composite] + list(composite_variables.keys()))
+            temp_df = df.filter(
+                variable=[composite] + list(composite_variables.keys())
+            ).data
             for contributor, factor in composite_variables.items():
-                temp_df.data.loc[
-                    temp_df.data["variable"] == contributor, "value"
-                ] *= factor
+                temp_df.loc[temp_df["variable"] == contributor, "value"] *= factor
 
             composite_df = _construct_consistent_values(
-                composite, composite_variables, temp_df
+                composite, composite_variables, IamDataFrame(temp_df)
             )
 
         else:

src/silicone/utils.py

@@ -526,3 +526,17 @@ def _construct_consistent_values(aggregate_name, components, db_to_generate):
     use["unit"] = units.iloc[0]
     use["variable"] = aggregate_name
     return pyam.IamDataFrame(use)
+
+
+def _remove_deprecation_warnings(caplog_warnings):
+    """
+    Takes a caplog warnings list of tuples and removes any deprecation notices.
+    Purely for use in testing to ensure inter-version compatibility.
+    :param caplog_warnings: list[tuple]
+    :return: list[tuple]
+    """
+    return [
+        ctuple
+        for ctuple in caplog_warnings
+        if ctuple[2][:25] != "This method is deprecated"
+    ]

tests/integration/crunchers/test_cruncher_constant_ratio.py

@@ -100,8 +100,9 @@ def test_relationship_usage(self, test_downscale_df, add_col):
         if add_col:
             # what should happen if there's more than one value in the `add_col`?
             add_col_val = "blah"
+            test_downscale_df = test_downscale_df.data
             test_downscale_df[add_col] = add_col_val
-            test_downscale_df = IamDataFrame(test_downscale_df.data)
+            test_downscale_df = IamDataFrame(test_downscale_df)
             assert test_downscale_df.extra_cols[0] == add_col
 
         lead = ["Emissions|HFC|C2F6"]
@@ -110,9 +111,11 @@ def test_relationship_usage(self, test_downscale_df, add_col):
         res = filler(test_downscale_df)
 
         exp = test_downscale_df.filter(variable=lead)
-        exp.data["variable"] = follow
-        exp.data["value"] = exp.data["value"] * 2
-        exp.data["unit"] = units
+        exp = exp.data
+        exp["variable"] = follow
+        exp["value"] = exp["value"] * 2
+        exp["unit"] = units
+        exp = IamDataFrame(exp)
 
         pd.testing.assert_frame_equal(
             res.timeseries(), exp.timeseries(), check_like=True
@@ -139,7 +142,9 @@ def test_negative_val_warning(self, test_db, test_downscale_df, caplog):
         with caplog.at_level(logging.INFO, logger="silicone.crunchers"):
             filler(test_downscale_df)
         assert len(caplog.record_tuples) == 0
-        test_downscale_df.data["value"].iloc[0] = -1
+        test_downscale_df = test_downscale_df.data
+        test_downscale_df["value"].iloc[0] = -1
+        test_downscale_df = IamDataFrame(test_downscale_df)
         with caplog.at_level(logging.INFO, logger="silicone.crunchers"):
             filler(test_downscale_df)
         assert len(caplog.record_tuples) == 1
@@ -180,10 +185,11 @@ def test_multiple_units_breaks_infillee(self, test_downscale_df):
             "Emissions|HFC|C5F12", ["Emissions|HFC|C2F6"], ratio=0, units="units"
         )
 
-        test_downscale_df = test_downscale_df.filter(year=[2010, 2015])
+        test_downscale_df = test_downscale_df.filter(year=[2010, 2015]).data
         test_downscale_df["unit"].iloc[0] = "bad units"
+        test_downscale_df = IamDataFrame(test_downscale_df)
         with pytest.raises(
             AssertionError,
             match="There are multiple or no units for the lead variable.",
         ):
-            res = filler(test_downscale_df)
+            filler(test_downscale_df)

tests/integration/crunchers/test_cruncher_equal_quantile_walk.py

@@ -8,7 +8,6 @@
 from base import _DataBaseCruncherTester
 from pyam import IamDataFrame
 
-import silicone.multiple_infillers
 from silicone.database_crunchers import EqualQuantileWalk
 
 _ma = "model_a"
@@ -124,8 +123,9 @@ def test_relationship_usage(self, test_db, simple_df, add_col):
         res = tcruncher.derive_relationship(follow, lead)
         if add_col:
             add_col_val = "blah"
+            simple_df = simple_df.data
             simple_df[add_col] = add_col_val
-            simple_df = IamDataFrame(simple_df.data)
+            simple_df = IamDataFrame(simple_df)
             assert simple_df.extra_cols[0] == add_col
 
         infilled = res(simple_df)
@@ -208,12 +208,12 @@ def test_uneven_lead_timeseries(self, test_db):
         test_db.filter(model=_ma, inplace=True)
         lead = ["Emissions|CH4"]
         follow = "Emissions|CO2"
-        one_follow = test_db.copy()
+        one_follow = test_db.data
         # We remove two of the CH4 values at the same time
-        one_follow.data = one_follow.data.drop([0, 16])
-        one_follow.data["scenario"] += "_orig"
+        one_follow = one_follow.drop([0, 16])
+        one_follow["scenario"] += "_orig"
+        one_follow = IamDataFrame(one_follow)
         one_follow.append(test_db, inplace=True)
-        one_follow = IamDataFrame(one_follow.data)
         tcruncher = self.tclass(one_follow)
         res = tcruncher.derive_relationship(follow, lead)
         infilled = res(test_db)
@@ -256,7 +256,9 @@ def test_extreme_values_relationship(self):
             modify_extreme_db["value"] == max(modify_extreme_db["value"])
         ]
         ind = modify_extreme_db["value"].idxmax
+        modify_extreme_db = modify_extreme_db.data
         modify_extreme_db["value"].loc[ind] += 10
+        modify_extreme_db = IamDataFrame(modify_extreme_db)
         extreme_crunched = res(modify_extreme_db)
         # Check results are the same
         assert crunched.equals(extreme_crunched)
@@ -272,7 +274,9 @@ def test_extreme_values_relationship(self):
             modify_extreme_db["value"] == min(modify_extreme_db["value"])
         ]
         ind = modify_extreme_db["value"].idxmin
+        modify_extreme_db = modify_extreme_db.data
         modify_extreme_db["value"].loc[ind] -= 10
+        modify_extreme_db = IamDataFrame(modify_extreme_db)
         extreme_crunched = res(modify_extreme_db)
         assert crunched.filter(scenario=min_scen)["value"].iloc[0] != min(
             large_db_int.filter(variable=follow)["value"].values

tests/integration/crunchers/test_cruncher_interpolate_specified.py

@@ -157,8 +157,9 @@ def test_relationship_usage(self, simple_df, add_col):
         res = tcruncher.derive_relationship(follow, lead, required_scenario="scen_a")
         if add_col:
             add_col_val = "blah"
+            simple_df = simple_df.data
             simple_df[add_col] = add_col_val
-            simple_df = IamDataFrame(simple_df.data)
+            simple_df = IamDataFrame(simple_df)
             assert simple_df.extra_cols[0] == add_col
 
         expect_00 = res(simple_df)

tests/integration/crunchers/test_cruncher_latest_time_ratio.py

@@ -8,6 +8,7 @@
 from pyam import IamDataFrame
 
 from silicone.database_crunchers import LatestTimeRatio
+from silicone.utils import _remove_deprecation_warnings
 
 _msa = ["model_a", "scen_a"]
 
@@ -19,7 +20,16 @@ class TestDatabaseCruncherLatestTimeRatio(_DataBaseCruncherTester):
             _msa + ["World", "Emissions|HFC|C5F12", "kt C5F12/yr", "", np.nan, 3.14],
             _msa + ["World", "Emissions|HFC|C2F6", "kt C2F6/yr", "", 1.2, 1.5],
         ],
-        columns=["model", "scenario", "region", "variable", "unit", "meta", 2010, 2015],
+        columns=[
+            "model",
+            "scenario",
+            "region",
+            "variable",
+            "unit",
+            "meta1",
+            2010,
+            2015,
+        ],
     )
     tdownscale_df = pd.DataFrame(
         [
@@ -89,7 +99,7 @@ def test_derive_relationship_error_no_info_follower(self, test_db):
                     "region",
                     "variable",
                     "unit",
-                    "meta",
+                    "meta1",
                     2010,
                     2015,
                 ],
@@ -105,7 +115,7 @@ def test_derive_relationship_error_no_info_follower(self, test_db):
                     "region",
                     "variable",
                     "unit",
-                    "meta",
+                    "meta1",
                     2015,
                 ],
             ),
@@ -155,8 +165,9 @@ def test_relationship_usage(self, test_db, test_downscale_df, add_col):
         filler = tcruncher.derive_relationship(follow, lead)
         if add_col:
             add_col_val = "blah"
+            test_downscale_df = test_downscale_df.data
             test_downscale_df[add_col] = add_col_val
-            test_downscale_df = IamDataFrame(test_downscale_df.data)
+            test_downscale_df = IamDataFrame(test_downscale_df)
             assert test_downscale_df.extra_cols[0] == add_col
         test_downscale_df = self._adjust_time_style_to_match(test_downscale_df, test_db)
         res = filler(test_downscale_df)
@@ -196,13 +207,15 @@ def test_negative_val_warning(self, test_db, test_downscale_df, caplog):
         test_downscale_df = self._adjust_time_style_to_match(test_downscale_df, test_db)
         with caplog.at_level(logging.INFO, logger="silicone.crunchers"):
             filler(test_downscale_df)
-        assert len(caplog.record_tuples) == 0
-        test_downscale_df.data["value"].iloc[0] = -1
+        assert len(_remove_deprecation_warnings(caplog.record_tuples)) == 0
+        test_downscale_df = test_downscale_df.data
+        test_downscale_df["value"].iloc[0] = -1
+        test_downscale_df = IamDataFrame(test_downscale_df)
         with caplog.at_level(logging.INFO, logger="silicone.crunchers"):
             filler(test_downscale_df)
-        assert len(caplog.record_tuples) == 1
+        assert len(_remove_deprecation_warnings(caplog.record_tuples)) == 1
         warn_str = "Note that the lead variable {} goes negative.".format(lead)
-        assert caplog.record_tuples[0][2] == warn_str
+        assert _remove_deprecation_warnings(caplog.record_tuples)[0][2] == warn_str
 
     @pytest.mark.parametrize("interpolate", [True, False])
     def test_relationship_usage_interpolation(