Update for pyam 07

CHANGELOG.rst

@@ -3,6 +3,7 @@ Changelog
 
 master
 ------
+- (`#116 <https://github.com/GranthamImperial/silicone/pull/116>`_) Updated to work with pyam v0.8
 - (`#115 <https://github.com/GranthamImperial/silicone/pull/115>`_) Pinned black
 - (`#113 <https://github.com/GranthamImperial/silicone/pull/113>`_) Added a warning for using ratio-based crunchers with negative values. Fixed some unit conversion todos (not user-facing).
 - (`#112 <https://github.com/GranthamImperial/silicone/pull/112>`_) Enabled more general unit conversion, bug fix and improvement for infill_composite_values.

src/silicone/database_crunchers/constant_ratio.py

@@ -96,7 +96,7 @@ def filler(in_iamdf):
             ValueError
                 The key year for filling is not in ``in_iamdf``.
             """
-            output_ts = in_iamdf.filter(variable=variable_leaders)
+            output_ts = in_iamdf.filter(variable=variable_leaders).data
             if any(output_ts["value"] < 0):
                 warn_str = "Note that the lead variable {} goes negative.".format(
                     variable_leaders
@@ -110,6 +110,6 @@ def filler(in_iamdf):
             output_ts["variable"] = variable_follower
             output_ts["unit"] = units
 
-            return output_ts
+            return IamDataFrame(output_ts)
 
         return filler

src/silicone/multiple_infillers/decompose_collection_with_time_dep_ratio.py

@@ -72,6 +72,8 @@ def _construct_consistent_values(self, aggregate_name, components, db_to_generat
         # Units are sorted in alphabetical order so we choose the first to get -equiv
         use["unit"] = units.iloc[0]
         use["variable"] = aggregate_name
+        for col in relevant_db.extra_cols:
+            use[col] = ""
         return pyam.IamDataFrame(use)
 
     def _set_of_units_without_equiv(self, df):

src/silicone/multiple_infillers/infill_all_required_emissions_for_openscm.py

@@ -1,5 +1,4 @@
 import logging
-import re
 import warnings
 
 import pandas as pd
@@ -122,8 +121,14 @@ def infill_all_required_variables(
             != to_fill_old_prefix
         ):
             raise ValueError("Not all of the data begins with the expected prefix")
-        to_fill.data["variable"] = to_fill.data["variable"].str.replace(
-            re.escape(to_fill_old_prefix + "|"), ""
+        to_fill.rename(
+            {
+                "variable": {
+                    var: var.replace(to_fill_old_prefix + "|", "")
+                    for var in to_fill.variables()
+                }
+            },
+            inplace=True,
         )
     if infilled_data_prefix:
         if any(
@@ -209,7 +214,14 @@ def infill_all_required_variables(
             **kwargs,
         )
     if infilled_data_prefix:
-        to_fill.data["variable"] = infilled_data_prefix + "|" + to_fill.data["variable"]
+        to_fill.rename(
+            {
+                "variable": {
+                    var: infilled_data_prefix + "|" + var for var in to_fill.variables()
+                }
+            },
+            inplace=True,
+        )
     return to_fill
 
 

src/silicone/multiple_infillers/infill_composite_values.py

@@ -1,5 +1,7 @@
 import logging
 
+from pyam import IamDataFrame
+
 from silicone.utils import _construct_consistent_values
 
 logger = logging.getLogger(__name__)
@@ -69,14 +71,14 @@ def infill_composite_values(df, composite_dic=None):
     df = df.filter(variable=composite_dic.keys(), keep=False)
     for composite, composite_variables in composite_dic.items():
         if isinstance(composite_variables, dict):
-            temp_df = df.filter(variable=[composite] + list(composite_variables.keys()))
+            temp_df = df.filter(
+                variable=[composite] + list(composite_variables.keys())
+            ).data
             for contributor, factor in composite_variables.items():
-                temp_df.data.loc[
-                    temp_df.data["variable"] == contributor, "value"
-                ] *= factor
+                temp_df.loc[temp_df["variable"] == contributor, "value"] *= factor
 
             composite_df = _construct_consistent_values(
-                composite, composite_variables, temp_df
+                composite, composite_variables, IamDataFrame(temp_df)
             )
 
         else:

tests/integration/crunchers/test_cruncher_constant_ratio.py

@@ -100,8 +100,9 @@ def test_relationship_usage(self, test_downscale_df, add_col):
         if add_col:
             # what should happen if there's more than one value in the `add_col`?
             add_col_val = "blah"
+            test_downscale_df = test_downscale_df.data
             test_downscale_df[add_col] = add_col_val
-            test_downscale_df = IamDataFrame(test_downscale_df.data)
+            test_downscale_df = IamDataFrame(test_downscale_df)
             assert test_downscale_df.extra_cols[0] == add_col
 
         lead = ["Emissions|HFC|C2F6"]
@@ -110,9 +111,11 @@ def test_relationship_usage(self, test_downscale_df, add_col):
         res = filler(test_downscale_df)
 
         exp = test_downscale_df.filter(variable=lead)
-        exp.data["variable"] = follow
-        exp.data["value"] = exp.data["value"] * 2
-        exp.data["unit"] = units
+        exp = exp.data
+        exp["variable"] = follow
+        exp["value"] = exp["value"] * 2
+        exp["unit"] = units
+        exp = IamDataFrame(exp)
 
         pd.testing.assert_frame_equal(
             res.timeseries(), exp.timeseries(), check_like=True
@@ -139,7 +142,9 @@ def test_negative_val_warning(self, test_db, test_downscale_df, caplog):
         with caplog.at_level(logging.INFO, logger="silicone.crunchers"):
             filler(test_downscale_df)
         assert len(caplog.record_tuples) == 0
-        test_downscale_df.data["value"].iloc[0] = -1
+        test_downscale_df = test_downscale_df.data
+        test_downscale_df["value"].iloc[0] = -1
+        test_downscale_df = IamDataFrame(test_downscale_df)
         with caplog.at_level(logging.INFO, logger="silicone.crunchers"):
             filler(test_downscale_df)
         assert len(caplog.record_tuples) == 1
@@ -180,10 +185,11 @@ def test_multiple_units_breaks_infillee(self, test_downscale_df):
             "Emissions|HFC|C5F12", ["Emissions|HFC|C2F6"], ratio=0, units="units"
         )
 
-        test_downscale_df = test_downscale_df.filter(year=[2010, 2015])
+        test_downscale_df = test_downscale_df.filter(year=[2010, 2015]).data
         test_downscale_df["unit"].iloc[0] = "bad units"
+        test_downscale_df = IamDataFrame(test_downscale_df)
         with pytest.raises(
             AssertionError,
             match="There are multiple or no units for the lead variable.",
         ):
-            res = filler(test_downscale_df)
+            filler(test_downscale_df)

tests/integration/crunchers/test_cruncher_equal_quantile_walk.py

@@ -8,7 +8,6 @@
 from base import _DataBaseCruncherTester
 from pyam import IamDataFrame
 
-import silicone.multiple_infillers
 from silicone.database_crunchers import EqualQuantileWalk
 
 _ma = "model_a"
@@ -124,8 +123,9 @@ def test_relationship_usage(self, test_db, simple_df, add_col):
         res = tcruncher.derive_relationship(follow, lead)
         if add_col:
             add_col_val = "blah"
+            simple_df = simple_df.data
             simple_df[add_col] = add_col_val
-            simple_df = IamDataFrame(simple_df.data)
+            simple_df = IamDataFrame(simple_df)
             assert simple_df.extra_cols[0] == add_col
 
         infilled = res(simple_df)
@@ -208,12 +208,12 @@ def test_uneven_lead_timeseries(self, test_db):
         test_db.filter(model=_ma, inplace=True)
         lead = ["Emissions|CH4"]
         follow = "Emissions|CO2"
-        one_follow = test_db.copy()
+        one_follow = test_db.data
         # We remove two of the CH4 values at the same time
-        one_follow.data = one_follow.data.drop([0, 16])
-        one_follow.data["scenario"] += "_orig"
+        one_follow = one_follow.drop([0, 16])
+        one_follow["scenario"] += "_orig"
+        one_follow = IamDataFrame(one_follow)
         one_follow.append(test_db, inplace=True)
-        one_follow = IamDataFrame(one_follow.data)
         tcruncher = self.tclass(one_follow)
         res = tcruncher.derive_relationship(follow, lead)
         infilled = res(test_db)
@@ -256,7 +256,9 @@ def test_extreme_values_relationship(self):
             modify_extreme_db["value"] == max(modify_extreme_db["value"])
         ]
         ind = modify_extreme_db["value"].idxmax
+        modify_extreme_db = modify_extreme_db.data
         modify_extreme_db["value"].loc[ind] += 10
+        modify_extreme_db = IamDataFrame(modify_extreme_db)
         extreme_crunched = res(modify_extreme_db)
         # Check results are the same
         assert crunched.equals(extreme_crunched)
@@ -272,7 +274,9 @@ def test_extreme_values_relationship(self):
             modify_extreme_db["value"] == min(modify_extreme_db["value"])
         ]
         ind = modify_extreme_db["value"].idxmin
+        modify_extreme_db = modify_extreme_db.data
         modify_extreme_db["value"].loc[ind] -= 10
+        modify_extreme_db = IamDataFrame(modify_extreme_db)
         extreme_crunched = res(modify_extreme_db)
         assert crunched.filter(scenario=min_scen)["value"].iloc[0] != min(
             large_db_int.filter(variable=follow)["value"].values

tests/integration/crunchers/test_cruncher_interpolate_specified.py

@@ -157,8 +157,9 @@ def test_relationship_usage(self, simple_df, add_col):
         res = tcruncher.derive_relationship(follow, lead, required_scenario="scen_a")
         if add_col:
             add_col_val = "blah"
+            simple_df = simple_df.data
             simple_df[add_col] = add_col_val
-            simple_df = IamDataFrame(simple_df.data)
+            simple_df = IamDataFrame(simple_df)
             assert simple_df.extra_cols[0] == add_col
 
         expect_00 = res(simple_df)

tests/integration/crunchers/test_cruncher_latest_time_ratio.py

@@ -19,7 +19,16 @@ class TestDatabaseCruncherLatestTimeRatio(_DataBaseCruncherTester):
             _msa + ["World", "Emissions|HFC|C5F12", "kt C5F12/yr", "", np.nan, 3.14],
             _msa + ["World", "Emissions|HFC|C2F6", "kt C2F6/yr", "", 1.2, 1.5],
         ],
-        columns=["model", "scenario", "region", "variable", "unit", "meta", 2010, 2015],
+        columns=[
+            "model",
+            "scenario",
+            "region",
+            "variable",
+            "unit",
+            "meta1",
+            2010,
+            2015,
+        ],
     )
     tdownscale_df = pd.DataFrame(
         [
@@ -89,7 +98,7 @@ def test_derive_relationship_error_no_info_follower(self, test_db):
                     "region",
                     "variable",
                     "unit",
-                    "meta",
+                    "meta1",
                     2010,
                     2015,
                 ],
@@ -105,7 +114,7 @@ def test_derive_relationship_error_no_info_follower(self, test_db):
                     "region",
                     "variable",
                     "unit",
-                    "meta",
+                    "meta1",
                     2015,
                 ],
             ),
@@ -155,8 +164,9 @@ def test_relationship_usage(self, test_db, test_downscale_df, add_col):
         filler = tcruncher.derive_relationship(follow, lead)
         if add_col:
             add_col_val = "blah"
+            test_downscale_df = test_downscale_df.data
             test_downscale_df[add_col] = add_col_val
-            test_downscale_df = IamDataFrame(test_downscale_df.data)
+            test_downscale_df = IamDataFrame(test_downscale_df)
             assert test_downscale_df.extra_cols[0] == add_col
         test_downscale_df = self._adjust_time_style_to_match(test_downscale_df, test_db)
         res = filler(test_downscale_df)
@@ -189,6 +199,9 @@ def test_relationship_usage(self, test_db, test_downscale_df, add_col):
             assert all(append_df[add_col] == add_col_val)
 
     def test_negative_val_warning(self, test_db, test_downscale_df, caplog):
+        # quiet pyam
+        caplog.set_level(logging.ERROR, logger="pyam")
+
         tcruncher = self.tclass(test_db)
         lead = ["Emissions|HFC|C2F6"]
         follow = "Emissions|HFC|C5F12"
@@ -197,7 +210,9 @@ def test_negative_val_warning(self, test_db, test_downscale_df, caplog):
         with caplog.at_level(logging.INFO, logger="silicone.crunchers"):
             filler(test_downscale_df)
         assert len(caplog.record_tuples) == 0
-        test_downscale_df.data["value"].iloc[0] = -1
+        test_downscale_df = test_downscale_df.data
+        test_downscale_df["value"].iloc[0] = -1
+        test_downscale_df = IamDataFrame(test_downscale_df)
         with caplog.at_level(logging.INFO, logger="silicone.crunchers"):
             filler(test_downscale_df)
         assert len(caplog.record_tuples) == 1

tests/integration/crunchers/test_cruncher_quantile_rolling_windows.py

@@ -116,6 +116,9 @@ def test_derive_relationship_with_multicolumns(self):
 
     @pytest.mark.parametrize("use_ratio", [True, False])
     def test_relationship_usage(self, simple_df, use_ratio, caplog):
+        # quiet pyam
+        caplog.set_level(logging.ERROR, logger="pyam")
+
         # This tests that using the cruncher for a simple case (no averages, just
         # choosing one of two values) produces the expected results. We test the
         # quantiles that should result in a flip between the two states.
@@ -286,23 +289,24 @@ def test_limit_of_similar_xs(self, test_db):
         )
         range_db = IamDataFrame(range_db)
         range_db = self._adjust_time_style_to_match(range_db, test_db)
-        same_db = range_db.copy()
+        same_db = range_db.copy().data
         same_db["variable"] = _eco2
         same_db["value"] = 1
+        same_db = IamDataFrame(same_db)
         same_db.append(range_db, inplace=True)
-        nearly_same_db = same_db.copy()
+        nearly_same_db = same_db.copy().data
         # We change the value of one point on the nearly_same and remove it on the same
-        nearly_same_db.data["value"].iloc[1] = 0
+        nearly_same_db["value"].iloc[1] = 0
+        same_db = IamDataFrame(same_db)
         same_db.filter(scenario="0", keep=False, inplace=True)
-        same_db = IamDataFrame(same_db.data)
-        nearly_same_db = IamDataFrame(nearly_same_db.data)
+        nearly_same_db = IamDataFrame(nearly_same_db)
         # Derive crunchers and compare results from the two values
         same_cruncher = self.tclass(same_db)
         nearly_same_cruncher = self.tclass(nearly_same_db)
         if test_db.time_col == "year":
-            single_date_df = test_db.filter(year=int(same_db[same_db.time_col][0]))
+            single_date_df = test_db.filter(year=int(same_db[same_db.time_col].iloc[0]))
         else:
-            single_date_df = test_db.filter(time=(same_db[same_db.time_col][0]))
+            single_date_df = test_db.filter(time=(same_db[same_db.time_col].iloc[0]))
         # We choose the case model a, which has only values over 1.
         single_date_df.filter(model=_mb, scenario=_sa, keep=False, inplace=True)
         same_res = same_cruncher.derive_relationship(_ech4, [_eco2])(single_date_df)
@@ -326,8 +330,9 @@ def test_extreme_values_relationship(self, add_col):
         assert callable(res)
         if add_col:
             add_col_val = "blah"
+            large_db = large_db.data
             large_db[add_col] = add_col_val
-            large_db = IamDataFrame(large_db.data)
+            large_db = IamDataFrame(large_db)
             assert large_db.extra_cols[0] == add_col
         crunched = res(large_db)
 
@@ -364,6 +369,9 @@ def test_derive_relationship_same_gas(self, test_db, test_downscale_df):
         )
 
     def test_negative_val_warning(self, test_db, test_downscale_df, caplog):
+        # quiet pyam
+        caplog.set_level(logging.ERROR, logger="pyam")
+
         tcruncher = self.tclass(test_db)
         lead = [_eco2]
         follow = _ech4
@@ -372,7 +380,9 @@ def test_negative_val_warning(self, test_db, test_downscale_df, caplog):
         with caplog.at_level(logging.INFO, logger="silicone.crunchers"):
             filler(test_downscale_df)
         assert len(caplog.record_tuples) == 0
-        test_downscale_df.data["value"].iloc[0] = -1
+        test_downscale_df = test_downscale_df.data
+        test_downscale_df["value"].iloc[0] = -1
+        test_downscale_df = IamDataFrame(test_downscale_df)
         with caplog.at_level(logging.INFO, logger="silicone.crunchers"):
             filler(test_downscale_df)
         assert len(caplog.record_tuples) == 1