Merge pull request #63 from OSeMOSYS/fix_index

Lack of indexes on dataframe break results

src/otoole/input.py

@@ -236,6 +236,16 @@ class ReadStrategy(Strategy):
     Strategies.
     """
 
+    def _check_index(self, input_data: Dict):
+        """Checks and applied that an index is applied to the parameter DataFrame
+        """
+        for name, df in input_data.items():
+            details = self.config[name]
+            try:
+                df.set_index(details["indices"], inplace=True)
+            except KeyError:
+                logger.debug("Parameter %s is indexed", name)
+
     @abstractmethod
     def read(self, filepath: str) -> Tuple[Dict[str, pd.DataFrame], Dict[str, Any]]:
         raise NotImplementedError()

src/otoole/read_strategies.py

@@ -41,7 +41,7 @@ def read(
 
         config = self.config
         default_values = self._read_default_values(config)
-
+        self._check_index(self._parameters)
         return self._parameters, default_values
 
 
@@ -133,6 +133,8 @@ def read(self, filepath) -> Tuple[Dict[str, pd.DataFrame], Dict[str, Any]]:
 
             input_data[mod_name] = narrow_checked
 
+        self._check_index(input_data)
+
         return input_data, default_values
 
 
@@ -176,6 +178,8 @@ def read(self, filepath) -> Tuple[Dict[str, pd.DataFrame], Dict[str, Any]]:
 
             input_data[parameter] = narrow_checked
 
+        self._check_index(input_data)
+
         return input_data, default_values
 
 
@@ -184,6 +188,7 @@ def read(self, filepath) -> Tuple[Dict[str, pd.DataFrame], Dict[str, Any]]:
         inputs = read_datapackage(filepath)
         default_resource = inputs.pop("default_values").set_index("name").to_dict()
         default_values = default_resource["default_value"]
+        self._check_index(inputs)
         return inputs, default_values
 
 
@@ -194,7 +199,7 @@ def read(self, filepath) -> Tuple[Dict[str, pd.DataFrame], Dict[str, Any]]:
         default_values = self._read_default_values(config)
         amply_datafile = self.read_in_datafile(filepath, config)
         inputs = self._convert_amply_to_dataframe(amply_datafile, config)
-
+        self._check_index(inputs)
         return inputs, default_values
 
     def read_in_datafile(self, path_to_datafile: str, config: Dict) -> Amply:
@@ -258,7 +263,7 @@ def _convert_amply_to_dataframe(
         for name in datafile_parser.symbols.keys():
             logger.debug("Extracting data for %s", name)
             if config[name]["type"] == "param":
-                indices = config[name]["indices"]
+                indices = config[name]["indices"].copy()
                 indices_dtypes = [config[index]["dtype"] for index in indices]
                 indices.append("VALUE")
                 indices_dtypes.append("float")

src/otoole/results/convert.py

@@ -206,7 +206,10 @@ def convert_dataframe_to_csv(
 
             df = df.rename(columns={"Value": "VALUE"})
 
-            results[name] = df[indices + ["VALUE"]].set_index(indices)
+            columns = indices + ["VALUE"]
+
+            df = df[columns]
+            results[name] = df.set_index(details["indices"])
 
         else:
             not_found.append(name)

src/otoole/results/result_package.py

@@ -190,7 +190,7 @@ def annual_emissions(self) -> pd.DataFrame:
         except KeyError as ex:
             raise KeyError(self._msg("AnnualEmissions", str(ex)))
 
-        mid = emission_activity_ratio.mul(yearsplit, fill_value=0.0)
+        mid = emission_activity_ratio.mul(yearsplit)
         data = mid.mul(rate_of_activity, fill_value=0.0)
 
         if not data.empty:

tests/results/test_results_package.py

@@ -282,7 +282,6 @@ def test_missing_tech(self, two_tech):
             data=[["SIMPLICITY", "CO2", 2014, 1.0]],
             columns=["REGION", "EMISSION", "YEAR", "VALUE"],
         ).set_index(["REGION", "EMISSION", "YEAR"])
-
         assert_frame_equal(actual, expected)
 
 

tests/test_read_strategies.py

@@ -21,7 +21,7 @@ def test_read_memory(self):
         expected = {
             "AccumulatedAnnualDemand": pd.DataFrame(
                 data=data, columns=["REGION", "FUEL", "YEAR", "VALUE"]
-            )
+            ).set_index(["REGION", "FUEL", "YEAR"])
         }
 
         assert "AccumulatedAnnualDemand" in actual.keys()
@@ -118,7 +118,7 @@ def test_convert_amply_to_dataframe(self):
             ],
             columns=["REGION", "TECHNOLOGY", "MODE_OF_OPERATION", "YEAR", "VALUE"],
         )
-
+        print(actual, expected)
         pd.testing.assert_frame_equal(actual["VariableCost"], expected)
 
     def test_convert_amply_data_to_list_of_lists(self):