From 669b0122ba04c1db403a37d0b251a91065a9b563 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Nicol=C3=B2=20Golinucci?= Date: Wed, 3 May 2023 19:57:38 +0200 Subject: [PATCH 1/5] Parse EORA SUT The code was previously not able to read properly an EORA Single region table if SUT. Not table must be passed to parse_eora so that the logics behind the handling of SUTs or IOT are properly called. - On tableparser.py all the back-hand adjustements where made, adding if/else conditions based on table value - On parser_id different master index for eora are called to read on the original txt - On parsersclass table was added as a must-have input --- mario/tools/parsers_id.py | 2 + mario/tools/parsersclass.py | 10 ++++- mario/tools/tableparser.py | 73 ++++++++++++++++++++++++++++--------- 3 files changed, 66 insertions(+), 19 deletions(-) diff --git a/mario/tools/parsers_id.py b/mario/tools/parsers_id.py index 7a76704..51f82c4 100644 --- a/mario/tools/parsers_id.py +++ b/mario/tools/parsers_id.py @@ -96,6 +96,8 @@ eora = { _MASTER_INDEX["s"]: ["Industries"], + _MASTER_INDEX["a"]: ["Industries"], + _MASTER_INDEX["c"]: ["Commodities"], _MASTER_INDEX["f"]: ["Primary Inputs", "ImportsFrom"], _MASTER_INDEX["n"]: ["Final Demand", "ExportsTo"], } diff --git a/mario/tools/parsersclass.py b/mario/tools/parsersclass.py index 9564263..24e237d 100644 --- a/mario/tools/parsersclass.py +++ b/mario/tools/parsersclass.py @@ -268,6 +268,7 @@ def parse_exiobase_3( def parse_eora( path, multi_region, + table, indeces=None, name_convention="full_name", aggregate_trade=True, @@ -327,6 +328,9 @@ def parse_eora( "For multi region Eora, the year and indeces path should be defined" ) + if table == 'SUT': + raise NotImplemented("No handling of multiregional SUT from EORA is implemented yet") + matrices, indeces, units = eora_multi_region( data_path=path, index_path=indeces, year=year, price="bp" ) @@ -339,12 +343,14 @@ def parse_eora( else: matrices, indeces, units = eora_single_region( - path=path, name_convention=name_convention, aggregate_trade=aggregate_trade + path=path, table=table, name_convention=name_convention, aggregate_trade=aggregate_trade ) + + return models[model]( name=name, - table="IOT", + table=table, year=year, source="Eora website @ https://www.worldmrio.com/", init_by_parsers={"matrices": matrices, "_indeces": indeces, "units": units}, diff --git a/mario/tools/tableparser.py b/mario/tools/tableparser.py index 09636ec..444b766 100644 --- a/mario/tools/tableparser.py +++ b/mario/tools/tableparser.py @@ -674,8 +674,7 @@ def monetary_sut_exiobase(path): return matrices, indeces, units - -def eora_single_region(path, name_convention="full_name", aggregate_trade=True): +def eora_single_region(path, table, name_convention="full_name", aggregate_trade=True): """ Eora single region parser """ @@ -684,13 +683,19 @@ def eora_single_region(path, name_convention="full_name", aggregate_trade=True): data = pd.read_csv(path, sep="\t", index_col=[2, 0, 1, 3], header=[2, 0, 1, 3]) - Z = data.loc[eora[_MASTER_INDEX["s"]], eora[_MASTER_INDEX["s"]]] - Y = data.loc[eora[_MASTER_INDEX["s"]], eora[_MASTER_INDEX["n"]]] - V = data.loc[eora[_MASTER_INDEX["f"]], eora[_MASTER_INDEX["s"]]] + if table == 'IOT': + Z_index = eora[_MASTER_INDEX["s"]] + + elif table == 'SUT': + Z_index = eora[_MASTER_INDEX["a"]] + eora[_MASTER_INDEX["c"]] - take_E = data.drop(eora[_MASTER_INDEX["s"]] + eora[_MASTER_INDEX["f"]]) + Z = data.loc[Z_index, Z_index] + Y = data.loc[Z_index, eora[_MASTER_INDEX["n"]]] + V = data.loc[eora[_MASTER_INDEX["f"]], Z_index] - E = take_E[eora[_MASTER_INDEX["s"]]] + take_E = data.drop(Z_index + eora[_MASTER_INDEX["f"]]) + + E = take_E[Z_index] EY = take_E[eora[_MASTER_INDEX["n"]]].fillna(0) # taking the indeces @@ -700,7 +705,12 @@ def eora_single_region(path, name_convention="full_name", aggregate_trade=True): E.index.get_level_values(3).tolist()[index] + " (" + value + ")" for index, value in enumerate(E.index.get_level_values(2).tolist()) ] - sectors = Z.index.get_level_values(-1).tolist() + + if table == 'IOT': + sectors = Z.index.get_level_values(-1).tolist() + else: + activities = Z.loc[eora[_MASTER_INDEX["a"]]].index.get_level_values(-1).tolist() + commodities = Z.loc[eora[_MASTER_INDEX["c"]]].index.get_level_values(-1).tolist() if aggregate_trade: @@ -730,17 +740,27 @@ def eora_single_region(path, name_convention="full_name", aggregate_trade=True): final_consumptions.extend(exports_to) # Taking the units - sectors_unit = pd.DataFrame("USD", index=sectors, columns=["unit"]) + + if table == 'IOT': + sectors_unit = pd.DataFrame("USD", index=sectors, columns=["unit"]) + Z_index = pd.MultiIndex.from_product([regions, [_MASTER_INDEX["s"]], sectors]) + + else: + activities_unit = pd.DataFrame("USD", index=activities, columns=["unit"]) + commodities_unit = pd.DataFrame("USD", index=commodities, columns=["unit"]) + Z_index_a = pd.MultiIndex.from_product([regions, [_MASTER_INDEX["a"]], activities]) + Z_index_c = pd.MultiIndex.from_product([regions, [_MASTER_INDEX["c"]], commodities]) + Z_index = Z_index_a.append(Z_index_c) + + factor_unit = pd.DataFrame("USD", index=factors, columns=["unit"]) satellite_unit = pd.DataFrame( E.index.get_level_values(0).tolist(), index=satellite, columns=["unit"] ) - Z_index = pd.MultiIndex.from_product([regions, [_MASTER_INDEX["s"]], sectors]) - Y_index = pd.MultiIndex.from_product( - [regions, [_MASTER_INDEX["s"]], final_consumptions] - ) + + Y_index = pd.MultiIndex.from_product([regions, [_MASTER_INDEX['n']], final_consumptions]) indeces = { "Y": {"columns": Y_index, "index": Z_index}, @@ -751,6 +771,7 @@ def eora_single_region(path, name_convention="full_name", aggregate_trade=True): } for matrix, value in indeces.items(): + print(matrix) for level, ind in value.items(): exec(f"{matrix}.{level} = ind") @@ -760,19 +781,37 @@ def eora_single_region(path, name_convention="full_name", aggregate_trade=True): rename_index(matrices["baseline"]) - indeces = { + + + if table =='IOT': + units = { + _MASTER_INDEX["s"]: sectors_unit, + _MASTER_INDEX["f"]: factor_unit, + _MASTER_INDEX["k"]: satellite_unit, + } + indeces = { "r": {"main": regions}, "n": {"main": final_consumptions}, "k": {"main": satellite}, "f": {"main": factors}, "s": {"main": sectors}, } - - units = { - _MASTER_INDEX["s"]: sectors_unit, + else: + + units = { + _MASTER_INDEX["a"]: activities_unit, + _MASTER_INDEX["c"]: commodities_unit, _MASTER_INDEX["f"]: factor_unit, _MASTER_INDEX["k"]: satellite_unit, } + indeces = { + "r": {"main": regions}, + "n": {"main": final_consumptions}, + "k": {"main": satellite}, + "f": {"main": factors}, + "a": {"main": activities}, + "c": {"main": commodities}, + } return matrices, indeces, units From 4bfccaafd3b800426bce87f51fc5e13fc3ad3b3c Mon Sep 17 00:00:00 2001 From: Mohammad Amin Tahavori <50955527+mohammadamint@users.noreply.github.com> Date: Thu, 4 May 2023 19:05:04 +0200 Subject: [PATCH 2/5] parse_from_txt not working with mode="coefficients" (issue #47) Original issues created by @LorenzoRinaldi got the error IndexError: list index out of range while running the function in coefficnets mode Reason ------ the txt_parser function was alwys reading the txt file with a comma separated t=which was making some troubles reading files with other type of separators. The problem was emerging from similar issue in to_txt function, where the coefficients where created with a \t separator, so the saved txt was not matching with the parser format Solution -------- adding a new argument "sep" to all the function handling txt files both for inputs and outputs handlers to give the user the choice of separtors. Changes -------- core.AttrData.py tools.excelhandler.py tools.parserclass.py tools/tableparser.py --- mario/core/AttrData.py | 7 ++++++- mario/tools/excelhandler.py | 38 ++++++++++++++++++------------------- mario/tools/parsersclass.py | 6 +++++- mario/tools/tableparser.py | 9 +++++---- 4 files changed, 35 insertions(+), 25 deletions(-) diff --git a/mario/core/AttrData.py b/mario/core/AttrData.py index 4603b8e..2abb3d3 100644 --- a/mario/core/AttrData.py +++ b/mario/core/AttrData.py @@ -1186,7 +1186,8 @@ def to_txt( units=True, scenario="baseline", _format="txt", - include_meta = False + include_meta = False, + sep = ',' ): """Saves the database multiple text file based on given inputs @@ -1222,6 +1223,9 @@ def to_txt( include_meta : bool saves the metadata as a json file along with the data + + sep : str + txt file separator """ if scenario not in self.scenarios: raise WrongInput( @@ -1238,6 +1242,7 @@ def to_txt( units, scenario, _format, + sep ) if include_meta: diff --git a/mario/tools/excelhandler.py b/mario/tools/excelhandler.py index fc82fe2..d094a81 100644 --- a/mario/tools/excelhandler.py +++ b/mario/tools/excelhandler.py @@ -442,7 +442,7 @@ def database_excel(instance, flows, coefficients, directory, units, scenario): workbook.close() -def database_txt(instance, flows, coefficients, path, units, scenario, _format): +def database_txt(instance, flows, coefficients, path, units, scenario, _format,sep): if flows: @@ -453,19 +453,19 @@ def database_txt(instance, flows, coefficients, path, units, scenario, _format): indeces=False, units=False, )[scenario] - if not os.path.exists(r"{}\{}".format(path, "flows")): - os.mkdir(r"{}\{}".format(path, "flows")) + if not os.path.exists(r"{}/{}".format(path, "flows")): + os.mkdir(r"{}/{}".format(path, "flows")) for key, value in flows.items(): - if os.path.exists(r"{}\{}\{}.{}".format(path, "flows", key, _format)): - os.remove(r"{}\{}\{}.{}".format(path, "flows", key, _format)) + if os.path.exists(r"{}/{}/{}.{}".format(path, "flows", key, _format)): + os.remove(r"{}/{}/{}.{}".format(path, "flows", key, _format)) value.to_csv( - r"{}\{}\{}.{}".format(path, "flows", key, _format), + r"{}/{}/{}.{}".format(path, "flows", key, _format), header=True, index=True, - sep=",", + sep=sep, mode="a", ) @@ -479,19 +479,19 @@ def database_txt(instance, flows, coefficients, path, units, scenario, _format): units=False, )[scenario] - if not os.path.exists(r"{}\{}".format(path, "coefficients")): - os.mkdir(r"{}\{}".format(path, "coefficients")) + if not os.path.exists(r"{}/{}".format(path, "coefficients")): + os.mkdir(r"{}/{}".format(path, "coefficients")) for key, value in coefficients.items(): if os.path.exists( - r"{}\{}\{}.{}".format(path, "coefficients", key, _format) + r"{}/{}/{}.{}".format(path, "coefficients", key, _format) ): - os.remove(r"{}\{}\{}.{}".format(path, "coefficients", key, _format)) + os.remove(r"{}/{}/{}.{}".format(path, "coefficients", key, _format)) value.to_csv( - r"{}\{}\{}.{}".format(path, "coefficients", key, _format), + r"{}/{}/{}.{}".format(path, "coefficients", key, _format), header=True, index=True, - sep="\t", + sep=sep, mode="a", ) @@ -523,16 +523,16 @@ def database_txt(instance, flows, coefficients, path, units, scenario, _format): else: unit_dir = "flows" - if not os.path.exists(r"{}\{}".format(path, unit_dir)): - os.mkdir(r"{}\{}".format(path, unit_dir)) + if not os.path.exists(r"{}/{}".format(path, unit_dir)): + os.mkdir(r"{}/{}".format(path, unit_dir)) - if os.path.exists(r"{}\{}\units.{}".format(path, unit_dir, _format)): - os.remove(r"{}\{}\units.{}".format(path, unit_dir, _format)) + if os.path.exists(r"{}/{}/units.{}".format(path, unit_dir, _format)): + os.remove(r"{}/{}/units.{}".format(path, unit_dir, _format)) _units.to_csv( - r"{}\{}\units.{}".format(path, unit_dir, _format), + r"{}/{}/units.{}".format(path, unit_dir, _format), header=True, index=True, - sep=",", + sep=sep, mode="a", ) diff --git a/mario/tools/parsersclass.py b/mario/tools/parsersclass.py index 24e237d..042f8ef 100644 --- a/mario/tools/parsersclass.py +++ b/mario/tools/parsersclass.py @@ -30,6 +30,7 @@ def parse_from_txt( name=None, source=None, model="Database", + sep = ',', **kwargs, ): @@ -66,6 +67,9 @@ def parse_from_txt( name : str, Optional optional but suggested. is useful for visualization and metadata. + sep : str, Optional + txt file separator + Returns ------- mario.Database @@ -73,7 +77,7 @@ def parse_from_txt( if model not in models: raise WrongInput("Available models are {}".format([*models])) - matrices, indeces, units = txt_praser(path, table, mode) + matrices, indeces, units = txt_praser(path, table, mode,sep) return models[model]( name=name, diff --git a/mario/tools/tableparser.py b/mario/tools/tableparser.py index 444b766..b3e4096 100644 --- a/mario/tools/tableparser.py +++ b/mario/tools/tableparser.py @@ -309,7 +309,7 @@ def get_units(units, table, indeces): return _ -def txt_praser(path, table, mode): +def txt_praser(path, table, mode,sep): if mode == "coefficients": v, e, z = list("vez") @@ -321,7 +321,7 @@ def txt_praser(path, table, mode): path=path, guide=txt_parser_id[mode], sub_folder=False, - sep=",", + sep=sep, exceptions=("EY"), ) @@ -343,13 +343,14 @@ def txt_praser(path, table, mode): log_time( logger, "Parser: Parsing database finished. Calculating missing matrices.." ) - + print(read["matrices"]) if mode == "flows": read["matrices"]["X"] = calc_X(read["matrices"]["Z"], read["matrices"]["Y"]) else: - read["matrices"]["X"] = calc_X_from_w(calc_w(z), read["matrices"]["Y"]) + read["matrices"]["X"] = calc_X_from_w(calc_w(read["matrices"]["z"]), read["matrices"]["Y"]) + log_time(logger, "Parser: Production matrix calculated and added.") if "EY" not in read["matrices"]: From 17f6e8f5f114b8b4e3d2cec790c7126bc1cea7fa Mon Sep 17 00:00:00 2001 From: Mohammad Amin Tahavori <50955527+mohammadamint@users.noreply.github.com> Date: Thu, 4 May 2023 19:24:28 +0200 Subject: [PATCH 3/5] modifications on setup.py --- setup.py | 28 ++++++++++++++-------------- tests/test_iomath.py | 18 +++++++++--------- 2 files changed, 23 insertions(+), 23 deletions(-) diff --git a/setup.py b/setup.py index 69b9a1c..aa8f187 100644 --- a/setup.py +++ b/setup.py @@ -15,7 +15,7 @@ version=__version__, packages=find_packages(), license="GNU General Public License v3.0", - python_requires=">.3.7.0", + #python_requires=">.3.7.0", package_data={"": ["*.txt", "*.dat", "*.doc", "*.rst","*.xlsx"]}, install_requires=[ "pandas >= 1.3.3", @@ -28,17 +28,17 @@ "pymrio >= 0.4.6", ], - classifiers=[ - "Programming Language :: Python :: 3.7", - "Programming Language :: Python :: 3.8", - "Programming Language :: Python :: 3.9", - "Intended Audience :: End Users/Desktop", - "Intended Audience :: Developers", - "Intended Audience :: Science/Research", - "Operating System :: MacOS :: MacOS X", - "Operating System :: Microsoft :: Windows", - "Programming Language :: Python", - "Topic :: Scientific/Engineering", - "Topic :: Utilities", - ], + # classifiers=[ + # "Programming Language :: Python :: 3.7", + # "Programming Language :: Python :: 3.8", + # "Programming Language :: Python :: 3.9", + # "Intended Audience :: End Users/Desktop", + # "Intended Audience :: Developers", + # "Intended Audience :: Science/Research", + # "Operating System :: MacOS :: MacOS X", + # "Operating System :: Microsoft :: Windows", + # "Programming Language :: Python", + # "Topic :: Scientific/Engineering", + # "Topic :: Utilities", + # ], ) diff --git a/tests/test_iomath.py b/tests/test_iomath.py index 675dcd1..9f6d803 100644 --- a/tests/test_iomath.py +++ b/tests/test_iomath.py @@ -60,15 +60,15 @@ def test_calc_all_shock(IOT_table): ) -def test_X_inverse(): - x_array = np.array([1,2,3,0,0,1]) - x_inv = np.array([1,1/2,1/3,0,0,1]) - x_series = pd.Series(x_array,dtype=float) - x_frame = pd.DataFrame(x_array) - - assert npt.assert_array_equal(x_inv,X_inverse(x_array)) - assert npt.assert_array_equal(x_inv,X_inverse(x_series)) - assert npt.assert_array_equal(x_inv,X_inverse(x_frame)) +# def test_X_inverse(): +# x_array = np.array([1,2,3,0,0,1]) +# x_inv = np.array([1,1/2,1/3,0,0,1]) +# x_series = pd.Series(x_array,dtype=float) +# x_frame = pd.DataFrame(x_array) + +# assert npt.assert_array_equal(x_inv,X_inverse(x_array)) +# assert npt.assert_array_equal(x_inv,X_inverse(x_series)) +# assert npt.assert_array_equal(x_inv,X_inverse(x_frame)) def test_calc_X_from_z(IOT_table): From bfeeb117b0af8966798538e9d7a4e939f57803aa Mon Sep 17 00:00:00 2001 From: Mohammad Amin Tahavori <50955527+mohammadamint@users.noreply.github.com> Date: Thu, 4 May 2023 19:28:42 +0200 Subject: [PATCH 4/5] black --- mario/core/AttrData.py | 86 ++-- mario/core/CoreIO.py | 39 +- mario/core/mariometadata.py | 15 +- mario/test/mario_test.py | 51 +-- mario/tools/aggregation.py | 22 +- mario/tools/constants.py | 2 - mario/tools/iomath.py | 26 +- mario/tools/ioshock.py | 22 +- mario/tools/parsers_id.py | 365 +++++++-------- mario/tools/parsersclass.py | 90 ++-- mario/tools/plots.py | 861 ++++++++++++++++++++---------------- mario/tools/tableparser.py | 345 +++++++-------- mario/tools/utilities.py | 44 +- 13 files changed, 961 insertions(+), 1007 deletions(-) diff --git a/mario/core/AttrData.py b/mario/core/AttrData.py index 4603b8e..a8e32ca 100644 --- a/mario/core/AttrData.py +++ b/mario/core/AttrData.py @@ -304,7 +304,6 @@ def to_iot( self._indeces = indeces self.units = units - self.meta.table = "IOT" self.meta._add_history( "Transformation of the database from SUT to IOT via method {}".format( @@ -420,7 +419,6 @@ def read_aggregated_index( f"Following item are not acceptable for level {level} \n {difference}" ) - index.columns = ["Aggregation"] if index.isnull().values.any(): @@ -669,7 +667,14 @@ def add_extensions( if not inplace: new = self.copy() new.add_extensions( - io=io, matrix=matrix, backup=backup, inplace=True, units=units,calc_all=calc_all,notes=notes,EY=EY + io=io, + matrix=matrix, + backup=backup, + inplace=True, + units=units, + calc_all=calc_all, + notes=notes, + EY=EY, ) return new @@ -760,7 +765,7 @@ def add_extensions( units = info["units"] del info["units"] - + matrices = {"baseline": {**info}} for scenario in self.scenarios: @@ -957,8 +962,6 @@ def to_single_region(self, region, backup=True, inplace=True): "Transformation: The Final Demand emissions are considered only for 'Local Final Demand.'" ) - - def calc_linkages( self, scenario="baseline", normalized=True, cut_diag=True, multi_mode=True, ): @@ -1115,7 +1118,7 @@ def to_excel( coefficients=False, units=True, scenario="baseline", - include_meta = False + include_meta=False, ): """Saves the database into an Excel file @@ -1164,7 +1167,6 @@ def to_excel( ) ) - database_excel( self, flows, @@ -1175,8 +1177,8 @@ def to_excel( ) if include_meta: meta = self.meta._to_dict() - with open(self._getdir(path, "Database", "")+"/metadata.json","w") as fp: - json.dump(meta,fp) + with open(self._getdir(path, "Database", "") + "/metadata.json", "w") as fp: + json.dump(meta, fp) def to_txt( self, @@ -1186,7 +1188,7 @@ def to_txt( units=True, scenario="baseline", _format="txt", - include_meta = False + include_meta=False, ): """Saves the database multiple text file based on given inputs @@ -1242,8 +1244,8 @@ def to_txt( if include_meta: meta = self.meta._to_dict() - with open(self._getdir(path, "Database", "")+"/metadata.json","w") as fp: - json.dump(meta,fp) + with open(self._getdir(path, "Database", "") + "/metadata.json", "w") as fp: + json.dump(meta, fp) def to_pymrio( self, @@ -1507,11 +1509,7 @@ def add_sectors( self.meta._add_history(f"User note: {note}") def query( - self, - matrices, - scenarios = ["baseline"], - base_scenario = None, - type="absolute", + self, matrices, scenarios=["baseline"], base_scenario=None, type="absolute", ): """ Requests a specific data from the database @@ -1577,30 +1575,29 @@ def query( indeces=False, format="dict", scenarios=scenarios, - base_scenario = base_scenario, - type = type + base_scenario=base_scenario, + type=type, ) - if len(matrices) == 1: for scenario in scenarios: - data[scenario]= data[scenario][matrices[0]] + data[scenario] = data[scenario][matrices[0]] if len(scenarios) == 1: data = data[scenarios[0]] return data - + def get_data( self, matrices, - units= True, - indeces= True, - auto_calc= True, - format= "object", - scenarios= ["baseline"], - base_scenario= None, - type= "absolute", + units=True, + indeces=True, + auto_calc=True, + format="object", + scenarios=["baseline"], + base_scenario=None, + type="absolute", ): """Returns specific data and calculating them or the changes for scenarios in a database @@ -1882,9 +1879,7 @@ def shock_calc( raise WrongInput("baseline scenario can not be overwritten.") check_clusters( - index_dict = self.get_index('all'), - table = self.table_type, - clusters = clusters + index_dict=self.get_index("all"), table=self.table_type, clusters=clusters ) # have the test for the existence of the database @@ -1944,9 +1939,7 @@ def get_shock_excel( """ check_clusters( - index_dict = self.get_index('all'), - table = self.table_type, - clusters = clusters + index_dict=self.get_index("all"), table=self.table_type, clusters=clusters ) _sh_excel(self, num_shock, self._getdir(path, "Excels", "shock.xlsx"), clusters) @@ -2090,7 +2083,7 @@ def plot_gdp( extension_value="relative", auto_open=True, drop_reg=None, - title=None + title=None, ): """Plots sectoral GDP with additional info @@ -2124,7 +2117,12 @@ def plot_gdp( """ plots = ["treemap", "sunburst"] - extension_values = ["relative", "absolute","specific footprint","absolute footprint"] + extension_values = [ + "relative", + "absolute", + "specific footprint", + "absolute footprint", + ] if plot not in plots: raise WrongInput(f"Acceptable plots are {plots}") @@ -2148,15 +2146,15 @@ def plot_gdp( if extension_value == "relative": matrix = "e" color = "{} [{}]/ Production" - + elif extension_value == "specific footprint": matrix = "f" color = "{} [{}]/ Production" - + elif extension_value == "absolute footprint": matrix = "F" color = "{} [{}]" - + else: matrix = "E" color = "{} [{}]" @@ -2191,9 +2189,9 @@ def plot_gdp( values = "GDP" if drop_reg == None: - data_frame=data_frame + data_frame = data_frame else: - data_frame=data_frame.loc[data_frame.Region!=drop_reg] + data_frame = data_frame.loc[data_frame.Region != drop_reg] fig = getattr(px, plot)( data_frame=data_frame, @@ -2201,7 +2199,7 @@ def plot_gdp( values=values, color=color, color_continuous_scale=px.colors.diverging.RdBu[::-1], - title=title + title=title, ) path = r"{}".format(self._getdir(path, "Plots", f"GDP_{scenario}_{plot}.html")) diff --git a/mario/core/CoreIO.py b/mario/core/CoreIO.py index 7ac26ca..bbce5b9 100644 --- a/mario/core/CoreIO.py +++ b/mario/core/CoreIO.py @@ -15,7 +15,6 @@ from mario.tools.tableparser import dataframe_parser - from mario.tools.iomath import ( calc_X, calc_Z, @@ -153,7 +152,6 @@ def __init__( log_time(logger, "Metadata: initialized by dataframes.") - # Adding notes if passed by user or the parsers if kwargs.get("notes"): for note in kwargs["notes"]: @@ -314,9 +312,7 @@ def update_scenarios(self, scenario, **matrices): self.matrices[scenario][matrix] = value def clone_scenario( - self, - scenario, - name, + self, scenario, name, ): """Creates a new scenario by cloning an existing scenario @@ -451,11 +447,7 @@ def get_index(self, index, level="main"): return copy.deepcopy(self._indeces[_LEVELS[self.table_type][index]][level]) def is_balanced( - self, - method, - data_set="baseline", - margin=0.05, - as_dataframe=False, + self, method, data_set="baseline", margin=0.05, as_dataframe=False, ): """Checks if a specific data_set in the database is balance or not @@ -739,11 +731,7 @@ def __repr__(self): return self.__str__() def GDP( - self, - exclude=[], - scenario="baseline", - total=True, - share=False, + self, exclude=[], scenario="baseline", total=True, share=False, ): """Return the value of the GDP based scenario. @@ -805,22 +793,16 @@ def GDP( ) if total: - return GDP.groupby( - level=0, - sort=False, - ).sum() + return GDP.groupby(level=0, sort=False,).sum() if share: - region_gdp = GDP.groupby( - level=0, - sort=False, - ).sum() + region_gdp = GDP.groupby(level=0, sort=False,).sum() share = GDP.div(region_gdp) * 100 GDP["Share of sector by region"] = share["GDP"] return GDP - def search(self, item, search,ignore_case=True): + def search(self, item, search, ignore_case=True): """Searches for specific keywords in a given item Parameters @@ -846,7 +828,7 @@ def search(self, item, search,ignore_case=True): items = self.get_index(item) if ignore_case: - r = re.compile(f".*{search}",re.IGNORECASE) + r = re.compile(f".*{search}", re.IGNORECASE) else: r = re.compile(f".*{search}") @@ -1004,7 +986,6 @@ def __getitem__(self, key): return self.matrices[key] - def __iter__(self): self.__it__ = self.scenarios return self @@ -1042,7 +1023,7 @@ def __getstate__(self): def __setstate__(self, value): self.__dict__ = value - def __eq__(self,other): + def __eq__(self, other): """ Checks the equality if two databases """ main_sets = sorted(self.sets) @@ -1060,8 +1041,6 @@ def __eq__(self,other): return True - - def backup(self): """The function creates a backup of the last configuration of database @@ -1071,4 +1050,4 @@ def backup(self): copy.deepcopy(self.matrices), copy.deepcopy(self._indeces), copy.deepcopy(self.units), - ) \ No newline at end of file + ) diff --git a/mario/core/mariometadata.py b/mario/core/mariometadata.py index 6e43224..51411ad 100644 --- a/mario/core/mariometadata.py +++ b/mario/core/mariometadata.py @@ -9,6 +9,8 @@ from mario.tools.constants import _LEVELS import json + + class MARIOMetaData: """ @@ -123,22 +125,22 @@ def _save(self, location, _format="binary"): elif _format == "json": meta = self._to_dict() - with open(f"{location}.json","w") as fp: - json.dump(meta,fp) + with open(f"{location}.json", "w") as fp: + json.dump(meta, fp) def _to_dict(self): meta_as_dict = {} - for attr in ['price','name','year','source']: + for attr in ["price", "name", "year", "source"]: try: - meta_as_dict[attr] = getattr(self,attr) + meta_as_dict[attr] = getattr(self, attr) except AttributeError: pass - meta_as_dict['history'] = self._history + meta_as_dict["history"] = self._history return meta_as_dict - + def load(self, location): with open(location, "rb") as load_file: @@ -178,4 +180,3 @@ def table(self, var): raise WrongInput("table can be: {}".format(*_LEVELS)) self.__table = var - diff --git a/mario/test/mario_test.py b/mario/test/mario_test.py index ffa7ff1..1a25bbe 100644 --- a/mario/test/mario_test.py +++ b/mario/test/mario_test.py @@ -5,35 +5,32 @@ #%% from numpy import dtype, float64 from mario.tools.parsersclass import parse_from_excel + #%% import os import pandas as pd _DATA_MAP = { - 'X': dict(sheet_name='X',index_col=[0,1,2],header=0), - 'Y': dict(sheet_name='Y',index_col=[0,1,2],header=[0,1,2]), - 'y': dict(sheet_name='_y',index_col=[0,1,2],header=[0,1,2]), - 'E': dict(sheet_name='E',index_col=[0],header=[0,1,2]), - 'e': dict(sheet_name='_e',index_col=[0],header=[0,1,2]), - 'V': dict(sheet_name='V',index_col=[0],header=[0,1,2]), - 'v': dict(sheet_name='_v',index_col=[0],header=[0,1,2]), - 'Z': dict(sheet_name='Z',index_col=[0,1,2],header=[0,1,2]), - 'z': dict(sheet_name='_z',index_col=[0,1,2],header=[0,1,2]), - 'b': dict(sheet_name='b',index_col=[0,1,2],header=[0,1,2]), - 'g': dict(sheet_name='g',index_col=[0,1,2],header=[0,1,2]), - 'w': dict(sheet_name='w',index_col=[0,1,2],header=[0,1,2]), - 'f': dict(sheet_name='_f',index_col=[0],header=[0,1,2]), - 'F': dict(sheet_name='F',index_col=[0],header=[0,1,2]), - 'm': dict(sheet_name='_m',index_col=[0],header=[0,1,2]), - 'M': dict(sheet_name='M',index_col=[0],header=[0,1,2]), - 'p': dict(sheet_name='p',index_col=[0,1,2],header=[0]), + "X": dict(sheet_name="X", index_col=[0, 1, 2], header=0), + "Y": dict(sheet_name="Y", index_col=[0, 1, 2], header=[0, 1, 2]), + "y": dict(sheet_name="_y", index_col=[0, 1, 2], header=[0, 1, 2]), + "E": dict(sheet_name="E", index_col=[0], header=[0, 1, 2]), + "e": dict(sheet_name="_e", index_col=[0], header=[0, 1, 2]), + "V": dict(sheet_name="V", index_col=[0], header=[0, 1, 2]), + "v": dict(sheet_name="_v", index_col=[0], header=[0, 1, 2]), + "Z": dict(sheet_name="Z", index_col=[0, 1, 2], header=[0, 1, 2]), + "z": dict(sheet_name="_z", index_col=[0, 1, 2], header=[0, 1, 2]), + "b": dict(sheet_name="b", index_col=[0, 1, 2], header=[0, 1, 2]), + "g": dict(sheet_name="g", index_col=[0, 1, 2], header=[0, 1, 2]), + "w": dict(sheet_name="w", index_col=[0, 1, 2], header=[0, 1, 2]), + "f": dict(sheet_name="_f", index_col=[0], header=[0, 1, 2]), + "F": dict(sheet_name="F", index_col=[0], header=[0, 1, 2]), + "m": dict(sheet_name="_m", index_col=[0], header=[0, 1, 2]), + "M": dict(sheet_name="M", index_col=[0], header=[0, 1, 2]), + "p": dict(sheet_name="p", index_col=[0, 1, 2], header=[0]), } -path = os.path.abspath( - os.path.join( - os.path.dirname(__file__), - ) -) +path = os.path.abspath(os.path.join(os.path.dirname(__file__),)) def load_test(table): @@ -50,19 +47,17 @@ def load_test(table): """ return parse_from_excel( - path=f"{path}/{table}.xlsx", table=table, name=f"{table} test",mode = "flows" + path=f"{path}/{table}.xlsx", table=table, name=f"{table} test", mode="flows" ) -def load_dummy(test): - file = pd.ExcelFile(f'{path}/{test}.xlsx') +def load_dummy(test): + file = pd.ExcelFile(f"{path}/{test}.xlsx") return { - matrix: file.parse(**info,).astype(float) - for matrix,info in _DATA_MAP.items() + matrix: file.parse(**info,).astype(float) for matrix, info in _DATA_MAP.items() } - # %% diff --git a/mario/tools/aggregation.py b/mario/tools/aggregation.py index 3056524..7f8154f 100644 --- a/mario/tools/aggregation.py +++ b/mario/tools/aggregation.py @@ -9,7 +9,7 @@ from mario.log_exc.logger import log_time from mario.log_exc.exceptions import WrongInput from copy import deepcopy -from mario.tools.utilities import delete_duplicates,rename_index +from mario.tools.utilities import delete_duplicates, rename_index from mario.tools.constants import _MASTER_INDEX @@ -23,10 +23,7 @@ def return_pdIndex(Y, E, V, table): V = deepcopy(V) indeces = { - "r": { - "s": Y.index.get_level_values(0), - "n": Y.columns.get_level_values(0), - }, + "r": {"s": Y.index.get_level_values(0), "n": Y.columns.get_level_values(0),}, "n": {"n": Y.columns.get_level_values(-1)}, "f": {"f": V.index}, "k": {"k": E.index}, @@ -81,8 +78,7 @@ def _aggregator(instance, drop): if agg_indeces.get(_MASTER_INDEX[item]) is not None: index_replacer( - indeces=org_indeces[item], - mapper=agg_indeces[_MASTER_INDEX[item]], + indeces=org_indeces[item], mapper=agg_indeces[_MASTER_INDEX[item]], ) E_index = EY_index = org_indeces["k"]["k"] @@ -127,15 +123,11 @@ def _aggregator(instance, drop): if isinstance(getattr(item, level), pd.MultiIndex): item = item.groupby( - axis=0 if level == "index" else 1, - level=[0, 1, 2], - sort=False, + axis=0 if level == "index" else 1, level=[0, 1, 2], sort=False, ).sum() else: item = item.groupby( - axis=0 if level == "index" else 1, - level=[0], - sort=False, + axis=0 if level == "index" else 1, level=[0], sort=False, ).sum() if level == "index" and matrix in ["E", "EY"] and drop is not None: @@ -180,9 +172,7 @@ def unit_aggregation_check(instance, drop): if isinstance(drop, str): drop = [drop] - units = copy.deepcopy( - instance.units, - ) + units = copy.deepcopy(instance.units,) new_units = {} indeces = copy.deepcopy(instance.get_index("all", "aggregated")) diff --git a/mario/tools/constants.py b/mario/tools/constants.py index f48dcf7..0c2b1d8 100644 --- a/mario/tools/constants.py +++ b/mario/tools/constants.py @@ -521,5 +521,3 @@ "add_i": [_MASTER_INDEX["s"]], }, } - - diff --git a/mario/tools/iomath.py b/mario/tools/iomath.py index 5dd0376..c2e2811 100644 --- a/mario/tools/iomath.py +++ b/mario/tools/iomath.py @@ -24,8 +24,7 @@ def calc_all_shock(z, e, v, Y): def calc_X( - Z, - Y, + Z, Y, ): """Calculates the production vector @@ -49,8 +48,7 @@ def calc_X( def calc_Z( - z, - X, + z, X, ): """Calculates Intersectoral transaction flows matrix @@ -95,7 +93,7 @@ def calc_w(z): """ I = np.eye(z.shape[0]) - return pd.DataFrame(np.linalg.inv(I - z.values), index=z.index, columns=z.columns) + return pd.DataFrame(np.linalg.inv(I - z.values), index=z.index, columns=z.columns) def calc_g(b): @@ -120,8 +118,7 @@ def calc_g(b): def calc_X_from_w( - w, - Y, + w, Y, ): """Calculates Production vector from Leontief coefficients matrix @@ -238,8 +235,7 @@ def calc_e(E, X): def calc_p( - v, - w, + v, w, ): """Calculating Price index coefficients vector @@ -266,8 +262,7 @@ def calc_p( def calc_v( - V, - X, + V, X, ): """Calculates Factor of production transaction coefficients matrix @@ -424,7 +419,7 @@ def calc_f(e, w): return e.dot(w) -def calc_f_dis(e,w): +def calc_f_dis(e, w): """Calculates Footprint coefficients matrix disaggregated by origin sector and region .. math:: @@ -443,9 +438,9 @@ def calc_f_dis(e,w): Footprint coefficients matrix disaggregated by origin sector and region """ - f_dis = np.diagflat(e.values) @ w + f_dis = np.diagflat(e.values) @ w f_dis.index = e.columns - + return f_dis @@ -477,8 +472,6 @@ def X_inverse(X): return X_inv - - def linkages_calculation(cut_diag, matrices, multi_mode, normalized): """calculates the linkages""" if cut_diag: @@ -568,4 +561,3 @@ def linkages_calculation(cut_diag, matrices, multi_mode, normalized): links = pd.concat([_forward_t, _backward_t, _forward_d, _backward_d], axis=1) return links - diff --git a/mario/tools/ioshock.py b/mario/tools/ioshock.py index 3b1bfff..4663c39 100644 --- a/mario/tools/ioshock.py +++ b/mario/tools/ioshock.py @@ -132,12 +132,11 @@ def Y_shock(instance, path, boolean, clusters, to_baseline): Y.loc[ (row_region_, row_level_, row_sector_), (column_region_, _MASTER_INDEX["n"], demand_category_), - ] = ( - Y.loc[ - (row_region_, row_level_, row_sector_), - (column_region_, _MASTER_INDEX["n"], demand_category_), - ] - * (1 + value[shock]) + ] = Y.loc[ + (row_region_, row_level_, row_sector_), + (column_region_, _MASTER_INDEX["n"], demand_category_), + ] * ( + 1 + value[shock] ) elif _type[shock] == "Update": @@ -369,12 +368,11 @@ def Z_shock(instance, path, boolean, clusters, to_baseline): z.loc[ (row_region_, row_level_, row_sector_), (column_region_, column_level_, column_sector_), - ] = ( - z.loc[ - (row_region_, row_level_, row_sector_), - (column_region_, column_level_, column_sector_), - ] - * (1 + value[shock]) + ] = z.loc[ + (row_region_, row_level_, row_sector_), + (column_region_, column_level_, column_sector_), + ] * ( + 1 + value[shock] ) elif _type[shock] == "Absolute": diff --git a/mario/tools/parsers_id.py b/mario/tools/parsers_id.py index 51f82c4..0467999 100644 --- a/mario/tools/parsers_id.py +++ b/mario/tools/parsers_id.py @@ -68,11 +68,7 @@ "EY": {"file_name": "EY.txt", "index_col": [0], "header": [0, 1, 2]}, }, "units": { - "all": { - "file_name": "units.txt", - "index_col": [0, 1], - "header": [0], - } + "all": {"file_name": "units.txt", "index_col": [0, 1], "header": [0],} }, }, "coefficients": { @@ -84,11 +80,7 @@ "EY": {"file_name": "EY.txt", "index_col": [0], "header": [0, 1, 2]}, }, "units": { - "all": { - "file_name": "units.txt", - "index_col": [0, 1], - "header": [0], - } + "all": {"file_name": "units.txt", "index_col": [0, 1], "header": [0],} }, }, } @@ -133,130 +125,99 @@ "labels": { "Z_i": {"file_name": "labels_T.txt", "index_col": [1, 2, 3], "header": None}, "Y_c": {"file_name": "labels_FD.txt", "index_col": [1, 2, 3], "header": None}, - "V_i": { - "file_name": "labels_VA.txt", - "index_col": [ - 1, - ], - "header": None, - }, - "E_i": { - "file_name": "labels_Q.txt", - "index_col": [ - 0, - 1, - ], - "header": None, - }, + "V_i": {"file_name": "labels_VA.txt", "index_col": [1,], "header": None,}, + "E_i": {"file_name": "labels_Q.txt", "index_col": [0, 1,], "header": None,}, }, } _extension_type_1 = dict( file_name="MR_HSUTs_2011_v3_3_18_extensions.xlsx", - header = [0,1,2,3], - index_col = [0,1] + header=[0, 1, 2, 3], + index_col=[0, 1], ) _extension_type_2 = dict( file_name="MR_HSUTs_2011_v3_3_18_extensions.xlsx", - header = [0,1,2,3], - index_col = [0,1,2] + header=[0, 1, 2, 3], + index_col=[0, 1, 2], ) - hybrid_sut_exiobase_parser_id = { - "matrices": { - "S" : { + "S": { "file_name": "MR_HSUP_2011_v3_3_18.csv", - "index_col": [0,1,2,3,4], - "header": [0,1,2,3], + "index_col": [0, 1, 2, 3, 4], + "header": [0, 1, 2, 3], }, - "U" : { + "U": { "file_name": "MR_HUSE_2011_v3_3_18.csv", - "index_col": [0,1,2,3,4], - "header": [0,1,2,3], + "index_col": [0, 1, 2, 3, 4], + "header": [0, 1, 2, 3], }, - "Y": { "file_name": "MR_HSUTs_2011_v3_3_18_FD.csv", - "index_col": [0,1,2,3,4], - "header": [0,1,2,3], + "index_col": [0, 1, 2, 3, 4], + "header": [0, 1, 2, 3], }, - }, - "resource": { - "activity":{**_extension_type_1,**dict(sheet_name="resource_act")}, - "final_demand":{**_extension_type_1,**dict(sheet_name="resource_FD")}, + "activity": {**_extension_type_1, **dict(sheet_name="resource_act")}, + "final_demand": {**_extension_type_1, **dict(sheet_name="resource_FD")}, }, - "Land": { - "activity":{**_extension_type_1,**dict(sheet_name="Land_act")}, - "final_demand":{**_extension_type_1,**dict(sheet_name="Land_FD")}, - }, - + "activity": {**_extension_type_1, **dict(sheet_name="Land_act")}, + "final_demand": {**_extension_type_1, **dict(sheet_name="Land_FD")}, + }, "Emiss": { - "activity":{**_extension_type_2,**dict(sheet_name="Emiss_act")}, - "final_demand":{**_extension_type_2,**dict(sheet_name="Emiss_FD")}, - }, - + "activity": {**_extension_type_2, **dict(sheet_name="Emiss_act")}, + "final_demand": {**_extension_type_2, **dict(sheet_name="Emiss_FD")}, + }, "Emis_unreg_w": { - "activity":{**_extension_type_2,**dict(sheet_name="Emis_unreg_w_act")}, - "final_demand":{**_extension_type_2,**dict(sheet_name="Emis_unreg_w_FD")}, - }, - + "activity": {**_extension_type_2, **dict(sheet_name="Emis_unreg_w_act")}, + "final_demand": {**_extension_type_2, **dict(sheet_name="Emis_unreg_w_FD")}, + }, "waste_sup": { - "activity":{**_extension_type_1,**dict(sheet_name="waste_sup_act")}, - "final_demand":{**_extension_type_1,**dict(sheet_name="waste_sup_FD")}, - }, - + "activity": {**_extension_type_1, **dict(sheet_name="waste_sup_act")}, + "final_demand": {**_extension_type_1, **dict(sheet_name="waste_sup_FD")}, + }, "waste_use": { - "activity":{**_extension_type_1,**dict(sheet_name="waste_use_act")}, - "final_demand":{**_extension_type_1,**dict(sheet_name="waste_use_FD")}, - }, - + "activity": {**_extension_type_1, **dict(sheet_name="waste_use_act")}, + "final_demand": {**_extension_type_1, **dict(sheet_name="waste_use_FD")}, + }, "pack_sup_waste": { - "activity":{**_extension_type_1,**dict(sheet_name="pack_sup_waste_act")}, - "final_demand":{**_extension_type_1,**dict(sheet_name="pack_sup_waste_fd")}, - }, - + "activity": {**_extension_type_1, **dict(sheet_name="pack_sup_waste_act")}, + "final_demand": {**_extension_type_1, **dict(sheet_name="pack_sup_waste_fd")}, + }, "pack_use_waste": { - "activity":{**_extension_type_1,**dict(sheet_name="pack_use_waste_act")}, - "final_demand":{**_extension_type_1,**dict(sheet_name="pack_use_waste_fd")}, - }, - + "activity": {**_extension_type_1, **dict(sheet_name="pack_use_waste_act")}, + "final_demand": {**_extension_type_1, **dict(sheet_name="pack_use_waste_fd")}, + }, "mach_sup_waste": { - "activity":{**_extension_type_1,**dict(sheet_name="mach_use_waste_act")}, - "final_demand":{**_extension_type_1,**dict(sheet_name="mach_use_waste_fd")}, - }, - + "activity": {**_extension_type_1, **dict(sheet_name="mach_use_waste_act")}, + "final_demand": {**_extension_type_1, **dict(sheet_name="mach_use_waste_fd")}, + }, "mach_use_waste": { - "activity":{**_extension_type_1,**dict(sheet_name="mach_use_waste_act")}, - "final_demand":{**_extension_type_1,**dict(sheet_name="mach_use_waste_fd")}, - }, - + "activity": {**_extension_type_1, **dict(sheet_name="mach_use_waste_act")}, + "final_demand": {**_extension_type_1, **dict(sheet_name="mach_use_waste_fd")}, + }, "stock_addition": { - "activity":{**_extension_type_1,**dict(sheet_name="stock_addition_act")}, - "final_demand":{**_extension_type_1,**dict(sheet_name="stock_addition_fd")}, - }, - + "activity": {**_extension_type_1, **dict(sheet_name="stock_addition_act")}, + "final_demand": {**_extension_type_1, **dict(sheet_name="stock_addition_fd")}, + }, "crop_res": { - "activity":{**_extension_type_1,**dict(sheet_name="crop_res_act")}, - "final_demand":{**_extension_type_1,**dict(sheet_name="crop_res_FD")}, - }, - - "Unreg_w":{ - "activity":{**_extension_type_1,**dict(sheet_name="Unreg_w_act")}, - "final_demand":{**_extension_type_1,**dict(sheet_name="Unreg_w_FD")}, + "activity": {**_extension_type_1, **dict(sheet_name="crop_res_act")}, + "final_demand": {**_extension_type_1, **dict(sheet_name="crop_res_FD")}, }, - } + "Unreg_w": { + "activity": {**_extension_type_1, **dict(sheet_name="Unreg_w_act")}, + "final_demand": {**_extension_type_1, **dict(sheet_name="Unreg_w_FD")}, + }, +} # letters follow the _MASTER_INDEX definitions eurostat_id = { - "a": [ "Crop and animal production, hunting and related service activities", "Forestry and logging", @@ -324,119 +285,107 @@ "Activities of households as employers; undifferentiated goods- and services-producing activities of households for own use", "Activities of extraterritorial organisations and bodies", ], - "c": [ -"Products of agriculture, hunting and related services", -"Products of forestry, logging and related services", -"Fish and other fishing products; aquaculture products; support services to fishing", -"Mining and quarrying", -"Food, beverages and tobacco products", -"Textiles, wearing apparel, leather and related products", -"Wood and of products of wood and cork, except furniture; articles of straw and plaiting materials", -"Paper and paper products", -"Printing and recording services", -"Coke and refined petroleum products", -"Chemicals and chemical products", -"Basic pharmaceutical products and pharmaceutical preparations", -"Rubber and plastic products", -"Other non-metallic mineral products", -"Basic metals", -"Fabricated metal products, except machinery and equipment", -"Computer, electronic and optical products", -"Electrical equipment", -"Machinery and equipment n.e.c.", -"Motor vehicles, trailers and semi-trailers", -"Other transport equipment", -"Furniture and other manufactured goods", -"Repair and installation services of machinery and equipment", -"Electricity, gas, steam and air conditioning", -"Natural water; water treatment and supply services", -"Sewerage services; sewage sludge; waste collection, treatment and disposal services; materials recovery services; remediation services and other waste management services", -"Constructions and construction works", -"Wholesale and retail trade and repair services of motor vehicles and motorcycles", -"Wholesale trade services, except of motor vehicles and motorcycles", -"Retail trade services, except of motor vehicles and motorcycles", -"Land transport services and transport services via pipelines", -"Water transport services", -"Air transport services", -"Warehousing and support services for transportation", -"Postal and courier services", -"Accommodation and food services", -"Publishing services", -"Motion picture, video and television programme production services, sound recording and music publishing; programming and broadcasting services", -"Telecommunications services", -"Computer programming, consultancy and related services; Information services", -"Financial services, except insurance and pension funding", -"Insurance, reinsurance and pension funding services, except compulsory social security", -"Services auxiliary to financial services and insurance services", -"Imputed rents of owner-occupied dwellings", -"Real estate services excluding imputed rents", -"Legal and accounting services; services of head offices; management consultancy services", -"Architectural and engineering services; technical testing and analysis services", -"Scientific research and development services", -"Advertising and market research services", -"Other professional, scientific and technical services and veterinary services", -"Rental and leasing services", -"Employment services", -"Travel agency, tour operator and other reservation services and related services", -"Security and investigation services; services to buildings and landscape; office administrative, office support and other business support services", -"Human health services", -"Residential care services; social work services without accommodation", -"Creative, arts, entertainment, library, archive, museum, other cultural services; gambling and betting services", -"Sporting services and amusement and recreation services", -"Services furnished by membership organisations", -"Repair services of computers and personal and household goods", -"Other personal services", -"Services of households as employers; undifferentiated goods and services produced by households for own use", -"Services provided by extraterritorial organisations and bodies", -"Public administration and defence services; compulsory social security services", -"Education services", -], - -"n": [ -"Final consumption expenditure by households", -"Final consumption expenditure by government", -"Final consumption expenditure by non-profit organisations serving households (NPISH)", -"Final consumption expediture", -"Gross fixed capital formation", -"Acquisitions less disposals of valuables", -"Changes in inventories", -"Changes in inventories and acquisition less disposals of valuables", -"Gross Capital formation", -"Exports to EU members states", -"Exports to non-member of the EU", -"Exports to members of the euro area", -"Exports to non-members of the euro area", -"Exports of goods and services", -], - -"f": [ -"Compensation of employees", -"Wages and salaries", -"Other taxes less other subsidies on production", -"Consumption of fixed capital", -"Operating surplus and mixed income, net", -], -"c_import":[ -"Imports from EU member states", -"Imports from non-members of the EU", -"Imports from members of the euro area", -"Imports from non-members of the euro area", -"Imports of goods and services", -"Total supply at basic prices", -"Taxes less subsidies on products", -"Trade and transport margins", -], - -"use": {"index_col":0,"header":11,"sheet_name":"Sheet 1"}, -"supply": {"index_col":0,"header":10,"sheet_name":"Sheet 1"}, -"meta_info": { - "year": (7,2,int), - "country": (6,2,str), - "table": (0,1,str) -} - + "Products of agriculture, hunting and related services", + "Products of forestry, logging and related services", + "Fish and other fishing products; aquaculture products; support services to fishing", + "Mining and quarrying", + "Food, beverages and tobacco products", + "Textiles, wearing apparel, leather and related products", + "Wood and of products of wood and cork, except furniture; articles of straw and plaiting materials", + "Paper and paper products", + "Printing and recording services", + "Coke and refined petroleum products", + "Chemicals and chemical products", + "Basic pharmaceutical products and pharmaceutical preparations", + "Rubber and plastic products", + "Other non-metallic mineral products", + "Basic metals", + "Fabricated metal products, except machinery and equipment", + "Computer, electronic and optical products", + "Electrical equipment", + "Machinery and equipment n.e.c.", + "Motor vehicles, trailers and semi-trailers", + "Other transport equipment", + "Furniture and other manufactured goods", + "Repair and installation services of machinery and equipment", + "Electricity, gas, steam and air conditioning", + "Natural water; water treatment and supply services", + "Sewerage services; sewage sludge; waste collection, treatment and disposal services; materials recovery services; remediation services and other waste management services", + "Constructions and construction works", + "Wholesale and retail trade and repair services of motor vehicles and motorcycles", + "Wholesale trade services, except of motor vehicles and motorcycles", + "Retail trade services, except of motor vehicles and motorcycles", + "Land transport services and transport services via pipelines", + "Water transport services", + "Air transport services", + "Warehousing and support services for transportation", + "Postal and courier services", + "Accommodation and food services", + "Publishing services", + "Motion picture, video and television programme production services, sound recording and music publishing; programming and broadcasting services", + "Telecommunications services", + "Computer programming, consultancy and related services; Information services", + "Financial services, except insurance and pension funding", + "Insurance, reinsurance and pension funding services, except compulsory social security", + "Services auxiliary to financial services and insurance services", + "Imputed rents of owner-occupied dwellings", + "Real estate services excluding imputed rents", + "Legal and accounting services; services of head offices; management consultancy services", + "Architectural and engineering services; technical testing and analysis services", + "Scientific research and development services", + "Advertising and market research services", + "Other professional, scientific and technical services and veterinary services", + "Rental and leasing services", + "Employment services", + "Travel agency, tour operator and other reservation services and related services", + "Security and investigation services; services to buildings and landscape; office administrative, office support and other business support services", + "Human health services", + "Residential care services; social work services without accommodation", + "Creative, arts, entertainment, library, archive, museum, other cultural services; gambling and betting services", + "Sporting services and amusement and recreation services", + "Services furnished by membership organisations", + "Repair services of computers and personal and household goods", + "Other personal services", + "Services of households as employers; undifferentiated goods and services produced by households for own use", + "Services provided by extraterritorial organisations and bodies", + "Public administration and defence services; compulsory social security services", + "Education services", + ], + "n": [ + "Final consumption expenditure by households", + "Final consumption expenditure by government", + "Final consumption expenditure by non-profit organisations serving households (NPISH)", + "Final consumption expediture", + "Gross fixed capital formation", + "Acquisitions less disposals of valuables", + "Changes in inventories", + "Changes in inventories and acquisition less disposals of valuables", + "Gross Capital formation", + "Exports to EU members states", + "Exports to non-member of the EU", + "Exports to members of the euro area", + "Exports to non-members of the euro area", + "Exports of goods and services", + ], + "f": [ + "Compensation of employees", + "Wages and salaries", + "Other taxes less other subsidies on production", + "Consumption of fixed capital", + "Operating surplus and mixed income, net", + ], + "c_import": [ + "Imports from EU member states", + "Imports from non-members of the EU", + "Imports from members of the euro area", + "Imports from non-members of the euro area", + "Imports of goods and services", + "Total supply at basic prices", + "Taxes less subsidies on products", + "Trade and transport margins", + ], + "use": {"index_col": 0, "header": 11, "sheet_name": "Sheet 1"}, + "supply": {"index_col": 0, "header": 10, "sheet_name": "Sheet 1"}, + "meta_info": {"year": (7, 2, int), "country": (6, 2, str), "table": (0, 1, str)}, } - - - diff --git a/mario/tools/parsersclass.py b/mario/tools/parsersclass.py index 24e237d..09ef235 100644 --- a/mario/tools/parsersclass.py +++ b/mario/tools/parsersclass.py @@ -13,7 +13,7 @@ eora_multi_region, eurostat_sut, parse_pymrio, - hybrid_sut_exiobase_reader + hybrid_sut_exiobase_reader, ) from mario.log_exc.exceptions import WrongInput, LackOfInput @@ -328,8 +328,10 @@ def parse_eora( "For multi region Eora, the year and indeces path should be defined" ) - if table == 'SUT': - raise NotImplemented("No handling of multiregional SUT from EORA is implemented yet") + if table == "SUT": + raise NotImplemented( + "No handling of multiregional SUT from EORA is implemented yet" + ) matrices, indeces, units = eora_multi_region( data_path=path, index_path=indeces, year=year, price="bp" @@ -343,11 +345,12 @@ def parse_eora( else: matrices, indeces, units = eora_single_region( - path=path, table=table, name_convention=name_convention, aggregate_trade=aggregate_trade + path=path, + table=table, + name_convention=name_convention, + aggregate_trade=aggregate_trade, ) - - return models[model]( name=name, table=table, @@ -358,15 +361,9 @@ def parse_eora( **kwargs, ) + def parse_exiobase( - table, - unit, - path, - model = "Database", - name = None, - year = None, - calc_all = False, - **kwargs + table, unit, path, model="Database", name=None, year=None, calc_all=False, **kwargs ): """A unique function for parsing all exiobase databases @@ -397,27 +394,27 @@ def parse_exiobase( WrongInput if non-valid values are passed to the arguments. """ - - if table not in ["IOT","SUT"]: + + if table not in ["IOT", "SUT"]: raise WrongInput("table only accpets 'IOT' or 'SUT'.") - - if unit not in ["Hybrid","Monetary"]: + + if unit not in ["Hybrid", "Monetary"]: raise WrongInput("unit only accpets 'Hybrid' or 'Monetary.'") - + if table == "IOT": if unit == "Monetary": parser = parse_exiobase_3 - else: + else: raise WrongInput("Hybrid IOT exiobase is not supported by mario.") - + else: if unit == "Monetary": parser = parse_exiobase_sut else: parser = hybrid_sut_exiobase - + kwargs["path"] = path kwargs["model"] = model kwargs["name"] = name @@ -425,16 +422,10 @@ def parse_exiobase( kwargs["year"] = year return parser(**kwargs) - - - + + def hybrid_sut_exiobase( - path, - extensions = [], - model = "Database", - name=None, - calc_all=False, - **kwargs + path, extensions=[], model="Database", name=None, calc_all=False, **kwargs ): """reads hybrid supply and use exiobase @@ -466,14 +457,15 @@ def hybrid_sut_exiobase( For more informatio refer to https://zenodo.org/record/7244919#.Y6hEfi8w2L1 """ if model not in models: - raise WrongInput("Available models are {}".format([*models])) + raise WrongInput("Available models are {}".format([*models])) - matrices,indeces,units = hybrid_sut_exiobase_reader( - path = path, - extensions = extensions, + matrices, indeces, units = hybrid_sut_exiobase_reader( + path=path, extensions=extensions, ) - notes = ["The name of extensions are changed to avoid confusion of same satellite account category for different extensions. For example 'Food' in 'pack_use_waste_act' is changed to 'Food (pack_use_waste)' to avoid confusion with 'Food' in 'pack_sup_waste'"] + notes = [ + "The name of extensions are changed to avoid confusion of same satellite account category for different extensions. For example 'Food' in 'pack_use_waste_act' is changed to 'Food (pack_use_waste)' to avoid confusion with 'Food' in 'pack_sup_waste'" + ] return models[model]( name=name, @@ -482,17 +474,13 @@ def hybrid_sut_exiobase( year=2011, init_by_parsers={"matrices": matrices, "_indeces": indeces, "units": units}, calc_all=calc_all, - notes = notes, + notes=notes, **kwargs, ) + def parse_eurostat_sut( - supply_path, - use_path, - model="Database", - name=None, - calc_all=False, - **kwargs, + supply_path, use_path, model="Database", name=None, calc_all=False, **kwargs, ) -> object: """Parsing Eurostat databases @@ -526,11 +514,7 @@ def parse_eurostat_sut( if model not in models: raise WrongInput("Available models are {}".format([*models])) - - matrices, indeces, units,meta = eurostat_sut( - supply_path, - use_path, - ) + matrices, indeces, units, meta = eurostat_sut(supply_path, use_path,) return models[model]( name=name, @@ -543,12 +527,7 @@ def parse_eurostat_sut( ) -def parse_from_pymrio( - io, - value_added, - satellite_account, - include_meta=True - ): +def parse_from_pymrio(io, value_added, satellite_account, include_meta=True): """Parsing a pymrio database Parameters @@ -571,8 +550,6 @@ def parse_from_pymrio( mario.Database """ - - matrices, units, indeces = parse_pymrio(io, value_added, satellite_account) notes = [ @@ -587,4 +564,3 @@ def parse_from_pymrio( init_by_parsers={"matrices": matrices, "_indeces": indeces, "units": units}, notes=notes, ) - diff --git a/mario/tools/plots.py b/mario/tools/plots.py index ff536c9..291740a 100644 --- a/mario/tools/plots.py +++ b/mario/tools/plots.py @@ -8,9 +8,7 @@ from mario.tools.plots_manager import _PLOTS_LAYOUT, Color, _PALETTES -from mario.tools.utilities import ( - run_from_jupyter, -) +from mario.tools.utilities import run_from_jupyter import plotly.graph_objects as go import plotly.express as px @@ -60,7 +58,7 @@ def _plot_linkages( path: str, multi_mode: bool, plot: str, - annotations = True, + annotations=True, auto_open: bool = False, **config, ): @@ -158,8 +156,7 @@ def _plot_linkages( legends.add(region) # Show in hover text for each trace the region, value - fig.update_traces( - hovertemplate="%{y}
%{x:.4f}") + fig.update_traces(hovertemplate="%{y}
%{x:.4f}") # geo_type * links_type * regions counter.append(2 * 2 * len(data.index.unique(level=0))) @@ -283,10 +280,7 @@ def _set_layout(fig, layout, mode, counter, iterator, prefix, x, y): ) ) - modifications = dict( - active=0, - pad={"t": 50}, - ) + modifications = dict(active=0, pad={"t": 50},) if mode == "sliders": modifications["steps"] = steps @@ -308,80 +302,99 @@ def _set_layout(fig, layout, mode, counter, iterator, prefix, x, y): def _plotX( - instance, - matrix, - x, - y, - color, - facet_row, - facet_col, - animation_frame, - base_scenario, - path, - item_from, - chart, - mode, - auto_open, - layout, - shared_yaxes, - shared_xaxes, - filters, + instance, + matrix, + x, + y, + color, + facet_row, + facet_col, + animation_frame, + base_scenario, + path, + item_from, + chart, + mode, + auto_open, + layout, + shared_yaxes, + shared_xaxes, + filters, ): - + # Extracting raw data scenarios = instance.scenarios if base_scenario != None: - scenarios.remove(base_scenario) - to_plot = instance.get_data(matrices=matrix, format="dict", scenarios=scenarios, base_scenario=base_scenario) - + scenarios.remove(base_scenario) + to_plot = instance.get_data( + matrices=matrix, format="dict", scenarios=scenarios, base_scenario=base_scenario + ) + # Processing raw data for plottinh data = pd.DataFrame() - for scenario in scenarios: + for scenario in scenarios: to_append = to_plot[scenario][matrix] to_append.columns = [scenario] to_append = to_append.stack(level=0).to_frame() - to_append.columns = [f'Value_{scenario}'] + to_append.columns = [f"Value_{scenario}"] data = pd.concat([data, to_append], axis=1) - data.fillna(0,inplace=True) + data.fillna(0, inplace=True) data = data.sum(1).to_frame() - data.columns = ['Value'] - + data.columns = ["Value"] + # Slicing according to filters - data.index.names = [f"{_MASTER_INDEX['r']}_from", 'Level_from', "Item_from", 'Scenario'] - if instance.table_type == 'IOT': - data = data.loc[(filters[f"filter_{_MASTER_INDEX['r']}_from"], - slice(None), - filters[f"filter_{_MASTER_INDEX['s']}_from"], - slice(None)), - :] - elif instance.table_type == 'SUT': - data1 = data.loc[(filters[f"filter_{_MASTER_INDEX['r']}_from"], - _MASTER_INDEX['a'], - filters[f"filter_{_MASTER_INDEX['a']}_from"], - slice(None)), - :] - data2 = data.loc[(filters[f"filter_{_MASTER_INDEX['r']}_from"], - _MASTER_INDEX['c'], - filters[f"filter_{_MASTER_INDEX['c']}_from"], - slice(None)), - :] + data.index.names = [ + f"{_MASTER_INDEX['r']}_from", + "Level_from", + "Item_from", + "Scenario", + ] + if instance.table_type == "IOT": + data = data.loc[ + ( + filters[f"filter_{_MASTER_INDEX['r']}_from"], + slice(None), + filters[f"filter_{_MASTER_INDEX['s']}_from"], + slice(None), + ), + :, + ] + elif instance.table_type == "SUT": + data1 = data.loc[ + ( + filters[f"filter_{_MASTER_INDEX['r']}_from"], + _MASTER_INDEX["a"], + filters[f"filter_{_MASTER_INDEX['a']}_from"], + slice(None), + ), + :, + ] + data2 = data.loc[ + ( + filters[f"filter_{_MASTER_INDEX['r']}_from"], + _MASTER_INDEX["c"], + filters[f"filter_{_MASTER_INDEX['c']}_from"], + slice(None), + ), + :, + ] data = data1.append(data2) data.reset_index(inplace=True) - data = data[data["Level_from"]==item_from] + data = data[data["Level_from"] == item_from] cols = [] for col in data.columns: if col == "Item_from": - if instance.table_type == 'SUT': - if item_from == _MASTER_INDEX['c']: + if instance.table_type == "SUT": + if item_from == _MASTER_INDEX["c"]: cols += [f"{_MASTER_INDEX['c']}_from"] else: cols += [f"{_MASTER_INDEX['a']}_from"] else: - cols += [f"{_MASTER_INDEX['s']}_from"] + cols += [f"{_MASTER_INDEX['s']}_from"] elif col == "Item_to": - if instance.table_type == 'SUT': - if item_from == _MASTER_INDEX['c']: + if instance.table_type == "SUT": + if item_from == _MASTER_INDEX["c"]: cols += [f"{_MASTER_INDEX['a']}_to"] else: cols += [f"{_MASTER_INDEX['c']}_to"] @@ -391,200 +404,248 @@ def _plotX( cols += [col] data.columns = cols - # Other input management if animation_frame.capitalize() not in data.columns: - raise WrongInput(f"'{animation_frame}' not a valid option for 'animation_frame'. Valid options are: {data.columns}") - - plot_parameters_to_cap = {'x': x, - 'y': y, - 'color': color, - 'facet_row': facet_row, - 'facet_col': facet_col, - 'animation_frame': animation_frame, - } - plot_parameters_to_low = {'chart': chart, - 'mode': mode, - } - - for param,given in plot_parameters_to_cap.items(): + raise WrongInput( + f"'{animation_frame}' not a valid option for 'animation_frame'. Valid options are: {data.columns}" + ) + + plot_parameters_to_cap = { + "x": x, + "y": y, + "color": color, + "facet_row": facet_row, + "facet_col": facet_col, + "animation_frame": animation_frame, + } + plot_parameters_to_low = { + "chart": chart, + "mode": mode, + } + + for param, given in plot_parameters_to_cap.items(): if given != None: plot_parameters_to_cap[param] = given.capitalize() - for param,given in plot_parameters_to_low.items(): + for param, given in plot_parameters_to_low.items(): if given != None: plot_parameters_to_low[param] = given.lower() - + plot_parameters = plot_parameters_to_cap.copy() plot_parameters.update(plot_parameters_to_low) # for param,given in plot_parameters.items(): - # if given != None: - # if _MASTER_INDEX['s'] in given: - # plot_parameters[param] = given.replace(_MASTER_INDEX['s'],'Item') - # if _MASTER_INDEX['a'] in given: - # plot_parameters[param] = given.replace(_MASTER_INDEX['a'],'Item') - # if _MASTER_INDEX['c'] in given: - # plot_parameters[param] = given.replace(_MASTER_INDEX['c'],'Item') - - - for key,value in plot_parameters.items(): + # if given != None: + # if _MASTER_INDEX['s'] in given: + # plot_parameters[param] = given.replace(_MASTER_INDEX['s'],'Item') + # if _MASTER_INDEX['a'] in given: + # plot_parameters[param] = given.replace(_MASTER_INDEX['a'],'Item') + # if _MASTER_INDEX['c'] in given: + # plot_parameters[param] = given.replace(_MASTER_INDEX['c'],'Item') + + for key, value in plot_parameters.items(): if value != None: - if value.split("_")[-1] == 'from': - indices = _INDECES[instance.table_type][matrix]['indices'] + if value.split("_")[-1] == "from": + indices = _INDECES[instance.table_type][matrix]["indices"] elements = [] for i in indices: elements += [i] if value.split("_")[0] not in elements: - raise WrongInput(f"Matrix {matrix} does not accept '{value}' as a valid option for '{key}'. Please rearrange your inputs") - - if value.split("_")[-1] == 'to': - columns = _INDECES[instance.table_type][matrix]['columns'] + raise WrongInput( + f"Matrix {matrix} does not accept '{value}' as a valid option for '{key}'. Please rearrange your inputs" + ) + + if value.split("_")[-1] == "to": + columns = _INDECES[instance.table_type][matrix]["columns"] elements = [] for i in columns: elements += [i] if value.split("_")[0] not in elements: - raise WrongInput(f"Matrix {matrix} does not accept '{value}' as a valid option for '{key}'. Please rearrange your inputs") - + raise WrongInput( + f"Matrix {matrix} does not accept '{value}' as a valid option for '{key}'. Please rearrange your inputs" + ) # Plotting colors = Color() - colors.has_enough_colors(plot_parameters['color']) - if chart=='bar': - fig = px.bar(data, - x=plot_parameters['x'], - y=plot_parameters['y'], - color=plot_parameters['color'], - animation_frame=plot_parameters['animation_frame'], - facet_row = plot_parameters['facet_row'], - facet_col = plot_parameters['facet_col'], - barmode=plot_parameters['mode'], - color_discrete_sequence = colors, - ) + colors.has_enough_colors(plot_parameters["color"]) + if chart == "bar": + fig = px.bar( + data, + x=plot_parameters["x"], + y=plot_parameters["y"], + color=plot_parameters["color"], + animation_frame=plot_parameters["animation_frame"], + facet_row=plot_parameters["facet_row"], + facet_col=plot_parameters["facet_col"], + barmode=plot_parameters["mode"], + color_discrete_sequence=colors, + ) for key in layout: try: fig["layout"][key] = layout[key] except: pass - + _plotter(fig, path, auto_open=auto_open) - - + + def _plotZYUS( - instance, - matrix, - x, - y, - color, - facet_row, - facet_col, - animation_frame, - base_scenario, - path, - item_from, - chart, - mode, - auto_open, - layout, - shared_yaxes, - shared_xaxes, - filters, + instance, + matrix, + x, + y, + color, + facet_row, + facet_col, + animation_frame, + base_scenario, + path, + item_from, + chart, + mode, + auto_open, + layout, + shared_yaxes, + shared_xaxes, + filters, ): - + # Extracting raw data scenarios = instance.scenarios if base_scenario != None: - scenarios.remove(base_scenario) - to_plot = instance.get_data(matrices=matrix, format="dict", scenarios=scenarios, base_scenario=base_scenario) - + scenarios.remove(base_scenario) + to_plot = instance.get_data( + matrices=matrix, format="dict", scenarios=scenarios, base_scenario=base_scenario + ) + # Processing raw data for plottinh data = pd.DataFrame() - for scenario in scenarios: + for scenario in scenarios: to_append = to_plot[scenario][matrix] - to_append = to_append.stack(level=[0,1,2]).to_frame() + to_append = to_append.stack(level=[0, 1, 2]).to_frame() to_append.columns = [scenario] - to_append = to_append.stack(level=[0]).to_frame() - to_append.columns = [f'Value_{scenario}'] + to_append = to_append.stack(level=[0]).to_frame() + to_append.columns = [f"Value_{scenario}"] data = pd.concat([data, to_append], axis=1) - data.fillna(0,inplace=True) + data.fillna(0, inplace=True) data = data.sum(1).to_frame() - data.columns = ['Value'] - + data.columns = ["Value"] + # Slicing according to filters - if matrix in ['Z','z','U','u','S','s','f_dis']: - data.index.names = [f"{_MASTER_INDEX['r']}_from", 'Level_from', "Item_from", f"{_MASTER_INDEX['r']}_to", 'Level_to', "Item_to", 'Scenario'] - if instance.table_type == 'IOT': - data = data.loc[(filters[f"filter_{_MASTER_INDEX['r']}_from"], - slice(None), - filters[f"filter_{_MASTER_INDEX['s']}_from"], - filters[f"filter_{_MASTER_INDEX['r']}_to"], - slice(None), - filters[f"filter_{_MASTER_INDEX['s']}_to"], - slice(None)), - :] - elif instance.table_type == 'SUT': - if matrix == 'S' or matrix == 's': - data = data.loc[(filters[f"filter_{_MASTER_INDEX['r']}_from"], - _MASTER_INDEX['a'], - filters[f"filter_{_MASTER_INDEX['a']}_from"], - filters[f"filter_{_MASTER_INDEX['r']}_to"], - _MASTER_INDEX['c'], - filters[f"filter_{_MASTER_INDEX['c']}_to"], - slice(None)), - :] - if matrix == 'U' or matrix == 'u': - data = data.loc[(filters[f"filter_{_MASTER_INDEX['r']}_from"], - _MASTER_INDEX['c'], - filters[f"filter_{_MASTER_INDEX['c']}_from"], - filters[f"filter_{_MASTER_INDEX['r']}_to"], - _MASTER_INDEX['a'], - filters[f"filter_{_MASTER_INDEX['a']}_to"], - slice(None)), - :] - - elif matrix in ['Y']: - data.index.names = [f"{_MASTER_INDEX['r']}_from", 'Level_from', "Item_from", f"{_MASTER_INDEX['r']}_to", 'Level_to', f"{_MASTER_INDEX['n']}".replace(" ","_"), 'Scenario'] - if instance.table_type == 'IOT': - data = data.loc[(filters[f"filter_{_MASTER_INDEX['r']}_from"], - slice(None), - filters[f"filter_{_MASTER_INDEX['s']}_from"], - filters[f"filter_{_MASTER_INDEX['r']}_to"], - slice(None), - filters[f"filter_{_MASTER_INDEX['n']}".replace(" ","_")], - slice(None)), - :] - elif instance.table_type == 'SUT': - data1 = data.loc[(filters[f"filter_{_MASTER_INDEX['r']}_from"], - _MASTER_INDEX['a'], - filters[f"filter_{_MASTER_INDEX['a']}_from"], - filters[f"filter_{_MASTER_INDEX['r']}_to"], - slice(None), - filters[f"filter_{_MASTER_INDEX['n']}".replace(" ","_")], - slice(None)), - :] - data2 = data.loc[(filters[f"filter_{_MASTER_INDEX['r']}_from"], - _MASTER_INDEX['c'], - filters[f"filter_{_MASTER_INDEX['c']}_from"], - filters[f"filter_{_MASTER_INDEX['r']}_to"], - slice(None), - filters[f"filter_{_MASTER_INDEX['n']}".replace(" ","_")], - slice(None)), - :] + if matrix in ["Z", "z", "U", "u", "S", "s", "f_dis"]: + data.index.names = [ + f"{_MASTER_INDEX['r']}_from", + "Level_from", + "Item_from", + f"{_MASTER_INDEX['r']}_to", + "Level_to", + "Item_to", + "Scenario", + ] + if instance.table_type == "IOT": + data = data.loc[ + ( + filters[f"filter_{_MASTER_INDEX['r']}_from"], + slice(None), + filters[f"filter_{_MASTER_INDEX['s']}_from"], + filters[f"filter_{_MASTER_INDEX['r']}_to"], + slice(None), + filters[f"filter_{_MASTER_INDEX['s']}_to"], + slice(None), + ), + :, + ] + elif instance.table_type == "SUT": + if matrix == "S" or matrix == "s": + data = data.loc[ + ( + filters[f"filter_{_MASTER_INDEX['r']}_from"], + _MASTER_INDEX["a"], + filters[f"filter_{_MASTER_INDEX['a']}_from"], + filters[f"filter_{_MASTER_INDEX['r']}_to"], + _MASTER_INDEX["c"], + filters[f"filter_{_MASTER_INDEX['c']}_to"], + slice(None), + ), + :, + ] + if matrix == "U" or matrix == "u": + data = data.loc[ + ( + filters[f"filter_{_MASTER_INDEX['r']}_from"], + _MASTER_INDEX["c"], + filters[f"filter_{_MASTER_INDEX['c']}_from"], + filters[f"filter_{_MASTER_INDEX['r']}_to"], + _MASTER_INDEX["a"], + filters[f"filter_{_MASTER_INDEX['a']}_to"], + slice(None), + ), + :, + ] + + elif matrix in ["Y"]: + data.index.names = [ + f"{_MASTER_INDEX['r']}_from", + "Level_from", + "Item_from", + f"{_MASTER_INDEX['r']}_to", + "Level_to", + f"{_MASTER_INDEX['n']}".replace(" ", "_"), + "Scenario", + ] + if instance.table_type == "IOT": + data = data.loc[ + ( + filters[f"filter_{_MASTER_INDEX['r']}_from"], + slice(None), + filters[f"filter_{_MASTER_INDEX['s']}_from"], + filters[f"filter_{_MASTER_INDEX['r']}_to"], + slice(None), + filters[f"filter_{_MASTER_INDEX['n']}".replace(" ", "_")], + slice(None), + ), + :, + ] + elif instance.table_type == "SUT": + data1 = data.loc[ + ( + filters[f"filter_{_MASTER_INDEX['r']}_from"], + _MASTER_INDEX["a"], + filters[f"filter_{_MASTER_INDEX['a']}_from"], + filters[f"filter_{_MASTER_INDEX['r']}_to"], + slice(None), + filters[f"filter_{_MASTER_INDEX['n']}".replace(" ", "_")], + slice(None), + ), + :, + ] + data2 = data.loc[ + ( + filters[f"filter_{_MASTER_INDEX['r']}_from"], + _MASTER_INDEX["c"], + filters[f"filter_{_MASTER_INDEX['c']}_from"], + filters[f"filter_{_MASTER_INDEX['r']}_to"], + slice(None), + filters[f"filter_{_MASTER_INDEX['n']}".replace(" ", "_")], + slice(None), + ), + :, + ] data = data1.append(data2) data.reset_index(inplace=True) - data = data[data["Level_from"]==item_from] + data = data[data["Level_from"] == item_from] cols = [] for col in data.columns: if col == "Item_from": - if instance.table_type == 'SUT': - if item_from == _MASTER_INDEX['c']: + if instance.table_type == "SUT": + if item_from == _MASTER_INDEX["c"]: cols += [f"{_MASTER_INDEX['c']}_from"] else: cols += [f"{_MASTER_INDEX['a']}_from"] else: - cols += [f"{_MASTER_INDEX['s']}_from"] + cols += [f"{_MASTER_INDEX['s']}_from"] elif col == "Item_to": - if instance.table_type == 'SUT': - if item_from == _MASTER_INDEX['c']: + if instance.table_type == "SUT": + if item_from == _MASTER_INDEX["c"]: cols += [f"{_MASTER_INDEX['a']}_to"] else: cols += [f"{_MASTER_INDEX['c']}_to"] @@ -594,181 +655,239 @@ def _plotZYUS( cols += [col] data.columns = cols - # Other input management if animation_frame.capitalize() not in data.columns: - raise WrongInput(f"'{animation_frame}' not a valid option for 'animation_frame'. Valid options are: {data.columns}") - - plot_parameters_to_cap = {'x': x, - 'y': y, - 'color': color, - 'facet_row': facet_row, - 'facet_col': facet_col, - 'animation_frame': animation_frame, - } - plot_parameters_to_low = {'chart': chart, - 'mode': mode, - } - - for param,given in plot_parameters_to_cap.items(): + raise WrongInput( + f"'{animation_frame}' not a valid option for 'animation_frame'. Valid options are: {data.columns}" + ) + + plot_parameters_to_cap = { + "x": x, + "y": y, + "color": color, + "facet_row": facet_row, + "facet_col": facet_col, + "animation_frame": animation_frame, + } + plot_parameters_to_low = { + "chart": chart, + "mode": mode, + } + + for param, given in plot_parameters_to_cap.items(): if given != None: plot_parameters_to_cap[param] = given.capitalize() - for param,given in plot_parameters_to_low.items(): + for param, given in plot_parameters_to_low.items(): if given != None: plot_parameters_to_low[param] = given.lower() - + plot_parameters = plot_parameters_to_cap.copy() plot_parameters.update(plot_parameters_to_low) - - for key,value in plot_parameters.items(): + + for key, value in plot_parameters.items(): if value != None: - if value.split("_")[-1] == 'from': - indices = _INDECES[instance.table_type][matrix]['indices'] + if value.split("_")[-1] == "from": + indices = _INDECES[instance.table_type][matrix]["indices"] elements = [] for i in indices: elements += [i] if value.split("_")[0] not in elements: - raise WrongInput(f"Matrix {matrix} does not accept '{value}' as a valid option for '{key}'. Please rearrange your inputs") - - if value.split("_")[-1] == 'to': - columns = _INDECES[instance.table_type][matrix]['columns'] + raise WrongInput( + f"Matrix {matrix} does not accept '{value}' as a valid option for '{key}'. Please rearrange your inputs" + ) + + if value.split("_")[-1] == "to": + columns = _INDECES[instance.table_type][matrix]["columns"] elements = [] for i in columns: elements += [i] if value.split("_")[0] not in elements: - raise WrongInput(f"Matrix {matrix} does not accept '{value}' as a valid option for '{key}'. Please rearrange your inputs") + raise WrongInput( + f"Matrix {matrix} does not accept '{value}' as a valid option for '{key}'. Please rearrange your inputs" + ) - # Plotting colors = Color() - colors.has_enough_colors(plot_parameters['color']) - if chart=='bar': - fig = px.bar(data, - x=plot_parameters['x'], - y=plot_parameters['y'], - color=plot_parameters['color'], - animation_frame=plot_parameters['animation_frame'], - facet_row = plot_parameters['facet_row'], - facet_col = plot_parameters['facet_col'], - barmode=plot_parameters['mode'], - color_discrete_sequence = colors - ) + colors.has_enough_colors(plot_parameters["color"]) + if chart == "bar": + fig = px.bar( + data, + x=plot_parameters["x"], + y=plot_parameters["y"], + color=plot_parameters["color"], + animation_frame=plot_parameters["animation_frame"], + facet_row=plot_parameters["facet_row"], + facet_col=plot_parameters["facet_col"], + barmode=plot_parameters["mode"], + color_discrete_sequence=colors, + ) for key in layout: try: fig["layout"][key] = layout[key] except: pass - + _plotter(fig, path, auto_open=auto_open) - - + def _plotVEMF( - instance, - matrix, - x, - y, - color, - facet_row, - facet_col, - animation_frame, - base_scenario, - path, - item_from, - chart, - mode, - auto_open, - layout, - shared_yaxes, - shared_xaxes, - filters, + instance, + matrix, + x, + y, + color, + facet_row, + facet_col, + animation_frame, + base_scenario, + path, + item_from, + chart, + mode, + auto_open, + layout, + shared_yaxes, + shared_xaxes, + filters, ): - + # Extracting raw data scenarios = instance.scenarios if base_scenario != None: - scenarios.remove(base_scenario) - to_plot = instance.get_data(matrices=matrix, format="dict", scenarios=scenarios, base_scenario=base_scenario) - + scenarios.remove(base_scenario) + to_plot = instance.get_data( + matrices=matrix, format="dict", scenarios=scenarios, base_scenario=base_scenario + ) + # Processing raw data for plottinh data = pd.DataFrame() - for scenario in scenarios: + for scenario in scenarios: to_append = to_plot[scenario][matrix] - to_append = to_append.stack(level=[0,1,2]).to_frame() + to_append = to_append.stack(level=[0, 1, 2]).to_frame() to_append.columns = [scenario] - to_append = to_append.stack(level=[0]).to_frame() - to_append.columns = [f'Value_{scenario}'] + to_append = to_append.stack(level=[0]).to_frame() + to_append.columns = [f"Value_{scenario}"] data = pd.concat([data, to_append], axis=1) - data.fillna(0,inplace=True) + data.fillna(0, inplace=True) data = data.sum(1).to_frame() - data.columns = ['Value'] - + data.columns = ["Value"] + # Slicing according to filters - if matrix in ['V','v','M']: - data.index.names = [f"{_MASTER_INDEX['f']}", f"{_MASTER_INDEX['r']}_to", 'Level_to', "Item_to", 'Scenario'] - if instance.table_type == 'IOT': - data = data.loc[(filters[f"filter_{_MASTER_INDEX['f']}".replace(" ","_")], - filters[f"filter_{_MASTER_INDEX['r']}_to"], - slice(None), - filters[f"filter_{_MASTER_INDEX['s']}_to"], - slice(None)), - :] - elif instance.table_type == 'SUT': - data = data.loc[(filters[f"filter_{_MASTER_INDEX['f']}".replace(" ","_")], - filters[f"filter_{_MASTER_INDEX['r']}_to"], - slice(None), - filters[f"filter_{_MASTER_INDEX['a']}_to"]+filters[f"filter_{_MASTER_INDEX['c']}_to"], - slice(None)), - :] - elif matrix in ['E','e','F']: - data.index.names = [f"{_MASTER_INDEX['k']}", f"{_MASTER_INDEX['r']}_to", 'Level_to', "Item_to", 'Scenario'] - if instance.table_type == 'IOT': - data = data.loc[(filters[f"filter_{_MASTER_INDEX['k']}".replace(" ","_")], - filters[f"filter_{_MASTER_INDEX['r']}_to"], - slice(None), - filters[f"filter_{_MASTER_INDEX['s']}_to"], - slice(None)), - :] - elif instance.table_type == 'SUT': - data = data.loc[(filters[f"filter_{_MASTER_INDEX['k']}".replace(" ","_")], - filters[f"filter_{_MASTER_INDEX['r']}_to"], - slice(None), - filters[f"filter_{_MASTER_INDEX['a']}_to"]+filters[f"filter_{_MASTER_INDEX['c']}_to"], - slice(None)), - :] - elif matrix in ['EY']: - data.index.names = [f"{_MASTER_INDEX['k']}", f"{_MASTER_INDEX['r']}_to", 'Level_to', f"{_MASTER_INDEX['n']}".replace(" ","_"), 'Scenario'] - data = data.loc[(filters[f"filter_{_MASTER_INDEX['k']}".replace(" ","_")], - filters[f"filter_{_MASTER_INDEX['r']}_to"], - slice(None), - filters[f"filter_{_MASTER_INDEX['n']}_to"], - slice(None)), - :] + if matrix in ["V", "v", "M"]: + data.index.names = [ + f"{_MASTER_INDEX['f']}", + f"{_MASTER_INDEX['r']}_to", + "Level_to", + "Item_to", + "Scenario", + ] + if instance.table_type == "IOT": + data = data.loc[ + ( + filters[f"filter_{_MASTER_INDEX['f']}".replace(" ", "_")], + filters[f"filter_{_MASTER_INDEX['r']}_to"], + slice(None), + filters[f"filter_{_MASTER_INDEX['s']}_to"], + slice(None), + ), + :, + ] + elif instance.table_type == "SUT": + data = data.loc[ + ( + filters[f"filter_{_MASTER_INDEX['f']}".replace(" ", "_")], + filters[f"filter_{_MASTER_INDEX['r']}_to"], + slice(None), + filters[f"filter_{_MASTER_INDEX['a']}_to"] + + filters[f"filter_{_MASTER_INDEX['c']}_to"], + slice(None), + ), + :, + ] + elif matrix in ["E", "e", "F"]: + data.index.names = [ + f"{_MASTER_INDEX['k']}", + f"{_MASTER_INDEX['r']}_to", + "Level_to", + "Item_to", + "Scenario", + ] + if instance.table_type == "IOT": + data = data.loc[ + ( + filters[f"filter_{_MASTER_INDEX['k']}".replace(" ", "_")], + filters[f"filter_{_MASTER_INDEX['r']}_to"], + slice(None), + filters[f"filter_{_MASTER_INDEX['s']}_to"], + slice(None), + ), + :, + ] + elif instance.table_type == "SUT": + data = data.loc[ + ( + filters[f"filter_{_MASTER_INDEX['k']}".replace(" ", "_")], + filters[f"filter_{_MASTER_INDEX['r']}_to"], + slice(None), + filters[f"filter_{_MASTER_INDEX['a']}_to"] + + filters[f"filter_{_MASTER_INDEX['c']}_to"], + slice(None), + ), + :, + ] + elif matrix in ["EY"]: + data.index.names = [ + f"{_MASTER_INDEX['k']}", + f"{_MASTER_INDEX['r']}_to", + "Level_to", + f"{_MASTER_INDEX['n']}".replace(" ", "_"), + "Scenario", + ] + data = data.loc[ + ( + filters[f"filter_{_MASTER_INDEX['k']}".replace(" ", "_")], + filters[f"filter_{_MASTER_INDEX['r']}_to"], + slice(None), + filters[f"filter_{_MASTER_INDEX['n']}_to"], + slice(None), + ), + :, + ] data.reset_index(inplace=True) - + item_units = [] - if len(set(to_plot[list(to_plot.keys())[0]]['units'][list(data.columns)[0]].T.values[0])) > 1: + if ( + len( + set( + to_plot[list(to_plot.keys())[0]]["units"][ + list(data.columns)[0] + ].T.values[0] + ) + ) + > 1 + ): for i in range(data.shape[0]): - s = data.iloc[i,list(data.columns).index("Scenario")] - item_units += [f"{data.iloc[i,0]} [{to_plot[s]['units'][list(data.columns)[0]].loc[data.iloc[i,0],'unit']}]"] + s = data.iloc[i, list(data.columns).index("Scenario")] + item_units += [ + f"{data.iloc[i,0]} [{to_plot[s]['units'][list(data.columns)[0]].loc[data.iloc[i,0],'unit']}]" + ] data[list(data.columns)[0]] = item_units - - + # data = data[data["Level_from"]==item_from] cols = [] for col in data.columns: if col == "Item_from": - if instance.table_type == 'SUT': - if item_from == _MASTER_INDEX['c']: + if instance.table_type == "SUT": + if item_from == _MASTER_INDEX["c"]: cols += [f"{_MASTER_INDEX['c']}_from"] else: cols += [f"{_MASTER_INDEX['a']}_from"] else: - cols += [f"{_MASTER_INDEX['s']}_from"] + cols += [f"{_MASTER_INDEX['s']}_from"] elif col == "Item_to": - if instance.table_type == 'SUT': - if item_from == _MASTER_INDEX['c']: + if instance.table_type == "SUT": + if item_from == _MASTER_INDEX["c"]: cols += [f"{_MASTER_INDEX['c']}_to"] else: cols += [f"{_MASTER_INDEX['a']}_to"] @@ -778,73 +897,77 @@ def _plotVEMF( cols += [col] data.columns = cols - # Other input management if animation_frame.capitalize() not in data.columns: - raise WrongInput(f"'{animation_frame}' not a valid option for 'animation_frame'. Valid options are: {data.columns}") - - plot_parameters_to_cap = {'x': x, - 'y': y, - 'color': color, - 'facet_row': facet_row, - 'facet_col': facet_col, - 'animation_frame': animation_frame, - } - plot_parameters_to_low = {'chart': chart, - 'mode': mode, - } - - for param,given in plot_parameters_to_cap.items(): + raise WrongInput( + f"'{animation_frame}' not a valid option for 'animation_frame'. Valid options are: {data.columns}" + ) + + plot_parameters_to_cap = { + "x": x, + "y": y, + "color": color, + "facet_row": facet_row, + "facet_col": facet_col, + "animation_frame": animation_frame, + } + plot_parameters_to_low = { + "chart": chart, + "mode": mode, + } + + for param, given in plot_parameters_to_cap.items(): if given != None: plot_parameters_to_cap[param] = given.capitalize() - for param,given in plot_parameters_to_low.items(): + for param, given in plot_parameters_to_low.items(): if given != None: plot_parameters_to_low[param] = given.lower() - + plot_parameters = plot_parameters_to_cap.copy() plot_parameters.update(plot_parameters_to_low) - - for key,value in plot_parameters.items(): + for key, value in plot_parameters.items(): if value != None: - if value.split("_")[-1] == 'from': - indices = _INDECES[instance.table_type][matrix]['indices'] + if value.split("_")[-1] == "from": + indices = _INDECES[instance.table_type][matrix]["indices"] elements = [] for i in indices: elements += [i] if value.split("_")[0] not in elements: - raise WrongInput(f"Matrix {matrix} does not accept '{value}' as a valid option for '{key}'. Please rearrange your inputs") - - if value.split("_")[-1] == 'to': - columns = _INDECES[instance.table_type][matrix]['columns'] + raise WrongInput( + f"Matrix {matrix} does not accept '{value}' as a valid option for '{key}'. Please rearrange your inputs" + ) + + if value.split("_")[-1] == "to": + columns = _INDECES[instance.table_type][matrix]["columns"] elements = [] for i in columns: elements += [i] if value.split("_")[0] not in elements: - raise WrongInput(f"Matrix {matrix} does not accept '{value}' as a valid option for '{key}'. Please rearrange your inputs") - + raise WrongInput( + f"Matrix {matrix} does not accept '{value}' as a valid option for '{key}'. Please rearrange your inputs" + ) # Plotting colors = Color() - colors.has_enough_colors(plot_parameters['color']) - if chart=='bar': - fig = px.bar(data, - x=plot_parameters['x'], - y=plot_parameters['y'], - color=plot_parameters['color'], - animation_frame=plot_parameters['animation_frame'], - facet_row = plot_parameters['facet_row'], - facet_col = plot_parameters['facet_col'], - barmode=plot_parameters['mode'], - color_discrete_sequence = colors - ) + colors.has_enough_colors(plot_parameters["color"]) + if chart == "bar": + fig = px.bar( + data, + x=plot_parameters["x"], + y=plot_parameters["y"], + color=plot_parameters["color"], + animation_frame=plot_parameters["animation_frame"], + facet_row=plot_parameters["facet_row"], + facet_col=plot_parameters["facet_col"], + barmode=plot_parameters["mode"], + color_discrete_sequence=colors, + ) for key in layout: try: fig["layout"][key] = layout[key] except: pass - + _plotter(fig, path, auto_open=auto_open) - - \ No newline at end of file diff --git a/mario/tools/tableparser.py b/mario/tools/tableparser.py index 444b766..277b1fd 100644 --- a/mario/tools/tableparser.py +++ b/mario/tools/tableparser.py @@ -33,7 +33,7 @@ eora, eora_parser_id, hybrid_sut_exiobase_parser_id, - eurostat_id + eurostat_id, ) from mario.tools.iomath import ( @@ -674,6 +674,7 @@ def monetary_sut_exiobase(path): return matrices, indeces, units + def eora_single_region(path, table, name_convention="full_name", aggregate_trade=True): """ Eora single region parser @@ -683,10 +684,10 @@ def eora_single_region(path, table, name_convention="full_name", aggregate_trade data = pd.read_csv(path, sep="\t", index_col=[2, 0, 1, 3], header=[2, 0, 1, 3]) - if table == 'IOT': + if table == "IOT": Z_index = eora[_MASTER_INDEX["s"]] - elif table == 'SUT': + elif table == "SUT": Z_index = eora[_MASTER_INDEX["a"]] + eora[_MASTER_INDEX["c"]] Z = data.loc[Z_index, Z_index] @@ -705,12 +706,14 @@ def eora_single_region(path, table, name_convention="full_name", aggregate_trade E.index.get_level_values(3).tolist()[index] + " (" + value + ")" for index, value in enumerate(E.index.get_level_values(2).tolist()) ] - - if table == 'IOT': + + if table == "IOT": sectors = Z.index.get_level_values(-1).tolist() else: activities = Z.loc[eora[_MASTER_INDEX["a"]]].index.get_level_values(-1).tolist() - commodities = Z.loc[eora[_MASTER_INDEX["c"]]].index.get_level_values(-1).tolist() + commodities = ( + Z.loc[eora[_MASTER_INDEX["c"]]].index.get_level_values(-1).tolist() + ) if aggregate_trade: @@ -741,26 +744,30 @@ def eora_single_region(path, table, name_convention="full_name", aggregate_trade # Taking the units - if table == 'IOT': + if table == "IOT": sectors_unit = pd.DataFrame("USD", index=sectors, columns=["unit"]) Z_index = pd.MultiIndex.from_product([regions, [_MASTER_INDEX["s"]], sectors]) else: activities_unit = pd.DataFrame("USD", index=activities, columns=["unit"]) commodities_unit = pd.DataFrame("USD", index=commodities, columns=["unit"]) - Z_index_a = pd.MultiIndex.from_product([regions, [_MASTER_INDEX["a"]], activities]) - Z_index_c = pd.MultiIndex.from_product([regions, [_MASTER_INDEX["c"]], commodities]) + Z_index_a = pd.MultiIndex.from_product( + [regions, [_MASTER_INDEX["a"]], activities] + ) + Z_index_c = pd.MultiIndex.from_product( + [regions, [_MASTER_INDEX["c"]], commodities] + ) Z_index = Z_index_a.append(Z_index_c) - factor_unit = pd.DataFrame("USD", index=factors, columns=["unit"]) satellite_unit = pd.DataFrame( E.index.get_level_values(0).tolist(), index=satellite, columns=["unit"] ) - - Y_index = pd.MultiIndex.from_product([regions, [_MASTER_INDEX['n']], final_consumptions]) + Y_index = pd.MultiIndex.from_product( + [regions, [_MASTER_INDEX["n"]], final_consumptions] + ) indeces = { "Y": {"columns": Y_index, "index": Z_index}, @@ -781,37 +788,35 @@ def eora_single_region(path, table, name_convention="full_name", aggregate_trade rename_index(matrices["baseline"]) - - - if table =='IOT': + if table == "IOT": units = { _MASTER_INDEX["s"]: sectors_unit, _MASTER_INDEX["f"]: factor_unit, _MASTER_INDEX["k"]: satellite_unit, - } + } indeces = { - "r": {"main": regions}, - "n": {"main": final_consumptions}, - "k": {"main": satellite}, - "f": {"main": factors}, - "s": {"main": sectors}, - } + "r": {"main": regions}, + "n": {"main": final_consumptions}, + "k": {"main": satellite}, + "f": {"main": factors}, + "s": {"main": sectors}, + } else: - + units = { - _MASTER_INDEX["a"]: activities_unit, - _MASTER_INDEX["c"]: commodities_unit, - _MASTER_INDEX["f"]: factor_unit, - _MASTER_INDEX["k"]: satellite_unit, - } + _MASTER_INDEX["a"]: activities_unit, + _MASTER_INDEX["c"]: commodities_unit, + _MASTER_INDEX["f"]: factor_unit, + _MASTER_INDEX["k"]: satellite_unit, + } indeces = { - "r": {"main": regions}, - "n": {"main": final_consumptions}, - "k": {"main": satellite}, - "f": {"main": factors}, - "a": {"main": activities}, - "c": {"main": commodities}, - } + "r": {"main": regions}, + "n": {"main": final_consumptions}, + "k": {"main": satellite}, + "f": {"main": factors}, + "a": {"main": activities}, + "c": {"main": commodities}, + } return matrices, indeces, units @@ -978,10 +983,8 @@ def eora_multi_region(data_path, index_path, year, price): return matrices, indeces, units - def eurostat_sut( - supply_path, - use_path, + supply_path, use_path, ): supply_file = pd.ExcelFile(supply_path) use_file = pd.ExcelFile(use_path) @@ -990,19 +993,30 @@ def eurostat_sut( use_meta = extract_metadata_from_eurostat(use_file) if "Supply table at basic prices" not in supply_meta["table"]: - raise WrongInput("specified supply table dataset is {}. Acceptable dataset is 'Supply table at basic prices'." - "Please refer to the documents for proper download of the dataset".format(supply_meta["table"])) + raise WrongInput( + "specified supply table dataset is {}. Acceptable dataset is 'Supply table at basic prices'." + "Please refer to the documents for proper download of the dataset".format( + supply_meta["table"] + ) + ) if "Use table at basic prices " not in use_meta["table"]: - raise WrongInput("specified use table dataset is {}. Acceptable dataset is 'Use table at basic prices '." - "Please refer to the documents for proper download of the dataset".format(use_meta["table"])) - - if supply_meta["country"]!=use_meta["country"] or supply_meta["year"]!=use_meta["year"]: - raise WrongInput("there are mismatched between the country/year of supply and use datasets.\nSupply Dataset: {}\nUse Dataset:{}".format( - supply_meta, - use_meta - )) - + raise WrongInput( + "specified use table dataset is {}. Acceptable dataset is 'Use table at basic prices '." + "Please refer to the documents for proper download of the dataset".format( + use_meta["table"] + ) + ) + + if ( + supply_meta["country"] != use_meta["country"] + or supply_meta["year"] != use_meta["year"] + ): + raise WrongInput( + "there are mismatched between the country/year of supply and use datasets.\nSupply Dataset: {}\nUse Dataset:{}".format( + supply_meta, use_meta + ) + ) supply_data = supply_file.parse( sheet_name=eurostat_id["supply"]["sheet_name"], @@ -1018,83 +1032,59 @@ def eurostat_sut( # build Z_matrix z_index_c = pd.MultiIndex.from_product( - [ - [supply_meta["country"]], - [_MASTER_INDEX["c"]], - eurostat_id["c"] - ] + [[supply_meta["country"]], [_MASTER_INDEX["c"]], eurostat_id["c"]] ) z_index_a = pd.MultiIndex.from_product( - [ - [supply_meta["country"]], - [_MASTER_INDEX["a"]], - eurostat_id["a"] - ] + [[supply_meta["country"]], [_MASTER_INDEX["a"]], eurostat_id["a"]] ) z_index = z_index_c.append(z_index_a) - Z = pd.DataFrame( - data = 0, - index = z_index, - columns=z_index - ) + Z = pd.DataFrame(data=0, index=z_index, columns=z_index) # fill supply side Z.loc[ - (supply_meta["country"],_MASTER_INDEX["a"],eurostat_id["a"]), - (supply_meta["country"],_MASTER_INDEX["c"],eurostat_id["c"]) - ] = supply_data.loc[eurostat_id["a"],eurostat_id["c"]].values + (supply_meta["country"], _MASTER_INDEX["a"], eurostat_id["a"]), + (supply_meta["country"], _MASTER_INDEX["c"], eurostat_id["c"]), + ] = supply_data.loc[eurostat_id["a"], eurostat_id["c"]].values # fill use side Z.loc[ - (supply_meta["country"],_MASTER_INDEX["c"],eurostat_id["c"]), - (supply_meta["country"],_MASTER_INDEX["a"],eurostat_id["a"]) - ] = use_data.loc[eurostat_id["c"],eurostat_id["a"]].values - + (supply_meta["country"], _MASTER_INDEX["c"], eurostat_id["c"]), + (supply_meta["country"], _MASTER_INDEX["a"], eurostat_id["a"]), + ] = use_data.loc[eurostat_id["c"], eurostat_id["a"]].values # build V_matrix V = pd.DataFrame( - data = 0, - index = eurostat_id['f'] + eurostat_id["c_import"], - columns = z_index + data=0, index=eurostat_id["f"] + eurostat_id["c_import"], columns=z_index ) # Activity VA V.loc[ - eurostat_id['f'], - (supply_meta["country"],_MASTER_INDEX["a"],eurostat_id["a"]) - ] = use_data.loc[eurostat_id['f'],eurostat_id['a']].values + eurostat_id["f"], (supply_meta["country"], _MASTER_INDEX["a"], eurostat_id["a"]) + ] = use_data.loc[eurostat_id["f"], eurostat_id["a"]].values # Commodity VA V.loc[ eurostat_id["c_import"], - (supply_meta["country"],_MASTER_INDEX["c"],eurostat_id["c"]) - ] = supply_data.loc[eurostat_id["c_import"],eurostat_id["c"]].values + (supply_meta["country"], _MASTER_INDEX["c"], eurostat_id["c"]), + ] = supply_data.loc[eurostat_id["c_import"], eurostat_id["c"]].values # Building Y matrix Y_columns = pd.MultiIndex.from_product( - [ - [supply_meta["country"]], - [_MASTER_INDEX["n"]], - eurostat_id["n"] - ] - ) - Y = pd.DataFrame( - data = 0 , - index = z_index, - columns = Y_columns + [[supply_meta["country"]], [_MASTER_INDEX["n"]], eurostat_id["n"]] ) + Y = pd.DataFrame(data=0, index=z_index, columns=Y_columns) Y.loc[ - (supply_meta["country"],_MASTER_INDEX["c"],eurostat_id["c"]), - (supply_meta["country"],_MASTER_INDEX["n"],eurostat_id["n"]) - ] = use_data.loc[eurostat_id["c"],eurostat_id["n"]].values + (supply_meta["country"], _MASTER_INDEX["c"], eurostat_id["c"]), + (supply_meta["country"], _MASTER_INDEX["n"], eurostat_id["n"]), + ] = use_data.loc[eurostat_id["c"], eurostat_id["n"]].values # Building E and EY - E = pd.DataFrame(data=0,index=["None"],columns=z_index) - EY = pd.DataFrame(data=0,index=["None"],columns=Y_columns) + E = pd.DataFrame(data=0, index=["None"], columns=z_index) + EY = pd.DataFrame(data=0, index=["None"], columns=Y_columns) # Units units = { @@ -1105,30 +1095,19 @@ def eurostat_sut( use_meta["unit"], index=eurostat_id["c"], columns=["unit"] ), _MASTER_INDEX["f"]: pd.DataFrame( - use_meta["unit"], index=eurostat_id["c_import"] + eurostat_id["f"], columns=["unit"] - ), - _MASTER_INDEX["k"]: pd.DataFrame( - "None", - index=E.index, + use_meta["unit"], + index=eurostat_id["c_import"] + eurostat_id["f"], columns=["unit"], ), + _MASTER_INDEX["k"]: pd.DataFrame("None", index=E.index, columns=["unit"],), } X = calc_X(Z, Y) - for matrix in [Z,V,E,EY,Y,X]: - matrix.replace(":",0,inplace=True) - - matrices = { - "baseline": { - "Z": Z, - "V": V, - "E": E, - "EY": EY, - "Y": Y, - "X": X, - } - } + for matrix in [Z, V, E, EY, Y, X]: + matrix.replace(":", 0, inplace=True) + + matrices = {"baseline": {"Z": Z, "V": V, "E": E, "EY": EY, "Y": Y, "X": X,}} sort_frames(matrices["baseline"]) indeces = { "r": {"main": Z.index.unique(0).tolist()}, @@ -1140,16 +1119,16 @@ def eurostat_sut( } rename_index(matrices["baseline"]) - return matrices, indeces, units,use_meta + return matrices, indeces, units, use_meta def parse_pymrio(io, value_added, satellite_account): """Extracts the data from pymrio in mario format """ - + # be sure that system is calculated io = io.calc_all() - + extensions = {} for value in dir(io): obj = getattr(io, value) @@ -1279,13 +1258,11 @@ def parse_pymrio(io, value_added, satellite_account): return matrices, units, indeces +def hybrid_sut_exiobase_reader(path, extensions): -def hybrid_sut_exiobase_reader(path,extensions): - _ACCEPTABLE_EXIOBASE_EXTENSIONS = [*hybrid_sut_exiobase_parser_id] - - _ACCEPTABLE_EXIOBASE_EXTENSIONS.remove("matrices") + _ACCEPTABLE_EXIOBASE_EXTENSIONS.remove("matrices") if extensions: if extensions == "all": @@ -1295,68 +1272,56 @@ def hybrid_sut_exiobase_reader(path,extensions): differnce = set(extensions).difference(_ACCEPTABLE_EXIOBASE_EXTENSIONS) if differnce: - raise WrongInput(f"Following items are not valid for extensions: \n {differnce}.\n Valid items are: \n {_ACCEPTABLE_EXIOBASE_EXTENSIONS}") + raise WrongInput( + f"Following items are not valid for extensions: \n {differnce}.\n Valid items are: \n {_ACCEPTABLE_EXIOBASE_EXTENSIONS}" + ) + main_files = dict(matrices=hybrid_sut_exiobase_parser_id["matrices"],) - main_files = dict( - matrices = hybrid_sut_exiobase_parser_id["matrices"], - ) - - extensions_files = {extension: hybrid_sut_exiobase_parser_id[extension] for extension in extensions} + extensions_files = { + extension: hybrid_sut_exiobase_parser_id[extension] for extension in extensions + } main_files = {**main_files, **extensions_files} - # reading the files - read = all_file_reader(path, main_files, sub_folder=False,sep=',') + read = all_file_reader(path, main_files, sub_folder=False, sep=",") S = read["matrices"]["S"] U = read["matrices"]["U"] Y = read["matrices"]["Y"] - # Commodity index - c_index= pd.MultiIndex.from_product( - [ - S.index.unique(0), - [_MASTER_INDEX["c"]], - S.index.unique(1) - ] + c_index = pd.MultiIndex.from_product( + [S.index.unique(0), [_MASTER_INDEX["c"]], S.index.unique(1)] ) # Activity index a_index = pd.MultiIndex.from_product( - [ - S.columns.unique(0), - [_MASTER_INDEX['a']], - S.columns.unique(1) - ] + [S.columns.unique(0), [_MASTER_INDEX["a"]], S.columns.unique(1)] ) # # Demand index n_index = pd.MultiIndex.from_product( - [ - Y.columns.unique(0), - [_MASTER_INDEX["n"]], - Y.columns.unique(1) - ] + [Y.columns.unique(0), [_MASTER_INDEX["n"]], Y.columns.unique(1)] ) - - - if extensions: + if extensions: E = [] EY = [] - for extension in extensions: + for extension in extensions: dfs = read[extension] - e = dfs["activity"] + e = dfs["activity"] ey = dfs["final_demand"] if e.index.nlevels == 3: - + idx = pd.MultiIndex.from_arrays( [ - e.index.get_level_values(0) + " (" + e.index.get_level_values(-1) + f" - {extension})", - e.index.get_level_values(1) + e.index.get_level_values(0) + + " (" + + e.index.get_level_values(-1) + + f" - {extension})", + e.index.get_level_values(1), ] ) @@ -1364,55 +1329,47 @@ def hybrid_sut_exiobase_reader(path,extensions): idx = pd.MultiIndex.from_arrays( [ e.index.get_level_values(0) + f" ({extension})", - e.index.get_level_values(1) + e.index.get_level_values(1), ] ) e.index = idx - ey.index= idx + ey.index = idx E.append(e) EY.append(ey) - E = pd.concat(E,axis=0) - EY = pd.concat(EY,axis=0) - + E = pd.concat(E, axis=0) + EY = pd.concat(EY, axis=0) + else: - E = pd.DataFrame(data = 0,index=[["None"],["None"]],columns=a_index) - EY =pd.DataFrame(data = 0,index=[["None"],["None"]],columns=n_index) - - - + E = pd.DataFrame(data=0, index=[["None"], ["None"]], columns=a_index) + EY = pd.DataFrame(data=0, index=[["None"], ["None"]], columns=n_index) + # # Satellite accounts index k_index = E.index.get_level_values(0) - # units commodities_unit = pd.DataFrame( - data = S.index.get_level_values(-1)[0:len(c_index.unique(2))], - index = c_index.unique(2), - columns = ["unit"] + data=S.index.get_level_values(-1)[0 : len(c_index.unique(2))], + index=c_index.unique(2), + columns=["unit"], ) activities_unit = pd.DataFrame( - data = ["None"] * len(a_index.unique(2)), - index = a_index.unique(2), - columns = ["unit"] - ) - - factors_unit = pd.DataFrame( - ["None"], - index=["None"], + data=["None"] * len(a_index.unique(2)), + index=a_index.unique(2), columns=["unit"], ) + factors_unit = pd.DataFrame(["None"], index=["None"], columns=["unit"],) + extensions_unit = pd.DataFrame( E.index.get_level_values(1), index=E.index.get_level_values(0), columns=["unit"], ) - # reshape the indeces S.index = c_index S.columns = a_index @@ -1420,21 +1377,27 @@ def hybrid_sut_exiobase_reader(path,extensions): U.index = c_index U.columns = a_index - - V = pd.DataFrame(data=0,index=["None"],columns=c_index.append(a_index)) + + V = pd.DataFrame(data=0, index=["None"], columns=c_index.append(a_index)) Y.index = c_index Y.columns = n_index - + E.index = k_index E.columns = U.columns - E = pd.concat([pd.DataFrame(np.zeros((E.shape[0],S.shape[1])),index=E.index,columns=S.columns), - E], axis=1) - + E = pd.concat( + [ + pd.DataFrame( + np.zeros((E.shape[0], S.shape[1])), index=E.index, columns=S.columns + ), + E, + ], + axis=1, + ) + EY.index = k_index EY.columns = n_index - - + # Creating the missing parts of the z matrix z_upper = pd.DataFrame( np.zeros((len(c_index), len(c_index))), index=c_index, columns=c_index @@ -1445,13 +1408,13 @@ def hybrid_sut_exiobase_reader(path,extensions): ) z_lower = U.append(z_lower) Z = z_upper.join(z_lower) - + # adding the lower part to Y y_lower = pd.DataFrame( np.zeros((len(a_index), len(n_index))), index=a_index, columns=n_index ) Y = Y.append(y_lower) - + X = calc_X(Z, Y) indeces = { @@ -1464,16 +1427,7 @@ def hybrid_sut_exiobase_reader(path,extensions): "c": {"main": c_index.unique(-1).tolist()}, } - matrices = { - "baseline": { - "Z": Z, - "V": V, - "E": E, - "EY": EY, - "Y": Y, - "X": X, - } - } + matrices = {"baseline": {"Z": Z, "V": V, "E": E, "EY": EY, "Y": Y, "X": X,}} units = { _MASTER_INDEX["a"]: activities_unit, @@ -1485,4 +1439,3 @@ def hybrid_sut_exiobase_reader(path,extensions): rename_index(matrices["baseline"]) return matrices, indeces, units - diff --git a/mario/tools/utilities.py b/mario/tools/utilities.py index ff05a12..2f74fb6 100644 --- a/mario/tools/utilities.py +++ b/mario/tools/utilities.py @@ -86,7 +86,7 @@ def run_from_jupyter(): ipy_str = str(type(get_ipython())) if "zmqshell" in ipy_str: return True - + return False @@ -121,7 +121,8 @@ def _manage_indeces(instance, case, **kwargs): for key, value in kwargs.items(): instance._indeces[key] = {"main": value} -def check_clusters(index_dict,table,clusters): + +def check_clusters(index_dict, table, clusters): differences = set(clusters).difference(set(_LEVELS[table])) @@ -134,14 +135,15 @@ def check_clusters(index_dict,table,clusters): for level, level_cluster in clusters.items(): for cluster, values in level_cluster.items(): - differences = set(values).difference(set(index_dict[level])) + differences = set(values).difference(set(index_dict[level])) if differences: raise WrongInput( "{} in cluster {} for level {} is/are not a valid item/s.".format( - differences, cluster, level + differences, cluster, level ) ) + def all_file_reader( path, guide, sub_folder=False, sep="\t", exceptions=[], engine=None ): @@ -208,10 +210,10 @@ def readers(file_to_read, file): def return_index(df, item, multi_index, del_duplicate, reindex=None, level=None): if multi_index: - index = list(getattr(df,item).get_level_values(level)) + index = list(getattr(df, item).get_level_values(level)) else: - index = list(getattr(df,item)) + index = list(getattr(df, item)) if del_duplicate: index = delete_duplicates(index) @@ -230,7 +232,9 @@ def multiindex_contain(inner_index, outer_index, file, check_levels=None): if check_levels is None: if inner_index.nlevels != outer_index.nlevels: raise WrongInput(f"number levels for {file} are not valid.") - levels = list(range(inner_index.nlevels))# [i for i in range(inner_index.nlevels)] + levels = list( + range(inner_index.nlevels) + ) # [i for i in range(inner_index.nlevels)] else: levels = check_levels @@ -239,9 +243,7 @@ def multiindex_contain(inner_index, outer_index, file, check_levels=None): for level in levels: diff = ( - outer_index.levels[level] - .difference(inner_index.levels[level]) - .tolist() + outer_index.levels[level].difference(inner_index.levels[level]).tolist() ) differences[level] = diff @@ -261,11 +263,10 @@ def rename_index(_dict): for key, value in _dict.items(): for item in ["index", "columns"]: - if isinstance(getattr(value,item), pd.MultiIndex): - getattr(value,item).names= _INDEX_NAMES['3levels'] + if isinstance(getattr(value, item), pd.MultiIndex): + getattr(value, item).names = _INDEX_NAMES["3levels"] else: - getattr(value,item).name = _INDEX_NAMES['1level'] - + getattr(value, item).name = _INDEX_NAMES["1level"] def filtering(instance, filters): @@ -328,6 +329,7 @@ def to_single_index(df): return df + def extract_metadata_from_eurostat(file): """extracts some info such as country,table_info, and the year of the data from an xlsx @@ -342,15 +344,15 @@ def extract_metadata_from_eurostat(file): metadata with country,table_type, and year info """ meta_info = { - "year": (7,2,int), - "country": (6,2,str), - "table": (0,1,str), - "unit": (4,2,str) + "year": (7, 2, int), + "country": (6, 2, str), + "table": (0, 1, str), + "unit": (4, 2, str), } initial_data = file.parse(sheet_name="Sheet 1") metadata = {} - for item,info in meta_info.items(): - metadata[item] = info[2](initial_data.iloc[info[0],info[1]]) + for item, info in meta_info.items(): + metadata[item] = info[2](initial_data.iloc[info[0], info[1]]) - return metadata \ No newline at end of file + return metadata From 48fe8e868ea35db083583926e0592faa0af774ff Mon Sep 17 00:00:00 2001 From: Lorenzo Rinaldi Date: Wed, 10 May 2023 14:11:08 +0200 Subject: [PATCH 5/5] Fix aggregation bug for multiple scenarios calc_all check at the beginning was checking only the baseline scenario --- mario/core/AttrData.py | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) diff --git a/mario/core/AttrData.py b/mario/core/AttrData.py index c41be1a..7b70654 100644 --- a/mario/core/AttrData.py +++ b/mario/core/AttrData.py @@ -503,8 +503,9 @@ def aggregate( """ - # insure of Y,E,V,Z,EY exist - self.calc_all(["E", "V", "Z"]) + # ensure of Y,E,V,Z,EY exist + for scenario in self.scenarios: + self.calc_all(["E", "V", "Z"],scenario=scenario) if not inplace: new = self.copy()