diff --git a/Imports Script/API_Imports_Data_Script.py b/Imports Script/API_Imports_Data_Script.py index 537086d..944dbbf 100644 --- a/Imports Script/API_Imports_Data_Script.py +++ b/Imports Script/API_Imports_Data_Script.py @@ -3,15 +3,12 @@ import yaml import numpy as np import requests -import json from pathlib import Path apiPath = Path(__file__).parent / 'API' dataPath = Path(__file__).parent / 'Data' conPath = Path(__file__).parent / 'Concordances' -data_years = ['2020'] - request_data = False #0 for no, 1 for yes #%% @@ -143,14 +140,14 @@ def make_reqs(file, reqs, data_years): print('Successfully Collected All',file,'Requests') return d -def get_census_df(d, c_d): +def get_census_df(d, c_d, data_years): ''' Creates a dataframe for Census response data for a given year. ''' df = pd.DataFrame() country_code = {v:k for k,v in c_d.items()} - for a,b in d.items(): - for k,v in b.items(): + for year in data_years: + for k, v in d[year].items(): v_d = v['data'] cty = country_code.get(v['cty']) value_df = pd.DataFrame(data=v_d[1:], columns=v_d[0]) @@ -163,29 +160,34 @@ def get_census_df(d, c_d): .set_index('NAICS') ) df = pd.concat([df, cols], axis=1) + df = df.assign(Year=year) df = df.replace(np.nan, 0).reset_index() ## Merge in BEA Codes and flatten c_b = pd.read_csv(apiPath / 'Census_API_Mappings.csv') df = df.merge(c_b, how='left', on='NAICS') df = (df.drop(columns='NAICS') - .groupby('BEA Sector').agg(sum) + .groupby(['BEA Sector', 'Year']).agg(sum) .reset_index() - .melt(id_vars=['BEA Sector'], var_name='CountryCode', + .melt(id_vars=['BEA Sector', 'Year'], var_name='CountryCode', value_name='Import Quantity') .assign(Unit='USD') .assign(Source='Census') - # .assign(Year='') ) return df -def get_bea_df(d, b_d): +def get_bea_df(d, b_d, data_years): ''' Creates a dataframe for BEA response data for a given year. ''' e_t_d = {v:k for k,v in b_d.items()} n_d = {} - for a,b in d.items(): - for k,v in b.items(): + df_all = pd.DataFrame() + b_b = (pd.read_csv(apiPath / 'BEA_API_Mappings.csv') + .filter(['API BEA Service', 'BEA Sector']) + .rename(columns={'API BEA Service': 'BEA Service'}) + ) + for year in data_years: + for k, v in d[year].items(): cty = v['cty'] cty = e_t_d[cty] d_n = {} @@ -195,37 +197,38 @@ def get_bea_df(d, b_d): value = item['DataValue'] d_n[sector] = value n_d[cty] = d_n - df = (pd.DataFrame(n_d) - .apply(pd.to_numeric) - .dropna(how='all') - .replace(np.nan,0) - .reset_index() - .rename(columns={'index':'BEA Service'})) - ## Merge in BEA codes and flatten - b_b = (pd.read_csv(apiPath / 'BEA_API_Mappings.csv') - .filter(['API BEA Service', 'BEA Sector']) - .rename(columns={'API BEA Service': 'BEA Service'}) - ) - df = (df.merge(b_b, how='right', on='BEA Service', validate='1:m') - .fillna(0) - .drop(columns='BEA Service') - ) - if(len(df['BEA Sector'].unique()) != len(df)): - raise ValueError("Duplicate BEA sectors") - df = (df.melt(id_vars=['BEA Sector'], - var_name='CountryCode', - value_name='Import Quantity') - .assign(Unit='USD') - .assign(Source='BEA') - # .assign(Year='') - ) - df['Import Quantity'] = df['Import Quantity'].apply(lambda x: x*1000000) - return df + df = (pd.DataFrame(n_d) + .apply(pd.to_numeric) + .dropna(how='all') + .replace(np.nan,0) + .reset_index() + .rename(columns={'index':'BEA Service'}) + ) + ## Merge in BEA codes and flatten + df = (df.merge(b_b, how='right', on='BEA Service', validate='1:m') + .fillna(0) + .drop(columns='BEA Service') + ) + if(len(df['BEA Sector'].unique()) != len(df)): + raise ValueError("Duplicate BEA sectors") + df = (df.melt(id_vars=['BEA Sector'], + var_name='CountryCode', + value_name='Import Quantity') + .assign(Unit='USD') + .assign(Source='BEA') + .assign(Year=year) + ) + df['Import Quantity'] = df['Import Quantity'].apply(lambda x: x*1000000) + df_all = pd.concat([df_all, df], ignore_index=True) + return df_all -def get_imports_data(request_data): +def get_imports_data(request_data, data_years=['2020']): ''' A function to call from other scripts. ''' + # TODO Update data call with more years, until then set as 2020 + data_years=['2020'] + b_d, c_d = get_country_schema() if request_data == True: b_reqs = create_Reqs('BEA_API.yml', b_d) @@ -234,13 +237,14 @@ def get_imports_data(request_data): pkl.dump(b_resp, open(dataPath / 'bea_responses.pkl', 'wb')) c_resp = make_reqs('Census', c_reqs, data_years) pkl.dump(c_resp, open(dataPath / 'census_responses.pkl', 'wb')) - + c_responses = pkl.load(open(dataPath / 'census_responses.pkl', 'rb')) b_responses = pkl.load(open(dataPath / 'bea_responses.pkl', 'rb')) - b_df = get_bea_df(b_responses, b_d) - c_df = get_census_df(c_responses, c_d) + b_df = get_bea_df(b_responses, b_d, data_years) + c_df = get_census_df(c_responses, c_d, data_years) i_df = pd.concat([c_df, b_df], ignore_index=True, axis=0) i_df['Country'] = i_df['CountryCode'].map(b_d) return(i_df) -id_f = get_imports_data(request_data=request_data) +if __name__ == '__main__': + id_f = get_imports_data(request_data=request_data) diff --git a/Imports Script/Data/Import Matrix, Canada, After Redefinitions.csv b/Imports Script/Data/Import Matrix, Canada, After Redefinitions_2021.csv similarity index 100% rename from Imports Script/Data/Import Matrix, Canada, After Redefinitions.csv rename to Imports Script/Data/Import Matrix, Canada, After Redefinitions_2021.csv diff --git a/Imports Script/Data/Import Matrix, China, After Redefinitions.csv b/Imports Script/Data/Import Matrix, China, After Redefinitions_2021.csv similarity index 100% rename from Imports Script/Data/Import Matrix, China, After Redefinitions.csv rename to Imports Script/Data/Import Matrix, China, After Redefinitions_2021.csv diff --git a/Imports Script/Data/Import Matrix, Europe, After Redefinitions.csv b/Imports Script/Data/Import Matrix, Europe, After Redefinitions_2021.csv similarity index 100% rename from Imports Script/Data/Import Matrix, Europe, After Redefinitions.csv rename to Imports Script/Data/Import Matrix, Europe, After Redefinitions_2021.csv diff --git a/Imports Script/Data/Import Matrix, Japan, After Redefinitions.csv b/Imports Script/Data/Import Matrix, Japan, After Redefinitions_2021.csv similarity index 100% rename from Imports Script/Data/Import Matrix, Japan, After Redefinitions.csv rename to Imports Script/Data/Import Matrix, Japan, After Redefinitions_2021.csv diff --git a/Imports Script/Data/Import Matrix, Mexico, After Redefinitions.csv b/Imports Script/Data/Import Matrix, Mexico, After Redefinitions_2021.csv similarity index 100% rename from Imports Script/Data/Import Matrix, Mexico, After Redefinitions.csv rename to Imports Script/Data/Import Matrix, Mexico, After Redefinitions_2021.csv diff --git a/Imports Script/Data/Import Matrix, Rest of Asia and Pacific, After Redefinitions.csv b/Imports Script/Data/Import Matrix, Rest of Asia and Pacific, After Redefinitions_2021.csv similarity index 100% rename from Imports Script/Data/Import Matrix, Rest of Asia and Pacific, After Redefinitions.csv rename to Imports Script/Data/Import Matrix, Rest of Asia and Pacific, After Redefinitions_2021.csv diff --git a/Imports Script/Data/Import Matrix, Rest of World, After Redefinitions.csv b/Imports Script/Data/Import Matrix, Rest of World, After Redefinitions_2021.csv similarity index 100% rename from Imports Script/Data/Import Matrix, Rest of World, After Redefinitions.csv rename to Imports Script/Data/Import Matrix, Rest of World, After Redefinitions_2021.csv diff --git a/Imports Script/Data/exio3_indout.pkl b/Imports Script/Data/exio3_indout.pkl deleted file mode 100644 index 35675af..0000000 Binary files a/Imports Script/Data/exio3_indout.pkl and /dev/null differ diff --git a/Imports Script/Data/exio3_multipliers_2021.pkl b/Imports Script/Data/exio3_multipliers_2021.pkl new file mode 100644 index 0000000..1be9c3d Binary files /dev/null and b/Imports Script/Data/exio3_multipliers_2021.pkl differ diff --git a/Imports Script/Data/exio3_multipliers.pkl b/Imports Script/Data/exio3_multipliers_2022.pkl similarity index 100% rename from Imports Script/Data/exio3_multipliers.pkl rename to Imports Script/Data/exio3_multipliers_2022.pkl diff --git a/Imports Script/Data/exio_config.yml b/Imports Script/Data/exio_config.yml index 1e9253e..0c9e33c 100644 --- a/Imports Script/Data/exio_config.yml +++ b/Imports Script/Data/exio_config.yml @@ -2,6 +2,6 @@ fields: region: CountryCode sector: Exiobase Sector flows: - Carbon dioxide (CO2) IPCC categories 1 to 4 and 6 to 7 (excl land use, land use change and forestry): Carbon Dioxide (CO2) - Methane (CH4) IPCC categories 1 to 4 and 6 to 7 (excl land use, land use change and forestry): Methane (CH4) - Nitrous Oxide (N2O) IPCC categories 1 to 4 and 6 to 7 (excl land use, land use change and forestry): Nitrous Oxide (N2O) + Carbon dioxide (CO2) IPCC categories 1 to 4 and 6 to 7 (excl land use, land use change and forestry): Carbon dioxide + Methane (CH4) IPCC categories 1 to 4 and 6 to 7 (excl land use, land use change and forestry): Methane + Nitrous Oxide (N2O) IPCC categories 1 to 4 and 6 to 7 (excl land use, land use change and forestry): Nitrous oxide diff --git a/Imports Script/Data/metadata.json b/Imports Script/Data/metadata.json index d596150..05fca02 100644 --- a/Imports Script/Data/metadata.json +++ b/Imports Script/Data/metadata.json @@ -4,6 +4,7 @@ "system": "pxp", "version": "10.5281/zenodo.3583070", "history": [ + "20230713 08:58:35 - FILEIO - Downloaded https://zenodo.org/record/5589597/files/IOT_2021_pxp.zip to IOT_2021_pxp.zip", "20230607 11:54:22 - FILEIO - Downloaded https://zenodo.org/record/5589597/files/IOT_2022_pxp.zip to IOT_2022_pxp.zip" ] } \ No newline at end of file diff --git a/Imports Script/imports_multipliers.csv b/Imports Script/imports_multipliers.csv index b5c8454..5258711 100644 --- a/Imports Script/imports_multipliers.csv +++ b/Imports Script/imports_multipliers.csv @@ -1,130 +1,130 @@ -BEA Summary,Flow,Amount,Unit,CurrencyYear,DataYear,PriceType -111CA,Carbon Dioxide (CO2),2.608889444268707,kg / Euro,2021,2021,Producer -111CA,Methane (CH4),0.02867161127823696,kg / Euro,2021,2021,Producer -111CA,Nitrous Oxide (N2O),5.2050806996613446e-05,kg / Euro,2021,2021,Producer -113FF,Carbon Dioxide (CO2),0.4241117162642016,kg / Euro,2021,2021,Producer -113FF,Methane (CH4),0.002318137159458142,kg / Euro,2021,2021,Producer -113FF,Nitrous Oxide (N2O),3.4650094490642824e-05,kg / Euro,2021,2021,Producer -211,Carbon Dioxide (CO2),2.7503144683312666,kg / Euro,2021,2021,Producer -211,Methane (CH4),0.04094744372237971,kg / Euro,2021,2021,Producer -211,Nitrous Oxide (N2O),8.653303865626393e-06,kg / Euro,2021,2021,Producer -212,Carbon Dioxide (CO2),166.43665575877674,kg / Euro,2021,2021,Producer -212,Methane (CH4),0.01399255103992939,kg / Euro,2021,2021,Producer -212,Nitrous Oxide (N2O),0.003580402461978241,kg / Euro,2021,2021,Producer -213,Carbon Dioxide (CO2),228.63344836887174,kg / Euro,2021,2021,Producer -213,Methane (CH4),0.10341300237189549,kg / Euro,2021,2021,Producer -213,Nitrous Oxide (N2O),0.004838794829258269,kg / Euro,2021,2021,Producer -22,Carbon Dioxide (CO2),1557.0154600969786,kg / Euro,2021,2021,Producer -22,Methane (CH4),0.11730663754642004,kg / Euro,2021,2021,Producer -22,Nitrous Oxide (N2O),0.018656442688659382,kg / Euro,2021,2021,Producer -311FT,Carbon Dioxide (CO2),0.9483587232531949,kg / Euro,2021,2021,Producer -311FT,Methane (CH4),0.008717106886123214,kg / Euro,2021,2021,Producer -311FT,Nitrous Oxide (N2O),4.267924596974192e-05,kg / Euro,2021,2021,Producer -313TT,Carbon Dioxide (CO2),0.640517155062235,kg / Euro,2021,2021,Producer -313TT,Methane (CH4),0.006564668253831874,kg / Euro,2021,2021,Producer -313TT,Nitrous Oxide (N2O),1.2506522089869479e-05,kg / Euro,2021,2021,Producer -315AL,Carbon Dioxide (CO2),0.5045070084559461,kg / Euro,2021,2021,Producer -315AL,Methane (CH4),0.007417880069411865,kg / Euro,2021,2021,Producer -315AL,Nitrous Oxide (N2O),1.1997317647281825e-05,kg / Euro,2021,2021,Producer -321,Carbon Dioxide (CO2),0.28723622367803725,kg / Euro,2021,2021,Producer -321,Methane (CH4),0.0019707246748674814,kg / Euro,2021,2021,Producer -321,Nitrous Oxide (N2O),9.432938248811588e-06,kg / Euro,2021,2021,Producer -322,Carbon Dioxide (CO2),0.35906267522365864,kg / Euro,2021,2021,Producer -322,Methane (CH4),0.003300096441328368,kg / Euro,2021,2021,Producer -322,Nitrous Oxide (N2O),3.697407806708972e-05,kg / Euro,2021,2021,Producer -323,Carbon Dioxide (CO2),0.5419568184452583,kg / Euro,2021,2021,Producer -323,Methane (CH4),0.002860859704279354,kg / Euro,2021,2021,Producer -323,Nitrous Oxide (N2O),1.8450919880041254e-05,kg / Euro,2021,2021,Producer -324,Carbon Dioxide (CO2),174702.8724760784,kg / Euro,2021,2021,Producer -324,Methane (CH4),34.629831494557465,kg / Euro,2021,2021,Producer -324,Nitrous Oxide (N2O),5.501706986489365,kg / Euro,2021,2021,Producer -325,Carbon Dioxide (CO2),2.9479294380886234,kg / Euro,2021,2021,Producer -325,Methane (CH4),0.017280654214439025,kg / Euro,2021,2021,Producer -325,Nitrous Oxide (N2O),4.593084607102359e-05,kg / Euro,2021,2021,Producer -326,Carbon Dioxide (CO2),0.5211595122925854,kg / Euro,2021,2021,Producer -326,Methane (CH4),0.003801649814389313,kg / Euro,2021,2021,Producer -326,Nitrous Oxide (N2O),1.1337880211940854e-05,kg / Euro,2021,2021,Producer -327,Carbon Dioxide (CO2),1.8307434910154436,kg / Euro,2021,2021,Producer -327,Methane (CH4),0.00578074933813609,kg / Euro,2021,2021,Producer -327,Nitrous Oxide (N2O),3.388480424315313e-05,kg / Euro,2021,2021,Producer -331,Carbon Dioxide (CO2),0.6684668401009526,kg / Euro,2021,2021,Producer -331,Methane (CH4),0.007088714780519518,kg / Euro,2021,2021,Producer -331,Nitrous Oxide (N2O),1.937681097534625e-05,kg / Euro,2021,2021,Producer -332,Carbon Dioxide (CO2),0.7846696364208352,kg / Euro,2021,2021,Producer -332,Methane (CH4),0.003831519430144668,kg / Euro,2021,2021,Producer -332,Nitrous Oxide (N2O),1.4454748172421288e-05,kg / Euro,2021,2021,Producer -333,Carbon Dioxide (CO2),0.4172934387357458,kg / Euro,2021,2021,Producer -333,Methane (CH4),0.0023618171667701055,kg / Euro,2021,2021,Producer -333,Nitrous Oxide (N2O),8.676159895827239e-06,kg / Euro,2021,2021,Producer -334,Carbon Dioxide (CO2),0.38746420481510413,kg / Euro,2021,2021,Producer -334,Methane (CH4),0.0010725539909152027,kg / Euro,2021,2021,Producer -334,Nitrous Oxide (N2O),5.614308271826552e-06,kg / Euro,2021,2021,Producer -335,Carbon Dioxide (CO2),0.512547292253821,kg / Euro,2021,2021,Producer -335,Methane (CH4),0.004425825061925481,kg / Euro,2021,2021,Producer -335,Nitrous Oxide (N2O),1.1275639466860172e-05,kg / Euro,2021,2021,Producer -3361MV,Carbon Dioxide (CO2),0.5600583081317574,kg / Euro,2021,2021,Producer -3361MV,Methane (CH4),0.002410126787694695,kg / Euro,2021,2021,Producer -3361MV,Nitrous Oxide (N2O),1.2888351993213271e-05,kg / Euro,2021,2021,Producer -3364OT,Carbon Dioxide (CO2),0.48723512038019956,kg / Euro,2021,2021,Producer -3364OT,Methane (CH4),0.0024232059008238247,kg / Euro,2021,2021,Producer -3364OT,Nitrous Oxide (N2O),9.788695553762485e-06,kg / Euro,2021,2021,Producer -337,Carbon Dioxide (CO2),0.9251637147936786,kg / Euro,2021,2021,Producer -337,Methane (CH4),0.003947978332629133,kg / Euro,2021,2021,Producer -337,Nitrous Oxide (N2O),2.0396635120016366e-05,kg / Euro,2021,2021,Producer -339,Carbon Dioxide (CO2),0.8440036037857748,kg / Euro,2021,2021,Producer -339,Methane (CH4),0.002474448435160721,kg / Euro,2021,2021,Producer -339,Nitrous Oxide (N2O),2.2263871505694962e-05,kg / Euro,2021,2021,Producer -481,Carbon Dioxide (CO2),0.0,kg / Euro,2021,2021,Producer -481,Methane (CH4),0.0,kg / Euro,2021,2021,Producer -481,Nitrous Oxide (N2O),0.0,kg / Euro,2021,2021,Producer -487OS,Carbon Dioxide (CO2),0.0924443310332,kg / Euro,2021,2021,Producer -487OS,Methane (CH4),0.000573470930092,kg / Euro,2021,2021,Producer -487OS,Nitrous Oxide (N2O),2.72594073699e-06,kg / Euro,2021,2021,Producer -511,Carbon Dioxide (CO2),0.09213944666206428,kg / Euro,2021,2021,Producer -511,Methane (CH4),0.0007096061366611835,kg / Euro,2021,2021,Producer -511,Nitrous Oxide (N2O),3.445591324383084e-06,kg / Euro,2021,2021,Producer -512,Carbon Dioxide (CO2),0.0,kg / Euro,2021,2021,Producer -512,Methane (CH4),0.0,kg / Euro,2021,2021,Producer -512,Nitrous Oxide (N2O),0.0,kg / Euro,2021,2021,Producer -513,Carbon Dioxide (CO2),0.1881968396119082,kg / Euro,2021,2021,Producer -513,Methane (CH4),0.0015359359278169925,kg / Euro,2021,2021,Producer -513,Nitrous Oxide (N2O),5.3866690319613905e-06,kg / Euro,2021,2021,Producer -514,Carbon Dioxide (CO2),0.1398181663408508,kg / Euro,2021,2021,Producer -514,Methane (CH4),0.0014149098984083137,kg / Euro,2021,2021,Producer -514,Nitrous Oxide (N2O),4.947148027491555e-06,kg / Euro,2021,2021,Producer -521CI,Carbon Dioxide (CO2),0.0,kg / Euro,2021,2021,Producer -521CI,Methane (CH4),0.0,kg / Euro,2021,2021,Producer -521CI,Nitrous Oxide (N2O),0.0,kg / Euro,2021,2021,Producer -523,Carbon Dioxide (CO2),0.11596599449730921,kg / Euro,2021,2021,Producer -523,Methane (CH4),0.0013531093507989167,kg / Euro,2021,2021,Producer -523,Nitrous Oxide (N2O),3.0316392651385453e-06,kg / Euro,2021,2021,Producer -524,Carbon Dioxide (CO2),0.061082571368966,kg / Euro,2021,2021,Producer -524,Methane (CH4),0.001267431683807749,kg / Euro,2021,2021,Producer -524,Nitrous Oxide (N2O),1.295688106596385e-06,kg / Euro,2021,2021,Producer -5411,Carbon Dioxide (CO2),0.11888509516964246,kg / Euro,2021,2021,Producer -5411,Methane (CH4),0.0022599145082580077,kg / Euro,2021,2021,Producer -5411,Nitrous Oxide (N2O),2.8071208851585024e-06,kg / Euro,2021,2021,Producer -5412OP,Carbon Dioxide (CO2),0.14111725410139564,kg / Euro,2021,2021,Producer -5412OP,Methane (CH4),0.0025219462066891933,kg / Euro,2021,2021,Producer -5412OP,Nitrous Oxide (N2O),3.377706565763931e-06,kg / Euro,2021,2021,Producer -5415,Carbon Dioxide (CO2),0.2853497542454429,kg / Euro,2021,2021,Producer -5415,Methane (CH4),0.0020076984153747737,kg / Euro,2021,2021,Producer -5415,Nitrous Oxide (N2O),1.1658128096474116e-05,kg / Euro,2021,2021,Producer -561,Carbon Dioxide (CO2),0.15258169088794277,kg / Euro,2021,2021,Producer -561,Methane (CH4),0.0026024540601753614,kg / Euro,2021,2021,Producer -561,Nitrous Oxide (N2O),3.919683125658802e-06,kg / Euro,2021,2021,Producer -61,Carbon Dioxide (CO2),0.1730791372401374,kg / Euro,2021,2021,Producer -61,Methane (CH4),0.004904019908921073,kg / Euro,2021,2021,Producer -61,Nitrous Oxide (N2O),3.1496439788624544e-06,kg / Euro,2021,2021,Producer -711AS,Carbon Dioxide (CO2),0.026907847692241398,kg / Euro,2021,2021,Producer -711AS,Methane (CH4),0.000562864227342592,kg / Euro,2021,2021,Producer -711AS,Nitrous Oxide (N2O),6.242662255199597e-07,kg / Euro,2021,2021,Producer -81,Carbon Dioxide (CO2),0.12201025862827189,kg / Euro,2021,2021,Producer -81,Methane (CH4),0.0014372517629645106,kg / Euro,2021,2021,Producer -81,Nitrous Oxide (N2O),2.9148569510762745e-06,kg / Euro,2021,2021,Producer -GFE,Carbon Dioxide (CO2),0.07351353777907273,kg / Euro,2021,2021,Producer -GFE,Methane (CH4),0.0007026617625373571,kg / Euro,2021,2021,Producer -GFE,Nitrous Oxide (N2O),1.6326128274983356e-06,kg / Euro,2021,2021,Producer -Used,Carbon Dioxide (CO2),0.0,kg / Euro,2021,2021,Producer -Used,Methane (CH4),0.0,kg / Euro,2021,2021,Producer -Used,Nitrous Oxide (N2O),0.0,kg / Euro,2021,2021,Producer +BEA Summary,Unit,CurrencyYear,EmissionYear,PriceType,Flowable,Context,Flow UUID,Amount +111CA,kg / USD,2021,2019,Basic,Carbon dioxide,emission/air,b6f010fb-a764-3063-af2d-bcb8309a97b7,2.2907987785719057 +111CA,kg / USD,2021,2019,Basic,Methane,emission/air,aab83476-ec6c-3742-af85-15d320b7ce80,0.024083972780082244 +111CA,kg / USD,2021,2019,Basic,Nitrous oxide,emission/air,cfee0524-7ad6-300b-b050-6249135a2492,4.4370773948525836e-05 +113FF,kg / USD,2021,2019,Basic,Carbon dioxide,emission/air,b6f010fb-a764-3063-af2d-bcb8309a97b7,0.3877230156932219 +113FF,kg / USD,2021,2019,Basic,Methane,emission/air,aab83476-ec6c-3742-af85-15d320b7ce80,0.002188542765820177 +113FF,kg / USD,2021,2019,Basic,Nitrous oxide,emission/air,cfee0524-7ad6-300b-b050-6249135a2492,2.8858204787533435e-05 +211,kg / USD,2021,2019,Basic,Carbon dioxide,emission/air,b6f010fb-a764-3063-af2d-bcb8309a97b7,2.2584820658295492 +211,kg / USD,2021,2019,Basic,Methane,emission/air,aab83476-ec6c-3742-af85-15d320b7ce80,0.03328747268675482 +211,kg / USD,2021,2019,Basic,Nitrous oxide,emission/air,cfee0524-7ad6-300b-b050-6249135a2492,6.746999735648209e-06 +212,kg / USD,2021,2019,Basic,Carbon dioxide,emission/air,b6f010fb-a764-3063-af2d-bcb8309a97b7,141.3726902956554 +212,kg / USD,2021,2019,Basic,Methane,emission/air,aab83476-ec6c-3742-af85-15d320b7ce80,0.01163852046088586 +212,kg / USD,2021,2019,Basic,Nitrous oxide,emission/air,cfee0524-7ad6-300b-b050-6249135a2492,0.003044200030768877 +213,kg / USD,2021,2019,Basic,Carbon dioxide,emission/air,b6f010fb-a764-3063-af2d-bcb8309a97b7,193.99204165444914 +213,kg / USD,2021,2019,Basic,Methane,emission/air,aab83476-ec6c-3742-af85-15d320b7ce80,0.08693846073902439 +213,kg / USD,2021,2019,Basic,Nitrous oxide,emission/air,cfee0524-7ad6-300b-b050-6249135a2492,0.004106951079875334 +22,kg / USD,2021,2019,Basic,Carbon dioxide,emission/air,b6f010fb-a764-3063-af2d-bcb8309a97b7,1301.3787081260543 +22,kg / USD,2021,2019,Basic,Methane,emission/air,aab83476-ec6c-3742-af85-15d320b7ce80,0.09607147155906812 +22,kg / USD,2021,2019,Basic,Nitrous oxide,emission/air,cfee0524-7ad6-300b-b050-6249135a2492,0.015599849197362951 +311FT,kg / USD,2021,2019,Basic,Carbon dioxide,emission/air,b6f010fb-a764-3063-af2d-bcb8309a97b7,0.8127472196017552 +311FT,kg / USD,2021,2019,Basic,Methane,emission/air,aab83476-ec6c-3742-af85-15d320b7ce80,0.007020568087471978 +311FT,kg / USD,2021,2019,Basic,Nitrous oxide,emission/air,cfee0524-7ad6-300b-b050-6249135a2492,3.599873244184416e-05 +313TT,kg / USD,2021,2019,Basic,Carbon dioxide,emission/air,b6f010fb-a764-3063-af2d-bcb8309a97b7,0.5555272860387396 +313TT,kg / USD,2021,2019,Basic,Methane,emission/air,aab83476-ec6c-3742-af85-15d320b7ce80,0.0052126300754897865 +313TT,kg / USD,2021,2019,Basic,Nitrous oxide,emission/air,cfee0524-7ad6-300b-b050-6249135a2492,1.0610071398058957e-05 +315AL,kg / USD,2021,2019,Basic,Carbon dioxide,emission/air,b6f010fb-a764-3063-af2d-bcb8309a97b7,0.5261178063079806 +315AL,kg / USD,2021,2019,Basic,Methane,emission/air,aab83476-ec6c-3742-af85-15d320b7ce80,0.006352845881051778 +315AL,kg / USD,2021,2019,Basic,Nitrous oxide,emission/air,cfee0524-7ad6-300b-b050-6249135a2492,1.0836331926808741e-05 +321,kg / USD,2021,2019,Basic,Carbon dioxide,emission/air,b6f010fb-a764-3063-af2d-bcb8309a97b7,0.25719348315095214 +321,kg / USD,2021,2019,Basic,Methane,emission/air,aab83476-ec6c-3742-af85-15d320b7ce80,0.0016591991258946934 +321,kg / USD,2021,2019,Basic,Nitrous oxide,emission/air,cfee0524-7ad6-300b-b050-6249135a2492,8.686011407629435e-06 +322,kg / USD,2021,2019,Basic,Carbon dioxide,emission/air,b6f010fb-a764-3063-af2d-bcb8309a97b7,0.3326116306472205 +322,kg / USD,2021,2019,Basic,Methane,emission/air,aab83476-ec6c-3742-af85-15d320b7ce80,0.002916766217832784 +322,kg / USD,2021,2019,Basic,Nitrous oxide,emission/air,cfee0524-7ad6-300b-b050-6249135a2492,3.209071033031136e-05 +323,kg / USD,2021,2019,Basic,Carbon dioxide,emission/air,b6f010fb-a764-3063-af2d-bcb8309a97b7,0.5015889795575358 +323,kg / USD,2021,2019,Basic,Methane,emission/air,aab83476-ec6c-3742-af85-15d320b7ce80,0.0028830010676250553 +323,kg / USD,2021,2019,Basic,Nitrous oxide,emission/air,cfee0524-7ad6-300b-b050-6249135a2492,1.6457102260888202e-05 +324,kg / USD,2021,2019,Basic,Carbon dioxide,emission/air,b6f010fb-a764-3063-af2d-bcb8309a97b7,155967.26998614686 +324,kg / USD,2021,2019,Basic,Methane,emission/air,aab83476-ec6c-3742-af85-15d320b7ce80,32.53096524574326 +324,kg / USD,2021,2019,Basic,Nitrous oxide,emission/air,cfee0524-7ad6-300b-b050-6249135a2492,4.909370772429924 +325,kg / USD,2021,2019,Basic,Carbon dioxide,emission/air,b6f010fb-a764-3063-af2d-bcb8309a97b7,2.3837986443462458 +325,kg / USD,2021,2019,Basic,Methane,emission/air,aab83476-ec6c-3742-af85-15d320b7ce80,0.02725123241079535 +325,kg / USD,2021,2019,Basic,Nitrous oxide,emission/air,cfee0524-7ad6-300b-b050-6249135a2492,3.8650939428323616e-05 +326,kg / USD,2021,2019,Basic,Carbon dioxide,emission/air,b6f010fb-a764-3063-af2d-bcb8309a97b7,0.5602585257945825 +326,kg / USD,2021,2019,Basic,Methane,emission/air,aab83476-ec6c-3742-af85-15d320b7ce80,0.0036461577078385813 +326,kg / USD,2021,2019,Basic,Nitrous oxide,emission/air,cfee0524-7ad6-300b-b050-6249135a2492,1.3971185218189926e-05 +327,kg / USD,2021,2019,Basic,Carbon dioxide,emission/air,b6f010fb-a764-3063-af2d-bcb8309a97b7,1.5515273328587702 +327,kg / USD,2021,2019,Basic,Methane,emission/air,aab83476-ec6c-3742-af85-15d320b7ce80,0.0043909483839260025 +327,kg / USD,2021,2019,Basic,Nitrous oxide,emission/air,cfee0524-7ad6-300b-b050-6249135a2492,2.8085311634606002e-05 +331,kg / USD,2021,2019,Basic,Carbon dioxide,emission/air,b6f010fb-a764-3063-af2d-bcb8309a97b7,0.565342120908402 +331,kg / USD,2021,2019,Basic,Methane,emission/air,aab83476-ec6c-3742-af85-15d320b7ce80,0.005847971169762493 +331,kg / USD,2021,2019,Basic,Nitrous oxide,emission/air,cfee0524-7ad6-300b-b050-6249135a2492,1.589799071259424e-05 +332,kg / USD,2021,2019,Basic,Carbon dioxide,emission/air,b6f010fb-a764-3063-af2d-bcb8309a97b7,0.8679043092159773 +332,kg / USD,2021,2019,Basic,Methane,emission/air,aab83476-ec6c-3742-af85-15d320b7ce80,0.004048842665206094 +332,kg / USD,2021,2019,Basic,Nitrous oxide,emission/air,cfee0524-7ad6-300b-b050-6249135a2492,2.0472571528900198e-05 +333,kg / USD,2021,2019,Basic,Carbon dioxide,emission/air,b6f010fb-a764-3063-af2d-bcb8309a97b7,0.40236484878090073 +333,kg / USD,2021,2019,Basic,Methane,emission/air,aab83476-ec6c-3742-af85-15d320b7ce80,0.002298824287881408 +333,kg / USD,2021,2019,Basic,Nitrous oxide,emission/air,cfee0524-7ad6-300b-b050-6249135a2492,9.018885403412663e-06 +334,kg / USD,2021,2019,Basic,Carbon dioxide,emission/air,b6f010fb-a764-3063-af2d-bcb8309a97b7,0.3465553490595418 +334,kg / USD,2021,2019,Basic,Methane,emission/air,aab83476-ec6c-3742-af85-15d320b7ce80,0.0011668211332355868 +334,kg / USD,2021,2019,Basic,Nitrous oxide,emission/air,cfee0524-7ad6-300b-b050-6249135a2492,6.58538487779637e-06 +335,kg / USD,2021,2019,Basic,Carbon dioxide,emission/air,b6f010fb-a764-3063-af2d-bcb8309a97b7,0.49959566891989293 +335,kg / USD,2021,2019,Basic,Methane,emission/air,aab83476-ec6c-3742-af85-15d320b7ce80,0.004113989777911597 +335,kg / USD,2021,2019,Basic,Nitrous oxide,emission/air,cfee0524-7ad6-300b-b050-6249135a2492,1.18423765485288e-05 +3361MV,kg / USD,2021,2019,Basic,Carbon dioxide,emission/air,b6f010fb-a764-3063-af2d-bcb8309a97b7,0.5042953638727882 +3361MV,kg / USD,2021,2019,Basic,Methane,emission/air,aab83476-ec6c-3742-af85-15d320b7ce80,0.002381144744189119 +3361MV,kg / USD,2021,2019,Basic,Nitrous oxide,emission/air,cfee0524-7ad6-300b-b050-6249135a2492,1.1607574276214475e-05 +3364OT,kg / USD,2021,2019,Basic,Carbon dioxide,emission/air,b6f010fb-a764-3063-af2d-bcb8309a97b7,0.41207381988436936 +3364OT,kg / USD,2021,2019,Basic,Methane,emission/air,aab83476-ec6c-3742-af85-15d320b7ce80,0.0020033414667239052 +3364OT,kg / USD,2021,2019,Basic,Nitrous oxide,emission/air,cfee0524-7ad6-300b-b050-6249135a2492,8.418029508362433e-06 +337,kg / USD,2021,2019,Basic,Carbon dioxide,emission/air,b6f010fb-a764-3063-af2d-bcb8309a97b7,0.8036226865445536 +337,kg / USD,2021,2019,Basic,Methane,emission/air,aab83476-ec6c-3742-af85-15d320b7ce80,0.0030603056540319733 +337,kg / USD,2021,2019,Basic,Nitrous oxide,emission/air,cfee0524-7ad6-300b-b050-6249135a2492,2.071803363452175e-05 +339,kg / USD,2021,2019,Basic,Carbon dioxide,emission/air,b6f010fb-a764-3063-af2d-bcb8309a97b7,0.4656505820730562 +339,kg / USD,2021,2019,Basic,Methane,emission/air,aab83476-ec6c-3742-af85-15d320b7ce80,0.0017103499825275913 +339,kg / USD,2021,2019,Basic,Nitrous oxide,emission/air,cfee0524-7ad6-300b-b050-6249135a2492,9.241529728653146e-06 +481,kg / USD,2021,2019,Basic,Carbon dioxide,emission/air,b6f010fb-a764-3063-af2d-bcb8309a97b7,0.0 +481,kg / USD,2021,2019,Basic,Methane,emission/air,aab83476-ec6c-3742-af85-15d320b7ce80,0.0 +481,kg / USD,2021,2019,Basic,Nitrous oxide,emission/air,cfee0524-7ad6-300b-b050-6249135a2492,0.0 +487OS,kg / USD,2021,2019,Basic,Carbon dioxide,emission/air,b6f010fb-a764-3063-af2d-bcb8309a97b7,0.0897928551474403 +487OS,kg / USD,2021,2019,Basic,Methane,emission/air,aab83476-ec6c-3742-af85-15d320b7ce80,0.0005714270227624544 +487OS,kg / USD,2021,2019,Basic,Nitrous oxide,emission/air,cfee0524-7ad6-300b-b050-6249135a2492,2.5772555096235905e-06 +511,kg / USD,2021,2019,Basic,Carbon dioxide,emission/air,b6f010fb-a764-3063-af2d-bcb8309a97b7,0.07953335722515441 +511,kg / USD,2021,2019,Basic,Methane,emission/air,aab83476-ec6c-3742-af85-15d320b7ce80,0.0006134601013131587 +511,kg / USD,2021,2019,Basic,Nitrous oxide,emission/air,cfee0524-7ad6-300b-b050-6249135a2492,2.826545751506698e-06 +512,kg / USD,2021,2019,Basic,Carbon dioxide,emission/air,b6f010fb-a764-3063-af2d-bcb8309a97b7,0.0 +512,kg / USD,2021,2019,Basic,Methane,emission/air,aab83476-ec6c-3742-af85-15d320b7ce80,0.0 +512,kg / USD,2021,2019,Basic,Nitrous oxide,emission/air,cfee0524-7ad6-300b-b050-6249135a2492,0.0 +513,kg / USD,2021,2019,Basic,Carbon dioxide,emission/air,b6f010fb-a764-3063-af2d-bcb8309a97b7,0.14639115646032028 +513,kg / USD,2021,2019,Basic,Methane,emission/air,aab83476-ec6c-3742-af85-15d320b7ce80,0.0013130765559578553 +513,kg / USD,2021,2019,Basic,Nitrous oxide,emission/air,cfee0524-7ad6-300b-b050-6249135a2492,4.170065777111074e-06 +514,kg / USD,2021,2019,Basic,Carbon dioxide,emission/air,b6f010fb-a764-3063-af2d-bcb8309a97b7,0.121186750371057 +514,kg / USD,2021,2019,Basic,Methane,emission/air,aab83476-ec6c-3742-af85-15d320b7ce80,0.0012218562409480277 +514,kg / USD,2021,2019,Basic,Nitrous oxide,emission/air,cfee0524-7ad6-300b-b050-6249135a2492,4.104580767661098e-06 +521CI,kg / USD,2021,2019,Basic,Carbon dioxide,emission/air,b6f010fb-a764-3063-af2d-bcb8309a97b7,0.0 +521CI,kg / USD,2021,2019,Basic,Methane,emission/air,aab83476-ec6c-3742-af85-15d320b7ce80,0.0 +521CI,kg / USD,2021,2019,Basic,Nitrous oxide,emission/air,cfee0524-7ad6-300b-b050-6249135a2492,0.0 +523,kg / USD,2021,2019,Basic,Carbon dioxide,emission/air,b6f010fb-a764-3063-af2d-bcb8309a97b7,0.0962108023806956 +523,kg / USD,2021,2019,Basic,Methane,emission/air,aab83476-ec6c-3742-af85-15d320b7ce80,0.0011871649334671735 +523,kg / USD,2021,2019,Basic,Nitrous oxide,emission/air,cfee0524-7ad6-300b-b050-6249135a2492,2.4185521353869875e-06 +524,kg / USD,2021,2019,Basic,Carbon dioxide,emission/air,b6f010fb-a764-3063-af2d-bcb8309a97b7,0.05128243718105348 +524,kg / USD,2021,2019,Basic,Methane,emission/air,aab83476-ec6c-3742-af85-15d320b7ce80,0.001023829547891879 +524,kg / USD,2021,2019,Basic,Nitrous oxide,emission/air,cfee0524-7ad6-300b-b050-6249135a2492,1.0640003603651764e-06 +5411,kg / USD,2021,2019,Basic,Carbon dioxide,emission/air,b6f010fb-a764-3063-af2d-bcb8309a97b7,0.10901552818620319 +5411,kg / USD,2021,2019,Basic,Methane,emission/air,aab83476-ec6c-3742-af85-15d320b7ce80,0.001973453440681415 +5411,kg / USD,2021,2019,Basic,Nitrous oxide,emission/air,cfee0524-7ad6-300b-b050-6249135a2492,2.6143645436264932e-06 +5412OP,kg / USD,2021,2019,Basic,Carbon dioxide,emission/air,b6f010fb-a764-3063-af2d-bcb8309a97b7,0.12858042942937947 +5412OP,kg / USD,2021,2019,Basic,Methane,emission/air,aab83476-ec6c-3742-af85-15d320b7ce80,0.00221604215328738 +5412OP,kg / USD,2021,2019,Basic,Nitrous oxide,emission/air,cfee0524-7ad6-300b-b050-6249135a2492,3.096411302762921e-06 +5415,kg / USD,2021,2019,Basic,Carbon dioxide,emission/air,b6f010fb-a764-3063-af2d-bcb8309a97b7,0.2501439663485212 +5415,kg / USD,2021,2019,Basic,Methane,emission/air,aab83476-ec6c-3742-af85-15d320b7ce80,0.0017208252217660999 +5415,kg / USD,2021,2019,Basic,Nitrous oxide,emission/air,cfee0524-7ad6-300b-b050-6249135a2492,9.638928802335164e-06 +561,kg / USD,2021,2019,Basic,Carbon dioxide,emission/air,b6f010fb-a764-3063-af2d-bcb8309a97b7,0.13736353187602388 +561,kg / USD,2021,2019,Basic,Methane,emission/air,aab83476-ec6c-3742-af85-15d320b7ce80,0.0022674465386572037 +561,kg / USD,2021,2019,Basic,Nitrous oxide,emission/air,cfee0524-7ad6-300b-b050-6249135a2492,3.5217385159697063e-06 +61,kg / USD,2021,2019,Basic,Carbon dioxide,emission/air,b6f010fb-a764-3063-af2d-bcb8309a97b7,0.14611532701383514 +61,kg / USD,2021,2019,Basic,Methane,emission/air,aab83476-ec6c-3742-af85-15d320b7ce80,0.004030697870012313 +61,kg / USD,2021,2019,Basic,Nitrous oxide,emission/air,cfee0524-7ad6-300b-b050-6249135a2492,2.6776754460712777e-06 +711AS,kg / USD,2021,2019,Basic,Carbon dioxide,emission/air,b6f010fb-a764-3063-af2d-bcb8309a97b7,0.03181982825924432 +711AS,kg / USD,2021,2019,Basic,Methane,emission/air,aab83476-ec6c-3742-af85-15d320b7ce80,0.0005414540603098257 +711AS,kg / USD,2021,2019,Basic,Nitrous oxide,emission/air,cfee0524-7ad6-300b-b050-6249135a2492,7.887170955025412e-07 +81,kg / USD,2021,2019,Basic,Carbon dioxide,emission/air,b6f010fb-a764-3063-af2d-bcb8309a97b7,0.09544382524937779 +81,kg / USD,2021,2019,Basic,Methane,emission/air,aab83476-ec6c-3742-af85-15d320b7ce80,0.0013703236240304847 +81,kg / USD,2021,2019,Basic,Nitrous oxide,emission/air,cfee0524-7ad6-300b-b050-6249135a2492,2.29392341110287e-06 +GFE,kg / USD,2021,2019,Basic,Carbon dioxide,emission/air,b6f010fb-a764-3063-af2d-bcb8309a97b7,0.0718987126733196 +GFE,kg / USD,2021,2019,Basic,Methane,emission/air,aab83476-ec6c-3742-af85-15d320b7ce80,0.0006531104954037798 +GFE,kg / USD,2021,2019,Basic,Nitrous oxide,emission/air,cfee0524-7ad6-300b-b050-6249135a2492,1.6066683707397594e-06 +Used,kg / USD,2021,2019,Basic,Carbon dioxide,emission/air,b6f010fb-a764-3063-af2d-bcb8309a97b7,0.0 +Used,kg / USD,2021,2019,Basic,Methane,emission/air,aab83476-ec6c-3742-af85-15d320b7ce80,0.0 +Used,kg / USD,2021,2019,Basic,Nitrous oxide,emission/air,cfee0524-7ad6-300b-b050-6249135a2492,0.0 diff --git a/Imports Script/useeio_imports_script.py b/Imports Script/useeio_imports_script.py index 22fe0f7..5603a4c 100644 --- a/Imports Script/useeio_imports_script.py +++ b/Imports Script/useeio_imports_script.py @@ -3,7 +3,7 @@ import pickle as pkl import yaml import statistics -#from currency_converter import CurrencyConverter +from currency_converter import CurrencyConverter from datetime import date from pathlib import Path from API_Imports_Data_Script import get_imports_data @@ -38,6 +38,10 @@ dataPath = Path(__file__).parent / 'Data' conPath = Path(__file__).parent / 'Concordances' +flow_cols = ('Flow', 'Compartment', 'Unit', + 'CurrencyYear', 'EmissionYear', 'PriceType', + 'Flowable', 'Context', 'Flow UUID') + #%% with open(dataPath.parent / "Data" / "exio_config.yml", "r") as file: @@ -49,7 +53,7 @@ def run_script(io_level='Summary', year=2021): Runs through script to produce emission factors for U.S. imports. ''' # Country imports by detail sector - sr_i = get_subregion_imports() + sr_i = get_subregion_imports(year) if io_level == 'Summary': u_c = get_detail_to_summary_useeio_concordance() @@ -67,7 +71,7 @@ def run_script(io_level='Summary', year=2021): print('Error calculating country coefficients by detail sector') e_u = get_exio_to_useeio_concordance() - e_d = pull_exiobase_multipliers() + e_d = pull_exiobase_multipliers(year) e_d = (e_d.merge(e_u, on='Exiobase Sector', how='left') .groupby(['BEA Detail', 'CountryCode']) .agg('mean') @@ -84,6 +88,30 @@ def run_script(io_level='Summary', year=2021): var_name = 'Flow', value_name = 'EF') + multiplier_df = ( + multiplier_df + .assign(Compartment='emission/air') + .assign(Unit='kg / Euro') + .assign(CurrencyYear=str(year)) + .assign(EmissionYear='2019' if year > 2019 else str(year)) + # ^^ GHG data stops at 2019 + .assign(PriceType='Basic') + ) + + import fedelemflowlist as fedelem + fl = (fedelem.get_flows() + .query('Flowable in @multiplier_df.Flow') + .filter(['Flowable', 'Context', 'Flow UUID']) + ) + multiplier_df = ( + multiplier_df + .merge(fl, how='left', + left_on=['Flow', 'Compartment'], + right_on=['Flowable', 'Context'], + ) + .drop(columns=['Flow', 'Compartment']) + ) + weighted_multipliers_bea_detail, weighted_multipliers_bea_summary = ( calculate_specific_emission_factors(multiplier_df)) @@ -92,38 +120,21 @@ def run_script(io_level='Summary', year=2021): imports_multipliers = (calculateWeightedEFsImportsData( weighted_multipliers_bea_summary, t_c)) + # Currency adjustment + c = CurrencyConverter(fallback_on_missing_rate=True) + exch = statistics.mean([c.convert(1, 'EUR', 'USD', date=date(year, 1, 1)), + c.convert(1, 'EUR', 'USD', date=date(year, 12, 30))]) imports_multipliers = ( imports_multipliers - # .assign(Compartment='air') - .rename(columns={'Weighted_Import_EF': 'Amount'}) - .assign(Unit='kg / Euro') - .assign(CurrencyYear=str(year)) - .assign(DataYear=str(year)) # Emissions year - .assign(PriceType='Producer') + .assign(Amount=lambda x: x['Amount']/exch) + .assign(Unit='kg / USD') ) - - # Currency adjustment - # c = CurrencyConverter(fallback_on_missing_rate=True) - # exch = statistics.mean([c.convert(1, 'EUR', 'USD', date=date(year, 1, 1)), - # c.convert(1, 'EUR', 'USD', date=date(year, 12, 30))]) - # imports_multipliers = ( - # imports_multipliers - # .assign(Amount=lambda x: x['Amount']/exch) - # .assign(Unit='kg / USD') - # ) - - - #TODO Pricetype adjustment - - #TODO Flow Mapping - - return (p_d, imports_multipliers, weighted_multipliers_bea_detail, + return (sr_i, imports_multipliers, weighted_multipliers_bea_detail, weighted_multipliers_bea_summary) -# TODO reflect the year of the data in the csv file -def get_tiva_data(year='2020'): +def get_tiva_data(year): ''' Iteratively pulls BEA imports data matricies from stored csv file, extracts the Total Imports columns by region, and consolidates @@ -132,7 +143,7 @@ def get_tiva_data(year='2020'): https://apps.bea.gov/iTable/?reqid=157&step=1 ''' - f_n = 'Import Matrix, __region__, After Redefinitions.csv' + f_n = f'Import Matrix, __region__, After Redefinitions_{year}.csv' regions = {'Canada': 'CA', 'China': 'CN', 'Europe': 'EU', @@ -151,7 +162,7 @@ def get_tiva_data(year='2020'): .apply(pd.to_numeric) ) df[abbv] = df[list(df.columns)].sum(axis=1) - df = df.reset_index(inplace=False) + df = df.reset_index() ri_r = df[['IOCode', abbv]] if ri_df.empty: ri_df = ri_r @@ -195,18 +206,18 @@ def calc_tiva_coefficients(year): return t_c -def download_and_store_mrio(): +def download_and_store_mrio(year): ''' If MRIO object not already present in directory, downloads MRIO object. ''' - file = dataPath / 'IOT_2022_pxp.zip' + file = dataPath / f'IOT_{year}_pxp.zip' if not file.exists(): exio3 = pymrio.download_exiobase3(storage_folder=dataPath, system='pxp', - years=[2022]) + years=[year]) e = pymrio.parse_exiobase3(file) exio_m = e.impacts.M - pkl.dump(exio_m, open(dataPath / 'exio3_multipliers.pkl', 'wb')) + pkl.dump(exio_m, open(dataPath / f'exio3_multipliers_{year}.pkl', 'wb')) def remove_exports(dataframe): @@ -235,6 +246,7 @@ def get_exio_to_useeio_concordance(): Transforms wide-form Exiobase to USEEIO concordance into long form, extracts all mappings to create new, two column concordance consisting of USEEIO detail and mappings to Exiobase. + modified slightly from: https://ntnu.app.box.com/v/EXIOBASEconcordances/file/983477211189 ''' path = conPath / "exio_to_bea_commodity_concordance.csv" e_u_b = (pd.read_csv(path, dtype=str) @@ -263,16 +275,16 @@ def get_detail_to_summary_useeio_concordance(): return u_c -def get_subregion_imports(): #TO-DO: Reconstruct using census and BEA data +def get_subregion_imports(year): ''' - Extracts industry output vector from exiobase pkl file. + Generates dataset of imports by country by sector from BEA and Census ''' - sr_i = get_imports_data(False) - sr_i = sr_i[['BEA Sector','CountryCode','Import Quantity']] + sr_i = get_imports_data(False, data_years=[year]) path = conPath / 'exio_tiva_concordance.csv' - regions = (pd.read_csv(path, dtype=str, usecols=['ISO 3166-alpha-2', - 'TiVA Region']) - .rename(columns={'ISO 3166-alpha-2': 'CountryCode'})) + regions = (pd.read_csv(path, dtype=str, + usecols=['ISO 3166-alpha-2', 'TiVA Region']) + .rename(columns={'ISO 3166-alpha-2': 'CountryCode'}) + ) sr_i = (sr_i.merge(regions, on='CountryCode', how='left') .rename(columns={'BEA Sector':'BEA Detail'})) # sr_i['Subregion Contribution'] = sr_i['Import Quantity']/sr_i.groupby('BEA Sector')['Import Quantity'].transform('sum') @@ -280,14 +292,14 @@ def get_subregion_imports(): #TO-DO: Reconstruct using census and BEA data return sr_i -def pull_exiobase_multipliers(): +def pull_exiobase_multipliers(year): ''' Extracts multiplier matrix from stored Exiobase model. ''' - file = dataPath/'exio3_multipliers.pkl' + file = dataPath/f'exio3_multipliers_{year}.pkl' if not file.exists(): - download_and_store_mrio() + download_and_store_mrio(year) M_df = pkl.load(open(file,'rb')) fields = {**config['fields'], **config['flows']} @@ -368,18 +380,20 @@ def calculate_specific_emission_factors(multiplier_df): ''' multiplier_df = (multiplier_df - .assign(Weighted_BEA_Detail = (multiplier_df['EF'] * + .assign(Amount = (multiplier_df['EF'] * multiplier_df['Subregion Contribution to Detail'])) - .assign(Weighted_BEA_Summary = (multiplier_df['EF'] * + .assign(Amount = (multiplier_df['EF'] * multiplier_df['Subregion Contribution to Summary'])) ) + col = [c for c in multiplier_df if c in flow_cols] + weighted_multipliers_bea_detail = (multiplier_df - .groupby(['TiVA Region','BEA Detail', 'Flow']) - .agg({'Weighted_BEA_Detail': 'sum'}).reset_index()) + .groupby(['TiVA Region','BEA Detail'] + col) + .agg({'Amount': 'sum'}).reset_index()) weighted_multipliers_bea_summary = (multiplier_df - .groupby(['TiVA Region','BEA Summary', 'Flow']) - .agg({'Weighted_BEA_Summary': 'sum'}).reset_index()) + .groupby(['TiVA Region','BEA Summary'] + col) + .agg({'Amount': 'sum'}).reset_index()) return(weighted_multipliers_bea_detail, weighted_multipliers_bea_summary) @@ -395,7 +409,6 @@ def calculateWeightedEFsImportsData(weighted_multipliers, are dropped from the dataframe. Other than weighted burden columns, the output dataframe only continues to include 'USEEIO Summary' codes. ''' - print("") weighted_df_imports = ( weighted_multipliers .merge(import_contribution_coeffs, how='left', validate='m:1', @@ -405,15 +418,17 @@ def calculateWeightedEFsImportsData(weighted_multipliers, ) weighted_df_imports = ( - weighted_df_imports.assign(Weighted_Import_EF=lambda x: - x['Weighted_BEA_Summary'] * + weighted_df_imports.assign(Amount=lambda x: + x['Amount'] * x['region_contributions_imports']) ) + col = [c for c in weighted_df_imports if c in flow_cols] + imports_multipliers = ( weighted_df_imports - .groupby(['BEA Summary', 'Flow']) - .agg({'Weighted_Import_EF': 'sum'}) + .groupby(['BEA Summary'] + col) + .agg({'Amount': 'sum'}) .reset_index() ) @@ -422,7 +437,7 @@ def calculateWeightedEFsImportsData(weighted_multipliers, #%% if __name__ == '__main__': - (prepared_dataframe, imports_multipliers, weighted_multipliers_bea_detail, - weighted_multipliers_bea_summary) = run_script() + (import_totals, imports_multipliers, weighted_multipliers_bea_detail, + weighted_multipliers_bea_summary) = run_script(year=2021) imports_multipliers.to_csv('imports_multipliers.csv', index=False)