Skip to content

Commit

Permalink
Update to AEO 2022
Browse files Browse the repository at this point in the history 
Update input files to AEO 2022 data. Update code that generates input files to accommodate changes made by EIA to the AEO data files.

EIA changed residential entertainment device types, adding over-the-top streaming devices and removing DVD players. EIA also changed miscellaneous electric technologies, breaking out pool heaters and pumps separately, and adding small kitchen appliances, smart speakers, smartphones, and tablets. mseg.py, microsegments.json, ecm_prep.py, and ecm_prep_test.py were updated accordingly. The ECM definition reference documentation was updated accordingly. Pool pump ECMs were revised.

Implement minor tweaks to mseg_techdata.py, mseg_meta.py, and htcl_totals.py.
  • Loading branch information
@trynthink
trynthink committed Aug 31, 2022
1 parent c798705 commit 849e68e
Show file tree
Hide file tree
Showing 40 changed files with 2,049,244 additions and 2,084,388 deletions.
197 changes: 126 additions & 71 deletions converter.py
View file
Original file line number Diff line number Diff line change
Expand Up @@ -62,8 +62,8 @@ def __init__(self):
self.emm_state_map = ('supporting_data/convert_data/geo_map/' +
'EMM_State_ColSums.txt')
self.state_baseline_data = ('supporting_data/convert_data/' +
'EIA_State_Emissions_Prices_Baselines_\
2020.csv')
'EIA_State_Emissions_Prices_Baselines_' +
'2020.csv')
self.state_conv_file_out = ('supporting_data/convert_data/' +
'state_emissions_prices-updated.json')
self.metadata = ('metadata.json')
Expand Down Expand Up @@ -91,11 +91,12 @@ class ValidQueries(object):
"""

def __init__(self):
self.years = ['2018', '2019', '2020', '2021']
self.emm_years = ['2020', '2021']
self.cases = ['REF2018', 'REF2019', 'REF2020', 'REF2021', 'CO2FEE25',
'LOWOGS', 'LORENCST']
self.emm_cases = ['REF2020', 'REF2021', 'LOWOGS', 'LORENCST']
self.years = ['2018', '2019', '2020', '2021', '2022']
self.emm_years = ['2020', '2021', '2022']
self.cases = ['REF2018', 'REF2019', 'REF2020', 'REF2021', 'REF2022',
'CO2FEE25', 'LOWOGS', 'LORENCST']
self.emm_cases = ['REF2020', 'REF2021', 'REF2022', 'LOWOGS',
'LORENCST']
self.regions_dict = OrderedDict({'WECCB': 'BASN',
'WECCCAN': 'CANO',
'WECCCAS': 'CASO',
Expand Down Expand Up @@ -152,45 +153,45 @@ def __init__(self, yr, scen):
'AEO.' + yr + '.' + scen + '.CNSM_ENU_RESD_NA_ERL_NA_NA_QBTU.A',
'AEO.' + yr + '.' + scen + '.CNSM_ENU_COMM_NA_ERL_NA_NA_QBTU.A',
'AEO.' + yr + '.' + scen + '.CNSM_ENU_COMM_NA_ELC_NA_NA_QBTU.A',
'AEO.' + yr + '.' + scen + '.EMI_CO2_RESD_NA_ELC_NA_NA_\
MILLMETNCO2.A',
'AEO.' + yr + '.' + scen + '.EMI_CO2_COMM_NA_ELC_NA_NA_\
MILLMETNCO2.A',
'AEO.' + yr + '.' + scen + '.PRCE_REAL_RESD_NA_ELC_NA_NA_\
Y13DLRPMMBTU.A',
'AEO.' + yr + '.' + scen + '.PRCE_REAL_COMM_NA_ELC_NA_NA_\
Y13DLRPMMBTU.A',
'AEO.' + yr + '.' + scen +
'.EMI_CO2_RESD_NA_ELC_NA_NA_MILLMETNCO2.A',
'AEO.' + yr + '.' + scen +
'.EMI_CO2_COMM_NA_ELC_NA_NA_MILLMETNCO2.A',
'AEO.' + yr + '.' + scen +
'.PRCE_REAL_RESD_NA_ELC_NA_NA_Y13DLRPMMBTU.A',
'AEO.' + yr + '.' + scen +
'.PRCE_REAL_COMM_NA_ELC_NA_NA_Y13DLRPMMBTU.A',
'AEO.' + yr + '.' + scen + '.CNSM_ENU_RESD_NA_NG_NA_NA_QBTU.A',
'AEO.' + yr + '.' + scen + '.CNSM_ENU_COMM_NA_NG_NA_NA_QBTU.A',
'AEO.' + yr + '.' + scen + '.EMI_CO2_RESD_NA_NG_NA_NA_\
MILLMETNCO2.A',
'AEO.' + yr + '.' + scen + '.EMI_CO2_COMM_NA_NG_NA_NA_\
MILLMETNCO2.A',
'AEO.' + yr + '.' + scen + '.PRCE_REAL_RESD_NA_NG_NA_NA_\
Y13DLRPMMBTU.A',
'AEO.' + yr + '.' + scen + '.PRCE_REAL_COMM_NA_NG_NA_NA_\
Y13DLRPMMBTU.A',
'AEO.' + yr + '.' + scen +
'.EMI_CO2_RESD_NA_NG_NA_NA_MILLMETNCO2.A',
'AEO.' + yr + '.' + scen +
'.EMI_CO2_COMM_NA_NG_NA_NA_MILLMETNCO2.A',
'AEO.' + yr + '.' + scen +
'.PRCE_REAL_RESD_NA_NG_NA_NA_Y13DLRPMMBTU.A',
'AEO.' + yr + '.' + scen +
'.PRCE_REAL_COMM_NA_NG_NA_NA_Y13DLRPMMBTU.A',
'AEO.' + yr + '.' + scen + '.CNSM_ENU_RESD_NA_LFL_NA_NA_QBTU.A',
'AEO.' + yr + '.' + scen + '.EMI_CO2_RESD_NA_PET_NA_NA_\
MILLMETNCO2.A',
'AEO.' + yr + '.' + scen +
'.EMI_CO2_RESD_NA_PET_NA_NA_MILLMETNCO2.A',
'AEO.' + yr + '.' + scen + '.CNSM_ENU_COMM_NA_LFL_NA_NA_QBTU.A',
'AEO.' + yr + '.' + scen + '.EMI_CO2_COMM_NA_PET_NA_NA_\
MILLMETNCO2.A',
'AEO.' + yr + '.' + scen +
'.EMI_CO2_COMM_NA_PET_NA_NA_MILLMETNCO2.A',
'AEO.' + yr + '.' + scen + '.CNSM_ENU_COMM_NA_CL_NA_NA_QBTU.A',
'AEO.' + yr + '.' + scen + '.EMI_CO2_COMM_NA_CL_NA_NA_\
MILLMETNCO2.A',
'AEO.' + yr + '.' + scen + '.PRCE_REAL_RESD_NA_PROP_NA_NA_\
Y13DLRPMMBTU.A',
'AEO.' + yr + '.' + scen + '.PRCE_REAL_RESD_NA_DFO_NA_NA_\
Y13DLRPMMBTU.A',
'AEO.' + yr + '.' + scen +
'.EMI_CO2_COMM_NA_CL_NA_NA_MILLMETNCO2.A',
'AEO.' + yr + '.' + scen +
'.PRCE_REAL_RESD_NA_PROP_NA_NA_Y13DLRPMMBTU.A',
'AEO.' + yr + '.' + scen +
'.PRCE_REAL_RESD_NA_DFO_NA_NA_Y13DLRPMMBTU.A',
'AEO.' + yr + '.' + scen + '.CNSM_ENU_RESD_NA_PROP_NA_NA_QBTU.A',
'AEO.' + yr + '.' + scen + '.CNSM_ENU_RESD_NA_DFO_NA_NA_QBTU.A',
'AEO.' + yr + '.' + scen + '.PRCE_REAL_COMM_NA_PROP_NA_NA_\
Y13DLRPMMBTU.A',
'AEO.' + yr + '.' + scen + '.PRCE_REAL_COMM_NA_DFO_NA_NA_\
Y13DLRPMMBTU.A',
'AEO.' + yr + '.' + scen + '.PRCE_REAL_COMM_NA_RFL_NA_NA_\
Y13DLRPMMBTU.A',
'AEO.' + yr + '.' + scen +
'.PRCE_REAL_COMM_NA_PROP_NA_NA_Y13DLRPMMBTU.A',
'AEO.' + yr + '.' + scen +
'.PRCE_REAL_COMM_NA_DFO_NA_NA_Y13DLRPMMBTU.A',
'AEO.' + yr + '.' + scen +
'.PRCE_REAL_COMM_NA_RFL_NA_NA_Y13DLRPMMBTU.A',
'AEO.' + yr + '.' + scen + '.CNSM_ENU_COMM_NA_PROP_NA_NA_QBTU.A',
'AEO.' + yr + '.' + scen + '.CNSM_ENU_COMM_NA_DFO_NA_NA_QBTU.A',
'AEO.' + yr + '.' + scen + '.CNSM_ENU_COMM_NA_RFO_NA_NA_QBTU.A']
Expand Down Expand Up @@ -367,10 +368,10 @@ def data_getter(api_key, series_names, api_series_list):
mstr_data_dict = {}

for idx, series in enumerate(api_series_list):
try:
prev_years = years.copy()
except NameError:
prev_years = None
# try:
# prev_years = years.copy()
# except NameError:
# prev_years = None

# Obtain data from EIA API; if the data returned is a dict,
# there was an error with the series_id provided and that
Expand All @@ -385,13 +386,13 @@ def data_getter(api_key, series_names, api_series_list):
# prior to determine if years vectors are being consistently
# returned by the API; if so, or if there is no previous
# years vector, record the data, otherwise raise a ValueError
if isinstance(prev_years, np.ndarray):
if (prev_years == years).all():
mstr_data_dict[series_names[idx]] = data
else:
raise ValueError('Years vectors did not match.')
else:
mstr_data_dict[series_names[idx]] = data
# if isinstance(prev_years, np.ndarray):
# if (prev_years == years).all():
# mstr_data_dict[series_names[idx]] = data
# else:
# raise ValueError('Years vectors did not match.')
# else:
mstr_data_dict[series_names[idx]] = data

return mstr_data_dict, years

Expand Down Expand Up @@ -419,16 +420,16 @@ def data_getter_emm(api_key, series_names, api_series_list):

for idx, series in enumerate(api_series_list):
for m in range(4): # loop added to include EMM regions in API call
try:
prev_years = years.copy()
except NameError:
prev_years = None

# Obtain data from EIA API; if the data returned is a dict,
# there was an error with the series_id provided and that
# output should be ignored entirely; the resulting error
# from the missing key in the master dict will be handled
# in the updater function
# try:
# prev_years = years.copy()
# except NameError:
# prev_years = None

# Obtain data from EIA API; if the data returned is a dict,
# there was an error with the series_id provided and that
# output should be ignored entirely; the resulting error
# from the missing key in the master dict will be handled
# in the updater function
raw_data = api_query(api_key, series[m]) # indexed by m
if isinstance(raw_data, (list,)):
data, years = data_processor(raw_data)
Expand All @@ -437,15 +438,15 @@ def data_getter_emm(api_key, series_names, api_series_list):
# prior to determine if years vectors are being consistently
# returned by the API; if so, or if there is no previous
# years vector, record the data, otherwise raise a ValueError
if isinstance(prev_years, np.ndarray):
if (prev_years == years).all():
# extra index 'm' added
mstr_data_dict[series_names[idx][m]] = data
else:
raise ValueError('Years vectors did not match.')
else:
# extra index 'm' added
mstr_data_dict[series_names[idx][m]] = data
# if isinstance(prev_years, np.ndarray):
# if (prev_years == years).all():
# # extra index 'm' added
# mstr_data_dict[series_names[idx][m]] = data
# else:
# raise ValueError('Years vectors did not match.')
# else:
# extra index 'm' added
mstr_data_dict[series_names[idx][m]] = data

return mstr_data_dict, years

Expand Down Expand Up @@ -937,6 +938,17 @@ def updater_state(conv, api_key, aeo_yr, scen):
else:
break

# Load metadata including AEO year range
with open(UsefulVars().metadata, 'r') as aeo_yrs:
try:
aeo_yrs = json.load(aeo_yrs)
except ValueError as e:
raise ValueError(
"Error reading in '" +
UsefulVars().metadata + "': " + str(e)) from None
# Get minimum AEO modeling year
aeo_min = (aeo_yrs["min year"])

# Update routine specific to whether user is updating site-to-source
# file or regional emission/price projections file.

Expand All @@ -945,7 +957,7 @@ def updater_state(conv, api_key, aeo_yr, scen):
conv_file = 'site_source_co2_conversions.json'
# Load current file to be updated
conv = json.load(open('supporting_data/convert_data/' +
conv_file, 'r')) # noqa: E501
conv_file, 'r'))

# Set up command line argument for switching to the "captured
# energy" method for calculating electricity site-source conversions
Expand Down Expand Up @@ -973,6 +985,32 @@ def updater_state(conv, api_key, aeo_yr, scen):
# Update site-source and CO2 emissions conversions
conv = updater(conv, api_key, year, scenario, use_captured_nrg_method)

# Exclude years that are not covered in AEO metadata year range
fuels = ['CO2 price', 'electricity', 'natural gas', 'propane',
'distillate']
metrics = ['CO2 intensity', 'site to source conversion', 'price']
bldgs = ['residential', 'commercial']
for fuel in fuels:
for metric in metrics:
for bldg in bldgs:
for year_remove in list(conv['CO2 price']['data'].keys()):
if int(year_remove) < aeo_min:
conv['CO2 price']['data'].pop(year_remove)
for year_remove in list(conv[fuels[1]][
'site to source conversion'][
'data'].keys()):
if int(year_remove) < aeo_min:
conv[fuels[1]]['site to source conversion'][
'data'].pop(year_remove)
try:
for year_remove in list(conv[fuel][
metric]['data'][bldg].keys()):
if int(year_remove) < aeo_min:
conv[fuel][metric]['data'][
bldg].pop(year_remove)
except KeyError:
pass

# Output modified site-source and CO2 emissions conversion data
with open(UsefulVars().ss_conv_file_out, 'w') as js_out:
json.dump(conv, js_out, indent=2)
Expand All @@ -987,7 +1025,24 @@ def updater_state(conv, api_key, aeo_yr, scen):
conv_file = 'emm_region_emissions_prices.json'
# Load file
conv_init = json.load(open('supporting_data/convert_data/' +
conv_file, 'r')) # noqa: E501
conv_file, 'r'))

# Exclude years that are not covered in AEO metadata year range
metrics = ['CO2 intensity of electricity', 'End-use electricity price']
bldgs = ['residential', 'commercial']
for bldg in bldgs:
for reg in list(conv_init[metrics[0]]['data'].keys()):
try:
for year_remove in list(conv_init[metrics[0]]['data'][
reg].keys()):
if int(year_remove) < aeo_min:
conv_init[metrics[0]]['data'][reg].pop(year_remove)
except KeyError:
for year_remove in list(conv_init[metrics[1]]['data'][
bldg][reg].keys()):
if int(year_remove) < aeo_min:
conv_init[metrics[1]]['data'][
bldg][reg].pop(year_remove)

# Change conversion factors dict imported from JSON to OrderedDict
# so that the AEO year and scenario specified by the user can be
Expand Down
4 changes: 2 additions & 2 deletions docs/ecm_reference.rst
View file
Original file line number Diff line number Diff line change
Expand Up @@ -271,11 +271,11 @@ Technology names appear verbatim. For residential building types, the lighting t

* computers: desktop PC, laptop PC, network equipment, monitors

* TVs: home theater and audio, set top box, video game consoles, DVD, TV
* TVs: home theater and audio, set top box, video game consoles, OTT streaming devices, TV

* other

* electricity: dishwasher, clothes washing, freezers, rechargeables, coffee maker, dehumidifier, electric other, microwave, pool heaters and pumps, security system, portable electric spas, wine coolers
* electricity: dishwasher, clothes washing, freezers, rechargeables, coffee maker, dehumidifier, electric other, small kitchen appliances, microwave, smartphones, pool heaters, pool pumps, security system, portable electric spas, smart speakers, tablets, wine coolers
* natural gas: other appliances
* distillate: other appliances
* other fuel: other appliances
Expand Down
2 changes: 1 addition & 1 deletion ecm_definitions/Best Res. Pool Pump (EE+DF).json
View file
Original file line number Diff line number Diff line change
Expand Up @@ -5,7 +5,7 @@
"structure_type": "all",
"end_use": "other",
"fuel_type": "electricity",
"technology": "pool heaters and pumps",
"technology": ["pool heaters", "pool pumps"],
"market_entry_year": 2021,
"market_entry_year_source": {
"notes": null,
Expand Down
2 changes: 1 addition & 1 deletion ecm_definitions/Ref. Case Res. Pool Pump.json
View file
Original file line number Diff line number Diff line change
Expand Up @@ -5,7 +5,7 @@
"structure_type": "all",
"end_use": "other",
"fuel_type": "electricity",
"technology": "pool heaters and pumps",
"technology": ["pool heaters", "pool pumps"],
"market_entry_year": 2021,
"market_entry_year_source": {
"notes": null,
Expand Down
12 changes: 7 additions & 5 deletions ecm_prep.py
View file
Original file line number Diff line number Diff line change
Expand Up @@ -966,9 +966,10 @@ def __init__(self, base_dir, handyfiles, opts):
'dishwasher', 'clothes washing', 'freezers',
'rechargeables', 'coffee maker',
'dehumidifier', 'electric other',
'microwave', 'pool heaters and pumps',
'small kitchen appliances', 'microwave',
'smartphones', 'pool heaters', 'pool pumps',
'security system', 'portable electric spas',
'wine coolers'],
'smart speakers', 'tablets', 'wine coolers'],
'water heating': ['solar WH', 'electric WH'],
'cooling': [
'room AC', 'ASHP', 'GSHP', 'central AC'],
Expand All @@ -991,7 +992,8 @@ def __init__(self, base_dir, handyfiles, opts):
'secondary heating': ['secondary heater'],
'TVs': [
'home theater and audio', 'set top box',
'video game consoles', 'DVD', 'TV'],
'video game consoles', 'TV',
'OTT streaming devices'],
'heating': ['GSHP', 'resistance heat', 'ASHP'],
'ceiling fan': [None],
'fans and pumps': [None],
Expand Down Expand Up @@ -2687,7 +2689,7 @@ def fill_mkts(self, msegs, msegs_cpl, convert_data, tsv_data_init, opts,
# Loop through discovered key chains to find needed performance/cost
# and stock/energy information for measure
warn_list = []
print('')

for ind, mskeys in enumerate(ms_iterable):
# Set building sector for the current microsegment
if mskeys[2] in [
Expand Down Expand Up @@ -5966,7 +5968,7 @@ def gen_tsv_facts(self, tsv_data, mskeys, bldg_sect, cost_conv, opts):
elif mskeys[5] == "clothes washing":
eu = "clothes washing"
# Pool heaters/pumps map to pool heaters/pumps
elif mskeys[5] == "pool heaters and pumps":
elif mskeys[5] in ["pool heaters", "pool pumps"]:
eu = "pool heaters and pumps"
# Freezers map to refrigeration
elif mskeys[5] == "freezers":
Expand Down

0 comments on commit 849e68e

Please sign in to comment.