Skip to content

Commit

Permalink
first draft of enhancement to allow output aggregated to daily, month…
Browse files Browse the repository at this point in the history 
…ly, or annual
  • Loading branch information
@PaulDudaRESPEC
PaulDudaRESPEC committed Nov 1, 2023
1 parent e78f84c commit 4431180
Show file tree
Hide file tree
Showing 5 changed files with 79 additions and 45 deletions.
26 changes: 21 additions & 5 deletions HSP2/main.py
View file
Original file line number Diff line number Diff line change
Expand Up @@ -238,14 +238,30 @@ def main(io_manager:IOManager, saveall:bool=False, jupyterlab:bool=True) -> None
for errorcnt, errormsg in zip(errors, errmessages):
if errorcnt > 0:
msg(4, f'Error count {errorcnt}: {errormsg}')

# default to hourly output
outstep = 2
outstep_oxrx = 2
outstep_nutrx = 2
outstep_plank = 2
outstep_phcarb = 2
if 'BINOUT' in uci[(operation, 'GENERAL', segment)]:
if activity in uci[(operation, 'GENERAL', segment)]['BINOUT']:
outstep = uci[(operation, 'GENERAL', segment)]['BINOUT'][activity]
elif activity == 'RQUAL':
outstep_oxrx = uci[(operation, 'GENERAL', segment)]['BINOUT']['OXRX']
outstep_nutrx = uci[(operation, 'GENERAL', segment)]['BINOUT']['NUTRX']
outstep_plank = uci[(operation, 'GENERAL', segment)]['BINOUT']['PLANK']
outstep_phcarb = uci[(operation, 'GENERAL', segment)]['BINOUT']['PHCARB']

if 'SAVE' in ui:
save_timeseries(io_manager,ts,ui['SAVE'],siminfo,saveall,operation,segment,activity,jupyterlab)
save_timeseries(io_manager,ts,ui['SAVE'],siminfo,saveall,operation,segment,activity,jupyterlab,outstep)

if (activity == 'RQUAL'):
if 'SAVE' in ui_oxrx: save_timeseries(io_manager,ts,ui_oxrx['SAVE'],siminfo,saveall,operation,segment,'OXRX',jupyterlab)
if 'SAVE' in ui_nutrx and flags['NUTRX'] == 1: save_timeseries(io_manager,ts,ui_nutrx['SAVE'],siminfo,saveall,operation,segment,'NUTRX',jupyterlab)
if 'SAVE' in ui_plank and flags['PLANK'] == 1: save_timeseries(io_manager,ts,ui_plank['SAVE'],siminfo,saveall,operation,segment,'PLANK',jupyterlab)
if 'SAVE' in ui_phcarb and flags['PHCARB'] == 1: save_timeseries(io_manager,ts,ui_phcarb['SAVE'],siminfo,saveall,operation,segment,'PHCARB',jupyterlab)
if 'SAVE' in ui_oxrx: save_timeseries(io_manager,ts,ui_oxrx['SAVE'],siminfo,saveall,operation,segment,'OXRX',jupyterlab,outstep_oxrx)
if 'SAVE' in ui_nutrx and flags['NUTRX'] == 1: save_timeseries(io_manager,ts,ui_nutrx['SAVE'],siminfo,saveall,operation,segment,'NUTRX',jupyterlab,outstep_nutrx)
if 'SAVE' in ui_plank and flags['PLANK'] == 1: save_timeseries(io_manager,ts,ui_plank['SAVE'],siminfo,saveall,operation,segment,'PLANK',jupyterlab,outstep_plank)
if 'SAVE' in ui_phcarb and flags['PHCARB'] == 1: save_timeseries(io_manager,ts,ui_phcarb['SAVE'],siminfo,saveall,operation,segment,'PHCARB',jupyterlab,outstep_phcarb)

msglist = msg(1, 'Done', final=True)

Expand Down
5 changes: 3 additions & 2 deletions HSP2/utilities.py
View file
Original file line number Diff line number Diff line change
Expand Up @@ -280,7 +280,7 @@ def get_timeseries(timeseries_inputs:SupportsReadTS, ext_sourcesdd, siminfo):
ts[tname] = t
return ts

def save_timeseries(timeseries:SupportsWriteTS, ts, savedict, siminfo, saveall, operation, segment, activity, compress=True):
def save_timeseries(timeseries:SupportsWriteTS, ts, savedict, siminfo, saveall, operation, segment, activity, compress=True, outstep=2):
df = pd.DataFrame(index=siminfo['tindex'])
if (operation == 'IMPLND' and activity == 'IQUAL') or (operation == 'PERLND' and activity == 'PQUAL'):
for y in savedict.keys():
Expand Down Expand Up @@ -316,7 +316,8 @@ def save_timeseries(timeseries:SupportsWriteTS, ts, savedict, siminfo, saveall,
operation=operation,
segment=segment,
activity=activity,
compress=compress
compress=compress,
outstep=outstep
)
else:
print(f'DataFrame Empty for {operation}|{activity}|{segment}')
Expand Down
16 changes: 16 additions & 0 deletions HSP2IO/io.py
View file
Original file line number Diff line number Diff line change
Expand Up @@ -62,6 +62,7 @@ def write_ts(self,
operation:Union[str,None]=None,
segment:Union[str,None]=None,
activity:Union[str,None]=None,
outstep:int=2,
*args, **kwargs) -> None:
key = (category, operation, segment, activity)
self._in_memory[key] = data_frame.copy(deep=True)
Expand All @@ -70,6 +71,21 @@ def write_ts(self,
if drop_columns:
data_frame = data_frame.drop(columns=drop_columns)

if outstep == 3:
# change time step of output to daily
sumdf1 = data_frame.resample('D',kind='timestamp',origin='start').sum()
lastdf2 = data_frame.resample('D', kind='timestamp', origin='start').last()
data_frame= pd.merge(lastdf2.add_suffix('_last'), sumdf1.add_suffix('_sum'), left_index=True, right_index=True)
elif outstep == 4:
# change to monthly
sumdf1 = data_frame.resample('M',kind='timestamp',origin='start').sum()
lastdf2 = data_frame.resample('M', kind='timestamp', origin='start').last()
data_frame = pd.merge(lastdf2.add_suffix('_last'), sumdf1.add_suffix('_sum'), left_index=True, right_index=True)
elif outstep == 5:
# change to annual
sumdf1 = data_frame.resample('Y',kind='timestamp',origin='start').sum()
lastdf2 = data_frame.resample('Y', kind='timestamp', origin='start').last()
data_frame = pd.merge(lastdf2.add_suffix('_last'), sumdf1.add_suffix('_sum'), left_index=True, right_index=True)
self._output.write_ts(data_frame, category, operation, segment, activity)

def read_ts(self,
Expand Down
74 changes: 37 additions & 37 deletions HSP2tools/data/ParseTable.csv
View file
Original file line number Diff line number Diff line change
Expand Up @@ -27,21 +27,21 @@
25,PERLND,PRINT-INFO,GENERAL,SKIP,TRACPR,I,65,70,4
26,PERLND,PRINT-INFO,GENERAL,SKIP,PIVL,I,70,75,1
27,PERLND,PRINT-INFO,GENERAL,SKIP,PYREND,I,75,80,9
28,PERLND,BINARY-INFO,GENERAL,SKIP,OPNID,C,0,10,9
29,PERLND,BINARY-INFO,GENERAL,SKIP,AIRTPR,I,10,15,4
30,PERLND,BINARY-INFO,GENERAL,SKIP,SNOWPR,I,15,20,4
31,PERLND,BINARY-INFO,GENERAL,SKIP,PWATPR,I,20,25,4
32,PERLND,BINARY-INFO,GENERAL,SKIP,SEDPR,I,25,30,4
33,PERLND,BINARY-INFO,GENERAL,SKIP,PSTPR,I,30,35,4
34,PERLND,BINARY-INFO,GENERAL,SKIP,PWGPR,I,35,40,4
35,PERLND,BINARY-INFO,GENERAL,SKIP,PQALPR,I,40,45,4
36,PERLND,BINARY-INFO,GENERAL,SKIP,MSTLPR,I,45,50,4
37,PERLND,BINARY-INFO,GENERAL,SKIP,PESTPR,I,50,55,4
38,PERLND,BINARY-INFO,GENERAL,SKIP,NITRPR,I,55,60,4
39,PERLND,BINARY-INFO,GENERAL,SKIP,PHOSPR,I,60,65,4
40,PERLND,BINARY-INFO,GENERAL,SKIP,TRACPR,I,65,70,4
41,PERLND,BINARY-INFO,GENERAL,SKIP,PIVL,I,70,75,1
42,PERLND,BINARY-INFO,GENERAL,SKIP,PYREND,I,75,80,9
28,PERLND,BINARY-INFO,GENERAL,BINOUT,OPNID,C,0,10,None
29,PERLND,BINARY-INFO,GENERAL,BINOUT,ATEMP,I,10,15,4
30,PERLND,BINARY-INFO,GENERAL,BINOUT,SNOW,I,15,20,4
31,PERLND,BINARY-INFO,GENERAL,BINOUT,PWATER,I,20,25,4
32,PERLND,BINARY-INFO,GENERAL,BINOUT,SEDMNT,I,25,30,4
33,PERLND,BINARY-INFO,GENERAL,BINOUT,PSTEMP,I,30,35,4
34,PERLND,BINARY-INFO,GENERAL,BINOUT,PWTGAS,I,35,40,4
35,PERLND,BINARY-INFO,GENERAL,BINOUT,PQUAL,I,40,45,4
36,PERLND,BINARY-INFO,GENERAL,BINOUT,MSTLAY,I,45,50,4
37,PERLND,BINARY-INFO,GENERAL,BINOUT,PEST,I,50,55,4
38,PERLND,BINARY-INFO,GENERAL,BINOUT,NITR,I,55,60,4
39,PERLND,BINARY-INFO,GENERAL,BINOUT,PHOS,I,60,65,4
40,PERLND,BINARY-INFO,GENERAL,BINOUT,TRACER,I,65,70,4
41,PERLND,BINARY-INFO,GENERAL,BINOUT,PIVL,I,70,75,1
42,PERLND,BINARY-INFO,GENERAL,BINOUT,PYREND,I,75,80,9
43,PERLND,GEN-INFO,GENERAL,INFO,OPNID,C,0,10,9
44,PERLND,GEN-INFO,GENERAL,INFO,LSID,C,10,30,9
45,PERLND,GEN-INFO,GENERAL,INFO,IUNITS,I,40,45,1
Expand Down Expand Up @@ -1172,15 +1172,15 @@
1170,IMPLND,PRINT-INFO,GENERAL,SKIP,IQALPR,I,35,40,4
1171,IMPLND,PRINT-INFO,GENERAL,SKIP,PIVL,I,40,45,1
1172,IMPLND,PRINT-INFO,GENERAL,SKIP,PYREND,I,45,50,9
1173,IMPLND,BINARY-INFO,GENERAL,SKIP,OPNID,C,0,10,9
1174,IMPLND,BINARY-INFO,GENERAL,SKIP,ATMPPR,I,10,15,4
1175,IMPLND,BINARY-INFO,GENERAL,SKIP,SNOWPR,I,15,20,4
1176,IMPLND,BINARY-INFO,GENERAL,SKIP,IWATPR,I,20,25,4
1177,IMPLND,BINARY-INFO,GENERAL,SKIP,SLDPR,I,25,30,4
1178,IMPLND,BINARY-INFO,GENERAL,SKIP,IWGPR,I,30,35,4
1179,IMPLND,BINARY-INFO,GENERAL,SKIP,IQALPR,I,35,40,4
1180,IMPLND,BINARY-INFO,GENERAL,SKIP,PIVL,I,40,45,1
1181,IMPLND,BINARY-INFO,GENERAL,SKIP,PYREND,I,45,50,9
1173,IMPLND,BINARY-INFO,GENERAL,BINOUT,OPNID,C,0,10,None
1174,IMPLND,BINARY-INFO,GENERAL,BINOUT,ATEMP,I,10,15,4
1175,IMPLND,BINARY-INFO,GENERAL,BINOUT,SNOW,I,15,20,4
1176,IMPLND,BINARY-INFO,GENERAL,BINOUT,IWATER,I,20,25,4
1177,IMPLND,BINARY-INFO,GENERAL,BINOUT,SOLIDS,I,25,30,4
1178,IMPLND,BINARY-INFO,GENERAL,BINOUT,IWTGAS,I,30,35,4
1179,IMPLND,BINARY-INFO,GENERAL,BINOUT,IQUAL,I,35,40,4
1180,IMPLND,BINARY-INFO,GENERAL,BINOUT,PIVL,I,40,45,1
1181,IMPLND,BINARY-INFO,GENERAL,BINOUT,PYREND,I,45,50,9
1182,IMPLND,GEN-INFO,GENERAL,INFO,OPNID,C,0,10,9
1183,IMPLND,GEN-INFO,GENERAL,INFO,LSID,C,10,30,9
1184,IMPLND,GEN-INFO,GENERAL,INFO,IUNITS,I,35,40,1
Expand Down Expand Up @@ -1455,19 +1455,19 @@
1453,RCHRES,PRINT-INFO,GENERAL,SKIP,PHCBPR,I,55,60,4
1454,RCHRES,PRINT-INFO,GENERAL,SKIP,PIVLPR,I,60,65,1
1455,RCHRES,PRINT-INFO,GENERAL,SKIP,PYREND,I,65,70,9
1456,RCHRES,BINARY-INFO,GENERAL,SKIP,OPNID,C,0,10,9
1457,RCHRES,BINARY-INFO,GENERAL,SKIP,HYDRPR,I,10,15,4
1458,RCHRES,BINARY-INFO,GENERAL,SKIP,ADCAPR,I,15,20,4
1459,RCHRES,BINARY-INFO,GENERAL,SKIP,CONSPR,I,20,25,4
1460,RCHRES,BINARY-INFO,GENERAL,SKIP,HEATPR,I,25,30,4
1461,RCHRES,BINARY-INFO,GENERAL,SKIP,SEDPR,I,30,35,4
1462,RCHRES,BINARY-INFO,GENERAL,SKIP,GQLPR,I,35,40,4
1463,RCHRES,BINARY-INFO,GENERAL,SKIP,OXRXPR,I,40,45,4
1464,RCHRES,BINARY-INFO,GENERAL,SKIP,NUTRPR,I,45,50,4
1465,RCHRES,BINARY-INFO,GENERAL,SKIP,PLNKPR,I,50,55,4
1466,RCHRES,BINARY-INFO,GENERAL,SKIP,PHCBPR,I,55,60,4
1467,RCHRES,BINARY-INFO,GENERAL,SKIP,PIVLPR,I,60,65,1
1468,RCHRES,BINARY-INFO,GENERAL,SKIP,PYREND,I,65,70,9
1456,RCHRES,BINARY-INFO,GENERAL,BINOUT,OPNID,C,0,10,None
1457,RCHRES,BINARY-INFO,GENERAL,BINOUT,HYDR,I,10,15,4
1458,RCHRES,BINARY-INFO,GENERAL,BINOUT,ADCALC,I,15,20,4
1459,RCHRES,BINARY-INFO,GENERAL,BINOUT,CONS,I,20,25,4
1460,RCHRES,BINARY-INFO,GENERAL,BINOUT,HTRCH,I,25,30,4
1461,RCHRES,BINARY-INFO,GENERAL,BINOUT,SEDTRN,I,30,35,4
1462,RCHRES,BINARY-INFO,GENERAL,BINOUT,GQUAL,I,35,40,4
1463,RCHRES,BINARY-INFO,GENERAL,BINOUT,OXRX,I,40,45,4
1464,RCHRES,BINARY-INFO,GENERAL,BINOUT,NUTRX,I,45,50,4
1465,RCHRES,BINARY-INFO,GENERAL,BINOUT,PLANK,I,50,55,4
1466,RCHRES,BINARY-INFO,GENERAL,BINOUT,PHCARB,I,55,60,4
1467,RCHRES,BINARY-INFO,GENERAL,BINOUT,PIVLPR,I,60,65,1
1468,RCHRES,BINARY-INFO,GENERAL,BINOUT,PYREND,I,65,70,9
1469,RCHRES,GEN-INFO,GENERAL,INFO,OPNID,C,0,10,9
1470,RCHRES,GEN-INFO,GENERAL,INFO,RCHID,C,10,30,9
1471,RCHRES,GEN-INFO,GENERAL,INFO,NEXITS,I,30,35,1
Expand Down
3 changes: 2 additions & 1 deletion HSP2tools/readUCI.py
View file
Original file line number Diff line number Diff line change
Expand Up @@ -544,6 +544,7 @@ def operation(info, llines, op):
tokens = line.split()
if len(tokens) == 1:
table = tokens[0]
print(table)
if dcat[op,table] == 'EXTENDED':
rows = {}
extended_line = 0
Expand Down Expand Up @@ -582,7 +583,7 @@ def operation(info, llines, op):
counter.add(path)
if cat == 'SKIP':
continue
if cat in {'PARAMETERS', 'STATES', 'FLAGS', 'ACTIVITY','INFO'}:
if cat in {'PARAMETERS', 'STATES', 'FLAGS', 'ACTIVITY', 'INFO', 'BINOUT'}:
df = concat([temp[1] for temp in history[path,cat]], axis='columns', sort=False)
df = fix_df(df, op, path, ddfaults, valid)
if cat == 'ACTIVITY' and op == 'PERLND':
Expand Down

0 comments on commit 4431180

Please sign in to comment.