From 71eaab01e3949e072aae85db13f062d4cd2167cf Mon Sep 17 00:00:00 2001 From: Lewis Staples Date: Wed, 22 Jul 2020 23:18:23 -0400 Subject: [PATCH 1/6] Installed flask_sqlalchemy --- requirements.txt | Bin 641 -> 632 bytes 1 file changed, 0 insertions(+), 0 deletions(-) diff --git a/requirements.txt b/requirements.txt index 56d11b6df135a4600c5b64f4052417faf0255719..878c58b800cac15a6687998a0bef24c89468247b 100644 GIT binary patch literal 632 zcmZ{hJx{|h7=&*|;-?fPDOJnBz`}s&fI2XzX`6n!QPNa|9}nEUPGc`9vK0Bw=jUGk z{^Y3mt*}Fl2G9IbJmAsRJ?>E8NUgxZv?=$cXON~-4KC(bnx0S_n@9SF`Vnic^N=7_ zuC&j1rJ6Iv=PNtp2W^A}KJbngx(jTXc|vD5W;_vSccf{^mBXQ=tLDvZ)JxfkvA>bc zVNYi^f*F6uk3aj10~u&VfS??V)OYa z*ehd3*~?zgS~&U17%d;95>(RS?Uj9Tt1${Ol$631Jj(&dIxha?(a`H2`2)J_nRs%= z7!=3{VS~39W1ggvTG$j4(?Z77>OuSrv?8fmvVrdH$NR@=3$DNFSP!K<5-YnBcbJ z;hvWJjB>AA;hhqHZl5o&ziUTb#O~jQ0pOvfuD4wMchgv}jKjAJsM=R<*-ah(#%_ud v8wa>PRHzR)#MNH?RL)fcK92b7_;Pm} Date: Mon, 3 Aug 2020 23:19:30 -0400 Subject: [PATCH 2/6] 49: allowing 1 or 48 hrs of data to be pulled from db; 50: displaying detailed output from db --- flagging_site/blueprints/flagging.py | 76 +++++++++++++------ flagging_site/data/model.py | 24 +++--- ...sql => return_1_hour_of_model_outputs.sql} | 0 .../return_48_hours_of_model_outputs.sql | 10 +++ 4 files changed, 79 insertions(+), 31 deletions(-) rename flagging_site/data/queries/{latest_model.sql => return_1_hour_of_model_outputs.sql} (100%) create mode 100644 flagging_site/data/queries/return_48_hours_of_model_outputs.sql diff --git a/flagging_site/blueprints/flagging.py b/flagging_site/blueprints/flagging.py index 880bbb5d..d98eea91 100644 --- a/flagging_site/blueprints/flagging.py +++ b/flagging_site/blueprints/flagging.py @@ -38,6 +38,11 @@ def stylize_model_output(df: pd.DataFrame) -> str: Returns: HTML table. """ + + # print('\n\ndf conv to html\n\n') + # print(df) + # print('\n\n') + def _apply_flag(x: bool) -> str: flag_class = 'blue-flag' if x else 'red-flag' return f'{x}' @@ -68,15 +73,23 @@ def add_to_dict(models, df, reach) -> None: @bp.route('/') def index() -> str: """ - Retrieves data from hobolink and usgs and processes data, then displays data - on `index_model.html` + Retrieves data from database, + then displays data on `index_model.html` returns: render model on index.html """ + + df = latest_model_outputs() + + print('\n\nlatest_model_outputs:\n\n') + print( latest_model_outputs(48) ) + print('\n\n') + + df = df.set_index('reach') flags = { key: val['safe'] for key, val - in latest_model_outputs().items() + in df.to_dict(orient='index').items() } return render_template('index.html', flags=flags) @@ -113,26 +126,45 @@ def output_model() -> str: # Look at no more than 72 hours. hours = min(max(hours, 1), 72) - df = get_data() + df = latest_model_outputs(hours) - reach_model_mapping = { - 2: reach_2_model, - 3: reach_3_model, - 4: reach_4_model, - 5: reach_5_model - } + + print('\n\ndf_list\n\n') + print( df.reach.unique() ) + print('\n\n') + # go through each reach in df and create a subset of df of that reach only + # then add that to reach_html_tables + + reach_html_tables = {} + for i in df.reach.unique(): + print( i ) + reach_html_tables[i] = stylize_model_output( df.loc[df['reach'] == i ] ) - if reach in reach_model_mapping: - reach_func = reach_model_mapping[int(reach)] - reach_html_tables = { - reach: stylize_model_output(reach_func(df, rows=hours)) - } - else: - reach_html_tables = { - reach: stylize_model_output(reach_func(df, rows=hours)) - for reach, reach_func - in reach_model_mapping.items() - } + + # df = get_data() + + # reach_model_mapping = { + # 2: reach_2_model, + # 3: reach_3_model, + # 4: reach_4_model, + # 5: reach_5_model + # } + + # if reach in reach_model_mapping: + # reach_func = reach_model_mapping[int(reach)] + # reach_html_tables = { + # reach: stylize_model_output(reach_func(df, rows=hours)) + # } + # else: + # reach_html_tables = { + # reach: stylize_model_output(reach_func(df, rows=hours)) + # for reach, reach_func + # in reach_model_mapping.items() + # } + + print('\n\nreach_html_tables\n\n') + print(reach_html_tables) + print('\n\n') return render_template('output_model.html', tables=reach_html_tables) @@ -174,4 +206,4 @@ def get(self): return self.model_api() -api.add_resource(ReachApi, '/api/v1/model') +api.add_resource(ReachApi, '/api/v1/model') \ No newline at end of file diff --git a/flagging_site/data/model.py b/flagging_site/data/model.py index adbb5ce0..e1a2d87e 100644 --- a/flagging_site/data/model.py +++ b/flagging_site/data/model.py @@ -145,7 +145,7 @@ def process_data( return df -def reach_2_model(df: pd.DataFrame, rows: int = 24) -> pd.DataFrame: +def reach_2_model(df: pd.DataFrame, rows: int = 48) -> pd.DataFrame: """Model params: a- rainfall sum 0-24 hrs d- Days since last rain @@ -172,7 +172,7 @@ def reach_2_model(df: pd.DataFrame, rows: int = 24) -> pd.DataFrame: return df[['reach', 'time', 'log_odds', 'probability', 'safe']] -def reach_3_model(df: pd.DataFrame, rows: int = 24) -> pd.DataFrame: +def reach_3_model(df: pd.DataFrame, rows: int = 48) -> pd.DataFrame: """ a- rainfall sum 0-24 hrs b- rainfall sum 24-48 hr @@ -201,7 +201,7 @@ def reach_3_model(df: pd.DataFrame, rows: int = 24) -> pd.DataFrame: return df[['reach', 'time', 'log_odds', 'probability', 'safe']] -def reach_4_model(df: pd.DataFrame, rows: int = 24) -> pd.DataFrame: +def reach_4_model(df: pd.DataFrame, rows: int = 48) -> pd.DataFrame: """ a- rainfall sum 0-24 hrs b- rainfall sum 24-48 hr @@ -230,7 +230,7 @@ def reach_4_model(df: pd.DataFrame, rows: int = 24) -> pd.DataFrame: return df[['reach', 'time', 'log_odds', 'probability', 'safe']] -def reach_5_model(df: pd.DataFrame, rows: int = 24) -> pd.DataFrame: +def reach_5_model(df: pd.DataFrame, rows: int = 48) -> pd.DataFrame: """ c- rainfall sum 0-48 hr d- Days since last rain @@ -269,9 +269,15 @@ def all_models(df: pd.DataFrame, *args, **kwargs): def latest_model_outputs(hours: int = 1) -> dict: - if hours != 1: - raise NotImplementedError('Need to work on this!') from .database import execute_sql_from_file - df = execute_sql_from_file('latest_model.sql') - df = df.set_index('reach') - return df.to_dict(orient='index') + + if hours == 1: + df = execute_sql_from_file('return_1_hour_of_model_outputs.sql') + elif hours > 1: + df = execute_sql_from_file('return_48_hours_of_model_outputs.sql') + # pull out all data older than parameter hours + + else: + raise ValueError('hours of data to pull cannot be less than one') + + return df \ No newline at end of file diff --git a/flagging_site/data/queries/latest_model.sql b/flagging_site/data/queries/return_1_hour_of_model_outputs.sql similarity index 100% rename from flagging_site/data/queries/latest_model.sql rename to flagging_site/data/queries/return_1_hour_of_model_outputs.sql diff --git a/flagging_site/data/queries/return_48_hours_of_model_outputs.sql b/flagging_site/data/queries/return_48_hours_of_model_outputs.sql new file mode 100644 index 00000000..5dc8c348 --- /dev/null +++ b/flagging_site/data/queries/return_48_hours_of_model_outputs.sql @@ -0,0 +1,10 @@ +-- This query returns 48 hours of data, +-- (unless the time interval in the data is less than 48 hours, +-- in which case it will return data for all timestamps) + +SELECT * +FROM model_outputs +WHERE time BETWEEN + (SELECT MAX(time) - interval '47 hours' FROM model_outputs) + AND + (SELECT MAX(time) FROM model_outputs) \ No newline at end of file From 6edc594c4f1f7fe8540277c7c91798262bb778ae Mon Sep 17 00:00:00 2001 From: Lewis Staples Date: Tue, 4 Aug 2020 18:35:38 -0400 Subject: [PATCH 3/6] Issue#49 complete: latest_model_outputs returns 1 thru 48, depending on input parameter; also cleaned up code and documentation --- flagging_site/blueprints/flagging.py | 47 +++++----------------------- flagging_site/data/model.py | 8 +++-- 2 files changed, 13 insertions(+), 42 deletions(-) diff --git a/flagging_site/blueprints/flagging.py b/flagging_site/blueprints/flagging.py index d98eea91..ece7f5f0 100644 --- a/flagging_site/blueprints/flagging.py +++ b/flagging_site/blueprints/flagging.py @@ -38,11 +38,6 @@ def stylize_model_output(df: pd.DataFrame) -> str: Returns: HTML table. """ - - # print('\n\ndf conv to html\n\n') - # print(df) - # print('\n\n') - def _apply_flag(x: bool) -> str: flag_class = 'blue-flag' if x else 'red-flag' return f'{x}' @@ -128,44 +123,18 @@ def output_model() -> str: df = latest_model_outputs(hours) - - print('\n\ndf_list\n\n') - print( df.reach.unique() ) - print('\n\n') - # go through each reach in df and create a subset of df of that reach only - # then add that to reach_html_tables - + # reach_html_tables is a dict where the index is the reach number + # and the values are HTML code for the table of data to display for + # that particular reach reach_html_tables = {} + + # loop through each reach in df + # extract the subset from the df for that reach + # then convert that df subset to HTML code + # and then add that HTML subset to reach_html_tables for i in df.reach.unique(): - print( i ) reach_html_tables[i] = stylize_model_output( df.loc[df['reach'] == i ] ) - - # df = get_data() - - # reach_model_mapping = { - # 2: reach_2_model, - # 3: reach_3_model, - # 4: reach_4_model, - # 5: reach_5_model - # } - - # if reach in reach_model_mapping: - # reach_func = reach_model_mapping[int(reach)] - # reach_html_tables = { - # reach: stylize_model_output(reach_func(df, rows=hours)) - # } - # else: - # reach_html_tables = { - # reach: stylize_model_output(reach_func(df, rows=hours)) - # for reach, reach_func - # in reach_model_mapping.items() - # } - - print('\n\nreach_html_tables\n\n') - print(reach_html_tables) - print('\n\n') - return render_template('output_model.html', tables=reach_html_tables) diff --git a/flagging_site/data/model.py b/flagging_site/data/model.py index e1a2d87e..a1542389 100644 --- a/flagging_site/data/model.py +++ b/flagging_site/data/model.py @@ -274,9 +274,11 @@ def latest_model_outputs(hours: int = 1) -> dict: if hours == 1: df = execute_sql_from_file('return_1_hour_of_model_outputs.sql') elif hours > 1: - df = execute_sql_from_file('return_48_hours_of_model_outputs.sql') - # pull out all data older than parameter hours - + df = execute_sql_from_file('return_48_hours_of_model_outputs.sql') # pull out 48 hours of model outputs + latest_time = max(df['time']) # find most recent timestamp + time_interval = pd.Timedelta(str(hours) + ' hours') # create pandas Timedelta, based on input parameter hours + df = df[ latest_time - df['time']< time_interval ] # reset df to exclude anything from before time_interval ago + else: raise ValueError('hours of data to pull cannot be less than one') From 99f70a6d193e7546857ce54e7e4bdb5163a87d27 Mon Sep 17 00:00:00 2001 From: Lewis Staples Date: Tue, 4 Aug 2020 21:40:00 -0400 Subject: [PATCH 4/6] Fix to get full 48 hours of numeric data populated --- flagging_site/data/database.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/flagging_site/data/database.py b/flagging_site/data/database.py index fec31e51..4151ff47 100644 --- a/flagging_site/data/database.py +++ b/flagging_site/data/database.py @@ -61,7 +61,7 @@ def update_database(): # Populate the `hobolink` table. from .hobolink import get_live_hobolink_data - df_hobolink = get_live_hobolink_data() + df_hobolink = get_live_hobolink_data('code_for_boston_export_21d') df_hobolink.to_sql('hobolink', **options) from .model import process_data From b1d5e64ffe86e87189ba34f66c0e836677058966 Mon Sep 17 00:00:00 2001 From: Lewis Staples Date: Tue, 4 Aug 2020 21:48:57 -0400 Subject: [PATCH 5/6] cleaned up code (print statements) --- flagging_site/blueprints/flagging.py | 5 ----- 1 file changed, 5 deletions(-) diff --git a/flagging_site/blueprints/flagging.py b/flagging_site/blueprints/flagging.py index ece7f5f0..95c97cbd 100644 --- a/flagging_site/blueprints/flagging.py +++ b/flagging_site/blueprints/flagging.py @@ -75,11 +75,6 @@ def index() -> str: """ df = latest_model_outputs() - - print('\n\nlatest_model_outputs:\n\n') - print( latest_model_outputs(48) ) - print('\n\n') - df = df.set_index('reach') flags = { key: val['safe'] From dd5ee0608754a8f5829d044719096389be7f2968 Mon Sep 17 00:00:00 2001 From: Lewis Staples Date: Tue, 4 Aug 2020 21:55:02 -0400 Subject: [PATCH 6/6] cleaned up comments in sql query --- .../data/queries/return_48_hours_of_model_outputs.sql | 4 +--- 1 file changed, 1 insertion(+), 3 deletions(-) diff --git a/flagging_site/data/queries/return_48_hours_of_model_outputs.sql b/flagging_site/data/queries/return_48_hours_of_model_outputs.sql index 5dc8c348..9e1cd8fd 100644 --- a/flagging_site/data/queries/return_48_hours_of_model_outputs.sql +++ b/flagging_site/data/queries/return_48_hours_of_model_outputs.sql @@ -1,6 +1,4 @@ --- This query returns 48 hours of data, --- (unless the time interval in the data is less than 48 hours, --- in which case it will return data for all timestamps) +-- This query returns up to 48 hours of the latest data SELECT * FROM model_outputs