From 33d3e83dc153c11e09a28e7d9078fa670e27ac06 Mon Sep 17 00:00:00 2001 From: Lewis Staples Date: Tue, 30 Jun 2020 10:27:26 -0400 Subject: [PATCH 001/118] this shows the format for storing db credentials --- flagging_site/data/db_creds_dummy.json | 7 +++++++ 1 file changed, 7 insertions(+) create mode 100644 flagging_site/data/db_creds_dummy.json diff --git a/flagging_site/data/db_creds_dummy.json b/flagging_site/data/db_creds_dummy.json new file mode 100644 index 00000000..964ee6cb --- /dev/null +++ b/flagging_site/data/db_creds_dummy.json @@ -0,0 +1,7 @@ +{ + "host":"dummy_host", + "port_num": 1234, + "dbname":"dummy_dbname", + "user":"dummy_user", + "password":"dummy_password" +} From b715cb6399a708803709ca05162dadebeb8a8f84 Mon Sep 17 00:00:00 2001 From: Lewis Staples Date: Tue, 30 Jun 2020 10:33:26 -0400 Subject: [PATCH 002/118] ignore file with database credentials --- .gitignore | 5 ++++- 1 file changed, 4 insertions(+), 1 deletion(-) diff --git a/.gitignore b/.gitignore index ff98b242..33a3cb88 100644 --- a/.gitignore +++ b/.gitignore @@ -144,4 +144,7 @@ dmypy.json # Cython debug symbols cython_debug/l -flagging_site/keys.json \ No newline at end of file +flagging_site/keys.json + +# ignore file with database credentials +flagging_site/data/db_creds.json From 941423c1cb1293a8bef78b8f71e4220bf5972e8e Mon Sep 17 00:00:00 2001 From: Lewis Staples Date: Tue, 30 Jun 2020 15:04:03 -0400 Subject: [PATCH 003/118] gitignore will ignore Pipfile.lock --- .gitignore | 1 + 1 file changed, 1 insertion(+) diff --git a/.gitignore b/.gitignore index 33a3cb88..8693ad97 100644 --- a/.gitignore +++ b/.gitignore @@ -4,6 +4,7 @@ keys.json .DS_Store Pipfile +Pipfile.lock # Byte-compiled / optimized / DLL files __pycache__/ From 424b1d2ad9c2d1dd52511f9a1a3e4387b19c3a7e Mon Sep 17 00:00:00 2001 From: Lewis Staples Date: Mon, 6 Jul 2020 10:42:12 -0400 Subject: [PATCH 004/118] Implemented creating db schema (also includes a Python file for testing) --- flagging_site/data/database.py | 94 ++++++++++++++++++++++++++++++++- flagging_site/data/db_tester.py | 12 +++++ flagging_site/data/schema.sql | 36 +++++++++++++ 3 files changed, 141 insertions(+), 1 deletion(-) create mode 100644 flagging_site/data/db_tester.py create mode 100644 flagging_site/data/schema.sql diff --git a/flagging_site/data/database.py b/flagging_site/data/database.py index fe2bdea2..46124e5d 100644 --- a/flagging_site/data/database.py +++ b/flagging_site/data/database.py @@ -1,4 +1,96 @@ """ This file should handle all database connection stuff, namely: writing and retrieving data. -""" \ No newline at end of file +""" + +# using json to pull in credentials from a JSON-formatted text file +import json + +import psycopg2 # adapter between Python and postgresql + +# import click +# from flask import current_app +from flask import g +from flask.cli import with_appcontext +from flask import Flask +# from flask_sqlalchemy import SQLAlchemy + +conn_string = None # store database connection string +app = Flask(__name__) +# db = SQLAlchemy(app) + +# this reads the database credentials and constructs the +# string to connect to the database +def db_init_str(): + print('-------------------------------') + print ("db_init_str has been called!\n") # TEST ONLY + + global conn_string + + if conn_string is not None: + print("Whoops! The database string was already initialized. It's shown below:") + print(conn_string) + return + + # get database credentials + with open('db_creds.json', 'r') as f: + creds = json.load(f) + + # create connection string for database + conn_string = "host=" + creds['host'] + " port="+ str(creds['port_num']) \ + + " user=" + creds['user'] + " password=" + creds['password'] \ + + " dbname=" + creds['dbname'] + + # this command could include the db name (below), but then we could't create db + # + " dbname=" + creds['dbname'] + + # BELOW BLOCK IS TEST ONLY + print ('db_init_str connection string is:') + print(conn_string) + print() + + + + +def db_get(): + print('-------------------------------') + print ("db_get has been called!\n") # TEST ONLY + + # only read database credentials if needed + # note if d.b. credentials were to change in the future, + # this wouldn't auto-magically detect that change + if conn_string is None: + db_init_str() + + with app.app_context(): + if "db" not in g: # set to cursor + g.db = psycopg2.connect(conn_string).cursor() + print("cursor created!\n") + else: + print("no cursor created\n") + return g.db + + +def db_close(): # note: Flask sample code adds an unused parameter e + print('-------------------------------') + print ("db_close has been called!\n") # TEST ONLY + + with app.app_context(): + db = g.pop("db", None) + if db is not None: + db.close() + print ("the database was closed\n") + else: + print ("the database was not closed\n") + +def db_init(): + print('-------------------------------') + print ("db_init has been called!\n") # TEST ONLY + + db = db_get() + + sqlfile = open('schema.sql', 'r') + db.execute(sqlfile.read()) + + + diff --git a/flagging_site/data/db_tester.py b/flagging_site/data/db_tester.py new file mode 100644 index 00000000..3ed5b4e4 --- /dev/null +++ b/flagging_site/data/db_tester.py @@ -0,0 +1,12 @@ +# this file is solely used to test the functions in database.py +# to run these tests: +# place this file in the same folder as database.py +# at the command line prompt, enter python db_tester.py +from database import db_init_str, db_get, db_close, db_init + +db_get() + +db_init() + +db_close() + diff --git a/flagging_site/data/schema.sql b/flagging_site/data/schema.sql new file mode 100644 index 00000000..7e6337a6 --- /dev/null +++ b/flagging_site/data/schema.sql @@ -0,0 +1,36 @@ +DROP TABLE IF EXISTS raw_usgs; +CREATE TABLE IF NOT EXISTS raw_usgs ( + time timestamp, + stream_flow decimal, + gage_height decimal +); + +DROP TABLE IF EXISTS raw_hobolink; +CREATE TABLE IF NOT EXISTS raw_hobolink ( + time timestamp, + pressure decimal, + par decimal, /* photosynthetically active radiation */ + rain decimal, + rh decimal, /* relative humidity */ + dew_point decimal, + wind_speed decimal, + gust_speed decimal, + wind_dir decimal, + water_temp decimal, + air_temp decimal +); + +DROP TABLE IF EXISTS ecoli_predictions; +CREATE TABLE IF NOT EXISTS ecoli_predictions ( + timestamp timestamp, + r2_probability decimal, + r3_probability decimal, + r4_probability decimal, + r5_probability decimal, + r2_safe boolean, + r3_safe boolean, + r4_safe boolean, + r5_safe boolean +); + +COMMIT; \ No newline at end of file From 3d11f3c77bb8838a623646abe43d87ca7a9650e3 Mon Sep 17 00:00:00 2001 From: Lewis Staples Date: Mon, 6 Jul 2020 11:08:41 -0400 Subject: [PATCH 005/118] Improved comments to the functions --- flagging_site/data/database.py | 5 ++++- 1 file changed, 4 insertions(+), 1 deletion(-) diff --git a/flagging_site/data/database.py b/flagging_site/data/database.py index 46124e5d..b8bb9456 100644 --- a/flagging_site/data/database.py +++ b/flagging_site/data/database.py @@ -51,7 +51,7 @@ def db_init_str(): - +# this function connects to the database and creates a cursor def db_get(): print('-------------------------------') print ("db_get has been called!\n") # TEST ONLY @@ -71,6 +71,7 @@ def db_get(): return g.db +# this function closes the database connection / cursor def db_close(): # note: Flask sample code adds an unused parameter e print('-------------------------------') print ("db_close has been called!\n") # TEST ONLY @@ -83,6 +84,8 @@ def db_close(): # note: Flask sample code adds an unused parameter e else: print ("the database was not closed\n") +# this function intializes the database +# (it connects via db_get and clears/creates the tables) def db_init(): print('-------------------------------') print ("db_init has been called!\n") # TEST ONLY From e3c04f9ac8d9a577201111c7091b3e8de50fb5fd Mon Sep 17 00:00:00 2001 From: Daniel Reeves Date: Tue, 7 Jul 2020 20:55:20 -0400 Subject: [PATCH 006/118] Horrific bugfix. Issue is explained in comments --- flagging_site/data/hobolink.py | 56 ++++++++++++++++++++++------------ 1 file changed, 37 insertions(+), 19 deletions(-) diff --git a/flagging_site/data/hobolink.py b/flagging_site/data/hobolink.py index 26361bb4..df3721b4 100644 --- a/flagging_site/data/hobolink.py +++ b/flagging_site/data/hobolink.py @@ -21,16 +21,16 @@ # Each key is the original column name; the value is the renamed column. HOBOLINK_COLUMNS = { 'Time, GMT-04:00': 'time', - 'Pressure, inHg, Charles River Weather Station': 'pressure', - 'PAR, uE, Charles River Weather Station': 'par', - 'Rain, in, Charles River Weather Station': 'rain', - 'RH, %, Charles River Weather Station': 'rh', - 'DewPt, *F, Charles River Weather Station': 'dew_point', - 'Wind Speed, mph, Charles River Weather Station': 'wind_speed', - 'Gust Speed, mph, Charles River Weather Station': 'gust_speed', - 'Wind Dir, *, Charles River Weather Station': 'wind_dir', - 'Water Temp, *F, Charles River Weather Station': 'water_temp', - 'Temp, *F, Charles River Weather Station Air Temp': 'air_temp', + 'Pressure': 'pressure', + 'PAR': 'par', + 'Rain': 'rain', + 'RH': 'rh', + 'DewPt': 'dew_point', + 'Wind Speed': 'wind_speed', + 'Gust Speed': 'gust_speed', + 'Wind Dir': 'wind_dir', + 'Water Temp': 'water_temp', + 'Temp': 'air_temp', # 'Batt, V, Charles River Weather Station': 'battery' } STATIC_FILE_NAME = 'hobolink.pickle' @@ -104,15 +104,33 @@ def parse_hobolink_data(res: str) -> pd.DataFrame: str_table = res[res.find(split_by) + len(split_by):] df = pd.read_csv(io.StringIO(str_table), sep=',') - # Remove all unnecessary columns - df = df[HOBOLINK_COLUMNS.keys()] - - # Rename the columns to have shorter, friendlier names. - df = df.rename(columns=HOBOLINK_COLUMNS) - - # Remove rows with missing data (i.e. the 05, 15, 25, 35, 45, and 55 min - # timestamps, which only include the battery status.) - df = df.loc[df['water_temp'].notna(), :] + # There is a weird issue in the HOBOlink data where it sometimes returns + # multiple columns with the same name and spreads real data out across + # those two columns. It is VERY weird. I promise this code used to be much + # simpler before we ran into this issue and it broke the website. Please + # trust us that it does have to be this complicated. + for old_col_startswith, new_col in HOBOLINK_COLUMNS.items(): + + # Only look at rows that start with `old_col_startswith` + subset_df = df.loc[ + :, + filter(lambda x: x.startswith(old_col_startswith), df.columns) + ] + + # Remove rows with missing data (i.e. the 05, 15, 25, 35, 45, and 55 min + # timestamps, which only include the battery status.) + subset_df = subset_df.loc[~subset_df.isna().all(axis=1)] + + # Take the first nonmissing column value within the subset of rows we've + # selected. This trick is similar to doing a COALESCE in sql. + df[new_col] = subset_df \ + .apply(lambda x: x[x.first_valid_index()], axis=1) + + # Only keep these columns + df = df[HOBOLINK_COLUMNS.values()] + + # Remove the rows with all missing values again. + df = df.loc[df['water_temp'].notna()] # Convert time column to Pandas datetime df['time'] = pd.to_datetime(df['time']) From c47af8bdfe386535068f35c7b915af438ee3b4d9 Mon Sep 17 00:00:00 2001 From: Daniel Reeves Date: Sun, 19 Jul 2020 13:29:55 -0400 Subject: [PATCH 007/118] MVP of database --- README.md | 19 ++++ flagging_site/app.py | 12 ++- flagging_site/blueprints/cyanobacteria.py | 18 +++- flagging_site/config.py | 24 ++++- flagging_site/data/__init__.py | 1 + flagging_site/data/database.py | 119 ++++++++-------------- flagging_site/data/db_creds_dummy.json | 7 -- flagging_site/data/schema.sql | 8 +- run_windows_dev.bat | 4 +- 9 files changed, 117 insertions(+), 95 deletions(-) delete mode 100644 flagging_site/data/db_creds_dummy.json diff --git a/README.md b/README.md index d043842f..31f43f13 100644 --- a/README.md +++ b/README.md @@ -60,3 +60,22 @@ python -m pytest ./tests -s ``` Note: the test may require you to enter the vault password if it is not already in your environment variables. + + +## Start DB + +Set environment variable `POSTGRES_PASSWORD` to be whatever you want. + + +Powershell and CMD (Windows): + +```commandline +psql -U postgres -c "DROP USER IF EXISTS flagging_admin; CREATE USER flagging_admin SUPERUSER PASSWORD '%POSTGRES_PASSWORD%'" +``` + +Bash (Mac OSX / Linux): + +```shell script +psql -U postgres -c "DROP USER IF EXISTS flagging_admin; CREATE USER flagging_admin SUPERUSER PASSWORD '${POSTGRES_PASSWORD}'" +``` + diff --git a/flagging_site/app.py b/flagging_site/app.py index 9351b392..d2f3b80a 100644 --- a/flagging_site/app.py +++ b/flagging_site/app.py @@ -2,6 +2,7 @@ This file handles the construction of the Flask application object. """ import os +import click from typing import Optional from flask import Flask @@ -43,8 +44,15 @@ def create_app(config: Optional[Config] = None) -> Flask: register_blueprints_from_module(app, blueprints) # Register the database commands - # from .data import db - # db.init_app(app) + from .data import db + db.init_app(app) + + @app.cli.command('init-db') + def init_db_command(): + """Clear existing data and create new tables.""" + from .data.database import init_db + init_db() + click.echo('Initialized the database.') # And we're all set! We can hand the app over to flask at this point. return app diff --git a/flagging_site/blueprints/cyanobacteria.py b/flagging_site/blueprints/cyanobacteria.py index 350818db..2dd8437e 100644 --- a/flagging_site/blueprints/cyanobacteria.py +++ b/flagging_site/blueprints/cyanobacteria.py @@ -1,8 +1,20 @@ +import pandas as pd from flask import Blueprint +from ..data.usgs import get_usgs_data +from ..data import db +from sqlalchemy.dialects.postgresql import insert +from sqlalchemy import create_engine bp = Blueprint('cyanobacteria', __name__, url_prefix='/cyanobacteria') -# @bp.route('/') -# def index() -> str: -# return 'Hello, world!' +@bp.route('/') +def index() -> str: + df = get_usgs_data() + try: + df.to_sql('raw_usgs', con=db.engine, if_exists='replace') + except ValueError: + pass + df = pd.read_sql('''select * from raw_usgs''', con=db.engine) + print(df) + return 'hello world' \ No newline at end of file diff --git a/flagging_site/config.py b/flagging_site/config.py index 8e2a9852..f85afddf 100644 --- a/flagging_site/config.py +++ b/flagging_site/config.py @@ -44,7 +44,24 @@ def __repr__(self): # Not currently used, but soon we'll want to start using the config to set # up references to the database, data storage, and data retrieval. # ========================================================================== - DATABASE: str = None + POSTGRES_USER: str = os.getenv('POSTGRES_USER', 'flagging_admin') + POSTGRES_PASSWORD: str = os.getenv('POSTGRES_PASSWORD') + POSTGRES_HOST: str = 'localhost' + POSTGRES_PORT: int = 5432 + POSTGRES_DBNAME: str = 'flagging' + + @property + def SQLALCHEMY_DATABASE_URI(self) -> str: + user = self.POSTGRES_USER + password = self.POSTGRES_PASSWORD + host = self.POSTGRES_HOST + port = self.POSTGRES_PORT + db = self.POSTGRES_DBNAME + return f'postgres://{user}:{password}@{host}:{port}/{db}' + + SQLALCHEMY_ECHO: bool = True + SQLALCHEMY_RECORD_QUERIES: bool = True + SQLALCHEMY_TRACK_MODIFICATIONS: bool = False # ========================================================================== # MISC. CUSTOM CONFIG OPTIONS @@ -158,3 +175,8 @@ def get_config_from_env(env: str) -> Config: raise KeyError('Bad config passed; the config must be production, ' 'development, or testing.') return config() + + +def postgres_uri_from_params(user: str, password: str, host: str, db: str): + """postgres://username:password@server/db""" + return f'postgres://{user}:{password}@{host}/{db}' diff --git a/flagging_site/data/__init__.py b/flagging_site/data/__init__.py index cbeb04b6..a5b67987 100644 --- a/flagging_site/data/__init__.py +++ b/flagging_site/data/__init__.py @@ -2,3 +2,4 @@ The data module contains exactly what you'd expect: everything related to data processing, collection, and storage. """ +from .database import db diff --git a/flagging_site/data/database.py b/flagging_site/data/database.py index b8bb9456..b94a13a2 100644 --- a/flagging_site/data/database.py +++ b/flagging_site/data/database.py @@ -2,98 +2,63 @@ This file should handle all database connection stuff, namely: writing and retrieving data. """ +import os +import click +import psycopg2 +import psycopg2.extensions -# using json to pull in credentials from a JSON-formatted text file -import json - -import psycopg2 # adapter between Python and postgresql - -# import click -# from flask import current_app from flask import g -from flask.cli import with_appcontext +from flask import current_app from flask import Flask -# from flask_sqlalchemy import SQLAlchemy - -conn_string = None # store database connection string -app = Flask(__name__) -# db = SQLAlchemy(app) - -# this reads the database credentials and constructs the -# string to connect to the database -def db_init_str(): - print('-------------------------------') - print ("db_init_str has been called!\n") # TEST ONLY - - global conn_string - - if conn_string is not None: - print("Whoops! The database string was already initialized. It's shown below:") - print(conn_string) - return - - # get database credentials - with open('db_creds.json', 'r') as f: - creds = json.load(f) - - # create connection string for database - conn_string = "host=" + creds['host'] + " port="+ str(creds['port_num']) \ - + " user=" + creds['user'] + " password=" + creds['password'] \ - + " dbname=" + creds['dbname'] - - # this command could include the db name (below), but then we could't create db - # + " dbname=" + creds['dbname'] - # BELOW BLOCK IS TEST ONLY - print ('db_init_str connection string is:') - print(conn_string) - print() +from ..config import ROOT_DIR +from flask_sqlalchemy import SQLAlchemy +SCHEMA_FILE = os.path.join(ROOT_DIR, 'data', 'schema.sql') +db = SQLAlchemy() -# this function connects to the database and creates a cursor -def db_get(): - print('-------------------------------') - print ("db_get has been called!\n") # TEST ONLY - # only read database credentials if needed - # note if d.b. credentials were to change in the future, - # this wouldn't auto-magically detect that change - if conn_string is None: - db_init_str() +# # this function connects to the database and creates a cursor +# def db_get() -> SQLAlchemy: # psycopg2.extensions.connection: +# with current_app.app_context(): +# if 'db' not in g: +# +# g.db = conn +# return g.db - with app.app_context(): - if "db" not in g: # set to cursor - g.db = psycopg2.connect(conn_string).cursor() - print("cursor created!\n") - else: - print("no cursor created\n") - return g.db +def close_db(e=None): + """If this request connected to the database, close the + connection. + """ + db = g.pop('db', None) + if db is not None: + db.close() -# this function closes the database connection / cursor -def db_close(): # note: Flask sample code adds an unused parameter e - print('-------------------------------') - print ("db_close has been called!\n") # TEST ONLY - with app.app_context(): - db = g.pop("db", None) - if db is not None: - db.close() - print ("the database was closed\n") - else: - print ("the database was not closed\n") +def execute_sql(query: str): + with db.engine.connect() as conn: + return conn.execute(query) -# this function intializes the database -# (it connects via db_get and clears/creates the tables) -def db_init(): - print('-------------------------------') - print ("db_init has been called!\n") # TEST ONLY - db = db_get() +def init_db(): + """Clear existing data and create new tables.""" + with current_app.app_context(): - sqlfile = open('schema.sql', 'r') - db.execute(sqlfile.read()) + # Read the `schema.sql` file, which initializes the database. + with current_app.open_resource(SCHEMA_FILE) as f: + schema = f.read().decode('utf8') + # Run `schema.sql` + execute_sql(schema) + # Populate the `usgs` table. + from .usgs import get_usgs_data + df = get_usgs_data() + df.to_sql('usgs', con=db.engine, index=False, if_exists='append') + # Populate the `hobolink` table. + from .hobolink import get_hobolink_data + df = get_hobolink_data() + df.to_sql('hobolink', con=db.engine, index=False, if_exists='append') diff --git a/flagging_site/data/db_creds_dummy.json b/flagging_site/data/db_creds_dummy.json deleted file mode 100644 index 964ee6cb..00000000 --- a/flagging_site/data/db_creds_dummy.json +++ /dev/null @@ -1,7 +0,0 @@ -{ - "host":"dummy_host", - "port_num": 1234, - "dbname":"dummy_dbname", - "user":"dummy_user", - "password":"dummy_password" -} diff --git a/flagging_site/data/schema.sql b/flagging_site/data/schema.sql index 7e6337a6..d1752618 100644 --- a/flagging_site/data/schema.sql +++ b/flagging_site/data/schema.sql @@ -1,12 +1,12 @@ -DROP TABLE IF EXISTS raw_usgs; -CREATE TABLE IF NOT EXISTS raw_usgs ( +DROP TABLE IF EXISTS usgs; +CREATE TABLE IF NOT EXISTS usgs ( time timestamp, stream_flow decimal, gage_height decimal ); -DROP TABLE IF EXISTS raw_hobolink; -CREATE TABLE IF NOT EXISTS raw_hobolink ( +DROP TABLE IF EXISTS hobolink; +CREATE TABLE IF NOT EXISTS hobolink ( time timestamp, pressure decimal, par decimal, /* photosynthetically active radiation */ diff --git a/run_windows_dev.bat b/run_windows_dev.bat index 0a418e6c..59dae58e 100644 --- a/run_windows_dev.bat +++ b/run_windows_dev.bat @@ -3,9 +3,11 @@ python -m venv venv call venv\Scripts\activate.bat python -m pip install -r requirements.txt -set FLASK_APP=flagging_site +set FLASK_APP=flagging_site:create_app set FLASK_ENV=development set /p OFFLINE_MODE="Offline mode? [y/n]: " set /p VAULT_PASSWORD="Enter vault password: " +set /p POSTGRES_PASSWORD="Enter postgres PW: " +flask init-db flask run \ No newline at end of file From b280273c8b369bcb48ef1a4fec819929eee3a5e9 Mon Sep 17 00:00:00 2001 From: Daniel Reeves Date: Sun, 19 Jul 2020 18:03:52 -0400 Subject: [PATCH 008/118] Homepage data comes in from the db now --- README.md | 9 ++- flagging_site/app.py | 6 ++ flagging_site/blueprints/cyanobacteria.py | 17 ---- flagging_site/blueprints/flagging.py | 30 +++---- flagging_site/config.py | 13 +-- flagging_site/data/_store/refresh.py | 8 +- flagging_site/data/database.py | 89 ++++++++++++--------- flagging_site/data/db_tester.py | 12 --- flagging_site/data/hobolink.py | 3 +- flagging_site/data/model.py | 32 +++++++- flagging_site/data/queries/latest_model.sql | 5 ++ flagging_site/data/{ => queries}/schema.sql | 20 ++--- flagging_site/data/task_queue.py | 4 - flagging_site/data/usgs.py | 2 +- 14 files changed, 128 insertions(+), 122 deletions(-) delete mode 100644 flagging_site/data/db_tester.py create mode 100644 flagging_site/data/queries/latest_model.sql rename flagging_site/data/{ => queries}/schema.sql (63%) delete mode 100644 flagging_site/data/task_queue.py diff --git a/README.md b/README.md index 31f43f13..e3f0c73f 100644 --- a/README.md +++ b/README.md @@ -66,16 +66,19 @@ Note: the test may require you to enter the vault password if it is not already Set environment variable `POSTGRES_PASSWORD` to be whatever you want. - Powershell and CMD (Windows): ```commandline -psql -U postgres -c "DROP USER IF EXISTS flagging_admin; CREATE USER flagging_admin SUPERUSER PASSWORD '%POSTGRES_PASSWORD%'" +set POSTGRES_PASSWORD=enter_password_here +createdb -U postgres flagging +psql -U postgres -d flagging -c "DROP USER IF EXISTS flagging; CREATE USER flagging SUPERUSER PASSWORD '%POSTGRES_PASSWORD%'" ``` Bash (Mac OSX / Linux): ```shell script -psql -U postgres -c "DROP USER IF EXISTS flagging_admin; CREATE USER flagging_admin SUPERUSER PASSWORD '${POSTGRES_PASSWORD}'" +export POSTGRES_PASSWORD=enter_password_here +createdb -U postgres flagging +psql -U postgres -d flagging -c "DROP USER IF EXISTS flagging; CREATE USER flagging SUPERUSER PASSWORD '${POSTGRES_PASSWORD}'" ``` diff --git a/flagging_site/app.py b/flagging_site/app.py index d2f3b80a..01b83a81 100644 --- a/flagging_site/app.py +++ b/flagging_site/app.py @@ -54,6 +54,12 @@ def init_db_command(): init_db() click.echo('Initialized the database.') + @app.cli.command('update-db') + def update_db_command(): + from .data.database import update_database + update_database() + click.echo('Updated the database.') + # And we're all set! We can hand the app over to flask at this point. return app diff --git a/flagging_site/blueprints/cyanobacteria.py b/flagging_site/blueprints/cyanobacteria.py index 2dd8437e..ff383efb 100644 --- a/flagging_site/blueprints/cyanobacteria.py +++ b/flagging_site/blueprints/cyanobacteria.py @@ -1,20 +1,3 @@ -import pandas as pd from flask import Blueprint -from ..data.usgs import get_usgs_data -from ..data import db -from sqlalchemy.dialects.postgresql import insert -from sqlalchemy import create_engine bp = Blueprint('cyanobacteria', __name__, url_prefix='/cyanobacteria') - - -@bp.route('/') -def index() -> str: - df = get_usgs_data() - try: - df.to_sql('raw_usgs', con=db.engine, if_exists='replace') - except ValueError: - pass - df = pd.read_sql('''select * from raw_usgs''', con=db.engine) - print(df) - return 'hello world' \ No newline at end of file diff --git a/flagging_site/blueprints/flagging.py b/flagging_site/blueprints/flagging.py index 9b84b1f8..880bbb5d 100644 --- a/flagging_site/blueprints/flagging.py +++ b/flagging_site/blueprints/flagging.py @@ -1,16 +1,18 @@ import pandas as pd from flask import Blueprint from flask import render_template -from flagging_site.data.hobolink import get_hobolink_data -from flagging_site.data.usgs import get_usgs_data -from flagging_site.data.model import process_data -from flagging_site.data.model import reach_2_model -from flagging_site.data.model import reach_3_model -from flagging_site.data.model import reach_4_model -from flagging_site.data.model import reach_5_model +from flask import request from flask_restful import Resource, Api -from flask import request +from ..data.hobolink import get_live_hobolink_data +from ..data.usgs import get_live_usgs_data +from ..data.model import process_data +from ..data.model import reach_2_model +from ..data.model import reach_3_model +from ..data.model import reach_4_model +from ..data.model import reach_5_model +from ..data.model import latest_model_outputs + bp = Blueprint('flagging', __name__) api = Api(bp) @@ -18,8 +20,8 @@ def get_data() -> pd.DataFrame: """Retrieves the data that gets plugged into the the model.""" - df_hobolink = get_hobolink_data('code_for_boston_export_21d') - df_usgs = get_usgs_data() + df_hobolink = get_live_hobolink_data('code_for_boston_export_21d') + df_usgs = get_live_usgs_data() df = process_data(df_hobolink, df_usgs) return df @@ -71,12 +73,10 @@ def index() -> str: returns: render model on index.html """ - df = get_data() flags = { - 2: reach_2_model(df, rows=1)['safe'].iloc[0], - 3: reach_3_model(df, rows=1)['safe'].iloc[0], - 4: reach_4_model(df, rows=1)['safe'].iloc[0], - 5: reach_5_model(df, rows=1)['safe'].iloc[0] + key: val['safe'] + for key, val + in latest_model_outputs().items() } return render_template('index.html', flags=flags) diff --git a/flagging_site/config.py b/flagging_site/config.py index f85afddf..e70979ab 100644 --- a/flagging_site/config.py +++ b/flagging_site/config.py @@ -13,6 +13,7 @@ # ~~~~~~~~~ ROOT_DIR = os.path.abspath(os.path.dirname(__file__)) +QUERIES_DIR = os.path.join(ROOT_DIR, 'data', 'queries') DATA_STORE = os.path.join(ROOT_DIR, 'data', '_store') VAULT_FILE = os.path.join(ROOT_DIR, 'vault.zip') @@ -40,11 +41,8 @@ def __repr__(self): # ========================================================================== # DATABASE CONFIG OPTIONS - # - # Not currently used, but soon we'll want to start using the config to set - # up references to the database, data storage, and data retrieval. # ========================================================================== - POSTGRES_USER: str = os.getenv('POSTGRES_USER', 'flagging_admin') + POSTGRES_USER: str = os.getenv('POSTGRES_USER', 'flagging') POSTGRES_PASSWORD: str = os.getenv('POSTGRES_PASSWORD') POSTGRES_HOST: str = 'localhost' POSTGRES_PORT: int = 5432 @@ -63,6 +61,8 @@ def SQLALCHEMY_DATABASE_URI(self) -> str: SQLALCHEMY_RECORD_QUERIES: bool = True SQLALCHEMY_TRACK_MODIFICATIONS: bool = False + QUERIES_DIR: str = QUERIES_DIR + # ========================================================================== # MISC. CUSTOM CONFIG OPTIONS # @@ -175,8 +175,3 @@ def get_config_from_env(env: str) -> Config: raise KeyError('Bad config passed; the config must be production, ' 'development, or testing.') return config() - - -def postgres_uri_from_params(user: str, password: str, host: str, db: str): - """postgres://username:password@server/db""" - return f'postgres://{user}:{password}@{host}/{db}' diff --git a/flagging_site/data/_store/refresh.py b/flagging_site/data/_store/refresh.py index a1b612b9..fb38dfb5 100644 --- a/flagging_site/data/_store/refresh.py +++ b/flagging_site/data/_store/refresh.py @@ -33,14 +33,14 @@ def refresh_data_store(vault_password: Optional[str] = None) -> None: from flagging_site.data.keys import get_data_store_file_path - from flagging_site.data.hobolink import get_hobolink_data + from flagging_site.data.hobolink import get_live_hobolink_data from flagging_site.data.hobolink import STATIC_FILE_NAME as hobolink_file - get_hobolink_data('code_for_boston_export_21d')\ + get_live_hobolink_data('code_for_boston_export_21d')\ .to_pickle(get_data_store_file_path(hobolink_file)) - from flagging_site.data.usgs import get_usgs_data + from flagging_site.data.usgs import get_live_usgs_data from flagging_site.data.usgs import STATIC_FILE_NAME as usgs_file - get_usgs_data().to_pickle(get_data_store_file_path(usgs_file)) + get_live_usgs_data().to_pickle(get_data_store_file_path(usgs_file)) if __name__ == '__main__': diff --git a/flagging_site/data/database.py b/flagging_site/data/database.py index b94a13a2..fec31e51 100644 --- a/flagging_site/data/database.py +++ b/flagging_site/data/database.py @@ -3,43 +3,34 @@ retrieving data. """ import os -import click -import psycopg2 -import psycopg2.extensions - -from flask import g +import pandas as pd +from typing import Optional from flask import current_app -from flask import Flask - -from ..config import ROOT_DIR from flask_sqlalchemy import SQLAlchemy +from sqlalchemy.exc import ResourceClosedError -SCHEMA_FILE = os.path.join(ROOT_DIR, 'data', 'schema.sql') db = SQLAlchemy() -# # this function connects to the database and creates a cursor -# def db_get() -> SQLAlchemy: # psycopg2.extensions.connection: -# with current_app.app_context(): -# if 'db' not in g: -# -# g.db = conn -# return g.db - - -def close_db(e=None): - """If this request connected to the database, close the - connection. - """ - db = g.pop('db', None) - if db is not None: - db.close() +def execute_sql(query: str) -> Optional[pd.DataFrame]: + """Execute arbitrary SQL in the database. This works for both read and write + operations. If it is a write operation, it will return None; otherwise it + returns a Pandas dataframe.""" + with db.engine.connect() as conn: + res = conn.execute(query) + try: + df = pd.DataFrame(res.fetchall()) + df.columns = res.keys() + return df + except ResourceClosedError: + return None -def execute_sql(query: str): - with db.engine.connect() as conn: - return conn.execute(query) +def execute_sql_from_file(file_name: str): + path = os.path.join(current_app.config['QUERIES_DIR'], file_name) + with current_app.open_resource(path) as f: + return execute_sql(f.read().decode('utf8')) def init_db(): @@ -47,18 +38,36 @@ def init_db(): with current_app.app_context(): # Read the `schema.sql` file, which initializes the database. - with current_app.open_resource(SCHEMA_FILE) as f: - schema = f.read().decode('utf8') + execute_sql_from_file('schema.sql') - # Run `schema.sql` - execute_sql(schema) + update_database() - # Populate the `usgs` table. - from .usgs import get_usgs_data - df = get_usgs_data() - df.to_sql('usgs', con=db.engine, index=False, if_exists='append') - # Populate the `hobolink` table. - from .hobolink import get_hobolink_data - df = get_hobolink_data() - df.to_sql('hobolink', con=db.engine, index=False, if_exists='append') +def update_database(): + """At the moment this overwrites the entire database. In the future we want + this to simply update it. + """ + + options = { + 'con': db.engine, + 'index': False, + 'if_exists': 'replace' + } + + # Populate the `usgs` table. + from .usgs import get_live_usgs_data + df_usgs = get_live_usgs_data() + df_usgs.to_sql('usgs', **options) + + # Populate the `hobolink` table. + from .hobolink import get_live_hobolink_data + df_hobolink = get_live_hobolink_data() + df_hobolink.to_sql('hobolink', **options) + + from .model import process_data + df = process_data(df_hobolink=df_hobolink, df_usgs=df_usgs) + df.to_sql('processed_data', **options) + + from .model import all_models + model_outs = all_models(df) + model_outs.to_sql('model_outputs', **options) diff --git a/flagging_site/data/db_tester.py b/flagging_site/data/db_tester.py deleted file mode 100644 index 3ed5b4e4..00000000 --- a/flagging_site/data/db_tester.py +++ /dev/null @@ -1,12 +0,0 @@ -# this file is solely used to test the functions in database.py -# to run these tests: -# place this file in the same folder as database.py -# at the command line prompt, enter python db_tester.py -from database import db_init_str, db_get, db_close, db_init - -db_get() - -db_init() - -db_close() - diff --git a/flagging_site/data/hobolink.py b/flagging_site/data/hobolink.py index df3721b4..39589248 100644 --- a/flagging_site/data/hobolink.py +++ b/flagging_site/data/hobolink.py @@ -6,6 +6,7 @@ # Pandas is inefficient. It should go to SQL, not to Pandas. I am currently # using pandas because we do not have any cron jobs or any caching or SQL, but # I think in future versions we should not be using Pandas at all. +from .database import db import io import requests import pandas as pd @@ -37,7 +38,7 @@ # ~ ~ ~ ~ -def get_hobolink_data(export_name: str = EXPORT_NAME) -> pd.DataFrame: +def get_live_hobolink_data(export_name: str = EXPORT_NAME) -> pd.DataFrame: """This function runs through the whole process for retrieving data from HOBOlink: first we perform the request, and then we clean the data. diff --git a/flagging_site/data/model.py b/flagging_site/data/model.py index a0b46e23..adbb5ce0 100644 --- a/flagging_site/data/model.py +++ b/flagging_site/data/model.py @@ -168,7 +168,8 @@ def reach_2_model(df: pd.DataFrame, rows: int = 24) -> pd.DataFrame: ) df['probability'] = sigmoid(df['log_odds']) df['safe'] = df['probability'] <= SAFETY_THRESHOLD - return df[['time', 'log_odds', 'probability', 'safe']] + df['reach'] = 2 + return df[['reach', 'time', 'log_odds', 'probability', 'safe']] def reach_3_model(df: pd.DataFrame, rows: int = 24) -> pd.DataFrame: @@ -196,7 +197,8 @@ def reach_3_model(df: pd.DataFrame, rows: int = 24) -> pd.DataFrame: ) df['probability'] = sigmoid(df['log_odds']) df['safe'] = df['probability'] <= SAFETY_THRESHOLD - return df[['time', 'log_odds', 'probability', 'safe']] + df['reach'] = 3 + return df[['reach', 'time', 'log_odds', 'probability', 'safe']] def reach_4_model(df: pd.DataFrame, rows: int = 24) -> pd.DataFrame: @@ -224,7 +226,8 @@ def reach_4_model(df: pd.DataFrame, rows: int = 24) -> pd.DataFrame: ) df['probability'] = sigmoid(df['log_odds']) df['safe'] = df['probability'] <= SAFETY_THRESHOLD - return df[['time', 'log_odds', 'probability', 'safe']] + df['reach'] = 4 + return df[['reach', 'time', 'log_odds', 'probability', 'safe']] def reach_5_model(df: pd.DataFrame, rows: int = 24) -> pd.DataFrame: @@ -250,4 +253,25 @@ def reach_5_model(df: pd.DataFrame, rows: int = 24) -> pd.DataFrame: ) df['probability'] = sigmoid(df['log_odds']) df['safe'] = df['probability'] <= SAFETY_THRESHOLD - return df[['time', 'log_odds', 'probability', 'safe']] + df['reach'] = 5 + return df[['reach', 'time', 'log_odds', 'probability', 'safe']] + + +def all_models(df: pd.DataFrame, *args, **kwargs): + out = pd.concat([ + reach_2_model(df, *args, **kwargs), + reach_3_model(df, *args, **kwargs), + reach_4_model(df, *args, **kwargs), + reach_5_model(df, *args, **kwargs), + ], axis=0) + out = out.sort_values(['reach', 'time']) + return out + + +def latest_model_outputs(hours: int = 1) -> dict: + if hours != 1: + raise NotImplementedError('Need to work on this!') + from .database import execute_sql_from_file + df = execute_sql_from_file('latest_model.sql') + df = df.set_index('reach') + return df.to_dict(orient='index') diff --git a/flagging_site/data/queries/latest_model.sql b/flagging_site/data/queries/latest_model.sql new file mode 100644 index 00000000..3d7a773c --- /dev/null +++ b/flagging_site/data/queries/latest_model.sql @@ -0,0 +1,5 @@ +-- This query returns the latest values for the model. + +SELECT * +FROM model_outputs +WHERE time = (SELECT MAX(time) FROM model_outputs); diff --git a/flagging_site/data/schema.sql b/flagging_site/data/queries/schema.sql similarity index 63% rename from flagging_site/data/schema.sql rename to flagging_site/data/queries/schema.sql index d1752618..75a1778f 100644 --- a/flagging_site/data/schema.sql +++ b/flagging_site/data/queries/schema.sql @@ -20,17 +20,13 @@ CREATE TABLE IF NOT EXISTS hobolink ( air_temp decimal ); -DROP TABLE IF EXISTS ecoli_predictions; -CREATE TABLE IF NOT EXISTS ecoli_predictions ( - timestamp timestamp, - r2_probability decimal, - r3_probability decimal, - r4_probability decimal, - r5_probability decimal, - r2_safe boolean, - r3_safe boolean, - r4_safe boolean, - r5_safe boolean +DROP TABLE IF EXISTS model_outputs; +CREATE TABLE IF NOT EXISTS model_outputs ( + reach int, + time timestamp, + log_odds decimal, + probability decimal, + safe boolean ); -COMMIT; \ No newline at end of file +COMMIT; diff --git a/flagging_site/data/task_queue.py b/flagging_site/data/task_queue.py deleted file mode 100644 index df5a61b3..00000000 --- a/flagging_site/data/task_queue.py +++ /dev/null @@ -1,4 +0,0 @@ -""" -This file should define and set up all the tasks that need to be pushed through -the task queue, and should also handle the logic of setting up the task queue. -""" \ No newline at end of file diff --git a/flagging_site/data/usgs.py b/flagging_site/data/usgs.py index fde1fea5..936d2bf5 100644 --- a/flagging_site/data/usgs.py +++ b/flagging_site/data/usgs.py @@ -18,7 +18,7 @@ # ~ ~ ~ ~ -def get_usgs_data() -> pd.DataFrame: +def get_live_usgs_data() -> pd.DataFrame: """This function runs through the whole process for retrieving data from usgs: first we perform the request, and then we clean the data. From b6bcb20bd70f9e744e08d6d0ddd1e88b2b460879 Mon Sep 17 00:00:00 2001 From: Daniel Reeves Date: Sun, 19 Jul 2020 18:08:09 -0400 Subject: [PATCH 009/118] updated dev scripts --- run_unix_dev.sh | 6 +++++- run_windows_dev.bat | 2 +- 2 files changed, 6 insertions(+), 2 deletions(-) diff --git a/run_unix_dev.sh b/run_unix_dev.sh index dfa0ce69..c268d1d2 100644 --- a/run_unix_dev.sh +++ b/run_unix_dev.sh @@ -23,10 +23,14 @@ source venv/bin/activate $PYEXEC -m pip install -r requirements.txt # Set up and run the Flask application -export FLASK_APP=flagging_site +export FLASK_APP=flagging_site:create_app export FLASK_ENV=development read -p "Offline mode? [y/n]: " offline_mode export OFFLINE_MODE=${offline_mode} read -p "Enter vault password: " vault_pw export VAULT_PASSWORD=${vault_pw} +read -p "Enter Postgres password: " postgres_pw +export VAULT_PASSWORD=${postgres_pw} + +flask init-db flask run \ No newline at end of file diff --git a/run_windows_dev.bat b/run_windows_dev.bat index 59dae58e..361e6de8 100644 --- a/run_windows_dev.bat +++ b/run_windows_dev.bat @@ -7,7 +7,7 @@ set FLASK_APP=flagging_site:create_app set FLASK_ENV=development set /p OFFLINE_MODE="Offline mode? [y/n]: " set /p VAULT_PASSWORD="Enter vault password: " -set /p POSTGRES_PASSWORD="Enter postgres PW: " +set /p POSTGRES_PASSWORD="Enter Postgres password: " flask init-db flask run \ No newline at end of file From e4d6c2b9047bbb15683199b9b77bd51c27e0bd6d Mon Sep 17 00:00:00 2001 From: Daniel Reeves Date: Sun, 19 Jul 2020 18:29:50 -0400 Subject: [PATCH 010/118] bugfix --- flagging_site/config.py | 3 ++- flagging_site/data/database.py | 2 ++ flagging_site/data/hobolink.py | 1 + 3 files changed, 5 insertions(+), 1 deletion(-) diff --git a/flagging_site/config.py b/flagging_site/config.py index e70979ab..0c6439c3 100644 --- a/flagging_site/config.py +++ b/flagging_site/config.py @@ -7,6 +7,7 @@ """ import os from typing import Dict, Any, Optional, List +from distutils.util import strtobool # Constants @@ -138,7 +139,7 @@ class DevelopmentConfig(Config): VAULT_OPTIONAL: bool = True DEBUG: bool = True TESTING: bool = True - OFFLINE_MODE = os.getenv('OFFLINE_MODE', 'false') + OFFLINE_MODE = strtobool(os.getenv('OFFLINE_MODE', 'false')) class TestingConfig(Config): diff --git a/flagging_site/data/database.py b/flagging_site/data/database.py index fec31e51..32eeab89 100644 --- a/flagging_site/data/database.py +++ b/flagging_site/data/database.py @@ -61,7 +61,9 @@ def update_database(): # Populate the `hobolink` table. from .hobolink import get_live_hobolink_data + print('AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA') df_hobolink = get_live_hobolink_data() + print('AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA') df_hobolink.to_sql('hobolink', **options) from .model import process_data diff --git a/flagging_site/data/hobolink.py b/flagging_site/data/hobolink.py index 39589248..e13c181c 100644 --- a/flagging_site/data/hobolink.py +++ b/flagging_site/data/hobolink.py @@ -49,6 +49,7 @@ def get_live_hobolink_data(export_name: str = EXPORT_NAME) -> pd.DataFrame: Returns: Pandas Dataframe containing the cleaned-up Hobolink data. """ + print('OFFLINE MODE', offline_mode()) if offline_mode(): df = pd.read_pickle(get_data_store_file_path(STATIC_FILE_NAME)) else: From 066d6900c1b4491721f8dc5834925dd0891bc9b0 Mon Sep 17 00:00:00 2001 From: Daniel Reeves Date: Sun, 19 Jul 2020 18:31:21 -0400 Subject: [PATCH 011/118] remove prints --- flagging_site/data/database.py | 2 -- flagging_site/data/hobolink.py | 1 - 2 files changed, 3 deletions(-) diff --git a/flagging_site/data/database.py b/flagging_site/data/database.py index 32eeab89..fec31e51 100644 --- a/flagging_site/data/database.py +++ b/flagging_site/data/database.py @@ -61,9 +61,7 @@ def update_database(): # Populate the `hobolink` table. from .hobolink import get_live_hobolink_data - print('AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA') df_hobolink = get_live_hobolink_data() - print('AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA') df_hobolink.to_sql('hobolink', **options) from .model import process_data diff --git a/flagging_site/data/hobolink.py b/flagging_site/data/hobolink.py index e13c181c..39589248 100644 --- a/flagging_site/data/hobolink.py +++ b/flagging_site/data/hobolink.py @@ -49,7 +49,6 @@ def get_live_hobolink_data(export_name: str = EXPORT_NAME) -> pd.DataFrame: Returns: Pandas Dataframe containing the cleaned-up Hobolink data. """ - print('OFFLINE MODE', offline_mode()) if offline_mode(): df = pd.read_pickle(get_data_store_file_path(STATIC_FILE_NAME)) else: From 71eaab01e3949e072aae85db13f062d4cd2167cf Mon Sep 17 00:00:00 2001 From: Lewis Staples Date: Wed, 22 Jul 2020 23:18:23 -0400 Subject: [PATCH 012/118] Installed flask_sqlalchemy --- requirements.txt | Bin 641 -> 632 bytes 1 file changed, 0 insertions(+), 0 deletions(-) diff --git a/requirements.txt b/requirements.txt index 56d11b6df135a4600c5b64f4052417faf0255719..878c58b800cac15a6687998a0bef24c89468247b 100644 GIT binary patch literal 632 zcmZ{hJx{|h7=&*|;-?fPDOJnBz`}s&fI2XzX`6n!QPNa|9}nEUPGc`9vK0Bw=jUGk z{^Y3mt*}Fl2G9IbJmAsRJ?>E8NUgxZv?=$cXON~-4KC(bnx0S_n@9SF`Vnic^N=7_ zuC&j1rJ6Iv=PNtp2W^A}KJbngx(jTXc|vD5W;_vSccf{^mBXQ=tLDvZ)JxfkvA>bc zVNYi^f*F6uk3aj10~u&VfS??V)OYa z*ehd3*~?zgS~&U17%d;95>(RS?Uj9Tt1${Ol$631Jj(&dIxha?(a`H2`2)J_nRs%= z7!=3{VS~39W1ggvTG$j4(?Z77>OuSrv?8fmvVrdH$NR@=3$DNFSP!K<5-YnBcbJ z;hvWJjB>AA;hhqHZl5o&ziUTb#O~jQ0pOvfuD4wMchgv}jKjAJsM=R<*-ah(#%_ud v8wa>PRHzR)#MNH?RL)fcK92b7_;Pm} Date: Sun, 26 Jul 2020 07:03:18 -0400 Subject: [PATCH 013/118] preliminary design of widget on front page --- flagging_site/templates/flags.html | 21 +++++++++++++++++++++ flagging_site/templates/index.html | 3 +++ 2 files changed, 24 insertions(+) create mode 100644 flagging_site/templates/flags.html diff --git a/flagging_site/templates/flags.html b/flagging_site/templates/flags.html new file mode 100644 index 00000000..93796917 --- /dev/null +++ b/flagging_site/templates/flags.html @@ -0,0 +1,21 @@ +{% block content %} + {% for row in flags %} +
+ {% for boathouse, flag in row.items() %} +
+ {% if flag %} + + {% else %} + + {% endif %} + +
+
+ {{ boathouse }} +
+ {% endfor %} +
+
+
+ {% endfor %} +{% endblock %} \ No newline at end of file diff --git a/flagging_site/templates/index.html b/flagging_site/templates/index.html index 8caa1519..170d69e2 100644 --- a/flagging_site/templates/index.html +++ b/flagging_site/templates/index.html @@ -26,5 +26,8 @@

Current Water Quality

{% endif %} {% endfor %} +
+ +
{% endblock %} \ No newline at end of file From 0564c7ffeb725c386466481909ab817c9f2b7129 Mon Sep 17 00:00:00 2001 From: Edwin Sun <34005715+edsun123@users.noreply.github.com> Date: Sun, 26 Jul 2020 07:19:24 -0400 Subject: [PATCH 014/118] first design of widget on front page --- flagging_site/blueprints/api.py | 1 - flagging_site/blueprints/flagging.py | 31 ++++++++++++++- flagging_site/blueprints/reach_api.yml | 52 -------------------------- 3 files changed, 30 insertions(+), 54 deletions(-) delete mode 100644 flagging_site/blueprints/reach_api.yml diff --git a/flagging_site/blueprints/api.py b/flagging_site/blueprints/api.py index a1f7ed7b..e37cd672 100644 --- a/flagging_site/blueprints/api.py +++ b/flagging_site/blueprints/api.py @@ -86,7 +86,6 @@ def model_api(reach_param: list = None, hour: str = 48) -> dict: return main class ReachesApi(Resource): - @swag_from('reach_api.yml') def get(self): reach = request.args.getlist('reach', None) hour = request.args.get('hour', 48) diff --git a/flagging_site/blueprints/flagging.py b/flagging_site/blueprints/flagging.py index 125086f1..539d23fe 100644 --- a/flagging_site/blueprints/flagging.py +++ b/flagging_site/blueprints/flagging.py @@ -112,4 +112,33 @@ def output_model() -> str: in reach_model_mapping.items() } - return render_template('output_model.html', tables=reach_html_tables) \ No newline at end of file + return render_template('output_model.html', tables=reach_html_tables) + +@bp.route('/flags') +def flags() -> str: + + df = get_data() + + flags_1 = { + 'Newton Yacht Club': reach_2_model(df, rows=1)['safe'].iloc[0], + 'Watertown Yacht Club': reach_2_model(df, rows=1)['safe'].iloc[0], + 'Community Rowing, Inc.': reach_2_model(df, rows=1)['safe'].iloc[0], + 'Northeastern\'s Henderson Boathouse': reach_2_model(df, rows=1)['safe'].iloc[0] + } + + flags_2 = { + 'Paddle Boston at Herter Park': reach_3_model(df, rows=1)['safe'].iloc[0], + 'Harvard\'s Weld Boathouse': reach_3_model(df, rows=1)['safe'].iloc[0], + 'Riverside Boat Club': reach_4_model(df, rows=1)['safe'].iloc[0], + 'Charles River Yacht Club': reach_5_model(df, rows=1)['safe'].iloc[0], + } + + flags_3 = { + 'Union Boat Club': reach_5_model(df, rows=1)['safe'].iloc[0], + 'Community Boating': reach_5_model(df, rows=1)['safe'].iloc[0], + 'Paddle Boston at Kendall Square': reach_5_model(df, rows=1)['safe'].iloc[0] + } + + flags = [flags_1, flags_2, flags_3] + + return render_template('flags.html', flags=flags) \ No newline at end of file diff --git a/flagging_site/blueprints/reach_api.yml b/flagging_site/blueprints/reach_api.yml deleted file mode 100644 index fe0f816a..00000000 --- a/flagging_site/blueprints/reach_api.yml +++ /dev/null @@ -1,52 +0,0 @@ -Endpoint returning json of the output model ---- -tags: - - Reach Model API -parameters: - - name: reach - description: type of reach model - in: query - type: int - enum: [2, 3, 4, 5] - required: false - default: - - name: hour - description: type of reach model - in: query - type: int - enum: [1, 2, 3, 6, 12, 24, 36, 48] - required: false - default: -responses: - 200: - description: Dictionary-like json of the output model - schema: - id: model_api - type: object - properties: - version: - description: the year schema is made - type: string - time_returned: - description: time data is returned - type: string - models: - description: the year schema is made - type: object - properties: - model_num: - description: type of reach model - type: object - properties: - time: - description: time of data - type: string - log_odds: - description: logistical odds - type: number - probability: - description: probability - type: number - safe: - description: whether or not river is safe - type: boolean From b79a80570dac480253d9c1e1902373383c6dd757 Mon Sep 17 00:00:00 2001 From: Edwin Sun <34005715+edsun123@users.noreply.github.com> Date: Mon, 27 Jul 2020 08:44:50 -0400 Subject: [PATCH 015/118] added define_boathouse.sql and create new boathouse table --- flagging_site/data/database.py | 10 +-- .../data/queries/define_boathouse.sql | 68 +++++++++++++++++++ flagging_site/data/queries/schema.sql | 10 ++- requirements.txt | 1 + 4 files changed, 80 insertions(+), 9 deletions(-) create mode 100644 flagging_site/data/queries/define_boathouse.sql diff --git a/flagging_site/data/database.py b/flagging_site/data/database.py index fec31e51..f87cdaae 100644 --- a/flagging_site/data/database.py +++ b/flagging_site/data/database.py @@ -9,10 +9,8 @@ from flask_sqlalchemy import SQLAlchemy from sqlalchemy.exc import ResourceClosedError - db = SQLAlchemy() - def execute_sql(query: str) -> Optional[pd.DataFrame]: """Execute arbitrary SQL in the database. This works for both read and write operations. If it is a write operation, it will return None; otherwise it @@ -26,28 +24,22 @@ def execute_sql(query: str) -> Optional[pd.DataFrame]: except ResourceClosedError: return None - def execute_sql_from_file(file_name: str): path = os.path.join(current_app.config['QUERIES_DIR'], file_name) with current_app.open_resource(path) as f: return execute_sql(f.read().decode('utf8')) - def init_db(): """Clear existing data and create new tables.""" with current_app.app_context(): - # Read the `schema.sql` file, which initializes the database. execute_sql_from_file('schema.sql') - update_database() - def update_database(): """At the moment this overwrites the entire database. In the future we want this to simply update it. """ - options = { 'con': db.engine, 'index': False, @@ -71,3 +63,5 @@ def update_database(): from .model import all_models model_outs = all_models(df) model_outs.to_sql('model_outputs', **options) + + execute_sql_from_file('define_boathouse.sql') \ No newline at end of file diff --git a/flagging_site/data/queries/define_boathouse.sql b/flagging_site/data/queries/define_boathouse.sql new file mode 100644 index 00000000..872ed39c --- /dev/null +++ b/flagging_site/data/queries/define_boathouse.sql @@ -0,0 +1,68 @@ +INSERT INTO boathouses (reach, boathouse, latitude, longitude) +VALUES ( + 2, + 'Newton Yacht Club', + 42.358698, + 71.172850 + ), + ( + 2, + 'Watertown Yacht Club', + 42.361952, + 71.167791 + ), + ( + 2, + 'Community Rowing, Inc.', + 42.358633, + 71.165467 + ), + ( + 3, + 'Northeastern''s Henderson Boathouse', + 42.364135, + 71.141571 + ), + ( + 3, + 'Paddle Boston at Herter Park', + 42.369182, + 71.131301 + ), + ( + 3, + 'Harvard''s Weld Boathouse', + 42.369566, + 71.122083 + ), + ( + 4, + 'Riverside Boat Club', + 42.358272, + 71.115763 + ), + ( + 5, + 'Charles River Yacht Club', + 42.360526, + 71.084760 + ), + ( + 5, + 'Union Boat Club', + 42.357816, + 71.073319 + ), + ( + 5, + 'Community Boating', + 42.359935, + 71.073035 + ), + ( + 5, + 'Paddle Boston at Kendall Square', + 42.362964, + 71.082112 + ); + diff --git a/flagging_site/data/queries/schema.sql b/flagging_site/data/queries/schema.sql index 75a1778f..a1c17b4c 100644 --- a/flagging_site/data/queries/schema.sql +++ b/flagging_site/data/queries/schema.sql @@ -20,6 +20,14 @@ CREATE TABLE IF NOT EXISTS hobolink ( air_temp decimal ); +DROP TABLE IF EXISTS boathouses; +CREATE TABLE IF NOT EXISTS boathouses ( + reach int, + boathouse varchar(255), + latitude decimal, + longitude decimal +); + DROP TABLE IF EXISTS model_outputs; CREATE TABLE IF NOT EXISTS model_outputs ( reach int, @@ -29,4 +37,4 @@ CREATE TABLE IF NOT EXISTS model_outputs ( safe boolean ); -COMMIT; +COMMIT; \ No newline at end of file diff --git a/requirements.txt b/requirements.txt index 56d11b6d..1ddbaeee 100644 --- a/requirements.txt +++ b/requirements.txt @@ -38,3 +38,4 @@ wcwidth==0.2.4 Werkzeug==1.0.1 wrapt==1.12.1 zipp==3.1.0 +psycopg2-binary==2.8.5 \ No newline at end of file From d00ce9e100bfafbec36ced77cf48e728027c905c Mon Sep 17 00:00:00 2001 From: Edwin Sun <34005715+edsun123@users.noreply.github.com> Date: Tue, 28 Jul 2020 00:39:30 -0400 Subject: [PATCH 016/118] edited README.md; added database.md to /docs --- README.md | 56 ++++++++++++++++++++++------------ docs/docs/database.md | 49 +++++++++++++++++++++++++++++ flagging_site/data/README.md | 1 + flagging_site/data/database.py | 5 +++ 4 files changed, 91 insertions(+), 20 deletions(-) create mode 100644 docs/docs/database.md diff --git a/README.md b/README.md index e3f0c73f..3895266a 100644 --- a/README.md +++ b/README.md @@ -14,7 +14,22 @@ For strict documentation of the website sans project management stuff, read the These are the steps to set the code up in development mode. -**On Windows:** open up a Command Prompt terminal window (the default in PyCharm on Windows), point the terminal to the project directory if it's not already there, and enter: +**On Windows:** (in progress) +First install Postgressql using this link: https://www.postgresql.org/download/ + +Then activate the Postgressql services in the Windows Services Panel. + +Set environment variable `POSTGRES_PASSWORD` to be whatever you want. + +Powershell and CMD (Windows): + +```commandline +set POSTGRES_PASSWORD=enter_password_here +createdb -U postgres flagging +psql -U postgres -d flagging -c "DROP USER IF EXISTS flagging; CREATE USER flagging SUPERUSER PASSWORD '%POSTGRES_PASSWORD%'" +``` + +open up a Command Prompt terminal window (the default in PyCharm on Windows), point the terminal to the project directory if it's not already there, and enter: ```commandline run_windows_dev @@ -22,7 +37,24 @@ run_windows_dev If you are in PowerShell (default VSC terminal), use `start-process run_windows_dev.bat` instead. -**On OSX or Linux:** open up a Bash terminal, and in the project directory enter: +**On OSX or Linux:** We need to setup postgres database first thus developers should install PostgreSQL with Homebrew then start the PostgreSQL database service. + +``` +brew install postgresql +brew services start postgresql +``` + +To begin initialize a database which we call `flagging`, enter into the bash terminal: + +```shell script +export POSTGRES_PASSWORD=*enter_password_here* +createdb -U *enter_username_here* flagging +psql -U *enter_username_here* -d flagging -c "DROP USER IF EXISTS flagging; CREATE USER flagging SUPERUSER PASSWORD '${POSTGRES_PASSWORD}'" +``` + +Note postgres password can be any password and postgres usernme can be default username `postgres` or your OS username. + +To run the website, in the project directory `flagging` enter: ```shell script sh run_unix_dev.sh @@ -34,6 +66,8 @@ After that, it will ask if you have the vault password. If you do, enter it here Note that the website will _not_ without either the vault password or offline mode turned on; you must do one or the other. +Next it will ask you to for the postgres password that you exported earlier. If you are in online mode, you can skip this as you do in the password section. + ## Deploy 1. Download Heroku. @@ -64,21 +98,3 @@ Note: the test may require you to enter the vault password if it is not already ## Start DB -Set environment variable `POSTGRES_PASSWORD` to be whatever you want. - -Powershell and CMD (Windows): - -```commandline -set POSTGRES_PASSWORD=enter_password_here -createdb -U postgres flagging -psql -U postgres -d flagging -c "DROP USER IF EXISTS flagging; CREATE USER flagging SUPERUSER PASSWORD '%POSTGRES_PASSWORD%'" -``` - -Bash (Mac OSX / Linux): - -```shell script -export POSTGRES_PASSWORD=enter_password_here -createdb -U postgres flagging -psql -U postgres -d flagging -c "DROP USER IF EXISTS flagging; CREATE USER flagging SUPERUSER PASSWORD '${POSTGRES_PASSWORD}'" -``` - diff --git a/docs/docs/database.md b/docs/docs/database.md new file mode 100644 index 00000000..c1d984ee --- /dev/null +++ b/docs/docs/database.md @@ -0,0 +1,49 @@ +# Database Project In-depth Guide + +We will be using PostgreSQL, a free, open-source database management system sucessor to UC Berkeley's Ingres Database but also support SQL language + +**On OSX or Linux:** + +We need to setup postgres database first thus enter into the bash terminals: + +``` +brew install postgresql +brew services start postgresql +``` +Explanation: We will need to install postgresql in order to create our database. With postgresql installed, we can start up database locally or in our computer. We use `brew` from homebrew to install and start postgresql services. To get homebrew, consult with this link: https://brew.sh/ + +To begin initialize a database, enter into the bash terminal: + +```shell script +export POSTGRES_PASSWORD=*enter_password_here* +createdb -U *enter_username_here* flagging +psql -U *enter_username_here* -d flagging -c "DROP USER IF EXISTS flagging; CREATE USER flagging SUPERUSER PASSWORD '${POSTGRES_PASSWORD}'" +``` +Explanation: Postgres password can be any password you choose. We exported your chosen postgres password into `POSTGRES_PASSWORD`, an environment variable, which is a variable set outside a program and is independent in each session. Next, we created a database called `flagging` using a username/rolename, which needs to be a Superuser or having all accesses of postgres. By default, the Superuser rolename can be `postgres` or the username for you OS. To find out, you can go into psql terminal, which we will explain below, and enter `\du` to see all usernames. Finally, we add the database `flagging` using the env variable in which we save our password. + +You can see the results using the postgresql terminal which you can open by entering: +``` +psql +``` + +Below are a couple of helpful commands you can use in the postgresql: + +``` +\q --to quit +\c *database_name* --to connect to database +\d --show what tables in current database +\du --show database users +\dt --show tables of current database +``` + +To run the website, in the project directory `flagging` enter: + +```shell script +sh run_unix_dev.sh +``` + +Running the bash script `run_unix_dev.sh` found in the `flagging` folder. Inside the scirpt, it defines environment variables `FLASK_APP` and `FLASK_ENV` which we need to find app.py. We also export the user input for offline mode, vault password, and postgres password for validation. Finally we initialize a database with a custom flask command `flask init-db` and finally run the flask application `flask run`. + +Regarding in how flask application connects to postgresql, `database.py` creates an object `db = SQLAlchemy()` which we will refer again in `app.py` to configure the flask application to support postgressql `from .data import db` `db.init_app(app)`. (We can import the `db` object beecause `__init__.py` make the object available as a global variable) + +Flask supports creating custom commands `init-db` for initializing database and `update-db` for updating database. `init-db` command calls `init_db` function from `database.py` and essentially calls `execute_sql()` which executes the sql file `schema.sql` that creates all the tables. Then calls `update_database()` which fills the database with data from usgs, hobolink, etc. `update-db` command primarily just udpates the table thus does not create new tables. Note: currently we are creating and deleting the database everytime the bashscript and program runs. \ No newline at end of file diff --git a/flagging_site/data/README.md b/flagging_site/data/README.md index 2921d1a8..b8ddad45 100644 --- a/flagging_site/data/README.md +++ b/flagging_site/data/README.md @@ -8,3 +8,4 @@ - `model.py`: outputs table model by processing usgs and hobolink data - `task_queue`: set up task queue - `usgs.py`: retrieve hobolink by requesting a response and parsing data from usgs + diff --git a/flagging_site/data/database.py b/flagging_site/data/database.py index f87cdaae..c2a07d3e 100644 --- a/flagging_site/data/database.py +++ b/flagging_site/data/database.py @@ -9,8 +9,10 @@ from flask_sqlalchemy import SQLAlchemy from sqlalchemy.exc import ResourceClosedError + db = SQLAlchemy() + def execute_sql(query: str) -> Optional[pd.DataFrame]: """Execute arbitrary SQL in the database. This works for both read and write operations. If it is a write operation, it will return None; otherwise it @@ -24,11 +26,13 @@ def execute_sql(query: str) -> Optional[pd.DataFrame]: except ResourceClosedError: return None + def execute_sql_from_file(file_name: str): path = os.path.join(current_app.config['QUERIES_DIR'], file_name) with current_app.open_resource(path) as f: return execute_sql(f.read().decode('utf8')) + def init_db(): """Clear existing data and create new tables.""" with current_app.app_context(): @@ -36,6 +40,7 @@ def init_db(): execute_sql_from_file('schema.sql') update_database() + def update_database(): """At the moment this overwrites the entire database. In the future we want this to simply update it. From 2594b0eaeaa4453eb60b902ac59cab90b7ff18ed Mon Sep 17 00:00:00 2001 From: Edwin Sun <34005715+edsun123@users.noreply.github.com> Date: Tue, 28 Jul 2020 20:36:05 -0400 Subject: [PATCH 017/118] corrected reach_model for henderson --- flagging_site/blueprints/flagging.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/flagging_site/blueprints/flagging.py b/flagging_site/blueprints/flagging.py index 539d23fe..1ae4b622 100644 --- a/flagging_site/blueprints/flagging.py +++ b/flagging_site/blueprints/flagging.py @@ -127,7 +127,7 @@ def flags() -> str: } flags_2 = { - 'Paddle Boston at Herter Park': reach_3_model(df, rows=1)['safe'].iloc[0], + 'Paddle Boston at Herter Park': reach_2_model(df, rows=1)['safe'].iloc[0], 'Harvard\'s Weld Boathouse': reach_3_model(df, rows=1)['safe'].iloc[0], 'Riverside Boat Club': reach_4_model(df, rows=1)['safe'].iloc[0], 'Charles River Yacht Club': reach_5_model(df, rows=1)['safe'].iloc[0], From 9832aa18e09d140b0c43a79ce8773fde90feb160 Mon Sep 17 00:00:00 2001 From: Edwin Sun <34005715+edsun123@users.noreply.github.com> Date: Tue, 28 Jul 2020 20:55:34 -0400 Subject: [PATCH 018/118] corrected reach model for henderson in sql file --- flagging_site/data/queries/define_boathouse.sql | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/flagging_site/data/queries/define_boathouse.sql b/flagging_site/data/queries/define_boathouse.sql index 872ed39c..d733bff2 100644 --- a/flagging_site/data/queries/define_boathouse.sql +++ b/flagging_site/data/queries/define_boathouse.sql @@ -18,7 +18,7 @@ VALUES ( 71.165467 ), ( - 3, + 2, 'Northeastern''s Henderson Boathouse', 42.364135, 71.141571 From 64a407d2dd79f43b7b2233836673de3940760d5c Mon Sep 17 00:00:00 2001 From: Edwin Sun <34005715+edsun123@users.noreply.github.com> Date: Tue, 28 Jul 2020 21:31:12 -0400 Subject: [PATCH 019/118] fixed issues in the docs, sql file, and database.py --- flagging_site/data/database.py | 5 ++--- flagging_site/data/queries/define_boathouse.sql | 2 +- 2 files changed, 3 insertions(+), 4 deletions(-) diff --git a/flagging_site/data/database.py b/flagging_site/data/database.py index c2a07d3e..d3ce6d82 100644 --- a/flagging_site/data/database.py +++ b/flagging_site/data/database.py @@ -38,6 +38,7 @@ def init_db(): with current_app.app_context(): # Read the `schema.sql` file, which initializes the database. execute_sql_from_file('schema.sql') + execute_sql_from_file('define_boathouse.sql') update_database() @@ -67,6 +68,4 @@ def update_database(): from .model import all_models model_outs = all_models(df) - model_outs.to_sql('model_outputs', **options) - - execute_sql_from_file('define_boathouse.sql') \ No newline at end of file + model_outs.to_sql('model_outputs', **options) \ No newline at end of file diff --git a/flagging_site/data/queries/define_boathouse.sql b/flagging_site/data/queries/define_boathouse.sql index d733bff2..8f15342c 100644 --- a/flagging_site/data/queries/define_boathouse.sql +++ b/flagging_site/data/queries/define_boathouse.sql @@ -24,7 +24,7 @@ VALUES ( 71.141571 ), ( - 3, + 2, 'Paddle Boston at Herter Park', 42.369182, 71.131301 From 24e423f011cae1aac6fdc1d395bd6ef8715c313a Mon Sep 17 00:00:00 2001 From: Lewis Staples Date: Mon, 3 Aug 2020 23:19:30 -0400 Subject: [PATCH 020/118] 49: allowing 1 or 48 hrs of data to be pulled from db; 50: displaying detailed output from db --- flagging_site/blueprints/flagging.py | 76 +++++++++++++------ flagging_site/data/model.py | 24 +++--- ...sql => return_1_hour_of_model_outputs.sql} | 0 .../return_48_hours_of_model_outputs.sql | 10 +++ 4 files changed, 79 insertions(+), 31 deletions(-) rename flagging_site/data/queries/{latest_model.sql => return_1_hour_of_model_outputs.sql} (100%) create mode 100644 flagging_site/data/queries/return_48_hours_of_model_outputs.sql diff --git a/flagging_site/blueprints/flagging.py b/flagging_site/blueprints/flagging.py index 880bbb5d..d98eea91 100644 --- a/flagging_site/blueprints/flagging.py +++ b/flagging_site/blueprints/flagging.py @@ -38,6 +38,11 @@ def stylize_model_output(df: pd.DataFrame) -> str: Returns: HTML table. """ + + # print('\n\ndf conv to html\n\n') + # print(df) + # print('\n\n') + def _apply_flag(x: bool) -> str: flag_class = 'blue-flag' if x else 'red-flag' return f'{x}' @@ -68,15 +73,23 @@ def add_to_dict(models, df, reach) -> None: @bp.route('/') def index() -> str: """ - Retrieves data from hobolink and usgs and processes data, then displays data - on `index_model.html` + Retrieves data from database, + then displays data on `index_model.html` returns: render model on index.html """ + + df = latest_model_outputs() + + print('\n\nlatest_model_outputs:\n\n') + print( latest_model_outputs(48) ) + print('\n\n') + + df = df.set_index('reach') flags = { key: val['safe'] for key, val - in latest_model_outputs().items() + in df.to_dict(orient='index').items() } return render_template('index.html', flags=flags) @@ -113,26 +126,45 @@ def output_model() -> str: # Look at no more than 72 hours. hours = min(max(hours, 1), 72) - df = get_data() + df = latest_model_outputs(hours) - reach_model_mapping = { - 2: reach_2_model, - 3: reach_3_model, - 4: reach_4_model, - 5: reach_5_model - } + + print('\n\ndf_list\n\n') + print( df.reach.unique() ) + print('\n\n') + # go through each reach in df and create a subset of df of that reach only + # then add that to reach_html_tables + + reach_html_tables = {} + for i in df.reach.unique(): + print( i ) + reach_html_tables[i] = stylize_model_output( df.loc[df['reach'] == i ] ) - if reach in reach_model_mapping: - reach_func = reach_model_mapping[int(reach)] - reach_html_tables = { - reach: stylize_model_output(reach_func(df, rows=hours)) - } - else: - reach_html_tables = { - reach: stylize_model_output(reach_func(df, rows=hours)) - for reach, reach_func - in reach_model_mapping.items() - } + + # df = get_data() + + # reach_model_mapping = { + # 2: reach_2_model, + # 3: reach_3_model, + # 4: reach_4_model, + # 5: reach_5_model + # } + + # if reach in reach_model_mapping: + # reach_func = reach_model_mapping[int(reach)] + # reach_html_tables = { + # reach: stylize_model_output(reach_func(df, rows=hours)) + # } + # else: + # reach_html_tables = { + # reach: stylize_model_output(reach_func(df, rows=hours)) + # for reach, reach_func + # in reach_model_mapping.items() + # } + + print('\n\nreach_html_tables\n\n') + print(reach_html_tables) + print('\n\n') return render_template('output_model.html', tables=reach_html_tables) @@ -174,4 +206,4 @@ def get(self): return self.model_api() -api.add_resource(ReachApi, '/api/v1/model') +api.add_resource(ReachApi, '/api/v1/model') \ No newline at end of file diff --git a/flagging_site/data/model.py b/flagging_site/data/model.py index adbb5ce0..e1a2d87e 100644 --- a/flagging_site/data/model.py +++ b/flagging_site/data/model.py @@ -145,7 +145,7 @@ def process_data( return df -def reach_2_model(df: pd.DataFrame, rows: int = 24) -> pd.DataFrame: +def reach_2_model(df: pd.DataFrame, rows: int = 48) -> pd.DataFrame: """Model params: a- rainfall sum 0-24 hrs d- Days since last rain @@ -172,7 +172,7 @@ def reach_2_model(df: pd.DataFrame, rows: int = 24) -> pd.DataFrame: return df[['reach', 'time', 'log_odds', 'probability', 'safe']] -def reach_3_model(df: pd.DataFrame, rows: int = 24) -> pd.DataFrame: +def reach_3_model(df: pd.DataFrame, rows: int = 48) -> pd.DataFrame: """ a- rainfall sum 0-24 hrs b- rainfall sum 24-48 hr @@ -201,7 +201,7 @@ def reach_3_model(df: pd.DataFrame, rows: int = 24) -> pd.DataFrame: return df[['reach', 'time', 'log_odds', 'probability', 'safe']] -def reach_4_model(df: pd.DataFrame, rows: int = 24) -> pd.DataFrame: +def reach_4_model(df: pd.DataFrame, rows: int = 48) -> pd.DataFrame: """ a- rainfall sum 0-24 hrs b- rainfall sum 24-48 hr @@ -230,7 +230,7 @@ def reach_4_model(df: pd.DataFrame, rows: int = 24) -> pd.DataFrame: return df[['reach', 'time', 'log_odds', 'probability', 'safe']] -def reach_5_model(df: pd.DataFrame, rows: int = 24) -> pd.DataFrame: +def reach_5_model(df: pd.DataFrame, rows: int = 48) -> pd.DataFrame: """ c- rainfall sum 0-48 hr d- Days since last rain @@ -269,9 +269,15 @@ def all_models(df: pd.DataFrame, *args, **kwargs): def latest_model_outputs(hours: int = 1) -> dict: - if hours != 1: - raise NotImplementedError('Need to work on this!') from .database import execute_sql_from_file - df = execute_sql_from_file('latest_model.sql') - df = df.set_index('reach') - return df.to_dict(orient='index') + + if hours == 1: + df = execute_sql_from_file('return_1_hour_of_model_outputs.sql') + elif hours > 1: + df = execute_sql_from_file('return_48_hours_of_model_outputs.sql') + # pull out all data older than parameter hours + + else: + raise ValueError('hours of data to pull cannot be less than one') + + return df \ No newline at end of file diff --git a/flagging_site/data/queries/latest_model.sql b/flagging_site/data/queries/return_1_hour_of_model_outputs.sql similarity index 100% rename from flagging_site/data/queries/latest_model.sql rename to flagging_site/data/queries/return_1_hour_of_model_outputs.sql diff --git a/flagging_site/data/queries/return_48_hours_of_model_outputs.sql b/flagging_site/data/queries/return_48_hours_of_model_outputs.sql new file mode 100644 index 00000000..5dc8c348 --- /dev/null +++ b/flagging_site/data/queries/return_48_hours_of_model_outputs.sql @@ -0,0 +1,10 @@ +-- This query returns 48 hours of data, +-- (unless the time interval in the data is less than 48 hours, +-- in which case it will return data for all timestamps) + +SELECT * +FROM model_outputs +WHERE time BETWEEN + (SELECT MAX(time) - interval '47 hours' FROM model_outputs) + AND + (SELECT MAX(time) FROM model_outputs) \ No newline at end of file From 6edc594c4f1f7fe8540277c7c91798262bb778ae Mon Sep 17 00:00:00 2001 From: Lewis Staples Date: Tue, 4 Aug 2020 18:35:38 -0400 Subject: [PATCH 021/118] Issue#49 complete: latest_model_outputs returns 1 thru 48, depending on input parameter; also cleaned up code and documentation --- flagging_site/blueprints/flagging.py | 47 +++++----------------------- flagging_site/data/model.py | 8 +++-- 2 files changed, 13 insertions(+), 42 deletions(-) diff --git a/flagging_site/blueprints/flagging.py b/flagging_site/blueprints/flagging.py index d98eea91..ece7f5f0 100644 --- a/flagging_site/blueprints/flagging.py +++ b/flagging_site/blueprints/flagging.py @@ -38,11 +38,6 @@ def stylize_model_output(df: pd.DataFrame) -> str: Returns: HTML table. """ - - # print('\n\ndf conv to html\n\n') - # print(df) - # print('\n\n') - def _apply_flag(x: bool) -> str: flag_class = 'blue-flag' if x else 'red-flag' return f'{x}' @@ -128,44 +123,18 @@ def output_model() -> str: df = latest_model_outputs(hours) - - print('\n\ndf_list\n\n') - print( df.reach.unique() ) - print('\n\n') - # go through each reach in df and create a subset of df of that reach only - # then add that to reach_html_tables - + # reach_html_tables is a dict where the index is the reach number + # and the values are HTML code for the table of data to display for + # that particular reach reach_html_tables = {} + + # loop through each reach in df + # extract the subset from the df for that reach + # then convert that df subset to HTML code + # and then add that HTML subset to reach_html_tables for i in df.reach.unique(): - print( i ) reach_html_tables[i] = stylize_model_output( df.loc[df['reach'] == i ] ) - - # df = get_data() - - # reach_model_mapping = { - # 2: reach_2_model, - # 3: reach_3_model, - # 4: reach_4_model, - # 5: reach_5_model - # } - - # if reach in reach_model_mapping: - # reach_func = reach_model_mapping[int(reach)] - # reach_html_tables = { - # reach: stylize_model_output(reach_func(df, rows=hours)) - # } - # else: - # reach_html_tables = { - # reach: stylize_model_output(reach_func(df, rows=hours)) - # for reach, reach_func - # in reach_model_mapping.items() - # } - - print('\n\nreach_html_tables\n\n') - print(reach_html_tables) - print('\n\n') - return render_template('output_model.html', tables=reach_html_tables) diff --git a/flagging_site/data/model.py b/flagging_site/data/model.py index e1a2d87e..a1542389 100644 --- a/flagging_site/data/model.py +++ b/flagging_site/data/model.py @@ -274,9 +274,11 @@ def latest_model_outputs(hours: int = 1) -> dict: if hours == 1: df = execute_sql_from_file('return_1_hour_of_model_outputs.sql') elif hours > 1: - df = execute_sql_from_file('return_48_hours_of_model_outputs.sql') - # pull out all data older than parameter hours - + df = execute_sql_from_file('return_48_hours_of_model_outputs.sql') # pull out 48 hours of model outputs + latest_time = max(df['time']) # find most recent timestamp + time_interval = pd.Timedelta(str(hours) + ' hours') # create pandas Timedelta, based on input parameter hours + df = df[ latest_time - df['time']< time_interval ] # reset df to exclude anything from before time_interval ago + else: raise ValueError('hours of data to pull cannot be less than one') From 99f70a6d193e7546857ce54e7e4bdb5163a87d27 Mon Sep 17 00:00:00 2001 From: Lewis Staples Date: Tue, 4 Aug 2020 21:40:00 -0400 Subject: [PATCH 022/118] Fix to get full 48 hours of numeric data populated --- flagging_site/data/database.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/flagging_site/data/database.py b/flagging_site/data/database.py index fec31e51..4151ff47 100644 --- a/flagging_site/data/database.py +++ b/flagging_site/data/database.py @@ -61,7 +61,7 @@ def update_database(): # Populate the `hobolink` table. from .hobolink import get_live_hobolink_data - df_hobolink = get_live_hobolink_data() + df_hobolink = get_live_hobolink_data('code_for_boston_export_21d') df_hobolink.to_sql('hobolink', **options) from .model import process_data From b1d5e64ffe86e87189ba34f66c0e836677058966 Mon Sep 17 00:00:00 2001 From: Lewis Staples Date: Tue, 4 Aug 2020 21:48:57 -0400 Subject: [PATCH 023/118] cleaned up code (print statements) --- flagging_site/blueprints/flagging.py | 5 ----- 1 file changed, 5 deletions(-) diff --git a/flagging_site/blueprints/flagging.py b/flagging_site/blueprints/flagging.py index ece7f5f0..95c97cbd 100644 --- a/flagging_site/blueprints/flagging.py +++ b/flagging_site/blueprints/flagging.py @@ -75,11 +75,6 @@ def index() -> str: """ df = latest_model_outputs() - - print('\n\nlatest_model_outputs:\n\n') - print( latest_model_outputs(48) ) - print('\n\n') - df = df.set_index('reach') flags = { key: val['safe'] From dd5ee0608754a8f5829d044719096389be7f2968 Mon Sep 17 00:00:00 2001 From: Lewis Staples Date: Tue, 4 Aug 2020 21:55:02 -0400 Subject: [PATCH 024/118] cleaned up comments in sql query --- .../data/queries/return_48_hours_of_model_outputs.sql | 4 +--- 1 file changed, 1 insertion(+), 3 deletions(-) diff --git a/flagging_site/data/queries/return_48_hours_of_model_outputs.sql b/flagging_site/data/queries/return_48_hours_of_model_outputs.sql index 5dc8c348..9e1cd8fd 100644 --- a/flagging_site/data/queries/return_48_hours_of_model_outputs.sql +++ b/flagging_site/data/queries/return_48_hours_of_model_outputs.sql @@ -1,6 +1,4 @@ --- This query returns 48 hours of data, --- (unless the time interval in the data is less than 48 hours, --- in which case it will return data for all timestamps) +-- This query returns up to 48 hours of the latest data SELECT * FROM model_outputs From c367d94162f5e1891126f1572bdafeb4b24d91d8 Mon Sep 17 00:00:00 2001 From: Lewis Staples Date: Sun, 9 Aug 2020 21:18:15 -0400 Subject: [PATCH 025/118] param. of single-reach display restored; Issue#50 implemented for API --- flagging_site/blueprints/flagging.py | 88 +++++++++++++++++++++------- 1 file changed, 67 insertions(+), 21 deletions(-) diff --git a/flagging_site/blueprints/flagging.py b/flagging_site/blueprints/flagging.py index 95c97cbd..d34a3737 100644 --- a/flagging_site/blueprints/flagging.py +++ b/flagging_site/blueprints/flagging.py @@ -124,11 +124,13 @@ def output_model() -> str: reach_html_tables = {} # loop through each reach in df - # extract the subset from the df for that reach - # then convert that df subset to HTML code - # and then add that HTML subset to reach_html_tables + # compare with reach to determine whether to display that reach + # extract the subset from the df for that reach + # then convert that df subset to HTML code + # and then add that HTML subset to reach_html_tables for i in df.reach.unique(): - reach_html_tables[i] = stylize_model_output( df.loc[df['reach'] == i ] ) + if (reach==-1 or reach==i): + reach_html_tables[i] = stylize_model_output( df.loc[df['reach'] == i ] ) return render_template('output_model.html', tables=reach_html_tables) @@ -136,35 +138,79 @@ def output_model() -> str: class ReachApi(Resource): def model_api(self) -> dict: + """ + This class method retrieves data from the database, + and then returns a json-like dictionary, prim_dict + + (note that there are three levels of dictionaries: + prim_dict, sec_dict, and tri_dict) + + prim_dict has three items: + key version with value version number, + key time_returned with value timestamp, + key models, and with value of a dictionary, sec_dict + sec_dict has four items, + the key for each of one of the reach models (model_2 ... model_5), and + the value for each item is another dictionary, tri_dict + tri_dict has five items, + the keys are: reach, time, log_odds, probability, and safe + the value for each is a list of values + """ + + # get model output data from database + df = latest_model_outputs(48) + + # converts time column to type string because of conversion to json error + df['time'] = df['time'].astype(str) + + # construct secondary dictionary from the file (tertiary dicts will be built along the way) + sec_dict = {} + for reach_num in df.reach.unique(): + tri_dict = {} + for field in df.columns: + tri_dict[field] = df[df['reach']==reach_num][field].tolist() + sec_dict["model_"+str(reach_num)] = tri_dict + + # create return value (primary dictionary) + prim_dict = { + "version" : "2020", + "time_returned":str( pd.to_datetime('today') ), + "models": sec_dict + } + + return prim_dict + + # to be replaced ...... """ Class method that retrieves data from hobolink and usgs and processes data, then creates json-like dictionary structure for model output. returns: json-like dictionary """ - df = get_data() - dfs = { - 2: reach_2_model(df), - 3: reach_3_model(df), - 4: reach_4_model(df), - 5: reach_5_model(df) - } + # df = get_data() + + # dfs = { + # 2: reach_2_model(df), + # 3: reach_3_model(df), + # 4: reach_4_model(df), + # 5: reach_5_model(df) + # } - main = {} - models = {} + # main = {} + # models = {} - # adds metadata - main['version'] = '2020' - main['time_returned'] = str(pd.to_datetime('today')) + # # adds metadata + # main['version'] = '2020' + # main['time_returned'] = str(pd.to_datetime('today')) - for reach, df in dfs.items(): - add_to_dict(models, df, reach) + # for reach, df in dfs.items(): + # add_to_dict(models, df, reach) - # adds models dict to main dict - main['models'] = models + # # adds models dict to main dict + # main['models'] = models - return main + # return main def get(self): return self.model_api() From 2e8d5c2430858de0576d2ba5e8b24cd5a4f162f1 Mon Sep 17 00:00:00 2001 From: Lewis Staples Date: Sun, 9 Aug 2020 21:47:23 -0400 Subject: [PATCH 026/118] Cleaned up comments, removed commented-out code --- flagging_site/blueprints/flagging.py | 48 ---------------------------- 1 file changed, 48 deletions(-) diff --git a/flagging_site/blueprints/flagging.py b/flagging_site/blueprints/flagging.py index d34a3737..50133cc3 100644 --- a/flagging_site/blueprints/flagging.py +++ b/flagging_site/blueprints/flagging.py @@ -48,23 +48,6 @@ def _apply_flag(x: bool) -> str: return df.to_html(index=False, escape=False) -def add_to_dict(models, df, reach) -> None: - """ - Iterates through dataframe from model output, adds to model dict where - key equals column name, value equals column values as list type - - args: - models: dictionary - df: pd.DataFrame - reach:int - - returns: None - """ - # converts time column to type string because of conversion to json error - df['time'] = df['time'].astype(str) - models[f'model_{reach}'] = df.to_dict(orient='list') - - @bp.route('/') def index() -> str: """ @@ -180,37 +163,6 @@ def model_api(self) -> dict: return prim_dict - # to be replaced ...... - """ - Class method that retrieves data from hobolink and usgs and processes - data, then creates json-like dictionary structure for model output. - - returns: json-like dictionary - """ - - # df = get_data() - - # dfs = { - # 2: reach_2_model(df), - # 3: reach_3_model(df), - # 4: reach_4_model(df), - # 5: reach_5_model(df) - # } - - # main = {} - # models = {} - - # # adds metadata - # main['version'] = '2020' - # main['time_returned'] = str(pd.to_datetime('today')) - - # for reach, df in dfs.items(): - # add_to_dict(models, df, reach) - - # # adds models dict to main dict - # main['models'] = models - - # return main def get(self): return self.model_api() From d4dfdbb92b4b8c15efb45e013596a65f6de3be1f Mon Sep 17 00:00:00 2001 From: Bertie Ancona Date: Tue, 11 Aug 2020 19:47:23 -0400 Subject: [PATCH 027/118] Make functions available in Flask shell (#61) --- docs/docs/shell.md | 62 ++++++++++++++++++++++++++++++++++ flagging_site/app.py | 15 ++++++++ flagging_site/data/hobolink.py | 4 +-- flagging_site/data/keys.py | 7 ---- flagging_site/data/usgs.py | 4 +-- 5 files changed, 81 insertions(+), 11 deletions(-) create mode 100644 docs/docs/shell.md diff --git a/docs/docs/shell.md b/docs/docs/shell.md new file mode 100644 index 00000000..0cd807e1 --- /dev/null +++ b/docs/docs/shell.md @@ -0,0 +1,62 @@ +# Flask Shell Documentation + +The shell is used to access app functions and data, such as Hobolink and USGS +data and access to the database. + +## Available Shell Functions and Variables + +- `db` (*flask_sqlalchemy.SQLAlchemy*): + The object used to interact with the Postgres database. +- `get_live_hobolink_data` (*(Optional[str]) -> pd.DataFrame*): + Gets the Hobolink data table based on the given "export" name. + See `flagging_site/data/hobolink.py` for details. +- `get_live_usgs_data` (*() -> pd.DataFrame*): + Gets the USGS data table. + See `flagging_site/data/usgs.py` for details. +- `get_data` (*() -> pd.DataFrame*): + Gets the Hobolink and USGS data tables and returns a combined table. +- `process_data` (*(pd.DataFrame, pd.DataFrame) -> pd.DataFrame*): + Combines the Hobolink and USGS tables. + See `flagging_site/data/model.py` for details. + +To add more functions and variables, simply add an entry to the dictionary +returned by the function `make_shell_context()` in `flagging_site/app.py:creat_app()`. + +## Running the Shell + +First, open up a terminal at the `flagging` folder. + +Make sure you have Python 3 installed. Set up your environment with the following commands: + +```shell +python3 -m venv venv +source venv/bin/activate +python3 -m pip install -r requirements.txt +``` + +Export the following environment variables like so: + +```shell +export VAULT_PASSWORD=replace_me_with_pw +export FLASK_APP=flagging_site:create_app +export FLASK_ENV=development +``` + +Finally, start the Flask shell: + +```shell +flask shell +``` + +And you should be good to go! The functions listed above should be available for use. See below for an example. + +## Example: Export Hobolink Data to CSV + +Here we assume you have already started the Flask shell. +This example shows how to download the Hobolink data and +save it as a CSV file. + +```shell +>>> hobolink_data = get_live_hobolink_data() +>>> hobolink_data.to_csv('path/where/to/save/my-CSV-file.csv') +``` \ No newline at end of file diff --git a/flagging_site/app.py b/flagging_site/app.py index 01b83a81..8a773903 100644 --- a/flagging_site/app.py +++ b/flagging_site/app.py @@ -6,7 +6,11 @@ from typing import Optional from flask import Flask +from .blueprints.flagging import get_data +from .data.hobolink import get_live_hobolink_data from .data.keys import get_keys +from .data.model import process_data +from .data.usgs import get_live_usgs_data from .config import Config from .config import get_config_from_env @@ -60,6 +64,17 @@ def update_db_command(): update_database() click.echo('Updated the database.') + # Make a few useful functions available in Flask shell without imports + @app.shell_context_processor + def make_shell_context(): + return { + 'db': db, + 'get_data': get_data, + 'get_live_hobolink_data': get_live_hobolink_data, + 'get_live_usgs_data': get_live_usgs_data, + 'process_data': process_data, + } + # And we're all set! We can hand the app over to flask at this point. return app diff --git a/flagging_site/data/hobolink.py b/flagging_site/data/hobolink.py index 39589248..e5c1b3d4 100644 --- a/flagging_site/data/hobolink.py +++ b/flagging_site/data/hobolink.py @@ -11,8 +11,8 @@ import requests import pandas as pd from flask import abort +from flask import current_app from .keys import get_keys -from .keys import offline_mode from .keys import get_data_store_file_path # Constants @@ -49,7 +49,7 @@ def get_live_hobolink_data(export_name: str = EXPORT_NAME) -> pd.DataFrame: Returns: Pandas Dataframe containing the cleaned-up Hobolink data. """ - if offline_mode(): + if current_app.config['OFFLINE_MODE']: df = pd.read_pickle(get_data_store_file_path(STATIC_FILE_NAME)) else: res = request_to_hobolink(export_name=export_name) diff --git a/flagging_site/data/keys.py b/flagging_site/data/keys.py index d4492524..8db3653a 100644 --- a/flagging_site/data/keys.py +++ b/flagging_site/data/keys.py @@ -69,13 +69,6 @@ def load_keys_from_vault( return d -def offline_mode() -> bool: - if current_app: - return current_app.config['OFFLINE_MODE'] - else: - return bool(strtobool(os.getenv('OFFLINE_MODE', 'false'))) - - def get_data_store_file_path(file_name: str) -> str: if current_app: return os.path.join(current_app.config['DATA_STORE'], file_name) diff --git a/flagging_site/data/usgs.py b/flagging_site/data/usgs.py index 936d2bf5..ab41c70c 100644 --- a/flagging_site/data/usgs.py +++ b/flagging_site/data/usgs.py @@ -8,7 +8,7 @@ import pandas as pd import requests from flask import abort -from .keys import offline_mode +from flask import current_app from .keys import get_data_store_file_path # Constants @@ -25,7 +25,7 @@ def get_live_usgs_data() -> pd.DataFrame: Returns: Pandas Dataframe containing the usgs data. """ - if offline_mode(): + if current_app.config['OFFLINE_MODE']: df = pd.read_pickle(get_data_store_file_path(STATIC_FILE_NAME)) else: res = request_to_usgs() From 05546a7c9fe8bdf38345467116e5979088eab145 Mon Sep 17 00:00:00 2001 From: Lewis Staples Date: Fri, 14 Aug 2020 14:35:49 -0400 Subject: [PATCH 028/118] verifying db isn't there, creating db --- flagging_site/app.py | 10 +++++++++ flagging_site/config.py | 4 ++-- flagging_site/data/database.py | 41 +++++++++++++++++++++++++++++++++- run_windows_dev.bat | 1 + 4 files changed, 53 insertions(+), 3 deletions(-) diff --git a/flagging_site/app.py b/flagging_site/app.py index 01b83a81..1fb7f31e 100644 --- a/flagging_site/app.py +++ b/flagging_site/app.py @@ -47,6 +47,16 @@ def create_app(config: Optional[Config] = None) -> Flask: from .data import db db.init_app(app) + @app.cli.command('create-db') + def create_db_command(): + """Create database (after verifying that it isn't already there)""" + from .data.database import create_db + if create_db(): + click.echo('The database was created.') + else: + click.echo('The database was already there.') + + @app.cli.command('init-db') def init_db_command(): """Clear existing data and create new tables.""" diff --git a/flagging_site/config.py b/flagging_site/config.py index 0c6439c3..b38623c8 100644 --- a/flagging_site/config.py +++ b/flagging_site/config.py @@ -47,7 +47,7 @@ def __repr__(self): POSTGRES_PASSWORD: str = os.getenv('POSTGRES_PASSWORD') POSTGRES_HOST: str = 'localhost' POSTGRES_PORT: int = 5432 - POSTGRES_DBNAME: str = 'flagging' + POSTGRES_DBNAME: str = 'the_test_db2' @property def SQLALCHEMY_DATABASE_URI(self) -> str: @@ -58,7 +58,7 @@ def SQLALCHEMY_DATABASE_URI(self) -> str: db = self.POSTGRES_DBNAME return f'postgres://{user}:{password}@{host}:{port}/{db}' - SQLALCHEMY_ECHO: bool = True + SQLALCHEMY_ECHO: bool = False SQLALCHEMY_RECORD_QUERIES: bool = True SQLALCHEMY_TRACK_MODIFICATIONS: bool = False diff --git a/flagging_site/data/database.py b/flagging_site/data/database.py index 4151ff47..1a847bc6 100644 --- a/flagging_site/data/database.py +++ b/flagging_site/data/database.py @@ -8,7 +8,7 @@ from flask import current_app from flask_sqlalchemy import SQLAlchemy from sqlalchemy.exc import ResourceClosedError - +from psycopg2 import connect db = SQLAlchemy() @@ -33,6 +33,45 @@ def execute_sql_from_file(file_name: str): return execute_sql(f.read().decode('utf8')) +def create_db(): + """If database doesn't exist, create it and a and return True, + otherwise leave it alone and return False""" + + # create local variables for database values in the config file: + db_user = current_app.config['POSTGRES_USER'] + db_pswd = current_app.config['POSTGRES_PASSWORD'] + db_host = current_app.config['POSTGRES_HOST'] + db_port = current_app.config['POSTGRES_PORT'] + db_name = current_app.config['POSTGRES_DBNAME'] + + # connect to postgres database, get cursor + conn = connect(dbname='postgres', user=db_user, host=db_host, password=db_pswd) + cursor = conn.cursor() + + # get a list of all databases: + cursor.execute('SELECT datname FROM pg_database;') + db_list = cursor.fetchall() # db_list here is a list of one-element tuples + db_list = [d[0] for d in db_list] # this converts db_list to a list of db names + + # if that database is already there, exit out of this function + if db_name in db_list: + return False + # since the database isn't already there, proceed ... + + # create the database + cursor.execute('commit;') + cursor.execute('create database ' + db_name) + cursor.execute('commit;') + + # ask the user for the password to use for the flagging user + # (the user name will be the same as the database name, + # which is db_name, which is set in the config as POSTGRES_DBNAME) + + # create flagging user + + return True + + def init_db(): """Clear existing data and create new tables.""" with current_app.app_context(): diff --git a/run_windows_dev.bat b/run_windows_dev.bat index 361e6de8..179953d8 100644 --- a/run_windows_dev.bat +++ b/run_windows_dev.bat @@ -9,5 +9,6 @@ set /p OFFLINE_MODE="Offline mode? [y/n]: " set /p VAULT_PASSWORD="Enter vault password: " set /p POSTGRES_PASSWORD="Enter Postgres password: " +flask create-db flask init-db flask run \ No newline at end of file From 341a3782def2aeec98cf7026d32e10c6c574cffb Mon Sep 17 00:00:00 2001 From: Lewis Staples Date: Fri, 14 Aug 2020 22:30:47 -0400 Subject: [PATCH 029/118] creating flagging user --- flagging_site/config.py | 2 +- flagging_site/data/database.py | 17 ++++++++++------- 2 files changed, 11 insertions(+), 8 deletions(-) diff --git a/flagging_site/config.py b/flagging_site/config.py index b38623c8..340b862f 100644 --- a/flagging_site/config.py +++ b/flagging_site/config.py @@ -47,7 +47,7 @@ def __repr__(self): POSTGRES_PASSWORD: str = os.getenv('POSTGRES_PASSWORD') POSTGRES_HOST: str = 'localhost' POSTGRES_PORT: int = 5432 - POSTGRES_DBNAME: str = 'the_test_db2' + POSTGRES_DBNAME: str = 'flagging' @property def SQLALCHEMY_DATABASE_URI(self) -> str: diff --git a/flagging_site/data/database.py b/flagging_site/data/database.py index 1a847bc6..9afd0333 100644 --- a/flagging_site/data/database.py +++ b/flagging_site/data/database.py @@ -45,7 +45,7 @@ def create_db(): db_name = current_app.config['POSTGRES_DBNAME'] # connect to postgres database, get cursor - conn = connect(dbname='postgres', user=db_user, host=db_host, password=db_pswd) + conn = connect(dbname='postgres', user='postgres', host=db_host, password=db_pswd) cursor = conn.cursor() # get a list of all databases: @@ -59,16 +59,19 @@ def create_db(): # since the database isn't already there, proceed ... # create the database - cursor.execute('commit;') - cursor.execute('create database ' + db_name) - cursor.execute('commit;') - + cursor.execute('COMMIT;') + cursor.execute('CREATE DATABASE ' + db_name) + cursor.execute('COMMIT;') + # ask the user for the password to use for the flagging user # (the user name will be the same as the database name, # which is db_name, which is set in the config as POSTGRES_DBNAME) - - # create flagging user + new_db_user_pswd = 'super' + # create flagging user (same as db_name) + cursor.execute("DROP USER IF EXISTS " + db_name + ";" ) + cursor.execute("COMMIT;") + cursor.execute("CREATE USER " + db_name + " WITH SUPERUSER PASSWORD '" + new_db_user_pswd + "';") return True From ea67ea35951584446f1dfa0d682517cc2e621839 Mon Sep 17 00:00:00 2001 From: Lewis Staples Date: Sun, 16 Aug 2020 10:44:37 -0400 Subject: [PATCH 030/118] early attempt at modifying flagging password --- flagging_site/data/database.py | 23 ++++++++++++++++++++--- 1 file changed, 20 insertions(+), 3 deletions(-) diff --git a/flagging_site/data/database.py b/flagging_site/data/database.py index 9afd0333..3177cb65 100644 --- a/flagging_site/data/database.py +++ b/flagging_site/data/database.py @@ -66,12 +66,29 @@ def create_db(): # ask the user for the password to use for the flagging user # (the user name will be the same as the database name, # which is db_name, which is set in the config as POSTGRES_DBNAME) - new_db_user_pswd = 'super' - + + + print('\n\n') + print('original flagging password: ' + current_app.config['POSTGRES_PASSWORD'] ) + print('\n\n') + print('original sqlalchemy_db_uri: ' + current_app.config['SQLALCHEMY_DATABASE_URI'] ) + + # modify the password in the local variable and in the config file: + db_pswd = 'shush' + current_app.config['POSTGRES_PASSWORD'] = db_pswd + current_app.config['SQLALCHEMY_DATABASE_URI'] = f'postgres://{db_user}:{db_pswd}@{db_host}:{db_port}/{db_name}' + # change this, instead + + print('\n\n') + print('modified flagging password: ' + current_app.config['POSTGRES_PASSWORD'] ) + print('\n\n') + print('modified sqlalchemy_db_uri: ' + current_app.config['SQLALCHEMY_DATABASE_URI'] ) + print('\n\n') + # create flagging user (same as db_name) cursor.execute("DROP USER IF EXISTS " + db_name + ";" ) cursor.execute("COMMIT;") - cursor.execute("CREATE USER " + db_name + " WITH SUPERUSER PASSWORD '" + new_db_user_pswd + "';") + cursor.execute("CREATE USER " + db_name + " WITH SUPERUSER PASSWORD '" + db_pswd + "';") return True From 569b3e580a613998e51d2370d9f6e7713911eb9b Mon Sep 17 00:00:00 2001 From: Lewis Staples Date: Tue, 18 Aug 2020 21:13:28 -0400 Subject: [PATCH 031/118] fix to environment variable error --- run_unix_dev.sh | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/run_unix_dev.sh b/run_unix_dev.sh index c268d1d2..0b0c2ec8 100644 --- a/run_unix_dev.sh +++ b/run_unix_dev.sh @@ -30,7 +30,7 @@ export OFFLINE_MODE=${offline_mode} read -p "Enter vault password: " vault_pw export VAULT_PASSWORD=${vault_pw} read -p "Enter Postgres password: " postgres_pw -export VAULT_PASSWORD=${postgres_pw} +export POSTGRES_PASSWORD=${postgres_pw} flask init-db flask run \ No newline at end of file From 81299cf74d9c6cd21e1e53ba8eab4140eb96bcd5 Mon Sep 17 00:00:00 2001 From: Dan Eder Date: Thu, 20 Aug 2020 16:38:47 -0400 Subject: [PATCH 032/118] revised tests --- tests/test_api.py | 31 ++++++++++++++++++++++++++++++- 1 file changed, 30 insertions(+), 1 deletion(-) diff --git a/tests/test_api.py b/tests/test_api.py index b873aad5..43b055ba 100644 --- a/tests/test_api.py +++ b/tests/test_api.py @@ -1,5 +1,34 @@ -from flagging_site.blueprints.api import model_api from flask import current_app +from flagging_site.blueprints.api import model_api +from flagging_site.data.model import reach_2_model +from flagging_site.data.model import process_data +from flagging_site.data.usgs import get_live_usgs_data +from flagging_site.data.hobolink import get_live_hobolink_data + +def test_model_output_api_schema(app): + """test_model_output_api_schema() should test that the keys are a particular way and the + values are of a particular type.""" + with app.app_context(): + schema = { + str: list + } + res = model_api([3], 10) + for key, value in res.items(): + assert type(key) is list(schema.keys())[0] + assert isinstance(value, schema[key]) + + +def test_model_output_api_parameters(app): + """test_model_output_api_parameters() should test that hours=10 returns 10 rows of data, + that setting the reach returns only that reach.""" + with app.app_context(): + df = reach_2_model(process_data(df_usgs=get_live_usgs_data(), df_hobolink=get_live_hobolink_data('code_for_boston_export_21d')), 10) + row_count = len(df) + assert row_count == 10 + test_reach = model_api([3], 10) + test_reach = list(test_reach.values()) + expected_reach = list(test_reach[2].items())[0][0] + assert 'reach_3' == expected_reach def test_max_hours(app): From ac02838178f1bb399d08de701eaebab27ec66ddc Mon Sep 17 00:00:00 2001 From: Bertie Ancona Date: Tue, 18 Aug 2020 23:53:24 -0400 Subject: [PATCH 033/118] Add cyanobacteria override administration --- flagging_site/admin.py | 99 +++++++++++++++++++ flagging_site/app.py | 10 +- flagging_site/auth.py | 21 ++++ flagging_site/blueprints/flagging.py | 10 +- flagging_site/config.py | 2 + flagging_site/data/cyano_overrides.py | 22 +++++ flagging_site/data/database.py | 9 +- .../queries/currently_overridden_reaches.sql | 3 + flagging_site/data/queries/schema.sql | 7 ++ requirements.txt | 10 ++ 10 files changed, 188 insertions(+), 5 deletions(-) create mode 100644 flagging_site/admin.py create mode 100644 flagging_site/auth.py create mode 100644 flagging_site/data/cyano_overrides.py create mode 100644 flagging_site/data/queries/currently_overridden_reaches.sql diff --git a/flagging_site/admin.py b/flagging_site/admin.py new file mode 100644 index 00000000..791a0783 --- /dev/null +++ b/flagging_site/admin.py @@ -0,0 +1,99 @@ +import os + +from flask import Flask +from flask import redirect +from flask import request +from flask import Response +from flask_admin import Admin +from flask_admin import BaseView +from flask_admin import expose +from flask_admin.contrib import sqla +from werkzeug.exceptions import HTTPException + +from .auth import AuthException +from .auth import basic_auth +from .data import db +from .data.cyano_overrides import CyanoOverrides + + +admin = Admin(template_mode='bootstrap3') + +def init_admin(app: Flask): + with app.app_context(): + # Register /admin + admin.init_app(app) + + # Register /admin sub-views + admin.add_view(AdminModelView(CyanoOverrides, db.session)) + admin.add_view(LogoutView(name="Logout")) + + +# Adapted from https://computableverse.com/blog/flask-admin-using-basicauth +class AdminModelView(sqla.ModelView): + """ + Extension of SQLAlchemy ModelView that requires BasicAuth authentication, + and shows all columns in the form (including primary keys). + """ + + def __init__(self, model, *args, **kwargs): + # Show all columns in form + self.column_list = [c.key for c in model.__table__.columns] + self.form_columns = self.column_list + + super().__init__(model, *args, **kwargs) + + def is_accessible(self): + """ + Protect admin pages with basic_auth. + If logged out and current page is /admin/, then ask for credentials. + Otherwise, raises HTTP 401 error and redirects user to /admin/ on the + frontend (redirecting with HTTP redirect causes user to always be + redirected to /admin/ even after logging in). + + We redirect to /admin/ because our logout method only works if the path to + /logout is the same as the path to where we put in our credentials. So if + we put in credentials at /admin/cyanooverride, then we would need to logout + at /admin/cyanooverride/logout, which would be difficult to arrange. Instead, + we always redirect to /admin/ to put in credentials, and then logout at + /admin/logout. + """ + if not basic_auth.authenticate(): + if '/admin/' == request.path: + raise AuthException('Not authenticated. Refresh the page.') + else: + raise HTTPException( + 'Attempted to visit admin page but not authenticated.', + Response( + ''' + Not authenticated. Navigate to /admin/ to login. + + ''', + status=401 # 'Forbidden' status + ) + ) + else: + return True + + def inaccessible_callback(self, name, **kwargs): + """Ask for credentials when access fails""" + return redirect(basic_auth.challenge()) + + +class LogoutView(BaseView): + @expose('/') + def index(self): + """ + To log out of basic auth for admin pages, + we raise an HTTP 401 error (there isn't really a cleaner way) + and then redirect on the frontend to home. + """ + raise HTTPException( + 'Logged out.', + Response( + ''' + Successfully logged out. + + ''', + status=401 + ) + ) \ No newline at end of file diff --git a/flagging_site/app.py b/flagging_site/app.py index 01b83a81..bf2953c7 100644 --- a/flagging_site/app.py +++ b/flagging_site/app.py @@ -47,6 +47,14 @@ def create_app(config: Optional[Config] = None) -> Flask: from .data import db db.init_app(app) + # Register auth + from .auth import init_auth + init_auth(app) + + # Register admin + from .admin import init_admin + init_admin(app) + @app.cli.command('init-db') def init_db_command(): """Clear existing data and create new tables.""" @@ -87,7 +95,7 @@ def update_config_from_vault(app: Flask) -> None: else: print(f'Warning: {msg}') app.config['KEYS'] = None - app.config['SECRET_KEY'] = None + app.config['SECRET_KEY'] = os.urandom(16) else: app.config['SECRET_KEY'] = app.config['KEYS']['flask']['secret_key'] diff --git a/flagging_site/auth.py b/flagging_site/auth.py new file mode 100644 index 00000000..6880d0e3 --- /dev/null +++ b/flagging_site/auth.py @@ -0,0 +1,21 @@ +from flask import Flask +from flask import Response +from flask_basicauth import BasicAuth +from werkzeug.exceptions import HTTPException + + +basic_auth = BasicAuth() + +def init_auth(app: Flask): + with app.app_context(): + basic_auth.init_app(app) + +# Taken from https://computableverse.com/blog/flask-admin-using-basicauth +class AuthException(HTTPException): + def __init__(self, message): + """HTTP Forbidden error that prompts for login""" + super().__init__(message, Response( + 'You could not be authenticated. Please refresh the page.', + status=401, + headers={'WWW-Authenticate': 'Basic realm="Login Required"'} + )) \ No newline at end of file diff --git a/flagging_site/blueprints/flagging.py b/flagging_site/blueprints/flagging.py index 95c97cbd..8ccf649c 100644 --- a/flagging_site/blueprints/flagging.py +++ b/flagging_site/blueprints/flagging.py @@ -2,8 +2,11 @@ from flask import Blueprint from flask import render_template from flask import request -from flask_restful import Resource, Api +from flask import Response +from flask_restful import Api +from flask_restful import Resource +from ..data.cyano_overrides import get_currently_overridden_reaches from ..data.hobolink import get_live_hobolink_data from ..data.usgs import get_live_usgs_data from ..data.model import process_data @@ -76,8 +79,11 @@ def index() -> str: df = latest_model_outputs() df = df.set_index('reach') + + overridden_reaches = get_currently_overridden_reaches() + flags = { - key: val['safe'] + key: val['safe'] and key not in overridden_reaches for key, val in df.to_dict(orient='index').items() } diff --git a/flagging_site/config.py b/flagging_site/config.py index 0c6439c3..e4c109f7 100644 --- a/flagging_site/config.py +++ b/flagging_site/config.py @@ -140,6 +140,8 @@ class DevelopmentConfig(Config): DEBUG: bool = True TESTING: bool = True OFFLINE_MODE = strtobool(os.getenv('OFFLINE_MODE', 'false')) + BASIC_AUTH_USERNAME: str = 'admin' + BASIC_AUTH_PASSWORD: str = 'password' class TestingConfig(Config): diff --git a/flagging_site/data/cyano_overrides.py b/flagging_site/data/cyano_overrides.py new file mode 100644 index 00000000..5e1e6fb2 --- /dev/null +++ b/flagging_site/data/cyano_overrides.py @@ -0,0 +1,22 @@ +from typing import Set + +from sqlalchemy import Column +from sqlalchemy import Integer +from sqlalchemy import TIMESTAMP +from sqlalchemy.ext.declarative import declarative_base + +from .database import Base +from .database import execute_sql_from_file + +class CyanoOverrides(Base): + __tablename__ = 'cyano_overrides' + reach = Column(Integer, primary_key=True) + start_time = Column(TIMESTAMP, primary_key=True) + end_time = Column(TIMESTAMP, primary_key=True) + +def get_currently_overridden_reaches() -> Set[int]: + return set( + execute_sql_from_file( + 'currently_overridden_reaches.sql' + )["reach"].unique() + ) diff --git a/flagging_site/data/database.py b/flagging_site/data/database.py index bab392b2..2b8ee20e 100644 --- a/flagging_site/data/database.py +++ b/flagging_site/data/database.py @@ -7,10 +7,12 @@ from typing import Optional from flask import current_app from flask_sqlalchemy import SQLAlchemy +from flask_sqlalchemy import declarative_base from sqlalchemy.exc import ResourceClosedError db = SQLAlchemy() +Base = declarative_base() def execute_sql(query: str) -> Optional[pd.DataFrame]: @@ -20,8 +22,10 @@ def execute_sql(query: str) -> Optional[pd.DataFrame]: with db.engine.connect() as conn: res = conn.execute(query) try: - df = pd.DataFrame(res.fetchall()) - df.columns = res.keys() + df = pd.DataFrame( + res.fetchall(), + columns=res.keys() + ) return df except ResourceClosedError: return None @@ -40,6 +44,7 @@ def init_db(): execute_sql_from_file('schema.sql') execute_sql_from_file('define_boathouse.sql') update_database() + Base.metadata.create_all(db.engine) def update_database(): diff --git a/flagging_site/data/queries/currently_overridden_reaches.sql b/flagging_site/data/queries/currently_overridden_reaches.sql new file mode 100644 index 00000000..d6a3ee79 --- /dev/null +++ b/flagging_site/data/queries/currently_overridden_reaches.sql @@ -0,0 +1,3 @@ +SELECT reach +FROM cyano_overrides +WHERE current_timestamp BETWEEN start_time AND end_time \ No newline at end of file diff --git a/flagging_site/data/queries/schema.sql b/flagging_site/data/queries/schema.sql index a1c17b4c..19eac1d1 100644 --- a/flagging_site/data/queries/schema.sql +++ b/flagging_site/data/queries/schema.sql @@ -37,4 +37,11 @@ CREATE TABLE IF NOT EXISTS model_outputs ( safe boolean ); +DROP TABLE IF EXISTS cyano_overrides; +CREATE TABLE IF NOT EXISTS cyano_overrides ( + reach int, + start_time timestamp, + end_time timestamp +); + COMMIT; \ No newline at end of file diff --git a/requirements.txt b/requirements.txt index 2dae0f25..a05aad28 100644 --- a/requirements.txt +++ b/requirements.txt @@ -1,9 +1,16 @@ +aniso8601==8.0.0 appdirs==1.4.4 +certifi==2020.6.20 +chardet==3.0.4 click==7.1.2 distlib==0.3.1 filelock==3.0.12 Flask==1.1.2 +Flask-Admin==1.5.6 +Flask-BasicAuth==0.2.0 +Flask-RESTful==0.3.8 Flask-SQLAlchemy==2.4.4 +idna==2.10 itsdangerous==1.1.0 Jinja2==2.11.2 MarkupSafe==1.1.1 @@ -13,7 +20,10 @@ psycopg2==2.8.5 psycopg2-binary==2.8.5 python-dateutil==2.8.1 pytz==2020.1 +requests==2.24.0 six==1.15.0 SQLAlchemy==1.3.18 +urllib3==1.25.10 virtualenv==20.0.27 Werkzeug==1.0.1 +WTForms==2.3.3 From 35b2fdf51864cb836aabf5910bd84a704d479a78 Mon Sep 17 00:00:00 2001 From: Bertie Ancona Date: Sat, 22 Aug 2020 03:35:07 -0400 Subject: [PATCH 034/118] Make / code clearer --- flagging_site/blueprints/flagging.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/flagging_site/blueprints/flagging.py b/flagging_site/blueprints/flagging.py index 8ccf649c..c44f858d 100644 --- a/flagging_site/blueprints/flagging.py +++ b/flagging_site/blueprints/flagging.py @@ -83,8 +83,8 @@ def index() -> str: overridden_reaches = get_currently_overridden_reaches() flags = { - key: val['safe'] and key not in overridden_reaches - for key, val + reach: val['safe'] and reach not in overridden_reaches + for reach, val in df.to_dict(orient='index').items() } return render_template('index.html', flags=flags) From 013a1bcb3449746bf23be186c6d1375f0480665c Mon Sep 17 00:00:00 2001 From: Bertie Ancona Date: Sat, 22 Aug 2020 03:41:43 -0400 Subject: [PATCH 035/118] Add documentation --- docs/docs/admin.md | 13 +++++++++++++ flagging_site/blueprints/__init__.py | 1 - flagging_site/blueprints/cyanobacteria.py | 3 --- 3 files changed, 13 insertions(+), 4 deletions(-) create mode 100644 docs/docs/admin.md delete mode 100644 flagging_site/blueprints/cyanobacteria.py diff --git a/docs/docs/admin.md b/docs/docs/admin.md new file mode 100644 index 00000000..42dc1a54 --- /dev/null +++ b/docs/docs/admin.md @@ -0,0 +1,13 @@ +# Admin Pages + +You can reach all admin pages by going to /admin and inputting a username and password. +In development, the username is 'admin' and the password is 'password', but these should +be set as config variables `BASIC_AUTH_USERNAME` and `BASIC_AUTH_PASSWORD`. Remember to log out +by clicking the `Logout` button in the nav bar. + +## Cyanobacteria Overrides + +There should be a link to this page in the admin navigation bar. +On this page, one can add an override for a reach with a start time and end time, +and if the current time is between those times then the reach will be marked as +unsafe on the main website, regardless of the model data. \ No newline at end of file diff --git a/flagging_site/blueprints/__init__.py b/flagging_site/blueprints/__init__.py index 4a0093a0..20f2ab82 100644 --- a/flagging_site/blueprints/__init__.py +++ b/flagging_site/blueprints/__init__.py @@ -1,2 +1 @@ -from . import cyanobacteria from . import flagging diff --git a/flagging_site/blueprints/cyanobacteria.py b/flagging_site/blueprints/cyanobacteria.py deleted file mode 100644 index ff383efb..00000000 --- a/flagging_site/blueprints/cyanobacteria.py +++ /dev/null @@ -1,3 +0,0 @@ -from flask import Blueprint - -bp = Blueprint('cyanobacteria', __name__, url_prefix='/cyanobacteria') From 8da2d8ecb104d4c9ab55f27c05ba1e959e4a1d0a Mon Sep 17 00:00:00 2001 From: Bertie Ancona Date: Sat, 22 Aug 2020 17:03:37 -0400 Subject: [PATCH 036/118] Add reason field --- flagging_site/admin.py | 5 +++-- flagging_site/data/cyano_overrides.py | 17 +++++++++++++++++ flagging_site/data/queries/schema.sql | 3 ++- 3 files changed, 22 insertions(+), 3 deletions(-) diff --git a/flagging_site/admin.py b/flagging_site/admin.py index 791a0783..1d4c4c5a 100644 --- a/flagging_site/admin.py +++ b/flagging_site/admin.py @@ -13,7 +13,6 @@ from .auth import AuthException from .auth import basic_auth from .data import db -from .data.cyano_overrides import CyanoOverrides admin = Admin(template_mode='bootstrap3') @@ -24,7 +23,9 @@ def init_admin(app: Flask): admin.init_app(app) # Register /admin sub-views - admin.add_view(AdminModelView(CyanoOverrides, db.session)) + from .data.cyano_overrides import CyanoOverridesModelView + admin.add_view(CyanoOverridesModelView(db.session)) + admin.add_view(LogoutView(name="Logout")) diff --git a/flagging_site/data/cyano_overrides.py b/flagging_site/data/cyano_overrides.py index 5e1e6fb2..0853c54f 100644 --- a/flagging_site/data/cyano_overrides.py +++ b/flagging_site/data/cyano_overrides.py @@ -3,8 +3,10 @@ from sqlalchemy import Column from sqlalchemy import Integer from sqlalchemy import TIMESTAMP +from sqlalchemy import VARCHAR from sqlalchemy.ext.declarative import declarative_base +from ..admin import AdminModelView from .database import Base from .database import execute_sql_from_file @@ -13,6 +15,21 @@ class CyanoOverrides(Base): reach = Column(Integer, primary_key=True) start_time = Column(TIMESTAMP, primary_key=True) end_time = Column(TIMESTAMP, primary_key=True) + reason = Column(VARCHAR(255)) + + +class CyanoOverridesModelView(AdminModelView): + form_choices = { + 'reason': [ + ('cyanobacteria', 'Cyanobacteria'), + ('sewage', 'Sewage'), + ('other', 'Other'), + ] + } + + def __init__(self, session): + super().__init__(CyanoOverrides, session) + def get_currently_overridden_reaches() -> Set[int]: return set( diff --git a/flagging_site/data/queries/schema.sql b/flagging_site/data/queries/schema.sql index 19eac1d1..3b2f95c5 100644 --- a/flagging_site/data/queries/schema.sql +++ b/flagging_site/data/queries/schema.sql @@ -41,7 +41,8 @@ DROP TABLE IF EXISTS cyano_overrides; CREATE TABLE IF NOT EXISTS cyano_overrides ( reach int, start_time timestamp, - end_time timestamp + end_time timestamp, + reason varchar(255) ); COMMIT; \ No newline at end of file From 8d21276ec8fc5f9829e9fdc0ba401c90472f57cf Mon Sep 17 00:00:00 2001 From: Bertie Ancona Date: Sat, 22 Aug 2020 17:16:53 -0400 Subject: [PATCH 037/118] unused import --- flagging_site/data/cyano_overrides.py | 1 - 1 file changed, 1 deletion(-) diff --git a/flagging_site/data/cyano_overrides.py b/flagging_site/data/cyano_overrides.py index 0853c54f..00a6a97c 100644 --- a/flagging_site/data/cyano_overrides.py +++ b/flagging_site/data/cyano_overrides.py @@ -4,7 +4,6 @@ from sqlalchemy import Integer from sqlalchemy import TIMESTAMP from sqlalchemy import VARCHAR -from sqlalchemy.ext.declarative import declarative_base from ..admin import AdminModelView from .database import Base From 5fbb1200a23deb25c9bfc76d13315908126e0eda Mon Sep 17 00:00:00 2001 From: Dan Eder Date: Sat, 22 Aug 2020 23:17:31 -0400 Subject: [PATCH 038/118] initial commit --- scheduler.py | 0 tests/test_api.py | 29 ----------------------------- 2 files changed, 29 deletions(-) create mode 100644 scheduler.py diff --git a/scheduler.py b/scheduler.py new file mode 100644 index 00000000..e69de29b diff --git a/tests/test_api.py b/tests/test_api.py index 43b055ba..6dd0b486 100644 --- a/tests/test_api.py +++ b/tests/test_api.py @@ -1,34 +1,5 @@ from flask import current_app from flagging_site.blueprints.api import model_api -from flagging_site.data.model import reach_2_model -from flagging_site.data.model import process_data -from flagging_site.data.usgs import get_live_usgs_data -from flagging_site.data.hobolink import get_live_hobolink_data - -def test_model_output_api_schema(app): - """test_model_output_api_schema() should test that the keys are a particular way and the - values are of a particular type.""" - with app.app_context(): - schema = { - str: list - } - res = model_api([3], 10) - for key, value in res.items(): - assert type(key) is list(schema.keys())[0] - assert isinstance(value, schema[key]) - - -def test_model_output_api_parameters(app): - """test_model_output_api_parameters() should test that hours=10 returns 10 rows of data, - that setting the reach returns only that reach.""" - with app.app_context(): - df = reach_2_model(process_data(df_usgs=get_live_usgs_data(), df_hobolink=get_live_hobolink_data('code_for_boston_export_21d')), 10) - row_count = len(df) - assert row_count == 10 - test_reach = model_api([3], 10) - test_reach = list(test_reach.values()) - expected_reach = list(test_reach[2].items())[0][0] - assert 'reach_3' == expected_reach def test_max_hours(app): From eabb0dd889727634b3df638e0159dc4808bfaecc Mon Sep 17 00:00:00 2001 From: Dan Eder Date: Sat, 22 Aug 2020 23:52:21 -0400 Subject: [PATCH 039/118] Time Scheduler Directions Added --- docs/docs/deployment.md | 25 +++++++++++++++++++++++++ 1 file changed, 25 insertions(+) diff --git a/docs/docs/deployment.md b/docs/docs/deployment.md index bfe57dcc..f8d1ad51 100644 --- a/docs/docs/deployment.md +++ b/docs/docs/deployment.md @@ -91,3 +91,28 @@ heroku logs --app crwa-flagging-staging --tail ``` 4. Check out the website and make sure it looks right. + +## Scheduling periodic database updates + +Refreshing the database on a schedule ensures the most accurate data is available to users. + +Fortunately, the app is equipped with a CLI that triggers a database refresh. +```flask update-db``` + +In addition, Heroku has an add-on that lets us schedule db refreshes to avoid doing the work manually. + +```Install Time Scheduler Add-On``` +1. Go to dashboard.com/heroku/apps/[app name], click on "Configure Add-ons" + +2. Choose Heroku Time Scheduler from the list of Add-ons. This should now appear in your list of Add-ons. + +```Add Job``` +1. On dashboard.heroku.com/apps/[app name]/scheduler, click on Add Job in the Time Scheduler section. + +2. Choose to update db every 10 minutes, hour, or day. + +3. Enter command: python flask update-db and click Save Job. + +4. Confirm it runs then you should be good to go! + +```For more info: https://devcenter.heroku.com/articles/scheduler``` \ No newline at end of file From d71d7a64ea59311f136879cc3d3493ca38f80913 Mon Sep 17 00:00:00 2001 From: Dan Eder Date: Sun, 23 Aug 2020 00:10:08 -0400 Subject: [PATCH 040/118] revised md --- docs/docs/deployment.md | 11 ++++------- 1 file changed, 4 insertions(+), 7 deletions(-) diff --git a/docs/docs/deployment.md b/docs/docs/deployment.md index f8d1ad51..7aa73a20 100644 --- a/docs/docs/deployment.md +++ b/docs/docs/deployment.md @@ -96,22 +96,19 @@ heroku logs --app crwa-flagging-staging --tail Refreshing the database on a schedule ensures the most accurate data is available to users. -Fortunately, the app is equipped with a CLI that triggers a database refresh. -```flask update-db``` +Fortunately, the app is equipped with a CLI that triggers a database refresh: ```flask update-db``` In addition, Heroku has an add-on that lets us schedule db refreshes to avoid doing the work manually. -```Install Time Scheduler Add-On``` 1. Go to dashboard.com/heroku/apps/[app name], click on "Configure Add-ons" 2. Choose Heroku Time Scheduler from the list of Add-ons. This should now appear in your list of Add-ons. -```Add Job``` -1. On dashboard.heroku.com/apps/[app name]/scheduler, click on Add Job in the Time Scheduler section. +3. On dashboard.heroku.com/apps/[app name]/scheduler, click on Add Job in the Time Scheduler section. -2. Choose to update db every 10 minutes, hour, or day. +4. Choose to update db every 10 minutes, hour, or day. -3. Enter command: python flask update-db and click Save Job. +5. Enter command: python flask update-db and click Save Job. 4. Confirm it runs then you should be good to go! From 020c58551136ad481c89960fa0c878a45bc77a3b Mon Sep 17 00:00:00 2001 From: Dan Eder Date: Sun, 23 Aug 2020 00:14:05 -0400 Subject: [PATCH 041/118] revised md --- docs/docs/deployment.md | 6 ++---- 1 file changed, 2 insertions(+), 4 deletions(-) diff --git a/docs/docs/deployment.md b/docs/docs/deployment.md index 7aa73a20..4c1b6a12 100644 --- a/docs/docs/deployment.md +++ b/docs/docs/deployment.md @@ -106,10 +106,8 @@ In addition, Heroku has an add-on that lets us schedule db refreshes to avoid do 3. On dashboard.heroku.com/apps/[app name]/scheduler, click on Add Job in the Time Scheduler section. -4. Choose to update db every 10 minutes, hour, or day. +4. Choose to update db every 10 minutes, hour, or day. -5. Enter command: python flask update-db and click Save Job. - -4. Confirm it runs then you should be good to go! +5. Enter command: ```python flask update-db``` and Save Job. ```For more info: https://devcenter.heroku.com/articles/scheduler``` \ No newline at end of file From 0a333e5ffdc34df625377e6fa4631b6a07c9b48f Mon Sep 17 00:00:00 2001 From: Lewis Staples Date: Mon, 24 Aug 2020 16:08:01 -0400 Subject: [PATCH 042/118] removing flagging user --- flagging_site/config.py | 4 ++-- flagging_site/data/database.py | 43 +++++----------------------------- 2 files changed, 8 insertions(+), 39 deletions(-) diff --git a/flagging_site/config.py b/flagging_site/config.py index 340b862f..00663645 100644 --- a/flagging_site/config.py +++ b/flagging_site/config.py @@ -43,7 +43,7 @@ def __repr__(self): # ========================================================================== # DATABASE CONFIG OPTIONS # ========================================================================== - POSTGRES_USER: str = os.getenv('POSTGRES_USER', 'flagging') + POSTGRES_USER: str = os.getenv('POSTGRES_USER', 'postgres') POSTGRES_PASSWORD: str = os.getenv('POSTGRES_PASSWORD') POSTGRES_HOST: str = 'localhost' POSTGRES_PORT: int = 5432 @@ -58,7 +58,7 @@ def SQLALCHEMY_DATABASE_URI(self) -> str: db = self.POSTGRES_DBNAME return f'postgres://{user}:{password}@{host}:{port}/{db}' - SQLALCHEMY_ECHO: bool = False + SQLALCHEMY_ECHO: bool = True SQLALCHEMY_RECORD_QUERIES: bool = True SQLALCHEMY_TRACK_MODIFICATIONS: bool = False diff --git a/flagging_site/data/database.py b/flagging_site/data/database.py index 3177cb65..0921a663 100644 --- a/flagging_site/data/database.py +++ b/flagging_site/data/database.py @@ -34,18 +34,13 @@ def execute_sql_from_file(file_name: str): def create_db(): - """If database doesn't exist, create it and a and return True, + """If database doesn't exist, create it and return True, otherwise leave it alone and return False""" - # create local variables for database values in the config file: - db_user = current_app.config['POSTGRES_USER'] - db_pswd = current_app.config['POSTGRES_PASSWORD'] - db_host = current_app.config['POSTGRES_HOST'] - db_port = current_app.config['POSTGRES_PORT'] - db_name = current_app.config['POSTGRES_DBNAME'] - # connect to postgres database, get cursor - conn = connect(dbname='postgres', user='postgres', host=db_host, password=db_pswd) + conn = connect(dbname='postgres', user=current_app.config['POSTGRES_USER'], + host=current_app.config['POSTGRES_HOST'], + password=current_app.config['POSTGRES_PASSWORD']) cursor = conn.cursor() # get a list of all databases: @@ -54,41 +49,15 @@ def create_db(): db_list = [d[0] for d in db_list] # this converts db_list to a list of db names # if that database is already there, exit out of this function - if db_name in db_list: + if current_app.config['POSTGRES_DBNAME'] in db_list: return False # since the database isn't already there, proceed ... # create the database cursor.execute('COMMIT;') - cursor.execute('CREATE DATABASE ' + db_name) + cursor.execute('CREATE DATABASE ' + current_app.config['POSTGRES_DBNAME']) cursor.execute('COMMIT;') - # ask the user for the password to use for the flagging user - # (the user name will be the same as the database name, - # which is db_name, which is set in the config as POSTGRES_DBNAME) - - - print('\n\n') - print('original flagging password: ' + current_app.config['POSTGRES_PASSWORD'] ) - print('\n\n') - print('original sqlalchemy_db_uri: ' + current_app.config['SQLALCHEMY_DATABASE_URI'] ) - - # modify the password in the local variable and in the config file: - db_pswd = 'shush' - current_app.config['POSTGRES_PASSWORD'] = db_pswd - current_app.config['SQLALCHEMY_DATABASE_URI'] = f'postgres://{db_user}:{db_pswd}@{db_host}:{db_port}/{db_name}' - # change this, instead - - print('\n\n') - print('modified flagging password: ' + current_app.config['POSTGRES_PASSWORD'] ) - print('\n\n') - print('modified sqlalchemy_db_uri: ' + current_app.config['SQLALCHEMY_DATABASE_URI'] ) - print('\n\n') - - # create flagging user (same as db_name) - cursor.execute("DROP USER IF EXISTS " + db_name + ";" ) - cursor.execute("COMMIT;") - cursor.execute("CREATE USER " + db_name + " WITH SUPERUSER PASSWORD '" + db_pswd + "';") return True From 050d09c4567b11568586a0fea412dc642e640f33 Mon Sep 17 00:00:00 2001 From: Lewis Staples Date: Mon, 24 Aug 2020 16:43:17 -0400 Subject: [PATCH 043/118] removed reach as a superfluous displayed column --- flagging_site/blueprints/flagging.py | 3 +++ 1 file changed, 3 insertions(+) diff --git a/flagging_site/blueprints/flagging.py b/flagging_site/blueprints/flagging.py index 50133cc3..7b5f7548 100644 --- a/flagging_site/blueprints/flagging.py +++ b/flagging_site/blueprints/flagging.py @@ -45,6 +45,9 @@ def _apply_flag(x: bool) -> str: df['safe'] = df['safe'].apply(_apply_flag) df.columns = [i.title().replace('_', ' ') for i in df.columns] + # remove reach number + df = df.drop('Reach', 1) + return df.to_html(index=False, escape=False) From d70d303824cebdc0cf99cc168950578078fefbe5 Mon Sep 17 00:00:00 2001 From: Bertie Ancona Date: Tue, 25 Aug 2020 18:05:54 -0400 Subject: [PATCH 044/118] Add newlines at end of files, and add basic auth config --- docs/docs/admin.md | 2 +- flagging_site/admin.py | 2 +- flagging_site/auth.py | 2 +- flagging_site/blueprints/flagging.py | 2 +- flagging_site/config.py | 12 ++++++++++-- flagging_site/data/database.py | 2 +- .../data/queries/currently_overridden_reaches.sql | 2 +- flagging_site/data/queries/schema.sql | 2 +- 8 files changed, 17 insertions(+), 9 deletions(-) diff --git a/docs/docs/admin.md b/docs/docs/admin.md index 42dc1a54..5bae89b8 100644 --- a/docs/docs/admin.md +++ b/docs/docs/admin.md @@ -10,4 +10,4 @@ by clicking the `Logout` button in the nav bar. There should be a link to this page in the admin navigation bar. On this page, one can add an override for a reach with a start time and end time, and if the current time is between those times then the reach will be marked as -unsafe on the main website, regardless of the model data. \ No newline at end of file +unsafe on the main website, regardless of the model data. diff --git a/flagging_site/admin.py b/flagging_site/admin.py index 1d4c4c5a..fdee8e90 100644 --- a/flagging_site/admin.py +++ b/flagging_site/admin.py @@ -97,4 +97,4 @@ def index(self): ''', status=401 ) - ) \ No newline at end of file + ) diff --git a/flagging_site/auth.py b/flagging_site/auth.py index 6880d0e3..2e4c4b17 100644 --- a/flagging_site/auth.py +++ b/flagging_site/auth.py @@ -18,4 +18,4 @@ def __init__(self, message): 'You could not be authenticated. Please refresh the page.', status=401, headers={'WWW-Authenticate': 'Basic realm="Login Required"'} - )) \ No newline at end of file + )) diff --git a/flagging_site/blueprints/flagging.py b/flagging_site/blueprints/flagging.py index c44f858d..914a86a1 100644 --- a/flagging_site/blueprints/flagging.py +++ b/flagging_site/blueprints/flagging.py @@ -176,4 +176,4 @@ def get(self): return self.model_api() -api.add_resource(ReachApi, '/api/v1/model') \ No newline at end of file +api.add_resource(ReachApi, '/api/v1/model') diff --git a/flagging_site/config.py b/flagging_site/config.py index e4c109f7..4a996e8f 100644 --- a/flagging_site/config.py +++ b/flagging_site/config.py @@ -64,6 +64,12 @@ def SQLALCHEMY_DATABASE_URI(self) -> str: QUERIES_DIR: str = QUERIES_DIR + # ========================================================================== + # BASIC AUTH CONFIG OPTIONS + # ========================================================================== + BASIC_AUTH_USERNAME: str = os.environ['BASIC_AUTH_USERNAME'] + BASIC_AUTH_PASSWORD: str = os.environ['BASIC_AUTH_PASSWORD'] + # ========================================================================== # MISC. CUSTOM CONFIG OPTIONS # @@ -140,8 +146,8 @@ class DevelopmentConfig(Config): DEBUG: bool = True TESTING: bool = True OFFLINE_MODE = strtobool(os.getenv('OFFLINE_MODE', 'false')) - BASIC_AUTH_USERNAME: str = 'admin' - BASIC_AUTH_PASSWORD: str = 'password' + BASIC_AUTH_USERNAME: str = os.getenv('BASIC_AUTH_USERNAME', 'admin') + BASIC_AUTH_PASSWORD: str = os.getenv('BASIC_AUTH_PASSWORD', 'password') class TestingConfig(Config): @@ -149,6 +155,8 @@ class TestingConfig(Config): website. """ TESTING: bool = True + BASIC_AUTH_USERNAME: str = os.getenv('BASIC_AUTH_USERNAME', 'admin') + BASIC_AUTH_PASSWORD: str = os.getenv('BASIC_AUTH_PASSWORD', 'password') def get_config_from_env(env: str) -> Config: diff --git a/flagging_site/data/database.py b/flagging_site/data/database.py index 2b8ee20e..41e7dff1 100644 --- a/flagging_site/data/database.py +++ b/flagging_site/data/database.py @@ -73,4 +73,4 @@ def update_database(): from .model import all_models model_outs = all_models(df) - model_outs.to_sql('model_outputs', **options) \ No newline at end of file + model_outs.to_sql('model_outputs', **options) diff --git a/flagging_site/data/queries/currently_overridden_reaches.sql b/flagging_site/data/queries/currently_overridden_reaches.sql index d6a3ee79..f8a660a4 100644 --- a/flagging_site/data/queries/currently_overridden_reaches.sql +++ b/flagging_site/data/queries/currently_overridden_reaches.sql @@ -1,3 +1,3 @@ SELECT reach FROM cyano_overrides -WHERE current_timestamp BETWEEN start_time AND end_time \ No newline at end of file +WHERE current_timestamp BETWEEN start_time AND end_time diff --git a/flagging_site/data/queries/schema.sql b/flagging_site/data/queries/schema.sql index 3b2f95c5..704457d0 100644 --- a/flagging_site/data/queries/schema.sql +++ b/flagging_site/data/queries/schema.sql @@ -45,4 +45,4 @@ CREATE TABLE IF NOT EXISTS cyano_overrides ( reason varchar(255) ); -COMMIT; \ No newline at end of file +COMMIT; From f9224ee2a6a870f7d66dca280d95a5f1660802bc Mon Sep 17 00:00:00 2001 From: Bertie Ancona Date: Tue, 25 Aug 2020 18:12:44 -0400 Subject: [PATCH 045/118] Fix config --- flagging_site/config.py | 10 ++++------ 1 file changed, 4 insertions(+), 6 deletions(-) diff --git a/flagging_site/config.py b/flagging_site/config.py index 4a996e8f..16a6c6b7 100644 --- a/flagging_site/config.py +++ b/flagging_site/config.py @@ -64,12 +64,6 @@ def SQLALCHEMY_DATABASE_URI(self) -> str: QUERIES_DIR: str = QUERIES_DIR - # ========================================================================== - # BASIC AUTH CONFIG OPTIONS - # ========================================================================== - BASIC_AUTH_USERNAME: str = os.environ['BASIC_AUTH_USERNAME'] - BASIC_AUTH_PASSWORD: str = os.environ['BASIC_AUTH_PASSWORD'] - # ========================================================================== # MISC. CUSTOM CONFIG OPTIONS # @@ -128,6 +122,10 @@ class ProductionConfig(Config): """ BLUEPRINTS: Optional[List[str]] = ['flagging'] + def __init__(self): + self.BASIC_AUTH_USERNAME: str = os.environ['BASIC_AUTH_USERNAME'] + self.BASIC_AUTH_PASSWORD: str = os.environ['BASIC_AUTH_PASSWORD'] + class DevelopmentConfig(Config): """The Development Config is used for running the website on your own From 56738d5de391b3dc7fabfb914b50861bf0c3be5d Mon Sep 17 00:00:00 2001 From: Edwin Sun <34005715+edsun123@users.noreply.github.com> Date: Tue, 25 Aug 2020 19:04:25 -0400 Subject: [PATCH 046/118] revised --- docs/docs/export.md | 8 ++++++++ 1 file changed, 8 insertions(+) create mode 100644 docs/docs/export.md diff --git a/docs/docs/export.md b/docs/docs/export.md new file mode 100644 index 00000000..07f1e263 --- /dev/null +++ b/docs/docs/export.md @@ -0,0 +1,8 @@ +# Export Data + +## HTML iFrame + +The outputs of the model can be exported as an iFrame, which allows the website's live data to be viewed on a statically rendered web page (such as those hosted by Weebly). To export the data using an iFrame, use the following HTML: + +
+ From 2c5631e693423c4c5b48bd81bf50045a71981ea2 Mon Sep 17 00:00:00 2001 From: Lewis Staples Date: Sun, 30 Aug 2020 23:32:45 -0400 Subject: [PATCH 047/118] Fixing to allow merge to run --- flagging_site/blueprints/flagging.py | 10 +++++----- 1 file changed, 5 insertions(+), 5 deletions(-) diff --git a/flagging_site/blueprints/flagging.py b/flagging_site/blueprints/flagging.py index 88ae082f..9df3891d 100644 --- a/flagging_site/blueprints/flagging.py +++ b/flagging_site/blueprints/flagging.py @@ -12,6 +12,7 @@ from ..data.model import reach_3_model from ..data.model import reach_4_model from ..data.model import reach_5_model +from ..data.model import latest_model_outputs bp = Blueprint('flagging', __name__) @@ -62,6 +63,7 @@ def index() -> str: key: val['safe'] for key, val in df.to_dict(orient='index').items() + } homepage = { 2: { @@ -70,7 +72,7 @@ def index() -> str: 'Newton Yacht Club', 'Watertown Yacht Club', 'Community Rowing, Inc.', - 'Northeastern\s Henderson Boathouse', + 'Northeastern\'s Henderson Boathouse', 'Paddle Boston at Herter Park' ] }, @@ -97,11 +99,9 @@ def index() -> str: } } - model_last_updated_time = reach_5_model(df, rows=1)['time'].iloc[0] + model_last_updated_time = df['time'].iloc[0] return render_template('index.html', homepage=homepage, model_last_updated_time=model_last_updated_time) - # return render_template('index.html', flags=flags) - @bp.route('/about') @@ -197,4 +197,4 @@ def get(self): return self.model_api() -api.add_resource(ReachApi, '/api/v1/model') +# api.add_resource(ReachApi, '/api/v1/model') From 17d5e2f3762eb8f48c92c413f98f7cd958782bc9 Mon Sep 17 00:00:00 2001 From: Lewis Staples Date: Tue, 1 Sep 2020 00:31:07 -0400 Subject: [PATCH 048/118] merge continued: brought in api logic --- flagging_site/blueprints/api.py | 79 +++++++++++++-------------- flagging_site/blueprints/flagging.py | 82 ++++++++++++++-------------- flagging_site/config.py | 2 +- 3 files changed, 80 insertions(+), 83 deletions(-) diff --git a/flagging_site/blueprints/api.py b/flagging_site/blueprints/api.py index 1367279a..0b70cf9d 100644 --- a/flagging_site/blueprints/api.py +++ b/flagging_site/blueprints/api.py @@ -9,13 +9,14 @@ from flask_restful import Resource from flask import current_app -from ..data.hobolink import get_live_hobolink_data -from ..data.usgs import get_live_usgs_data -from ..data.model import process_data -from ..data.model import reach_2_model -from ..data.model import reach_3_model -from ..data.model import reach_4_model -from ..data.model import reach_5_model +# from ..data.hobolink import get_live_hobolink_data +# from ..data.usgs import get_live_usgs_data +# from ..data.model import process_data +# from ..data.model import reach_2_model +# from ..data.model import reach_3_model +# from ..data.model import reach_4_model +# from ..data.model import reach_5_model +from ..data.model import latest_model_outputs from flasgger import swag_from @@ -28,14 +29,6 @@ def index() -> str: return render_template('api/index.html') -def get_data() -> pd.DataFrame: - """Retrieves the data that gets plugged into the the model.""" - df_hobolink = get_live_hobolink_data('code_for_boston_export_21d') - df_usgs = get_live_usgs_data() - df = process_data(df_hobolink, df_usgs) - return df - - def add_to_dict(models, df, reach) -> None: """ Iterates through dataframe from model output, adds to model dict where @@ -60,43 +53,47 @@ def model_api(reaches: Optional[List[int]], hours: Optional[int]) -> dict: returns: json-like dictionary """ - # Set defaults - if reaches is None: - reaches = [2, 3, 4, 5] + + # set default `hours` must be between 1 and `API_MAX_HOURS` if hours is None: hours = 24 - + # `hours` must be between 1 and `API_MAX_HOURS` if hours > current_app.config['API_MAX_HOURS']: hours = current_app.config['API_MAX_HOURS'] elif hours < 1: hours = 1 - df = get_data() - - dfs = { - 2: reach_2_model, - 3: reach_3_model, - 4: reach_4_model, - 5: reach_5_model - } + # get model output data from database + df = latest_model_outputs(hours) - main = {} - models = {} - - # adds metadata - main['version'] = '2020' - main['time_returned'] = str(pd.to_datetime('today')) - - for reach, model_func in dfs.items(): - if reach in reaches: - _df = model_func(df, hours) - add_to_dict(models, _df, reach) + # converts time column to type string because of conversion to json error + df['time'] = df['time'].astype(str) - # adds models dict to main dict - main['models'] = models + # set default `reaches`: all reach values in the data + if not reaches: + reaches = df.reach.unique() + + print('\n\nreaches') + print(reaches) + print('\n\n') + + # construct secondary dictionary from the file (tertiary dicts will be built along the way) + sec_dict = {} + for reach_num in reaches: + tri_dict = {} + for field in df.columns: + tri_dict[field] = df[df['reach']==reach_num][field].tolist() + sec_dict["model_"+str(reach_num)] = tri_dict + + # create return value (primary dictionary) + prim_dict = { + "version" : "2020", + "time_returned":str( pd.to_datetime('today') ), + "models": sec_dict + } - return main + return prim_dict class ReachesApi(Resource): diff --git a/flagging_site/blueprints/flagging.py b/flagging_site/blueprints/flagging.py index 9df3891d..d31b6ca2 100644 --- a/flagging_site/blueprints/flagging.py +++ b/flagging_site/blueprints/flagging.py @@ -2,16 +2,16 @@ from flask import Blueprint from flask import render_template from flask import request -from flask_restful import Resource, Api +# from flask_restful import Resource, Api from flask import current_app from ..data.hobolink import get_live_hobolink_data from ..data.usgs import get_live_usgs_data from ..data.model import process_data -from ..data.model import reach_2_model -from ..data.model import reach_3_model -from ..data.model import reach_4_model -from ..data.model import reach_5_model +# from ..data.model import reach_2_model +# from ..data.model import reach_3_model +# from ..data.model import reach_4_model +# from ..data.model import reach_5_model from ..data.model import latest_model_outputs bp = Blueprint('flagging', __name__) @@ -147,54 +147,54 @@ def output_model() -> str: return render_template('output_model.html', tables=reach_html_tables) -class ReachApi(Resource): +# class ReachApi(Resource): - def model_api(self) -> dict: - """ - This class method retrieves data from the database, - and then returns a json-like dictionary, prim_dict +# def model_api(self) -> dict: +# """ +# This class method retrieves data from the database, +# and then returns a json-like dictionary, prim_dict - (note that there are three levels of dictionaries: - prim_dict, sec_dict, and tri_dict) +# (note that there are three levels of dictionaries: +# prim_dict, sec_dict, and tri_dict) - prim_dict has three items: - key version with value version number, - key time_returned with value timestamp, - key models, and with value of a dictionary, sec_dict - sec_dict has four items, - the key for each of one of the reach models (model_2 ... model_5), and - the value for each item is another dictionary, tri_dict - tri_dict has five items, - the keys are: reach, time, log_odds, probability, and safe - the value for each is a list of values - """ +# prim_dict has three items: +# key version with value version number, +# key time_returned with value timestamp, +# key models, and with value of a dictionary, sec_dict +# sec_dict has four items, +# the key for each of one of the reach models (model_2 ... model_5), and +# the value for each item is another dictionary, tri_dict +# tri_dict has five items, +# the keys are: reach, time, log_odds, probability, and safe +# the value for each is a list of values +# """ # get model output data from database - df = latest_model_outputs(48) + # df = latest_model_outputs(48) # converts time column to type string because of conversion to json error - df['time'] = df['time'].astype(str) + # df['time'] = df['time'].astype(str) # construct secondary dictionary from the file (tertiary dicts will be built along the way) - sec_dict = {} - for reach_num in df.reach.unique(): - tri_dict = {} - for field in df.columns: - tri_dict[field] = df[df['reach']==reach_num][field].tolist() - sec_dict["model_"+str(reach_num)] = tri_dict - - # create return value (primary dictionary) - prim_dict = { - "version" : "2020", - "time_returned":str( pd.to_datetime('today') ), - "models": sec_dict - } + # sec_dict = {} + # for reach_num in df.reach.unique(): + # tri_dict = {} + # for field in df.columns: + # tri_dict[field] = df[df['reach']==reach_num][field].tolist() + # sec_dict["model_"+str(reach_num)] = tri_dict + + # # create return value (primary dictionary) + # prim_dict = { + # "version" : "2020", + # "time_returned":str( pd.to_datetime('today') ), + # "models": sec_dict + # } - return prim_dict + # return prim_dict - def get(self): - return self.model_api() + # def get(self): + # return self.model_api() # api.add_resource(ReachApi, '/api/v1/model') diff --git a/flagging_site/config.py b/flagging_site/config.py index 5f2fa47b..ea27ae41 100644 --- a/flagging_site/config.py +++ b/flagging_site/config.py @@ -68,7 +68,7 @@ def SQLALCHEMY_DATABASE_URI(self) -> str: db = self.POSTGRES_DBNAME return f'postgres://{user}:{password}@{host}:{port}/{db}' - SQLALCHEMY_ECHO: bool = True + SQLALCHEMY_ECHO: bool = False SQLALCHEMY_RECORD_QUERIES: bool = True SQLALCHEMY_TRACK_MODIFICATIONS: bool = False From 34d49d14ca18267e774807062b16d1deb7dca762 Mon Sep 17 00:00:00 2001 From: dwreeves Date: Tue, 1 Sep 2020 20:55:02 -0400 Subject: [PATCH 049/118] Fixed requirements.txt being binary and added filter over psycopg2 in bash script --- requirements.txt | 67 ++++++++++++++++++++++++++++++++++++++++++++--- run_unix_dev.sh | 2 +- tests/conftest.py | 2 +- 3 files changed, 66 insertions(+), 5 deletions(-) diff --git a/requirements.txt b/requirements.txt index a05aad28..5022b20c 100644 --- a/requirements.txt +++ b/requirements.txt @@ -1,29 +1,90 @@ +alembic==1.4.2 aniso8601==8.0.0 appdirs==1.4.4 +astroid==2.4.1 +atomicwrites==1.4.0 +attrs==19.3.0 +Babel==2.8.0 +bcrypt==3.2.0 +blinker==1.4 certifi==2020.6.20 +cffi==1.14.2 chardet==3.0.4 click==7.1.2 +colorama==0.4.3 +coverage==5.2 +cryptography==3.0 distlib==0.3.1 +dnspython==2.0.0 +email-validator==1.1.1 filelock==3.0.12 +flasgger==0.9.4 Flask==1.1.2 Flask-Admin==1.5.6 +Flask-BabelEx==0.9.4 Flask-BasicAuth==0.2.0 +Flask-Compress==1.4.0 +Flask-Gravatar==0.5.0 +Flask-Login==0.4.1 +Flask-Mail==0.9.1 +Flask-Migrate==2.4.0 +Flask-Paranoid==0.2.0 +Flask-Principal==0.4.0 Flask-RESTful==0.3.8 +Flask-Security-Too==3.4.4 Flask-SQLAlchemy==2.4.4 +Flask-WTF==0.14.3 +gunicorn==20.0.4 idna==2.10 +importlib-metadata==1.6.1 +isort==4.3.21 itsdangerous==1.1.0 Jinja2==2.11.2 +jsonschema==3.2.0 +lazy-object-proxy==1.4.3 +ldap3==2.8 +Mako==1.1.3 MarkupSafe==1.1.1 +mccabe==0.6.1 +mistune==0.8.4 +more-itertools==8.3.0 numpy==1.19.1 +packaging==20.4 pandas==1.0.5 +paramiko==2.7.1 +passlib==1.7.2 +pgadmin4==4.24 +pluggy==0.13.1 +psutil==5.7.2 psycopg2==2.8.5 psycopg2-binary==2.8.5 +py==1.8.1 +pyasn1==0.4.8 +pycparser==2.20 +pylint==2.5.2 +PyNaCl==1.4.0 +pyparsing==2.4.7 +pyrsistent==0.16.0 +pytest==5.4.3 +pytest-cov==2.10.0 python-dateutil==2.8.1 -pytz==2020.1 -requests==2.24.0 +python-dotenv==0.14.0 +python-editor==1.0.4 +pytz==2018.9 +PyYAML==5.3.1 +requests==2.23.0 +simplejson==3.16.0 six==1.15.0 +speaklater==1.3 SQLAlchemy==1.3.18 -urllib3==1.25.10 +sqlparse==0.2.4 +sshtunnel==0.1.5 +toml==0.10.1 +typed-ast==1.4.1 +urllib3==1.25.9 virtualenv==20.0.27 +wcwidth==0.2.4 Werkzeug==1.0.1 +wrapt==1.12.1 WTForms==2.3.3 +zipp==3.1.0 diff --git a/run_unix_dev.sh b/run_unix_dev.sh index 0b0c2ec8..26b963a9 100644 --- a/run_unix_dev.sh +++ b/run_unix_dev.sh @@ -20,7 +20,7 @@ PYEXEC=$(get_python_exec) # Set up virtual environment in /venv $PYEXEC -m venv venv source venv/bin/activate -$PYEXEC -m pip install -r requirements.txt +$PYEXEC -m pip install $(cat requirements.txt | grep -v "psycopg2==") # Set up and run the Flask application export FLASK_APP=flagging_site:create_app diff --git a/tests/conftest.py b/tests/conftest.py index 3d69a910..f1391267 100644 --- a/tests/conftest.py +++ b/tests/conftest.py @@ -5,7 +5,7 @@ from flagging_site.config import TestingConfig -@pytest.fixture +@pytest.fixture(scope='session') def app(): """Create and configure a new app instance for each test.""" if 'VAULT_PASSWORD' not in os.environ: From b261f9745827ee1a9a2795e91056675cd48db1ed Mon Sep 17 00:00:00 2001 From: dwreeves Date: Wed, 2 Sep 2020 00:47:48 -0400 Subject: [PATCH 050/118] Updates to docs --- docs/README.md | 6 +++ docs/docs/about.md | 32 ++++++++++++ docs/docs/admin.md | 2 + docs/docs/background.md | 16 +----- docs/docs/deployment.md | 83 +++++++++++++++++++++++++------- docs/docs/development/history.md | 35 ++++++++++++++ docs/docs/index.md | 9 ++-- docs/mkdocs.yml | 41 +++++++++++++--- 8 files changed, 180 insertions(+), 44 deletions(-) create mode 100644 docs/docs/about.md create mode 100644 docs/docs/development/history.md diff --git a/docs/README.md b/docs/README.md index d56b933d..04d68053 100644 --- a/docs/README.md +++ b/docs/README.md @@ -7,6 +7,9 @@ The full docs are available at: https://codeforboston.github.io/flagging/ If you have write permission to the upstream repository (i.e. you are a project manager), point your terminal to this directory and run the following: ```shell script +pip install mkdocs +pip install pymdown-extensions +pip install mkdocs-material mkdocs gh-deploy --remote-branch upstream ``` @@ -18,6 +21,9 @@ If you do not have write permission to the upstream repository, you can do one o If you are a project manager but you're having issues, you can do a more manual git approach to updating the docs: ```shell script +pip install mkdocs +pip install pymdown-extensions +pip install mkdocs-material mkdocs gh-deploy git checkout gh-pages git push upstream gh-pages diff --git a/docs/docs/about.md b/docs/docs/about.md new file mode 100644 index 00000000..ff45d769 --- /dev/null +++ b/docs/docs/about.md @@ -0,0 +1,32 @@ +# About + +## Flagging Website + +Of the many services that the CRWA provides to the greater Boston community, one of those is monitoring whether it is safe to swim and/or boat in the Charles River. The CRWA Flagging Program uses a system of color-coded flags to indicate whether or not the river's water quality is safe for boating at eleven boating locations between Watertown and Boston. Flag colors are based on E. coli and cyanobacteria (blue-green algae) levels; blue flags indicate suitable boating conditions and red flags indicate potential health risks. + +See the website's [about page](https://crwa-flagging.herokuapp.com/about) for more about the website functionality and its relationship to + +See the [development history](development/history) document for more information on how this project started and how we came to make the design decisions that you see here today. + +## Code for Boston + +Code for Boston is the group that built the CRWA's flagging website. You can find a list of individual contributors [here](https://github.com/codeforboston/flagging/graphs/contributors) + +Code for Boston is a volunteer Civic Technology meetup. We are part of the [Code for America Brigade network](http://www.codeforamerica.org/brigade/about), and are made up of developers, designers, data geeks, citizen activists, and many others who use creative technology to solve civic and social problems. + +## Charles River + +![](https://www.epa.gov/sites/production/files/styles/large/public/2015-04/cr-watershed-map.jpg) + +[Via the EPA:](https://www.epa.gov/charlesriver/about-charles-river#HistoricalTimeline) + +> The Charles River flows 80 miles from Hopkinton, Mass. to Boston Harbor. The Charles River is the most prominent urban river in New England. It is a major source of recreation and a readily-available connection to the natural world for residents of the Boston metropolitan area. The entire Charles River drains rain and melted snow from a watershed area of 310 square miles. + +## Charles River Watershed Association (CRWA) + +The Charles River Watershed Association ("CRWA") was formed in 1965, the same year that Dirty Water peaked at #11 on the Billboard singles chart. [Via the CRWA's website:](https://www.crwa.org/about.html) + +> CRWA is one of the country’s oldest watershed organizations and has figured prominently in major cleanup and protection efforts. Since our earliest days of advocacy, we have worked with government officials and citizen groups from 35 Massachusetts watershed towns from Hopkinton to Boston. + +The EPA also relies on sample data collected by the CRWA to construct its report card. + diff --git a/docs/docs/admin.md b/docs/docs/admin.md index 5bae89b8..ba513f1c 100644 --- a/docs/docs/admin.md +++ b/docs/docs/admin.md @@ -5,6 +5,8 @@ In development, the username is 'admin' and the password is 'password', but thes be set as config variables `BASIC_AUTH_USERNAME` and `BASIC_AUTH_PASSWORD`. Remember to log out by clicking the `Logout` button in the nav bar. +!!! tip foo bar + ## Cyanobacteria Overrides There should be a link to this page in the admin navigation bar. diff --git a/docs/docs/background.md b/docs/docs/background.md index ecd71956..76388e5d 100644 --- a/docs/docs/background.md +++ b/docs/docs/background.md @@ -1,11 +1,6 @@ # Background # Charles River -![](https://www.epa.gov/sites/production/files/styles/large/public/2015-04/cr-watershed-map.jpg) - -[Via the EPA:](https://www.epa.gov/charlesriver/about-charles-river#HistoricalTimeline) - -> The Charles River flows 80 miles from Hopkinton, Mass. to Boston Harbor. The Charles River is the most prominent urban river in New England. It is a major source of recreation and a readily-available connection to the natural world for residents of the Boston metropolitan area. The entire Charles River drains rain and melted snow from a watershed area of 310 square miles. Throughout most of the 20th century, the Charles River in Boston was known for its contaminated water. The reputation of the Charles River was popularized out of state by the song [Dirty Water by the Standells](https://en.wikipedia.org/wiki/Dirty_Water), which peaked at #11 on the Billboard singles chart on June 11, 1965. (The song has a chorus containing the lines "Well I love that dirty water / Boston you're my home.") @@ -21,7 +16,7 @@ The EPA also relies on sample data collected by the CRWA to construct its report ## Flagging Program -Of the many services that the CRWA provides to the greater Boston community, one of those is monitoring whether it is safe to swim and/or boat in the Charles River. Traditionally, this was accomplished by running some data through a predictive model hosted on a PHP website and outputting the results through that PHP website. However, that website is currently out of commission. At Code for Boston, we attempted to fix the website, although we have had trouble maintaining a steady stream of PHP expertise inside the "Safe Water Project" (the flagging website's parent project). So we are going to be focusing now on building the website from scratch in Python. See the "Stack Justification" documentation for why we chose this path, and why we chose Python + Flask. +Of the many services that the CRWA provides to the greater Boston community, one of those is monitoring whether it is safe to swim and/or boat in the Charles River. ## Code for Boston @@ -35,12 +30,3 @@ Code for Boston is a volunteer Civic Technology meetup. We are part of the Code ## More Information on Code For Boston -If you are interested more about our organization, go to our [website](https://www.codeforboston.org/about/). - -If you wish to contact the project team - -- Join our [Slack channel](https://cfb-public.slack.com): `#water`. - - - Meet us on Tuesday Night for our general meetings by [Signing up to attend Code for Boston events here](https://www.meetup.com/Code-for-Boston/). - - - More contact information on various projects in this [tab of our website](https://www.codeforboston.org/projects/). Search for our Safe Water Project which will provide link to our github project and google hangout/video chat link. diff --git a/docs/docs/deployment.md b/docs/docs/deployment.md index bfe57dcc..b4e32db2 100644 --- a/docs/docs/deployment.md +++ b/docs/docs/deployment.md @@ -1,4 +1,7 @@ -# Production Deployment +# Deployment + +!!! note + This guide is an instruction manual on how to deploy the flagging website to internet via Heroku. If you just want to run the website locally, you do not need Heroku. Instead, check out the [development](/development) guide. The following tools are required to deploy the website: @@ -7,17 +10,29 @@ The following tools are required to deploy the website: ## Deploying for the first time -If you've never deployed the app from your computer, follow these instructions. +!!! note + In this section, the project name is assumed to be `crwa-flagging`. If you are deploying to another URL, such as `crwa-flagging-staging` or another personal domain, then replace each reference to `crwa-flagging` with that. -Note: the project name here is assumed to be `crwa-flagging`. +If you've never deployed the app from your computer, follow these instructions. -1. If you have not already done so, pull the repository to your computer, and then change your directory to it. +1. If you have not already done so, pull the repository to your computer, and then change your directory to it: ```shell git clone https://github.com/codeforboston/flagging.git cd ./flagging ``` + Additionally, make sure the `VAULT_PASSWORD` environment variable is set if it has not already been: + +=== "Windows (CMD)" + ```shell + set VAULT_PASSWORD=replace_me_with_pw + ``` +=== "OSX or Linux (Bash)" + ```shell + export VAULT_PASSWORD=replace_me_with_pw + ``` + 2. Login to Heroku, and add Heroku as a remote repo using Heroku's CLI: ```shell @@ -27,9 +42,14 @@ heroku git:remote -a crwa-flagging 3. Add the vault password as an environment variable to Heroku. -```shell -heroku config:set VAULT_PASSWORD=replace_me_with_pw -``` +=== "Windows (CMD)" + ```shell + heroku config:set VAULT_PASSWORD=%VAULT_PASSWORD% + ``` +=== "OSX or Linux (Bash)" + ```shell + heroku config:set VAULT_PASSWORD=${VAULT_PASSWORD} + ``` 4. Now deploy the app! @@ -45,11 +65,12 @@ heroku logs --tail 6. If everything worked out, you should see the following at or near the bottom of the log: -```shell +``` 2020-06-13T23:17:54.000000+00:00 app[api]: Build succeeded ``` -If you see insted see something like `2020-06-13T23:17:54.000000+00:00 heroku[web.1]: State changed from starting to crashed`, then read the rest of the output to see what happened (there will likely be a lot of stuff, so dig through it). The most common error when deploying to production will be a `RuntimeError: Unable to load the vault; bad password provided` which is self-explanatory. Update the password, and the website will automatically attempt to redeploy. If you don't see that error, then try to self-diagnose. +!!! note + If you see instead see something like `[...] State changed from starting to crashed`, then read the rest of the output to see what happened. The most common error when deploying to production will be a `RuntimeError: Unable to load the vault; bad password provided` which is self-explanatory. Update the password, and the website will automatically attempt to redeploy. If you don't see that error, then try to self-diagnose. 7. Go see the website for yourself! @@ -57,7 +78,10 @@ If you see insted see something like `2020-06-13T23:17:54.000000+00:00 heroku[we 1. Heroku doesn't allow you to redeploy the website unless you create a new commit. Add some updates if you need to with `git add .` then `git commit -m "describe your changes here"`. -2. Once you have done that, Heroku will simply redeploy the site when you merge your working branch: +!!! note + In the _very_ rare case you simply need to redeploy without making any changes to the site, in lieu of the above, simply do `git commit --allow-empty -m "redeploy"`. + +2. Once you have done that, Heroku will redeploy the site when you merge your working branch: ```shell git push heroku master @@ -65,9 +89,9 @@ git push heroku master ## Staging and Production Split -It is recommended, though not required, that you have both "staging" and "production" environments for the website (see [here](https://en.wikipedia.org/wiki/Deployment_environment#Staging) for an explanation), and furthermore it is recommended you deploy to staging and play around with the website to see if it looks right before you deploy to prodution. +It is recommended, though not required, that you have both "staging" and "production" environments for the website (see [here](https://en.wikipedia.org/wiki/Deployment_environment#Staging) for an explanation), and furthermore it is recommended you deploy to staging and play around with the website to see if it looks right before you ever deploy to production. -Managing effectively two separate Heroku apps from a single repository requires a bit of knowledge about how git works. Basically what you're doing is connecting to two separate remote git repositories. The default remote repo is called `heroku` and it was created by Heroku's CLI. But since you now have two Heroku remotes, the Heroku CLI doesn't know what it's supposed to name the 2nd one. So you have to manually name it using git. +Managing effectively two separate Heroku apps from a single repository requires a bit of knowledge about how git works. Basically what you're doing is connecting to two separate remote git repositories. The default remote repo is called `heroku` and it was created by Heroku's CLI. But since you now have two Heroku remote repositories, the Heroku CLI doesn't know what it's supposed to name the 2nd one. So you have to manually name it using git. 1. Run the following command to create a staging environment if it does not already exist. @@ -82,12 +106,35 @@ git remote add staging https://git.heroku.com/crwa-flagging-staging.git git remote -v ``` +!!! tip + The above command should output something like this: + + ```shell + heroku https://git.heroku.com/crwa-flagging.git (fetch) + heroku https://git.heroku.com/crwa-flagging.git (push) + origin https://github.com//flagging.git (fetch) + origin https://github.com//flagging.git (push) + staging https://git.heroku.com/crwa-flagging-staging.git (fetch) + staging https://git.heroku.com/crwa-flagging-staging.git (push) + upstream https://github.com/codeforboston/flagging.git (fetch) + upstream https://github.com/codeforboston/flagging.git (push) + ``` + 3. Now all of your `heroku` commands are going to require specifying the app, but the steps to deploy in staging are otherwise similar to the production deployment: -```shell -heroku config:set --app crwa-flagging-staging VAULT_PASSWORD=replace_me_with_pw -git push staging master -heroku logs --app crwa-flagging-staging --tail -``` -4. Check out the website and make sure it looks right. +=== "Windows (CMD)" + ```shell + heroku config:set --app crwa-flagging-staging VAULT_PASSWORD=%VAULT_PASSWORD% + git push staging master + heroku logs --app crwa-flagging-staging --tail + ``` + +=== "OSX or Linux (Bash)" + ```shell + heroku config:set --app crwa-flagging-staging VAULT_PASSWORD=${VAULT_PASSWORD} + git push staging master + heroku logs --app crwa-flagging-staging --tail + ``` + +4. Check out the website in the staging environment and make sure it looks right. diff --git a/docs/docs/development/history.md b/docs/docs/development/history.md new file mode 100644 index 00000000..c7f0f602 --- /dev/null +++ b/docs/docs/development/history.md @@ -0,0 +1,35 @@ +Traditionally, the CRWA Flagging Program was hosted on a PHP-built website that hosted a predictive model and ran it. However, that website was out of commission due to some bugs and the CRWA's lack of PHP development resources. + +We at Code for Boston attempted to fix the website, although we have had trouble maintaining a steady stream of PHP expertise, so we rebuilt the website from scratch in Python. The project's source code is available [on GitHub](https://github.com/codeforboston/flagging/wiki), and the docs we used for project management and some dev stuff are available in [the repo's wiki](https://github.com/codeforboston/flagging/wiki). + +## Why Python? + +Python proves to be an excellent choice for the development of this website. Due to how the CRWA tends to staff its team (academics and scientists), Python is the most viable language that a website can be built in while still being maintainable by the CRWA. The two most popular coding languages in academia are R and Python. You can't really build a website in R (you technically can, but really really shouldn't for a lot of reasons). So the next best option is Python. + +Even if the CRWA does not staff people based on their Python knowledge (we do not expect that they will do this), they are very likely have connections to various people who know Python. It is unlikely that the CRWA will have as many direct ties to people who have Javascript or PHP knowledge. Because long-term maintainability is such a high priority, Python is the sensible technical solution. + +Not only is Python way more popular than PHP in academia, it's [the most popular programming language](http://pypl.github.io/PYPL.html) _in general_. This means that Python is a natural fit for any organization's coding projects that do not have specialized needs for a particular coding language. + +## Why Flask? + +Once we have decided on Python for web development, we need to make a determination on whether to use Django or Flask, the two leading frameworks for building websites in Python. + +Django is designed for much more complicated websites than what we would be building. Django has its own idiom that takes a lot of time to learn and get used to. On the other hand, Flask is a very simple and lightweight framework built mainly around the use of its "`app.route()`" decorator. + +## Why Heroku? + +Heroku's main advantage is that we can run it for free; the CRWA does not want to spend money if they can avoid doing so. + +One alternative was [Google Cloud](https://cloud.google.com/free/docs/gcp-free-tier#always-free), specifically the [Google App Engine](https://cloud.google.com/appengine/docs/standard/python3/building-app). + +We did not do this mainly as it is more work to set up for developers and controlling costs requires extra care. E.g. the always free tier of Google Cloud still requires users to plug in a payment method. Developers who want to test Google Cloud functionality would also run into some of those limitations too, depending on their past history with Google Cloud. + +With that said, Heroku does provide some excellent benefits focused around how lightweight it is. Google Cloud is not shy about the fact that it can host massive enterprise websites with extremely complicated infrastructural needs. Don't get me wrong: Heroku can host large websites too. But Heroku supports small to medium sites extremely well, and it is really nice for open source websites in particular. + +- Heroku is less opinionated about how you manage your website, whereas Google Cloud products tend to push you toward Google's various Python integrations and APIs. +- Google Cloud is a behemoth of various services that can overwhelm users, whereas Heroku is conceptually easier to understand. +- Heroku integrates much more nicely into Flask's extensive use of CLIs. For example, Heroku's task scheduler tool (which is very easy to set up) can simply run a command line script built in Flask. Google App Engine lets you do a simple cron job setup that [sends GET requests to your app](https://cloud.google.com/appengine/docs/flexible/python/scheduling-jobs-with-cron-yaml), but doing something that doesn't publicly expose the interface requires use of [three additional services](https://cloud.google.com/python/getting-started/background-processing): Pub/Sub, Firestore, and Cloud Scheduler. +- We want to publicly host this website, but we don't want to expose the keys we use for various things. This is a bit easier to do with Heroku, as it has the concept of an environment that lives on the instance's memory and can be set through the CLI. Google App Engine lets you configure the environment [only](https://cloud.google.com/appengine/docs/flexible/python/reference/app-yaml) through `app.yaml`, which is an issue because it means we'd need to gitignore the `app.yaml`. (We want to just gitignore the keys, not the whole cloud deployment config!) + +!!! warning + If you ever want to run this website on Google App Engine, you'll have to make some changes to the repository (such as adding an `app.yaml`) and may also involve making changes to the code-- mainly the data backend and the task scheduler interface. diff --git a/docs/docs/index.md b/docs/docs/index.md index cc8e042b..dc914d3d 100644 --- a/docs/docs/index.md +++ b/docs/docs/index.md @@ -1,9 +1,10 @@ -# Safe Water Project Guide +Welcome to the CRWA Flagging Website Documentation! -Welcome to the Safe Water MKDocs! +This site provides developers and maintainers information about the CRWA's flagging website, including information on: deploying the website, developing the website, using the website's code locally, and using the website's admin panel. -This site provides first-time users background about this project and guide to deploying and using the website. +!!! note + This project is still currently under development. If you are interested in joining our team and contributing, [read our project wiki](https://github.com/codeforboston/flagging/wiki). ## Quick Blurb of this Project -Charles River Watershed Association monitors Charles river for diseases and originally had out-of-comissioned PHP website that keep tracks of its temperature and volume flow. This project aims to update the website wih a new website using flask framework. More information about deployment, background, and how it works in other web pages. +The Charles River Watershed Association (CRWA) monitors the Charles River for diseases and originally had out-of-comissioned PHP website that keep tracks of its temperature and volume flow. This project aims to update the website wih a new website using flask framework. More information about deployment, background, and how it works in other web pages. diff --git a/docs/mkdocs.yml b/docs/mkdocs.yml index 24e52fba..f7141225 100644 --- a/docs/mkdocs.yml +++ b/docs/mkdocs.yml @@ -1,8 +1,35 @@ -site_name: Water MKDocs +site_name: CRWA Flagging Website Documentation nav: - - Home: index.md - - Background: background.md - - Deployment: deployment.md - - System: system.md - -theme: readthedocs +- Home: index.md +- Admin: admin.md +- Background: background.md +- Deployment: deployment.md +- System: system.md +- Development: + - History: development/history.md +theme: + name: material + palette: + scheme: default + primary: teal + accent: cyan + font: + text: Opens Sans + code: Roboto Mono +markdown_extensions: +- admonition +- pymdownx.tabbed # https://facelessuser.github.io/pymdown-extensions/ +- pymdownx.superfences: + custom_fences: + - name: mermaid + class: mermaid + format: !!python/name:pymdownx.superfences.fence_div_format +- sane_lists +extra: + social: + - icon: fontawesome/brands/github + link: https://github.com/codeforboston/flagging + - icon: fontawesome/brands/meetup + link: https://www.meetup.com/Code-for-Boston/ + - icon: fontawesome/brands/twitter + link: https://twitter.com/codeforboston From 0598e9a9384693547d3d0773168b106e6d7c167f Mon Sep 17 00:00:00 2001 From: dwreeves Date: Wed, 2 Sep 2020 00:54:14 -0400 Subject: [PATCH 051/118] Bugfix --- docs/README.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/docs/README.md b/docs/README.md index 04d68053..d959440d 100644 --- a/docs/README.md +++ b/docs/README.md @@ -10,7 +10,7 @@ If you have write permission to the upstream repository (i.e. you are a project pip install mkdocs pip install pymdown-extensions pip install mkdocs-material -mkdocs gh-deploy --remote-branch upstream +mkdocs gh-deploy --remote-name upstream ``` If you do not have write permission to the upstream repository, you can do one of the following: From c4aa43be4f70ae7bc8c4256270f71418cd55b9b3 Mon Sep 17 00:00:00 2001 From: dwreeves Date: Wed, 2 Sep 2020 00:56:20 -0400 Subject: [PATCH 052/118] routing update --- docs/docs/admin.md | 2 -- docs/mkdocs.yml | 4 +++- 2 files changed, 3 insertions(+), 3 deletions(-) diff --git a/docs/docs/admin.md b/docs/docs/admin.md index ba513f1c..5bae89b8 100644 --- a/docs/docs/admin.md +++ b/docs/docs/admin.md @@ -5,8 +5,6 @@ In development, the username is 'admin' and the password is 'password', but thes be set as config variables `BASIC_AUTH_USERNAME` and `BASIC_AUTH_PASSWORD`. Remember to log out by clicking the `Logout` button in the nav bar. -!!! tip foo bar - ## Cyanobacteria Overrides There should be a link to this page in the admin navigation bar. diff --git a/docs/mkdocs.yml b/docs/mkdocs.yml index f7141225..3cc8f8a6 100644 --- a/docs/mkdocs.yml +++ b/docs/mkdocs.yml @@ -1,10 +1,12 @@ site_name: CRWA Flagging Website Documentation nav: - Home: index.md +- About: about.md - Admin: admin.md -- Background: background.md - Deployment: deployment.md - System: system.md +- Database: database.md +- Shell: shell.md - Development: - History: development/history.md theme: From 2aca4bd858ad1093a109ef02ee39d7bf80698e59 Mon Sep 17 00:00:00 2001 From: Lewis Staples Date: Wed, 2 Sep 2020 23:14:49 -0400 Subject: [PATCH 053/118] switching from bool to strtobool --- flagging_site/blueprints/api.py | 12 ------------ flagging_site/config.py | 4 ++-- 2 files changed, 2 insertions(+), 14 deletions(-) diff --git a/flagging_site/blueprints/api.py b/flagging_site/blueprints/api.py index 0b70cf9d..cfd52788 100644 --- a/flagging_site/blueprints/api.py +++ b/flagging_site/blueprints/api.py @@ -8,14 +8,6 @@ from flask_restful import Api from flask_restful import Resource from flask import current_app - -# from ..data.hobolink import get_live_hobolink_data -# from ..data.usgs import get_live_usgs_data -# from ..data.model import process_data -# from ..data.model import reach_2_model -# from ..data.model import reach_3_model -# from ..data.model import reach_4_model -# from ..data.model import reach_5_model from ..data.model import latest_model_outputs from flasgger import swag_from @@ -74,10 +66,6 @@ def model_api(reaches: Optional[List[int]], hours: Optional[int]) -> dict: if not reaches: reaches = df.reach.unique() - print('\n\nreaches') - print(reaches) - print('\n\n') - # construct secondary dictionary from the file (tertiary dicts will be built along the way) sec_dict = {} for reach_num in reaches: diff --git a/flagging_site/config.py b/flagging_site/config.py index ea27ae41..226c1f16 100644 --- a/flagging_site/config.py +++ b/flagging_site/config.py @@ -8,7 +8,7 @@ import os from typing import Dict, Any, Optional, List from flask.cli import load_dotenv - +from distutils.util import strtobool # Constants # ~~~~~~~~~ @@ -155,7 +155,7 @@ class DevelopmentConfig(Config): VAULT_OPTIONAL: bool = True DEBUG: bool = True TESTING: bool = True - OFFLINE_MODE = bool(os.getenv('OFFLINE_MODE', 'false')) + OFFLINE_MODE = strtobool(os.getenv('OFFLINE_MODE', 'false')) class TestingConfig(Config): From 09e39e60295df0e3d5d07f931131c8cefc5dd302 Mon Sep 17 00:00:00 2001 From: Edwin Sun <34005715+edsun123@users.noreply.github.com> Date: Tue, 8 Sep 2020 06:02:03 -0400 Subject: [PATCH 054/118] added back the changes --- flagging_site/blueprints/flagging.py | 10 ++-------- flagging_site/templates/flags.html | 26 +++++++++++++------------- flagging_site/templates/index.html | 4 +--- 3 files changed, 16 insertions(+), 24 deletions(-) diff --git a/flagging_site/blueprints/flagging.py b/flagging_site/blueprints/flagging.py index 1ae4b622..f203940a 100644 --- a/flagging_site/blueprints/flagging.py +++ b/flagging_site/blueprints/flagging.py @@ -123,22 +123,16 @@ def flags() -> str: 'Newton Yacht Club': reach_2_model(df, rows=1)['safe'].iloc[0], 'Watertown Yacht Club': reach_2_model(df, rows=1)['safe'].iloc[0], 'Community Rowing, Inc.': reach_2_model(df, rows=1)['safe'].iloc[0], - 'Northeastern\'s Henderson Boathouse': reach_2_model(df, rows=1)['safe'].iloc[0] - } - - flags_2 = { + 'Northeastern\'s Henderson Boathouse': reach_2_model(df, rows=1)['safe'].iloc[0], 'Paddle Boston at Herter Park': reach_2_model(df, rows=1)['safe'].iloc[0], 'Harvard\'s Weld Boathouse': reach_3_model(df, rows=1)['safe'].iloc[0], 'Riverside Boat Club': reach_4_model(df, rows=1)['safe'].iloc[0], 'Charles River Yacht Club': reach_5_model(df, rows=1)['safe'].iloc[0], - } - - flags_3 = { 'Union Boat Club': reach_5_model(df, rows=1)['safe'].iloc[0], 'Community Boating': reach_5_model(df, rows=1)['safe'].iloc[0], 'Paddle Boston at Kendall Square': reach_5_model(df, rows=1)['safe'].iloc[0] } - flags = [flags_1, flags_2, flags_3] + flags = [flags_1] return render_template('flags.html', flags=flags) \ No newline at end of file diff --git a/flagging_site/templates/flags.html b/flagging_site/templates/flags.html index 93796917..6a5d1ef2 100644 --- a/flagging_site/templates/flags.html +++ b/flagging_site/templates/flags.html @@ -1,20 +1,20 @@ {% block content %} {% for row in flags %} -
- {% for boathouse, flag in row.items() %} -
- {% if flag %} - - {% else %} - - {% endif %} + {% for boathouse, flag in row.items() %} -
-
- {{ boathouse }} -
- {% endfor %} +
+ {% if flag %} + + {% else %} + + {% endif %} + +
+
+ {{ boathouse }}
+ {% endfor %} +

{% endfor %} diff --git a/flagging_site/templates/index.html b/flagging_site/templates/index.html index 170d69e2..64c342ce 100644 --- a/flagging_site/templates/index.html +++ b/flagging_site/templates/index.html @@ -26,8 +26,6 @@

Current Water Quality

{% endif %}
{% endfor %} -
- -
+ {% endblock %} \ No newline at end of file From 993af007c7510985676f77cce3a9a7cbf58e9176 Mon Sep 17 00:00:00 2001 From: Lewis Staples Date: Wed, 9 Sep 2020 10:08:15 -0400 Subject: [PATCH 055/118] cleaning: removed disused commented-out code --- flagging_site/blueprints/flagging.py | 59 +--------------------------- 1 file changed, 1 insertion(+), 58 deletions(-) diff --git a/flagging_site/blueprints/flagging.py b/flagging_site/blueprints/flagging.py index d31b6ca2..20e53974 100644 --- a/flagging_site/blueprints/flagging.py +++ b/flagging_site/blueprints/flagging.py @@ -1,17 +1,13 @@ import pandas as pd + from flask import Blueprint from flask import render_template from flask import request -# from flask_restful import Resource, Api from flask import current_app from ..data.hobolink import get_live_hobolink_data from ..data.usgs import get_live_usgs_data from ..data.model import process_data -# from ..data.model import reach_2_model -# from ..data.model import reach_3_model -# from ..data.model import reach_4_model -# from ..data.model import reach_5_model from ..data.model import latest_model_outputs bp = Blueprint('flagging', __name__) @@ -145,56 +141,3 @@ def output_model() -> str: reach_html_tables[i] = stylize_model_output( df.loc[df['reach'] == i ] ) return render_template('output_model.html', tables=reach_html_tables) - - -# class ReachApi(Resource): - -# def model_api(self) -> dict: -# """ -# This class method retrieves data from the database, -# and then returns a json-like dictionary, prim_dict - -# (note that there are three levels of dictionaries: -# prim_dict, sec_dict, and tri_dict) - -# prim_dict has three items: -# key version with value version number, -# key time_returned with value timestamp, -# key models, and with value of a dictionary, sec_dict -# sec_dict has four items, -# the key for each of one of the reach models (model_2 ... model_5), and -# the value for each item is another dictionary, tri_dict -# tri_dict has five items, -# the keys are: reach, time, log_odds, probability, and safe -# the value for each is a list of values -# """ - - # get model output data from database - # df = latest_model_outputs(48) - - # converts time column to type string because of conversion to json error - # df['time'] = df['time'].astype(str) - - # construct secondary dictionary from the file (tertiary dicts will be built along the way) - # sec_dict = {} - # for reach_num in df.reach.unique(): - # tri_dict = {} - # for field in df.columns: - # tri_dict[field] = df[df['reach']==reach_num][field].tolist() - # sec_dict["model_"+str(reach_num)] = tri_dict - - # # create return value (primary dictionary) - # prim_dict = { - # "version" : "2020", - # "time_returned":str( pd.to_datetime('today') ), - # "models": sec_dict - # } - - # return prim_dict - - - # def get(self): - # return self.model_api() - - -# api.add_resource(ReachApi, '/api/v1/model') From 820429755f2c5db11350398940edf8b9158a3483 Mon Sep 17 00:00:00 2001 From: Edwin Sun <34005715+edsun123@users.noreply.github.com> Date: Tue, 8 Sep 2020 04:53:56 -0400 Subject: [PATCH 056/118] combined commit to debug heroku dev_postgres commented out init_db and auth uncommented line 52 recommented lines 51-52 test app test script uri uri test test test combined test test 1 test 2 --- flagging_site/config.py | 19 ++++++++++--------- 1 file changed, 10 insertions(+), 9 deletions(-) diff --git a/flagging_site/config.py b/flagging_site/config.py index 8d51b44d..86f40c69 100644 --- a/flagging_site/config.py +++ b/flagging_site/config.py @@ -49,15 +49,16 @@ def __repr__(self): POSTGRES_PORT: int = 5432 POSTGRES_DBNAME: str = 'flagging' - @property - def SQLALCHEMY_DATABASE_URI(self) -> str: - user = self.POSTGRES_USER - password = self.POSTGRES_PASSWORD - host = self.POSTGRES_HOST - port = self.POSTGRES_PORT - db = self.POSTGRES_DBNAME - return f'postgres://{user}:{password}@{host}:{port}/{db}' - + # @property + # def SQLALCHEMY_DATABASE_URI(self) -> str: + # user = self.POSTGRES_USER + # password = self.POSTGRES_PASSWORD + # host = self.POSTGRES_HOST + # port = self.POSTGRES_PORT + # db = self.POSTGRES_DBNAME + # return f'postgres://{user}:{password}@{host}:{port}/{db}' + + SQLALCHEMY_DATABASE_URI: str = 'postgres://yuqwhsktykmrqa:34cec8b5de36ee64db9bbc6e6390e1a9ab961008cfcb1640931cd8199fedd971@ec2-34-232-212-164.compute-1.amazonaws.com:5432/devrsq7prgrrfc' SQLALCHEMY_ECHO: bool = True SQLALCHEMY_RECORD_QUERIES: bool = True SQLALCHEMY_TRACK_MODIFICATIONS: bool = False From b2cace1355ebe317c73ec18d0a8ee1c6bbed3cdd Mon Sep 17 00:00:00 2001 From: Edwin Sun <34005715+edsun123@users.noreply.github.com> Date: Thu, 10 Sep 2020 14:43:29 -0400 Subject: [PATCH 057/118] added two options between heroku uri and local uri --- flagging_site/config.py | 30 +++++++++++++++++++----------- 1 file changed, 19 insertions(+), 11 deletions(-) diff --git a/flagging_site/config.py b/flagging_site/config.py index 86f40c69..cff2e181 100644 --- a/flagging_site/config.py +++ b/flagging_site/config.py @@ -49,16 +49,24 @@ def __repr__(self): POSTGRES_PORT: int = 5432 POSTGRES_DBNAME: str = 'flagging' - # @property - # def SQLALCHEMY_DATABASE_URI(self) -> str: - # user = self.POSTGRES_USER - # password = self.POSTGRES_PASSWORD - # host = self.POSTGRES_HOST - # port = self.POSTGRES_PORT - # db = self.POSTGRES_DBNAME - # return f'postgres://{user}:{password}@{host}:{port}/{db}' - - SQLALCHEMY_DATABASE_URI: str = 'postgres://yuqwhsktykmrqa:34cec8b5de36ee64db9bbc6e6390e1a9ab961008cfcb1640931cd8199fedd971@ec2-34-232-212-164.compute-1.amazonaws.com:5432/devrsq7prgrrfc' + @property + def SQLALCHEMY_DATABASE_URI(self) -> str: + if (self.OFFLINE_MODE == True): + user = self.POSTGRES_USER + password = self.POSTGRES_PASSWORD + host = self.POSTGRES_HOST + port = self.POSTGRES_PORT + db = self.POSTGRES_DBNAME + return f'postgres://{user}:{password}@{host}:{port}/{db}' + else: + return f'postgres://yuqwhsktykmrqa:34cec8b5de36ee64 db9bbc6e6390e1a'\ + '9ab961008cfcb1640931cd8199fedd971@ec2-34-232-212-164.compute-1'\ + '.amazonaws.com:5432/devrsq7prgrrfc' + + # SQLALCHEMY_DATABASE_URI: str = 'postgres://yuqwhsktykmrqa:34cec8b5de36ee64'\ + # 'db9bbc6e6390e1a9ab961008cfcb1640931cd8199fedd971@ec2-34-232-212-164.compu'\ + # 'te-1.amazonaws.com:5432/devrsq7prgrrfc' + SQLALCHEMY_ECHO: bool = True SQLALCHEMY_RECORD_QUERIES: bool = True SQLALCHEMY_TRACK_MODIFICATIONS: bool = False @@ -184,4 +192,4 @@ def get_config_from_env(env: str) -> Config: except KeyError: raise KeyError('Bad config passed; the config must be production, ' 'development, or testing.') - return config() + return config() \ No newline at end of file From 5a8fe640a311c6a37a612e38d528649749ee3194 Mon Sep 17 00:00:00 2001 From: Edwin Sun <34005715+edsun123@users.noreply.github.com> Date: Thu, 10 Sep 2020 14:47:21 -0400 Subject: [PATCH 058/118] added two options between heroku uri and local uri --- flagging_site/config.py | 32 ++++++++++++++++---------------- 1 file changed, 16 insertions(+), 16 deletions(-) diff --git a/flagging_site/config.py b/flagging_site/config.py index cff2e181..b091ae07 100644 --- a/flagging_site/config.py +++ b/flagging_site/config.py @@ -49,23 +49,23 @@ def __repr__(self): POSTGRES_PORT: int = 5432 POSTGRES_DBNAME: str = 'flagging' - @property - def SQLALCHEMY_DATABASE_URI(self) -> str: - if (self.OFFLINE_MODE == True): - user = self.POSTGRES_USER - password = self.POSTGRES_PASSWORD - host = self.POSTGRES_HOST - port = self.POSTGRES_PORT - db = self.POSTGRES_DBNAME - return f'postgres://{user}:{password}@{host}:{port}/{db}' - else: - return f'postgres://yuqwhsktykmrqa:34cec8b5de36ee64 db9bbc6e6390e1a'\ - '9ab961008cfcb1640931cd8199fedd971@ec2-34-232-212-164.compute-1'\ - '.amazonaws.com:5432/devrsq7prgrrfc' + # @property + # def SQLALCHEMY_DATABASE_URI(self) -> str: + # if (self.OFFLINE_MODE == True): + # user = self.POSTGRES_USER + # password = self.POSTGRES_PASSWORD + # host = self.POSTGRES_HOST + # port = self.POSTGRES_PORT + # db = self.POSTGRES_DBNAME + # return f'postgres://{user}:{password}@{host}:{port}/{db}' + # else: + # return f'postgres://yuqwhsktykmrqa:34cec8b5de36ee64 db9bbc6e6390e1a'\ + # '9ab961008cfcb1640931cd8199fedd971@ec2-34-232-212-164.compute-1'\ + # '.amazonaws.com:5432/devrsq7prgrrfc' - # SQLALCHEMY_DATABASE_URI: str = 'postgres://yuqwhsktykmrqa:34cec8b5de36ee64'\ - # 'db9bbc6e6390e1a9ab961008cfcb1640931cd8199fedd971@ec2-34-232-212-164.compu'\ - # 'te-1.amazonaws.com:5432/devrsq7prgrrfc' + SQLALCHEMY_DATABASE_URI: str = 'postgres://yuqwhsktykmrqa:34cec8b5de36ee64'\ + 'db9bbc6e6390e1a9ab961008cfcb1640931cd8199fedd971@ec2-34-232-212-164.compu'\ + 'te-1.amazonaws.com:5432/devrsq7prgrrfc' SQLALCHEMY_ECHO: bool = True SQLALCHEMY_RECORD_QUERIES: bool = True From 5c17119b6a1fd7e9e6dd20b97423d61e52963681 Mon Sep 17 00:00:00 2001 From: Edwin Sun <34005715+edsun123@users.noreply.github.com> Date: Thu, 10 Sep 2020 14:49:59 -0400 Subject: [PATCH 059/118] added two options between heroku uri and local uri --- flagging_site/config.py | 32 ++++++++++++++++---------------- 1 file changed, 16 insertions(+), 16 deletions(-) diff --git a/flagging_site/config.py b/flagging_site/config.py index b091ae07..4c2d3054 100644 --- a/flagging_site/config.py +++ b/flagging_site/config.py @@ -49,23 +49,23 @@ def __repr__(self): POSTGRES_PORT: int = 5432 POSTGRES_DBNAME: str = 'flagging' - # @property - # def SQLALCHEMY_DATABASE_URI(self) -> str: - # if (self.OFFLINE_MODE == True): - # user = self.POSTGRES_USER - # password = self.POSTGRES_PASSWORD - # host = self.POSTGRES_HOST - # port = self.POSTGRES_PORT - # db = self.POSTGRES_DBNAME - # return f'postgres://{user}:{password}@{host}:{port}/{db}' - # else: - # return f'postgres://yuqwhsktykmrqa:34cec8b5de36ee64 db9bbc6e6390e1a'\ - # '9ab961008cfcb1640931cd8199fedd971@ec2-34-232-212-164.compute-1'\ - # '.amazonaws.com:5432/devrsq7prgrrfc' + @property + def SQLALCHEMY_DATABASE_URI(self) -> str: + if (self.OFFLINE_MODE == True): + user = self.POSTGRES_USER + password = self.POSTGRES_PASSWORD + host = self.POSTGRES_HOST + port = self.POSTGRES_PORT + db = self.POSTGRES_DBNAME + return f'postgres://{user}:{password}@{host}:{port}/{db}' + else: + return 'postgres://yuqwhsktykmrqa:34cec8b5de36ee64 db9bbc6e6390e1a'\ + '9ab961008cfcb1640931cd8199fedd971@ec2-34-232-212-164.compute-1'\ + '.amazonaws.com:5432/devrsq7prgrrfc' - SQLALCHEMY_DATABASE_URI: str = 'postgres://yuqwhsktykmrqa:34cec8b5de36ee64'\ - 'db9bbc6e6390e1a9ab961008cfcb1640931cd8199fedd971@ec2-34-232-212-164.compu'\ - 'te-1.amazonaws.com:5432/devrsq7prgrrfc' + # SQLALCHEMY_DATABASE_URI: str = 'postgres://yuqwhsktykmrqa:34cec8b5de36ee64'\ + # 'db9bbc6e6390e1a9ab961008cfcb1640931cd8199fedd971@ec2-34-232-212-164.compu'\ + # 'te-1.amazonaws.com:5432/devrsq7prgrrfc' SQLALCHEMY_ECHO: bool = True SQLALCHEMY_RECORD_QUERIES: bool = True From 361cc55ac62d157b0c752b846a8dc37c32e92b14 Mon Sep 17 00:00:00 2001 From: Edwin Sun <34005715+edsun123@users.noreply.github.com> Date: Thu, 10 Sep 2020 14:54:19 -0400 Subject: [PATCH 060/118] fixing pw --- flagging_site/config.py | 16 ++++++++-------- 1 file changed, 8 insertions(+), 8 deletions(-) diff --git a/flagging_site/config.py b/flagging_site/config.py index 4c2d3054..8489985b 100644 --- a/flagging_site/config.py +++ b/flagging_site/config.py @@ -51,14 +51,14 @@ def __repr__(self): @property def SQLALCHEMY_DATABASE_URI(self) -> str: - if (self.OFFLINE_MODE == True): - user = self.POSTGRES_USER - password = self.POSTGRES_PASSWORD - host = self.POSTGRES_HOST - port = self.POSTGRES_PORT - db = self.POSTGRES_DBNAME - return f'postgres://{user}:{password}@{host}:{port}/{db}' - else: + # if (self.OFFLINE_MODE == True): + # user = self.POSTGRES_USER + # password = self.POSTGRES_PASSWORD + # host = self.POSTGRES_HOST + # port = self.POSTGRES_PORT + # db = self.POSTGRES_DBNAME + # return f'postgres://{user}:{password}@{host}:{port}/{db}' + # else: return 'postgres://yuqwhsktykmrqa:34cec8b5de36ee64 db9bbc6e6390e1a'\ '9ab961008cfcb1640931cd8199fedd971@ec2-34-232-212-164.compute-1'\ '.amazonaws.com:5432/devrsq7prgrrfc' From 79980987887563f87e182fbdd20f8c3c7a1f1644 Mon Sep 17 00:00:00 2001 From: Edwin Sun <34005715+edsun123@users.noreply.github.com> Date: Thu, 10 Sep 2020 14:56:24 -0400 Subject: [PATCH 061/118] fixing pw --- flagging_site/config.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/flagging_site/config.py b/flagging_site/config.py index 8489985b..d6086133 100644 --- a/flagging_site/config.py +++ b/flagging_site/config.py @@ -59,7 +59,7 @@ def SQLALCHEMY_DATABASE_URI(self) -> str: # db = self.POSTGRES_DBNAME # return f'postgres://{user}:{password}@{host}:{port}/{db}' # else: - return 'postgres://yuqwhsktykmrqa:34cec8b5de36ee64 db9bbc6e6390e1a'\ + return 'postgres://yuqwhsktykmrqa:34cec8b5de36ee64db9bbc6e6390e1a'\ '9ab961008cfcb1640931cd8199fedd971@ec2-34-232-212-164.compute-1'\ '.amazonaws.com:5432/devrsq7prgrrfc' From 6c5239b57ba53d2979037bd860f4042c169b191c Mon Sep 17 00:00:00 2001 From: Edwin Sun <34005715+edsun123@users.noreply.github.com> Date: Thu, 10 Sep 2020 14:58:40 -0400 Subject: [PATCH 062/118] fixing pw --- flagging_site/config.py | 20 ++++++++++---------- 1 file changed, 10 insertions(+), 10 deletions(-) diff --git a/flagging_site/config.py b/flagging_site/config.py index d6086133..61c3f543 100644 --- a/flagging_site/config.py +++ b/flagging_site/config.py @@ -51,16 +51,16 @@ def __repr__(self): @property def SQLALCHEMY_DATABASE_URI(self) -> str: - # if (self.OFFLINE_MODE == True): - # user = self.POSTGRES_USER - # password = self.POSTGRES_PASSWORD - # host = self.POSTGRES_HOST - # port = self.POSTGRES_PORT - # db = self.POSTGRES_DBNAME - # return f'postgres://{user}:{password}@{host}:{port}/{db}' - # else: - return 'postgres://yuqwhsktykmrqa:34cec8b5de36ee64db9bbc6e6390e1a'\ - '9ab961008cfcb1640931cd8199fedd971@ec2-34-232-212-164.compute-1'\ + if (self.OFFLINE_MODE == True): + user = self.POSTGRES_USER + password = self.POSTGRES_PASSWORD + host = self.POSTGRES_HOST + port = self.POSTGRES_PORT + db = self.POSTGRES_DBNAME + return f'postgres://{user}:{password}@{host}:{port}/{db}' + else: + return 'postgres://yuqwhsktykmrqa:34cec8b5de36ee64db9bbc6e6390e1a9'\ + 'ab961008cfcb1640931cd8199fedd971@ec2-34-232-212-164.compute-1'\ '.amazonaws.com:5432/devrsq7prgrrfc' # SQLALCHEMY_DATABASE_URI: str = 'postgres://yuqwhsktykmrqa:34cec8b5de36ee64'\ From e9599d5653bcdd2fdddceca62892c5c2cec01eee Mon Sep 17 00:00:00 2001 From: Lewis Staples Date: Tue, 15 Sep 2020 14:14:43 -0400 Subject: [PATCH 063/118] Home page populated with boathouse names, reach numbers in db table (previously was hardcoded) --- flagging_site/blueprints/flagging.py | 44 +++++++--------------------- flagging_site/data/database.py | 33 ++++++++++++++++++++- 2 files changed, 42 insertions(+), 35 deletions(-) diff --git a/flagging_site/blueprints/flagging.py b/flagging_site/blueprints/flagging.py index 20e53974..a159e89c 100644 --- a/flagging_site/blueprints/flagging.py +++ b/flagging_site/blueprints/flagging.py @@ -9,6 +9,7 @@ from ..data.usgs import get_live_usgs_data from ..data.model import process_data from ..data.model import latest_model_outputs +from ..data.database import get_boathouse_dict bp = Blueprint('flagging', __name__) @@ -60,40 +61,15 @@ def index() -> str: for key, val in df.to_dict(orient='index').items() } - - homepage = { - 2: { - 'flag': flags[2], - 'boathouses': [ - 'Newton Yacht Club', - 'Watertown Yacht Club', - 'Community Rowing, Inc.', - 'Northeastern\'s Henderson Boathouse', - 'Paddle Boston at Herter Park' - ] - }, - 3: { - 'flag': flags[3], - 'boathouses': [ - 'Harvard\'s Weld Boathouse' - ] - }, - 4: { - 'flag': flags[4], - 'boathouses': [ - 'Riverside Boat Club' - ] - }, - 5: { - 'flag': flags[5], - 'boathouses': [ - 'Charles River Yacht Club', - 'Union Boat Club', - 'Community Boating', - 'Paddle Boston at Kendall Square' - ] - } - } + + homepage = get_boathouse_dict() + + # verify that the same reaches are in boathouse list and model outputs + if flags.keys() != homepage.keys(): + print('ERROR! the reaches are\'t identical between boathouse list and model outputs!') + + for (flag_reach, flag_safe) in flags.items(): + homepage[flag_reach]['flag']=flag_safe model_last_updated_time = df['time'].iloc[0] diff --git a/flagging_site/data/database.py b/flagging_site/data/database.py index d4210bb8..27de7d18 100644 --- a/flagging_site/data/database.py +++ b/flagging_site/data/database.py @@ -4,11 +4,13 @@ """ import os import pandas as pd +import re from typing import Optional from flask import current_app from flask_sqlalchemy import SQLAlchemy from sqlalchemy.exc import ResourceClosedError from psycopg2 import connect +from sqlalchemy import Column db = SQLAlchemy() @@ -96,4 +98,33 @@ def update_database(): from .model import all_models model_outs = all_models(df) - model_outs.to_sql('model_outputs', **options) \ No newline at end of file + model_outs.to_sql('model_outputs', **options) + +class boathouses(db.Model): + reach = Column(db.Integer, unique=False) + boathouse = Column(db.String(255), primary_key=True) + latitude = Column(db.Numeric, unique=False) + longitude = Column(db.Numeric, unique=False) + def __repr__(self): + return ''.format(self.boathouse) + +def get_boathouse_dict(): + """ + Return a dict of boathouses + """ + # return value is an outer dictionary with the reach number as the keys + # and the a sub-dict as the values each sub-dict has the string 'boathouses' + # as the key, and an array of boathouse names as the value + boathouse_dict = {} + + # outer boathouse loop: take one reach at a time + for bh_out in boathouses.query.distinct(boathouses.reach): + bh_list = [] + # inner boathouse loop: get all boathouse names within + # the reach (the reach that was selected by outer loop) + for bh_in in boathouses.query.filter(boathouses.reach == bh_out.reach).all(): + bh_list.append( bh_in.boathouse ) + + boathouse_dict[ bh_out.reach ] = {'boathouses': bh_list} + + return boathouse_dict From b62f7b23ba28fc175029612db0fc69e5b3224fec Mon Sep 17 00:00:00 2001 From: Lewis Staples Date: Tue, 15 Sep 2020 15:17:57 -0400 Subject: [PATCH 064/118] cleaning: delete unneeded import, restore sql echoing --- flagging_site/config.py | 2 +- flagging_site/data/database.py | 9 ++++----- 2 files changed, 5 insertions(+), 6 deletions(-) diff --git a/flagging_site/config.py b/flagging_site/config.py index 226c1f16..fe4280bb 100644 --- a/flagging_site/config.py +++ b/flagging_site/config.py @@ -68,7 +68,7 @@ def SQLALCHEMY_DATABASE_URI(self) -> str: db = self.POSTGRES_DBNAME return f'postgres://{user}:{password}@{host}:{port}/{db}' - SQLALCHEMY_ECHO: bool = False + SQLALCHEMY_ECHO: bool = True SQLALCHEMY_RECORD_QUERIES: bool = True SQLALCHEMY_TRACK_MODIFICATIONS: bool = False diff --git a/flagging_site/data/database.py b/flagging_site/data/database.py index 27de7d18..a3c7133f 100644 --- a/flagging_site/data/database.py +++ b/flagging_site/data/database.py @@ -10,7 +10,6 @@ from flask_sqlalchemy import SQLAlchemy from sqlalchemy.exc import ResourceClosedError from psycopg2 import connect -from sqlalchemy import Column db = SQLAlchemy() @@ -101,10 +100,10 @@ def update_database(): model_outs.to_sql('model_outputs', **options) class boathouses(db.Model): - reach = Column(db.Integer, unique=False) - boathouse = Column(db.String(255), primary_key=True) - latitude = Column(db.Numeric, unique=False) - longitude = Column(db.Numeric, unique=False) + reach = db.Column(db.Integer, unique=False) + boathouse = db.Column(db.String(255), primary_key=True) + latitude = db.Column(db.Numeric, unique=False) + longitude = db.Column(db.Numeric, unique=False) def __repr__(self): return ''.format(self.boathouse) From 563aa4e229760b9dd76ab98de612ead72295a39a Mon Sep 17 00:00:00 2001 From: dwreeves Date: Tue, 15 Sep 2020 18:35:40 -0400 Subject: [PATCH 065/118] Some updates to our docs (still WIP) --- docs/README.md | 2 +- docs/docs/about.md | 6 +- docs/docs/admin.md | 17 ++- docs/docs/background.md | 32 ---- docs/docs/database.md | 49 ------- docs/docs/deployment.md | 34 +++-- docs/docs/development/data.md | 105 ++++++++++++++ docs/docs/development/database.md | 0 docs/docs/development/history.md | 14 +- docs/docs/development/index.md | 6 + docs/docs/development/learning_resources.md | 39 +++++ docs/docs/development/predictive_models.md | 137 ++++++++++++++++++ docs/docs/img/github_fork.png | Bin 0 -> 45775 bytes docs/docs/index.md | 21 ++- docs/docs/setup.md | 153 ++++++++++++++++++++ docs/docs/shell.md | 2 +- 16 files changed, 508 insertions(+), 109 deletions(-) delete mode 100644 docs/docs/background.md delete mode 100644 docs/docs/database.md create mode 100644 docs/docs/development/data.md create mode 100644 docs/docs/development/database.md create mode 100644 docs/docs/development/index.md create mode 100644 docs/docs/development/learning_resources.md create mode 100644 docs/docs/development/predictive_models.md create mode 100644 docs/docs/img/github_fork.png create mode 100644 docs/docs/setup.md diff --git a/docs/README.md b/docs/README.md index d959440d..a218c68b 100644 --- a/docs/README.md +++ b/docs/README.md @@ -10,7 +10,7 @@ If you have write permission to the upstream repository (i.e. you are a project pip install mkdocs pip install pymdown-extensions pip install mkdocs-material -mkdocs gh-deploy --remote-name upstream +mkdocs gh-d`eploy --remote-name upstream ``` If you do not have write permission to the upstream repository, you can do one of the following: diff --git a/docs/docs/about.md b/docs/docs/about.md index ff45d769..0174e125 100644 --- a/docs/docs/about.md +++ b/docs/docs/about.md @@ -2,11 +2,11 @@ ## Flagging Website -Of the many services that the CRWA provides to the greater Boston community, one of those is monitoring whether it is safe to swim and/or boat in the Charles River. The CRWA Flagging Program uses a system of color-coded flags to indicate whether or not the river's water quality is safe for boating at eleven boating locations between Watertown and Boston. Flag colors are based on E. coli and cyanobacteria (blue-green algae) levels; blue flags indicate suitable boating conditions and red flags indicate potential health risks. +Of the many services that the CRWA provides to the greater Boston community, one of those is monitoring whether it is safe to swim and/or boat in the Charles River. The CRWA Flagging Program uses a system of color-coded flags to indicate whether or not the river's water quality is safe for boating at various boating locations between Watertown and Boston. Flag colors are based on E. coli and cyanobacteria (blue-green algae) levels; blue flags indicate suitable boating conditions and red flags indicate potential health risks. -See the website's [about page](https://crwa-flagging.herokuapp.com/about) for more about the website functionality and its relationship to +See the website's [about page](https://crwa-flagging.herokuapp.com/about) for more about the website functionality and how it relates to the flagging program's objectives. -See the [development history](development/history) document for more information on how this project started and how we came to make the design decisions that you see here today. +See the [development history](../development/history) document for more information on how this project started and how we came to make the design decisions that you see here today. ## Code for Boston diff --git a/docs/docs/admin.md b/docs/docs/admin.md index 5bae89b8..2e4ee80e 100644 --- a/docs/docs/admin.md +++ b/docs/docs/admin.md @@ -1,9 +1,16 @@ -# Admin Pages +# Admin Panel -You can reach all admin pages by going to /admin and inputting a username and password. -In development, the username is 'admin' and the password is 'password', but these should -be set as config variables `BASIC_AUTH_USERNAME` and `BASIC_AUTH_PASSWORD`. Remember to log out -by clicking the `Logout` button in the nav bar. +???+ Note + This page discusses how to use the admin panel for the website. For how to set up the admin page username and password during deployment, see the [deployment](../deployment) documentation. + +The admin panel is used to manually override the model outputs during events and advisories that would adversely effect the river quality. + +You can reach the admin panel by going to `/admin` after the base URL for the flagging website. (You need to it in manually.) + +You will be asked a username and password, which will be provided to you by the person who deployed the website. Enter the correct credentials to enter the admin panel. + +???+ note + In "development" mode, the default username is `admin` and the password is `password`. In production, the environment variables `BASIC_AUTH_USERNAME` and `BASIC_AUTH_PASSWORD` are used to set the credentials. ## Cyanobacteria Overrides diff --git a/docs/docs/background.md b/docs/docs/background.md deleted file mode 100644 index 76388e5d..00000000 --- a/docs/docs/background.md +++ /dev/null @@ -1,32 +0,0 @@ -# Background -# Charles River - - -Throughout most of the 20th century, the Charles River in Boston was known for its contaminated water. The reputation of the Charles River was popularized out of state by the song [Dirty Water by the Standells](https://en.wikipedia.org/wiki/Dirty_Water), which peaked at #11 on the Billboard singles chart on June 11, 1965. (The song has a chorus containing the lines "Well I love that dirty water / Boston you're my home.") - -Starting in the late 80s, efforts were made to start cleaning up the Charles River. In 1988, as the result of a lawsuit from the Conservation Law Foundation (CLF), the Massachusetts Water Resources Authority (MWRA) created a combined sewer overflow system to address sewage in the Charles River. In 1995, the CRWA, EPA, municipalities, and Massachusetts state agencies launched the Clean Charles Initiative, which included a report card for the Charles River that is issued by EPA scientists annually. The first grade the Charles River received was a D for the year 1995. The Charles River's grade [peaked](https://www.wbur.org/earthwhile/2019/06/12/charles-river-water-quality-report-card-2018) at A- in 2013 and 2018. - -## Charles River Watershed Association - -The Charles River Watershed Association ("CRWA") was formed in 1965, the same year that Dirty Water peaked at #11 on the Billboard singles chart. [Via the CRWA's website:](https://www.crwa.org/about.html) - -> CRWA is one of the country’s oldest watershed organizations and has figured prominently in major cleanup and protection efforts. Since our earliest days of advocacy, we have worked with government officials and citizen groups from 35 Massachusetts watershed towns from Hopkinton to Boston. - -The EPA also relies on sample data collected by the CRWA to construct its report card. - -## Flagging Program - -Of the many services that the CRWA provides to the greater Boston community, one of those is monitoring whether it is safe to swim and/or boat in the Charles River. - -## Code for Boston - -Code for Boston is a volunteer Civic Technology meetup. We are part of the Code for America Brigade network, and are made up of developers, designers, data geeks, citizen activists, and many others who use creative technology to solve civic and social problems. We aim to find creative means of technology to help better the lives of individuals in our communities. We meet every Tuesday - -## More Information on CRWA - -- [This WBUR article](https://www.wbur.org/news/2017/09/08/charles-river-water-quality-swimming) provides a great overview of the CRWA and its monitoring programs. All volunteers should read it! - -- The CRWA periodically sends a report to the Governor of Massachusetts on the status of the Charles River. [This is the CRWA's latest report.](https://drive.google.com/file/d/1dnDQYMbYvY7U40Fn33Y9xiL7oYcPO6L_/view?usp=sharing) - -## More Information on Code For Boston - diff --git a/docs/docs/database.md b/docs/docs/database.md deleted file mode 100644 index c1d984ee..00000000 --- a/docs/docs/database.md +++ /dev/null @@ -1,49 +0,0 @@ -# Database Project In-depth Guide - -We will be using PostgreSQL, a free, open-source database management system sucessor to UC Berkeley's Ingres Database but also support SQL language - -**On OSX or Linux:** - -We need to setup postgres database first thus enter into the bash terminals: - -``` -brew install postgresql -brew services start postgresql -``` -Explanation: We will need to install postgresql in order to create our database. With postgresql installed, we can start up database locally or in our computer. We use `brew` from homebrew to install and start postgresql services. To get homebrew, consult with this link: https://brew.sh/ - -To begin initialize a database, enter into the bash terminal: - -```shell script -export POSTGRES_PASSWORD=*enter_password_here* -createdb -U *enter_username_here* flagging -psql -U *enter_username_here* -d flagging -c "DROP USER IF EXISTS flagging; CREATE USER flagging SUPERUSER PASSWORD '${POSTGRES_PASSWORD}'" -``` -Explanation: Postgres password can be any password you choose. We exported your chosen postgres password into `POSTGRES_PASSWORD`, an environment variable, which is a variable set outside a program and is independent in each session. Next, we created a database called `flagging` using a username/rolename, which needs to be a Superuser or having all accesses of postgres. By default, the Superuser rolename can be `postgres` or the username for you OS. To find out, you can go into psql terminal, which we will explain below, and enter `\du` to see all usernames. Finally, we add the database `flagging` using the env variable in which we save our password. - -You can see the results using the postgresql terminal which you can open by entering: -``` -psql -``` - -Below are a couple of helpful commands you can use in the postgresql: - -``` -\q --to quit -\c *database_name* --to connect to database -\d --show what tables in current database -\du --show database users -\dt --show tables of current database -``` - -To run the website, in the project directory `flagging` enter: - -```shell script -sh run_unix_dev.sh -``` - -Running the bash script `run_unix_dev.sh` found in the `flagging` folder. Inside the scirpt, it defines environment variables `FLASK_APP` and `FLASK_ENV` which we need to find app.py. We also export the user input for offline mode, vault password, and postgres password for validation. Finally we initialize a database with a custom flask command `flask init-db` and finally run the flask application `flask run`. - -Regarding in how flask application connects to postgresql, `database.py` creates an object `db = SQLAlchemy()` which we will refer again in `app.py` to configure the flask application to support postgressql `from .data import db` `db.init_app(app)`. (We can import the `db` object beecause `__init__.py` make the object available as a global variable) - -Flask supports creating custom commands `init-db` for initializing database and `update-db` for updating database. `init-db` command calls `init_db` function from `database.py` and essentially calls `execute_sql()` which executes the sql file `schema.sql` that creates all the tables. Then calls `update_database()` which fills the database with data from usgs, hobolink, etc. `update-db` command primarily just udpates the table thus does not create new tables. Note: currently we are creating and deleting the database everytime the bashscript and program runs. \ No newline at end of file diff --git a/docs/docs/deployment.md b/docs/docs/deployment.md index b4e32db2..909520b0 100644 --- a/docs/docs/deployment.md +++ b/docs/docs/deployment.md @@ -1,6 +1,6 @@ # Deployment -!!! note +???+ note This guide is an instruction manual on how to deploy the flagging website to internet via Heroku. If you just want to run the website locally, you do not need Heroku. Instead, check out the [development](/development) guide. The following tools are required to deploy the website: @@ -8,9 +8,9 @@ The following tools are required to deploy the website: - [Heroku CLI](https://devcenter.heroku.com/articles/heroku-cli) - [Git](https://git-scm.com/book/en/v2/Getting-Started-Installing-Git) -## Deploying for the first time +## First Time Deployment -!!! note +???+ note In this section, the project name is assumed to be `crwa-flagging`. If you are deploying to another URL, such as `crwa-flagging-staging` or another personal domain, then replace each reference to `crwa-flagging` with that. If you've never deployed the app from your computer, follow these instructions. @@ -28,7 +28,7 @@ cd ./flagging ```shell set VAULT_PASSWORD=replace_me_with_pw ``` -=== "OSX or Linux (Bash)" +=== "OSX (Bash)" ```shell export VAULT_PASSWORD=replace_me_with_pw ``` @@ -46,7 +46,7 @@ heroku git:remote -a crwa-flagging ```shell heroku config:set VAULT_PASSWORD=%VAULT_PASSWORD% ``` -=== "OSX or Linux (Bash)" +=== "OSX (Bash)" ```shell heroku config:set VAULT_PASSWORD=${VAULT_PASSWORD} ``` @@ -69,16 +69,18 @@ heroku logs --tail 2020-06-13T23:17:54.000000+00:00 app[api]: Build succeeded ``` -!!! note +???+ note If you see instead see something like `[...] State changed from starting to crashed`, then read the rest of the output to see what happened. The most common error when deploying to production will be a `RuntimeError: Unable to load the vault; bad password provided` which is self-explanatory. Update the password, and the website will automatically attempt to redeploy. If you don't see that error, then try to self-diagnose. 7. Go see the website for yourself! -## Subsequent deployments +8. You are still not done; you need to do one more step, which is to set up the task scheduler. + +## Subsequent Deployments 1. Heroku doesn't allow you to redeploy the website unless you create a new commit. Add some updates if you need to with `git add .` then `git commit -m "describe your changes here"`. -!!! note +???+ note In the _very_ rare case you simply need to redeploy without making any changes to the site, in lieu of the above, simply do `git commit --allow-empty -m "redeploy"`. 2. Once you have done that, Heroku will redeploy the site when you merge your working branch: @@ -87,6 +89,14 @@ heroku logs --tail git push heroku master ``` +???+ tip + If you are having any issues here related to merge conflicts, instead of deleting everything and starting over, try to pull the data from the `heroku` branch in and merge it into your local branch. + + ```shell + git fetch heroku + git pull heroku master + ``` + ## Staging and Production Split It is recommended, though not required, that you have both "staging" and "production" environments for the website (see [here](https://en.wikipedia.org/wiki/Deployment_environment#Staging) for an explanation), and furthermore it is recommended you deploy to staging and play around with the website to see if it looks right before you ever deploy to production. @@ -106,14 +116,14 @@ git remote add staging https://git.heroku.com/crwa-flagging-staging.git git remote -v ``` -!!! tip +???+ success The above command should output something like this: ```shell heroku https://git.heroku.com/crwa-flagging.git (fetch) heroku https://git.heroku.com/crwa-flagging.git (push) - origin https://github.com//flagging.git (fetch) - origin https://github.com//flagging.git (push) + origin https://github.com/YOUR_USERNAME_HERE/flagging.git (fetch) + origin https://github.com/YOUR_USERNAME_HERE/flagging.git (push) staging https://git.heroku.com/crwa-flagging-staging.git (fetch) staging https://git.heroku.com/crwa-flagging-staging.git (push) upstream https://github.com/codeforboston/flagging.git (fetch) @@ -130,7 +140,7 @@ git remote -v heroku logs --app crwa-flagging-staging --tail ``` -=== "OSX or Linux (Bash)" +=== "OSX (Bash)" ```shell heroku config:set --app crwa-flagging-staging VAULT_PASSWORD=${VAULT_PASSWORD} git push staging master diff --git a/docs/docs/development/data.md b/docs/docs/development/data.md new file mode 100644 index 00000000..5c22cf64 --- /dev/null +++ b/docs/docs/development/data.md @@ -0,0 +1,105 @@ +# Data + +Here is a "TLDR" of the data engineering for this website: + +- To get data, we ping two different APIs, combine the responses from those API requests, do some processing and feature engineering of the data, and then run a predictive model on the processed data. + +- To store the data and then later retrieve it for the front-end of the website, we use PostgreSQL database. + +- To actually run the functionality that gets data, processes it, and stores it. we run a [scheduled job](https://en.wikipedia.org/wiki/Job_scheduler) that runs the command `flask update-db` at a set time intervals. + +## Sources + +There are two sources of data for our website: + +1. An API hosted by the USGS National Water Information System API that's hooked up to a Waltham based stream gauge (herein "USGS" data); +2. An API for a HOBOlink RX3000 Remote Monitoring Station device stationed on the Charles River (herein "HOBOlink"). + +### USGS + +The code for retrieving and processing the HOBOlink data is in `flagging_site/data/usgs.py`. + +The USGS API very is straightforward. It's a very typical REST API that takes "GET" requests and return well-formatted json data. Our preprocessing of the USGS API consists of parsing the JSON into a Pandas dataframe. + +The data returned by the USGS API is in 15 minute increments, and it measures the stream flow (cubic feet per second) of the Charles River out in Waltham. + +### HOBOlink + +The code for retrieving and processing the HOBOlink data is in `flagging_site/data/hobolink.py`. + +The HOBOlink device captures various information about the Charles River at the location it's stationed: + +- Air temperature +- Water temperature +- Wind speed +- Photosynthetically active radiation (i.e. sunlight) +- Rainfall + +The HOBOlink data is accessed through a REST API using some credentials stored in the `vault.zip` file. + +The data actually returned by the API is a combination of a yaml file with a CSV below it, and we just use the CSV part. We then do the following to preprocess the CSV: + +- We remove all timestamps ending `:05`, `:15`, `:25`, `:35`, `:45`, and `:55`. These only contain battery information, not weather information. The final dataframe returned is ultimately in 10 minute incremenets. +- We make the timestamp consistently report eastern standard times. There is a weird issue in which the HOBOlink API returns slightly different datetime formats that messes with Pandas's timestamp parser. We are able to coerce the timestamp into something consistent and predictable. +- We consolidate duplicative columns. The HOBOlink API has a weird issue where sometimes it splits columns of data with the same name, seemingly at random. This issue causes serious data issues if untreated (at one point, it caused our model to fail to update for a couple days), so our function cleans the data. + +As you can see from the above, the HOBOlink API is a bit finicky for whatever reason, but we have a good data processing solution for these problems. + +The HOBOlink data is also notoriously slow to retrieve (regardless of whether you ask for 1 hour of data or multiple weeks of data), which is why we belabored building the database portion of the flagging website out in the first place. + +???+ tip + You can manually download the latest raw data from this device [here](https://www.hobolink.com/p/0cdac4a6910cef5a8883deb005d73ae1). If you want some preprocessed data that implements the above modifications to the output, there is a better way to get that data explained in the shell guide. + +### Combining the data + +Additional information related to combining the data and how the models work is in the [Predictive Models](../predictive_models.md) page. + +## Postgres Database + +PostgresSQL is a free, open-source database management system, and it's what our website uses to store data. + +**On OSX or Linux:** + +We need to setup postgres database first thus enter into the bash terminals: + +``` +brew install postgresql +brew services start postgresql +``` +Explanation: We will need to install postgresql in order to create our database. With postgresql installed, we can start up database locally or in our computer. We use `brew` from homebrew to install and start postgresql services. To get homebrew, consult with this link: https://brew.sh/ + +To begin initialize a database, enter into the bash terminal: + +```shell script +export POSTGRES_PASSWORD=*enter_password_here* +createdb -U *enter_username_here* flagging +psql -U *enter_username_here* -d flagging -c "DROP USER IF EXISTS flagging; CREATE USER flagging SUPERUSER PASSWORD '${POSTGRES_PASSWORD}'" +``` +Explanation: Postgres password can be any password you choose. We exported your chosen postgres password into `POSTGRES_PASSWORD`, an environment variable, which is a variable set outside a program and is independent in each session. Next, we created a database called `flagging` using a username/rolename, which needs to be a Superuser or having all accesses of postgres. By default, the Superuser rolename can be `postgres` or the username for you OS. To find out, you can go into psql terminal, which we will explain below, and enter `\du` to see all usernames. Finally, we add the database `flagging` using the env variable in which we save our password. + +You can see the results using the postgresql terminal which you can open by entering: +``` +psql +``` + +Below are a couple of helpful commands you can use in the postgresql: + +``` +\q --to quit +\c *database_name* --to connect to database +\d --show what tables in current database +\du --show database users +\dt --show tables of current database +``` + +To run the website, in the project directory `flagging` enter: + +```shell script +sh run_unix_dev.sh +``` + +Running the bash script `run_unix_dev.sh` found in the `flagging` folder. Inside the scirpt, it defines environment variables `FLASK_APP` and `FLASK_ENV` which we need to find app.py. We also export the user input for offline mode, vault password, and postgres password for validation. Finally we initialize a database with a custom flask command `flask init-db` and finally run the flask application `flask run`. + +Regarding in how flask application connects to postgresql, `database.py` creates an object `db = SQLAlchemy()` which we will refer again in `app.py` to configure the flask application to support postgressql `from .data import db` `db.init_app(app)`. (We can import the `db` object beecause `__init__.py` make the object available as a global variable) + +Flask supports creating custom commands `init-db` for initializing database and `update-db` for updating database. `init-db` command calls `init_db` function from `database.py` and essentially calls `execute_sql()` which executes the sql file `schema.sql` that creates all the tables. Then calls `update_database()` which fills the database with data from usgs, hobolink, etc. `update-db` command primarily just udpates the table thus does not create new tables. Note: currently we are creating and deleting the database everytime the bashscript and program runs. \ No newline at end of file diff --git a/docs/docs/development/database.md b/docs/docs/development/database.md new file mode 100644 index 00000000..e69de29b diff --git a/docs/docs/development/history.md b/docs/docs/development/history.md index c7f0f602..72c19e63 100644 --- a/docs/docs/development/history.md +++ b/docs/docs/development/history.md @@ -31,5 +31,15 @@ With that said, Heroku does provide some excellent benefits focused around how l - Heroku integrates much more nicely into Flask's extensive use of CLIs. For example, Heroku's task scheduler tool (which is very easy to set up) can simply run a command line script built in Flask. Google App Engine lets you do a simple cron job setup that [sends GET requests to your app](https://cloud.google.com/appengine/docs/flexible/python/scheduling-jobs-with-cron-yaml), but doing something that doesn't publicly expose the interface requires use of [three additional services](https://cloud.google.com/python/getting-started/background-processing): Pub/Sub, Firestore, and Cloud Scheduler. - We want to publicly host this website, but we don't want to expose the keys we use for various things. This is a bit easier to do with Heroku, as it has the concept of an environment that lives on the instance's memory and can be set through the CLI. Google App Engine lets you configure the environment [only](https://cloud.google.com/appengine/docs/flexible/python/reference/app-yaml) through `app.yaml`, which is an issue because it means we'd need to gitignore the `app.yaml`. (We want to just gitignore the keys, not the whole cloud deployment config!) -!!! warning - If you ever want to run this website on Google App Engine, you'll have to make some changes to the repository (such as adding an `app.yaml`) and may also involve making changes to the code-- mainly the data backend and the task scheduler interface. +???+ warning + If you ever want to run this website on Google App Engine, you'll have to make some changes to the repository (such as adding an `app.yaml`), and it may also involve making changes to the code-- mainly the data backend and the task scheduler interface. + +## Why Pandas? + +We made the decision to use Pandas to manipulate data in the backend of the website because it has an interface that should feel familiar to users of Stata, R, or other statistical software packages commonly used by scientists and academics. This ultimately helps with the maintainability of the website for its intended audience. Data manipulation in SQL can sometimes be unintuitive and require advanced trickery (CTEs, window functions) compared to Pandas code that achieves the same results. Additionally, SQL code tends to be formatted in a non-chronological way, e.g. subqueries run before the query that references them, but occur somewhere in the middle of a query. This isn't hard if you use SQL a bit, but it's not intuitive until you've done a bit of SQL. + +It's true that Pandas is not as efficient as SQL, but we're not processing millions of rows of data, we're only processing a few hundred rows at a time and at infrequent intervals. (And even if efficiency was a concern, we'd sooner use something like [Dask](https://dask.org/) than SQL.) + +One possible downside of Pandas compared to SQL is that SQL has been around for a very long time, and is more of a "standardized" thing than Pandas is or perhaps ever will be. We went with the choice for Pandas after discussing it with some academic friends, but we are aware that in the non-academic world, there are more people who know SQL than Pandas. + +We do use SQL in this website, but primarily to store and retrieve data and to access some features that integrate nicely with the SQLAlchemy ORM (notably the Flask-Admin extension). diff --git a/docs/docs/development/index.md b/docs/docs/development/index.md new file mode 100644 index 00000000..776766a7 --- /dev/null +++ b/docs/docs/development/index.md @@ -0,0 +1,6 @@ +# Development + +The Development guide is aimed at users who wish to understand the code base and make changes to it if need be. + +!!! tip + Make sure to go through the [setup guide](../../setup) before doing anything in the development guide. \ No newline at end of file diff --git a/docs/docs/development/learning_resources.md b/docs/docs/development/learning_resources.md new file mode 100644 index 00000000..ffd7ba06 --- /dev/null +++ b/docs/docs/development/learning_resources.md @@ -0,0 +1,39 @@ +# Learning Resources + +???+ tip + Unless you want to overhaul the website or do some bugfixing, you _probably_ don't need to learn any of the frameworks here. + + The Flagging Website documentation is detailed, self-contained, and should cover the vast majority of use cases for the Flagging website from an administrative perspective, such as updating the predictive model and deploying the website to Heroku. + +The code base is mainly built with the following frameworks; all of these but the last one are Python frameworks: + +- Pandas (data manipulation framework, built on top of another framework called `numpy`.) +- Flask (web framework that handles routing of the website) +- Jinja2 (text markup framework that is used for adding programmatic logic to statically rendered HTML pages.) +- Click (CLI building framework) +- Postgresql + +These frameworks may be intimidating if this is your first time seeing them and you want to make changes to the site. This page has some learning resources that can help you learn these frameworks. + +## Flask & Jinja2 + +The [official Flask tutorial](https://flask.palletsprojects.com/en/1.1.x/tutorial/) is excellent and worth following if you want to learn both Flask and Jinja2. + +???+ tip + Our website's code base is organized somewhat similar to the code base built in the official Flask tutorial. If you are confused by how the code base is organized, going through the tutorial may help clarify some of our design choices. For more examples of larger Flask websites, check out the [flask-bones](https://github.com/cburmeister/flask-bones) template; we did not explicitly reference it in constructing our website but it nevertheless follows a lot of the same ideas we use. + +## Pandas + +The Pandas documentation has excellent resources for users who are coming from R, Stata, or SAS: [https://pandas.pydata.org/docs/getting_started/comparison/comparison_with_r.html](https://pandas.pydata.org/docs/getting_started/comparison/comparison_with_r.html) + +## Click + +Click is pretty easy to understand: it lets you wrap your Python functions with decorators to make the code run on the command line. We use Click to do a lot of our database management. + +The [homepage for Click's documentation](https://click.palletsprojects.com/en/7.x/) should give you a good idea of what Click is all about. Additionally, Flask's documentation has a page [here](https://flask.palletsprojects.com/en/1.1.x/cli/) that discusses Flask's integration with Click. + +## Postgresql + +We do not do anything crazy with Postgresql. We made a deliberate decision to only use SQL for retrieving and storing data, and to avoid some of the more intermediate to advanced aspects of Postgres such as CTEs, views, and so on. Actual data manipulation is done in Pandas. + +A simple [intro SQL tutorial](https://www.khanacademy.org/computing/computer-programming/sql) should be more than sufficient for understanding the SQL we use in this code base. diff --git a/docs/docs/development/predictive_models.md b/docs/docs/development/predictive_models.md new file mode 100644 index 00000000..048847be --- /dev/null +++ b/docs/docs/development/predictive_models.md @@ -0,0 +1,137 @@ +# Predictive Models + +The Flagging Website is basically just a deployed predictive model, so in a sense this document covers the real core of the code base. This page explains the models and the data transformations that occur from the original data. At the bottom, there are some notes on how to change the model coefficients and rerun the website with new coefficients. + +The predictive models are stored in the file `/flagging_site/data/models.py`. These models are run as part of the `update-db` command. The input for the models are a combination of the HOBOlink and USGS data with some transformations of the data. The outputs are stored in the SQL table named `model_outputs`. + +???+ tip + There is a fair bit of Pandas in this document and it may be intimidating. However, if you only want to change the model's coefficients and nothing more, you won't need to touch the Pandas directly. + +## Data Transformations + +The model combines data from the HOBOlink device and USGS. These two data sources are run through the function `process_data()`, in which the data is aggregated to hourly intervals (USGS is every 15 minutes and HOBOlink is every 10 minutes). + +Once the data sources are aligned, additional feature transformations are performed such as rolling averages, rolling sums, and a measure of when the last significant rainfall was. + +The feature transformations the CRWA uses depends on the year of the model, so by the time you may be reading this, this information may be outdated. Previous versions included rolling average wind speeds and air/water temperatures over 24 hours. The current version of the model (as of 2020) calculates the following: + +- Rolling 24 hours of the PAR (photosynthetically active radiation) and the stream flow (cubic feet per second). +- Rolling sum of rainfall over the following intervals: 0-24h, 0-48h, and 24-48h. +- The numbers of days since the last "significant rainfall," where significant rain is defined as when the rolling sum of the last 24 hours of rainfall is at least 0.20 inches. + +???+ tip + If you look at the code, you'll see a lot of stuff like `#!python rolling(24)`. The reason `rolling` works is because the dataframe is sorted already by timestamp at that point by `#!python df = df.sort_values('time')`. + +???+ note + We use 28 days of HOBOlink data to process the model. For most features, we only need the last 48 hours worth of data to calculate the most recent value, however the last significant rainfall feature requires a lot of historic data because it is not technically bounded or transformed otherwise. This means that even when calculating 1 row of output data, i.e. the latest hour of data, we still need 28 days. + + In the deployed model, if we do not see any significant rainfall in the last 28 days, we return the difference between the timestamp and the earliest time in the dataframe, `#!python df['time'].min()`. In this scenario, the data will no longer be temporally consistent: a calculation right now will have `28.0` for `'days_since_sig_rain'`, but 12 hours from now it will be 27.5. This is fine though because the model will basically never predict E. coli blooms with 28+ days since significant rain, even when the data is not censored. + + Unfortunately there's no pretty way to implement `days_since_sig_rain`, so the Pandas code that does all of this is one of the more inscrutable parts of the codebase. Note that `'last_sig_rain'` is calculating the timestamp of the last significant rain, and `'days_since_sig_rain'` calculates the time delta and translates into days: + + ```python + df['sig_rain'] = df['rain_0_to_24h_sum'] >= SIGNIFICANT_RAIN + df['last_sig_rain'] = ( + df['time'] + .where(df['sig_rain']) + .ffill() + .fillna(df['time'].min()) + ) + df['days_since_sig_rain'] = ( + (df['time'] - df['last_sig_rain']).dt.seconds / 60 / 60 / 24 + ) + ``` + +## Model Overviews + +Each model function defined in `models.py` is formatted like this: + +1. Take the last few rows of the input dataframe (I discuss what the input dataframe is later on this page). Each row is an hour of data on the condition of the Charles River and its surrounding environment, so for example, taking the last 24 rows is equivalent to taking the last 24 hours of data. +2. Predict the probability of the water being unsafe using a logistic regression fit, with the coefficients in the log odds form (so the dot product of the parameters and the data returns a predicted log odds of the target variable). +3. To get the probability of a log odds, we run it through a logistic function (`sigmoid()`, defined at the top of `models.py`). +4. We check whether the function is above or below the target threshold for safety, defined by `SAFETY_THRESHOLD`. +5. Lastly, we return a dataframe with 5 columns of data: `'reach'`, `'time'`, `'log_odds'` (step 2), `'probability'` (step 3), and `'safe'` (step 4). Each row in output corresponds to a row of input data. + +Here is an example function. It should be pretty easy to track the steps outlined above with the code below. + +```python +def reach_3_model(df: pd.DataFrame, rows: int = 48) -> pd.DataFrame: + """ + a- rainfall sum 0-24 hrs + b- rainfall sum 24-48 hr + d- Days since last rain + 0.267*a + 0.1681*b - 0.02855*d + 0.5157 + + Args: + df: (pd.DataFrame) Input data from `process_data()` + rows: (int) Number of rows to return. + + Returns: + Outputs for model as a dataframe. + """ + df = df.tail(n=rows).copy() + + df['log_odds'] = ( + 0.5157 + + 0.267 * df['rain_0_to_24h_sum'] + + 0.1681 * df['rain_24_to_48h_sum'] + - 0.02855 * df['days_since_sig_rain'] + ) + + df['probability'] = sigmoid(df['log_odds']) + df['safe'] = df['probability'] <= SAFETY_THRESHOLD + df['reach'] = 3 + + return df[['reach', 'time', 'log_odds', 'probability', 'safe']] +``` + +## Editing the Models + +???+ note + + This section covers making changes to the following: + + - The coefficients for the models. + - The safety threshold. + - The model features. + + If you want to do anything more complicated, such as adding a new source of information to the model, that is outside the scope of this document. To accomplish that, you'll need to do more sleuthing into the code to really understand it. + +???+ note + Making any changes covered in this section is relatively easy, but you'll still need to actually deploy the changes to Heroku if you want them to be on the live site. Read the [deployment guide](../../deployment) for more. + +### Model coefficients + +As covered in the last section, each model's coefficients are represented as log odds ratios. Don't be confused by this statement though: this is how logistic regression is represented in all statistical software packages-- `Logit` in Python's Statsmodels, `logit` in Stata, and `glm` in R-- since that's what's being calculated mathematically when a logistic regression is calculated. I only emphasize this to point out that to get a probability, the final log odds needs to be logistically transformed (which is done via the `sigmoid()` function) after the linear terms are summed up. + +The code representing the logistic model prediction was organized for maximum legibility: the first number is the constant term, and the remaining coefficients are aligned next to the column name. Note the final coefficient in this particular example is a negative coefficient and is thus subtracted. + +```python +df['log_odds'] = ( + 0.5157 + + 0.267 * df['rain_0_to_24h_sum'] + + 0.1681 * df['rain_24_to_48h_sum'] + - 0.02855 * df['days_since_sig_rain'] +) +``` + +Changing the coefficients is as simple as just changing one of those numbers next to its respective column name, inside of its respective model for any particular reach. + +### Safety threshold + +The safety threshold is defined near the top of the document: + +```python +SAFETY_THRESHOLD = 0.65 +``` + +This represents a 65% threshold for whether or not we consider the water safe or not. The `SAFETY_THRESHOLD` value is just used as a placeholder/convenience for whatever the default threshold should be. You can always change this value to be lower or higher, and additionally you can replace `SAFETY_THRESHOLD` inside of a model function + +???+ warning + Hopefully this goes without saying, but if you are going to change the threshold, please have a good, scientifically and statistically justifiable reason for doing so! + +### Feature transformations + +Feature transformations occur in the `process_data()` function after the data has been aggregated by hour, merged, and sorted by timestamp. + +If you want to add some feature transformations, my suggestion is you try to learn from existing examples and copy+paste with the necessary replacements. If you have a feature that can't be built from a copy+paste, that's where you'll possibly need to learn a bit of Pandas. diff --git a/docs/docs/img/github_fork.png b/docs/docs/img/github_fork.png new file mode 100644 index 0000000000000000000000000000000000000000..c0b9e32ac983799ba547aa3d7998f5f6a5c4ae04 GIT binary patch literal 45775 zcmYhi1ymftwlzF>kl+qMf;+(-g1fs1cNjdl1h?Ss7Cb<33+^5uxH}B)_I2L7@BaT< z4a_Qfx~oo|I(7CwyCReorQRbEB7s1l_cGGrsvr=|67c!t9W?M8wo^tB1cLfzEheTc zBPK@X763rhDYV}~fI71QG&@Q3r5{ZPaemms#siBu;4gH*+r)_#qa z!9hvK92;cY-JRQC9voF)janT-bRy zh+~y&cKBP`cvWP+*7e7CWZ%-p7%7-#=)SmVv_*b~d=Uk8znc544mVllRKu5oN^X*FH4XWaHej-uh8~ zn8acxsLAUHo~P%WFHd}9&05e(X;Jism0>hN#1yCfkspm@*LUzoruKWu)3-SPzc+F_ zszVy^6ob&^Q)F#^=LFA?Dvh*^>x3S}yr+J!kfgOpn@Bx`XMHesRsbq!@^|vutrpSm z6Pm-DMqEK?_{PAflq78^MZ<`0Vp^0 zZZO2e(54b2hIbDhN|x)dzOzT`Xo!Q(zQyhqD!{ZLI!WuefI!R?|9+r6i-p~RMg&(G z1qp;5L>Lef(`HhnI?zPmDxvKv=4fwk?%)a%b2k6tYHmj6VeM)~CMBbwtQClg2Lh3S zWW+zKdoG=1dl(b#KHTT>p97!V4Z*MZw+2&C6=n_^;;ry@|QST}IZ3p5icb2V~>wVom z`Q;0SjR+a~J6PyI;14%6F(=gjeEQemO8l?wfBwZ6`+uLvzQ!oT8#<6ow-73z@zyzm4mcl-T1*~-g1#0;HK!)wkq9;dQ)JuxCP1R~dw-h|_tA_DotZW)_@{QI z<}%Y=@P~h+bRd5lsIGM~#X1;w{eIjEwVt{C2MlF&K3nSFM_C-5+<1HIvG(#b7d8wg zPLwT@?!yQd;UO;SHqN6p+xa*0W-jMwIEKl;3RlcQm}g-4dB?oiin$|8zf1M=&tRi5 zi>(@e@2-TxV@?;e}Dt^)Vl(zIC&KsU{Bext&a8{F9v_I;GfCnq1X^7bAq zJ|YZMCd1a(sYDFoG18Sp7ncsl1o03n>gvj>T$Ej$Hc3{scw2#A-#20T8(FQ5>{=#rtM!`9wZIHvKk&y-Tr(0rrf?SVv4uktqgU({krxOF#t}I_6m+`D*`jKa6i7PtnmCRIzYrtZXFWl5*$uF&``m z5f$R-OnIu=G~^aqoX zb^)(}sQuDqVt0BRTfDrKH4mTR{-9r`j0wAW)V#R1cDQ*5cZGx;7gMN56H4AIva#*_%88jr#Dk8ccafHU~9Wx_)WR6QseFEW=#Q_yUg@41`XY?;}t^xSnOeDT|tNSjg9dNHK2G4}pjGw|du0U8T`^gk7v z5l1ktMKx39!5a2GO!ydqy(uUvJd{+^6ENfBF+XB$ihx_&-W4LU<0HOoX1V@OL>!9j%f zRg56wv~{ouE<_NU3^(>6o7-65$Bcu8+j27~$do4X^doWL_qf8=mc^i) z#O-WguTsg*2OIQSe`tD^Jav6w6pL>xGHZC24Nqt_#*iEm5s#)+G^-}2x}E%myeB18 z5m%2o;ai3R^^*7oV^;sJGKYe3fGH}n&0UV#v zcXv%=Z59@|QLP%}Q+R`wX@{h9`C>s|St+Tg$HsCeHl5O@Tih@8tq+TjmLmS9Xfqo4dYqkwv8=Ju^MKTyJ}~HZ5G5 zp04KN;%t>y!EBI`o?G*M)aLl8IyshxhDP`*krV@WlqJ`9Q`5TFW6dvW+=7#ftK(&2 zT^~zKG^y)pyar4x>~gz8ll#K*cQi_z!+%X5^v*Ej>dI2OFF}opO0p1~v?wZ?r4!Dr zJeVGQ%z5$mqJU|+lkm8FMM?CUPLd1*jcltE7m^viCDiZorD>Y8LhBG7;q9W^e}$&= zEe@@&j*X=KZ4J-=2?^I~eITcp&04kKYyyR-#s8uh#x4y0w~t|=bsW(*>0~oi{pNmo zCk>NCsLFYA;vsFfqJB|ZH;W*$6KrKh;@$w~zv?#b>gq+}E5@w40Xh#uYsTHuUw_IJ zcjjMI8TqouW;ynVo~&)Jda(1-eGQrzY+tcv#C_T+n_ahCSMjNXwX7}K-*e_^(hlik zX|!8ycU!($)YR7gVWF?&A*!#GfQhxkcKFR92z&um)mUGT+$-_{=VoV{h^Dl7?MKtt zmwHfui(E3ddA$zkds-Sg4#8nYvs1TEi79jDc=}Y;dW*$ewyjpb_&PMH<=zIgh?Z8< zaU99qs1(@%-4;nQ!V67#^Nb2KWq~J^w`Xxw^U%l`kN<0s66QNY1B1-udB zs4B(a>snf67M4@vE5R(Yn0eJE0cTNUXf@sfnO}(W!J)I36GG1$Fc7r@P8BS$tSpsD zU_j06`}sffPyr6gaogJAGdjhe^ouh-32HH?Cc1ljTke-7%*wBwjuFgK6n^ac2zGRI zv^Nk;T(5I}Pz~$t-gE-egkp0%2Kow%i8K-{tnuv&FE6hnBErlJ7WRsmh%hi4u%0PR&_0iDK{Cs24jyQ8>ArUbny-*qHrSzRqtLGPWGs;*~qWM+dmg1YS zMVEMt&6ZhbZ0D9>=j+=Wyj;dlnd1jG9;V%3eM(GD4laW*d*92$H(*Sj`fH zopnj(Z#4G%{Fp@d=~D(&RdrcedW%uaJ2azTPJYhLY`KFSD@$HSe-5K|agg`$#xmL4 z3d)a;9XsIy_~)x9MuvEtum46An1Gs}s`+P2n;5laZ)$F3UtR}+=QKoHhYmam0?E| zlDEiJM{ObZ8Fn;y4!ka(T=8|p*Vi|=Vuc)Z4)+|v^C=8PGPs*Ibrg>!%ScUcVQpH+ zKy9Dx##|0>o2PpA@ZQHy)dV`TUkGrA%4dSIvP5|JU#A~FsKm$T=jTVqMp2p6N{c7N z=HwVwY4Tg%j>!zm6lI^Bp1wzZ9}^w@=Quks>)W^ceDYw$QniY?kMZ}Jjvk(;bC>2( zyKL!;W?bmvR#rnZR8gIaoiLz2lXz{$I^C)&dwXi7U7V63wl6GAI?>FfeBHWoQ{;@R3Vc^Aav_82Wqrc93`TWlij_!EYJq*}Mr?r^V) zqzmru`g&y7xmNqn^mN&rO)=&L@Ir0I6h-c#ZRbMHqvX+4hbJ%o*1c8j9*z8t4#5SN zD9y}{Z?u0J zu&JNOX4j*{Z0&MAdvv?nn<%l)AY;4$*8*2?aIi!_4d)RpNnELF8R++xCRfhjuHa4Y z84WcxK1!&8_pQXYFkrJTO=A_1P)VFsEzI>hy5U|&(ke@Pz(HzBXN3_l6X%H=Y;!q1 z+JvEbxIpWAyJYC{SWnM7seC#!Km%E@V^kwWz0*Pr=~2k_!NnnXNF;p;=yx9YiG{Vr zEp$00$I``SdCO?wmr(-e^E&o?=p1s=aredxd6I9D-y{6+=a|KivCH(7zD2&#Z*_Gr z7H3y!5+p>H%O0%p-y_|Jn3|OxwwdZfXwTr$XM5Mm7+%$s^Ood2lWj#sG{ar%O z($tBqEE5*>FxiL)Ho?su;?EUW-vv+t4H`Z`H%2~GDX{V{n-YC8X&b39I9fOj=vb31U5+BE`ESLKQ z5c^XMAqU+&`Y849@qD@kiijnVy6XkE-kKGtOT}04TEy{fcM8|u`w~5PJe+l^q5a>vGs?}diKg0up3ck z22?=7L3W$_G5ifNs`v1ggl2`?Xk*Q(JDWZZmj~qJz>3p!Cp&u;NN8aLsG@MirEJYq zBfMexxTtlR6W!e1KfSQk*Z;dp3^#SzW_d9RKL+L=t7FsU73PD(Xs zKG3CP=Gt?XV`#l!73t|wPu5tVW6`8f{`QTQot+JHYa5hjd7@$;!xwqjezD-OPV>%W zZe*9kt>FW?Ng*d_YIgaqRJiK{R^jmQFc(3L0J9PfE)s~}{ZvRrz1@ABK2Ri2c?hOT zoDo-PbQnH>rOA$8McwD@(R*(^o85Avk|JdA-!i~>NtvJjo0j&~#gbIO#|)CC19EzB zmebc~ygCYS&$~C1G1t)eDzO{tc83jYYl19IKkg~7Z1lXJ_Kh5@?d?r-a^OKqLM@*@ zeF`KCzk9u?>Fn%8AZ^Yaw{UawczdYoLV|r>@4OBY|2RA`a=*xu>wR^jMTI&1^mf0Y z>K0>b$;aQu<$I|kE`DE#COl+OzdxQm?(^5+;&3aGs>8P<3>DA9+(O{texPy;1*BbP zMIO~p_~-avk_wN2VC3_vg%l#-p*v9kcLfz76ErzI;pMYUVNFN(<;xde-|a);;O+b4 znwJUA$ITKYxzUkEyO*OUC4m$nb6HiBkDx4Gm+GS#DurKHPgYN{tHtKUchyGR=FUsk z?fM=cQb)~zAOW>;A5n0+4$*C}=`VnTOHV(nPJs;o8U*|b(t3B>A08_@Eum+?eO1g` zP(7Q4y3PJ@g96!Jl+FN7&}M9`ac?{3fu*(>NjVS$Q3?{q;KOi zcZl&sh@y0v`F%KSQ5l*&a@8wQ6+hy{Hss ztF!a1y%r!HZSH3`M-bi#@v-ll2rjW`=HZu*S*7EtdKoNDMe?$mQWDl2M@2>A$+8Wn z$ET`gC?Hj8D)t0&+=#SM4=s--Wm^4qr%q*BGn3nDhdjP)MeZM#7QNugii$89hr`3u z<=Z)TP%qfG0(0H(g3U78<==HlYAur!x0o(S-;wOyWT*P-^# z?=nILeaJQ#6Ynr&*+5>|18eqJ+uN#D6q?OUU&f_N`>3hSo_{>GaZ4Iw@(F2vm8LSO zUp#5*yca+3GmUEv1xOXA11E9MzO;Ge^(pBDRIbdMOF5S7>)W{n?h|3pbNajT&V2CN z3NOZHxp2qBQ|!bi!$V13eaDw*p2?`O=;3m=S1LAUYwFW9;a-KwVuFMm zF4K3};xU>(HubUAf?7M`)5CueOe`idzP1!ge84Ft`eRgH20pxxz@o7nAITMVTszWW zNK?wHKwtt9^XVVB=}61TKm`EU%fZnRAt3x64oF;5{P{7&s3pY>jmR7Pc(OQAT@73z z$Vgk;TRozJdyF+d%<6etX6thOxx&8Avth-WXnK~lhAtjNv2=`WUsiG*Q=s{xfl7AG zXsG+zpdElzczYHMGBzQdkb`WHf%F+`Xqnp560_+xuEIi5j2s9KB~01UQAzkKpcv?*dL4%Z~cwH#k+Pu?dsXu*MJ4+9!<_cey z>>IS+e0B5Lso|}+bmqxg>l89Ng`A%5I`P1{>=U!l-*#nw{=pm7Z{i~P_}7WYE~6x! zL_Au&9^9yuB7*kHe*JZm0Q5@W_(E-oqE0*Eo@N;2l9Obe@xs zu?8Q?jv&1Sy;Rm2_n}K(nG*EqNC6Vdr.ukVSxc%QKuj;ECF8cRlx9nO-pFAYAu7I~z@9l93OO zd#5Gwt)i};J=c4)xMHzlrVg1H)w28fE08YMe)Y0}INOGOeXU zZTQ50+L%8>=)|fk2wVsdKq<>}!)`w5+$K$n>1U!^BjZy}TGJ{Mu&`SxI2Y49%W)yz z8BLQtey;kAm~W2{`^-(hNO+uv6!x;CPL}=81E0=bPUTqE8eTg$w-mLfNFY1e9_uco zFB_z1(7)sSF6%LRFQ)NPHhSZP`*kX5NW5V|uP>W}nL@8gi-Jq*d(gY73!F{oROF08 z>sAL*F`AO)N$=l}Q~P#F(EZf4vy0qOb0npMoS@zUko0*KYyqEi;=hp$-%3{p^d}B8;~sXvpF!xPcrYNzLh>OGv!-Q$hZJR=eSegU z6e3OeOPkTuCb9Xi_Np;2=m)0?$e08FMVBY6w{dxxJeVBmqp4{@g{_T&3RQ<88puaz zy}`OnT)>Q#4eJ9c+fH5k_R(y`o zBjMJz2^JRCL=J=R8!73I;^^Z2keEq4t@%vRQfn*tOheHKl7$b4asHIpNOf%NX;_@h%HE>#H}!mFpL*ZH zsj2Vs)D_ib=Jc4%I_+t_U%0usje6OY3#Z32`7;SLX?OlHpNFxCAhvWy2KMcq{>$fQ z0+5JG(o-D%hwO>A`L9=d!~(9K^OZh($F%_H#i@2)7tdpdGH^oYu8NA2=@Tp_ymH+1 zb6rvE_TAczG2HpsMP@Z}+wN#mU{9QE}(6MNO&qI;UxKp?7F$EK7WP znO=W+@a^0Q_pebh-s9F4HfmZ@@YJ8H#hM`6m)d)EE~26zedoDb*s1ta6PVF! z&_S`J5r7*#x_NZ$T;=UDb`bT^2P!tVu@ittHEJ_Lx~M6tnN%Z+Dk}B|AYv{qS9yjk zsl$G+v!n=DY7ru2&8Fk@ptST{8q5gLZ<~fzb!(qKaVZv^_9>eNE)FgLE-)AB)>-8% z)28L5+OnmmO_bQ%pPqimuF$GNj}SlWdcun=(0^K4SXh|)o0gPze?K6_$Hy<0L@t`2 zS5RL5>m+FXMt>ZYRM^jC_mWDVxv8#vqywpW&f@fpRJp8Ye${0m)XhU~h0g|I>pZ?^ z3VpY|#RWhD2J2JG>OO2V{Or`aS9`}bM`0+0ZrCz><;)s)H#cr~*Rqre-V5uADG8UG zgSo6xG*r}`FY}pWO|TLwR8*ELT5C)5Bsc`Tf-D}gy5-)ITNhi2yu3PJzT`W0zLxi! z00m3x#Fl~rdbW=y4|P;q9cUhaf3yI!Os?7&^^Q5P6VrVFeIo`-JKVSMrErV3Zuyj7{*>hrV zh}8aA4h}A^tje!(8$s{tUp4*#llyAer+L-iD`#ewX8sW46iyE%0yJT45%HL&bV5-m zmT_98NmwsIL5cN<;AC@El|te429E_>dR#8;ZHyT=2M16pv*xgA%VrdCZ~t+>bG>4{ zSq`LHuC1R-!U!aIrDq7>nwC!}4}UJ7VoNNZa%HZDGN)sOX_mG*EIacMGp4-R{c!X+ zF|4(#l`RG+EC8FRs!@NMwbvvj4S_rSggtj46@j<)_OYgfRm^9+lG9l(zo&7H1XU}a4G+=v)oVSvvz6y!j&Gox&nX`ER zE9cW${c7xX^QuMDQY8W^No-6MSjaf4gf6m9`yJ?St?eBE2QXfbD2O!*=~O_zO8X7+mYz@~rAMzVl1n zA_|~(bNyS)Pct-3;=>2GIk`B2g1tq9jt69o0vpM4JS$%{WbyR&@iDZfYxNTLSH%V@u4eJHY?-SnP~GeA@wfKcjY9gXBupZNCfu^!745 zZ#FG6J5IZ0Tu~({=4VEriObiMz|V=Rkd4{DyE%s2-xn24`GPozV(hr&3Q%J9qK@>I z7;!}@u`}#Bu;NH#{buFmW;jlBx_eII^{Q}CkfW3lW|5#fHyX73wlyH=AosJyCP2?X zmg&43xm%&hc$FcI-sfoOXJ2W3AiMjQFPI(&Dd88|!;VEWjX|@qDrb;R)p~z$;8$!F zR=v%u2cSM7$jbWD`|K5bk{|VOwMR+W=YM@+x&8Na;|>@69s>~$TLJ^u_Iv3^#R_iy31RzA%DSMF1723R^D3r&gN`{Q?Li2%+-u48vb)O$rK&O=OH5y4hJt%}%cZT{qf_$yps4n0s0Tn%B0)-QN56`U3X~=MT?MT$KRXFn85- zS;4Vg3*o}NAcFov)m_zff0Em+=nBP$``k7BugIwi#sC$lTZB!dX#$Y=z1ROci3 zXWWv5K{q$rmxTP&K-N-pGJ@p?UW?W)zHkGJl>ReN){!fEcD zp-g+-wiGAy>PpN9Au8=+BA3DJ{fUe>vU=q)w?0XN_x?w1XR{mKwi!=B8;=M?YdOcHW1M@PmH;4$!eas=dzTkv z0f?^~f<*2Vx@az^q09U?OPTSDp?dKxQr%XBFj?l@Xf~qec3Dhlh(LW~=KW#@pQ}l_ zY+)?P4AJ8w0*^jw=4e7xzXj3i95;v0B)gAUsAHghupvx!w{fBdzWyS$1^dMS={j@( zUhZ4*XtePCT^JyAGK~5$cZ8SwWS)luS!?qLl8S9B>tazT{7|8qcjD-}N^s?=VSpYe zJaeWW%4amJ?Je#BF~(cGPY`<3aFMQhgEchpt(6g-Opbc$b+vW<`zxlFq+uOa=$`}S z8O0I>g{A~9ez;_f7y%(+x=$q)^}*0cgP*vFnwEYH%nLP9;%Eg$4q`?8ritt4WDPyw z0sZ0TFe@pQD^^7f10pV<(D8=&5PYcxnp3ib)O6juJurg1f7}|B{iCXcxVF7}MU0)JW6ZobwUVqWk95VH?6hl5%E8f` zY4tM6nD3N-w2zsqXGE>UV0XRJKI!Ydrj*nm4&K+d)vkL*Vk0y~Jb-7W#LZV?f(A>- zOU}BD21@>uRn4_Euje&;(1E#kuZT!IkV8!URL9AU4^~YbH_8h((z%B8DOb!1dq2_*3@IgN zfpP5Lshjj`4I&2JkG7p56kZmRCCeHpsfm%H`;p5yTUotpfC1D%D+$abd7~wh>uX3P zWD@4T-e67#|BgTHjTz?;9v?U)t2`@Rf4Eg~&NX4ERwH^6 zZO>?ffc2eEh5B{qF-$`R;a8sho1wCid5+0Vv~YCS>OG8PhSTA$kS+H1WA-4hP$0Zf z24FXBeDADf7OI56U-1S>^lg}0S$*A6$aWwk0T3Q7qyL$$!7J+*%u51 zFSEaKNwe}HSMdlQs5M}pVS2-N_dxg1r5lows6L}FZ0ZRAM~&DN6V+C(9{iTagG*-M z!-0XuMjX3}>V{H7!WV9bn@gDzA94_I$dEHz9rIr?V|7^+!Y`+qy04uCdxn#zuFE8K z=ZII%H9FOlyO8Z4q?Ev`{+W7wV+*2tdc2i0R}KT;4~2w*L>00Rm2HXvHrvG7*rgzZ93D8RqJq-e5@!D^ zrgU(}^XMOXf$fJ8T1AMs7x(_^q$(tEG+#AKp&k3*2g0}V+_&B*HdlaQ!eDo}1= zk?jVyujq}u;2J|ERR;a9785LHK#aW9FO?)9kp^8a;;Xrl(-fmMEeHp7_BF@HsE z#;lNfZM&2!f3D2aM-Q`)0JFQvbJc{9MzeuqhbB4_UuT3>ec!BjN1)$p2t)hdjx((i zU(wSGCHQQai4R|VS+!`nUOwDH>2BuXO86X0j?uM-;PdC8X#u7BOO4q7H)%H4{8PyI z&6f&U(_j?Rh&XmpNW^R}m<7-B)k0vcCJKQ7Tk50Vm2^v6D#d?oiOUS3Q83pY&K8Pg zmlUkr@=IP3^p>58C zVfHH;fUXod0ga}XhiZjEV`ZUVMfrLD@=`=Kwh&(QKTb2y3c)1|i=P5)h&6GE?Hsq3 zEib;3zJU3onaBA;Pi5qV5Ica~iOREF-KWh-yBzok7^3P{%b+CDi zDTxNERDz~4b(AJN$z0>#q^|Oi@7cCmarih6R9#3`RL4ib#tha5UxTo$_#+A{= z_^AI*nh5N?OKNa117GZ>g9hsAAMc_CYWA%Wb*S7a9%t}>{d)+ol4*4o_)Z>2OPhEQ zf1E^7v)jjhMMtWi-IjO-2^^ik6RS^r$n(GBfW34*A66%bb%NPL@@u0BZ`+M%GrUP; z5o&d!z?1qv=L~eJzl@j1acboenkk;8+D^0mtzqIrS!+(oEaa7e^#6PPI^6=vjfQYK z{f1XHb?les%xJUrH__^5S@yw-oS555xBe%B-Zo$}$sOQcCX+{eHG7LgeD z?CuLhWkD1kJFZ-ojf;n-BdD8H-q;omO>kVuHqSth4iB7Y5ImFe7f%Lnf!{ z6Z-bg#lk;?q;VmpzjvMcO(LWz31^+*3%m2Uwe1_1H%)q4NuaElQyEgCTeAPhoB9-x z6x8@m$)ZB!iZDr1uU73~E5;qz2oSOIl}A3INX5AO9zY;5;qP+8IhXCY5{J!d(b>{b z9EM>Cf6^4|(Iv%;W6!e^m-@Nz5Tk}|6Fb$r-~#y;Zp+F{#&>l8eyO);jS(i?D8*8C z!uRya^l5SbssjeAj9d(q9OA44%cYs$2TJw;P0I04a!Y<9l=AbuC4&E%k(ECOzk)^M zfdDkcp?vM}3xfyXrSn+{75Gl`*u*0UPc_3vcb!}G3#Wf99Gg4$SGkFN0)dv|vSvZ} zQyhC^>QID(tb{9j1FVFsOI5Ft4B2q3(eQatg@GmMoN&{=F2n#77#fn!R$N$KCL$pL z@9Nc6JCw=O$4gsflvrKs5PI|9U584E!*-F4{96~{nW#PTfRq@RpY>pl_a?RPEWdmU z0p`k^3^xi)++=td0LYp24g$PrVuQVF6rKFJXqw@sS4B6VqKF=s=Xn$i@|Ax&t(dXJ z3N3$M`)yKBY4RyXNN(G~az1(aKZ9Q3Q$!byf9H$U8VxAWu&ALqm+XV3DeJTu3nxv< z8M$wuZhr7UURV{+WsSkTO;6O(gxFn;Uz|fkzi=3S#)^>))@PO?Ev~S)?8SF|UZSkf zu9_DM86?VNM#vy%WZZeZ@hI#K28`M*l0X?2tePtYq=#B3!(oJ{xIv@4OOBZ^kq`B* zTIyMc>xXnZ0dPB!NPfxg6Z=>8T1Oey4U3)vBz$r-t=Tmlv1e=EKb!oGB@JWOwfOm3 zm-yQpq+Qfq(S5~sere~+48;u5#7|D}yQ}=z09v7; zp^1iuTBZoICe`cu_R>9S;v_t&c3%SI+gYtZ%Ppe-3pfWAbNj?qY#iRG4~?eBbB~`@ zlx{x9%v`Wc9Pr;1=?_gryqArA^YzJVjsStk@rZI`n~v>rbLPXs&zD68Jx&+BH+%Y1 z4SfuuN%#zxMBDXvexo3_{9TdRDn~soExiFbdEhp1$+@rII*s!BHU*w=eP?*Ay!djU zN%Ak|fk1#T5{iu9<*32sbi=ZL<#dICbH$)rDhIo7s7a|GY(yv3Fc2TR{;JH8)hZAA zeaWHHy0^!4e6{b}5d+Bc7!egvbETn*4@XAmYqg1SarbUUa*$xy(dkVTV%y^}vq)Kf zoYhA(YyV1!QZ(-Kxb870kJ;xtSrtRgPA^7}bpG=UO9j%-_*F84YVnaqlzu{E+sCzA z%7=y(Ujc6f=o{T9!;iSD0fP9A7M3#3HFh}Xin1-mRBeUz$Ayzzwr;_2^_FWb7U{N+ zx7kmzIH|NamxCm&S|4yC6y=JxyH(fi^J%{$oG}r?8%I&g?I~co{H4_mxQ7%h>zv|qxNtZ-ME5LB3A;A0&0~CC zmbO(BX5K<%o7)K(R$a9}BSF&H-`@s^IY!MWY1&7J#+-2rTPqs}>(c|sAzVPXlZIV1 zA02tlXot+%xy+2;rszYF%&=ciA-*y_Hjkg$pD<(nEP^zC+X1&}M^Kl!#&335 zh_h4fw+k|?!54ifcmzat47Y8cJ7CP2h#!5Ok7~EnRN^;hUFty_V=c&ok#aLMrSsgJ z{@06p>jHOU4NLj}u8!Z_EGwg1c(A{{xq}hWJ`gt_SaanwoR?Mi&~>#4)9p@Aq(n*f zr2N=)tv9;F_<_&+TT&v%_}zQ7)F!N5d)%n(kw^?Pi6C8Ny782d8a?op@0(eERlw(@V{aj5_@c(ikxCx>#S^CyHojK|Cs!HYW{C@;GvE z6Tj|}?Zw?nC2XDh!kk^UVd=~2552-pXo$um=p9UG&wP+o7GWAr_<+yE8Tnc)AmY6YgTK7p3_=MSI+F5CD+?HI)4{ z_H(_f3L6$is{FzM8n>=ZD^x%#I=x=fL3<8|br|Xb(Znyg5?jg6ez(?fy~eruITdUi z42)W@xsmHKwNACFh1FT_tsU#a=^|y?I<^vM5jnlV!+LX#IX4%}@$h$A56pEh%h{!} zvak08$JKPgPyIETRr;Nt(*?G5a$X&sKkm)C+gnU(|DK^x*FV(1`Mw6zv&&}h z;A3`I$K;oKzw6#NWIq>$@o_*kE0@YR0Iki(Ww+@d$S;MmCbmBqzN#qncz-vU!;c2q z+$_h-^>^>Dqgv?dZfNbkrYjr8#ZlKUpFV;&z=ASf+=AwT#YwgCN%cCIrlDtGFen^N zVe$p^RKbfYjz)fN4|T;|ew&LCt&FaB&l_c>6&1EzfTRd{65#Z_o*qfOt|RvLMnT_` zlJ1A4-ZI|Nbw#+YSb?9v`Wr<}EY3aJRqQT1_XdBhlSs^t7$wEAj^j=<=xnW@`Chmw z*uWZnVT}NHMtsC{=6Y+&Wb;wo`{re?{o`AGMb8w6 zF#@PQX3cM!42!4DIkt1!-dxNph|eYc;7=lmmEkk(-Zn^N_Izm~$D%4*=ftXm_fy~f zN+fG4c5LTTC;1cdvbX-`Spl`lQ-3};B+_bjsmaKdIx@?;RwRmwDToH=r#$Xp?(JRE z!AW(=vM$#zX1P{+H6+luZ{sQ6`wLmch3~6uEAcPJF^3SwVcM^205!xDt0XIX*$lbX zVR6;Y#)C%CKlmw&aN_5h6{@weCIq_9B%VWf)KbEz8BJtoZ*Q##vvvXt;u2dS4&I9Q^+9` zuU_Nb?|5EzOG=ur!UL0=PM$cP`LrXO%@$o6HVsQK2xz-zmT)r{nK3{mq^EEY3k!3; z((r51`9aH7F`(az!o^+*m<$>(nS0;b)_jG0y<<&nE>ZT~`{m%X3N~m8dzZTTw$`Dk zq!F@_-wXJ%yNj(>ve*Lcp62`M^cLsc@Dj~+cC53czhCmzV)bK zGHShl-0#31n-JF2S5?vI-M_jlG5>^hRbu{uLFi__jg*!BzU7K|MoRqK+r}_W+H@NJ z__&jh5l{qldh3;!z+Q;BsMt5ObLf1&vh)+3Jx>J)?vpDP`^QOPN6QcXT~2utQgz^F zem(llO^Nq=ONPsqme(@$4tBJTM#X^_3lek%bSlp!Ldy_(G3&OQQTEoeT?Wub=2dWX z6pJE8vE-#qYh{>?0!gY0(ra_qy68+T(nPsyZEz6FrHFsOM;GJ9H6Fdcg&n}kMM8SEV<4VucJt4hzvCL4;y;J8`5HDu#L{rl>n z+XQZ8OPlL7`awk}{cf`|lbVQ)|J6gYhT?|;O+(y-s!#`+*{n=e? zjgFo`-0bY$pticAO-y&0#Pm;>_pH?nWUG}8_29H@vTzk=v;}<1^9=Ncme} z6MiM@ymzkex1kyn_Z1U!R-F;_Sf@fiH}c<(ixawZ(zTJ@oGyI2C4F9XG{k%dN9y-0 zVsuyNdgIRhWW{aoHq+S`m2AwarnPI^Lu z!%!0`A9`y1+b-h60L@CapjPYI@9wpn96&G+)KP#l7=nHUo=;8uE-s}X1YGa?@p1sg zMM*Att2eCRy?6eITzWb|%?pGZKQ!5(x%um8BLMe``o0~nyn0mYI5cWc5#zu;W9R@JV>4^Pd=P!N$g{(RHq7c zf-*Kr#sD=YA3<&8AJ&Cg&^w#J9Xp!~8{k8OfHjXx1jlhw#s&TRh;9qPlHPgy8`^}r z!1v>6ZEuEa+=RXQ&1C+bQkrlJ>Ig%_D(C=I!NtF}G4FSaCC@=3ef=&baPl7qak%*8 zQ!?&mZ02UlFk75CI;}vSiX<$76k*!G`+4zK29nsav{jd z!;H(mER^wjAPjKc0ywfak&$j>pmqg_MBzX^Cet@t=P} zivTI&&HMLKNXY5ekfkMHnKCdmX)`w50C=*jm0Cccy`b(0kk|0mUujiq`$1M(fP)Gn z=9hq21H{TMEX+%X_)mRET{zwB+|M@i`E|cgL9^m&aJZ*Xik+R^#MCqvo5bGH@vcpn zMk)FE;q9gWcyd^cYN7EHa17zW=XJiO@ocH%4oFXd(2wg3^l0Aes0%oEF^*GkF3Auh}T6qs@)BqUcn)S4=3M8>z4xgxPvD{+risSyVXP>o>%w~|Vd$v64WD2{s#zJz-d3c= zqyZN!NGl}Xo)b{xA*vaUC?+F=eEY4>o+3R;6x;&=@OwWVm`o(ByIcgFo}S9_y_|Mr z`al36RSNXsco9KFo^S6|2_9<>Cf>G_Cd$>@j&YkgfKi6zr`yDdH5#?r+Ik3Iniv}~ z9eSX)u5P1s>JQ>xLcv60l*gZAdKxrn<(7oOzkSn_<$^8JA;{QgpI|KUG4mpoR!71x zC0rZ3BwON&Lwvrq8j!rmabo8xl+UaaC_ z6dswkSOGs#y6qVLdfAjetF^gx?%K+1)pXVMSYl$5nKT>@dPCuCnyNHr`WWnGXf5$J zF~Py?orx$H^y%fnw_Iy{T{HzIHPgk7{uox3z;Mu3a<8_p_$3c8V8coUqopBy4wYp? zD`#{&v2FCShnDN?109~8b*!!CFNjcrZIysB^yOv8_(1$Na|9!jAwnqjWwI#mg>(yu zo{4nsg~>^HKnjyVr_CJZdK$!gT{Jt;0ng`i3&S+jAWJxnniK535&2k>0h27+o>%cF zCNC63E7+(w;C!yZ`pMkXG{g-BB#xPgoLxayo^bwRp~rt4!G{sEr7mP zgRuhu7A@~;I{r2-RM?`)sJmgjzX5})mSQrJy|Lbl_PFgD_{VzBnwrv9^QmXfF2WCN zPvUR)UY(Uce*73NmI8jJq$q4m9ugB+omqj{KUP-O7*y0xUu9)wA|t{)PvmUAV;eK2 zf}Ef>*Q#v2^;I*@B@o$UB0T|;;y?>yNgd<&J`NuG`V|D2VG{C|FPyk}5(2aujkbGX zMk*z@O9#58g{9M`&(D+m7RG!4dM3|a3X{jYwDC5p2qlWNtP~wxOQp*j1OT#nzfo4W z&_Rc))!Gkr^TI_hTIJ-W*fypmNMS}5O@=)+m9cY7SP@32q7dCbmQ+j%*e#~z=JxK7 zBpqOl8rgSmc%MXT(I*jyoR?=949GQ(KJINib8&JGofL^JP*?1#PIN3zP5n7w*X@wk zHd|9vT-~~MQ=}+>)i_>#xMnsUHwt>Yy32H^t_Cr1gd2YkCw%T$09x%Tay3C5Qwk)D9tfna%B?o)2LG6jH_iy77ECW3~b@|hlOhLo#QXf z5k8HikQ}lEA3E8UeI-SU6JzF5X zQxR0_`16WsP_ICQk5_nDukgx5UFQ5*A6Z0ebPgE<*iKk&-qMN(1Jd8y&MsH{gBHZw z2+w<@4R1p;vF3;Ueo_MQ)#8xrStyy8{448wzLfUl;TE$N;k1G~H5FR7LyvnKHJAH5 zskxTBQ(o6YkP~RRpVzy!(E`Gje{Bp)e+IKqSmrMzeC>aCsnW-B*rGxr0N`_mVa;;( ztbC2h7?1OFZ`BS3DH+*ChfAA7QGn}$fT$Y`b7_==tNmr;s|M6Ar`~IrjH2}d z>+m&hL6ib>5C$%;hUx|YC{9neFSh1A*fU4%j3m;ox0%>zSt6y5&701G5c$*wYa?SQ z-vH#XI=}D@E58M%OC}}wBPj!RX z!7*NscE^EnZBM?ptcx`tP;f)CZ~b(CHl3ojOt^HvhBAI_Z9pu26tQy zbMpTlip%}_EFEYPoXPa5YAQV?JtBr+yx;KD;yY07|D}_$<8e{( zvK8%dcMe))k4vexR}Q_1w7KM79wN(yDz@TdgQ{m|tF8Wa|H+2~U`epei#n#>EEDrw_L!kF{2AcTc0oJRy% zvgR1P%8?gJ7ks;t?$|50cUirCAIdd`6`wi`8k~{93g~YUD(%r=U=$;o z$Nj|wcz+3rL6CM70q?UTXQN$Qo{^Ra3xJRVP`Kw8M5Nez5P{#3Q!|4Dd3XrzE01ty zrPg6RSB$|rSGNMHa$GYsFmfgl5Tq*>NbAFju-9gOckYkph4Z*0q-`}wxHdWOyFSR^ zaKi;w!?({kzezx)r*His%hICHp)AhJBWGj71kBCNAp|fRsT34G%cDCT+HQgP)7shD zH?KcNW=nm&0nnY5AgHpgzV7Y$R{>}luUF$9gsL`Sut1E&D2uA4WhuC0i^^t9v-=;7 zwGsGG=|A+aHiGOEQWW?^FhjGr&$tR3#Joa8 zG7MCzg^Z7npPrV=*V}`XM3o8+d2Pj(vyzIp`@Vf${A5RTY=m&S$)DAeKSSU; zYt8z2b$y$*yerwMBGCfkREY-q329`Yf?2og4BA?wGzZQ;Y0q230EBq2-W@5S^e^73 z6;C`u^v3m5Ebp;2$2jeT_5~OpJ{hE#l#0B63$m?RyPCWd$?qhGvD6-*t<_;czrDTD-^B8|fdIgkK|K8jxu}7zSiQJEYLOPnN zE(_@{0Dv}EAGaHty*)lI`=ui%SF6!!%k`r+2jsD^-sXc`hl?R<0`|%*`V*=Tv6zUeLP}XH~irlFOv?=_eine<^<0T&80m5bKCmw0(LE}W%ji1-@a9BZvM7W zDZjhErK6;6_2fLrg1czoKJZOGEXZ(mJdRkBzeA!6wQ5E5n_%tuc3Wqa#w*^<5TjVxW6lcaSK7f12WZMJ%-}$yg8xkC z6FMbt4OHI8me1njlohhS-acLVyeZjbN!vUnE_+VR^G^+QsQeaukCEO;}qE^Ejo~3MW6`!W@&ITj@RE z&&6VBEVi!>PH7K@Q{uL`#%EFcI(r_Nyu9Kn_Z+pOjG*0J8;|0TYJ}%H3|UVn=3h7J z5lZ$rnB_chmL?*N=ar8GkbRn*X~plyV$5*|#eWhal*s=bjWSxBt>dOYbfEj-{>7RN$C}5T=Ub1ljm1k;Qx(*A*#7rgPixH$Z%_PS!`OD6M=yDR z$g8(cv_^j?U)N`apOMXZd6o+3tfpPfC24&W7mdQ<%&shSUC7wTt;UPLD#2*Edsb0X zvTT^t|5l+Be$k>@GGq1l8VpWia8px>+%F+)Y#2(S$u1(<8QEZg?{K{U(DoOFTMq!F zn;fde%3ckRs{P@RXQWVo&5HvpGy<=i#VW6k*ZsYt+s&77u+iP|Xl&$s*C!f|>FVw- zDOzH+@|tgZpAF*=*O$qedi!VENIW|4Lr7@T1#(A+x=ns_bG>+A@KXwa}&VGL9*Vnw$tK-XK!yG z7YE|$1AQKDhZ9QL^OS=^YJ}ENLAadw1R=lrb!pYd*DfJ1Ti zrmMYJPp)qTQJUb#isux+4P}|dN8MjIC^&i_H>CC+8@$VND@s%Z3LA=Be-uo(yQj(7 z>$00*K2416kvp~e1ttq%Fmxvj8|wrNeX+ieyn*nQyHn1fo@Pc0aHDzUm}2Nud~i_d zd5m+aJ>r?uvP++$^z{oQcY$j$YpkVdp7<~)@|RLKeMQRr?CnL%uDtbOeM`rs-)(R~ zNLhLA#HN+i^&8JwjZqSuD(1|B`y2gC!VJ{XvW%d=EI$<9anWX8}d^kM*t^>`Eaec&_e^F0e z_Qz&J7sfYPU~YsN5<+j}h^xq%JJ9(cuj;yCA1-U1WbH>-3C0iEc!M~UTONW8MKrwK zj9EOOmO|o3mG(j?E<-oX7CU$j0l%+v{_I30N>71sRv1xH{AbqKAU$ zJDyhYki}KY+G6d+*~MHXDlV<%)-g6So1Cvbz{BHX!2QvC0H*IBukHNYIw}YvmQP#R zh_#2wyS@DbS<3$W@4R7x1UnXNq-XB^mJ*=q83 zbi|z<+8Fl3BNdM$l4@FS2x|>*wpmTPqDmbSz)$m^!GU*Xc~aJ3=rvGO^v&$($v_H_ z5TpD!y}DW|Uy?_?-RVmlA{pqXj~!%`8aYOfixsuQ5v5ESj{sBfjS_^UD-?aBR+h~3 zTUVw;M+o%W;Ry6IP>(WD4>D3u9GhSW35~MPKX(lt8XH4=Z$URe#=yYO#FX6@J~lM= z2JR-XCV>JO@jl-Bf8cT+xwIr|rUn56_DXHtjo(v+1d$OV*E}QDwKL`p4$O=Zl1q0a z~?2sCy)-m3q2wNA^RXi^p{6TJCW!ytE?q@V~JWcun|kXaHob^WQJ z`WNskX%m&;cQ(Dm!@;PI!DBNc((VGb)Um6Oo@?@c9cO-9y8d2)@aAZ^eS}Q1pFTi5 zE9E8^iv{&lajBFw#9TByoV#YxXB<8PQQ;418jVf6+7vB{#D|4dgK|f2msr)G!G0)| zgB0w_c#h+bl#hE>2yhzA^C(3nAbdl63I~J8>-f?3i2!Zb3f@K*TDba#VD~bF! ztA-tv4t{(+0@A;absTs_mcQd-JiTUbF6vN|!-2kad@XL5l~TmY$tioxEjjUL+6%G~ zD-0j2!4=<5wz%`~URNY+tr3gAv%7o$>lWN(_ zAq8sXp|8UG<7vz8X68ZqFHp}f&(9i)`q#qv19b`tYvw1ky2G%70uwOil-*J%)-jJT;-=Q+o4V0Y z70%IZ#3}%3C_1X&8y1mwTKSONkH0k8yCfWS$R z+=~czK_DUNIBrr=iDC_9NUu3crkKCl3gyj} zf;|E6#G~oZbnA0#!^U$om_9m{4;W`CnY*<_dJ=0&aVw6?no`Zb?xuXlMSQz>)@f^v zA0M4rZ%|qpe2ZTK0DZpUKTy)?L*7H?f6omG21yU$?Pttvv?}`~B#-OF5rIwzA>9?F zD>hWI&+%&75hf(XZ{M)d2w=wF zk1l<&{^p1eL}iXHwt8U!qnWI{oK#g$meUJF1RbbnmUbyq?~$RcamQtLE>i`H^mglp ztp3<;@%)Zth@CfTgJL*_*JT$C#u}-b9L}M?lWb`$R#$ZzsBp!O><3@c=H}*!3%E>F z$tX5YXIfeot1j|z00lzIK7~7DCw^;bPR3c5aghl(vs?>19mf3$IYA`ft_tfwuxa(k zdpHN_oWh7mdno9mtwIlv;)`Qf^-obdo-M7`EcU6>OEF&J-Dg|p^X2wC1pbQ;0s#Pu z+iog~P;QF~^WA1*l7y_}sS4)0|2I7}VmcFhw+Ddnlig*ccUGWqn=IB5jgwHDj#h#O zg^Y>1>#HpP!GRfrB+T_DyiLl!_on^Mj}ban<)9n+=uj~N@#e8C^)8&$)C6h_HkiKO z3pC3Jiv~Hj#5eA|1>aVgXyW%T(EwtBTs-K~Sfvm*Ym3nt)63LBg*k2laD_~1x`gwT zK$`J9)6CCtQV2I>|snTu$3ob%CPD_leoB zp!GoXBilw_s#^1?tE%;!xXI_}L9I5qg=Y8*YuiEzG$WhW~=6|~m^BIjSgS&kp2yXi8#U!jtue^$#WOz#6avSJe+E@1aAyvnCr<;t-dj|ORax2s9;NTY7lCq+rKYzMr>!RwJ8UFswjv|BiyD{Vl?C$Bw z$;#>)#9Px^vtEHlGH|FVm#09;ZV7IwE?b4w!Ua$UKi1SR2Z;5`OxZ!!3t8vv+wT;NuQgH8xVNqEZtJXv)jN|%?L0rf z6jPTV!L~ItAO{Vttc|yBp6$$(YY4AZRU`Nj!$SsuCJhdpe=l6O+IrOg5b7#o#m!U5 zQYcQNJs!9g^Ur#4wyu@^VPqq4&IE-JC{(6UG>c0iBZ@AX!Hyg0*`Ssck^U6X+K#}8 zG%^WELpxGQJ75Q12#<1EN|Qu^MfjX1da47o;so2`pm*)W~ImfgCDVkOMrenHO7b?bzU`; zaLxv`RcAHQjBzHvl)tJyet}AfhMM!!@2knSG{s3d?^>s-+b)r$rfR5hI zj`WCb(JYCwIRg0Ej(2AJj0}Jwamd&OE<<&dA|(Vr0W2fcUhd7k=@O1N^hmKBC2Ce& zndtJM^}S%)pz?X=Ogg<45_msHT^jQPc1j^~oh-d7f^wIYlg~k&jBAEBh4WI;{-1V? z2@cc}S&tf(JXf{sS5J5Me{f>po4CV9#*`>gw?JH-e6i5#iOKnA+5hhL8M$FdHKzZs*-^{VN1P<$ zEAg{|LCq=9wf*loey)r*Tb)a>PMz(Q)b0kO#W^kqYd8M^Z03 za3(9_j)F}f{_l-JA2oXW?>tiHl1s?aLTjOm$fHBgL{q96Rt#RN?2cPG^$ZW%Dc@)@ z{_hE=KfG9zqsZh2zmp(?_#*O&R2Vy7v>ztMNJeEGp0p< zVLu`KzaQv*Qr{})%P64Oj^pO5^$B_u!d}t3k&1R4!LJj1I`A38{D>Km$ikEgij-?; zEfNArahk}k#mFuMG3kH&cbBF9b7Fz`4E!1N)u`ZA@YcItcq1RGc4Q$YT(*1!|O`<@KQBVx%NSVj*|y#XdxN zI4`q0Ls^& zgmXCWiTvN{HFfn{fA~3463vVRE;Bq*kAxV#K5Jhs@ldzf6viW7TA?>Q_U8+4P=ARN z_5bz=msCSK($ASGUlCGBebW#vLu|%5g(^l{O$RoaY!J92rWbmFd2`5ONOOEUBsvmBMd7508Kn-LWSYnO!hz!;YL`vX#@jLLMlOGU*m=t8@JL(V4IeZ z1f`EXZ%5cU7C?{2MW1Y-(|2%t=* z39qM|e??@@D@76Fh=Ns<`94rxTB?@a2M8H+eco$GP2#rMI>WGgXTJNNi)0`Fw*Z_* zf%?+IPGLYPJSj~zOZaMe@Cek-NBjW@k^r&*Tl`8|8qwh+Q$c&dT0(`vjQECy5t}cp zHc5UBW?N+;jrj8D^8Z^<>!=v1H7;_ph6Fpv&R;gkX{(>ypbaHI1fWDE9K?XulOXlw ziY*U+nFz8$VLHR;;Oy2$eB#(QWQ2CrQxRSCKlg}dv5SeE+~V|+l-V9K8@}2)wYPM8 zwxcLI6*`@ag>bxzc8ensB{A%=S z#^uxpMC1)|P{#UHKYb}+8~;JeCmS+YoknF5NFK1mL35pxVMX8gU3|D~wiT^y_3OKH zE%grBt@Fx2fB)q`=bkOPW4f^Hz-Z9Iz&9Kr)|Q2nwkRwT6GA$1-MT&r*6QEJt>eak z1#UlL)7{xYL1;aQQMcXfQ=HkWM$EAtc#;4sRNE+{F@5{_8%h$|k;x7u1F}6JH|Nil z?Cc>P$t3%sHYfA%Xl<+p47;$=p->hWr&i6$+yyL*Bv4;b-pbDbv*@N#(nLcsBOa5CTb? zbqytK87x_HF>q^_tFOW=66OGu%?mty^uw)l?~#)xGBOHjsR>zmNyRVsJG&fT?>XY) zMxgdesZ;~41QFsir8tu<;zr<*?E6QyR8Gf^4tswS^q7AvX|Jby*fL(Rp0bu_ENavm z3k~5~f8XfJ*-)5^4nV%VKt6jx2$alWU?yQ=CI#KT3ynJV&}?kPd0Z`LsR>VV7%5kw z#0cy919iwQ=&#~IO=i)ZjK0dzV#$2?c3kH>fbk z8z>}rKLGf2XbyI!gZ42YD`p5GBGmbu>AV#0x4pVfCcg7MTlfiD@d8slkt{8DheeG( z>@!VLZW^`g2;Hw3sph4yq=xK$-obIrccTsJjo+FrFg`RCYnvuuTQk|rtn35)WkLnkZod_*cGJ2|n|6i8Cd~T4K#7frTB^gokiLvI zL47h=twwruQj+6$1C}eJJ&ezY6e#%Ul-ij^;Ct%RxYD}ItG)ueMey*Kw|}l;qpHN# zUC%U}WVE&W>WZhYN=O9TU_TuCc;Ir9L5_rwo<+Z#mz4G2W@W9LV`c3Ue5D;hl{e&+ zMHh|Qk^W&fej^-WnrI&*Hee<$R}x^qqOe!5v)-J-J-wQGayXvb%b{S~f@z(#`JqO! z)It9AJIPM=Ae~n2p)u=B@!ZDwVwJ<{V%2nf$Lra{Rf#GHeooU8()Q9e5(PV1r=6;< zf`Wp}e|B)|mS<+Jjq5u5Mw$|o=b#K8!A9A>n(QmefIZ*34sKm@Zt1G&?#qj(d#_-@ z*9y3T>n_~d_vQ)9T@CGpDn-qk?{!Ox%ZtjwiLebAu^pT9LoWFqVEXEuoPckNnl5F< z&CSi_s=qVmmXFfve2x_M>Lh)9-tznNXKQN#pEu3VKF!*xT{7wyYlaIWgM+1|r3gq! zr&Jbf>>ZN>D~8hJ`hO-`xMg*HCQ^$X$J=ZbEvx)WJd!3jp;oC-$HTw?YZ-!H_X#vjpr#Hk3fhd= z$f|F7_L+Nj+|pM4l@*pCil{tznZDOSk!GIcw9#<%oIAFO&dXo_VwdgtT}Y_AYE}tw z-F)pU<9~U6K-?%-2L;uIFFxRL-IR+Wwzajb*s+X`th5@H1h!h(%PcX-9||U>%y-e$&&JH(MS;0^rXG zgzz%4{(>%X2(6#Rw9brzxm>%%`Z|SM8*zG#ugwF5G1uIpjm*xn2yVh5g_upUmA+VO zzj@1-mzS?G1TEO!oi^p>>W^0-il_dt+|YI4Jh~9y z*=9U8Bge<-dimSA?%ij79I-Vno)!4T(} zd;ZZBPKWu?&GLszm8bYQtjsf{sPE<=>^n9iMvy%#es|YwCZE!Ju}}J8eYr|zFciC~ zh>+7u4LS%S5fuVE1|o$v$OLmPpJVN1~A^c*Phj?G#L)qbJ@( zBJH?kfbD5i9EZm+D!V|Vi`O161M$e`>&!PpJcfp0+N9vo899^Ir;d9#WZZZrQXEUj zgZP;OW*wiMF*k>U#rpm&GV!%lt>90eyCYBMkEC@tI4Ft0?A%%6wDaNFqdMEtO(xYa2Li_Mzr&`(1gt zm!xCs;N(vzq%The1l+x=f3BZb9|Ju(2YW|lWucOHVf9)>voMj74!R{p22Vn}(TPfe zBvmv55vRN3Xz~<@rC;;e>;#tsMzc4pvWN7H!{v6KF_u?qWLdUsKu3Y|jxAM1<<6t3hZA`#> z-JbKHS||$z`l-gL%e`UkcHB_=5M&4$0aoYkbX?xF%4PGp`T6xlJ1S=n+Z&m|D|(Y= z9y&sR-k-L6G4Xr$!}pv`s`ECBrjsWJW2q;*ANR*nnXQ&*KQwXxrCN25p#y{`)?Bjk z_Gf<;qC!LYY%leMz5Ig<-~cTS5Iah*HJHz}KD(&&P0eRA-1)6rw`HWGf=?D=j;1*81y27D6U<(V7SO=vHh{L7$69h7oSiYQ3E5I=S3zI_Yv-wW110mOaknM zA+{L0Xbii8T`DPQ{1S*WRqv!>&vZDyr6zY5X^8`s+tL3&0R=WYE8;z&{q1Rzn?$%d;(XKmy zoXE$Xgq!=?alddXCnfbgxAQ3oh;S74XRDRV1jTp7#Klu*FMEg9JnrY_=8dbnZ0!8r52nanx$*`n?gUj4pP-Z+{x9o6%ecIj3 zZ!*CSmj=UPrt@_B;NoC=`y&bp3J#9jb8qMj4i1Q@plN@mrY_#-eTR0)W7Gb!vBYdr zdt?P%4wqEOq;YKR4&9{DaAtbmkN}ozcQZvdGk%@OxGbPpF${ei-~=(xD^!zY9wuc2Y%4&>+k13KJmhl9@u{I1QtX?THSnlOZgfyw(s z5sk}xJX>`09uHK*)x3CSr?=kpx?eZojy_w#BsgizTB>v#N!;D)K|~D4ADe+WS#w?} zDJ(56EUg|UbFp72X;g`EAU^=DShM|mpvubTxpniZ5BdG@{@6-bSa@3-55=0@K%%a=hx;sE?w=pFlf{o9F&78N>0w>n#Tq24b$dQ zX(^d_l+Iu~HWs(uHUZkB_-!k#CIdr-uP*=`qAgQ^ld-eEp`d&yA+I~_$@9Z+owg4| zd_2~xjq1!Mi&nqlC}yJZd9RZA-*A9pkQ8XW>`3yI-H>!pC{K2g#`7!co0_6x%~3EB z2pg_Y0ks0(r?T6~YHPp5$|XW}Hs78!Sgt)TmsD^#7@pLN+($$D(lj-NhQ0%AKhx0g z(Vvh0Vz!+i^3}J$wlAuN^7WL8Yy6qs=KT@``pgeFv46W_qfI7KzPL{Q&bV5#urNQt z!wo_vl10|_hkO6HFv5TbqnL!zvhf|T{W}=*@`S8aQi|~ILk7Q%tfOOXRntRc!@Bq7 zYhR4l6Qn@jTv#;!%gsXv-@)0Ll@oNn6$g`1fG67H0{`2e`IWbHJ}XK4>Lb6N>Ipt% zWaLavgU4I!CaOtkqVLj0y8-&Xi~%tHr@kzEf`;z~s*et;!=*-Wg1)fe#RVaKSHHXY z?b{`-&(B*uLG4$= z9sZC|KlDG~^W5z0kAX;qNE}xCKieL`6y#v6eRk z;y^&Wa;lcM-uC+xj-M|!ozzSuVPRs(WjGF;R6MVYt$}E_H3u%To!%iukNcNq%cB7( z*j~>T=d-)B76gPV3b~Z$*|=!U14Snmg!92B&FDb?Hg~GnBuyl18{T9&DcMFD$JK3; za7r$^pEG?elXqv_8g_O=gW~zQ_);=!?OwrWIiMJ3LP7$7g2ye80wOz+Uv~1a40gUl z=8er4UK@uN`zxEct4D>i;4zP*FAQ2`SV+^lk&cRzp+>- zYf({WwLF~k>-L*S^0E7~-A5k+*T|i8zJGrHSsTkborvA$s|+`v&-yCPOu*Hs501hoU^c!XaFOQmxH}!5D}ngkv~<}ePPDg&HJ;2jwyX6xj1LB zfQf-2CK4c*!txH-AOFSbZWy-ambQvS*>SoA2PhV-FAW!*``wV!r0`^i;K*R3_*&-q@=ftHe^Sxwd}xa0|SE&%?>^W_Jv~Jf2hX#tab=s zczAgIX;au=umcp#0||c@^A-G_H2J;wxgU4A=0iTdF5nNb7IppXT zEV}G1-#|nx4mZ$kG`4sI1P31M?oJZ4R#)4DGUgzah8)D+{fu)U=KzIoTfb_i^d``# zCVD;(U2gRRBYlPT&CV9`U!U~n^SA)jNPONRQ``9+4{Z$$$asswBSI|qG*1_+W?3yd zuJ7(%L3?RxH067-bW>DBgRrm5?w+)~awW3?OQ>AUoH^9w`mDLnxk}Xj!hVV~Q8NXq z%F5}S9&E+B8*B!<_4i+;yj@3FZ5$X?&_gBQ&89dVmZ2EH27KBmhvqjm{Ez61+>NR_ zf?J_CqdNsm_Sc9{0XMizro=WZS7Pth2d{Z1u7yTyQs_zGg$xu8%)AiOy;F=d-Tv=(9a%%{hF_Cb}>wK6P9)ypG+t zJcS<6TIEuhtghmZ0!8~2vhSyjGT%$$*E=pTWgN&9ZLp1*?rLhJu)o=uIg(9Wi>;el zx{+#!QAKMVx+N$VmlOlUEG(lkiFyb7=du@x2?=>7hJ|w}4eOq3V{=v;6&==AZm>u7*(1=ze*eVZ5=u~^NkBJ!>^1i!1VcJ5&l z)~iZ;Vg?Zb0TwAFOO=|8v>QgpmSqaHq8$FI9P<}92oF;ib^kIa2P1p9_ooZZ#uXKP-gfa z-*i0a$jF%K>CeqY`)cXoER!3(ax$)nN3!(--BbG4HLSq?c})`UCyIy0}hn2V0i z3QSp|9xt5XmOV(~QT|+Rkum4XK?QxqiSgV$1Vcmr^b0)j`sXT~g{lvnY8EycYHhS% z-PJB%%$kplFtiQYt%7L;|C8s#M2im&vpb8IBN$3@#KEBa*c=oQ5dnt$cvUSJUvMu% zLqm&*i(_D5p^>haik&RxeG1Lb&#y6=$dHsvayjw2i=LR|bNB~8lN4fJs)i^R*V6X#%u~G)_%*^?zA?bVLNSuTC2zgLZT2yy+oKc1Q?3?z{8Ln6gwx z({Ea0VxIv+-BvK`!uuRq@}*4t_rbZ~hn*UxJZ5E1!xnk|phUB**5b51sD z#K3WGkw3g!^2Xz_sf2wxH;3A05O0~l;TCeC-)`bgZTwJNqUQ|+0Ls9-NKC^I#X@YJHLOcWOGEpfhGY4> z?j=23A4K8sn$I0~US3{;1qGCV^IhH~pBG5A`mIEbh6am;jv1EiN)T)Zm!M#XEHQNs zdh!pz2%qNbO`P215Ez5CG!!HO@73zPA_~HS$9B&L1VHVzPc@q7=I1kdXLEj|gholu zcXjhLl$a2@J=7MB0h6&5@cg|scjw0VF(!+7=@oGdNf^Ks5BHR>q4b#=_=FTy@O2l= z(3Wprkiq1Cb{2xzZ1G!=J})mX*giuhZja4>#wfCc!fl)(574;PnUsLq+on3__i)3G@VoN!sQ;DW&r#09i#*VL2SdY3UR zjnRD}(ce3^kxQ414v&^1J}0%rRJ<{I?-;51-nT*2M4MO3ez-|3wPb=Mn*@8Jq@|)@%fr-=&Y7QBk*^knsUuh%Zy1mA-lJ&Bahrfhl<<*Yff0(s*_-l8XTP=L|i9#HABQ8E3eIKRH%lqE4C zsbnstl5qxsfXi@jl0-O-$M}vIwzt=Eqe`)0E`!^}`kN;{fP{Vxc4yK-k zPY_*TU!I8Q-+`(2g=4<=!S>OXingYSo1`(|pi6+8Z9RE(_zfsSK}G!o63npB(KEPx zwvu=~_@7OOdXV{krlnclYeo=0ewJ(Ymz6J?1(Cr?I(m-#@SrvulmS>h20CEH22tb{ zWXj^iZ10YnE)1F_r;*h5htVda0X;z03->#+Y9u-Xx{&WyUS1O4Hh(Xg3ZjHiejvoo z+2jb)ha2d-6eAm{qV1{1A7RW-t(TM)Aw?jKT!JJK4*qn-Gr=frX*n7ciRYF<#O4@M z5q%$_4w}O+R^yw)p}5b~=#t%?*UDrHbanU@_n@PGp*O$V`(^U324TOgS`33pX1AE{o zT;H8EMPvK3$bja=2OHiRe8BVN*|f3}Omnyl!I=RiJ^ec*coescbF+MHS&(RM>@Rp_ zna~uG(w|0Sx$6e1qOy9wzB)?k>FHr9XZZdFlM?YJC^-~+9B9bIX z_StFaRrq%L3bYI$x}P6>LbBk}n)#hx5siGS3TC90m5dAwazC(UsmaIUVSof93W=44 zya7S*FU>F}GFWMtNFf12pVKL@31Bjwwgw)vu}`<6lYAcLblP84-$tx7t=QJ~;05*5>r1GdJWYbcZO%jRW zug*uTNa6jIXr;OQ-tE(R=DEo$eC0@xe92tU;Y~?#2e$);uGbt5<0W{Aj<_o3FcQ+G zMUAIq>S!__FAr^mKDI7V=FfGKY2(qV(l~JIWfM>HcHh8kop8He5)+G{i`Gd}>Be6V zj1uPVfLoe{CBtnW$!Tm60;=9>#rw5e6sSCl6MlC#LjjghOAZIQxyR+73RU$)jg-xn zl#6PoemP(PdRGBoZqDm0=6ELHIHu%d^%ud;J@`cxN74558VrX_08(M=pXkIGKz(~h zq@T*nB+k*DWxth*f+*fgO^in1pg0^|?FAz4H;_eHSnTRmI8gjdWaLzKM3ttZ&E~NG z`z=u)TK>1$0VxU8<2~2`J%C9hFQsJ4m!v5*J=&U6IN*kqBDPQIS8V(|J@$hj*W%8(3P%G1)W}oN^6HF6&zqn)zj$aOTeZ*sY4kKZ-qMiY+!KNJ-`RaOrayq_(} z6Y2XsSeV=4@pcY_MxEPLUas~*d!E{|ogZzGU~c6=%UY+~;eAvf7oNSy z&eT0_^pigHj`irac39u+K{;w?SCvA;ZBU<*oNv5-);~CyTV@a1cYAv~s!h(vhaZ4P zM3njZHEYGw%Brj(DXF8w2UIR`Q|F94HqvdKeLN|iGnYd~27#bdI+}`#JKNb`a3jI5 zo**O~Hb+xMgBcKD|0~h?dlzJ!`8A{**-6>x%)W8u-}R^5svlrR(YLbV(SKvcZq&AN z?rsN_eU`>wTX(|Bj`x9p^SPzvLh>uy8RC=o)9q%LaE25U*eX3Faj?e|5ioOU}$> z_kMQ0exSR!OkPsv(os}QLZoYBIAli!losaahW!RuS<50JKzT4USmgYLUJ#fM5Yw_q zm(4%l%-n(dRq5_y513(J`@D>Vfvk?J5}VhTiKLe?yTWpwB+%0~JCaZI^hApFf@$&MF>kj7slFgA6$@$)ANAtrciy1Js~^`p=Fqx0A@L&s}M zKR=+SB#8i&sz0q^fjRZCMTKrzA*1-tB30Z4v@bH>vnkln)-C;e<#XXUtgcM zx3@C|qyR{B?fSjZZg8-dHiE}y87owuBbGNiJMVeF76hJGt9W!Y(Q&CDW}KfBxCs3& zKI;3stKG|A%=rT%;unu@_aJi!6#K#B`eYX{c2Cs9q>Xk=LsPNo<}IS|{dWfu0mIs_ z;AK%yPfsypw>&jN@!GnBS3kxq#Y*f4(cMUQQz!Ctelf6X6 z#L)O(QT7z2ZPk7r5V`{nJI}%TAvQMl|JC*tUR8bD_lJ-cK}u;6M7ld9q@+Z;1Ze~= zUD6;TsUqE-(s8*oBGMq;-Q8Wk?R)R@-XHKCKe&T&$IZju=j^@KTyxHq{NnEV@Wq|F zws!W&djro4swwYD6?OIK?;hD8M(m!#hSoJt(j{d;#!k_@v}^{zcYKPO)LU0)`;)^% zEUvY2r-Kjhx-IR6o?_Qk1!4%4?S|V%0&hWJW#J{+nk^sO9!6=yz8>pSw&>8e0sv`H zFv7kpdTOhDk?}i>(v*E&)APg`h`3mEt8*w`h`w7qA7$mxuRp%VW=j@~eygwUwfoZ@ z+CMz@hl#x!{JQle0bXc|MvwERkOxTITmf8|l$7K;>t)+L%JR2IdzyxoMzK!vxA^#I z=iQ?f1HC#8xzxd2*S#frWnvN%A$OBH@ii76XJ=5G9lSdZs{nVwZR4@|%8=FOrcOUL zDL;Glk9uZ~Cw?*0tNg4S@9ebQwWpi?ie5&8G=tKXt$&57vG&$JN~pPc{8nsUO`3Rz zkwxD+J%9$q&#%lz4GU{$C^s+9ZlwSWENB$->g?un>wthhS1EaMT^9%1t#^F$;lt(G zi|e@9)U%~8i2!t)-(6_2pDs<8d_c}0746K*!(wx`E7Di*V@Z(VnwfFw{=v<9w%lg8 zBMP??&0e@uxKdeOD;DCB5|jeS)5~tk6o2meuwnJDE|lB}-elpGFVgda(g*&MSnmlE zVJw8_#H0G5BMbNEW+%r98jrFyZ8Na43OF4<$6H+5VUcQgPciVE69G|!_kOy}3*Fnc z9}|xHdit37`4K2j{Kc+Vup$ev{6SP-cGuhj6sbJCyxxtTZAtFfO6DfZtE<_mf;($l zMX#Tus#M<1AsxPR?YUuL(eXQt%VIb@^p2~3qzkKZ`km$Db|Er^s%ku#8gnm>%-|MH zkxs)xRcpQ0jjPT<)b7p*!alm4zd1{CH)?9UxjmgWXuLIxrtJ+_T-;FOKxVah~9!YL;)3#;jC(WOlnJGG{VZOm&ZY^c@m}@u#5ap_- z!QB^~BDqK8FMMu=>j;pjX=zO+qZqEdZ*10z^)Um`zsM@(bZ=0N_<)in?D{rHQU>=5 zd`Tjw>Gt}-@wiPLo{&9BbQkQs%pt%hzeldi*C1+H8~5Id;}uxF2b7SV)1~c%^v6hI znHnf!Z@pEVi=6w@H7Fe_iML(kC?dbwt#n047dGAV1rW_EC9#sq1S1MVvL{G|`42?* zn>TI8JR2NK-X6cnuv^?j$9)M(F(wU62Kk4A?ES1L*O~k(Z zy|SV9B<{`SiHG6A;t3gF@+)O@-%i(Q`?ajjLE@ird?;vQMWc|Ik;Knsx$nHz(^_+X z@JLBr!0>+z3KCat-Muw(W?joOT^^IaJIpoLD84;yrK19w>v@e|)78ujlr~Uvy@Pn+ zQ{lif!vl8vL|z*#uPz`ouf9DV0&wchMY(daaQV#VBjQ4az&5&AheL>3kdgC;i2AI> zU0Y3!ALm54qLBA=1{1V7Q{ z*>2Ck**Iwq2ZE?h4^EMMh3J(h%bm9dO}jz(aP960irgKXKX&l7W3e6uEk>c@h5y#m8HLt8w11B-A1c{D=KeLvLm6OxcNDEM?BIr7tQ2uHnIwG zUzX2{#rkz$CZP|gfk*!p|Bb9rA4>1U~Yw5Xwg%oG|N&vp%VefGbNFg1Nx{ucD= zRmh(HH8m7Q8h^O!GpABGqG7CB8L2q?UQf?Ovw$4N^E_YJ^jG!kP<3bfDF~755HyN{ zT7~*6SY4rhB`NfqtLwV`@W>d?Fwgn<y*h%8&O~7z)yb49!6G$r z#7W?|o;07Hd5u)Q+I#LzR+!Hvg&_~+Tc!ujCfU!U$L8iDT!hOT(27QQYw8eVeBAZ> zgWYVRcYHdM@aQojRB_usQ@DvzTPRwJ%kEE6!YXHX`W$Z(#Ysxe$jY`RjNo+z>K;11 zCHD6AE<#UUjt!4T8>jBf_>8xAcem&61eho)>IZ1>y|aF0-cYID4Iq`H;e01cu~)U) z?}tgJa;&Va)e1+dhgO!F1VMQe+2UJoUqk($WRAn%)Abn{k#7CaEp;mRDRw8XB=6>j@tJv=VBfh3tN456v3b`w* zjz`1abE|YH8kD{Ch#y6g2}0~CDU?gn_st*0f0Q?N&~Xzm<_O^aE9$xj6)CN)Y*?~f z;h2~v6TGySHKGyGtT8`8K{3iO@L0Js@hgadKW{6=c!Cz-=dx`4h@I3O$Yrd^CSY$4 zgHfKs5kR+r#LNkw%1f~@9_QVluikh*in*B4HkC$TYRckK8!^WTIA4KM*K~^y4g+`-R9O-kyCxHH5#8)!<epQ-7cRa$jw9HJYnTw8!y1tyceJb-s^ypYMd5|Z@(m7J0 zOIKlR@RuKwW0KzO*!D<_4>N_Hck66G1hAo4Z@Rtd+VghRl(l=UGkK7iZ&}!QTMlr% zA+pHuDof2?7Evp7=@lpDrNeY?anZbIVnPM+LwRt;(&?_cl>ih0jK;*-FWH|HLwf#F zZJ%ww<0ef=l=JxH^iE5<=#8_f^Jm>Jt7feO#`%&D2^#9_KXjS-Zc$jWM>--LxHen>FDug=uGJOs>-40k0l27W2cv1BBKnaWQ zO_!S66aFvotzjUwYpnbTk8&IirDOhs@&R+fiZC!-jo&_=XuvJ3;&(Wk-%Q zF5t^Mcx1PFB1^X4SRdF`#KV78_xd?9^!0XKeZnfAGIS(kCNOFCuEyejrC~YJWh1ft z%P1px^OJ@sjEn_-_f#+X=5 z0(M-{j2%?O=YX3ZM7lh-4B{{it1VLI>QfleX?3|B!WFLh>%V+_X9oEFn5i>|G)(4a z&hX4t;yxQ);WqJ9#i;>bW?SS+E9hCN%UeWHKu>E?MdwP3!xU~TZ!#Oj^O|D`C9RsO z&qBqQ{}t+d&q}hF7?+a|Cn_{VRK+G9Y3W4jQDmw)iI_S1=`*dQg10Q zy>ueCMqd848LNmEYAQRWQVplL7yF`{=4so2I}$%W?tf)C6VkYSa)`iKf_xbGtGis* z;Wd|%Ku|izF=W=PXmt;sxy*t*+p^B@>%2cv5SivxSlw$~rh6|?>$#5%^E4)YZpA-S zGoA4UwzN`COPuH?;M?$l*?|ZB>x94!Qi2OE@5-;ozkfIG5R;o1FAYt7cXa>xIXp#( zw0+104IK;C{xkh%!LCsIA7u(mg#muxfZ(tG)!mC=vdKfVy+p>5Eptrx04sN{$P7p} zbfb@}L0O{XRhq5UBG9#OV!h|u_j9K&@IQJaheCM){;=49hOY=O9=L;?QTF!BQLkZg zK3jKUW9ROOxax}inufq}>E2+kOqc!ArI&GiaJzr>4^zrrx+e~$81YjD0^?OjF=KR* z>WLZMVb|)9>x{Z88K(QXiZU6Wjjrx=G)-mBE9us7as5kI3DjrP-fv+2-MXrd2xWN{ zXNJZ;-^^0ZC0Y5$IbIr8Yd1m!Z0P90t5DH1PXZa+uu2d8%FdHbx1m#8`?}nk9-Xjo z-@g(z*|TqlE*|bo?^%^CIHqdDS0}lwbDZNFXN!}P-)$u4&r|TXz6Qy)L4Vii{3&cz zQ1{`#v}(&}El#Clv_rQfMzss=^jRnzZiDG0n;lcCsORnmV{3_55Nwg~fi?2Kv`R4T zbaU86zbpoVp(GUX&l+*1b~0Lel(;hYk4!~&E!5i8iM%1pVA<>94l__BH>CvrGJ&KVTXkaJ`nZo{C#OQk~{Qv(fIwLT$@THpz*iz+L7KO{p%BqL@^3pEOjwMV4_+IRbc@a$Y&jkTo5#y=PR{Re9Y z+6kVFi^J!$HWg0kH&tpIu5yw{3*I z0SXMZJNXGsfkp;TSZro}36obdC~?<%L0hQ}pT$Y_QZ|3I2WR%2o%Q{!_Ogwj zUQiT71*QAnEIRCWIU6+t9Zb=225u-ozB`_dCB6@h-23;T=!aLoS<3%a>eg;Z^jg)x z{b@{6ygM&FA78D{aN0S)wRzSM10(CK&%7Z>R-yf=+kmH$K|!Y*LoE%EF2UdHtC*li z1re+WKU$mV7rvV2>KBboaM2SL5qY{aHg^hKcSaDlT7+#Nk^+P_04M5s!Jovk8n805 zqWOa49Hm(wdu^`DuF(meS95;2l61en0cuM56^WxY)8D7-QXzmxBB1*f;ys!8_& zgo6J2gNOUOt2qtX35KZ0L%_tR7W$i~o>K9j_f+%`Y!LhgI;a2WB0E@=E=EEU5_CRX zy@J7vjR{6FUBofoJwhWRu%t*x=6rWu0|@e(nwn>Ob8NLfK9_&M@zCQ$G!;sQ0d-|6o7{e>NJ9xD8Z~Dykv0{ zm2eb)yI7F?t#?r~SdM@Fg$nVIH{vk`Rs9t`(=cZiDvqfZW=ob)X204S&vy_px2%Y@A1= zp7Czn_j=l@Y@X8a)$5y5{Z}uJB?XPWuk+6u-mUF6)GXqoA}sOVxEGb@9@ z^1B}nSEp8!h5r`uI0Kv^P`1gl9j|b;?PNKKHqd!fYrZ#odHG=~2p8b!mrkE{oKy(e zObn(^hfV=kmZ`8H#;dwM4RlTg7#1a91C36td+K?4>3IoVJK4g;NUCJeQ`R$1WjBOG zhv5catjfSJOx6lQvgE>U=cVa?=QV9;gZcEw>gr!^;$AX*_u^&`(B3Y*ns(5Aoyqu+ zd`n99Uc9}$60&WiB;mvR%eELXWUo97F>Vk0ur~wW*x|c7+C<;GA$br3e^6Y^+Ang| z{>+OY{Bv#&g)*h+wO`wraOrspU=gbhDM8KgT-u74Icq65#&wZ7FrD&Opl;LDh~L@z zX?c}sn9rh@pbBEAv3xG}!Cr1c(M|K@B;LI5*KeHtc0YFfWtTP!?%Nu7tM@vj;qicg z4QSX3jZ3(8f>o6@dHf~RJ}(dLqF?RQ-!xrbk^zh#C`MeB5-RVSpA=~TKxjIlHepA| zg(13Nw!y=7O?t`wcCMs;bQ<90M?PC4On-KTV@vcV8jUsY@}4UzD;EY;!>;mO_huzO z(dsA~oa`iog(+0MA{r?%$W(GY7|h<^F9qZkU^q`gOdLVQN6e=Qy|ycOZ10$HedWN6 z4(SB!itVN!(d8x^NU-&PE%TBwjWHuqH`vfl6*)P#hX9qx6@a&6B_$H+^X|85mm7c? z-9gypyn37uv42qOTF8_EuR$J)>$ z%w6(0=>5Puw zz&lSV9vQlG+qjpqMifFh)UdWa@#FVz>A%;bpzUr_pN_S(w3(aRIjC4Oix_VJt~NV6 z9-j)+uCn)dfA8Ael+aOgPQmt<)cP###4Ndl$ZE?CT>W*4^ek(ZsZv4}-geVwo7Xb! zbNnx4d|=m3Njq!)i^y1A*RRzWv5F1xyWVsS-m8@y6PwKX2$PH5-lIEeXX0?VpFPgF zc7!erb1-;|%eLmVhgeIQve_CSbv8=TdVzTZ;mZnMcld_f!fvsF@md1(fv z1^vmz#u8a^Q#1K^#V^QMP*#@TYRDb0aGdz`Y3oL;e*p4EKy|gt^oIwJ@hT?DFLqe; zSot_Okf2p_Gx81rS@icCW+Bgs(vdBbG%w3GAbXkE|#1c9R!O+G-UE2+1+7t{(4!eDzw^R0>{e2NIG zE!p+Q{ri|F!jh-vsU$=sD9u(Swkg8ygSB;gq}AP>s=19*cOuX-;OKkh%(sR2oBLGf zwitC#ZFxJ_EA>u zFIILL8ymw8Z0xQmzf66dI|Qgf=nyv#Ps9GA{Mk$$SWWBUHvv;)#p$XaXDX|zy{;{* zb5xdEjp*p;fZKXV>%#T%WM7i-h-Mt{*%#%~=TZ7WzbIeVLov&JN-x5x`t0kYp5Mv$ zgk-yLA-N}(P4Sh6X5%o|V)){XNV{IMVZFr}u|x@OwB z?I_8)dm* ztU4^v=Z_M%>Mcb>JV8Og(APtzq^ivbAD9sP(#nNo#3?+39)(FeeUa664^+y%8Z!R! zXX?g;MHd-zs=6|jCmy=PQDLN-w7W~RwPj%{k=1msBQ#-_WTQ!EV`&i~+el(D{c1G6 z>mp>yn|S8g2v&rUwhW+Fx${~m!SVkbhEbAVbag&7)7cq}gLbg7yFFNb`0<>dHP)jz zORc2ou;)$~OmemLjC7>Q?`eUeqN4VrkN%1H5QOI?_flL(_V?d0+1h><%gYH8&3(qVk95R^3#J`z1r$N6Z}x3ea*Yr&=Q)N8|ak`THT%O{DL) zwvec~9E={t$B4>Vnn}$X}+B@shA>ITnbB02IOI2wC;DccjWnirA~zCqGoV0 zy?kBZTZJrWg8M+hoA4>J%hIkO(^F`FqfNA<9^0+9XI#fu(<@-lA4MhcG4lNn02NA6XOe_J* z4i^MpUP#0F$(MhV7$EyDI?^3+e(-5S3n(ktl<@Sl{;r|>*Ms|&DSJQ*=sf{uFSm~{ zCr4?ebfP`ttXLXD?YV5ZEBa``(MYiBVpXkK!`r=Ey~Rz7imGg)t%y0b=h5OoVx%%L z@Xlh7{iTc)IP)Y5?V!9OV7a5&o%OIt2_51X7%P2bylpN4-#PzdR^d^7e$`F$v$cLZ zg`aK#j0@QP z)R0W6Ku=FsTw^K#Ma63F-@DH1koBjoZoGQ?J86P(1P3}7T|>uByn#2tOxQoibE(@l zG@Oi=ExBHwx?E;b-L$rHa12O(3jOUNizfb6bp3<{qK-YuDWnke9N*YhL|4lnWPT)pco@9=eP-e!S82#Wn2(PHqMY8pIF`r958&LgGey_qx9RL9Hin?3!e;;bxgM#jV0lPxICBaJ5w91Ykj z5`gxkC*?j=($I*g&loh}#XR?KA-r&N{Mjq5SjXU^w7gC5Vq|eA=hQ6IY<$;Sj|0-% ze(k~g*19O?!JtGgikP^u{o9= zQtM-aA!y!E5$3L%{?m4`*3y-Zi2(ik5?IX`#O1t5+>Ip;0d|7c9^!!kldV}tO zyG(v;eb%^sUh{RSMcmKfj;ch9_Bb5l!4}sDKkC83fz$ab#%N?{$6r!X0&0F=g(Rl8 zy&Te2kW971d2*~b3Wt0rtg-H60^i1L)0KWr?|C=l*opUDB_+xH|rI5>inR0y=7w8sh%|Fhsb zGm+}96nM5){??~RAI$`u5Yqyje0>MfsvdIa91`!^GriqD2U?=o@B{y6Gw(g zVsB1cM+C6x+2RQVrj#a;sAb8=Veen)B`@K+JwRH?f_%5Q>_Q5{*pPfH!lj@;qh8um z6T&JF7uoh>_i3Tm_K2=k_6c&k^lUSinyw0Y9ohE<<*ueIp@s+i5}`$PcNdluJNX?x zye!2dJZHzpP&1(Qi%bm-=8O%GT2|HWrcnA*Lx#8YJxnYt>gsD7=4X3UHJUWvATdwR z?VGdWJ8q9JBk!wpicrOw<6hjw{Fs$pE6}x_skIT3i!`eca5*%g{Nh@i_3@Snr8#7! zTlDf=wQyuIt?@JLD93+xH>iCn@Okvq?413!+uPAWI>?TC&2Ta5V-TeDIWu&pO2XMZrUxLDC$ z5yPTWGf&SNNxP9+-1Gy!9i_+WSVD8?5Io|B`*(^{eSLisCMQEIW9-Jl?(*&AtAV!A*MxGPM zJ~hGTiS3>V4}M|^6(8o%e8`oHbU60@`W+cU3owLGAX$2M^`Y;(jZP$)ZL+n`f#Z6Z zt{Ju-yf1zV4%ej#w8y?MH_*4%23E6kjdcXBa;>oM!*}r9-T-z?QEcHSa9MQ}_iY z7`pFCNp3VR?zU_j!y`fr%RIH1VS z#C(nFHe@=!xSF~0O`77vSoBDt#vr~-B(1CWxyo(+P=3CuDQCQNCa~UnmHiOv0(5v3 zOehcNTzcZLAz-*M#^s6Zdw%XV62=X&lM3DcjeBPm|CG`;gdD0LTZHV&Zr(&O)4U}T(18p!hU??sF@B%Xn^haO+=$wzv-eYdMcQxYiTY%b8e&hlpFFArtt?}HFJOyB z5~~#M^7^SHANz4n(;HE)VSof-w&X{JmJ29I>ILZf-&yN;(mDjSZDzL3;EXEaeN+%s zdkmaA(GiyEj+zk^KywI|!>0s`#>`+0l@0QW@A=N8?$tT&6p&?g1U5<#hpOxAJ8rwB zDCIJhtb09*V_DUF>mfo-Z8#Cx20F^n$_+9VuS=QS!!6 zu7*a+3!jCbwROAX;`d(?E@B+YKVjtLd?r{>U3^Ogx>-t3R>s7PMh5<$%uQB|l}Oc8$g!MK{q+|u3j<@qn6 zvEs^!Z4zlh(kB>bH)BwVzE?Sic2AX+1_A29il%|4Msg~b2k{K+_s-E`Z5#N6i+3f{ z+I;#i2F#|qDgB_Si-1lbohjgX{yA13UB0)8m zjcAe~=YSoK`f$^+Nj>>=(F^w@4p2o}BV zB#8MS`*umfBewgf%$zsTT^L4Xv*RSFuSD{Fh0L$Pa_2uKvg;LLl}&51h5k zuH|Gf;GCDIFl*&W?lPfpDokkMKEspSe33{jgJ;sST%)(^vsWv${{s4p$i@$l@qA$0 z1mAhcghr6|Z#z0X)Rq5O8TzM@H)&`w=d{9Pgl74zHJCyJqZRQ{DTeQ#(OVSbC-x~* zJHBD99PCIVS9;-xv*VjaP9-2r*>BC8NnuM)TqrfqBql_SA(*FfKrbw6Yv*_{peKni zYDvRZHV!b7OC%`b=1jshcd+1`L=qPebvXB=lv2q_q2yi(Zyp|6%1ZKI2zfw_!AVVy zOGcqa<@0j|M`xyOEcVMP=}#xgVLz3?++f5TwdJS2l>$ zHjtNs?s3cVfjhG3+*NrUhCj}JhpQk3ZqUp^O*{qRj`Fyus5NgQ$8?M!t(?vejpMbH zoF#iiD~$c6ikhpk^2$oV;O{q?`4ozL%-+b_4i*j)hV_5OT*pPU@ljW5;QhjpwEC=J zSYfyd&%Sq>aMq89;);Jww@2s^N(m(QC60=7Ji_s(uAeI#+;9T^uptrxe_u(B!PZK{ z8r%K83IsNhvA0b7CJqQa&%q}hc{V7(S`nl-0W=t#xNYnAXM}FX&{@O=ES}a3 zv2P%YfBojB&;8#rBKf~vG{y0v{#!gu`#(Gu7Ig1DBmq5uB_ DZqx6w literal 0 HcmV?d00001 diff --git a/docs/docs/index.md b/docs/docs/index.md index dc914d3d..650586dd 100644 --- a/docs/docs/index.md +++ b/docs/docs/index.md @@ -1,10 +1,23 @@ +!!! note + This project is still currently under development. If you are interested in joining our team and contributing, [read our project wiki](https://github.com/codeforboston/flagging/wiki) for more info. + Welcome to the CRWA Flagging Website Documentation! This site provides developers and maintainers information about the CRWA's flagging website, including information on: deploying the website, developing the website, using the website's code locally, and using the website's admin panel. -!!! note - This project is still currently under development. If you are interested in joining our team and contributing, [read our project wiki](https://github.com/codeforboston/flagging/wiki). +## For Website Administrators + +If the website is already deployed and you want to implement a manual override, you do not need to follow the setup guide. All you need to do is read the [admin](../admin) guide to manage the website while it's deployed. + +## Connecting to Weebly + +Work in progress + +## For Developers -## Quick Blurb of this Project +Start by following the [setup guide](../setup). Once you have the website setup locally, you now have access to the following: -The Charles River Watershed Association (CRWA) monitors the Charles River for diseases and originally had out-of-comissioned PHP website that keep tracks of its temperature and volume flow. This project aims to update the website wih a new website using flask framework. More information about deployment, background, and how it works in other web pages. +- Deploy the website to Heroku (guide [here](../deployment)) +- Manually run commands and download data through the [shell](../shell). +- Make changes to the predictive model, including revising its coefficients. (Guide is currently WIP) +- (Advanced) Make other changes to the website. diff --git a/docs/docs/setup.md b/docs/docs/setup.md new file mode 100644 index 00000000..7ac7e63b --- /dev/null +++ b/docs/docs/setup.md @@ -0,0 +1,153 @@ +# Setup + +This is a guide on how to do your first-time setup for running the website locally and getting ready to make changes to the code base. If you are a developer, you should follow this guide before doing anything else! + +This guide assumes you're a complete beginner at Python, Git, Postgres, etc. so don't be intimidated if you don't know what all of these things are. The main thing this guide assumes is that you know how to open up a terminal in your respective operating system (command prompt or "CMD" in Windows, and bash in OSX). + +## Dependencies + +Install all of the following programs onto your computer: + +**Required:** + +- [Python 3](https://www.python.org/downloads/) - specifically 3.7 or higher +- [Git](https://git-scm.com/book/en/v2/Getting-Started-Installing-Git) (first time setup guide [here](https://git-scm.com/book/en/v2/Getting-Started-First-Time-Git-Setup)) +- [Postgres](https://www.postgresql.org/) _(see installation instructions below)_ +- _(OSX only)_ [Homebrew](https://brew.sh/) + +**Recommended:** + +- A good text editor or IDE, such as [Atom.io](https://atom.io/) (which is lightweight and beginner friendly) or [PyCharm](https://www.jetbrains.com/pycharm/) (which is powerful but bulky and geared toward advanced users). +- [Heroku CLI](https://git-scm.com/book/en/v2/Getting-Started-Installing-Git) _(required for remote deployment to Heroku.)_ + +**Other:** + +- It is strongly recommend that you create a [GitHub account](https://github.com/) if you haven't done so already. The GitHub account should have the same email as the one registered to your `git config --global user.email` that you set in the first time git setup. + +???+ warning + _(Windows users only)_ At least two Windows users have had problems getting Python working in Windows for the first time. Check out some people troubleshooting various Python installation related issues [on StackOverflow](https://stackoverflow.com/questions/13596505/python-not-working-in-command-prompt). Also note that the command to run Python in Windows may be `python`, `python3`, `py`, or `py3`. Figure out which one works for you. + +### Postgres installation + +=== "Windows (CMD)" + 1. Download [here](https://www.postgresql.org/download/windows/) and install via the executable. + + 2. (If you had any terminals open, close out and reopen after Postgres installation.) + + 3. Open command prompt and try the following (case-sensitive): `psql -V` If it returns the version number then you're set. + + 4. If you get an error about the command not being recognized, then it might mean you need to manually add Postgres's bin to your PATH ([see here](https://stackoverflow.com/a/11567231)). + +=== "OSX (Bash)" + + 1. If you do not have Homebrew installed, install it from [here](https://brew.sh/). + + 2. Via a bash terminal: `brew install postgres` + + 3. Test that it works by running (case-sensitive): `psql -V`. If it returns the version number then you're set. + +???+ tip + Chances are you are not going to need Postgres to run in the background constantly, so you should learn how to turn it off and back on. + + === "Windows (CMD)" + + **Turn Postgres on/off:** + + 1. Go to the Start menu and open up "Run..." + + 2. `services.msc` -> ++enter++. This opens the Services panel. + + 3. Look for the name _postgresql_ and start/stop Postgres. + + **Keep Postgres from running at startup:** + + (Via the Services panel) As long as the service is "manual" and not automatic, it will not load at startup. + + === "OSX (Bash)" + **Turn Postgres on:** + + ```shell + pg_ctl -D /usr/local/var/postgres start + ``` + + **Turn Postgres off:** + + ```shell + pg_ctl -D /usr/local/var/postgres stop + ``` + + **Keep Postgres from running at startup:** + + Some solutions [here](https://superuser.com/questions/244589/prevent-postgresql-from-running-at-startup). + +## Download and Setup the Code Base + +The flagging website is open source; the whole website's source code is available on GitHub. This section of the setup guide shows you the preferred way to install it and set up the code on a local computer. + +1. Fork the [main GitHub repo](https://github.com/codeforboston/flagging/) to your personal GitHub account. You can do that by going to the Code For Boston flagging repo and clicking on this button: + +![](img/github_fork.png) + +2. Point your terminal (Bash on OSX, command prompt on Windows) to the directory you want to put the `/flagging` project folder inside of. E.g. if you want the project folder to be located at `/Documents/flagging`, then point your directory to `/Documents`. You can change directories using the `cd` command: `cd "path/goes/here"` + +3. Run the following to download your fork and setup the connection to the upstream remote. Replace `YOUR_USERNAME_HERE` (in the first line) with your actual GitHub username. + +```shell +git clone https://github.com/YOUR_USERNAME_HERE/flagging/ +cd flagging +git remote add upstream https://github.com/codeforboston/flagging.git +git fetch upstream +``` + +4. In your newly created `flagging` folder, create a file called `.env` and add the vault password to it. (Replace `vault_password_here` with the actual vault password if you have it; otherwise just copy+paste and run the command as-is): + +```shell +echo "VAULT_PASSWORD=vault_password_here" > .env +``` + +???+ danger + If you do any commits to the repo, _please make sure `.env` is properly gitignored!_ (`.flaskenv` does not need to be gitignored, only `.env`.) The vault password is sensitive information; it should not be shared with others and it should not be posted online publicly. + +## Run the Website Locally + +???+ note + From here on in the documentation, each terminal command assumes your terminal's working directory is pointed toward the `flagging` directory. + +After you get everything set up, you should run the website at least once. Te process of running the website installs the remaining dependencies, and sets up a virtual environment to work in. + +1. Run the following: + +=== "Windows (CMD)" + ```shell + run_windows_dev + ``` + +=== "OSX (Bash)" + ```shell + sh run_unix_dev.sh + ``` + +???+ note + The script being run is doing the following, in order: + + 1. Set up a "virtual environment" (basically an isolated folder inside your project directory that we install the Python packages into), + 2. install the packages inside of `requirements.txt`; this can take a while during your first time. + 3. Set up some environment variables that Flask needs. + 4. Prompts the user to set some options for the deployment. (See step 2 below.) + 5. Set up the Postgres database and update it with data. + 6. Run the actual website. + +2. For the first prompt, type `y` to run the website in offline mode. For the subsequent two prompts, ++enter++ through them without inputting anything. + +???+ success + You should be good if you eventually see something like the following in your terminal: + + ``` + * Serving Flask app "flagging_site:create_app" (lazy loading) + * Environment: development + * Debug mode: on + * Running on http://127.0.0.1:5000/ (Press CTRL+C to quit) + * Restarting with stat + ``` + +3. Point your browser of choice to the URL shown in the terminal output. If everything worked out, the website should be running on your local computer! diff --git a/docs/docs/shell.md b/docs/docs/shell.md index 0cd807e1..ce0a345e 100644 --- a/docs/docs/shell.md +++ b/docs/docs/shell.md @@ -56,7 +56,7 @@ Here we assume you have already started the Flask shell. This example shows how to download the Hobolink data and save it as a CSV file. -```shell +```python >>> hobolink_data = get_live_hobolink_data() >>> hobolink_data.to_csv('path/where/to/save/my-CSV-file.csv') ``` \ No newline at end of file From 7fcc585045fdae35fcff03b66be319144a8b6ee1 Mon Sep 17 00:00:00 2001 From: dwreeves Date: Tue, 15 Sep 2020 20:09:00 -0400 Subject: [PATCH 066/118] minor updates --- README.md | 4 +-- docs/README.md | 10 ++----- docs/mkdocs.yml | 25 +++++++++++++++-- flagging_site/app.py | 7 +++++ flagging_site/config.py | 2 +- flagging_site/data/hobolink.py | 2 +- flagging_site/data/model.py | 46 ++++++++++++++----------------- flagging_site/templates/base.html | 2 ++ 8 files changed, 60 insertions(+), 38 deletions(-) diff --git a/README.md b/README.md index 3895266a..e5f7f021 100644 --- a/README.md +++ b/README.md @@ -47,8 +47,8 @@ brew services start postgresql To begin initialize a database which we call `flagging`, enter into the bash terminal: ```shell script -export POSTGRES_PASSWORD=*enter_password_here* -createdb -U *enter_username_here* flagging +export POSTGRES_PASSWORD=enter_password_here +createdb flagging psql -U *enter_username_here* -d flagging -c "DROP USER IF EXISTS flagging; CREATE USER flagging SUPERUSER PASSWORD '${POSTGRES_PASSWORD}'" ``` diff --git a/docs/README.md b/docs/README.md index a218c68b..5d88704a 100644 --- a/docs/README.md +++ b/docs/README.md @@ -7,10 +7,8 @@ The full docs are available at: https://codeforboston.github.io/flagging/ If you have write permission to the upstream repository (i.e. you are a project manager), point your terminal to this directory and run the following: ```shell script -pip install mkdocs -pip install pymdown-extensions -pip install mkdocs-material -mkdocs gh-d`eploy --remote-name upstream +pip install mkdocs pymdown-extensions mkdocs-material +mkdocs gh-deploy --remote-name upstream ``` If you do not have write permission to the upstream repository, you can do one of the following: @@ -21,9 +19,7 @@ If you do not have write permission to the upstream repository, you can do one o If you are a project manager but you're having issues, you can do a more manual git approach to updating the docs: ```shell script -pip install mkdocs -pip install pymdown-extensions -pip install mkdocs-material +pip install mkdocs pymdown-extensions mkdocs-material mkdocs gh-deploy git checkout gh-pages git push upstream gh-pages diff --git a/docs/mkdocs.yml b/docs/mkdocs.yml index 3cc8f8a6..f97e536a 100644 --- a/docs/mkdocs.yml +++ b/docs/mkdocs.yml @@ -1,26 +1,47 @@ site_name: CRWA Flagging Website Documentation +site_description: Guide on developing and deploying the flagging website nav: - Home: index.md - About: about.md +- Setup: setup.md - Admin: admin.md - Deployment: deployment.md - System: system.md -- Database: database.md - Shell: shell.md -- Development: +- Development Background: - History: development/history.md + - Learning Resources: development/learning_resources.md +- Development: + # - Overview: development/index.md + # ^ (Explain the vault here too) + # - Config: development/config.md + - Data: development/data.md + - Predictive Models: development/predictive_models.md + # - Front-End: development/front-end.md theme: name: material palette: scheme: default primary: teal accent: cyan + icon: + logo: fontawesome/regular/flag font: text: Opens Sans code: Roboto Mono +repo_name: codeforboston/flagging +repo_url: https://github.com/codeforboston/flagging +edit_uri: "" markdown_extensions: - admonition - pymdownx.tabbed # https://facelessuser.github.io/pymdown-extensions/ +- pymdownx.keys +- pymdownx.details +- pymdownx.inlinehilite +- pymdownx.magiclink: + repo_url_shorthand: true + user: squidfunk + repo: mkdocs-material - pymdownx.superfences: custom_fences: - name: mermaid diff --git a/flagging_site/app.py b/flagging_site/app.py index 65d088b8..6ef6df5a 100644 --- a/flagging_site/app.py +++ b/flagging_site/app.py @@ -3,6 +3,7 @@ """ import os import click +import time from typing import Optional from flask import Flask @@ -59,6 +60,12 @@ def create_app(config: Optional[Config] = None) -> Flask: from .admin import init_admin init_admin(app) + @app.before_request + def before_request(): + from flask import g + g.request_start_time = time.time() + g.request_time = lambda: '%.5fs' % (time.time() - g.request_start_time) + @app.cli.command('create-db') def create_db_command(): """Create database (after verifying that it isn't already there)""" diff --git a/flagging_site/config.py b/flagging_site/config.py index 8d51b44d..9333111c 100644 --- a/flagging_site/config.py +++ b/flagging_site/config.py @@ -143,7 +143,7 @@ class DevelopmentConfig(Config): VAULT_OPTIONAL: bool = True DEBUG: bool = True TESTING: bool = True - OFFLINE_MODE = strtobool(os.getenv('OFFLINE_MODE', 'false')) + OFFLINE_MODE = strtobool(os.getenv('OFFLINE_MODE') or 'false') BASIC_AUTH_USERNAME: str = os.getenv('BASIC_AUTH_USERNAME', 'admin') BASIC_AUTH_PASSWORD: str = os.getenv('BASIC_AUTH_PASSWORD', 'password') diff --git a/flagging_site/data/hobolink.py b/flagging_site/data/hobolink.py index e5c1b3d4..9ac598d7 100644 --- a/flagging_site/data/hobolink.py +++ b/flagging_site/data/hobolink.py @@ -72,7 +72,7 @@ def request_to_hobolink( """ data = { 'query': export_name, - 'authentication': get_keys()['hobolink'] + 'authentication': current_app.config['KEYS']['hobolink'] } res = requests.post(HOBOLINK_URL, json=data) diff --git a/flagging_site/data/model.py b/flagging_site/data/model.py index a1542389..ad8a2d40 100644 --- a/flagging_site/data/model.py +++ b/flagging_site/data/model.py @@ -79,7 +79,7 @@ def process_data( .agg({ 'pressure': np.mean, 'par': np.mean, - 'rain': sum, + 'rain': np.sum, 'rh': np.mean, 'dew_point': np.mean, 'wind_speed': np.mean, @@ -105,28 +105,16 @@ def process_data( if df.iloc[-1, :][['stream_flow', 'rain']].isna().any(): df = df.drop(df.index[-1]) - # Next, do the following: - # - # 1 day avg of: - # - wind_speed - # - water_temp - # - air_temp - # - stream_flow (via USGS) - # 2 day avg of: - # - par - # - stream_flow (via USGS) - # sum of rain at following increments: - # - 1 day - # - 2 day - # - 7 day - for col in ['par', 'stream_flow']: - df[f'{col}_1d_mean'] = df[col].rolling(24).mean() - - for incr in [24, 48]: - df[f'rain_0_to_{str(incr)}h_sum'] = df['rain'].rolling(incr).sum() - df[f'rain_24_to_48h_sum'] = ( - df[f'rain_0_to_48h_sum'] - df[f'rain_0_to_24h_sum'] - ) + # The code from here on consists of feature transformations. + + # Calculate rolling means + df['par_1d_mean'] = df['par'].rolling(24).mean() + df['stream_flow_1d_mean'] = df['stream_flow'].rolling(24).mean() + + # Calculate rolling sums + df[f'rain_0_to_24h_sum'] = df['rain'].rolling(24).sum() + df[f'rain_0_to_48h_sum'] = df['rain'].rolling(48).sum() + df[f'rain_24_to_48h_sum'] = df[f'rain_0_to_48h_sum'] - df[f'rain_0_to_24h_sum'] # Lastly, they measure the "time since last significant rain." Significant # rain is defined as a cumulative sum of 0.2 in over a 24 hour time period. @@ -134,7 +122,6 @@ def process_data( df['last_sig_rain'] = ( df['time'] .where(df['sig_rain']) - .shift() .ffill() .fillna(df['time'].min()) ) @@ -166,9 +153,11 @@ def reach_2_model(df: pd.DataFrame, rows: int = 48) -> pd.DataFrame: - 0.0362 * df['days_since_sig_rain'] - 0.000312 * df['par_1d_mean'] ) + df['probability'] = sigmoid(df['log_odds']) df['safe'] = df['probability'] <= SAFETY_THRESHOLD df['reach'] = 2 + return df[['reach', 'time', 'log_odds', 'probability', 'safe']] @@ -186,7 +175,6 @@ def reach_3_model(df: pd.DataFrame, rows: int = 48) -> pd.DataFrame: Returns: Outputs for model as a dataframe. """ - df = df.tail(n=rows).copy() df['log_odds'] = ( @@ -195,9 +183,11 @@ def reach_3_model(df: pd.DataFrame, rows: int = 48) -> pd.DataFrame: + 0.1681 * df['rain_24_to_48h_sum'] - 0.02855 * df['days_since_sig_rain'] ) + df['probability'] = sigmoid(df['log_odds']) df['safe'] = df['probability'] <= SAFETY_THRESHOLD df['reach'] = 3 + return df[['reach', 'time', 'log_odds', 'probability', 'safe']] @@ -217,6 +207,7 @@ def reach_4_model(df: pd.DataFrame, rows: int = 48) -> pd.DataFrame: Outputs for model as a dataframe. """ df = df.tail(n=rows).copy() + df['log_odds'] = ( 0.5791 + 0.30276 * df['rain_0_to_24h_sum'] @@ -224,9 +215,11 @@ def reach_4_model(df: pd.DataFrame, rows: int = 48) -> pd.DataFrame: - 0.02267 * df['days_since_sig_rain'] - 0.000427 * df['par_1d_mean'] ) + df['probability'] = sigmoid(df['log_odds']) df['safe'] = df['probability'] <= SAFETY_THRESHOLD df['reach'] = 4 + return df[['reach', 'time', 'log_odds', 'probability', 'safe']] @@ -245,15 +238,18 @@ def reach_5_model(df: pd.DataFrame, rows: int = 48) -> pd.DataFrame: Outputs for model as a dataframe. """ df = df.tail(n=rows).copy() + df['log_odds'] = ( 0.3333 + 0.1091 * df['rain_0_to_48h_sum'] - 0.01355 * df['days_since_sig_rain'] + 0.000342 * df['stream_flow_1d_mean'] ) + df['probability'] = sigmoid(df['log_odds']) df['safe'] = df['probability'] <= SAFETY_THRESHOLD df['reach'] = 5 + return df[['reach', 'time', 'log_odds', 'probability', 'safe']] diff --git a/flagging_site/templates/base.html b/flagging_site/templates/base.html index 5de5a2ab..7c0a8a07 100644 --- a/flagging_site/templates/base.html +++ b/flagging_site/templates/base.html @@ -47,6 +47,8 @@
This website was built by Code for Boston in collaboration with Charles River Watershed Association.
+
+ Page loaded in {{ g.request_time() }} seconds.
{% endblock %} From 08f28cd5467ff239f240874000084517058da872 Mon Sep 17 00:00:00 2001 From: dwreeves Date: Tue, 15 Sep 2020 22:48:30 -0400 Subject: [PATCH 067/118] Refactor config related stuff --- flagging_site/app.py | 107 +++++++++++++++------------ flagging_site/config.py | 12 +-- flagging_site/data/_store/refresh.py | 8 +- flagging_site/data/hobolink.py | 19 +++-- flagging_site/data/keys.py | 77 ------------------- flagging_site/data/usgs.py | 9 ++- run_unix_dev.sh | 6 +- 7 files changed, 85 insertions(+), 153 deletions(-) delete mode 100644 flagging_site/data/keys.py diff --git a/flagging_site/app.py b/flagging_site/app.py index 9cec66c3..39a73237 100644 --- a/flagging_site/app.py +++ b/flagging_site/app.py @@ -4,14 +4,12 @@ import os import click import time +import json +import zipfile from typing import Optional + from flask import Flask -from .blueprints.flagging import get_data -from .data.hobolink import get_live_hobolink_data -from .data.keys import get_keys -from .data.model import process_data -from .data.usgs import get_live_usgs_data from .config import Config from .config import get_config_from_env @@ -45,8 +43,11 @@ def create_app(config: Optional[Config] = None) -> Flask: # blueprints are imported is: If BLUEPRINTS is in the config, then import # only from that list. Otherwise, import everything that's inside of # `blueprints/__init__.py`. - from . import blueprints - register_blueprints_from_module(app, blueprints) + from .blueprints.api import bp as api_bp + app.register_blueprint(api_bp) + + from .blueprints.flagging import bp as flagging_bp + app.register_blueprint(flagging_bp) # Add Swagger to the app. Swagger automates the API documentation and # provides an interface for users to query the API on the website. @@ -68,7 +69,7 @@ def create_app(config: Optional[Config] = None) -> Flask: def before_request(): from flask import g g.request_start_time = time.time() - g.request_time = lambda: '%.5fs' % (time.time() - g.request_start_time) + g.request_time = lambda: '%.3fs' % (time.time() - g.request_start_time) @app.cli.command('create-db') def create_db_command(): @@ -95,6 +96,11 @@ def update_db_command(): # Make a few useful functions available in Flask shell without imports @app.shell_context_processor def make_shell_context(): + from .blueprints.flagging import get_data + from .data.hobolink import get_live_hobolink_data + from .data.model import process_data + from .data.usgs import get_live_usgs_data + return { 'db': db, 'get_data': get_data, @@ -107,34 +113,6 @@ def make_shell_context(): return app -def update_config_from_vault(app: Flask) -> None: - """ - This updates the state of the `app` to have the keys from the vault. The - vault also stores the "SECRET_KEY", which is a Flask builtin configuration - variable (i.e. Flask treats the "SECRET_KEY" as special). So we also - populate the "SECRET_KEY" in this step. - - If we fail to load the vault in development mode, then the user is warned - that the vault was not loaded successfully. In production mode, failing to - load the vault raises a RuntimeError. - - Args: - app: A Flask application instance. - """ - try: - app.config['KEYS'] = get_keys() - except (RuntimeError, KeyError): - msg = 'Unable to load the vault; bad password provided.' - if app.env == 'production': - raise RuntimeError(msg) - else: - print(f'Warning: {msg}') - app.config['KEYS'] = None - app.config['SECRET_KEY'] = os.urandom(16) - else: - app.config['SECRET_KEY'] = app.config['KEYS']['flask']['secret_key'] - - def add_swagger_plugin_to_app(app: Flask): """This function hnadles all the logic for adding Swagger automated documentation to the application instance. @@ -178,20 +156,53 @@ def add_swagger_plugin_to_app(app: Flask): Swagger(app, config=swagger_config, template=template) -def register_blueprints_from_module(app: Flask, module: object) -> None: +def _load_keys_from_vault( + vault_password: str, + vault_file: str +) -> dict: + """This code loads the keys directly from the vault zip file. + + Args: + vault_password: (str) Password for opening up the `vault_file`. + vault_file: (str) File path of the zip file containing `keys.json`. + + Returns: + Dict of credentials. + """ + pwd = bytes(vault_password, 'utf-8') + with zipfile.ZipFile(vault_file) as f: + with f.open('keys.json', pwd=pwd, mode='r') as keys_file: + d = json.load(keys_file) + return d + + +def update_config_from_vault(app: Flask) -> None: """ - This function looks within the submodules of a module for objects - specifically named `bp`. It then assumes those objects are blueprints, and - registers them to the app. + This updates the state of the `app` to have the keys from the vault. The + vault also stores the "SECRET_KEY", which is a Flask builtin configuration + variable (i.e. Flask treats the "SECRET_KEY" as special). So we also + populate the "SECRET_KEY" in this step. + + If we fail to load the vault in development mode, then the user is warned + that the vault was not loaded successfully. In production mode, failing to + load the vault raises a RuntimeError. Args: - app: (Flask) Flask instance to which we will register blueprints. - module: (object) A module that contains submodules which themselves - contain `bp` objects. + app: A Flask application instance. """ - if app.config.get('BLUEPRINTS'): - blueprint_list = app.config['BLUEPRINTS'] + try: + keys = _load_keys_from_vault( + vault_password=app.config['VAULT_PASSWORD'], + vault_file=app.config['VAULT_FILE'] + ) + except (RuntimeError, KeyError): + msg = 'Unable to load the vault; bad password provided.' + if app.env == 'production': + raise RuntimeError(msg) + else: + print(f'Warning: {msg}') + app.config['KEYS'] = None + app.config['SECRET_KEY'] = os.urandom(16) else: - blueprint_list = filter(lambda x: not x.startswith('_'), dir(module)) - for submodule in blueprint_list: - app.register_blueprint(getattr(module, submodule).bp) + app.config['KEYS'] = keys + app.config['SECRET_KEY'] = keys['flask']['secret_key'] diff --git a/flagging_site/config.py b/flagging_site/config.py index 7210dc83..5fecfcf2 100644 --- a/flagging_site/config.py +++ b/flagging_site/config.py @@ -28,7 +28,6 @@ load_dotenv(os.path.join(ROOT_DIR, '..', '.flaskenv')) load_dotenv(os.path.join(ROOT_DIR, '..', '.env')) - # Configs # ~~~~~~~ @@ -87,6 +86,8 @@ def SQLALCHEMY_DATABASE_URI(self) -> str: wasn't opened. Usually set alongside DEBUG mode. """ + VAULT_PASSWORD: str = os.getenv('VAULT_PASSWORD') + KEYS: Dict[str, Dict[str, Any]] = None """These are where the keys from the vault are stored. It should be a dict of dicts. Each key in the first level dict corresponds to a different @@ -117,13 +118,6 @@ def SQLALCHEMY_DATABASE_URI(self) -> str: when doing requests. """ - BLUEPRINTS: Optional[List[str]] = None - """Names of the blueprints available to the app. We can use this to turn - parts of the website off or on depending on if they're fully developed - or not. If BLUEPRINTS is `None`, then it imports all the blueprints it can - find in the `blueprints` module. - """ - API_MAX_HOURS: int = 48 """The maximum number of hours of data that the API will return. We are not trying to be stingy about our data, we just want this in order to avoid any odd behaviors @@ -136,8 +130,6 @@ class ProductionConfig(Config): internet. Currently the only part of the website that's pretty fleshed out is the `flagging` part, so that's the only blueprint we import. """ - BLUEPRINTS: Optional[List[str]] = ['flagging', 'api'] - def __init__(self): self.BASIC_AUTH_USERNAME: str = os.environ['BASIC_AUTH_USERNAME'] self.BASIC_AUTH_PASSWORD: str = os.environ['BASIC_AUTH_PASSWORD'] diff --git a/flagging_site/data/_store/refresh.py b/flagging_site/data/_store/refresh.py index fb38dfb5..933d22f1 100644 --- a/flagging_site/data/_store/refresh.py +++ b/flagging_site/data/_store/refresh.py @@ -34,13 +34,13 @@ def refresh_data_store(vault_password: Optional[str] = None) -> None: from flagging_site.data.keys import get_data_store_file_path from flagging_site.data.hobolink import get_live_hobolink_data - from flagging_site.data.hobolink import STATIC_FILE_NAME as hobolink_file + from flagging_site.data.hobolink import HOBOLINK_STATIC_FILE_NAME get_live_hobolink_data('code_for_boston_export_21d')\ - .to_pickle(get_data_store_file_path(hobolink_file)) + .to_pickle(get_data_store_file_path(HOBOLINK_STATIC_FILE_NAME)) from flagging_site.data.usgs import get_live_usgs_data - from flagging_site.data.usgs import STATIC_FILE_NAME as usgs_file - get_live_usgs_data().to_pickle(get_data_store_file_path(usgs_file)) + from flagging_site.data.usgs import USGS_STATIC_FILE_NAME + get_live_usgs_data().to_pickle(get_data_store_file_path(USGS_STATIC_FILE_NAME)) if __name__ == '__main__': diff --git a/flagging_site/data/hobolink.py b/flagging_site/data/hobolink.py index e513fb82..8c07aae0 100644 --- a/flagging_site/data/hobolink.py +++ b/flagging_site/data/hobolink.py @@ -6,19 +6,17 @@ # Pandas is inefficient. It should go to SQL, not to Pandas. I am currently # using pandas because we do not have any cron jobs or any caching or SQL, but # I think in future versions we should not be using Pandas at all. -from .database import db +import os import io import requests import pandas as pd from flask import abort from flask import current_app -from .keys import get_keys -from .keys import get_data_store_file_path # Constants HOBOLINK_URL = 'http://webservice.hobolink.com/restv2/data/custom/file' -EXPORT_NAME = 'code_for_boston_export' +DEFAULT_HOBOLINK_EXPORT_NAME = 'code_for_boston_export' # Each key is the original column name; the value is the renamed column. HOBOLINK_COLUMNS = { 'Time, GMT-04:00': 'time', @@ -34,11 +32,13 @@ 'Temp': 'air_temp', # 'Batt, V, Charles River Weather Station': 'battery' } -STATIC_FILE_NAME = 'hobolink.pickle' +HOBOLINK_STATIC_FILE_NAME = 'hobolink.pickle' # ~ ~ ~ ~ -def get_live_hobolink_data(export_name: str = EXPORT_NAME) -> pd.DataFrame: +def get_live_hobolink_data( + export_name: str = DEFAULT_HOBOLINK_EXPORT_NAME +) -> pd.DataFrame: """This function runs through the whole process for retrieving data from HOBOlink: first we perform the request, and then we clean the data. @@ -50,7 +50,10 @@ def get_live_hobolink_data(export_name: str = EXPORT_NAME) -> pd.DataFrame: Pandas Dataframe containing the cleaned-up Hobolink data. """ if current_app.config['OFFLINE_MODE']: - df = pd.read_pickle(get_data_store_file_path(STATIC_FILE_NAME)) + fpath = os.path.join( + current_app.config['DATA_STORE'], HOBOLINK_STATIC_FILE_NAME + ) + df = pd.read_pickle(fpath) else: res = request_to_hobolink(export_name=export_name) df = parse_hobolink_data(res.text) @@ -58,7 +61,7 @@ def get_live_hobolink_data(export_name: str = EXPORT_NAME) -> pd.DataFrame: def request_to_hobolink( - export_name: str = EXPORT_NAME, + export_name: str = DEFAULT_HOBOLINK_EXPORT_NAME, ) -> requests.models.Response: """ Get a request from the Hobolink server. diff --git a/flagging_site/data/keys.py b/flagging_site/data/keys.py deleted file mode 100644 index 8db3653a..00000000 --- a/flagging_site/data/keys.py +++ /dev/null @@ -1,77 +0,0 @@ -""" -This file handles their access credentials and tokens for various APIs required -to retrieve data for the website. This file also handles retrieving config -variables, which are either stored in the application config or (if there is no -active Flask app context) the system environment. - -The file that contains the credentials is called "vault.zip", and is referenced -by a constant, `VAULT_FILE`. This file is accessed using a password stored in -the config called `VAULT_PASSWORD`. - -Inside the "vault.zip" file, there is a file named "keys.yml." And this is the -file with all the credentials (plus a Flask secret key). It looks like this: - - flask: - secret_key: "" - hobolink: - password: "" - user: "crwa" - token: "" -""" -import os -import zipfile -import json -from distutils.util import strtobool -from flask import current_app - -from flagging_site.config import VAULT_FILE - - -def get_keys() -> dict: - """Retrieves the keys from the `current_app` if it exists. If not, then this - function tries to load directly from the vault. The reason this function - exists is so that you can use the API wrappers regardless of whether or not - the Flask app is running. - - Note that this function requires that you assign the vault password to the - environmental variable named `VAULT_PASSWORD`. - - Returns: - The full keys dict. - """ - if current_app: - d = current_app.config['KEYS'] - else: - vault_file = os.getenv('VAULT_FILE') or VAULT_FILE - d = load_keys_from_vault(vault_password=os.environ['VAULT_PASSWORD'], - vault_file=vault_file) - return d.copy() - - -def load_keys_from_vault( - vault_password: str, - vault_file: str = VAULT_FILE -) -> dict: - """This code loads the keys directly from the vault zip file. Users should - preference using the `get_keys()` function over this function. - - Args: - vault_password: (str) Password for opening up the `vault_file`. - vault_file: (str) File path of the zip file containing `keys.json`. - - Returns: - Dict of credentials. - """ - pwd = bytes(vault_password, 'utf-8') - with zipfile.ZipFile(vault_file) as f: - with f.open('keys.json', pwd=pwd, mode='r') as keys_file: - d = json.load(keys_file) - return d - - -def get_data_store_file_path(file_name: str) -> str: - if current_app: - return os.path.join(current_app.config['DATA_STORE'], file_name) - else: - from ..config import DATA_STORE - return os.path.join(DATA_STORE, file_name) diff --git a/flagging_site/data/usgs.py b/flagging_site/data/usgs.py index ab41c70c..fc93aa50 100644 --- a/flagging_site/data/usgs.py +++ b/flagging_site/data/usgs.py @@ -5,16 +5,16 @@ Link to the web interface (not the api) https://waterdata.usgs.gov/nwis/uv?site_no=01104500 """ +import os import pandas as pd import requests from flask import abort from flask import current_app -from .keys import get_data_store_file_path # Constants USGS_URL = 'https://waterservices.usgs.gov/nwis/iv/' -STATIC_FILE_NAME = 'usgs.pickle' +USGS_STATIC_FILE_NAME = 'usgs.pickle' # ~ ~ ~ ~ @@ -26,7 +26,10 @@ def get_live_usgs_data() -> pd.DataFrame: Pandas Dataframe containing the usgs data. """ if current_app.config['OFFLINE_MODE']: - df = pd.read_pickle(get_data_store_file_path(STATIC_FILE_NAME)) + fpath = os.path.join( + current_app.config['DATA_STORE'], USGS_STATIC_FILE_NAME + ) + df = pd.read_pickle(fpath) else: res = request_to_usgs() df = parse_usgs_data(res) diff --git a/run_unix_dev.sh b/run_unix_dev.sh index 9e6efd34..5015d885 100644 --- a/run_unix_dev.sh +++ b/run_unix_dev.sh @@ -26,11 +26,11 @@ $PYEXEC -m pip install $(cat requirements.txt | grep -v "psycopg2==") export FLASK_APP=flagging_site:create_app export FLASK_ENV=development read -p "Offline mode? [y/n]: " offline_mode -export OFFLINE_MODE=${offline_mode} +export OFFLINE_MODE=${offline_mode:-${OFFLINE_MODE}} read -p "Enter vault password: " vault_pw -export VAULT_PASSWORD=${vault_pw} +export VAULT_PASSWORD=${vault_pw:-${VAULT_PASSWORD}} read -p "Enter Postgres password: " postgres_pw -export POSTGRES_PASSWORD=${postgres_pw} +export $POSTGRES_PASSWORD=${postgres_pw:-${POSTGRES_PASSWORD}} flask create-db flask init-db From 405b5be1d9cbcb9dce1f4ff878d1d07e4343b898 Mon Sep 17 00:00:00 2001 From: dwreeves Date: Tue, 15 Sep 2020 22:49:01 -0400 Subject: [PATCH 068/118] added procfile comments --- Procfile | 9 ++++++++- 1 file changed, 8 insertions(+), 1 deletion(-) diff --git a/Procfile b/Procfile index 81305c19..dc798cc4 100644 --- a/Procfile +++ b/Procfile @@ -1 +1,8 @@ -web: gunicorn "flagging_site:create_app()" \ No newline at end of file +# ~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ +# This file is used to deploy the website to Heroku +# +# See here for more: +# https://devcenter.heroku.com/articles/procfile +# ~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ + +web: gunicorn "flagging_site:create_app()" From 6012d0c86687b39873e4fa1e20adb5299d01214a Mon Sep 17 00:00:00 2001 From: dwreeves Date: Tue, 15 Sep 2020 22:49:53 -0400 Subject: [PATCH 069/118] gitignore pgadmin4 --- .gitignore | 1 + 1 file changed, 1 insertion(+) diff --git a/.gitignore b/.gitignore index 18d828a8..e0a6fcb5 100644 --- a/.gitignore +++ b/.gitignore @@ -6,6 +6,7 @@ keys.json .DS_Store Pipfile Pipfile.lock +pgadmin4/ # Byte-compiled / optimized / DLL files __pycache__/ From b8aba68ea10f67883e9d57bc51aa49ef319a14b4 Mon Sep 17 00:00:00 2001 From: dwreeves Date: Tue, 15 Sep 2020 22:53:29 -0400 Subject: [PATCH 070/118] single source of truth for versioning --- flagging_site/__init__.py | 1 - setup.py | 11 +++++++---- 2 files changed, 7 insertions(+), 5 deletions(-) diff --git a/flagging_site/__init__.py b/flagging_site/__init__.py index 11ea373b..338fc771 100644 --- a/flagging_site/__init__.py +++ b/flagging_site/__init__.py @@ -1,4 +1,3 @@ - __version__ = '0.3.0' from .app import create_app diff --git a/setup.py b/setup.py index 2fdd1c2e..77303776 100644 --- a/setup.py +++ b/setup.py @@ -1,9 +1,12 @@ # -*- coding: utf-8 -*- -import io +import os from setuptools import setup, find_packages -with io.open('README.md', 'rt', encoding='utf8') as f: +with open(os.path.join('flagging_site', '__init__.py'), encoding='utf8') as f: + version = re.search(r"__version__ = '(.*?)'", f.read()).group(1) + +with open('README.md', encoding='utf8') as f: readme = f.read() @@ -27,8 +30,8 @@ 'pandas', 'flask', 'flasgger', - # 'psycopg2', - # 'Flask-SQLAlchemy' + 'psycopg2', + 'Flask-SQLAlchemy' ], url='https://github.com/codeforboston/flagging', description='Flagging website for the CRWA', From 6f29187c662ade6c9fd696449db9c7adf7284c8f Mon Sep 17 00:00:00 2001 From: dwreeves Date: Tue, 15 Sep 2020 22:54:01 -0400 Subject: [PATCH 071/118] single source of truth for versioning --- setup.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/setup.py b/setup.py index 77303776..f73176aa 100644 --- a/setup.py +++ b/setup.py @@ -12,7 +12,7 @@ setup( name='CRWA Flagging Website', - version='0.3.0', + version=version, packages=find_packages(), author='Code for Boston', python_requires='>=3.7.1', From 72de7be1b52811e1813b764b72524560bba9955c Mon Sep 17 00:00:00 2001 From: dwreeves Date: Tue, 15 Sep 2020 23:26:17 -0400 Subject: [PATCH 072/118] refactoring data portion of code --- flagging_site/blueprints/flagging.py | 4 +++- flagging_site/data/database.py | 2 +- flagging_site/data/hobolink.py | 13 +++++++------ flagging_site/data/model.py | 22 ++++++++++++++++------ 4 files changed, 27 insertions(+), 14 deletions(-) diff --git a/flagging_site/blueprints/flagging.py b/flagging_site/blueprints/flagging.py index e424c630..490072ad 100644 --- a/flagging_site/blueprints/flagging.py +++ b/flagging_site/blueprints/flagging.py @@ -36,6 +36,8 @@ def stylize_model_output(df: pd.DataFrame) -> str: Returns: HTML table. """ + df = df.copy() + def _apply_flag(x: bool) -> str: flag_class = 'blue-flag' if x else 'red-flag' return f'{x}' @@ -44,7 +46,7 @@ def _apply_flag(x: bool) -> str: df.columns = [i.title().replace('_', ' ') for i in df.columns] # remove reach number - df = df.drop('Reach', 1) + df = df.drop(columns=['Reach']) return df.to_html(index=False, escape=False) diff --git a/flagging_site/data/database.py b/flagging_site/data/database.py index 00034dd4..61eeaada 100644 --- a/flagging_site/data/database.py +++ b/flagging_site/data/database.py @@ -93,7 +93,7 @@ def update_database(): # Populate the `hobolink` table. from .hobolink import get_live_hobolink_data - df_hobolink = get_live_hobolink_data('code_for_boston_export_21d') + df_hobolink = get_live_hobolink_data() df_hobolink.to_sql('hobolink', **options) from .model import process_data diff --git a/flagging_site/data/hobolink.py b/flagging_site/data/hobolink.py index 8c07aae0..8415211a 100644 --- a/flagging_site/data/hobolink.py +++ b/flagging_site/data/hobolink.py @@ -16,7 +16,7 @@ # Constants HOBOLINK_URL = 'http://webservice.hobolink.com/restv2/data/custom/file' -DEFAULT_HOBOLINK_EXPORT_NAME = 'code_for_boston_export' +DEFAULT_HOBOLINK_EXPORT_NAME = 'code_for_boston_export_21d' # Each key is the original column name; the value is the renamed column. HOBOLINK_COLUMNS = { 'Time, GMT-04:00': 'time', @@ -97,15 +97,16 @@ def parse_hobolink_data(res: str) -> pd.DataFrame: Returns: Pandas DataFrame containing the HOBOlink data. """ - # TODO: - # The first half of the output is a yaml-formatted text stream. Is there - # anything useful in it? Can we store it and make use of it somehow? if isinstance(res, requests.models.Response): res = res.text - # Turn the text from the API response into a Pandas DataFrame. + # The first half of the text from the response is a yaml. The part below + # the yaml is the actual data. The following lines split the text and grab + # the csv: split_by = '------------' str_table = res[res.find(split_by) + len(split_by):] + + # Turn the text from the API response into a Pandas DataFrame. df = pd.read_csv(io.StringIO(str_table), sep=',') # There is a weird issue in the HOBOlink data where it sometimes returns @@ -137,6 +138,6 @@ def parse_hobolink_data(res: str) -> pd.DataFrame: df = df.loc[df['water_temp'].notna()] # Convert time column to Pandas datetime - df['time'] = pd.to_datetime(df['time']) + df['time'] = pd.to_datetime(df['time'], format='%m/%d/%y %H:%M:%S') return df diff --git a/flagging_site/data/model.py b/flagging_site/data/model.py index ad8a2d40..b8fdac7f 100644 --- a/flagging_site/data/model.py +++ b/flagging_site/data/model.py @@ -141,6 +141,7 @@ def reach_2_model(df: pd.DataFrame, rows: int = 48) -> pd.DataFrame: Args: df: Input data from `process_data()` + rows: (int) Number of rows to return. Returns: Outputs for model as a dataframe. @@ -269,13 +270,22 @@ def latest_model_outputs(hours: int = 1) -> dict: if hours == 1: df = execute_sql_from_file('return_1_hour_of_model_outputs.sql') + elif hours > 1: - df = execute_sql_from_file('return_48_hours_of_model_outputs.sql') # pull out 48 hours of model outputs - latest_time = max(df['time']) # find most recent timestamp - time_interval = pd.Timedelta(str(hours) + ' hours') # create pandas Timedelta, based on input parameter hours - df = df[ latest_time - df['time']< time_interval ] # reset df to exclude anything from before time_interval ago + # pull out 48 hours of model outputs + df = execute_sql_from_file('return_48_hours_of_model_outputs.sql') + + # find most recent timestamp + latest_time = df['time'].max() + + # create pandas Timedelta, based on input parameter hours + time_interval = pd.Timedelta(str(hours) + ' hours') + + # reset df to exclude anything from before time_interval ago + df = df[latest_time - df['time'] < time_interval] else: - raise ValueError('hours of data to pull cannot be less than one') + raise ValueError('Hours of data to pull must be a number and it ' + 'cannot be less than one') - return df \ No newline at end of file + return df From d4bf6760586864dca2a24302c5ca249b96a19646 Mon Sep 17 00:00:00 2001 From: dwreeves Date: Tue, 15 Sep 2020 23:27:26 -0400 Subject: [PATCH 073/118] delete depreciated todo --- flagging_site/data/hobolink.py | 4 ---- 1 file changed, 4 deletions(-) diff --git a/flagging_site/data/hobolink.py b/flagging_site/data/hobolink.py index 8415211a..c6bf351a 100644 --- a/flagging_site/data/hobolink.py +++ b/flagging_site/data/hobolink.py @@ -2,10 +2,6 @@ This file handles connections to the HOBOlink API, including cleaning and formatting of the data that we receive from it. """ -# TODO: -# Pandas is inefficient. It should go to SQL, not to Pandas. I am currently -# using pandas because we do not have any cron jobs or any caching or SQL, but -# I think in future versions we should not be using Pandas at all. import os import io import requests From 630d2b8f49045acbc742552b1a9bab7862d7a0a6 Mon Sep 17 00:00:00 2001 From: dwreeves Date: Wed, 16 Sep 2020 23:08:31 -0400 Subject: [PATCH 074/118] changed docs setup --- .gitignore | 1 + docs/README.md | 13 +++++++------ docs/docs/development/data.md | 2 +- .../data/{model.py => predictive_models.py} | 0 4 files changed, 9 insertions(+), 7 deletions(-) rename flagging_site/data/{model.py => predictive_models.py} (100%) diff --git a/.gitignore b/.gitignore index e0a6fcb5..275d1fc5 100644 --- a/.gitignore +++ b/.gitignore @@ -7,6 +7,7 @@ keys.json Pipfile Pipfile.lock pgadmin4/ +mkdocs_env/ # Byte-compiled / optimized / DLL files __pycache__/ diff --git a/docs/README.md b/docs/README.md index 5d88704a..dc6f21a6 100644 --- a/docs/README.md +++ b/docs/README.md @@ -7,20 +7,21 @@ The full docs are available at: https://codeforboston.github.io/flagging/ If you have write permission to the upstream repository (i.e. you are a project manager), point your terminal to this directory and run the following: ```shell script +python3 -m venv mkdocs_env +source mkdocs_env/bin/activate pip install mkdocs pymdown-extensions mkdocs-material mkdocs gh-deploy --remote-name upstream +deactivate +source ../venv/bin/activate ``` If you do not have write permission to the upstream repository, you can do one of the following: 1. (Preferred) Ask a project manager to refresh the pages after you've made changes to the docs. - 2. Run `mkdocs gh-deploy` on your own fork, and then do a pull request to `codeforboston:gh-pages` - - If you are a project manager but you're having issues, you can do a more manual git approach to updating the docs: + 2. Run `mkdocs gh-deploy` on your own fork, and then do a pull request to `codeforboston:gh-pages`: ```shell script -pip install mkdocs pymdown-extensions mkdocs-material mkdocs gh-deploy git checkout gh-pages -git push upstream gh-pages -``` \ No newline at end of file +git push origin gh-pages +``` diff --git a/docs/docs/development/data.md b/docs/docs/development/data.md index 5c22cf64..9c1599cf 100644 --- a/docs/docs/development/data.md +++ b/docs/docs/development/data.md @@ -52,7 +52,7 @@ The HOBOlink data is also notoriously slow to retrieve (regardless of whether yo ### Combining the data -Additional information related to combining the data and how the models work is in the [Predictive Models](../predictive_models.md) page. +Additional information related to combining the data and how the models work is in the [Predictive Models](../predictive_models) page. ## Postgres Database diff --git a/flagging_site/data/model.py b/flagging_site/data/predictive_models.py similarity index 100% rename from flagging_site/data/model.py rename to flagging_site/data/predictive_models.py From 8a138aa25a202321d0bf6f2da4512f08156aef42 Mon Sep 17 00:00:00 2001 From: dwreeves Date: Wed, 16 Sep 2020 23:09:22 -0400 Subject: [PATCH 075/118] Refactors and code cleaning --- .flaskenv | 3 +- flagging_site/app.py | 11 ++- flagging_site/blueprints/api.py | 2 +- flagging_site/blueprints/flagging.py | 4 +- flagging_site/config.py | 41 +++++--- flagging_site/data/_store/refresh.py | 18 ++-- flagging_site/data/database.py | 94 ++++++++++++++----- .../data/queries/define_boathouse.sql | 82 +++------------- .../return_48_hours_of_model_outputs.sql | 6 +- flagging_site/data/queries/schema.sql | 4 +- flagging_site/data/usgs.py | 13 ++- requirements.txt | 11 --- setup.py | 16 +++- 13 files changed, 158 insertions(+), 147 deletions(-) diff --git a/.flaskenv b/.flaskenv index 1698ff4c..dfe2a1a6 100644 --- a/.flaskenv +++ b/.flaskenv @@ -1 +1,2 @@ -FLASK_APP=flagging_site \ No newline at end of file +FLASK_APP=flagging_site +FLASK_ENV=development diff --git a/flagging_site/app.py b/flagging_site/app.py index 39a73237..92a0109c 100644 --- a/flagging_site/app.py +++ b/flagging_site/app.py @@ -89,6 +89,7 @@ def init_db_command(): @app.cli.command('update-db') def update_db_command(): + """Update the database with the latest live data.""" from .data.database import update_database update_database() click.echo('Updated the database.') @@ -96,12 +97,17 @@ def update_db_command(): # Make a few useful functions available in Flask shell without imports @app.shell_context_processor def make_shell_context(): + import pandas as pd + import numpy as np from .blueprints.flagging import get_data + from .data import db from .data.hobolink import get_live_hobolink_data - from .data.model import process_data + from .data.predictive_models import process_data from .data.usgs import get_live_usgs_data return { + 'pd': pd, + 'np': np, 'db': db, 'get_data': get_data, 'get_live_hobolink_data': get_live_hobolink_data, @@ -150,7 +156,8 @@ def add_swagger_plugin_to_app(app: Flask): } app.config['SWAGGER'] = { 'uiversion': 3, - 'favicon': LazyString(lambda: url_for('static', filename='favicon/favicon.ico')) + 'favicon': LazyString( + lambda: url_for('static', filename='favicon/favicon.ico')) } Swagger(app, config=swagger_config, template=template) diff --git a/flagging_site/blueprints/api.py b/flagging_site/blueprints/api.py index cfd52788..9bda2ea5 100644 --- a/flagging_site/blueprints/api.py +++ b/flagging_site/blueprints/api.py @@ -8,7 +8,7 @@ from flask_restful import Api from flask_restful import Resource from flask import current_app -from ..data.model import latest_model_outputs +from ..data.predictive_models import latest_model_outputs from flasgger import swag_from diff --git a/flagging_site/blueprints/flagging.py b/flagging_site/blueprints/flagging.py index 490072ad..12cbed68 100644 --- a/flagging_site/blueprints/flagging.py +++ b/flagging_site/blueprints/flagging.py @@ -9,8 +9,8 @@ from ..data.cyano_overrides import get_currently_overridden_reaches from ..data.hobolink import get_live_hobolink_data from ..data.usgs import get_live_usgs_data -from ..data.model import process_data -from ..data.model import latest_model_outputs +from ..data.predictive_models import process_data +from ..data.predictive_models import latest_model_outputs from ..data.database import get_boathouse_dict bp = Blueprint('flagging', __name__) diff --git a/flagging_site/config.py b/flagging_site/config.py index 5fecfcf2..a0f22e24 100644 --- a/flagging_site/config.py +++ b/flagging_site/config.py @@ -10,6 +10,7 @@ from flask.cli import load_dotenv from distutils.util import strtobool + # Constants # ~~~~~~~~~ @@ -18,15 +19,6 @@ DATA_STORE = os.path.join(ROOT_DIR, 'data', '_store') VAULT_FILE = os.path.join(ROOT_DIR, 'vault.zip') -# Dotenv -# ~~~~~~ - -# If you are using a .env file, please double check that it is gitignored. -# The `.flaskenv` file should not be gitignored, only `.env`. -# See this for more: -# https://flask.palletsprojects.com/en/1.1.x/cli/ -load_dotenv(os.path.join(ROOT_DIR, '..', '.flaskenv')) -load_dotenv(os.path.join(ROOT_DIR, '..', '.env')) # Configs # ~~~~~~~ @@ -60,6 +52,13 @@ def __repr__(self): @property def SQLALCHEMY_DATABASE_URI(self) -> str: + """ + Returns the URI for the Postgres database. + + Example: + >>> Config().SQLALCHEMY_DATABASE_URI + 'postgres://postgres:password_here@localhost:5432/flagging' + """ user = self.POSTGRES_USER password = self.POSTGRES_PASSWORD host = self.POSTGRES_HOST @@ -72,6 +71,9 @@ def SQLALCHEMY_DATABASE_URI(self) -> str: SQLALCHEMY_TRACK_MODIFICATIONS: bool = False QUERIES_DIR: str = QUERIES_DIR + """Directory that contains various queries that are accessible throughout + the rest of the code base. + """ # ========================================================================== # MISC. CUSTOM CONFIG OPTIONS @@ -87,6 +89,7 @@ def SQLALCHEMY_DATABASE_URI(self) -> str: """ VAULT_PASSWORD: str = os.getenv('VAULT_PASSWORD') + """Password """ KEYS: Dict[str, Dict[str, Any]] = None """These are where the keys from the vault are stored. It should be a dict @@ -131,8 +134,20 @@ class ProductionConfig(Config): is the `flagging` part, so that's the only blueprint we import. """ def __init__(self): - self.BASIC_AUTH_USERNAME: str = os.environ['BASIC_AUTH_USERNAME'] - self.BASIC_AUTH_PASSWORD: str = os.environ['BASIC_AUTH_PASSWORD'] + """Initializing the production config allows us to ensure the existence + of these variables in the environment.""" + try: + self.VAULT_PASSWORD: str = os.environ['VAULT_PASSWORD'] + self.BASIC_AUTH_USERNAME: str = os.environ['BASIC_AUTH_USERNAME'] + self.BASIC_AUTH_PASSWORD: str = os.environ['BASIC_AUTH_PASSWORD'] + except KeyError: + msg = ( + 'You did not set all of the environment variables required to ' + 'initiate the app in production mode. If you are deploying ' + 'the website to Heroku, read the Deployment docs page to ' + 'learn how to set env variables in Heroku.' + ) + raise KeyError(msg) class DevelopmentConfig(Config): @@ -152,8 +167,6 @@ class DevelopmentConfig(Config): DEBUG: bool = True TESTING: bool = True OFFLINE_MODE = strtobool(os.getenv('OFFLINE_MODE') or 'false') - BASIC_AUTH_USERNAME: str = os.getenv('BASIC_AUTH_USERNAME', 'admin') - BASIC_AUTH_PASSWORD: str = os.getenv('BASIC_AUTH_PASSWORD', 'password') class TestingConfig(Config): @@ -161,8 +174,6 @@ class TestingConfig(Config): website. """ TESTING: bool = True - BASIC_AUTH_USERNAME: str = os.getenv('BASIC_AUTH_USERNAME', 'admin') - BASIC_AUTH_PASSWORD: str = os.getenv('BASIC_AUTH_PASSWORD', 'password') def get_config_from_env(env: str) -> Config: diff --git a/flagging_site/data/_store/refresh.py b/flagging_site/data/_store/refresh.py index 933d22f1..21db83f6 100644 --- a/flagging_site/data/_store/refresh.py +++ b/flagging_site/data/_store/refresh.py @@ -5,6 +5,8 @@ This file is a CLI to refresh the data store. You can run it with: `python flagging_site/data/_store/refresh.py` + + """ import os import sys @@ -12,6 +14,8 @@ import click +DATA_STORE_PATH = os.path.dirname(__file__) + @click.command() @click.option('--vault_password', prompt=True, @@ -31,21 +35,17 @@ def refresh_data_store(vault_password: Optional[str] = None) -> None: raise Exception('The app should not be running when the data store is ' 'being refreshed.') - from flagging_site.data.keys import get_data_store_file_path - from flagging_site.data.hobolink import get_live_hobolink_data from flagging_site.data.hobolink import HOBOLINK_STATIC_FILE_NAME get_live_hobolink_data('code_for_boston_export_21d')\ - .to_pickle(get_data_store_file_path(HOBOLINK_STATIC_FILE_NAME)) + .to_pickle(os.path.join(DATA_STORE_PATH, HOBOLINK_STATIC_FILE_NAME)) from flagging_site.data.usgs import get_live_usgs_data from flagging_site.data.usgs import USGS_STATIC_FILE_NAME - get_live_usgs_data().to_pickle(get_data_store_file_path(USGS_STATIC_FILE_NAME)) + get_live_usgs_data()\ + .to_pickle(os.path.join(DATA_STORE_PATH, USGS_STATIC_FILE_NAME)) if __name__ == '__main__': - try: - sys.path.append('.') - refresh_data_store() - finally: - sys.path.remove('.') + sys.path.append('.') + refresh_data_store() diff --git a/flagging_site/data/database.py b/flagging_site/data/database.py index 61eeaada..e352640e 100644 --- a/flagging_site/data/database.py +++ b/flagging_site/data/database.py @@ -1,6 +1,13 @@ -""" -This file should handle all database connection stuff, namely: writing and -retrieving data. +"""This file handles all database stuff, i.e. writing and retrieving data to +the Postgres database. Note that of the functionality in this file is available +directly in the command line. + +While the app is running, the database connection is managed by SQLAlchemy. The +`db` object defined near the top of the file is that connector, and is used +throughout both this file and other files in the code base. The `db` object is +connected to the actual database in the `create_app` function: the app instance +is passed in via `db.init_app(app)`, and the `db` object looks for the config +variable `SQLALCHEMY_DATABASE_URI`. """ import os import pandas as pd @@ -17,9 +24,17 @@ def execute_sql(query: str) -> Optional[pd.DataFrame]: - """Execute arbitrary SQL in the database. This works for both read and write - operations. If it is a write operation, it will return None; otherwise it - returns a Pandas dataframe.""" + """Execute arbitrary SQL in the database. This works for both read and + write operations. If it is a write operation, it will return None; + otherwise it returns a Pandas dataframe. + + Args: + query: (str) A string that contains the contents of a SQL query. + + Returns: + Either a Pandas Dataframe the selected data for read queries, or None + for write queries. + """ with db.engine.connect() as conn: res = conn.execute(query) try: @@ -32,32 +47,53 @@ def execute_sql(query: str) -> Optional[pd.DataFrame]: return None -def execute_sql_from_file(file_name: str): +def execute_sql_from_file(file_name: str) -> Optional[pd.DataFrame]: + """Execute SQL from a file in the `QUERIES_DIR` directory, which should be + located at `flagging_site/data/queries`. + + Args: + file_name: (str) A file name inside the `QUERIES_DIR` directory. It + should be only the file name alone and not the full path. + + Returns: + Either a Pandas Dataframe the selected data for read queries, or None + for write queries. + """ path = os.path.join(current_app.config['QUERIES_DIR'], file_name) with current_app.open_resource(path) as f: return execute_sql(f.read().decode('utf8')) -def create_db(): - """If database doesn't exist, create it and return True, - otherwise leave it alone and return False""" +def create_db() -> bool: + """If the database defined by `POSTGRES_DBNAME` doesn't exist, create it + and return True, otherwise do nothing and return False. By default, the + config variable `POSTGRES_DBNAME` is set to "flagging". + + Returns: + bool for whether the database needed to be created. + """ # connect to postgres database, get cursor - conn = connect(dbname='postgres', user=current_app.config['POSTGRES_USER'], + conn = connect( + dbname='postgres', + user=current_app.config['POSTGRES_USER'], host=current_app.config['POSTGRES_HOST'], - password=current_app.config['POSTGRES_PASSWORD']) + password=current_app.config['POSTGRES_PASSWORD'] + ) cursor = conn.cursor() # get a list of all databases: cursor.execute('SELECT datname FROM pg_database;') - db_list = cursor.fetchall() # db_list here is a list of one-element tuples - db_list = [d[0] for d in db_list] # this converts db_list to a list of db names + + # create a list of all available database names: + db_list = cursor.fetchall() + db_list = [d[0] for d in db_list] # if that database is already there, exit out of this function if current_app.config['POSTGRES_DBNAME'] in db_list: return False - # since the database isn't already there, proceed ... - + # if the database isn't already there, proceed ... + # create the database cursor.execute('COMMIT;') cursor.execute('CREATE DATABASE ' + current_app.config['POSTGRES_DBNAME']) @@ -67,18 +103,30 @@ def create_db(): def init_db(): - """Clear existing data and create new tables.""" + """This data clears and then populates the database from scratch. You only + need to run this function once per instance of the database. + """ with current_app.app_context(): - # Read the `schema.sql` file, which initializes the database. + # This file drops the tables if they already exist, and then defines + # the tables. This is the only query that CREATES tables. execute_sql_from_file('schema.sql') + + # The boathouses table is populated. This table doesn't change, so it + # only needs to be populated once. execute_sql_from_file('define_boathouse.sql') + + # The function that updates the database periodically is run for the + # first time. update_database() + + # The models available in Base are given corresponding tables if they + # do not already exist. Base.metadata.create_all(db.engine) def update_database(): - """At the moment this overwrites the entire database. In the future we want - this to simply update it. + """This function basically controls all of our data refreshes. The + following tables """ options = { 'con': db.engine, @@ -96,11 +144,13 @@ def update_database(): df_hobolink = get_live_hobolink_data() df_hobolink.to_sql('hobolink', **options) - from .model import process_data + # Populate the `processed_data` table. + from .predictive_models import process_data df = process_data(df_hobolink=df_hobolink, df_usgs=df_usgs) df.to_sql('processed_data', **options) - from .model import all_models + # Populate the `model_outputs` table. + from .predictive_models import all_models model_outs = all_models(df) model_outs.to_sql('model_outputs', **options) diff --git a/flagging_site/data/queries/define_boathouse.sql b/flagging_site/data/queries/define_boathouse.sql index 8f15342c..f850a5a8 100644 --- a/flagging_site/data/queries/define_boathouse.sql +++ b/flagging_site/data/queries/define_boathouse.sql @@ -1,68 +1,14 @@ -INSERT INTO boathouses (reach, boathouse, latitude, longitude) -VALUES ( - 2, - 'Newton Yacht Club', - 42.358698, - 71.172850 - ), - ( - 2, - 'Watertown Yacht Club', - 42.361952, - 71.167791 - ), - ( - 2, - 'Community Rowing, Inc.', - 42.358633, - 71.165467 - ), - ( - 2, - 'Northeastern''s Henderson Boathouse', - 42.364135, - 71.141571 - ), - ( - 2, - 'Paddle Boston at Herter Park', - 42.369182, - 71.131301 - ), - ( - 3, - 'Harvard''s Weld Boathouse', - 42.369566, - 71.122083 - ), - ( - 4, - 'Riverside Boat Club', - 42.358272, - 71.115763 - ), - ( - 5, - 'Charles River Yacht Club', - 42.360526, - 71.084760 - ), - ( - 5, - 'Union Boat Club', - 42.357816, - 71.073319 - ), - ( - 5, - 'Community Boating', - 42.359935, - 71.073035 - ), - ( - 5, - 'Paddle Boston at Kendall Square', - 42.362964, - 71.082112 - ); - +INSERT INTO boathouses + (reach, boathouse, latitude, longitude) +VALUES + (2, 'Newton Yacht Club', 42.358698, 71.172850), + (2, 'Watertown Yacht Club', 42.361952, 71.167791), + (2, 'Community Rowing, Inc.', 42.358633, 71.165467), + (2, 'Northeastern''s Henderson Boathouse', 42.364135, 71.141571), + (2, 'Paddle Boston at Herter Park', 42.369182, 71.131301), + (3, 'Harvard''s Weld Boathouse', 42.369566, 71.122083), + (4, 'Riverside Boat Club', 42.358272, 71.115763), + (5, 'Charles River Yacht Club', 42.360526, 71.084760), + (5, 'Union Boat Club', 42.357816, 71.073319), + (5, 'Community Boating', 42.359935, 71.073035), + (5, 'Paddle Boston at Kendall Square', 42.362964, 71.082112); diff --git a/flagging_site/data/queries/return_48_hours_of_model_outputs.sql b/flagging_site/data/queries/return_48_hours_of_model_outputs.sql index 9e1cd8fd..9f1d364c 100644 --- a/flagging_site/data/queries/return_48_hours_of_model_outputs.sql +++ b/flagging_site/data/queries/return_48_hours_of_model_outputs.sql @@ -3,6 +3,6 @@ SELECT * FROM model_outputs WHERE time BETWEEN - (SELECT MAX(time) - interval '47 hours' FROM model_outputs) - AND - (SELECT MAX(time) FROM model_outputs) \ No newline at end of file + (SELECT MAX(time) - interval '47 hours' FROM model_outputs) + AND + (SELECT MAX(time) FROM model_outputs) diff --git a/flagging_site/data/queries/schema.sql b/flagging_site/data/queries/schema.sql index 704457d0..76c2fa53 100644 --- a/flagging_site/data/queries/schema.sql +++ b/flagging_site/data/queries/schema.sql @@ -9,9 +9,9 @@ DROP TABLE IF EXISTS hobolink; CREATE TABLE IF NOT EXISTS hobolink ( time timestamp, pressure decimal, - par decimal, /* photosynthetically active radiation */ + par decimal, -- photosynthetically active radiation rain decimal, - rh decimal, /* relative humidity */ + rh decimal, -- relative humidity dew_point decimal, wind_speed decimal, gust_speed decimal, diff --git a/flagging_site/data/usgs.py b/flagging_site/data/usgs.py index fc93aa50..5da92650 100644 --- a/flagging_site/data/usgs.py +++ b/flagging_site/data/usgs.py @@ -19,8 +19,8 @@ def get_live_usgs_data() -> pd.DataFrame: - """This function runs through the whole process for retrieving data from - usgs: first we perform the request, and then we clean the data. + """This function runs through the whole process for retrieving data from + usgs: first we perform the request, and then we parse the data. Returns: Pandas Dataframe containing the usgs data. @@ -37,13 +37,12 @@ def get_live_usgs_data() -> pd.DataFrame: def request_to_usgs() -> requests.models.Response: - """ - Get a request from the USGS. + """Get a request from the USGS. Returns: Request Response containing the data from the request. """ - + payload = { 'format': 'json', 'sites': '01104500', @@ -51,7 +50,7 @@ def request_to_usgs() -> requests.models.Response: 'parameterCd': '00060,00065', 'siteStatus': 'all' } - + res = requests.get(USGS_URL, params=payload) if res.status_code // 100 in [4, 5]: error_msg = 'API request to the USGS endpoint failed with status code '\ @@ -92,7 +91,7 @@ def parse_usgs_data(res) -> pd.DataFrame: df['time'] = ( pd.to_datetime(df['time']) # Convert to Pandas datetime format .dt.tz_localize('UTC') # This is UTC; define it as such - .dt.tz_convert('US/Eastern') # Take the UTC time and convert to EST + .dt.tz_convert('US/Eastern') # Take UTC time and convert to EST .dt.tz_localize(None) # Remove the timezone from the datetime ) except TypeError: diff --git a/requirements.txt b/requirements.txt index 091f13b5..a1ffe11f 100644 --- a/requirements.txt +++ b/requirements.txt @@ -16,26 +16,17 @@ coverage==5.2 cryptography==3.0 distlib==0.3.1 dnspython==2.0.0 -email-validator==1.1.1 filelock==3.0.12 flasgger==0.9.4 Flask-Admin==1.5.6 Flask-BabelEx==0.9.4 Flask-BasicAuth==0.2.0 -Flask-Compress==1.4.0 -Flask-Gravatar==0.5.0 -Flask-Login==0.4.1 -Flask-Mail==0.9.1 -Flask-Migrate==2.4.0 -Flask-Paranoid==0.2.0 -Flask-Principal==0.4.0 Flask-RESTful==0.3.8 Flask-Security-Too==3.4.4 Flask-SQLAlchemy==2.4.4 Flask-WTF==0.14.3 Flask==1.1.2 gunicorn==20.0.4 -gunicorn==20.0.4 idna==2.10 importlib-metadata==1.6.1 isort==4.3.21 @@ -54,8 +45,6 @@ packaging==20.4 pandas==1.0.5 paramiko==2.7.1 passlib==1.7.2 -pgadmin4==4.24 -pluggy==0.13.1 pluggy==0.13.1 psutil==5.7.2 psycopg2-binary==2.8.5 diff --git a/setup.py b/setup.py index f73176aa..4cd05491 100644 --- a/setup.py +++ b/setup.py @@ -17,6 +17,7 @@ author='Code for Boston', python_requires='>=3.7.1', maintainer='Charles River Watershed Association', + license='MIT', include_package_data=True, setup_requires=[ 'pytest-runner', @@ -26,14 +27,21 @@ 'pytest-cov' ], install_requires=[ - 'pyyaml', 'pandas', 'flask', + 'jinja2', 'flasgger', - 'psycopg2', - 'Flask-SQLAlchemy' + 'requests', + 'Flask-SQLAlchemy', + 'Flask-Admin', + 'Flask-BasicAuth' ], + extras_require={ + 'windows': ['psycopg2'], + 'osx': ['psycopg2-binary'] + }, url='https://github.com/codeforboston/flagging', description='Flagging website for the CRWA', - long_description=readme + long_description=readme, + long_description_content_type='text/markdown', ) From 53eebe11e9eb6324eac41f0f32d5a565de4812ca Mon Sep 17 00:00:00 2001 From: dwreeves Date: Thu, 17 Sep 2020 18:28:37 -0400 Subject: [PATCH 076/118] Added Twitter bot and new archiving strategy --- docs/docs/development/twitter_bot.md | 57 +++++++++++++++++++ docs/mkdocs.yml | 9 ++- flagging_site/app.py | 80 +++++++++++++++++++++------ flagging_site/twitter_bot.py | 70 +++++++++++++---------- flagging_site/vault.7z | Bin 0 -> 594 bytes flagging_site/vault.zip | Bin 339 -> 0 bytes 6 files changed, 165 insertions(+), 51 deletions(-) create mode 100644 docs/docs/development/twitter_bot.md create mode 100644 flagging_site/vault.7z delete mode 100644 flagging_site/vault.zip diff --git a/docs/docs/development/twitter_bot.md b/docs/docs/development/twitter_bot.md new file mode 100644 index 00000000..43f291f9 --- /dev/null +++ b/docs/docs/development/twitter_bot.md @@ -0,0 +1,57 @@ +## First Time Setup + +Follow these steps to set up the Twitter bot for the first time, such as on a new Twitter account. + +1. Create a [Twitter](https://twitter.com/) account that will host the bot, or login to an account you already have that you want to send automated tweets from. + +2. Go to [https://apps.twitter.com/](https://apps.twitter.com/) and sign up for a development account. Note that you will need both a valid phone number and a valid email tied to the developer account in order to use development features. + +???+ note + You will have to wait an hour or two for Twitter.com to get back to you and approve your developer account. + +3. Once you are approved, go to the [Twitter Developer Portal](https://developer.twitter.com/). Click on the app you created, and in the `Settings` tab, ensure that the App permissions are set to *Read and Write* instead of only *Read*. + +???+ tip + If at some point during step 3 Twitter starts throwing API keys at you, ignore it for now. We'll get all the keys we need in next couple steps. + +4. In the code base, use the `VAULT_PASSWORD` to unzip the `vault.7z` manually. You should have a file called `secrets.json`. Open up `secrets.json` in the plaintext editor of your choosing. + +???+ critical + Make sure that you delete the unencrpyted, unarchived version of the `secrets.json` file after you are done with it. + +5. Now go back to your browser with the Twitter Developer Portal. At the top of the screen, flip to the `Keys and tokens`. Now it's time to go through the dashboard and get your copy+paste ready. We will be inserting these values into the `secrets.json` (remember to wrap the keys in double quotes `"like this"` when you insert them). + + - The `API Key & Secret` should should go in the corresponding fields for `"api_key": "..."` and `"api_key_secret": "..."`. + - The `Bearer Token` should go in the field `"bearer_token": "..."`. + - The `Access Token & Secret` should go in the corresponding fields for `"access_token": "..."` and `"access_token_secret": "..."`. _But first, you will need to regenerate the `Access Token & Secret` so that it has both read and write permissions._ + +???+ success + The `secrets.json` file should look something like this, with the ellipses replacing the actual values: + + ```json + { + "SECRET_KEY": "...", + "HOBOLINK_AUTH": { + "password": "...", + "user": "...", + "token": "..." + }, + "TWITTER_AUTH": { + "api_key": "...", + "api_key_secret": "...", + "bearer_token": "...", + "access_token": "...", + "access_token_secret": "..." + } + } + ``` + +6. Rezip the file. Enter the `VAULT_PASSWORD` when prompted (happens twice). + +```shell +cd flagging_site +7z a -p vault.7z secrets.json +cd .. +``` + +7. Delete delete the unencrpyted, unarchived version of the `secrets.json` file. diff --git a/docs/mkdocs.yml b/docs/mkdocs.yml index f97e536a..1c84dd87 100644 --- a/docs/mkdocs.yml +++ b/docs/mkdocs.yml @@ -5,18 +5,19 @@ nav: - About: about.md - Setup: setup.md - Admin: admin.md -- Deployment: deployment.md +- Remote Deployment: deployment.md - System: system.md - Shell: shell.md - Development Background: - History: development/history.md - Learning Resources: development/learning_resources.md - Development: - # - Overview: development/index.md + - Overview: development/index.md # ^ (Explain the vault here too) - # - Config: development/config.md + - Config: development/config.md - Data: development/data.md - Predictive Models: development/predictive_models.md + - Twitter Bot: development/twitter_bot.md # - Front-End: development/front-end.md theme: name: material @@ -56,3 +57,5 @@ extra: link: https://www.meetup.com/Code-for-Boston/ - icon: fontawesome/brands/twitter link: https://twitter.com/codeforboston +extra_javascript: +- https://unpkg.com/mermaid@8.4.6/dist/mermaid.min.js diff --git a/flagging_site/app.py b/flagging_site/app.py index 92a0109c..b5fd1c2e 100644 --- a/flagging_site/app.py +++ b/flagging_site/app.py @@ -7,9 +7,14 @@ import json import zipfile from typing import Optional +from typing import Dict +from typing import Union from flask import Flask +import py7zr +from lzma import LZMAError + from .config import Config from .config import get_config_from_env @@ -25,7 +30,7 @@ def create_app(config: Optional[Config] = None) -> Flask: Returns: The fully configured Flask app instance. """ - app = Flask(__name__, instance_relative_config=True) + app = Flask(__name__) # Get a config for the website. If one was not passed in the function, then # a config will be used depending on the `FLASK_ENV`. @@ -51,7 +56,7 @@ def create_app(config: Optional[Config] = None) -> Flask: # Add Swagger to the app. Swagger automates the API documentation and # provides an interface for users to query the API on the website. - add_swagger_plugin_to_app(app) + init_swagger(app) # Register the database commands from .data import db @@ -65,6 +70,10 @@ def create_app(config: Optional[Config] = None) -> Flask: from .admin import init_admin init_admin(app) + # Register Twitter bot + from .twitter_bot import init_tweepy + init_tweepy(app) + @app.before_request def before_request(): from flask import g @@ -94,6 +103,13 @@ def update_db_command(): update_database() click.echo('Updated the database.') + @app.cli.command('update-website') + def update_db_command(): + """Update the database with the latest live data.""" + update_db_command() + from .twitter_bot import tweet_out_status + tweet_out_status() + # Make a few useful functions available in Flask shell without imports @app.shell_context_processor def make_shell_context(): @@ -119,8 +135,8 @@ def make_shell_context(): return app -def add_swagger_plugin_to_app(app: Flask): - """This function hnadles all the logic for adding Swagger automated +def init_swagger(app: Flask): + """This function handles all the logic for adding Swagger automated documentation to the application instance. """ from flasgger import Swagger @@ -163,12 +179,30 @@ def add_swagger_plugin_to_app(app: Flask): Swagger(app, config=swagger_config, template=template) -def _load_keys_from_vault( - vault_password: str, +def _load_secrets_from_vault( + password: str, vault_file: str -) -> dict: +) -> Dict[str, Union[str, Dict[str, str]]]: """This code loads the keys directly from the vault zip file. + The schema of the vault's `secrets.json` file looks like this: + + >>> { + >>> "SECRET_KEY": str, + >>> "HOBOLINK_AUTH": { + >>> "password": str, + >>> "user": str, + >>> "token": str + >>> }, + >>> "TWITTER_AUTH": { + >>> "api_key": str, + >>> "api_key_secret": str, + >>> "access_token": str, + >>> "access_token_secret": str, + >>> "bearer_token": str + >>> } + >>> } + Args: vault_password: (str) Password for opening up the `vault_file`. vault_file: (str) File path of the zip file containing `keys.json`. @@ -176,16 +210,15 @@ def _load_keys_from_vault( Returns: Dict of credentials. """ - pwd = bytes(vault_password, 'utf-8') - with zipfile.ZipFile(vault_file) as f: - with f.open('keys.json', pwd=pwd, mode='r') as keys_file: - d = json.load(keys_file) + with py7zr.SevenZipFile(vault_file, mode='r', password=password) as f: + archive = f.readall() + d = json.load(archive['secrets.json']) return d def update_config_from_vault(app: Flask) -> None: """ - This updates the state of the `app` to have the keys from the vault. The + This updates the state of the `app` to have the secrets from the vault. The vault also stores the "SECRET_KEY", which is a Flask builtin configuration variable (i.e. Flask treats the "SECRET_KEY" as special). So we also populate the "SECRET_KEY" in this step. @@ -198,18 +231,29 @@ def update_config_from_vault(app: Flask) -> None: app: A Flask application instance. """ try: - keys = _load_keys_from_vault( - vault_password=app.config['VAULT_PASSWORD'], + secrets = _load_secrets_from_vault( + password=app.config['VAULT_PASSWORD'], vault_file=app.config['VAULT_FILE'] ) - except (RuntimeError, KeyError): + except (LZMAError, KeyError): msg = 'Unable to load the vault; bad password provided.' if app.env == 'production': raise RuntimeError(msg) else: print(f'Warning: {msg}') - app.config['KEYS'] = None + app.config['HOBOLINK_AUTH'] = { + 'password': None, + 'user': None, + 'token': None + } + app.config['TWITTER_AUTH'] = { + 'api_key': None, + 'api_key_secret': None, + 'access_token': None, + 'access_token_secret': None, + 'bearer_token': None + } app.config['SECRET_KEY'] = os.urandom(16) else: - app.config['KEYS'] = keys - app.config['SECRET_KEY'] = keys['flask']['secret_key'] + # Add 'SECRET_KEY', 'HOBOLINK_AUTH', AND 'TWITTER_AUTH' to the config. + app.config.update(secrets) diff --git a/flagging_site/twitter_bot.py b/flagging_site/twitter_bot.py index 806f4614..07f7735d 100644 --- a/flagging_site/twitter_bot.py +++ b/flagging_site/twitter_bot.py @@ -1,42 +1,52 @@ -""" -This is a basic script to output a message to twitter -User must input public and private key from Consumer -and access. They will be imported from keys file - -Future Plans: -Allow more functionality allowing user input +import pandas as pd +import tweepy +from flask import Flask -""" -import tweepy +tweepy_api = tweepy.API() -# Constants for secret and public keys -CONSUMER_KEY = '' -CONSUMER_SECRET = '' -ACCESS_KEY = '' -ACCESS_SECRET = '' +def init_tweepy(app: Flask): + # Pass Twitter API tokens into Tweepy's OAuthHandler + auth = tweepy.OAuthHandler( + consumer_key=app.config['TWITTER_AUTH']['api_key'], + consumer_secret=app.config['TWITTER_AUTH']['api_key_secret'] + ) + auth.set_access_token( + key=app.config['TWITTER_AUTH']['access_token'], + secret=app.config['TWITTER_AUTH']['access_token_secret'] + ) -def post_tweet(msg): - """ - Posts tweet onto twitter handle + # Register the auth defined above + tweepy_api.auth = auth - arg: accept string message msg - returns: string message that have been inputed - """ +def tweet_out_status(): + from .data.predictive_models import latest_model_outputs + from .data.cyano_overrides import get_currently_overridden_reaches - # Authenticates using consumer key and secret - auth=tweepy.OAuthHandler(CONSUMER_KEY, CONSUMER_SECRET) + df = latest_model_outputs() + df = df.set_index('reach') - # Authenticates using access key and secret - auth.set_access_token(ACCESS_KEY, ACCESS_SECRET) - api = tweepy.API(auth) + overridden_reaches = get_currently_overridden_reaches() + flags = { + reach: val['safe'] and reach not in overridden_reaches + for reach, val + in df.to_dict(orient='index').items() + } - #print message on terminal and update status - # to twitter handle - print ('Tweeting' + msg) - api.update_status(msg) + current_time = pd.to_datetime('today').strftime('%I:%M:%S %p, %m/%d/%Y') - return msg \ No newline at end of file + if all(i for i in flags.values()): + msg = ( + 'Our predictive model is reporting all reaches are safe for ' + f'recreational activities as of {current_time}.' + ) + else: + unsafe = ', '.join([str(k) for k, v in flags.items() if v is False]) + msg = ( + 'Our predictive model is reporting that the following reaches are ' + f'unsafe as of {current_time}: {unsafe}.' + ) + tweepy_api.update_status(msg) diff --git a/flagging_site/vault.7z b/flagging_site/vault.7z new file mode 100644 index 0000000000000000000000000000000000000000..ae587e015145e6376e9ddf48250dcb5e200e2513 GIT binary patch literal 594 zcmV-Y0QjdT_QnqXrOkFlvsBgvl*C31CdcuZfD&B*q`E7`? zm(K%SX>jh*FFe<`5sUfW1rR%`F~Pz$r!wqMNc!3)T46!h5i;|8w4^Sl$_ULE#g+X>|dm)&HTEUse1BHid-H9kKs(@iKr4xpC(?AI#MLgp6~chn^(c z@|YgYrAYLP4AwiS>wecRQ*mD-wQMbhY&;oDB;A!y?dUSlC3ga#6c%$iT!Q1un_;^K zvVv=(k-Ro4|GnBXiXR)G7Wemlj&}Bv!-9BUgyPjEYl;E@&gD`Xxw(%qe~erlAtNbE zXE{0&J1;B0T~t~I6T%ex(dvBny`+@{{<_^$9b%eK;USaH)qi1bIIgBoR-a&BhD6vm zgvnSmi}-pJ@)#tOC0>NgGblYwpGOw_9OJSQo{V)q9!n?9x=dCa4=0g;B?vZPc@;yZrdum>qrz(~ z-xY=GUf6VIEaAJ2N94WbY_1Ri1O@;B34y=>2MYlJ0wf0U2LTdO514-3EKnIhYuFW- z0*hJ-%ON2F0096DfwzKp00;^J-QSy~000F65gPz=0A&DU0CE6j0CWIz04@M(0CNCu g0B!&P02u%j3IPBBc~w&~jn)Ab1_1ydfTV!{0C;ftyZ`_I literal 0 HcmV?d00001 diff --git a/flagging_site/vault.zip b/flagging_site/vault.zip deleted file mode 100644 index 0fe88ce411d380187a75343eab2035d8243aeb51..0000000000000000000000000000000000000000 GIT binary patch literal 0 HcmV?d00001 literal 339 zcmWIWW@Zs#;AG%nXsW&t@LfO0Yc>M|!(kxiWRPLVPOU7~%PP*#3k~69V0M;$83)3p z72FJrEMFNJ7=T(C7>pm6UQgF)WO#i(k3;BNb;FIvLYekn`mx(CaIoht)MK1>R$^6=8I7ythM2|x7BtI8!qk?cKiM&N zb*lgW#L}OEEHcygFwVcQf9uodH>5jv?F%bEEFh7nCdbSuXMW7Qc^l{4RmvJcMW=Ti zZswMD6P^B{GKBY-tAqbK9=V%A0p9E!$bJg&W@M6M#^qlLpl28ufZk$Q(g From 5497766b1d75a129819d4f9a44544efd0630c7ea Mon Sep 17 00:00:00 2001 From: dwreeves Date: Thu, 17 Sep 2020 18:57:13 -0400 Subject: [PATCH 077/118] Updated docs --- docs/docs/deployment.md | 5 ++- docs/docs/development/data.md | 21 +++++++++++-- docs/docs/development/index.md | 56 ++++++++++++++++++++++++++++++++-- docs/docs/setup.md | 1 + setup.py | 3 +- 5 files changed, 77 insertions(+), 9 deletions(-) diff --git a/docs/docs/deployment.md b/docs/docs/deployment.md index d97b5de3..2aeb6e8a 100644 --- a/docs/docs/deployment.md +++ b/docs/docs/deployment.md @@ -1,7 +1,7 @@ -# Deployment +# Remote Deployment ???+ note - This guide is an instruction manual on how to deploy the flagging website to internet via Heroku. If you just want to run the website locally, you do not need Heroku. Instead, check out the [development](/development) guide. + This guide is an instruction manual on how to deploy the flagging website to the internet via Heroku. If you just want to run the website locally, you do not need Heroku. Instead, check out the [development](/development) guide. The following tools are required to deploy the website: @@ -133,7 +133,6 @@ git remote -v 3. Now all of your `heroku` commands are going to require specifying the app, but the steps to deploy in staging are otherwise similar to the production deployment: -<<<<<<< HEAD === "Windows (CMD)" ```shell heroku config:set --app crwa-flagging-staging VAULT_PASSWORD=%VAULT_PASSWORD% diff --git a/docs/docs/development/data.md b/docs/docs/development/data.md index 9c1599cf..1d10ad80 100644 --- a/docs/docs/development/data.md +++ b/docs/docs/development/data.md @@ -1,5 +1,7 @@ # Data +# High-Level Overview + Here is a "TLDR" of the data engineering for this website: - To get data, we ping two different APIs, combine the responses from those API requests, do some processing and feature engineering of the data, and then run a predictive model on the processed data. @@ -8,6 +10,19 @@ Here is a "TLDR" of the data engineering for this website: - To actually run the functionality that gets data, processes it, and stores it. we run a [scheduled job](https://en.wikipedia.org/wiki/Job_scheduler) that runs the command `flask update-db` at a set time intervals. +Actually setting up the database requires a few additional steps during either remote or local deployment (`flask create-db` and `flask init-db`), however those steps are covered elsewhere in the docs. + +The `update_database()` inside of `database.py` runs four functions elsewhere in the data folder. This flow chart shows how those functions relate to one another (each block is a function; the arrows represent that the function's output is used as an input in the function being pointed at). + +```mermaid +graph TD +A(get_live_hobolink_data) --> C(process_data) +B(get_live_usgs_data) --> C +C --> D(all_models) +``` + +The rest of this document explains in more detail what's happening in these functions individually. + ## Sources There are two sources of data for our website: @@ -39,13 +54,13 @@ The HOBOlink data is accessed through a REST API using some credentials stored i The data actually returned by the API is a combination of a yaml file with a CSV below it, and we just use the CSV part. We then do the following to preprocess the CSV: -- We remove all timestamps ending `:05`, `:15`, `:25`, `:35`, `:45`, and `:55`. These only contain battery information, not weather information. The final dataframe returned is ultimately in 10 minute incremenets. -- We make the timestamp consistently report eastern standard times. There is a weird issue in which the HOBOlink API returns slightly different datetime formats that messes with Pandas's timestamp parser. We are able to coerce the timestamp into something consistent and predictable. +- We remove all timestamps ending `:05`, `:15`, `:25`, `:35`, `:45`, and `:55`. These only contain battery information, not weather information. The final dataframe returned is ultimately in 10 minute increments. +- We make the timestamp consistently report eastern standard times. - We consolidate duplicative columns. The HOBOlink API has a weird issue where sometimes it splits columns of data with the same name, seemingly at random. This issue causes serious data issues if untreated (at one point, it caused our model to fail to update for a couple days), so our function cleans the data. As you can see from the above, the HOBOlink API is a bit finicky for whatever reason, but we have a good data processing solution for these problems. -The HOBOlink data is also notoriously slow to retrieve (regardless of whether you ask for 1 hour of data or multiple weeks of data), which is why we belabored building the database portion of the flagging website out in the first place. +The HOBOlink data is also notoriously slow to retrieve (regardless of whether you ask for 1 hour of data or multiple weeks of data), which is why we belabored building the database portion of the flagging website out in the first place. The HOBOlink API does not seem to be rate limited or subject to fees that scale with usage. ???+ tip You can manually download the latest raw data from this device [here](https://www.hobolink.com/p/0cdac4a6910cef5a8883deb005d73ae1). If you want some preprocessed data that implements the above modifications to the output, there is a better way to get that data explained in the shell guide. diff --git a/docs/docs/development/index.md b/docs/docs/development/index.md index 776766a7..fe15e970 100644 --- a/docs/docs/development/index.md +++ b/docs/docs/development/index.md @@ -1,6 +1,58 @@ -# Development +# Development - Overview The Development guide is aimed at users who wish to understand the code base and make changes to it if need be. +This overview page describes at a high-level what the website's infrastructure is, how it all relates, and why those things are in the app. + !!! tip - Make sure to go through the [setup guide](../../setup) before doing anything in the development guide. \ No newline at end of file + Make sure to go through the [setup guide](../../setup) before doing anything in the development guide. + +## Dependency Diagram + +```mermaid +classDiagram +Heroku <.. gunicorn +gunicorn <.. Flask : create_app() +gunicorn : /../Procfile +Heroku <.. PostgreSQL +class Flask +Flask : /app.py +Flask : create_app() +Flask : app = Flask(...) +class Config +Config : /config.py +Config : config = get_config_from_env(...) +class vault +vault : /vault.zip +vault : /app.py +Config <.. vault : update_config_from_vault(app) +class Swagger +Swagger : /app.py +Swagger : Swagger(app, ...) +Flask <.. Swagger : init_swagger(app) +Swagger ..> blueprints : wraps RESTful API +Flask <.. Config : app.config.from_object(config) +class SQLAlchemy +SQLAlchemy : /data/database.py +SQLAlchemy : db = SqlAlchemy() +class Jinja2 +Jinja2 : /app.py +Flask <.. Jinja2 : Built-in Flask +SQLAlchemy <.. PostgreSQL: Connected via psycopg2 +Flask <.. SQLAlchemy : db.init_app(app) +class blueprints +blueprints : blueprints/flagging.py +blueprints : blueprints/api.py +blueprints : app.register_blueprint(...) +Flask <.. blueprints +Jinja2 <.. blueprints : Renders HTML +class Admin +Admin : /admin.py +Admin: admin = Admin(...) +SQLAlchemy <.. Admin +Flask <.. Admin : init_admin(app) +class BasicAuth +BasicAuth : /auth.py +BasicAuth : auth = BasicAuth() +BasicAuth ..> Admin : init_auth(app) +``` diff --git a/docs/docs/setup.md b/docs/docs/setup.md index 7ac7e63b..79f419d3 100644 --- a/docs/docs/setup.md +++ b/docs/docs/setup.md @@ -13,6 +13,7 @@ Install all of the following programs onto your computer: - [Python 3](https://www.python.org/downloads/) - specifically 3.7 or higher - [Git](https://git-scm.com/book/en/v2/Getting-Started-Installing-Git) (first time setup guide [here](https://git-scm.com/book/en/v2/Getting-Started-First-Time-Git-Setup)) - [Postgres](https://www.postgresql.org/) _(see installation instructions below)_ +- [7zip](https://www.7-zip.org/) (If on OSX, install via Homebrew: `brew install p7zip`) - _(OSX only)_ [Homebrew](https://brew.sh/) **Recommended:** diff --git a/setup.py b/setup.py index 4cd05491..5f60bfbb 100644 --- a/setup.py +++ b/setup.py @@ -34,7 +34,8 @@ 'requests', 'Flask-SQLAlchemy', 'Flask-Admin', - 'Flask-BasicAuth' + 'Flask-BasicAuth', + 'py7zr' ], extras_require={ 'windows': ['psycopg2'], From 786495a1c84b58f4bc28e25bcb6ca7cd3cc86320 Mon Sep 17 00:00:00 2001 From: dwreeves Date: Thu, 17 Sep 2020 18:57:25 -0400 Subject: [PATCH 078/118] slimmed down requirements.txt --- requirements.txt | 66 ++---------------------------------------------- 1 file changed, 2 insertions(+), 64 deletions(-) diff --git a/requirements.txt b/requirements.txt index a1ffe11f..9b1d0ebf 100644 --- a/requirements.txt +++ b/requirements.txt @@ -1,82 +1,20 @@ -alembic==1.4.2 -aniso8601==8.0.0 -appdirs==1.4.4 -astroid==2.4.1 -atomicwrites==1.4.0 -attrs==19.3.0 -Babel==2.8.0 -bcrypt==3.2.0 -blinker==1.4 -certifi==2020.6.20 -cffi==1.14.2 -chardet==3.0.4 click==7.1.2 -colorama==0.4.3 -coverage==5.2 -cryptography==3.0 -distlib==0.3.1 -dnspython==2.0.0 -filelock==3.0.12 flasgger==0.9.4 Flask-Admin==1.5.6 -Flask-BabelEx==0.9.4 Flask-BasicAuth==0.2.0 Flask-RESTful==0.3.8 -Flask-Security-Too==3.4.4 Flask-SQLAlchemy==2.4.4 -Flask-WTF==0.14.3 Flask==1.1.2 gunicorn==20.0.4 -idna==2.10 -importlib-metadata==1.6.1 -isort==4.3.21 -itsdangerous==1.1.0 Jinja2==2.11.2 -jsonschema==3.2.0 -lazy-object-proxy==1.4.3 -ldap3==2.8 -Mako==1.1.3 -MarkupSafe==1.1.1 -mccabe==0.6.1 -mistune==0.8.4 -more-itertools==8.3.0 -numpy==1.19.1 -packaging==20.4 pandas==1.0.5 -paramiko==2.7.1 -passlib==1.7.2 -pluggy==0.13.1 -psutil==5.7.2 psycopg2-binary==2.8.5 psycopg2==2.8.5 -py==1.8.1 -pyasn1==0.4.8 -pycparser==2.20 -pylint==2.5.2 -PyNaCl==1.4.0 -pyparsing==2.4.7 -pyparsing==2.4.7 -pyrsistent==0.16.0 +py7zr==0.10.0a6 pytest-cov==2.10.0 pytest==5.4.3 -python-dateutil==2.8.1 python-dotenv==0.14.0 -python-editor==1.0.4 -pytz==2018.9 PyYAML==5.3.1 requests==2.23.0 -simplejson==3.16.0 -six==1.15.0 -speaklater==1.3 SQLAlchemy==1.3.18 -sqlparse==0.2.4 -sshtunnel==0.1.5 -toml==0.10.1 -typed-ast==1.4.1 -urllib3==1.25.9 -virtualenv==20.0.27 -wcwidth==0.2.4 -Werkzeug==1.0.1 -wrapt==1.12.1 -WTForms==2.3.3 -zipp==3.1.0 \ No newline at end of file +tweepy==3.9.0 From 815eb9b4de708fb2f3e0260706c6bacd8daafa93 Mon Sep 17 00:00:00 2001 From: dwreeves Date: Thu, 17 Sep 2020 18:57:38 -0400 Subject: [PATCH 079/118] refactoring --- .flaskenv | 1 - .gitignore | 8 ++----- flagging_site/app.py | 12 ---------- flagging_site/auth.py | 2 ++ flagging_site/blueprints/flagging.py | 2 +- flagging_site/config.py | 34 ++++++++++++++++++++-------- flagging_site/data/database.py | 10 +++++++- flagging_site/data/hobolink.py | 2 +- run_unix_dev.sh | 5 +--- run_windows_dev.bat | 2 -- 10 files changed, 41 insertions(+), 37 deletions(-) diff --git a/.flaskenv b/.flaskenv index dfe2a1a6..cf2c40e5 100644 --- a/.flaskenv +++ b/.flaskenv @@ -1,2 +1 @@ FLASK_APP=flagging_site -FLASK_ENV=development diff --git a/.gitignore b/.gitignore index 275d1fc5..21770205 100644 --- a/.gitignore +++ b/.gitignore @@ -2,12 +2,13 @@ docs/site/ .vscode/ *.code-workspace .idea/ -keys.json .DS_Store Pipfile Pipfile.lock pgadmin4/ mkdocs_env/ +secrets.json +keys.json # Byte-compiled / optimized / DLL files __pycache__/ @@ -147,8 +148,3 @@ dmypy.json # Cython debug symbols cython_debug/l - -flagging_site/keys.json - -# ignore file with database credentials -flagging_site/data/db_creds.json diff --git a/flagging_site/app.py b/flagging_site/app.py index b5fd1c2e..b6ca548a 100644 --- a/flagging_site/app.py +++ b/flagging_site/app.py @@ -241,18 +241,6 @@ def update_config_from_vault(app: Flask) -> None: raise RuntimeError(msg) else: print(f'Warning: {msg}') - app.config['HOBOLINK_AUTH'] = { - 'password': None, - 'user': None, - 'token': None - } - app.config['TWITTER_AUTH'] = { - 'api_key': None, - 'api_key_secret': None, - 'access_token': None, - 'access_token_secret': None, - 'bearer_token': None - } app.config['SECRET_KEY'] = os.urandom(16) else: # Add 'SECRET_KEY', 'HOBOLINK_AUTH', AND 'TWITTER_AUTH' to the config. diff --git a/flagging_site/auth.py b/flagging_site/auth.py index 2e4c4b17..201f4e5e 100644 --- a/flagging_site/auth.py +++ b/flagging_site/auth.py @@ -6,10 +6,12 @@ basic_auth = BasicAuth() + def init_auth(app: Flask): with app.app_context(): basic_auth.init_app(app) + # Taken from https://computableverse.com/blog/flask-admin-using-basicauth class AuthException(HTTPException): def __init__(self, message): diff --git a/flagging_site/blueprints/flagging.py b/flagging_site/blueprints/flagging.py index 12cbed68..77171a74 100644 --- a/flagging_site/blueprints/flagging.py +++ b/flagging_site/blueprints/flagging.py @@ -76,7 +76,7 @@ def index() -> str: print('ERROR! the reaches are\'t identical between boathouse list and model outputs!') for (flag_reach, flag_safe) in flags.items(): - homepage[flag_reach]['flag']=flag_safe + homepage[flag_reach]['flag'] = flag_safe model_last_updated_time = df['time'].iloc[0] diff --git a/flagging_site/config.py b/flagging_site/config.py index a0f22e24..1139d939 100644 --- a/flagging_site/config.py +++ b/flagging_site/config.py @@ -17,9 +17,15 @@ ROOT_DIR = os.path.abspath(os.path.dirname(__file__)) QUERIES_DIR = os.path.join(ROOT_DIR, 'data', 'queries') DATA_STORE = os.path.join(ROOT_DIR, 'data', '_store') -VAULT_FILE = os.path.join(ROOT_DIR, 'vault.zip') +VAULT_FILE = os.path.join(ROOT_DIR, 'vault.7z') +# Load dotenv +# ~~~~~~~~~~~ +if os.getenv('FLASK_ENV') == 'development': + load_dotenv(os.path.join(ROOT_DIR, '..', '.flaskenv')) + load_dotenv(os.path.join(ROOT_DIR, '..', '.env')) + # Configs # ~~~~~~~ @@ -89,15 +95,25 @@ def SQLALCHEMY_DATABASE_URI(self) -> str: """ VAULT_PASSWORD: str = os.getenv('VAULT_PASSWORD') - """Password """ - KEYS: Dict[str, Dict[str, Any]] = None - """These are where the keys from the vault are stored. It should be a dict - of dicts. Each key in the first level dict corresponds to a different - service that needs keys / secured credentials stored. - - Currently, HOBOlink and Flask's `SECRET_KEY` are the two services that pass - through the vault. + HOBOLINK_AUTH: dict = { + 'password': None, + 'user': None, + 'token': None + } + """Note: Do not fill these out manually; the HOBOlink auth gets populated + from the vault. + """ + + TWITTER_AUTH: dict = { + 'api_key': None, + 'api_key_secret': None, + 'access_token': None, + 'access_token_secret': None, + 'bearer_token': None + } + """Note: Do not fill these out manually; the Twitter auth gets populated + from the vault. """ VAULT_FILE: str = VAULT_FILE diff --git a/flagging_site/data/database.py b/flagging_site/data/database.py index e352640e..6faaf4f8 100644 --- a/flagging_site/data/database.py +++ b/flagging_site/data/database.py @@ -126,7 +126,15 @@ def init_db(): def update_database(): """This function basically controls all of our data refreshes. The - following tables + following tables are updated in order: + + - usgs + - hobolink + - processed_data + - model_outputs + + The functions run to calculate the data are imported from other files + within the data folder. """ options = { 'con': db.engine, diff --git a/flagging_site/data/hobolink.py b/flagging_site/data/hobolink.py index c6bf351a..838d66e9 100644 --- a/flagging_site/data/hobolink.py +++ b/flagging_site/data/hobolink.py @@ -71,7 +71,7 @@ def request_to_hobolink( """ data = { 'query': export_name, - 'authentication': current_app.config['KEYS']['hobolink'] + 'authentication': current_app.config['HOBOLINK_AUTH'] } res = requests.post(HOBOLINK_URL, json=data) diff --git a/run_unix_dev.sh b/run_unix_dev.sh index 5015d885..349a2abf 100644 --- a/run_unix_dev.sh +++ b/run_unix_dev.sh @@ -25,12 +25,9 @@ $PYEXEC -m pip install $(cat requirements.txt | grep -v "psycopg2==") # Set up and run the Flask application export FLASK_APP=flagging_site:create_app export FLASK_ENV=development + read -p "Offline mode? [y/n]: " offline_mode export OFFLINE_MODE=${offline_mode:-${OFFLINE_MODE}} -read -p "Enter vault password: " vault_pw -export VAULT_PASSWORD=${vault_pw:-${VAULT_PASSWORD}} -read -p "Enter Postgres password: " postgres_pw -export $POSTGRES_PASSWORD=${postgres_pw:-${POSTGRES_PASSWORD}} flask create-db flask init-db diff --git a/run_windows_dev.bat b/run_windows_dev.bat index 063f64f9..d01dc450 100644 --- a/run_windows_dev.bat +++ b/run_windows_dev.bat @@ -6,8 +6,6 @@ python -m pip install -r requirements.txt set FLASK_APP=flagging_site:create_app set FLASK_ENV=development set /p OFFLINE_MODE="Offline mode? [y/n]: " -set /p VAULT_PASSWORD="Enter vault password: " -set /p POSTGRES_PASSWORD="Enter Postgres password: " flask create-db flask init-db From 9ca1c20cd94d5e44cace8228521ca16f318421f9 Mon Sep 17 00:00:00 2001 From: dwreeves Date: Thu, 17 Sep 2020 19:02:32 -0400 Subject: [PATCH 080/118] bugfix --- flagging_site/app.py | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) diff --git a/flagging_site/app.py b/flagging_site/app.py index b6ca548a..d125e60f 100644 --- a/flagging_site/app.py +++ b/flagging_site/app.py @@ -104,9 +104,10 @@ def update_db_command(): click.echo('Updated the database.') @app.cli.command('update-website') - def update_db_command(): + @click.pass_context + def update_website_command(ctx): """Update the database with the latest live data.""" - update_db_command() + ctx.invoke(update_db_command) from .twitter_bot import tweet_out_status tweet_out_status() From 439947d69f18b584cd859dff3ac8d801d2f71457 Mon Sep 17 00:00:00 2001 From: dwreeves Date: Thu, 17 Sep 2020 20:01:40 -0400 Subject: [PATCH 081/118] made tweet transparent in cli --- flagging_site/app.py | 3 ++- flagging_site/twitter_bot.py | 1 + 2 files changed, 3 insertions(+), 1 deletion(-) diff --git a/flagging_site/app.py b/flagging_site/app.py index d125e60f..684edde0 100644 --- a/flagging_site/app.py +++ b/flagging_site/app.py @@ -109,7 +109,8 @@ def update_website_command(ctx): """Update the database with the latest live data.""" ctx.invoke(update_db_command) from .twitter_bot import tweet_out_status - tweet_out_status() + msg = tweet_out_status() + click.echo(f'Sent out tweet: {msg!r}') # Make a few useful functions available in Flask shell without imports @app.shell_context_processor diff --git a/flagging_site/twitter_bot.py b/flagging_site/twitter_bot.py index 07f7735d..820d9d42 100644 --- a/flagging_site/twitter_bot.py +++ b/flagging_site/twitter_bot.py @@ -50,3 +50,4 @@ def tweet_out_status(): f'unsafe as of {current_time}: {unsafe}.' ) tweepy_api.update_status(msg) + return msg From 80421417654cb94f1cc4d81375dcb6c71a4f20fe Mon Sep 17 00:00:00 2001 From: dwreeves Date: Thu, 17 Sep 2020 21:28:26 -0400 Subject: [PATCH 082/118] Twitter bot updates and refactoring --- Procfile | 2 +- docs/docs/development/index.md | 6 ++-- docs/docs/development/twitter_bot.md | 4 +++ docs/docs/shell.md | 29 ++++++++++------ flagging_site/admin.py | 34 ++++++++++++++----- flagging_site/app.py | 35 +++++++++++--------- flagging_site/auth.py | 23 ------------- flagging_site/config.py | 16 ++++++--- flagging_site/{twitter_bot.py => twitter.py} | 24 ++++++++++++-- 9 files changed, 106 insertions(+), 67 deletions(-) delete mode 100644 flagging_site/auth.py rename flagging_site/{twitter_bot.py => twitter.py} (68%) diff --git a/Procfile b/Procfile index dc798cc4..8c440d4e 100644 --- a/Procfile +++ b/Procfile @@ -5,4 +5,4 @@ # https://devcenter.heroku.com/articles/procfile # ~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ -web: gunicorn "flagging_site:create_app()" +web: gunicorn "flagging_site:create_app('production')" diff --git a/docs/docs/development/index.md b/docs/docs/development/index.md index fe15e970..b3a0cbda 100644 --- a/docs/docs/development/index.md +++ b/docs/docs/development/index.md @@ -23,7 +23,7 @@ class Config Config : /config.py Config : config = get_config_from_env(...) class vault -vault : /vault.zip +vault : /vault.7z vault : /app.py Config <.. vault : update_config_from_vault(app) class Swagger @@ -52,7 +52,7 @@ Admin: admin = Admin(...) SQLAlchemy <.. Admin Flask <.. Admin : init_admin(app) class BasicAuth -BasicAuth : /auth.py +BasicAuth : /admin.py BasicAuth : auth = BasicAuth() -BasicAuth ..> Admin : init_auth(app) +BasicAuth ..> Admin ``` diff --git a/docs/docs/development/twitter_bot.md b/docs/docs/development/twitter_bot.md index 43f291f9..926787d9 100644 --- a/docs/docs/development/twitter_bot.md +++ b/docs/docs/development/twitter_bot.md @@ -1,3 +1,7 @@ +# Twitter Bot + +Every time the website updates, it sends out a tweet. + ## First Time Setup Follow these steps to set up the Twitter bot for the first time, such as on a new Twitter account. diff --git a/docs/docs/shell.md b/docs/docs/shell.md index ce0a345e..e2409ee0 100644 --- a/docs/docs/shell.md +++ b/docs/docs/shell.md @@ -3,24 +3,31 @@ The shell is used to access app functions and data, such as Hobolink and USGS data and access to the database. +The reason why the shell is useful is because there may be cases where you want to play around with the app's functions. For example, maybe you see something that seems fishy in the data, so you want to have direct access to the function the website is running. You may also want to + +The way Flask works makes it impossible to run the website's functions outside the Flask app context, which means importing the functions into a naked shell doesn't work as intended. The `flask shell` provides all the tools needed to let coders access the functions the exact same way the website does, except in a shell environment. + ## Available Shell Functions and Variables -- `db` (*flask_sqlalchemy.SQLAlchemy*): +- **`app`** (*flask.Flask*): + The actual Flask app instance. +- **`db`** (*flask_sqlalchemy.SQLAlchemy*): The object used to interact with the Postgres database. -- `get_live_hobolink_data` (*(Optional[str]) -> pd.DataFrame*): - Gets the Hobolink data table based on the given "export" name. - See `flagging_site/data/hobolink.py` for details. -- `get_live_usgs_data` (*() -> pd.DataFrame*): +- **`get_live_hobolink_data`** (*(Optional[str]) -> pd.DataFrame*): + Gets the HOBOlink data table based on the given "export" name. +- **`get_live_usgs_data`** (*() -> pd.DataFrame*): Gets the USGS data table. - See `flagging_site/data/usgs.py` for details. -- `get_data` (*() -> pd.DataFrame*): +- **`get_data`** (*() -> pd.DataFrame*): Gets the Hobolink and USGS data tables and returns a combined table. -- `process_data` (*(pd.DataFrame, pd.DataFrame) -> pd.DataFrame*): +- **`process_data`** (*(pd.DataFrame, pd.DataFrame) -> pd.DataFrame*): Combines the Hobolink and USGS tables. - See `flagging_site/data/model.py` for details. +- **`compose_tweet`** (*() -> str*): + Generates a message for Twitter that represents the current status of the flagging program (note: this function does not actually send the Tweet to Twitter.com). + +Additionally, Pandas and Numpy are already pre-imported via `import pandas as pd` and `import numpy as np`. -To add more functions and variables, simply add an entry to the dictionary -returned by the function `make_shell_context()` in `flagging_site/app.py:creat_app()`. +???+ tip + To add more functions and variables that pre-load in the Flask shell, simply add another entry to the dictionary returned by the function `make_shell_context()` in `flagging_site/app.py:creat_app()`. ## Running the Shell diff --git a/flagging_site/admin.py b/flagging_site/admin.py index fdee8e90..b58e6b73 100644 --- a/flagging_site/admin.py +++ b/flagging_site/admin.py @@ -1,5 +1,4 @@ import os - from flask import Flask from flask import redirect from flask import request @@ -10,22 +9,41 @@ from flask_admin.contrib import sqla from werkzeug.exceptions import HTTPException -from .auth import AuthException -from .auth import basic_auth -from .data import db +from flask_basicauth import BasicAuth +from werkzeug.exceptions import HTTPException +from .data import db admin = Admin(template_mode='bootstrap3') +basic_auth = BasicAuth() + + +# Taken from https://computableverse.com/blog/flask-admin-using-basicauth +class AuthException(HTTPException): + def __init__(self, message): + """HTTP Forbidden error that prompts for login""" + super().__init__(message, Response( + 'You could not be authenticated. Please refresh the page.', + status=401, + headers={'WWW-Authenticate': 'Basic realm="Login Required"'} + )) + + def init_admin(app: Flask): - with app.app_context(): - # Register /admin - admin.init_app(app) + """Registers the Flask-Admin extensions to the app, and attaches the + model views to the admin panel. + Args: + app: A Flask application instance. + """ + basic_auth.init_app(app) + admin.init_app(app) + + with app.app_context(): # Register /admin sub-views from .data.cyano_overrides import CyanoOverridesModelView admin.add_view(CyanoOverridesModelView(db.session)) - admin.add_view(LogoutView(name="Logout")) diff --git a/flagging_site/app.py b/flagging_site/app.py index 684edde0..746bb050 100644 --- a/flagging_site/app.py +++ b/flagging_site/app.py @@ -19,7 +19,7 @@ from .config import get_config_from_env -def create_app(config: Optional[Config] = None) -> Flask: +def create_app(config: Optional[Union[Config, str]] = None) -> Flask: """Create and configure an instance of the Flask application. We use the `create_app` scheme over defining the `app` directly at the module level so the app isn't loaded immediately by importing the module. @@ -34,9 +34,12 @@ def create_app(config: Optional[Config] = None) -> Flask: # Get a config for the website. If one was not passed in the function, then # a config will be used depending on the `FLASK_ENV`. - if not config: + if config is None: # Determine the config based on the `FLASK_ENV`. config = get_config_from_env(app.env) + elif isinstance(config, str): + # If config is string, parse it as if it's an env. + config = get_config_from_env(config) app.config.from_object(config) @@ -62,16 +65,12 @@ def create_app(config: Optional[Config] = None) -> Flask: from .data import db db.init_app(app) - # Register auth - from .auth import init_auth - init_auth(app) - # Register admin from .admin import init_admin init_admin(app) # Register Twitter bot - from .twitter_bot import init_tweepy + from .twitter import init_tweepy init_tweepy(app) @app.before_request @@ -108,8 +107,8 @@ def update_db_command(): def update_website_command(ctx): """Update the database with the latest live data.""" ctx.invoke(update_db_command) - from .twitter_bot import tweet_out_status - msg = tweet_out_status() + from .twitter import tweet_current_status + msg = tweet_current_status() click.echo(f'Sent out tweet: {msg!r}') # Make a few useful functions available in Flask shell without imports @@ -117,20 +116,24 @@ def update_website_command(ctx): def make_shell_context(): import pandas as pd import numpy as np + from .flask import current_app from .blueprints.flagging import get_data from .data import db from .data.hobolink import get_live_hobolink_data from .data.predictive_models import process_data from .data.usgs import get_live_usgs_data + from .twitter import compose_tweet return { 'pd': pd, 'np': np, + 'app': current_app, 'db': db, 'get_data': get_data, 'get_live_hobolink_data': get_live_hobolink_data, 'get_live_usgs_data': get_live_usgs_data, 'process_data': process_data, + 'compose_tweet': compose_tweet } # And we're all set! We can hand the app over to flask at this point. @@ -140,6 +143,9 @@ def make_shell_context(): def init_swagger(app: Flask): """This function handles all the logic for adding Swagger automated documentation to the application instance. + + Args: + app: A Flask application instance. """ from flasgger import Swagger from flasgger import LazyString @@ -237,13 +243,12 @@ def update_config_from_vault(app: Flask) -> None: password=app.config['VAULT_PASSWORD'], vault_file=app.config['VAULT_FILE'] ) + # Add 'SECRET_KEY', 'HOBOLINK_AUTH', AND 'TWITTER_AUTH' to the config. + app.config.update(secrets) except (LZMAError, KeyError): msg = 'Unable to load the vault; bad password provided.' - if app.env == 'production': - raise RuntimeError(msg) - else: + if app.config.get('VAULT_OPTIONAL'): print(f'Warning: {msg}') app.config['SECRET_KEY'] = os.urandom(16) - else: - # Add 'SECRET_KEY', 'HOBOLINK_AUTH', AND 'TWITTER_AUTH' to the config. - app.config.update(secrets) + else: + raise RuntimeError(msg) diff --git a/flagging_site/auth.py b/flagging_site/auth.py deleted file mode 100644 index 201f4e5e..00000000 --- a/flagging_site/auth.py +++ /dev/null @@ -1,23 +0,0 @@ -from flask import Flask -from flask import Response -from flask_basicauth import BasicAuth -from werkzeug.exceptions import HTTPException - - -basic_auth = BasicAuth() - - -def init_auth(app: Flask): - with app.app_context(): - basic_auth.init_app(app) - - -# Taken from https://computableverse.com/blog/flask-admin-using-basicauth -class AuthException(HTTPException): - def __init__(self, message): - """HTTP Forbidden error that prompts for login""" - super().__init__(message, Response( - 'You could not be authenticated. Please refresh the page.', - status=401, - headers={'WWW-Authenticate': 'Basic realm="Login Required"'} - )) diff --git a/flagging_site/config.py b/flagging_site/config.py index 1139d939..ffb0418d 100644 --- a/flagging_site/config.py +++ b/flagging_site/config.py @@ -138,9 +138,15 @@ def SQLALCHEMY_DATABASE_URI(self) -> str: """ API_MAX_HOURS: int = 48 - """The maximum number of hours of data that the API will return. We are not trying - to be stingy about our data, we just want this in order to avoid any odd behaviors - if the user requests more data than exists. + """The maximum number of hours of data that the API will return. We are not + trying to be stingy about our data, we just want this in order to avoid any + odd behaviors if the user requests more data than exists. + """ + + SEND_TWEETS: bool = strtobool(os.getenv('SEND_TWEETS') or 'false') + """If True, the website behaves normally. If False, any time the app would + send a Tweet, it does not do so. It is useful to turn this off when + developing to test Twitter messages. """ @@ -149,6 +155,8 @@ class ProductionConfig(Config): internet. Currently the only part of the website that's pretty fleshed out is the `flagging` part, so that's the only blueprint we import. """ + SEND_TWEETS: str = True + def __init__(self): """Initializing the production config allows us to ensure the existence of these variables in the environment.""" @@ -211,7 +219,7 @@ def get_config_from_env(env: str) -> Config: config_mapping = { 'production': ProductionConfig, 'development': DevelopmentConfig, - 'testing': TestingConfig + 'testing': TestingConfig, } try: config = config_mapping[env] diff --git a/flagging_site/twitter_bot.py b/flagging_site/twitter.py similarity index 68% rename from flagging_site/twitter_bot.py rename to flagging_site/twitter.py index 820d9d42..4b12fb1c 100644 --- a/flagging_site/twitter_bot.py +++ b/flagging_site/twitter.py @@ -21,7 +21,16 @@ def init_tweepy(app: Flask): tweepy_api.auth = auth -def tweet_out_status(): +def compose_tweet() -> str: + """Generates the message that gets tweeted out. This function does not + actually send the Tweet out; this function is separated from the function + that sends the Tweet in order to assist with testing and development, in + addition to addressing separation of concerns. + + Returns: + Message intended to be tweeted that conveys the status of the Charles + River. + """ from .data.predictive_models import latest_model_outputs from .data.cyano_overrides import get_currently_overridden_reaches @@ -49,5 +58,16 @@ def tweet_out_status(): 'Our predictive model is reporting that the following reaches are ' f'unsafe as of {current_time}: {unsafe}.' ) - tweepy_api.update_status(msg) + return msg + + +def tweet_current_status() -> str: + """Tweet a message about the status of the Charles River. + + Returns: + The message that was tweeted out. + """ + msg = compose_tweet() + if current_app.config['SEND_TWEETS']: + tweepy_api.update_status(msg) return msg From a10a25b913f2298318eca895467dcbdee273517c Mon Sep 17 00:00:00 2001 From: dwreeves Date: Thu, 17 Sep 2020 21:31:11 -0400 Subject: [PATCH 083/118] forgot basicauth user/pw in dev --- flagging_site/config.py | 3 +++ 1 file changed, 3 insertions(+) diff --git a/flagging_site/config.py b/flagging_site/config.py index ffb0418d..af5756f4 100644 --- a/flagging_site/config.py +++ b/flagging_site/config.py @@ -149,6 +149,9 @@ def SQLALCHEMY_DATABASE_URI(self) -> str: developing to test Twitter messages. """ + BASIC_AUTH_USERNAME: str = os.getenv('BASIC_AUTH_USERNAME', 'admin') + BASIC_AUTH_PASSWORD: str = os.getenv('BASIC_AUTH_PASSWORD', 'password') + class ProductionConfig(Config): """The Production Config is used for deployment of the website to the From 2cae6fcd2f0dfc936d16a2c10f4b0171236c635d Mon Sep 17 00:00:00 2001 From: dwreeves Date: Thu, 17 Sep 2020 22:05:24 -0400 Subject: [PATCH 084/118] More docs changes and minor touchups --- .flaskenv | 2 +- docs/docs/shell.md | 83 +++++++++++++++++++++++++--------------- flagging_site/app.py | 4 +- flagging_site/twitter.py | 3 ++ 4 files changed, 59 insertions(+), 33 deletions(-) diff --git a/.flaskenv b/.flaskenv index cf2c40e5..23a11b6b 100644 --- a/.flaskenv +++ b/.flaskenv @@ -1 +1 @@ -FLASK_APP=flagging_site +FLASK_APP="flagging_site:create_app" diff --git a/docs/docs/shell.md b/docs/docs/shell.md index e2409ee0..4b7e8643 100644 --- a/docs/docs/shell.md +++ b/docs/docs/shell.md @@ -7,6 +7,35 @@ The reason why the shell is useful is because there may be cases where you want The way Flask works makes it impossible to run the website's functions outside the Flask app context, which means importing the functions into a naked shell doesn't work as intended. The `flask shell` provides all the tools needed to let coders access the functions the exact same way the website does, except in a shell environment. +## Run the Shell + +1. Open up a terminal at the `flagging` folder. + +2. Activate a Python virtual environment: + +```shell +python3 -m venv venv +source venv/bin/activate +python3 -m pip install -r requirements.txt +``` + +3. Set up the `FLASK_ENV` environment variable: + +```shell +export FLASK_ENV=development +``` + +4. Run the shell: + +```shell +flask shell +``` + +And you should be good to go! The functions listed below should be available for use, and the section below contains some example use cases for the shell. + +???+ tip + To exit from the shell, type `exit()` then ++enter++. + ## Available Shell Functions and Variables - **`app`** (*flask.Flask*): @@ -28,42 +57,36 @@ Additionally, Pandas and Numpy are already pre-imported via `import pandas as pd ???+ tip To add more functions and variables that pre-load in the Flask shell, simply add another entry to the dictionary returned by the function `make_shell_context()` in `flagging_site/app.py:creat_app()`. + +???+ tip + All of the website's functions can be run in the Flask shell, even those that are not pre-loaded in the shell's global context. All you need to do is import it. For example, let's say you want to get the un-parsed request object from USGS.gov. You can import the function we use and run it like this: + + ```python + # (in Flask shell) + from flagging_site.data.usgs import request_to_usgs + res = request_to_usgs() + print(res.json()) + ``` -## Running the Shell - -First, open up a terminal at the `flagging` folder. - -Make sure you have Python 3 installed. Set up your environment with the following commands: - -```shell -python3 -m venv venv -source venv/bin/activate -python3 -m pip install -r requirements.txt -``` - -Export the following environment variables like so: +## Example 1: Export Hobolink Data to CSV -```shell -export VAULT_PASSWORD=replace_me_with_pw -export FLASK_APP=flagging_site:create_app -export FLASK_ENV=development -``` - -Finally, start the Flask shell: +Here we assume you have already started the Flask shell. +This example shows how to download the Hobolink data and +save it as a CSV file. -```shell -flask shell +```python +# (in Flask shell) +hobolink_data = get_live_hobolink_data() +hobolink_data.to_csv('path/where/to/save/my-CSV-file.csv') ``` -And you should be good to go! The functions listed above should be available for use. See below for an example. +Downloading the data may be useful if you want to see -## Example: Export Hobolink Data to CSV +## Example 2: Preview Tweet -Here we assume you have already started the Flask shell. -This example shows how to download the Hobolink data and -save it as a CSV file. +Let's say you want to preview a Tweet that would be sent out without actually sending it. The `compose_tweet()` function returns a string of this message: ```python ->>> hobolink_data = get_live_hobolink_data() ->>> hobolink_data.to_csv('path/where/to/save/my-CSV-file.csv') -``` \ No newline at end of file +# (in Flask shell) +print(compose_tweet()) +``` diff --git a/flagging_site/app.py b/flagging_site/app.py index 746bb050..3c31ef13 100644 --- a/flagging_site/app.py +++ b/flagging_site/app.py @@ -81,7 +81,7 @@ def before_request(): @app.cli.command('create-db') def create_db_command(): - """Create database (after verifying that it isn't already there)""" + """Create database (after verifying that it isn't already there).""" from .data.database import create_db if create_db(): click.echo('The database was created.') @@ -105,7 +105,7 @@ def update_db_command(): @app.cli.command('update-website') @click.pass_context def update_website_command(ctx): - """Update the database with the latest live data.""" + """Updates the database, then Tweets a message.""" ctx.invoke(update_db_command) from .twitter import tweet_current_status msg = tweet_current_status() diff --git a/flagging_site/twitter.py b/flagging_site/twitter.py index 4b12fb1c..ee836ba6 100644 --- a/flagging_site/twitter.py +++ b/flagging_site/twitter.py @@ -7,6 +7,9 @@ def init_tweepy(app: Flask): + """Uses the app instance's config to add requisite credentials to the + tweepy API instance. + """ # Pass Twitter API tokens into Tweepy's OAuthHandler auth = tweepy.OAuthHandler( consumer_key=app.config['TWITTER_AUTH']['api_key'], From 26b64fb46c95b509e40103d234d83f882dd2dfa1 Mon Sep 17 00:00:00 2001 From: dwreeves Date: Thu, 17 Sep 2020 22:08:18 -0400 Subject: [PATCH 085/118] bugfix --- flagging_site/app.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/flagging_site/app.py b/flagging_site/app.py index 3c31ef13..b06c8f30 100644 --- a/flagging_site/app.py +++ b/flagging_site/app.py @@ -116,7 +116,7 @@ def update_website_command(ctx): def make_shell_context(): import pandas as pd import numpy as np - from .flask import current_app + from flask import current_app from .blueprints.flagging import get_data from .data import db from .data.hobolink import get_live_hobolink_data From b37f6fb875f985e9f19dc922147b3861ef91b53d Mon Sep 17 00:00:00 2001 From: dwreeves Date: Thu, 17 Sep 2020 22:15:17 -0400 Subject: [PATCH 086/118] fix --- docs/docs/development/twitter_bot.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/docs/docs/development/twitter_bot.md b/docs/docs/development/twitter_bot.md index 926787d9..17c16656 100644 --- a/docs/docs/development/twitter_bot.md +++ b/docs/docs/development/twitter_bot.md @@ -20,7 +20,7 @@ Follow these steps to set up the Twitter bot for the first time, such as on a ne 4. In the code base, use the `VAULT_PASSWORD` to unzip the `vault.7z` manually. You should have a file called `secrets.json`. Open up `secrets.json` in the plaintext editor of your choosing. -???+ critical +???+ danger Make sure that you delete the unencrpyted, unarchived version of the `secrets.json` file after you are done with it. 5. Now go back to your browser with the Twitter Developer Portal. At the top of the screen, flip to the `Keys and tokens`. Now it's time to go through the dashboard and get your copy+paste ready. We will be inserting these values into the `secrets.json` (remember to wrap the keys in double quotes `"like this"` when you insert them). From 52e20f827d6f0aa977c19cd6690f43df13dc632f Mon Sep 17 00:00:00 2001 From: dwreeves Date: Thu, 17 Sep 2020 22:57:25 -0400 Subject: [PATCH 087/118] modified docs --- .../heroku_deployment.md} | 0 docs/docs/cloud/index.md | 5 +++++ .../docs/{development => cloud}/twitter_bot.md | 0 docs/docs/development/database.md | 0 docs/docs/development/index.md | 2 +- .../index.md} | 4 ++++ .../learning_resources.md | 0 docs/docs/{ => development_resources}/shell.md | 0 docs/docs/system.md | 8 -------- docs/mkdocs.yml | 18 +++++++++--------- 10 files changed, 19 insertions(+), 18 deletions(-) rename docs/docs/{deployment.md => cloud/heroku_deployment.md} (100%) create mode 100644 docs/docs/cloud/index.md rename docs/docs/{development => cloud}/twitter_bot.md (100%) delete mode 100644 docs/docs/development/database.md rename docs/docs/{development/history.md => development_resources/index.md} (99%) rename docs/docs/{development => development_resources}/learning_resources.md (100%) rename docs/docs/{ => development_resources}/shell.md (100%) delete mode 100644 docs/docs/system.md diff --git a/docs/docs/deployment.md b/docs/docs/cloud/heroku_deployment.md similarity index 100% rename from docs/docs/deployment.md rename to docs/docs/cloud/heroku_deployment.md diff --git a/docs/docs/cloud/index.md b/docs/docs/cloud/index.md new file mode 100644 index 00000000..e7bed2c8 --- /dev/null +++ b/docs/docs/cloud/index.md @@ -0,0 +1,5 @@ +# Overview + +The flagging website is designed to be hosted on [Heroku](https://heroku.com/). The guide for how to set up deployment is available [here](../cloud/heroku_deployment.md). + +The full cloud deployment depends not only on Heroku but also Twitter's development API. The Twitter bot only needs to be set up once and, notwithstanding exigent circumstances (losing the API key, migrating the bot, or handling a Twitter ban), the Twitter bot does not need any additional maintenance. Nevertheless, there is documentation for how to set up the Twitter bot [here](../cloud/twitter_bot.md). diff --git a/docs/docs/development/twitter_bot.md b/docs/docs/cloud/twitter_bot.md similarity index 100% rename from docs/docs/development/twitter_bot.md rename to docs/docs/cloud/twitter_bot.md diff --git a/docs/docs/development/database.md b/docs/docs/development/database.md deleted file mode 100644 index e69de29b..00000000 diff --git a/docs/docs/development/index.md b/docs/docs/development/index.md index b3a0cbda..e93419da 100644 --- a/docs/docs/development/index.md +++ b/docs/docs/development/index.md @@ -5,7 +5,7 @@ The Development guide is aimed at users who wish to understand the code base and This overview page describes at a high-level what the website's infrastructure is, how it all relates, and why those things are in the app. !!! tip - Make sure to go through the [setup guide](../../setup) before doing anything in the development guide. + Make sure to go through the [setup guide](../setup.md) before doing anything in the development guide. ## Dependency Diagram diff --git a/docs/docs/development/history.md b/docs/docs/development_resources/index.md similarity index 99% rename from docs/docs/development/history.md rename to docs/docs/development_resources/index.md index 72c19e63..59578e67 100644 --- a/docs/docs/development/history.md +++ b/docs/docs/development_resources/index.md @@ -1,3 +1,7 @@ +# Stack + +## Project History + Traditionally, the CRWA Flagging Program was hosted on a PHP-built website that hosted a predictive model and ran it. However, that website was out of commission due to some bugs and the CRWA's lack of PHP development resources. We at Code for Boston attempted to fix the website, although we have had trouble maintaining a steady stream of PHP expertise, so we rebuilt the website from scratch in Python. The project's source code is available [on GitHub](https://github.com/codeforboston/flagging/wiki), and the docs we used for project management and some dev stuff are available in [the repo's wiki](https://github.com/codeforboston/flagging/wiki). diff --git a/docs/docs/development/learning_resources.md b/docs/docs/development_resources/learning_resources.md similarity index 100% rename from docs/docs/development/learning_resources.md rename to docs/docs/development_resources/learning_resources.md diff --git a/docs/docs/shell.md b/docs/docs/development_resources/shell.md similarity index 100% rename from docs/docs/shell.md rename to docs/docs/development_resources/shell.md diff --git a/docs/docs/system.md b/docs/docs/system.md deleted file mode 100644 index dad6054f..00000000 --- a/docs/docs/system.md +++ /dev/null @@ -1,8 +0,0 @@ -# Website Explained - -## Diagram -![Image of Yaktocat](Flaggin_System.png) - -## Explanation - -Here is a tentative explanantion of how the website works. Currently it is a flask web application that creates a main web application using `create_app()` function and retrieve configuration options from `config.py` and keys from the `vault.zip`. Then joins mini web apps by registering blueprints found inside the `blueprints` directory. Particularly the main web app will be joining web app `flagging.py` to retrieve data from USGS and Hobolink api. With this information, we generate predictive data based on multiple logistic models to determine if river is safe or not. The website displays that data calling `render_template()` which renders `output_model.html` with the Jinja template engine. Moreover, we save that data inside a SQL database hosted in heroku, which will also where we deploy the flask web application. diff --git a/docs/mkdocs.yml b/docs/mkdocs.yml index 1c84dd87..c99a1973 100644 --- a/docs/mkdocs.yml +++ b/docs/mkdocs.yml @@ -5,20 +5,20 @@ nav: - About: about.md - Setup: setup.md - Admin: admin.md -- Remote Deployment: deployment.md -- System: system.md -- Shell: shell.md -- Development Background: - - History: development/history.md - - Learning Resources: development/learning_resources.md +- Cloud: + - Overview: cloud/index.md + - Heroku Deployment: cloud/heroku_deployment.md + - Twitter Bot: cloud/twitter_bot.md - Development: - Overview: development/index.md - # ^ (Explain the vault here too) - - Config: development/config.md + # - Config: development/config.md - Data: development/data.md - Predictive Models: development/predictive_models.md - - Twitter Bot: development/twitter_bot.md # - Front-End: development/front-end.md +- Development Resources: + - Overview: development_resources/index.md + - Learning Resources: development_resources/learning_resources.md + - Shell: development_resources/shell.md theme: name: material palette: From cbd703dd78f034baf31d849212fa7f8a3e47b009 Mon Sep 17 00:00:00 2001 From: dwreeves Date: Thu, 17 Sep 2020 22:59:45 -0400 Subject: [PATCH 088/118] typo --- docs/docs/about.md | 2 +- docs/docs/admin.md | 2 +- docs/docs/cloud/heroku_deployment.md | 4 ++-- docs/docs/cloud/index.md | 4 ++-- docs/docs/cloud/twitter_bot.md | 10 +++++----- docs/docs/development/index.md | 2 +- docs/docs/development_resources/shell.md | 4 ++-- docs/docs/index.md | 8 ++++---- 8 files changed, 18 insertions(+), 18 deletions(-) diff --git a/docs/docs/about.md b/docs/docs/about.md index 0174e125..1494e2ba 100644 --- a/docs/docs/about.md +++ b/docs/docs/about.md @@ -6,7 +6,7 @@ Of the many services that the CRWA provides to the greater Boston community, one See the website's [about page](https://crwa-flagging.herokuapp.com/about) for more about the website functionality and how it relates to the flagging program's objectives. -See the [development history](../development/history) document for more information on how this project started and how we came to make the design decisions that you see here today. +See the [development resources overview](development/history) for more information on how this project started and how we came to make the design decisions that you see here today. ## Code for Boston diff --git a/docs/docs/admin.md b/docs/docs/admin.md index 2e4ee80e..26d7226e 100644 --- a/docs/docs/admin.md +++ b/docs/docs/admin.md @@ -1,7 +1,7 @@ # Admin Panel ???+ Note - This page discusses how to use the admin panel for the website. For how to set up the admin page username and password during deployment, see the [deployment](../deployment) documentation. + This page discusses how to use the admin panel for the website. For how to set up the admin page username and password during deployment, see the [Heroku deployment](cloud/heroku_deployment) documentation. The admin panel is used to manually override the model outputs during events and advisories that would adversely effect the river quality. diff --git a/docs/docs/cloud/heroku_deployment.md b/docs/docs/cloud/heroku_deployment.md index 2aeb6e8a..b8d03b7e 100644 --- a/docs/docs/cloud/heroku_deployment.md +++ b/docs/docs/cloud/heroku_deployment.md @@ -1,7 +1,7 @@ # Remote Deployment ???+ note - This guide is an instruction manual on how to deploy the flagging website to the internet via Heroku. If you just want to run the website locally, you do not need Heroku. Instead, check out the [development](/development) guide. + This guide is an instruction manual on how to deploy the flagging website to the internet via Heroku. If you just want to run the website locally, you do not need Heroku. The following tools are required to deploy the website: @@ -119,7 +119,7 @@ git remote -v ???+ success The above command should output something like this: - ```shell + ``` heroku https://git.heroku.com/crwa-flagging.git (fetch) heroku https://git.heroku.com/crwa-flagging.git (push) origin https://github.com/YOUR_USERNAME_HERE/flagging.git (fetch) diff --git a/docs/docs/cloud/index.md b/docs/docs/cloud/index.md index e7bed2c8..3d4a6147 100644 --- a/docs/docs/cloud/index.md +++ b/docs/docs/cloud/index.md @@ -1,5 +1,5 @@ # Overview -The flagging website is designed to be hosted on [Heroku](https://heroku.com/). The guide for how to set up deployment is available [here](../cloud/heroku_deployment.md). +The flagging website is designed to be hosted on [Heroku](https://heroku.com/). The guide for how to set up deployment is available [here](heroku_deployment). -The full cloud deployment depends not only on Heroku but also Twitter's development API. The Twitter bot only needs to be set up once and, notwithstanding exigent circumstances (losing the API key, migrating the bot, or handling a Twitter ban), the Twitter bot does not need any additional maintenance. Nevertheless, there is documentation for how to set up the Twitter bot [here](../cloud/twitter_bot.md). +The full cloud deployment depends not only on Heroku but also Twitter's development API. The Twitter bot only needs to be set up once and, notwithstanding exigent circumstances (losing the API key, migrating the bot, or handling a Twitter ban), the Twitter bot does not need any additional maintenance. Nevertheless, there is documentation for how to set up the Twitter bot [here](twitter_bot). diff --git a/docs/docs/cloud/twitter_bot.md b/docs/docs/cloud/twitter_bot.md index 17c16656..b92beff1 100644 --- a/docs/docs/cloud/twitter_bot.md +++ b/docs/docs/cloud/twitter_bot.md @@ -1,6 +1,6 @@ # Twitter Bot -Every time the website updates, it sends out a tweet. +Every time the website updates, it sends out a tweet. In order for it to do that though, you need to set up a Twitter account. ## First Time Setup @@ -21,13 +21,13 @@ Follow these steps to set up the Twitter bot for the first time, such as on a ne 4. In the code base, use the `VAULT_PASSWORD` to unzip the `vault.7z` manually. You should have a file called `secrets.json`. Open up `secrets.json` in the plaintext editor of your choosing. ???+ danger - Make sure that you delete the unencrpyted, unarchived version of the `secrets.json` file after you are done with it. + Make sure that you delete the unencrypted, unarchived version of the `secrets.json` file after you are done with it. 5. Now go back to your browser with the Twitter Developer Portal. At the top of the screen, flip to the `Keys and tokens`. Now it's time to go through the dashboard and get your copy+paste ready. We will be inserting these values into the `secrets.json` (remember to wrap the keys in double quotes `"like this"` when you insert them). - - The `API Key & Secret` should should go in the corresponding fields for `"api_key": "..."` and `"api_key_secret": "..."`. - - The `Bearer Token` should go in the field `"bearer_token": "..."`. - - The `Access Token & Secret` should go in the corresponding fields for `"access_token": "..."` and `"access_token_secret": "..."`. _But first, you will need to regenerate the `Access Token & Secret` so that it has both read and write permissions._ + - The `API Key & Secret` should should go in the corresponding fields for `#!json "api_key": "..."` and `#!json "api_key_secret": "..."`. + - The `Bearer Token` should go in the field `#!json "bearer_token": "..."`. + - The `Access Token & Secret` should go in the corresponding fields for `#!json "access_token": "..."` and `#!json "access_token_secret": "..."`. _But first, you will need to regenerate the `Access Token & Secret` so that it has both read and write permissions._ ???+ success The `secrets.json` file should look something like this, with the ellipses replacing the actual values: diff --git a/docs/docs/development/index.md b/docs/docs/development/index.md index e93419da..ebc22901 100644 --- a/docs/docs/development/index.md +++ b/docs/docs/development/index.md @@ -5,7 +5,7 @@ The Development guide is aimed at users who wish to understand the code base and This overview page describes at a high-level what the website's infrastructure is, how it all relates, and why those things are in the app. !!! tip - Make sure to go through the [setup guide](../setup.md) before doing anything in the development guide. + Make sure to go through the [setup guide](../setup) before doing anything in the development guide. ## Dependency Diagram diff --git a/docs/docs/development_resources/shell.md b/docs/docs/development_resources/shell.md index 4b7e8643..7c01f8c9 100644 --- a/docs/docs/development_resources/shell.md +++ b/docs/docs/development_resources/shell.md @@ -53,10 +53,10 @@ And you should be good to go! The functions listed below should be available for - **`compose_tweet`** (*() -> str*): Generates a message for Twitter that represents the current status of the flagging program (note: this function does not actually send the Tweet to Twitter.com). -Additionally, Pandas and Numpy are already pre-imported via `import pandas as pd` and `import numpy as np`. +Additionally, Pandas and Numpy are already pre-imported via `#!python import pandas as pd` and `#!python import numpy as np`. ???+ tip - To add more functions and variables that pre-load in the Flask shell, simply add another entry to the dictionary returned by the function `make_shell_context()` in `flagging_site/app.py:creat_app()`. + To add more functions and variables that pre-load in the Flask shell, simply add another entry to the dictionary returned by the function `#!python make_shell_context()` in `flagging_site/app.py:creat_app()`. ???+ tip All of the website's functions can be run in the Flask shell, even those that are not pre-loaded in the shell's global context. All you need to do is import it. For example, let's say you want to get the un-parsed request object from USGS.gov. You can import the function we use and run it like this: diff --git a/docs/docs/index.md b/docs/docs/index.md index 650586dd..ad3514bf 100644 --- a/docs/docs/index.md +++ b/docs/docs/index.md @@ -7,7 +7,7 @@ This site provides developers and maintainers information about the CRWA's flagg ## For Website Administrators -If the website is already deployed and you want to implement a manual override, you do not need to follow the setup guide. All you need to do is read the [admin](../admin) guide to manage the website while it's deployed. +If the website is already deployed and you want to implement a manual override, you do not need to follow the setup guide. All you need to do is read the [admin](admin) guide to manage the website while it's deployed. ## Connecting to Weebly @@ -15,9 +15,9 @@ Work in progress ## For Developers -Start by following the [setup guide](../setup). Once you have the website setup locally, you now have access to the following: +Start by following the [setup guide](setup). Once you have the website setup locally, you now have access to the following: -- Deploy the website to Heroku (guide [here](../deployment)) -- Manually run commands and download data through the [shell](../shell). +- Deploy the website to Heroku (guide [here](deployment)) +- Manually run commands and download data through the [shell](shell). - Make changes to the predictive model, including revising its coefficients. (Guide is currently WIP) - (Advanced) Make other changes to the website. From 4b07be9a727b50c4c72fab292e20a8fa1f65fe97 Mon Sep 17 00:00:00 2001 From: Lewis Staples Date: Sun, 20 Sep 2020 18:34:55 -0400 Subject: [PATCH 089/118] issue 75: api for boathouse metadata --- flagging_site/blueprints/api.py | 10 ++++++++++ flagging_site/blueprints/flagging.py | 4 ++-- flagging_site/data/database.py | 23 +++++++++++++++++++---- flagging_site/templates/api/index.html | 3 +++ 4 files changed, 34 insertions(+), 6 deletions(-) diff --git a/flagging_site/blueprints/api.py b/flagging_site/blueprints/api.py index cfd52788..c02cdd60 100644 --- a/flagging_site/blueprints/api.py +++ b/flagging_site/blueprints/api.py @@ -9,6 +9,7 @@ from flask_restful import Resource from flask import current_app from ..data.model import latest_model_outputs +from ..data.database import get_boathouse_metadata_dict from flasgger import swag_from @@ -93,3 +94,12 @@ def get(self): api.add_resource(ReachesApi, '/v1/model') + + +class BoathousesApi(Resource): + def get(self): + boathouse_metadata_dict = get_boathouse_metadata_dict() + return boathouse_metadata_dict + + +api.add_resource(BoathousesApi, '/v1/boathouses') diff --git a/flagging_site/blueprints/flagging.py b/flagging_site/blueprints/flagging.py index e424c630..3adc1d42 100644 --- a/flagging_site/blueprints/flagging.py +++ b/flagging_site/blueprints/flagging.py @@ -11,7 +11,7 @@ from ..data.usgs import get_live_usgs_data from ..data.model import process_data from ..data.model import latest_model_outputs -from ..data.database import get_boathouse_dict +from ..data.database import get_boathouse_by_reach_dict bp = Blueprint('flagging', __name__) @@ -67,7 +67,7 @@ def index() -> str: in df.to_dict(orient='index').items() } - homepage = get_boathouse_dict() + homepage = get_boathouse_by_reach_dict() # verify that the same reaches are in boathouse list and model outputs if flags.keys() != homepage.keys(): diff --git a/flagging_site/data/database.py b/flagging_site/data/database.py index 00034dd4..e0de4e52 100644 --- a/flagging_site/data/database.py +++ b/flagging_site/data/database.py @@ -7,10 +7,12 @@ import re from typing import Optional from flask import current_app +from flask import jsonify from flask_sqlalchemy import SQLAlchemy from flask_sqlalchemy import declarative_base from sqlalchemy.exc import ResourceClosedError from psycopg2 import connect +from dataclasses import dataclass db = SQLAlchemy() Base = declarative_base() @@ -105,17 +107,22 @@ def update_database(): model_outs.to_sql('model_outputs', **options) +@dataclass class boathouses(db.Model): + reach: int + boathouse: str + latitude: float + longitude: float + reach = db.Column(db.Integer, unique=False) boathouse = db.Column(db.String(255), primary_key=True) latitude = db.Column(db.Numeric, unique=False) longitude = db.Column(db.Numeric, unique=False) - def __repr__(self): - return ''.format(self.boathouse) -def get_boathouse_dict(): + +def get_boathouse_by_reach_dict(): """ - Return a dict of boathouses + Return a dict of boathouses, indexed by reach """ # return value is an outer dictionary with the reach number as the keys # and the a sub-dict as the values each sub-dict has the string 'boathouses' @@ -133,3 +140,11 @@ def get_boathouse_dict(): boathouse_dict[ bh_out.reach ] = {'boathouses': bh_list} return boathouse_dict + +def get_boathouse_metadata_dict(): + """ + Return a dictionary of boathouses' metadata + """ + boathouse_query = (boathouses.query.all()) + return jsonify({"boathouses" : boathouse_query}) + diff --git a/flagging_site/templates/api/index.html b/flagging_site/templates/api/index.html index d9ac11a7..f6619204 100644 --- a/flagging_site/templates/api/index.html +++ b/flagging_site/templates/api/index.html @@ -25,4 +25,7 @@

Available Endpoints

+ {% endblock %} \ No newline at end of file From c70990745d9dc487822db38d3577c03f8e9a5736 Mon Sep 17 00:00:00 2001 From: Lewis Staples Date: Sun, 20 Sep 2020 23:59:42 -0400 Subject: [PATCH 090/118] added documentation for boathouse metadata API --- flagging_site/blueprints/api.py | 1 + flagging_site/blueprints/boathouses_api.yml | 25 +++++++++++++++++++++ 2 files changed, 26 insertions(+) create mode 100644 flagging_site/blueprints/boathouses_api.yml diff --git a/flagging_site/blueprints/api.py b/flagging_site/blueprints/api.py index c02cdd60..f8be8355 100644 --- a/flagging_site/blueprints/api.py +++ b/flagging_site/blueprints/api.py @@ -97,6 +97,7 @@ def get(self): class BoathousesApi(Resource): + @swag_from('boathouses_api.yml') def get(self): boathouse_metadata_dict = get_boathouse_metadata_dict() return boathouse_metadata_dict diff --git a/flagging_site/blueprints/boathouses_api.yml b/flagging_site/blueprints/boathouses_api.yml new file mode 100644 index 00000000..c1ebfaa6 --- /dev/null +++ b/flagging_site/blueprints/boathouses_api.yml @@ -0,0 +1,25 @@ +Endpoint returning json of boathouse metadata +--- +tags: + - Boathouse API +responses: + 200: + description: Dictionary-like json of the boathouses + schema: + id: boathouse_metadata_dict + type: object + properties: + boathouses: + description: + type: array + items: + type: object + properties: + boathouse: + type: string + latitude: + type: number + longitude: + type: number + reach: + type: integer From 292a92ec314ebc45d146faa3fd0286dc42c3876d Mon Sep 17 00:00:00 2001 From: Bertie Ancona Date: Tue, 22 Sep 2020 00:21:25 -0400 Subject: [PATCH 091/118] Organize CSS and prettify --- flagging_site/static/style.css | 169 +++++++++++++++-------------- flagging_site/templates/about.html | 2 +- flagging_site/templates/base.html | 47 ++++---- flagging_site/templates/index.html | 6 +- 4 files changed, 115 insertions(+), 109 deletions(-) diff --git a/flagging_site/static/style.css b/flagging_site/static/style.css index f0cf052a..25373ad9 100644 --- a/flagging_site/static/style.css +++ b/flagging_site/static/style.css @@ -1,149 +1,154 @@ +/***** Start Element Styling *****/ html { font-family: sans-serif; - background: #dce9f2; - padding: 1rem; } + body { max-width: 1260px; margin: 0 auto; - background: white; + background: #dce9f2; } + h1, h2, h3, h4 { color: #27475e; - margin: 1rem 0; } + a { color: #377ba8; + text-decoration: none; } + hr { border: none; - border-top: 1px solid lightgray; + border-top: thin solid #dce9f2; } + p { line-height: 25px; } -nav h1 { - flex: auto; - margin: 0; -} -nav h1 a { - text-decoration: none; - padding: 0.25rem 0.5rem; + +ul { + line-height: 25px; + list-style-type: square; } -nav ul { +/***** End Element Styling *****/ + +/***** Start Common Classes *****/ +.above-fold { + /* Everything above the bottom of the screen */ + min-height: 100vh; display: flex; - list-style: none; - margin: 0; - padding: 0; -} -nav ul li a, -nav ul li span, -header .action { - display: block; - padding: 0.5rem; + flex-direction: column; } -ul { - line-height: 25px; + +.centered { + text-align: center; } -ul.home-boat-house { - list-style-type: square; - padding-left: 15px; - padding-right: 15px; - text-align: left; + +.non-breaking { + display: inline-block; } -section.page-header { +/***** End Common Classes *****/ + +/***** Start Header Section *****/ +.page-header { text-align: center; - border-bottom: 1px solid lightgray; - padding-bottom: 0.7em; + border-bottom: thin solid #dce9f2; + padding: 15px; + background: white; } + .page-header > header { padding: 0.4em 0 0.4em 0; text-align: center; font-size: 1.5em; } + +.header-logo { + max-width: 100%; +} +/***** End Header Section *****/ + +/***** Start Content Section *****/ .content { - padding: 0 1rem 1rem; + background: white; + padding: 24px; + flex-grow: 1; +} + +@media(min-width: 1024px) { + .content { + padding: 48px 10%; + } } + .content > header h1 { flex: auto; margin: 1rem 0 0.25rem 0; } -.flash { - margin: 1em 0; - padding: 1em; - background: #cae6f6; - border: 1px solid #377ba8; -} -.flag { - width: 2rem; - height: auto; - padding-top: 1rem; - padding-left: 1rem; - padding-right: 1rem; -} -.post > header { - display: flex; - align-items: flex-end; - font-size: 0.85em; -} -.post > header > div:first-of-type { - flex: auto; -} -.post > header h1 { - font-size: 1.5em; - margin-bottom: 0; -} -.post .about { - color: slategray; - font-style: italic; -} -.post .body { - white-space: pre-line; -} + .content:last-child { margin-bottom: 0; } + .content form { margin: 1em 0; display: flex; flex-direction: column; } + .content label { font-weight: bold; margin-bottom: 0.5em; } + .content input, .content textarea { margin-bottom: 1em; } -.footer { - padding: 3em 1em 3em 1em; - font-size: 0.85em; -} -.footer > .footer-content { - width: 30em; - text-align: center; - margin: 0 auto; -} .content textarea { min-height: 12em; resize: vertical; } +/***** End Content Section *****/ -.home-centered{ +/***** Start Footer Section *****/ +.footer { text-align: center; + padding: 3em 1em 3em 1em; + font-size: 0.85em; + background: white; + border-top: thin solid #dce9f2; } -.home-reach{ + +.footer p { + line-height: normal; +} + +.footer .footer-section:not(:first-child) { + padding-top: 30px; +} +/***** End Footer *****/ + +/***** Start Home Page *****/ +.home-boat-house { + margin: 5px 0; + text-align: left; +} + +.home-reach { border: 1px solid black; border-collapse: collapse; padding: 15px; margin: 0 auto; } -input.danger { - color: #cc2f2e; -} -input[type="submit"] { - align-self: start; - min-width: 10em; + +.flag { + width: 2rem; + height: auto; + padding-top: 1rem; + padding-left: 1rem; + padding-right: 1rem; } +/***** End Home Page *****/ diff --git a/flagging_site/templates/about.html b/flagging_site/templates/about.html index 512997c4..a9d0c84e 100644 --- a/flagging_site/templates/about.html +++ b/flagging_site/templates/about.html @@ -10,7 +10,7 @@

Flagging Program

Water Contamination

- CRWA conducts two water quality monitoring events per week during the peak recreational season in the Lower Basin from late June to late October. + CRWA conducts two water quality monitoring events per week during the peak recreational season in the Lower Basin from late June to late October. During monitoring events, CRWA measures water temperature and depth and collects water quality samples at four sampling locations in the Lower Basin. G & L Laboratory in Quincy analyzes the samples for E. coli bacteria. Water temperature and depth are measured in situ with a digital field thermometer and a digital depth finder, respectively. Sampling locations are center channel sites, upstream of the following bridges: North Beacon Street, diff --git a/flagging_site/templates/base.html b/flagging_site/templates/base.html index 0a233ff5..ed13ae1f 100644 --- a/flagging_site/templates/base.html +++ b/flagging_site/templates/base.html @@ -23,32 +23,33 @@ -

- {% block header %} - -
{{ self.title() }}
-
- Home -   |   About -   |   Detailed Outputs -   |   API +
+
+ {% block header %} + +
{{ self.title() }}
+ + {% endblock %}
- {% endblock %} -
-
{% block content %}{% endblock %}
+
{% block content %}{% endblock %}
+
{% block footer %} -
-
- Charles River Watershed Association
- 190 Park Rd, Weston, MA 02493
- t (781) 788-0007     - e charles@crwa.org
-
- This website was built by Code for Boston in - collaboration with Charles River Watershed Association.
-
- Page loaded in {{ g.request_time() }} seconds. +
+

Charles River Watershed Association

+

190 Park Rd, Weston, MA 02493

+

+ Phone: (781) 788-0007     + Email: charles@crwa.org +

+
+
+

This website was built by Code for Boston in collaboration with Charles River Watershed Association.

{% endblock %}
diff --git a/flagging_site/templates/index.html b/flagging_site/templates/index.html index 7260e8c4..175eb208 100644 --- a/flagging_site/templates/index.html +++ b/flagging_site/templates/index.html @@ -7,9 +7,9 @@

Overview

safe for boating at eleven boating locations between Watertown and Boston. Flag colors are based on E. coli and cyanobacteria (blue-green algae) levels:

-