From f248ffcd6efba1fea1513a4a83d98789871b5c80 Mon Sep 17 00:00:00 2001 From: Dmytro Trotsko Date: Wed, 16 Aug 2023 07:41:08 +0300 Subject: [PATCH 1/3] Replaced old admin endpoint with Flask-Admin lib. Added new internal endpoints for API Key registration and removal request. Added two new tables to track registration/removal requests. Moved /diagnostics route to the main.py. Added general `send_email` function to the _common.py and reused it for sending emails in places where it is needed. --- requirements.api.txt | 1 + src/ddl/api_user.sql | 16 ++ src/maintenance/remove_outdated_keys.py | 31 +-- src/server/_common.py | 16 +- src/server/_config.py | 5 + src/server/_security.py | 4 +- src/server/admin/models.py | 57 +++-- src/server/admin/templates/index.html | 81 ------- src/server/endpoints/__init__.py | 4 + src/server/endpoints/admin.py | 217 ++++++++---------- .../endpoints/api_key_removal_request.py | 31 +++ src/server/endpoints/registration.py | 45 ++++ src/server/main.py | 30 ++- src/server/static/css/style.css | 41 ++++ src/server/templates/registration.html | 74 ++++++ src/server/templates/removal_request.html | 62 +++++ 16 files changed, 467 insertions(+), 248 deletions(-) delete mode 100644 src/server/admin/templates/index.html create mode 100644 src/server/endpoints/api_key_removal_request.py create mode 100644 src/server/endpoints/registration.py create mode 100644 src/server/static/css/style.css create mode 100644 src/server/templates/registration.html create mode 100644 src/server/templates/removal_request.html diff --git a/requirements.api.txt b/requirements.api.txt index 10fed2eec..0d5a123cf 100644 --- a/requirements.api.txt +++ b/requirements.api.txt @@ -1,6 +1,7 @@ delphi_utils==0.3.15 epiweeks==2.1.2 Flask==2.2.2 +Flask-Admin==1.6.1 Flask-Limiter==3.3.0 itsdangerous<2.1 jinja2==3.0.3 diff --git a/src/ddl/api_user.sql b/src/ddl/api_user.sql index 90e56539f..263fedb4b 100644 --- a/src/ddl/api_user.sql +++ b/src/ddl/api_user.sql @@ -29,3 +29,19 @@ CREATE TABLE IF NOT EXISTS `user_role_link` ( `role_id` int(11) UNSIGNED NOT NULL, PRIMARY KEY (`user_id`, `role_id`) ) ENGINE=InnoDB DEFAULT CHARSET=utf8; + + +-- `registration_responses` table + +CREATE TABLE IF NOT EXISTS `registration_responses` ( + `email` varchar(320) UNIQUE NOT NULL, + `organization` varchar(120), + `purpose` varchar(320) +) ENGINE=InnoDB DEFAULT CHARSET=utf8; + +-- `removal_requests` table + +CREATE TABLE IF NOT EXISTS `removal_requests` ( + `api_key` varchar(50) UNIQUE NOT NULL, + `comment` varchar(320) +) ENGINE=InnoDB DEFAULT CHARSET=utf8; \ No newline at end of file diff --git a/src/maintenance/remove_outdated_keys.py b/src/maintenance/remove_outdated_keys.py index 56fea0cf9..ff41cc9a3 100644 --- a/src/maintenance/remove_outdated_keys.py +++ b/src/maintenance/remove_outdated_keys.py @@ -4,19 +4,10 @@ import delphi.operations.secrets as secrets import mysql.connector from delphi.epidata.server._config import API_KEY_REGISTRATION_FORM_LINK_LOCAL +from delphi.epidata.server._common import send_email ApiUserRecord = namedtuple("APIUserRecord", ("api_key", "email", "date_diff")) -SMTP_HOST = "relay.andrew.cmu.edu" -SMTP_PORT = 25 - -EMAIL_SUBJECT = "Your API Key was deleted." -EMAIL_FROM = "noreply@andrew.cmu.edu" -ALERT_EMAIL_MESSAGE = f"""Hi! \n Your API Key is going to be removed due to inactivity. -To renew it, pelase use it within one month from now.""" -DELETED_EMAIL_MESSAGE = f"""Hi! \n Your API Key was removed due to inactivity. -To get new one, please use registration form ({API_KEY_REGISTRATION_FORM_LINK_LOCAL}) or contact us.""" - def get_old_keys(cur): cur.execute( @@ -41,14 +32,6 @@ def remove_outdated_key(cur, api_key): ) -def send_notification(to_addr, alert=True): - message = ALERT_EMAIL_MESSAGE if alert else DELETED_EMAIL_MESSAGE - BODY = "\r\n".join((f"FROM: {EMAIL_FROM}", f"TO: {to_addr}", f"Subject: {EMAIL_SUBJECT}", "", message)) - smtp_server = SMTP(host=SMTP_HOST, port=SMTP_PORT) - smtp_server.starttls() - smtp_server.sendmail(EMAIL_FROM, to_addr, BODY) - - def main(): u, p = secrets.db.epi cnx = mysql.connector.connect(database="epidata", user=u, password=p, host=secrets.db.host) @@ -56,10 +39,18 @@ def main(): outdated_keys_list = get_old_keys(cur) for item in outdated_keys_list: if item.date_diff == 5: - send_notification(item.email) + send_email( + to_addr=item.email, + subject="API Key will be expired soon", + message=f"Hi! \n Your API Key: {item.api_key}, is going to be expired due to inactivity.", + ) else: remove_outdated_key(cur, item.api_key) - send_notification(item.email, alert=False) + send_email( + to_addr=item.email, + subject="Your API Key was expired", + message=f"Hi! \n Your API Key: {item.api_key}, was removed due to inactivity. To get new one, please use registration form ({API_KEY_REGISTRATION_FORM_LINK_LOCAL}) or contact us."" + ) cur.close() cnx.commit() cnx.close() diff --git a/src/server/_common.py b/src/server/_common.py index 8633d07fd..fb42e413e 100644 --- a/src/server/_common.py +++ b/src/server/_common.py @@ -6,9 +6,10 @@ from sqlalchemy.engine import Connection, Engine from werkzeug.exceptions import Unauthorized from werkzeug.local import LocalProxy +from smtplib import SMTP from delphi.epidata.common.logger import get_structured_logger -from ._config import SECRET, REVERSE_PROXY_DEPTH +from ._config import SECRET, REVERSE_PROXY_DEPTH, SMTP_HOST, SMTP_PORT, EMAIL_FROM from ._db import engine from ._exceptions import DatabaseErrorException, EpiDataException from ._security import current_user, _is_public_route, resolve_auth_token, update_key_last_time_used, ERROR_MSG_INVALID_KEY @@ -210,3 +211,16 @@ def set_compatibility_mode(): sets the compatibility mode for this request """ g.compatibility = True + + +def send_email(to_addr: str, subject: str, message: str): + """Send email messages + Args: + to_addr (str): Reciever email address + subject (str): Email subject + message (str): Email message + """ + smtp_server = SMTP(host=SMTP_HOST, port=SMTP_PORT) + smtp_server.starttls() + body = "\r\n".join((f"FROM: {EMAIL_FROM}", f"TO: {to_addr}", f"Subject: {subject}", "", message)) + smtp_server.sendmail(EMAIL_FROM, to_addr, body) \ No newline at end of file diff --git a/src/server/_config.py b/src/server/_config.py index 841e93b9e..a0914e04f 100644 --- a/src/server/_config.py +++ b/src/server/_config.py @@ -111,3 +111,8 @@ # ^ shortcut to "https://docs.google.com/forms/d/e/1FAIpQLSff30tsq4xwPCoUbvaIygLSMs_Mt8eDhHA0rifBoIrjo8J5lw/viewform" API_KEY_REMOVAL_REQUEST_LINK_LOCAL = "https://api.delphi.cmu.edu/epidata/admin/removal_request" # ^ redirects to API_KEY_REMOVAL_REQUEST_LINK + +# STMP credentials +SMTP_HOST = "relay.andrew.cmu.edu" +SMTP_PORT = 25 +EMAIL_FROM = "noreply@andrew.cmu.edu" \ No newline at end of file diff --git a/src/server/_security.py b/src/server/_security.py index 38294eb10..e6b8708e6 100644 --- a/src/server/_security.py +++ b/src/server/_security.py @@ -54,10 +54,12 @@ def _get_current_user(): def _is_public_route() -> bool: - public_routes_list = ["lib", "admin", "version"] + public_routes_list = ["lib", "version", "diagnostics"] for route in public_routes_list: if request.path.startswith(f"{URL_PREFIX}/{route}"): return True + if "admin" in request.path: + return True return False diff --git a/src/server/admin/models.py b/src/server/admin/models.py index f5c0d54ed..62b418bab 100644 --- a/src/server/admin/models.py +++ b/src/server/admin/models.py @@ -6,8 +6,8 @@ from .._db import Session, WriteSession, default_session from delphi.epidata.common.logger import get_structured_logger -from typing import Set, Optional, List -from datetime import datetime as dtime +from typing import Set, Optional +from datetime import date Base = declarative_base() @@ -20,7 +20,7 @@ ) def _default_date_now(): - return dtime.strftime(dtime.now(), "%Y-%m-%d") + return date.today() class User(Base): __tablename__ = "api_user" @@ -35,16 +35,6 @@ def __init__(self, api_key: str, email: str = None) -> None: self.api_key = api_key self.email = email - @property - def as_dict(self): - return { - "id": self.id, - "api_key": self.api_key, - "email": self.email, - "roles": set(role.name for role in self.roles), - "created": self.created, - "last_time_used": self.last_time_used - } def has_role(self, required_role: str) -> bool: return required_role in set(role.name for role in self.roles) @@ -121,6 +111,9 @@ class UserRole(Base): id = Column(Integer, primary_key=True, autoincrement=True) name = Column(String(50), unique=True) + def __repr__(self): + return self.name + @staticmethod @default_session(WriteSession) def create_role(name: str, session) -> None: @@ -142,3 +135,41 @@ def create_role(name: str, session) -> None: def list_all_roles(session): roles = session.query(UserRole).all() return [role.name for role in roles] + + +class RegistrationResponse(Base): + __tablename__ = "registration_responses" + + email = Column(String(320), unique=True, nullable=False, primary_key=True) + organization = Column(String(120), unique=False, nullable=True) + purpose = Column(String(320), unique=False, nullable=True) + + def __init__(self, email: str, organization: str = None, purpose: str = None) -> None: + self.email = email + self.organization = organization + self.purpose = purpose + + @staticmethod + @default_session(WriteSession) + def add_response(email: str, organization: str, purpose: str, session): + new_response = RegistrationResponse(email, organization, purpose) + session.add(new_response) + session.commit() + + +class RemovalRequest(Base): + __tablename__ = "removal_requests" + + api_key = Column(String(50), unique=True, nullable=False, primary_key=True) + comment = Column(String(320), unique=False, nullable=True) + + def __init__(self, api_key: str, comment: str = None) -> None: + self.api_key = api_key + self.comment = comment + + @staticmethod + @default_session(WriteSession) + def add_request(api_key: str, comment: str, session): + new_request = RemovalRequest(api_key, comment) + session.add(new_request) + session.commit() \ No newline at end of file diff --git a/src/server/admin/templates/index.html b/src/server/admin/templates/index.html deleted file mode 100644 index 518211199..000000000 --- a/src/server/admin/templates/index.html +++ /dev/null @@ -1,81 +0,0 @@ - - - - - - API Keys - - - - -
-

- API Key Admin Interface -

- {% if flags.banner %} - - {% endif %} - {% if mode == 'overview' %} -

Registered Users

- - - - - - - - - - {% for user in users %} - - - - - - - - {% endfor %} - -
IDAPI KeyEmailRolesActions
{{ user.id }}{{ user.api_key }}{{ user.email }}{{ ','.join(user.roles) }} - Edit - Delete -
-

Register New User

- - {% else %} -

- < Back - Edit User {{user.id}} -

- {% endif %} -
- -
- - -
-
- - -
-
- - {% for role in roles %} - - {% endfor %} -
- {% if mode == 'overview' %} - - {% else %} - - - {% endif %} -
-
- - \ No newline at end of file diff --git a/src/server/endpoints/__init__.py b/src/server/endpoints/__init__.py index 94f1de5b8..1891a42c4 100644 --- a/src/server/endpoints/__init__.py +++ b/src/server/endpoints/__init__.py @@ -31,6 +31,8 @@ wiki, signal_dashboard_status, signal_dashboard_coverage, + registration, + api_key_removal_request ) endpoints = [ @@ -66,6 +68,8 @@ wiki, signal_dashboard_status, signal_dashboard_coverage, + registration, + api_key_removal_request ] __all__ = ["endpoints"] diff --git a/src/server/endpoints/admin.py b/src/server/endpoints/admin.py index a6f941b48..a71a3b297 100644 --- a/src/server/endpoints/admin.py +++ b/src/server/endpoints/admin.py @@ -1,135 +1,108 @@ -from pathlib import Path -from typing import Dict, List, Set +from datetime import timedelta +from functools import wraps -from flask import Blueprint, make_response, render_template_string, request -from werkzeug.exceptions import NotFound, Unauthorized -from werkzeug.utils import redirect +from flask import session +from flask_admin import Admin, AdminIndexView, expose +from flask_admin.contrib.sqla import ModelView +from werkzeug.exceptions import Unauthorized -from .._common import log_info_with_request -from .._config import ADMIN_PASSWORD, API_KEY_REGISTRATION_FORM_LINK, API_KEY_REMOVAL_REQUEST_LINK, REGISTER_WEBHOOK_TOKEN +from .._common import app +from .._config import ADMIN_PASSWORD from .._db import WriteSession from .._security import resolve_auth_token from ..admin.models import User, UserRole -self_dir = Path(__file__).parent -# first argument is the endpoint name -bp = Blueprint("admin", __name__) +# set optional bootswatch theme +app.config["FLASK_ADMIN_SWATCH"] = "cerulean" +# set app secret key to enable session +app.secret_key = "SOME_RANDOM_SECRET_KEY" -templates_dir = Path(__file__).parent.parent / "admin" / "templates" +def require_auth(func): + @wraps(func) + def check_token(*args, **kwargs): + # Check to see if it's in user's session + if "admin_auth_token" not in session: + raise Unauthorized() + return func(*args, **kwargs) -def enable_admin() -> bool: - # only enable admin endpoint if we have a password for it, so it is not exposed to the world - return bool(ADMIN_PASSWORD) + return check_token -def _require_admin(): +def require_admin(): token = resolve_auth_token() if token is None or token != ADMIN_PASSWORD: - raise Unauthorized() - return token - - -def _render(mode: str, token: str, flags: Dict, session, **kwargs): - template = (templates_dir / "index.html").read_text("utf8") - return render_template_string( - template, mode=mode, token=token, flags=flags, roles=UserRole.list_all_roles(session), **kwargs - ) - - -# ~~~~ PUBLIC ROUTES ~~~~ - - -@bp.route("/registration_form", methods=["GET"]) -def registration_form_redirect(): - # TODO: replace this with our own hosted registration form instead of external - return redirect(API_KEY_REGISTRATION_FORM_LINK, code=302) - - -@bp.route("/removal_request", methods=["GET"]) -def removal_request_redirect(): - # TODO: replace this with our own hosted form instead of external - return redirect(API_KEY_REMOVAL_REQUEST_LINK, code=302) - - -# ~~~~ PRIVLEGED ROUTES ~~~~ - - -@bp.route("/", methods=["GET", "POST"]) -def _index(): - token = _require_admin() - flags = dict() - with WriteSession() as session: - if request.method == "POST": - # register a new user - if not User.find_user( - user_email=request.values["email"], api_key=request.values["api_key"], - session=session): - User.create_user( - api_key=request.values["api_key"], - email=request.values["email"], - user_roles=set(request.values.getlist("roles")), - session=session - ) - flags["banner"] = "Successfully Added" - else: - flags["banner"] = "User with such email and/or api key already exists." - users = [user.as_dict for user in session.query(User).all()] - return _render("overview", token, flags, session=session, users=users, user=dict()) - - -@bp.route("/", methods=["GET", "PUT", "POST", "DELETE"]) -def _detail(user_id: int): - token = _require_admin() - with WriteSession() as session: - user = User.find_user(user_id=user_id, session=session) - if not user: - raise NotFound() - if request.method == "DELETE" or "delete" in request.values: - User.delete_user(user.id, session=session) - return redirect(f"./?auth={token}") - flags = dict() - if request.method in ["PUT", "POST"]: - user_check = User.find_user(api_key=request.values["api_key"], user_email=request.values["email"], session=session) - if user_check and user_check.id != user.id: - flags["banner"] = "Could not update user; same api_key and/or email already exists." - else: - user = User.update_user( - user=user, - api_key=request.values["api_key"], - email=request.values["email"], - roles=set(request.values.getlist("roles")), - session=session - ) - flags["banner"] = "Successfully Saved" - return _render("detail", token, flags, session=session, user=user.as_dict) - - -@bp.route("/register", methods=["POST"]) -def _register(): - body = request.get_json() - token = body.get("token") - if token is None or token != REGISTER_WEBHOOK_TOKEN: - raise Unauthorized() - - user_api_key = body["user_api_key"] - user_email = body["user_email"] - with WriteSession() as session: - if User.find_user(user_email=user_email, api_key=user_api_key, session=session): - return make_response( - "User with email and/or API Key already exists, use different parameters or contact us for help", - 409, - ) - User.create_user(api_key=user_api_key, email=user_email, session=session) - return make_response(f"Successfully registered API key '{user_api_key}'", 200) - - -@bp.route("/diagnostics", methods=["GET", "PUT", "POST", "DELETE"]) -def diags(): - # allows us to get useful diagnostic information written into server logs, - # such as a full current "X-Forwarded-For" path as inserted into headers by intermediate proxies... - # (but only when initiated purposefully by us to keep junk out of the logs) - _require_admin() - log_info_with_request("diagnostics", headers=request.headers) - response_text = f"request path: {request.headers.get('X-Forwarded-For', 'idk')}" - return make_response(response_text, 200, {'content-type': 'text/plain'}) + if "admin_auth_token" not in session: + raise Unauthorized() + session["admin_auth_token"] = token + + +class AuthModelView(ModelView): + @require_auth + def is_accessible(self): + return True + + +@app.before_first_request # runs before FIRST request (only once) +def make_session_permanent(): + session.permanent = True + app.permanent_session_lifetime = timedelta(minutes=30) + + +class AuthAdminIndexView(AdminIndexView): + """ + Admin view main page + require_admin() is used for authentication using one of key words("auth", "api_key", "token") with ADMIN_PASSWORD + """ + + @expose("/") + def index(self): + require_admin() + return super().index() + + +class UserView(AuthModelView): + """ + User model view: + - form_columns: list of columns that will be available in CRUD forms + - column_list: list of columns that are displayed on user model view page + - column_filters: list of available filters + - page_size: number of items on page + """ + + form_columns = ["api_key", "email", "roles"] + column_list = ("api_key", "email", "created", "last_time_used", "roles") + column_filters = ("api_key", "email") + + page_size = 10 + + +class UserRoleView(AuthModelView): + """ + User role view: + - colums_filters: list of available filters + - page_size: number of items on page + """ + + column_filters = ["name"] + + page_size = 10 + + +# init admin view, default endpoint is /admin +admin = Admin(app, name="EpiData admin", template_mode="bootstrap4", index_view=AuthAdminIndexView()) +# database session +admin_session = WriteSession() + +# add views +admin.add_view(UserView(User, admin_session)) +admin.add_view(UserRoleView(UserRole, admin_session)) + + +@app.teardown_request +def teardown_request(*args, **kwargs): + """ + Remove the session after each request. + That is used to protect from dirty read. + """ + admin_session.close() diff --git a/src/server/endpoints/api_key_removal_request.py b/src/server/endpoints/api_key_removal_request.py new file mode 100644 index 000000000..4c613b5ec --- /dev/null +++ b/src/server/endpoints/api_key_removal_request.py @@ -0,0 +1,31 @@ +from flask import Blueprint, render_template, request + +from .._common import send_email +from .._db import WriteSession +from ..admin.models import RemovalRequest, User + +bp = Blueprint("removal_request", __name__, template_folder="/app/delphi/epidata/server/templates", static_url_path="") +alias = None + + +REMOVAL_REQUEST_MESSAGE = """ +Your API Key: {}, removal request will be processed soon. +To verify, we will send you an email message after processing your request. +Best, +Delphi Team +""" + + +@bp.route("/", methods=["GET", "POST"]) +def handle(): + flags = dict() + with WriteSession() as session: + if request.method == "POST": + api_key = request.values["api_key"] + comment = request.values.get("comment") + user = User.find_user(api_key=api_key) + # User.delete_user(user.id, session) + RemovalRequest.add_request(api_key, comment, session) + flags["banner"] = "Your request has been successfully recorded." + send_email(user.email, "API Key removal request", REMOVAL_REQUEST_MESSAGE.format(api_key)) + return render_template("removal_request.html", flags=flags) diff --git a/src/server/endpoints/registration.py b/src/server/endpoints/registration.py new file mode 100644 index 000000000..cf72c6a49 --- /dev/null +++ b/src/server/endpoints/registration.py @@ -0,0 +1,45 @@ +import random +import string + +from flask import Blueprint, render_template, request + +from .._common import send_email +from .._db import WriteSession +from ..admin.models import RegistrationResponse, User + +# first argument is the endpoint name +bp = Blueprint("registration_form", __name__, template_folder="/app/delphi/epidata/server/templates", static_url_path="") +alias = None + +NEW_KEY_MESSAGE = """ +Thank you for registering with the Delphi Epidata API. +Your API key is: {} +For usage information, see the API Keys section of the documentation: https://cmu-delphi.github.io/delphi-epidata/api/api_keys.html +Best, +Delphi Team +""" + + +@bp.route("/", methods=["GET", "POST"]) +def handle(): + flags = dict() + with WriteSession() as session: + if request.method == "POST": + email = request.values["email"] + if not User.find_user(user_email=email): + # Use a separate table for email, purpose, organization + api_key = "".join(random.choices(string.ascii_letters + string.digits, k=13)) + user = User.create_user(api_key=api_key, email=email, session=session) + RegistrationResponse.add_response( + email=email, + organization=request.values["organization"], + purpose=request.values["purpose"], + session=session, + ) + send_email( + to_addr=email, subject="Delphi API Key Registration", message=NEW_KEY_MESSAGE.format(api_key) + ) + flags["banner"] = "Successfully sent" + else: + flags["error"] = "User with such email already exists. Please try another email address or contact us." + return render_template("registration.html", flags=flags) diff --git a/src/server/main.py b/src/server/main.py index a91a91ee2..4b9bee2f0 100644 --- a/src/server/main.py +++ b/src/server/main.py @@ -2,16 +2,17 @@ import logging from typing import Dict, Callable -from flask import request, send_file, Response, send_from_directory, jsonify +from flask import request, send_file, Response, send_from_directory, jsonify, make_response from delphi.epidata.common.logger import get_structured_logger from ._config import URL_PREFIX, VERSION -from ._common import app, set_compatibility_mode +from ._common import app, set_compatibility_mode, log_info_with_request +from .endpoints.admin import require_admin from ._exceptions import MissingOrWrongSourceException from .endpoints import endpoints -from .endpoints.admin import bp as admin_bp, enable_admin -from ._limiter import limiter, apply_limit +from .endpoints.admin import * +from ._limiter import apply_limit __all__ = ["app"] @@ -30,12 +31,6 @@ logger.info("endpoint has alias", bp_name=endpoint.bp.name, alias=alias) endpoint_map[alias] = endpoint.handle -if enable_admin(): - logger.info("admin endpoint enabled") - limiter.exempt(admin_bp) - app.register_blueprint(admin_bp, url_prefix=f"{URL_PREFIX}/admin") - - @app.route(f"{URL_PREFIX}/api.php", methods=["GET", "POST"]) @apply_limit def handle_generic(): @@ -63,6 +58,21 @@ def send_lib_file(path: str): return send_from_directory(pathlib.Path(__file__).parent / "lib", path) +@app.route('/static/') +def send_static(path): + return send_from_directory("/app/delphi/epidata/server/static", path) + +@app.route(f"{URL_PREFIX}/diagnostics", methods=["GET", "PUT", "POST", "DELETE"]) +def diags(): + # allows us to get useful diagnostic information written into server logs, + # such as a full current "X-Forwarded-For" path as inserted into headers by intermediate proxies... + # (but only when initiated purposefully by us to keep junk out of the logs) + require_admin() + log_info_with_request("diagnostics", headers=request.headers) + response_text = f"request path: {request.headers.get('X-Forwarded-For', 'idk')}" + return make_response(response_text, 200, {"content-type": "text/plain"}) + + if __name__ == "__main__": app.run(host="0.0.0.0", port=5000) else: diff --git a/src/server/static/css/style.css b/src/server/static/css/style.css new file mode 100644 index 000000000..26ba2862b --- /dev/null +++ b/src/server/static/css/style.css @@ -0,0 +1,41 @@ +.divider-text { + position: relative; + text-align: center; + margin-top: 15px; + margin-bottom: 15px; +} + +.divider-text span { + padding: 7px; + font-size: 12px; + position: relative; + z-index: 2; +} + +.divider-text:after { + content: ""; + position: absolute; + width: 100%; + border-bottom: 1px solid #ddd; + top: 55%; + left: 0; + z-index: 1; +} + +.btn-facebook { + background-color: #405D9D; + color: #fff; +} + +.btn-twitter { + background-color: #42AEEC; + color: #fff; +} + +.text-description { + padding: 3%; +} + +.card-body { + width: 100%; +} \ No newline at end of file diff --git a/src/server/templates/registration.html b/src/server/templates/registration.html new file mode 100644 index 000000000..826099040 --- /dev/null +++ b/src/server/templates/registration.html @@ -0,0 +1,74 @@ + + + + + + + API Keys registration page + + + + + + + +
+
+
+

Delphi: Register API Key

+ {% if flags.banner %} + + {% endif %} + {% if flags.error %} + + {% endif %} +

This form allows you to register your API usage in order to lift the + restrictions + applied to anonymous access. If you regularly or frequently use our system, please consider using an + API key even if your usage falls within the anonymous usage limits. API key usage helps us + understand who and how others are using our Delphi Epidata API, which may in turn inform our future + research, data partnerships, and funding. +
+ It is important for us to be able to contact our high-traffic users in case of excessive or abnormal + activity that may adversely affect our systems. +
+ The questions about your organization and use case are optional, but we hope you will answer in + detail to help us better understand who and how others are using our API. +

+
+
+
+ + +
+
+
+ + +
+
+
+ + +
+
+
+ +
+
+
+
+
+ + + \ No newline at end of file diff --git a/src/server/templates/removal_request.html b/src/server/templates/removal_request.html new file mode 100644 index 000000000..67d1c8aaa --- /dev/null +++ b/src/server/templates/removal_request.html @@ -0,0 +1,62 @@ + + + + + + + API Key removal request page + + + + + + +
+
+
+

Delphi: API Key removal request

+ {% if flags.banner %} + + {% endif %} +

+ Submit this form if you would like Delphi to disable your API key and destroy all information + associating that key with your identity. We will confirm the request by emailing the address you used at + registration time. Since this is a destructive operation, we handle these requests manually; you should + expect a response within 1-2 business days. +

+
+
+
+ + +
+
+
+ + +
+
+
+
+
+ + +
+
+
+ +
+
+
+
+
+ + + \ No newline at end of file From a9899d114631a1e4c790d65234a156a4f86a7a5b Mon Sep 17 00:00:00 2001 From: Dmytro Trotsko Date: Tue, 22 Aug 2023 22:58:08 +0300 Subject: [PATCH 2/3] Added login page. Changed admin authentication --- requirements.api.txt | 1 + src/server/_config.py | 4 +- src/server/endpoints/admin.py | 71 ++++++++++++++++++-------- src/server/endpoints/registration.py | 3 +- src/server/main.py | 4 +- src/server/templates/login.html | 36 +++++++++++++ src/server/templates/registration.html | 8 ++- 7 files changed, 101 insertions(+), 26 deletions(-) create mode 100644 src/server/templates/login.html diff --git a/requirements.api.txt b/requirements.api.txt index 0d5a123cf..f18f1ec61 100644 --- a/requirements.api.txt +++ b/requirements.api.txt @@ -2,6 +2,7 @@ delphi_utils==0.3.15 epiweeks==2.1.2 Flask==2.2.2 Flask-Admin==1.6.1 +Flask-Login==0.6.2 Flask-Limiter==3.3.0 itsdangerous<2.1 jinja2==3.0.3 diff --git a/src/server/_config.py b/src/server/_config.py index a0914e04f..6ea9a6723 100644 --- a/src/server/_config.py +++ b/src/server/_config.py @@ -115,4 +115,6 @@ # STMP credentials SMTP_HOST = "relay.andrew.cmu.edu" SMTP_PORT = 25 -EMAIL_FROM = "noreply@andrew.cmu.edu" \ No newline at end of file +EMAIL_FROM = "noreply@andrew.cmu.edu" + +RECAPTCHA_SITE_KEY = os.environ.get("RECAPTCHA_SITE_KEY") diff --git a/src/server/endpoints/admin.py b/src/server/endpoints/admin.py index a71a3b297..75775459c 100644 --- a/src/server/endpoints/admin.py +++ b/src/server/endpoints/admin.py @@ -1,8 +1,8 @@ from datetime import timedelta -from functools import wraps -from flask import session -from flask_admin import Admin, AdminIndexView, expose +from flask import session, request, redirect, url_for, render_template +import flask_login +from flask_admin import Admin, AdminIndexView, expose, BaseView from flask_admin.contrib.sqla import ModelView from werkzeug.exceptions import Unauthorized @@ -17,28 +17,51 @@ # set app secret key to enable session app.secret_key = "SOME_RANDOM_SECRET_KEY" +login_manager = flask_login.LoginManager() +login_manager.login_view = "login" +login_manager.init_app(app) -def require_auth(func): - @wraps(func) - def check_token(*args, **kwargs): - # Check to see if it's in user's session - if "admin_auth_token" not in session: - raise Unauthorized() - return func(*args, **kwargs) - return check_token - - -def require_admin(): +def _require_admin(): token = resolve_auth_token() if token is None or token != ADMIN_PASSWORD: - if "admin_auth_token" not in session: - raise Unauthorized() - session["admin_auth_token"] = token + raise Unauthorized() + return token + + +class AdminUser(flask_login.UserMixin): + pass + + +@login_manager.user_loader +def user_loader(admin_token): + if admin_token != ADMIN_PASSWORD: + return + + user = AdminUser() + user.id = admin_token + return user + + +@login_manager.unauthorized_handler +def unauthorized_handler(): + return "Unauthorized", 401 + + +@app.route("/login", methods=["GET", "POST"]) +def login(): + if request.method == "POST": + admin_token = request.form["admin_token"] + if admin_token == ADMIN_PASSWORD: + user = AdminUser() + user.id = admin_token + flask_login.login_user(user) + return redirect(url_for("admin.index")) + return render_template("login.html") class AuthModelView(ModelView): - @require_auth + @flask_login.login_required def is_accessible(self): return True @@ -52,12 +75,11 @@ def make_session_permanent(): class AuthAdminIndexView(AdminIndexView): """ Admin view main page - require_admin() is used for authentication using one of key words("auth", "api_key", "token") with ADMIN_PASSWORD """ @expose("/") + @flask_login.login_required def index(self): - require_admin() return super().index() @@ -89,6 +111,14 @@ class UserRoleView(AuthModelView): page_size = 10 +class LogoutView(BaseView): + @expose("/") + @flask_login.login_required + def logout(self): + flask_login.logout_user() + return redirect(url_for("login")) + + # init admin view, default endpoint is /admin admin = Admin(app, name="EpiData admin", template_mode="bootstrap4", index_view=AuthAdminIndexView()) # database session @@ -97,6 +127,7 @@ class UserRoleView(AuthModelView): # add views admin.add_view(UserView(User, admin_session)) admin.add_view(UserRoleView(UserRole, admin_session)) +admin.add_view(LogoutView(name="Logout", endpoint="logout")) @app.teardown_request diff --git a/src/server/endpoints/registration.py b/src/server/endpoints/registration.py index cf72c6a49..2af3999d0 100644 --- a/src/server/endpoints/registration.py +++ b/src/server/endpoints/registration.py @@ -6,6 +6,7 @@ from .._common import send_email from .._db import WriteSession from ..admin.models import RegistrationResponse, User +from .._config import RECAPTCHA_SITE_KEY # first argument is the endpoint name bp = Blueprint("registration_form", __name__, template_folder="/app/delphi/epidata/server/templates", static_url_path="") @@ -42,4 +43,4 @@ def handle(): flags["banner"] = "Successfully sent" else: flags["error"] = "User with such email already exists. Please try another email address or contact us." - return render_template("registration.html", flags=flags) + return render_template("registration.html", flags=flags, recaptcha_key=RECAPTCHA_SITE_KEY) diff --git a/src/server/main.py b/src/server/main.py index 4b9bee2f0..e178e7997 100644 --- a/src/server/main.py +++ b/src/server/main.py @@ -8,7 +8,7 @@ from ._config import URL_PREFIX, VERSION from ._common import app, set_compatibility_mode, log_info_with_request -from .endpoints.admin import require_admin +from .endpoints.admin import _require_admin from ._exceptions import MissingOrWrongSourceException from .endpoints import endpoints from .endpoints.admin import * @@ -67,7 +67,7 @@ def diags(): # allows us to get useful diagnostic information written into server logs, # such as a full current "X-Forwarded-For" path as inserted into headers by intermediate proxies... # (but only when initiated purposefully by us to keep junk out of the logs) - require_admin() + _require_admin() log_info_with_request("diagnostics", headers=request.headers) response_text = f"request path: {request.headers.get('X-Forwarded-For', 'idk')}" return make_response(response_text, 200, {"content-type": "text/plain"}) diff --git a/src/server/templates/login.html b/src/server/templates/login.html new file mode 100644 index 000000000..3996ae0aa --- /dev/null +++ b/src/server/templates/login.html @@ -0,0 +1,36 @@ + + + + + + + API Keys registration page + + + + + + + +
+
+
+
+
+
+ + +
+
+
+ +
+
+
+
+
+ + + \ No newline at end of file diff --git a/src/server/templates/registration.html b/src/server/templates/registration.html index 826099040..cd7c36405 100644 --- a/src/server/templates/registration.html +++ b/src/server/templates/registration.html @@ -7,6 +7,7 @@ API Keys registration page + @@ -41,7 +42,7 @@

Delphi: Register API Key

detail to help us better understand who and how others are using our API.

-
+
Delphi: Register API Key value="{{ request.form['purpose'] }}">
+
+
- +
From d9c8ea823fc79a3722678059c1f663c438b087c1 Mon Sep 17 00:00:00 2001 From: Dmytro Trotsko Date: Thu, 7 Sep 2023 23:55:24 +0300 Subject: [PATCH 3/3] Added Slack notifications. Added table to show API Key registration request on the admin main page. Fixed reCaptcha (added backend verification) --- dev/local/Makefile | 3 + src/server/_common.py | 26 +++++- src/server/_config.py | 3 + src/server/endpoints/admin.py | 18 +++- .../endpoints/api_key_removal_request.py | 40 ++++++++- src/server/endpoints/registration.py | 90 +++++++++++++++---- src/server/templates/admin/index.html | 31 +++++++ 7 files changed, 186 insertions(+), 25 deletions(-) create mode 100644 src/server/templates/admin/index.html diff --git a/dev/local/Makefile b/dev/local/Makefile index e7e896aa6..ecfa93f37 100644 --- a/dev/local/Makefile +++ b/dev/local/Makefile @@ -95,6 +95,7 @@ web: @# Run the web server @# MODULE_NAME specifies the location of the `app` variable, the actual WSGI application object to run. @# see https://github.com/tiangolo/meinheld-gunicorn-docker#module_name + @# To receive Slack notification, please add SLACK_WEBHOOK_URL with proper webhook link to the list of env variables. @docker run --rm -p 127.0.0.1:10080:80 \ $(M1) \ --env "MODULE_NAME=delphi.epidata.server.main" \ @@ -104,6 +105,8 @@ web: --env "REDIS_PASSWORD=1234" \ --env "API_KEY_ADMIN_PASSWORD=test_admin_password" \ --env "API_KEY_REGISTER_WEBHOOK_TOKEN=abc" \ + --env "RECAPTCHA_SITE_KEY=6Le3wcMnAAAAAL0phFNTaWIHWDDb78I8aSzA34f2" \ + --env "RECAPTCHA_SECRET_KEY=6Le3wcMnAAAAANlmnp_oApGewLwOmiNFjxlTe2Zo" \ --network delphi-net --name delphi_web_epidata \ delphi_web_epidata >$(LOG_WEB) 2>&1 & diff --git a/src/server/_common.py b/src/server/_common.py index fb42e413e..df3b33f52 100644 --- a/src/server/_common.py +++ b/src/server/_common.py @@ -1,5 +1,8 @@ from typing import cast import time +from datetime import datetime +import requests +import json from flask import Flask, g, request from sqlalchemy import event @@ -9,13 +12,13 @@ from smtplib import SMTP from delphi.epidata.common.logger import get_structured_logger -from ._config import SECRET, REVERSE_PROXY_DEPTH, SMTP_HOST, SMTP_PORT, EMAIL_FROM +from ._config import SECRET, REVERSE_PROXY_DEPTH, SMTP_HOST, SMTP_PORT, EMAIL_FROM, SLACK_WEBHOOK_URL, RECAPTCHA_SECRET_KEY from ._db import engine from ._exceptions import DatabaseErrorException, EpiDataException from ._security import current_user, _is_public_route, resolve_auth_token, update_key_last_time_used, ERROR_MSG_INVALID_KEY -app = Flask("EpiData", static_url_path="") +app = Flask("EpiData", static_url_path="", template_folder="/app/delphi/epidata/server/templates") app.config["SECRET"] = SECRET @@ -223,4 +226,21 @@ def send_email(to_addr: str, subject: str, message: str): smtp_server = SMTP(host=SMTP_HOST, port=SMTP_PORT) smtp_server.starttls() body = "\r\n".join((f"FROM: {EMAIL_FROM}", f"TO: {to_addr}", f"Subject: {subject}", "", message)) - smtp_server.sendmail(EMAIL_FROM, to_addr, body) \ No newline at end of file + smtp_server.sendmail(EMAIL_FROM, to_addr, body) + + +def send_slack_notification(payload: dict): + headers = {"Accept": "application/json", "Content-Type": "application/json"} + response = requests.post( + SLACK_WEBHOOK_URL, + headers=headers, + data=json.dumps(payload) + ) + return response + + +def verify_recaptcha_response(response_token: str): + verification_url = "https://www.google.com/recaptcha/api/siteverify" + payload = {"secret": RECAPTCHA_SECRET_KEY, "response": response_token} + response = requests.post(verification_url, data=payload) + return response.json()["success"] diff --git a/src/server/_config.py b/src/server/_config.py index 6ea9a6723..7d8ebc41b 100644 --- a/src/server/_config.py +++ b/src/server/_config.py @@ -118,3 +118,6 @@ EMAIL_FROM = "noreply@andrew.cmu.edu" RECAPTCHA_SITE_KEY = os.environ.get("RECAPTCHA_SITE_KEY") +RECAPTCHA_SECRET_KEY = os.environ.get("RECAPTCHA_SECRET_KEY") + +SLACK_WEBHOOK_URL = os.environ.get("SLACK_WEBHOOK_URL") diff --git a/src/server/endpoints/admin.py b/src/server/endpoints/admin.py index 75775459c..7f42abed6 100644 --- a/src/server/endpoints/admin.py +++ b/src/server/endpoints/admin.py @@ -6,7 +6,7 @@ from flask_admin.contrib.sqla import ModelView from werkzeug.exceptions import Unauthorized -from .._common import app +from .._common import app, db from .._config import ADMIN_PASSWORD from .._db import WriteSession from .._security import resolve_auth_token @@ -80,7 +80,21 @@ class AuthAdminIndexView(AdminIndexView): @expose("/") @flask_login.login_required def index(self): - return super().index() + users_info_query = """ + SELECT + au.email, + au.api_key, + rr.organization, + rr.purpose, + au.created, + au.last_time_used + FROM api_user au + JOIN registration_responses rr + ON au.email = rr.email + """ + users_info = db.execute(users_info_query) + user_data = users_info.fetchall() + return self.render("admin/index.html", user_data=user_data) class UserView(AuthModelView): diff --git a/src/server/endpoints/api_key_removal_request.py b/src/server/endpoints/api_key_removal_request.py index 4c613b5ec..2d991b33b 100644 --- a/src/server/endpoints/api_key_removal_request.py +++ b/src/server/endpoints/api_key_removal_request.py @@ -1,10 +1,12 @@ +from datetime import datetime from flask import Blueprint, render_template, request -from .._common import send_email +from .._common import send_email, send_slack_notification +from .._config import SLACK_WEBHOOK_URL from .._db import WriteSession from ..admin.models import RemovalRequest, User -bp = Blueprint("removal_request", __name__, template_folder="/app/delphi/epidata/server/templates", static_url_path="") +bp = Blueprint("removal_request", __name__) alias = None @@ -16,6 +18,37 @@ """ +def generate_api_key_removal_notification(email: str, comment: str): + payload = { + "blocks": [ + { + "type": "header", + "text": {"type": "plain_text", "text": "New request", "emoji": True}, + }, + { + "type": "section", + "fields": [ + {"type": "mrkdwn", "text": "*Type:*\nAPI Key deletion."}, + {"type": "mrkdwn", "text": "*Created by:*\n{}".format(email)}, + ], + }, + { + "type": "section", + "fields": [ + {"type": "mrkdwn", "text": "*Comment:*\n{}".format(comment)}, + ], + }, + { + "type": "section", + "fields": [ + {"type": "mrkdwn", "text": "*Timestamp:*\n{}".format(datetime.now())}, + ], + }, + ] + } + return payload + + @bp.route("/", methods=["GET", "POST"]) def handle(): flags = dict() @@ -26,6 +59,9 @@ def handle(): user = User.find_user(api_key=api_key) # User.delete_user(user.id, session) RemovalRequest.add_request(api_key, comment, session) + if SLACK_WEBHOOK_URL: + slack_message = generate_api_key_removal_notification(user.email, comment) + send_slack_notification(slack_message) flags["banner"] = "Your request has been successfully recorded." send_email(user.email, "API Key removal request", REMOVAL_REQUEST_MESSAGE.format(api_key)) return render_template("removal_request.html", flags=flags) diff --git a/src/server/endpoints/registration.py b/src/server/endpoints/registration.py index 2af3999d0..55dff7434 100644 --- a/src/server/endpoints/registration.py +++ b/src/server/endpoints/registration.py @@ -1,15 +1,17 @@ import random import string +from datetime import datetime from flask import Blueprint, render_template, request -from .._common import send_email +from .._common import send_email, send_slack_notification, verify_recaptcha_response +from .._config import SLACK_WEBHOOK_URL from .._db import WriteSession from ..admin.models import RegistrationResponse, User from .._config import RECAPTCHA_SITE_KEY # first argument is the endpoint name -bp = Blueprint("registration_form", __name__, template_folder="/app/delphi/epidata/server/templates", static_url_path="") +bp = Blueprint("registration_form", __name__) alias = None NEW_KEY_MESSAGE = """ @@ -21,26 +23,78 @@ """ +def generate_registration_notification(email: str, organization: str, purpose: str): + payload = { + "blocks": [ + { + "type": "header", + "text": {"type": "plain_text", "text": "New request", "emoji": True}, + }, + { + "type": "section", + "fields": [ + {"type": "mrkdwn", "text": "*Type:*\nNew registration."}, + {"type": "mrkdwn", "text": "*Created by:*\n{}".format(email)}, + ], + }, + { + "type": "section", + "fields": [ + {"type": "mrkdwn", "text": "*Organization:*\n{}".format(organization)}, + {"type": "mrkdwn", "text": "*Purpose:*\n{}".format(purpose)}, + ], + }, + { + "type": "section", + "fields": [ + {"type": "mrkdwn", "text": "*Timestamp:*\n{}".format(datetime.now())}, + ], + }, + ] + } + return payload + + @bp.route("/", methods=["GET", "POST"]) def handle(): flags = dict() with WriteSession() as session: if request.method == "POST": email = request.values["email"] - if not User.find_user(user_email=email): - # Use a separate table for email, purpose, organization - api_key = "".join(random.choices(string.ascii_letters + string.digits, k=13)) - user = User.create_user(api_key=api_key, email=email, session=session) - RegistrationResponse.add_response( - email=email, - organization=request.values["organization"], - purpose=request.values["purpose"], - session=session, - ) - send_email( - to_addr=email, subject="Delphi API Key Registration", message=NEW_KEY_MESSAGE.format(api_key) - ) - flags["banner"] = "Successfully sent" + organization = request.values["organization"] + purpose = request.values["purpose"] + verified = verify_recaptcha_response(request.values["g-recaptcha-response"]) + if verified: + if not User.find_user(user_email=email): + # Use a separate table for email, purpose, organization + api_key = "".join( + random.choices(string.ascii_letters + string.digits, k=13) + ) + user = User.create_user(api_key=api_key, email=email, session=session) + RegistrationResponse.add_response( + email=email, + organization=request.values["organization"], + purpose=request.values["purpose"], + session=session, + ) + send_email( + to_addr=email, + subject="Delphi API Key Registration", + message=NEW_KEY_MESSAGE.format(api_key), + ) + if SLACK_WEBHOOK_URL: + slack_message = generate_registration_notification(email, organization, purpose) + send_slack_notification(slack_message) + flags["banner"] = "Successfully sent" + + else: + flags[ + "error" + ] = "User with such email already exists. Please try another email address or contact us." else: - flags["error"] = "User with such email already exists. Please try another email address or contact us." - return render_template("registration.html", flags=flags, recaptcha_key=RECAPTCHA_SITE_KEY) + flags[ + "error" + ] = "Captcha was not submitted. Please try again." + return render_template( + "registration.html", flags=flags, recaptcha_key=RECAPTCHA_SITE_KEY + ) diff --git a/src/server/templates/admin/index.html b/src/server/templates/admin/index.html new file mode 100644 index 000000000..45e1f0edf --- /dev/null +++ b/src/server/templates/admin/index.html @@ -0,0 +1,31 @@ +{% extends 'admin/master.html' %} + +{% block body %} + + + + + + + + + + + + + + + {% for i in range(user_data | length ) %} + + + + + + + + + + {% endfor %} + +
List of users
#EmailAPI KeyOrganizationPurposeCreatedLast time used
{{ i }}{{ user_data[i].email }}{{ user_data[i].api_key }}{{ user_data[i].organization }}{{ user_data[i].purpose }}{{ user_data[i].created }}{{ user_data[i].last_time_used }}
+{% endblock %} \ No newline at end of file