wsgi: Add middleware for delaying and rejecting requests

- Fix #7

Co-Authored-By: Bence Nagy <bence@underyx.me>

.bandit

@@ -0,0 +1,2 @@
+[bandit]
+exclude: /test

.pre-commit-config.yaml

@@ -11,14 +11,19 @@ repos:
       - id: flake8
         language_version: python3.7
 
+  - repo: https://github.com/asottile/seed-isort-config
+    rev: v1.9.1
+    hooks:
+      - id: seed-isort-config
+
   - repo: https://github.com/pre-commit/mirrors-isort
-    rev: v4.3.16
+    rev: v4.3.20
     hooks:
       - id: isort
         language_version: python3.7
 
   - repo: https://github.com/pre-commit/pre-commit-hooks
-    rev: v2.1.0
+    rev: v2.2.3
     hooks:
       - id: trailing-whitespace
       - id: end-of-file-fixer

kw/platform/settings.py

@@ -0,0 +1,24 @@
+"""
+Settings
+========
+
+Configuration of this library.
+"""
+
+import os
+
+
+#: Datetime when to start slowing down requests from services which do not comply with
+#: the ``KW-RFC-22`` standard. See :obj:`kw.platform.wsgi.user_agent_middleware`.
+KIWI_REQUESTS_SLOWDOWN_DATETIME = os.getenv(
+    "KIWI_REQUESTS_SLOWDOWN_DATETIME", "2019-07-24T13:00:00"
+)
+
+#: Datetime when to start refusing requests from services which do not comply with
+#: the ``KW-RFC-22`` standard. See :obj:`kw.platform.wsgi.user_agent_middleware`.
+KIWI_REQUESTS_RESTRICT_DATETIME = os.getenv(
+    "KIWI_REQUESTS_RESTRICT_DATETIME", "2019-08-01T13:00:00"
+)
+
+#: Status message sent in response to requests with invalid `User-Agent`.
+KIWI_RESTRICT_USER_AGENT_MESSAGE = "Invalid User-Agent: does not comply with KW-RFC-22"

kw/platform/utils.py

@@ -0,0 +1,37 @@
+"""
+Utils
+=====
+"""
+
+import re
+from datetime import datetime
+
+from dateutil.parser import parse
+
+from . import settings
+
+
+USER_AGENT_RE = re.compile(
+    r"^(?P<name>\S.+?)\/(?P<version>\S.+?) "
+    r"\([Kk]iwi\.com (?P<environment>\S.+?)\)(?: ?(?P<system_info>.*))$"
+)
+REQ_SLOWDOWN_DATETIME = parse(settings.KIWI_REQUESTS_SLOWDOWN_DATETIME)
+REQ_RESTRICT_DATETIME = parse(settings.KIWI_REQUESTS_RESTRICT_DATETIME)
+
+
+class UserAgentValidator:
+    def __init__(self, value):
+        self.value = value
+        self.is_valid = bool(USER_AGENT_RE.match(self.value))
+
+    @property
+    def ok(self):
+        return datetime.utcnow() < REQ_SLOWDOWN_DATETIME or self.is_valid
+
+    @property
+    def slowdown(self):
+        return not self.ok and datetime.utcnow() < REQ_RESTRICT_DATETIME
+
+    @property
+    def restrict(self):
+        return not self.ok and not self.slowdown

kw/platform/wsgi.py

@@ -0,0 +1,79 @@
+"""
+WSGI helpers
+============
+
+Various helpers for WSGI applications.
+"""
+
+import time
+
+import webob
+from webob.dec import wsgify
+
+from . import settings, utils
+
+
+def _refuse_request(req, app):
+    return webob.exc.HTTPBadRequest(settings.KIWI_RESTRICT_USER_AGENT_MESSAGE)
+
+
+def _slowdown_request(req, app):
+    before_time = time.time()
+    resp = req.get_response(app)
+    seconds = time.time() - before_time
+    time.sleep(seconds)
+    return resp
+
+
+def _validate_user_agent(req, app):
+    user_agent = utils.UserAgentValidator(req.user_agent)
+
+    if user_agent.slowdown:
+        return _slowdown_request(req, app)
+    elif user_agent.restrict:
+        return _refuse_request(req, app)
+
+    return app
+
+
+#: Validate client's User-Agent header and modify response based on that.
+#:
+#: If the User-Agent header is invalid, there are three possible outcomes:
+#:
+#: 1. The current time is less then :obj:`settings.KIWI_REQUESTS_SLOWDOWN_DATETIME`,
+#:    do nothing in this case.
+#: 2. The current time is less then :obj:`settings.KIWI_REQUESTS_RESTRICT_DATETIME`,
+#:    slow down the response twice the normal responce time.
+#: 3. The current time is more then :obj:`settings.KIWI_REQUESTS_RESTRICT_DATETIME`,
+#:    refuse the request, return ``HTTP 400`` to the client.
+#:
+#: Usage::
+#:
+#:     from your_app import wsgi_app
+#:
+#:     wsgi_app = user_agent_middleware(wsgi_app)
+#:
+#: For example, in Flask, the middleware can be applied like this::
+#:
+#:     from flask import Flask
+#:
+#:     app = Flask(__name__)
+#:     app.wsgi_app = user_agent_middleware(app)
+#:
+#:     app.run()
+#:
+#: In Django, you can apply the middleware like this::
+#:
+#:     from django.core.wsgi import get_wsgi_application
+#:
+#:     application = user_agent_middleware(get_wsgi_application())
+#:
+#: For more information see
+#: `Django docs <https://docs.djangoproject.com/en/dev/howto/deployment/wsgi/>`_.
+#:
+#: .. warning::
+#:
+#:     The middleware slows down requests by calling :meth:`time.sleep()`
+#:     (in the time frame when requests with invalid user-agent are being delayed).
+#:     This can increase worker busyness which can overload a service.
+user_agent_middleware = wsgify.middleware(_validate_user_agent)

pyproject.toml

@@ -42,7 +42,10 @@ include = ["*.md", "*.toml", "*.txt", "*.yaml", ".coveragerc", "tox.ini"]
 
 [tool.poetry.dependencies]
 python = "~2.7 || ^3.5"
+webob = "^1.8"
+python-dateutil = "^2.8"
 
 [tool.poetry.dev-dependencies]
 pytest = "^4.6"
+pytest-freezegun = "^0.3.0"
 coverage = "^4.5"

test/test_utils.py

@@ -0,0 +1,18 @@
+import pytest
+
+from kw.platform import utils as uut
+
+
+@pytest.mark.parametrize(
+    "user_agent,should_pass",
+    [
+        ("invalid", False),
+        ("mambo/1a (Kiwi.com dev)", True),
+        ("mambo/1a (kiwi.com dev)", True),
+        ("mambo/1a (kiwicom dev)", False),
+        ("zoo/1.0 (Kiwi.com production)", True),
+        ("zoo/git-123ad4 (Kiwi.com production) thief requests/2.22", True),
+    ],
+)
+def test_user_agent_re(user_agent, should_pass):
+    assert bool(uut.USER_AGENT_RE.match(user_agent)) is should_pass

test/test_wsgi.py

@@ -0,0 +1,65 @@
+import time
+
+import pytest
+from freezegun import freeze_time
+from webob.request import BaseRequest
+
+from kw.platform import wsgi as uut
+
+
+def create_app(sleep_seconds=0):
+    def simple_app(environ, start_response):
+        if sleep_seconds:
+            time.sleep(sleep_seconds)
+        start_response("200 OK", [("Content-Type", "text/html; charset=UTF-8")])
+        return ["OK"]
+
+    return simple_app
+
+
+@pytest.mark.parametrize(
+    "user_agent,expected_status,current_time",
+    [
+        ("invalid", 200, "2019-05-21"),
+        ("invalid", 400, "2020-01-01"),
+        ("mambo/1a (Kiwi.com dev)", 200, "2020-01-01"),
+    ],
+)
+def test_user_agent_middleware__restrict(user_agent, expected_status, current_time):
+    app = create_app()
+
+    req = BaseRequest.blank("/")
+    req.user_agent = user_agent
+
+    with freeze_time(current_time, tick=True):
+        app = uut.user_agent_middleware(app)
+        res = req.get_response(app)
+
+    assert res.status_code == expected_status
+
+
+@pytest.mark.parametrize(
+    "user_agent,sleep_seconds,expected_time,current_time",
+    [
+        ("invalid", 0.1, 0.2, "2019-07-26"),
+        ("mambo/1a (Kiwi.com dev)", 0.1, 0.1, "2019-07-26"),
+        ("invalid", 0.1, 0.1, "2019-05-07"),
+    ],
+)
+def test_user_agent_middleware__slowdown(
+    user_agent, sleep_seconds, expected_time, current_time
+):
+    app = create_app(sleep_seconds=sleep_seconds)
+
+    req = BaseRequest.blank("/")
+    req.user_agent = user_agent
+
+    with freeze_time(current_time, tick=True):
+        app = uut.user_agent_middleware(app)
+
+        before_time = time.time()
+        res = req.get_response(app)
+        request_time = time.time() - before_time
+
+    assert res.status_code == 200
+    assert request_time >= expected_time

tox.ini

@@ -40,11 +40,10 @@ atomic = true
 force_grid_wrap = 0
 include_trailing_comma = true
 lines_after_imports = 2
-lines_between_types = 1
 multi_line_output = 3
 not_skip = __init__.py
 use_parentheses = true
-known_first_party = kw.platform
+known_third_party = dateutil,freezegun,pytest,webob
 
 [flake8]
 max-line-length = 88