From 350b86579d8bfe75874210daa59015361c9deb65 Mon Sep 17 00:00:00 2001 From: Pavel Dedik Date: Fri, 7 Jun 2019 14:02:16 +0200 Subject: [PATCH] wsgi: Add middleware for delaying and rejecting requests - Fix https://github.com/kiwicom/kiwi-platform-py/issues/7 Co-Authored-By: Bence Nagy --- .bandit | 2 ++ .pre-commit-config.yaml | 9 +++-- kw/platform/settings.py | 24 +++++++++++++ kw/platform/utils.py | 37 +++++++++++++++++++ kw/platform/wsgi.py | 79 +++++++++++++++++++++++++++++++++++++++++ poetry.lock | 55 ++++++++++++++++++++++++++-- pyproject.toml | 3 ++ test/test_nothing.py | 7 ---- test/test_utils.py | 18 ++++++++++ test/test_wsgi.py | 65 +++++++++++++++++++++++++++++++++ tox.ini | 3 +- 11 files changed, 289 insertions(+), 13 deletions(-) create mode 100644 .bandit create mode 100644 kw/platform/settings.py create mode 100644 kw/platform/utils.py create mode 100644 kw/platform/wsgi.py delete mode 100644 test/test_nothing.py create mode 100644 test/test_utils.py create mode 100644 test/test_wsgi.py diff --git a/.bandit b/.bandit new file mode 100644 index 0000000..ddc4ceb --- /dev/null +++ b/.bandit @@ -0,0 +1,2 @@ +[bandit] +exclude: /test diff --git a/.pre-commit-config.yaml b/.pre-commit-config.yaml index 4eb86d8..a830005 100644 --- a/.pre-commit-config.yaml +++ b/.pre-commit-config.yaml @@ -11,14 +11,19 @@ repos: - id: flake8 language_version: python3.7 + - repo: https://github.com/asottile/seed-isort-config + rev: v1.9.1 + hooks: + - id: seed-isort-config + - repo: https://github.com/pre-commit/mirrors-isort - rev: v4.3.16 + rev: v4.3.20 hooks: - id: isort language_version: python3.7 - repo: https://github.com/pre-commit/pre-commit-hooks - rev: v2.1.0 + rev: v2.2.3 hooks: - id: trailing-whitespace - id: end-of-file-fixer diff --git a/kw/platform/settings.py b/kw/platform/settings.py new file mode 100644 index 0000000..bb599ff --- /dev/null +++ b/kw/platform/settings.py @@ -0,0 +1,24 @@ +""" +Settings +======== + +Configuration of this library. +""" + +import os + + +#: Datetime when to start slowing down requests from services which do not comply with +#: the ``KW-RFC-22`` standard. See :obj:`kw.platform.wsgi.user_agent_middleware`. +KIWI_REQUESTS_SLOWDOWN_DATETIME = os.getenv( + "KIWI_REQUESTS_SLOWDOWN_DATETIME", "2019-07-24T13:00:00" +) + +#: Datetime when to start refusing requests from services which do not comply with +#: the ``KW-RFC-22`` standard. See :obj:`kw.platform.wsgi.user_agent_middleware`. +KIWI_REQUESTS_RESTRICT_DATETIME = os.getenv( + "KIWI_REQUESTS_RESTRICT_DATETIME", "2019-08-01T13:00:00" +) + +#: Status message sent in response to requests with invalid `User-Agent`. +KIWI_RESTRICT_USER_AGENT_MESSAGE = "Invalid User-Agent: does not comply with KW-RFC-22" diff --git a/kw/platform/utils.py b/kw/platform/utils.py new file mode 100644 index 0000000..e42ae3a --- /dev/null +++ b/kw/platform/utils.py @@ -0,0 +1,37 @@ +""" +Utils +===== +""" + +import re +from datetime import datetime + +from dateutil.parser import parse + +from . import settings + + +USER_AGENT_RE = re.compile( + r"^(?P\S.+?)\/(?P\S.+?) " + r"\([Kk]iwi\.com (?P\S.+?)\)(?: ?(?P.*))$" +) +REQ_SLOWDOWN_DATETIME = parse(settings.KIWI_REQUESTS_SLOWDOWN_DATETIME) +REQ_RESTRICT_DATETIME = parse(settings.KIWI_REQUESTS_RESTRICT_DATETIME) + + +class UserAgentValidator: + def __init__(self, value): + self.value = value + self.is_valid = bool(USER_AGENT_RE.match(self.value)) + + @property + def ok(self): + return datetime.utcnow() < REQ_SLOWDOWN_DATETIME or self.is_valid + + @property + def slowdown(self): + return not self.ok and datetime.utcnow() < REQ_RESTRICT_DATETIME + + @property + def restrict(self): + return not self.ok and not self.slowdown diff --git a/kw/platform/wsgi.py b/kw/platform/wsgi.py new file mode 100644 index 0000000..f81848e --- /dev/null +++ b/kw/platform/wsgi.py @@ -0,0 +1,79 @@ +""" +WSGI helpers +============ + +Various helpers for WSGI applications. +""" + +import time + +import webob +from webob.dec import wsgify + +from . import settings, utils + + +def _refuse_request(req, app): + return webob.exc.HTTPBadRequest(settings.KIWI_RESTRICT_USER_AGENT_MESSAGE) + + +def _slowdown_request(req, app): + before_time = time.time() + resp = req.get_response(app) + seconds = time.time() - before_time + time.sleep(seconds) + return resp + + +def _validate_user_agent(req, app): + user_agent = utils.UserAgentValidator(req.user_agent) + + if user_agent.slowdown: + return _slowdown_request(req, app) + elif user_agent.restrict: + return _refuse_request(req, app) + + return app + + +#: Validate client's User-Agent header and modify response based on that. +#: +#: If the User-Agent header is invalid, there are three possible outcomes: +#: +#: 1. The current time is less then :obj:`settings.KIWI_REQUESTS_SLOWDOWN_DATETIME`, +#: do nothing in this case. +#: 2. The current time is less then :obj:`settings.KIWI_REQUESTS_RESTRICT_DATETIME`, +#: slow down the response twice the normal responce time. +#: 3. The current time is more then :obj:`settings.KIWI_REQUESTS_RESTRICT_DATETIME`, +#: refuse the request, return ``HTTP 400`` to the client. +#: +#: Usage:: +#: +#: from your_app import wsgi_app +#: +#: wsgi_app = user_agent_middleware(wsgi_app) +#: +#: For example, in Flask, the middleware can be applied like this:: +#: +#: from flask import Flask +#: +#: app = Flask(__name__) +#: app.wsgi_app = user_agent_middleware(app) +#: +#: app.run() +#: +#: In Django, you can apply the middleware like this:: +#: +#: from django.core.wsgi import get_wsgi_application +#: +#: application = user_agent_middleware(get_wsgi_application()) +#: +#: For more information see +#: `Django docs `_. +#: +#: .. warning:: +#: +#: The middleware slows down requests by calling :meth:`time.sleep()` +#: (in the time frame when requests with invalid user-agent are being delayed). +#: This can increase worker busyness which can overload a service. +user_agent_middleware = wsgify.middleware(_validate_user_agent) diff --git a/poetry.lock b/poetry.lock index c372495..4912d92 100644 --- a/poetry.lock +++ b/poetry.lock @@ -58,6 +58,18 @@ optional = false python-versions = ">=2.6, !=3.0.*, !=3.1.*, !=3.2.*, <4" version = "4.5.3" +[[package]] +category = "dev" +description = "Let your Python tests travel through time" +name = "freezegun" +optional = false +python-versions = ">=2.7, !=3.0.*, !=3.1.*, !=3.2.*, !=3.3.*" +version = "0.3.12" + +[package.dependencies] +python-dateutil = ">=1.0,<2.0 || >2.0" +six = "*" + [[package]] category = "dev" description = "Python function signatures from PEP362 for Python 2.6, 2.7 and 3.2+" @@ -206,6 +218,29 @@ version = ">=2.2.0" [package.extras] testing = ["argcomplete", "hypothesis (>=3.56)", "nose", "requests", "mock"] +[[package]] +category = "dev" +description = "Wrap tests with fixtures in freeze_time" +name = "pytest-freezegun" +optional = false +python-versions = "*" +version = "0.3.0.post1" + +[package.dependencies] +freezegun = ">0.3" +pytest = ">=3.0.0" + +[[package]] +category = "main" +description = "Extensions to the standard Python datetime module" +name = "python-dateutil" +optional = false +python-versions = ">=2.7, !=3.0.*, !=3.1.*, !=3.2.*" +version = "2.8.0" + +[package.dependencies] +six = ">=1.5" + [[package]] category = "dev" description = "scandir, a better directory iterator and faster os.walk()" @@ -216,7 +251,7 @@ python-versions = "*" version = "1.10.0" [[package]] -category = "dev" +category = "main" description = "Python 2 and 3 compatibility utilities" name = "six" optional = false @@ -231,6 +266,18 @@ optional = false python-versions = "*" version = "0.1.7" +[[package]] +category = "main" +description = "WSGI request and response object" +name = "webob" +optional = false +python-versions = ">=2.7,!=3.0.*,!=3.1.*,!=3.2.*" +version = "1.8.5" + +[package.extras] +docs = ["Sphinx (>=1.7.5)", "pylons-sphinx-themes"] +testing = ["pytest (>=3.1.0)", "coverage", "pytest-cov", "pytest-xdist"] + [[package]] category = "dev" description = "Backport of pathlib-compatible object wrapper for zip files" @@ -244,7 +291,7 @@ docs = ["sphinx", "jaraco.packaging (>=3.2)", "rst.linker (>=1.9)"] testing = ["pathlib2", "contextlib2", "unittest2"] [metadata] -content-hash = "77a35ae3b956aa7a4f7a6d15daaa4e4ede35f11487e3d31ce8e49e2244ffe1fe" +content-hash = "cabebb12adc5aa7de3dea5897391fc740c547cd371723e21063bae88a214fd1a" python-versions = "~2.7 || ^3.5" [metadata.hashes] @@ -254,6 +301,7 @@ colorama = ["05eed71e2e327246ad6b38c540c4a3117230b19679b875190486ddd2d721422d", configparser = ["8be81d89d6e7b4c0d4e44bcc525845f6da25821de80cb5e06e7e0238a2899e32", "da60d0014fd8c55eb48c1c5354352e363e2d30bbf7057e5e171a468390184c75"] contextlib2 = ["509f9419ee91cdd00ba34443217d5ca51f5a364a404e1dce9e8979cea969ca48", "f5260a6e679d2ff42ec91ec5252f4eeffdcf21053db9113bd0a8e4d953769c00"] coverage = ["0c5fe441b9cfdab64719f24e9684502a59432df7570521563d7b1aff27ac755f", "2b412abc4c7d6e019ce7c27cbc229783035eef6d5401695dccba80f481be4eb3", "3684fabf6b87a369017756b551cef29e505cb155ddb892a7a29277b978da88b9", "39e088da9b284f1bd17c750ac672103779f7954ce6125fd4382134ac8d152d74", "3c205bc11cc4fcc57b761c2da73b9b72a59f8d5ca89979afb0c1c6f9e53c7390", "42692db854d13c6c5e9541b6ffe0fe921fe16c9c446358d642ccae1462582d3b", "465ce53a8c0f3a7950dfb836438442f833cf6663d407f37d8c52fe7b6e56d7e8", "48020e343fc40f72a442c8a1334284620f81295256a6b6ca6d8aa1350c763bbe", "4ec30ade438d1711562f3786bea33a9da6107414aed60a5daa974d50a8c2c351", "5296fc86ab612ec12394565c500b412a43b328b3907c0d14358950d06fd83baf", "5f61bed2f7d9b6a9ab935150a6b23d7f84b8055524e7be7715b6513f3328138e", "6899797ac384b239ce1926f3cb86ffc19996f6fa3a1efbb23cb49e0c12d8c18c", "68a43a9f9f83693ce0414d17e019daee7ab3f7113a70c79a3dd4c2f704e4d741", "6b8033d47fe22506856fe450470ccb1d8ba1ffb8463494a15cfc96392a288c09", "7ad7536066b28863e5835e8cfeaa794b7fe352d99a8cded9f43d1161be8e9fbd", "7bacb89ccf4bedb30b277e96e4cc68cd1369ca6841bde7b005191b54d3dd1034", "839dc7c36501254e14331bcb98b27002aa415e4af7ea039d9009409b9d2d5420", "8e679d1bde5e2de4a909efb071f14b472a678b788904440779d2c449c0355b27", "8f9a95b66969cdea53ec992ecea5406c5bd99c9221f539bca1e8406b200ae98c", "932c03d2d565f75961ba1d3cec41ddde00e162c5b46d03f7423edcb807734eab", "93f965415cc51604f571e491f280cff0f5be35895b4eb5e55b47ae90c02a497b", "988529edadc49039d205e0aa6ce049c5ccda4acb2d6c3c5c550c17e8c02c05ba", "998d7e73548fe395eeb294495a04d38942edb66d1fa61eb70418871bc621227e", "9de60893fb447d1e797f6bf08fdf0dbcda0c1e34c1b06c92bd3a363c0ea8c609", "9e80d45d0c7fcee54e22771db7f1b0b126fb4a6c0a2e5afa72f66827207ff2f2", "a545a3dfe5082dc8e8c3eb7f8a2cf4f2870902ff1860bd99b6198cfd1f9d1f49", "a5d8f29e5ec661143621a8f4de51adfb300d7a476224156a39a392254f70687b", "a9abc8c480e103dc05d9b332c6cc9fb1586330356fc14f1aa9c0ca5745097d19", "aca06bfba4759bbdb09bf52ebb15ae20268ee1f6747417837926fae990ebc41d", "bb23b7a6fd666e551a3094ab896a57809e010059540ad20acbeec03a154224ce", "bfd1d0ae7e292105f29d7deaa9d8f2916ed8553ab9d5f39ec65bcf5deadff3f9", "c22ab9f96cbaff05c6a84e20ec856383d27eae09e511d3e6ac4479489195861d", "c62ca0a38958f541a73cf86acdab020c2091631c137bd359c4f5bddde7b75fd4", "c709d8bda72cf4cd348ccec2a4881f2c5848fd72903c185f363d361b2737f773", "c968a6aa7e0b56ecbd28531ddf439c2ec103610d3e2bf3b75b813304f8cb7723", "ca58eba39c68010d7e87a823f22a081b5290e3e3c64714aac3c91481d8b34d22", "df785d8cb80539d0b55fd47183264b7002077859028dfe3070cf6359bf8b2d9c", "f406628ca51e0ae90ae76ea8398677a921b36f0bd71aab2099dfed08abd0322f", "f46087bbd95ebae244a0eda01a618aff11ec7a069b15a3ef8f6b520db523dcf1", "f8019c5279eb32360ca03e9fac40a12667715546eed5c5eb59eb381f2f501260", "fc5f4d209733750afd2714e9109816a29500718b32dd9a5db01c0cb3a019b96a"] +freezegun = ["2a4d9c8cd3c04a201e20c313caf8b6338f1cfa4cda43f46a94cc4a9fd13ea5e7", "edfdf5bc6040969e6ed2e36eafe277963bdc8b7c01daeda96c5c8594576c9390"] funcsigs = ["330cc27ccbf7f1e992e69fef78261dc7c6569012cf397db8d3de0234e6c937ca", "a7bb0f2cf3a3fd1ab2732cb49eba4252c2af4240442415b4abce3b87022a8f50"] importlib-metadata = ["a9f185022cfa69e9ca5f7eabfd5a58b689894cb78a11e3c8c89398a8ccbb8e7f", "df1403cd3aebeb2b1dcd3515ca062eecb5bd3ea7611f18cba81130c68707e879"] more-itertools = ["38a936c0a6d98a38bcc2d03fdaaedaba9f412879461dd2ceff8d37564d6522e4", "c0a5785b1109a6bd7fac76d6837fd1feca158e54e521ccd2ae8bfe393cc9d4fc", "fe7a7cae1ccb57d33952113ff4fa1bc5f879963600ed74918f1236e212ee50b9", "2112d2ca570bb7c3e53ea1a35cd5df42bb0fd10c45f0fb97178679c3c03d64c7", "c3e4748ba1aad8dba30a4886b0b1a2004f9a863837b8654e7059eebf727afa5a"] @@ -263,7 +311,10 @@ pluggy = ["0825a152ac059776623854c1543d65a4ad408eb3d33ee114dff91e57ec6ae6fc", "b py = ["64f65755aee5b381cea27766a3a147c3f15b9b6b9ac88676de66ba2ae36793fa", "dc639b046a6e2cff5bbe40194ad65936d6ba360b52b3c3fe1d08a82dd50b5e53"] pyparsing = ["1873c03321fc118f4e9746baf201ff990ceb915f433f23b395f5580d1840cb2a", "9b6323ef4ab914af344ba97510e966d64ba91055d6b9afa6b30799340e89cc03"] pytest = ["6032845e68a17a96e8da3088037f899b56357769a724122056265ca2ea1890ee", "bea27a646a3d74cbbcf8d3d4a06b2dfc336baf3dc2cc85cf70ad0157e73e8322"] +pytest-freezegun = ["94c370a2cd3db9692962522cb74525d908e669df7cb53a448e01bb47c21a8173", "b86b13ef75959bedf4c32f1fd81fec66fa4502d9892e0ef6ad1717a34fe1560e"] +python-dateutil = ["7e6584c74aeed623791615e26efd690f29817a27c73085b78e4bad02493df2fb", "c89805f6f4d64db21ed966fda138f8a5ed7a4fdbc1a8ee329ce1b74e3c74da9e"] scandir = ["2586c94e907d99617887daed6c1d102b5ca28f1085f90446554abf1faf73123e", "2ae41f43797ca0c11591c0c35f2f5875fa99f8797cb1a1fd440497ec0ae4b022", "2b8e3888b11abb2217a32af0766bc06b65cc4a928d8727828ee68af5a967fa6f", "2c712840c2e2ee8dfaf36034080108d30060d759c7b73a01a52251cc8989f11f", "4d4631f6062e658e9007ab3149a9b914f3548cb38bfb021c64f39a025ce578ae", "67f15b6f83e6507fdc6fca22fedf6ef8b334b399ca27c6b568cbfaa82a364173", "7d2d7a06a252764061a020407b997dd036f7bd6a175a5ba2b345f0a357f0b3f4", "8c5922863e44ffc00c5c693190648daa6d15e7c1207ed02d6f46a8dcc2869d32", "92c85ac42f41ffdc35b6da57ed991575bdbe69db895507af88b9f499b701c188", "b24086f2375c4a094a6b51e78b4cf7ca16c721dcee2eddd7aa6494b42d6d519d", "cb925555f43060a1745d0a321cca94bcea927c50114b623d73179189a4e100ac"] six = ["3350809f0555b11f552448330d0b52d5f24c91a322ea4a15ef22629740f3761c", "d16a0141ec1a18405cd4ce8b4613101da75da0e9a7aec5bdd4fa804d0e0eba73"] wcwidth = ["3df37372226d6e63e1b1e1eda15c594bca98a22d33a23832a90998faa96bc65e", "f4ebe71925af7b40a864553f761ed559b43544f8f71746c2d756c7fe788ade7c"] +webob = ["05aaab7975e0ee8af2026325d656e5ce14a71f1883c52276181821d6d5bf7086", "36db8203c67023d68c1b00208a7bf55e3b10de2aa317555740add29c619de12b"] zipp = ["8c1019c6aad13642199fbe458275ad6a84907634cc9f0989877ccc4a2840139d", "ca943a7e809cc12257001ccfb99e3563da9af99d52f261725e96dfe0f9275bc3"] diff --git a/pyproject.toml b/pyproject.toml index 0b21522..6d10d84 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -42,7 +42,10 @@ include = ["*.md", "*.toml", "*.txt", "*.yaml", ".coveragerc", "tox.ini"] [tool.poetry.dependencies] python = "~2.7 || ^3.5" +webob = "^1.8" +python-dateutil = "^2.8" [tool.poetry.dev-dependencies] pytest = "^4.6" +pytest-freezegun = "^0.3.0" coverage = "^4.5" diff --git a/test/test_nothing.py b/test/test_nothing.py deleted file mode 100644 index c0b124d..0000000 --- a/test/test_nothing.py +++ /dev/null @@ -1,7 +0,0 @@ -"""Placeholder test file.""" - -from kw import platform as uut - - -def test_import_succeeded(): - assert uut diff --git a/test/test_utils.py b/test/test_utils.py new file mode 100644 index 0000000..c631504 --- /dev/null +++ b/test/test_utils.py @@ -0,0 +1,18 @@ +import pytest + +from kw.platform import utils as uut + + +@pytest.mark.parametrize( + "user_agent,should_pass", + [ + ("invalid", False), + ("mambo/1a (Kiwi.com dev)", True), + ("mambo/1a (kiwi.com dev)", True), + ("mambo/1a (kiwicom dev)", False), + ("zoo/1.0 (Kiwi.com production)", True), + ("zoo/git-123ad4 (Kiwi.com production) thief requests/2.22", True), + ], +) +def test_user_agent_re(user_agent, should_pass): + assert bool(uut.USER_AGENT_RE.match(user_agent)) is should_pass diff --git a/test/test_wsgi.py b/test/test_wsgi.py new file mode 100644 index 0000000..6b9f4d8 --- /dev/null +++ b/test/test_wsgi.py @@ -0,0 +1,65 @@ +import time + +import pytest +from freezegun import freeze_time +from webob.request import BaseRequest + +from kw.platform import wsgi as uut + + +def create_app(sleep_seconds=0): + def simple_app(environ, start_response): + if sleep_seconds: + time.sleep(sleep_seconds) + start_response("200 OK", [("Content-Type", "text/html; charset=UTF-8")]) + return ["OK"] + + return simple_app + + +@pytest.mark.parametrize( + "user_agent,expected_status,current_time", + [ + ("invalid", 200, "2019-05-21"), + ("invalid", 400, "2020-01-01"), + ("mambo/1a (Kiwi.com dev)", 200, "2020-01-01"), + ], +) +def test_user_agent_middleware__restrict(user_agent, expected_status, current_time): + app = create_app() + + req = BaseRequest.blank("/") + req.user_agent = user_agent + + with freeze_time(current_time, tick=True): + app = uut.user_agent_middleware(app) + res = req.get_response(app) + + assert res.status_code == expected_status + + +@pytest.mark.parametrize( + "user_agent,sleep_seconds,expected_time,current_time", + [ + ("invalid", 0.1, 0.2, "2019-07-26"), + ("mambo/1a (Kiwi.com dev)", 0.1, 0.1, "2019-07-26"), + ("invalid", 0.1, 0.1, "2019-05-07"), + ], +) +def test_user_agent_middleware__slowdown( + user_agent, sleep_seconds, expected_time, current_time +): + app = create_app(sleep_seconds=sleep_seconds) + + req = BaseRequest.blank("/") + req.user_agent = user_agent + + with freeze_time(current_time, tick=True): + app = uut.user_agent_middleware(app) + + before_time = time.time() + res = req.get_response(app) + request_time = time.time() - before_time + + assert res.status_code == 200 + assert request_time >= expected_time diff --git a/tox.ini b/tox.ini index 9c2c369..90005ca 100644 --- a/tox.ini +++ b/tox.ini @@ -40,11 +40,10 @@ atomic = true force_grid_wrap = 0 include_trailing_comma = true lines_after_imports = 2 -lines_between_types = 1 multi_line_output = 3 not_skip = __init__.py use_parentheses = true -known_first_party = kw.platform +known_third_party = dateutil,freezegun,pytest,webob [flake8] max-line-length = 88