Feat: improve traffic filtering at nginx

appcontainer/Dockerfile

@@ -3,6 +3,10 @@ FROM ghcr.io/cal-itp/docker-python-web:main
 # upgrade pip
 RUN python -m pip install --upgrade pip
 
+# overwrite default nginx.conf
+COPY appcontainer/nginx.conf /etc/nginx/nginx.conf
+COPY appcontainer/proxy.conf /home/calitp/run/proxy.conf
+
 # copy source files
 COPY manage.py manage.py
 COPY bin bin

appcontainer/nginx.conf

@@ -0,0 +1,94 @@
+worker_processes auto;
+error_log stderr warn;
+pid /var/run/nginx.pid;
+
+events {
+  worker_connections 1024;
+  accept_mutex on;
+}
+
+http {
+  include mime.types;
+  default_type application/octet-stream;
+  sendfile on;
+  gzip on;
+  keepalive_timeout 5;
+
+  log_format main '[$time_local] "$request" '
+                  '$status $body_bytes_sent "$http_referer" '
+                  '"$http_user_agent" "$http_x_forwarded_for"';
+  access_log /dev/stdout main;
+
+  upstream app_server {
+    # fail_timeout=0 means we always retry an upstream even if it failed
+    # to return a good HTTP response
+    server unix:/home/calitp/run/gunicorn.sock fail_timeout=0;
+  }
+
+  # maps $binary_ip_address to $limit variable if request is of type POST
+  map $request_method $limit {
+    default         "";
+    POST            $binary_remote_addr;
+  }
+
+  # define a zone with 10mb memory, rate limit to 12 requests/min (~= 1 request/5 seconds) on applied locations
+  # $limit will eval to $binary_remote_addr for POST requests using the above map
+  # requests with an empty key value (e.g. GET) are not affected
+  # http://nginx.org/en/docs/http/ngx_http_limit_req_module.html#limit_req_zone
+  limit_req_zone $limit zone=rate_limit:10m rate=12r/m;
+
+  server {
+    listen 8000;
+
+    keepalive_timeout 65;
+
+    # 404 known scraping path targets
+    # case-insensitive regex matches the given path fragment anywhere in the request path
+    # e.g. /something/api and /login
+    location ~* /(\.?git|api|app|assets|ats|auth|bootstrap|bower|cgi|content|credentials|docker|doc|env|example|login|swagger|web) {
+        access_log off;
+        log_not_found off;
+        return 404;
+    }
+
+    # 404 known scraping file targets
+    # case-insensitive regex matches the given file extension anywhere in the request path
+    # e.g. /something/admin.php and /secrets.yaml
+    location ~* /.*\.(asp|axd|cgi|com|env|json|log|php|xml|ya?ml) {
+        access_log off;
+        log_not_found off;
+        return 404;
+    }
+
+    location /favicon.ico {
+      access_log off;
+      log_not_found off;
+      expires 1y;
+      add_header Cache-Control public;
+    }
+
+    # path for static files
+    location /static/ {
+      alias /home/calitp/app/static/;
+      expires 1y;
+      add_header Cache-Control public;
+    }
+
+    location / {
+      # checks for static file, if not found proxy to app
+      try_files $uri @proxy_to_app;
+    }
+
+    # apply rate limit to these paths
+    # case-insensitive regex matches path
+    location ~* ^/(eligibility/confirm)$ {
+        limit_req zone=rate_limit;
+        include /home/calitp/run/proxy.conf;
+    }
+
+    # app path
+    location @proxy_to_app {
+      include /home/calitp/run/proxy.conf;
+    }
+  }
+}

appcontainer/proxy.conf

@@ -0,0 +1,8 @@
+# the core app proxy directives
+proxy_set_header X-Forwarded-For $proxy_add_x_forwarded_for;
+proxy_set_header X-Forwarded-Proto $scheme;
+proxy_set_header Host $http_host;
+# we don't want nginx trying to do something clever with
+# redirects, we set the Host: header above already.
+proxy_redirect off;
+proxy_pass http://app_server;

benefits/core/middleware.py

@@ -2,12 +2,10 @@
 The core application: middleware definitions for request/response cycle.
 """
 import logging
-import time
 
 from django.conf import settings
-from django.http import HttpResponse, HttpResponseBadRequest
+from django.http import HttpResponse
 from django.shortcuts import redirect
-from django.template import loader
 from django.template.response import TemplateResponse
 from django.utils.decorators import decorator_from_middleware
 from django.utils.deprecation import MiddlewareMixin
@@ -40,43 +38,6 @@ def process_request(self, request):
             return user_error(request)
 
 
-class RateLimit(MiddlewareMixin):
-    """Middleware checks settings and session to ensure rate limit is respected."""
-
-    def process_request(self, request):
-        if not settings.RATE_LIMIT_ENABLED:
-            logger.debug("Rate Limiting is not configured")
-            return None
-
-        if request.method in settings.RATE_LIMIT_METHODS:
-            session.increment_rate_limit_counter(request)
-        else:
-            # bail early if the request method doesn't match
-            return None
-
-        counter = session.rate_limit_counter(request)
-        reset_time = session.rate_limit_time(request)
-        now = int(time.time())
-
-        if counter > settings.RATE_LIMIT:
-            if reset_time > now:
-                logger.warning("Rate limit exceeded")
-                home = viewmodels.Button.home(request)
-                page = viewmodels.ErrorPage.server_error(
-                    title="Rate limit error",
-                    headline="Rate limit error",
-                    paragraphs=["You have reached the rate limit. Please try again."],
-                    button=home,
-                )
-                t = loader.get_template("400.html")
-                return HttpResponseBadRequest(t.render(page.context_dict()))
-            else:
-                # enough time has passed, reset the rate limit
-                session.reset_rate_limit(request)
-
-        return None
-
-
 class EligibleSessionRequired(MiddlewareMixin):
     """Middleware raises an exception for sessions lacking confirmed eligibility."""
 

benefits/core/session.py

@@ -6,7 +6,6 @@
 import time
 import uuid
 
-from django.conf import settings
 from django.urls import reverse
 
 from . import models
@@ -22,8 +21,6 @@
 _ENROLLMENT_TOKEN = "enrollment_token"
 _ENROLLMENT_TOKEN_EXP = "enrollment_token_exp"
 _LANG = "lang"
-_LIMITCOUNTER = "limitcounter"
-_LIMITUNTIL = "limituntil"
 _OAUTH_CLAIM = "oauth_claim"
 _OAUTH_TOKEN = "oauth_token"
 _ORIGIN = "origin"
@@ -54,7 +51,6 @@ def context_dict(request):
     logger.debug("Get session context dict")
     return {
         _AGENCY: agency(request).slug if active_agency(request) else None,
-        _LIMITCOUNTER: rate_limit_counter(request),
         _DEBUG: debug(request),
         _DID: did(request),
         _ELIGIBILITY: eligibility(request),
@@ -64,7 +60,6 @@ def context_dict(request):
         _OAUTH_TOKEN: oauth_token(request),
         _OAUTH_CLAIM: oauth_claim(request),
         _ORIGIN: origin(request),
-        _LIMITUNTIL: rate_limit_time(request),
         _START: start(request),
         _UID: uid(request),
         _VERIFIER: verifier(request),
@@ -174,33 +169,6 @@ def origin(request):
     return request.session.get(_ORIGIN)
 
 
-def rate_limit_counter(request):
-    """Get this session's rate limit counter."""
-    logger.debug("Get rate limit counter")
-    return request.session.get(_LIMITCOUNTER)
-
-
-def increment_rate_limit_counter(request):
-    """Adds 1 to this session's rate limit counter."""
-    logger.debug("Increment rate limit counter")
-    c = rate_limit_counter(request)
-    request.session[_LIMITCOUNTER] = int(c) + 1
-
-
-def reset_rate_limit(request):
-    """Reset this session's rate limit counter and time."""
-    logger.debug("Reset rate limit")
-    request.session[_LIMITCOUNTER] = 0
-    # get the current time in Unix seconds, then add RATE_LIMIT_PERIOD seconds
-    request.session[_LIMITUNTIL] = int(time.time()) + settings.RATE_LIMIT_PERIOD
-
-
-def rate_limit_time(request):
-    """Get this session's rate limit time, a Unix timestamp after which the session's rate limt resets."""
-    logger.debug("Get rate limit time")
-    return request.session.get(_LIMITUNTIL)
-
-
 def reset(request):
     """Reset the session for the request."""
     logger.debug("Reset session")
@@ -219,7 +187,6 @@ def reset(request):
         u = str(uuid.uuid4())
         request.session[_UID] = u
         request.session[_DID] = str(uuid.UUID(hashlib.sha512(bytes(u, "utf8")).hexdigest()[:32]))
-        reset_rate_limit(request)
 
 
 def start(request):

benefits/eligibility/views.py

@@ -10,7 +10,7 @@
 from django.utils.translation import pgettext, gettext as _
 
 from benefits.core import recaptcha, session, viewmodels
-from benefits.core.middleware import AgencySessionRequired, LoginRequired, RateLimit, RecaptchaEnabled, VerifierSessionRequired
+from benefits.core.middleware import AgencySessionRequired, LoginRequired, RecaptchaEnabled, VerifierSessionRequired
 from benefits.core.models import EligibilityVerifier
 from benefits.core.views import ROUTE_HELP
 from . import analytics, forms, verify
@@ -152,7 +152,6 @@ def start(request):
 
 @decorator_from_middleware(AgencySessionRequired)
 @decorator_from_middleware(LoginRequired)
-@decorator_from_middleware(RateLimit)
 @decorator_from_middleware(RecaptchaEnabled)
 @decorator_from_middleware(VerifierSessionRequired)
 def confirm(request):

benefits/settings.py

@@ -241,21 +241,6 @@ def _filter_empty(ls):
 
 ANALYTICS_KEY = os.environ.get("ANALYTICS_KEY")
 
-# rate limit configuration
-# these should match the values in rate-limit.cy.js
-
-# number of requests allowed in the given period
-RATE_LIMIT = int(os.environ.get("DJANGO_RATE_LIMIT", 5))
-
-# HTTP request methods to rate limit
-RATE_LIMIT_METHODS = os.environ.get("DJANGO_RATE_LIMIT_METHODS", "POST").upper().split(",")
-
-# number of seconds before additional requests are denied
-RATE_LIMIT_PERIOD = int(os.environ.get("DJANGO_RATE_LIMIT_PERIOD", 60))
-
-# Rate Limit feature flag
-RATE_LIMIT_ENABLED = all((RATE_LIMIT > 0, len(RATE_LIMIT_METHODS) > 0, RATE_LIMIT_PERIOD > 0))
-
 # reCAPTCHA configuration
 
 RECAPTCHA_API_URL = os.environ.get("DJANGO_RECAPTCHA_API_URL", "https://www.google.com/recaptcha/api.js")

docs/configuration/data.md

@@ -33,7 +33,6 @@ The sample data included in the repository is enough to bootstrap the applicatio
 Some configuration data is not available with the samples in the repository:
 
 - OAuth configuration to enable authentication (read more about [OAuth configuration](oauth.md))
-- Rate Limiting configuration for eligibility
 - reCAPTCHA configuration for user-submitted forms
 - Payment processor configuration for the enrollment phase
 - Amplitude configuration for capturing analytics events

docs/configuration/rate-limit.md

@@ -1,47 +1,56 @@
 # Configuring Rate Limiting
 
-The benefits application has a simple, single-configuration Rate Limit feature that acts per-session to limit the
-number of consecutive requests in a given time period.
+The benefits application has a simple, single-configuration Rate Limit that acts
+per-IP to limit the number of consecutive requests in a given time period, via
+nginx [`limit_req_zone`](http://nginx.org/en/docs/http/ngx_http_limit_req_module.html#limit_req_zone)
+and [`limit_req`](http://nginx.org/en/docs/http/ngx_http_limit_req_module.html#limit_req) directives.
 
-## Applying to Django code
+The configured rate limit is 12 requests/minute, or 1 request/5 seconds:
 
-The [`RateLimit` middleware][benefits-middleware] can be installed globally for all requests with the
-[`MIDDLEWARE` setting][django-middleware], or per-view with a function decorator.
-
-The latter approach is how the Benefits application rate-limits the Eligibility verification form (which is shown on the
-`confirm` route/view):
-
-```python
-from django.utils.decorators import decorator_from_middleware
-
-from benefits.core.middleware import RateLimit
-
-
-@decorator_from_middleware(RateLimit)
-def confirm(request):
-    """View handler for the eligibility verification form."""
-    # ...
+```nginx
+limit_req_zone $limit zone=rate_limit:10m rate=12r/m;
 ```
 
-## Environment variables
+## HTTP method selection
 
-!!! warning
+An NGINX [map](http://nginx.org/en/docs/http/ngx_http_map_module.html#map)
+variable lists HTTP methods that will be rate limited:
 
-    The following environment variables are all required to activate the Rate Limit feature
+```nginx
+map $request_method $limit {
+    default         "";
+    POST            $binary_remote_addr;
+}
+```
 
-### `DJANGO_RATE_LIMIT`
+The `default` means don't apply a rate limit.
 
-Number of requests allowed in the given [`DJANGO_RATE_LIMIT_PERIOD`](#DJANGO_RATE_LIMIT_PERIOD).
+To add a new method, add a new line:
 
-Must be greater than `0`.
+```nginx
+map $request_method $limit {
+    default         "";
+    OPTIONS         $binary_remote_addr;
+    POST            $binary_remote_addr;
+}
+```
 
-### `DJANGO_RATE_LIMIT_METHODS`
+## App path selection
 
-Comma-separated list of HTTP Methods for which requests are rate limited.
+The `limit_req` is applied to an NGINX [`location`](https://nginx.org/en/docs/http/ngx_http_core_module.html#location) block with a case-insensitive regex to match paths:
 
-### `DJANGO_RATE_LIMIT_PERIOD`
+```nginx
+location ~* ^/(eligibility/confirm)$ {
+    limit_req zone=rate_limit;
+    # config...
+}
+```
 
-Number of seconds before additional requests are denied.
+To add a new path, add a regex OR `|` with the new path (omitting the leading slash):
 
-[benefits-middleware]: https://github.com/cal-itp/benefits/blob/dev/benefits/core/middleware.py
-[django-middleware]: https://docs.djangoproject.com/en/4.0/ref/settings/#std:setting-MIDDLEWARE
+```nginx
+location ~* ^/(eligibility/confirm|new/path)$ {
+    limit_req zone=rate_limit;
+    # config...
+}
+```

terraform/app_service.tf

@@ -63,10 +63,6 @@ resource "azurerm_linux_web_app" "main" {
     "DJANGO_DEBUG"         = local.is_prod ? null : "${local.secret_prefix}django-debug)",
     "DJANGO_LOG_LEVEL"     = "${local.secret_prefix}django-log-level)",
 
-    "DJANGO_RATE_LIMIT"         = local.is_dev ? null : "${local.secret_prefix}django-rate-limit)",
-    "DJANGO_RATE_LIMIT_METHODS" = local.is_dev ? null : "${local.secret_prefix}django-rate-limit-methods)",
-    "DJANGO_RATE_LIMIT_PERIOD"  = local.is_dev ? null : "${local.secret_prefix}django-rate-limit-period)",
-
     "DJANGO_RECAPTCHA_SECRET_KEY" = local.is_dev ? null : "${local.secret_prefix}django-recaptcha-secret-key)",
     "DJANGO_RECAPTCHA_SITE_KEY"   = local.is_dev ? null : "${local.secret_prefix}django-recaptcha-site-key)",