Merge pull request #223 from release-engineering/prometheus-metrics

Add metrics for request latency and count

docker/Dockerfile-api

@@ -11,6 +11,7 @@ RUN dnf -y install \
     python3-flask-oidc \
     python3-gunicorn \
     python3-neomodel \
+    python3-prometheus_client \
     python3-six \
     && dnf clean all
 # This will allow a non-root user to install a custom root CA at run-time

docker/Dockerfile-tests

@@ -17,12 +17,14 @@ RUN dnf -y install \
   python3-mock \
   python2-neomodel \
   python3-neomodel \
+  python3-prometheus_client \
   python2-pytest \
   python3-pytest \
   python2-pytest-cov \
   python3-pytest-cov \
   && dnf clean all
-RUN pip install flake8-docstrings
+# We must install prometheus_client for Python 2 since it is no longer packaged for Fedora
+RUN pip install flake8-docstrings prometheus_client
 
 VOLUME /src
 WORKDIR /src

estuary/api/monitoring.py

@@ -0,0 +1,75 @@
+# SPDX-License-Identifier: GPL-3.0+
+
+from __future__ import unicode_literals
+
+import time
+
+from flask import Response, Blueprint, request
+import prometheus_client
+
+
+REQUEST_COUNT = prometheus_client.Counter(
+    'request_count', 'App Request Count',
+    ['app_name', 'method', 'endpoint', 'query_string', 'http_status'])
+
+REQUEST_LATENCY = prometheus_client.Histogram(
+    'request_latency_seconds', 'Request latency',
+    ['app_name', 'endpoint', 'query_string'])
+
+
+def start_request_timer():
+    """Start the request timer."""
+    request.start_time = time.time()
+
+
+def stop_request_timer(response):
+    """
+    Stop the request timer.
+
+    :param flask.Response response: the Flask response to stop the timer on
+    :return: the Flask response
+    :rtype: flask.Response
+    """
+    resp_time = time.time() - request.start_time
+    REQUEST_LATENCY.labels(
+        'estuary-api', request.path, request.query_string.decode('utf-8')).observe(resp_time)
+    return response
+
+
+def record_request_metadata(response):
+    """
+    Record metadata about the request.
+
+    :param flask.Response response: the Flask response to record metadata about
+    :return: the Flask response
+    :rtype: flask.Response
+    """
+    REQUEST_COUNT.labels(
+        'estuary-api', request.method, request.path, request.query_string,
+        response.status_code).inc()
+    return response
+
+
+def configure_monitoring(app):
+    """Configure monitoring on the Flask app.
+
+    :param flask.Flask app: the Flask application to configure
+    """
+    app.before_request(start_request_timer)
+    app.after_request(stop_request_timer)
+    app.after_request(record_request_metadata)
+
+
+monitoring_api = Blueprint('monitoring', __name__)
+
+
+@monitoring_api.route('/metrics')
+def metrics():
+    """
+    Display Prometheus metrics about the app.
+
+    :rtype: flask.Response
+    """
+    return Response(
+        prometheus_client.generate_latest(),
+        content_type=prometheus_client.CONTENT_TYPE_LATEST)

estuary/app.py

@@ -10,6 +10,7 @@
 from neomodel import config as neomodel_config
 from neo4j.exceptions import ServiceUnavailable, AuthError
 
+from estuary import log
 from estuary.logger import init_logging
 from estuary.error import json_error, ValidationError
 from estuary.api.v1 import api_v1
@@ -109,6 +110,15 @@ def create_app(config_obj=None):
     app.register_error_handler(ServiceUnavailable, json_error)
     app.register_error_handler(AuthError, json_error)
     app.register_blueprint(api_v1, url_prefix='/api/v1')
+    try:
+        from estuary.api.monitoring import monitoring_api, configure_monitoring
+        app.register_blueprint(monitoring_api, url_prefix='/monitoring')
+        configure_monitoring(app)
+    except ImportError as e:
+        # If prometheus_client isn't installed, then don't register the monitoring blueprint
+        log.warning('The promethus_client is not installed, so metrics will be disabled')
+        if 'prometheus_client' not in str(e):
+            raise
 
     app.after_request(insert_headers)
 

tests/api/test_monitoring.py

@@ -0,0 +1,25 @@
+# SPDX-License-Identifier: GPL-3.0+
+
+from __future__ import unicode_literals
+
+
+def test_monitoring(client):
+    """Test the Prometheus metrics endpoint is working."""
+    # First request something even if it doesn't exist to generate a metric
+    client.get('/api/v1/story/containeradvisory/123123123?fallback=advisory')
+    # Then check the metrics
+    rv = client.get('/monitoring/metrics')
+    assert rv.status_code == 200
+    rv_data = rv.data.decode('utf-8')
+    for le in ('0.005', '0.01', '0.025', '0.05', '0.075', '0.1', '0.25', '0.5', '0.75', '1.0',
+               '2.5', '5.0', '7.5', '10.0', '+Inf'):
+        expected = (
+            'request_latency_seconds_bucket{{app_name="estuary-api",endpoint="/api/v1/story/'
+            'containeradvisory/123123123",le="{}",query_string="fallback=advisory"}}'.format(le))
+        assert expected in rv_data
+    expected = ('request_latency_seconds_count{app_name="estuary-api",endpoint="/api/v1/story/'
+                'containeradvisory/123123123",query_string="fallback=advisory"}')
+    assert expected in rv_data
+    expected = ('request_latency_seconds_sum{app_name="estuary-api",endpoint="/api/v1/story/'
+                'containeradvisory/123123123",query_string="fallback=advisory"}')
+    assert expected in rv_data