Fix 404 not found error causing app to exit

README.md

@@ -85,7 +85,7 @@ flask "karrot:create_app()" --bind 127.0.0.1:5000 -w 4
 ## Global Karrot env vars:
 
 | env                | value                   | description                              |
-| ------------------ | ----------------------- | ---------------------------------------- |
+|--------------------|-------------------------|------------------------------------------|
 | `KARROT_LOG`       | `INFO, DEBUG, ERROR`    | The log level to use for the Karrot app  |
 | `KARROT_REPORTERS` | `prometheus,cloudwatch` | A CSV list of reporters to use in Karrot |
 
@@ -94,7 +94,7 @@ flask "karrot:create_app()" --bind 127.0.0.1:5000 -w 4
 * Cloudwatch
 
 | env                           | value                             | description                                                             |
-| ----------------------------- | --------------------------------- | ----------------------------------------------------------------------- |
+|-------------------------------|-----------------------------------|-------------------------------------------------------------------------|
 | `KARROT_CLOUDWATCH_NAMESPACE` | `GumGum/Kafka/Burrow/ConsumerLag` | The Cloudwatch namespace prefix to use for lag reporting                |
 | `KARROT_CLOUDWATCH_INTERVAL`  | `30`                              | The Cloudwatch flush interval to execute the `put_metric_data` api call |
 
@@ -151,12 +151,12 @@ The documentation can be genered with [Sphinx](https://www.sphinx-doc.org/en/2.0
 
 ```bash
 # Start the stack locally using docker-compose
-cd tests/
+cd docker/
 docker-compose up
 
 # Send messages to a test topic
 docker exec -it \
-  tests_kafka_1 \
+  docker_kafka_1 \
   kafka-producer-perf-test.sh \
   --producer-props bootstrap.servers=localhost:9092 \
   --throughput 10 \
@@ -166,7 +166,7 @@ docker exec -it \
 
 # Run a consumer polling from the test topic
 docker exec -it \
-   tests_kafka_1 \
+   docker_kafka_1 \
    kafka-console-consumer.sh \
    --bootstrap-server kafka:9092 \
    --topic topicA \

docker/config/burrow.toml

@@ -1,3 +1,6 @@
+[general]
+pidfile="/tmp/burrow.pid"
+
 [zookeeper]
 servers=[ "zookeeper:2181" ]
 timeout=6

docker/docker-compose.yml

@@ -1,4 +1,4 @@
-version: "2"
+version: "3"
 services:
   karrot:
     image: karrot
@@ -7,8 +7,10 @@ services:
     environment:
       FLASK_APP: karrot.wsgi
       FLASK_ENV: production
+      KARROT_REPORTERS: stdout,
     volumes:
       - ~/.aws:/root/.aws
+      - ../:/app
     command:
       - "karrot:create_app()"
       - -b :5000
@@ -22,25 +24,42 @@ services:
   burrow:
     image: solsson/burrow:latest
     volumes:
-      - ${PWD}/config/dsci:/etc/burrow
+      - ${PWD}/config:/etc/burrow
+      - ${PWD}/tmp:/tmp
     ports:
       - 8000:8000
-    # depends_on:
-    #   - zookeeper
-    #   - kafka
+    depends_on:
+      - zookeeper
+      - kafka
     restart: always
 
-  # zookeeper:
-  #   image: wurstmeister/zookeeper
-  #   ports:
-  #     - 2181:2181
+  zookeeper:
+    image: wurstmeister/zookeeper
+    ports:
+      - 2181:2181
 
-  # kafka:
-  #   image: wurstmeister/kafka:1.1.0
-  #   ports:
-  #     - 9092:9092
-  #   environment:
-  #     KAFKA_ZOOKEEPER_CONNECT: zookeeper:2181/local
-  #     KAFKA_ADVERTISED_HOST_NAME: kafka
-  #     KAFKA_ADVERTISED_PORT: 9092
-  #     KAFKA_CREATE_TOPICS: "test-topic:2:1,test-topic2:1:1,test-topic3:1:1"
+  kafka:
+    image: wurstmeister/kafka:1.1.0
+    ports:
+      - 9092:9092
+    environment:
+      KAFKA_ZOOKEEPER_CONNECT: zookeeper:2181/local
+      KAFKA_ADVERTISED_HOST_NAME: kafka
+      KAFKA_ADVERTISED_PORT: 9092
+      KAFKA_CREATE_TOPICS: "test-topic:2:1,test-topic2:2:1,test-topic3:2:1"
+#   producer:
+#     image: wurstmeister/kafka:1.1.0
+#     depends_on:
+#       - kafka
+#     links:
+#       - kafka
+#     entrypoint:
+#       - /bin/bash
+#       - -c
+#     command:
+#       - "sleep 10; kafka-producer-perf-test.sh \
+#         --producer-props bootstrap.servers=kafka:9092 \
+#         --throughput 10 \
+#         --num-record 100000000 \
+#         --record-size 100 \
+#         --topic topicA"

docker/tmp/burrow.pid

@@ -0,0 +1 @@
+1

karrot/__init__.py

@@ -4,8 +4,8 @@
 import os
 import logging
 import structlog
-
-from flask import Flask, redirect, url_for, render_template, send_from_directory
+from datetime import datetime
+from flask import Flask, redirect, url_for, send_from_directory, jsonify
 from karrot.reporters.factory import ReporterFactory
 from karrot.config.logger import Logger
 
@@ -44,8 +44,11 @@ def create_app():
     app.config["REPORTERS"] = {}
 
     for reporter in app.config["KARROT_REPORTERS"]:
-        logger.debug("Initializing reporter", reporter=reporter)
-        app.config["REPORTERS"][reporter] = ReporterFactory.get(reporter=reporter)
+        try:
+            logger.debug("Initializing reporter", reporter=reporter)
+            app.config["REPORTERS"][reporter] = ReporterFactory.get(reporter=reporter)
+        except ValueError:
+            logger.exception(f"{reporter} is not a valid reporter")
 
     logger.info(
         "Karrot initialized with the following reporters",
@@ -88,14 +91,26 @@ def favicon():
 
         @app.errorhandler(500)
         def internal_server_error(error):
-            return render_template("error.html", error=str(error), code=500), 500
+            return jsonify(
+                status="error",
+                msg=f"Karrot failed processing request {error}",
+                time=str(datetime.now()),
+            )
 
         @app.errorhandler(404)
         def page_not_found(error):
-            return render_template("error.html", error=str(error), code=404), 404
+            return jsonify(
+                status="error",
+                msg=f"Karrot endpoint not found {error}",
+                time=str(datetime.now()),
+            )
 
         @app.errorhandler(Exception)
         def exception_handler(error):
-            return render_template("error.html", error=error)
+            return jsonify(
+                status="error",
+                msg=f"Karrot raised an exception {error}",
+                time=str(datetime.now()),
+            )
 
     return app

karrot/burrow/controllers.py

@@ -23,10 +23,10 @@
 @burrow.route("", methods=["POST"])
 def webhook_handler():
     """
-        Process an incoming event from Burrow and call
-        ``event_handler()`` to process it.
+    Process an incoming event from Burrow and call
+    ``event_handler()`` to process it.
 
-        :param str event: A valid Burrow Json event POSTed to this endpoint
+    :param str event: A valid Burrow Json event POSTed to this endpoint
     """
     logger.debug("Hit on /burrow endpoint")
     data = request.get_json()
@@ -37,11 +37,11 @@ def webhook_handler():
 
 def event_handler(event):
     """
-        For each enabled reporter, call the ``process(event)`` function.
+    For each enabled reporter, call the ``process(event)`` function.
 
-        If ``prometheus`` is enabled, it tracks the reporter update.
+    If ``prometheus`` is enabled, it tracks the reporter update.
 
-        :param str event: A valid Burrow Json event
+    :param str event: A valid Burrow Json event
     """
     prom = app.config["REPORTERS"].get("prometheus", None)
 

karrot/config/config.py

@@ -32,3 +32,4 @@ class ProductionConfig(Config):
 
 class DevelopmentConfig(Config):
     DEBUG = True
+    KARROT_LOG = "DEBUG"

karrot/config/controllers.py

@@ -21,10 +21,10 @@
 @config.route("", methods=["GET"])
 def display():
     """
-        Returns a simple JSON string with the current application configuration.
+    Returns a simple JSON string with the current application configuration.
 
-        :returns: json -- A JSON with the following format:
-        ``{"status": "success", "settings": "{...}"}``
+    :returns: json -- A JSON with the following format:
+    ``{"status": "success", "settings": "{...}"}``
     """
     logger.debug("Hit on /config endpoint")
 

karrot/heartbeat/controllers.py

@@ -19,14 +19,12 @@
 @heartbeat.route("", methods=["GET"])
 def health():
     """
-        Returns a simple JSON string when the application is healthy.
+    Returns a simple JSON string when the application is healthy.
 
-        :returns: json -- A JSON with the following format:
-        ``{"status": "success",
-           "msg": "Burrow-reporter is healthy",
-           "time": "<datetime.now()>"}``
+    :returns: json -- A JSON with the following format:
+    ``{"status": "success",
+       "msg": "Burrow-reporter is healthy",
+       "time": "<datetime.now()>"}``
     """
     app.logger.debug("Healthcheck")
-    return jsonify(
-        status="success", msg="Burrow-reporter is healthy", time=str(datetime.now())
-    )
+    return jsonify(status="success", msg="Karrot is healthy", time=str(datetime.now()))

karrot/reporters/controllers.py

@@ -15,16 +15,17 @@
 # Define a blueprint
 reporters = Blueprint("reporters", __name__, url_prefix="/reporters")
 
+
 # http://<hostname>/metrics endpoint
 @reporters.route("/", defaults={"reporter": None}, methods=["GET"])
 @reporters.route("/<reporter>", methods=["GET"])
 def display(reporter):
     """
-        Returns a simple JSON with the current application reporters.
+    Returns a simple JSON with the current application reporters.
 
-        :param str reporter: If provided returns details about this reporter only
-        :returns: json -- A JSON with the following format:
-        ``{"reporters": ["reporter1", "reporter2"]}``
+    :param str reporter: If provided returns details about this reporter only
+    :returns: json -- A JSON with the following format:
+    ``{"reporters": ["reporter1", "reporter2"]}``
     """
     logger.debug(f"{reporter}")
     data = {"reporters": []}

karrot/reporters/factory.py

@@ -3,25 +3,28 @@
 
 from karrot.reporters.prometheus.models import PrometheusReporter
 from karrot.reporters.cloudwatch.models import CloudwatchReporter
+from karrot.reporters.stdout.models import StdoutReporter
 
 
 class ReporterFactory(object):
     """
-        ReporterFactory used to create a Reporter object based on it's name.
+    ReporterFactory used to create a Reporter object based on it's name.
     """
 
     @staticmethod
     def get(reporter):
         """
-            Return the proper notifier object based on the name.
+        Return the proper notifier object based on the name.
 
-            :params str reporter: The type of reporter to create
-            :returns: Reporter -- A reporter matching ``reporter``
-            :raises: ValueError -- When the requested reporter is not found.
+        :params str reporter: The type of reporter to create
+        :returns: Reporter -- A reporter matching ``reporter``
+        :raises: ValueError -- When the requested reporter is not found.
         """
         if reporter == "prometheus":
             return PrometheusReporter("prometheus")
         elif reporter == "cloudwatch":
             return CloudwatchReporter("cloudwatch")
+        elif reporter == "stdout":
+            return StdoutReporter("stdout")
         else:
             raise ValueError(reporter)

karrot/reporters/models.py

@@ -11,28 +11,28 @@
 class Reporter(object):
     def __init__(self, name):
         """
-            Parent class defining common method and attributes for child Reporters.
+        Parent class defining common method and attributes for child Reporters.
         """
         self._name = name
         self._event = None
         self._last_event_ts = None
 
     def process(self, event):
         """
-            Refresh the object with the event details.
-            A child class should call this parent function
-            and then implement the proper processing steps for the event.
+        Refresh the object with the event details.
+        A child class should call this parent function
+        and then implement the proper processing steps for the event.
 
-            :param str event: A
+        :param str event: A
         """
         self._event = munchify(event["Event"])
         self._last_event_ts = datetime.now()
 
     def stats(self, reporter):
         """
-            Report stats of an other reporter. Useful for Prometheus to collect
-            stats about other reporters.
+        Report stats of an other reporter. Useful for Prometheus to collect
+        stats about other reporters.
 
-            This method must be implemented in child classes.
+        This method must be implemented in child classes.
         """
         pass

karrot/reporters/prometheus/controllers.py

@@ -9,6 +9,7 @@
 # Define a blueprint
 prometheus = Blueprint("prometheus", __name__, url_prefix="/metrics")
 
+
 # http://<hostname>/metrics endpoint
 @prometheus.route("", methods=["GET"])
 def metrics():

karrot/reporters/prometheus/models.py

@@ -33,7 +33,7 @@ def process(self, event):
 
     def stats(self, reporter):
         """
-            Increment the number of events processed by the reporter.
+        Increment the number of events processed by the reporter.
         """
         REPORTER_EVENTS_COUNT.labels(reporter=reporter._name).inc()
 

karrot/reporters/stdout/__init__.py

@@ -0,0 +1,2 @@
+#!/usr/bin/env python
+# -*- coding: utf-8 -*-

karrot/reporters/stdout/models.py

@@ -0,0 +1,18 @@
+#!/usr/bin/env python
+# -*- coding: utf-8 -*-
+
+from karrot.reporters.models import Reporter
+from structlog import get_logger
+import json
+
+logger = get_logger()
+
+
+class StdoutReporter(Reporter):
+    def __init__(self, name):
+        super().__init__(name)
+
+    def process(self, event):
+        super().process(event)
+        logger.info(json.dumps(event))
+        logger.info("Successfully processed burrow event", reporter=self._name)

tox.ini

@@ -35,7 +35,7 @@ passenv =
 ### tox -e lint
 [testenv:lint]
 deps = -r{toxinidir}/tests/requirements.txt
-commands = flake8 lib/ tests/
+commands = flake8 karrot/ tests/
 
 ### tox -e checkstyle
 [testenv:checkstyle]