From 018ddd14a1dd8b75a8ce8f100c1fabb139f26e34 Mon Sep 17 00:00:00 2001 From: Florian Dambrine Date: Thu, 1 Oct 2020 07:14:47 -0700 Subject: [PATCH] Fix 404 not found error causing app to exit --- README.md | 10 ++-- docker/config/burrow.toml | 3 ++ docker/docker-compose.yml | 55 +++++++++++++++------- docker/tmp/burrow.pid | 1 + karrot/__init__.py | 29 +++++++++--- karrot/burrow/controllers.py | 12 ++--- karrot/config/config.py | 1 + karrot/config/controllers.py | 6 +-- karrot/heartbeat/controllers.py | 14 +++--- karrot/reporters/controllers.py | 9 ++-- karrot/reporters/factory.py | 13 +++-- karrot/reporters/models.py | 16 +++---- karrot/reporters/prometheus/controllers.py | 1 + karrot/reporters/prometheus/models.py | 2 +- karrot/reporters/stdout/__init__.py | 2 + karrot/reporters/stdout/models.py | 18 +++++++ tox.ini | 2 +- 17 files changed, 128 insertions(+), 66 deletions(-) create mode 100644 docker/tmp/burrow.pid create mode 100644 karrot/reporters/stdout/__init__.py create mode 100644 karrot/reporters/stdout/models.py diff --git a/README.md b/README.md index 41954b3..a39e8e7 100644 --- a/README.md +++ b/README.md @@ -85,7 +85,7 @@ flask "karrot:create_app()" --bind 127.0.0.1:5000 -w 4 ## Global Karrot env vars: | env | value | description | -| ------------------ | ----------------------- | ---------------------------------------- | +|--------------------|-------------------------|------------------------------------------| | `KARROT_LOG` | `INFO, DEBUG, ERROR` | The log level to use for the Karrot app | | `KARROT_REPORTERS` | `prometheus,cloudwatch` | A CSV list of reporters to use in Karrot | @@ -94,7 +94,7 @@ flask "karrot:create_app()" --bind 127.0.0.1:5000 -w 4 * Cloudwatch | env | value | description | -| ----------------------------- | --------------------------------- | ----------------------------------------------------------------------- | +|-------------------------------|-----------------------------------|-------------------------------------------------------------------------| | `KARROT_CLOUDWATCH_NAMESPACE` | `GumGum/Kafka/Burrow/ConsumerLag` | The Cloudwatch namespace prefix to use for lag reporting | | `KARROT_CLOUDWATCH_INTERVAL` | `30` | The Cloudwatch flush interval to execute the `put_metric_data` api call | @@ -151,12 +151,12 @@ The documentation can be genered with [Sphinx](https://www.sphinx-doc.org/en/2.0 ```bash # Start the stack locally using docker-compose -cd tests/ +cd docker/ docker-compose up # Send messages to a test topic docker exec -it \ - tests_kafka_1 \ + docker_kafka_1 \ kafka-producer-perf-test.sh \ --producer-props bootstrap.servers=localhost:9092 \ --throughput 10 \ @@ -166,7 +166,7 @@ docker exec -it \ # Run a consumer polling from the test topic docker exec -it \ - tests_kafka_1 \ + docker_kafka_1 \ kafka-console-consumer.sh \ --bootstrap-server kafka:9092 \ --topic topicA \ diff --git a/docker/config/burrow.toml b/docker/config/burrow.toml index 9205432..0ef97a9 100644 --- a/docker/config/burrow.toml +++ b/docker/config/burrow.toml @@ -1,3 +1,6 @@ +[general] +pidfile="/tmp/burrow.pid" + [zookeeper] servers=[ "zookeeper:2181" ] timeout=6 diff --git a/docker/docker-compose.yml b/docker/docker-compose.yml index 4dc7ef4..76b6bb1 100644 --- a/docker/docker-compose.yml +++ b/docker/docker-compose.yml @@ -1,4 +1,4 @@ -version: "2" +version: "3" services: karrot: image: karrot @@ -7,8 +7,10 @@ services: environment: FLASK_APP: karrot.wsgi FLASK_ENV: production + KARROT_REPORTERS: stdout, volumes: - ~/.aws:/root/.aws + - ../:/app command: - "karrot:create_app()" - -b :5000 @@ -22,25 +24,42 @@ services: burrow: image: solsson/burrow:latest volumes: - - ${PWD}/config/dsci:/etc/burrow + - ${PWD}/config:/etc/burrow + - ${PWD}/tmp:/tmp ports: - 8000:8000 - # depends_on: - # - zookeeper - # - kafka + depends_on: + - zookeeper + - kafka restart: always - # zookeeper: - # image: wurstmeister/zookeeper - # ports: - # - 2181:2181 + zookeeper: + image: wurstmeister/zookeeper + ports: + - 2181:2181 - # kafka: - # image: wurstmeister/kafka:1.1.0 - # ports: - # - 9092:9092 - # environment: - # KAFKA_ZOOKEEPER_CONNECT: zookeeper:2181/local - # KAFKA_ADVERTISED_HOST_NAME: kafka - # KAFKA_ADVERTISED_PORT: 9092 - # KAFKA_CREATE_TOPICS: "test-topic:2:1,test-topic2:1:1,test-topic3:1:1" + kafka: + image: wurstmeister/kafka:1.1.0 + ports: + - 9092:9092 + environment: + KAFKA_ZOOKEEPER_CONNECT: zookeeper:2181/local + KAFKA_ADVERTISED_HOST_NAME: kafka + KAFKA_ADVERTISED_PORT: 9092 + KAFKA_CREATE_TOPICS: "test-topic:2:1,test-topic2:2:1,test-topic3:2:1" +# producer: +# image: wurstmeister/kafka:1.1.0 +# depends_on: +# - kafka +# links: +# - kafka +# entrypoint: +# - /bin/bash +# - -c +# command: +# - "sleep 10; kafka-producer-perf-test.sh \ +# --producer-props bootstrap.servers=kafka:9092 \ +# --throughput 10 \ +# --num-record 100000000 \ +# --record-size 100 \ +# --topic topicA" diff --git a/docker/tmp/burrow.pid b/docker/tmp/burrow.pid new file mode 100644 index 0000000..d00491f --- /dev/null +++ b/docker/tmp/burrow.pid @@ -0,0 +1 @@ +1 diff --git a/karrot/__init__.py b/karrot/__init__.py index 601dc16..c01ae19 100644 --- a/karrot/__init__.py +++ b/karrot/__init__.py @@ -4,8 +4,8 @@ import os import logging import structlog - -from flask import Flask, redirect, url_for, render_template, send_from_directory +from datetime import datetime +from flask import Flask, redirect, url_for, send_from_directory, jsonify from karrot.reporters.factory import ReporterFactory from karrot.config.logger import Logger @@ -44,8 +44,11 @@ def create_app(): app.config["REPORTERS"] = {} for reporter in app.config["KARROT_REPORTERS"]: - logger.debug("Initializing reporter", reporter=reporter) - app.config["REPORTERS"][reporter] = ReporterFactory.get(reporter=reporter) + try: + logger.debug("Initializing reporter", reporter=reporter) + app.config["REPORTERS"][reporter] = ReporterFactory.get(reporter=reporter) + except ValueError: + logger.exception(f"{reporter} is not a valid reporter") logger.info( "Karrot initialized with the following reporters", @@ -88,14 +91,26 @@ def favicon(): @app.errorhandler(500) def internal_server_error(error): - return render_template("error.html", error=str(error), code=500), 500 + return jsonify( + status="error", + msg=f"Karrot failed processing request {error}", + time=str(datetime.now()), + ) @app.errorhandler(404) def page_not_found(error): - return render_template("error.html", error=str(error), code=404), 404 + return jsonify( + status="error", + msg=f"Karrot endpoint not found {error}", + time=str(datetime.now()), + ) @app.errorhandler(Exception) def exception_handler(error): - return render_template("error.html", error=error) + return jsonify( + status="error", + msg=f"Karrot raised an exception {error}", + time=str(datetime.now()), + ) return app diff --git a/karrot/burrow/controllers.py b/karrot/burrow/controllers.py index 909b265..817eb28 100644 --- a/karrot/burrow/controllers.py +++ b/karrot/burrow/controllers.py @@ -23,10 +23,10 @@ @burrow.route("", methods=["POST"]) def webhook_handler(): """ - Process an incoming event from Burrow and call - ``event_handler()`` to process it. + Process an incoming event from Burrow and call + ``event_handler()`` to process it. - :param str event: A valid Burrow Json event POSTed to this endpoint + :param str event: A valid Burrow Json event POSTed to this endpoint """ logger.debug("Hit on /burrow endpoint") data = request.get_json() @@ -37,11 +37,11 @@ def webhook_handler(): def event_handler(event): """ - For each enabled reporter, call the ``process(event)`` function. + For each enabled reporter, call the ``process(event)`` function. - If ``prometheus`` is enabled, it tracks the reporter update. + If ``prometheus`` is enabled, it tracks the reporter update. - :param str event: A valid Burrow Json event + :param str event: A valid Burrow Json event """ prom = app.config["REPORTERS"].get("prometheus", None) diff --git a/karrot/config/config.py b/karrot/config/config.py index 1ae5e59..4d75c95 100644 --- a/karrot/config/config.py +++ b/karrot/config/config.py @@ -32,3 +32,4 @@ class ProductionConfig(Config): class DevelopmentConfig(Config): DEBUG = True + KARROT_LOG = "DEBUG" diff --git a/karrot/config/controllers.py b/karrot/config/controllers.py index 24e9214..4041fce 100644 --- a/karrot/config/controllers.py +++ b/karrot/config/controllers.py @@ -21,10 +21,10 @@ @config.route("", methods=["GET"]) def display(): """ - Returns a simple JSON string with the current application configuration. + Returns a simple JSON string with the current application configuration. - :returns: json -- A JSON with the following format: - ``{"status": "success", "settings": "{...}"}`` + :returns: json -- A JSON with the following format: + ``{"status": "success", "settings": "{...}"}`` """ logger.debug("Hit on /config endpoint") diff --git a/karrot/heartbeat/controllers.py b/karrot/heartbeat/controllers.py index cfb2d7d..244c543 100644 --- a/karrot/heartbeat/controllers.py +++ b/karrot/heartbeat/controllers.py @@ -19,14 +19,12 @@ @heartbeat.route("", methods=["GET"]) def health(): """ - Returns a simple JSON string when the application is healthy. + Returns a simple JSON string when the application is healthy. - :returns: json -- A JSON with the following format: - ``{"status": "success", - "msg": "Burrow-reporter is healthy", - "time": ""}`` + :returns: json -- A JSON with the following format: + ``{"status": "success", + "msg": "Burrow-reporter is healthy", + "time": ""}`` """ app.logger.debug("Healthcheck") - return jsonify( - status="success", msg="Burrow-reporter is healthy", time=str(datetime.now()) - ) + return jsonify(status="success", msg="Karrot is healthy", time=str(datetime.now())) diff --git a/karrot/reporters/controllers.py b/karrot/reporters/controllers.py index 000effc..fab059e 100644 --- a/karrot/reporters/controllers.py +++ b/karrot/reporters/controllers.py @@ -15,16 +15,17 @@ # Define a blueprint reporters = Blueprint("reporters", __name__, url_prefix="/reporters") + # http:///metrics endpoint @reporters.route("/", defaults={"reporter": None}, methods=["GET"]) @reporters.route("/", methods=["GET"]) def display(reporter): """ - Returns a simple JSON with the current application reporters. + Returns a simple JSON with the current application reporters. - :param str reporter: If provided returns details about this reporter only - :returns: json -- A JSON with the following format: - ``{"reporters": ["reporter1", "reporter2"]}`` + :param str reporter: If provided returns details about this reporter only + :returns: json -- A JSON with the following format: + ``{"reporters": ["reporter1", "reporter2"]}`` """ logger.debug(f"{reporter}") data = {"reporters": []} diff --git a/karrot/reporters/factory.py b/karrot/reporters/factory.py index 5b29caf..2714a2b 100644 --- a/karrot/reporters/factory.py +++ b/karrot/reporters/factory.py @@ -3,25 +3,28 @@ from karrot.reporters.prometheus.models import PrometheusReporter from karrot.reporters.cloudwatch.models import CloudwatchReporter +from karrot.reporters.stdout.models import StdoutReporter class ReporterFactory(object): """ - ReporterFactory used to create a Reporter object based on it's name. + ReporterFactory used to create a Reporter object based on it's name. """ @staticmethod def get(reporter): """ - Return the proper notifier object based on the name. + Return the proper notifier object based on the name. - :params str reporter: The type of reporter to create - :returns: Reporter -- A reporter matching ``reporter`` - :raises: ValueError -- When the requested reporter is not found. + :params str reporter: The type of reporter to create + :returns: Reporter -- A reporter matching ``reporter`` + :raises: ValueError -- When the requested reporter is not found. """ if reporter == "prometheus": return PrometheusReporter("prometheus") elif reporter == "cloudwatch": return CloudwatchReporter("cloudwatch") + elif reporter == "stdout": + return StdoutReporter("stdout") else: raise ValueError(reporter) diff --git a/karrot/reporters/models.py b/karrot/reporters/models.py index cc5316b..18ce7c1 100644 --- a/karrot/reporters/models.py +++ b/karrot/reporters/models.py @@ -11,7 +11,7 @@ class Reporter(object): def __init__(self, name): """ - Parent class defining common method and attributes for child Reporters. + Parent class defining common method and attributes for child Reporters. """ self._name = name self._event = None @@ -19,20 +19,20 @@ def __init__(self, name): def process(self, event): """ - Refresh the object with the event details. - A child class should call this parent function - and then implement the proper processing steps for the event. + Refresh the object with the event details. + A child class should call this parent function + and then implement the proper processing steps for the event. - :param str event: A + :param str event: A """ self._event = munchify(event["Event"]) self._last_event_ts = datetime.now() def stats(self, reporter): """ - Report stats of an other reporter. Useful for Prometheus to collect - stats about other reporters. + Report stats of an other reporter. Useful for Prometheus to collect + stats about other reporters. - This method must be implemented in child classes. + This method must be implemented in child classes. """ pass diff --git a/karrot/reporters/prometheus/controllers.py b/karrot/reporters/prometheus/controllers.py index e0d3249..dcd1c5a 100644 --- a/karrot/reporters/prometheus/controllers.py +++ b/karrot/reporters/prometheus/controllers.py @@ -9,6 +9,7 @@ # Define a blueprint prometheus = Blueprint("prometheus", __name__, url_prefix="/metrics") + # http:///metrics endpoint @prometheus.route("", methods=["GET"]) def metrics(): diff --git a/karrot/reporters/prometheus/models.py b/karrot/reporters/prometheus/models.py index 1d5da67..083483d 100644 --- a/karrot/reporters/prometheus/models.py +++ b/karrot/reporters/prometheus/models.py @@ -33,7 +33,7 @@ def process(self, event): def stats(self, reporter): """ - Increment the number of events processed by the reporter. + Increment the number of events processed by the reporter. """ REPORTER_EVENTS_COUNT.labels(reporter=reporter._name).inc() diff --git a/karrot/reporters/stdout/__init__.py b/karrot/reporters/stdout/__init__.py new file mode 100644 index 0000000..faa18be --- /dev/null +++ b/karrot/reporters/stdout/__init__.py @@ -0,0 +1,2 @@ +#!/usr/bin/env python +# -*- coding: utf-8 -*- diff --git a/karrot/reporters/stdout/models.py b/karrot/reporters/stdout/models.py new file mode 100644 index 0000000..69810d7 --- /dev/null +++ b/karrot/reporters/stdout/models.py @@ -0,0 +1,18 @@ +#!/usr/bin/env python +# -*- coding: utf-8 -*- + +from karrot.reporters.models import Reporter +from structlog import get_logger +import json + +logger = get_logger() + + +class StdoutReporter(Reporter): + def __init__(self, name): + super().__init__(name) + + def process(self, event): + super().process(event) + logger.info(json.dumps(event)) + logger.info("Successfully processed burrow event", reporter=self._name) diff --git a/tox.ini b/tox.ini index 4edf51d..4873201 100644 --- a/tox.ini +++ b/tox.ini @@ -35,7 +35,7 @@ passenv = ### tox -e lint [testenv:lint] deps = -r{toxinidir}/tests/requirements.txt -commands = flake8 lib/ tests/ +commands = flake8 karrot/ tests/ ### tox -e checkstyle [testenv:checkstyle]