Merge pull request #2255 from willkg/1636210-eliot

bug 1636210: implement eliot microservice for symbolication

CONTRIBUTING.rst

@@ -13,7 +13,7 @@ etiquette guidelines. For more details please see the `CODE_OF_CONDUCT.md file
 Bugs
 ====
 
-All bugs are tracked in `<https://bugzilla.mozilla.org/>`_.
+All bugs are tracked in `Bugzilla <https://bugzilla.mozilla.org/>`_.
 
 Write up a new bug:
 

Makefile

@@ -45,7 +45,7 @@ setup: .env  ## | Initialize services.
 
 .PHONY: run
 run: .env .docker-build  ## | Run the web app and services.
-	docker-compose up web worker frontend
+	docker-compose up web eliot worker frontend
 
 .PHONY: stop
 stop: .env  ## | Stop docker containers.

bin/entrypoint.sh

@@ -31,6 +31,9 @@ case ${SERVICE} in
 web)  ## Run Tecken web service
     exec ./bin/run_web.sh $@
     ;;
+eliot)  ## Run Eliot service
+    exec circusd circus_eliot.ini
+    ;;
 worker)  ## Run Celery worker
     exec ${CMD_PREFIX} celery -A tecken.celery:app worker --loglevel INFO
     ;;

bin/run_eliot_disk_manager.sh

@@ -0,0 +1,9 @@
+#!/bin/bash
+
+# This Source Code Form is subject to the terms of the Mozilla Public
+# License, v. 2.0. If a copy of the MPL was not distributed with this
+# file, You can obtain one at http://mozilla.org/MPL/2.0/.
+
+cd /app/eliot-service/
+
+PYTHONPATH=.:$PYTHONPATH python eliot/cache_manager.py

bin/run_eliot_web.sh

@@ -0,0 +1,22 @@
+#!/bin/bash
+
+# This Source Code Form is subject to the terms of the Mozilla Public
+# License, v. 2.0. If a copy of the MPL was not distributed with this
+# file, You can obtain one at http://mozilla.org/MPL/2.0/.
+
+# default variables
+: "${ELIOT_PORT:=8000}"
+: "${ELIOT_GUNICORN_WORKERS:=1}"
+: "${ELIOT_GUNICORN_TIMEOUT:=300}"
+
+(set -o posix; set) | grep ELIOT
+
+cd /app/eliot-service/
+
+${CMD_PREFIX} gunicorn \
+    --bind 0.0.0.0:"${ELIOT_PORT}" \
+    --timeout "${ELIOT_GUNICORN_TIMEOUT}" \
+    --workers "${ELIOT_GUNICORN_WORKERS}" \
+    --preload \
+    --access-logfile - \
+    eliot.wsgi:application

bin/run_lint.sh

@@ -12,7 +12,7 @@
 
 set -e
 
-BLACKARGS=("--line-length=88" "--target-version=py36" bin tecken systemtests)
+BLACKARGS=("--line-length=88" "--target-version=py36" bin tecken eliot-service systemtests)
 
 if [[ $1 == "--fix" ]]; then
     echo ">>> black fix"

bin/run_test.sh

@@ -16,6 +16,13 @@ export DJANGO_CONFIGURATION=Test
 if [ "$1" = "--shell" ]; then
     bash
 else
-    # python manage.py collectstatic --noinput
+    # Run tecken tests
+    pushd tecken
     pytest
+    popd
+
+    # Run eliot-service tests
+    pushd eliot-service
+    pytest
+    popd
 fi

circus_eliot.ini

@@ -0,0 +1,13 @@
+[watcher:eliot_web]
+cmd = bin/run_eliot_web.sh
+numprocesses = 1
+copy_env = True
+singleton = True
+respawn = False
+
+[watcher:eliot_disk_manager]
+cmd = bin/run_eliot_disk_manager.sh
+numprocesses = 1
+copy_env = True
+singleton = True
+respawn = False

docker-compose.yml

@@ -1,3 +1,4 @@
+---
 version: '2'
 
 services:
@@ -38,6 +39,20 @@ services:
       - $PWD:/app
     command: ["web", "--dev"]
 
+  # Symbolicator service
+  eliot:
+    extends:
+      service: base
+    ports:
+      - "8050:8000"
+    depends_on:
+      - statsd
+    links:
+      - statsd
+    volumes:
+      - $PWD:/app
+    command: ["eliot"]
+
   # Container specifically for running tests.
   test:
     extends:

docker/config/local_dev.env

@@ -9,6 +9,7 @@
 
 # Statsd things
 DJANGO_STATSD_HOST=statsd
+DJANGO_STATSD_PORT=8125
 DJANGO_STATSD_NAMESPACE=mcboatface
 
 DJANGO_ALLOWED_HOSTS=web,localhost
@@ -21,3 +22,17 @@ DATABASE_URL=postgresql://postgres:postgres@db/tecken
 # Django-configuration variables, so don't prefix them with DJANGO_
 AWS_ACCESS_KEY_ID=minio
 AWS_SECRET_ACCESS_KEY=miniostorage
+
+# Eliot
+# -----
+
+# Local development flag
+ELIOT_LOCAL_DEV_ENV=True
+
+# Logging
+ELIOT_LOGGING_LEVEL=INFO
+
+# Statsd things
+ELIOT_STATSD_HOST=statsd
+ELIOT_STATSD_NAMESPACE=mcboatface
+ELIOT_STATSD_PORT=8125

docker/config/test.env

@@ -2,6 +2,9 @@
 # Tecken configuration
 # --------------------
 
+# Tecken settings
+# ---------------
+
 DJANGO_DEBUG=False
 
 # Tells the code to swap the ThreadPoolExecutor in for an executor
@@ -31,3 +34,13 @@ DJANGO_SYMBOL_FILE_PREFIX=v0
 DJANGO_UPLOAD_DEFAULT_URL=https://s3.example.com/private/prefix/
 DJANGO_UPLOAD_TRY_SYMBOLS_URL=https://s3.example.com/try/prefix
 DJANGO_UPLOAD_URL_EXCEPTIONS={"*peterbe.com": "https://s3.example.com/peterbe-com"}
+
+
+# Eliot settings
+# --------------
+
+ELIOT_HOST_ID=testnode
+
+ELIOT_SYMBOLS_URLS=http://symbols.example.com/
+ELIOT_TMP_DIR=/tmp/test/junk/
+ELIOT_SYMBOLS_CACHE_DIR=/tmp/test/cache/

docs/conf.py

@@ -16,8 +16,12 @@
 from pathlib import Path
 
 BASEDIR = Path(__file__).parent.parent
+# Insert repo base dir which will pick up Tecken webapp things
 sys.path.insert(0, str(BASEDIR))
 
+# Insert Eliot base dir for Eliot things
+sys.path.insert(0, str(BASEDIR / "eliot-service"))
+
 # If extensions (or modules to document with autodoc) are in another directory,
 # add these directories to sys.path here. If the directory is relative to the
 # documentation root, use os.path.abspath to make it absolute, like shown here.
@@ -43,6 +47,7 @@
 extensions = [
     "sphinx.ext.extlinks",
     "sphinxcontrib.httpdomain",
+    "everett.sphinxext",
 ]
 
 # Add any paths that contain templates here, relative to this directory.