diff --git a/ara/server/settings.py b/ara/server/settings.py index f4659209..c617e5d6 100644 --- a/ara/server/settings.py +++ b/ara/server/settings.py @@ -91,6 +91,18 @@ def get_secret_key(): # Under which URL should requests be delegated to the distributed sqlite wsgi application DISTRIBUTED_SQLITE_PREFIX = settings.get("DISTRIBUTED_SQLITE_PREFIX", "ara-report") +# Map distributed sqlite mode to a Zuul API +# Databases are downloaded from log urls retrieved from a zuul api +# /zuul_tenant/job_id/ara-report will query DISTRIBUTED_SQLITE_ZUUL_API +# and download the sqlite file from the log storage +DISTRIBUTED_SQLITE_ZUUL = settings.get("DISTRIBUTED_SQLITE_ZUUL", False) + +# Zuul API endpoint for distributed sqlite zuul mode +DISTRIBUTED_SQLITE_ZUUL_API = settings.get("DISTRIBUTED_SQLITE_ZUUL_API", "zuul.example.com/api") + +# Location of the sqlite file in a zuul job log +DISTRIBUTED_SQLITE_ZUUL_DB_PATH = settings.get("DISTRIBUTED_SQLITE_ZUUL_DB_PATH", "logs/ara-report/ansible.sqlite") + # Root directory under which databases will be found relative to the requested URLs. # This will restrict where the WSGI application will go to seek out databases. # For example, the URL "example.org/some/path/ara-report" would translate to @@ -278,6 +290,9 @@ def get_secret_key(): DISTRIBUTED_SQLITE=DISTRIBUTED_SQLITE, DISTRIBUTED_SQLITE_PREFIX=DISTRIBUTED_SQLITE_PREFIX, DISTRIBUTED_SQLITE_ROOT=DISTRIBUTED_SQLITE_ROOT, + DISTRIBUTED_SQLITE_ZUUL=DISTRIBUTED_SQLITE_ZUUL, + DISTRIBUTED_SQLITE_ZUUL_API=DISTRIBUTED_SQLITE_ZUUL_API, + DISTRIBUTED_SQLITE_ZUUL_DB_PATH=DISTRIBUTED_SQLITE_ZUUL_DB_PATH, TIME_ZONE=TIME_ZONE, ) with open(DEFAULT_SETTINGS, "w+") as settings_file: diff --git a/ara/server/wsgi.py b/ara/server/wsgi.py index 697210c6..1a1d6753 100644 --- a/ara/server/wsgi.py +++ b/ara/server/wsgi.py @@ -3,6 +3,9 @@ import logging import os +import requests +import subprocess +import urllib from ara.setup.exceptions import MissingDjangoException @@ -63,15 +66,37 @@ def distributed_sqlite(environ, start_response): # Make sure we aren't escaping outside the root and the directory exists db_dir = os.path.abspath(os.path.join(root, fs_path.lstrip("/"))) + db_file = os.path.join(db_dir, "ansible.sqlite") if not db_dir.startswith(root): logger.warn("Ignoring request: path is outside the root (%s)" % db_dir) return handle_404(start_response) + elif settings.DISTRIBUTED_SQLITE_ZUUL: + if not os.path.exists(db_file): + # find the URL of the zuul sqlite file for ara location /{tenant}/{build-uuid}/ara-report + logger.info("Querying zuul api for (%s)" % db_file) + parts = path_info.split('/') + api_url = settings.DISTRIBUTED_SQLITE_ZUUL_API + "/tenant/" + parts[1] + "/builds?uuid=" + parts[2] + api_resp = requests.get(api_url) + if not api_resp.ok: + return handle_404(start_response) + log_url = api_resp.json()[0].get('log_url') + if not log_url: + return handle_404(start_response) + zuul_sqlite_url = api_resp.json()[0].get('log_url') + "/" + settings.DISTRIBUTED_SQLITE_ZUUL_DB_PATH + + # download the sqlite file + logger.warn("Downloading %s" % zuul_sqlite_url) + os.makedirs(os.path.dirname(db_file), exist_ok=True) + try: + urllib.request.urlretrieve(zuul_sqlite_url, db_file) + except urllib.error.URLError as e: + logger.warn("Could not download sqlite '%s'" % e.reason) + return handle_404(start_response) + elif not os.path.exists(db_dir): logger.warn("Ignoring request: database directory not found (%s)" % db_dir) return handle_404(start_response) - # Find the database file and make sure it exists - db_file = os.path.join(db_dir, "ansible.sqlite") if not os.path.exists(db_file): logger.warn("Ignoring request: database file not found (%s)" % db_file) return handle_404(start_response)