Added migrations to aggregator, which now will run them in test.

.travis.yml

@@ -9,6 +9,12 @@ install:
   - pip install tox-travis
   - pip install coveralls
   - pip install -r requirements.txt
+addons:
+  postgresql: "10"
+env:
+  - TEST_POSTGRES_URL="postgres://postgres@127.0.0.1/cron_tools_aggregator_test"
+before_script:
+  - psql -c 'create database cron_tools_aggregator_test'
 script: tox
 after_success:
   - coveralls

cron_tools/aggregator/__init__.py

@@ -0,0 +1,4 @@
+import os
+
+SQL_DIR = os.path.join(os.path.abspath(os.path.dirname(__file__)), 'sql')
+MIGRATIONS_DIR = os.path.join(SQL_DIR, 'migrations')

cron_tools/aggregator/app.py

@@ -4,12 +4,16 @@
 
 
 from cron_tools.aggregator.config import AggregatorConfiguration
+from cron_tools.aggregator.queries import apply_migrations
 
 
 aggregator_argument_parser = argparse.ArgumentParser()
 aggregator_argument_parser.add_argument(
     "-f", "--config-file", type=str, default=None, help="JSON Configuration file for the agent."
 )
+aggregator_argument_parser.add_argument(
+    "--auto-apply-migrations", action="store_true", help="Automatically apply any pending database migrations."
+)
 
 
 def build_app(config):
@@ -27,5 +31,8 @@ def main(args=None, config=None):
     else:
         config = AggregatorConfiguration.default()
 
+    if args.auto_apply_migrations:
+        apply_migrations(config.postgres_database_url)
+
     app = build_app(config)
     app.run(host=config.bind_address, port=config.bind_port)

cron_tools/aggregator/queries.py

@@ -1,10 +1,14 @@
 from psycopg2.extras import DictRow, DictCursor
 from psycopg2.pool import ThreadedConnectionPool
+from yoyo import read_migrations, get_backend
+
 import psycopg2
 from six import string_types
 from functools import wraps
 from contextlib import contextmanager
 
+from cron_tools.aggregator import MIGRATIONS_DIR, SQL_DIR
+
 
 class AlteredDictRow(DictRow):
     def __getitem__(self, item):
@@ -67,3 +71,11 @@ def cursor_factory(*cursor_args, **cursor_kwargs):
 
     kwargs.update(cursor_factory=cursor_factory)
     return ThreadedConnectionPool(*args, **kwargs)
+
+
+def apply_migrations(database_url):
+    backend = get_backend(database_url)
+    migrations = read_migrations(MIGRATIONS_DIR)
+
+    with backend.lock(60):
+        backend.apply_migrations(backend.to_apply(migrations))

cron_tools/aggregator/sql/migrations/20190905_01_Hv8dj-initial-schema-for-aggregator-application.py

@@ -0,0 +1,28 @@
+"""
+Initial schema for aggregator application.
+"""
+
+from yoyo import step
+
+__depends__ = {}
+
+steps = [
+    step(
+        """
+CREATE TABLE IF NOT EXISTS job(
+  job_id BIGSERIAL,
+  job_uuid UUID UNIQUE NOT NULL,
+  job_name TEXT NOT NULL,
+  job_args JSONB NOT NULL,
+  job_user TEXT NOT NULL,
+  job_host TEXT NOT NULL,
+  job_tags JSONB NOT NULL,
+  job_status_code INTEGER,
+  job_start_time TIMESTAMP WITH TIME ZONE,
+  job_end_time TIMESTAMP WITH TIME ZONE,
+  created_time TIMESTAMP WITH TIME ZONE,
+  last_updated_time TIMESTAMP WITH TIME ZONE
+);
+        """
+    )
+]

cron_tools/aggregator/sql/schema.sql

@@ -0,0 +1,14 @@
+CREATE TABLE IF NOT EXISTS job(
+  job_id BIGSERIAL,
+  job_uuid UUID UNIQUE NOT NULL,
+  job_name TEXT NOT NULL,
+  job_args JSONB NOT NULL,
+  job_user TEXT NOT NULL,
+  job_host TEXT NOT NULL,
+  job_tags JSONB NOT NULL,
+  job_status_code INTEGER,
+  job_start_time TIMESTAMP WITH TIME ZONE,
+  job_end_time TIMESTAMP WITH TIME ZONE,
+  created_time TIMESTAMP WITH TIME ZONE,
+  last_updated_time TIMESTAMP WITH TIME ZONE
+);

requirements.txt

@@ -4,4 +4,5 @@ bottle
 bottle-swagger-2
 requests
 psycopg2-binary
-python-dateutil
+python-dateutil
+yoyo-migrations

test_cron_tools/test_aggregator/test_app.py

@@ -1,17 +1,37 @@
 import unittest
 from webtest import TestApp
 from assertpy import assert_that
+import os
 
 
 from cron_tools.aggregator.app import build_app
 from cron_tools.aggregator.config import AggregatorConfiguration
+from cron_tools.aggregator.queries import apply_migrations, transaction_wrapper, create_pg_connection
+
+from test_cron_tools.test_aggregator.utils import clear_postgres_database
 
 
 class CronToolsAggregatorApplicationUnitTest(unittest.TestCase):
-    @staticmethod
-    def build_test_app(config=None):
+    DATABASE_URL = os.environ.get("TEST_POSTGRES_URL")
+
+    def setUp(self):
+        if self.DATABASE_URL is None:
+            raise AssertionError("No Postgres Database to test against!")
+        apply_migrations(self.DATABASE_URL)
+
+    def tearDown(self):
+        conn = None
+        try:
+            conn = create_pg_connection(self.DATABASE_URL)
+            with transaction_wrapper(conn) as t:
+                clear_postgres_database(t)
+        finally:
+            if conn:
+                conn.close()
+
+    def build_test_app(self, config=None):
         app = build_app(config or AggregatorConfiguration.default())
         return TestApp(app)
 
     def test_basic_aggregator_start_up(self):
-        app = self.build_test_app()
+        app = self.build_test_app()

test_cron_tools/test_aggregator/utils.py

@@ -0,0 +1,128 @@
+# -*- coding: utf-8 -*-
+from cron_tools.aggregator.queries import cursor_manager
+
+
+# https://stackoverflow.com/questions/2056876/postgres-clear-entire-database-before-re-creating-re-populating-from-bash-scr
+CLEANUP_DATABASE_SQL_SCRIPT = """
+-- Copyright © 2019
+--      mirabilos <t.glaser@tarent.de>
+--
+-- Provided that these terms and disclaimer and all copyright notices
+-- are retained or reproduced in an accompanying document, permission
+-- is granted to deal in this work without restriction, including un‐
+-- limited rights to use, publicly perform, distribute, sell, modify,
+-- merge, give away, or sublicence.
+--
+-- This work is provided “AS IS” and WITHOUT WARRANTY of any kind, to
+-- the utmost extent permitted by applicable law, neither express nor
+-- implied; without malicious intent or gross negligence. In no event
+-- may a licensor, author or contributor be held liable for indirect,
+-- direct, other damage, loss, or other issues arising in any way out
+-- of dealing in the work, even if advised of the possibility of such
+-- damage or existence of a defect, except proven that it results out
+-- of said person’s immediate fault when using the work as intended.
+-- -
+-- Drop everything from the PostgreSQL database.
+
+DO $$
+DECLARE
+        r RECORD;
+BEGIN
+        -- triggers
+        FOR r IN (SELECT pns.nspname, pc.relname, pt.tgname
+                FROM pg_trigger pt, pg_class pc, pg_namespace pns
+                WHERE pns.oid=pc.relnamespace AND pc.oid=pt.tgrelid
+                    AND pns.nspname NOT IN ('information_schema', 'pg_catalog', 'pg_toast')
+                    AND pt.tgisinternal=false
+            ) LOOP
+                EXECUTE format('DROP TRIGGER %I ON %I.%I;',
+                    r.tgname, r.nspname, r.relname);
+        END LOOP;
+        -- constraints #1: foreign key
+        FOR r IN (SELECT pns.nspname, pc.relname, pcon.conname
+                FROM pg_constraint pcon, pg_class pc, pg_namespace pns
+                WHERE pns.oid=pc.relnamespace AND pc.oid=pcon.conrelid
+                    AND pns.nspname NOT IN ('information_schema', 'pg_catalog', 'pg_toast')
+                    AND pcon.contype='f'
+            ) LOOP
+                EXECUTE format('ALTER TABLE ONLY %I.%I DROP CONSTRAINT %I;',
+                    r.nspname, r.relname, r.conname);
+        END LOOP;
+        -- constraints #2: the rest
+        FOR r IN (SELECT pns.nspname, pc.relname, pcon.conname
+                FROM pg_constraint pcon, pg_class pc, pg_namespace pns
+                WHERE pns.oid=pc.relnamespace AND pc.oid=pcon.conrelid
+                    AND pns.nspname NOT IN ('information_schema', 'pg_catalog', 'pg_toast')
+                    AND pcon.contype<>'f'
+            ) LOOP
+                EXECUTE format('ALTER TABLE ONLY %I.%I DROP CONSTRAINT %I;',
+                    r.nspname, r.relname, r.conname);
+        END LOOP;
+        -- indicēs
+        FOR r IN (SELECT pns.nspname, pc.relname
+                FROM pg_class pc, pg_namespace pns
+                WHERE pns.oid=pc.relnamespace
+                    AND pns.nspname NOT IN ('information_schema', 'pg_catalog', 'pg_toast')
+                    AND pc.relkind='i'
+            ) LOOP
+                EXECUTE format('DROP INDEX %I.%I;',
+                    r.nspname, r.relname);
+        END LOOP;
+        -- normal and materialised views
+        FOR r IN (SELECT pns.nspname, pc.relname
+                FROM pg_class pc, pg_namespace pns
+                WHERE pns.oid=pc.relnamespace
+                    AND pns.nspname NOT IN ('information_schema', 'pg_catalog', 'pg_toast')
+                    AND pc.relkind IN ('v', 'm')
+            ) LOOP
+                EXECUTE format('DROP VIEW %I.%I;',
+                    r.nspname, r.relname);
+        END LOOP;
+        -- tables
+        FOR r IN (SELECT pns.nspname, pc.relname
+                FROM pg_class pc, pg_namespace pns
+                WHERE pns.oid=pc.relnamespace
+                    AND pns.nspname NOT IN ('information_schema', 'pg_catalog', 'pg_toast')
+                    AND pc.relkind='r'
+            ) LOOP
+                EXECUTE format('DROP TABLE %I.%I;',
+                    r.nspname, r.relname);
+        END LOOP;
+        -- sequences
+        FOR r IN (SELECT pns.nspname, pc.relname
+                FROM pg_class pc, pg_namespace pns
+                WHERE pns.oid=pc.relnamespace
+                    AND pns.nspname NOT IN ('information_schema', 'pg_catalog', 'pg_toast')
+                    AND pc.relkind='S'
+            ) LOOP
+                EXECUTE format('DROP SEQUENCE %I.%I;',
+                    r.nspname, r.relname);
+        END LOOP;
+        -- functions / procedures
+        FOR r IN (SELECT pns.nspname, pp.proname, pp.oid
+                FROM pg_proc pp, pg_namespace pns
+                WHERE pns.oid=pp.pronamespace
+                    AND pns.nspname NOT IN ('information_schema', 'pg_catalog', 'pg_toast')
+            ) LOOP
+                EXECUTE format('DROP FUNCTION %I.%I(%s);',
+                    r.nspname, r.proname,
+                    pg_get_function_identity_arguments(r.oid));
+        END LOOP;
+        -- nōn-default schemata we own; assume to be run by a not-superuser
+        FOR r IN (SELECT pns.nspname
+                FROM pg_namespace pns, pg_roles pr
+                WHERE pr.oid=pns.nspowner
+                    AND pns.nspname NOT IN ('information_schema', 'pg_catalog', 'pg_toast', 'public')
+                    AND pr.rolname=current_user
+            ) LOOP
+                EXECUTE format('DROP SCHEMA %I;', r.nspname);
+        END LOOP;
+        -- voilà
+        RAISE NOTICE 'Database cleared!';
+END; $$;
+"""
+
+
+def clear_postgres_database(transaction):
+    with cursor_manager(transaction) as c:
+        c.execute(CLEANUP_DATABASE_SQL_SCRIPT)

tox.ini

@@ -2,6 +2,7 @@
 envlist=py27,py34,py35,py36,py37
 
 [testenv]
+passenv = TEST_POSTGRES_URL
 commands=coverage run  --source cron_tools -m unittest discover test_cron_tools
 deps=-r dev-requirements.txt