feat(cms): database migration tool

.github/workflows/docker.yml

@@ -9,14 +9,16 @@ jobs:
     strategy:
       fail-fast: true
       matrix:
-        component: [backend, frontend, next]
+        component: [backend, frontend, next, postgres]
         include:
           - component: backend
             name: cms-backend
           - component: frontend
             name: cms-frontend
           - component: next
             name: website-frontend
+          - component: postgres
+            name: cms-migrations
     permissions:
       contents: read
       packages: write

docker-compose.yml

@@ -11,6 +11,7 @@ services:
     stdin_open: true
     ports:
       - 3001:3001
+
   frontend:
     container_name: frontend
     build:
@@ -21,13 +22,14 @@ services:
     stdin_open: true
     ports:
       - 3000:3000
+
   backend:
     container_name: go_backend
     build:
       context: ./backend
       dockerfile: ./Dockerfile.development
     depends_on:
-      - db
+      - migration
     volumes:
       - './backend:/go/src/cms.csesoc.unsw.edu.au'
       - 'unpublished_document_data:/var/lib/documents/unpublished/data'
@@ -41,6 +43,7 @@ services:
       - POSTGRES_DB=${PG_DB}
       - POSTGRES_PORT=${PG_PORT}
       - POSTGRES_HOST=${PG_HOST}
+
   db:
     container_name: pg_container
     image: postgres
@@ -52,8 +55,21 @@ services:
     ports:
       - ${PG_PORT}:5432
     volumes:
-      - './postgres:/docker-entrypoint-initdb.d/'
       - 'pg_data:/var/lib/postgresql/data'
+
+  migration:
+    container_name: migration
+    build:
+      context: ./postgres
+      dockerfile: ./Dockerfile
+    depends_on:
+      - db
+    environment:
+      - POSTGRES_HOST=db
+      - POSTGRES_DB=${PG_DB}
+      - POSTGRES_USER=${PG_USER}
+      - POSTGRES_PASSWORD=${PG_PASSWORD}
+
   staging_db:
     container_name: pg_container_testing
     image: postgres
@@ -65,7 +81,7 @@ services:
     ports:
       - 1234:5432
     volumes:
-      - './postgres:/docker-entrypoint-initdb.d/'
+      - './postgres/up:/docker-entrypoint-initdb.d/'
       - 'staging_pg_db:/var/lib/postgresql/data'
 volumes:
   pg_data:

postgres/Dockerfile

@@ -0,0 +1,8 @@
+FROM python:slim
+
+COPY requirements.txt .
+RUN pip install -r requirements.txt
+
+COPY . .
+
+ENTRYPOINT [ "python", "migrate.py" ]

postgres/dbver.txt

@@ -0,0 +1 @@
+1

postgres/down/down.sql

@@ -0,0 +1,6 @@
+DROP TABLE IF EXISTS frontend CASCADE;
+DROP TABLE IF EXISTS groups CASCADE;
+DROP TABLE IF EXISTS person CASCADE;
+DROP TABLE IF EXISTS filesystem CASCADE;
+
+DROP TYPE IF EXISTS permissions_enum;

postgres/migrate.py

@@ -0,0 +1,99 @@
+# Simple migration script to migrate the database in two phases
+#   - Phase 1: involves completely destroying the current state of the database (down)
+#   - Phase 2: involves recreating the entire database (up)
+
+import os
+import sys
+import psycopg2
+import glob
+
+# get_db acquires a connection to the database
+def get_db():
+    connection = psycopg2.connect(
+        host = os.environ["POSTGRES_HOST"],
+        database = os.environ["POSTGRES_DB"],
+        user = os.environ["POSTGRES_USER"],
+        password = os.environ["POSTGRES_PASSWORD"]
+    )
+
+    connection.autocommit = False
+    return connection
+
+
+# UpgradeJob represents a single attempt to upgrade the DB, it does everything within a transaction
+class UpgradeJob:
+    def __init__(self):
+        self.connection = get_db()
+        self.cursor = self.connection.cursor()
+
+    def run_script(self, script: str):
+        self.cursor.execute(script)
+
+    def cancel_job(self):
+        self.cursor.close()
+        self.connection.rollback()
+
+    def finish_job(self):
+        self.cursor.execute("update migrations SET VersionID = VersionID + 1 WHERE MigrationID = 1;")
+        self.connection.commit()
+        self.cursor.close()
+
+
+# Completely destroy the current state of the DB
+def down(job: UpgradeJob):
+    down_script = open("down/down.sql", "r").read()
+    job.run_script(down_script)
+
+
+# Recreate the database from the defined schema files
+def up(job: UpgradeJob):
+    up_jobs = glob.glob('up/*.sql')
+    for script in sorted(up_jobs): 
+        job.run_script(open(script, "r").read())
+
+
+# requires_update determines if the current database requires an update, it does so by querying an update table and comparing the result
+# with the contents of the dbver.txt file
+def get_db_versions():
+    db = get_db()
+
+    git_version_file = open("dbver.txt", "r")
+    git_version = int(git_version_file.readline())
+    container_version = 0
+
+    # acquire the db version
+    cursor = db.cursor()
+    try:
+        cursor.execute("select VersionID from migrations where MigrationID = 1;")
+        migration_records = cursor.fetchall()
+        container_version = 0 if len(migration_records) == 0 else migration_records[0][0]
+    except Exception as e:
+        print(e)
+        pass
+    finally:
+        cursor.close()        
+
+    return (container_version, git_version)
+
+
+if __name__ == '__main__':
+    (container_version, git_version) = get_db_versions()
+
+    if git_version <= container_version:
+        print("Container DB is up to date, skipping upgrade :)")
+        sys.exit()
+
+    # run the upgrade now :D
+    upgradeJob = UpgradeJob()
+    try:
+        down(upgradeJob)
+        up(upgradeJob)
+    except Exception as e:
+        print(f"""
+            Failed to upgrade the database from version {container_version} to {git_version}, check logs for additional information.
+            Database is currently on version {container_version}.""")
+        upgradeJob.cancel_job()
+        raise e
+
+    upgradeJob.finish_job()
+    print(f"Successfully updated DB from version {container_version} to {git_version}.")

postgres/requirements.txt

@@ -0,0 +1 @@
+psycopg2-binary==2.9.3

postgres/up/01-create_migrations_table.sql

@@ -0,0 +1,11 @@
+CREATE TABLE IF NOT EXISTS migrations (
+   MigrationID SERIAL PRIMARY KEY,
+   VersionID INTEGER default 0
+);
+
+DO LANGUAGE plpgsql $$
+BEGIN
+   IF NOT EXISTS (SELECT FROM migrations WHERE MigrationID = 1) THEN
+      INSERT INTO migrations (MigrationID, VersionID) VALUES (1, 0);
+   END IF;
+END $$;

postgres/01-create_groups_table.sql → postgres/up/03-create_groups_table.sql

@@ -1,9 +1,9 @@
-CREATE EXTENSION hstore;
+CREATE EXTENSION IF NOT EXISTS hstore;
 SET timezone = 'Australia/Sydney';
 
 CREATE TYPE permissions_enum as ENUM ('read', 'write', 'delete');
 
-CREATE TABLE groups (
+CREATE TABLE IF NOT EXISTS groups (
   UID   SERIAL PRIMARY KEY,
   Name  VARCHAR(50) NOT NULL,
   Permission permissions_enum UNIQUE NOT NULL