diff --git a/.github/workflows/docker.yml b/.github/workflows/docker.yml index 08b5dc21..a1d5022a 100644 --- a/.github/workflows/docker.yml +++ b/.github/workflows/docker.yml @@ -9,7 +9,7 @@ jobs: strategy: fail-fast: true matrix: - component: [backend, frontend, next] + component: [backend, frontend, next, postgres] include: - component: backend name: cms-backend @@ -17,6 +17,8 @@ jobs: name: cms-frontend - component: next name: website-frontend + - component: postgres + name: cms-migrations permissions: contents: read packages: write diff --git a/docker-compose.yml b/docker-compose.yml index f6daae93..eba518cd 100644 --- a/docker-compose.yml +++ b/docker-compose.yml @@ -11,6 +11,7 @@ services: stdin_open: true ports: - 3001:3001 + frontend: container_name: frontend build: @@ -21,13 +22,14 @@ services: stdin_open: true ports: - 3000:3000 + backend: container_name: go_backend build: context: ./backend dockerfile: ./Dockerfile.development depends_on: - - db + - migration volumes: - './backend:/go/src/cms.csesoc.unsw.edu.au' - 'unpublished_document_data:/var/lib/documents/unpublished/data' @@ -41,6 +43,7 @@ services: - POSTGRES_DB=${PG_DB} - POSTGRES_PORT=${PG_PORT} - POSTGRES_HOST=${PG_HOST} + db: container_name: pg_container image: postgres @@ -52,8 +55,21 @@ services: ports: - ${PG_PORT}:5432 volumes: - - './postgres:/docker-entrypoint-initdb.d/' - 'pg_data:/var/lib/postgresql/data' + + migration: + container_name: migration + build: + context: ./postgres + dockerfile: ./Dockerfile + depends_on: + - db + environment: + - POSTGRES_HOST=db + - POSTGRES_DB=${PG_DB} + - POSTGRES_USER=${PG_USER} + - POSTGRES_PASSWORD=${PG_PASSWORD} + staging_db: container_name: pg_container_testing image: postgres @@ -65,7 +81,7 @@ services: ports: - 1234:5432 volumes: - - './postgres:/docker-entrypoint-initdb.d/' + - './postgres/up:/docker-entrypoint-initdb.d/' - 'staging_pg_db:/var/lib/postgresql/data' volumes: pg_data: diff --git a/postgres/Dockerfile b/postgres/Dockerfile new file mode 100644 index 00000000..b432f298 --- /dev/null +++ b/postgres/Dockerfile @@ -0,0 +1,8 @@ +FROM python:slim + +COPY requirements.txt . +RUN pip install -r requirements.txt + +COPY . . + +ENTRYPOINT [ "python", "migrate.py" ] \ No newline at end of file diff --git a/postgres/dbver.txt b/postgres/dbver.txt new file mode 100644 index 00000000..56a6051c --- /dev/null +++ b/postgres/dbver.txt @@ -0,0 +1 @@ +1 \ No newline at end of file diff --git a/postgres/down/down.sql b/postgres/down/down.sql new file mode 100644 index 00000000..224983dd --- /dev/null +++ b/postgres/down/down.sql @@ -0,0 +1,6 @@ +DROP TABLE IF EXISTS frontend CASCADE; +DROP TABLE IF EXISTS groups CASCADE; +DROP TABLE IF EXISTS person CASCADE; +DROP TABLE IF EXISTS filesystem CASCADE; + +DROP TYPE IF EXISTS permissions_enum; \ No newline at end of file diff --git a/postgres/migrate.py b/postgres/migrate.py new file mode 100644 index 00000000..038c492f --- /dev/null +++ b/postgres/migrate.py @@ -0,0 +1,99 @@ +# Simple migration script to migrate the database in two phases +# - Phase 1: involves completely destroying the current state of the database (down) +# - Phase 2: involves recreating the entire database (up) + +import os +import sys +import psycopg2 +import glob + +# get_db acquires a connection to the database +def get_db(): + connection = psycopg2.connect( + host = os.environ["POSTGRES_HOST"], + database = os.environ["POSTGRES_DB"], + user = os.environ["POSTGRES_USER"], + password = os.environ["POSTGRES_PASSWORD"] + ) + + connection.autocommit = False + return connection + + +# UpgradeJob represents a single attempt to upgrade the DB, it does everything within a transaction +class UpgradeJob: + def __init__(self): + self.connection = get_db() + self.cursor = self.connection.cursor() + + def run_script(self, script: str): + self.cursor.execute(script) + + def cancel_job(self): + self.cursor.close() + self.connection.rollback() + + def finish_job(self): + self.cursor.execute("update migrations SET VersionID = VersionID + 1 WHERE MigrationID = 1;") + self.connection.commit() + self.cursor.close() + + +# Completely destroy the current state of the DB +def down(job: UpgradeJob): + down_script = open("down/down.sql", "r").read() + job.run_script(down_script) + + +# Recreate the database from the defined schema files +def up(job: UpgradeJob): + up_jobs = glob.glob('up/*.sql') + for script in sorted(up_jobs): + job.run_script(open(script, "r").read()) + + +# requires_update determines if the current database requires an update, it does so by querying an update table and comparing the result +# with the contents of the dbver.txt file +def get_db_versions(): + db = get_db() + + git_version_file = open("dbver.txt", "r") + git_version = int(git_version_file.readline()) + container_version = 0 + + # acquire the db version + cursor = db.cursor() + try: + cursor.execute("select VersionID from migrations where MigrationID = 1;") + migration_records = cursor.fetchall() + container_version = 0 if len(migration_records) == 0 else migration_records[0][0] + except Exception as e: + print(e) + pass + finally: + cursor.close() + + return (container_version, git_version) + + +if __name__ == '__main__': + (container_version, git_version) = get_db_versions() + + if git_version <= container_version: + print("Container DB is up to date, skipping upgrade :)") + sys.exit() + + # run the upgrade now :D + upgradeJob = UpgradeJob() + try: + down(upgradeJob) + up(upgradeJob) + except Exception as e: + print(f""" + Failed to upgrade the database from version {container_version} to {git_version}, check logs for additional information. + Database is currently on version {container_version}.""") + upgradeJob.cancel_job() + raise e + + upgradeJob.finish_job() + print(f"Successfully updated DB from version {container_version} to {git_version}.") \ No newline at end of file diff --git a/postgres/requirements.txt b/postgres/requirements.txt new file mode 100644 index 00000000..1d773548 --- /dev/null +++ b/postgres/requirements.txt @@ -0,0 +1 @@ +psycopg2-binary==2.9.3 \ No newline at end of file diff --git a/postgres/up/01-create_migrations_table.sql b/postgres/up/01-create_migrations_table.sql new file mode 100644 index 00000000..130a6202 --- /dev/null +++ b/postgres/up/01-create_migrations_table.sql @@ -0,0 +1,11 @@ +CREATE TABLE IF NOT EXISTS migrations ( + MigrationID SERIAL PRIMARY KEY, + VersionID INTEGER default 0 +); + +DO LANGUAGE plpgsql $$ +BEGIN + IF NOT EXISTS (SELECT FROM migrations WHERE MigrationID = 1) THEN + INSERT INTO migrations (MigrationID, VersionID) VALUES (1, 0); + END IF; +END $$; \ No newline at end of file diff --git a/postgres/02-create_frontend_table.sql b/postgres/up/02-create_frontend_table.sql similarity index 100% rename from postgres/02-create_frontend_table.sql rename to postgres/up/02-create_frontend_table.sql diff --git a/postgres/01-create_groups_table.sql b/postgres/up/03-create_groups_table.sql similarity index 73% rename from postgres/01-create_groups_table.sql rename to postgres/up/03-create_groups_table.sql index 95934cd4..a9291e68 100644 --- a/postgres/01-create_groups_table.sql +++ b/postgres/up/03-create_groups_table.sql @@ -1,9 +1,9 @@ -CREATE EXTENSION hstore; +CREATE EXTENSION IF NOT EXISTS hstore; SET timezone = 'Australia/Sydney'; CREATE TYPE permissions_enum as ENUM ('read', 'write', 'delete'); -CREATE TABLE groups ( +CREATE TABLE IF NOT EXISTS groups ( UID SERIAL PRIMARY KEY, Name VARCHAR(50) NOT NULL, Permission permissions_enum UNIQUE NOT NULL diff --git a/postgres/03-create_person_table.sql b/postgres/up/04-create_person_table.sql similarity index 100% rename from postgres/03-create_person_table.sql rename to postgres/up/04-create_person_table.sql diff --git a/postgres/05-create_filesystem_table.sql b/postgres/up/05-create_filesystem_table.sql similarity index 100% rename from postgres/05-create_filesystem_table.sql rename to postgres/up/05-create_filesystem_table.sql diff --git a/postgres/06-create_dummy_data.sql b/postgres/up/06-create_dummy_data.sql similarity index 100% rename from postgres/06-create_dummy_data.sql rename to postgres/up/06-create_dummy_data.sql