From 70202038222d02a5d7d4aff5f6557a77e790e39e Mon Sep 17 00:00:00 2001 From: Varun Sethu Date: Mon, 26 Sep 2022 20:25:25 +1000 Subject: [PATCH 1/5] setup migration tool --- docker-compose.yml | 20 +++- postgres/01-create_migrations_table.sql | 11 +++ ...s_table.sql => 03-create_groups_table.sql} | 4 +- ...n_table.sql => 04-create_person_table.sql} | 0 postgres/Dockerfile | 6 ++ postgres/dbver.txt | 1 + postgres/down.sql | 6 ++ postgres/migrate.py | 98 +++++++++++++++++++ postgres/requirements.txt | 1 + 9 files changed, 143 insertions(+), 4 deletions(-) create mode 100644 postgres/01-create_migrations_table.sql rename postgres/{01-create_groups_table.sql => 03-create_groups_table.sql} (73%) rename postgres/{03-create_person_table.sql => 04-create_person_table.sql} (100%) create mode 100644 postgres/Dockerfile create mode 100644 postgres/dbver.txt create mode 100644 postgres/down.sql create mode 100644 postgres/migrate.py create mode 100644 postgres/requirements.txt diff --git a/docker-compose.yml b/docker-compose.yml index f6daae93..89366d27 100644 --- a/docker-compose.yml +++ b/docker-compose.yml @@ -11,6 +11,7 @@ services: stdin_open: true ports: - 3001:3001 + frontend: container_name: frontend build: @@ -21,13 +22,14 @@ services: stdin_open: true ports: - 3000:3000 + backend: container_name: go_backend build: context: ./backend dockerfile: ./Dockerfile.development depends_on: - - db + - migration volumes: - './backend:/go/src/cms.csesoc.unsw.edu.au' - 'unpublished_document_data:/var/lib/documents/unpublished/data' @@ -41,6 +43,7 @@ services: - POSTGRES_DB=${PG_DB} - POSTGRES_PORT=${PG_PORT} - POSTGRES_HOST=${PG_HOST} + db: container_name: pg_container image: postgres @@ -52,8 +55,21 @@ services: ports: - ${PG_PORT}:5432 volumes: - - './postgres:/docker-entrypoint-initdb.d/' - 'pg_data:/var/lib/postgresql/data' + + migration: + container_name: migration + build: + context: ./postgres + dockerfile: ./Dockerfile + depends_on: + - db + environment: + - POSTGRES_HOST=db + - POSTGRES_DB=${PG_DB} + - POSTGRES_USER=${PG_USER} + - POSTGRES_PASSWORD=${PG_PASSWORD} + staging_db: container_name: pg_container_testing image: postgres diff --git a/postgres/01-create_migrations_table.sql b/postgres/01-create_migrations_table.sql new file mode 100644 index 00000000..130a6202 --- /dev/null +++ b/postgres/01-create_migrations_table.sql @@ -0,0 +1,11 @@ +CREATE TABLE IF NOT EXISTS migrations ( + MigrationID SERIAL PRIMARY KEY, + VersionID INTEGER default 0 +); + +DO LANGUAGE plpgsql $$ +BEGIN + IF NOT EXISTS (SELECT FROM migrations WHERE MigrationID = 1) THEN + INSERT INTO migrations (MigrationID, VersionID) VALUES (1, 0); + END IF; +END $$; \ No newline at end of file diff --git a/postgres/01-create_groups_table.sql b/postgres/03-create_groups_table.sql similarity index 73% rename from postgres/01-create_groups_table.sql rename to postgres/03-create_groups_table.sql index 95934cd4..a9291e68 100644 --- a/postgres/01-create_groups_table.sql +++ b/postgres/03-create_groups_table.sql @@ -1,9 +1,9 @@ -CREATE EXTENSION hstore; +CREATE EXTENSION IF NOT EXISTS hstore; SET timezone = 'Australia/Sydney'; CREATE TYPE permissions_enum as ENUM ('read', 'write', 'delete'); -CREATE TABLE groups ( +CREATE TABLE IF NOT EXISTS groups ( UID SERIAL PRIMARY KEY, Name VARCHAR(50) NOT NULL, Permission permissions_enum UNIQUE NOT NULL diff --git a/postgres/03-create_person_table.sql b/postgres/04-create_person_table.sql similarity index 100% rename from postgres/03-create_person_table.sql rename to postgres/04-create_person_table.sql diff --git a/postgres/Dockerfile b/postgres/Dockerfile new file mode 100644 index 00000000..0618ba58 --- /dev/null +++ b/postgres/Dockerfile @@ -0,0 +1,6 @@ +FROM python:latest + +COPY . . +RUN pip install -r requirements.txt + +ENTRYPOINT [ "python", "migrate.py" ] \ No newline at end of file diff --git a/postgres/dbver.txt b/postgres/dbver.txt new file mode 100644 index 00000000..d8263ee9 --- /dev/null +++ b/postgres/dbver.txt @@ -0,0 +1 @@ +2 \ No newline at end of file diff --git a/postgres/down.sql b/postgres/down.sql new file mode 100644 index 00000000..224983dd --- /dev/null +++ b/postgres/down.sql @@ -0,0 +1,6 @@ +DROP TABLE IF EXISTS frontend CASCADE; +DROP TABLE IF EXISTS groups CASCADE; +DROP TABLE IF EXISTS person CASCADE; +DROP TABLE IF EXISTS filesystem CASCADE; + +DROP TYPE IF EXISTS permissions_enum; \ No newline at end of file diff --git a/postgres/migrate.py b/postgres/migrate.py new file mode 100644 index 00000000..fba4ed85 --- /dev/null +++ b/postgres/migrate.py @@ -0,0 +1,98 @@ +# Simple migration script to migrate the database in two phases +# - Phase 1: involves completely destroying the current state of the database (down) +# - Phase 2: involves recreating the entire database (up) + +import os +import sys +import psycopg2 +import glob + +# get_db acquires a connection to the database +def get_db(): + connection = psycopg2.connect( + host = os.environ["POSTGRES_HOST"], + database = os.environ["POSTGRES_DB"], + user = os.environ["POSTGRES_USER"], + password = os.environ["POSTGRES_PASSWORD"] + ) + + connection.autocommit = False + return connection + + +# UpgradeJob represents a single attempt to upgrade the DB, it does everything within a transaction +class UpgradeJob: + def __init__(self): + self.connection = get_db() + self.cursor = self.connection.cursor() + pass + + def run_script(self, script: str): + self.cursor.execute(script) + + def cancel_job(self): + self.cursor.close() + self.connection.rollback() + + def finish_job(self): + self.cursor.execute("update migrations SET VersionID = VersionID + 1 WHERE MigrationID = 1;") + self.connection.commit() + self.cursor.close() + + +# Completely destroy the current state of the DB +def down(job: UpgradeJob): + down_script = open("down.sql", "r").read() + job.run_script(down_script) + + +# Recreate the database from the defined schema files +def up(job: UpgradeJob): + up_jobs = [x for x in glob.glob('*.sql') if x != "down.sql"] + up_jobs.sort() + for script in up_jobs: job.run_script(open(script, "r").read()) + + +# requires_update determines if the current database requires an update, it does so by querying an update table and comparing the result +# with the contents of the dbver.txt file +def get_db_versions(): + db = get_db() + + git_version_file = open("dbver.txt", "r") + git_version = int(git_version_file.readlines()[0]) + container_version = 0 + + # acquire the db version + cursor = db.cursor() + try: + cursor.execute("select VersionID from migrations where MigrationID = 0;") + migration_records = cursor.fetchall() + container_version = 0 if len(migration_records) == 0 else migration_records[0] + except: pass + finally: + cursor.close() + + return (container_version, git_version) + + +if __name__ == '__main__': + container_version, git_version = get_db_versions() + + if git_version <= container_version: + print("Container DB is up to date, skipping upgrade :)") + sys.exit() + + # run the upgrade now :D + upgradeJob = UpgradeJob() + try: + down(upgradeJob) + up(upgradeJob) + except Exception as e: + print(f""" + Failed to upgrade the database from version {container_version} to {git_version}, check logs for additional information. + Database is currently on version {container_version}.""") + upgradeJob.cancel_job() + raise e + + upgradeJob.finish_job() + print(f"Successfully updated DB from version {container_version} to {git_version}.") \ No newline at end of file diff --git a/postgres/requirements.txt b/postgres/requirements.txt new file mode 100644 index 00000000..1d773548 --- /dev/null +++ b/postgres/requirements.txt @@ -0,0 +1 @@ +psycopg2-binary==2.9.3 \ No newline at end of file From f871894214d1691cea90a51c59a3344a57f33c6b Mon Sep 17 00:00:00 2001 From: Varun Sethu Date: Mon, 26 Sep 2022 20:29:22 +1000 Subject: [PATCH 2/5] removed pass --- postgres/migrate.py | 1 - 1 file changed, 1 deletion(-) diff --git a/postgres/migrate.py b/postgres/migrate.py index fba4ed85..a3321f8d 100644 --- a/postgres/migrate.py +++ b/postgres/migrate.py @@ -25,7 +25,6 @@ class UpgradeJob: def __init__(self): self.connection = get_db() self.cursor = self.connection.cursor() - pass def run_script(self, script: str): self.cursor.execute(script) From 1900b145ab9687ceffd955cf0b222e25d22c8ddc Mon Sep 17 00:00:00 2001 From: Varun Sethu Date: Mon, 26 Sep 2022 20:46:24 +1000 Subject: [PATCH 3/5] migration script complete + review :D --- docker-compose.yml | 2 +- postgres/dbver.txt | 2 +- postgres/{ => down}/down.sql | 0 postgres/migrate.py | 20 ++++++++++--------- .../{ => up}/01-create_migrations_table.sql | 0 .../{ => up}/02-create_frontend_table.sql | 0 postgres/{ => up}/03-create_groups_table.sql | 0 postgres/{ => up}/04-create_person_table.sql | 0 .../{ => up}/05-create_filesystem_table.sql | 0 postgres/{ => up}/06-create_dummy_data.sql | 0 10 files changed, 13 insertions(+), 11 deletions(-) rename postgres/{ => down}/down.sql (100%) rename postgres/{ => up}/01-create_migrations_table.sql (100%) rename postgres/{ => up}/02-create_frontend_table.sql (100%) rename postgres/{ => up}/03-create_groups_table.sql (100%) rename postgres/{ => up}/04-create_person_table.sql (100%) rename postgres/{ => up}/05-create_filesystem_table.sql (100%) rename postgres/{ => up}/06-create_dummy_data.sql (100%) diff --git a/docker-compose.yml b/docker-compose.yml index 89366d27..eba518cd 100644 --- a/docker-compose.yml +++ b/docker-compose.yml @@ -81,7 +81,7 @@ services: ports: - 1234:5432 volumes: - - './postgres:/docker-entrypoint-initdb.d/' + - './postgres/up:/docker-entrypoint-initdb.d/' - 'staging_pg_db:/var/lib/postgresql/data' volumes: pg_data: diff --git a/postgres/dbver.txt b/postgres/dbver.txt index d8263ee9..56a6051c 100644 --- a/postgres/dbver.txt +++ b/postgres/dbver.txt @@ -1 +1 @@ -2 \ No newline at end of file +1 \ No newline at end of file diff --git a/postgres/down.sql b/postgres/down/down.sql similarity index 100% rename from postgres/down.sql rename to postgres/down/down.sql diff --git a/postgres/migrate.py b/postgres/migrate.py index a3321f8d..038c492f 100644 --- a/postgres/migrate.py +++ b/postgres/migrate.py @@ -41,15 +41,15 @@ def finish_job(self): # Completely destroy the current state of the DB def down(job: UpgradeJob): - down_script = open("down.sql", "r").read() + down_script = open("down/down.sql", "r").read() job.run_script(down_script) # Recreate the database from the defined schema files def up(job: UpgradeJob): - up_jobs = [x for x in glob.glob('*.sql') if x != "down.sql"] - up_jobs.sort() - for script in up_jobs: job.run_script(open(script, "r").read()) + up_jobs = glob.glob('up/*.sql') + for script in sorted(up_jobs): + job.run_script(open(script, "r").read()) # requires_update determines if the current database requires an update, it does so by querying an update table and comparing the result @@ -58,16 +58,18 @@ def get_db_versions(): db = get_db() git_version_file = open("dbver.txt", "r") - git_version = int(git_version_file.readlines()[0]) + git_version = int(git_version_file.readline()) container_version = 0 # acquire the db version cursor = db.cursor() try: - cursor.execute("select VersionID from migrations where MigrationID = 0;") + cursor.execute("select VersionID from migrations where MigrationID = 1;") migration_records = cursor.fetchall() - container_version = 0 if len(migration_records) == 0 else migration_records[0] - except: pass + container_version = 0 if len(migration_records) == 0 else migration_records[0][0] + except Exception as e: + print(e) + pass finally: cursor.close() @@ -75,7 +77,7 @@ def get_db_versions(): if __name__ == '__main__': - container_version, git_version = get_db_versions() + (container_version, git_version) = get_db_versions() if git_version <= container_version: print("Container DB is up to date, skipping upgrade :)") diff --git a/postgres/01-create_migrations_table.sql b/postgres/up/01-create_migrations_table.sql similarity index 100% rename from postgres/01-create_migrations_table.sql rename to postgres/up/01-create_migrations_table.sql diff --git a/postgres/02-create_frontend_table.sql b/postgres/up/02-create_frontend_table.sql similarity index 100% rename from postgres/02-create_frontend_table.sql rename to postgres/up/02-create_frontend_table.sql diff --git a/postgres/03-create_groups_table.sql b/postgres/up/03-create_groups_table.sql similarity index 100% rename from postgres/03-create_groups_table.sql rename to postgres/up/03-create_groups_table.sql diff --git a/postgres/04-create_person_table.sql b/postgres/up/04-create_person_table.sql similarity index 100% rename from postgres/04-create_person_table.sql rename to postgres/up/04-create_person_table.sql diff --git a/postgres/05-create_filesystem_table.sql b/postgres/up/05-create_filesystem_table.sql similarity index 100% rename from postgres/05-create_filesystem_table.sql rename to postgres/up/05-create_filesystem_table.sql diff --git a/postgres/06-create_dummy_data.sql b/postgres/up/06-create_dummy_data.sql similarity index 100% rename from postgres/06-create_dummy_data.sql rename to postgres/up/06-create_dummy_data.sql From a86175895fcefe13dc78a38fb73fda7255356db9 Mon Sep 17 00:00:00 2001 From: Jared L <48422312+lhjt@users.noreply.github.com> Date: Mon, 26 Sep 2022 23:01:41 +1000 Subject: [PATCH 4/5] chore(build): optimise `Dockerfile` --- postgres/Dockerfile | 6 ++++-- 1 file changed, 4 insertions(+), 2 deletions(-) diff --git a/postgres/Dockerfile b/postgres/Dockerfile index 0618ba58..b432f298 100644 --- a/postgres/Dockerfile +++ b/postgres/Dockerfile @@ -1,6 +1,8 @@ -FROM python:latest +FROM python:slim -COPY . . +COPY requirements.txt . RUN pip install -r requirements.txt +COPY . . + ENTRYPOINT [ "python", "migrate.py" ] \ No newline at end of file From 7964e27d78bf33b18c22907fecbe6dde2b20e0a4 Mon Sep 17 00:00:00 2001 From: Jared L <48422312+lhjt@users.noreply.github.com> Date: Mon, 26 Sep 2022 23:04:19 +1000 Subject: [PATCH 5/5] chore(ci): add migration build pipeline --- .github/workflows/docker.yml | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/.github/workflows/docker.yml b/.github/workflows/docker.yml index 08b5dc21..a1d5022a 100644 --- a/.github/workflows/docker.yml +++ b/.github/workflows/docker.yml @@ -9,7 +9,7 @@ jobs: strategy: fail-fast: true matrix: - component: [backend, frontend, next] + component: [backend, frontend, next, postgres] include: - component: backend name: cms-backend @@ -17,6 +17,8 @@ jobs: name: cms-frontend - component: next name: website-frontend + - component: postgres + name: cms-migrations permissions: contents: read packages: write