chore: Add Migration scripts along with extensions test (supabase#365)

After building AMI, Perform extension install test. If tests
passed, push migrations files to S3. Also create migration-output.sql
file.

Signed-off-by: Lakshmipathi <lakshmi@supabase.io>

Author: Lakshmipathi

.github/workflows/ami-release.yml

@@ -17,3 +17,17 @@ jobs:
       - name: Build AMI
         run: |
           packer build -var-file="development-arm.vars.pkr.hcl" -var-file="common.vars.pkr.hcl" amazon-arm64.pkr.hcl
+
+      - name: Merging migration files
+        run: cat $(ls -1) > ../migration-output.sql
+        working-directory: ${{ github.workspace }}/migrations/db/migrations
+
+      - name: Push migration files to S3
+        uses: jakejarvis/s3-sync-action@master
+        env:
+          AWS_S3_BUCKET: ${{ secrets.PG_INIT_SCRIPT_S3_BUCKET_STAGING }}
+          AWS_ACCESS_KEY_ID: ${{ secrets.PG_INIT_SCRIPT_ACCESS_KEY_ID_STAGING }}
+          AWS_SECRET_ACCESS_KEY: ${{ secrets.PG_INIT_SCRIPT_SECRET_ACCESS_KEY_STAGING }}
+          AWS_REGION: ap-southeast-1
+          SOURCE_DIR: migrations/db
+          DEST_DIR: migrations/db

ansible/playbook.yml

@@ -153,10 +153,15 @@
       shell:
         cmd: "for fl in /usr/lib/postgresql/bin/* ; do ln -sf $fl /usr/bin/$(basename $fl) ; done"
 
+    - name: Run migrations
+      import_tasks: tasks/setup-migrations.yml
+
     - name: Stop Postgres Database without Systemd
       become: yes
       become_user: postgres
       shell:
         cmd: /usr/bin/pg_ctl -D /var/lib/postgresql/data stop
       when: ebssurrogate_mode
 
+    - name: Run unit tests
+      import_tasks: tasks/test-image.yml

ansible/tasks/postgres-extensions/12-pljava.yml

@@ -50,6 +50,11 @@
     update_cache: yes
     install_recommends: no
 
+- name: Hold jre package 
+  dpkg_selections:
+    name: default-jre-headless
+    selection: hold
+
 - name: pljava - set pljava.libjvm_location
   become: yes
   lineinfile:

ansible/tasks/setup-extensions.yml

@@ -67,11 +67,10 @@
 # - name: Install vault
 #   import_tasks: tasks/postgres-extensions/23-vault.yml
 
-#- name: Install PGroonga
-#  import_tasks: tasks/postgres-extensions/24-pgroonga.yml
+- name: Install PGroonga
+  import_tasks: tasks/postgres-extensions/24-pgroonga.yml
 
 - name: Verify async task status
   import_tasks: tasks/postgres-extensions/99-finish_async_tasks.yml
   when: async_mode
 
-

ansible/tasks/setup-migrations.yml

@@ -5,3 +5,9 @@
   args:
     chdir: /tmp/migrations/db
   failed_when: retval.rc != 0
+
+- name: Create /root/MIGRATION-AMI file
+  file:
+    path: "/root/MIGRATION-AMI"
+    state: touch
+  when: ebssurrogate_mode

ansible/tasks/test-image.yml

@@ -21,12 +21,29 @@
   shell: /usr/lib/postgresql/bin/psql -U postgres -h localhost -d postgres -c "CREATE extension pgtap";
   when: ebssurrogate_mode
 
+- name: Create function for testing extensions
+  shell: /usr/lib/postgresql/bin/psql -U postgres -h localhost -d postgres -f /tmp/unit-tests/test-extensions.sql;
+  when: ebssurrogate_mode
+
+- name: Extension Installation tests
+  shell: /usr/bin/pg_prove -U supabase_admin -h localhost -d postgres -v /tmp/unit-tests/verify-extensions.sql
+  register: retval
+  when: ebssurrogate_mode
+
 - name: Run Unit tests (with filename unit-test-*) on Postgres Database 
   shell: /usr/bin/pg_prove -U postgres -h localhost -d postgres -v /tmp/unit-tests/unit-test-*.sql
   register: retval
   failed_when: retval.rc != 0
   when: ebssurrogate_mode
 
+- name: Run migrations tests
+  shell: /usr/bin/pg_prove -U postgres -h localhost -d postgres -v tests/test.sql
+  register: retval
+  failed_when: retval.rc != 0
+  when: ebssurrogate_mode
+  args:
+    chdir: /tmp/migrations
+
 - name: Re-enable PG Sodium references in config
   become: yes
   become_user: postgres

ebssurrogate/files/apparmor_profiles/usr.lib.postgresql.bin.postgres

@@ -9,25 +9,35 @@ profile /usr/lib/postgresql/bin/postgres flags=(attach_disconnected) {
 #include <abstractions/ssl_keys>
 #include <abstractions/user-tmp>
 
-/dev/shm rw,
+capability dac_override,
+capability dac_read_search,
+
 deny @{HOME}/** rwx,
+
+/data/pgdata/** r,
+/dev/shm rw,
 /etc/postgresql-custom/** r,
 /etc/postgresql/** r,
+/etc/wal-g/config.json r,
 /run/systemd/notify rw,
-/var/run/systemd/notify rw,
 /usr/bin/cat rix,
 /usr/bin/dash rix,
 /usr/bin/mknod rix,
 /usr/lib/postgresql/bin/* mrix,
-/usr/local/lib/libgroonga.so.* mr,
+/usr/local/bin/wal-g rix,
 /usr/local/lib/libSFCGAL.so.* r,
+/usr/local/lib/libgroonga.so.* mr,
 /usr/share/postgresql/** r,
 /var/lib/postgresql/** rwl,
 /var/log/postgresql/** rw,
+/var/log/wal-g/** w,
+/var/run/systemd/notify rw,
 /{,var/}run/postgresql/** rw,
 owner /data/pgdata/** rwl,
 owner /data/pgdata/pgroonga.log k,
 owner /dev/shm/ rw,
 owner /dev/shm/PostgreSQL.* rw,
+owner /var/log/wal-g/** rw,
 owner @{PROC}/[0-9]*/oom_adj rw,
+
 }

ebssurrogate/files/unit-tests/test-extensions.sql

@@ -1,15 +1,20 @@
 CREATE OR REPLACE FUNCTION install_available_extensions_and_test() RETURNS boolean AS $$
 DECLARE extension_name TEXT;
-name TEXT;
 allowed_extentions TEXT[] := string_to_array(current_setting('supautils.privileged_extensions'), ',');
 BEGIN 
   FOREACH extension_name IN ARRAY allowed_extentions 
   LOOP
-    RAISE notice '%', extension_name;
     SELECT trim(extension_name) INTO extension_name;
+    /* skip below extensions check for now */
+    CONTINUE WHEN extension_name = 'pgroonga' OR  extension_name = 'pgroonga_database' OR extension_name = 'pgsodium';
+    CONTINUE WHEN extension_name = 'plpgsql' OR  extension_name = 'plpgsql_check' OR extension_name = 'pgtap';
+    CONTINUE WHEN extension_name = 'supabase_vault';
+    RAISE notice 'START TEST FOR: %', extension_name;
     EXECUTE format('DROP EXTENSION IF EXISTS %s CASCADE', quote_ident(extension_name));
     EXECUTE format('CREATE EXTENSION %s CASCADE', quote_ident(extension_name));
+    RAISE notice 'END TEST FOR: %', extension_name;
   END LOOP;
-return true;
+    RAISE notice 'EXTENSION TESTS COMPLETED..';
+    return true;
 END;
 $$ LANGUAGE plpgsql;

ebssurrogate/files/unit-tests/unit-test-01.sql

@@ -1,5 +1,5 @@
 BEGIN;
-SELECT plan( 7 );
+SELECT plan( 5 );
 
 -- Check installed extensions
 SELECT extensions_are(
@@ -20,8 +20,6 @@ SELECT extensions_are(
 SELECT has_schema('pg_toast');
 SELECT has_schema('pg_catalog');
 SELECT has_schema('information_schema');
-SELECT has_schema('pgsodium');
-SELECT has_schema('pgsodium_masks');
 SELECT has_schema('public');
 
 SELECT * from finish();

scripts/90-cleanup.sh

@@ -20,7 +20,6 @@ if [ -n "$(command -v yum)" ]; then
 elif [ -n "$(command -v apt-get)" ]; then
   # Cleanup more packages
   apt-get -y remove --purge \
-	libgl1-mesa-dri \
 	automake \
  	autoconf \
 	autotools-dev \
@@ -34,7 +33,6 @@ elif [ -n "$(command -v apt-get)" ]; then
 	git  \
 	git-man  \
 	ansible \
-	libasound2 \
 	libicu-dev \
 	libcgal-dev \
 	libgcc-9-dev \
@@ -45,7 +43,6 @@ elif [ -n "$(command -v apt-get)" ]; then
   apt-get -y autoremove
   apt-get -y autoclean
 fi
-
 rm -rf /tmp/* /var/tmp/*
 history -c
 cat /dev/null > /root/.bash_history