Fix build and speed it up.

Addresses current breakage by modifying perms on the
/etc/postgresql-common dir to allow adminapi to write to it.

Additionally, speeds up the build by applying a few fixes:

- using parallelism where applicable (make/maven parallelism)
- not building postgres docs
- using our own s3 bucket for postgis sources (origin is extremely
  slow)
- using a 16 core machine instead of 8 cores: plv8 is the slowest task
  right now and is mostly CPU constrained on v8 compilation
- links postgres binaries using a shell loop rather than
  ansible (saves a minute)
- switch to use1 (closer to a lot of external resources used)
- enable ssh pipelining

Old timing info (incomplete as the build was broken, but based on GH
actions it completed it in ~1hr 40mins; 20mins of that was likely
post-ansible AMI copying etc):

2022-03-21T17:05:46Z:     amazon-ebs: Monday 21 March 2022  17:05:46 +0000 (0:00:02.675)       1:15:00.775 **********
2022-03-21T17:05:46Z:     amazon-ebs: ===============================================================================
2022-03-21T17:05:46Z:     amazon-ebs: postgis - build SFCGAL ----------------------------------------------- 1101.32s
2022-03-21T17:05:46Z:     amazon-ebs: plv8 - build ---------------------------------------------------------- 801.52s
2022-03-21T17:05:46Z:     amazon-ebs: Postgres - build ------------------------------------------------------ 419.36s
2022-03-21T17:05:46Z:     amazon-ebs: pgRouting - build ----------------------------------------------------- 364.44s
2022-03-21T17:05:46Z:     amazon-ebs: pljava - build -------------------------------------------------------- 319.36s
2022-03-21T17:05:46Z:     amazon-ebs: postgis - download latest release ------------------------------------- 219.28s
2022-03-21T17:05:46Z:     amazon-ebs: System - apt update and apt upgrade ----------------------------------- 138.52s
2022-03-21T17:05:46Z:     amazon-ebs: postgis - build ------------------------------------------------------- 126.25s
2022-03-21T17:05:46Z:     amazon-ebs: pljava - download & install dependencies ------------------------------- 63.43s
2022-03-21T17:05:46Z:     amazon-ebs: libsodium - build ------------------------------------------------------ 61.62s
2022-03-21T17:05:46Z:     amazon-ebs: postgis - download & install dependencies ------------------------------ 61.46s
2022-03-21T17:05:46Z:     amazon-ebs: Install other useful tools --------------------------------------------- 30.84s
2022-03-21T17:05:46Z:     amazon-ebs: Download LLVM & Clang -------------------------------------------------- 26.68s
2022-03-21T17:05:46Z:     amazon-ebs: Postgres dependencies -------------------------------------------------- 18.55s
2022-03-21T17:05:46Z:     amazon-ebs: Postgres - configure --------------------------------------------------- 18.29s
2022-03-21T17:05:46Z:     amazon-ebs: plv8 - download & install dependencies --------------------------------- 17.32s
2022-03-21T17:05:46Z:     amazon-ebs: Download GCC 10 -------------------------------------------------------- 16.87s
2022-03-21T17:05:46Z:     amazon-ebs: pgAudit - download & install dependencies ------------------------------ 16.45s
2022-03-21T17:05:46Z:     amazon-ebs: install Vector for logging --------------------------------------------- 11.14s
2022-03-21T17:05:46Z:     amazon-ebs: libsodium - configure -------------------------------------------------- 10.69s

New timing info:

2022-03-22T09:36:08Z:     amazon-ebs: Tuesday 22 March 2022  09:36:08 +0000 (0:00:01.587)       0:36:11.036 *********
2022-03-22T09:36:08Z:     amazon-ebs: ===============================================================================
2022-03-22T09:36:08Z:     amazon-ebs: plv8 - build ---------------------------------------------------------- 513.34s
2022-03-22T09:36:08Z:     amazon-ebs: Postgres - build ------------------------------------------------------ 355.08s
2022-03-22T09:36:08Z:     amazon-ebs: wal-g - additional go dependencies ------------------------------------ 118.28s
2022-03-22T09:36:08Z:     amazon-ebs: postgis - build SFCGAL ------------------------------------------------ 101.59s
2022-03-22T09:36:08Z:     amazon-ebs: System - apt update and apt upgrade ------------------------------------ 87.44s
2022-03-22T09:36:08Z:     amazon-ebs: postgis - download & install dependencies ------------------------------ 58.85s
2022-03-22T09:36:08Z:     amazon-ebs: libsodium - build ------------------------------------------------------ 52.69s
2022-03-22T09:36:08Z:     amazon-ebs: wal-g - build ---------------------------------------------------------- 41.63s
2022-03-22T09:36:08Z:     amazon-ebs: pljava - download & install dependencies ------------------------------- 35.37s
2022-03-22T09:36:08Z:     amazon-ebs: pgRouting - build ------------------------------------------------------ 25.96s
2022-03-22T09:36:08Z:     amazon-ebs: Install other useful tools --------------------------------------------- 22.59s
2022-03-22T09:36:08Z:     amazon-ebs: pljava - build --------------------------------------------------------- 21.98s
2022-03-22T09:36:08Z:     amazon-ebs: Download LLVM & Clang -------------------------------------------------- 20.20s
2022-03-22T09:36:08Z:     amazon-ebs: postgis - build -------------------------------------------------------- 19.40s
2022-03-22T09:36:08Z:     amazon-ebs: Remove build dependencies ---------------------------------------------- 16.02s
2022-03-22T09:36:08Z:     amazon-ebs: ensure services are stopped -------------------------------------------- 15.34s
2022-03-22T09:36:08Z:     amazon-ebs: Postgres dependencies -------------------------------------------------- 14.54s
2022-03-22T09:36:08Z:     amazon-ebs: Postgres - configure --------------------------------------------------- 14.18s
2022-03-22T09:36:08Z:     amazon-ebs: PgBouncer - build ------------------------------------------------------ 13.92s
2022-03-22T09:36:08Z:     amazon-ebs: ensure services are stopped and disabled for first boot ---------------- 12.86s

Author: darora

ansible.cfg

@@ -0,0 +1,3 @@
+[defaults]
+
+callback_whitelist = profile_tasks

ansible/playbook.yml

@@ -157,9 +157,5 @@
 
     - name: Create symbolic links for Postgres binaries to /usr/bin/
       become: yes
-      file:
-        src: "{{ item.path }}"
-        path: "/usr/bin/{{ item.path | basename }}"
-        state: link
-        force: yes
-      with_items: "{{ postgresql_bin.files }}"
+      shell:
+        cmd: "for fl in /usr/lib/postgresql/bin/* ; do ln -sf $fl /usr/bin/$(basename $fl) ; done"

ansible/tasks/internal/admin-api.yml

@@ -1,7 +1,7 @@
 - name: adminapi - system user
   user:
     name: adminapi
-    groups: root,admin
+    groups: root,admin,postgres
     append: yes
 
 - name: disk script

ansible/tasks/postgres-extensions/01-postgis.yml

@@ -49,6 +49,7 @@
 - name: postgis - build SFCGAL
   make:
     chdir: /tmp/SFCGAL-v{{ sfcgal_release }}
+    jobs: 16
   become: yes
 
 - name: postgis - install SFCGAL
@@ -58,11 +59,8 @@
   become: yes
 
 - name: postgis - download latest release
-  get_url:
-    url: "https://download.osgeo.org/postgis/source/postgis-{{ postgis_release }}.tar.gz"
-    dest: /tmp/postgis-{{ postgis_release }}.tar.gz
-    checksum: "{{ postgis_release_checksum }}"
-    timeout: 60
+  shell:
+    cmd: "curl -sf -L https://supabase-public-artifacts-bucket.s3.amazonaws.com/postgis-{{ postgis_release }}.tar.gz -o /tmp/postgis-{{ postgis_release }}.tar.gz"
 
 - name: postgis - unpack archive
   unarchive:
@@ -80,10 +78,11 @@
 - name: postgis - build
   make:
     chdir: /tmp/postgis-{{ postgis_release }}
+    jobs: 16
   become: yes
 
 - name: postgis - install
   make:
     chdir: /tmp/postgis-{{ postgis_release }}
     target: install
-  become: yes
+  become: yes

ansible/tasks/postgres-extensions/02-pgrouting.yml

@@ -36,10 +36,11 @@
 - name: pgRouting - build
   make:
     chdir: /tmp/pgrouting-{{ pgrouting_release }}/build
+    jobs: 16
   become: yes
 
 - name: pgRouting - install
   make:
     chdir: /tmp/pgrouting-{{ pgrouting_release }}/build
     target: install
-  become: yes
+  become: yes

ansible/tasks/postgres-extensions/12-pljava.yml

@@ -26,7 +26,7 @@
 - name: pljava - build
   become: yes
   shell:
-    cmd: mvn clean install
+    cmd: mvn -T 1C clean install -Dmaven.test.skip -DskipTests -Dmaven.javadoc.skip=true
     chdir: /tmp/pljava-{{ pljava_release }}
 
 - name: pljava - install
@@ -61,4 +61,4 @@
   become: yes
   file:
     path: ~/.m2
-    state: absent
+    state: absent

ansible/tasks/postgres-extensions/13-plv8.yml

@@ -46,4 +46,4 @@
   make:
     chdir: /tmp/plv8
     target: install
-  become: yes
+  become: yes

ansible/tasks/setup-postgres.yml

@@ -63,12 +63,12 @@
 
 - name: Postgres - build
   make:
-    target: world
+    target: world-bin
     chdir: /tmp/postgresql-{{ postgresql_release }}
 
 - name: Postgres - install
   make:
-    target: install-world
+    target: install-world-bin
     chdir: /tmp/postgresql-{{ postgresql_release }}
 
 # Create postgres user
@@ -99,7 +99,7 @@
     state: directory
     owner: postgres
     group: postgres
-    mode: 0664
+    mode: 0775
   with_items:
     - '/etc/postgresql-custom'
 
@@ -109,6 +109,7 @@
     state: touch
     owner: postgres
     group: postgres
+    mode: 0664
   with_items:
     - 'generated-optimizations.conf'
     - 'custom-overrides.conf'

ansible/tasks/setup-wal-g.yml

@@ -44,6 +44,7 @@
   make:
     chdir: /tmp/wal-g
     target: pg_build
+    jobs: 16
     params:
       GOBIN: "/usr/local/bin"
       PATH: "{{ ansible_env.PATH }}:/usr/local/go/bin"

ansible/vars.yml

@@ -34,10 +34,10 @@ node_exporter_release_checksum:
   arm64: sha256:eb5e7d16f18bb3272d0d832986fc8ac6cb0b6c42d487c94e15dabb10feae8e04
   amd64: sha256:8c1f6a317457a658e0ae68ad710f6b4098db2cad10204649b51e3c043aa3e70d
 
-adminapi_release: 0.16.1
+adminapi_release: 0.16.2
 adminapi_release_checksum:
-  arm64: sha256:96ad2a67063882c77ff63b73a2bf82e9af1bae837a33070484593eaca28b05b5
-  amd64: sha256:5e82ac02ef43b9c962f840a0fb5e93b11969ab58e11b087afd8685d25d4c8cdb
+  arm64: sha256:94fc9d916ce7b54c20e4cb9e0898a2aed205436ee604663636f38a40183163f3
+  amd64: sha256:134be79a98416e8fcefe2bebf85c6fad56d23c0f99252c8566889c879b9651b5
 
 # Postgres Extensions
 postgis_release: "3.1.4"