Automate in place major upgrade for PostgreSQL

.github/workflows/molecule_pg_upgrade.yml

@@ -0,0 +1,81 @@
+---
+name: Molecule pg_upgrade
+
+on:
+  schedule:
+    - cron: "0 0 * * 6"
+
+jobs:
+  test:
+    runs-on: ubuntu-latest
+    strategy:
+      fail-fast: false
+      matrix:
+        config:
+          - distro: debian12
+            tag: latest
+            namespace: geerlingguy
+          - distro: debian11
+            tag: latest
+            namespace: geerlingguy
+          - distro: debian10
+            tag: latest
+            namespace: geerlingguy
+          - distro: ubuntu2204
+            tag: latest
+            namespace: geerlingguy
+          - distro: ubuntu2004
+            tag: latest
+            namespace: geerlingguy
+          - distro: ubuntu1804
+            tag: latest
+            namespace: geerlingguy
+          - distro: rockylinux8
+            tag: latest
+            namespace: geerlingguy
+          - distro: rockylinux9
+            tag: latest
+            namespace: geerlingguy
+          - distro: almalinux8
+            tag: latest
+            namespace: glillico
+          - distro: almalinux9
+            tag: latest
+            namespace: glillico
+          - distro: oraclelinux8
+            tag: latest
+            namespace: glillico
+          - distro: oraclelinux9
+            tag: latest
+            namespace: glillico
+          - distro: centosstream9
+            tag: latest
+            namespace: glillico
+          - distro: centosstream8
+            tag: latest
+            namespace: glillico
+
+    steps:
+      - name: Set TERM environment variable
+        run: echo "TERM=xterm" >> $GITHUB_ENV
+
+      - name: Checkout
+        uses: actions/checkout@v3
+
+      - name: Set up Python 3.10
+        uses: actions/setup-python@v4
+        with:
+          python-version: "3.10"
+
+      - name: Install dependencies
+        run: make bootstrap-dev
+
+      - name: Run Molecule tests for pg_upgrade
+        run: make molecule-test-scenario
+        env:
+          PY_COLORS: "1"
+          ANSIBLE_FORCE_COLOR: "1"
+          IMAGE_DISTRO: ${{ matrix.config.distro }}
+          IMAGE_TAG: ${{ matrix.config.tag }}
+          IMAGE_NAMESPACE: ${{ matrix.config.namespace }}
+          MOLECULE_SCENARIO: "pg_upgrade"

README.md

@@ -45,6 +45,7 @@ In addition to deploying new clusters, this playbook also support the deployment
     - [Point-In-Time-Recovery:](#point-in-time-recovery)
 - [Maintenance](#maintenance)
     - [Update the PostgreSQL HA Cluster](#update-the-postgresql-ha-cluster)
+    - [PostgreSQL major upgrade](#postgresql-major-upgrade)
     - [Using Git for cluster configuration management](#using-git-for-cluster-configuration-management-iacgitops)
 - [Disaster Recovery](#disaster-recovery)
     - [etcd](#etcd)
@@ -490,7 +491,7 @@ I recommend that you study the following materials for further maintenance of th
 - [Patroni documentation](https://patroni.readthedocs.io/en/latest/)
 - [etcd operations guide](https://etcd.io/docs/v3.5/op-guide/)
 
-#### Update the PostgreSQL HA Cluster
+### Update the PostgreSQL HA Cluster
 
 Use the `update_pgcluster.yml` playbook for update the PostgreSQL HA Cluster to a new minor version (for example 15.1->15.2, and etc).
 
@@ -523,7 +524,22 @@ ansible-playbook update_pgcluster.yml -e target=system
 
 More details [here](roles/update/README.md)
 
-#### Using Git for cluster configuration management (IaC/GitOps)
+### PostgreSQL major upgrade
+
+Use the `pg_upgrade.yml` playbook to upgrade the PostgreSQL to a new major version (for example 14->15, and etc).
+
+<details><summary>Upgrade PostgreSQL</summary>
+
+```
+ansible-playbook pg_upgrade.yml -e "pg_old_version=14 pg_new_version=15"
+```
+
+</details>
+
+More details [here](roles/upgrade/README.md)
+
+
+### Using Git for cluster configuration management (IaC/GitOps)
 
 Infrastructure as Code (IaC) is the managing and provisioning of infrastructure through code instead of through manual processes. \
 GitOps automates infrastructure updates using a Git workflow with continuous integration (CI) and continuous delivery (CI/CD). When new code is merged, the CI/CD pipeline enacts the change in the environment. Any configuration drift, such as manual changes or errors, is overwritten by GitOps automation so the environment converges on the desired state defined in Git. 

molecule/default/converge.yml

@@ -15,13 +15,13 @@
         with_haproxy_load_balancing: true
         consul_node_role: server # if dcs_type: "consul"
         consul_bootstrap_expect: true # if dcs_type: "consul"
-        postgresql_version: "15"
+        postgresql_version: "15" # to test custom WAL dir
         cacheable: true
 
     - name: Set variables for custom PostgreSQL data and WAL directory test
       ansible.builtin.set_fact:
-        postgresql_data_dir: "/data/{{ postgresql_version }}/main"
-        postgresql_wal_dir: "/wal/{{ postgresql_version }}/pg_wal"
+        postgresql_data_dir: "/pgdata/{{ postgresql_version }}/main"
+        postgresql_wal_dir: "/pgwal/{{ postgresql_version }}/pg_wal"
 
     - name: Set variables for TimescaleDB cluster deployment test
       ansible.builtin.set_fact:

molecule/pg_upgrade/converge.yml

@@ -0,0 +1,83 @@
+---
+- name: Converge
+  hosts: all
+  gather_facts: true
+
+  tasks:
+    - name: Set variables for Patroni cluster deployment test
+      ansible.builtin.set_fact:
+        firewall_enabled_at_boot: false
+        firewall_enable_ipv6: false # Added to prevent test failures in CI.
+        swap_file_create: false # Added to prevent test failures in CI.
+        sysctl_set: false # Added to prevent test failures in CI.
+        nameservers: ["8.8.8.8", "9.9.9.9"]
+        timezone: "Etc/UTC"
+        with_haproxy_load_balancing: true
+        consul_node_role: server # if dcs_type: "consul"
+        consul_bootstrap_expect: true # if dcs_type: "consul"
+        postgresql_version: "12" # redefine the version to install for the upgrade test
+        cacheable: true
+
+    - name: Set variables for custom PostgreSQL data and WAL directory test
+      ansible.builtin.set_fact:
+        postgresql_data_dir: "/pgdata/{{ postgresql_version }}/main"
+        postgresql_wal_dir: "/pgwal/{{ postgresql_version }}/pg_wal"
+
+    - name: Set variables for TimescaleDB cluster deployment test
+      ansible.builtin.set_fact:
+        enable_timescale: true
+      when:
+        - not (ansible_distribution == 'Ubuntu' and ansible_distribution_version is version('20.04', '<'))
+        - not (ansible_distribution == 'Debian' and ansible_distribution_version is version('11', '>')) # TODO Debian 12
+
+    - name: Set variables for PostgreSQL upgrade test
+      ansible.builtin.set_fact:
+        pg_old_version: "12"
+        pg_new_version: "15"
+
+    - name: Clean yum cache (molecule containers)
+      ansible.builtin.command: yum clean all
+      when:
+        - ansible_os_family == "RedHat"
+        - ansible_distribution_major_version == '7'
+
+    - name: Clean dnf cache (molecule containers)
+      ansible.builtin.command: dnf clean all
+      when:
+        - ansible_os_family == "RedHat"
+        - ansible_distribution_major_version is version('8', '>=')
+
+    - name: Update apt cache (molecule containers)
+      ansible.builtin.apt:
+        update_cache: true
+        cache_valid_time: 3600
+      register: apt_status
+      until: apt_status is success
+      delay: 5
+      retries: 3
+      when: ansible_os_family == "Debian"
+
+    - name: Install openssh-server package (molecule containers)
+      become: true
+      ansible.builtin.package:
+        name: openssh-server
+        state: present
+
+    - name: Start ssh service (molecule containers)
+      become: true
+      ansible.builtin.systemd:
+        name: "{{ 'ssh' if ansible_os_family == 'Debian' else 'sshd' }}"
+        state: started
+        enabled: true
+
+    - name: Delete "/run/nologin" file (if exists)
+      become: true
+      ansible.builtin.file:
+        path: /run/nologin
+        state: absent
+
+- name: Deploy PostgreSQL Cluster
+  ansible.builtin.import_playbook: ../../deploy_pgcluster.yml
+
+- name: PostgreSQL upgrade test
+  ansible.builtin.import_playbook: ../../pg_upgrade.yml

molecule/pg_upgrade/molecule.yml

@@ -0,0 +1,94 @@
+---
+platforms:
+  - name: 10.172.2.20
+    hostname: pgnode01
+    image: "${IMAGE_NAMESPACE:-geerlingguy}/docker-${IMAGE_DISTRO:-ubuntu2204}-ansible:${IMAGE_TAG:-latest}"
+    networks:
+      - name: upgrade_test_docker_network
+        ipv4_address: 10.172.2.20
+    exposed_ports:
+      - 22/tcp
+      - 2379/tcp # if dcs_type: "etcd"
+      - 2380/tcp # if dcs_type: "etcd"
+      - 8300/tcp # if dcs_type: "consul"
+      - 8301/tcp # if dcs_type: "consul"
+      - 8302/tcp # if dcs_type: "consul"
+      - 8500/tcp # if dcs_type: "consul"
+      - 8600/tcp # if dcs_type: "consul"
+      - 8008/tcp
+      - 5432/tcp
+      - 6432/tcp
+    command: ""
+    volumes:
+      - /sys/fs/cgroup:/sys/fs/cgroup:rw
+    cgroupns_mode: host
+    privileged: true
+    pre_build_image: true
+    groups:
+      - etcd_cluster # if dcs_type: "etcd"
+      - consul_instances # if dcs_type: "consul"
+      - master
+      - postgres_cluster
+      - balancers
+
+  - name: 10.172.2.21
+    hostname: pgnode02
+    image: "${IMAGE_NAMESPACE:-geerlingguy}/docker-${IMAGE_DISTRO:-ubuntu2204}-ansible:${IMAGE_TAG:-latest}"
+    networks:
+      - name: upgrade_test_docker_network
+        ipv4_address: 10.172.2.21
+    exposed_ports:
+      - 22/tcp
+      - 2379/tcp
+      - 2380/tcp
+      - 8300/tcp
+      - 8301/tcp
+      - 8302/tcp
+      - 8500/tcp
+      - 8600/tcp
+      - 8008/tcp
+      - 5432/tcp
+      - 6432/tcp
+    command: ""
+    volumes:
+      - /sys/fs/cgroup:/sys/fs/cgroup:rw
+    cgroupns_mode: host
+    privileged: true
+    pre_build_image: true
+    groups:
+      - etcd_cluster
+      - consul_instances
+      - replica
+      - postgres_cluster
+      - balancers
+
+  - name: 10.172.2.22
+    hostname: pgnode03
+    image: "${IMAGE_NAMESPACE:-geerlingguy}/docker-${IMAGE_DISTRO:-ubuntu2204}-ansible:${IMAGE_TAG:-latest}"
+    networks:
+      - name: upgrade_test_docker_network
+        ipv4_address: 10.172.2.22
+    exposed_ports:
+      - 22/tcp
+      - 2379/tcp
+      - 2380/tcp
+      - 8300/tcp
+      - 8301/tcp
+      - 8302/tcp
+      - 8500/tcp
+      - 8600/tcp
+      - 8008/tcp
+      - 5432/tcp
+      - 6432/tcp
+    command: ""
+    volumes:
+      - /sys/fs/cgroup:/sys/fs/cgroup:rw
+    cgroupns_mode: host
+    privileged: true
+    pre_build_image: true
+    groups:
+      - etcd_cluster
+      - consul_instances
+      - replica
+      - postgres_cluster
+      - balancers

molecule/pg_upgrade/prepare.yml

@@ -0,0 +1,28 @@
+---
+- name: "Update docker network(s)"
+  hosts: localhost
+  gather_facts: false
+  become: false
+  tasks:
+    - name: "Create docker network: upgrade_test_docker_network"
+      community.docker.docker_network:
+        name: upgrade_test_docker_network
+        driver: bridge
+        driver_options:
+          com.docker.network.driver.mtu: 1440
+        enable_ipv6: false
+        internal: false
+        ipam_config:
+          - subnet: 10.172.2.0/24
+            gateway: 10.172.2.1
+        force: true
+        state: present
+        labels:
+          owner: molecule
+
+    - name: "Install netaddr dependency on controlling host"
+      ansible.builtin.pip:
+        name: netaddr
+      become: false
+
+...

molecule/tests/etcd/etcd.yml

@@ -1,6 +1,6 @@
 ---
 - name: Check etcd health
-  uri:
+  ansible.builtin.uri:
     url: "http://{{ inventory_hostname }}:2379/health"
     return_content: true
   register: etcd_health_status

molecule/tests/patroni/patroni.yml

@@ -1,6 +1,6 @@
 ---
 - name: Check Patroni status
-  uri:
+  ansible.builtin.uri:
     url: "http://{{ inventory_hostname }}:8008/patroni"
     return_content: true
   register: patroni_status

molecule/tests/postgres/postgres.yml

@@ -5,14 +5,16 @@
   failed_when: result.rc != 0
 
 - name: Check if PostgreSQL is listening on the default port
-  wait_for:
+  ansible.builtin.wait_for:
     port: 5432
     timeout: 5
   register: is_listening
   failed_when: not is_listening
 
 - name: Try to connect to PostgreSQL
   postgresql_ping:
+    login_host: "127.0.0.1"
+    login_port: "{{ postgresql_port }}"
     login_user: "{{ patroni_superuser_username }}"
     login_password: "{{ patroni_superuser_password }}"
-    db: template1
+    login_db: template1

molecule/tests/postgres/replication.yml

@@ -1,10 +1,12 @@
 ---
 - name: Check PostgreSQL replication status
   postgresql_query:
-    db: postgres
+    query: "SELECT * FROM pg_stat_wal_receiver;"
+    login_host: "127.0.0.1"
+    login_port: "{{ postgresql_port }}"
     login_user: "{{ patroni_superuser_username }}"
     login_password: "{{ patroni_superuser_password }}"
-    query: "SELECT * FROM pg_stat_wal_receiver;"
+    login_db: template1
   register: pg_replication_status
   failed_when: "pg_replication_status.rowcount == 0"
   when: "'replica' in group_names"