Pmm 14517

pmm_qa/percona-distribution-postgresql/data/etcd.conf.yaml.j2

@@ -4,9 +4,9 @@ initial-cluster-token: PostgreSQL_HA_Cluster_1
 initial-cluster-state: new
 
 listen-peer-urls: http://0.0.0.0:2380
-initial-advertise-peer-urls: http://pdpgsql_pmm_patroni_{{ pg_version }}_{{ item }}:2380
+initial-advertise-peer-urls: http://pdpgsql_pmm_patroni_{{ pdpgsql_version }}_{{ item }}:2380
 
 listen-client-urls: http://0.0.0.0:2379
-advertise-client-urls: http://pdpgsql_pmm_patroni_{{ pg_version }}_{{ item }}:2379
+advertise-client-urls: http://pdpgsql_pmm_patroni_{{ pdpgsql_version }}_{{ item }}:2379
 
-initial-cluster: node1=http://pdpgsql_pmm_patroni_{{ pg_version }}_1:2380,node2=http://pdpgsql_pmm_patroni_{{ pg_version }}_2:2380,node3=http://pdpgsql_pmm_patroni_{{ pg_version }}_3:2380
+initial-cluster: node1=http://pdpgsql_pmm_patroni_{{ pdpgsql_version }}_1:2380,node2=http://pdpgsql_pmm_patroni_{{ pdpgsql_version }}_2:2380,node3=http://pdpgsql_pmm_patroni_{{ pdpgsql_version }}_3:2380

pmm_qa/percona-distribution-postgresql/data/patroni.yml.j2

@@ -4,10 +4,10 @@ name: pg_node_{{ item }}
 
 restapi:
   listen: 0.0.0.0:8008
-  connect_address: "pdpgsql_pmm_patroni_{{ pg_version }}_{{ item }}:8008"
+  connect_address: "pdpgsql_pmm_patroni_{{ pdpgsql_version }}_{{ item }}:8008"
 
 etcd3:
-  host: "pdpgsql_pmm_patroni_{{ pg_version }}_1:2379"
+  host: "pdpgsql_pmm_patroni_{{ pdpgsql_version }}_1:2379"
 
 bootstrap:
   dcs:
@@ -51,9 +51,9 @@ bootstrap:
 
 postgresql:
   listen: 0.0.0.0:5432
-  connect_address: "pdpgsql_pmm_patroni_{{ pg_version }}_{{ item }}:5432"
-  data_dir: /var/lib/pgsql/{{ pg_version}}/data
-  bin_dir: /usr/pgsql-{{ pg_version }}/bin
+  connect_address: "pdpgsql_pmm_patroni_{{ pdpgsql_version }}_{{ item }}:5432"
+  data_dir: /var/lib/postgresql/{{ pdpgsql_version }}/main
+  bin_dir: /usr/lib/postgresql/{{ pdpgsql_version }}/bin
   pgpass: /tmp/pgpass
 
   authentication:

pmm_qa/percona-distribution-postgresql/data/pgsql_load.sql

@@ -0,0 +1,115 @@
+-- ========================================
+-- CREATE TABLES
+-- ========================================
+
+CREATE TABLE students (
+    student_id SERIAL PRIMARY KEY,
+    first_name VARCHAR(50),
+    last_name VARCHAR(50),
+    birth_date DATE
+);
+
+CREATE TABLE classes (
+    class_id SERIAL PRIMARY KEY,
+    name VARCHAR(100),
+    teacher VARCHAR(100)
+);
+
+CREATE TABLE enrollments (
+    enrollment_id SERIAL PRIMARY KEY,
+    student_id INTEGER REFERENCES students(student_id),
+    class_id INTEGER REFERENCES classes(class_id),
+    enrollment_date DATE DEFAULT CURRENT_DATE
+);
+
+-- ========================================
+-- INSERT MOCK DATA
+-- ========================================
+
+INSERT INTO students (first_name, last_name, birth_date) VALUES
+('Alice', 'Smith', '2005-04-10'),
+('Bob', 'Johnson', '2006-08-15'),
+('Charlie', 'Brown', '2004-12-01');
+
+INSERT INTO classes (name, teacher) VALUES
+('Mathematics', 'Mrs. Taylor'),
+('History', 'Mr. Anderson'),
+('Science', 'Dr. Reynolds');
+
+INSERT INTO enrollments (student_id, class_id) VALUES
+(1, 1),
+(1, 2),
+(2, 2),
+(3, 1),
+(3, 3);
+
+-- ========================================
+-- SIMULATE DEAD TUPLES
+-- ========================================
+
+
+INSERT INTO students (first_name, last_name, birth_date)
+SELECT 'John', 'Doe', CURRENT_DATE - (random() * 5000)::int
+FROM generate_series(1, 100000);
+
+-- These updates and deletes will create dead tuples
+
+-- Update records (old versions become dead)
+UPDATE students
+SET last_name = last_name || '_updated'
+WHERE student_id IN (1, 2);
+
+-- Delete records (deleted rows become dead)
+DELETE FROM enrollments
+WHERE enrollment_id IN (SELECT enrollment_id FROM enrollments LIMIT 2);
+
+-- Disable autovacuum temporarily (for demo)
+ALTER TABLE students SET (autovacuum_enabled = false);
+ALTER TABLE enrollments SET (autovacuum_enabled = false);
+
+-- ========================================
+-- SELECT QUERIES
+-- ========================================
+
+-- Get all students
+SELECT * FROM students;
+
+-- Get all students enrolled in Mathematics
+SELECT s.first_name, s.last_name
+FROM students s
+JOIN enrollments e ON s.student_id = e.student_id
+JOIN classes c ON e.class_id = c.class_id
+WHERE c.name = 'Mathematics';
+
+-- Count students per class
+SELECT c.name, COUNT(e.student_id) AS student_count
+FROM classes c
+LEFT JOIN enrollments e ON c.class_id = e.class_id
+GROUP BY c.name;
+
+-- ========================================
+-- UPDATE QUERIES
+-- ========================================
+
+-- Change Bob's last name
+UPDATE students
+SET last_name = 'Williams'
+WHERE first_name = 'Bob' AND last_name = 'Johnson';
+
+-- Update the teacher for the History class
+UPDATE classes
+SET teacher = 'Ms. Carter'
+WHERE name = 'History';
+
+-- ========================================
+-- DELETE QUERIES
+-- ========================================
+
+-- Remove Charlie from Science class
+DELETE FROM enrollments
+WHERE student_id = (SELECT student_id FROM students WHERE first_name = 'Charlie')
+  AND class_id = (SELECT class_id FROM classes WHERE name = 'Science');
+
+-- Delete a student completely
+DELETE FROM students
+WHERE first_name = 'Alice' AND last_name = 'Smith';

pmm_qa/percona-distribution-postgresql/data/postgresql-primary.conf → pmm_qa/percona-distribution-postgresql/data/postgresql-primary.conf.j2

@@ -3,7 +3,8 @@ max_wal_senders = 10
 wal_keep_size = 64MB
 hot_standby = on
 listen_addresses = '*'
-hba_file = '/etc/postgresql/pg_hba.conf'
+hba_file = '/etc/postgresql/{{ pdpgsql_version }}/main/pg_hba.conf'
+data_directory = '/var/lib/postgresql/{{ pdpgsql_version }}/main'
 shared_preload_libraries = 'pg_stat_monitor'
 track_activity_query_size=2048
 track_io_timing=ON

pmm_qa/percona-distribution-postgresql/data/postgresql-replica.conf.j2

@@ -0,0 +1,17 @@
+hba_file = '/etc/postgresql/{{ pdpgsql_version }}/main/pg_hba.conf'
+data_directory = '/var/lib/postgresql/{{ pdpgsql_version }}/main'
+shared_preload_libraries = 'pg_stat_monitor'
+track_activity_query_size=2048
+track_io_timing=ON
+max_connections=1000
+listen_addresses = '*'
+pg_stat_monitor.pgsm_enable_query_plan = 'yes'
+pg_stat_monitor.pgsm_query_max_len = 2048
+pg_stat_monitor.pgsm_normalized_query=1
+pg_stat_monitor.pgsm_enable_query_plan=1
+log_connections = on
+log_disconnections = on
+log_replication_commands = on
+log_statement = 'all'
+primary_slot_name = 'replica{{ item }}'
+primary_conninfo = 'host={{ container_prefix }}1 user={{ replication_user }} password={{ replication_password }}'

pmm_qa/percona-distribution-postgresql/data/postgres-replica.conf → pmm_qa/percona-distribution-postgresql/data/postgresql-single.conf.j2

@@ -1,13 +1,11 @@
-hba_file = '/etc/postgresql/pg_hba.conf'
+listen_addresses = '*'
+hba_file = '/etc/postgresql/{{ pdpgsql_version }}/main/pg_hba.conf'
+data_directory = '/var/lib/postgresql/{{ pdpgsql_version }}/main'
 shared_preload_libraries = 'pg_stat_monitor'
 track_activity_query_size=2048
 track_io_timing=ON
 max_connections=1000
-listen_addresses = '*'
 pg_stat_monitor.pgsm_enable_query_plan = 'yes'
-pg_stat_monitor.pgsm_query_max_len = 2048
-pg_stat_monitor.pgsm_normalized_query=1
-pg_stat_monitor.pgsm_enable_query_plan=1
 log_connections = on
 log_disconnections = on
 log_replication_commands = on

pmm_qa/percona-distribution-postgresql/percona-distribution-postgres-setup.yml

@@ -5,12 +5,12 @@
   connection: local
   gather_facts: yes
   vars:
-    pg_version: "{{ lookup('env', 'PGSQL_VERSION') | default('17', true) }}"
+    pdpgsql_version: "{{ lookup('env', 'PDPGSQL_VERSION') | default('17', true) }}"
     replication_user: "repl_user"
     replication_password: "GRgrO9301RuF"
     root_password: "GRgrO9301RuF"
     pdpgsql_port: 5432
-    nodes_count: "{{ (lookup('env', 'NODES_COUNT') | default('2', true)) | int }}"
+    nodes_count: "{{ (lookup('env', 'NODES_COUNT') | default('1', true)) | int }}"
     network_name: "pmm-qa"
     data_dir: "{{ lookup('env', 'HOME') }}/pgsql_cluster_data"
     pmm_server_ip: "{{ lookup('vars', 'extra_pmm_server_ip', default=lookup('env','PMM_SERVER_IP') | default('127.0.0.1', true) ) }}"
@@ -21,12 +21,21 @@
     setup_type: "{{ lookup('env', 'SETUP_TYPE') }}"
     random_service_name_value: ""
     docker_repo: "percona/percona-distribution-postgresql"
+    container_prefix: "pdpgsql_pmm{{ (setup_type|default('')) and '_' ~ setup_type }}_{{ pdpgsql_version }}_"
 
   tasks:
-    - name: Display binary log status for primary
+    - name: Display setup type selected
       debug:
         msg: "{{ setup_type }}"
 
+    - name: Create Docker network
+      shell: docker network create {{ network_name }}
+      ignore_errors: yes
+
+    - name: Install Percona Distribution for Postgres
+      include_tasks: tasks/install-percona-distribution-postgresql-single.yml
+      when: setup_type != "replication" and setup_type != "patroni"
+
     - name: Install Percona Distribution for Postgres with Replication
       include_tasks: tasks/percona-distribution-postgres-replication-setup.yml
       when: setup_type == "replication"

pmm_qa/percona-distribution-postgresql/tasks/install-pdpgsql.yml

@@ -0,0 +1,39 @@
+- name: Start containers for Percona Distribution for Postgresql
+  shell: |
+    docker run -d \
+      --name={{ container_prefix }}{{ item }} \
+      --restart=always \
+      --network {{ network_name }} \
+      -e POSTGRES_PASSWORD={{ root_password }} \
+      --privileged --cgroupns=host \
+      -v /sys/fs/cgroup:/sys/fs/cgroup:rw \
+      -v /var/lib/containerd \
+      -p {{ pdpgsql_port + (item - 1) }}:5432 \
+      antmelekhin/docker-systemd:ubuntu-24.04
+  loop: "{{ range(1, nodes_count | int + 1) | list }}"
+
+- name: Install dependencies
+  shell: |
+    docker exec {{ container_prefix }}{{ item }} apt-get update
+    docker exec {{ container_prefix }}{{ item }} apt-get -y install wget gnupg2 lsb-release curl
+  loop: "{{ range(1, nodes_count | int + 1) | list }}"
+
+- name: Install Percona Release
+  shell: |
+    docker exec {{ container_prefix }}{{ item }} wget https://repo.percona.com/apt/percona-release_latest.generic_all.deb
+    docker exec {{ container_prefix }}{{ item }} dpkg -i ./percona-release_latest.generic_all.deb
+    docker exec {{ container_prefix }}{{ item }} percona-release enable ppg-{{ pdpgsql_version }}
+  loop: "{{ range(1, nodes_count | int + 1) | list }}"
+
+- name: Install Percona Distribution for PostgreSQL
+  shell: |
+    docker exec -u root {{ container_prefix }}{{ item }} apt-get install -y percona-postgresql-{{ pdpgsql_version }} percona-pgbackrest percona-pg-stat-monitor{{ pdpgsql_version }}
+  loop: "{{ range(1, nodes_count | int + 1) | list }}"
+
+- name: Start Percona distribution for Postgresql
+  shell: docker exec -u root {{ container_prefix }}{{ item }} systemctl start postgresql
+  loop: "{{ range(1, nodes_count | int + 1) | list }}"
+
+- name: Wait 5 seconds for Percona Distribution for postgresql to start
+  pause:
+    seconds: 5

pmm_qa/percona-distribution-postgresql/tasks/install-percona-distribution-postgresql-single.yml

@@ -0,0 +1,77 @@
+- name: Remove old data folders
+  shell: 'rm -fr {{ data_dir }}'
+
+- name: Create data directories
+  file:
+    path: "{{ data_dir }}/node{{ item }}/data"
+    state: directory
+    mode: '0755'
+  loop: "{{ range(1, nodes_count | int + 1) | list }}"
+
+- name: Generate pg_hba.conf for primary node
+  template:
+    src: data/pg_hba.conf.j2
+    dest: "{{ data_dir }}/node{{ item }}/pg_hba.conf"
+  loop: "{{ range(1, nodes_count | int + 1) | list }}"
+
+- name: Generate postgres.conf for primary node
+  template:
+    src: data/postgresql-single.conf.j2
+    dest: "{{ data_dir }}/node1/postgresql.conf"
+
+- name: Install Percona Distribution for Postgres
+  include_tasks: tasks/install-pdpgsql.yml
+
+- name: Install pg stat monitor.
+  include_tasks: ./tasks/install_pg_stat-monitor.yml
+
+- name: Install and add pmm client.
+  include_tasks: ../tasks/install_pmm_client.yml
+  vars:
+    container_name: "{{ container_prefix }}{{ item }}"
+  loop: "{{ range(1, nodes_count | int + 1) | list }}"
+
+- name: Copy config files
+  shell: |
+    docker cp {{ data_dir }}/node{{ item }}/postgresql.conf {{ container_prefix }}{{ item }}:/etc/postgresql/{{ pdpgsql_version }}/main/postgresql.conf
+    docker cp {{ data_dir }}/node{{ item }}/pg_hba.conf {{ container_prefix }}{{ item }}:/etc/postgresql/{{ pdpgsql_version }}/main/pg_hba.conf
+  loop: "{{ range(1, nodes_count | int + 1) | list }}"
+
+- name: Restart Percona distribution for Postgresql
+  shell: docker exec -u root {{ container_prefix }}{{ item }} systemctl restart postgresql
+  loop: "{{ range(1, nodes_count | int + 1) | list }}"
+
+- name: Create PostgreSQL user 'pmm' with password and grant pg_monitor
+  shell: |
+    docker exec -u postgres {{ container_prefix }}{{ item }} psql -U postgres -d {{ db_name | default('postgres') }} -c "
+      DO \$\$
+      BEGIN
+        IF NOT EXISTS (SELECT FROM pg_catalog.pg_roles WHERE rolname = 'pmm') THEN
+          CREATE ROLE pmm LOGIN PASSWORD 'pmm';
+        END IF;
+        GRANT pg_monitor TO pmm;
+      END
+      \$\$;
+    "
+  loop: "{{ range(1, nodes_count | int + 1) | list }}"
+
+- name: Add service to pmm server
+  shell: |
+    docker exec {{ container_prefix }}{{ item }} pmm-admin add postgresql --username=pmm --password=pmm --cluster=pdpgsql_replication_cluster --environment=pdpgsql_replication_environment --query-source=pgstatmonitor {{ container_prefix }}{{ item }}{{ random_service_name_value }} --debug 127.0.0.1:5432
+  loop: "{{ range(1, nodes_count | int + 1) | list }}"
+
+- name: Run queries for generating data
+  shell: docker exec {{ container_prefix }}{{ item }} bash ./data/pgsm_run_queries.sh &
+  loop: "{{ range(1, nodes_count | int + 1) | list }}"
+
+- name: Copy a file into the container
+  shell: docker cp ./data/pgsql_load.sql {{ container_prefix }}{{ item }}:/pgsql_load.sql
+  loop: "{{ range(1, nodes_count | int + 1) | list }}"
+
+- name: Create database if it doesn't exist
+  shell: docker exec {{ container_prefix }}{{ item }} psql -U postgres -c "CREATE DATABASE school;"
+  loop: "{{ range(1, nodes_count | int + 1) | list }}"
+
+- name: Run SQL script using docker exec
+  shell: docker exec {{ container_prefix }}{{ item }} psql -U postgres -d school -f /pgsql_load.sql > /dev/null
+  loop: "{{ range(1, nodes_count | int + 1) | list }}"