PMM-7: New PMM QA Framework flag --bucket

pmm_psmdb-pbm_setup/start-sharded.sh

@@ -237,10 +237,10 @@ do
     echo "congiguring pmm agent on $node"
     rs=$(echo $node | awk -F "0" '{print $1}')
     docker compose -f docker-compose-sharded.yaml exec -T -e PMM_AGENT_SETUP_NODE_NAME=${node}_${random_number} $node pmm-agent setup
-    docker compose -f docker-compose-sharded.yaml exec -T $node pmm-admin add mongodb --agent-password=mypass --cluster=sharded --username=${pmm_user} --password=${pmm_pass} ${node}_${random_number} 127.0.0.1:27017
+    docker compose -f docker-compose-sharded.yaml exec -T $node pmm-admin add mongodb --agent-password=mypass --cluster=sharded --environment=mongo-sharded-dev --username=${pmm_user} --password=${pmm_pass} ${node}_${random_number} 127.0.0.1:27017
 done
 echo "configuring pmm-agent on primary rscfg01 for mongos instance"
-docker compose -f docker-compose-sharded.yaml exec -T rscfg01 pmm-admin add mongodb --agent-password=mypass --cluster=sharded --username=${pmm_user} --password=${pmm_pass} mongos_${random_number} mongos:27017
+docker compose -f docker-compose-sharded.yaml exec -T rscfg01 pmm-admin add mongodb --agent-password=mypass --cluster=sharded --environment=mongo-sharded-dev --username=${pmm_user} --password=${pmm_pass} mongos_${random_number} mongos:27017
 
 echo "adding some data"
 docker compose -f docker-compose-sharded.yaml exec -T mongos mgodatagen -f /etc/datagen/sharded.json --uri=mongodb://root:root@127.0.0.1:27017

pmm_qa/pmm-framework.py

@@ -82,6 +82,9 @@
     "DOCKERCLIENTS": {
         "configurations": {}  # Empty dictionary for consistency
     },
+    "BUCKET": {
+        "configurations": {"BUCKET_NAMES": 'bcp'}
+    }
 }
 
 
@@ -92,8 +95,6 @@ def run_ansible_playbook(playbook_filename, env_vars, args):
     playbook_path = script_dir + "/" + playbook_filename
     verbosity_level = 1
 
-
-
     if args.verbosity_level is not None:
         verbosity_level = int(args.verbosity_level)
 
@@ -212,7 +213,8 @@ def setup_ps(db_type, db_version=None, db_config=None, args=None):
             'PS_NODES': no_of_nodes,
             'PS_VERSION': ps_version,
             'PMM_SERVER_IP': args.pmm_server_ip or container_name or '127.0.0.1',
-            'PS_CONTAINER': 'ps_pmm_' + str(ps_version) + ('_replica' if setup_type_value in ("replication", "replica") else ''),
+            'PS_CONTAINER': 'ps_pmm_' + str(ps_version) + (
+                '_replica' if setup_type_value in ("replication", "replica") else ''),
             'PS_PORT': 3318 if setup_type_value in ("replication", "replica") else 3317,
             'CLIENT_VERSION': get_value('CLIENT_VERSION', db_type, args, db_config),
             'QUERY_SOURCE': get_value('QUERY_SOURCE', db_type, args, db_config),
@@ -227,6 +229,7 @@ def setup_ps(db_type, db_version=None, db_config=None, args=None):
         # Call the function to run the Ansible playbook
         run_ansible_playbook(playbook_filename, env_vars, args)
 
+
 def setup_mysql(db_type, db_version=None, db_config=None, args=None):
     # Check if PMM server is running
     container_name = get_running_container_name()
@@ -470,6 +473,7 @@ def setup_mlaunch_psmdb(db_type, db_version=None, db_config=None, args=None):
     # Call the function to run the Ansible playbook
     run_ansible_playbook(playbook_filename, env_vars, args)
 
+
 def setup_mlaunch_modb(db_type, db_version=None, db_config=None, args=None):
     # Check if PMM server is running
     container_name = get_running_container_name()
@@ -479,7 +483,7 @@ def setup_mlaunch_modb(db_type, db_version=None, db_config=None, args=None):
 
     # Gather Version details
     modb_version = os.getenv('MODB_VERSION') or db_version or \
-                    database_configs[db_type]["versions"][-1]
+                   database_configs[db_type]["versions"][-1]
 
     # Define environment variables for playbook
     env_vars = {
@@ -867,12 +871,24 @@ def setup_database(db_type, db_version=None, db_config=None, args=None):
         setup_mlaunch_modb(db_type, db_version, db_config, args)
     elif db_type == 'SSL_MLAUNCH':
         setup_ssl_mlaunch(db_type, db_version, db_config, args)
-
+    elif db_type == 'BUCKET':
+        setup_bucket(db_type, db_version, db_config, args)
     else:
         print(f"Database type {db_type} is not recognised, Exiting...")
         exit(1)
 
 
+def setup_bucket(db_type, db_version=None, db_config=None, args=None):
+    print("Setting up bucket")
+    bucket_names_value = get_value('BUCKET_NAMES', db_type, args, db_config).lower().split(';')
+    print(bucket_names_value)
+    env_vars = {
+        'BUCKETS': bucket_names_value
+    }
+
+    run_ansible_playbook('tasks/create_minio_container.yml', env_vars, args)
+
+
 # Main
 if __name__ == "__main__":
     parser = argparse.ArgumentParser(description='PMM Framework Script to setup Multiple Databases',

pmm_qa/scripts/pgsql_load.sql

@@ -43,6 +43,30 @@ INSERT INTO enrollments (student_id, class_id) VALUES
 (3, 1),
 (3, 3);
 
+-- ========================================
+-- SIMULATE DEAD TUPLES
+-- ========================================
+
+
+INSERT INTO students (first_name, last_name, birth_date)
+SELECT 'John', 'Doe', CURRENT_DATE - (random() * 5000)::int
+FROM generate_series(1, 100000);
+
+-- These updates and deletes will create dead tuples
+
+-- Update records (old versions become dead)
+UPDATE students
+SET last_name = last_name || '_updated'
+WHERE student_id IN (1, 2);
+
+-- Delete records (deleted rows become dead)
+DELETE FROM enrollments
+WHERE enrollment_id IN (SELECT enrollment_id FROM enrollments LIMIT 2);
+
+-- Disable autovacuum temporarily (for demo)
+ALTER TABLE students SET (autovacuum_enabled = false);
+ALTER TABLE enrollments SET (autovacuum_enabled = false);
+
 -- ========================================
 -- SELECT QUERIES
 -- ========================================

pmm_qa/tasks/create_minio_container.yml

@@ -0,0 +1,80 @@
+---
+- name: Deploy MinIO bucket.
+  hosts: localhost
+  connection: local
+  gather_facts: yes
+  vars:
+    network_name: "pmm-qa"
+    buckets: "{{ lookup('env', 'BUCKETS') | default('bcp', true) }}"
+    minio_access_key: minio1234
+    minio_secret_key: minio1234
+    minio_volume_name: minio_backups
+    minio_ports:
+      - "9010:9000"
+      - "9001:9001"
+
+  tasks:
+    - name: Run docker ps with port info
+      shell: docker ps -a
+      register: docker_ps_output
+
+    - name: Create Docker network
+      community.docker.docker_network:
+        name: "{{ network_name }}"
+        state: present
+      ignore_errors: yes
+
+    - name: Remove old MinIO docker container
+      community.docker.docker_container:
+        name: minio
+        image: minio/minio
+        restart_policy: always
+        state: absent
+      ignore_errors: yes
+
+    - name: Remove MinIO Docker volume
+      community.docker.docker_volume:
+        name: "{{ minio_volume_name }}"
+        state: absent
+      ignore_errors: true
+
+    - name: Create MinIO Docker volume
+      community.docker.docker_volume:
+        name: "{{ minio_volume_name }}"
+
+    - name: Run MinIO container
+      community.docker.docker_container:
+        name: minio
+        image: minio/minio
+        restart_policy: unless-stopped
+        ports: "{{ minio_ports }}"
+        networks:
+          - name: "{{ network_name }}"
+        volumes:
+          - "{{ minio_volume_name }}:/backups"
+        env:
+          MINIO_ACCESS_KEY: "{{ minio_access_key }}"
+          MINIO_SECRET_KEY: "{{ minio_secret_key }}"
+        command: server /backups --address 0.0.0.0:9000 --console-address 0.0.0.0:9001
+
+    - name: Show the list of buckets
+      debug:
+        var: buckets
+
+    - name: Set MinIO alias inside the container
+      community.docker.docker_container_exec:
+        container: "minio"
+        command: >
+          /bin/sh -c "
+          sleep 5;
+          /usr/bin/mc alias set myminio http://127.0.0.1:9000 minio1234 minio1234;
+          exit 0;"
+
+    - name: Create MinIO buckets
+      community.docker.docker_container_exec:
+        container: "minio"
+        command: >
+          /bin/sh -c "
+          /usr/bin/mc mb myminio/{{ item }} || echo 'Bucket {{ item }} already exists';
+          exit 0;"
+      loop: "{{ buckets }}"