diff --git a/pmm_psmdb-pbm_setup/start-sharded.sh b/pmm_psmdb-pbm_setup/start-sharded.sh index 4fe16b1b..e63274eb 100755 --- a/pmm_psmdb-pbm_setup/start-sharded.sh +++ b/pmm_psmdb-pbm_setup/start-sharded.sh @@ -237,10 +237,10 @@ do echo "congiguring pmm agent on $node" rs=$(echo $node | awk -F "0" '{print $1}') docker compose -f docker-compose-sharded.yaml exec -T -e PMM_AGENT_SETUP_NODE_NAME=${node}_${random_number} $node pmm-agent setup - docker compose -f docker-compose-sharded.yaml exec -T $node pmm-admin add mongodb --agent-password=mypass --cluster=sharded --username=${pmm_user} --password=${pmm_pass} ${node}_${random_number} 127.0.0.1:27017 + docker compose -f docker-compose-sharded.yaml exec -T $node pmm-admin add mongodb --agent-password=mypass --cluster=sharded --environment=mongo-sharded-dev --username=${pmm_user} --password=${pmm_pass} ${node}_${random_number} 127.0.0.1:27017 done echo "configuring pmm-agent on primary rscfg01 for mongos instance" -docker compose -f docker-compose-sharded.yaml exec -T rscfg01 pmm-admin add mongodb --agent-password=mypass --cluster=sharded --username=${pmm_user} --password=${pmm_pass} mongos_${random_number} mongos:27017 +docker compose -f docker-compose-sharded.yaml exec -T rscfg01 pmm-admin add mongodb --agent-password=mypass --cluster=sharded --environment=mongo-sharded-dev --username=${pmm_user} --password=${pmm_pass} mongos_${random_number} mongos:27017 echo "adding some data" docker compose -f docker-compose-sharded.yaml exec -T mongos mgodatagen -f /etc/datagen/sharded.json --uri=mongodb://root:root@127.0.0.1:27017 diff --git a/pmm_qa/pmm-framework.py b/pmm_qa/pmm-framework.py index 5538e53b..edca1182 100755 --- a/pmm_qa/pmm-framework.py +++ b/pmm_qa/pmm-framework.py @@ -82,6 +82,9 @@ "DOCKERCLIENTS": { "configurations": {} # Empty dictionary for consistency }, + "BUCKET": { + "configurations": {"BUCKET_NAMES": 'bcp'} + } } @@ -92,8 +95,6 @@ def run_ansible_playbook(playbook_filename, env_vars, args): playbook_path = script_dir + "/" + playbook_filename verbosity_level = 1 - - if args.verbosity_level is not None: verbosity_level = int(args.verbosity_level) @@ -212,7 +213,8 @@ def setup_ps(db_type, db_version=None, db_config=None, args=None): 'PS_NODES': no_of_nodes, 'PS_VERSION': ps_version, 'PMM_SERVER_IP': args.pmm_server_ip or container_name or '127.0.0.1', - 'PS_CONTAINER': 'ps_pmm_' + str(ps_version) + ('_replica' if setup_type_value in ("replication", "replica") else ''), + 'PS_CONTAINER': 'ps_pmm_' + str(ps_version) + ( + '_replica' if setup_type_value in ("replication", "replica") else ''), 'PS_PORT': 3318 if setup_type_value in ("replication", "replica") else 3317, 'CLIENT_VERSION': get_value('CLIENT_VERSION', db_type, args, db_config), 'QUERY_SOURCE': get_value('QUERY_SOURCE', db_type, args, db_config), @@ -227,6 +229,7 @@ def setup_ps(db_type, db_version=None, db_config=None, args=None): # Call the function to run the Ansible playbook run_ansible_playbook(playbook_filename, env_vars, args) + def setup_mysql(db_type, db_version=None, db_config=None, args=None): # Check if PMM server is running container_name = get_running_container_name() @@ -470,6 +473,7 @@ def setup_mlaunch_psmdb(db_type, db_version=None, db_config=None, args=None): # Call the function to run the Ansible playbook run_ansible_playbook(playbook_filename, env_vars, args) + def setup_mlaunch_modb(db_type, db_version=None, db_config=None, args=None): # Check if PMM server is running container_name = get_running_container_name() @@ -479,7 +483,7 @@ def setup_mlaunch_modb(db_type, db_version=None, db_config=None, args=None): # Gather Version details modb_version = os.getenv('MODB_VERSION') or db_version or \ - database_configs[db_type]["versions"][-1] + database_configs[db_type]["versions"][-1] # Define environment variables for playbook env_vars = { @@ -867,12 +871,24 @@ def setup_database(db_type, db_version=None, db_config=None, args=None): setup_mlaunch_modb(db_type, db_version, db_config, args) elif db_type == 'SSL_MLAUNCH': setup_ssl_mlaunch(db_type, db_version, db_config, args) - + elif db_type == 'BUCKET': + setup_bucket(db_type, db_version, db_config, args) else: print(f"Database type {db_type} is not recognised, Exiting...") exit(1) +def setup_bucket(db_type, db_version=None, db_config=None, args=None): + print("Setting up bucket") + bucket_names_value = get_value('BUCKET_NAMES', db_type, args, db_config).lower().split(';') + print(bucket_names_value) + env_vars = { + 'BUCKETS': bucket_names_value + } + + run_ansible_playbook('tasks/create_minio_container.yml', env_vars, args) + + # Main if __name__ == "__main__": parser = argparse.ArgumentParser(description='PMM Framework Script to setup Multiple Databases', diff --git a/pmm_qa/scripts/pgsql_load.sql b/pmm_qa/scripts/pgsql_load.sql index bd70b7f2..81a4fa90 100644 --- a/pmm_qa/scripts/pgsql_load.sql +++ b/pmm_qa/scripts/pgsql_load.sql @@ -43,6 +43,30 @@ INSERT INTO enrollments (student_id, class_id) VALUES (3, 1), (3, 3); +-- ======================================== +-- SIMULATE DEAD TUPLES +-- ======================================== + + +INSERT INTO students (first_name, last_name, birth_date) +SELECT 'John', 'Doe', CURRENT_DATE - (random() * 5000)::int +FROM generate_series(1, 100000); + +-- These updates and deletes will create dead tuples + +-- Update records (old versions become dead) +UPDATE students +SET last_name = last_name || '_updated' +WHERE student_id IN (1, 2); + +-- Delete records (deleted rows become dead) +DELETE FROM enrollments +WHERE enrollment_id IN (SELECT enrollment_id FROM enrollments LIMIT 2); + +-- Disable autovacuum temporarily (for demo) +ALTER TABLE students SET (autovacuum_enabled = false); +ALTER TABLE enrollments SET (autovacuum_enabled = false); + -- ======================================== -- SELECT QUERIES -- ======================================== diff --git a/pmm_qa/tasks/create_minio_container.yml b/pmm_qa/tasks/create_minio_container.yml new file mode 100644 index 00000000..326043c6 --- /dev/null +++ b/pmm_qa/tasks/create_minio_container.yml @@ -0,0 +1,80 @@ +--- +- name: Deploy MinIO bucket. + hosts: localhost + connection: local + gather_facts: yes + vars: + network_name: "pmm-qa" + buckets: "{{ lookup('env', 'BUCKETS') | default('bcp', true) }}" + minio_access_key: minio1234 + minio_secret_key: minio1234 + minio_volume_name: minio_backups + minio_ports: + - "9010:9000" + - "9001:9001" + + tasks: + - name: Run docker ps with port info + shell: docker ps -a + register: docker_ps_output + + - name: Create Docker network + community.docker.docker_network: + name: "{{ network_name }}" + state: present + ignore_errors: yes + + - name: Remove old MinIO docker container + community.docker.docker_container: + name: minio + image: minio/minio + restart_policy: always + state: absent + ignore_errors: yes + + - name: Remove MinIO Docker volume + community.docker.docker_volume: + name: "{{ minio_volume_name }}" + state: absent + ignore_errors: true + + - name: Create MinIO Docker volume + community.docker.docker_volume: + name: "{{ minio_volume_name }}" + + - name: Run MinIO container + community.docker.docker_container: + name: minio + image: minio/minio + restart_policy: unless-stopped + ports: "{{ minio_ports }}" + networks: + - name: "{{ network_name }}" + volumes: + - "{{ minio_volume_name }}:/backups" + env: + MINIO_ACCESS_KEY: "{{ minio_access_key }}" + MINIO_SECRET_KEY: "{{ minio_secret_key }}" + command: server /backups --address 0.0.0.0:9000 --console-address 0.0.0.0:9001 + + - name: Show the list of buckets + debug: + var: buckets + + - name: Set MinIO alias inside the container + community.docker.docker_container_exec: + container: "minio" + command: > + /bin/sh -c " + sleep 5; + /usr/bin/mc alias set myminio http://127.0.0.1:9000 minio1234 minio1234; + exit 0;" + + - name: Create MinIO buckets + community.docker.docker_container_exec: + container: "minio" + command: > + /bin/sh -c " + /usr/bin/mc mb myminio/{{ item }} || echo 'Bucket {{ item }} already exists'; + exit 0;" + loop: "{{ buckets }}"