From 06f0a613cb6946b8e6b270fa85c619118c91eca2 Mon Sep 17 00:00:00 2001 From: Peter Sirotnak Date: Wed, 4 Jun 2025 23:55:15 +0200 Subject: [PATCH 01/41] PMM-7: New buckets --- pmm_psmdb-pbm_setup/start-rs-only.sh | 2 ++ pmm_psmdb-pbm_setup/start-sharded.sh | 2 ++ 2 files changed, 4 insertions(+) diff --git a/pmm_psmdb-pbm_setup/start-rs-only.sh b/pmm_psmdb-pbm_setup/start-rs-only.sh index d7f95dfc..c038bdba 100755 --- a/pmm_psmdb-pbm_setup/start-rs-only.sh +++ b/pmm_psmdb-pbm_setup/start-rs-only.sh @@ -35,3 +35,5 @@ if [ $profile = "extra" ]; then fi bash -x ./configure-extra-agents.sh fi + +docker exec minio /bin/sh -c " sleep 5; /usr/bin/mc alias set myminio http://minio:9000 minio1234 minio1234; /usr/bin/mc mb myminio/bcp-replicaset; exit 0; " diff --git a/pmm_psmdb-pbm_setup/start-sharded.sh b/pmm_psmdb-pbm_setup/start-sharded.sh index 4fe16b1b..0798fe48 100755 --- a/pmm_psmdb-pbm_setup/start-sharded.sh +++ b/pmm_psmdb-pbm_setup/start-sharded.sh @@ -259,3 +259,5 @@ if [ $cleanup != "no" ]; then else echo "skipping cleanup" fi + +docker exec minio /bin/sh -c " sleep 5; /usr/bin/mc alias set myminio http://minio:9000 minio1234 minio1234; /usr/bin/mc mb myminio/bcp-sharded; exit 0; " From 3238db92fcac0c25fe7cd51e8be8df2d3e3919c2 Mon Sep 17 00:00:00 2001 From: Peter Sirotnak Date: Thu, 5 Jun 2025 22:32:23 +0200 Subject: [PATCH 02/41] PMM-7: Configure agents --- pmm_psmdb-pbm_setup/configure-agents.sh | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/pmm_psmdb-pbm_setup/configure-agents.sh b/pmm_psmdb-pbm_setup/configure-agents.sh index 93e9fa96..8505ce67 100644 --- a/pmm_psmdb-pbm_setup/configure-agents.sh +++ b/pmm_psmdb-pbm_setup/configure-agents.sh @@ -33,9 +33,9 @@ do echo "congiguring pmm agent on $node" docker compose -f docker-compose-rs.yaml exec -T -e PMM_AGENT_SETUP_NODE_NAME=${node}._${random_number} $node pmm-agent setup if [[ $mongo_setup_type == "psa" && $node == "rs103" ]]; then - docker compose -f docker-compose-rs.yaml exec -T $node pmm-admin add mongodb --enable-all-collectors --agent-password=mypass --cluster=replicaset --replication-set=rs ${node}_${random_number} 127.0.0.1:27017 + docker compose -f docker-compose-rs.yaml exec -T $node pmm-admin add mongodb --enable-all-collectors --agent-password=mypass --cluster=replicaset --environment=mongo-sharded-dev--replication-set=rs ${node}_${random_number} 127.0.0.1:27017 else - docker compose -f docker-compose-rs.yaml exec -T $node pmm-admin add mongodb --enable-all-collectors --agent-password=mypass --cluster=replicaset --replication-set=rs --username=${pmm_mongo_user} --password=${pmm_mongo_user_pass} ${node}_${random_number} 127.0.0.1:27017 + docker compose -f docker-compose-rs.yaml exec -T $node pmm-admin add mongodb --enable-all-collectors --agent-password=mypass --cluster=replicaset --environment=mongo-sharded-dev --replication-set=rs --username=${pmm_mongo_user} --password=${pmm_mongo_user_pass} ${node}_${random_number} 127.0.0.1:27017 fi done echo From aae13c60f16c2f040ebb6e4151b76f565f387f23 Mon Sep 17 00:00:00 2001 From: Peter Sirotnak Date: Thu, 5 Jun 2025 22:52:27 +0200 Subject: [PATCH 03/41] PMM-7: Configure agents --- pmm_psmdb-pbm_setup/configure-agents.sh | 4 ++-- pmm_psmdb-pbm_setup/start-sharded.sh | 4 ++-- 2 files changed, 4 insertions(+), 4 deletions(-) diff --git a/pmm_psmdb-pbm_setup/configure-agents.sh b/pmm_psmdb-pbm_setup/configure-agents.sh index 8505ce67..93e9fa96 100644 --- a/pmm_psmdb-pbm_setup/configure-agents.sh +++ b/pmm_psmdb-pbm_setup/configure-agents.sh @@ -33,9 +33,9 @@ do echo "congiguring pmm agent on $node" docker compose -f docker-compose-rs.yaml exec -T -e PMM_AGENT_SETUP_NODE_NAME=${node}._${random_number} $node pmm-agent setup if [[ $mongo_setup_type == "psa" && $node == "rs103" ]]; then - docker compose -f docker-compose-rs.yaml exec -T $node pmm-admin add mongodb --enable-all-collectors --agent-password=mypass --cluster=replicaset --environment=mongo-sharded-dev--replication-set=rs ${node}_${random_number} 127.0.0.1:27017 + docker compose -f docker-compose-rs.yaml exec -T $node pmm-admin add mongodb --enable-all-collectors --agent-password=mypass --cluster=replicaset --replication-set=rs ${node}_${random_number} 127.0.0.1:27017 else - docker compose -f docker-compose-rs.yaml exec -T $node pmm-admin add mongodb --enable-all-collectors --agent-password=mypass --cluster=replicaset --environment=mongo-sharded-dev --replication-set=rs --username=${pmm_mongo_user} --password=${pmm_mongo_user_pass} ${node}_${random_number} 127.0.0.1:27017 + docker compose -f docker-compose-rs.yaml exec -T $node pmm-admin add mongodb --enable-all-collectors --agent-password=mypass --cluster=replicaset --replication-set=rs --username=${pmm_mongo_user} --password=${pmm_mongo_user_pass} ${node}_${random_number} 127.0.0.1:27017 fi done echo diff --git a/pmm_psmdb-pbm_setup/start-sharded.sh b/pmm_psmdb-pbm_setup/start-sharded.sh index 0798fe48..78e72af6 100755 --- a/pmm_psmdb-pbm_setup/start-sharded.sh +++ b/pmm_psmdb-pbm_setup/start-sharded.sh @@ -237,10 +237,10 @@ do echo "congiguring pmm agent on $node" rs=$(echo $node | awk -F "0" '{print $1}') docker compose -f docker-compose-sharded.yaml exec -T -e PMM_AGENT_SETUP_NODE_NAME=${node}_${random_number} $node pmm-agent setup - docker compose -f docker-compose-sharded.yaml exec -T $node pmm-admin add mongodb --agent-password=mypass --cluster=sharded --username=${pmm_user} --password=${pmm_pass} ${node}_${random_number} 127.0.0.1:27017 + docker compose -f docker-compose-sharded.yaml exec -T $node pmm-admin add mongodb --agent-password=mypass --cluster=sharded --environment=mongo-sharded-dev --username=${pmm_user} --password=${pmm_pass} ${node}_${random_number} 127.0.0.1:27017 done echo "configuring pmm-agent on primary rscfg01 for mongos instance" -docker compose -f docker-compose-sharded.yaml exec -T rscfg01 pmm-admin add mongodb --agent-password=mypass --cluster=sharded --username=${pmm_user} --password=${pmm_pass} mongos_${random_number} mongos:27017 +docker compose -f docker-compose-sharded.yaml exec -T rscfg01 pmm-admin add mongodb --agent-password=mypass --cluster=sharded --environment=mongo-sharded-dev --username=${pmm_user} --password=${pmm_pass} mongos_${random_number} mongos:27017 echo "adding some data" docker compose -f docker-compose-sharded.yaml exec -T mongos mgodatagen -f /etc/datagen/sharded.json --uri=mongodb://root:root@127.0.0.1:27017 From ab67e16f3d4ccfbbb1e43e11fae009a44c22f9df Mon Sep 17 00:00:00 2001 From: Peter Sirotnak Date: Fri, 6 Jun 2025 10:41:32 +0200 Subject: [PATCH 04/41] PMM-7: Minio --- pmm_psmdb-pbm_setup/docker-compose-sharded.yaml | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/pmm_psmdb-pbm_setup/docker-compose-sharded.yaml b/pmm_psmdb-pbm_setup/docker-compose-sharded.yaml index 4f4598a5..af16ad5d 100644 --- a/pmm_psmdb-pbm_setup/docker-compose-sharded.yaml +++ b/pmm_psmdb-pbm_setup/docker-compose-sharded.yaml @@ -229,7 +229,8 @@ services: image: minio/minio container_name: minio ports: - - "9001:9000" + - "9000:9000" + - "9001:9001" networks: - test-network volumes: From e3c4421f97061e464acecc6b764782e1059553f9 Mon Sep 17 00:00:00 2001 From: Peter Sirotnak Date: Sun, 8 Jun 2025 00:35:30 +0200 Subject: [PATCH 05/41] PMM-7: Minio --- pmm_psmdb-pbm_setup/start-sharded.sh | 1 + 1 file changed, 1 insertion(+) diff --git a/pmm_psmdb-pbm_setup/start-sharded.sh b/pmm_psmdb-pbm_setup/start-sharded.sh index 78e72af6..5ab2a50e 100755 --- a/pmm_psmdb-pbm_setup/start-sharded.sh +++ b/pmm_psmdb-pbm_setup/start-sharded.sh @@ -261,3 +261,4 @@ if [ $cleanup != "no" ]; then fi docker exec minio /bin/sh -c " sleep 5; /usr/bin/mc alias set myminio http://minio:9000 minio1234 minio1234; /usr/bin/mc mb myminio/bcp-sharded; exit 0; " +docker ps -a From 4498168accd9075e4d6889812910c723e91886dc Mon Sep 17 00:00:00 2001 From: Peter Sirotnak Date: Sun, 8 Jun 2025 00:39:03 +0200 Subject: [PATCH 06/41] PMM-7: Minio --- pmm_psmdb-pbm_setup/docker-compose-rs.yaml | 2 +- pmm_psmdb-pbm_setup/docker-compose-sharded.yaml | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/pmm_psmdb-pbm_setup/docker-compose-rs.yaml b/pmm_psmdb-pbm_setup/docker-compose-rs.yaml index 9a328eaf..817092c3 100644 --- a/pmm_psmdb-pbm_setup/docker-compose-rs.yaml +++ b/pmm_psmdb-pbm_setup/docker-compose-rs.yaml @@ -200,7 +200,7 @@ services: environment: - "MINIO_ACCESS_KEY=minio1234" - "MINIO_SECRET_KEY=minio1234" - command: server /backups + command: server /backups --address 0.0.0.0:9000 --console-address 0.0.0.0:9001 createbucket: container_name: createbucket diff --git a/pmm_psmdb-pbm_setup/docker-compose-sharded.yaml b/pmm_psmdb-pbm_setup/docker-compose-sharded.yaml index af16ad5d..6f0b5ca4 100644 --- a/pmm_psmdb-pbm_setup/docker-compose-sharded.yaml +++ b/pmm_psmdb-pbm_setup/docker-compose-sharded.yaml @@ -238,7 +238,7 @@ services: environment: - "MINIO_ACCESS_KEY=minio1234" - "MINIO_SECRET_KEY=minio1234" - command: server /backups + command: server /backups --address 0.0.0.0:9000 --console-address 0.0.0.0:9001 createbucket: container_name: createbucket From da934d0386a0b82498f74991808fd7aa5313704f Mon Sep 17 00:00:00 2001 From: Peter Sirotnak Date: Sun, 8 Jun 2025 13:26:25 +0200 Subject: [PATCH 07/41] PMM-7: Minio --- pmm_qa/pmm-framework.py | 3 +++ pmm_qa/tasks/create_minio_container.yml | 30 +++++++++++++++++++++++++ 2 files changed, 33 insertions(+) create mode 100644 pmm_qa/tasks/create_minio_container.yml diff --git a/pmm_qa/pmm-framework.py b/pmm_qa/pmm-framework.py index 5538e53b..b3635890 100755 --- a/pmm_qa/pmm-framework.py +++ b/pmm_qa/pmm-framework.py @@ -894,12 +894,15 @@ def setup_database(db_type, db_version=None, db_config=None, args=None): parser.add_argument("--client-version", nargs='?', help='PMM Client version/tarball') parser.add_argument("--verbose", "--v", action='store_true', help='Display verbose information') parser.add_argument("--verbosity-level", nargs='?', help='Display verbose information level') + parser.add_argument("--bucket", nargs='?', help='Create MinIO S3 compatible bucket') args = parser.parse_args() if args.verbosity_level is not None and not args.verbosity_level.isnumeric(): print(f"Option {args.verbosity_level} is invalid verbosity level option, please provide number 1-5") exit(1) + print(args) + # Parse arguments try: for db in args.database: diff --git a/pmm_qa/tasks/create_minio_container.yml b/pmm_qa/tasks/create_minio_container.yml new file mode 100644 index 00000000..d9e234e1 --- /dev/null +++ b/pmm_qa/tasks/create_minio_container.yml @@ -0,0 +1,30 @@ +--- +- name: Deploy MinIO bucket. + hosts: localhost + connection: local + gather_facts: yes + vars: + minio_access_key: minio1234 + minio_secret_key: minio1234 + minio_volume_name: minio_backups + minio_ports: + - "9000:9000" + - "9001:9001" + + tasks: + - name: Create MinIO Docker volume + community.docker.docker_volume: + name: "{{ minio_volume_name }}" + + - name: Run MinIO container + community.docker.docker_container: + name: minio + image: minio/minio + restart_policy: unless-stopped + ports: "{{ minio_ports }}" + volumes: + - "{{ minio_volume_name }}:/backups" + env: + MINIO_ACCESS_KEY: "{{ minio_access_key }}" + MINIO_SECRET_KEY: "{{ minio_secret_key }}" + command: server /backups --address 0.0.0.0:9000 --console-address 0.0.0.0:9001 \ No newline at end of file From 4febefeda68e6a67f5ecf7eb726657ce720b66a1 Mon Sep 17 00:00:00 2001 From: Peter Sirotnak Date: Sun, 8 Jun 2025 13:36:24 +0200 Subject: [PATCH 08/41] PMM-7: Minio --- pmm_qa/pmm-framework.py | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/pmm_qa/pmm-framework.py b/pmm_qa/pmm-framework.py index b3635890..f63d6432 100755 --- a/pmm_qa/pmm-framework.py +++ b/pmm_qa/pmm-framework.py @@ -894,7 +894,7 @@ def setup_database(db_type, db_version=None, db_config=None, args=None): parser.add_argument("--client-version", nargs='?', help='PMM Client version/tarball') parser.add_argument("--verbose", "--v", action='store_true', help='Display verbose information') parser.add_argument("--verbosity-level", nargs='?', help='Display verbose information level') - parser.add_argument("--bucket", nargs='?', help='Create MinIO S3 compatible bucket') + parser.add_argument("--bucket", nargs='store_true', help='Create MinIO S3 compatible bucket') args = parser.parse_args() if args.verbosity_level is not None and not args.verbosity_level.isnumeric(): @@ -938,6 +938,7 @@ def setup_database(db_type, db_version=None, db_config=None, args=None): if args.verbose: print(f"Option {key} is not recognised, will be using default option") continue + # if args except KeyError as e: print(f"Option {key} is not recognised with error {e}, Please check and try again") parser.print_help() From 03c862563fc2279624094f61de3bf8a3df2c3025 Mon Sep 17 00:00:00 2001 From: Peter Sirotnak Date: Sun, 8 Jun 2025 13:39:51 +0200 Subject: [PATCH 09/41] PMM-7: Minio --- pmm_qa/pmm-framework.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/pmm_qa/pmm-framework.py b/pmm_qa/pmm-framework.py index f63d6432..a1557e34 100755 --- a/pmm_qa/pmm-framework.py +++ b/pmm_qa/pmm-framework.py @@ -894,7 +894,7 @@ def setup_database(db_type, db_version=None, db_config=None, args=None): parser.add_argument("--client-version", nargs='?', help='PMM Client version/tarball') parser.add_argument("--verbose", "--v", action='store_true', help='Display verbose information') parser.add_argument("--verbosity-level", nargs='?', help='Display verbose information level') - parser.add_argument("--bucket", nargs='store_true', help='Create MinIO S3 compatible bucket') + parser.add_argument("--bucket", nargs='+', help='Create MinIO S3 compatible bucket') args = parser.parse_args() if args.verbosity_level is not None and not args.verbosity_level.isnumeric(): From b77b0bfb44aa8d0147e18a363215e14c86c7d220 Mon Sep 17 00:00:00 2001 From: Peter Sirotnak Date: Sun, 8 Jun 2025 13:41:17 +0200 Subject: [PATCH 10/41] PMM-7: Minio --- pmm_qa/pmm-framework.py | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/pmm_qa/pmm-framework.py b/pmm_qa/pmm-framework.py index a1557e34..2d9ef44e 100755 --- a/pmm_qa/pmm-framework.py +++ b/pmm_qa/pmm-framework.py @@ -938,7 +938,8 @@ def setup_database(db_type, db_version=None, db_config=None, args=None): if args.verbose: print(f"Option {key} is not recognised, will be using default option") continue - # if args + if args.bucket: + print(f"Buckets are: {args.bucket}") except KeyError as e: print(f"Option {key} is not recognised with error {e}, Please check and try again") parser.print_help() From 8fd7b0e6e1ffbd6ff49e060c05199280e4529b79 Mon Sep 17 00:00:00 2001 From: Peter Sirotnak Date: Sun, 8 Jun 2025 13:43:14 +0200 Subject: [PATCH 11/41] PMM-7: Minio --- pmm_qa/pmm-framework.py | 1 + 1 file changed, 1 insertion(+) diff --git a/pmm_qa/pmm-framework.py b/pmm_qa/pmm-framework.py index 2d9ef44e..18237aa3 100755 --- a/pmm_qa/pmm-framework.py +++ b/pmm_qa/pmm-framework.py @@ -902,6 +902,7 @@ def setup_database(db_type, db_version=None, db_config=None, args=None): exit(1) print(args) + print(f"Buckets are {args.bucket}") # Parse arguments try: From 15c79302622b4d916727b51cb723b502bc4e67a3 Mon Sep 17 00:00:00 2001 From: Peter Sirotnak Date: Sun, 8 Jun 2025 13:45:10 +0200 Subject: [PATCH 12/41] PMM-7: Minio --- pmm_qa/pmm-framework.py | 5 ++--- 1 file changed, 2 insertions(+), 3 deletions(-) diff --git a/pmm_qa/pmm-framework.py b/pmm_qa/pmm-framework.py index 18237aa3..55871edf 100755 --- a/pmm_qa/pmm-framework.py +++ b/pmm_qa/pmm-framework.py @@ -902,10 +902,11 @@ def setup_database(db_type, db_version=None, db_config=None, args=None): exit(1) print(args) - print(f"Buckets are {args.bucket}") # Parse arguments try: + if args.bucket: + print(f"Buckets are: {args.bucket}") for db in args.database: db_parts = db[0].split(',') configs = db_parts[0:] if len(db_parts) > 1 else db[0:] @@ -939,8 +940,6 @@ def setup_database(db_type, db_version=None, db_config=None, args=None): if args.verbose: print(f"Option {key} is not recognised, will be using default option") continue - if args.bucket: - print(f"Buckets are: {args.bucket}") except KeyError as e: print(f"Option {key} is not recognised with error {e}, Please check and try again") parser.print_help() From 353b5f85a3e127954c90ad49055254020c19a727 Mon Sep 17 00:00:00 2001 From: Peter Sirotnak Date: Sun, 8 Jun 2025 13:58:42 +0200 Subject: [PATCH 13/41] PMM-7: Minio --- pmm_qa/pmm-framework.py | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/pmm_qa/pmm-framework.py b/pmm_qa/pmm-framework.py index 55871edf..73b22492 100755 --- a/pmm_qa/pmm-framework.py +++ b/pmm_qa/pmm-framework.py @@ -872,6 +872,9 @@ def setup_database(db_type, db_version=None, db_config=None, args=None): print(f"Database type {db_type} is not recognised, Exiting...") exit(1) +def setup_bucket(args=None): + print("Setting up bucket") + print(args) # Main if __name__ == "__main__": @@ -907,6 +910,7 @@ def setup_database(db_type, db_version=None, db_config=None, args=None): try: if args.bucket: print(f"Buckets are: {args.bucket}") + setup_bucket(args) for db in args.database: db_parts = db[0].split(',') configs = db_parts[0:] if len(db_parts) > 1 else db[0:] From 4a253839f1ec80b9c5e65b97db131a9304a94a37 Mon Sep 17 00:00:00 2001 From: Peter Sirotnak Date: Sun, 8 Jun 2025 14:00:06 +0200 Subject: [PATCH 14/41] PMM-7: Minio --- pmm_qa/pmm-framework.py | 93 ++++++++++++++++++++++------------------- 1 file changed, 49 insertions(+), 44 deletions(-) diff --git a/pmm_qa/pmm-framework.py b/pmm_qa/pmm-framework.py index 73b22492..489b2f1d 100755 --- a/pmm_qa/pmm-framework.py +++ b/pmm_qa/pmm-framework.py @@ -92,8 +92,6 @@ def run_ansible_playbook(playbook_filename, env_vars, args): playbook_path = script_dir + "/" + playbook_filename verbosity_level = 1 - - if args.verbosity_level is not None: verbosity_level = int(args.verbosity_level) @@ -212,7 +210,8 @@ def setup_ps(db_type, db_version=None, db_config=None, args=None): 'PS_NODES': no_of_nodes, 'PS_VERSION': ps_version, 'PMM_SERVER_IP': args.pmm_server_ip or container_name or '127.0.0.1', - 'PS_CONTAINER': 'ps_pmm_' + str(ps_version) + ('_replica' if setup_type_value in ("replication", "replica") else ''), + 'PS_CONTAINER': 'ps_pmm_' + str(ps_version) + ( + '_replica' if setup_type_value in ("replication", "replica") else ''), 'PS_PORT': 3318 if setup_type_value in ("replication", "replica") else 3317, 'CLIENT_VERSION': get_value('CLIENT_VERSION', db_type, args, db_config), 'QUERY_SOURCE': get_value('QUERY_SOURCE', db_type, args, db_config), @@ -227,6 +226,7 @@ def setup_ps(db_type, db_version=None, db_config=None, args=None): # Call the function to run the Ansible playbook run_ansible_playbook(playbook_filename, env_vars, args) + def setup_mysql(db_type, db_version=None, db_config=None, args=None): # Check if PMM server is running container_name = get_running_container_name() @@ -470,6 +470,7 @@ def setup_mlaunch_psmdb(db_type, db_version=None, db_config=None, args=None): # Call the function to run the Ansible playbook run_ansible_playbook(playbook_filename, env_vars, args) + def setup_mlaunch_modb(db_type, db_version=None, db_config=None, args=None): # Check if PMM server is running container_name = get_running_container_name() @@ -479,7 +480,7 @@ def setup_mlaunch_modb(db_type, db_version=None, db_config=None, args=None): # Gather Version details modb_version = os.getenv('MODB_VERSION') or db_version or \ - database_configs[db_type]["versions"][-1] + database_configs[db_type]["versions"][-1] # Define environment variables for playbook env_vars = { @@ -599,7 +600,6 @@ def get_latest_psmdb_version(psmdb_version): # Extract the version number using regular expression version_number = [v.split('|')[0] for v in re.findall(r'value="([^"]*)"', response.text)] - if version_number: # Sort the version numbers and extract the latest one latest_version = sorted(version_number, key=lambda x: tuple(map(int, x.split('-')[-1].split('.'))))[-1] @@ -689,7 +689,9 @@ def mongo_ssl_setup(script_filename, args): subprocess.run( ['cp', f'{scripts_path}docker-compose-pmm-psmdb.yml', f'{compose_file_path}']) admin_password = os.getenv('ADMIN_PASSWORD') or args.pmm_server_password or 'admin' - subprocess.run(['sed', '-i', f's/PMM_AGENT_SERVER_PASSWORD=admin/PMM_AGENT_SERVER_PASSWORD={admin_password}/g', f'{compose_file_path}']) + subprocess.run( + ['sed', '-i', f's/PMM_AGENT_SERVER_PASSWORD=admin/PMM_AGENT_SERVER_PASSWORD={admin_password}/g', + f'{compose_file_path}']) subprocess.run(['sed', '-i', '/container_name/a\ networks:\\\n \\- pmm-qa', f'{compose_file_path}']) subprocess.run(['sed', '-i', '$a\\\nnetworks:\\\n pmm-qa:\\\n name: pmm-qa\\\n external: true', f'{compose_file_path}']) @@ -872,10 +874,12 @@ def setup_database(db_type, db_version=None, db_config=None, args=None): print(f"Database type {db_type} is not recognised, Exiting...") exit(1) + def setup_bucket(args=None): print("Setting up bucket") print(args) + # Main if __name__ == "__main__": parser = argparse.ArgumentParser(description='PMM Framework Script to setup Multiple Databases', @@ -911,45 +915,46 @@ def setup_bucket(args=None): if args.bucket: print(f"Buckets are: {args.bucket}") setup_bucket(args) - for db in args.database: - db_parts = db[0].split(',') - configs = db_parts[0:] if len(db_parts) > 1 else db[0:] - db_type = None - db_version = None - db_config = {} - - if configs: - for config in configs: - if "=" in config: - key, value = config.split('=') - else: - key, value = config, None - - # Convert all arguments/options only to uppercase - key = key.upper() - - try: - if key in database_configs: - db_type = key - if "versions" in database_configs[db_type]: - if value in database_configs[db_type]["versions"]: - db_version = value - else: - if args.verbose: - print( - f"Value {value} is not recognised for Option {key}, will be using Default value") - elif key in database_configs[db_type]["configurations"]: - db_config[key] = value + if args.database: + for db in args.database: + db_parts = db[0].split(',') + configs = db_parts[0:] if len(db_parts) > 1 else db[0:] + db_type = None + db_version = None + db_config = {} + + if configs: + for config in configs: + if "=" in config: + key, value = config.split('=') else: - if args.verbose: - print(f"Option {key} is not recognised, will be using default option") - continue - except KeyError as e: - print(f"Option {key} is not recognised with error {e}, Please check and try again") - parser.print_help() - exit(1) - # Set up the specified databases - setup_database(db_type, db_version, db_config, args) + key, value = config, None + + # Convert all arguments/options only to uppercase + key = key.upper() + + try: + if key in database_configs: + db_type = key + if "versions" in database_configs[db_type]: + if value in database_configs[db_type]["versions"]: + db_version = value + else: + if args.verbose: + print( + f"Value {value} is not recognised for Option {key}, will be using Default value") + elif key in database_configs[db_type]["configurations"]: + db_config[key] = value + else: + if args.verbose: + print(f"Option {key} is not recognised, will be using default option") + continue + except KeyError as e: + print(f"Option {key} is not recognised with error {e}, Please check and try again") + parser.print_help() + exit(1) + # Set up the specified databases + setup_database(db_type, db_version, db_config, args) except argparse.ArgumentError as e: print(f"Option is not recognised:", e) parser.print_help() From 9fe9408a207db00cd200eddd913d69fd83fc9306 Mon Sep 17 00:00:00 2001 From: Peter Sirotnak Date: Sun, 8 Jun 2025 14:01:45 +0200 Subject: [PATCH 15/41] PMM-7: Minio --- pmm_qa/pmm-framework.py | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) diff --git a/pmm_qa/pmm-framework.py b/pmm_qa/pmm-framework.py index 489b2f1d..2339a44e 100755 --- a/pmm_qa/pmm-framework.py +++ b/pmm_qa/pmm-framework.py @@ -877,8 +877,10 @@ def setup_database(db_type, db_version=None, db_config=None, args=None): def setup_bucket(args=None): print("Setting up bucket") - print(args) + print(args.bucket) + env_vars = {} + run_ansible_playbook('tasks/create_minio_container.yml', env_vars, args) # Main if __name__ == "__main__": @@ -913,7 +915,6 @@ def setup_bucket(args=None): # Parse arguments try: if args.bucket: - print(f"Buckets are: {args.bucket}") setup_bucket(args) if args.database: for db in args.database: From aacad1b6d12a27c513befe49fc29821895a5672e Mon Sep 17 00:00:00 2001 From: Peter Sirotnak Date: Sun, 8 Jun 2025 14:03:37 +0200 Subject: [PATCH 16/41] PMM-7: Minio --- pmm_qa/tasks/create_minio_container.yml | 14 ++++++++++++++ 1 file changed, 14 insertions(+) diff --git a/pmm_qa/tasks/create_minio_container.yml b/pmm_qa/tasks/create_minio_container.yml index d9e234e1..e74583eb 100644 --- a/pmm_qa/tasks/create_minio_container.yml +++ b/pmm_qa/tasks/create_minio_container.yml @@ -12,6 +12,20 @@ - "9001:9001" tasks: + - name: Remove old MinIO docker container + community.docker.docker_container: + name: minio + image: minio/minio + restart_policy: always + state: absent + ignore_errors: yes + + - name: Remove MinIO Docker volume + community.docker.docker_volume: + name: "{{ minio_volume_name }}" + state: absent + ignore_errors: true + - name: Create MinIO Docker volume community.docker.docker_volume: name: "{{ minio_volume_name }}" From 1ab1024787cef988b3195d7e2771d4ce9050a690 Mon Sep 17 00:00:00 2001 From: Peter Sirotnak Date: Sun, 8 Jun 2025 14:04:28 +0200 Subject: [PATCH 17/41] PMM-7: Minio --- pmm_qa/pmm-framework.py | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/pmm_qa/pmm-framework.py b/pmm_qa/pmm-framework.py index 2339a44e..08c8d8d6 100755 --- a/pmm_qa/pmm-framework.py +++ b/pmm_qa/pmm-framework.py @@ -878,7 +878,9 @@ def setup_database(db_type, db_version=None, db_config=None, args=None): def setup_bucket(args=None): print("Setting up bucket") print(args.bucket) - env_vars = {} + env_vars = { + 'BUCKETS': args.bucket + } run_ansible_playbook('tasks/create_minio_container.yml', env_vars, args) From 875ce5f1c18ff7fc2ea76979b829096beb6aa908 Mon Sep 17 00:00:00 2001 From: Peter Sirotnak Date: Sun, 8 Jun 2025 14:05:33 +0200 Subject: [PATCH 18/41] PMM-7: Minio --- pmm_qa/tasks/create_minio_container.yml | 7 ++++++- 1 file changed, 6 insertions(+), 1 deletion(-) diff --git a/pmm_qa/tasks/create_minio_container.yml b/pmm_qa/tasks/create_minio_container.yml index e74583eb..667df81f 100644 --- a/pmm_qa/tasks/create_minio_container.yml +++ b/pmm_qa/tasks/create_minio_container.yml @@ -4,6 +4,7 @@ connection: local gather_facts: yes vars: + buckets: "{{ lookup('env', 'BUCKETS') | default('bcp', true) }}" minio_access_key: minio1234 minio_secret_key: minio1234 minio_volume_name: minio_backups @@ -41,4 +42,8 @@ env: MINIO_ACCESS_KEY: "{{ minio_access_key }}" MINIO_SECRET_KEY: "{{ minio_secret_key }}" - command: server /backups --address 0.0.0.0:9000 --console-address 0.0.0.0:9001 \ No newline at end of file + command: server /backups --address 0.0.0.0:9000 --console-address 0.0.0.0:9001 + + - name: Show the list of buckets + debug: + var: buckets \ No newline at end of file From 4c0d568d9fa21f9bd577f546456ecdc91607b657 Mon Sep 17 00:00:00 2001 From: Peter Sirotnak Date: Sun, 8 Jun 2025 14:09:13 +0200 Subject: [PATCH 19/41] PMM-7: Minio --- pmm_qa/tasks/create_minio_container.yml | 13 ++++++++++++- 1 file changed, 12 insertions(+), 1 deletion(-) diff --git a/pmm_qa/tasks/create_minio_container.yml b/pmm_qa/tasks/create_minio_container.yml index 667df81f..44a6aae1 100644 --- a/pmm_qa/tasks/create_minio_container.yml +++ b/pmm_qa/tasks/create_minio_container.yml @@ -46,4 +46,15 @@ - name: Show the list of buckets debug: - var: buckets \ No newline at end of file + var: buckets + + - name: Create MinIO buckets + community.docker.docker_container_exec: + container: "minio" + command: > + /bin/sh -c " + sleep 5; + /usr/bin/mc alias set myminio http://minio:9000 minio1234 minio1234; + /usr/bin/mc mb myminio/{{ item }}; + exit 0;" + loop: "{{ buckets }}" From 2085ac0ec222a3302b7ff0332bfe8329264d04ad Mon Sep 17 00:00:00 2001 From: Peter Sirotnak Date: Sun, 8 Jun 2025 14:11:56 +0200 Subject: [PATCH 20/41] PMM-7: Minio --- pmm_qa/tasks/create_minio_container.yml | 13 ++++++++++--- 1 file changed, 10 insertions(+), 3 deletions(-) diff --git a/pmm_qa/tasks/create_minio_container.yml b/pmm_qa/tasks/create_minio_container.yml index 44a6aae1..18dd05b9 100644 --- a/pmm_qa/tasks/create_minio_container.yml +++ b/pmm_qa/tasks/create_minio_container.yml @@ -48,13 +48,20 @@ debug: var: buckets - - name: Create MinIO buckets + - name: Set MinIO alias inside the container community.docker.docker_container_exec: container: "minio" command: > /bin/sh -c " sleep 5; /usr/bin/mc alias set myminio http://minio:9000 minio1234 minio1234; - /usr/bin/mc mb myminio/{{ item }}; exit 0;" - loop: "{{ buckets }}" + + - name: Create MinIO buckets + community.docker.docker_container_exec: + container: "minio" + command: > + /bin/sh -c " + /usr/bin/mc mb myminio/{{ item }} || echo 'Bucket {{ item }} already exists'; + exit 0;" + loop: "{{ buckets }}" From 7ac3513b9ee2ea844232e8c97a52835a433b6e07 Mon Sep 17 00:00:00 2001 From: Peter Sirotnak Date: Sun, 8 Jun 2025 14:13:18 +0200 Subject: [PATCH 21/41] PMM-7: Minio --- pmm_qa/tasks/create_minio_container.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/pmm_qa/tasks/create_minio_container.yml b/pmm_qa/tasks/create_minio_container.yml index 18dd05b9..ed8162c0 100644 --- a/pmm_qa/tasks/create_minio_container.yml +++ b/pmm_qa/tasks/create_minio_container.yml @@ -54,7 +54,7 @@ command: > /bin/sh -c " sleep 5; - /usr/bin/mc alias set myminio http://minio:9000 minio1234 minio1234; + /usr/bin/mc alias set myminio http://127.0.0.1:9000 minio1234 minio1234; exit 0;" - name: Create MinIO buckets From 6ba7e6e7510252789f86987eb3942becccfb060e Mon Sep 17 00:00:00 2001 From: Peter Sirotnak Date: Sun, 8 Jun 2025 15:49:35 +0200 Subject: [PATCH 22/41] PMM-7: Minio --- pmm_qa/tasks/create_minio_container.yml | 9 +++++++++ 1 file changed, 9 insertions(+) diff --git a/pmm_qa/tasks/create_minio_container.yml b/pmm_qa/tasks/create_minio_container.yml index ed8162c0..0e6cc1e5 100644 --- a/pmm_qa/tasks/create_minio_container.yml +++ b/pmm_qa/tasks/create_minio_container.yml @@ -4,6 +4,7 @@ connection: local gather_facts: yes vars: + network_name: "pmm-qa" buckets: "{{ lookup('env', 'BUCKETS') | default('bcp', true) }}" minio_access_key: minio1234 minio_secret_key: minio1234 @@ -13,6 +14,12 @@ - "9001:9001" tasks: + - name: Create Docker network + community.docker.docker_network: + name: "{{ network_name }}" + state: present + ignore_errors: yes + - name: Remove old MinIO docker container community.docker.docker_container: name: minio @@ -37,6 +44,8 @@ image: minio/minio restart_policy: unless-stopped ports: "{{ minio_ports }}" + networks: + - name: "{{ network_name }}" volumes: - "{{ minio_volume_name }}:/backups" env: From 9244f6f2fdf4ad224ae94a41afc979c08bafa105 Mon Sep 17 00:00:00 2001 From: Peter Sirotnak Date: Sun, 8 Jun 2025 16:20:41 +0200 Subject: [PATCH 23/41] PMM-7: Minio --- pmm_qa/tasks/create_minio_container.yml | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/pmm_qa/tasks/create_minio_container.yml b/pmm_qa/tasks/create_minio_container.yml index 0e6cc1e5..cda41e7c 100644 --- a/pmm_qa/tasks/create_minio_container.yml +++ b/pmm_qa/tasks/create_minio_container.yml @@ -14,6 +14,10 @@ - "9001:9001" tasks: + - name: List running Docker containers + community.docker.docker_container_info: + gather_facts: false + - name: Create Docker network community.docker.docker_network: name: "{{ network_name }}" From 932aad098135174de0636a2c4a0d777de6a40c61 Mon Sep 17 00:00:00 2001 From: Peter Sirotnak Date: Sun, 8 Jun 2025 16:31:49 +0200 Subject: [PATCH 24/41] PMM-7: Minio --- pmm_qa/tasks/create_minio_container.yml | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/pmm_qa/tasks/create_minio_container.yml b/pmm_qa/tasks/create_minio_container.yml index cda41e7c..60c4fa64 100644 --- a/pmm_qa/tasks/create_minio_container.yml +++ b/pmm_qa/tasks/create_minio_container.yml @@ -14,9 +14,9 @@ - "9001:9001" tasks: - - name: List running Docker containers - community.docker.docker_container_info: - gather_facts: false + - name: Run docker ps with port info + shell: docker ps -a + register: docker_ps_output - name: Create Docker network community.docker.docker_network: From 2183beb876839e50f059444b623206ae6c05120d Mon Sep 17 00:00:00 2001 From: Peter Sirotnak Date: Sun, 8 Jun 2025 16:46:22 +0200 Subject: [PATCH 25/41] PMM-7: Minio --- pmm_qa/tasks/create_minio_container.yml | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/pmm_qa/tasks/create_minio_container.yml b/pmm_qa/tasks/create_minio_container.yml index 60c4fa64..2e355b74 100644 --- a/pmm_qa/tasks/create_minio_container.yml +++ b/pmm_qa/tasks/create_minio_container.yml @@ -10,8 +10,8 @@ minio_secret_key: minio1234 minio_volume_name: minio_backups minio_ports: - - "9000:9000" - - "9001:9001" + - "90000:9000" + - "90001:9001" tasks: - name: Run docker ps with port info @@ -55,7 +55,7 @@ env: MINIO_ACCESS_KEY: "{{ minio_access_key }}" MINIO_SECRET_KEY: "{{ minio_secret_key }}" - command: server /backups --address 0.0.0.0:9000 --console-address 0.0.0.0:9001 + command: server /backups --address 0.0.0.0:90000 --console-address 0.0.0.0:90001 - name: Show the list of buckets debug: @@ -67,7 +67,7 @@ command: > /bin/sh -c " sleep 5; - /usr/bin/mc alias set myminio http://127.0.0.1:9000 minio1234 minio1234; + /usr/bin/mc alias set myminio http://127.0.0.1:90000 minio1234 minio1234; exit 0;" - name: Create MinIO buckets From 168b161f238e7a0e8bd73f8092f822c38375185e Mon Sep 17 00:00:00 2001 From: Peter Sirotnak Date: Sun, 8 Jun 2025 16:48:25 +0200 Subject: [PATCH 26/41] PMM-7: Minio --- pmm_qa/tasks/create_minio_container.yml | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/pmm_qa/tasks/create_minio_container.yml b/pmm_qa/tasks/create_minio_container.yml index 2e355b74..d5fc806d 100644 --- a/pmm_qa/tasks/create_minio_container.yml +++ b/pmm_qa/tasks/create_minio_container.yml @@ -55,7 +55,7 @@ env: MINIO_ACCESS_KEY: "{{ minio_access_key }}" MINIO_SECRET_KEY: "{{ minio_secret_key }}" - command: server /backups --address 0.0.0.0:90000 --console-address 0.0.0.0:90001 + command: server /backups --address 0.0.0.0:9000 --console-address 0.0.0.0:9001 - name: Show the list of buckets debug: @@ -67,7 +67,7 @@ command: > /bin/sh -c " sleep 5; - /usr/bin/mc alias set myminio http://127.0.0.1:90000 minio1234 minio1234; + /usr/bin/mc alias set myminio http://127.0.0.1:9000 minio1234 minio1234; exit 0;" - name: Create MinIO buckets From 1135804b606e71922d9d8c035d3bfe855f0eb456 Mon Sep 17 00:00:00 2001 From: Peter Sirotnak Date: Sun, 8 Jun 2025 16:50:03 +0200 Subject: [PATCH 27/41] PMM-7: Minio --- pmm_qa/tasks/create_minio_container.yml | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/pmm_qa/tasks/create_minio_container.yml b/pmm_qa/tasks/create_minio_container.yml index d5fc806d..326043c6 100644 --- a/pmm_qa/tasks/create_minio_container.yml +++ b/pmm_qa/tasks/create_minio_container.yml @@ -10,8 +10,8 @@ minio_secret_key: minio1234 minio_volume_name: minio_backups minio_ports: - - "90000:9000" - - "90001:9001" + - "9010:9000" + - "9001:9001" tasks: - name: Run docker ps with port info From fe762fb26cbee0d464398be88a3c348a93d600cf Mon Sep 17 00:00:00 2001 From: Peter Sirotnak Date: Mon, 9 Jun 2025 07:58:37 +0200 Subject: [PATCH 28/41] PMM-7: Minio --- pmm_qa/scripts/pgsql_load.sql | 24 ++++++++++++++++++++++++ 1 file changed, 24 insertions(+) diff --git a/pmm_qa/scripts/pgsql_load.sql b/pmm_qa/scripts/pgsql_load.sql index bd70b7f2..81a4fa90 100644 --- a/pmm_qa/scripts/pgsql_load.sql +++ b/pmm_qa/scripts/pgsql_load.sql @@ -43,6 +43,30 @@ INSERT INTO enrollments (student_id, class_id) VALUES (3, 1), (3, 3); +-- ======================================== +-- SIMULATE DEAD TUPLES +-- ======================================== + + +INSERT INTO students (first_name, last_name, birth_date) +SELECT 'John', 'Doe', CURRENT_DATE - (random() * 5000)::int +FROM generate_series(1, 100000); + +-- These updates and deletes will create dead tuples + +-- Update records (old versions become dead) +UPDATE students +SET last_name = last_name || '_updated' +WHERE student_id IN (1, 2); + +-- Delete records (deleted rows become dead) +DELETE FROM enrollments +WHERE enrollment_id IN (SELECT enrollment_id FROM enrollments LIMIT 2); + +-- Disable autovacuum temporarily (for demo) +ALTER TABLE students SET (autovacuum_enabled = false); +ALTER TABLE enrollments SET (autovacuum_enabled = false); + -- ======================================== -- SELECT QUERIES -- ======================================== From 7f9258fe654aa4714c45cb7beb9db205c1d67851 Mon Sep 17 00:00:00 2001 From: Peter Sirotnak Date: Tue, 10 Jun 2025 09:02:26 +0200 Subject: [PATCH 29/41] PMM-7: Code cleanup --- pmm_psmdb-pbm_setup/docker-compose-rs.yaml | 2 +- pmm_psmdb-pbm_setup/docker-compose-sharded.yaml | 5 ++--- pmm_psmdb-pbm_setup/start-rs-only.sh | 2 -- pmm_psmdb-pbm_setup/start-sharded.sh | 3 --- pmm_qa/pmm-framework.py | 10 +++------- 5 files changed, 6 insertions(+), 16 deletions(-) diff --git a/pmm_psmdb-pbm_setup/docker-compose-rs.yaml b/pmm_psmdb-pbm_setup/docker-compose-rs.yaml index 817092c3..9a328eaf 100644 --- a/pmm_psmdb-pbm_setup/docker-compose-rs.yaml +++ b/pmm_psmdb-pbm_setup/docker-compose-rs.yaml @@ -200,7 +200,7 @@ services: environment: - "MINIO_ACCESS_KEY=minio1234" - "MINIO_SECRET_KEY=minio1234" - command: server /backups --address 0.0.0.0:9000 --console-address 0.0.0.0:9001 + command: server /backups createbucket: container_name: createbucket diff --git a/pmm_psmdb-pbm_setup/docker-compose-sharded.yaml b/pmm_psmdb-pbm_setup/docker-compose-sharded.yaml index 6f0b5ca4..4f4598a5 100644 --- a/pmm_psmdb-pbm_setup/docker-compose-sharded.yaml +++ b/pmm_psmdb-pbm_setup/docker-compose-sharded.yaml @@ -229,8 +229,7 @@ services: image: minio/minio container_name: minio ports: - - "9000:9000" - - "9001:9001" + - "9001:9000" networks: - test-network volumes: @@ -238,7 +237,7 @@ services: environment: - "MINIO_ACCESS_KEY=minio1234" - "MINIO_SECRET_KEY=minio1234" - command: server /backups --address 0.0.0.0:9000 --console-address 0.0.0.0:9001 + command: server /backups createbucket: container_name: createbucket diff --git a/pmm_psmdb-pbm_setup/start-rs-only.sh b/pmm_psmdb-pbm_setup/start-rs-only.sh index c038bdba..d7f95dfc 100755 --- a/pmm_psmdb-pbm_setup/start-rs-only.sh +++ b/pmm_psmdb-pbm_setup/start-rs-only.sh @@ -35,5 +35,3 @@ if [ $profile = "extra" ]; then fi bash -x ./configure-extra-agents.sh fi - -docker exec minio /bin/sh -c " sleep 5; /usr/bin/mc alias set myminio http://minio:9000 minio1234 minio1234; /usr/bin/mc mb myminio/bcp-replicaset; exit 0; " diff --git a/pmm_psmdb-pbm_setup/start-sharded.sh b/pmm_psmdb-pbm_setup/start-sharded.sh index 5ab2a50e..e63274eb 100755 --- a/pmm_psmdb-pbm_setup/start-sharded.sh +++ b/pmm_psmdb-pbm_setup/start-sharded.sh @@ -259,6 +259,3 @@ if [ $cleanup != "no" ]; then else echo "skipping cleanup" fi - -docker exec minio /bin/sh -c " sleep 5; /usr/bin/mc alias set myminio http://minio:9000 minio1234 minio1234; /usr/bin/mc mb myminio/bcp-sharded; exit 0; " -docker ps -a diff --git a/pmm_qa/pmm-framework.py b/pmm_qa/pmm-framework.py index 08c8d8d6..739dfdd1 100755 --- a/pmm_qa/pmm-framework.py +++ b/pmm_qa/pmm-framework.py @@ -470,7 +470,6 @@ def setup_mlaunch_psmdb(db_type, db_version=None, db_config=None, args=None): # Call the function to run the Ansible playbook run_ansible_playbook(playbook_filename, env_vars, args) - def setup_mlaunch_modb(db_type, db_version=None, db_config=None, args=None): # Check if PMM server is running container_name = get_running_container_name() @@ -600,6 +599,7 @@ def get_latest_psmdb_version(psmdb_version): # Extract the version number using regular expression version_number = [v.split('|')[0] for v in re.findall(r'value="([^"]*)"', response.text)] + if version_number: # Sort the version numbers and extract the latest one latest_version = sorted(version_number, key=lambda x: tuple(map(int, x.split('-')[-1].split('.'))))[-1] @@ -622,7 +622,7 @@ def setup_psmdb(db_type, db_version=None, db_config=None, args=None): # Gather Version details psmdb_version = os.getenv('PSMDB_VERSION') or get_latest_psmdb_version(db_version) or \ - database_configs[db_type]["versions"][-1] + database_configs[db_type]["versions"][-1] # Handle port address for external or internal address server_hostname = container_name @@ -689,9 +689,7 @@ def mongo_ssl_setup(script_filename, args): subprocess.run( ['cp', f'{scripts_path}docker-compose-pmm-psmdb.yml', f'{compose_file_path}']) admin_password = os.getenv('ADMIN_PASSWORD') or args.pmm_server_password or 'admin' - subprocess.run( - ['sed', '-i', f's/PMM_AGENT_SERVER_PASSWORD=admin/PMM_AGENT_SERVER_PASSWORD={admin_password}/g', - f'{compose_file_path}']) + subprocess.run(['sed', '-i', f's/PMM_AGENT_SERVER_PASSWORD=admin/PMM_AGENT_SERVER_PASSWORD={admin_password}/g', f'{compose_file_path}']) subprocess.run(['sed', '-i', '/container_name/a\ networks:\\\n \\- pmm-qa', f'{compose_file_path}']) subprocess.run(['sed', '-i', '$a\\\nnetworks:\\\n pmm-qa:\\\n name: pmm-qa\\\n external: true', f'{compose_file_path}']) @@ -912,8 +910,6 @@ def setup_bucket(args=None): print(f"Option {args.verbosity_level} is invalid verbosity level option, please provide number 1-5") exit(1) - print(args) - # Parse arguments try: if args.bucket: From db661f2c7fa2cca3a8e84cf8da3e0fd3284d4c3d Mon Sep 17 00:00:00 2001 From: Peter Sirotnak Date: Tue, 10 Jun 2025 09:04:03 +0200 Subject: [PATCH 30/41] PMM-7: Code cleanup --- pmm_qa/pmm-framework.py | 10 +++++----- 1 file changed, 5 insertions(+), 5 deletions(-) diff --git a/pmm_qa/pmm-framework.py b/pmm_qa/pmm-framework.py index 739dfdd1..7c225a86 100755 --- a/pmm_qa/pmm-framework.py +++ b/pmm_qa/pmm-framework.py @@ -92,6 +92,8 @@ def run_ansible_playbook(playbook_filename, env_vars, args): playbook_path = script_dir + "/" + playbook_filename verbosity_level = 1 + + if args.verbosity_level is not None: verbosity_level = int(args.verbosity_level) @@ -210,8 +212,7 @@ def setup_ps(db_type, db_version=None, db_config=None, args=None): 'PS_NODES': no_of_nodes, 'PS_VERSION': ps_version, 'PMM_SERVER_IP': args.pmm_server_ip or container_name or '127.0.0.1', - 'PS_CONTAINER': 'ps_pmm_' + str(ps_version) + ( - '_replica' if setup_type_value in ("replication", "replica") else ''), + 'PS_CONTAINER': 'ps_pmm_' + str(ps_version) + ('_replica' if setup_type_value in ("replication", "replica") else ''), 'PS_PORT': 3318 if setup_type_value in ("replication", "replica") else 3317, 'CLIENT_VERSION': get_value('CLIENT_VERSION', db_type, args, db_config), 'QUERY_SOURCE': get_value('QUERY_SOURCE', db_type, args, db_config), @@ -226,7 +227,6 @@ def setup_ps(db_type, db_version=None, db_config=None, args=None): # Call the function to run the Ansible playbook run_ansible_playbook(playbook_filename, env_vars, args) - def setup_mysql(db_type, db_version=None, db_config=None, args=None): # Check if PMM server is running container_name = get_running_container_name() @@ -479,7 +479,7 @@ def setup_mlaunch_modb(db_type, db_version=None, db_config=None, args=None): # Gather Version details modb_version = os.getenv('MODB_VERSION') or db_version or \ - database_configs[db_type]["versions"][-1] + database_configs[db_type]["versions"][-1] # Define environment variables for playbook env_vars = { @@ -622,7 +622,7 @@ def setup_psmdb(db_type, db_version=None, db_config=None, args=None): # Gather Version details psmdb_version = os.getenv('PSMDB_VERSION') or get_latest_psmdb_version(db_version) or \ - database_configs[db_type]["versions"][-1] + database_configs[db_type]["versions"][-1] # Handle port address for external or internal address server_hostname = container_name From dbea822495a2f15c34654cfaff6c6e14bf1c482c Mon Sep 17 00:00:00 2001 From: Peter Sirotnak Date: Tue, 10 Jun 2025 11:13:04 +0200 Subject: [PATCH 31/41] PMM-7: Default value for bucket --- pmm_qa/pmm-framework.py | 3 +++ 1 file changed, 3 insertions(+) diff --git a/pmm_qa/pmm-framework.py b/pmm_qa/pmm-framework.py index 7c225a86..ac869822 100755 --- a/pmm_qa/pmm-framework.py +++ b/pmm_qa/pmm-framework.py @@ -82,6 +82,9 @@ "DOCKERCLIENTS": { "configurations": {} # Empty dictionary for consistency }, + "BUCKET": { + "configurations": { "BUCKET_NAME": 'bcp' } + } } From f689b3a0b6e063d98235e719a50af23385665be3 Mon Sep 17 00:00:00 2001 From: Peter Sirotnak Date: Tue, 10 Jun 2025 11:14:38 +0200 Subject: [PATCH 32/41] PMM-7: Default value for bucket --- pmm_qa/pmm-framework.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/pmm_qa/pmm-framework.py b/pmm_qa/pmm-framework.py index ac869822..667dc73a 100755 --- a/pmm_qa/pmm-framework.py +++ b/pmm_qa/pmm-framework.py @@ -83,7 +83,7 @@ "configurations": {} # Empty dictionary for consistency }, "BUCKET": { - "configurations": { "BUCKET_NAME": 'bcp' } + "configurations": { "BUCKET_NAMES": 'bcp' } } } From b7ff844875ce2ed660838c615fcc7f3838759b6f Mon Sep 17 00:00:00 2001 From: Peter Sirotnak Date: Tue, 10 Jun 2025 11:17:07 +0200 Subject: [PATCH 33/41] PMM-7: Default value for bucket --- pmm_qa/pmm-framework.py | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/pmm_qa/pmm-framework.py b/pmm_qa/pmm-framework.py index 667dc73a..5da0b667 100755 --- a/pmm_qa/pmm-framework.py +++ b/pmm_qa/pmm-framework.py @@ -870,7 +870,8 @@ def setup_database(db_type, db_version=None, db_config=None, args=None): setup_mlaunch_modb(db_type, db_version, db_config, args) elif db_type == 'SSL_MLAUNCH': setup_ssl_mlaunch(db_type, db_version, db_config, args) - + elif db_type == 'BUCKET': + setup_bucket(args) else: print(f"Database type {db_type} is not recognised, Exiting...") exit(1) From 1f9ed10cb92f2fd8fed026a9dd3d82b38da14d7f Mon Sep 17 00:00:00 2001 From: Peter Sirotnak Date: Tue, 10 Jun 2025 11:19:55 +0200 Subject: [PATCH 34/41] PMM-7: Default value for bucket --- pmm_qa/pmm-framework.py | 1 + 1 file changed, 1 insertion(+) diff --git a/pmm_qa/pmm-framework.py b/pmm_qa/pmm-framework.py index 5da0b667..860883a7 100755 --- a/pmm_qa/pmm-framework.py +++ b/pmm_qa/pmm-framework.py @@ -871,6 +871,7 @@ def setup_database(db_type, db_version=None, db_config=None, args=None): elif db_type == 'SSL_MLAUNCH': setup_ssl_mlaunch(db_type, db_version, db_config, args) elif db_type == 'BUCKET': + print(f'Args Are: {args}') setup_bucket(args) else: print(f"Database type {db_type} is not recognised, Exiting...") From 53769196709fd682c08f6d784ccc948f2f2f3d62 Mon Sep 17 00:00:00 2001 From: Peter Sirotnak Date: Tue, 10 Jun 2025 11:22:08 +0200 Subject: [PATCH 35/41] PMM-7: Default value for bucket --- pmm_qa/pmm-framework.py | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/pmm_qa/pmm-framework.py b/pmm_qa/pmm-framework.py index 860883a7..be372fab 100755 --- a/pmm_qa/pmm-framework.py +++ b/pmm_qa/pmm-framework.py @@ -880,7 +880,8 @@ def setup_database(db_type, db_version=None, db_config=None, args=None): def setup_bucket(args=None): print("Setting up bucket") - print(args.bucket) + bucket_names_value = get_value('BUCKET_NAMES', db_type, args, db_config).lower() + print(bucket_names_value) env_vars = { 'BUCKETS': args.bucket } From bbbd4a6fb0121670c0a7632e6bbdbd182874a6c3 Mon Sep 17 00:00:00 2001 From: Peter Sirotnak Date: Tue, 10 Jun 2025 11:22:44 +0200 Subject: [PATCH 36/41] PMM-7: Default value for bucket --- pmm_qa/pmm-framework.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/pmm_qa/pmm-framework.py b/pmm_qa/pmm-framework.py index be372fab..c7d660fc 100755 --- a/pmm_qa/pmm-framework.py +++ b/pmm_qa/pmm-framework.py @@ -880,7 +880,7 @@ def setup_database(db_type, db_version=None, db_config=None, args=None): def setup_bucket(args=None): print("Setting up bucket") - bucket_names_value = get_value('BUCKET_NAMES', db_type, args, db_config).lower() + bucket_names_value = get_value('BUCKET_NAMES', '', args, '').lower() print(bucket_names_value) env_vars = { 'BUCKETS': args.bucket From f91fe8ed2e9a3596cd29a50507302cab90851f51 Mon Sep 17 00:00:00 2001 From: Peter Sirotnak Date: Tue, 10 Jun 2025 11:24:17 +0200 Subject: [PATCH 37/41] PMM-7: Default value for bucket --- pmm_qa/pmm-framework.py | 102 ++++++++++++++++++++-------------------- 1 file changed, 51 insertions(+), 51 deletions(-) diff --git a/pmm_qa/pmm-framework.py b/pmm_qa/pmm-framework.py index c7d660fc..3a7a1586 100755 --- a/pmm_qa/pmm-framework.py +++ b/pmm_qa/pmm-framework.py @@ -83,7 +83,7 @@ "configurations": {} # Empty dictionary for consistency }, "BUCKET": { - "configurations": { "BUCKET_NAMES": 'bcp' } + "configurations": {"BUCKET_NAMES": 'bcp'} } } @@ -95,8 +95,6 @@ def run_ansible_playbook(playbook_filename, env_vars, args): playbook_path = script_dir + "/" + playbook_filename verbosity_level = 1 - - if args.verbosity_level is not None: verbosity_level = int(args.verbosity_level) @@ -215,7 +213,8 @@ def setup_ps(db_type, db_version=None, db_config=None, args=None): 'PS_NODES': no_of_nodes, 'PS_VERSION': ps_version, 'PMM_SERVER_IP': args.pmm_server_ip or container_name or '127.0.0.1', - 'PS_CONTAINER': 'ps_pmm_' + str(ps_version) + ('_replica' if setup_type_value in ("replication", "replica") else ''), + 'PS_CONTAINER': 'ps_pmm_' + str(ps_version) + ( + '_replica' if setup_type_value in ("replication", "replica") else ''), 'PS_PORT': 3318 if setup_type_value in ("replication", "replica") else 3317, 'CLIENT_VERSION': get_value('CLIENT_VERSION', db_type, args, db_config), 'QUERY_SOURCE': get_value('QUERY_SOURCE', db_type, args, db_config), @@ -230,6 +229,7 @@ def setup_ps(db_type, db_version=None, db_config=None, args=None): # Call the function to run the Ansible playbook run_ansible_playbook(playbook_filename, env_vars, args) + def setup_mysql(db_type, db_version=None, db_config=None, args=None): # Check if PMM server is running container_name = get_running_container_name() @@ -473,6 +473,7 @@ def setup_mlaunch_psmdb(db_type, db_version=None, db_config=None, args=None): # Call the function to run the Ansible playbook run_ansible_playbook(playbook_filename, env_vars, args) + def setup_mlaunch_modb(db_type, db_version=None, db_config=None, args=None): # Check if PMM server is running container_name = get_running_container_name() @@ -482,7 +483,7 @@ def setup_mlaunch_modb(db_type, db_version=None, db_config=None, args=None): # Gather Version details modb_version = os.getenv('MODB_VERSION') or db_version or \ - database_configs[db_type]["versions"][-1] + database_configs[db_type]["versions"][-1] # Define environment variables for playbook env_vars = { @@ -602,7 +603,6 @@ def get_latest_psmdb_version(psmdb_version): # Extract the version number using regular expression version_number = [v.split('|')[0] for v in re.findall(r'value="([^"]*)"', response.text)] - if version_number: # Sort the version numbers and extract the latest one latest_version = sorted(version_number, key=lambda x: tuple(map(int, x.split('-')[-1].split('.'))))[-1] @@ -692,7 +692,9 @@ def mongo_ssl_setup(script_filename, args): subprocess.run( ['cp', f'{scripts_path}docker-compose-pmm-psmdb.yml', f'{compose_file_path}']) admin_password = os.getenv('ADMIN_PASSWORD') or args.pmm_server_password or 'admin' - subprocess.run(['sed', '-i', f's/PMM_AGENT_SERVER_PASSWORD=admin/PMM_AGENT_SERVER_PASSWORD={admin_password}/g', f'{compose_file_path}']) + subprocess.run( + ['sed', '-i', f's/PMM_AGENT_SERVER_PASSWORD=admin/PMM_AGENT_SERVER_PASSWORD={admin_password}/g', + f'{compose_file_path}']) subprocess.run(['sed', '-i', '/container_name/a\ networks:\\\n \\- pmm-qa', f'{compose_file_path}']) subprocess.run(['sed', '-i', '$a\\\nnetworks:\\\n pmm-qa:\\\n name: pmm-qa\\\n external: true', f'{compose_file_path}']) @@ -872,15 +874,15 @@ def setup_database(db_type, db_version=None, db_config=None, args=None): setup_ssl_mlaunch(db_type, db_version, db_config, args) elif db_type == 'BUCKET': print(f'Args Are: {args}') - setup_bucket(args) + setup_bucket(db_type, db_version, db_config, args) else: print(f"Database type {db_type} is not recognised, Exiting...") exit(1) -def setup_bucket(args=None): +def setup_bucket(db_type, db_version=None, db_config=None, args=None): print("Setting up bucket") - bucket_names_value = get_value('BUCKET_NAMES', '', args, '').lower() + bucket_names_value = get_value('BUCKET_NAMES', db_type, args, db_config).lower() print(bucket_names_value) env_vars = { 'BUCKETS': args.bucket @@ -888,6 +890,7 @@ def setup_bucket(args=None): run_ansible_playbook('tasks/create_minio_container.yml', env_vars, args) + # Main if __name__ == "__main__": parser = argparse.ArgumentParser(description='PMM Framework Script to setup Multiple Databases', @@ -918,48 +921,45 @@ def setup_bucket(args=None): # Parse arguments try: - if args.bucket: - setup_bucket(args) - if args.database: - for db in args.database: - db_parts = db[0].split(',') - configs = db_parts[0:] if len(db_parts) > 1 else db[0:] - db_type = None - db_version = None - db_config = {} - - if configs: - for config in configs: - if "=" in config: - key, value = config.split('=') + for db in args.database: + db_parts = db[0].split(',') + configs = db_parts[0:] if len(db_parts) > 1 else db[0:] + db_type = None + db_version = None + db_config = {} + + if configs: + for config in configs: + if "=" in config: + key, value = config.split('=') + else: + key, value = config, None + + # Convert all arguments/options only to uppercase + key = key.upper() + + try: + if key in database_configs: + db_type = key + if "versions" in database_configs[db_type]: + if value in database_configs[db_type]["versions"]: + db_version = value + else: + if args.verbose: + print( + f"Value {value} is not recognised for Option {key}, will be using Default value") + elif key in database_configs[db_type]["configurations"]: + db_config[key] = value else: - key, value = config, None - - # Convert all arguments/options only to uppercase - key = key.upper() - - try: - if key in database_configs: - db_type = key - if "versions" in database_configs[db_type]: - if value in database_configs[db_type]["versions"]: - db_version = value - else: - if args.verbose: - print( - f"Value {value} is not recognised for Option {key}, will be using Default value") - elif key in database_configs[db_type]["configurations"]: - db_config[key] = value - else: - if args.verbose: - print(f"Option {key} is not recognised, will be using default option") - continue - except KeyError as e: - print(f"Option {key} is not recognised with error {e}, Please check and try again") - parser.print_help() - exit(1) - # Set up the specified databases - setup_database(db_type, db_version, db_config, args) + if args.verbose: + print(f"Option {key} is not recognised, will be using default option") + continue + except KeyError as e: + print(f"Option {key} is not recognised with error {e}, Please check and try again") + parser.print_help() + exit(1) + # Set up the specified databases + setup_database(db_type, db_version, db_config, args) except argparse.ArgumentError as e: print(f"Option is not recognised:", e) parser.print_help() From 48e47d8dce0e5af012d26b02f054dd2f12edaefb Mon Sep 17 00:00:00 2001 From: Peter Sirotnak Date: Tue, 10 Jun 2025 11:25:42 +0200 Subject: [PATCH 38/41] PMM-7: Default value for bucket --- pmm_qa/pmm-framework.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/pmm_qa/pmm-framework.py b/pmm_qa/pmm-framework.py index 3a7a1586..0c9fe70b 100755 --- a/pmm_qa/pmm-framework.py +++ b/pmm_qa/pmm-framework.py @@ -885,7 +885,7 @@ def setup_bucket(db_type, db_version=None, db_config=None, args=None): bucket_names_value = get_value('BUCKET_NAMES', db_type, args, db_config).lower() print(bucket_names_value) env_vars = { - 'BUCKETS': args.bucket + 'BUCKETS': bucket_names_value } run_ansible_playbook('tasks/create_minio_container.yml', env_vars, args) From f5c3e1efb93446ea0186a4c738801b6f114f7fab Mon Sep 17 00:00:00 2001 From: Peter Sirotnak Date: Tue, 10 Jun 2025 11:28:31 +0200 Subject: [PATCH 39/41] PMM-7: Default value for bucket --- pmm_qa/pmm-framework.py | 2 ++ 1 file changed, 2 insertions(+) diff --git a/pmm_qa/pmm-framework.py b/pmm_qa/pmm-framework.py index 0c9fe70b..129c7fac 100755 --- a/pmm_qa/pmm-framework.py +++ b/pmm_qa/pmm-framework.py @@ -884,6 +884,8 @@ def setup_bucket(db_type, db_version=None, db_config=None, args=None): print("Setting up bucket") bucket_names_value = get_value('BUCKET_NAMES', db_type, args, db_config).lower() print(bucket_names_value) + bucket_names_value = bucket_names_value if isinstance(bucket_names_value, list) else [bucket_names_value] + print(bucket_names_value) env_vars = { 'BUCKETS': bucket_names_value } From 8aa72138af14b0c3ec8b46746863e42e330aace1 Mon Sep 17 00:00:00 2001 From: Peter Sirotnak Date: Tue, 10 Jun 2025 11:42:15 +0200 Subject: [PATCH 40/41] PMM-7: Default value for bucket --- pmm_qa/pmm-framework.py | 4 +--- 1 file changed, 1 insertion(+), 3 deletions(-) diff --git a/pmm_qa/pmm-framework.py b/pmm_qa/pmm-framework.py index 129c7fac..d2b583b5 100755 --- a/pmm_qa/pmm-framework.py +++ b/pmm_qa/pmm-framework.py @@ -882,9 +882,7 @@ def setup_database(db_type, db_version=None, db_config=None, args=None): def setup_bucket(db_type, db_version=None, db_config=None, args=None): print("Setting up bucket") - bucket_names_value = get_value('BUCKET_NAMES', db_type, args, db_config).lower() - print(bucket_names_value) - bucket_names_value = bucket_names_value if isinstance(bucket_names_value, list) else [bucket_names_value] + bucket_names_value = get_value('BUCKET_NAMES', db_type, args, db_config).lower().split(';') print(bucket_names_value) env_vars = { 'BUCKETS': bucket_names_value From 799d8c415610e1c2aae40630d165c88e13d65e5a Mon Sep 17 00:00:00 2001 From: Peter Sirotnak Date: Tue, 10 Jun 2025 11:44:42 +0200 Subject: [PATCH 41/41] PMM-7: Default value for bucket --- pmm_qa/pmm-framework.py | 7 ++----- 1 file changed, 2 insertions(+), 5 deletions(-) diff --git a/pmm_qa/pmm-framework.py b/pmm_qa/pmm-framework.py index d2b583b5..edca1182 100755 --- a/pmm_qa/pmm-framework.py +++ b/pmm_qa/pmm-framework.py @@ -603,6 +603,7 @@ def get_latest_psmdb_version(psmdb_version): # Extract the version number using regular expression version_number = [v.split('|')[0] for v in re.findall(r'value="([^"]*)"', response.text)] + if version_number: # Sort the version numbers and extract the latest one latest_version = sorted(version_number, key=lambda x: tuple(map(int, x.split('-')[-1].split('.'))))[-1] @@ -692,9 +693,7 @@ def mongo_ssl_setup(script_filename, args): subprocess.run( ['cp', f'{scripts_path}docker-compose-pmm-psmdb.yml', f'{compose_file_path}']) admin_password = os.getenv('ADMIN_PASSWORD') or args.pmm_server_password or 'admin' - subprocess.run( - ['sed', '-i', f's/PMM_AGENT_SERVER_PASSWORD=admin/PMM_AGENT_SERVER_PASSWORD={admin_password}/g', - f'{compose_file_path}']) + subprocess.run(['sed', '-i', f's/PMM_AGENT_SERVER_PASSWORD=admin/PMM_AGENT_SERVER_PASSWORD={admin_password}/g', f'{compose_file_path}']) subprocess.run(['sed', '-i', '/container_name/a\ networks:\\\n \\- pmm-qa', f'{compose_file_path}']) subprocess.run(['sed', '-i', '$a\\\nnetworks:\\\n pmm-qa:\\\n name: pmm-qa\\\n external: true', f'{compose_file_path}']) @@ -873,7 +872,6 @@ def setup_database(db_type, db_version=None, db_config=None, args=None): elif db_type == 'SSL_MLAUNCH': setup_ssl_mlaunch(db_type, db_version, db_config, args) elif db_type == 'BUCKET': - print(f'Args Are: {args}') setup_bucket(db_type, db_version, db_config, args) else: print(f"Database type {db_type} is not recognised, Exiting...") @@ -912,7 +910,6 @@ def setup_bucket(db_type, db_version=None, db_config=None, args=None): parser.add_argument("--client-version", nargs='?', help='PMM Client version/tarball') parser.add_argument("--verbose", "--v", action='store_true', help='Display verbose information') parser.add_argument("--verbosity-level", nargs='?', help='Display verbose information level') - parser.add_argument("--bucket", nargs='+', help='Create MinIO S3 compatible bucket') args = parser.parse_args() if args.verbosity_level is not None and not args.verbosity_level.isnumeric():