diff --git a/pmm_qa/pmm-framework.py b/pmm_qa/pmm-framework.py index 24f4b769..7b477332 100755 --- a/pmm_qa/pmm-framework.py +++ b/pmm_qa/pmm-framework.py @@ -800,6 +800,7 @@ def setup_valkey(db_type, db_version=None, db_config=None, args=None): # Gather Version details valkey_version = os.getenv('VALKEY_VERSION') or db_version or database_configs[db_type]["versions"][-1] + setup_type_value = get_value('SETUP_TYPE', db_type, args, db_config).lower() # Define environment variables for playbook env_vars = { @@ -807,11 +808,15 @@ def setup_valkey(db_type, db_version=None, db_config=None, args=None): 'VALKEY_VERSION': valkey_version, 'CLIENT_VERSION': get_value('CLIENT_VERSION', db_type, args, db_config), 'ADMIN_PASSWORD': os.getenv('ADMIN_PASSWORD') or args.pmm_server_password or 'admin', - 'PMM_QA_GIT_BRANCH': os.getenv('PMM_QA_GIT_BRANCH') or 'v3' + 'PMM_QA_GIT_BRANCH': os.getenv('PMM_QA_GIT_BRANCH') or 'v3', + 'SETUP_TYPE': setup_type_value } - # Ansible playbook filename - playbook_filename = 'valkey/valkey.yml' + # Choose playbook based on SETUP_TYPE (cluster is default; sentinel only when explicitly requested) + if setup_type_value in ("sentinel", "sentinels"): + playbook_filename = 'valkey/valkey-sentinel.yml' + else: + playbook_filename = 'valkey/valkey-cluster.yml' # Call the function to run the Ansible playbook run_ansible_playbook(playbook_filename, env_vars, args) diff --git a/pmm_qa/valkey/valkey-cluster.yml b/pmm_qa/valkey/valkey-cluster.yml new file mode 100644 index 00000000..56208a0d --- /dev/null +++ b/pmm_qa/valkey/valkey-cluster.yml @@ -0,0 +1,259 @@ +--- +- name: Deploy Valkey Native Cluster + hosts: localhost + gather_facts: false + vars: + pmm_server_ip: "{{ lookup('vars', 'extra_pmm_server_ip', default=lookup('env','PMM_SERVER_IP') | default('127.0.0.1', true) ) }}" + metrics_mode: "{{ lookup('env', 'metrics_mode') }}" + client_version: "{{ lookup('vars', 'extra_client_version', default=lookup('env','CLIENT_VERSION') | default('3-dev-latest', true) ) }}" + admin_password: "{{ lookup('vars', 'extra_admin_password', default=lookup('env','ADMIN_PASSWORD') | default('admin', true) ) }}" + valkey_version: "{{ lookup('env', 'VALKEY_VERSION') | default('7', true) }}" + valkey_image: "valkey/valkey:{{ valkey_version }}" + valkey_network_name: "pmm-qa" + valkey_password: "VKvl41568AsE" + valkey_cluster_node_count: 6 # total nodes + valkey_cluster_primaries: 3 # number of primary nodes + valkey_cluster_replicas: 1 # replicas per primary + valkey_cluster_start_port: 6379 # Base host port to map sequentially + valkey_config_dir: "{{ lookup('env', 'HOME') }}/valkey/cluster-config" + valkey_primary_prefix: "valkey-primary-" + valkey_replica_prefix: "valkey-replica-" + pmm_server_name: "pmm-server" + + tasks: + - name: Set Random Number Fact + set_fact: + random_number: "{{ (10000 | random) | int }}" + + - name: Validate cluster counts + assert: + that: + - valkey_cluster_node_count == valkey_cluster_primaries + (valkey_cluster_primaries * valkey_cluster_replicas) + fail_msg: "Mismatch: total nodes must equal primaries + primaries*replicas" + + - name: Create Docker network + community.docker.docker_network: + name: "{{ valkey_network_name }}" + driver: bridge + state: present + + - name: Create cluster config directory + file: + path: "{{ valkey_config_dir }}" + state: directory + mode: "0755" + + - name: Create per-primary config directories + file: + path: "{{ valkey_config_dir }}/{{ valkey_primary_prefix }}{{ item }}" + state: directory + mode: "0755" + loop: "{{ range(1, valkey_cluster_primaries + 1) | list }}" + + - name: Create per-replica config directories + file: + path: "{{ valkey_config_dir }}/{{ valkey_replica_prefix }}{{ item }}" + state: directory + mode: "0755" + loop: "{{ range(valkey_cluster_primaries + 1, valkey_cluster_node_count + 1) | list }}" + + - name: Generate base configuration for each primary + copy: + dest: "{{ valkey_config_dir }}/{{ valkey_primary_prefix }}{{ item }}/valkey.conf" + mode: "0644" + content: | + port 6379 + requirepass {{ valkey_password }} + masterauth {{ valkey_password }} + protected-mode no + appendonly yes + cluster-enabled yes + cluster-config-file nodes.conf + cluster-node-timeout 5000 + # Minimal persistence & logging + save 900 1 + save 300 10 + save 60 10000 + loglevel notice + loop: "{{ range(1, valkey_cluster_primaries + 1) | list }}" + + - name: Generate base configuration for each replica + copy: + dest: "{{ valkey_config_dir }}/{{ valkey_replica_prefix }}{{ item }}/valkey.conf" + mode: "0644" + content: | + port 6379 + requirepass {{ valkey_password }} + masterauth {{ valkey_password }} + protected-mode no + appendonly yes + cluster-enabled yes + cluster-config-file nodes.conf + cluster-node-timeout 5000 + # Minimal persistence & logging + save 900 1 + save 300 10 + save 60 10000 + loglevel notice + loop: "{{ range(valkey_cluster_primaries + 1, valkey_cluster_node_count + 1) | list }}" + + - name: Create docker volumes for primary nodes + community.docker.docker_volume: + name: "{{ valkey_primary_prefix }}{{ item }}-data" + state: present + loop: "{{ range(1, valkey_cluster_primaries + 1) | list }}" + + - name: Create docker volumes for replica nodes + community.docker.docker_volume: + name: "{{ valkey_replica_prefix }}{{ item }}-data" + state: present + loop: "{{ range(valkey_cluster_primaries + 1, valkey_cluster_node_count + 1) | list }}" + + - name: Start primary node containers + community.docker.docker_container: + name: "{{ valkey_primary_prefix }}{{ item }}" + hostname: "{{ valkey_primary_prefix }}{{ item }}-node-{{ random_number }}" + image: "{{ valkey_image }}" + state: started + restart_policy: unless-stopped + networks: + - name: "{{ valkey_network_name }}" + ports: + - "{{ valkey_cluster_start_port + item - 1 }}:6379" + volumes: + - "{{ valkey_primary_prefix }}{{ item }}-data:/data" + - "{{ valkey_config_dir }}/{{ valkey_primary_prefix }}{{ item }}/valkey.conf:/usr/local/etc/valkey/valkey.conf:ro" + command: ["valkey-server", "/usr/local/etc/valkey/valkey.conf"] + healthcheck: + test: ["CMD", "valkey-cli", "-a", "{{ valkey_password }}", "ping"] + interval: 10s + timeout: 5s + retries: 5 + loop: "{{ range(1, valkey_cluster_primaries + 1) | list }}" + + - name: Start replica node containers + community.docker.docker_container: + name: "{{ valkey_replica_prefix }}{{ item }}" + hostname: "{{ valkey_replica_prefix }}{{ item }}-node-{{ random_number }}" + image: "{{ valkey_image }}" + state: started + restart_policy: unless-stopped + networks: + - name: "{{ valkey_network_name }}" + ports: + - "{{ (valkey_cluster_start_port + valkey_cluster_primaries - 1) + item }}:6379" + volumes: + - "{{ valkey_replica_prefix }}{{ item }}-data:/data" + - "{{ valkey_config_dir }}/{{ valkey_replica_prefix }}{{ item }}/valkey.conf:/usr/local/etc/valkey/valkey.conf:ro" + command: ["valkey-server", "/usr/local/etc/valkey/valkey.conf"] + healthcheck: + test: ["CMD", "valkey-cli", "-a", "{{ valkey_password }}", "ping"] + interval: 10s + timeout: 5s + retries: 5 + loop: "{{ range(valkey_cluster_primaries + 1, valkey_cluster_node_count + 1) | list }}" + + - name: Wait for primary node ports + wait_for: + host: localhost + port: "{{ valkey_cluster_start_port + item - 1 }}" + timeout: 30 + delay: 1 + loop: "{{ range(1, valkey_cluster_primaries + 1) | list }}" + + - name: Wait for replica node ports + wait_for: + host: localhost + port: "{{ (valkey_cluster_start_port + valkey_cluster_primaries - 1) + item }}" + timeout: 30 + delay: 1 + loop: "{{ range(valkey_cluster_primaries + 1, valkey_cluster_node_count + 1) | list }}" + + - name: Build list of internal container addresses + set_fact: + primary_nodes: "{{ range(1, valkey_cluster_primaries + 1) | map('string') | map('regex_replace', '^(.*)$', valkey_primary_prefix ~ '\\1') | list }}" + replica_nodes: "{{ range(valkey_cluster_primaries + 1, valkey_cluster_node_count + 1) | map('string') | map('regex_replace', '^(.*)$', valkey_replica_prefix ~ '\\1') | list }}" + + - name: Build combined list of internal container addresses + set_fact: + cluster_node_addresses: "{{ (primary_nodes | map('regex_replace', '^(.*)$', '\\1:6379') | list) + (replica_nodes | map('regex_replace', '^(.*)$', '\\1:6379') | list) }}" + + - name: Display cluster node addresses + debug: + var: cluster_node_addresses + + - name: Create the cluster (run once) + community.docker.docker_container_exec: + container: "{{ valkey_primary_prefix }}1" + command: >- + bash -c "yes 'yes' | valkey-cli --cluster create {{ cluster_node_addresses | join(' ') }} --cluster-replicas {{ valkey_cluster_replicas }} -a {{ valkey_password }}" + register: cluster_create_output + changed_when: "'[OK]' in cluster_create_output.stdout" + + - name: Show cluster creation output + debug: + msg: "{{ cluster_create_output.stdout_lines }}" + + - name: Check cluster info on first primary + community.docker.docker_container_exec: + container: "{{ valkey_primary_prefix }}1" + command: valkey-cli -a "{{ valkey_password }}" cluster info + register: cluster_info + + - name: Display cluster info + debug: + msg: "{{ cluster_info.stdout_lines }}" + + - name: Install PMM Client in each container + ansible.builtin.include_tasks: ../tasks/install_pmm_client.yml + loop: "{{ primary_nodes + replica_nodes }}" + loop_control: + loop_var: current_container_name + vars: + container_name: "{{ current_container_name }}" + + - name: Add primary nodes to monitoring + community.docker.docker_container_exec: + container: "{{ item }}" + command: >- + pmm-admin add valkey --cluster=valkey-native-cluster --environment=valkey-test --username=default + --password="{{ valkey_password }}" --service-name={{ item }}-svc-{{ random_number }} + --host={{ item }} --port=6379 --custom-labels='role=primary' + loop: "{{ primary_nodes }}" + ignore_errors: yes + + - name: Add replica nodes to monitoring + community.docker.docker_container_exec: + container: "{{ item }}" + command: >- + pmm-admin add valkey --cluster=valkey-native-cluster --environment=valkey-test --username=default + --password="{{ valkey_password }}" --service-name={{ item }}-svc-{{ random_number }} + --host={{ item }} --port=6379 --custom-labels='role=replica' + loop: "{{ replica_nodes }}" + ignore_errors: yes + + - name: Seed sample list data on first primary + community.docker.docker_container_exec: + container: "{{ valkey_primary_prefix }}1" + command: valkey-cli -a "{{ valkey_password }}" RPUSH mylist "one" "two" "three" "four" "five" + + - name: Pop one item from sample list on first primary + community.docker.docker_container_exec: + container: "{{ valkey_primary_prefix }}1" + command: valkey-cli -a "{{ valkey_password }}" RPOP mylist + + - name: Generate workload on primary nodes (latency metrics) + community.docker.docker_container_exec: + container: "{{ item }}" + command: >- + bash -c "for i in $(seq 1 50); do valkey-cli -a {{ valkey_password }} SET k$i v$i >/dev/null; valkey-cli -a {{ valkey_password }} GET k$i >/dev/null; valkey-cli -a {{ valkey_password }} HSET h$i f v >/dev/null; valkey-cli -a {{ valkey_password }} LPUSH l$i a b c >/dev/null; valkey-cli -a {{ valkey_password }} RPUSH l$i d e f >/dev/null; valkey-cli -a {{ valkey_password }} LRANGE l$i 0 -1 >/dev/null; valkey-cli -a {{ valkey_password }} LPOP l$i >/dev/null || true; valkey-cli -a {{ valkey_password }} RPOP l$i >/dev/null || true; done" + loop: "{{ primary_nodes }}" + ignore_errors: yes + + - name: Generate read workload on replica nodes (latency metrics) + community.docker.docker_container_exec: + container: "{{ item }}" + command: >- + bash -c "for i in $(seq 1 50); do valkey-cli -a {{ valkey_password }} GET k$i >/dev/null || true; valkey-cli -a {{ valkey_password }} LRANGE l$i 0 -1 >/dev/null || true; done" + loop: "{{ replica_nodes }}" + ignore_errors: yes diff --git a/pmm_qa/valkey/valkey.yml b/pmm_qa/valkey/valkey-sentinel.yml similarity index 69% rename from pmm_qa/valkey/valkey.yml rename to pmm_qa/valkey/valkey-sentinel.yml index eeaa4b80..0f730b49 100644 --- a/pmm_qa/valkey/valkey.yml +++ b/pmm_qa/valkey/valkey-sentinel.yml @@ -37,39 +37,39 @@ file: path: "{{ valkey_config_dir }}" state: directory - mode: '0755' + mode: "0755" - name: Create a config directory per Sentinel file: path: "{{ valkey_config_dir }}/sentinel-{{ item }}" state: directory - mode: '0755' + mode: "0755" loop: "{{ range(1, sentinel_count + 1) | list }}" - name: Create data directory file: path: "{{ valkey_data_dir }}" state: directory - mode: '0755' + mode: "0755" - name: Generate Valkey primary configuration template: src: valkey-primary.conf.j2 dest: "{{ valkey_config_dir }}/valkey-primary.conf" - mode: '0644' + mode: "0644" - name: Generate Valkey replica configurations template: src: valkey-replica.conf.j2 dest: "{{ valkey_config_dir }}/valkey-replica-{{ item }}.conf" - mode: '0644' + mode: "0644" loop: "{{ range(1, valkey_replica_count + 1) | list }}" - name: Generate Sentinel configurations template: src: sentinel.conf.j2 dest: "{{ valkey_config_dir }}/sentinel-{{ item }}/sentinel.conf" - mode: '0664' + mode: "0664" loop: "{{ range(1, sentinel_count + 1) | list }}" - name: Create Docker volume for primary data @@ -86,6 +86,7 @@ - name: Start Valkey primary container community.docker.docker_container: name: "valkey-primary" + hostname: "valkey-primary-node-{{ random_number }}" image: "{{ valkey_image }}" state: started restart_policy: unless-stopped @@ -113,6 +114,7 @@ - name: Start Valkey replica containers community.docker.docker_container: name: "valkey-replica-{{ item }}" + hostname: "valkey-replica-{{ item }}-node-{{ random_number }}" image: "{{ valkey_image }}" state: started restart_policy: unless-stopped @@ -142,6 +144,7 @@ - name: Start Sentinel containers community.docker.docker_container: name: "sentinel-{{ item }}" + hostname: "sentinel-{{ item }}-node-{{ random_number }}" image: "{{ valkey_image }}" state: started restart_policy: unless-stopped @@ -190,31 +193,62 @@ - name: Install PMM Client in each container ansible.builtin.include_tasks: ../tasks/install_pmm_client.yml loop: >- - {{ ['valkey-primary'] - + (range(1, valkey_replica_count + 1) | map('string') | map('regex_replace', '^(.*)$', 'valkey-replica-\1') | list) - + (range(1, sentinel_count + 1) | map('string') | map('regex_replace', '^(.*)$', 'sentinel-\1') | list) - }} + {{ ['valkey-primary'] + + (range(1, valkey_replica_count + 1) | map('string') | map('regex_replace', '^(.*)$', 'valkey-replica-\1') | list) + + (range(1, sentinel_count + 1) | map('string') | map('regex_replace', '^(.*)$', 'sentinel-\1') | list) + }} loop_control: loop_var: current_container_name vars: container_name: "{{ current_container_name }}" - - name: Add the primary to monitoring community.docker.docker_container_exec: container: "valkey-primary" - command: pmm-admin add valkey --cluster=valkey-cluster --replication-set=valkey-repl --environment=valkey-test --username=default --password="{{ valkey_password }}" --service-name=valkey-primary-svc-{{ random_number }} --host=valkey-primary --port=6379 --custom-labels='role=primary' + command: >- + pmm-admin add valkey --cluster=valkey-cluster --replication-set=valkey-repl --environment=valkey-test + --username=default --password="{{ valkey_password }}" --service-name=valkey-primary-svc-{{ random_number }} + --host=valkey-primary --port=6379 --custom-labels='role=primary' ignore_errors: yes - - name: Add the replicas to monitoring community.docker.docker_container_exec: container: "valkey-replica-{{ item }}" - command: pmm-admin add valkey --cluster=valkey-cluster --replication-set=valkey-repl --environment=valkey-test --username=default --password="{{ valkey_password }}" --service-name=valkey-replica{{ item }}-svc-{{ random_number }} --host=valkey-replica-{{ item }} --port=6379 --custom-labels='role=replica' + command: >- + pmm-admin add valkey --cluster=valkey-cluster --replication-set=valkey-repl --environment=valkey-test + --username=default --password="{{ valkey_password }}" --service-name=valkey-replica{{ item }}-svc-{{ random_number }} + --host=valkey-replica-{{ item }} --port=6379 --custom-labels='role=replica' loop: "{{ range(1, valkey_replica_count + 1) | list }}" ignore_errors: yes - - name: Add Sentinels to monitoring community.docker.docker_container_exec: container: "sentinel-{{ item }}" - command: pmm-admin add valkey --cluster=valkey-cluster --environment=valkey-test --username=default --password="{{ valkey_password }}" --service-name=sentinel{{ item }}-svc-{{ random_number }} --host=sentinel-{{ item }} --port={{ sentinel_start_port }} --custom-labels='role=sentinel' + command: >- + pmm-admin add valkey --cluster=valkey-cluster --environment=valkey-test --username=default + --password="{{ valkey_password }}" --service-name=sentinel{{ item }}-svc-{{ random_number }} + --host=sentinel-{{ item }} --port={{ sentinel_start_port }} --custom-labels='role=sentinel' loop: "{{ range(1, sentinel_count + 1) | list }}" ignore_errors: yes + + - name: Seed sample list data on primary (sentinel setup) + community.docker.docker_container_exec: + container: "valkey-primary" + command: valkey-cli -a "{{ valkey_password }}" RPUSH mylist "one" "two" "three" "four" "five" + + - name: Pop one item from sample list (sentinel setup) + community.docker.docker_container_exec: + container: "valkey-primary" + command: valkey-cli -a "{{ valkey_password }}" RPOP mylist + + - name: Generate workload on primary (sentinel setup) + community.docker.docker_container_exec: + container: "valkey-primary" + command: >- + bash -c "for i in $(seq 1 50); do valkey-cli -a {{ valkey_password }} SET k$i v$i >/dev/null; valkey-cli -a {{ valkey_password }} GET k$i >/dev/null; valkey-cli -a {{ valkey_password }} HSET h$i f v >/dev/null; valkey-cli -a {{ valkey_password }} LPUSH l$i a b c >/dev/null; valkey-cli -a {{ valkey_password }} RPUSH l$i d e f >/dev/null; valkey-cli -a {{ valkey_password }} LRANGE l$i 0 -1 >/dev/null; valkey-cli -a {{ valkey_password }} LPOP l$i >/dev/null || true; valkey-cli -a {{ valkey_password }} RPOP l$i >/dev/null || true; done" + ignore_errors: yes + + - name: Generate read workload on replicas (sentinel setup) + community.docker.docker_container_exec: + container: "valkey-replica-{{ item }}" + command: >- + bash -c "for i in $(seq 1 50); do valkey-cli -a {{ valkey_password }} GET k$i >/dev/null || true; valkey-cli -a {{ valkey_password }} LRANGE l$i 0 -1 >/dev/null || true; done" + loop: "{{ range(1, valkey_replica_count + 1) | list }}" + ignore_errors: yes