refactor Ansible files

.gitignore

@@ -3,6 +3,7 @@
 /tf/terraform.tfstate*
 /tf/.terraform/*
 /tf/.terraform.lock.hcl
+/ansible/.retry
 /ansible/testnet.toml
 /ansible/testnet/*
 /ansible/rotating/*

Makefile

@@ -18,8 +18,6 @@ endif
 
 RUNNER_COMMIT_HASH ?= $(VERSION_TAG)
 LOAD_RUNNER_CMD=go run github.com/cometbft/cometbft/test/e2e/runner@$(RUNNER_COMMIT_HASH)
-ANSIBLE_SSH_RETRIES=5
-ANSIBLE_FORKS=150
 export DO_INSTANCE_TAGNAME
 export DO_VPC_SUBNET
 export EPHEMERAL_SIZE
@@ -47,20 +45,18 @@ configgen:
 
 .PHONY: ansible-install
 ansible-install:
-	cd ansible && \
-		ANSIBLE_SSH_RETRIES=$(ANSIBLE_SSH_RETRIES) ansible-playbook -i hosts -u root testapp-install.yaml -f $(ANSIBLE_FORKS) -e "version_tag=$(VERSION_TAG)" -e "go_modules_token=$(GO_MODULES_TOKEN)" -e "vpc_subnet=$(DO_VPC_SUBNET)"
+	ansible-playbook ./ansible/testapp-install.yaml -e "version_tag=$(VERSION_TAG)"
 ifneq ($(VERSION2_WEIGHT), 0)
-	cd ansible && \
-		ANSIBLE_SSH_RETRIES=$(ANSIBLE_SSH_RETRIES) ansible-playbook -i hosts --limit validators2 -u root testapp-update.yaml -f $(ANSIBLE_FORKS) -e "version_tag=$(VERSION2_TAG)" -e "go_modules_token=$(GO_MODULES_TOKEN)"
+	ansible-playbook ./ansible/testapp-update.yaml -e "version_tag=$(VERSION2_TAG)" --limit validators2
 endif
 
 .PHONY: prometheus-init
 prometheus-init:
-	cd ansible && ANSIBLE_SSH_RETRIES=$(ANSIBLE_SSH_RETRIES) ansible-playbook -i hosts  -u root prometheus-init.yaml -f 10
+	ansible-playbook ./ansible/prometheus-init.yaml
 
 .PHONY: loadrunners-init
 loadrunners-init:
-	cd ansible && ANSIBLE_SSH_RETRIES=$(ANSIBLE_SSH_RETRIES) ansible-playbook -i hosts -u root loader-init.yaml -f 10
+	ansible-playbook ./ansible/loader-init.yaml -e "version_tag=$(VERSION_TAG)"
 
 .PHONY: start-network
 start-network:
@@ -74,23 +70,21 @@ stop-network:
 
 .PHONY: runload
 runload:
-	cd ansible && \
-		endpoints=$$(scripts/get-endpoints.sh) && \
-		ANSIBLE_SSH_RETRIES=$(ANSIBLE_SSH_RETRIES) ansible-playbook loader-run.yaml -i hosts -u root \
-			-e endpoints=$$endpoints \
-			-e connections=$(LOAD_CONNECTIONS) \
-			-e time_seconds=$(LOAD_TOTAL_TIME) \
-			-e tx_per_second=$(LOAD_TX_RATE) \
-			-e iterations=$(ITERATIONS)
+	ansible-playbook ./ansible/loader-run.yaml \
+		-e endpoints=`./ansible/scripts/get-endpoints.sh` \
+		-e load.connections=$(LOAD_CONNECTIONS) \
+		-e load.time_seconds=$(LOAD_TOTAL_TIME) \
+		-e load.tx_per_second=$(LOAD_TX_RATE) \
+		-e load.iterations=$(ITERATIONS)
 
 .PHONY: restart
 restart:
-	cd ansible && ANSIBLE_SSH_RETRIES=$(ANSIBLE_SSH_RETRIES) ansible-playbook -i hosts -u root testapp-update.yaml -f $(ANSIBLE_FORKS) -e "version_tag=$(VERSION_TAG)" -e "go_modules_token=$(GO_MODULES_TOKEN)"
+	ansible-playbook ./ansible/testapp-update.yaml -e "version_tag=$(VERSION_TAG)"
 ifneq ($(VERSION2_WEIGHT), 0)
-	cd ansible && ANSIBLE_SSH_RETRIES=$(ANSIBLE_SSH_RETRIES) ansible-playbook -i hosts --limit validators2 -u root testapp-update.yaml -f $(ANSIBLE_FORKS) -e "version_tag=$(VERSION2_TAG)" -e "go_modules_token=$(GO_MODULES_TOKEN)"
+	ansible-playbook ./ansible/testapp-update.yaml -e "version_tag=$(VERSION2_TAG)" --limit validators2
 endif
-	cd ansible && ANSIBLE_SSH_RETRIES=$(ANSIBLE_SSH_RETRIES) ansible-playbook prometheus-restart.yaml -i hosts -u root
-	cd ansible && ANSIBLE_SSH_RETRIES=$(ANSIBLE_SSH_RETRIES) ansible-playbook testapp-reinit.yaml -i hosts -u root -f $(ANSIBLE_FORKS)
+	ansible-playbook ./ansible/prometheus-restart.yaml
+	ansible-playbook ./ansible/testapp-reinit.yaml
 
 .PHONY: rotate
 rotate:
@@ -106,19 +100,17 @@ perturb-nodes:
 retrieve-blockstore:
 	mkdir -p "./experiments/$(EXPERIMENT_DIR)"
 ifeq ($(RETRIEVE_TARGET_HOST), any)
-	cd ansible && \
-		last_val=$$(ansible -i hosts --list-hosts validators | tail -1 | sed  "s/ //g") && \
-		retrieve_target_host=$$(ansible-inventory -i hosts -y --host $$last_val | grep 'name' | cut -d ' ' -f2) && \
-		ansible-playbook -i hosts -u root retrieve-blockstore.yaml -e "dir=../experiments/$(EXPERIMENT_DIR)/" -e "target_host=$$retrieve_target_host"
+	last_val=$$(ansible --list-hosts validators | tail -1 | sed  "s/ //g") && \
+	retrieve_target_host=$$(ansible-inventory -y --host $$last_val | grep 'name' | cut -d ' ' -f2) && \
+	ansible-playbook ./ansible/retrieve-blockstore.yaml -e "dir=../experiments/$(EXPERIMENT_DIR)/" -e "target_host=$$retrieve_target_host"
 else
-	cd ansible && \
-		ansible-playbook -i hosts -u root retrieve-blockstore.yaml -e "dir=../experiments/$(EXPERIMENT_DIR)/" -e "target_host=$(RETRIEVE_TARGET_HOST)"
+	ansible-playbook ./ansible/retrieve-blockstore.yaml -e "dir=../experiments/$(EXPERIMENT_DIR)/" -e "target_host=$(RETRIEVE_TARGET_HOST)"
 endif
 
 .PHONY: retrieve-prometheus-data
 retrieve-prometheus-data:
 	mkdir -p "./experiments/$(EXPERIMENT_DIR)"; \
-	cd ansible && ansible-playbook -i hosts -u root retrieve-prometheus.yaml --limit `ansible -i hosts --list-hosts prometheus | tail -1 | sed  's/ //g'` -e "dir=../experiments/$(EXPERIMENT_DIR)/";
+	ansible-playbook ./ansible/retrieve-prometheus.yaml --limit `ansible --list-hosts prometheus | tail -1 | sed  's/ //g'` -e "dir=../experiments/$(EXPERIMENT_DIR)/"
 
 .PHONY: retrieve-data
 retrieve-data: retrieve-prometheus-data retrieve-blockstore

ansible.cfg

@@ -0,0 +1,9 @@
+[defaults]
+inventory = ./ansible/hosts
+remote_user = root
+forks = 150
+retry_files_enabled = true
+retry_files_save_path = ./ansible/.retry
+
+[ssh_connection]
+retries = 5

ansible/group_vars/all.yaml

@@ -0,0 +1,13 @@
+---
+testnet_dir: testnet
+cmt_home: /root/.testapp/
+ansible_host_key_checking: false
+
+go_modules_token: ""
+
+load:
+  connections: 2
+  time_seconds: 91
+  tx_per_second: 200
+  size_bytes: 1024
+  iterations: 5

ansible/loader-init.yaml

@@ -1,21 +1,15 @@
 - name: Initialize loadrunners
+  hosts: loadrunners
   become: false
   gather_facts: true
-  hosts: loadrunners
 
   tasks:
-    - name: Check if load tool exists
-      ansible.builtin.stat:
-        path: /root/go/bin/load
-      register: load_tool
+    - ansible.builtin.import_tasks: tasks/testapp-clone-repo.yaml
 
     - name: Install load tool
       ansible.builtin.command: /usr/lib/go-1.21/bin/go install
       args:
         chdir: /root/cometbft/test/loadtime/cmd/load/
-      when: not load_tool.stat.exists
-      register: result
-      changed_when: result.rc != 0
 
     - name: Copy script files
       ansible.builtin.copy:

ansible/loader-run.yaml

@@ -1,21 +1,24 @@
 - name: runload
-  become: false
-  gather_facts: yes
   hosts: loadrunners
+  become: false
+  gather_facts: true
   vars:
-    ansible_host_key_checking: false
     endpoints: ws://{{ hostvars[groups[validators][0]].internal_ip }}:26657/v1/websocket # comma separated list of endpoints
-    connections: 25
-    time_seconds: 60
-    tx_per_second: 1000
-    size_bytes: 1024
-    iterations: 1
 
   tasks:
+    - debug: 
+        msg:
+          connections: "{{ load.connections }}"
+          tx_per_second: "{{ load.tx_per_second }}"
+          size_bytes: "{{ load.size_bytes }}"
+          time_seconds: "{{ load.time_seconds }}"
+          endpoints: "{{ endpoints }}"
+
     - name: install load tool
       shell: "cd cometbft/test/loadtime/cmd/load/ && /usr/lib/go-1.21/bin/go install"
+
     - name: run the load tool
-      shell: "/root/go/bin/load -c {{ connections }} -T {{ time_seconds }} -r {{ tx_per_second }} -s {{ size_bytes }} --broadcast-tx-method sync --endpoints {{endpoints}}"
-      loop: "{{ range(0, iterations| int, 1)| list }}"
+      shell: "/root/go/bin/load -c {{ load.connections }} -T {{ load.time_seconds }} -r {{ load.tx_per_second }} -s {{ load.size_bytes }} --broadcast-tx-method sync --endpoints {{ endpoints }}"
+      loop: "{{ range(0, load.iterations| int, 1)| list }}"
       loop_control:
         pause: 300

ansible/prometheus-init.yaml

@@ -1,9 +1,8 @@
 - name: create prometheus
-  become: false
-  gather_facts: yes
   hosts: prometheus
-  vars:
-    ansible_host_key_checking: false
+  become: false
+  gather_facts: false
+
   tasks:
     - name: create unit file
       template:

ansible/prometheus-restart.yaml

@@ -1,9 +1,7 @@
 - name: delete and restart prometheus
   hosts: prometheus
   become_method: sudo
-  gather_facts: yes
-  vars:
-    ansible_host_key_checking: false
+  gather_facts: true
 
   tasks:
     - name: stop prometheus

ansible/retrieve-blockstore.yaml

@@ -1,11 +1,10 @@
 - name: retrieve blockstore
   hosts: validators
   become: false
-  gather_facts: yes
+  gather_facts: true
   vars:
-    cmt_home: /root/.testapp/
-    ansible_host_key_checking: false
     target_host: all
+
   tasks:
     - name: stop app
       ansible.builtin.systemd:

ansible/retrieve-prometheus.yaml

@@ -1,9 +1,8 @@
 - name: retrieve prometheus
   hosts: prometheus
   become: false
-  gather_facts: yes
-  vars:
-    ansible_host_key_checking: false
+  gather_facts: true
+
   tasks:
     - name: Zip the prometheus directory
       archive:

ansible/scripts/get-endpoints.sh

@@ -1,5 +1,5 @@
 NUM_CONNECTIONS=1
-endpoint_list=$(ansible-inventory -i hosts --export --list | jq '[.[] | .hostvars][0]' | grep name -B 1  | grep internal_ip | sed 's/\"//g' | cut -w -f3 | sed 's/,//' | sed 's/\(.*\)/ws:\/\/\1:26657\/v1\/websocket/')
+endpoint_list=$(ansible-inventory --export --list | jq '[.[] | .hostvars][0]' | grep name -B 1  | grep internal_ip | sed 's/\"//g' | cut -w -f3 | sed 's/,//' | sed 's/\(.*\)/ws:\/\/\1:26657\/v1\/websocket/')
 
 IFS='
 '

ansible/tasks/testapp-build.yaml

@@ -0,0 +1,5 @@
+- name: redirect https
+  shell: "git config --global url.https://{{ go_modules_token }}@github.com/.insteadOf https://github.com/"
+
+- name: build testapp
+  shell: "cd cometbft/test/e2e/node && GOPRIVATE=github.com/cometbft /usr/lib/go-1.21/bin/go install"

ansible/tasks/testapp-clone-repo.yaml

@@ -0,0 +1,10 @@
+- name: remove directory with CometBFT repo
+  ansible.builtin.file:
+    path: "{{ ansible_user_dir }}/cometbft"
+    state: absent          
+
+- name: "clone CometBFT repo hash={{ version_tag }}"
+  ansible.builtin.git:
+    repo: https://{{ go_modules_token }}@github.com/cometbft/cometbft
+    dest: "{{ ansible_user_dir }}/cometbft"
+    version: "{{ version_tag }}"

ansible/tasks/testapp-copy-config-files.yaml

@@ -0,0 +1,4 @@
+- name: copy configuration files
+  ansible.builtin.copy:
+    src: "./{{ testnet_dir }}/{{ hostvars[inventory_hostname].name }}/"
+    dest: "{{ cmt_home }}/"

ansible/tasks/testapp-remove-data.yaml

@@ -0,0 +1,11 @@
+- name: delete data in home directory
+  ansible.builtin.file:
+    path: "{{ cmt_home }}"
+    state: absent
+  become: true
+
+- name: delete app data
+  ansible.builtin.file:
+    path: "{{ ansible_user_dir }}/data"
+    state: absent
+  become: true

ansible/tasks/testapp-start.yaml

@@ -0,0 +1,5 @@
+- name: start the systemd-unit
+  ansible.builtin.systemd:
+    name: testappd
+    state: started
+    enabled: yes

ansible/tasks/testapp-stop.yaml

@@ -0,0 +1,5 @@
+- name: stop the systemd-unit
+  ansible.builtin.systemd:
+    name: testappd
+    state: stopped
+    enabled: yes

ansible/tasks/testapp-update-unit-file.yaml

@@ -0,0 +1,5 @@
+- name: update unit file
+  template:
+    src: templates/testappd.service.j2
+    dest: /lib/systemd/system/testappd.service
+  become: true

ansible/testapp-init.yaml

@@ -1,10 +1,7 @@
 - name: initialize app
   hosts: validators
   become: false
-  gather_facts: yes
-  vars:
-    cmt_home: /root/.testapp/
-    ansible_host_key_checking: false
+  gather_facts: true
 
   tasks:
     - name: copy configuration files

ansible/testapp-install.yaml

@@ -1,9 +1,22 @@
-- hosts: validators,loadrunners,ephemeral
+- name: build testapp
+  hosts: validators,ephemeral
+  become_method: sudo
+  gather_facts: true
   vars:
-    ansible_host_key_checking: false
+    is_same_version: hostvars[inventory_hostname].version_tag == "{{ version_tag }}"
 
-- name: Init test app playbook
-  ansible.builtin.import_playbook: testapp-init.yaml
+  tasks:
+    - ansible.builtin.import_tasks: tasks/testapp-copy-config-files.yaml
+      when: "'ephemeral' not in group_names" # initially there are no config files for ephemeral nodes
 
-- name: Update test app playbook 
-  ansible.builtin.import_playbook: testapp-update.yaml
+    - ansible.builtin.import_tasks: tasks/testapp-clone-repo.yaml
+      when: not is_same_version
+
+    - ansible.builtin.import_tasks: tasks/testapp-build.yaml
+      when: not is_same_version
+
+    - ansible.builtin.import_tasks: tasks/testapp-update-unit-file.yaml
+      when: not is_restarting
+
+    - ansible.builtin.import_tasks: tasks/testapp-start.yaml
+      when: is_restarting

ansible/testapp-reinit.yaml

@@ -1,39 +1,10 @@
 - name: re-init testapp
-  hosts: ephemeral,validators
+  hosts: validators,ephemeral
   become_method: sudo
-  gather_facts: yes
-  vars:
-    ansible_host_key_checking: false
-    cmt_home: /root/.testapp/
-    testnet_dir: ./testnet
+  gather_facts: true
 
-  # TODO: the first three tasks are testapp-remove-data.yaml
   tasks:
-    - name: stop app
-      ansible.builtin.systemd:
-        name: testappd
-        state: stopped
-      become: yes
-
-    - name: delete data in home directory
-      ansible.builtin.file:
-        path: "{{ cmt_home }}"
-        state: absent
-      become: yes
-
-    - name: delete app data
-      ansible.builtin.file:
-        path: "{{ ansible_user_dir }}/data"
-        state: absent
-      become: yes
-
-    - name: copy configuration files
-      ansible.builtin.copy:
-        src: "{{ testnet_dir }}/{{ hostvars[inventory_hostname].name }}/"
-        dest: "{{ cmt_home }}/"
-
-    - name: start the systemd-unit
-      ansible.builtin.systemd:
-        name: testappd
-        state: started
-        enabled: yes
+    - ansible.builtin.import_tasks: tasks/testapp-stop.yaml
+    - ansible.builtin.import_tasks: tasks/testapp-remove-data.yaml
+    - ansible.builtin.import_tasks: tasks/testapp-copy-config-files.yaml
+    - ansible.builtin.import_tasks: tasks/testapp-start.yaml