ansible-scylla-node: Make sure all nodes are up before adjusting repl…

…ication

Before this patch, we were assuming that if we get to the adjust_keyspace_replication
task and start_scylla_service is set to true, it means that all the nodes are up,
since the 'start_scylla_service dependent tasks' block was already executed.
However, the role is responsible for starting scylla only for the nodes in
ansible_play_hosts, meaning that if the user has a cluster with 6 nodes, but is
running the role for 3 nodes (by using --limit), the role should start only these 3
nodes and the other 3 nodes might or might not have already been started.
Having that in mind, this patch checks if ALL the nodes in the inventory (and not only
the ones in ansible_play_hosts) are up before adjusting the replication.

ansible-scylla-node/tasks/common.yml

@@ -390,14 +390,37 @@
           {% if (_scylla_yaml_map.audit is defined and _scylla_yaml_map.audit == 'table') %}True{% else %}False{% endif %}
   run_once: true
 
+- name: Check if all nodes are up
+  block:
+    - name: Use Scylla API to see if the nodes are up
+      uri:
+        url: "http://{{ scylla_api_address }}:{{ scylla_api_port }}/failure_detector/endpoints/"
+        follow_redirects: none
+        method: GET
+      register: _failure_detector_result
+      until: _failure_detector_result.status == 200
+      retries: 3
+      delay: 1
+      ignore_errors: true
+      delegate_to: "{{ item }}"
+      loop: "{{ groups['scylla'] }}"
+      run_once: true
+
+    - name: Set all_nodes_up as a fact
+      set_fact:
+        all_nodes_up: "{% if item.status is defined and item.status == 200 %}{{ True }}{% else %}{{ False }}{% endif %}"
+      when: all_nodes_up is not defined or all_nodes_up|bool
+      loop: "{{ _failure_detector_result.results }}"
+      run_once: true
+
 - name: Adjust replication for system_auth keyspace
   include_tasks: adjust_keyspace_replication.yml
   vars:
     _keyspace: "system_auth"
     _keyspace_replication_strategy: "{{ system_auth_replication_strategy }}"
     _keyspace_rf: "{{ system_auth_rf }}"
   when:
-    - start_scylla_service|bool
+    - all_nodes_up | bool
     - adjust_system_auth_replication is defined and adjust_system_auth_replication|bool
     - _authentication_enabled is defined and _authentication_enabled|bool
     - system_auth_rf is defined and system_auth_replication_strategy is defined
@@ -409,7 +432,7 @@
     _keyspace_replication_strategy: "{{ audit_replication_strategy }}"
     _keyspace_rf: "{{ audit_rf }}"
   when:
-    - start_scylla_service|bool
+    - all_nodes_up | bool
     - adjust_audit_replication is defined and adjust_audit_replication|bool
     - _audit_enabled is defined and _audit_enabled|bool
     - audit_rf is defined and audit_replication_strategy is defined
@@ -421,7 +444,7 @@
     _keyspace_replication_strategy: "{{ system_distributed_replication_strategy }}"
     _keyspace_rf: "{{ system_distributed_rf }}"
   when:
-    - start_scylla_service|bool
+    - all_nodes_up | bool
     - adjust_system_distributed_replication is defined and adjust_system_distributed_replication|bool
     - system_distributed_rf is defined and system_distributed_replication_strategy is defined
 
@@ -432,7 +455,7 @@
     _keyspace_replication_strategy: "{{ system_traces_replication_strategy }}"
     _keyspace_rf: "{{ system_traces_rf }}"
   when:
-    - start_scylla_service|bool
+    - all_nodes_up | bool
     - adjust_system_traces_replication is defined and adjust_system_traces_replication|bool
     - system_traces_rf is defined and system_traces_replication_strategy is defined