Ci fix small problems with integration tests

ci-infra/ntp-chrony/Readme.md

@@ -0,0 +1,19 @@
+We need a reliable 2 NTP servers for testing NTP reconfiguration.
+
+Setup them once in VMs
+
+```
+apt install -y chrony ntpdate
+cat /etc/chrony/chrony.conf
+  # See chrony.conf example file.
+  # Just add 'allow all' line if needed
+grep 'allow all' /etc/chrony/chrony.conf || echo 'allow all'
+
+systemctl enable chrony
+systemctl restart chrony
+chronyc tracking
+
+ntpdate -qup1 localhost
+```
+
+Check from outside if firewall allows NTP traffic - port 123/UDP.

ci-infra/ntp-chrony/chrony.conf

@@ -0,0 +1,47 @@
+# Welcome to the chrony configuration file. See chrony.conf(5) for more
+# information about usuable directives.
+
+# This will use (up to):
+# - 4 sources from ntp.ubuntu.com which some are ipv6 enabled
+# - 2 sources from 2.ubuntu.pool.ntp.org which is ipv6 enabled as well
+# - 1 source from [01].ubuntu.pool.ntp.org each (ipv4 only atm)
+# This means by default, up to 6 dual-stack and up to 2 additional IPv4-only
+# sources will be used.
+# At the same time it retains some protection against one of the entries being
+# down (compare to just using one of the lines). See (LP: #1754358) for the
+# discussion.
+#
+# About using servers from the NTP Pool Project in general see (LP: #104525).
+# Approved by Ubuntu Technical Board on 2011-02-08.
+# See http://www.pool.ntp.org/join.html for more information.
+pool ntp.ubuntu.com        iburst maxsources 4
+pool 0.ubuntu.pool.ntp.org iburst maxsources 1
+pool 1.ubuntu.pool.ntp.org iburst maxsources 1
+pool 2.ubuntu.pool.ntp.org iburst maxsources 2
+
+# This directive specify the location of the file containing ID/key pairs for
+# NTP authentication.
+keyfile /etc/chrony/chrony.keys
+
+# This directive specify the file into which chronyd will store the rate
+# information.
+driftfile /var/lib/chrony/chrony.drift
+
+# Uncomment the following line to turn logging on.
+#log tracking measurements statistics
+
+# Log files location.
+logdir /var/log/chrony
+
+# Stop bad estimates upsetting machine clock.
+maxupdateskew 100.0
+
+# This directive enables kernel synchronisation (every 11 minutes) of the
+# real-time clock. Note that it can't be used along with the 'rtcfile' directive.
+rtcsync
+
+# Step the system clock instead of slewing it if the adjustment is larger than
+# one second, but only in the first three clock updates.
+makestep 1 3
+
+allow all

tests/integration/integration_config.yml.j2

@@ -94,6 +94,9 @@ sc_config:
         # Can we update the host? VSNS hosts report update as available,
         # but if we try to update, the update fails (unsupported HW is reported back).
         can_be_applied: True
+        # If the cluster was updated before, then we can access
+        # https://IP/update/update_status.json (version_update_status_info module).
+        old_update_status_present: True
       virtual_disk:
         # Virtual disk feature is supported iff version ">=9.2.10"
         is_supported: True
@@ -126,6 +129,8 @@ sc_config:
         next_version: "9.1.23.210897"
         latest_version: "9.1.23.210897"
         can_be_applied: False
+        # We can try update, update will fail, and status.json will be present.
+        old_update_status_present: True
       virtual_disk:
         is_supported: False
         # replication_factor:
@@ -155,6 +160,7 @@ sc_config:
         next_version: ""
         latest_version: ""
         can_be_applied: False
+        old_update_status_present: False
       virtual_disk:
         is_supported: True
         replication_factor: 1

tests/integration/targets/email_alert/tasks/02_email_alert.yml

@@ -119,18 +119,30 @@
             ['alert_tag_uuid', 'email', 'latest_task_tag',
             'resend_delay', 'silent_period', 'uuid']
 
-
+    # Sending email can fail.
+    # We are using external SMTP server, and email address is from external domain.
+    # Either of two might not work.
+    # Try to resend after delay.
+    # Alternatives:
+    # - Set up a dedicated SMTP server, and use it during test.
+    #   It will have IP, but no DNS domain.
+    # - Use a temp email address from some public service (like https://temp-mail.org).
+    #   Again external service.
     - name: Send test email to an existing Email Alert Recipient
       scale_computing.hypercore.email_alert:
         email: "{{ new_email }}"
         state: test
       register: result
+      retries: 5
+      delay: 10
+      until: result is succeeded
     - scale_computing.hypercore.email_alert_info:
       register: info
     - ansible.builtin.debug:
         msg: "{{ result }}"
     - ansible.builtin.assert:
         that:
+          - result is succeeded
           - result.changed == False
           - result.diff.before == result.diff.after
           - info.records|length == 1|int

tests/integration/targets/inventory/runme.sh

@@ -6,9 +6,10 @@ eval "$(cat <<EOF | python
 import yaml
 with open("$vars_file") as fd:
     data = yaml.safe_load(fd)
-print("export SC_HOST='{}'".format(data["sc_host"]))
-print("export SC_USERNAME='{}'".format(data["sc_username"]))
-print("export SC_PASSWORD='{}'".format(data["sc_password"]))
+sc_host=data["sc_host"]
+print("export SC_HOST='{}'".format(sc_host))
+print("export SC_USERNAME='{}'".format(data["sc_config"][sc_host]["sc_username"]))
+print("export SC_PASSWORD='{}'".format(data["sc_config"][sc_host]["sc_password"]))
 EOF
 )"
 

tests/integration/targets/registration/tasks/main.yml

@@ -6,10 +6,10 @@
     SC_TIMEOUT: "{{ sc_timeout }}"
 
   vars:
-    default_company_name: ""
-    default_contact: ""
-    default_email: ""
-    default_phone: ""
+    default_company_name: "{{ sc_config[sc_host].registration.company_name }}"
+    default_contact: "{{ sc_config[sc_host].registration.contact }}"
+    default_email: "{{ sc_config[sc_host].registration.email }}"
+    default_phone: "{{ sc_config[sc_host].registration.phone }}"
 
 # ----------------------------------Setup------------------------------------------------------------------------
   block:
@@ -18,11 +18,9 @@
         endpoint: rest/v1/Registration
         action: get
       register: registration_original
-    - ansible.builtin.set_fact:
-        default_company_name: "{{ registration_original.record.0.companyName }}"
-        default_contact: "{{ registration_original.record.0.contact }}"
-        default_email: "{{ registration_original.record.0.email }}"
-        default_phone: "{{ registration_original.record.0.phone }}"
+    - name: Show Registration info (original info)
+      ansible.builtin.debug:
+        var: registration_original
 
     - name: Delete current registration
       scale_computing.hypercore.registration:

tests/integration/targets/remote_cluster_info/tasks/main.yml

@@ -9,24 +9,38 @@
     - name: Retrieve list of remote clusters
       scale_computing.hypercore.remote_cluster_info:
       register: initial
-    - ansible.builtin.assert:
-        that:
-          - initial.records != []
-          - "{{ initial.records[0].keys() | sort == ['connection_status', 'name', 'remote_node_ips', 'remote_node_uuids', 'replication_ok'] }}"
-          - initial.records[0].name
-          - initial.records[0].connection_status
-          - initial.records[0].replication_ok
-          - initial.records[0].remote_node_ips
-          - initial.records[0].remote_node_uuids
 
-    - name: Retrieve remote cluster by name
-      scale_computing.hypercore.remote_cluster_info:
-        remote_cluster: "{{ initial.records[0].name }}"
-      register: results
-    - ansible.builtin.assert:
-        that:
-          - results.records[0].name == initial.records[0].name
-
+    - name: If remote cluster is configured
+      when: sc_config[sc_host].sc_replication_dest_host
+      block:
+        - ansible.builtin.assert:
+            that:
+              - initial is succeeded
+              - initial.records != []
+              - "{{ initial.records[0].keys() | sort == ['connection_status', 'name', 'remote_node_ips', 'remote_node_uuids', 'replication_ok'] }}"
+              - initial.records[0].name
+              - initial.records[0].connection_status
+              - initial.records[0].replication_ok
+              - initial.records[0].remote_node_ips
+              - initial.records[0].remote_node_uuids
+
+        - name: Retrieve remote cluster by name
+          scale_computing.hypercore.remote_cluster_info:
+            remote_cluster: "{{ initial.records[0].name }}"
+          register: results
+        - ansible.builtin.assert:
+            that:
+              - results.records[0].name == initial.records[0].name
+
+    - name: If remote cluster is not configured
+      when: not sc_config[sc_host].sc_replication_dest_host
+      block:
+        - ansible.builtin.assert:
+            that:
+              - initial is succeeded
+              - initial.records == []
+
+    # Always
     - name: Retrieve nonexisting remote cluster
       scale_computing.hypercore.remote_cluster_info:
         remote_cluster: "DOES NOT EXIST"

tests/integration/targets/role_cluster_config/tasks/main.yml

@@ -29,7 +29,7 @@
         # certificate: plain_text_from_x509
         config_url: https://login.microsoftonline.com/76d4c62a-a9ca-4dc2-9187-e2cc4d9abe7f/v2.0/.well-known/openid-configuration
         scopes: openid+profile
-      time_server: 2.pool.ntp.org
+      time_server: 10.5.11.5
       time_zone: Europe/Ljubljana
       smtp:
         server: mail_a.example.com
@@ -71,7 +71,7 @@
         # certificate: plain_text_from_x509
         config_url: https://login.microsoftonline.com/76d4c62a-a9ca-4dc2-9187-e2cc4d9abe7f/v2.0/.well-known/openid-configuration
         scopes: openid+profile
-      time_server: 3.pool.ntp.org
+      time_server: 10.5.11.7
       time_zone: Europe/Zagreb
       smtp:
         server: mail_b.example.com

tests/integration/targets/time_server/tasks/01_time_server_info.yml

@@ -6,7 +6,9 @@
 
   vars:
     actual_uuid: "timesource_guid"
-    actual_host: "3.pool.ntp.org"
+    # We need 2 NTP servers.
+    # Set them up per ci-infra/ntp-chrony/Readme.md.
+    actual_host: "10.5.11.5"
 
   block:
     # -------------------------------------------------------

tests/integration/targets/time_server/tasks/02_time_server.yml

@@ -5,8 +5,10 @@
     SC_PASSWORD: "{{ sc_config[sc_host].sc_password }}"
     SC_TIMEOUT: "{{ sc_timeout * 10 }}"
   vars:
-    time_server_a: 2.pool.ntp.org
-    time_server_b: 3.pool.ntp.org
+    # We need 2 NTP servers.
+    # Set them up per ci-infra/ntp-chrony/Readme.md.
+    time_server_a: 10.5.11.5
+    time_server_b: 10.5.11.7
     actual_uuid: timesource_guid
 
   block:

tests/integration/targets/version_update_status_info/tasks/main.yml

@@ -9,7 +9,14 @@
     - name: Get status of the latest update applied
       scale_computing.hypercore.version_update_status_info:
       register: update_status
+
     - ansible.builtin.assert:
         that:
           - update_status.record != []
           - update_status.record.keys() | sort == ['from_build', 'percent', 'prepare_status', 'to_build', 'to_version', 'update_status', 'update_status_details', 'usernotes']
+      when: sc_config[sc_host].features.version_update.old_update_status_present
+
+    - ansible.builtin.assert:
+        that:
+          - update_status.record == None
+      when: not sc_config[sc_host].features.version_update.old_update_status_present