From b83e17920fa0af7ef80150b52d3a23fa762e9447 Mon Sep 17 00:00:00 2001 From: Justin Cinkelj Date: Fri, 7 Apr 2023 07:19:39 +0200 Subject: [PATCH 1/6] CI try to resend test email if it fails job on 200 VSNS: https://github.com/ScaleComputing/HyperCoreAnsibleCollection/actions/runs/4625947983/jobs/8182206970#step:8:182 job on 201 VSNS: https://github.com/ScaleComputing/HyperCoreAnsibleCollection/actions/runs/4625947983/jobs/8182207045#step:8:182 On 201 we were not able to send email. A guess - we are using a real external SMTP server, and sometimes it might reject out email (maybe it looks like spam). Signed-off-by: Justin Cinkelj --- .../targets/email_alert/tasks/02_email_alert.yml | 14 +++++++++++++- 1 file changed, 13 insertions(+), 1 deletion(-) diff --git a/tests/integration/targets/email_alert/tasks/02_email_alert.yml b/tests/integration/targets/email_alert/tasks/02_email_alert.yml index 4f6206e92..93617832c 100644 --- a/tests/integration/targets/email_alert/tasks/02_email_alert.yml +++ b/tests/integration/targets/email_alert/tasks/02_email_alert.yml @@ -119,18 +119,30 @@ ['alert_tag_uuid', 'email', 'latest_task_tag', 'resend_delay', 'silent_period', 'uuid'] - + # Sending email can fail. + # We are using external SMTP server, and email address is from external domain. + # Either of two might not work. + # Try to resend after delay. + # Alternatives: + # - Set up a dedicated SMTP server, and use it during test. + # It will have IP, but no DNS domain. + # - Use a temp email address from some public service (like https://temp-mail.org). + # Again external service. - name: Send test email to an existing Email Alert Recipient scale_computing.hypercore.email_alert: email: "{{ new_email }}" state: test register: result + retries: 5 + delay: 10 + until: result is succeeded - scale_computing.hypercore.email_alert_info: register: info - ansible.builtin.debug: msg: "{{ result }}" - ansible.builtin.assert: that: + - result is succeeded - result.changed == False - result.diff.before == result.diff.after - info.records|length == 1|int From 6463f1706c45d4b909ba7ebc42867c9914daaf04 Mon Sep 17 00:00:00 2001 From: Justin Cinkelj Date: Fri, 7 Apr 2023 07:35:01 +0200 Subject: [PATCH 2/6] CI fix how SC_USERNAME/PASSWORD is computed for inventory test https://github.com/ScaleComputing/HyperCoreAnsibleCollection/actions/runs/4625947983/jobs/8182204674#step:8:580 https://github.com/ScaleComputing/HyperCoreAnsibleCollection/actions/runs/4625947983/jobs/8182204811#step:8:578 Both failed. Signed-off-by: Justin Cinkelj --- tests/integration/targets/inventory/runme.sh | 7 ++++--- 1 file changed, 4 insertions(+), 3 deletions(-) diff --git a/tests/integration/targets/inventory/runme.sh b/tests/integration/targets/inventory/runme.sh index 4f6793416..7fc9786ad 100755 --- a/tests/integration/targets/inventory/runme.sh +++ b/tests/integration/targets/inventory/runme.sh @@ -6,9 +6,10 @@ eval "$(cat < Date: Fri, 7 Apr 2023 07:43:07 +0200 Subject: [PATCH 3/6] CI registration_info test should pass on empty cluster too https://github.com/ScaleComputing/HyperCoreAnsibleCollection/actions/runs/4625947983/jobs/8182204047#step:8:36 Test was failing Also set registration data as configured in sc_config. Signed-off-by: Justin Cinkelj --- .../targets/registration/tasks/main.yml | 16 +++++++--------- 1 file changed, 7 insertions(+), 9 deletions(-) diff --git a/tests/integration/targets/registration/tasks/main.yml b/tests/integration/targets/registration/tasks/main.yml index e94a7a4d4..38dbe2e59 100644 --- a/tests/integration/targets/registration/tasks/main.yml +++ b/tests/integration/targets/registration/tasks/main.yml @@ -6,10 +6,10 @@ SC_TIMEOUT: "{{ sc_timeout }}" vars: - default_company_name: "" - default_contact: "" - default_email: "" - default_phone: "" + default_company_name: "{{ sc_config[sc_host].registration.company_name }}" + default_contact: "{{ sc_config[sc_host].registration.contact }}" + default_email: "{{ sc_config[sc_host].registration.email }}" + default_phone: "{{ sc_config[sc_host].registration.phone }}" # ----------------------------------Setup------------------------------------------------------------------------ block: @@ -18,11 +18,9 @@ endpoint: rest/v1/Registration action: get register: registration_original - - ansible.builtin.set_fact: - default_company_name: "{{ registration_original.record.0.companyName }}" - default_contact: "{{ registration_original.record.0.contact }}" - default_email: "{{ registration_original.record.0.email }}" - default_phone: "{{ registration_original.record.0.phone }}" + - name: Show Registration info (original info) + ansible.builtin.debug: + var: registration_original - name: Delete current registration scale_computing.hypercore.registration: From b4056a77f5978d8f949c6ac693ec9cab54926a83 Mon Sep 17 00:00:00 2001 From: Justin Cinkelj Date: Fri, 7 Apr 2023 07:54:33 +0200 Subject: [PATCH 4/6] CI make remote_cluster_info pass also on cluster without remote cluster It failed before https://github.com/ScaleComputing/HyperCoreAnsibleCollection/actions/runs/4625947983/jobs/8182203573#step:8:46 Signed-off-by: Justin Cinkelj --- .../remote_cluster_info/tasks/main.yml | 48 ++++++++++++------- 1 file changed, 31 insertions(+), 17 deletions(-) diff --git a/tests/integration/targets/remote_cluster_info/tasks/main.yml b/tests/integration/targets/remote_cluster_info/tasks/main.yml index 911452286..fb1935ff1 100644 --- a/tests/integration/targets/remote_cluster_info/tasks/main.yml +++ b/tests/integration/targets/remote_cluster_info/tasks/main.yml @@ -9,24 +9,38 @@ - name: Retrieve list of remote clusters scale_computing.hypercore.remote_cluster_info: register: initial - - ansible.builtin.assert: - that: - - initial.records != [] - - "{{ initial.records[0].keys() | sort == ['connection_status', 'name', 'remote_node_ips', 'remote_node_uuids', 'replication_ok'] }}" - - initial.records[0].name - - initial.records[0].connection_status - - initial.records[0].replication_ok - - initial.records[0].remote_node_ips - - initial.records[0].remote_node_uuids - - name: Retrieve remote cluster by name - scale_computing.hypercore.remote_cluster_info: - remote_cluster: "{{ initial.records[0].name }}" - register: results - - ansible.builtin.assert: - that: - - results.records[0].name == initial.records[0].name - + - name: If remote cluster is configured + when: sc_config[sc_host].sc_replication_dest_host + block: + - ansible.builtin.assert: + that: + - initial is succeeded + - initial.records != [] + - "{{ initial.records[0].keys() | sort == ['connection_status', 'name', 'remote_node_ips', 'remote_node_uuids', 'replication_ok'] }}" + - initial.records[0].name + - initial.records[0].connection_status + - initial.records[0].replication_ok + - initial.records[0].remote_node_ips + - initial.records[0].remote_node_uuids + + - name: Retrieve remote cluster by name + scale_computing.hypercore.remote_cluster_info: + remote_cluster: "{{ initial.records[0].name }}" + register: results + - ansible.builtin.assert: + that: + - results.records[0].name == initial.records[0].name + + - name: If remote cluster is not configured + when: not sc_config[sc_host].sc_replication_dest_host + block: + - ansible.builtin.assert: + that: + - initial is succeeded + - initial.records == [] + + # Always - name: Retrieve nonexisting remote cluster scale_computing.hypercore.remote_cluster_info: remote_cluster: "DOES NOT EXIST" From 729ca655c30ad08105da8a06b54f37691e1a590e Mon Sep 17 00:00:00 2001 From: Justin Cinkelj Date: Fri, 7 Apr 2023 09:26:44 +0200 Subject: [PATCH 5/6] CI make version_update_status_info pass if cluster was never upgraded https://github.com/ScaleComputing/HyperCoreAnsibleCollection/actions/runs/4625947983/jobs/8182200158#step:8:39 On VSNS 201 we cannot even initiate upgrade, and we will never have update.json there. Signed-off-by: Justin Cinkelj --- tests/integration/integration_config.yml.j2 | 6 ++++++ .../targets/version_update_status_info/tasks/main.yml | 7 +++++++ 2 files changed, 13 insertions(+) diff --git a/tests/integration/integration_config.yml.j2 b/tests/integration/integration_config.yml.j2 index 5f39be9a9..ef279f0db 100644 --- a/tests/integration/integration_config.yml.j2 +++ b/tests/integration/integration_config.yml.j2 @@ -94,6 +94,9 @@ sc_config: # Can we update the host? VSNS hosts report update as available, # but if we try to update, the update fails (unsupported HW is reported back). can_be_applied: True + # If the cluster was updated before, then we can access + # https://IP/update/update_status.json (version_update_status_info module). + old_update_status_present: True virtual_disk: # Virtual disk feature is supported iff version ">=9.2.10" is_supported: True @@ -126,6 +129,8 @@ sc_config: next_version: "9.1.23.210897" latest_version: "9.1.23.210897" can_be_applied: False + # We can try update, update will fail, and status.json will be present. + old_update_status_present: True virtual_disk: is_supported: False # replication_factor: @@ -155,6 +160,7 @@ sc_config: next_version: "" latest_version: "" can_be_applied: False + old_update_status_present: False virtual_disk: is_supported: True replication_factor: 1 diff --git a/tests/integration/targets/version_update_status_info/tasks/main.yml b/tests/integration/targets/version_update_status_info/tasks/main.yml index e20189245..5ea4b61c4 100644 --- a/tests/integration/targets/version_update_status_info/tasks/main.yml +++ b/tests/integration/targets/version_update_status_info/tasks/main.yml @@ -9,7 +9,14 @@ - name: Get status of the latest update applied scale_computing.hypercore.version_update_status_info: register: update_status + - ansible.builtin.assert: that: - update_status.record != [] - update_status.record.keys() | sort == ['from_build', 'percent', 'prepare_status', 'to_build', 'to_version', 'update_status', 'update_status_details', 'usernotes'] + when: sc_config[sc_host].features.version_update.old_update_status_present + + - ansible.builtin.assert: + that: + - update_status.record == None + when: not sc_config[sc_host].features.version_update.old_update_status_present From 49ba0cca3d5650a682adf416e4d7846b840690ac Mon Sep 17 00:00:00 2001 From: Justin Cinkelj Date: Fri, 7 Apr 2023 10:01:57 +0200 Subject: [PATCH 6/6] CI Use dedicated NTP servers Failed job https://github.com/ScaleComputing/HyperCoreAnsibleCollection/actions/runs/4625947983/jobs/8182201867#step:8:120 Hopefully this will lessen load on pool.ntp.org, so we can continue to use it when not testing. Signed-off-by: Justin Cinkelj --- ci-infra/ntp-chrony/Readme.md | 19 ++++++++ ci-infra/ntp-chrony/chrony.conf | 47 +++++++++++++++++++ .../role_cluster_config/tasks/main.yml | 4 +- .../time_server/tasks/01_time_server_info.yml | 4 +- .../time_server/tasks/02_time_server.yml | 6 ++- 5 files changed, 75 insertions(+), 5 deletions(-) create mode 100644 ci-infra/ntp-chrony/Readme.md create mode 100644 ci-infra/ntp-chrony/chrony.conf diff --git a/ci-infra/ntp-chrony/Readme.md b/ci-infra/ntp-chrony/Readme.md new file mode 100644 index 000000000..fb1c66f09 --- /dev/null +++ b/ci-infra/ntp-chrony/Readme.md @@ -0,0 +1,19 @@ +We need a reliable 2 NTP servers for testing NTP reconfiguration. + +Setup them once in VMs + +``` +apt install -y chrony ntpdate +cat /etc/chrony/chrony.conf + # See chrony.conf example file. + # Just add 'allow all' line if needed +grep 'allow all' /etc/chrony/chrony.conf || echo 'allow all' + +systemctl enable chrony +systemctl restart chrony +chronyc tracking + +ntpdate -qup1 localhost +``` + +Check from outside if firewall allows NTP traffic - port 123/UDP. diff --git a/ci-infra/ntp-chrony/chrony.conf b/ci-infra/ntp-chrony/chrony.conf new file mode 100644 index 000000000..3a1b23834 --- /dev/null +++ b/ci-infra/ntp-chrony/chrony.conf @@ -0,0 +1,47 @@ +# Welcome to the chrony configuration file. See chrony.conf(5) for more +# information about usuable directives. + +# This will use (up to): +# - 4 sources from ntp.ubuntu.com which some are ipv6 enabled +# - 2 sources from 2.ubuntu.pool.ntp.org which is ipv6 enabled as well +# - 1 source from [01].ubuntu.pool.ntp.org each (ipv4 only atm) +# This means by default, up to 6 dual-stack and up to 2 additional IPv4-only +# sources will be used. +# At the same time it retains some protection against one of the entries being +# down (compare to just using one of the lines). See (LP: #1754358) for the +# discussion. +# +# About using servers from the NTP Pool Project in general see (LP: #104525). +# Approved by Ubuntu Technical Board on 2011-02-08. +# See http://www.pool.ntp.org/join.html for more information. +pool ntp.ubuntu.com iburst maxsources 4 +pool 0.ubuntu.pool.ntp.org iburst maxsources 1 +pool 1.ubuntu.pool.ntp.org iburst maxsources 1 +pool 2.ubuntu.pool.ntp.org iburst maxsources 2 + +# This directive specify the location of the file containing ID/key pairs for +# NTP authentication. +keyfile /etc/chrony/chrony.keys + +# This directive specify the file into which chronyd will store the rate +# information. +driftfile /var/lib/chrony/chrony.drift + +# Uncomment the following line to turn logging on. +#log tracking measurements statistics + +# Log files location. +logdir /var/log/chrony + +# Stop bad estimates upsetting machine clock. +maxupdateskew 100.0 + +# This directive enables kernel synchronisation (every 11 minutes) of the +# real-time clock. Note that it can't be used along with the 'rtcfile' directive. +rtcsync + +# Step the system clock instead of slewing it if the adjustment is larger than +# one second, but only in the first three clock updates. +makestep 1 3 + +allow all diff --git a/tests/integration/targets/role_cluster_config/tasks/main.yml b/tests/integration/targets/role_cluster_config/tasks/main.yml index 4c1f80e44..11f6c3df6 100644 --- a/tests/integration/targets/role_cluster_config/tasks/main.yml +++ b/tests/integration/targets/role_cluster_config/tasks/main.yml @@ -29,7 +29,7 @@ # certificate: plain_text_from_x509 config_url: https://login.microsoftonline.com/76d4c62a-a9ca-4dc2-9187-e2cc4d9abe7f/v2.0/.well-known/openid-configuration scopes: openid+profile - time_server: 2.pool.ntp.org + time_server: 10.5.11.5 time_zone: Europe/Ljubljana smtp: server: mail_a.example.com @@ -71,7 +71,7 @@ # certificate: plain_text_from_x509 config_url: https://login.microsoftonline.com/76d4c62a-a9ca-4dc2-9187-e2cc4d9abe7f/v2.0/.well-known/openid-configuration scopes: openid+profile - time_server: 3.pool.ntp.org + time_server: 10.5.11.7 time_zone: Europe/Zagreb smtp: server: mail_b.example.com diff --git a/tests/integration/targets/time_server/tasks/01_time_server_info.yml b/tests/integration/targets/time_server/tasks/01_time_server_info.yml index 06bba6121..e85558e7c 100644 --- a/tests/integration/targets/time_server/tasks/01_time_server_info.yml +++ b/tests/integration/targets/time_server/tasks/01_time_server_info.yml @@ -6,7 +6,9 @@ vars: actual_uuid: "timesource_guid" - actual_host: "3.pool.ntp.org" + # We need 2 NTP servers. + # Set them up per ci-infra/ntp-chrony/Readme.md. + actual_host: "10.5.11.5" block: # ------------------------------------------------------- diff --git a/tests/integration/targets/time_server/tasks/02_time_server.yml b/tests/integration/targets/time_server/tasks/02_time_server.yml index e950ac167..e06929993 100644 --- a/tests/integration/targets/time_server/tasks/02_time_server.yml +++ b/tests/integration/targets/time_server/tasks/02_time_server.yml @@ -5,8 +5,10 @@ SC_PASSWORD: "{{ sc_config[sc_host].sc_password }}" SC_TIMEOUT: "{{ sc_timeout * 10 }}" vars: - time_server_a: 2.pool.ntp.org - time_server_b: 3.pool.ntp.org + # We need 2 NTP servers. + # Set them up per ci-infra/ntp-chrony/Readme.md. + time_server_a: 10.5.11.5 + time_server_b: 10.5.11.7 actual_uuid: timesource_guid block: