Merge pull request #362 from ScaleComputing/hc-v955

Easier local integration testing, SMB server

Author: justinc1

.ansible-lint

@@ -10,3 +10,4 @@ exclude_paths:
   - .github/workflows
   - changelogs
   - .tox
+  - ci-infra/smb-server/compose.yml

DEVELOPMENT.md

@@ -2,7 +2,7 @@
 
 Create python venv and clone code.
 
-```
+```bash
 mkdir -p ansible_collections/scale_computing/
 cd ansible_collections/scale_computing/
 
@@ -109,6 +109,38 @@ Details:
 - IP 10.5.11.39 (see `tests/integration/integration_config.yml.j2`)
 - CI tests should use only `/cidata` and subdirectories
 
+#### Local SMB server
+
+Use `ci-infra/smb-server/compose.yml` to start a local SMB server.
+The HyperCore cluster needs to have access to the SMB server.
+Execute the commands on machine that is accessible to HyperCore cluster -
+e.g. VM on the HyperCore NUC.
+
+Usage:
+
+```bash
+cd ci-infra/smb-server/
+docker compose up
+
+# test it works
+smbclient "//IP_ADDRESS/Home" -U "alice%alipass" -D "/" -c "ls"
+smbclient "//IP_ADDRESS/Home" -U "alice%alipass" -D "/" -c "put compose.yml"
+smbclient "//IP_ADDRESS/Home" -U "alice%alipass" -D "/" -c "ls"
+```
+
+To use this SMB server in `ansible-test integration ...`,
+set in `tests/integration/integration_config.yml`:
+
+```yaml
+smb_server: "IP_ADDRESS"
+smb_share: "/home"
+smb_username: "alice"
+smb_password: "alipass"
+```
+
+Notice - windows SMB server username is `;administrator`, it starts with `;`.
+This Samba SMB server username does not start with `;`.
+
 ### CI NTP server
 
 NTP server is running on VM with github runner.

README.md

@@ -26,6 +26,7 @@ This collection has been tested against following HyperCore cluster versions:
 - v9.2.13.211102
 - v9.3.5.212852
 - v9.4.17.215487
+- v9.5.5.219383
 
 # Installation
 

ci-infra/helpers/run-tests.sh

@@ -0,0 +1,70 @@
+#!/bin/bash
+
+# Usage:
+# ./ci-infra/helpers/run-tests.sh outdir test-names.txt
+# Input:
+# - outdir will contain status/progress file, and log files of individual tests
+# - (optional) test-names.txt contains integration tests to be run, one per line.
+# Output:
+# - logs are one test per file, in outdir/log-timestamp/
+# - list of succeded/failed tests are in outdir/status.txt
+# If line starts with:
+# - PEND (or contains just test name), test will be run
+# - SKIP means skip this test
+# - OK or ERR are set after test is run
+
+set -ue
+# set -v
+OUTD1="$1"
+TNFILE="${2:-}"
+
+TIME=$(date +%Y%m%d-%H%M%S)
+TSTATUS="$OUTD1/status.txt"
+
+[ ! -d "$OUTD1" ] && mkdir "$OUTD1"
+OUTD2="$OUTD1/log-$TIME"
+mkdir "$OUTD2"
+
+if [ ! -f "$TSTATUS" ]
+then
+    if [ -n "$TNFILE" ]
+    then
+        /bin/cp "$TNFILE" "$TSTATUS"
+    else
+        /bin/ls tests/integration/targets/ >"$TSTATUS"
+    fi
+fi
+sed -i 's/^[a-z]/PEND\t&/' "$TSTATUS"
+if grep -q -v -E "^(OK|ERR|PEND|SKIP)" "$TSTATUS"
+then
+    echo "ERROR file content $TSTATUS" 1>&2
+    exit 1
+fi
+
+TEST_NAMES=$(grep "^PEND" "$TSTATUS" | awk '{print $2}')
+# shellcheck disable=SC2086
+echo "Pending tests: "$TEST_NAMES
+# shellcheck disable=SC2086
+for TN in $TEST_NAMES
+do
+    echo "Running test $TN"
+    (
+        echo ansible-test integration --local "$TN"
+        echo "======================"
+        set +e
+        ansible-test integration --local "$TN"
+        RET=$?
+        set -e
+        if [ "$RET" == "0" ]
+        then
+            status="OK"
+        else
+            status="ERR"
+        fi
+        sed -i "s/^PEND\t$TN\$/$status\t$TN/" "$TSTATUS"
+        echo "======================"
+        echo "RESULT $status"
+    ) >"$OUTD2/$TN.log" 2>&1
+    res=$(grep $'\t'"$TN\$" "$TSTATUS" | awk '{print $1}')
+    echo "  result $res $TN"
+done

ci-infra/smb-server/compose.yml

@@ -0,0 +1,75 @@
+---
+# docker-compose.yml example for https://github.com/ServerContainers/samba
+
+services:
+  samba:
+    # build: .
+    # image: ghcr.io/servercontainers/samba
+    image: servercontainers/samba:smbd-only-a3.22.1-s4.21.4-r4
+    restart: always
+    # note that this network_mode makes it super easy (especially for zeroconf) but is not as safe as exposing ports directly
+    # more about that here: https://book.hacktricks.wiki/en/linux-hardening/privilege-escalation/docker-security/docker-breakout-privilege-escalation/index.html#hostnetwork
+    network_mode: host
+    # uncomment to solve bug: https://github.com/ServerContainers/samba/issues/50 - wsdd2 only - not needed for samba
+    #cap_add:
+    #  - CAP_NET_ADMIN
+    environment:
+      # uncomment to enable fail fast (currently only fails fast if there are conflicts/errors during user/group creation)
+      #FAIL_FAST: 1
+
+      MODEL: 'TimeCapsule'
+      AVAHI_NAME: StorageServer
+
+      SAMBA_CONF_LOG_LEVEL: 3
+
+      # uncomment to disable optional services
+      WSDD2_DISABLE: 1
+      AVAHI_DISABLE: 1
+      NETBIOS_DISABLE: 1
+
+      GROUP_family: 1500
+
+      ACCOUNT_alice: alipass
+      UID_alice: 1000
+      GROUPS_alice: family
+
+      ACCOUNT_bob: bobpass
+      UID_bob: 1001
+      GROUPS_bob: family
+
+      # example for hashed password (user: foo | password: bar) - generated using create-hash.sh script.
+      ACCOUNT_foo: "foo:1000:XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX:86C156FC198B358CCCF6278D8BD49B6A:[U          ]:LCT-61B0859A:"
+      # example for password hashes in the list format:
+      # - "ACCOUNT_foo=foo:1000:XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX:8846F7EAEE8FB117AD06BDD830B7586C:[U          ]:LCT-5FE1F7DF:"
+      UID_foo: 1002
+      GROUPS_foo: family
+
+      SAMBA_VOLUME_CONFIG_shared_home: "[Home]; path=/shares/homes/%U; valid users = alice, bob, foo; guest ok = no; read only = no; browseable = yes"
+
+#      SAMBA_VOLUME_CONFIG_aliceonly: "[Alice Share]; path=/shares/alice; valid users = alice; guest ok = no; read only = no; browseable = yes"
+#      SAMBA_VOLUME_CONFIG_alicehidden: "[Alice Hidden Share]; path=/shares/alice-hidden; valid users = alice; guest ok = no; read only = no; browseable = no"
+
+#      SAMBA_VOLUME_CONFIG_bobonly: "[Bob Share]; path=/shares/bob; valid users = bob; guest ok = no; read only = no; browseable = yes"
+
+#      SAMBA_VOLUME_CONFIG_public: "[Public]; path=/shares/public; valid users = alice, bob, foo; guest ok = no; read only = no; browseable = yes; force group = family"
+#      SAMBA_VOLUME_CONFIG_public_ro: "[Public ReadOnly]; path=/shares/public; guest ok = yes; read only = yes; browseable = yes; force group = family"
+
+#      SAMBA_VOLUME_CONFIG_timemachine: "[TimeMachine]; path=/shares/timemachine/%U; valid users = alice, bob, foo; guest ok = no; read only = no; browseable = yes; fruit:time machine = yes; fruit:time machine max size = 500G"
+
+#      SAMBA_VOLUME_CONFIG_guestmultilineexample: |
+#        [Guest Share]
+#         path = /shares/guest
+#         guest ok = yes
+#         browseable = yes
+
+    volumes:
+#      - /etc/avahi/services/:/external/avahi
+
+      # avoid loops when mounting folders to /shares (I'd recommend explicit mapping for each share)
+#      - ./shares/alice:/shares/alice
+#      - ./shares/alice-hidden:/shares/alice-hidden
+#      - ./shares/bob:/shares/bob
+#      - ./shares/public:/shares/public
+#      - ./shares/homes:/shares/homes
+#      - ./shares/timemachine:/shares/timemachine
+      - ./shares-homes:/shares/homes

docs/rst/index.rst

@@ -53,3 +53,4 @@ This collection has been tested against following HyperCore cluster versions:
   - v9.2.13.211102
   - v9.3.5.212852
   - v9.4.17.215487
+  - v9.5.5.219383

tests/integration/integration_config.yml.j2

@@ -270,3 +270,40 @@ sc_config:
         is_supported: True
       cluster_name:
         is_writable: True
+
+  https://10.5.11.206:
+    <<: *base_cfg
+    ci_system_name: vsns206
+    sc_username: admin
+    sc_password: admin
+    sc_replication_dest_host: ""
+    sc_replication_dest_cluster_name: ""
+    sc_replication_dest_username: ""
+    sc_replication_dest_password: ""
+    cluster:
+      name: VSNS206
+    support_tunnel:
+      open: true
+      code: "4427"
+    smtp:
+      <<: *base_smtp
+      from_address: VSNS206@scalecomputing.com
+    version_update:
+      magic_allow_string: "oh-no-no"
+      vm_shutdown_restart_allow_string: "allow-vm-shutdown-restart-test"
+    syslog_server:
+      host: 10.5.11.222
+    features:
+      version_update:
+        current_version: "9.5.5.219383"
+        next_version: ""
+        latest_version: ""
+        can_be_applied: False
+        old_update_status_present: False
+      virtual_disk:
+        is_supported: True
+        replication_factor: 1
+      vtpm_disk:
+        is_supported: True
+      cluster_name:
+        is_writable: True

tests/integration/targets/vm_snapshot/tasks/03_vm_snapshot_attach_disk.yml

@@ -13,6 +13,7 @@
     slot_b: 0  # vm_2, ide_disk
 
     force_reboot: true  # allow forced vm shutdown
+    shutdown_timeout: 10
 
     # disk size gets rounded by HC3:
     #   0.1 GB to 107374182 B
@@ -83,6 +84,7 @@
         source_disk_type: virtio_disk
         source_disk_slot: 1
         force_reboot: "{{ force_reboot }}"
+        shutdown_timeout: "{{ shutdown_timeout }}"
       register: result
     - ansible.builtin.debug:
         var: result
@@ -128,6 +130,7 @@
         source_disk_type: virtio_disk
         source_disk_slot: 1
         force_reboot: "{{ force_reboot }}"
+        shutdown_timeout: "{{ shutdown_timeout }}"
       register: result
     - ansible.builtin.debug:
         var: result
@@ -157,7 +160,7 @@
         source_disk_type: virtio_disk
         source_disk_slot: 1
         force_reboot: "{{ force_reboot }}"
-        shutdown_timeout: 10  # For faster testing. VM has no OS, so it cannot react to ACPI shutdown.
+        shutdown_timeout: "{{ shutdown_timeout }}" # For faster testing. VM has no OS, so it cannot react to ACPI shutdown.
       register: result
     - ansible.builtin.debug:
         var: result
@@ -201,6 +204,7 @@
         source_disk_type: virtio_disk
         source_disk_slot: 1
         force_reboot: "{{ force_reboot }}"
+        shutdown_timeout: "{{ shutdown_timeout }}"
       register: result
     - ansible.builtin.debug:
         var: result
@@ -231,6 +235,7 @@
         source_disk_type: virtio_disk
         source_disk_slot: 1
         force_reboot: "{{ force_reboot }}"
+        shutdown_timeout: "{{ shutdown_timeout }}"
       register: result
     - ansible.builtin.debug:
         var: result
@@ -270,6 +275,7 @@
         source_disk_type: virtio_disk
         source_disk_slot: 1
         force_reboot: "{{ force_reboot }}"
+        shutdown_timeout: "{{ shutdown_timeout }}"
       register: result
     - ansible.builtin.debug:
         var: result

tests/integration/targets/vm_snapshot/tasks/main.yml

@@ -14,6 +14,11 @@
     #    greater than 0, is a newer snapshot
 
   block:
+    - name: Delete VMs before test
+      include_tasks: helper_api_vm_snapshot_delete_all.yml
+      vars:
+        vms_number: "{{ number_of_snapshot_testing_vms }}"
+
     - name: Create VMs
       include_tasks: helper_api_vm_snapshot_create.yml
       vars:
@@ -31,6 +36,7 @@
       vars:
         test_vms_number: "{{ number_of_snapshot_testing_vms }}"
   always:
-    - include_tasks: helper_api_vm_snapshot_delete_all.yml
+    - name: Delete VMs after test
+      include_tasks: helper_api_vm_snapshot_delete_all.yml
       vars:
         vms_number: "{{ number_of_snapshot_testing_vms }}"

tests/sanity/ignore-2.16.txt

@@ -0,0 +1 @@
+ignore.txt