diff --git a/.github/workflows/canary-integration-test.yml b/.github/workflows/canary-integration-test.yml index bb096e215db8..404ad506b80b 100644 --- a/.github/workflows/canary-integration-test.yml +++ b/.github/workflows/canary-integration-test.yml @@ -14,1146 +14,7 @@ defaults: shell: bash --noprofile --norc -eo pipefail -x {0} jobs: - canary: - runs-on: ubuntu-20.04 - if: "!contains(github.event.pull_request.labels.*.name, 'skip-ci')" - strategy: - matrix: - ceph-image: ${{ fromJson(inputs.ceph_images) }} - steps: - - name: checkout - uses: actions/checkout@v4 - with: - fetch-depth: 0 - - - name: consider debugging - uses: ./.github/workflows/tmate_debug - with: - use-tmate: ${{ secrets.USE_TMATE }} - - - name: setup cluster resources - uses: ./.github/workflows/canary-test-config - - - name: set Ceph version in CephCluster manifest - run: tests/scripts/github-action-helper.sh replace_ceph_image "deploy/examples/cluster-test.yaml" "${{ matrix.ceph-image }}" - - - name: validate-yaml - run: tests/scripts/github-action-helper.sh validate_yaml - - - name: use local disk and create partitions for osds - run: | - tests/scripts/github-action-helper.sh use_local_disk - tests/scripts/github-action-helper.sh create_partitions_for_osds - - - name: deploy cluster - run: tests/scripts/github-action-helper.sh deploy_cluster - - - name: setup csi-addons - run: tests/scripts/csiaddons.sh setup_csiaddons - - - name: wait for prepare pod - run: tests/scripts/github-action-helper.sh wait_for_prepare_pod 2 - - - name: wait for ceph to be ready - run: tests/scripts/github-action-helper.sh wait_for_ceph_to_be_ready all 2 - - - name: wait for ceph mgr to be ready - run: | - toolbox=$(kubectl get pod -l app=rook-ceph-tools -n rook-ceph -o jsonpath='{.items[*].metadata.name}') - timeout 15 sh -c "until kubectl -n rook-ceph exec $toolbox -- ceph mgr dump -f json|jq --raw-output .active_addr|grep -Eosq \"(25[0-5]|2[0-4][0-9]|[01]?[0-9][0-9]?)\.(25[0-5]|2[0-4][0-9]|[01]?[0-9][0-9]?)\.(25[0-5]|2[0-4][0-9]|[01]?[0-9][0-9]?)\.(25[0-5]|2[0-4][0-9]|[01]?[0-9][0-9]?)\" ; do sleep 1 && echo 'waiting for the manager IP to be available'; done" - mgr_raw=$(kubectl -n rook-ceph exec $toolbox -- ceph mgr dump -f json|jq --raw-output .active_addr) - timeout 60 sh -c "until kubectl -n rook-ceph exec $toolbox -- curl --silent --show-error ${mgr_raw%%:*}:9283; do echo 'waiting for mgr prometheus exporter to be ready' && sleep 1; done" - - - name: test external script create-external-cluster-resources.py - run: | - toolbox=$(kubectl get pod -l app=rook-ceph-tools -n rook-ceph -o jsonpath='{.items[*].metadata.name}') - kubectl -n rook-ceph exec $toolbox -- mkdir -p /etc/ceph/test-data - kubectl -n rook-ceph cp tests/ceph-status-out $toolbox:/etc/ceph/test-data/ - kubectl -n rook-ceph cp deploy/examples/create-external-cluster-resources.py $toolbox:/etc/ceph - kubectl -n rook-ceph cp deploy/examples/create-external-cluster-resources-tests.py $toolbox:/etc/ceph - timeout 10 sh -c "until kubectl -n rook-ceph exec $toolbox -- python3 /etc/ceph/create-external-cluster-resources.py --rbd-data-pool-name replicapool; do echo 'waiting for script to succeed' && sleep 1; done" - # print existing client auth - kubectl -n rook-ceph exec $toolbox -- ceph auth ls - - - name: test re-running of external script should result in same output - run: | - toolbox=$(kubectl get pod -l app=rook-ceph-tools -n rook-ceph -o jsonpath='{.items[*].metadata.name}') - kubectl -n rook-ceph exec $toolbox -- python3 /etc/ceph/create-external-cluster-resources.py --rbd-data-pool-name=replicapool | tee output1.txt - kubectl -n rook-ceph exec $toolbox -- python3 /etc/ceph/create-external-cluster-resources.py --rbd-data-pool-name=replicapool | tee output2.txt - if cmp output1.txt output2.txt; then - echo "files have same output" - rm output1.txt - rm output2.txt - else - echo "re-run with same flags changed the output, result in failure" - rm output1.txt - rm output2.txt - exit 1 - fi - - - name: dry run external script create-external-cluster-resources.py - run: | - toolbox=$(kubectl get pod -l app=rook-ceph-tools -n rook-ceph -o jsonpath='{.items[*].metadata.name}') - kubectl -n rook-ceph exec $toolbox -- python3 /etc/ceph/create-external-cluster-resources.py --rbd-data-pool-name=replicapool --dry-run - - - name: test external script create-external-cluster-resources.py if users already exist with different caps - run: | - toolbox=$(kubectl get pod -l app=rook-ceph-tools -n rook-ceph -o jsonpath='{.items[*].metadata.name}') - # update client.csi-rbd-provisioner csi user caps - # print client.csi-rbd-provisioner user before update - kubectl -n rook-ceph exec $toolbox -- ceph auth get client.csi-rbd-provisioner - kubectl -n rook-ceph exec $toolbox -- ceph auth caps client.csi-rbd-provisioner mon 'profile rbd, allow command "osd ls"' osd 'profile rbd' mgr 'allow rw' - # print client.csi-rbd-provisioner user after update - kubectl -n rook-ceph exec $toolbox -- ceph auth get client.csi-rbd-provisioner - kubectl -n rook-ceph exec $toolbox -- python3 /etc/ceph/create-external-cluster-resources.py --rbd-data-pool-name replicapool - # print client.csi-rbd-provisioner user after running script - kubectl -n rook-ceph exec $toolbox -- ceph auth get client.csi-rbd-provisioner - - - name: run external script create-external-cluster-resources.py unit tests - run: | - kubectl -n rook-ceph exec $(kubectl get pod -l app=rook-ceph-tools -n rook-ceph -o jsonpath='{.items[0].metadata.name}') -- python3 -m unittest /etc/ceph/create-external-cluster-resources-tests.py - - - name: wait for the subvolumegroup to be created - run: | - toolbox=$(kubectl get pod -l app=rook-ceph-tools -n rook-ceph -o jsonpath='{.items[*].metadata.name}') - timeout 60 sh -c "until kubectl -n rook-ceph exec $toolbox -- ceph fs subvolumegroup ls myfs|jq .[0].name|grep -q "group-a"; do sleep 1 && echo 'waiting for the subvolumegroup to be created'; done" - - - name: test subvolumegroup validation - run: | - toolbox=$(kubectl get pod -l app=rook-ceph-tools -n rook-ceph -o jsonpath='{.items[*].metadata.name}') - # pass the correct subvolumegroup and cephfs_filesystem flag name - kubectl -n rook-ceph exec $toolbox -- python3 /etc/ceph/create-external-cluster-resources.py --rbd-data-pool-name replicapool --subvolume-group group-a --cephfs-filesystem-name myfs - # pass the subvolumegroup name which doesn't exist - kubectl -n rook-ceph exec $toolbox -- python3 /etc/ceph/create-external-cluster-resources.py --rbd-data-pool-name replicapool --subvolume-group false-test-subvolume-group - - - name: dry run test skip monitoring endpoint - run: | - toolbox=$(kubectl get pod -l app=rook-ceph-tools -n rook-ceph -o jsonpath='{.items[*].metadata.name}') - kubectl -n rook-ceph exec $toolbox -- python3 /etc/ceph/create-external-cluster-resources.py --rbd-data-pool-name=replicapool --dry-run --skip-monitoring-endpoint - - - name: test of rados namespace - run: | - kubectl create -f deploy/examples/radosnamespace.yaml - toolbox=$(kubectl get pod -l app=rook-ceph-tools -n rook-ceph -o jsonpath='{.items[*].metadata.name}') - timeout 60 sh -c "until kubectl -n rook-ceph exec $toolbox -- rbd namespace ls replicapool --format=json|jq .[0].name|grep -q "namespace-a"; do sleep 1 && echo 'waiting for the rados namespace to be created'; done" - kubectl delete -f deploy/examples/radosnamespace.yaml - - - name: test rados namespace validation - run: | - toolbox=$(kubectl get pod -l app=rook-ceph-tools -n rook-ceph -o jsonpath='{.items[*].metadata.name}') - # create `radosNamespace1` rados-namespace for `replicapool` rbd data-pool - kubectl -n rook-ceph exec $toolbox -- rbd namespace create replicapool/radosnamespace1 - kubectl -n rook-ceph exec $toolbox -- python3 /etc/ceph/create-external-cluster-resources.py --rbd-data-pool-name replicapool --rados-namespace radosnamespace1 - # test the rados namespace which not exit for replicapool(false testing) - if output=$(kubectl -n rook-ceph exec $toolbox -- python3 /etc/ceph/create-external-cluster-resources.py --rbd-data-pool-name replicapool --rados-namespace false-test-namespace); then - echo "unexpectedly succeeded after passing the wrong rados namespace: $output" - exit 1 - else - echo "script failed because wrong rados namespace was passed" - fi - - - name: test external script with restricted_auth_permission flag and without having cephfs_filesystem flag - run: | - toolbox=$(kubectl get pod -l app=rook-ceph-tools -n rook-ceph -o jsonpath='{.items[*].metadata.name}') - kubectl -n rook-ceph exec $toolbox -- python3 /etc/ceph/create-external-cluster-resources.py --rbd-data-pool-name replicapool --k8s-cluster-name rookstorage --restricted-auth-permission true - - - name: test external script with restricted_auth_permission flag and with --cluster-name legacy flag - run: | - toolbox=$(kubectl get pod -l app=rook-ceph-tools -n rook-ceph -o jsonpath='{.items[*].metadata.name}') - kubectl -n rook-ceph exec $toolbox -- python3 /etc/ceph/create-external-cluster-resources.py --rbd-data-pool-name replicapool --cluster-name rookstorage --restricted-auth-permission true - - - name: test external script with restricted_auth_permission flag - run: | - toolbox=$(kubectl get pod -l app=rook-ceph-tools -n rook-ceph -o jsonpath='{.items[*].metadata.name}') - kubectl -n rook-ceph exec $toolbox -- python3 /etc/ceph/create-external-cluster-resources.py --cephfs-filesystem-name myfs --rbd-data-pool-name replicapool --k8s-cluster-name rookstorage --restricted-auth-permission true - - - name: test the upgrade flag - run: | - toolbox=$(kubectl get pod -l app=rook-ceph-tools -n rook-ceph -o jsonpath='{.items[*].metadata.name}') - # print existing client auth - kubectl -n rook-ceph exec $toolbox -- ceph auth ls - # update the existing non-restricted client auth with the new ones - kubectl -n rook-ceph exec $toolbox -- python3 /etc/ceph/create-external-cluster-resources.py --upgrade - # print upgraded client auth - kubectl -n rook-ceph exec $toolbox -- ceph auth ls - - - name: test the upgrade flag for restricted auth user - run: | - toolbox=$(kubectl get pod -l app=rook-ceph-tools -n rook-ceph -o jsonpath='{.items[*].metadata.name}') - # print existing client auth - kubectl -n rook-ceph exec $toolbox -- ceph auth get client.csi-rbd-node-rookstorage-replicapool - # restricted auth user need to provide --rbd-data-pool-name, - # --k8s-cluster-name and --run-as-user flag while upgrading - kubectl -n rook-ceph exec $toolbox -- python3 /etc/ceph/create-external-cluster-resources.py --upgrade --rbd-data-pool-name replicapool --k8s-cluster-name rookstorage --run-as-user client.csi-rbd-node-rookstorage-replicapool - # print upgraded client auth - kubectl -n rook-ceph exec $toolbox -- ceph auth get client.csi-rbd-node-rookstorage-replicapool - - - name: validate-rgw-endpoint - run: | - rgw_endpoint=$(kubectl get service -n rook-ceph -l rgw=store-a | awk '/rgw/ {print $3":80"}') - toolbox=$(kubectl get pod -l app=rook-ceph-tools -n rook-ceph -o jsonpath='{.items[*].metadata.name}') - # pass the valid rgw-endpoint of same ceph cluster - timeout 15 sh -c "until kubectl -n rook-ceph exec $toolbox -- python3 /etc/ceph/create-external-cluster-resources.py --rbd-data-pool-name replicapool --rgw-endpoint $rgw_endpoint 2> output.txt; do sleep 1 && echo 'waiting for the rgw endpoint to be validated'; done" - tests/scripts/github-action-helper.sh check_empty_file output.txt - rm -f output.txt - # pass the invalid rgw-endpoint of different ceph cluster - timeout 15 sh -c "until kubectl -n rook-ceph exec $toolbox -- python3 /etc/ceph/create-external-cluster-resources.py --rbd-data-pool-name replicapool --rgw-endpoint 10.108.96.128:80 2> output.txt; do sleep 1 && echo 'waiting for the rgw endpoint to be validated'; done" - if [ -s output.txt ]; then - echo "script run completed with stderr error after passing the wrong rgw-endpoint: $output" - rm -f output.txt - else - echo "no stderr error even wrong endpoint was provided" - rm -f output.txt - exit 1 - fi - # pass the valid rgw-endpoint of same ceph cluster with --rgw-tls-cert-path - timeout 15 sh -c "until kubectl -n rook-ceph exec $toolbox -- python3 /etc/ceph/create-external-cluster-resources.py --rbd-data-pool-name replicapool --rgw-endpoint $rgw_endpoint --rgw-tls-cert-path my-cert 2> output.txt; do sleep 1 && echo 'waiting for the rgw endpoint to be validated'; done" - tests/scripts/github-action-helper.sh check_empty_file output.txt - rm -f output.txt - # pass the valid rgw-endpoint of same ceph cluster with --rgw-skip-tls - timeout 15 sh -c "until kubectl -n rook-ceph exec $toolbox -- python3 /etc/ceph/create-external-cluster-resources.py --rbd-data-pool-name replicapool --rgw-endpoint $rgw_endpoint --rgw-skip-tls true 2> output.txt; do sleep 1 && echo 'waiting for the rgw endpoint to be validated'; done" - tests/scripts/github-action-helper.sh check_empty_file output.txt - rm -f output.txt - - - name: validate multisite - run: | - toolbox=$(kubectl get pod -l app=rook-ceph-tools -n rook-ceph -o jsonpath='{.items[*].metadata.name}') - # create realm - kubectl -n rook-ceph exec $toolbox -- radosgw-admin realm create --rgw-realm=realm1 - # pass correct realm - kubectl -n rook-ceph exec $toolbox -- python3 /etc/ceph/create-external-cluster-resources.py --rbd-data-pool-name replicapool --rgw-realm-name realm1 - # pass wrong realm - if output=$(kubectl -n rook-ceph exec $toolbox -- python3 /etc/ceph/create-external-cluster-resources.py --rbd-data-pool-name replicapool --rgw-realm-name realm3); then - echo "script run completed with stderr error after passing the wrong realm: $output" - else - echo "script failed because wrong realm was passed" - fi - - - name: test topology flags - run: | - toolbox=$(kubectl get pod -l app=rook-ceph-tools -n rook-ceph -o jsonpath='{.items[*].metadata.name}') - # create 3 replica-1 pools - sed -i 's/replicapool/replica1a/' deploy/examples/pool-test.yaml - kubectl create -f deploy/examples/pool-test.yaml - sed -i 's/replica1a/replica1b/' deploy/examples/pool-test.yaml - kubectl create -f deploy/examples/pool-test.yaml - sed -i 's/replica1b/replica1c/' deploy/examples/pool-test.yaml - kubectl create -f deploy/examples/pool-test.yaml - # bring back the original file - sed -i 's/replica1c/replicapool/' deploy/examples/pool-test.yaml - - # check and wait for the pools to get ready - kubectl wait --for='jsonpath={.status.phase}=Ready' Cephblockpool/replica1a -nrook-ceph - kubectl wait --for='jsonpath={.status.phase}=Ready' Cephblockpool/replica1b -nrook-ceph - kubectl wait --for='jsonpath={.status.phase}=Ready' Cephblockpool/replica1c -nrook-ceph - - # pass correct flags - kubectl -n rook-ceph exec $toolbox -- python3 /etc/ceph/create-external-cluster-resources.py --rbd-data-pool-name replicapool --topology-pools replica1a,replica1b,replica1c --topology-failure-domain-label hostname --topology-failure-domain-values minikube,minikube-m02,minikube-m03 - # pass the pool which is not exists - if output=$(kubectl -n rook-ceph exec $toolbox -- python3 /etc/ceph/create-external-cluster-resources.py --rbd-data-pool-name replicapool --topology-pools ab,cd,ef --topology-failure-domain-label hostname --topology-failure-domain-values minikube,minikube-m02,minikube-m03); then - echo "script run completed with stderr error after passing the wrong pools: $output" - else - echo "script failed because wrong pools doesn't exist" - fi - # dont pass all topology flags - if output=$(kubectl -n rook-ceph exec $toolbox -- python3 /etc/ceph/create-external-cluster-resources.py --rbd-data-pool-name replicapool --topology-pools replica1a,replica1b,replica1c --topology-failure-domain-values minikube,minikube-m02,minikube-m03); then - echo "script run completed with stderr error after passing the wrong flags: $output" - else - echo "script failed because topology-failure-domain-label is missing" - fi - - - name: test enable v2 mon port - run: | - toolbox=$(kubectl get pod -l app=rook-ceph-tools -n rook-ceph -o jsonpath='{.items[*].metadata.name}') - kubectl -n rook-ceph exec $toolbox -- python3 /etc/ceph/create-external-cluster-resources.py --rbd-data-pool-name replicapool --v2-port-enable - - - name: check-ownerreferences - run: tests/scripts/github-action-helper.sh check_ownerreferences - - - name: validate csi-addons - run: | - tests/scripts/csiaddons.sh verify_crd_created - tests/scripts/csiaddons.sh verify_container_is_running - - - name: test osd removal jobs - run: | - kubectl -n rook-ceph delete deploy/rook-ceph-operator - kubectl -n rook-ceph delete deploy/rook-ceph-osd-1 --grace-period=0 --force - sed -i 's//1/' deploy/examples/osd-purge.yaml - # the CI must force the deletion since we use replica 1 on 2 OSDs - sed -i 's/false/true/' deploy/examples/osd-purge.yaml - sed -i 's|rook/ceph:.*|rook/ceph:local-build|' deploy/examples/osd-purge.yaml - kubectl -n rook-ceph create -f deploy/examples/osd-purge.yaml - toolbox=$(kubectl get pod -l app=rook-ceph-tools -n rook-ceph -o jsonpath='{.items[*].metadata.name}') - kubectl -n rook-ceph exec $toolbox -- ceph status - # wait until osd.1 is removed from the osd tree - timeout 120 sh -c "while kubectl -n rook-ceph exec $toolbox -- ceph osd tree|grep -qE 'osd.1'; do echo 'waiting for ceph osd 1 to be purged'; sleep 1; done" - kubectl -n rook-ceph exec $toolbox -- ceph status - kubectl -n rook-ceph exec $toolbox -- ceph osd tree - - - name: collect common logs - if: always() - uses: ./.github/workflows/collect-logs - with: - name: ${{ github.job }}-${{ matrix.ceph-image }} - - raw-disk-with-object: - runs-on: ubuntu-20.04 - if: "!contains(github.event.pull_request.labels.*.name, 'skip-ci')" - strategy: - matrix: - ceph-image: ${{ fromJson(inputs.ceph_images) }} - steps: - - name: checkout - uses: actions/checkout@v4 - with: - fetch-depth: 0 - - - name: consider debugging - uses: ./.github/workflows/tmate_debug - with: - use-tmate: ${{ secrets.USE_TMATE }} - - - name: setup cluster resources - uses: ./.github/workflows/canary-test-config - - - name: set Ceph version in CephCluster manifest - run: tests/scripts/github-action-helper.sh replace_ceph_image "deploy/examples/cluster-test.yaml" "${{ github.event.inputs.ceph-image }}" - - - name: validate-yaml - run: tests/scripts/github-action-helper.sh validate_yaml - - - name: use local disk as OSD - run: | - tests/scripts/github-action-helper.sh use_local_disk - export BLOCK="/dev/$(tests/scripts/github-action-helper.sh find_extra_block_dev)" - tests/scripts/create-bluestore-partitions.sh --disk "$BLOCK" --wipe-only - - - name: prepare loop devices for osds - run: | - tests/scripts/github-action-helper.sh prepare_loop_devices 1 - - - name: deploy cluster - run: | - export ALLOW_LOOP_DEVICES=true - tests/scripts/github-action-helper.sh deploy_cluster loop - tests/scripts/github-action-helper.sh create_operator_toolbox - - - name: wait for prepare pod - run: tests/scripts/github-action-helper.sh wait_for_prepare_pod 2 - - - name: wait for ceph to be ready - run: | - tests/scripts/github-action-helper.sh wait_for_ceph_to_be_ready osd 2 - - - name: wait for object stores to be ready - run: | - tests/scripts/validate_cluster.sh rgw store-a - tests/scripts/validate_cluster.sh rgw store-b - - - name: test toolbox-operator-image pod - run: | - # waiting for toolbox operator image pod to get ready - kubectl -n rook-ceph wait --for=condition=ready pod -l app=rook-ceph-tools-operator-image --timeout=180s - - - name: check s5cmd version in the toolbox image - run: | - toolbox=$(kubectl get pod -l app=rook-ceph-tools-operator-image -n rook-ceph -o jsonpath='{.items[*].metadata.name}') - s5cmd_version="$(kubectl -n rook-ceph exec ${toolbox} -- /usr/local/bin/s5cmd version)" - echo ${s5cmd_version} | grep -q "^v2.2.1" || { - echo " Error: the version of s5cmd version in the toolbox is not the expected v2.2.1 but ${s5cmd_version}" - exit 1 - } - - - name: check-ownerreferences - run: tests/scripts/github-action-helper.sh check_ownerreferences - - - name: collect common logs - if: always() - uses: ./.github/workflows/collect-logs - with: - name: ${{ github.job }}-${{ matrix.ceph-image }} - - two-osds-in-device: - runs-on: ubuntu-20.04 - if: "!contains(github.event.pull_request.labels.*.name, 'skip-ci')" - strategy: - matrix: - ceph-image: ${{ fromJson(inputs.ceph_images) }} - steps: - - name: checkout - uses: actions/checkout@v4 - with: - fetch-depth: 0 - - name: consider debugging - uses: ./.github/workflows/tmate_debug - with: - use-tmate: ${{ secrets.USE_TMATE }} - - - name: setup cluster resources - uses: ./.github/workflows/canary-test-config - - - name: set Ceph version in CephCluster manifest - run: tests/scripts/github-action-helper.sh replace_ceph_image "deploy/examples/cluster-test.yaml" "${{ github.event.inputs.ceph-image }}" - - - name: validate-yaml - run: tests/scripts/github-action-helper.sh validate_yaml - - - name: use local disk as OSD - run: | - tests/scripts/github-action-helper.sh use_local_disk - export BLOCK="/dev/$(tests/scripts/github-action-helper.sh find_extra_block_dev)" - tests/scripts/create-bluestore-partitions.sh --disk "$BLOCK" --wipe-only - - - name: deploy cluster - run: tests/scripts/github-action-helper.sh deploy_cluster two_osds_in_device - - - name: wait for prepare pod - run: tests/scripts/github-action-helper.sh wait_for_prepare_pod 2 - - - name: wait for ceph to be ready - run: tests/scripts/github-action-helper.sh wait_for_ceph_to_be_ready osd 2 - - - name: check-ownerreferences - run: tests/scripts/github-action-helper.sh check_ownerreferences - - - name: collect common logs - if: always() - uses: ./.github/workflows/collect-logs - with: - name: ${{ github.job }}-${{ matrix.ceph-image }} - - osd-with-metadata-partition-device: - runs-on: ubuntu-20.04 - if: "!contains(github.event.pull_request.labels.*.name, 'skip-ci')" - strategy: - matrix: - ceph-image: ${{ fromJson(inputs.ceph_images) }} - steps: - - name: checkout - uses: actions/checkout@v4 - with: - fetch-depth: 0 - - - name: consider debugging - uses: ./.github/workflows/tmate_debug - with: - use-tmate: ${{ secrets.USE_TMATE }} - - - name: setup cluster resources - uses: ./.github/workflows/canary-test-config - - - name: validate-yaml - run: tests/scripts/github-action-helper.sh validate_yaml - - - name: use local disk as OSD metadata partition - run: | - export BLOCK="/dev/$(tests/scripts/github-action-helper.sh find_extra_block_dev)" - tests/scripts/github-action-helper.sh use_local_disk - tests/scripts/create-bluestore-partitions.sh --disk "$BLOCK" --bluestore-type block.db --osd-count 1 - - - name: deploy cluster - run: | - tests/scripts/github-action-helper.sh deploy_cluster osd_with_metadata_partition_device - - - name: wait for prepare pod - run: tests/scripts/github-action-helper.sh wait_for_prepare_pod 1 - - - name: wait for ceph to be ready - run: tests/scripts/github-action-helper.sh wait_for_ceph_to_be_ready osd 1 - - - name: check-ownerreferences - run: tests/scripts/github-action-helper.sh check_ownerreferences - - - name: collect common logs - if: always() - uses: ./.github/workflows/collect-logs - with: - name: ${{ github.job }}-${{ matrix.ceph-image }} - - osd-with-metadata-device: - runs-on: ubuntu-20.04 - if: "!contains(github.event.pull_request.labels.*.name, 'skip-ci')" - strategy: - matrix: - ceph-image: ${{ fromJson(inputs.ceph_images) }} - steps: - - name: checkout - uses: actions/checkout@v4 - with: - fetch-depth: 0 - - - name: consider debugging - uses: ./.github/workflows/tmate_debug - with: - use-tmate: ${{ secrets.USE_TMATE }} - - - name: setup cluster resources - uses: ./.github/workflows/canary-test-config - - - name: set Ceph version in CephCluster manifest - run: tests/scripts/github-action-helper.sh replace_ceph_image "deploy/examples/cluster-test.yaml" "${{ github.event.inputs.ceph-image }}" - - - name: validate-yaml - run: tests/scripts/github-action-helper.sh validate_yaml - - - name: use local disk as OSD - run: | - export BLOCK="/dev/$(tests/scripts/github-action-helper.sh find_extra_block_dev)" - tests/scripts/github-action-helper.sh use_local_disk - tests/scripts/create-bluestore-partitions.sh --disk "$BLOCK" --wipe-only - - - name: create LV on disk - run: | - dd if=/dev/zero of=test-rook.img bs=1 count=0 seek=10G - # If we use metadata device, both data devices and metadata devices should be logical volumes or raw devices - tests/scripts/github-action-helper.sh create_LV_on_disk $(sudo losetup --find --show test-rook.img) - - - name: deploy cluster - run: tests/scripts/github-action-helper.sh deploy_cluster osd_with_metadata_device - - - name: wait for prepare pod - run: tests/scripts/github-action-helper.sh wait_for_prepare_pod 1 - - - name: wait for ceph to be ready - run: tests/scripts/github-action-helper.sh wait_for_ceph_to_be_ready osd 1 - - - name: check-ownerreferences - run: tests/scripts/github-action-helper.sh check_ownerreferences - - - name: collect common logs - if: always() - uses: ./.github/workflows/collect-logs - with: - name: ${{ github.job }}-${{ matrix.ceph-image }} - - encryption: - runs-on: ubuntu-20.04 - if: "!contains(github.event.pull_request.labels.*.name, 'skip-ci')" - strategy: - matrix: - ceph-image: ${{ fromJson(inputs.ceph_images) }} - steps: - - name: checkout - uses: actions/checkout@v4 - with: - fetch-depth: 0 - - - name: consider debugging - uses: ./.github/workflows/tmate_debug - with: - use-tmate: ${{ secrets.USE_TMATE }} - - - name: setup cluster resources - uses: ./.github/workflows/canary-test-config - - - name: set Ceph version in CephCluster manifest - run: tests/scripts/github-action-helper.sh replace_ceph_image "deploy/examples/cluster-test.yaml" "${{ github.event.inputs.ceph-image }}" - - - name: validate-yaml - run: tests/scripts/github-action-helper.sh validate_yaml - - - name: use local disk as OSD - run: | - export BLOCK="/dev/$(tests/scripts/github-action-helper.sh find_extra_block_dev)" - tests/scripts/github-action-helper.sh use_local_disk - tests/scripts/create-bluestore-partitions.sh --disk "$BLOCK" --wipe-only - - - name: deploy cluster - run: tests/scripts/github-action-helper.sh deploy_cluster encryption - - - name: wait for prepare pod - run: tests/scripts/github-action-helper.sh wait_for_prepare_pod 1 - - - name: wait for ceph to be ready - run: tests/scripts/github-action-helper.sh wait_for_ceph_to_be_ready osd 1 - - - name: check-ownerreferences - run: tests/scripts/github-action-helper.sh check_ownerreferences - - - name: collect common logs - if: always() - uses: ./.github/workflows/collect-logs - with: - name: ${{ github.job }}-${{ matrix.ceph-image }} - - lvm: - runs-on: ubuntu-20.04 - if: "!contains(github.event.pull_request.labels.*.name, 'skip-ci')" - strategy: - matrix: - ceph-image: ${{ fromJson(inputs.ceph_images) }} - steps: - - name: checkout - uses: actions/checkout@v4 - with: - fetch-depth: 0 - - - name: consider debugging - uses: ./.github/workflows/tmate_debug - with: - use-tmate: ${{ secrets.USE_TMATE }} - - - name: setup cluster resources - uses: ./.github/workflows/canary-test-config - - - name: set Ceph version in CephCluster manifest - run: tests/scripts/github-action-helper.sh replace_ceph_image "deploy/examples/cluster-test.yaml" "${{ github.event.inputs.ceph-image }}" - - - name: validate-yaml - run: tests/scripts/github-action-helper.sh validate_yaml - - - name: use local disk as OSD - run: | - export BLOCK="/dev/$(tests/scripts/github-action-helper.sh find_extra_block_dev)" - tests/scripts/github-action-helper.sh use_local_disk - tests/scripts/create-bluestore-partitions.sh --disk "$BLOCK" --wipe-only - - - name: create LV on disk - run: | - export BLOCK="/dev/$(tests/scripts/github-action-helper.sh find_extra_block_dev)" - tests/scripts/github-action-helper.sh create_LV_on_disk $BLOCK - - - name: deploy cluster - run: tests/scripts/github-action-helper.sh deploy_cluster lvm - - - name: wait for prepare pod - run: tests/scripts/github-action-helper.sh wait_for_prepare_pod 1 - - - name: wait for ceph to be ready - run: tests/scripts/github-action-helper.sh wait_for_ceph_to_be_ready osd 1 - - - name: check-ownerreferences - run: tests/scripts/github-action-helper.sh check_ownerreferences - - - name: collect common logs - if: always() - uses: ./.github/workflows/collect-logs - with: - name: ${{ github.job }}-${{ matrix.ceph-image }} - - pvc: - runs-on: ubuntu-20.04 - if: "!contains(github.event.pull_request.labels.*.name, 'skip-ci')" - strategy: - matrix: - ceph-image: ${{ fromJson(inputs.ceph_images) }} - steps: - - name: checkout - uses: actions/checkout@v4 - with: - fetch-depth: 0 - - - name: consider debugging - uses: ./.github/workflows/tmate_debug - with: - use-tmate: ${{ secrets.USE_TMATE }} - - - name: setup cluster resources - uses: ./.github/workflows/canary-test-config - - - name: set Ceph version in CephCluster manifest - run: tests/scripts/github-action-helper.sh replace_ceph_image "tests/manifests/test-cluster-on-pvc-encrypted.yaml" "${{ github.event.inputs.ceph-image }}" - - - name: use local disk and create partitions for osds - run: | - tests/scripts/github-action-helper.sh use_local_disk - tests/scripts/github-action-helper.sh create_partitions_for_osds - - - name: prepare loop devices for osds - run: | - tests/scripts/github-action-helper.sh prepare_loop_devices 1 - - - name: create cluster prerequisites - run: | - export BLOCK="/dev/$(tests/scripts/github-action-helper.sh find_extra_block_dev)" - tests/scripts/localPathPV.sh "$BLOCK" - tests/scripts/loopDevicePV.sh 1 - tests/scripts/github-action-helper.sh create_cluster_prerequisites - - - name: deploy cluster - run: | - yq write -i deploy/examples/operator.yaml "data.ROOK_CEPH_ALLOW_LOOP_DEVICES" --style=double "true" - tests/scripts/github-action-helper.sh deploy_manifest_with_local_build deploy/examples/operator.yaml - yq write -i tests/manifests/test-cluster-on-pvc-encrypted.yaml "spec.storage.storageClassDeviceSets[0].encrypted" false - yq write -i tests/manifests/test-cluster-on-pvc-encrypted.yaml "spec.storage.storageClassDeviceSets[0].count" 3 - yq write -i tests/manifests/test-cluster-on-pvc-encrypted.yaml "spec.storage.storageClassDeviceSets[0].volumeClaimTemplates[0].spec.resources.requests.storage" 6Gi - kubectl create -f tests/manifests/test-cluster-on-pvc-encrypted.yaml - tests/scripts/github-action-helper.sh deploy_manifest_with_local_build deploy/examples/toolbox.yaml - - - name: wait for prepare pod - run: tests/scripts/github-action-helper.sh wait_for_prepare_pod 3 - - - name: wait for ceph to be ready - run: tests/scripts/github-action-helper.sh wait_for_ceph_to_be_ready osd 3 - - - name: check-ownerreferences - run: tests/scripts/github-action-helper.sh check_ownerreferences - - - name: teardown cluster with cleanup policy - run: | - kubectl -n rook-ceph patch cephcluster rook-ceph --type merge -p '{"spec":{"cleanupPolicy":{"confirmation":"yes-really-destroy-data"}}}' - kubectl -n rook-ceph delete cephcluster rook-ceph - kubectl -n rook-ceph logs deploy/rook-ceph-operator - tests/scripts/github-action-helper.sh wait_for_cleanup_pod - lsblk - export BLOCK="/dev/$(tests/scripts/github-action-helper.sh find_extra_block_dev)" - sudo head --bytes=60 ${BLOCK}1 - sudo head --bytes=60 ${BLOCK}2 - sudo head --bytes=60 /dev/loop1 - - - name: collect common logs - if: always() - uses: ./.github/workflows/collect-logs - with: - name: ${{ github.job }}-${{ matrix.ceph-image }} - - pvc-db: - runs-on: ubuntu-20.04 - if: "!contains(github.event.pull_request.labels.*.name, 'skip-ci')" - strategy: - matrix: - ceph-image: ${{ fromJson(inputs.ceph_images) }} - steps: - - name: checkout - uses: actions/checkout@v4 - with: - fetch-depth: 0 - - - name: consider debugging - uses: ./.github/workflows/tmate_debug - with: - use-tmate: ${{ secrets.USE_TMATE }} - - - name: setup cluster resources - uses: ./.github/workflows/canary-test-config - - - name: set Ceph version in CephCluster manifest - run: tests/scripts/github-action-helper.sh replace_ceph_image "tests/manifests/test-cluster-on-pvc-encrypted.yaml" "${{ github.event.inputs.ceph-image }}" - - - name: use local disk - run: tests/scripts/github-action-helper.sh use_local_disk - - - name: create bluestore partitions and PVCs - run: tests/scripts/github-action-helper.sh create_bluestore_partitions_and_pvcs - - - name: create cluster prerequisites - run: tests/scripts/github-action-helper.sh create_cluster_prerequisites - - - name: deploy cluster - run: | - tests/scripts/github-action-helper.sh deploy_manifest_with_local_build deploy/examples/operator.yaml - yq write -i tests/manifests/test-cluster-on-pvc-encrypted.yaml "spec.storage.storageClassDeviceSets[0].encrypted" false - cat tests/manifests/test-on-pvc-db.yaml >> tests/manifests/test-cluster-on-pvc-encrypted.yaml - kubectl create -f tests/manifests/test-cluster-on-pvc-encrypted.yaml - tests/scripts/github-action-helper.sh deploy_manifest_with_local_build deploy/examples/toolbox.yaml - - - name: wait for prepare pod - run: tests/scripts/github-action-helper.sh wait_for_prepare_pod 1 - - - name: wait for ceph to be ready - run: tests/scripts/github-action-helper.sh wait_for_ceph_to_be_ready osd 1 - - - name: collect common logs - if: always() - uses: ./.github/workflows/collect-logs - with: - name: ${{ github.job }}-${{ matrix.ceph-image }} - - pvc-db-wal: - runs-on: ubuntu-20.04 - if: "!contains(github.event.pull_request.labels.*.name, 'skip-ci')" - strategy: - matrix: - ceph-image: ${{ fromJson(inputs.ceph_images) }} - steps: - - name: checkout - uses: actions/checkout@v4 - with: - fetch-depth: 0 - - - name: consider debugging - uses: ./.github/workflows/tmate_debug - with: - use-tmate: ${{ secrets.USE_TMATE }} - - - name: setup cluster resources - uses: ./.github/workflows/canary-test-config - - - name: set Ceph version in CephCluster manifest - run: tests/scripts/github-action-helper.sh replace_ceph_image "tests/manifests/test-cluster-on-pvc-encrypted.yaml" "${{ github.event.inputs.ceph-image }}" - - - name: use local disk - run: tests/scripts/github-action-helper.sh use_local_disk - - - name: create bluestore partitions and PVCs for wal - run: tests/scripts/github-action-helper.sh create_bluestore_partitions_and_pvcs_for_wal - - - name: create cluster prerequisites - run: tests/scripts/github-action-helper.sh create_cluster_prerequisites - - - name: deploy rook - run: | - tests/scripts/github-action-helper.sh deploy_manifest_with_local_build deploy/examples/operator.yaml - yq write -i tests/manifests/test-cluster-on-pvc-encrypted.yaml "spec.storage.storageClassDeviceSets[0].encrypted" false - cat tests/manifests/test-on-pvc-db.yaml >> tests/manifests/test-cluster-on-pvc-encrypted.yaml - cat tests/manifests/test-on-pvc-wal.yaml >> tests/manifests/test-cluster-on-pvc-encrypted.yaml - kubectl create -f tests/manifests/test-cluster-on-pvc-encrypted.yaml - tests/scripts/github-action-helper.sh deploy_manifest_with_local_build deploy/examples/toolbox.yaml - - - name: wait for prepare pod - run: tests/scripts/github-action-helper.sh wait_for_prepare_pod 1 - - - name: wait for ceph to be ready - run: | - tests/scripts/github-action-helper.sh wait_for_ceph_to_be_ready osd 1 - kubectl -n rook-ceph get pods - - - name: collect common logs - if: always() - uses: ./.github/workflows/collect-logs - with: - name: ${{ github.job }}-${{ matrix.ceph-image }} - - encryption-pvc: - runs-on: ubuntu-20.04 - if: "!contains(github.event.pull_request.labels.*.name, 'skip-ci')" - strategy: - matrix: - ceph-image: ${{ fromJson(inputs.ceph_images) }} - steps: - - name: checkout - uses: actions/checkout@v4 - with: - fetch-depth: 0 - - - name: consider debugging - uses: ./.github/workflows/tmate_debug - with: - use-tmate: ${{ secrets.USE_TMATE }} - - - name: setup cluster resources - uses: ./.github/workflows/canary-test-config - - - name: set Ceph version in CephCluster manifest - run: tests/scripts/github-action-helper.sh replace_ceph_image "tests/manifests/test-cluster-on-pvc-encrypted.yaml" "${{ github.event.inputs.ceph-image }}" - - - name: use local disk and create partitions for osds - run: | - tests/scripts/github-action-helper.sh use_local_disk - tests/scripts/github-action-helper.sh create_partitions_for_osds - - - name: create cluster prerequisites - run: | - export BLOCK="/dev/$(tests/scripts/github-action-helper.sh find_extra_block_dev)" - tests/scripts/localPathPV.sh "$BLOCK" - tests/scripts/github-action-helper.sh create_cluster_prerequisites - - - name: deploy cluster - run: | - tests/scripts/github-action-helper.sh deploy_manifest_with_local_build deploy/examples/operator.yaml - yq write -i tests/manifests/test-cluster-on-pvc-encrypted.yaml "spec.storage.storageClassDeviceSets[0].count" 2 - yq write -i tests/manifests/test-cluster-on-pvc-encrypted.yaml "spec.storage.storageClassDeviceSets[0].volumeClaimTemplates[0].spec.resources.requests.storage" 6Gi - kubectl create -f tests/manifests/test-cluster-on-pvc-encrypted.yaml - tests/scripts/github-action-helper.sh deploy_manifest_with_local_build deploy/examples/toolbox.yaml - - - name: wait for prepare pod - run: tests/scripts/github-action-helper.sh wait_for_prepare_pod 2 - - - name: wait for ceph to be ready - run: | - tests/scripts/github-action-helper.sh wait_for_ceph_to_be_ready osd 2 - kubectl -n rook-ceph get secrets - sudo lsblk - - - name: teardown cluster with cleanup policy - run: | - kubectl -n rook-ceph patch cephcluster rook-ceph --type merge -p '{"spec":{"cleanupPolicy":{"confirmation":"yes-really-destroy-data"}}}' - kubectl -n rook-ceph delete cephcluster rook-ceph - kubectl -n rook-ceph logs deploy/rook-ceph-operator - tests/scripts/github-action-helper.sh wait_for_cleanup_pod - export BLOCK="/dev/$(tests/scripts/github-action-helper.sh find_extra_block_dev)" - sudo head --bytes=60 ${BLOCK}1 - sudo head --bytes=60 ${BLOCK}2 - sudo lsblk - - - name: collect common logs - if: always() - uses: ./.github/workflows/collect-logs - with: - name: ${{ github.job }}-${{ matrix.ceph-image }} - - encryption-pvc-db: - runs-on: ubuntu-20.04 - if: "!contains(github.event.pull_request.labels.*.name, 'skip-ci')" - strategy: - matrix: - ceph-image: ${{ fromJson(inputs.ceph_images) }} - steps: - - name: checkout - uses: actions/checkout@v4 - with: - fetch-depth: 0 - - - name: consider debugging - uses: ./.github/workflows/tmate_debug - with: - use-tmate: ${{ secrets.USE_TMATE }} - - - name: setup cluster resources - uses: ./.github/workflows/canary-test-config - - - name: set Ceph version in CephCluster manifest - run: tests/scripts/github-action-helper.sh replace_ceph_image "tests/manifests/test-cluster-on-pvc-encrypted.yaml" "${{ github.event.inputs.ceph-image }}" - - - name: use local disk - run: tests/scripts/github-action-helper.sh use_local_disk - - - name: create bluestore partitions and PVCs - run: tests/scripts/github-action-helper.sh create_bluestore_partitions_and_pvcs - - - name: create cluster prerequisites - run: tests/scripts/github-action-helper.sh create_cluster_prerequisites - - - name: deploy cluster - run: | - tests/scripts/github-action-helper.sh deploy_manifest_with_local_build deploy/examples/operator.yaml - cat tests/manifests/test-on-pvc-db.yaml >> tests/manifests/test-cluster-on-pvc-encrypted.yaml - kubectl create -f tests/manifests/test-cluster-on-pvc-encrypted.yaml - tests/scripts/github-action-helper.sh deploy_manifest_with_local_build deploy/examples/toolbox.yaml - - - name: wait for prepare pod - run: tests/scripts/github-action-helper.sh wait_for_prepare_pod 1 - - - name: wait for ceph to be ready - run: | - tests/scripts/github-action-helper.sh wait_for_ceph_to_be_ready osd 1 - kubectl -n rook-ceph get pods - kubectl -n rook-ceph get secrets - - - name: collect common logs - if: always() - uses: ./.github/workflows/collect-logs - with: - name: ${{ github.job }}-${{ matrix.ceph-image }} - - encryption-pvc-db-wal: - runs-on: ubuntu-20.04 - if: "!contains(github.event.pull_request.labels.*.name, 'skip-ci')" - strategy: - matrix: - ceph-image: ${{ fromJson(inputs.ceph_images) }} - steps: - - name: checkout - uses: actions/checkout@v4 - with: - fetch-depth: 0 - - - name: consider debugging - uses: ./.github/workflows/tmate_debug - with: - use-tmate: ${{ secrets.USE_TMATE }} - - - name: setup cluster resources - uses: ./.github/workflows/canary-test-config - - - name: set Ceph version in CephCluster manifest - run: tests/scripts/github-action-helper.sh replace_ceph_image "tests/manifests/test-cluster-on-pvc-encrypted.yaml" "${{ github.event.inputs.ceph-image }}" - - - name: use local disk - run: tests/scripts/github-action-helper.sh use_local_disk - - - name: create bluestore partitions and PVCs for wal - run: tests/scripts/github-action-helper.sh create_bluestore_partitions_and_pvcs_for_wal - - - name: create cluster prerequisites - run: tests/scripts/github-action-helper.sh create_cluster_prerequisites - - - name: deploy rook - run: | - tests/scripts/github-action-helper.sh deploy_manifest_with_local_build deploy/examples/operator.yaml - cat tests/manifests/test-on-pvc-db.yaml >> tests/manifests/test-cluster-on-pvc-encrypted.yaml - cat tests/manifests/test-on-pvc-wal.yaml >> tests/manifests/test-cluster-on-pvc-encrypted.yaml - kubectl create -f tests/manifests/test-cluster-on-pvc-encrypted.yaml - kubectl patch -n rook-ceph cephcluster rook-ceph --type merge -p '{"spec":{"security":{"keyRotation":{"enabled": true, "schedule":"*/1 * * * *"}}}}' - tests/scripts/github-action-helper.sh deploy_manifest_with_local_build deploy/examples/toolbox.yaml - - - name: wait for prepare pod - run: tests/scripts/github-action-helper.sh wait_for_prepare_pod 1 - - - name: wait for ceph to be ready - run: | - tests/scripts/github-action-helper.sh wait_for_ceph_to_be_ready osd 1 - kubectl -n rook-ceph get pods - kubectl -n rook-ceph get secrets - - - name: wait and verify key rotation - run: tests/scripts/github-action-helper.sh verify_key_rotation - - - name: test osd deployment removal and re-hydration - run: | - kubectl -n rook-ceph delete deploy/rook-ceph-osd-0 - tests/scripts/github-action-helper.sh wait_for_ceph_to_be_ready osd 1 - - - name: collect common logs - if: always() - uses: ./.github/workflows/collect-logs - with: - name: ${{ github.job }}-${{ matrix.ceph-image }} - - encryption-pvc-kms-vault-token-auth: - runs-on: ubuntu-20.04 - if: "!contains(github.event.pull_request.labels.*.name, 'skip-ci')" - strategy: - matrix: - ceph-image: ${{ fromJson(inputs.ceph_images) }} - steps: - - name: checkout - uses: actions/checkout@v4 - with: - fetch-depth: 0 - - - name: consider debugging - uses: ./.github/workflows/tmate_debug - with: - use-tmate: ${{ secrets.USE_TMATE }} - - - name: setup cluster resources - uses: ./.github/workflows/canary-test-config - - - name: set Ceph version in CephCluster manifest - run: tests/scripts/github-action-helper.sh replace_ceph_image "tests/manifests/test-cluster-on-pvc-encrypted.yaml" "${{ github.event.inputs.ceph-image }}" - - - name: use local disk and create partitions for osds - run: | - tests/scripts/github-action-helper.sh use_local_disk - tests/scripts/github-action-helper.sh create_partitions_for_osds - - - name: create cluster prerequisites - run: | - export BLOCK="/dev/$(tests/scripts/github-action-helper.sh find_extra_block_dev)" - tests/scripts/localPathPV.sh "$BLOCK" - tests/scripts/github-action-helper.sh create_cluster_prerequisites - - - name: deploy vault - run: tests/scripts/deploy-validate-vault.sh deploy - - - name: deploy cluster - run: | - tests/scripts/github-action-helper.sh deploy_manifest_with_local_build deploy/examples/operator.yaml - cat tests/manifests/test-kms-vault.yaml >> tests/manifests/test-cluster-on-pvc-encrypted.yaml - yq merge --inplace --arrays append tests/manifests/test-cluster-on-pvc-encrypted.yaml tests/manifests/test-kms-vault-spec-token-auth.yaml - yq write -i tests/manifests/test-cluster-on-pvc-encrypted.yaml "spec.storage.storageClassDeviceSets[0].count" 2 - yq write -i tests/manifests/test-cluster-on-pvc-encrypted.yaml "spec.storage.storageClassDeviceSets[0].volumeClaimTemplates[0].spec.resources.requests.storage" 6Gi - kubectl create -f tests/manifests/test-cluster-on-pvc-encrypted.yaml - yq merge --inplace --arrays append tests/manifests/test-object.yaml tests/manifests/test-kms-vault-spec-token-auth.yaml - yq write -i tests/manifests/test-object.yaml "spec.security.kms.connectionDetails.VAULT_BACKEND_PATH" rook/ver2 - kubectl create -f tests/manifests/test-object.yaml - tests/scripts/github-action-helper.sh deploy_manifest_with_local_build deploy/examples/toolbox.yaml - - - name: wait for prepare pod - run: tests/scripts/github-action-helper.sh wait_for_prepare_pod 2 - - - name: wait for ceph to be ready - run: | - tests/scripts/github-action-helper.sh wait_for_ceph_to_be_ready osd 2 - tests/scripts/validate_cluster.sh rgw my-store - kubectl -n rook-ceph get pods - kubectl -n rook-ceph get secrets - - - name: validate osd vault - run: | - tests/scripts/deploy-validate-vault.sh validate_osd - sudo lsblk - - - name: validate rgw vault kv - run: | - tests/scripts/deploy-validate-vault.sh validate_rgw - - - name: validate rgw vault transit - run: | - kubectl delete -f tests/manifests/test-object.yaml - yq write -i tests/manifests/test-object.yaml "spec.security.kms.connectionDetails.VAULT_SECRET_ENGINE" transit - timeout 120 bash -c 'while kubectl -n rook-ceph get cephobjectstore my-store; do echo "waiting for objectstore my-store to delete"; sleep 5; done' - echo "wait for rgw pod to be deleted" - kubectl wait --for=delete pod -l app=rook-ceph-rgw -n rook-ceph --timeout=100s - kubectl create -f tests/manifests/test-object.yaml - tests/scripts/validate_cluster.sh rgw my-store - tests/scripts/deploy-validate-vault.sh validate_rgw - - - name: collect common logs - if: always() - uses: ./.github/workflows/collect-logs - with: - name: ${{ github.job }}-${{ matrix.ceph-image }} - - encryption-pvc-kms-vault-k: - runs-on: ubuntu-20.04 - if: "!contains(github.event.pull_request.labels.*.name, 'skip-ci')" - strategy: - matrix: - ceph-image: ${{ fromJson(inputs.ceph_images) }} - steps: - - name: checkout - uses: actions/checkout@v4 - with: - fetch-depth: 0 - - - name: consider debugging - uses: ./.github/workflows/tmate_debug - with: - use-tmate: ${{ secrets.USE_TMATE }} - - - name: setup cluster resources - uses: ./.github/workflows/canary-test-config - - - name: set Ceph version in CephCluster manifest - run: tests/scripts/github-action-helper.sh replace_ceph_image "tests/manifests/test-cluster-on-pvc-encrypted.yaml" "${{ github.event.inputs.ceph-image }}" - - - name: use local disk and create partitions for osds - run: | - tests/scripts/github-action-helper.sh use_local_disk - tests/scripts/github-action-helper.sh create_partitions_for_osds - - - name: create cluster prerequisites - run: | - export BLOCK="/dev/$(tests/scripts/github-action-helper.sh find_extra_block_dev)" - tests/scripts/localPathPV.sh "$BLOCK" - tests/scripts/github-action-helper.sh create_cluster_prerequisites - - - name: deploy vault - run: KUBERNETES_AUTH=true tests/scripts/deploy-validate-vault.sh deploy - - - name: deploy cluster - run: | - tests/scripts/github-action-helper.sh deploy_manifest_with_local_build deploy/examples/operator.yaml - yq merge --inplace --arrays append tests/manifests/test-cluster-on-pvc-encrypted.yaml tests/manifests/test-kms-vault-spec-k8s-auth.yaml - yq write -i tests/manifests/test-cluster-on-pvc-encrypted.yaml "spec.storage.storageClassDeviceSets[0].count" 2 - yq write -i tests/manifests/test-cluster-on-pvc-encrypted.yaml "spec.storage.storageClassDeviceSets[0].volumeClaimTemplates[0].spec.resources.requests.storage" 6Gi - kubectl create -f tests/manifests/test-cluster-on-pvc-encrypted.yaml - tests/scripts/github-action-helper.sh deploy_manifest_with_local_build deploy/examples/toolbox.yaml - - - name: wait for prepare pod - run: tests/scripts/github-action-helper.sh wait_for_prepare_pod 2 - - - name: wait for ceph to be ready - run: | - tests/scripts/github-action-helper.sh wait_for_ceph_to_be_ready osd 2 - kubectl -n rook-ceph get pods - kubectl -n rook-ceph get secrets - - - name: validate osd vault - run: | - tests/scripts/deploy-validate-vault.sh validate_osd - sudo lsblk - - - name: collect common logs - if: always() - uses: ./.github/workflows/collect-logs - with: - name: ${{ github.job }}-${{ matrix.ceph-image }} - - lvm-pvc: + encryption-pvc-kms-vault-token-auth: runs-on: ubuntu-20.04 if: "!contains(github.event.pull_request.labels.*.name, 'skip-ci')" strategy: @@ -1168,7 +29,7 @@ jobs: - name: consider debugging uses: ./.github/workflows/tmate_debug with: - use-tmate: ${{ secrets.USE_TMATE }} + use-tmate: true - name: setup cluster resources uses: ./.github/workflows/canary-test-config @@ -1176,495 +37,60 @@ jobs: - name: set Ceph version in CephCluster manifest run: tests/scripts/github-action-helper.sh replace_ceph_image "tests/manifests/test-cluster-on-pvc-encrypted.yaml" "${{ github.event.inputs.ceph-image }}" - - name: create cluster prerequisites - run: tests/scripts/github-action-helper.sh create_cluster_prerequisites - - - name: use local disk - run: tests/scripts/github-action-helper.sh use_local_disk + - name: use local disk and create partitions for osds + run: | + tests/scripts/github-action-helper.sh use_local_disk + tests/scripts/github-action-helper.sh create_partitions_for_osds - - name: create LV on disk + - name: create cluster prerequisites run: | export BLOCK="/dev/$(tests/scripts/github-action-helper.sh find_extra_block_dev)" - tests/scripts/github-action-helper.sh create_LV_on_disk $BLOCK - tests/scripts/localPathPV.sh /dev/test-rook-vg/test-rook-lv + tests/scripts/localPathPV.sh "$BLOCK" + tests/scripts/github-action-helper.sh create_cluster_prerequisites + + - name: deploy vault + run: tests/scripts/deploy-validate-vault.sh deploy - name: deploy cluster run: | tests/scripts/github-action-helper.sh deploy_manifest_with_local_build deploy/examples/operator.yaml - yq write -i tests/manifests/test-cluster-on-pvc-encrypted.yaml "spec.storage.storageClassDeviceSets[0].encrypted" false + cat tests/manifests/test-kms-vault.yaml >> tests/manifests/test-cluster-on-pvc-encrypted.yaml + yq merge --inplace --arrays append tests/manifests/test-cluster-on-pvc-encrypted.yaml tests/manifests/test-kms-vault-spec-token-auth.yaml kubectl create -f tests/manifests/test-cluster-on-pvc-encrypted.yaml + yq merge --inplace --arrays append tests/manifests/test-object.yaml tests/manifests/test-kms-vault-spec-token-auth.yaml + yq write -i tests/manifests/test-object.yaml "spec.security.kms.connectionDetails.VAULT_BACKEND_PATH" rook/ver2 + kubectl create -f tests/manifests/test-object.yaml tests/scripts/github-action-helper.sh deploy_manifest_with_local_build deploy/examples/toolbox.yaml - name: wait for prepare pod run: tests/scripts/github-action-helper.sh wait_for_prepare_pod 1 - name: wait for ceph to be ready - run: tests/scripts/github-action-helper.sh wait_for_ceph_to_be_ready osd 1 - - - name: check-ownerreferences - run: tests/scripts/github-action-helper.sh check_ownerreferences - - - name: collect common logs - if: always() - uses: ./.github/workflows/collect-logs - with: - name: ${{ github.job }}-${{ matrix.ceph-image }} - - multi-cluster-mirroring: - runs-on: ubuntu-20.04 - if: "!contains(github.event.pull_request.labels.*.name, 'skip-ci')" - strategy: - matrix: - ceph-image: ${{ fromJson(inputs.ceph_images) }} - steps: - - name: checkout - uses: actions/checkout@v4 - with: - fetch-depth: 0 - - - name: consider debugging - uses: ./.github/workflows/tmate_debug - with: - use-tmate: ${{ secrets.USE_TMATE }} - - - name: setup cluster resources - uses: ./.github/workflows/canary-test-config - - - name: set Ceph version in CephCluster manifest - run: tests/scripts/github-action-helper.sh replace_ceph_image "deploy/examples/cluster-test.yaml" "${{ github.event.inputs.ceph-image }}" - - - name: use local disk into two partitions - run: | - tests/scripts/github-action-helper.sh use_local_disk - export BLOCK="/dev/$(tests/scripts/github-action-helper.sh find_extra_block_dev)" - tests/scripts/create-bluestore-partitions.sh --disk "$BLOCK" --osd-count 2 - sudo lsblk - - - name: deploy first cluster rook - run: | - tests/scripts/github-action-helper.sh deploy_first_rook_cluster - cd deploy/examples/ - sed -i "/resources:/,/ # priorityClassName:/d" rbdmirror.yaml - sed -i "/resources:/,/ # priorityClassName:/d" filesystem-mirror.yaml - kubectl create -f rbdmirror.yaml -f filesystem-mirror.yaml - - # cephfs-mirroring is a push operation - # running bootstrap create on secondary and bootstrap import on primary. mirror daemons on primary. - - name: deploy second cluster rook - run: | - tests/scripts/github-action-helper.sh deploy_second_rook_cluster - cd deploy/examples/ - sed -i 's/namespace: rook-ceph/namespace: rook-ceph-secondary/g' rbdmirror.yaml - kubectl create -f rbdmirror.yaml - - - name: wait for ceph cluster 1 to be ready run: | - mkdir -p test - tests/scripts/validate_cluster.sh osd 1 + tests/scripts/github-action-helper.sh wait_for_ceph_to_be_ready osd 1 + tests/scripts/validate_cluster.sh rgw kubectl -n rook-ceph get pods + kubectl -n rook-ceph get secrets - - name: create replicated mirrored pool on cluster 1 - run: | - cd deploy/examples/ - yq w -i pool-test.yaml spec.mirroring.enabled true - yq w -i pool-test.yaml spec.mirroring.mode image - kubectl create -f pool-test.yaml - timeout 180 sh -c 'until [ "$(kubectl -n rook-ceph get cephblockpool replicapool -o jsonpath='{.status.phase}'|grep -c "Ready")" -eq 1 ]; do echo "waiting for pool replicapool to be created on cluster 1" && sleep 1; done' - - - name: create replicated mirrored pool 2 on cluster 1 - run: | - cd deploy/examples/ - yq w -i pool-test.yaml metadata.name replicapool2 - kubectl create -f pool-test.yaml - timeout 180 sh -c 'until [ "$(kubectl -n rook-ceph get cephblockpool replicapool2 -o jsonpath='{.status.phase}'|grep -c "Ready")" -eq 1 ]; do echo "waiting for pool replicapool2 to be created on cluster 1" && sleep 1; done' - yq w -i pool-test.yaml metadata.name replicapool - - - name: create replicated mirrored pool on cluster 2 - run: | - cd deploy/examples/ - yq w -i pool-test.yaml metadata.namespace rook-ceph-secondary - kubectl create -f pool-test.yaml - timeout 180 sh -c 'until [ "$(kubectl -n rook-ceph-secondary get cephblockpool replicapool -o jsonpath='{.status.phase}'|grep -c "Ready")" -eq 1 ]; do echo "waiting for pool replicapool to be created on cluster 2" && sleep 1; done' - - - name: create replicated mirrored pool 2 on cluster 2 - run: | - cd deploy/examples/ - yq w -i pool-test.yaml metadata.name replicapool2 - kubectl create -f pool-test.yaml - timeout 180 sh -c 'until [ "$(kubectl -n rook-ceph-secondary get cephblockpool replicapool -o jsonpath='{.status.phase}'|grep -c "Ready")" -eq 1 ]; do echo "waiting for pool replicapool2 to be created on cluster 2" && sleep 1; done' - - - name: create images in the pools - run: | - kubectl exec -n rook-ceph deploy/rook-ceph-tools -ti -- rbd -p replicapool create test -s 1G - kubectl exec -n rook-ceph deploy/rook-ceph-tools -t -- rbd mirror image enable replicapool/test snapshot - kubectl exec -n rook-ceph deploy/rook-ceph-tools -t -- rbd -p replicapool info test - kubectl exec -n rook-ceph deploy/rook-ceph-tools -ti -- rbd -p replicapool2 create test -s 1G - kubectl exec -n rook-ceph deploy/rook-ceph-tools -t -- rbd mirror image enable replicapool2/test snapshot - kubectl exec -n rook-ceph deploy/rook-ceph-tools -t -- rbd -p replicapool2 info test - - - name: copy block mirror peer secret into the other cluster for replicapool - run: | - kubectl -n rook-ceph get secret pool-peer-token-replicapool -o yaml > pool-peer-token-replicapool.yaml - yq delete --inplace pool-peer-token-replicapool.yaml metadata.ownerReferences - yq write --inplace pool-peer-token-replicapool.yaml metadata.namespace rook-ceph-secondary - yq write --inplace pool-peer-token-replicapool.yaml metadata.name pool-peer-token-replicapool-config - kubectl create --namespace=rook-ceph-secondary -f pool-peer-token-replicapool.yaml - - - name: copy block mirror peer secret into the other cluster for replicapool2 (using cluster global peer) - run: | - kubectl -n rook-ceph get secret cluster-peer-token-my-cluster -o yaml > cluster-peer-token-my-cluster.yaml - yq delete --inplace cluster-peer-token-my-cluster.yaml metadata.ownerReferences - yq write --inplace cluster-peer-token-my-cluster.yaml metadata.namespace rook-ceph-secondary - yq write --inplace cluster-peer-token-my-cluster.yaml metadata.name cluster-peer-token-my-cluster-config - kubectl create --namespace=rook-ceph-secondary -f cluster-peer-token-my-cluster.yaml - - - name: add block mirror peer secret to the other cluster for replicapool - run: | - kubectl -n rook-ceph-secondary patch cephblockpool replicapool --type merge -p '{"spec":{"mirroring":{"peers": {"secretNames": ["pool-peer-token-replicapool-config"]}}}}' - - - name: add block mirror peer secret to the other cluster for replicapool2 (using cluster global peer) - run: | - kubectl -n rook-ceph-secondary patch cephblockpool replicapool2 --type merge -p '{"spec":{"mirroring":{"peers": {"secretNames": ["cluster-peer-token-my-cluster-config"]}}}}' - - - name: verify image has been mirrored for replicapool - run: | - # let's wait a bit for the image to be present - timeout 120 sh -c 'until [ "$(kubectl exec -n rook-ceph-secondary deploy/rook-ceph-tools -t -- rbd -p replicapool ls|grep -c test)" -eq 1 ]; do echo "waiting for image to be mirrored in pool replicapool" && sleep 1; done' - - - name: verify image has been mirrored for replicapool2 - run: | - # let's wait a bit for the image to be present - timeout 120 sh -c 'until [ "$(kubectl exec -n rook-ceph-secondary deploy/rook-ceph-tools -t -- rbd -p replicapool2 ls|grep -c test)" -eq 1 ]; do echo "waiting for image to be mirrored in pool replicapool2" && sleep 1; done' - - - name: display cephblockpool and image status - run: | - timeout 80 sh -c 'until [ "$(kubectl -n rook-ceph-secondary get cephblockpool replicapool -o jsonpath='{.status.mirroringStatus.summary.daemon_health}'|grep -c OK)" -eq 1 ]; do echo "waiting for mirroring status to be updated in replicapool" && sleep 1; done' - timeout 80 sh -c 'until [ "$(kubectl -n rook-ceph-secondary get cephblockpool replicapool2 -o jsonpath='{.status.mirroringStatus.summary.daemon_health}'|grep -c OK)" -eq 1 ]; do echo "waiting for mirroring status to be updated in replicapool2" && sleep 1; done' - kubectl -n rook-ceph-secondary get cephblockpool replicapool -o yaml - kubectl -n rook-ceph-secondary get cephblockpool replicapool2 -o yaml - kubectl exec -n rook-ceph deploy/rook-ceph-tools -t -- rbd -p replicapool info test - kubectl exec -n rook-ceph deploy/rook-ceph-tools -t -- rbd -p replicapool2 info test - - - name: copy block mirror peer secret into the primary cluster for replicapool - run: | - kubectl -n rook-ceph-secondary get secret pool-peer-token-replicapool -o yaml |\ - sed 's/namespace: rook-ceph-secondary/namespace: rook-ceph/g; s/name: pool-peer-token-replicapool/name: pool-peer-token-replicapool-config/g' |\ - kubectl create --namespace=rook-ceph -f - - - - name: add block mirror peer secret to the primary cluster for replicapool - run: | - kubectl -n rook-ceph patch cephblockpool replicapool --type merge -p '{"spec":{"mirroring":{"peers": {"secretNames": ["pool-peer-token-replicapool-config"]}}}}' - - - name: wait for rook-ceph-csi-mapping-config to be updated with cluster ID - run: | - timeout 60 sh -c 'until [ "$(kubectl get cm -n rook-ceph rook-ceph-csi-mapping-config -o jsonpath='{.data.csi-mapping-config-json}' | grep -c "rook-ceph-secondary")" -eq 1 ]; do echo "waiting for rook-ceph-csi-mapping-config to be created with cluster ID mappings" && sleep 1; done' - - - name: create replicated mirrored filesystem on cluster 1 - run: | - PRIMARY_YAML=deploy/examples/filesystem-test-primary.yaml - cp deploy/examples/filesystem-test.yaml "$PRIMARY_YAML" - yq merge --inplace --arrays append "$PRIMARY_YAML" tests/manifests/test-fs-mirror-spec.yaml - kubectl create -f "$PRIMARY_YAML" - - - name: create replicated mirrored filesystem on cluster 2 - run: | - cd deploy/examples/ - sed -i 's/namespace: rook-ceph/namespace: rook-ceph-secondary/g' filesystem-test.yaml - yq w -i filesystem-test.yaml spec.mirroring.enabled true - kubectl create -f filesystem-test.yaml - - - name: wait for filesystem on cluster 1 - run: | - timeout 300 sh -c 'until [ "$(kubectl -n rook-ceph get cephfilesystem myfs -o jsonpath='{.status.phase}'|grep -c "Ready")" -eq 1 ]; do echo "waiting for filesystem to be created" && sleep 1; done' - - - name: wait for filesystem on cluster 2 - run: | - timeout 300 sh -c 'until [ "$(kubectl -n rook-ceph-secondary get cephfilesystem myfs -o jsonpath='{.status.phase}'|grep -c "Ready")" -eq 1 ]; do echo "waiting for filesystem to be created" && sleep 1; done' - - - name: copy filesystem mirror peer secret from the secondary cluster to the primary one - run: | - kubectl -n rook-ceph-secondary get secret fs-peer-token-myfs -o yaml |\ - sed '/ownerReferences/,+6d' |\ - sed 's/namespace: rook-ceph-secondary/namespace: rook-ceph/g; s/name: fs-peer-token-myfs/name: fs-peer-token-myfs-config/g' |\ - kubectl create --namespace=rook-ceph -f - - - - name: add filesystem mirror peer secret to the primary cluster - run: | - kubectl -n rook-ceph patch cephfilesystem myfs --type merge -p '{"spec":{"mirroring":{"peers": {"secretNames": ["fs-peer-token-myfs-config"]}}}}' - - - name: verify fs mirroring is working - run: | - timeout 45 sh -c 'until [ "$(kubectl -n rook-ceph exec -t deploy/rook-ceph-fs-mirror -- ls -1 /var/run/ceph/|grep -c asok)" -gt 3 ]; do echo "waiting for connection to peer" && sleep 1; done' - sockets=$(kubectl -n rook-ceph exec -t deploy/rook-ceph-fs-mirror -- ls -1 /var/run/ceph/) - status=$(for socket in $sockets; do minikube kubectl -- -n rook-ceph exec -t deploy/rook-ceph-fs-mirror -- ceph --admin-daemon /var/run/ceph/$socket help|awk -F ":" '/get filesystem mirror status/ {print $1}'; done) - if [ "${#status}" -lt 1 ]; then echo "peer addition failed" && exit 1; fi - - - name: display cephfilesystem and fs mirror daemon status - run: | - kubectl -n rook-ceph get cephfilesystem myfs -o yaml - # the check is not super ideal since 'mirroring_failed' is only displayed when there is a failure but not when it's working... - timeout 60 sh -c 'while [ "$(kubectl exec -n rook-ceph deploy/rook-ceph-tools -t -- ceph fs snapshot mirror daemon status|jq -r '.[0].filesystems[0]'|grep -c "mirroring_failed")" -eq 1 ]; do echo "waiting for filesystem to be mirrored" && sleep 1; done' - - - name: Create subvolume on primary cluster - run: | - kubectl exec -n rook-ceph deploy/rook-ceph-tools -t -- ceph fs subvolume create myfs testsubvolume - - - name: Create subvolume of same name on secondary cluster - run: | - kubectl exec -n rook-ceph-secondary deploy/rook-ceph-tools -t -- ceph fs subvolume create myfs testsubvolume - - - name: Deploy Direct Tools pod on primary cluster - run: | - tests/scripts/github-action-helper.sh deploy_manifest_with_local_build deploy/examples/direct-mount.yaml - - - name: Deploy Direct Tools pod on secondary cluster - run: | - sed -i "s/rook-ceph # namespace/rook-ceph-secondary # namespace/" deploy/examples/direct-mount.yaml - tests/scripts/github-action-helper.sh deploy_manifest_with_local_build deploy/examples/direct-mount.yaml - - - name: Configure a directory for snapshot mirroring on primary cluster - run: | - kubectl exec -n rook-ceph deploy/rook-ceph-tools -t -- ceph fs snapshot mirror enable myfs - kubectl exec -n rook-ceph deploy/rook-ceph-tools -t -- ceph fs snapshot mirror add myfs /volumes/_nogroup/testsubvolume/ - - - name: make sure that snapshot mirror is enabled on the secondary cluster - run: | - kubectl exec -n rook-ceph-secondary deploy/rook-ceph-tools -t -- ceph fs snapshot mirror enable myfs - - - name: Create 3 snapshots on cluster primary cluster - run: | - kubectl exec -n rook-ceph deploy/rook-ceph-tools -t -- ceph fs subvolume snapshot create myfs testsubvolume snap1 - kubectl exec -n rook-ceph deploy/rook-ceph-tools -t -- ceph fs subvolume snapshot create myfs testsubvolume snap2 - - - name: Get the peer and verify the peer synchronization status that snaps have synced on secondary cluster + - name: validate osd vault run: | - exec_fs_mirror='kubectl -n rook-ceph exec deploy/rook-ceph-fs-mirror --' - mirror_daemon=$($exec_fs_mirror ls /var/run/ceph/ | grep "fs-mirror" | head -n 1) - # timeout 45 bash -x <"${LOG_DIR}"/ceph-report.txt NAMESPACES=("$CLUSTER_NAMESPACE") if [[ "$OPERATOR_NAMESPACE" != "$CLUSTER_NAMESPACE" ]]; then - NAMESPACES+=("$OPERATOR_NAMESPACE") + NAMESPACES+=("$OPERATOR_NAMESPACE") fi if [[ -n "${ADDITIONAL_NAMESPACE}" ]]; then - NAMESPACES+=("${ADDITIONAL_NAMESPACE}") + NAMESPACES+=("${ADDITIONAL_NAMESPACE}") fi for NAMESPACE in "${NAMESPACES[@]}"; do - # each namespace is a sub-directory for easier debugging - NS_DIR="${LOG_DIR}"/namespace-"${NAMESPACE}" - mkdir "${NS_DIR}" + # each namespace is a sub-directory for easier debugging + NS_DIR="${LOG_DIR}"/namespace-"${NAMESPACE}" + mkdir "${NS_DIR}" - # describe every one of the k8s resources in the namespace which rook commonly uses - for KIND in 'pod' 'deployment' 'job' 'daemonset' 'cm'; do - kubectl -n "$NAMESPACE" get "$KIND" -o wide >"${NS_DIR}"/"$KIND"-list.txt - for resource in $(kubectl -n "$NAMESPACE" get "$KIND" -o jsonpath='{.items[*].metadata.name}'); do - kubectl -n "$NAMESPACE" describe "$KIND" "$resource" >"${NS_DIR}"/"$KIND"-describe--"$resource".txt + # describe every one of the k8s resources in the namespace which rook commonly uses + for KIND in 'pod' 'deployment' 'job' 'daemonset' 'cm'; do + kubectl -n "$NAMESPACE" get "$KIND" -o wide >"${NS_DIR}"/"$KIND"-list.txt + for resource in $(kubectl -n "$NAMESPACE" get "$KIND" -o jsonpath='{.items[*].metadata.name}'); do + kubectl -n "$NAMESPACE" describe "$KIND" "$resource" >"${NS_DIR}"/"$KIND"-describe--"$resource".txt - # collect logs for pods along the way - if [[ "$KIND" == 'pod' ]]; then - kubectl -n "$NAMESPACE" logs --all-containers "$resource" >"${NS_DIR}"/logs--"$resource".txt - fi - done - done + # collect logs for pods along the way + if [[ "$KIND" == 'pod' ]]; then + kubectl -n "$NAMESPACE" logs --all-containers "$resource" >"${NS_DIR}"/logs--"$resource".txt + kubectl -n "$NAMESPACE" logs -p --all-containers "$resource" >"${NS_DIR}"/logs--"$resource"-p.txt + fi + done + done - # secret need `-oyaml` to read the content instead of `describe` since secrets `describe` will be encrypted. - # so keeping it in a different block. - for secret in $(kubectl -n "$NAMESPACE" get secrets -o jsonpath='{.items[*].metadata.name}'); do - kubectl -n "$NAMESPACE" get -o yaml secret "$secret" >"${NS_DIR}"/secret-describe--"$secret".txt - done + # secret need `-oyaml` to read the content instead of `describe` since secrets `describe` will be encrypted. + # so keeping it in a different block. + for secret in $(kubectl -n "$NAMESPACE" get secrets -o jsonpath='{.items[*].metadata.name}'); do + kubectl -n "$NAMESPACE" get -o yaml secret "$secret" >"${NS_DIR}"/secret-describe--"$secret".txt + done - # describe every one of the custom resources in the namespace since all should be rook-related and - # they aren't captured by 'kubectl get all' - for CRD in $(kubectl get crds -o jsonpath='{.items[*].metadata.name}'); do - for resource in $(kubectl -n "$NAMESPACE" get "$CRD" -o jsonpath='{.items[*].metadata.name}'); do - crd_main_type="${CRD%%.*}" # e.g., for cephclusters.ceph.rook.io, only use 'cephclusters' - kubectl -n "$NAMESPACE" get -o yaml "$CRD" "$resource" >"${NS_DIR}"/"$crd_main_type"-describe--"$resource".txt - done - done + # describe every one of the custom resources in the namespace since all should be rook-related and + # they aren't captured by 'kubectl get all' + for CRD in $(kubectl get crds -o jsonpath='{.items[*].metadata.name}'); do + for resource in $(kubectl -n "$NAMESPACE" get "$CRD" -o jsonpath='{.items[*].metadata.name}'); do + crd_main_type="${CRD%%.*}" # e.g., for cephclusters.ceph.rook.io, only use 'cephclusters' + kubectl -n "$NAMESPACE" get -o yaml "$CRD" "$resource" >"${NS_DIR}"/"$crd_main_type"-describe--"$resource".txt + done + done - # do simple 'get all' calls for resources we don't often want to look at - kubectl get all -n "$NAMESPACE" -o wide >"${NS_DIR}"/all-wide.txt - kubectl get all -n "$NAMESPACE" -o yaml >"${NS_DIR}"/all-yaml.txt + # do simple 'get all' calls for resources we don't often want to look at + kubectl get all -n "$NAMESPACE" -o wide >"${NS_DIR}"/all-wide.txt + kubectl get all -n "$NAMESPACE" -o yaml >"${NS_DIR}"/all-yaml.txt done sudo lsblk | sudo tee -a "${LOG_DIR}"/lsblk.txt