From eef2c2d9c7e3573cc89785f088a4a962371021d3 Mon Sep 17 00:00:00 2001 From: Martin Jackson Date: Mon, 22 Jul 2024 16:07:54 -0500 Subject: [PATCH 01/23] Update storageclass for 4.15 --- overrides/values-egv-4.15.yaml | 2 ++ values-hub.yaml | 2 ++ 2 files changed, 4 insertions(+) create mode 100644 overrides/values-egv-4.15.yaml diff --git a/overrides/values-egv-4.15.yaml b/overrides/values-egv-4.15.yaml new file mode 100644 index 00000000..2b9af3a9 --- /dev/null +++ b/overrides/values-egv-4.15.yaml @@ -0,0 +1,2 @@ +--- +defaultStorageClass: ocs-storagecluster-ceph-rbd-virtualization diff --git a/values-hub.yaml b/values-hub.yaml index f8a6d16f..bc00375d 100644 --- a/values-hub.yaml +++ b/values-hub.yaml @@ -95,6 +95,8 @@ clusterGroup: namespace: edge-gitops-vms project: hub path: charts/hub/edge-gitops-vms + extraValueFiles: + - '/overrides/values-egv-{{ $.Values.global.clusterVersion }}.yaml' # Only the hub cluster here - managed entities are edge nodes managedClusterGroups: [] From dfd76e1dc0f16c5c019a2fc2924a1b7e506ae4c8 Mon Sep 17 00:00:00 2001 From: Martin Jackson Date: Wed, 24 Jul 2024 13:52:26 -0500 Subject: [PATCH 02/23] Default storageclass to use the new hotness --- ansible/odf_clean_pvcs.yml | 65 ++++++++++++++++++++++++++ charts/hub/edge-gitops-vms/values.yaml | 2 +- 2 files changed, 66 insertions(+), 1 deletion(-) create mode 100644 ansible/odf_clean_pvcs.yml diff --git a/ansible/odf_clean_pvcs.yml b/ansible/odf_clean_pvcs.yml new file mode 100644 index 00000000..615b10c7 --- /dev/null +++ b/ansible/odf_clean_pvcs.yml @@ -0,0 +1,65 @@ +#!/usr/bin/env ansible-playbook +--- +- name: Determine if we have PVC clean-up to do + become: false + connection: local + hosts: localhost + gather_facts: false + vars: + kubeconfig: "{{ lookup('env', 'KUBECONFIG') }}" + pvc_cleanup: false + image_cleanup_namespace: "openshift-virtualization-os-images" + tasks: + - name: Find default storageclass + ansible.builtin.shell: | + oc get storageclass -o json | jq -r '.items[] | select(.metadata.annotations."storageclass.kubernetes.io/is-default-class")' + register: default_sc_output + changed_when: false + + - name: Find virtualization default storageclass + ansible.builtin.shell: | + oc get storageclass -o json | jq -r '.items[] | select(.metadata.annotations."storageclass.kubevirt.io/is-default-virt-class")' + register: default_virt_sc_output + changed_when: false + + - name: Trap errors + block: + + - name: Parse results + ansible.builtin.set_fact: + default_sc: '{{ default_sc_output.stdout | from_json }}' + default_virt_sc: '{{ default_virt_sc_output.stdout | from_json }}' + + - name: Commit to PVC cleanup + ansible.builtin.set_fact: + pvc_cleanup: true + when: default_sc.metadata.name != default_virt_sc.metadata.name + + - name: Find PVCs + kubernetes.core.k8s_info: + kind: pvc + namespace: '{{ image_cleanup_namespace }}' + register: pvc_cleanup_list + when: pvc_cleanup + + - name: Remove stray PVCs + kubernetes.core.k8s: + kind: pvc + namespace: '{{ image_cleanup_namespace }}' + name: '{{ item.metadata.name }}' + state: absent + loop: "{{ pvc_cleanup_list.resources }}" + when: pvc_cleanup + rescue: + - name: Note that we exited + ansible.builtin.debug: + msg: "Caught an error, exiting" + + # - name: Perform the PVC cleanup + # when: pvc_cleanup + # block: + # - name: Remove PVCs + # ansible.builtin.k8s: + # kind: PVC + # namespace: openshift-virtualization-os-images + # state: absent diff --git a/charts/hub/edge-gitops-vms/values.yaml b/charts/hub/edge-gitops-vms/values.yaml index d1f43b7a..3b6f9777 100644 --- a/charts/hub/edge-gitops-vms/values.yaml +++ b/charts/hub/edge-gitops-vms/values.yaml @@ -11,7 +11,7 @@ cloudInitDefaultSecretName: secret/data/hub/cloud-init # Or "RWO" and "gp2"; other choices are possible too but # these are defaults for ODF which is included in the pattern defaultAccessMode: "ReadWriteMany" -defaultStorageClassName: "ocs-storagecluster-ceph-rbd" +defaultStorageClassName: "ocs-storagecluster-ceph-rbd-virtualization" defaultVolumeMode: "Block" vmNamespace: edge-gitops-vms From 64eeaf955607476c32b7761403cbb333b602a868 Mon Sep 17 00:00:00 2001 From: Martin Jackson Date: Wed, 24 Jul 2024 13:58:45 -0500 Subject: [PATCH 03/23] Add job to imperative --- values-hub.yaml | 10 ++++++++++ 1 file changed, 10 insertions(+) diff --git a/values-hub.yaml b/values-hub.yaml index bc00375d..70740cc4 100644 --- a/values-hub.yaml +++ b/values-hub.yaml @@ -32,6 +32,10 @@ clusterGroup: - name: deploy-kubevirt-worker playbook: ansible/deploy_kubevirt_worker.yml verbosity: -vvv + - name: clean-golden-images + playbook: ansible/odf_clean_pvcs.yml + image: quay.io/hybridcloudpatterns/utility-container:latest + verbosity: -vvv - name: configure-aap-controller playbook: ansible/imperative_configure_controller.yml image: quay.io/hybridcloudpatterns/ansible-edge-gitops-ee:latest @@ -44,6 +48,12 @@ clusterGroup: - machinesets verbs: - "*" + - apiGroups: + - "*" + resources: + - pvcs + verbs: + - "*" - apiGroups: - "*" resources: From c85e0dad390dcf6bdc03fdda0eefbbe429e29585 Mon Sep 17 00:00:00 2001 From: Martin Jackson Date: Wed, 24 Jul 2024 14:36:27 -0500 Subject: [PATCH 04/23] Also make sure the virt default class is -virtualization --- ansible/odf_clean_pvcs.yml | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/ansible/odf_clean_pvcs.yml b/ansible/odf_clean_pvcs.yml index 615b10c7..426b74ef 100644 --- a/ansible/odf_clean_pvcs.yml +++ b/ansible/odf_clean_pvcs.yml @@ -33,7 +33,9 @@ - name: Commit to PVC cleanup ansible.builtin.set_fact: pvc_cleanup: true - when: default_sc.metadata.name != default_virt_sc.metadata.name + when: + - default_virt_sc.metadata.name == "ocs-storagecluster-ceph-rbd-virtualization" + - default_sc.metadata.name != default_virt_sc.metadata.name - name: Find PVCs kubernetes.core.k8s_info: From b3e589e400a6966fb75d991def27812f7171d3b3 Mon Sep 17 00:00:00 2001 From: Martin Jackson Date: Wed, 24 Jul 2024 14:36:56 -0500 Subject: [PATCH 05/23] Update tests --- .../hub-edge-gitops-vms-industrial-edge-factory.expected.yaml | 4 ++-- tests/hub-edge-gitops-vms-industrial-edge-hub.expected.yaml | 4 ++-- tests/hub-edge-gitops-vms-medical-diagnosis-hub.expected.yaml | 4 ++-- tests/hub-edge-gitops-vms-naked.expected.yaml | 4 ++-- tests/hub-edge-gitops-vms-normal.expected.yaml | 4 ++-- 5 files changed, 10 insertions(+), 10 deletions(-) diff --git a/tests/hub-edge-gitops-vms-industrial-edge-factory.expected.yaml b/tests/hub-edge-gitops-vms-industrial-edge-factory.expected.yaml index 8bbe2f1a..a9761701 100644 --- a/tests/hub-edge-gitops-vms-industrial-edge-factory.expected.yaml +++ b/tests/hub-edge-gitops-vms-industrial-edge-factory.expected.yaml @@ -181,7 +181,7 @@ items: resources: requests: storage: 30Gi - storageClassName: ocs-storagecluster-ceph-rbd + storageClassName: ocs-storagecluster-ceph-rbd-virtualization volumeMode: Block running: true template: @@ -319,7 +319,7 @@ items: resources: requests: storage: 30Gi - storageClassName: ocs-storagecluster-ceph-rbd + storageClassName: ocs-storagecluster-ceph-rbd-virtualization volumeMode: Block running: true template: diff --git a/tests/hub-edge-gitops-vms-industrial-edge-hub.expected.yaml b/tests/hub-edge-gitops-vms-industrial-edge-hub.expected.yaml index 8bbe2f1a..a9761701 100644 --- a/tests/hub-edge-gitops-vms-industrial-edge-hub.expected.yaml +++ b/tests/hub-edge-gitops-vms-industrial-edge-hub.expected.yaml @@ -181,7 +181,7 @@ items: resources: requests: storage: 30Gi - storageClassName: ocs-storagecluster-ceph-rbd + storageClassName: ocs-storagecluster-ceph-rbd-virtualization volumeMode: Block running: true template: @@ -319,7 +319,7 @@ items: resources: requests: storage: 30Gi - storageClassName: ocs-storagecluster-ceph-rbd + storageClassName: ocs-storagecluster-ceph-rbd-virtualization volumeMode: Block running: true template: diff --git a/tests/hub-edge-gitops-vms-medical-diagnosis-hub.expected.yaml b/tests/hub-edge-gitops-vms-medical-diagnosis-hub.expected.yaml index 8bbe2f1a..a9761701 100644 --- a/tests/hub-edge-gitops-vms-medical-diagnosis-hub.expected.yaml +++ b/tests/hub-edge-gitops-vms-medical-diagnosis-hub.expected.yaml @@ -181,7 +181,7 @@ items: resources: requests: storage: 30Gi - storageClassName: ocs-storagecluster-ceph-rbd + storageClassName: ocs-storagecluster-ceph-rbd-virtualization volumeMode: Block running: true template: @@ -319,7 +319,7 @@ items: resources: requests: storage: 30Gi - storageClassName: ocs-storagecluster-ceph-rbd + storageClassName: ocs-storagecluster-ceph-rbd-virtualization volumeMode: Block running: true template: diff --git a/tests/hub-edge-gitops-vms-naked.expected.yaml b/tests/hub-edge-gitops-vms-naked.expected.yaml index 8bbe2f1a..a9761701 100644 --- a/tests/hub-edge-gitops-vms-naked.expected.yaml +++ b/tests/hub-edge-gitops-vms-naked.expected.yaml @@ -181,7 +181,7 @@ items: resources: requests: storage: 30Gi - storageClassName: ocs-storagecluster-ceph-rbd + storageClassName: ocs-storagecluster-ceph-rbd-virtualization volumeMode: Block running: true template: @@ -319,7 +319,7 @@ items: resources: requests: storage: 30Gi - storageClassName: ocs-storagecluster-ceph-rbd + storageClassName: ocs-storagecluster-ceph-rbd-virtualization volumeMode: Block running: true template: diff --git a/tests/hub-edge-gitops-vms-normal.expected.yaml b/tests/hub-edge-gitops-vms-normal.expected.yaml index 8bbe2f1a..a9761701 100644 --- a/tests/hub-edge-gitops-vms-normal.expected.yaml +++ b/tests/hub-edge-gitops-vms-normal.expected.yaml @@ -181,7 +181,7 @@ items: resources: requests: storage: 30Gi - storageClassName: ocs-storagecluster-ceph-rbd + storageClassName: ocs-storagecluster-ceph-rbd-virtualization volumeMode: Block running: true template: @@ -319,7 +319,7 @@ items: resources: requests: storage: 30Gi - storageClassName: ocs-storagecluster-ceph-rbd + storageClassName: ocs-storagecluster-ceph-rbd-virtualization volumeMode: Block running: true template: From 50f48c9b4c8a53a40dc9420905d5e394de8c847b Mon Sep 17 00:00:00 2001 From: Martin Jackson Date: Wed, 24 Jul 2024 17:56:01 -0500 Subject: [PATCH 06/23] Remove extra blank line --- ansible/odf_clean_pvcs.yml | 1 - 1 file changed, 1 deletion(-) diff --git a/ansible/odf_clean_pvcs.yml b/ansible/odf_clean_pvcs.yml index 426b74ef..7ccc6c12 100644 --- a/ansible/odf_clean_pvcs.yml +++ b/ansible/odf_clean_pvcs.yml @@ -24,7 +24,6 @@ - name: Trap errors block: - - name: Parse results ansible.builtin.set_fact: default_sc: '{{ default_sc_output.stdout | from_json }}' From fc685431df189952892f3b202e1ccb2d0da4e37d Mon Sep 17 00:00:00 2001 From: Martin Jackson Date: Wed, 24 Jul 2024 17:57:03 -0500 Subject: [PATCH 07/23] Update RBAC --- values-hub.yaml | 7 +++++-- 1 file changed, 5 insertions(+), 2 deletions(-) diff --git a/values-hub.yaml b/values-hub.yaml index 70740cc4..ce237172 100644 --- a/values-hub.yaml +++ b/values-hub.yaml @@ -51,9 +51,12 @@ clusterGroup: - apiGroups: - "*" resources: - - pvcs + - pvc verbs: - - "*" + - get + - list + - watch + - delete - apiGroups: - "*" resources: From 808fdd12c93e3a783198c95f9bed76ae7e99b845 Mon Sep 17 00:00:00 2001 From: Martin Jackson Date: Wed, 24 Jul 2024 18:04:56 -0500 Subject: [PATCH 08/23] Tweak RBAC --- values-hub.yaml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/values-hub.yaml b/values-hub.yaml index ce237172..f5688a99 100644 --- a/values-hub.yaml +++ b/values-hub.yaml @@ -51,7 +51,7 @@ clusterGroup: - apiGroups: - "*" resources: - - pvc + - persistentvolumeclaims verbs: - get - list From 41623a8ce72ef61e0be5e9f4c5c0bfca0cdadf91 Mon Sep 17 00:00:00 2001 From: Martin Jackson Date: Wed, 24 Jul 2024 18:17:09 -0500 Subject: [PATCH 09/23] Also make sure PVC does not match default virt class --- ansible/odf_clean_pvcs.yml | 16 +++++----------- 1 file changed, 5 insertions(+), 11 deletions(-) diff --git a/ansible/odf_clean_pvcs.yml b/ansible/odf_clean_pvcs.yml index 7ccc6c12..6f272f36 100644 --- a/ansible/odf_clean_pvcs.yml +++ b/ansible/odf_clean_pvcs.yml @@ -41,7 +41,8 @@ kind: pvc namespace: '{{ image_cleanup_namespace }}' register: pvc_cleanup_list - when: pvc_cleanup + when: + - pvc_cleanup - name: Remove stray PVCs kubernetes.core.k8s: @@ -50,17 +51,10 @@ name: '{{ item.metadata.name }}' state: absent loop: "{{ pvc_cleanup_list.resources }}" - when: pvc_cleanup + when: + - pvc_cleanup + - item.spec.storageClassName != default_virt_sc.metadata.name rescue: - name: Note that we exited ansible.builtin.debug: msg: "Caught an error, exiting" - - # - name: Perform the PVC cleanup - # when: pvc_cleanup - # block: - # - name: Remove PVCs - # ansible.builtin.k8s: - # kind: PVC - # namespace: openshift-virtualization-os-images - # state: absent From b2fb4b9c66602d4adfe18e96e7874b9db13023ae Mon Sep 17 00:00:00 2001 From: Martin Jackson Date: Wed, 24 Jul 2024 18:18:52 -0500 Subject: [PATCH 10/23] Increase verbosity and use utility container --- values-hub.yaml | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/values-hub.yaml b/values-hub.yaml index f5688a99..27175af7 100644 --- a/values-hub.yaml +++ b/values-hub.yaml @@ -38,8 +38,8 @@ clusterGroup: verbosity: -vvv - name: configure-aap-controller playbook: ansible/imperative_configure_controller.yml - image: quay.io/hybridcloudpatterns/ansible-edge-gitops-ee:latest - verbosity: -vvv + image: quay.io/hybridcloudpatterns/utility-container:latest + verbosity: -vvvvv timeout: "900" clusterRoleYaml: - apiGroups: From 8573e8ac3a88c3c882b86dcb76d34cd1846f9bd4 Mon Sep 17 00:00:00 2001 From: Martin Jackson Date: Wed, 24 Jul 2024 18:26:15 -0500 Subject: [PATCH 11/23] Go back to previous image --- values-hub.yaml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/values-hub.yaml b/values-hub.yaml index 27175af7..0d4ea194 100644 --- a/values-hub.yaml +++ b/values-hub.yaml @@ -38,7 +38,7 @@ clusterGroup: verbosity: -vvv - name: configure-aap-controller playbook: ansible/imperative_configure_controller.yml - image: quay.io/hybridcloudpatterns/utility-container:latest + image: quay.io/hybridcloudpatterns/ansible-edge-gitops-ee:latest verbosity: -vvvvv timeout: "900" clusterRoleYaml: From dceda6fde2bd00ea53590a551a8ce8ea9d1b42d2 Mon Sep 17 00:00:00 2001 From: Martin Jackson Date: Mon, 22 Jul 2024 16:07:54 -0500 Subject: [PATCH 12/23] Update storageclass for 4.15 --- overrides/values-egv-4.15.yaml | 2 ++ values-hub.yaml | 2 ++ 2 files changed, 4 insertions(+) create mode 100644 overrides/values-egv-4.15.yaml diff --git a/overrides/values-egv-4.15.yaml b/overrides/values-egv-4.15.yaml new file mode 100644 index 00000000..2b9af3a9 --- /dev/null +++ b/overrides/values-egv-4.15.yaml @@ -0,0 +1,2 @@ +--- +defaultStorageClass: ocs-storagecluster-ceph-rbd-virtualization diff --git a/values-hub.yaml b/values-hub.yaml index f8a6d16f..bc00375d 100644 --- a/values-hub.yaml +++ b/values-hub.yaml @@ -95,6 +95,8 @@ clusterGroup: namespace: edge-gitops-vms project: hub path: charts/hub/edge-gitops-vms + extraValueFiles: + - '/overrides/values-egv-{{ $.Values.global.clusterVersion }}.yaml' # Only the hub cluster here - managed entities are edge nodes managedClusterGroups: [] From 44fa67e79e2537099b51e09abbb3ce82f4b25258 Mon Sep 17 00:00:00 2001 From: Martin Jackson Date: Thu, 25 Jul 2024 09:17:25 -0500 Subject: [PATCH 13/23] Consolidate RBACs --- values-hub.yaml | 11 ++--------- 1 file changed, 2 insertions(+), 9 deletions(-) diff --git a/values-hub.yaml b/values-hub.yaml index 0d4ea194..c5342552 100644 --- a/values-hub.yaml +++ b/values-hub.yaml @@ -46,17 +46,10 @@ clusterGroup: - "*" resources: - machinesets - verbs: - - "*" - - apiGroups: - - "*" - resources: - persistentvolumeclaims + - routes verbs: - - get - - list - - watch - - delete + - "*" - apiGroups: - "*" resources: From fcaddf15bb5c3fd2e82c44037eb8a680e0b4371c Mon Sep 17 00:00:00 2001 From: Martin Jackson Date: Thu, 25 Jul 2024 10:31:58 -0500 Subject: [PATCH 14/23] Add api_version to route query --- ansible/ansible_get_credentials.yml | 1 + 1 file changed, 1 insertion(+) diff --git a/ansible/ansible_get_credentials.yml b/ansible/ansible_get_credentials.yml index fcdf523b..b928bc6c 100644 --- a/ansible/ansible_get_credentials.yml +++ b/ansible/ansible_get_credentials.yml @@ -10,6 +10,7 @@ tasks: - name: Retrieve API hostname for AAP kubernetes.core.k8s_info: + api_version: route.openshift.io/v1 kind: Route namespace: ansible-automation-platform name: controller From f250c3fc58a4e7ae99c99f8f6e86da037233aa86 Mon Sep 17 00:00:00 2001 From: Martin Jackson Date: Thu, 25 Jul 2024 12:34:35 -0500 Subject: [PATCH 15/23] Add Changes notes and remove unneeded override --- Changes.md | 6 ++++++ overrides/values-egv-4.15.yaml | 2 -- 2 files changed, 6 insertions(+), 2 deletions(-) delete mode 100644 overrides/values-egv-4.15.yaml diff --git a/Changes.md b/Changes.md index ff095058..ce1d1036 100644 --- a/Changes.md +++ b/Changes.md @@ -42,3 +42,9 @@ * Remove support for 4.10 (since it is out of support) * Update platform level override using new templated valuefile name feature in common * Skip multicloud gateway (noobaa) installation in ODF by default + +## Changes in main (July 25, 2024) + +* Introduce clean-golden-images job to imperative. This is a workaround for a bug in CNV 4.15/ODF 4.15 where if the default StorageClass is not the same as the default virtualization storage class, CNV cannot properly provision datavolumes. +* Default storageclass for edge-gitops-vms to "ocs-storagecluster-ceph-rbd-virtualization", available since ODF 4.14. +* Use api_version for Route queries when discovering credentials for AAP instance. diff --git a/overrides/values-egv-4.15.yaml b/overrides/values-egv-4.15.yaml deleted file mode 100644 index 2b9af3a9..00000000 --- a/overrides/values-egv-4.15.yaml +++ /dev/null @@ -1,2 +0,0 @@ ---- -defaultStorageClass: ocs-storagecluster-ceph-rbd-virtualization From 46b23a5c979392427729a8f2736d5e5eebd9a1c9 Mon Sep 17 00:00:00 2001 From: Martin Jackson Date: Thu, 25 Jul 2024 14:27:42 -0500 Subject: [PATCH 16/23] Add datavolumes for workflow to remove stuck datavolumes --- values-hub.yaml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/values-hub.yaml b/values-hub.yaml index c5342552..b2c2a43b 100644 --- a/values-hub.yaml +++ b/values-hub.yaml @@ -47,7 +47,7 @@ clusterGroup: resources: - machinesets - persistentvolumeclaims - - routes + - datavolumes verbs: - "*" - apiGroups: From 5bbc082c4f6f3b05f0a703a6d08bd70d6af56fb3 Mon Sep 17 00:00:00 2001 From: Martin Jackson Date: Thu, 25 Jul 2024 16:07:57 -0500 Subject: [PATCH 17/23] Delete datavolumes if stray --- ansible/odf_clean_pvcs.yml | 41 ++++++++++++++++++++++++++++++++------ 1 file changed, 35 insertions(+), 6 deletions(-) diff --git a/ansible/odf_clean_pvcs.yml b/ansible/odf_clean_pvcs.yml index 6f272f36..200ae118 100644 --- a/ansible/odf_clean_pvcs.yml +++ b/ansible/odf_clean_pvcs.yml @@ -9,6 +9,10 @@ kubeconfig: "{{ lookup('env', 'KUBECONFIG') }}" pvc_cleanup: false image_cleanup_namespace: "openshift-virtualization-os-images" + dv_namespace: edge-gitops-vms + dv_remove_timeout: 1800 + dv_remove_states: [ "Pending" ] + ts_fmt: '%Y-%m-%dT%H:%M:%S.%fZ' tasks: - name: Find default storageclass ansible.builtin.shell: | @@ -22,7 +26,7 @@ register: default_virt_sc_output changed_when: false - - name: Trap errors + - name: Compare default virtualization storageclass and default storageclass to determine whether to clean PVCs block: - name: Parse results ansible.builtin.set_fact: @@ -35,16 +39,22 @@ when: - default_virt_sc.metadata.name == "ocs-storagecluster-ceph-rbd-virtualization" - default_sc.metadata.name != default_virt_sc.metadata.name + rescue: + - name: Note that we exited + ansible.builtin.debug: + msg: "Caught an error before we could determine to clean up PVCs, exiting" + - name: Cleanup incorrect datasourceimport images (PVCs) + when: + - pvc_cleanup + block: - name: Find PVCs kubernetes.core.k8s_info: kind: pvc namespace: '{{ image_cleanup_namespace }}' register: pvc_cleanup_list - when: - - pvc_cleanup - - name: Remove stray PVCs + - name: Remove stray datasource PVCs kubernetes.core.k8s: kind: pvc namespace: '{{ image_cleanup_namespace }}' @@ -52,9 +62,28 @@ state: absent loop: "{{ pvc_cleanup_list.resources }}" when: - - pvc_cleanup - item.spec.storageClassName != default_virt_sc.metadata.name + + - name: Check for stuck datavolumes + kubernetes.core.k8s_info: + namespace: '{{ dv_namespace }}' + kind: DataVolume + api_version: cdi.kubevirt.io/v1beta1 + register: vm_ds + + - name: Remove stuck datavolume if needed + kubernetes.core.k8s: + name: "{{ item.metadata.name }}" + namespace: "{{ item.metadata.namespace }}" + kind: "{{ item.kind }}" + api_version: "{{ item.apiVersion }}" + state: absent + when: + - item.status.phase in dv_remove_stats + - (now(utc=true) - (item.metadata.creationTimestamp|to_datetime(ts_fmt))).seconds >= dv_remote_timeout + loop: '{{ vm_ds.resources }}' + rescue: - name: Note that we exited ansible.builtin.debug: - msg: "Caught an error, exiting" + msg: "Caught an error while cleaning up PVCs, exiting" From 9416ed4eff13312bd2187f05ba30367d1ab3bc76 Mon Sep 17 00:00:00 2001 From: Martin Jackson Date: Thu, 25 Jul 2024 16:22:49 -0500 Subject: [PATCH 18/23] Correct misspelling in status --- ansible/odf_clean_pvcs.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/ansible/odf_clean_pvcs.yml b/ansible/odf_clean_pvcs.yml index 200ae118..a7705e13 100644 --- a/ansible/odf_clean_pvcs.yml +++ b/ansible/odf_clean_pvcs.yml @@ -79,7 +79,7 @@ api_version: "{{ item.apiVersion }}" state: absent when: - - item.status.phase in dv_remove_stats + - item.status.phase in dv_remove_status - (now(utc=true) - (item.metadata.creationTimestamp|to_datetime(ts_fmt))).seconds >= dv_remote_timeout loop: '{{ vm_ds.resources }}' From 9ae6989ac088bfc2f4fc98da566ad3cb1ca4ce6e Mon Sep 17 00:00:00 2001 From: Martin Jackson Date: Thu, 25 Jul 2024 16:32:11 -0500 Subject: [PATCH 19/23] Correct var spelling --- ansible/odf_clean_pvcs.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/ansible/odf_clean_pvcs.yml b/ansible/odf_clean_pvcs.yml index a7705e13..0f103990 100644 --- a/ansible/odf_clean_pvcs.yml +++ b/ansible/odf_clean_pvcs.yml @@ -11,7 +11,7 @@ image_cleanup_namespace: "openshift-virtualization-os-images" dv_namespace: edge-gitops-vms dv_remove_timeout: 1800 - dv_remove_states: [ "Pending" ] + dv_remove_status: ["Pending"] ts_fmt: '%Y-%m-%dT%H:%M:%S.%fZ' tasks: - name: Find default storageclass From 2863b19b4c4ecdb3f3cd63dc40b80cac5015a4af Mon Sep 17 00:00:00 2001 From: Martin Jackson Date: Thu, 25 Jul 2024 16:39:19 -0500 Subject: [PATCH 20/23] Set pipefail --- ansible/odf_clean_pvcs.yml | 2 ++ 1 file changed, 2 insertions(+) diff --git a/ansible/odf_clean_pvcs.yml b/ansible/odf_clean_pvcs.yml index 0f103990..008f85f2 100644 --- a/ansible/odf_clean_pvcs.yml +++ b/ansible/odf_clean_pvcs.yml @@ -16,12 +16,14 @@ tasks: - name: Find default storageclass ansible.builtin.shell: | + set -o pipefail oc get storageclass -o json | jq -r '.items[] | select(.metadata.annotations."storageclass.kubernetes.io/is-default-class")' register: default_sc_output changed_when: false - name: Find virtualization default storageclass ansible.builtin.shell: | + set -o pipefail oc get storageclass -o json | jq -r '.items[] | select(.metadata.annotations."storageclass.kubevirt.io/is-default-virt-class")' register: default_virt_sc_output changed_when: false From d44669723175daf387e7a715def73b61e7056f0d Mon Sep 17 00:00:00 2001 From: Martin Jackson Date: Thu, 25 Jul 2024 16:41:18 -0500 Subject: [PATCH 21/23] Remove fractional seconds --- ansible/odf_clean_pvcs.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/ansible/odf_clean_pvcs.yml b/ansible/odf_clean_pvcs.yml index 008f85f2..8d37160e 100644 --- a/ansible/odf_clean_pvcs.yml +++ b/ansible/odf_clean_pvcs.yml @@ -12,7 +12,7 @@ dv_namespace: edge-gitops-vms dv_remove_timeout: 1800 dv_remove_status: ["Pending"] - ts_fmt: '%Y-%m-%dT%H:%M:%S.%fZ' + ts_fmt: '%Y-%m-%dT%H:%M:%SZ' tasks: - name: Find default storageclass ansible.builtin.shell: | From e2d05cedfe64e1bcd0eb7f40673e717bb7f3c3ae Mon Sep 17 00:00:00 2001 From: Martin Jackson Date: Thu, 25 Jul 2024 16:51:41 -0500 Subject: [PATCH 22/23] Correct misspelling --- ansible/odf_clean_pvcs.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/ansible/odf_clean_pvcs.yml b/ansible/odf_clean_pvcs.yml index 8d37160e..2380fcb7 100644 --- a/ansible/odf_clean_pvcs.yml +++ b/ansible/odf_clean_pvcs.yml @@ -82,7 +82,7 @@ state: absent when: - item.status.phase in dv_remove_status - - (now(utc=true) - (item.metadata.creationTimestamp|to_datetime(ts_fmt))).seconds >= dv_remote_timeout + - (now(utc=true) - (item.metadata.creationTimestamp|to_datetime(ts_fmt))).seconds >= dv_remove_timeout loop: '{{ vm_ds.resources }}' rescue: From 54f18dc15328dd3ff90715738bcf0a3555fe9a04 Mon Sep 17 00:00:00 2001 From: Martin Jackson Date: Thu, 25 Jul 2024 17:03:38 -0500 Subject: [PATCH 23/23] Turn down verbosity a bit --- values-hub.yaml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/values-hub.yaml b/values-hub.yaml index b2c2a43b..50b710b0 100644 --- a/values-hub.yaml +++ b/values-hub.yaml @@ -39,7 +39,7 @@ clusterGroup: - name: configure-aap-controller playbook: ansible/imperative_configure_controller.yml image: quay.io/hybridcloudpatterns/ansible-edge-gitops-ee:latest - verbosity: -vvvvv + verbosity: -vvv timeout: "900" clusterRoleYaml: - apiGroups: