Fixes for 4.15+ for new ODF and CNV behaviors

Changes.md

@@ -42,3 +42,9 @@
 * Remove support for 4.10 (since it is out of support)
 * Update platform level override using new templated valuefile name feature in common
 * Skip multicloud gateway (noobaa) installation in ODF by default
+
+## Changes in main (July 25, 2024)
+
+* Introduce clean-golden-images job to imperative. This is a workaround for a bug in CNV 4.15/ODF 4.15 where if the default StorageClass is not the same as the default virtualization storage class, CNV cannot properly provision datavolumes.
+* Default storageclass for edge-gitops-vms to "ocs-storagecluster-ceph-rbd-virtualization", available since ODF 4.14.
+* Use api_version for Route queries when discovering credentials for AAP instance.

ansible/ansible_get_credentials.yml

@@ -10,6 +10,7 @@
   tasks:
     - name: Retrieve API hostname for AAP
       kubernetes.core.k8s_info:
+        api_version: route.openshift.io/v1
         kind: Route
         namespace: ansible-automation-platform
         name: controller

ansible/odf_clean_pvcs.yml

@@ -0,0 +1,91 @@
+#!/usr/bin/env ansible-playbook
+---
+- name: Determine if we have PVC clean-up to do
+  become: false
+  connection: local
+  hosts: localhost
+  gather_facts: false
+  vars:
+    kubeconfig: "{{ lookup('env', 'KUBECONFIG') }}"
+    pvc_cleanup: false
+    image_cleanup_namespace: "openshift-virtualization-os-images"
+    dv_namespace: edge-gitops-vms
+    dv_remove_timeout: 1800
+    dv_remove_status: ["Pending"]
+    ts_fmt: '%Y-%m-%dT%H:%M:%SZ'
+  tasks:
+    - name: Find default storageclass
+      ansible.builtin.shell: |
+        set -o pipefail
+        oc get storageclass -o json | jq -r '.items[] | select(.metadata.annotations."storageclass.kubernetes.io/is-default-class")'
+      register: default_sc_output
+      changed_when: false
+
+    - name: Find virtualization default storageclass
+      ansible.builtin.shell: |
+        set -o pipefail
+        oc get storageclass -o json | jq -r '.items[] | select(.metadata.annotations."storageclass.kubevirt.io/is-default-virt-class")'
+      register: default_virt_sc_output
+      changed_when: false
+
+    - name: Compare default virtualization storageclass and default storageclass to determine whether to clean PVCs
+      block:
+        - name: Parse results
+          ansible.builtin.set_fact:
+            default_sc: '{{ default_sc_output.stdout | from_json }}'
+            default_virt_sc: '{{ default_virt_sc_output.stdout | from_json }}'
+
+        - name: Commit to PVC cleanup
+          ansible.builtin.set_fact:
+            pvc_cleanup: true
+          when:
+            - default_virt_sc.metadata.name == "ocs-storagecluster-ceph-rbd-virtualization"
+            - default_sc.metadata.name != default_virt_sc.metadata.name
+      rescue:
+        - name: Note that we exited
+          ansible.builtin.debug:
+            msg: "Caught an error before we could determine to clean up PVCs, exiting"
+
+    - name: Cleanup incorrect datasourceimport images (PVCs)
+      when:
+        - pvc_cleanup
+      block:
+        - name: Find PVCs
+          kubernetes.core.k8s_info:
+            kind: pvc
+            namespace: '{{ image_cleanup_namespace }}'
+          register: pvc_cleanup_list
+
+        - name: Remove stray datasource PVCs
+          kubernetes.core.k8s:
+            kind: pvc
+            namespace: '{{ image_cleanup_namespace }}'
+            name: '{{ item.metadata.name }}'
+            state: absent
+          loop: "{{ pvc_cleanup_list.resources }}"
+          when:
+            - item.spec.storageClassName != default_virt_sc.metadata.name
+
+        - name: Check for stuck datavolumes
+          kubernetes.core.k8s_info:
+            namespace: '{{ dv_namespace }}'
+            kind: DataVolume
+            api_version: cdi.kubevirt.io/v1beta1
+          register: vm_ds
+
+        - name: Remove stuck datavolume if needed
+          kubernetes.core.k8s:
+            name: "{{ item.metadata.name }}"
+            namespace: "{{ item.metadata.namespace }}"
+            kind: "{{ item.kind }}"
+            api_version: "{{ item.apiVersion }}"
+            state: absent
+          when:
+            - item.status.phase in dv_remove_status
+            - (now(utc=true) - (item.metadata.creationTimestamp|to_datetime(ts_fmt))).seconds >= dv_remove_timeout
+          loop: '{{ vm_ds.resources }}'
+
+      rescue:
+        - name: Note that we exited
+          ansible.builtin.debug:
+            msg: "Caught an error while cleaning up PVCs, exiting"

charts/hub/edge-gitops-vms/values.yaml

@@ -11,7 +11,7 @@ cloudInitDefaultSecretName: secret/data/hub/cloud-init
 # Or "RWO" and "gp2"; other choices are possible too but
 # these are defaults for ODF which is included in the pattern
 defaultAccessMode: "ReadWriteMany"
-defaultStorageClassName: "ocs-storagecluster-ceph-rbd"
+defaultStorageClassName: "ocs-storagecluster-ceph-rbd-virtualization"
 defaultVolumeMode: "Block"
 
 vmNamespace: edge-gitops-vms

tests/hub-edge-gitops-vms-industrial-edge-factory.expected.yaml

@@ -181,7 +181,7 @@ items:
           resources:
             requests:
               storage: 30Gi
-          storageClassName: ocs-storagecluster-ceph-rbd
+          storageClassName: ocs-storagecluster-ceph-rbd-virtualization
           volumeMode: Block
     running: true
     template:
@@ -319,7 +319,7 @@ items:
           resources:
             requests:
               storage: 30Gi
-          storageClassName: ocs-storagecluster-ceph-rbd
+          storageClassName: ocs-storagecluster-ceph-rbd-virtualization
           volumeMode: Block
     running: true
     template:

tests/hub-edge-gitops-vms-industrial-edge-hub.expected.yaml

@@ -181,7 +181,7 @@ items:
           resources:
             requests:
               storage: 30Gi
-          storageClassName: ocs-storagecluster-ceph-rbd
+          storageClassName: ocs-storagecluster-ceph-rbd-virtualization
           volumeMode: Block
     running: true
     template:
@@ -319,7 +319,7 @@ items:
           resources:
             requests:
               storage: 30Gi
-          storageClassName: ocs-storagecluster-ceph-rbd
+          storageClassName: ocs-storagecluster-ceph-rbd-virtualization
           volumeMode: Block
     running: true
     template:

tests/hub-edge-gitops-vms-medical-diagnosis-hub.expected.yaml

@@ -181,7 +181,7 @@ items:
           resources:
             requests:
               storage: 30Gi
-          storageClassName: ocs-storagecluster-ceph-rbd
+          storageClassName: ocs-storagecluster-ceph-rbd-virtualization
           volumeMode: Block
     running: true
     template:
@@ -319,7 +319,7 @@ items:
           resources:
             requests:
               storage: 30Gi
-          storageClassName: ocs-storagecluster-ceph-rbd
+          storageClassName: ocs-storagecluster-ceph-rbd-virtualization
           volumeMode: Block
     running: true
     template:

tests/hub-edge-gitops-vms-naked.expected.yaml

@@ -181,7 +181,7 @@ items:
           resources:
             requests:
               storage: 30Gi
-          storageClassName: ocs-storagecluster-ceph-rbd
+          storageClassName: ocs-storagecluster-ceph-rbd-virtualization
           volumeMode: Block
     running: true
     template:
@@ -319,7 +319,7 @@ items:
           resources:
             requests:
               storage: 30Gi
-          storageClassName: ocs-storagecluster-ceph-rbd
+          storageClassName: ocs-storagecluster-ceph-rbd-virtualization
           volumeMode: Block
     running: true
     template:

tests/hub-edge-gitops-vms-normal.expected.yaml

@@ -181,7 +181,7 @@ items:
           resources:
             requests:
               storage: 30Gi
-          storageClassName: ocs-storagecluster-ceph-rbd
+          storageClassName: ocs-storagecluster-ceph-rbd-virtualization
           volumeMode: Block
     running: true
     template:
@@ -319,7 +319,7 @@ items:
           resources:
             requests:
               storage: 30Gi
-          storageClassName: ocs-storagecluster-ceph-rbd
+          storageClassName: ocs-storagecluster-ceph-rbd-virtualization
           volumeMode: Block
     running: true
     template:

values-hub.yaml

@@ -32,6 +32,10 @@ clusterGroup:
       - name: deploy-kubevirt-worker
         playbook: ansible/deploy_kubevirt_worker.yml
         verbosity: -vvv
+      - name: clean-golden-images
+        playbook: ansible/odf_clean_pvcs.yml
+        image: quay.io/hybridcloudpatterns/utility-container:latest
+        verbosity: -vvv
       - name: configure-aap-controller
         playbook: ansible/imperative_configure_controller.yml
         image: quay.io/hybridcloudpatterns/ansible-edge-gitops-ee:latest
@@ -42,6 +46,8 @@ clusterGroup:
           - "*"
         resources:
           - machinesets
+          - persistentvolumeclaims
+          - datavolumes
         verbs:
           - "*"
       - apiGroups:
@@ -95,6 +101,8 @@ clusterGroup:
       namespace: edge-gitops-vms
       project: hub
       path: charts/hub/edge-gitops-vms
+      extraValueFiles:
+        - '/overrides/values-egv-{{ $.Values.global.clusterVersion }}.yaml'
 
   # Only the hub cluster here - managed entities are edge nodes
   managedClusterGroups: []