[WIP] ci(cilium): migrate Cilium installation from OLM manifests to Cilium CLI

ci-operator/config/openshift/hypershift/openshift-hypershift-release-4.22__periodics.yaml

@@ -176,23 +176,6 @@ tests:
   minimum_interval: 168h
   steps:
     cluster_profile: hypershift-aws
-    env:
-      TEST_SKIPS: Netpol NetworkPolicy between server and client should allow egress
-        access to server in CIDR block \[Feature:NetworkPolicy\] \[Suite:openshift/conformance/parallel\]
-        \[Suite:k8s\]\| Netpol NetworkPolicy between server and client should ensure
-        an IP overlapping both IPBlock.CIDR and IPBlock.Except is allowed \[Feature:NetworkPolicy\]
-        \[Suite:openshift/conformance/parallel\] \[Suite:k8s\]\| Services should serve
-        endpoints on same port and different protocols \[Conformance\] \[Suite:openshift/conformance/parallel/minimal\]
-        \[Suite:k8s\]\| Netpol NetworkPolicy between server and client should enforce
-        except clause while egress access to server in CIDR block \[Feature:NetworkPolicy\]
-        \[Suite:openshift/conformance/parallel\] \[Suite:k8s\]\| Unidling \[apigroup:apps.openshift.io\]\[apigroup:route.openshift.io\]
-        should work with UDP \[Suite:openshift/conformance/parallel\]\| Unidling with
-        Deployments \[apigroup:route.openshift.io\] should work with TCP (when fully
-        idled) \[Suite:openshift/conformance/parallel\]\| Unidling \[apigroup:apps.openshift.io\]\[apigroup:route.openshift.io\]
-        should work with TCP (when fully idled) \[Suite:openshift/conformance/parallel\]\|
-        Unidling with Deployments \[apigroup:route.openshift.io\] should work with
-        UDP \[Suite:openshift/conformance/parallel\]\| DNS should answer queries using
-        the local DNS endpoint \[Suite:openshift/conformance/parallel\]
     workflow: hypershift-aws-conformance-cilium
 - as: e2e-aws-external-oidc
   minimum_interval: 12h

ci-operator/step-registry/cilium/conf/cilium-conf-commands.sh

@@ -5,8 +5,8 @@ set -o errexit
 set -o pipefail
 set -x
 
-cilium_olm_rev="main"
-cv="$CILIUM_VERSION"
+CILIUM_VERSION=${CILIUM_VERSION:-"1.19.1"}
+CILIUM_CLI_VERSION=${CILIUM_CLI_VERSION:-"0.19.2"}
 
 if [[ -f "${SHARED_DIR}/install-config.yaml" ]]; then
   sed -i "s/networkType: .*/networkType: Cilium/" "${SHARED_DIR}/install-config.yaml"
@@ -26,53 +26,41 @@ spec:
   - 172.30.0.0/16
 EOF
 
-# OLD -- Include all Cilium OLM manifest from https://github.com/cilium/cilium-olm/tree/${cilium_olm_rev}/manifests/cilium.v${cv}
-# New -- Migrating to new OLM ( https://github.com/isovalent/olm-for-cilium )
+export KUBECONFIG="${SHARED_DIR}/kubeconfig"
+if [[ -f "${SHARED_DIR}/nested_kubeconfig" ]]; then
+  export KUBECONFIG="${SHARED_DIR}/nested_kubeconfig"
+fi
 
-OLM_URL="https://github.com/isovalent/olm-for-cilium"
+mkdir -p /tmp/bin
+export PATH=/tmp/bin:$PATH
+curl --fail --retry 3 -sS -L "https://github.com/cilium/cilium-cli/releases/download/v${CILIUM_CLI_VERSION}/cilium-linux-amd64.tar.gz" | tar -xzC /tmp/bin/
+chmod +x /tmp/bin/cilium
 
-curl --silent --location --fail --show-error "${OLM_URL}/archive/${cilium_olm_rev}.tar.gz" --output /tmp/cilium-olm.tgz
-tar -C /tmp -xf /tmp/cilium-olm.tgz
+oc get ns cilium || oc create ns cilium
+oc adm policy add-scc-to-user privileged -z cilium -n cilium
+oc adm policy add-scc-to-user privileged -z cilium-operator -n cilium
+oc adm policy add-scc-to-user privileged -z cilium-envoy -n cilium
 
-cd "/tmp/olm-for-cilium-${cilium_olm_rev}/manifests/cilium.v${cv}"
-# Overwrite the CiliumConfig
-cat > cluster-network-07-cilium-ciliumconfig.yaml << EOF
-apiVersion: cilium.io/v1alpha1
-kind: CiliumConfig
-metadata:
-  name: cilium
-  namespace: cilium
-spec:
-  cni:
-    binPath: /var/lib/cni/bin
-    confPath: /var/run/multus/cni/net.d
-  endpointRoutes:
-    enabled: ${ENDPOINT_ROUTES}
-  hubble:
-    enabled: ${HUBBLE}
-  ipam:
-    mode: cluster-pool
-    operator:
-      clusterPoolIPv4MaskSize: "23"
-      clusterPoolIPv4PodCIDRList:
-      - 10.128.0.0/14
-  kubeProxyReplacement: disabled
-  nativeRoutingCIDR: 10.128.0.0/14
-  operator:
-    prometheus:
-      enabled: true
-      serviceMonitor:
-        enabled: true
-  prometheus:
-    enabled: true
-    serviceMonitor:
-      enabled: true
-  securityContext:
-    privileged: true
-  sessionAffinity: true
-  clusterHealthPort: 9940
-  tunnelPort: 4789
-EOF
-for manifest in *.yaml ; do
-  cp "${manifest}" "${SHARED_DIR}/manifest_${manifest}"
-done
+cilium install \
+    --namespace cilium \
+    --version "${CILIUM_VERSION}" \
+    --set debug.enabled=true \
+    --set k8s.requireIPv4PodCIDR=true \
+    --set logSystemLoad=true \
+    --set ipv6.enabled=false \
+    --set identityChangeGracePeriod=0s \
+    --set ipam.mode=cluster-pool \
+    --set "ipam.operator.clusterPoolIPv4PodCIDRList={10.128.0.0/14}" \
+    --set ipam.operator.clusterPoolIPv4MaskSize=23 \
+    --set ipv4NativeRoutingCIDR=10.128.0.0/14 \
+    --set cni.binPath=/var/lib/cni/bin \
+    --set cni.confPath=/var/run/multus/cni/net.d \
+    --set sessionAffinity=true \
+    --set hubble.enabled="${HUBBLE:-true}" \
+    --set endpointRoutes.enabled="${ENDPOINT_ROUTES:-true}" \
+    --set cni.chainingMode=portmap \
+    --set tunnelPort=4789 \
+    --set clusterHealthPort=9940 \
+    --set socketLB.enabled=true
+
+cilium status --namespace cilium --wait

ci-operator/step-registry/cilium/conf/cilium-conf-ref.yaml

@@ -8,9 +8,13 @@ ref:
       memory: 100Mi
   env:
   - name: CILIUM_VERSION
-    default: "1.13.9"
+    default: "1.19.1"
     documentation: |-
-      This var will default to 1.13.9, however users can override to a different Cilium version, ensure the manifests exists in the repo.
+      The Cilium version to install (Helm chart version).
+  - name: CILIUM_CLI_VERSION
+    default: "0.19.2"
+    documentation: |-
+      The Cilium CLI version to download from GitHub releases.
   - name: HUBBLE
     default: "true"
     documentation: |-
@@ -20,4 +24,4 @@ ref:
     documentation: |-
       This var will default to true, however users can override to disable endpoint routes.
   documentation: |-
-    This steps installs Cilium OLM manifests (https://github.com/cilium/cilium-olm)
+    This step installs Cilium CNI using the Cilium CLI (https://github.com/cilium/cilium-cli)

ci-operator/step-registry/cucushift/hypershift-extended/cilium/cucushift-hypershift-extended-cilium-commands.sh

@@ -2,6 +2,9 @@
 
 set -xeuo pipefail
 
+CILIUM_VERSION=${CILIUM_VERSION:-"1.19.1"}
+CILIUM_CLI_VERSION=${CILIUM_CLI_VERSION:-"0.19.2"}
+
 function set_proxy () {
     if test -s "${SHARED_DIR}/proxy-conf.sh" ; then
         echo "setting the proxy"
@@ -25,74 +28,43 @@ if [[ -f "${SHARED_DIR}/kubeconfig.kubeadmin" ]]; then
   export KUBECONFIG="${SHARED_DIR}/kubeconfig.kubeadmin"
 fi
 
-
-cilium_ns=$(oc get ns cilium --ignore-not-found)
-if [[ -z "$cilium_ns" ]]; then
-  oc create ns cilium
-fi
-
-oc label ns cilium security.openshift.io/scc.podSecurityLabelSync=false pod-security.kubernetes.io/enforce=privileged pod-security.kubernetes.io/audit=privileged pod-security.kubernetes.io/warn=privileged --overwrite
-
-# apply isovalent cilium CNI
-oc apply -f "https://raw.githubusercontent.com/isovalent/olm-for-cilium/main/manifests/cilium.v${CILIUM_VERSION}/cluster-network-03-cilium-ciliumconfigs-crd.yaml"
-oc apply -f "https://raw.githubusercontent.com/isovalent/olm-for-cilium/main/manifests/cilium.v${CILIUM_VERSION}/cluster-network-06-cilium-00000-cilium-namespace.yaml"
-oc apply -f "https://raw.githubusercontent.com/isovalent/olm-for-cilium/main/manifests/cilium.v${CILIUM_VERSION}/cluster-network-06-cilium-00001-cilium-olm-serviceaccount.yaml"
-oc apply -f "https://raw.githubusercontent.com/isovalent/olm-for-cilium/main/manifests/cilium.v${CILIUM_VERSION}/cluster-network-06-cilium-00002-cilium-olm-deployment.yaml"
-oc apply -f "https://raw.githubusercontent.com/isovalent/olm-for-cilium/main/manifests/cilium.v${CILIUM_VERSION}/cluster-network-06-cilium-00003-cilium-olm-service.yaml"
-oc apply -f "https://raw.githubusercontent.com/isovalent/olm-for-cilium/main/manifests/cilium.v${CILIUM_VERSION}/cluster-network-06-cilium-00004-cilium-olm-leader-election-role.yaml"
-oc apply -f "https://raw.githubusercontent.com/isovalent/olm-for-cilium/main/manifests/cilium.v${CILIUM_VERSION}/cluster-network-06-cilium-00005-cilium-olm-role.yaml"
-oc apply -f "https://raw.githubusercontent.com/isovalent/olm-for-cilium/main/manifests/cilium.v${CILIUM_VERSION}/cluster-network-06-cilium-00006-leader-election-rolebinding.yaml"
-oc apply -f "https://raw.githubusercontent.com/isovalent/olm-for-cilium/main/manifests/cilium.v${CILIUM_VERSION}/cluster-network-06-cilium-00007-cilium-olm-rolebinding.yaml"
-oc apply -f "https://raw.githubusercontent.com/isovalent/olm-for-cilium/main/manifests/cilium.v${CILIUM_VERSION}/cluster-network-06-cilium-00008-cilium-cilium-olm-clusterrole.yaml"
-oc apply -f "https://raw.githubusercontent.com/isovalent/olm-for-cilium/main/manifests/cilium.v${CILIUM_VERSION}/cluster-network-06-cilium-00009-cilium-cilium-clusterrole.yaml"
-oc apply -f "https://raw.githubusercontent.com/isovalent/olm-for-cilium/main/manifests/cilium.v${CILIUM_VERSION}/cluster-network-06-cilium-00010-cilium-cilium-olm-clusterrolebinding.yaml"
-oc apply -f "https://raw.githubusercontent.com/isovalent/olm-for-cilium/main/manifests/cilium.v${CILIUM_VERSION}/cluster-network-06-cilium-00011-cilium-cilium-clusterrolebinding.yaml"
+mkdir -p /tmp/bin
+export PATH=/tmp/bin:$PATH
+curl --fail --retry 3 -sS -L "https://github.com/cilium/cilium-cli/releases/download/v${CILIUM_CLI_VERSION}/cilium-linux-amd64.tar.gz" | tar -xzC /tmp/bin/
+chmod +x /tmp/bin/cilium
 
 PODCIDR=$(oc get network cluster -o jsonpath='{.spec.clusterNetwork[0].cidr}')
 HOSTPREFIX=$(oc get network cluster -o jsonpath='{.spec.clusterNetwork[0].hostPrefix}')
 export PODCIDR=$PODCIDR
 export HOSTPREFIX=$HOSTPREFIX
 
-echo '
-apiVersion: cilium.io/v1alpha1
-kind: CiliumConfig
-metadata:
-  name: cilium
-  namespace: cilium
-spec:
-  debug:
-    enabled: true
-  k8s:
-    requireIPv4PodCIDR: true
-  logSystemLoad: true
-  bpf:
-    preallocateMaps: true
-  etcd:
-    leaseTTL: 30s
-  ipv4:
-    enabled: true
-  ipv6:
-    enabled: false
-  identityChangeGracePeriod: 0s
-  ipam:
-    mode: "cluster-pool"
-    operator:
-      clusterPoolIPv4PodCIDRList:
-        - "${PODCIDR}"
-      clusterPoolIPv4MaskSize: "${HOSTPREFIX}"
-  nativeRoutingCIDR: "${PODCIDR}"
-  endpointRoutes: {enabled: true}
-  clusterHealthPort: 9940
-  tunnelPort: 4789
-  cni:
-    binPath: "/var/lib/cni/bin"
-    confPath: "/var/run/multus/cni/net.d"
-    chainingMode: portmap
-  prometheus:
-    serviceMonitor: {enabled: false}
-  hubble:
-    tls: {enabled: false}
-  sessionAffinity: true
-' | envsubst > /tmp/ciliumconfig.json
+oc get ns cilium || oc create ns cilium
+oc adm policy add-scc-to-user privileged -z cilium -n cilium
+oc adm policy add-scc-to-user privileged -z cilium-operator -n cilium
+oc adm policy add-scc-to-user privileged -z cilium-envoy -n cilium
+
+# Note: In order to test with a development version, use:
+# --repository oci://quay.io/cilium-charts-dev/cilium --version <version>
+# where <version> is a tag from https://quay.io/repository/cilium-charts-dev/cilium
+cilium install \
+    --namespace cilium \
+    --version "${CILIUM_VERSION}" \
+    --set debug.enabled=true \
+    --set k8s.requireIPv4PodCIDR=true \
+    --set logSystemLoad=true \
+    --set ipv6.enabled=false \
+    --set identityChangeGracePeriod=0s \
+    --set ipam.mode=cluster-pool \
+    --set "ipam.operator.clusterPoolIPv4PodCIDRList={${PODCIDR}}" \
+    --set ipam.operator.clusterPoolIPv4MaskSize=${HOSTPREFIX} \
+    --set ipv4NativeRoutingCIDR=${PODCIDR} \
+    --set cni.binPath=/var/lib/cni/bin \
+    --set cni.confPath=/var/run/multus/cni/net.d \
+    --set sessionAffinity=true \
+    --set endpointRoutes.enabled="true" \
+    --set cni.chainingMode=portmap \
+    --set tunnelPort=4789 \
+    --set clusterHealthPort=9940 \
+    --set socketLB.enabled=true
 
-oc apply -f /tmp/ciliumconfig.json
+cilium status --namespace cilium --wait

ci-operator/step-registry/cucushift/hypershift-extended/cilium/cucushift-hypershift-extended-cilium-ref.yaml

@@ -9,10 +9,14 @@ ref:
   commands: cucushift-hypershift-extended-cilium-commands.sh
   env:
   - name: CILIUM_VERSION
-    default: "1.15.1"
+    default: "1.19.1"
+  - name: CILIUM_CLI_VERSION
+    default: "0.19.2"
+    documentation: |-
+      The Cilium CLI version to download from GitHub releases.
   resources:
     requests:
       cpu: 100m
       memory: 100Mi
   documentation: |-
-    install cilium CNI for the hosted cluster. In this case, the HostedCluster.spec.networking.networkType should be Other
+    This step installs Cilium CNI using the Cilium CLI (https://github.com/cilium/cilium-cli)

ci-operator/step-registry/cucushift/hypershift-extended/cilium/dump-debug/OWNERS

@@ -0,0 +1,4 @@
+approvers:
+  - mgencur
+reviewers:
+  - mgencur

ci-operator/step-registry/cucushift/hypershift-extended/cilium/dump-debug/cucushift-hypershift-extended-cilium-dump-debug-commands.sh

@@ -0,0 +1,17 @@
+#!/bin/bash
+
+set -xuo pipefail
+export KUBECONFIG=${SHARED_DIR}/nested_kubeconfig
+mkdir -p ${ARTIFACT_DIR}/cilium-debug
+
+oc get ciliumclusterwidenetworkpolicy -A -o yaml > ${ARTIFACT_DIR}/cilium-debug/ciliumclusterwidenetworkpolicies.yaml 2>&1 || true
+oc get ciliumendpoint -A > ${ARTIFACT_DIR}/cilium-debug/ciliumendpoints.txt 2>&1 || true
+oc get ciliumconfig -n cilium -o yaml > ${ARTIFACT_DIR}/cilium-debug/ciliumconfig.yaml 2>&1 || true
+
+CILIUM_POD=$(oc get pods -n cilium -l k8s-app=cilium -o jsonpath='{.items[0].metadata.name}' 2>/dev/null) || true
+if [[ -n "${CILIUM_POD}" ]]; then
+  oc exec -n cilium ${CILIUM_POD} -- cilium status > ${ARTIFACT_DIR}/cilium-debug/cilium-status.txt 2>&1 || true
+  oc exec -n cilium ${CILIUM_POD} -- cilium service list > ${ARTIFACT_DIR}/cilium-debug/cilium-service-list.txt 2>&1 || true
+  oc exec -n cilium ${CILIUM_POD} -- cilium bpf lb list > ${ARTIFACT_DIR}/cilium-debug/cilium-bpf-lb-list.txt 2>&1 || true
+  oc exec -n cilium ${CILIUM_POD} -- cilium config > ${ARTIFACT_DIR}/cilium-debug/cilium-config.txt 2>&1 || true
+fi

ci-operator/step-registry/cucushift/hypershift-extended/cilium/dump-debug/cucushift-hypershift-extended-cilium-dump-debug-ref.metadata.json

@@ -0,0 +1,11 @@
+{
+	"path": "cucushift/hypershift-extended/cilium/dump-debug/cucushift-hypershift-extended-cilium-dump-debug-ref.yaml",
+	"owners": {
+		"approvers": [
+			"mgencur"
+		],
+		"reviewers": [
+			"mgencur"
+		]
+	}
+}

ci-operator/step-registry/cucushift/hypershift-extended/cilium/dump-debug/cucushift-hypershift-extended-cilium-dump-debug-ref.yaml

@@ -0,0 +1,10 @@
+ref:
+  as: cucushift-hypershift-extended-cilium-dump-debug
+  from: cli
+  commands: cucushift-hypershift-extended-cilium-dump-debug-commands.sh
+  grace_period: 1m0s
+  resources:
+    requests:
+      cpu: 100m
+      memory: 100Mi
+  timeout: 5m0s

ci-operator/step-registry/cucushift/hypershift-extended/cilium/health-check/cucushift-hypershift-extended-cilium-health-check-commands.sh

@@ -31,8 +31,6 @@ oc wait clusteroperators --all --for=condition=Progressing=False --timeout=30m
 oc wait clusteroperators --all --for=condition=Degraded=False --timeout=30m
 oc wait clusterversion/version --for=condition=Available=True --timeout=30m
 
-oc wait --for=condition=Ready pod -n cilium --all --timeout=5m
-
 echo "Performing Cilium connectivity tests"
 trap "dump_connectivity_test_namespace; cleanup_connectivity_test" EXIT
 oc apply -f - <<EOF

ci-operator/step-registry/cucushift/hypershift-extended/cilium/network-policies/OWNERS

@@ -0,0 +1,4 @@
+approvers:
+  - mgencur
+reviewers:
+  - mgencur