test/k8s: keep configmap across upgrade test

Documentation/cmdref/cilium-operator-generic.md

@@ -36,7 +36,7 @@ cilium-operator-generic [flags]
       --identity-allocation-mode string         Method to use for identity allocation (default "kvstore")
       --identity-gc-interval duration           GC interval for security identities (default 15m0s)
       --identity-heartbeat-timeout duration     Timeout after which identity expires on lack of heartbeat (default 30m0s)
-      --ipam string                             Backend to use for IPAM (default "cluster-pool")
+      --ipam string                             Backend to use for IPAM (default "hostscope-legacy")
       --k8s-api-server string                   Kubernetes API server URL
       --k8s-client-burst int                    Burst value allowed for the K8s client
       --k8s-client-qps float32                  Queries per second limit for the K8s client

Documentation/install/upgrade.rst

@@ -179,7 +179,9 @@ Kubernetes resources are updated accordingly to version you are upgrading to:
    Make sure that you are using the same options as for the initial deployment.
    Instead of using ``--set``, you can also modify the ``values.yaml`` in
    ``install/kubernetes/cilium/values.yaml`` and use it to regenerate the YAML
-   for the latest version.
+   for the latest version. Running any of the previous commands will overwrite
+   the existing cluster's `ConfigMap` which might not be ideal if you want to
+   keep your existing `ConfigMap`.
 
 Step 2: Option B: Preserve ConfigMap
 ------------------------------------
@@ -209,21 +211,16 @@ configuration options for each minor version.
 
   .. group-tab:: Helm
 
-    Deploy Cilium release via Helm:
-
-    .. parsed-literal::
-
-      helm upgrade cilium |CHART_RELEASE| \\
-        --namespace=kube-system \\
-        --set config.enabled=false
+    Keeping an existing `ConfigMap` with ``helm upgrade`` is currently not
+    supported.
 
 .. note::
 
    The above variant can not be used in combination with ``--set`` or providing
    ``values.yaml`` because all options are fed into the DaemonSets and
    Deployments using the `ConfigMap` which is not generated if
-   ``config.enabled=false`` is set. The above command *only* generates the
-   DaemonSet, Deployment and RBAC definitions.
+   ``config.enabled=false`` or ``config.keepCurrent=true`` are set. The above
+   command *only* generates the DaemonSet, Deployment and RBAC definitions.
 
 Step 3: Rolling Back
 --------------------
@@ -328,6 +325,48 @@ IMPORTANT: Changes required before upgrading to 1.8.0
    Do not upgrade to 1.8.0 before reading the following section and completing
    the required steps.
 
+* The ``cilium-agent`` container ``liveness`` and ``readiness`` probes have been
+  replaced with a ``httpGet`` instead of an ``exec`` probe. Unfortunately,
+  upgrading using ``kubectl apply`` does not work since the merge strategy done
+  by Kubernetes does not remove the old probe when replacing with a new one.
+  This causes ``kubectl apply`` command to return an error such as:
+
+::
+
+  The DaemonSet "cilium" is invalid:
+  * spec.template.spec.containers[0].livenessProbe.httpGet: Forbidden: may not specify more than 1 handler type
+  * spec.template.spec.containers[0].readinessProbe.httpGet: Forbidden: may not specify more than 1 handler type
+
+  Existing users must either choose to keep the ``exec`` probe in the
+  `DaemonSet` specification to safely upgrade or re-create the Cilium `DaemonSet`
+  without the deprecated probe. It is advisable to keep the probe when doing
+  an upgrade from ``v1.7.x`` to ``v1.8.x`` in the event of having to do a
+  downgrade. The removal of this probe should be done after a successful
+  upgrade.
+
+  The helm option ``agent.keepDeprecatedProbes=true`` will keep the
+  ``exec`` probe in the new `DaemonSet`:
+
+.. tabs::
+  .. group-tab:: kubectl
+
+    .. parsed-literal::
+
+      helm template cilium \
+      --namespace=kube-system \
+      ...
+      --set agent.keepDeprecatedProbes=true \
+      ...
+      > cilium.yaml
+      kubectl apply -f cilium.yaml
+
+  .. group-tab:: Helm
+
+    .. parsed-literal::
+
+      helm upgrade cilium --namespace=kube-system \
+      --set agent.keepDeprecatedProbes=true
+
 * **Important:** The masquerading behavior has changed, depending on how you
   have configured masquerading you need to take action to avoid potential
   NetworkPolicy related drops:

install/kubernetes/cilium/charts/agent/templates/daemonset.yaml

@@ -55,6 +55,13 @@ spec:
         command:
         - cilium-agent
         livenessProbe:
+{{- if .Values.keepDeprecatedProbes }}
+          exec:
+            command:
+            - cilium
+            - status
+            - --brief
+{{- else }}
           httpGet:
             host: '127.0.0.1'
             path: /healthz
@@ -63,6 +70,7 @@ spec:
             httpHeaders:
             - name: "brief"
               value: "true"
+{{- end }}
           failureThreshold: 10
           # The initial delay for the liveness probe is intentionally large to
           # avoid an endless kill & restart cycle if in the event that the initial
@@ -72,6 +80,13 @@ spec:
           successThreshold: 1
           timeoutSeconds: 5
         readinessProbe:
+{{- if .Values.keepDeprecatedProbes }}
+          exec:
+            command:
+            - cilium
+            - status
+            - --brief
+{{- else }}
           httpGet:
             host: '127.0.0.1'
             path: /healthz
@@ -80,6 +95,7 @@ spec:
             httpHeaders:
             - name: "brief"
               value: "true"
+{{- end }}
           failureThreshold: 3
           initialDelaySeconds: 5
           periodSeconds: 30

install/kubernetes/cilium/charts/operator/templates/deployment.yaml

@@ -37,8 +37,10 @@ spec:
         - cilium-operator-aws
 {{- else if .Values.global.azure.enabled }}
         - cilium-operator-azure
-{{- else }}
+{{- else if .Values.global.ipam.operator }}
         - cilium-operator-generic
+{{- else }}
+        - cilium-operator
 {{- end }}
         env:
         - name: K8S_NODE_NAME
@@ -114,8 +116,10 @@ spec:
         image: "{{ .Values.global.registry }}/{{ .Values.image }}-aws:{{ .Values.global.tag }}"
 {{- else if .Values.global.azure.enabled }}
         image: "{{ .Values.global.registry }}/{{ .Values.image }}-azure:{{ .Values.global.tag }}"
-{{- else }}
+{{- else if .Values.global.ipam.operator }}
         image: "{{ .Values.global.registry }}/{{ .Values.image }}-generic:{{ .Values.global.tag }}"
+{{- else }}
+        image: "{{ .Values.global.registry }}/{{ .Values.image }}:{{ .Values.global.tag }}"
 {{- end }}
 {{- end }}
         imagePullPolicy: {{ .Values.global.pullPolicy }}

install/kubernetes/cilium/values.yaml

@@ -7,6 +7,8 @@ agent:
   sleepAfterInit: false
   # Keep the deprecated selector labels when deploying Cilium DaemonSet
   keepDeprecatedLabels: false
+  # Keep the deprecated probes when deploying Cilium DaemonSet
+  keepDeprecatedProbes: false
 
 # Include the cilium-config ConfigMap
 config:

operator/flags.go

@@ -124,7 +124,9 @@ func init() {
 	case "cilium-operator-azure":
 		defaultIPAM = ipamOption.IPAMAzure
 	case "cilium-operator-generic":
-		defaultIPAM = ipamOption.IPAMOperator
+		// Default to Legacy for upgrade paths; new users should
+		// explicitly override the IPAM flag.
+		defaultIPAM = ipamOption.IPAMHostScopeLegacy
 	}
 
 	flags.String(option.IPAM, defaultIPAM, "Backend to use for IPAM")

test/helpers/kubectl.go

@@ -2355,6 +2355,24 @@ func (kub *Kubectl) RunHelm(action, repo, helmName, version, namespace string, o
 		"%s", action, helmName, repo, version, namespace, optionsString)), nil
 }
 
+// RunHelmTemplate runs the helm template command for a specific version.
+func (kub *Kubectl) RunHelmTemplateApply(repo, helmName, version, namespace string, options map[string]string) (*CmdRes, error) {
+	err := kub.overwriteHelmOptions(options)
+	if err != nil {
+		return nil, err
+	}
+	optionsString := ""
+
+	for k, v := range options {
+		optionsString += fmt.Sprintf(" --set %s=%s ", k, v)
+	}
+
+	return kub.ExecMiddle(fmt.Sprintf("helm template %s %s "+
+		"--version=%s "+
+		"--namespace=%s "+
+		"%s | %s apply -f -", helmName, repo, version, namespace, optionsString, KubectlCmd)), nil
+}
+
 // CiliumUninstall uninstalls Cilium with the provided Helm options.
 func (kub *Kubectl) CiliumUninstall(filename string, options map[string]string) error {
 	return kub.ciliumUninstallHelm(filename, options)

test/k8sT/Updates.go

@@ -445,16 +445,26 @@ func InstallAndValidateCiliumUpgrades(kubectl *helpers.Kubectl, oldHelmChartVers
 		By("Upgrading Cilium to %s", newHelmChartVersion)
 		opts = map[string]string{
 			"global.tag": newImageVersion,
+			// Do not use new configuration map as we are testing an upgrade
+			// scenario where the user deploys Cilium while keeping its existing
+			// configuration map.
+			"config.enabled": "false",
 		}
 		// We have removed the labels since >= 1.7 and we are only testing
 		// starting from 1.6.
 		if oldHelmChartVersion == "1.6-dev" {
 			opts["agent.keepDeprecatedLabels"] = "true"
 		}
+		// We have replaced the liveness and readiness probes since >= 1.8 and
+		// we need to keep those deprecated probes from <1.8-dev to >=1.8
+		// upgrades since kubernetes does not do `kubectl apply -f` correctly.
+		switch oldHelmChartVersion {
+		case "1.6-dev", "1.7-dev":
+			opts["agent.keepDeprecatedProbes"] = "true"
+		}
 
 		EventuallyWithOffset(1, func() (*helpers.CmdRes, error) {
-			return kubectl.RunHelm(
-				"upgrade",
+			return kubectl.RunHelmTemplateApply(
 				filepath.Join(kubectl.BasePath(), helpers.HelmTemplate),
 				"cilium",
 				newHelmChartVersion,
@@ -481,9 +491,11 @@ func InstallAndValidateCiliumUpgrades(kubectl *helpers.Kubectl, oldHelmChartVers
 		checkNoInteruptsInSVCFlows()
 
 		By("Downgrading cilium to %s image", oldHelmChartVersion)
-		// rollback cilium 1 because it's the version that we have started
-		// cilium with in this updates test.
-		cmd = kubectl.ExecMiddle("helm rollback cilium 1 --namespace=" + helpers.CiliumNamespace)
+		// Install the previous configuration using helm. This is a hack
+		// as we have previously upgrade Cilium using kubectl apply -f.
+		// helm get keeps the values used when we installed Cilium wihtout the
+		// upgrade changes that we have done with kubectl apply -f.
+		cmd = kubectl.ExecMiddle(fmt.Sprintf("helm get -n %s manifest cilium | kubectl apply -f -", helpers.CiliumNamespace))
 		ExpectWithOffset(1, cmd).To(helpers.CMDSuccess(), "Cilium %q was not able to be deployed", oldHelmChartVersion)
 
 		err = helpers.WithTimeout(