Upgrade Cilium from v1.7.5 to v1.8.3-rc3

conditional.tf

@@ -49,6 +49,7 @@ locals {
       {
         cilium_agent_image    = var.container_images["cilium_agent"]
         cilium_operator_image = var.container_images["cilium_operator"]
+        pod_cidr                        = var.pod_cidr
         daemonset_tolerations = var.daemonset_tolerations
       }
     )

resources/cilium/cluster-role-binding.yaml

@@ -1,14 +1,14 @@
 apiVersion: rbac.authorization.k8s.io/v1
 kind: ClusterRoleBinding
 metadata:
-  name: cilium
+  name: cilium-operator
 roleRef:
   apiGroup: rbac.authorization.k8s.io
   kind: ClusterRole
-  name: cilium
+  name: cilium-operator
 subjects:
 - kind: ServiceAccount
-  name: cilium
+  name: cilium-operator
   namespace: kube-system
 
 ---

resources/cilium/cluster-role.yaml

@@ -1,7 +1,7 @@
 apiVersion: rbac.authorization.k8s.io/v1
 kind: ClusterRole
 metadata:
-  name: cilium
+  name: cilium-operator
 rules:
 - apiGroups:
   - ""

resources/cilium/config.yaml

@@ -16,9 +16,24 @@ data:
   #   setting it to "kvstore".
   identity-allocation-mode: crd
 
+  # identity-change-grace-period is the grace period that needs to pass
+  # before an endpoint that has changed its identity will start using
+  # that new identity. During the grace period, the new identity has
+  # already been allocated and other nodes in the cluster have a chance
+  # to whitelist the new upcoming identity of the endpoint.
+  identity-change-grace-period: "5s"
+
   # If you want to run cilium in debug mode change this value to true
   debug: "false"
 
+  # TCP liveness and readiness probes
+  agent-health-port: "9876"
+
+  # Prometheus
+  # enable-metrics: "true"
+  # prometheus-serve-addr: ":foo"
+  # operator-prometheus-serve-addr: ":bar"
+
   # Enable IPv4 addressing. If enabled, all endpoints are allocated an IPv4
   # address.
   enable-ipv4: "true"
@@ -27,6 +42,9 @@ data:
   # address.
   enable-ipv6: "false"
 
+  # Enable probing for a more efficient clock source for the BPF datapath
+  enable-bpf-clock-probe: "true"
+
   # If you want cilium monitor to aggregate tracing for packets, set this level
   # to "low", "medium", or "maximum". The higher the level, the less packets
   # that will be seen in monitor output.
@@ -44,24 +62,14 @@ data:
   # Only effective when monitor aggregation is set to "medium" or higher.
   monitor-aggregation-flags: all
 
-  # ct-global-max-entries-* specifies the maximum number of connections
-  # supported across all endpoints, split by protocol: tcp or other. One pair
-  # of maps uses these values for IPv4 connections, and another pair of maps
-  # use these values for IPv6 connections.
-  #
-  # If these values are modified, then during the next Cilium startup the
-  # tracking of ongoing connections may be disrupted. This may lead to brief
-  # policy drops or a change in loadbalancing decisions for a connection.
-  #
-  # For users upgrading from Cilium 1.2 or earlier, to minimize disruption
-  # during the upgrade process, comment out these options.
-  bpf-ct-global-tcp-max: "524288"
-  bpf-ct-global-any-max: "262144"
-
   # bpf-policy-map-max specified the maximum number of entries in endpoint
   # policy map (per endpoint)
   bpf-policy-map-max: "16384"
 
+  # Specifies the ratio (0.0-1.0) of total system memory to use for dynamic
+  # sizing of the TCP CT, non-TCP CT, NAT and policy BPF maps.
+  bpf-map-dynamic-size-ratio: "0.0025"
+
   # Pre-allocation of map entries allows per-packet latency to be reduced, at
   # the expense of up-front memory allocation for the entries in the maps. The
   # default value below will minimize memory usage in the default installation;
@@ -114,20 +122,67 @@ data:
   # wait-bpf-mount makes init container wait until bpf filesystem is mounted
   wait-bpf-mount: "false"
 
+  # CNI
   # Enable chaining with another CNI plugin
   #  - portmap (Enables HostPort support for Cilium)
-  cni-chaining-mode: "portmap"
+  cni-chaining-mode: "none"
 
-  masquerade: "true"
+  auto-direct-node-routes: "false"
+
+  # enableXTSocketFallback enables the fallback compatibility solution
+  # when the xt_socket kernel module is missing and it is needed for
+  # the datapath L7 redirection to work properly.  See documentation
+  # for details on when this can be disabled:
+  # http://docs.cilium.io/en/latest/install/system_requirements/#admin-kernel-version.
   enable-xt-socket-fallback: "true"
+
+  # installIptablesRules enables installation of iptables rules to allow for
+  # TPROXY (L7 proxy injection), itpables based masquerading and compatibility
+  # with kube-proxy. See documentation for details on when this can be
+  # disabled.
   install-iptables-rules: "true"
-  auto-direct-node-routes: "false"
+
+  # masquerade traffic leaving the node destined for outside
+  masquerade: "true"
+  # bpfMasquerade enables masquerading with BPF instead of iptables
+  enable-bpf-masquerade: "true"
+
+  # kube-proxy
   kube-proxy-replacement:  "probe"
+  enable-session-affinity: "true"
+
+  # ClusterIPs from host namespace
   enable-host-reachable-services: "false"
-  enable-external-ips: "false"
-  enable-node-port: "false"
+
+  # NodePort
+  enable-node-port: "true"
   node-port-bind-protection: "true"
   enable-auto-protect-node-port-range: "true"
+
+  # HostPort (i.e. partial portmap)
+  enable-host-port: "true"
+
+  # ExternalIPs
+  enable-external-ips: "false"
+
+  # IPAM
+  ipam: "cluster-pool"
+  synchronize-k8s-nodes: "true"
+  disable-cnp-status-updates: "true"
+  k8s-require-ipv4-pod-cidr: "true"
+  k8s-require-ipv6-pod-cidr: "false"
+  cluster-pool-ipv4-cidr: "${pod_cidr}"
+  cluster-pool-ipv4-mask-size: "24"
+
+  # Health
+  enable-health-checking: "true"
   enable-endpoint-health-checking: "true"
+
+  # Identity
   enable-well-known-identities: "false"
   enable-remote-node-identity: "true"
+
+  # Disable unwanted
+  # enable-l7-proxy: "false"
+  policy-audit-mode: "false"
+

resources/cilium/daemonset.yaml

@@ -34,22 +34,10 @@ spec:
       %{~ endfor ~}
       initContainers:
       - name: clean-cilium-state
-        image: docker.io/cilium/cilium:v1.7.5
+        image: ${cilium_agent_image}
         command:
         - /init-container.sh
         env:
-        - name: CILIUM_ALL_STATE
-          valueFrom:
-            configMapKeyRef:
-              name: cilium
-              key: clean-cilium-state
-              optional: true
-        - name: CILIUM_BPF_STATE
-          valueFrom:
-            configMapKeyRef:
-              name: cilium
-              key: clean-cilium-bpf-state
-              optional: true
         - name: CILIUM_WAIT_BPF_MOUNT
           valueFrom:
             configMapKeyRef:
@@ -85,11 +73,11 @@ spec:
             fieldRef:
               apiVersion: v1
               fieldPath: metadata.namespace
-        - name: CILIUM_CNI_CHAINING_MODE
-          valueFrom:
-            configMapKeyRef:
-              key: cni-chaining-mode
-              name: cilium
+        ports:
+          # Not yet used, prefer exec's
+          - name: health
+            protocol: TCP
+            containerPort: 9876
         lifecycle:
           # Install Cilium CNI binary and CNI network config
           postStart:

resources/cilium/deployment.yaml

@@ -20,82 +20,52 @@ spec:
         seccomp.security.alpha.kubernetes.io/pod: 'docker/default'
     spec:
       hostNetwork: true
-      serviceAccountName: cilium
+      serviceAccountName: cilium-operator
       containers:
       - name: cilium-operator
         image: ${cilium_operator_image}
+        command:
+        - cilium-operator-generic
         args:
+        - --config-dir=/tmp/cilium/config-map
         - --debug=$(CILIUM_DEBUG)
-        - --identity-allocation-mode=$(CILIUM_IDENTITY_ALLOCATION_MODE)
-        - --synchronize-k8s-nodes=true
-        command:
-        - cilium-operator
         env:
-        - name: CILIUM_K8S_NAMESPACE
+        - name: K8S_NODE_NAME
           valueFrom:
             fieldRef:
               apiVersion: v1
-              fieldPath: metadata.namespace
-        - name: K8S_NODE_NAME
+              fieldPath: spec.nodeName
+        - name: CILIUM_K8S_NAMESPACE
           valueFrom:
             fieldRef:
               apiVersion: v1
-              fieldPath: spec.nodeName
+              fieldPath: metadata.namespace
         - name: CILIUM_DEBUG
           valueFrom:
             configMapKeyRef:
               name: cilium
               key: debug
               optional: true
-        - name: CILIUM_CLUSTER_NAME
-          valueFrom:
-            configMapKeyRef:
-              name: cilium
-              key: cluster-name
-              optional: true
-        - name: CILIUM_CLUSTER_ID
-          valueFrom:
-            configMapKeyRef:
-              name: cilium
-              key: cluster-id
-              optional: true
-        - name: CILIUM_IPAM
-          valueFrom:
-            configMapKeyRef:
-              name: cilium
-              key: ipam
-              optional: true
-        - name: CILIUM_DISABLE_ENDPOINT_CRD
-          valueFrom:
-            configMapKeyRef:
-              name: cilium
-              key: disable-endpoint-crd
-              optional: true
-        - name: CILIUM_KVSTORE
-          valueFrom:
-            configMapKeyRef:
-              name: cilium
-              key: kvstore
-              optional: true
-        - name: CILIUM_KVSTORE_OPT
-          valueFrom:
-            configMapKeyRef:
-              name: cilium
-              key: kvstore-opt
-              optional: true
-        - name: CILIUM_IDENTITY_ALLOCATION_MODE
-          valueFrom:
-            configMapKeyRef:
-              name: cilium
-              key: identity-allocation-mode
-              optional: true
+        ports:
+          - name: health
+            protocol: TCP
+            containerPort: 9234
         livenessProbe:
           httpGet:
             scheme: HTTP
             host: 127.0.0.1
-            path: /healthz
             port: 9234
+            path: /healthz
           initialDelaySeconds: 60
           periodSeconds: 10
           timeoutSeconds: 3
+        volumeMounts:
+        - name: config
+          mountPath: /tmp/cilium/config-map
+          readOnly: true
+      volumes:
+      # Read configuration
+      - name: config
+        configMap:
+          name: cilium
 

resources/cilium/service-account.yaml

@@ -1,8 +1,9 @@
 apiVersion: v1
 kind: ServiceAccount
 metadata:
-  name: cilium
+  name: cilium-operator
   namespace: kube-system
+
 ---
 apiVersion: v1
 kind: ServiceAccount

variables.tf

@@ -81,8 +81,8 @@ variable "container_images" {
     kube_proxy              = "k8s.gcr.io/kube-proxy:v1.18.3"
     # experimental
     kube_router     = "cloudnativelabs/kube-router:v0.3.2"
-    cilium_agent    = "docker.io/cilium/cilium:v1.7.5"
-    cilium_operator = "docker.io/cilium/operator:v1.7.5"
+    cilium_agent    = "docker.io/cilium/cilium:v1.8.0-rc3"
+    cilium_operator = "docker.io/cilium/operator-generic:v1.8.0-rc3"
   }
 }