diff --git a/conditional.tf b/conditional.tf index b0cb0ad0..097d43f5 100644 --- a/conditional.tf +++ b/conditional.tf @@ -49,6 +49,7 @@ locals { { cilium_agent_image = var.container_images["cilium_agent"] cilium_operator_image = var.container_images["cilium_operator"] + pod_cidr = var.pod_cidr daemonset_tolerations = var.daemonset_tolerations } ) diff --git a/resources/cilium/cluster-role-binding.yaml b/resources/cilium/cluster-role-binding.yaml index bda945f3..6133d05c 100644 --- a/resources/cilium/cluster-role-binding.yaml +++ b/resources/cilium/cluster-role-binding.yaml @@ -1,14 +1,14 @@ apiVersion: rbac.authorization.k8s.io/v1 kind: ClusterRoleBinding metadata: - name: cilium + name: cilium-operator roleRef: apiGroup: rbac.authorization.k8s.io kind: ClusterRole - name: cilium + name: cilium-operator subjects: - kind: ServiceAccount - name: cilium + name: cilium-operator namespace: kube-system --- diff --git a/resources/cilium/cluster-role.yaml b/resources/cilium/cluster-role.yaml index ea0d102d..1fce809b 100644 --- a/resources/cilium/cluster-role.yaml +++ b/resources/cilium/cluster-role.yaml @@ -1,7 +1,7 @@ apiVersion: rbac.authorization.k8s.io/v1 kind: ClusterRole metadata: - name: cilium + name: cilium-operator rules: - apiGroups: - "" diff --git a/resources/cilium/config.yaml b/resources/cilium/config.yaml index 571f5068..c8089628 100644 --- a/resources/cilium/config.yaml +++ b/resources/cilium/config.yaml @@ -16,9 +16,24 @@ data: # setting it to "kvstore". identity-allocation-mode: crd + # identity-change-grace-period is the grace period that needs to pass + # before an endpoint that has changed its identity will start using + # that new identity. During the grace period, the new identity has + # already been allocated and other nodes in the cluster have a chance + # to whitelist the new upcoming identity of the endpoint. + identity-change-grace-period: "5s" + # If you want to run cilium in debug mode change this value to true debug: "false" + # TCP liveness and readiness probes + agent-health-port: "9876" + + # Prometheus + # enable-metrics: "true" + # prometheus-serve-addr: ":foo" + # operator-prometheus-serve-addr: ":bar" + # Enable IPv4 addressing. If enabled, all endpoints are allocated an IPv4 # address. enable-ipv4: "true" @@ -27,6 +42,9 @@ data: # address. enable-ipv6: "false" + # Enable probing for a more efficient clock source for the BPF datapath + enable-bpf-clock-probe: "true" + # If you want cilium monitor to aggregate tracing for packets, set this level # to "low", "medium", or "maximum". The higher the level, the less packets # that will be seen in monitor output. @@ -44,24 +62,14 @@ data: # Only effective when monitor aggregation is set to "medium" or higher. monitor-aggregation-flags: all - # ct-global-max-entries-* specifies the maximum number of connections - # supported across all endpoints, split by protocol: tcp or other. One pair - # of maps uses these values for IPv4 connections, and another pair of maps - # use these values for IPv6 connections. - # - # If these values are modified, then during the next Cilium startup the - # tracking of ongoing connections may be disrupted. This may lead to brief - # policy drops or a change in loadbalancing decisions for a connection. - # - # For users upgrading from Cilium 1.2 or earlier, to minimize disruption - # during the upgrade process, comment out these options. - bpf-ct-global-tcp-max: "524288" - bpf-ct-global-any-max: "262144" - # bpf-policy-map-max specified the maximum number of entries in endpoint # policy map (per endpoint) bpf-policy-map-max: "16384" + # Specifies the ratio (0.0-1.0) of total system memory to use for dynamic + # sizing of the TCP CT, non-TCP CT, NAT and policy BPF maps. + bpf-map-dynamic-size-ratio: "0.0025" + # Pre-allocation of map entries allows per-packet latency to be reduced, at # the expense of up-front memory allocation for the entries in the maps. The # default value below will minimize memory usage in the default installation; @@ -114,20 +122,67 @@ data: # wait-bpf-mount makes init container wait until bpf filesystem is mounted wait-bpf-mount: "false" + # CNI # Enable chaining with another CNI plugin # - portmap (Enables HostPort support for Cilium) - cni-chaining-mode: "portmap" + cni-chaining-mode: "none" - masquerade: "true" + auto-direct-node-routes: "false" + + # enableXTSocketFallback enables the fallback compatibility solution + # when the xt_socket kernel module is missing and it is needed for + # the datapath L7 redirection to work properly. See documentation + # for details on when this can be disabled: + # http://docs.cilium.io/en/latest/install/system_requirements/#admin-kernel-version. enable-xt-socket-fallback: "true" + + # installIptablesRules enables installation of iptables rules to allow for + # TPROXY (L7 proxy injection), itpables based masquerading and compatibility + # with kube-proxy. See documentation for details on when this can be + # disabled. install-iptables-rules: "true" - auto-direct-node-routes: "false" + + # masquerade traffic leaving the node destined for outside + masquerade: "true" + # bpfMasquerade enables masquerading with BPF instead of iptables + enable-bpf-masquerade: "true" + + # kube-proxy kube-proxy-replacement: "probe" + enable-session-affinity: "true" + + # ClusterIPs from host namespace enable-host-reachable-services: "false" - enable-external-ips: "false" - enable-node-port: "false" + + # NodePort + enable-node-port: "true" node-port-bind-protection: "true" enable-auto-protect-node-port-range: "true" + + # HostPort (i.e. partial portmap) + enable-host-port: "true" + + # ExternalIPs + enable-external-ips: "false" + + # IPAM + ipam: "cluster-pool" + synchronize-k8s-nodes: "true" + disable-cnp-status-updates: "true" + k8s-require-ipv4-pod-cidr: "true" + k8s-require-ipv6-pod-cidr: "false" + cluster-pool-ipv4-cidr: "${pod_cidr}" + cluster-pool-ipv4-mask-size: "24" + + # Health + enable-health-checking: "true" enable-endpoint-health-checking: "true" + + # Identity enable-well-known-identities: "false" enable-remote-node-identity: "true" + + # Disable unwanted + # enable-l7-proxy: "false" + policy-audit-mode: "false" + diff --git a/resources/cilium/daemonset.yaml b/resources/cilium/daemonset.yaml index bcee7e0d..b4a54fa3 100644 --- a/resources/cilium/daemonset.yaml +++ b/resources/cilium/daemonset.yaml @@ -34,22 +34,10 @@ spec: %{~ endfor ~} initContainers: - name: clean-cilium-state - image: docker.io/cilium/cilium:v1.7.5 + image: ${cilium_agent_image} command: - /init-container.sh env: - - name: CILIUM_ALL_STATE - valueFrom: - configMapKeyRef: - name: cilium - key: clean-cilium-state - optional: true - - name: CILIUM_BPF_STATE - valueFrom: - configMapKeyRef: - name: cilium - key: clean-cilium-bpf-state - optional: true - name: CILIUM_WAIT_BPF_MOUNT valueFrom: configMapKeyRef: @@ -85,11 +73,11 @@ spec: fieldRef: apiVersion: v1 fieldPath: metadata.namespace - - name: CILIUM_CNI_CHAINING_MODE - valueFrom: - configMapKeyRef: - key: cni-chaining-mode - name: cilium + ports: + # Not yet used, prefer exec's + - name: health + protocol: TCP + containerPort: 9876 lifecycle: # Install Cilium CNI binary and CNI network config postStart: diff --git a/resources/cilium/deployment.yaml b/resources/cilium/deployment.yaml index ae3fcc5b..3a4249eb 100644 --- a/resources/cilium/deployment.yaml +++ b/resources/cilium/deployment.yaml @@ -20,82 +20,52 @@ spec: seccomp.security.alpha.kubernetes.io/pod: 'docker/default' spec: hostNetwork: true - serviceAccountName: cilium + serviceAccountName: cilium-operator containers: - name: cilium-operator image: ${cilium_operator_image} + command: + - cilium-operator-generic args: + - --config-dir=/tmp/cilium/config-map - --debug=$(CILIUM_DEBUG) - - --identity-allocation-mode=$(CILIUM_IDENTITY_ALLOCATION_MODE) - - --synchronize-k8s-nodes=true - command: - - cilium-operator env: - - name: CILIUM_K8S_NAMESPACE + - name: K8S_NODE_NAME valueFrom: fieldRef: apiVersion: v1 - fieldPath: metadata.namespace - - name: K8S_NODE_NAME + fieldPath: spec.nodeName + - name: CILIUM_K8S_NAMESPACE valueFrom: fieldRef: apiVersion: v1 - fieldPath: spec.nodeName + fieldPath: metadata.namespace - name: CILIUM_DEBUG valueFrom: configMapKeyRef: name: cilium key: debug optional: true - - name: CILIUM_CLUSTER_NAME - valueFrom: - configMapKeyRef: - name: cilium - key: cluster-name - optional: true - - name: CILIUM_CLUSTER_ID - valueFrom: - configMapKeyRef: - name: cilium - key: cluster-id - optional: true - - name: CILIUM_IPAM - valueFrom: - configMapKeyRef: - name: cilium - key: ipam - optional: true - - name: CILIUM_DISABLE_ENDPOINT_CRD - valueFrom: - configMapKeyRef: - name: cilium - key: disable-endpoint-crd - optional: true - - name: CILIUM_KVSTORE - valueFrom: - configMapKeyRef: - name: cilium - key: kvstore - optional: true - - name: CILIUM_KVSTORE_OPT - valueFrom: - configMapKeyRef: - name: cilium - key: kvstore-opt - optional: true - - name: CILIUM_IDENTITY_ALLOCATION_MODE - valueFrom: - configMapKeyRef: - name: cilium - key: identity-allocation-mode - optional: true + ports: + - name: health + protocol: TCP + containerPort: 9234 livenessProbe: httpGet: scheme: HTTP host: 127.0.0.1 - path: /healthz port: 9234 + path: /healthz initialDelaySeconds: 60 periodSeconds: 10 timeoutSeconds: 3 + volumeMounts: + - name: config + mountPath: /tmp/cilium/config-map + readOnly: true + volumes: + # Read configuration + - name: config + configMap: + name: cilium diff --git a/resources/cilium/service-account.yaml b/resources/cilium/service-account.yaml index 7715b9cd..5b551c94 100644 --- a/resources/cilium/service-account.yaml +++ b/resources/cilium/service-account.yaml @@ -1,8 +1,9 @@ apiVersion: v1 kind: ServiceAccount metadata: - name: cilium + name: cilium-operator namespace: kube-system + --- apiVersion: v1 kind: ServiceAccount diff --git a/variables.tf b/variables.tf index 0536322f..ea046a55 100644 --- a/variables.tf +++ b/variables.tf @@ -81,8 +81,8 @@ variable "container_images" { kube_proxy = "k8s.gcr.io/kube-proxy:v1.18.3" # experimental kube_router = "cloudnativelabs/kube-router:v0.3.2" - cilium_agent = "docker.io/cilium/cilium:v1.7.5" - cilium_operator = "docker.io/cilium/operator:v1.7.5" + cilium_agent = "docker.io/cilium/cilium:v1.8.0-rc3" + cilium_operator = "docker.io/cilium/operator-generic:v1.8.0-rc3" } }