From 4e713f6c471a3ed7bb269931dfb0457664d34434 Mon Sep 17 00:00:00 2001 From: Chris Knowles Date: Mon, 9 Jul 2018 08:37:18 +0800 Subject: [PATCH] Self hosted calico v3.1.3 (#1397) - Bump calico images for self hosted Calico - Include latest CRDs and RBAC for canal from https://docs.projectcalico.org/v3.1/getting-started/kubernetes/installation/flannel - Correct the self hosted flannel images to self hosted config (previously unused) - Mount calico dirs in canal and kubelet to fix pod init issue, ref https://github.com/projectcalico/calico/issues/1795#issuecomment-390171235 Fixes https://github.com/kubernetes-incubator/kube-aws/issues/1396 Changelog: * Self hosted calico v3.1.3 - Bump calico images for self hosted Calico (non-self hosted is EOL) - Include latest CRDs and RBAC for canal from https://docs.projectcalico.org/v3.1/getting-started/kubernetes/installation/flannel * Link self hosted flannel images to self hosted config Previously the older flannel config was used and the self hosted config in cluster.yaml did nothing * Update canal comment to a reference Our config is dynamic so would change anyway if a user configures the versions differently * Mount calico dirs in canal and kubelet Fixes pod init issue similar to https://github.com/projectcalico/calico/issues/1795#issuecomment-390171235: ``` Failed create pod sandbox: rpc error: code = Unknown desc = NetworkPlugin cni failed to set up pod "xxxx" network: stat /var/lib/calico/nodename: no such file or directory: check that the calico/node container is running and has mounted /var/lib/calico/ ``` --- core/controlplane/config/config.go | 6 +- .../config/templates/cloud-config-controller | 86 ++++++++++++++++--- .../config/templates/cloud-config-worker | 17 +++- core/root/config/templates/cluster.yaml | 20 ++--- 4 files changed, 102 insertions(+), 27 deletions(-) diff --git a/core/controlplane/config/config.go b/core/controlplane/config/config.go index 70b5c89c0..9ce0ace16 100644 --- a/core/controlplane/config/config.go +++ b/core/controlplane/config/config.go @@ -35,11 +35,11 @@ const ( userDataDir = "userdata" // Experimental SelfHosting feature default images. - kubeNetworkingSelfHostingDefaultCalicoNodeImageTag = "v3.0.6" - kubeNetworkingSelfHostingDefaultCalicoCniImageTag = "v2.0.5" + kubeNetworkingSelfHostingDefaultCalicoNodeImageTag = "v3.1.3" + kubeNetworkingSelfHostingDefaultCalicoCniImageTag = "v3.1.3" kubeNetworkingSelfHostingDefaultFlannelImageTag = "v0.9.1" kubeNetworkingSelfHostingDefaultFlannelCniImageTag = "v0.3.0" - kubeNetworkingSelfHostingDefaultTyphaImageTag = "v0.6.4" + kubeNetworkingSelfHostingDefaultTyphaImageTag = "v0.7.4" // ControlPlaneStackName is the logical name of a CloudFormation stack resource in a root stack template // This is not needed to be unique in an AWS account because the actual name of a nested stack is generated randomly diff --git a/core/controlplane/config/templates/cloud-config-controller b/core/controlplane/config/templates/cloud-config-controller index 09520d97c..80a218a53 100644 --- a/core/controlplane/config/templates/cloud-config-controller +++ b/core/controlplane/config/templates/cloud-config-controller @@ -364,13 +364,14 @@ coreos: EnvironmentFile=-/etc/etcd-environment Environment=KUBELET_IMAGE_TAG={{.K8sVer}} Environment=KUBELET_IMAGE_URL={{ .HyperkubeImage.RktRepoWithoutTag }} - Environment="RKT_RUN_ARGS=--volume dns,kind=host,source=/etc/resolv.conf {{.HyperkubeImage.Options}}\ + Environment="RKT_RUN_ARGS={{.HyperkubeImage.Options}}\ + --volume dns,kind=host,source=/etc/resolv.conf \ + --mount volume=dns,target=/etc/resolv.conf \ {{ if not .Kubernetes.Networking.SelfHosting.Enabled -}} --set-env=ETCD_CA_CERT_FILE=/etc/kubernetes/ssl/etcd-trusted-ca.pem \ --set-env=ETCD_CERT_FILE=/etc/kubernetes/ssl/etcd-client.pem \ --set-env=ETCD_KEY_FILE=/etc/kubernetes/ssl/etcd-client-key.pem \ {{ end -}} - --mount volume=dns,target=/etc/resolv.conf \ {{ if eq .ContainerRuntime "rkt" -}} --volume rkt,kind=host,source=/opt/bin/host-rkt \ --mount volume=rkt,target=/usr/bin/rkt \ @@ -381,6 +382,12 @@ coreos: {{ end -}} --volume var-lib-cni,kind=host,source=/var/lib/cni \ --mount volume=var-lib-cni,target=/var/lib/cni \ + {{ if .Kubernetes.Networking.SelfHosting.Enabled -}} + --volume var-run-calico,kind=host,source=/var/run/calico \ + --mount volume=var-run-calico,target=/var/run/calico \ + --volume var-lib-calico,kind=host,source=/var/lib/calico \ + --mount volume=var-lib-calico,target=/var/lib/calico \ + {{ end -}} --volume var-log,kind=host,source=/var/log \ --mount volume=var-log,target=/var/log \ --volume cni-bin,kind=host,source=/opt/cni/bin \ @@ -402,6 +409,10 @@ coreos: ExecStartPre=/usr/bin/mkdir -p /opt/cni/bin ExecStartPre=/usr/bin/mkdir -p /etc/kubernetes/manifests ExecStartPre=/usr/bin/mkdir -p /etc/kubernetes/cni/net.d + {{ if .Kubernetes.Networking.SelfHosting.Enabled -}} + ExecStartPre=/usr/bin/mkdir -p /var/run/calico + ExecStartPre=/usr/bin/mkdir -p /var/lib/calico + {{ end -}} ExecStartPre=/bin/sh -ec "find /etc/kubernetes/manifests /srv/kubernetes/manifests -maxdepth 1 -type f | xargs --no-run-if-empty sed -i 's|#ETCD_ENDPOINTS#|${ETCD_ENDPOINTS}|'" {{ if not .Kubernetes.Networking.SelfHosting.Enabled -}} {{ if .UseCalico -}} @@ -1135,12 +1146,7 @@ write_files: - path: /srv/kubernetes/manifests/canal.yaml content: | - # Canal Version v3.0.3 - # https://docs.projectcalico.org/v3.0/releases#v3.0.3 - # This manifest includes the following component versions: - # calico/node:v3.0.3 - # calico/cni:v2.0.1 - # coreos/flannel: + # Based on https://docs.projectcalico.org/v3.1/getting-started/kubernetes/installation/hosted/canal/canal.yaml # This ConfigMap can be used to configure a self-hosted Canal installation. kind: ConfigMap @@ -1449,6 +1455,9 @@ write_files: - mountPath: /var/run/calico name: var-run-calico readOnly: false + - mountPath: /var/lib/calico + name: var-lib-calico + readOnly: false # This container installs the Calico CNI binaries # and CNI network config file on each node. - name: install-cni @@ -1477,7 +1486,7 @@ write_files: # This container runs flannel using the kube-subnet-mgr backend # for allocating subnets. - name: flannel - image: {{ .FlannelImage.RepoWithTag }} + image: {{ .Kubernetes.Networking.SelfHosting.FlannelImage.RepoWithTag }} command: [ "/opt/bin/flanneld", "--ip-masq", "--kube-subnet-mgr" ] securityContext: privileged: true @@ -1513,6 +1522,9 @@ write_files: - name: var-run-calico hostPath: path: /var/run/calico + - name: var-lib-calico + hostPath: + path: /var/lib/calico # Used to install CNI. - name: cni-bin-dir hostPath: @@ -1669,6 +1681,9 @@ write_files: - mountPath: /var/run/calico name: var-run-calico readOnly: false + - mountPath: /var/lib/calico + name: var-lib-calico + readOnly: false # This container installs the Calico CNI binaries # and CNI network config file on each node. - name: install-cni @@ -1697,7 +1712,7 @@ write_files: # This container runs flannel using the kube-subnet-mgr backend # for allocating subnets. - name: flannel - image: {{ .FlannelImage.RepoWithTag }} + image: {{ .Kubernetes.Networking.SelfHosting.FlannelImage.RepoWithTag }} command: [ "/opt/bin/flanneld", "--ip-masq", "--kube-subnet-mgr" ] securityContext: privileged: true @@ -1733,6 +1748,9 @@ write_files: - name: var-run-calico hostPath: path: /var/run/calico + - name: var-lib-calico + hostPath: + path: /var/lib/calico # Used to install CNI. - name: cni-bin-dir hostPath: @@ -1846,6 +1864,36 @@ write_files: plural: networkpolicies singular: networkpolicy + --- + + apiVersion: apiextensions.k8s.io/v1beta1 + kind: CustomResourceDefinition + metadata: + name: globalnetworksets.crd.projectcalico.org + spec: + scope: Cluster + group: crd.projectcalico.org + version: v1 + names: + kind: GlobalNetworkSet + plural: globalnetworksets + singular: globalnetworkset + + --- + + apiVersion: apiextensions.k8s.io/v1beta1 + kind: CustomResourceDefinition + metadata: + name: hostendpoints.crd.projectcalico.org + spec: + scope: Cluster + group: crd.projectcalico.org + version: v1 + names: + kind: HostEndpoint + plural: hostendpoints + singular: hostendpoint + - path: /srv/kubernetes/rbac/network-daemonsets.yaml content: | apiVersion: v1 @@ -1907,6 +1955,7 @@ write_files: - list - update - watch + # remove when Calico < 3.1 is not supported any more - apiGroups: ["extensions"] resources: - networkpolicies @@ -1914,6 +1963,13 @@ write_files: - get - list - watch + - apiGroups: ["networking.k8s.io"] + resources: + - networkpolicies + verbs: + - get + - list + - watch - apiGroups: ["crd.projectcalico.org"] resources: - globalfelixconfigs @@ -1925,6 +1981,8 @@ write_files: - globalnetworkpolicies - networkpolicies - clusterinformations + - hostendpoints + - globalnetworksets verbs: - create - get @@ -2119,7 +2177,7 @@ write_files: - mountPath: /host/etc/cni/net.d name: cni-net-dir - name: flannel - image: {{ .FlannelImage.RepoWithTag }} + image: {{ .Kubernetes.Networking.SelfHosting.FlannelImage.RepoWithTag }} command: - /opt/bin/flanneld args: @@ -2298,6 +2356,9 @@ write_files: - mountPath: /var/run/calico name: var-run-calico readOnly: false + - mountPath: /var/lib/calico + name: var-lib-calico + readOnly: false - mountPath: /calico-secrets name: etcd-certs - mountPath: /etc/resolv.conf @@ -2310,6 +2371,9 @@ write_files: - name: var-run-calico hostPath: path: /var/run/calico + - name: var-lib-calico + hostPath: + path: /var/lib/calico - name: cni-bin-dir hostPath: path: /opt/cni/bin diff --git a/core/nodepool/config/templates/cloud-config-worker b/core/nodepool/config/templates/cloud-config-worker index 37cc22a01..20bc457f5 100644 --- a/core/nodepool/config/templates/cloud-config-worker +++ b/core/nodepool/config/templates/cloud-config-worker @@ -422,14 +422,15 @@ coreos: EnvironmentFile=-/etc/etcd-environment {{- end }} Environment=KUBELET_IMAGE_TAG={{.K8sVer}} - Environment=KUBELET_IMAGE_URL={{.HyperkubeImage.RktRepoWithoutTag}} - Environment="RKT_RUN_ARGS=--volume dns,kind=host,source=/etc/resolv.conf {{.HyperkubeImage.Options}}\ + Environment=KUBELET_IMAGE_URL={{ .HyperkubeImage.RktRepoWithoutTag }} + Environment="RKT_RUN_ARGS={{.HyperkubeImage.Options}}\ + --volume dns,kind=host,source=/etc/resolv.conf \ + --mount volume=dns,target=/etc/resolv.conf \ {{ if not .Kubernetes.Networking.SelfHosting.Enabled -}} --set-env=ETCD_CA_CERT_FILE=/etc/kubernetes/ssl/etcd-trusted-ca.pem \ --set-env=ETCD_CERT_FILE=/etc/kubernetes/ssl/etcd-client.pem \ --set-env=ETCD_KEY_FILE=/etc/kubernetes/ssl/etcd-client-key.pem \ {{ end -}} - --mount volume=dns,target=/etc/resolv.conf \ {{ if eq .ContainerRuntime "rkt" -}} --volume rkt,kind=host,source=/opt/bin/host-rkt \ --mount volume=rkt,target=/usr/bin/rkt \ @@ -440,6 +441,12 @@ coreos: {{ end -}} --volume var-lib-cni,kind=host,source=/var/lib/cni \ --mount volume=var-lib-cni,target=/var/lib/cni \ + {{ if .Kubernetes.Networking.SelfHosting.Enabled -}} + --volume var-run-calico,kind=host,source=/var/run/calico \ + --mount volume=var-run-calico,target=/var/run/calico \ + --volume var-lib-calico,kind=host,source=/var/lib/calico \ + --mount volume=var-lib-calico,target=/var/lib/calico \ + {{ end -}} --volume var-log,kind=host,source=/var/log \ --mount volume=var-log,target=/var/log \ --volume cni-bin,kind=host,source=/opt/cni/bin \ @@ -459,6 +466,10 @@ coreos: ExecStartPre=/usr/bin/mkdir -p /opt/cni/bin ExecStartPre=/usr/bin/mkdir -p /etc/kubernetes/manifests ExecStartPre=/usr/bin/mkdir -p /etc/kubernetes/cni/net.d + {{ if .Kubernetes.Networking.SelfHosting.Enabled -}} + ExecStartPre=/usr/bin/mkdir -p /var/run/calico + ExecStartPre=/usr/bin/mkdir -p /var/lib/calico + {{ end -}} {{ if not .Kubernetes.Networking.SelfHosting.Enabled -}} ExecStartPre=/bin/sh -ec "find /etc/kubernetes/manifests /etc/kubernetes/cni/net.d/ -maxdepth 1 -type f | xargs --no-run-if-empty sed -i 's|#ETCD_ENDPOINTS#|${ETCD_ENDPOINTS}|'" ExecStartPre=/usr/bin/etcdctl \ diff --git a/core/root/config/templates/cluster.yaml b/core/root/config/templates/cluster.yaml index 59062cf61..89a839ace 100644 --- a/core/root/config/templates/cluster.yaml +++ b/core/root/config/templates/cluster.yaml @@ -1019,6 +1019,12 @@ worker: # tag: v1.11.2 # rktPullDocker: false +# Calico Controller image repository to use. +#calicoCtlImage: +# repo: calico/ctl +# tag: v1.6.3 +# rktPullDocker: false + # Calico Kube Controllers image repository to use. #calicoKubeControllersImage: # repo: quay.io/calico/kube-controllers @@ -1115,12 +1121,6 @@ worker: # tag: v1.8.3 # rktPullDocker: false -# Calico Controller image repository to use. -#calicoCtlImage: -# repo: calico/ctl -# tag: v1.6.3 -# rktPullDocker: false - # Pause image repository to use.This works only if you are deploying your cluster in "cn-north-1" region. #pauseImage: # repo: k8s.gcr.io/pause-amd64 @@ -1187,19 +1187,19 @@ kubernetes: typha: false # enable for type 'canal' for 50+ node clusters # calicoNodeImage: # repo: quay.io/calico/node -# tag: v3.0.3 +# tag: v3.1.3 # calicoCniImage: # repo: quay.io/calico/cni -# tag: v2.0.1 +# tag: v3.1.3 # flannelImage: # repo: quay.io/coreos/flannel # tag: v0.9.1 # flannelCniImage: # repo: quay.io/coreos/flannel-cni # tag: v0.3.0 -# typhaImage: +# typhaImage: # repo: quay.io/calico/typha -# tag: v0.6.2 +# tag: v0.7.4 # Use Calico for network policy (legacy setting - ignored if networking self-hosting is enabled) # useCalico: false