Self hosted calico v3.1.3 (#1397)

- Bump calico images for self hosted Calico - Include latest CRDs and RBAC for canal from https://docs.projectcalico.org/v3.1/getting-started/kubernetes/installation/flannel - Correct the self hosted flannel images to self hosted config (previously unused) - Mount calico dirs in canal and kubelet to fix pod init issue, ref projectcalico/calico#1795 (comment) Fixes #1396 Changelog: * Self hosted calico v3.1.3 - Bump calico images for self hosted Calico (non-self hosted is EOL) - Include latest CRDs and RBAC for canal from https://docs.projectcalico.org/v3.1/getting-started/kubernetes/installation/flannel * Link self hosted flannel images to self hosted config Previously the older flannel config was used and the self hosted config in cluster.yaml did nothing * Update canal comment to a reference Our config is dynamic so would change anyway if a user configures the versions differently * Mount calico dirs in canal and kubelet Fixes pod init issue similar to projectcalico/calico#1795 (comment): ``` Failed create pod sandbox: rpc error: code = Unknown desc = NetworkPlugin cni failed to set up pod "xxxx" network: stat /var/lib/calico/nodename: no such file or directory: check that the calico/node container is running and has mounted /var/lib/calico/ ```
kubernetes-retired · Jul 9, 2018 · 4e713f6 · 4e713f6
1 parent acfa4b2
commit 4e713f6
Show file tree

Hide file tree

Showing 4 changed files with 102 additions and 27 deletions.
diff --git a/core/controlplane/config/config.go b/core/controlplane/config/config.go
@@ -35,11 +35,11 @@ const (
 	userDataDir    = "userdata"
 
 	// Experimental SelfHosting feature default images.
-	kubeNetworkingSelfHostingDefaultCalicoNodeImageTag = "v3.0.6"
-	kubeNetworkingSelfHostingDefaultCalicoCniImageTag  = "v2.0.5"
+	kubeNetworkingSelfHostingDefaultCalicoNodeImageTag = "v3.1.3"
+	kubeNetworkingSelfHostingDefaultCalicoCniImageTag  = "v3.1.3"
 	kubeNetworkingSelfHostingDefaultFlannelImageTag    = "v0.9.1"
 	kubeNetworkingSelfHostingDefaultFlannelCniImageTag = "v0.3.0"
-	kubeNetworkingSelfHostingDefaultTyphaImageTag      = "v0.6.4"
+	kubeNetworkingSelfHostingDefaultTyphaImageTag      = "v0.7.4"
 
 	// ControlPlaneStackName is the logical name of a CloudFormation stack resource in a root stack template
 	// This is not needed to be unique in an AWS account because the actual name of a nested stack is generated randomly

diff --git a/core/controlplane/config/templates/cloud-config-controller b/core/controlplane/config/templates/cloud-config-controller
@@ -364,13 +364,14 @@ coreos:
         EnvironmentFile=-/etc/etcd-environment
         Environment=KUBELET_IMAGE_TAG={{.K8sVer}}
         Environment=KUBELET_IMAGE_URL={{ .HyperkubeImage.RktRepoWithoutTag }}
-        Environment="RKT_RUN_ARGS=--volume dns,kind=host,source=/etc/resolv.conf {{.HyperkubeImage.Options}}\
+        Environment="RKT_RUN_ARGS={{.HyperkubeImage.Options}}\
+        --volume dns,kind=host,source=/etc/resolv.conf \
+        --mount volume=dns,target=/etc/resolv.conf \
         {{ if not .Kubernetes.Networking.SelfHosting.Enabled -}}
         --set-env=ETCD_CA_CERT_FILE=/etc/kubernetes/ssl/etcd-trusted-ca.pem \
         --set-env=ETCD_CERT_FILE=/etc/kubernetes/ssl/etcd-client.pem \
         --set-env=ETCD_KEY_FILE=/etc/kubernetes/ssl/etcd-client-key.pem \
         {{ end -}}
-        --mount volume=dns,target=/etc/resolv.conf \
         {{ if eq .ContainerRuntime "rkt" -}}
         --volume rkt,kind=host,source=/opt/bin/host-rkt \
         --mount volume=rkt,target=/usr/bin/rkt \
@@ -381,6 +382,12 @@ coreos:
         {{ end -}}
         --volume var-lib-cni,kind=host,source=/var/lib/cni \
         --mount volume=var-lib-cni,target=/var/lib/cni \
+        {{ if .Kubernetes.Networking.SelfHosting.Enabled -}}
+        --volume var-run-calico,kind=host,source=/var/run/calico \
+        --mount volume=var-run-calico,target=/var/run/calico \
+        --volume var-lib-calico,kind=host,source=/var/lib/calico \
+        --mount volume=var-lib-calico,target=/var/lib/calico \
+        {{ end -}}
         --volume var-log,kind=host,source=/var/log \
         --mount volume=var-log,target=/var/log \
         --volume cni-bin,kind=host,source=/opt/cni/bin \
@@ -402,6 +409,10 @@ coreos:
         ExecStartPre=/usr/bin/mkdir -p /opt/cni/bin
         ExecStartPre=/usr/bin/mkdir -p /etc/kubernetes/manifests
         ExecStartPre=/usr/bin/mkdir -p /etc/kubernetes/cni/net.d
+        {{ if .Kubernetes.Networking.SelfHosting.Enabled -}}
+        ExecStartPre=/usr/bin/mkdir -p /var/run/calico
+        ExecStartPre=/usr/bin/mkdir -p /var/lib/calico
+        {{ end -}}
         ExecStartPre=/bin/sh -ec "find /etc/kubernetes/manifests /srv/kubernetes/manifests  -maxdepth 1 -type f | xargs --no-run-if-empty sed -i 's|#ETCD_ENDPOINTS#|${ETCD_ENDPOINTS}|'"
         {{ if not .Kubernetes.Networking.SelfHosting.Enabled -}}
         {{ if .UseCalico -}}
@@ -1135,12 +1146,7 @@ write_files:
 
   - path: /srv/kubernetes/manifests/canal.yaml
     content: |
-      # Canal Version v3.0.3
-      # https://docs.projectcalico.org/v3.0/releases#v3.0.3
-      # This manifest includes the following component versions:
-      #   calico/node:v3.0.3
-      #   calico/cni:v2.0.1
-      #   coreos/flannel:
+      # Based on https://docs.projectcalico.org/v3.1/getting-started/kubernetes/installation/hosted/canal/canal.yaml
 
       # This ConfigMap can be used to configure a self-hosted Canal installation.
       kind: ConfigMap
@@ -1449,6 +1455,9 @@ write_files:
                   - mountPath: /var/run/calico
                     name: var-run-calico
                     readOnly: false
+                  - mountPath: /var/lib/calico
+                    name: var-lib-calico
+                    readOnly: false
               # This container installs the Calico CNI binaries
               # and CNI network config file on each node.
               - name: install-cni
@@ -1477,7 +1486,7 @@ write_files:
               # This container runs flannel using the kube-subnet-mgr backend
               # for allocating subnets.
               - name: flannel
-                image: {{ .FlannelImage.RepoWithTag }}
+                image: {{ .Kubernetes.Networking.SelfHosting.FlannelImage.RepoWithTag }}
                 command: [ "/opt/bin/flanneld", "--ip-masq", "--kube-subnet-mgr" ]
                 securityContext:
                   privileged: true
@@ -1513,6 +1522,9 @@ write_files:
               - name: var-run-calico
                 hostPath:
                   path: /var/run/calico
+              - name: var-lib-calico
+                hostPath:
+                  path: /var/lib/calico
               # Used to install CNI.
               - name: cni-bin-dir
                 hostPath:
@@ -1669,6 +1681,9 @@ write_files:
                   - mountPath: /var/run/calico
                     name: var-run-calico
                     readOnly: false
+                  - mountPath: /var/lib/calico
+                    name: var-lib-calico
+                    readOnly: false
               # This container installs the Calico CNI binaries
               # and CNI network config file on each node.
               - name: install-cni
@@ -1697,7 +1712,7 @@ write_files:
               # This container runs flannel using the kube-subnet-mgr backend
               # for allocating subnets.
               - name: flannel
-                image: {{ .FlannelImage.RepoWithTag }}
+                image: {{ .Kubernetes.Networking.SelfHosting.FlannelImage.RepoWithTag }}
                 command: [ "/opt/bin/flanneld", "--ip-masq", "--kube-subnet-mgr" ]
                 securityContext:
                   privileged: true
@@ -1733,6 +1748,9 @@ write_files:
               - name: var-run-calico
                 hostPath:
                   path: /var/run/calico
+              - name: var-lib-calico
+                hostPath:
+                  path: /var/lib/calico
               # Used to install CNI.
               - name: cni-bin-dir
                 hostPath:
@@ -1846,6 +1864,36 @@ write_files:
           plural: networkpolicies
           singular: networkpolicy
 
+      ---
+
+      apiVersion: apiextensions.k8s.io/v1beta1
+      kind: CustomResourceDefinition
+      metadata:
+        name: globalnetworksets.crd.projectcalico.org
+      spec:
+        scope: Cluster
+        group: crd.projectcalico.org
+        version: v1
+        names:
+          kind: GlobalNetworkSet
+          plural: globalnetworksets
+          singular: globalnetworkset
+
+      ---
+
+      apiVersion: apiextensions.k8s.io/v1beta1
+      kind: CustomResourceDefinition
+      metadata:
+        name: hostendpoints.crd.projectcalico.org
+      spec:
+        scope: Cluster
+        group: crd.projectcalico.org
+        version: v1
+        names:
+          kind: HostEndpoint
+          plural: hostendpoints
+          singular: hostendpoint
+
   - path: /srv/kubernetes/rbac/network-daemonsets.yaml
     content: |
       apiVersion: v1
@@ -1907,13 +1955,21 @@ write_files:
             - list
             - update
             - watch
+        # remove when Calico < 3.1 is not supported any more
         - apiGroups: ["extensions"]
           resources:
             - networkpolicies
           verbs:
             - get
             - list
             - watch
+        - apiGroups: ["networking.k8s.io"]
+          resources:
+            - networkpolicies
+          verbs:
+            - get
+            - list
+            - watch
         - apiGroups: ["crd.projectcalico.org"]
           resources:
             - globalfelixconfigs
@@ -1925,6 +1981,8 @@ write_files:
             - globalnetworkpolicies
             - networkpolicies
             - clusterinformations
+            - hostendpoints
+            - globalnetworksets
           verbs:
             - create
             - get
@@ -2119,7 +2177,7 @@ write_files:
                 - mountPath: /host/etc/cni/net.d
                   name: cni-net-dir
             - name: flannel
-              image: {{ .FlannelImage.RepoWithTag }}
+              image: {{ .Kubernetes.Networking.SelfHosting.FlannelImage.RepoWithTag }}
               command:
               - /opt/bin/flanneld
               args:
@@ -2298,6 +2356,9 @@ write_files:
                   - mountPath: /var/run/calico
                     name: var-run-calico
                     readOnly: false
+                  - mountPath: /var/lib/calico
+                    name: var-lib-calico
+                    readOnly: false
                   - mountPath: /calico-secrets
                     name: etcd-certs
                   - mountPath: /etc/resolv.conf
@@ -2310,6 +2371,9 @@ write_files:
               - name: var-run-calico
                 hostPath:
                   path: /var/run/calico
+              - name: var-lib-calico
+                hostPath:
+                  path: /var/lib/calico
               - name: cni-bin-dir
                 hostPath:
                   path: /opt/cni/bin

diff --git a/core/nodepool/config/templates/cloud-config-worker b/core/nodepool/config/templates/cloud-config-worker
@@ -422,14 +422,15 @@ coreos:
         EnvironmentFile=-/etc/etcd-environment
         {{- end }}
         Environment=KUBELET_IMAGE_TAG={{.K8sVer}}
-        Environment=KUBELET_IMAGE_URL={{.HyperkubeImage.RktRepoWithoutTag}}
-        Environment="RKT_RUN_ARGS=--volume dns,kind=host,source=/etc/resolv.conf {{.HyperkubeImage.Options}}\
+        Environment=KUBELET_IMAGE_URL={{ .HyperkubeImage.RktRepoWithoutTag }}
+        Environment="RKT_RUN_ARGS={{.HyperkubeImage.Options}}\
+        --volume dns,kind=host,source=/etc/resolv.conf \
+        --mount volume=dns,target=/etc/resolv.conf \
         {{ if not .Kubernetes.Networking.SelfHosting.Enabled -}}
         --set-env=ETCD_CA_CERT_FILE=/etc/kubernetes/ssl/etcd-trusted-ca.pem \
         --set-env=ETCD_CERT_FILE=/etc/kubernetes/ssl/etcd-client.pem \
         --set-env=ETCD_KEY_FILE=/etc/kubernetes/ssl/etcd-client-key.pem \
         {{ end -}}
-        --mount volume=dns,target=/etc/resolv.conf \
         {{ if eq .ContainerRuntime "rkt" -}}
         --volume rkt,kind=host,source=/opt/bin/host-rkt \
         --mount volume=rkt,target=/usr/bin/rkt \
@@ -440,6 +441,12 @@ coreos:
         {{ end -}}
         --volume var-lib-cni,kind=host,source=/var/lib/cni \
         --mount volume=var-lib-cni,target=/var/lib/cni \
+        {{ if .Kubernetes.Networking.SelfHosting.Enabled -}}
+        --volume var-run-calico,kind=host,source=/var/run/calico \
+        --mount volume=var-run-calico,target=/var/run/calico \
+        --volume var-lib-calico,kind=host,source=/var/lib/calico \
+        --mount volume=var-lib-calico,target=/var/lib/calico \
+        {{ end -}}
         --volume var-log,kind=host,source=/var/log \
         --mount volume=var-log,target=/var/log \
         --volume cni-bin,kind=host,source=/opt/cni/bin \
@@ -459,6 +466,10 @@ coreos:
         ExecStartPre=/usr/bin/mkdir -p /opt/cni/bin
         ExecStartPre=/usr/bin/mkdir -p /etc/kubernetes/manifests
         ExecStartPre=/usr/bin/mkdir -p /etc/kubernetes/cni/net.d
+        {{ if .Kubernetes.Networking.SelfHosting.Enabled -}}
+        ExecStartPre=/usr/bin/mkdir -p /var/run/calico
+        ExecStartPre=/usr/bin/mkdir -p /var/lib/calico
+        {{ end -}}
         {{ if not .Kubernetes.Networking.SelfHosting.Enabled -}}
         ExecStartPre=/bin/sh -ec "find /etc/kubernetes/manifests /etc/kubernetes/cni/net.d/  -maxdepth 1 -type f | xargs --no-run-if-empty sed -i 's|#ETCD_ENDPOINTS#|${ETCD_ENDPOINTS}|'"
         ExecStartPre=/usr/bin/etcdctl \

diff --git a/core/root/config/templates/cluster.yaml b/core/root/config/templates/cluster.yaml
@@ -1019,6 +1019,12 @@ worker:
 #  tag: v1.11.2
 #  rktPullDocker: false
 
+# Calico Controller image repository to use.
+#calicoCtlImage:
+#  repo: calico/ctl
+#  tag: v1.6.3
+#  rktPullDocker: false
+
 # Calico Kube Controllers image repository to use.
 #calicoKubeControllersImage:
 #  repo: quay.io/calico/kube-controllers
@@ -1115,12 +1121,6 @@ worker:
 #  tag: v1.8.3
 #  rktPullDocker: false
 
-# Calico Controller image repository to use.
-#calicoCtlImage:
-#  repo: calico/ctl
-#  tag: v1.6.3
-#  rktPullDocker: false
-
 # Pause image repository to use.This works only if you are deploying your cluster in "cn-north-1" region.
 #pauseImage:
 #  repo: k8s.gcr.io/pause-amd64
@@ -1187,19 +1187,19 @@ kubernetes:
       typha: false     # enable for type 'canal' for 50+ node clusters
 #      calicoNodeImage:
 #        repo: quay.io/calico/node
-#        tag: v3.0.3
+#        tag: v3.1.3
 #      calicoCniImage:
 #        repo: quay.io/calico/cni
-#        tag: v2.0.1
+#        tag: v3.1.3
 #      flannelImage:
 #        repo: quay.io/coreos/flannel
 #        tag: v0.9.1
 #      flannelCniImage:
 #        repo: quay.io/coreos/flannel-cni
 #        tag: v0.3.0
-#     typhaImage:
+#      typhaImage:
 #        repo: quay.io/calico/typha
-#        tag: v0.6.2
+#        tag: v0.7.4
 
 # Use Calico for network policy (legacy setting - ignored if networking self-hosting is enabled)
 # useCalico: false