From 8cc8a014b0da8daaf04f857608ba788da9f6bf57 Mon Sep 17 00:00:00 2001 From: Vinayak Goyal Date: Sat, 19 Jun 2021 11:51:05 -0700 Subject: [PATCH] Add e2e tests for rootless control-plane. --- kinder/ci/tools/update-workflows/config.yaml | 12 + .../testinfra/kubeadm-kinder-rootless.yaml | 38 +++ .../templates/workflows/rootless-tasks.yaml | 277 +++++++++++++++++ .../templates/workflows/rootless.yaml | 9 + kinder/ci/workflows/rootless-latest.yaml | 10 + kinder/ci/workflows/rootless-tasks.yaml | 278 ++++++++++++++++++ kinder/cmd/kinder/do/do.go | 7 + kinder/pkg/cluster/manager/actions/actions.go | 12 +- .../cluster/manager/actions/kubeadm-config.go | 17 +- .../cluster/manager/actions/kubeadm-init.go | 4 +- kinder/pkg/kubeadm/config.go | 11 + 11 files changed, 667 insertions(+), 8 deletions(-) create mode 100644 kinder/ci/tools/update-workflows/templates/testinfra/kubeadm-kinder-rootless.yaml create mode 100644 kinder/ci/tools/update-workflows/templates/workflows/rootless-tasks.yaml create mode 100644 kinder/ci/tools/update-workflows/templates/workflows/rootless.yaml create mode 100644 kinder/ci/workflows/rootless-latest.yaml create mode 100644 kinder/ci/workflows/rootless-tasks.yaml diff --git a/kinder/ci/tools/update-workflows/config.yaml b/kinder/ci/tools/update-workflows/config.yaml index 4989d855..f964395b 100644 --- a/kinder/ci/tools/update-workflows/config.yaml +++ b/kinder/ci/tools/update-workflows/config.yaml @@ -166,3 +166,15 @@ jobGroups: - ./templates/workflows/upgrade-latest-no-addon-config-maps.yaml jobs: - kubernetesVersion: latest + +- name: rootless + testInfraJobSpec: + targetFile: kubeadm-kinder-rootless.yaml + template: ./templates/testinfra/kubeadm-kinder-rootless.yaml + kinderWorkflowSpec: + targetFile: rootless-{{ .KubernetesVersion }}.yaml + template: ./templates/workflows/rootless.yaml + additionalFiles: + - ./templates/workflows/rootless-tasks.yaml + jobs: + - kubernetesVersion: latest diff --git a/kinder/ci/tools/update-workflows/templates/testinfra/kubeadm-kinder-rootless.yaml b/kinder/ci/tools/update-workflows/templates/testinfra/kubeadm-kinder-rootless.yaml new file mode 100644 index 00000000..3f321b92 --- /dev/null +++ b/kinder/ci/tools/update-workflows/templates/testinfra/kubeadm-kinder-rootless.yaml @@ -0,0 +1,38 @@ +- name: ci-kubernetes-e2e-kubeadm-kinder-rootless-{{ dashVer .KubernetesVersion }} + interval: {{ .JobInterval }} + decorate: true + labels: + preset-dind-enabled: "true" + preset-kind-volume-mounts: "true" + annotations: + testgrid-dashboards: sig-cluster-lifecycle-kubeadm + testgrid-tab-name: kubeadm-kinder-rootless-{{ dashVer .KubernetesVersion }} + testgrid-alert-email: sig-cluster-lifecycle-kubeadm-alerts@kubernetes.io + description: "OWNER: sig-cluster-lifecycle (kinder); Uses kubeadm/kinder to create a cluster with rootless control-plane and run kubeadm-e2e and the conformance suite" + testgrid-num-columns-recent: "20" +{{ .AlertAnnotations }} + decoration_config: + timeout: 60m + extra_refs: + - org: kubernetes + repo: kubernetes + base_ref: {{ branchFor .KubernetesVersion }} + path_alias: k8s.io/kubernetes + - org: kubernetes + repo: kubeadm + base_ref: master + path_alias: k8s.io/kubeadm + spec: + containers: + - image: gcr.io/k8s-testimages/kubekins-e2e:{{ .TestInfraImage }}-{{ imageVer .KubernetesVersion }} + command: + - runner.sh + - "../kubeadm/kinder/ci/kinder-run.sh" + args: + - {{ .WorkflowFile }} + securityContext: + privileged: true + resources: + requests: + memory: "9000Mi" + cpu: 2000m diff --git a/kinder/ci/tools/update-workflows/templates/workflows/rootless-tasks.yaml b/kinder/ci/tools/update-workflows/templates/workflows/rootless-tasks.yaml new file mode 100644 index 00000000..74e24031 --- /dev/null +++ b/kinder/ci/tools/update-workflows/templates/workflows/rootless-tasks.yaml @@ -0,0 +1,277 @@ +# IMPORTANT! this workflow is imported by regular-* workflows. +version: 1 +summary: | + This workflow implements a sequence of tasks used test the proper functioning + of kubeadm version X with Kubernetes version X. +vars: + # vars defines default values for variable used by tasks in this workflow; + # those values might be overridden when importing this files. + kubernetesVersion: v1.13.5 + controlPlaneNodes: 3 + workerNodes: 2 + baseImage: kindest/base:v20191105-ee880e9b # has containerd + image: kindest/node:test + clusterName: kinder-regular + kubeadmVerbosity: 6 +tasks: +- name: pull-base-image + description: | + pulls kindest/base image with docker in docker and all the prerequisites necessary for running kind(er) + cmd: docker + args: + - pull + - "{{ .vars.baseImage }}" +- name: add-kubernetes-versions + description: | + creates a node-image-variant by adding a Kubernetes version + cmd: kinder + args: + - build + - node-image-variant + - --base-image={{ .vars.baseImage }} + - --image={{ .vars.image }} + - --with-init-artifacts={{ .vars.kubernetesVersion }} + - --loglevel=debug + timeout: 15m +- name: create-cluster + description: | + create a set of nodes ready for hosting the Kubernetes cluster + cmd: kinder + args: + - create + - cluster + - --name={{ .vars.clusterName }} + - --image={{ .vars.image }} + - --control-plane-nodes={{ .vars.controlPlaneNodes }} + - --worker-nodes={{ .vars.workerNodes }} + - --loglevel=debug + timeout: 5m +- name: prepare verify-rootless.sh script + cmd: /bin/sh + args: + - -c + - | + cat </tmp/verify-rootless.sh + #!/usr/bin/env bash + res=0 + users=("kubeadm-kas" "kubeadm-ks" "kubeadm-kcm" "kubeadm-etcd") + for d in ${users[@]}; do + if grep -q "\$d" /etc/passwd ; then + echo "/etc/passwd has user \$d!" + else + echo "ERROR: /etc/passwd does not have user \$d" + res=1 + fi + done + + groups=("kubeadm-kas" "kubeadm-ks" "kubeadm-kcm" "kubeadm-etcd" kubeadm-sa-key-readers) + for d in ${groups[@]}; do + if grep -q "\$d" /etc/group ; then + echo "/etc/group has user \$d!" + else + echo "ERROR: /etc/group does not have user \$d" + res=1 + fi + done + + if pgrep kube-apiserver | xargs ps o user:16 --no-headers -p | grep -q kubeadm-kas ; then + echo "kube-apiserver is running as user kubeadm-kas" + else + echo "ERROR: kube-apiserver is not running as user kubeadm-kas" + res=1 + fi + + if pgrep kube-apiserver | xargs ps o group:16 --no-headers -p | grep -q kubeadm-kas ; then + echo "kube-apiserver is running as user kubeadm-kas" + else + echo "ERROR: kube-apiserver is not running as user kubeadm-kas" + res=1 + fi + + if pgrep kube-apiserver | xargs ps o supgrp:16 --no-headers -p | grep -q kubeadm-sa-key-readers ; then + echo "kube-apiserver is running as supplemental group kubeadm-sa-key-readers" + else + echo "ERROR: kube-apiserver is not running as supplemental group kubeadm-sa-key-readers" + res=1 + fi + + if pgrep kube-controller-manager | xargs ps o user:16 --no-headers -p | grep -q kubeadm-kcm ; then + echo "kube-controller-manager is running as user kubeadm-kcm" + else + echo "ERROR: kube-controller-manager is not running as user kubeadm-kcm" + res=1 + fi + + if pgrep kube-controller-manager | xargs ps o group:16 --no-headers -p | grep -q kubeadm-kcm ; then + echo "kube-controller-manager is running as user kubeadm-kcm" + else + echo "ERROR: kube-controller-manager is not running as user kubeadm-kcm" + res=1 + fi + + if pgrep kube-controller-manager | xargs ps o supgrp:16 --no-headers -p | grep -q kubeadm-sa-key-readers ; then + echo "kube-controller-manager is running as supplemental group kubeadm-sa-key-readers" + else + echo "ERROR: kube-controller-manager is not running as supplemental group kubeadm-sa-key-readers" + res=1 + fi + + if pgrep kube-scheduler | xargs ps o user:16 --no-headers -p | grep -q kubeadm-ks ; then + echo "kube-scheduler is running as user kubeadm-ks" + else + echo "ERROR: kube-scheduler is not running as user kubeadm-ks" + res=1 + fi + + if pgrep kube-scheduler | xargs ps o group:16 --no-headers -p | grep -q kubeadm-ks ; then + echo "kube-scheduler is running as user kubeadm-ks" + else + echo "ERROR: kube-scheduler is not running as user kubeadm-ks" + res=1 + fi + + if pgrep etcd | xargs ps o user:16 --no-headers -p | grep -q kubeadm-etcd ; then + echo "etcd is running as user kubeadm-etcd" + else + echo "ERROR: etcd is not running as user kubeadm-etcd" + res=1 + fi + + if pgrep etcd | xargs ps o group:16 --no-headers -p | grep -q kubeadm-etcd ; then + echo "etcd is running as user kubeadm-etcd" + else + echo "ERROR: etcd is not running as user kubeadm-etcd" + res=1 + fi + + if [[ "\${res}" = 0 ]]; then + echo "All verify checks passed, congrats!" + echo "" + else + echo "One or more verify checks failed! See output above..." + echo "" + exit 1 + fi + EOF + + chmod +x /tmp/verify-rootless.sh +- name: copy verify-rootless.sh on controlplane nodes + cmd: kinder + args: + - cp + - --name={{ .vars.clusterName }} + - /tmp/verify-rootless.sh + - "@cp*:/kinder/verify-rootless.sh" + - --loglevel=debug +- name: init + description: | + Initializes the Kubernetes cluster with version "initVersion" + by starting the boostrap control-plane nodes + cmd: kinder + args: + - do + - kubeadm-init + - --name={{ .vars.clusterName }} + - --loglevel=debug + - --kubeadm-verbosity={{ .vars.kubeadmVerbosity }} + - --kubeadm-feature-gate="RootlessControlPlane=true" + timeout: 5m +- name: join + description: | + Join the other nodes to the Kubernetes cluster + cmd: kinder + args: + - do + - kubeadm-join + - --name={{ .vars.clusterName }} + - --loglevel=debug + - --kubeadm-verbosity={{ .vars.kubeadmVerbosity }} + timeout: 10m +- name: run verify-rootless.sh on controlplane nodes before upgrades + cmd: kinder + args: + - exec + - --name={{ .vars.clusterName }} + - "@cp*" + - /kinder/verify-rootless.sh + - --loglevel=debug +- name: e2e-kubeadm + description: | + Runs kubeadm e2e tests + cmd: kinder + args: + - test + - e2e-kubeadm + - --test-flags=--report-dir={{ .env.ARTIFACTS }} --report-prefix=e2e-kubeadm + - --name={{ .vars.clusterName }} + - --loglevel=debug + timeout: 10m +- name: e2e + description: | + Runs Kubernetes e2e test (conformance) + cmd: kinder + args: + - test + - e2e + - --test-flags=--report-dir={{ .env.ARTIFACTS }} --report-prefix=e2e + - --parallel + - --name={{ .vars.clusterName }} + - --loglevel=debug + timeout: 35m +- name: upgrade + description: | + upgrades the cluster to Kubernetes "upgradeVersion" + cmd: kinder + args: + - do + - kubeadm-upgrade + - --upgrade-version={{ .vars.kubernetesVersion }} + - --name={{ .vars.clusterName }} + - --loglevel=debug + - --kubeadm-verbosity={{ .vars.kubeadmVerbosity }} + timeout: 15m +- name: run verify-rootless.sh on controlplane nodes before upgrades + cmd: kinder + args: + - exec + - --name={{ .vars.clusterName }} + - "@cp*" + - /kinder/verify-rootless.sh + - --loglevel=debug +- name: get-logs + description: | + Collects all the test logs + cmd: kinder + args: + - export + - logs + - --loglevel=debug + - --name={{ .vars.clusterName }} + - "{{ .env.ARTIFACTS }}" + force: true + timeout: 5m + # kind export log is know to be flaky, so we are temporary ignoring errors in order + # to make the test pass in case everything else passed + # see https://github.com/kubernetes-sigs/kind/issues/456 + ignoreError: true +- name: reset + description: | + Exec kubeadm reset + cmd: kinder + args: + - do + - kubeadm-reset + - --name={{ .vars.clusterName }} + - --loglevel=debug + - --kubeadm-verbosity={{ .vars.kubeadmVerbosity }} + force: true +- name: delete + description: | + Deletes the cluster + cmd: kinder + args: + - delete + - cluster + - --name={{ .vars.clusterName }} + - --loglevel=debug + force: true diff --git a/kinder/ci/tools/update-workflows/templates/workflows/rootless.yaml b/kinder/ci/tools/update-workflows/templates/workflows/rootless.yaml new file mode 100644 index 00000000..dfa343fc --- /dev/null +++ b/kinder/ci/tools/update-workflows/templates/workflows/rootless.yaml @@ -0,0 +1,9 @@ +version: 1 +summary: | + This workflow tests the proper functioning of the {{ .KubernetesVersion }} version of both kubeadm and Kubernetes + test grid > https://testgrid.k8s.io/sig-cluster-lifecycle-kubeadm#kubeadm-kinder-rootless{{ dashVer .KubernetesVersion }} + config > https://git.k8s.io/test-infra/config/jobs/kubernetes/sig-cluster-lifecycle/{{ .TargetFile }} +vars: + kubernetesVersion: "\{\{ resolve `ci/{{ ciLabelFor .KubernetesVersion }}` \}\}" +tasks: +- import: rootless-tasks.yaml diff --git a/kinder/ci/workflows/rootless-latest.yaml b/kinder/ci/workflows/rootless-latest.yaml new file mode 100644 index 00000000..ed361fdd --- /dev/null +++ b/kinder/ci/workflows/rootless-latest.yaml @@ -0,0 +1,10 @@ +# AUTOGENERATED by https://git.k8s.io/kubeadm/kinder/ci/tools/update-workflows +version: 1 +summary: | + This workflow tests the proper functioning of the latest version of both kubeadm and Kubernetes + test grid > https://testgrid.k8s.io/sig-cluster-lifecycle-kubeadm#kubeadm-kinder-rootlesslatest + config > https://git.k8s.io/test-infra/config/jobs/kubernetes/sig-cluster-lifecycle/kubeadm-kinder-rootless.yaml +vars: + kubernetesVersion: "{{ resolve `ci/latest` }}" +tasks: +- import: rootless-tasks.yaml diff --git a/kinder/ci/workflows/rootless-tasks.yaml b/kinder/ci/workflows/rootless-tasks.yaml new file mode 100644 index 00000000..9ed8d4e2 --- /dev/null +++ b/kinder/ci/workflows/rootless-tasks.yaml @@ -0,0 +1,278 @@ +# AUTOGENERATED by https://git.k8s.io/kubeadm/kinder/ci/tools/update-workflows +# IMPORTANT! this workflow is imported by regular-* workflows. +version: 1 +summary: | + This workflow implements a sequence of tasks used test the proper functioning + of kubeadm version X with Kubernetes version X. +vars: + # vars defines default values for variable used by tasks in this workflow; + # those values might be overridden when importing this files. + kubernetesVersion: v1.13.5 + controlPlaneNodes: 3 + workerNodes: 2 + baseImage: kindest/base:v20191105-ee880e9b # has containerd + image: kindest/node:test + clusterName: kinder-regular + kubeadmVerbosity: 6 +tasks: +- name: pull-base-image + description: | + pulls kindest/base image with docker in docker and all the prerequisites necessary for running kind(er) + cmd: docker + args: + - pull + - "{{ .vars.baseImage }}" +- name: add-kubernetes-versions + description: | + creates a node-image-variant by adding a Kubernetes version + cmd: kinder + args: + - build + - node-image-variant + - --base-image={{ .vars.baseImage }} + - --image={{ .vars.image }} + - --with-init-artifacts={{ .vars.kubernetesVersion }} + - --loglevel=debug + timeout: 15m +- name: create-cluster + description: | + create a set of nodes ready for hosting the Kubernetes cluster + cmd: kinder + args: + - create + - cluster + - --name={{ .vars.clusterName }} + - --image={{ .vars.image }} + - --control-plane-nodes={{ .vars.controlPlaneNodes }} + - --worker-nodes={{ .vars.workerNodes }} + - --loglevel=debug + timeout: 5m +- name: prepare verify-rootless.sh script + cmd: /bin/sh + args: + - -c + - | + cat </tmp/verify-rootless.sh + #!/usr/bin/env bash + res=0 + users=("kubeadm-kas" "kubeadm-ks" "kubeadm-kcm" "kubeadm-etcd") + for d in ${users[@]}; do + if grep -q "\$d" /etc/passwd ; then + echo "/etc/passwd has user \$d!" + else + echo "ERROR: /etc/passwd does not have user \$d" + res=1 + fi + done + + groups=("kubeadm-kas" "kubeadm-ks" "kubeadm-kcm" "kubeadm-etcd" kubeadm-sa-key-readers) + for d in ${groups[@]}; do + if grep -q "\$d" /etc/group ; then + echo "/etc/group has user \$d!" + else + echo "ERROR: /etc/group does not have user \$d" + res=1 + fi + done + + if pgrep kube-apiserver | xargs ps o user:16 --no-headers -p | grep -q kubeadm-kas ; then + echo "kube-apiserver is running as user kubeadm-kas" + else + echo "ERROR: kube-apiserver is not running as user kubeadm-kas" + res=1 + fi + + if pgrep kube-apiserver | xargs ps o group:16 --no-headers -p | grep -q kubeadm-kas ; then + echo "kube-apiserver is running as user kubeadm-kas" + else + echo "ERROR: kube-apiserver is not running as user kubeadm-kas" + res=1 + fi + + if pgrep kube-apiserver | xargs ps o supgrp:16 --no-headers -p | grep -q kubeadm-sa-key-readers ; then + echo "kube-apiserver is running as supplemental group kubeadm-sa-key-readers" + else + echo "ERROR: kube-apiserver is not running as supplemental group kubeadm-sa-key-readers" + res=1 + fi + + if pgrep kube-controller-manager | xargs ps o user:16 --no-headers -p | grep -q kubeadm-kcm ; then + echo "kube-controller-manager is running as user kubeadm-kcm" + else + echo "ERROR: kube-controller-manager is not running as user kubeadm-kcm" + res=1 + fi + + if pgrep kube-controller-manager | xargs ps o group:16 --no-headers -p | grep -q kubeadm-kcm ; then + echo "kube-controller-manager is running as user kubeadm-kcm" + else + echo "ERROR: kube-controller-manager is not running as user kubeadm-kcm" + res=1 + fi + + if pgrep kube-controller-manager | xargs ps o supgrp:16 --no-headers -p | grep -q kubeadm-sa-key-readers ; then + echo "kube-controller-manager is running as supplemental group kubeadm-sa-key-readers" + else + echo "ERROR: kube-controller-manager is not running as supplemental group kubeadm-sa-key-readers" + res=1 + fi + + if pgrep kube-scheduler | xargs ps o user:16 --no-headers -p | grep -q kubeadm-ks ; then + echo "kube-scheduler is running as user kubeadm-ks" + else + echo "ERROR: kube-scheduler is not running as user kubeadm-ks" + res=1 + fi + + if pgrep kube-scheduler | xargs ps o group:16 --no-headers -p | grep -q kubeadm-ks ; then + echo "kube-scheduler is running as user kubeadm-ks" + else + echo "ERROR: kube-scheduler is not running as user kubeadm-ks" + res=1 + fi + + if pgrep etcd | xargs ps o user:16 --no-headers -p | grep -q kubeadm-etcd ; then + echo "etcd is running as user kubeadm-etcd" + else + echo "ERROR: etcd is not running as user kubeadm-etcd" + res=1 + fi + + if pgrep etcd | xargs ps o group:16 --no-headers -p | grep -q kubeadm-etcd ; then + echo "etcd is running as user kubeadm-etcd" + else + echo "ERROR: etcd is not running as user kubeadm-etcd" + res=1 + fi + + if [[ "\${res}" = 0 ]]; then + echo "All verify checks passed, congrats!" + echo "" + else + echo "One or more verify checks failed! See output above..." + echo "" + exit 1 + fi + EOF + + chmod +x /tmp/verify-rootless.sh +- name: copy verify-rootless.sh on controlplane nodes + cmd: kinder + args: + - cp + - --name={{ .vars.clusterName }} + - /tmp/verify-rootless.sh + - "@cp*:/kinder/verify-rootless.sh" + - --loglevel=debug +- name: init + description: | + Initializes the Kubernetes cluster with version "initVersion" + by starting the boostrap control-plane nodes + cmd: kinder + args: + - do + - kubeadm-init + - --name={{ .vars.clusterName }} + - --loglevel=debug + - --kubeadm-verbosity={{ .vars.kubeadmVerbosity }} + - --kubeadm-feature-gate="RootlessControlPlane=true" + timeout: 5m +- name: join + description: | + Join the other nodes to the Kubernetes cluster + cmd: kinder + args: + - do + - kubeadm-join + - --name={{ .vars.clusterName }} + - --loglevel=debug + - --kubeadm-verbosity={{ .vars.kubeadmVerbosity }} + timeout: 10m +- name: run verify-rootless.sh on controlplane nodes before upgrades + cmd: kinder + args: + - exec + - --name={{ .vars.clusterName }} + - "@cp*" + - /kinder/verify-rootless.sh + - --loglevel=debug +- name: e2e-kubeadm + description: | + Runs kubeadm e2e tests + cmd: kinder + args: + - test + - e2e-kubeadm + - --test-flags=--report-dir={{ .env.ARTIFACTS }} --report-prefix=e2e-kubeadm + - --name={{ .vars.clusterName }} + - --loglevel=debug + timeout: 10m +- name: e2e + description: | + Runs Kubernetes e2e test (conformance) + cmd: kinder + args: + - test + - e2e + - --test-flags=--report-dir={{ .env.ARTIFACTS }} --report-prefix=e2e + - --parallel + - --name={{ .vars.clusterName }} + - --loglevel=debug + timeout: 35m +- name: upgrade + description: | + upgrades the cluster to Kubernetes "upgradeVersion" + cmd: kinder + args: + - do + - kubeadm-upgrade + - --upgrade-version={{ .vars.kubernetesVersion }} + - --name={{ .vars.clusterName }} + - --loglevel=debug + - --kubeadm-verbosity={{ .vars.kubeadmVerbosity }} + timeout: 15m +- name: run verify-rootless.sh on controlplane nodes before upgrades + cmd: kinder + args: + - exec + - --name={{ .vars.clusterName }} + - "@cp*" + - /kinder/verify-rootless.sh + - --loglevel=debug +- name: get-logs + description: | + Collects all the test logs + cmd: kinder + args: + - export + - logs + - --loglevel=debug + - --name={{ .vars.clusterName }} + - "{{ .env.ARTIFACTS }}" + force: true + timeout: 5m + # kind export log is know to be flaky, so we are temporary ignoring errors in order + # to make the test pass in case everything else passed + # see https://github.com/kubernetes-sigs/kind/issues/456 + ignoreError: true +- name: reset + description: | + Exec kubeadm reset + cmd: kinder + args: + - do + - kubeadm-reset + - --name={{ .vars.clusterName }} + - --loglevel=debug + - --kubeadm-verbosity={{ .vars.kubeadmVerbosity }} + force: true +- name: delete + description: | + Deletes the cluster + cmd: kinder + args: + - delete + - cluster + - --name={{ .vars.clusterName }} + - --loglevel=debug + force: true diff --git a/kinder/cmd/kinder/do/do.go b/kinder/cmd/kinder/do/do.go index b849d091..9da49a25 100644 --- a/kinder/cmd/kinder/do/do.go +++ b/kinder/cmd/kinder/do/do.go @@ -43,6 +43,7 @@ type flagpole struct { Wait time.Duration IgnorePreflightErrors string KubeadmConfigVersion string + FeatureGate string } // NewCommand returns a new cobra.Command for exec @@ -122,6 +123,11 @@ func NewCommand() *cobra.Command { "If not set, kubeadm will automatically choose the kubeadm config version "+ "according to the Kubernetes version in use", ) + cmd.Flags().StringVar( + &flags.FeatureGate, + "kubeadm-feature-gate", "", + "the kubeadm feature-gate to be used for init, join and upgrade", + ) return cmd } @@ -175,6 +181,7 @@ func runE(flags *flagpole, cmd *cobra.Command, args []string) (err error) { actions.PatchesDir(flags.PatchesDir), actions.IgnorePreflightErrors(flags.IgnorePreflightErrors), actions.KubeadmConfigVersion(flags.KubeadmConfigVersion), + actions.FeatureGate(flags.FeatureGate), ) if err != nil { return errors.Wrapf(err, "failed to exec action %s", action) diff --git a/kinder/pkg/cluster/manager/actions/actions.go b/kinder/pkg/cluster/manager/actions/actions.go index 6238d897..a3cec02b 100644 --- a/kinder/pkg/cluster/manager/actions/actions.go +++ b/kinder/pkg/cluster/manager/actions/actions.go @@ -39,10 +39,10 @@ var actionRegistry = map[string]func(*status.Cluster, *RunOptions) error{ "kubeadm-config": func(c *status.Cluster, flags *RunOptions) error { // Nb. this action is invoked automatically at kubeadm init/join time, but it is possible // to invoke it separately as well - return KubeadmConfig(c, flags.kubeadmConfigVersion, flags.copyCertsMode, flags.discoveryMode, c.K8sNodes().EligibleForActions()...) + return KubeadmConfig(c, flags.kubeadmConfigVersion, flags.copyCertsMode, flags.discoveryMode, flags.featureGate, c.K8sNodes().EligibleForActions()...) }, "kubeadm-init": func(c *status.Cluster, flags *RunOptions) error { - return KubeadmInit(c, flags.usePhases, flags.copyCertsMode, flags.kubeadmConfigVersion, flags.patchesDir, flags.ignorePreflightErrors, flags.wait, flags.vLevel) + return KubeadmInit(c, flags.usePhases, flags.copyCertsMode, flags.kubeadmConfigVersion, flags.patchesDir, flags.ignorePreflightErrors, flags.featureGate, flags.wait, flags.vLevel) }, "kubeadm-join": func(c *status.Cluster, flags *RunOptions) error { return KubeadmJoin(c, flags.usePhases, flags.copyCertsMode, flags.discoveryMode, flags.kubeadmConfigVersion, flags.patchesDir, flags.ignorePreflightErrors, flags.wait, flags.vLevel) @@ -145,6 +145,13 @@ func KubeadmConfigVersion(kubeadmConfigVersion string) Option { } } +// FeatureGate option sets the kubeadm feature-gates for the kubeadm commands +func FeatureGate(featureGate string) Option { + return func(r *RunOptions) { + r.featureGate = featureGate + } +} + // RunOptions holds options supplied to actions.Run type RunOptions struct { usePhases bool @@ -156,6 +163,7 @@ type RunOptions struct { patchesDir string ignorePreflightErrors string kubeadmConfigVersion string + featureGate string } // DiscoveryMode defines discovery mode supported by kubeadm join diff --git a/kinder/pkg/cluster/manager/actions/kubeadm-config.go b/kinder/pkg/cluster/manager/actions/kubeadm-config.go index b72abda0..eef9b74a 100644 --- a/kinder/pkg/cluster/manager/actions/kubeadm-config.go +++ b/kinder/pkg/cluster/manager/actions/kubeadm-config.go @@ -40,9 +40,9 @@ type kubeadmConfigOptions struct { // KubeadmInitConfig action writes the InitConfiguration into /kind/kubeadm.conf file on all the K8s nodes in the cluster. // Please note that this action is automatically executed at create time, but it is possible // to invoke it separately as well. -func KubeadmInitConfig(c *status.Cluster, kubeadmConfigVersion string, copyCertsMode CopyCertsMode, nodes ...*status.Node) error { +func KubeadmInitConfig(c *status.Cluster, kubeadmConfigVersion string, copyCertsMode CopyCertsMode, featureGate string, nodes ...*status.Node) error { // defaults everything not relevant for the Init Config - return KubeadmConfig(c, kubeadmConfigVersion, copyCertsMode, TokenDiscovery, nodes...) + return KubeadmConfig(c, kubeadmConfigVersion, copyCertsMode, TokenDiscovery, featureGate, nodes...) } // KubeadmJoinConfig action writes the JoinConfiguration into /kind/kubeadm.conf file on all the K8s nodes in the cluster. @@ -50,13 +50,13 @@ func KubeadmInitConfig(c *status.Cluster, kubeadmConfigVersion string, copyCerts // to invoke it separately as well. func KubeadmJoinConfig(c *status.Cluster, kubeadmConfigVersion string, copyCertsMode CopyCertsMode, discoveryMode DiscoveryMode, nodes ...*status.Node) error { // defaults everything not relevant for the join Config - return KubeadmConfig(c, kubeadmConfigVersion, copyCertsMode, discoveryMode, nodes...) + return KubeadmConfig(c, kubeadmConfigVersion, copyCertsMode, discoveryMode, "" /* feature-gates */, nodes...) } // KubeadmConfig action writes the /kind/kubeadm.conf file on all the K8s nodes in the cluster. // Please note that this action is automatically executed at create time, but it is possible // to invoke it separately as well. -func KubeadmConfig(c *status.Cluster, kubeadmConfigVersion string, copyCertsMode CopyCertsMode, discoveryMode DiscoveryMode, nodes ...*status.Node) error { +func KubeadmConfig(c *status.Cluster, kubeadmConfigVersion string, copyCertsMode CopyCertsMode, discoveryMode DiscoveryMode, featureGate string, nodes ...*status.Node) error { cp1 := c.BootstrapControlPlane() // get installed kubernetes version from the node image @@ -84,6 +84,13 @@ func KubeadmConfig(c *status.Cluster, kubeadmConfigVersion string, copyCertsMode controlPlaneEndpoint = controlPlaneEndpointIPv6 } + featureGateName := "" + featureGateValue := "" + if featureGate != "" { + featureGateName = strings.Split(featureGate, "=")[0] + featureGateValue = strings.Split(featureGate, "=")[1] + } + // create configData with all the configurations supported by the kubeadm config template implemented in kind configData := kubeadm.ConfigData{ ClusterName: c.Name(), @@ -95,6 +102,8 @@ func KubeadmConfig(c *status.Cluster, kubeadmConfigVersion string, copyCertsMode PodSubnet: "192.168.0.0/16", // default for kindnet ControlPlane: true, IPv6: c.Settings.IPFamily == status.IPv6Family, + FeatureGateName: featureGateName, + FeatureGateValue: featureGateValue, } // create configOptions with all the kinder flags that impact on the kubeadm config generation diff --git a/kinder/pkg/cluster/manager/actions/kubeadm-init.go b/kinder/pkg/cluster/manager/actions/kubeadm-init.go index 12e8ed20..917be49f 100644 --- a/kinder/pkg/cluster/manager/actions/kubeadm-init.go +++ b/kinder/pkg/cluster/manager/actions/kubeadm-init.go @@ -37,7 +37,7 @@ import ( // KubeadmInit executes the kubeadm init workflow including also post init task // like installing the CNI network plugin -func KubeadmInit(c *status.Cluster, usePhases bool, copyCertsMode CopyCertsMode, kubeadmConfigVersion, patchesDir, ignorePreflightErrors string, wait time.Duration, vLevel int) (err error) { +func KubeadmInit(c *status.Cluster, usePhases bool, copyCertsMode CopyCertsMode, kubeadmConfigVersion, patchesDir, ignorePreflightErrors, featureGates string, wait time.Duration, vLevel int) (err error) { cp1 := c.BootstrapControlPlane() // if patcheDir is defined, copy the patches to the node @@ -61,7 +61,7 @@ func KubeadmInit(c *status.Cluster, usePhases bool, copyCertsMode CopyCertsMode, } // prepares the kubeadm config on this node - if err := KubeadmInitConfig(c, kubeadmConfigVersion, copyCertsMode, cp1); err != nil { + if err := KubeadmInitConfig(c, kubeadmConfigVersion, copyCertsMode, featureGates, cp1); err != nil { return err } diff --git a/kinder/pkg/kubeadm/config.go b/kinder/pkg/kubeadm/config.go index 8537dcc9..b67846c8 100644 --- a/kinder/pkg/kubeadm/config.go +++ b/kinder/pkg/kubeadm/config.go @@ -95,6 +95,9 @@ type ConfigData struct { ServiceSubnet string // IPv4 values take precedence over IPv6 by default, if true set IPv6 default values IPv6 bool + // The kubeadm feature-gate + FeatureGateName string + FeatureGateValue string // DerivedConfigData is populated by Derive() // These auto-generated fields are available to Config templates, // but not meant to be set by hand @@ -149,6 +152,10 @@ scheduler: networking: podSubnet: "{{ .PodSubnet }}" serviceSubnet: "{{ .ServiceSubnet }}" +{{ if .FeatureGateName -}} +featureGates: + {{ .FeatureGateName }}: {{ .FeatureGateValue }} +{{- end }} --- apiVersion: kubeadm.k8s.io/v1beta2 kind: InitConfiguration @@ -248,6 +255,10 @@ scheduler: networking: podSubnet: "{{ .PodSubnet }}" serviceSubnet: "{{ .ServiceSubnet }}" +{{ if .FeatureGates -}} +featureGates: + {{ .FeatureGates }} + {{- end }} --- apiVersion: kubeadm.k8s.io/v1beta3 kind: InitConfiguration