diff --git a/pkg/apis/config/defaults/image.go b/pkg/apis/config/defaults/image.go index 5994c2cf9d..373e2e7870 100644 --- a/pkg/apis/config/defaults/image.go +++ b/pkg/apis/config/defaults/image.go @@ -18,4 +18,4 @@ limitations under the License. package defaults // Image is the default for the Config.Image field, aka the default node image. -const Image = "kindest/node:v1.16.3@sha256:70ce6ce09bee5c34ab14aec2b84d6edb260473a60638b1b095470a3a0f95ebec" +const Image = "kindest/node:v1.17.0@sha256:c1ccbb2d2a5358a7dfba838537baeed84c2f6c0c69ae8a15df8a4915b4dc9a14" diff --git a/pkg/build/node/cni.go b/pkg/build/node/cni.go index f0fba6284d..2ac304f731 100644 --- a/pkg/build/node/cni.go +++ b/pkg/build/node/cni.go @@ -16,13 +16,6 @@ limitations under the License. package node -// these are well known paths within the node image -const ( - // TODO: refactor kubernetesVersionLocation to a common internal package - kubernetesVersionLocation = "/kind/version" - defaultCNIManifestLocation = "/kind/manifests/default-cni.yaml" -) - /* The default CNI manifest and images are our own tiny kindnet */ diff --git a/pkg/build/node/const.go b/pkg/build/node/const.go new file mode 100644 index 0000000000..2ade6cd05e --- /dev/null +++ b/pkg/build/node/const.go @@ -0,0 +1,25 @@ +/* +Copyright 2019 The Kubernetes Authors. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package node + +// these are well known paths within the node image +const ( + // TODO: refactor kubernetesVersionLocation to a common internal package + kubernetesVersionLocation = "/kind/version" + defaultCNIManifestLocation = "/kind/manifests/default-cni.yaml" + defaultStorageManifestLocation = "/kind/manifests/default-storage.yaml" +) diff --git a/pkg/build/node/node.go b/pkg/build/node/node.go index 4ed3384e9c..cf6af87540 100644 --- a/pkg/build/node/node.go +++ b/pkg/build/node/node.go @@ -375,10 +375,6 @@ func (c *BuildContext) prePullImages(dir, containerID string) error { // helpers to run things in the build container cmder := docker.ContainerCmder(containerID) - inheritOutputAndRun := func(cmd exec.Cmd) error { - exec.InheritOutput(cmd) - return cmd.Run() - } // get the Kubernetes version we installed on the node // we need this to ask kubeadm what images we need @@ -396,7 +392,7 @@ func (c *BuildContext) prePullImages(dir, containerID string) error { // later releases use manifest list images // at node boot time we retag our images to handle this where necessary, // so we virtually re-tag them here. - ver, err := version.ParseGeneric(rawVersion[0]) + ver, err := version.ParseSemantic(rawVersion[0]) if err != nil { return err } @@ -418,25 +414,6 @@ func (c *BuildContext) prePullImages(dir, containerID string) error { builtImages = fixedImages c.logger.V(0).Info("Detected built images: " + strings.Join(builtImages.List(), ", ")) - // write the default CNI manifest - // NOTE: the paths inside the container should use the path package - // and not filepath (!), we want posixy paths in the linux container, NOT - // whatever path format the host uses. For paths on the host we use filepath - if err := inheritOutputAndRun(cmder.Command( - "mkdir", "-p", path.Dir(defaultCNIManifestLocation), - )); err != nil { - c.logger.Errorf("Image build Failed! Failed write default CNI Manifest: %v", err) - return err - } - if err := cmder.Command( - "cp", "/dev/stdin", defaultCNIManifestLocation, - ).SetStdin( - strings.NewReader(defaultCNIManifest), - ).Run(); err != nil { - c.logger.Errorf("Image build Failed! Failed write default CNI Manifest: %v", err) - return err - } - // gets the list of images required by kubeadm requiredImages, err := exec.OutputLines(cmder.Command( "kubeadm", "config", "images", "list", "--kubernetes-version", rawVersion[0], @@ -445,9 +422,32 @@ func (c *BuildContext) prePullImages(dir, containerID string) error { return err } - // all builds should isntall the default CNI images currently + // write the default CNI manifest + if err := writeManifest(cmder, defaultCNIManifestLocation, defaultCNIManifest); err != nil { + c.logger.Errorf("Image build Failed! Failed write default CNI Manifest: %v", err) + return err + } + // all builds should install the default CNI images from the above manifest currently requiredImages = append(requiredImages, defaultCNIImages...) + // for v1.12.0+ we support a nicer storage driver + if ver.LessThan(version.MustParseSemantic("v1.12.0")) { + // otherwise, we must use something built in and simpler, which is + // also the same as what kind previously used... + if err := writeManifest(cmder, legacyDefaultStorage, defaultStorageManifest); err != nil { + c.logger.Errorf("Image build Failed! Failed write default Storage Manifest: %v", err) + return err + } + } else { + // write the default Storage manifest + if err := writeManifest(cmder, defaultStorageManifestLocation, defaultStorageManifest); err != nil { + c.logger.Errorf("Image build Failed! Failed write default Storage Manifest: %v", err) + return err + } + // all builds should install the default storage driver images currently + requiredImages = append(requiredImages, defaultStorageImages...) + } + // Create "images" subdir. imagesDir := path.Join(dir, "bits", "images") if err := os.MkdirAll(imagesDir, 0777); err != nil { @@ -542,6 +542,23 @@ func (c *BuildContext) prePullImages(dir, containerID string) error { return nil } +func writeManifest(cmder exec.Cmder, manifestPath, manifestContents string) error { + // NOTE: the paths inside the container should use the path package + // and not filepath (!), we want posixy paths in the linux container, NOT + // whatever path format the host uses. For paths on the host we use filepath + cmdMkdir := cmder.Command("mkdir", "-p", path.Dir(manifestPath)) + exec.InheritOutput(cmdMkdir) + if err := cmdMkdir.Run(); err != nil { + return err + } + + return cmder.Command( + "cp", "/dev/stdin", manifestPath, + ).SetStdin( + strings.NewReader(manifestContents), + ).Run() +} + func repositoryCorrectorForVersion(kubeVersion *version.Version, arch string) func(string) string { archSuffix := "-" + arch diff --git a/pkg/build/node/storage.go b/pkg/build/node/storage.go new file mode 100644 index 0000000000..7ec790ef41 --- /dev/null +++ b/pkg/build/node/storage.go @@ -0,0 +1,160 @@ +/* +Copyright 2019 The Kubernetes Authors. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package node + +/* +The default PV driver manifest and images are provisionally rancher.io/local-path-provisioner +NOTE: we have customized it in the following ways: +- storage is under /var instead of /opt +- debian-base is used as the helper image (k8s already ships this upstream as the base for many images) instead of busybox +- schedule to "master" kubeadm nodes (control-plane host) +- install as the default storage class +*/ + +var defaultStorageImages = []string{"rancher/local-path-provisioner:v0.0.11", "k8s.gcr.io/debian-base:v2.0.0"} + +const defaultStorageManifest = ` +# kind customized https://github.com/rancher/local-path-provisioner manifest +apiVersion: v1 +kind: Namespace +metadata: + name: local-path-storage +--- +apiVersion: v1 +kind: ServiceAccount +metadata: + name: local-path-provisioner-service-account + namespace: local-path-storage +--- +apiVersion: rbac.authorization.k8s.io/v1 +kind: ClusterRole +metadata: + name: local-path-provisioner-role +rules: +- apiGroups: [""] + resources: ["nodes", "persistentvolumeclaims"] + verbs: ["get", "list", "watch"] +- apiGroups: [""] + resources: ["endpoints", "persistentvolumes", "pods"] + verbs: ["*"] +- apiGroups: [""] + resources: ["events"] + verbs: ["create", "patch"] +- apiGroups: ["storage.k8s.io"] + resources: ["storageclasses"] + verbs: ["get", "list", "watch"] +--- +apiVersion: rbac.authorization.k8s.io/v1 +kind: ClusterRoleBinding +metadata: + name: local-path-provisioner-bind +roleRef: + apiGroup: rbac.authorization.k8s.io + kind: ClusterRole + name: local-path-provisioner-role +subjects: +- kind: ServiceAccount + name: local-path-provisioner-service-account + namespace: local-path-storage +--- +apiVersion: apps/v1 +kind: Deployment +metadata: + name: local-path-provisioner + namespace: local-path-storage +spec: + replicas: 1 + selector: + matchLabels: + app: local-path-provisioner + template: + metadata: + labels: + app: local-path-provisioner + spec: + nodeSelector: + node-role.kubernetes.io/master: '' + tolerations: + - key: node-role.kubernetes.io/master + operator: Equal + effect: NoSchedule + serviceAccountName: local-path-provisioner-service-account + containers: + - name: local-path-provisioner + image: rancher/local-path-provisioner:v0.0.11 + imagePullPolicy: IfNotPresent + command: + - local-path-provisioner + - --debug + - start + - --helper-image + - k8s.gcr.io/debian-base:v2.0.0 + - --config + - /etc/config/config.json + volumeMounts: + - name: config-volume + mountPath: /etc/config/ + env: + - name: POD_NAMESPACE + valueFrom: + fieldRef: + fieldPath: metadata.namespace + volumes: + - name: config-volume + configMap: + name: local-path-config +--- +apiVersion: storage.k8s.io/v1 +kind: StorageClass +metadata: + namespace: kube-system + name: standard + annotations: + storageclass.kubernetes.io/is-default-class: "true" +provisioner: rancher.io/local-path +volumeBindingMode: WaitForFirstConsumer +reclaimPolicy: Delete +--- +kind: ConfigMap +apiVersion: v1 +metadata: + name: local-path-config + namespace: local-path-storage +data: + config.json: |- + { + "nodePathMap":[ + { + "node":"DEFAULT_PATH_FOR_NON_LISTED_NODES", + "paths":["/var/local-path-provisioner"] + } + ] + } +` + +// legacy default storage class for older than Kubernetes v1.12.0 +// we need this for e2es (StatefulSet) +// newer kind images ship a storage driver manifest +const legacyDefaultStorage = `# host-path based default storage class +apiVersion: storage.k8s.io/v1 +kind: StorageClass +metadata: + namespace: kube-system + name: standard + annotations: + storageclass.kubernetes.io/is-default-class: "true" +provisioner: kubernetes.io/host-path` diff --git a/pkg/cluster/internal/create/actions/installstorage/storage.go b/pkg/cluster/internal/create/actions/installstorage/storage.go index 7878a977bf..69db58109b 100644 --- a/pkg/cluster/internal/create/actions/installstorage/storage.go +++ b/pkg/cluster/internal/create/actions/installstorage/storage.go @@ -19,10 +19,12 @@ limitations under the License. package installstorage import ( + "bytes" "strings" "sigs.k8s.io/kind/pkg/cluster/nodes" "sigs.k8s.io/kind/pkg/errors" + "sigs.k8s.io/kind/pkg/log" "sigs.k8s.io/kind/pkg/cluster/internal/create/actions" "sigs.k8s.io/kind/pkg/cluster/nodeutils" @@ -53,7 +55,7 @@ func (a *action) Execute(ctx *actions.ActionContext) error { node := controlPlanes[0] // kind expects at least one always // add the default storage class - if err := addDefaultStorageClass(node); err != nil { + if err := addDefaultStorage(ctx.Logger, node); err != nil { return errors.Wrap(err, "failed to add default storage class") } @@ -62,9 +64,10 @@ func (a *action) Execute(ctx *actions.ActionContext) error { return nil } -// a default storage class +// legacy default storage class // we need this for e2es (StatefulSet) -const defaultStorageClassManifest = `# host-path based default storage class +// newer kind images ship a storage driver manifest +const defaultStorageManifest = `# host-path based default storage class apiVersion: storage.k8s.io/v1 kind: StorageClass metadata: @@ -74,8 +77,19 @@ metadata: storageclass.kubernetes.io/is-default-class: "true" provisioner: kubernetes.io/host-path` -func addDefaultStorageClass(controlPlane nodes.Node) error { - in := strings.NewReader(defaultStorageClassManifest) +func addDefaultStorage(logger log.Logger, controlPlane nodes.Node) error { + // start with fallback default, and then try to get the newer kind node + // storage manifest if present + manifest := defaultStorageManifest + var raw bytes.Buffer + if err := controlPlane.Command("cat", "/kind/manifests/default-storage.yaml").SetStdout(&raw).Run(); err != nil { + logger.Warn("Could not read storage manifest, falling back on old k8s.io/host-path default ...") + } else { + manifest = raw.String() + } + + // apply the manifest + in := strings.NewReader(manifest) cmd := controlPlane.Command( "kubectl", "--kubeconfig=/etc/kubernetes/admin.conf", "apply", "-f", "-",