Merge pull request kubernetes-sigs#1157 from BenTheElder/storage

ship a better storage solution

pkg/apis/config/defaults/image.go

@@ -18,4 +18,4 @@ limitations under the License.
 package defaults
 
 // Image is the default for the Config.Image field, aka the default node image.
-const Image = "kindest/node:v1.16.3@sha256:70ce6ce09bee5c34ab14aec2b84d6edb260473a60638b1b095470a3a0f95ebec"
+const Image = "kindest/node:v1.17.0@sha256:c1ccbb2d2a5358a7dfba838537baeed84c2f6c0c69ae8a15df8a4915b4dc9a14"

pkg/build/node/cni.go

@@ -16,13 +16,6 @@ limitations under the License.
 
 package node
 
-// these are well known paths within the node image
-const (
-	// TODO: refactor kubernetesVersionLocation to a common internal package
-	kubernetesVersionLocation  = "/kind/version"
-	defaultCNIManifestLocation = "/kind/manifests/default-cni.yaml"
-)
-
 /*
 The default CNI manifest and images are our own tiny kindnet
 */

pkg/build/node/const.go

@@ -0,0 +1,25 @@
+/*
+Copyright 2019 The Kubernetes Authors.
+
+Licensed under the Apache License, Version 2.0 (the "License");
+you may not use this file except in compliance with the License.
+You may obtain a copy of the License at
+
+    http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+See the License for the specific language governing permissions and
+limitations under the License.
+*/
+
+package node
+
+// these are well known paths within the node image
+const (
+	// TODO: refactor kubernetesVersionLocation to a common internal package
+	kubernetesVersionLocation      = "/kind/version"
+	defaultCNIManifestLocation     = "/kind/manifests/default-cni.yaml"
+	defaultStorageManifestLocation = "/kind/manifests/default-storage.yaml"
+)

pkg/build/node/node.go

@@ -375,10 +375,6 @@ func (c *BuildContext) prePullImages(dir, containerID string) error {
 
 	// helpers to run things in the build container
 	cmder := docker.ContainerCmder(containerID)
-	inheritOutputAndRun := func(cmd exec.Cmd) error {
-		exec.InheritOutput(cmd)
-		return cmd.Run()
-	}
 
 	// get the Kubernetes version we installed on the node
 	// we need this to ask kubeadm what images we need
@@ -396,7 +392,7 @@ func (c *BuildContext) prePullImages(dir, containerID string) error {
 	// later releases use manifest list images
 	// at node boot time we retag our images to handle this where necessary,
 	// so we virtually re-tag them here.
-	ver, err := version.ParseGeneric(rawVersion[0])
+	ver, err := version.ParseSemantic(rawVersion[0])
 	if err != nil {
 		return err
 	}
@@ -418,25 +414,6 @@ func (c *BuildContext) prePullImages(dir, containerID string) error {
 	builtImages = fixedImages
 	c.logger.V(0).Info("Detected built images: " + strings.Join(builtImages.List(), ", "))
 
-	// write the default CNI manifest
-	// NOTE: the paths inside the container should use the path package
-	// and not filepath (!), we want posixy paths in the linux container, NOT
-	// whatever path format the host uses. For paths on the host we use filepath
-	if err := inheritOutputAndRun(cmder.Command(
-		"mkdir", "-p", path.Dir(defaultCNIManifestLocation),
-	)); err != nil {
-		c.logger.Errorf("Image build Failed! Failed write default CNI Manifest: %v", err)
-		return err
-	}
-	if err := cmder.Command(
-		"cp", "/dev/stdin", defaultCNIManifestLocation,
-	).SetStdin(
-		strings.NewReader(defaultCNIManifest),
-	).Run(); err != nil {
-		c.logger.Errorf("Image build Failed! Failed write default CNI Manifest: %v", err)
-		return err
-	}
-
 	// gets the list of images required by kubeadm
 	requiredImages, err := exec.OutputLines(cmder.Command(
 		"kubeadm", "config", "images", "list", "--kubernetes-version", rawVersion[0],
@@ -445,9 +422,32 @@ func (c *BuildContext) prePullImages(dir, containerID string) error {
 		return err
 	}
 
-	// all builds should isntall the default CNI images currently
+	// write the default CNI manifest
+	if err := writeManifest(cmder, defaultCNIManifestLocation, defaultCNIManifest); err != nil {
+		c.logger.Errorf("Image build Failed! Failed write default CNI Manifest: %v", err)
+		return err
+	}
+	// all builds should install the default CNI images from the above manifest currently
 	requiredImages = append(requiredImages, defaultCNIImages...)
 
+	// for v1.12.0+ we support a nicer storage driver
+	if ver.LessThan(version.MustParseSemantic("v1.12.0")) {
+		// otherwise, we must use something built in and simpler, which is
+		// also the same as what kind previously used...
+		if err := writeManifest(cmder, legacyDefaultStorage, defaultStorageManifest); err != nil {
+			c.logger.Errorf("Image build Failed! Failed write default Storage Manifest: %v", err)
+			return err
+		}
+	} else {
+		// write the default Storage manifest
+		if err := writeManifest(cmder, defaultStorageManifestLocation, defaultStorageManifest); err != nil {
+			c.logger.Errorf("Image build Failed! Failed write default Storage Manifest: %v", err)
+			return err
+		}
+		// all builds should install the default storage driver images currently
+		requiredImages = append(requiredImages, defaultStorageImages...)
+	}
+
 	// Create "images" subdir.
 	imagesDir := path.Join(dir, "bits", "images")
 	if err := os.MkdirAll(imagesDir, 0777); err != nil {
@@ -542,6 +542,23 @@ func (c *BuildContext) prePullImages(dir, containerID string) error {
 	return nil
 }
 
+func writeManifest(cmder exec.Cmder, manifestPath, manifestContents string) error {
+	// NOTE: the paths inside the container should use the path package
+	// and not filepath (!), we want posixy paths in the linux container, NOT
+	// whatever path format the host uses. For paths on the host we use filepath
+	cmdMkdir := cmder.Command("mkdir", "-p", path.Dir(manifestPath))
+	exec.InheritOutput(cmdMkdir)
+	if err := cmdMkdir.Run(); err != nil {
+		return err
+	}
+
+	return cmder.Command(
+		"cp", "/dev/stdin", manifestPath,
+	).SetStdin(
+		strings.NewReader(manifestContents),
+	).Run()
+}
+
 func repositoryCorrectorForVersion(kubeVersion *version.Version, arch string) func(string) string {
 	archSuffix := "-" + arch
 

pkg/build/node/storage.go

@@ -0,0 +1,160 @@
+/*
+Copyright 2019 The Kubernetes Authors.
+
+Licensed under the Apache License, Version 2.0 (the "License");
+you may not use this file except in compliance with the License.
+You may obtain a copy of the License at
+
+    http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+See the License for the specific language governing permissions and
+limitations under the License.
+*/
+
+package node
+
+/*
+The default PV driver manifest and images are provisionally rancher.io/local-path-provisioner
+NOTE: we have customized it in the following ways:
+- storage is under /var instead of /opt
+- debian-base is used as the helper image (k8s already ships this upstream as the base for many images) instead of busybox
+- schedule to "master" kubeadm nodes (control-plane host)
+- install as the default storage class
+*/
+
+var defaultStorageImages = []string{"rancher/local-path-provisioner:v0.0.11", "k8s.gcr.io/debian-base:v2.0.0"}
+
+const defaultStorageManifest = `
+# kind customized https://github.com/rancher/local-path-provisioner manifest
+apiVersion: v1
+kind: Namespace
+metadata:
+  name: local-path-storage
+---
+apiVersion: v1
+kind: ServiceAccount
+metadata:
+  name: local-path-provisioner-service-account
+  namespace: local-path-storage
+---
+apiVersion: rbac.authorization.k8s.io/v1
+kind: ClusterRole
+metadata:
+  name: local-path-provisioner-role
+rules:
+- apiGroups: [""]
+  resources: ["nodes", "persistentvolumeclaims"]
+  verbs: ["get", "list", "watch"]
+- apiGroups: [""]
+  resources: ["endpoints", "persistentvolumes", "pods"]
+  verbs: ["*"]
+- apiGroups: [""]
+  resources: ["events"]
+  verbs: ["create", "patch"]
+- apiGroups: ["storage.k8s.io"]
+  resources: ["storageclasses"]
+  verbs: ["get", "list", "watch"]
+---
+apiVersion: rbac.authorization.k8s.io/v1
+kind: ClusterRoleBinding
+metadata:
+  name: local-path-provisioner-bind
+roleRef:
+  apiGroup: rbac.authorization.k8s.io
+  kind: ClusterRole
+  name: local-path-provisioner-role
+subjects:
+- kind: ServiceAccount
+  name: local-path-provisioner-service-account
+  namespace: local-path-storage
+---
+apiVersion: apps/v1
+kind: Deployment
+metadata:
+  name: local-path-provisioner
+  namespace: local-path-storage
+spec:
+  replicas: 1
+  selector:
+    matchLabels:
+      app: local-path-provisioner
+  template:
+    metadata:
+      labels:
+        app: local-path-provisioner
+    spec:
+      nodeSelector:
+        node-role.kubernetes.io/master: ''
+      tolerations:
+      - key: node-role.kubernetes.io/master
+        operator: Equal
+        effect: NoSchedule
+      serviceAccountName: local-path-provisioner-service-account
+      containers:
+      - name: local-path-provisioner
+        image: rancher/local-path-provisioner:v0.0.11
+        imagePullPolicy: IfNotPresent
+        command:
+        - local-path-provisioner
+        - --debug
+        - start
+        - --helper-image
+        - k8s.gcr.io/debian-base:v2.0.0
+        - --config
+        - /etc/config/config.json
+        volumeMounts:
+        - name: config-volume
+          mountPath: /etc/config/
+        env:
+        - name: POD_NAMESPACE
+          valueFrom:
+            fieldRef:
+              fieldPath: metadata.namespace
+      volumes:
+        - name: config-volume
+          configMap:
+            name: local-path-config
+---
+apiVersion: storage.k8s.io/v1
+kind: StorageClass
+metadata:
+  namespace: kube-system
+  name: standard
+  annotations:
+    storageclass.kubernetes.io/is-default-class: "true"
+provisioner: rancher.io/local-path
+volumeBindingMode: WaitForFirstConsumer
+reclaimPolicy: Delete
+---
+kind: ConfigMap
+apiVersion: v1
+metadata:
+  name: local-path-config
+  namespace: local-path-storage
+data:
+  config.json: |-
+        {
+                "nodePathMap":[
+                {
+                        "node":"DEFAULT_PATH_FOR_NON_LISTED_NODES",
+                        "paths":["/var/local-path-provisioner"]
+                }
+                ]
+        }
+`
+
+// legacy default storage class for older than Kubernetes v1.12.0
+// we need this for e2es (StatefulSet)
+// newer kind images ship a storage driver manifest
+const legacyDefaultStorage = `# host-path based default storage class
+apiVersion: storage.k8s.io/v1
+kind: StorageClass
+metadata:
+  namespace: kube-system
+  name: standard
+  annotations:
+    storageclass.kubernetes.io/is-default-class: "true"
+provisioner: kubernetes.io/host-path`

pkg/cluster/internal/create/actions/installstorage/storage.go

@@ -19,10 +19,12 @@ limitations under the License.
 package installstorage
 
 import (
+	"bytes"
 	"strings"
 
 	"sigs.k8s.io/kind/pkg/cluster/nodes"
 	"sigs.k8s.io/kind/pkg/errors"
+	"sigs.k8s.io/kind/pkg/log"
 
 	"sigs.k8s.io/kind/pkg/cluster/internal/create/actions"
 	"sigs.k8s.io/kind/pkg/cluster/nodeutils"
@@ -53,7 +55,7 @@ func (a *action) Execute(ctx *actions.ActionContext) error {
 	node := controlPlanes[0] // kind expects at least one always
 
 	// add the default storage class
-	if err := addDefaultStorageClass(node); err != nil {
+	if err := addDefaultStorage(ctx.Logger, node); err != nil {
 		return errors.Wrap(err, "failed to add default storage class")
 	}
 
@@ -62,9 +64,10 @@ func (a *action) Execute(ctx *actions.ActionContext) error {
 	return nil
 }
 
-// a default storage class
+// legacy default storage class
 // we need this for e2es (StatefulSet)
-const defaultStorageClassManifest = `# host-path based default storage class
+// newer kind images ship a storage driver manifest
+const defaultStorageManifest = `# host-path based default storage class
 apiVersion: storage.k8s.io/v1
 kind: StorageClass
 metadata:
@@ -74,8 +77,19 @@ metadata:
     storageclass.kubernetes.io/is-default-class: "true"
 provisioner: kubernetes.io/host-path`
 
-func addDefaultStorageClass(controlPlane nodes.Node) error {
-	in := strings.NewReader(defaultStorageClassManifest)
+func addDefaultStorage(logger log.Logger, controlPlane nodes.Node) error {
+	// start with fallback default, and then try to get the newer kind node
+	// storage manifest if present
+	manifest := defaultStorageManifest
+	var raw bytes.Buffer
+	if err := controlPlane.Command("cat", "/kind/manifests/default-storage.yaml").SetStdout(&raw).Run(); err != nil {
+		logger.Warn("Could not read storage manifest, falling back on old k8s.io/host-path default ...")
+	} else {
+		manifest = raw.String()
+	}
+
+	// apply the manifest
+	in := strings.NewReader(manifest)
 	cmd := controlPlane.Command(
 		"kubectl",
 		"--kubeconfig=/etc/kubernetes/admin.conf", "apply", "-f", "-",