Improve Performance

hack/verify-generated.sh

@@ -49,8 +49,8 @@ make_temp_root() {
   export -f quiet_recursive_cp
   # we need to copy everything but _output (which is .gitignore anyhow)
   find . \
-    -type d -path "./_output" -prune -o \
     -mindepth 1 -maxdepth 1 \
+    -type d -path "./_output" -prune -o \
     -exec bash -c 'quiet_recursive_cp "${0}" "${1}/${0}"' {} "${fake_root}" \;
 }
 

images/base/Dockerfile

@@ -25,12 +25,7 @@
 # https://developers.redhat.com/blog/2014/05/05/running-systemd-within-docker-container/
 # https://developers.redhat.com/blog/2016/09/13/running-systemd-in-a-non-privileged-container/
 
-# TODO(bentheelder): explore supporting other arches, IE support the ARCH arg.
-# For now this is still an ARG so we can reuse it throughout the build
-# without persisting it to the container.
-ARG ARCH="amd64"
-ARG BASE_IMAGE_VERSION="0.3.2"
-ARG BASE_IMAGE="k8s.gcr.io/debian-base-${ARCH}:${BASE_IMAGE_VERSION}"
+ARG BASE_IMAGE="ubuntu:18.04"
 FROM ${BASE_IMAGE}
 
 # NOTE: ARCH must be defined again after FROM
@@ -40,9 +35,11 @@ ARG ARCH="amd64"
 # setting DEBIAN_FRONTEND=noninteractive stops some apt warnings, this is not 
 # a real argument, we're (ab)using ARG to get a temporary ENV again.
 ARG DEBIAN_FRONTEND=noninteractive
+
+COPY clean-install /usr/local/bin/clean-install
+RUN chmod +x /usr/local/bin/clean-install
+
 # Get dependencies
-# First we add stretch-backports so we can get newer systemd which supports
-# supressing kmsg in journald
 # The base image already has: ssh, apt, snapd
 # This is broken down into (each on a line):
 # - packages necessary for installing docker
@@ -52,11 +49,9 @@ ARG DEBIAN_FRONTEND=noninteractive
 # Then we cleanup (removing unwanted systemd services)
 # Finally we disable kmsg in journald
 # https://developers.redhat.com/blog/2014/05/05/running-systemd-within-docker-container/
-RUN echo "deb http://ftp.debian.org/debian stretch-backports main" \
-        > /etc/apt/sources.list.d/backports.list \
-    && clean-install \
+RUN clean-install \
       apt-transport-https ca-certificates curl software-properties-common gnupg2 lsb-release \
-      systemd/stretch-backports systemd-sysv/stretch-backports libsystemd0/stretch-backports \
+      systemd systemd-sysv libsystemd0 \
       iptables iproute2 ethtool socat util-linux mount ebtables udev kmod aufs-tools \
       bash rsync \
     && find /lib/systemd/system/sysinit.target.wants/ -name "systemd-tmpfiles-setup.service" -delete \
@@ -75,9 +70,9 @@ RUN echo "deb http://ftp.debian.org/debian stretch-backports main" \
 # - add the fingerprint
 # - add the repository
 # - update apt, install docker, cleanup
-# NOTE: 17.03 is officially supported by Kubernetes currently, so we pin to that.
+# NOTE: 18.06 is officially supported by Kubernetes currently, so we pin to that.
 # https://kubernetes.io/docs/tasks/tools/install-kubeadm/
-ARG DOCKER_VERSION="17.03.2~ce-0~debian-stretch"
+ARG DOCKER_VERSION="18.06.*"
 # another temporary env, not a real argument. setting this to a non-zero value
 # silences this warning from apt-key:
 # "Warning: apt-key output should not be parsed (stdout is not a terminal)"

images/base/clean-install

@@ -0,0 +1,36 @@
+#!/bin/sh
+
+# Copyright 2017 The Kubernetes Authors.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+# A script encapsulating a common Dockerimage pattern for installing packages
+# and then cleaning up the unnecessary install artifacts.
+# e.g. clean-install iptables ebtables conntrack
+
+set -o errexit
+
+if [ $# = 0 ]; then
+  echo >&2 "No packages specified"
+  exit 1
+fi
+
+apt-get update
+apt-get install -y --no-install-recommends $@
+apt-get clean -y
+rm -rf \
+   /var/cache/debconf/* \
+   /var/lib/apt/lists/* \
+   /var/log/* \
+   /tmp/* \
+   /var/tmp/*

pkg/build/kube/bits.go

@@ -32,7 +32,7 @@ type Bits interface {
 	// Paths returns a map of path on host machine to desired path in the image
 	// These paths will be populated in the image relative to some base path,
 	// obtainable by NodeInstall.BasePath()
-	// Note: if Images are populated in iamges/, the cluster provisioning
+	// Note: if Images are populated to images/, the cluster provisioning
 	// will load these prior to calling kubeadm
 	Paths() map[string]string
 	// Install should install the built sources on the node, assuming paths

pkg/build/node/node.go

@@ -21,8 +21,14 @@ import (
 	"fmt"
 	"os"
 	"path"
+	"regexp"
+	"strings"
 	"time"
 
+	"k8s.io/apimachinery/pkg/util/version"
+
+	"k8s.io/apimachinery/pkg/util/sets"
+
 	"github.com/pkg/errors"
 	log "github.com/sirupsen/logrus"
 
@@ -160,6 +166,25 @@ func (c *BuildContext) populateBits(buildDir string) error {
 	return nil
 }
 
+// matches image tarballs kind will sideload
+var imageRegex = regexp.MustCompile(`images/[^/]+\.tar`)
+
+// returns a set of image tags that will be sideloaded
+func (c *BuildContext) getBuiltImages() (sets.String, error) {
+	bitPaths := c.bits.Paths()
+	images := sets.NewString()
+	for src, dest := range bitPaths {
+		if imageRegex.MatchString(dest) {
+			tags, err := docker.GetArchiveTags(src)
+			if err != nil {
+				return nil, err
+			}
+			images.Insert(tags...)
+		}
+	}
+	return images, nil
+}
+
 // BuildContainerLabelKey is applied to each build container
 const BuildContainerLabelKey = "io.k8s.sigs.kind.build"
 
@@ -250,6 +275,12 @@ func (c *BuildContext) buildImage(dir string) error {
 		return err
 	}
 
+	// pre-pull images that were not part of the build
+	if err = c.prePullImages(dir, containerID); err != nil {
+		log.Errorf("Image build Failed! %v", err)
+		return err
+	}
+
 	// Save the image changes to a new image
 	cmd := exec.Command("docker", "commit", containerID, c.image)
 	cmd.Debug = true
@@ -263,6 +294,92 @@ func (c *BuildContext) buildImage(dir string) error {
 	return nil
 }
 
+// must be run after kubernetes has been installed on the node
+func (c *BuildContext) prePullImages(dir, containerID string) error {
+	// first get the images we actually built
+	builtImages, err := c.getBuiltImages()
+	if err != nil {
+		log.Errorf("Image build Failed! %v", err)
+		return err
+	}
+
+	// helpers to run things in the build container
+	execInBuild := func(command ...string) error {
+		cmd := exec.Command("docker", "exec", containerID)
+		cmd.Args = append(cmd.Args, command...)
+		cmd.Debug = true
+		cmd.InheritOutput = true
+		return cmd.Run()
+	}
+	combinedOutputLinesInBuild := func(command ...string) ([]string, error) {
+		cmd := exec.Command("docker", "exec", containerID)
+		cmd.Args = append(cmd.Args, command...)
+		cmd.Debug = true
+		return cmd.CombinedOutputLines()
+	}
+
+	// get the Kubernetes version we installed on the node
+	// we need this to ask kubeadm what images we need
+	rawVersion, err := combinedOutputLinesInBuild("cat", "/kind/version")
+	if err != nil {
+		log.Errorf("Image build Failed! %v", err)
+		return err
+	}
+	if len(rawVersion) != 1 {
+		log.Errorf("Image build Failed! %v", err)
+		return fmt.Errorf("invalid kubernetes version file")
+	}
+
+	// before Kubernetes v1.12.0 kubeadm requires arch specific images, instead
+	// later releases use manifest list images
+	// at node boot time we retag our images to handle this where necessary,
+	// so we virtually re-tag them here.
+	ver, err := version.ParseGeneric(rawVersion[0])
+	if err != nil {
+		return err
+	}
+	if ver.LessThan(version.MustParseSemantic("v1.12.0")) {
+		archSuffix := fmt.Sprintf("-%s:", c.arch)
+		for _, image := range builtImages.List() {
+			if !strings.Contains(image, archSuffix) {
+				builtImages.Insert(strings.Replace(image, ":", archSuffix, 1))
+			}
+		}
+	}
+
+	// gets the list of images required by kubeadm
+	requiredImages, err := combinedOutputLinesInBuild(
+		"kubeadm", "config", "images", "list", "--kubernetes-version", rawVersion[0],
+	)
+	if err != nil {
+		return err
+	}
+
+	movePulled := []string{"mv"}
+	for i, image := range requiredImages {
+		if !builtImages.Has(image) {
+			fmt.Printf("Pulling: %s\n", image)
+			err := docker.Pull(image, 2)
+			if err != nil {
+				return err
+			}
+			// TODO(bentheelder): generate a friendlier name
+			pullName := fmt.Sprintf("%d.tar", i)
+			pullTo := fmt.Sprintf("%s/bits/images/%s", dir, pullName)
+			err = docker.Save(image, pullTo)
+			if err != nil {
+				return err
+			}
+			movePulled = append(movePulled, fmt.Sprintf("/build/bits/images/%s", pullName))
+		}
+	}
+	movePulled = append(movePulled, "/kind/images/")
+	if err := execInBuild(movePulled...); err != nil {
+		return err
+	}
+	return nil
+}
+
 func (c *BuildContext) createBuildContainer(buildDir string) (id string, err error) {
 	// attempt to explicitly pull the image if it doesn't exist locally
 	// we don't care if this errors, we'll still try to run which also pulls

pkg/cluster/cluster.go

@@ -106,7 +106,7 @@ func (c *Context) Create(cfg *config.Config) error {
 	}
 
 	fmt.Printf("Creating cluster '%s' ...\n", c.ClusterName())
-	c.status = logutil.NewStatus()
+	c.status = logutil.NewStatus(os.Stdout)
 	c.status.MaybeWrapLogrus(log.StandardLogger())
 
 	defer c.status.End(false)