Add ability to include images in custom node build

This adds a new `kind build add-image` command that provides the ability
to "preload" container images in a custom node image.

Signed-off-by: Sean McGinnis <sean.mcginnis@gmail.com>
Co-authored-by: Curt Bushko <cbushko@gmail.com>

pkg/build/addimage/build.go

@@ -0,0 +1,51 @@
+/*
+Copyright 2024 The Kubernetes Authors.
+
+Licensed under the Apache License, Version 2.0 (the "License");
+you may not use this file except in compliance with the License.
+You may obtain a copy of the License at
+
+    http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+See the License for the specific language governing permissions and
+limitations under the License.
+*/
+
+// Package addimage implements functionality to build a node image with container images included.
+package addimage
+
+import (
+	"runtime"
+
+	"sigs.k8s.io/kind/pkg/apis/config/defaults"
+	"sigs.k8s.io/kind/pkg/build/internal/build"
+	"sigs.k8s.io/kind/pkg/log"
+)
+
+// Build creates a new node image by combining an existing node image with a collection
+// of additional images.
+func Build(options ...Option) error {
+	// default options
+	ctx := &buildContext{
+		image:     DefaultImage,
+		baseImage: defaults.Image,
+		logger:    log.NoopLogger{},
+		arch:      runtime.GOARCH,
+	}
+
+	// apply user options
+	for _, option := range options {
+		if err := option.apply(ctx); err != nil {
+			return err
+		}
+	}
+
+	// verify that we're using a supported arch
+	if !build.SupportedArch(ctx.arch) {
+		ctx.logger.Warnf("unsupported architecture %q", ctx.arch)
+	}
+	return ctx.Build()
+}

pkg/build/addimage/buildcontext.go

@@ -0,0 +1,184 @@
+/*
+Copyrigh. 2024 The Kubernetes Authors.
+
+Licensed under the Apache License, Version 2.0 (the "License");
+you may not use this file except in compliance with the License.
+You may obtain a copy of the License at
+
+    http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+See the License for the specific language governing permissions and
+limitations under the License.
+*/
+
+package addimage
+
+import (
+	"fmt"
+	"math/rand"
+	"os"
+	"path/filepath"
+	"strings"
+	"time"
+
+	"sigs.k8s.io/kind/pkg/build/internal/build"
+	"sigs.k8s.io/kind/pkg/build/internal/container/docker"
+	"sigs.k8s.io/kind/pkg/errors"
+	"sigs.k8s.io/kind/pkg/exec"
+	"sigs.k8s.io/kind/pkg/fs"
+	"sigs.k8s.io/kind/pkg/log"
+)
+
+const (
+	// httpProxy is the HTTP_PROXY environment variable key
+	httpProxy = "HTTP_PROXY"
+	// httpsProxy is the HTTPS_PROXY environment variable key
+	httpsProxy = "HTTPS_PROXY"
+	// noProxy is the NO_PROXY environment variable key
+	noProxy = "NO_PROXY"
+)
+
+// buildContext the settings to use for rebuilding the node image.
+type buildContext struct {
+	// option fields
+	image            string
+	baseImage        string
+	additionalImages []string
+	logger           log.Logger
+	arch             string
+	alwaysPull       bool
+}
+
+// Build rebuilds the cluster node image using the buildContext to determine
+// which base image and additional images to package into a new node image.
+func (c *buildContext) Build() (err error) {
+	c.logger.V(0).Infof("Adding %v images to base image", c.additionalImages)
+
+	// Retrieve any necessary images
+	for _, imageName := range c.additionalImages {
+		if !c.alwaysPull {
+			// Check if the image already exists locally
+			if _, err := docker.ImageID(imageName, c.arch); err == nil {
+				continue
+			}
+		}
+
+		// Pull the image for the requested architecture, which may be different than this host
+		err = docker.Pull(c.logger, imageName, build.DockerBuildOsAndArch(c.arch), 3)
+		if err != nil {
+			c.logger.Errorf("add image build failed, unable to pull image %q: %v", imageName, err)
+			return err
+		}
+	}
+
+	c.logger.V(0).Infof("Creating build container based on %q", c.baseImage)
+
+	containerID, err := c.createBuildContainer()
+	if containerID != "" {
+		defer func() {
+			_ = exec.Command("docker", "rm", "-f", "-v", containerID).Run()
+		}()
+	}
+	if err != nil {
+		c.logger.Errorf("add image build failed, unable to create build container: %v", err)
+		return err
+	}
+	c.logger.V(1).Infof("Building in %s", containerID)
+
+	// Tar up the images to make the load easier (and follow the current load pattern)
+	// Setup the tar path where the images will be saved
+	dir, err := fs.TempDir("", "images-tar")
+	if err != nil {
+		return errors.Wrap(err, "failed to create tempdir")
+	}
+	defer os.RemoveAll(dir)
+
+	// Save the images into a tar file
+	imagesTarFile := filepath.Join(dir, "images.tar")
+	c.logger.V(1).Infof("Saving images into tar file at %q", imagesTarFile)
+	err = docker.SaveImages(c.additionalImages, imagesTarFile)
+	if err != nil {
+		return err
+	}
+
+	// Import the images from the tarfile into our build container
+	cmder := docker.ContainerCmder(containerID)
+	importer := build.NewContainerdImporter(cmder)
+
+	f, err := os.Open(imagesTarFile)
+	if err != nil {
+		return err
+	}
+	defer f.Close()
+
+	c.logger.V(0).Infof("Importing images into build container %s", containerID)
+	if err := importer.LoadCommand().SetStdout(os.Stdout).SetStderr(os.Stderr).SetStdin(f).Run(); err != nil {
+		return err
+	}
+
+	// Save the image changes to a new image
+	c.logger.V(0).Info("Saving new image " + c.image)
+	saveCmd := exec.Command(
+		"docker", "commit",
+		// we need to put this back after changing it when running the image
+		"--change", `ENTRYPOINT [ "/usr/local/bin/entrypoint", "/sbin/init" ]`,
+		// remove proxy settings since they're for the building process
+		// and should not be carried with the built image
+		"--change", `ENV HTTP_PROXY="" HTTPS_PROXY="" NO_PROXY=""`,
+		containerID, c.image,
+	)
+	exec.InheritOutput(saveCmd)
+	if err = saveCmd.Run(); err != nil {
+		c.logger.Errorf("add image build failed, unable to save destination image: %v", err)
+		return err
+	}
+
+	c.logger.V(0).Info("Add image build completed.")
+	return nil
+}
+
+func (c *buildContext) createBuildContainer() (id string, err error) {
+	// Attempt to explicitly pull the image if it doesn't exist locally
+	// we don't care if this errors, we'll still try to run which also pulls
+	_ = docker.Pull(c.logger, c.baseImage, build.DockerBuildOsAndArch(c.arch), 4)
+
+	// This should be good enough: a specific prefix, the current unix time,
+	// and a little random bits in case we have multiple builds simultaneously
+	random := rand.New(rand.NewSource(time.Now().UnixNano())).Int31()
+	id = fmt.Sprintf("kind-build-%d-%d", time.Now().UTC().Unix(), random)
+	runArgs := []string{
+		// make the client exit while the container continues to run
+		"-d",
+		// run containerd so that the cri command works
+		"--entrypoint=/usr/local/bin/containerd",
+		"--name=" + id,
+		"--platform=" + build.DockerBuildOsAndArch(c.arch),
+		"--security-opt", "seccomp=unconfined", // ignore seccomp
+	}
+
+	// Pass proxy settings from environment variables to the building container
+	// to make them work during the building process
+	for _, name := range []string{httpProxy, httpsProxy, noProxy} {
+		val := os.Getenv(name)
+		if val == "" {
+			val = os.Getenv(strings.ToLower(name))
+		}
+		if val != "" {
+			runArgs = append(runArgs, "--env", name+"="+val)
+		}
+	}
+
+	// Run it
+	err = docker.Run(
+		c.baseImage,
+		runArgs,
+		[]string{
+			"",
+		},
+	)
+
+	return id, errors.Wrap(err, "failed to create build container")
+}

pkg/build/nodeimage/internal/container/docker/save.go → pkg/build/addimage/defaults.go

@@ -1,5 +1,5 @@
 /*
-Copyright 2018 The Kubernetes Authors.
+Copyright 2024 The Kubernetes Authors.
 
 Licensed under the Apache License, Version 2.0 (the "License");
 you may not use this file except in compliance with the License.
@@ -14,13 +14,7 @@ See the License for the specific language governing permissions and
 limitations under the License.
 */
 
-package docker
+package addimage
 
-import (
-	"sigs.k8s.io/kind/pkg/exec"
-)
-
-// Save saves image to dest, as in `docker save`
-func Save(image, dest string) error {
-	return exec.Command("docker", "save", "-o", dest, image).Run()
-}
+// DefaultImage is the default name:tag for the built image
+const DefaultImage = "kindest/custom-node:latest"

pkg/build/addimage/options.go

@@ -0,0 +1,82 @@
+/*
+Copyright 2024 The Kubernetes Authors.
+
+Licensed under the Apache License, Version 2.0 (the "License");
+you may not use this file except in compliance with the License.
+You may obtain a copy of the License at
+
+    http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+See the License for the specific language governing permissions and
+limitations under the License.
+*/
+
+package addimage
+
+import (
+	"sigs.k8s.io/kind/pkg/log"
+)
+
+// Option is a configuration option supplied to Build
+type Option interface {
+	apply(*buildContext) error
+}
+
+type optionAdapter func(*buildContext) error
+
+func (c optionAdapter) apply(o *buildContext) error {
+	return c(o)
+}
+
+// WithImage configures a build to tag the built image with `image`
+func WithImage(image string) Option {
+	return optionAdapter(func(b *buildContext) error {
+		b.image = image
+		return nil
+	})
+}
+
+// WithBaseImage configures a build to use `image` as the base image
+func WithBaseImage(image string) Option {
+	return optionAdapter(func(b *buildContext) error {
+		b.baseImage = image
+		return nil
+	})
+}
+
+// WithAdditionalImages configures a build to add images to the node image
+func WithAdditonalImages(images []string) Option {
+	return optionAdapter(func(b *buildContext) error {
+		b.additionalImages = images
+		return nil
+	})
+}
+
+// WithLogger sets the logger
+func WithLogger(logger log.Logger) Option {
+	return optionAdapter(func(b *buildContext) error {
+		b.logger = logger
+		return nil
+	})
+}
+
+// WithArch sets the architecture to build for
+func WithArch(arch string) Option {
+	return optionAdapter(func(b *buildContext) error {
+		if arch != "" {
+			b.arch = arch
+		}
+		return nil
+	})
+}
+
+// WithPullPolicy sets whether to always pull the images
+func WithPullPolicy(pull bool) Option {
+	return optionAdapter(func(b *buildContext) error {
+		b.alwaysPull = pull
+		return nil
+	})
+}

pkg/build/internal/build/helpers.go

@@ -0,0 +1,34 @@
+/*
+Copyright 2020 The Kubernetes Authors.
+
+Licensed under the Apache License, Version 2.0 (the "License");
+you may not use this file except in compliance with the License.
+You may obtain a copy of the License at
+
+    http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+See the License for the specific language governing permissions and
+limitations under the License.
+*/
+
+package build
+
+// SupportedArch checks whether the requested arch is one that is officially supportedf
+// by the project.
+func SupportedArch(arch string) bool {
+	switch arch {
+	default:
+		return false
+	// currently we nominally support building node images for these
+	case "amd64":
+	case "arm64":
+	}
+	return true
+}
+
+func DockerBuildOsAndArch(arch string) string {
+	return "linux/" + arch
+}