Support persisting images pulled in KIND in tests when KINDNodeCache …

…is set. (#805)

go.mod

@@ -7,6 +7,10 @@ require (
 	github.com/Azure/go-autorest/autorest v0.5.0 // indirect
 	github.com/Masterminds/goutils v1.1.0 // indirect
 	github.com/Masterminds/semver v1.4.2
+	github.com/docker/distribution v2.7.1+incompatible // indirect
+	github.com/docker/docker v1.13.1
+	github.com/docker/go-connections v0.4.0 // indirect
+	github.com/docker/go-units v0.4.0 // indirect
 	github.com/dustinkirkland/golang-petname v0.0.0-20170921220637-d3c2ba80e75e
 	github.com/go-playground/locales v0.12.1 // indirect
 	github.com/go-playground/universal-translator v0.16.0 // indirect
@@ -33,6 +37,7 @@ require (
 	github.com/mattn/go-runewidth v0.0.4 // indirect
 	github.com/nbutton23/zxcvbn-go v0.0.0-20180912185939-ae427f1e4c1d // indirect
 	github.com/onsi/gomega v1.5.0
+	github.com/opencontainers/go-digest v1.0.0-rc1 // indirect
 	github.com/pborman/uuid v0.0.0-20180906182336-adf5a7427709 // indirect
 	github.com/pelletier/go-toml v1.4.0 // indirect
 	github.com/pkg/errors v0.8.1

go.sum

@@ -65,6 +65,14 @@ github.com/davecgh/go-spew v1.1.1/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSs
 github.com/dgrijalva/jwt-go v3.2.0+incompatible h1:7qlOGliEKZXTDg6OTjfoBKDXWrumCAMpl/TFQ4/5kLM=
 github.com/dgrijalva/jwt-go v3.2.0+incompatible/go.mod h1:E3ru+11k8xSBh+hMPgOLZmtrrCbhqsmaPHjLKYnJCaQ=
 github.com/dgryski/go-sip13 v0.0.0-20181026042036-e10d5fee7954/go.mod h1:vAd38F8PWV+bWy6jNmig1y/TA+kYO4g3RSRF0IAv0no=
+github.com/docker/distribution v2.7.1+incompatible h1:a5mlkVzth6W5A4fOsS3D2EO5BUmsJpcB+cRlLU7cSug=
+github.com/docker/distribution v2.7.1+incompatible/go.mod h1:J2gT2udsDAN96Uj4KfcMRqY0/ypR+oyYUYmja8H+y+w=
+github.com/docker/docker v1.13.1 h1:IkZjBSIc8hBjLpqeAbeE5mca5mNgeatLHBy3GO78BWo=
+github.com/docker/docker v1.13.1/go.mod h1:eEKB0N0r5NX/I1kEveEz05bcu8tLC/8azJZsviup8Sk=
+github.com/docker/go-connections v0.4.0 h1:El9xVISelRB7BuFusrZozjnkIM5YnzCViNKohAFqRJQ=
+github.com/docker/go-connections v0.4.0/go.mod h1:Gbd7IOopHjR8Iph03tsViu4nIes5XhDvyHbTtUxmeec=
+github.com/docker/go-units v0.4.0 h1:3uh0PgVws3nIA0Q+MwDC8yjEPf9zjRfZZWXZYDct3Tw=
+github.com/docker/go-units v0.4.0/go.mod h1:fgPhTUdO+D/Jk86RDLlptpiXQzgHJF7gydDDbaIK4Dk=
 github.com/dustinkirkland/golang-petname v0.0.0-20170921220637-d3c2ba80e75e h1:bRcq7ruHMqCVB/ugLbBylx+LrccNACFDEaqAD/aZ80Q=
 github.com/dustinkirkland/golang-petname v0.0.0-20170921220637-d3c2ba80e75e/go.mod h1:V+Qd57rJe8gd4eiGzZyg4h54VLHmYVVw54iMnlAMrF8=
 github.com/eapache/go-resiliency v1.1.0/go.mod h1:kFI+JgMyC7bLPUVY133qvEBtVayf5mFgVsvEsIPBvNs=
@@ -374,6 +382,8 @@ github.com/onsi/gomega v1.4.3 h1:RE1xgDvH7imwFD45h+u2SgIfERHlS2yNG4DObb5BSKU=
 github.com/onsi/gomega v1.4.3/go.mod h1:ex+gbHU/CVuBBDIJjb2X0qEXbFg53c61hWP/1CpauHY=
 github.com/onsi/gomega v1.5.0 h1:izbySO9zDPmjJ8rDjLvkA2zJHIo+HkYXHnf7eN7SSyo=
 github.com/onsi/gomega v1.5.0/go.mod h1:ex+gbHU/CVuBBDIJjb2X0qEXbFg53c61hWP/1CpauHY=
+github.com/opencontainers/go-digest v1.0.0-rc1 h1:WzifXhOVOEOuFYOJAW6aQqW0TooG2iki3E3Ii+WN7gQ=
+github.com/opencontainers/go-digest v1.0.0-rc1/go.mod h1:cMLVZDEM3+U2I4VmLI6N8jQYUd2OVphdqWwCJHrFt2s=
 github.com/openzipkin/zipkin-go v0.1.6/go.mod h1:QgAqvLzwWbR/WpD4A3cGpPtJrZXNIiJc5AZX7/PBEpw=
 github.com/pborman/uuid v0.0.0-20170612153648-e790cca94e6c/go.mod h1:VyrYX9gd7irzKovcSS6BIIEwPRkP2Wm2m9ufcdFSJ34=
 github.com/pborman/uuid v0.0.0-20180906182336-adf5a7427709 h1:zNBQb37RGLmJybyMcs983HfUfpkw9OTFD9tbBfAViHE=

keps/0008-operator-testing.md

@@ -161,6 +161,9 @@ type TestSuite struct {
 	KINDConfig string `json:"kindConfig"`
 	// KIND context to use.
 	KINDContext string `json:"kindContext"`
+	// If set, each node defined in the kind configuration will have a docker named volume mounted into it to persist
+	// pulled container images across test runs.
+	KINDNodeCache bool `json:"kindNodeCache"`
 	// Whether or not to start the KUDO controller for the tests.
 	StartKUDO         bool
 	// If set, do not delete the resources after running the tests (implies SkipClusterDelete).

pkg/apis/kudo/v1alpha1/test_types.go

@@ -28,6 +28,9 @@ type TestSuite struct {
 	KINDConfig string `json:"kindConfig"`
 	// KIND context to use.
 	KINDContext string `json:"kindContext"`
+	// If set, each node defined in the kind configuration will have a docker named volume mounted into it to persist
+	// pulled container images across test runs.
+	KINDNodeCache bool `json:"kindNodeCache"`
 	// Whether or not to start the KUDO controller for the tests.
 	StartKUDO bool `json:"startKUDO"`
 	// If set, do not delete the resources after running the tests (implies SkipClusterDelete).
@@ -99,3 +102,6 @@ type Command struct {
 	// If set, failures will be ignored.
 	IgnoreFailure bool `json:"ignoreFailure"`
 }
+
+// DefaultKINDContext defines the default kind context to use.
+const DefaultKINDContext = "kind"

pkg/kudoctl/cmd/test.go

@@ -124,7 +124,7 @@ For more detailed documentation, visit: https://kudo.dev/docs/testing`,
 			}
 
 			if options.KINDContext == "" {
-				options.KINDContext = "kind"
+				options.KINDContext = kudo.DefaultKINDContext
 			}
 
 			if options.StartControlPlane && options.StartKIND {

pkg/test/harness.go

@@ -11,6 +11,8 @@ import (
 	"testing"
 	"time"
 
+	volumetypes "github.com/docker/docker/api/types/volume"
+	docker "github.com/docker/docker/client"
 	kudo "github.com/kudobuilder/kudo/pkg/apis/kudo/v1alpha1"
 	"github.com/kudobuilder/kudo/pkg/controller"
 	testutils "github.com/kudobuilder/kudo/pkg/test/utils"
@@ -25,6 +27,7 @@ import (
 	kindConfig "sigs.k8s.io/kind/pkg/apis/config/v1alpha3"
 	kind "sigs.k8s.io/kind/pkg/cluster"
 	kindCreate "sigs.k8s.io/kind/pkg/cluster/create"
+	"sigs.k8s.io/kind/pkg/container/cri"
 )
 
 // Harness loads and runs tests based on the configuration provided.
@@ -35,6 +38,7 @@ type Harness struct {
 	logger        testutils.Logger
 	managerStopCh chan struct{}
 	config        *rest.Config
+	docker        testutils.DockerClient
 	client        client.Client
 	dclient       discovery.DiscoveryInterface
 	env           *envtest.Environment
@@ -126,6 +130,8 @@ func (h *Harness) RunKIND() (*rest.Config, error) {
 			}
 		}
 
+		h.addNodeCaches(kindCfg)
+
 		err := h.kind.Create(kindCreate.WithV1Alpha3(kindCfg))
 		if err != nil {
 			return nil, err
@@ -135,6 +141,45 @@ func (h *Harness) RunKIND() (*rest.Config, error) {
 	return clientcmd.BuildConfigFromFlags("", h.kind.KubeConfigPath())
 }
 
+func (h *Harness) addNodeCaches(kindCfg *kindConfig.Cluster) error {
+	if !h.TestSuite.KINDNodeCache {
+		return nil
+	}
+
+	dockerClient, err := h.DockerClient()
+	if err != nil {
+		return err
+	}
+
+	// add a default node if there are none specified.
+	if len(kindCfg.Nodes) == 0 {
+		kindCfg.Nodes = append(kindCfg.Nodes, kindConfig.Node{})
+	}
+
+	if h.TestSuite.KINDContext == "" {
+		h.TestSuite.KINDContext = kudo.DefaultKINDContext
+	}
+
+	for index := range kindCfg.Nodes {
+		volume, err := dockerClient.VolumeCreate(context.TODO(), volumetypes.VolumesCreateBody{
+			Driver: "local",
+			Name:   fmt.Sprintf("%s-%d", h.TestSuite.KINDContext, index),
+		})
+		if err != nil {
+			h.T.Log("error creating volume for node", err)
+			continue
+		}
+
+		h.T.Log("node mount point", volume.Mountpoint)
+		kindCfg.Nodes[index].ExtraMounts = append(kindCfg.Nodes[index].ExtraMounts, cri.Mount{
+			ContainerPath: "/var/lib/containerd",
+			HostPath:      volume.Mountpoint,
+		})
+	}
+
+	return nil
+}
+
 // RunTestEnv starts a Kubernetes API server and etcd server for use in the
 // tests and returns the Kubernetes configuration.
 func (h *Harness) RunTestEnv() (*rest.Config, error) {
@@ -254,6 +299,17 @@ func (h *Harness) DiscoveryClient() (discovery.DiscoveryInterface, error) {
 	return h.dclient, err
 }
 
+// DockerClient returns the Docker client to use for the test harness.
+func (h *Harness) DockerClient() (testutils.DockerClient, error) {
+	if h.docker != nil {
+		return h.docker, nil
+	}
+
+	var err error
+	h.docker, err = docker.NewEnvClient()
+	return h.docker, err
+}
+
 // RunTests should be called from within a Go test (t) and launches all of the KUDO integration
 // tests at dir.
 func (h *Harness) RunTests() {

pkg/test/harness_test.go

@@ -1,8 +1,14 @@
 package test
 
 import (
+	"context"
+	"fmt"
 	"testing"
 
+	kindConfig "sigs.k8s.io/kind/pkg/apis/config/v1alpha3"
+
+	dockertypes "github.com/docker/docker/api/types"
+	volumetypes "github.com/docker/docker/api/types/volume"
 	"github.com/stretchr/testify/assert"
 )
 
@@ -13,3 +19,46 @@ func TestGetTimeout(t *testing.T) {
 	h.TestSuite.Timeout = 45
 	assert.Equal(t, 45, h.GetTimeout())
 }
+
+type dockerMock struct{}
+
+func (d *dockerMock) VolumeCreate(ctx context.Context, body volumetypes.VolumesCreateBody) (dockertypes.Volume, error) {
+	return dockertypes.Volume{
+		Mountpoint: fmt.Sprintf("/var/lib/docker/data/%s", body.Name),
+	}, nil
+}
+
+func TestAddNodeCaches(t *testing.T) {
+	h := Harness{
+		T:      t,
+		docker: &dockerMock{},
+	}
+
+	kindCfg := &kindConfig.Cluster{}
+	h.addNodeCaches(kindCfg)
+	assert.Nil(t, kindCfg.Nodes)
+
+	h.TestSuite.KINDNodeCache = true
+	h.addNodeCaches(kindCfg)
+	assert.NotNil(t, kindCfg.Nodes)
+	assert.Equal(t, 1, len(kindCfg.Nodes))
+	assert.NotNil(t, kindCfg.Nodes[0].ExtraMounts)
+	assert.Equal(t, 1, len(kindCfg.Nodes[0].ExtraMounts))
+	assert.Equal(t, "/var/lib/containerd", kindCfg.Nodes[0].ExtraMounts[0].ContainerPath)
+	assert.Equal(t, "/var/lib/docker/data/kind-0", kindCfg.Nodes[0].ExtraMounts[0].HostPath)
+
+	kindCfg = &kindConfig.Cluster{
+		Nodes: []kindConfig.Node{
+			kindConfig.Node{},
+			kindConfig.Node{},
+		},
+	}
+	h.addNodeCaches(kindCfg)
+	assert.NotNil(t, kindCfg.Nodes)
+	assert.Equal(t, 2, len(kindCfg.Nodes))
+	assert.NotNil(t, kindCfg.Nodes[0].ExtraMounts)
+	assert.Equal(t, 1, len(kindCfg.Nodes[0].ExtraMounts))
+	assert.Equal(t, "/var/lib/containerd", kindCfg.Nodes[0].ExtraMounts[0].ContainerPath)
+	assert.Equal(t, "/var/lib/docker/data/kind-0", kindCfg.Nodes[0].ExtraMounts[0].HostPath)
+	assert.Equal(t, "/var/lib/docker/data/kind-1", kindCfg.Nodes[1].ExtraMounts[0].HostPath)
+}

pkg/test/utils/docker.go

@@ -0,0 +1,13 @@
+package utils
+
+import (
+	"context"
+
+	dockertypes "github.com/docker/docker/api/types"
+	volumetypes "github.com/docker/docker/api/types/volume"
+)
+
+// DockerClient is a wrapper interface for the Docker library to support unit testing.
+type DockerClient interface {
+	VolumeCreate(context.Context, volumetypes.VolumesCreateBody) (dockertypes.Volume, error)
+}