Initial provider for nerdctl/Finch

Adds implementation for a provider based on nerdctl. Several todos
in the code but the core functionality of creating/deleting clusters
is working and a simple application deployed works properly

Signed-off-by: Phil Estes <estesp@gmail.com>

pkg/cluster/internal/providers/nerdctl/OWNERS

@@ -0,0 +1,2 @@
+labels:
+- area/provider/nerdctl

pkg/cluster/internal/providers/nerdctl/constants.go

@@ -0,0 +1,24 @@
+/*
+Copyright 2019 The Kubernetes Authors.
+
+Licensed under the Apache License, Version 2.0 (the "License");
+you may not use this file except in compliance with the License.
+You may obtain a copy of the License at
+
+    http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or impliep.
+See the License for the specific language governing permissions and
+limitations under the License.
+*/
+
+package nerdctl
+
+// clusterLabelKey is applied to each "node" container for identification
+const clusterLabelKey = "io.x-k8s.kind.cluster"
+
+// nodeRoleLabelKey is applied to each "node" container for categorization
+// of nodes by role
+const nodeRoleLabelKey = "io.x-k8s.kind.role"

pkg/cluster/internal/providers/nerdctl/images.go

@@ -0,0 +1,91 @@
+/*
+Copyright 2019 The Kubernetes Authors.
+
+Licensed under the Apache License, Version 2.0 (the "License");
+you may not use this file except in compliance with the License.
+You may obtain a copy of the License at
+
+    http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+See the License for the specific language governing permissions and
+limitations under the License.
+*/
+
+package nerdctl
+
+import (
+	"fmt"
+	"strings"
+	"time"
+
+	"sigs.k8s.io/kind/pkg/errors"
+	"sigs.k8s.io/kind/pkg/exec"
+	"sigs.k8s.io/kind/pkg/log"
+
+	"sigs.k8s.io/kind/pkg/cluster/internal/providers/common"
+	"sigs.k8s.io/kind/pkg/internal/apis/config"
+	"sigs.k8s.io/kind/pkg/internal/cli"
+)
+
+// ensureNodeImages ensures that the node images used by the create
+// configuration are present
+func ensureNodeImages(logger log.Logger, status *cli.Status, cfg *config.Cluster, binaryName string) error {
+	// pull each required image
+	for _, image := range common.RequiredNodeImages(cfg).List() {
+		// prints user friendly message
+		friendlyImageName, image := sanitizeImage(image)
+		status.Start(fmt.Sprintf("Ensuring node image (%s) 🖼", friendlyImageName))
+		if _, err := pullIfNotPresent(logger, image, 4, binaryName); err != nil {
+			status.End(false)
+			return err
+		}
+	}
+	return nil
+}
+
+// pullIfNotPresent will pull an image if it is not present locally
+// retrying up to retries times
+// it returns true if it attempted to pull, and any errors from pulling
+func pullIfNotPresent(logger log.Logger, image string, retries int, binaryName string) (pulled bool, err error) {
+	// TODO(bentheelder): switch most (all) of the logging here to debug level
+	// once we have configurable log levels
+	// if this did not return an error, then the image exists locally
+	cmd := exec.Command(binaryName, "inspect", "--type=image", image)
+	if err := cmd.Run(); err == nil {
+		logger.V(1).Infof("Image: %s present locally", image)
+		return false, nil
+	}
+	// otherwise try to pull it
+	return true, pull(logger, image, retries, binaryName)
+}
+
+// pull pulls an image, retrying up to retries times
+func pull(logger log.Logger, image string, retries int, binaryName string) error {
+	logger.V(1).Infof("Pulling image: %s ...", image)
+	err := exec.Command(binaryName, "pull", image).Run()
+	// retry pulling up to retries times if necessary
+	if err != nil {
+		for i := 0; i < retries; i++ {
+			time.Sleep(time.Second * time.Duration(i+1))
+			logger.V(1).Infof("Trying again to pull image: %q ... %v", image, err)
+			// TODO(bentheelder): add some backoff / sleep?
+			err = exec.Command(binaryName, "pull", image).Run()
+			if err == nil {
+				break
+			}
+		}
+	}
+	return errors.Wrapf(err, "failed to pull image %q", image)
+}
+
+// sanitizeImage is a helper to return human readable image name and
+// the docker pullable image name from the provided image
+func sanitizeImage(image string) (string, string) {
+	if strings.Contains(image, "@sha256:") {
+		return strings.Split(image, "@sha256:")[0], image
+	}
+	return image, image
+}

pkg/cluster/internal/providers/nerdctl/network.go

@@ -0,0 +1,195 @@
+/*
+Copyright 2020 The Kubernetes Authors.
+
+Licensed under the Apache License, Version 2.0 (the "License");
+you may not use this file except in compliance with the License.
+You may obtain a copy of the License at
+
+    http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+See the License for the specific language governing permissions and
+limitations under the License.
+*/
+
+package nerdctl
+
+import (
+	"crypto/sha1"
+	"encoding/binary"
+	"fmt"
+	"net"
+	"strconv"
+	"strings"
+
+	"sigs.k8s.io/kind/pkg/errors"
+	"sigs.k8s.io/kind/pkg/exec"
+)
+
+// This may be overridden by KIND_EXPERIMENTAL_DOCKER_NETWORK env,
+// experimentally...
+//
+// By default currently picking a single network is equivalent to the previous
+// behavior *except* that we moved from the default bridge to a user defined
+// network because the default bridge is actually special versus any other
+// docker network and lacks the embedded DNS
+//
+// For now this also makes it easier for apps to join the same network, and
+// leaves users with complex networking desires to create and manage their own
+// networks.
+const fixedNetworkName = "kind"
+
+// ensureNetwork checks if docker network by name exists, if not it creates it
+func ensureNetwork(name, binaryName string) error {
+	// check if network exists already and remove any duplicate networks
+	exists, err := checkIfNetworkExists(name, binaryName)
+	if err != nil {
+		return err
+	}
+
+	// network already exists, we're good
+	// TODO: the network might already exist and not have ipv6 ... :|
+	// discussion: https://github.com/kubernetes-sigs/kind/pull/1508#discussion_r414594198
+	if exists {
+		return nil
+	}
+
+	// TODO: IPv6 support is coming in nerdctl v1.7.0 (https://github.com/containerd/nerdctl/pull/1558);
+	// for now we will disable IPv6 until that is released (and then may have to be conditional based
+	// on the nerdctl version)
+	//subnet := generateULASubnetFromName(name, 0)
+	var subnet string
+	mtu := getDefaultNetworkMTU(binaryName)
+	err = createNetwork(name, subnet, mtu, binaryName)
+	if err == nil {
+		// Success!
+		return nil
+	}
+
+	// On the first try check if ipv6 fails entirely on this machine
+	// https://github.com/kubernetes-sigs/kind/issues/1544
+	// Otherwise if it's not a pool overlap error, fail
+	// If it is, make more attempts below
+	if isIPv6UnavailableError(err) {
+		// only one attempt, IPAM is automatic in ipv4 only
+		return createNetwork(name, "", mtu, binaryName)
+	}
+	if isPoolOverlapError(err) {
+		// pool overlap suggests perhaps another process created the network
+		// check if network exists already and remove any duplicate networks
+		exists, err := checkIfNetworkExists(name, binaryName)
+		if err != nil {
+			return err
+		}
+		if exists {
+			return nil
+		}
+		// otherwise we'll start trying with different subnets
+	} else {
+		// unknown error ...
+		return err
+	}
+
+	// keep trying for ipv6 subnets
+	const maxAttempts = 5
+	for attempt := int32(1); attempt < maxAttempts; attempt++ {
+		subnet := generateULASubnetFromName(name, attempt)
+		err = createNetwork(name, subnet, mtu, binaryName)
+		if err == nil {
+			// success!
+			return nil
+		}
+		if isPoolOverlapError(err) {
+			// pool overlap suggests perhaps another process created the network
+			// check if network exists already and remove any duplicate networks
+			exists, err := checkIfNetworkExists(name, binaryName)
+			if err != nil {
+				return err
+			}
+			if exists {
+				return nil
+			}
+			// otherwise we'll try again
+			continue
+		}
+		// unknown error ...
+		return err
+	}
+	return errors.New("exhausted attempts trying to find a non-overlapping subnet")
+}
+
+func createNetwork(name, ipv6Subnet string, mtu int, binaryName string) error {
+	args := []string{"network", "create", "-d=bridge"}
+	// TODO: Not supported in nerdctl yet
+	//	"-o", "com.docker.network.bridge.enable_ip_masquerade=true",
+	if mtu > 0 {
+		args = append(args, "-o", fmt.Sprintf("com.docker.network.driver.mtu=%d", mtu))
+	}
+	if ipv6Subnet != "" {
+		args = append(args, "--ipv6", "--subnet", ipv6Subnet)
+	}
+	args = append(args, name)
+	return exec.Command(binaryName, args...).Run()
+}
+
+// getDefaultNetworkMTU obtains the MTU from the docker default network
+func getDefaultNetworkMTU(binaryName string) int {
+	cmd := exec.Command(binaryName, "network", "inspect", "bridge",
+		"-f", `{{ index .Options "com.docker.network.driver.mtu" }}`)
+	lines, err := exec.OutputLines(cmd)
+	if err != nil || len(lines) != 1 {
+		return 0
+	}
+	mtu, err := strconv.Atoi(lines[0])
+	if err != nil {
+		return 0
+	}
+	return mtu
+}
+
+func checkIfNetworkExists(name, binaryName string) (bool, error) {
+	out, err := exec.Output(exec.Command(
+		binaryName, "network", "inspect",
+		name, "--format={{.Name}}",
+	))
+	if err != nil {
+		return false, nil
+	}
+	return strings.HasPrefix(string(out), name), err
+}
+
+func isIPv6UnavailableError(err error) bool {
+	rerr := exec.RunErrorForError(err)
+	return rerr != nil && strings.HasPrefix(string(rerr.Output), "Error response from daemon: Cannot read IPv6 setup for bridge")
+}
+
+func isPoolOverlapError(err error) bool {
+	rerr := exec.RunErrorForError(err)
+	return rerr != nil && strings.HasPrefix(string(rerr.Output), "Error response from daemon: Pool overlaps with other one on this address space") || strings.Contains(string(rerr.Output), "networks have overlapping")
+}
+
+func deleteNetworks(binaryName string, networks ...string) error {
+	return exec.Command(binaryName, append([]string{"network", "rm"}, networks...)...).Run()
+}
+
+// generateULASubnetFromName generate an IPv6 subnet based on the
+// name and Nth probing attempt
+func generateULASubnetFromName(name string, attempt int32) string {
+	ip := make([]byte, 16)
+	ip[0] = 0xfc
+	ip[1] = 0x00
+	h := sha1.New()
+	_, _ = h.Write([]byte(name))
+	_ = binary.Write(h, binary.LittleEndian, attempt)
+	bs := h.Sum(nil)
+	for i := 2; i < 8; i++ {
+		ip[i] = bs[i]
+	}
+	subnet := &net.IPNet{
+		IP:   net.IP(ip),
+		Mask: net.CIDRMask(64, 128),
+	}
+	return subnet.String()
+}

pkg/cluster/internal/providers/nerdctl/network_integration_test.go

@@ -0,0 +1,83 @@
+//go:build !nointegration
+// +build !nointegration
+
+/*
+Copyright 2020 The Kubernetes Authors.
+
+Licensed under the Apache License, Version 2.0 (the "License");
+you may not use this file except in compliance with the License.
+You may obtain a copy of the License at
+
+    http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+See the License for the specific language governing permissions and
+limitations under the License.
+*/
+
+package nerdctl
+
+import (
+	"fmt"
+	"regexp"
+	"testing"
+
+	"sigs.k8s.io/kind/pkg/errors"
+	"sigs.k8s.io/kind/pkg/exec"
+
+	"sigs.k8s.io/kind/pkg/internal/integration"
+)
+
+func TestIntegrationEnsureNetworkConcurrent(t *testing.T) {
+	integration.MaybeSkip(t)
+
+	testNetworkName := "integration-test-ensure-kind-network"
+
+	// cleanup
+	cleanup := func() {
+		//ids, _ := networksWithName(testNetworkName, "nerdctl")
+		var ids []string
+		if len(ids) > 0 {
+			_ = deleteNetworks("nerdctl", ids...)
+		}
+	}
+	cleanup()
+	defer cleanup()
+
+	// this is more than enough to trigger race conditions
+	networkConcurrency := 10
+
+	// Create multiple networks concurrently
+	errCh := make(chan error, networkConcurrency)
+	for i := 0; i < networkConcurrency; i++ {
+		go func() {
+			errCh <- ensureNetwork(testNetworkName, "nerdctl")
+		}()
+	}
+	for i := 0; i < networkConcurrency; i++ {
+		if err := <-errCh; err != nil {
+			t.Errorf("error creating network: %v", err)
+			rerr := exec.RunErrorForError(err)
+			if rerr != nil {
+				t.Errorf("%q", rerr.Output)
+			}
+			t.Errorf("%+v", errors.StackTrace(err))
+		}
+	}
+
+	cmd := exec.Command(
+		"nerdctl", "network", "ls",
+		fmt.Sprintf("--filter=name=^%s$", regexp.QuoteMeta(testNetworkName)),
+		"--format={{.Name}}",
+	)
+
+	lines, err := exec.OutputLines(cmd)
+	if err != nil {
+		t.Errorf("obtaining the nerdctl networks")
+	}
+	if len(lines) != 1 {
+		t.Errorf("wrong number of networks created: %d", len(lines))
+	}
+}