Kola: update to handle torcx deprecation / docker and containerd sysext

kola/README.md

@@ -187,8 +187,8 @@ import (
         _ "github.com/flatcar/mantle/kola/tests/podman"
         _ "github.com/flatcar/mantle/kola/tests/rkt"
         _ "github.com/flatcar/mantle/kola/tests/rpmostree"
+        _ "github.com/flatcar/mantle/kola/tests/sysext"
         _ "github.com/flatcar/mantle/kola/tests/systemd"
-        _ "github.com/flatcar/mantle/kola/tests/torcx"
         _ "github.com/flatcar/mantle/kola/tests/update"
 )
 ```

kola/cluster/cluster.go

@@ -1,4 +1,5 @@
 // Copyright 2016 CoreOS, Inc.
+// Copyright 2023 by the Flatcar Maintainers
 //
 // Licensed under the Apache License, Version 2.0 (the "License");
 // you may not use this file except in compliance with the License.
@@ -21,10 +22,15 @@ import (
 	"path/filepath"
 	"strings"
 
+	"github.com/coreos/pkg/capnslog"
 	"github.com/flatcar/mantle/harness"
 	"github.com/flatcar/mantle/platform"
 )
 
+var (
+	logger = capnslog.NewPackageLogger("github.com/flatcar/mantle", "kola/cluster")
+)
+
 // TestCluster embedds a Cluster to provide platform independant helper
 // methods.
 type TestCluster struct {
@@ -56,6 +62,7 @@ func (t *TestCluster) Run(name string, f func(c TestCluster)) bool {
 // RunNative runs a registered NativeFunc on a remote machine
 func (t *TestCluster) RunNative(funcName string, m platform.Machine) bool {
 	command := fmt.Sprintf("./kolet run %q %q", t.H.Name(), funcName)
+	logger.Infof("RunNative: running command %s", command)
 	return t.Run(funcName, func(c TestCluster) {
 		client, err := m.SSHClient()
 		if err != nil {
@@ -110,11 +117,19 @@ func (t *TestCluster) DropFile(localPath string) error {
 // This ensures the output will be correctly accumulated under the correct
 // test.
 func (t *TestCluster) SSH(m platform.Machine, cmd string) ([]byte, error) {
+	logger.Infof("SSH: running command: %s", cmd)
 	stdout, stderr, err := m.SSH(cmd)
 
 	if len(stderr) > 0 {
 		for _, line := range strings.Split(string(stderr), "\n") {
 			t.Log(line)
+			logger.Debugf("SSH: stderr: %s", line)
+		}
+	}
+
+	if len(stdout) > 0 {
+		for _, line := range strings.Split(string(stdout), "\n") {
+			logger.Debugf("SSH: stdout: %s", line)
 		}
 	}
 

kola/registry/registry.go

@@ -18,7 +18,7 @@ import (
 	_ "github.com/flatcar/mantle/kola/tests/packages"
 	_ "github.com/flatcar/mantle/kola/tests/podman"
 	_ "github.com/flatcar/mantle/kola/tests/rpmostree"
+	_ "github.com/flatcar/mantle/kola/tests/sysext"
 	_ "github.com/flatcar/mantle/kola/tests/systemd"
-	_ "github.com/flatcar/mantle/kola/tests/torcx"
 	_ "github.com/flatcar/mantle/kola/tests/update"
 )

kola/tests/bpf/bpf.go

@@ -48,83 +48,78 @@ func init() {
 }
 
 func execsnoopTest(c cluster.TestCluster) {
-	c.Run("filter name and args", func(c cluster.TestCluster) {
-		m := c.Machines()[0]
-		containerName := "execsnoop"
-
-		// filter commands with `docker ps`
-		plog.Infof("running %s container", containerName)
-		cmd := fmt.Sprintf(cmdPrefix, containerName, "/usr/share/bcc/tools/execsnoop -n docker -l ps")
-		if _, err := c.SSH(m, cmd); err != nil {
-			c.Fatalf("unable to run SSH command '%s': %v", cmd, err)
+	m := c.Machines()[0]
+	containerName := "execsnoop"
+
+	// filter commands with `docker ps`
+	plog.Infof("running %s container", containerName)
+	cmd := fmt.Sprintf(cmdPrefix, containerName, "/usr/share/bcc/tools/execsnoop -n docker -l ps")
+	if _, err := c.SSH(m, cmd); err != nil {
+		c.Fatalf("unable to run SSH command '%s': %v", cmd, err)
+	}
+
+	// wait for the container and the `execsnoop` command to be correctly started before
+	// generating traffic.
+	if err := util.Retry(10, 2*time.Second, func() error {
+
+		// Run 'docker ps' to trigger log output. Execsnoop won't print anything, not even the header,
+		// before it's been triggered for the first time.
+		_ = c.MustSSH(m, "docker ps")
+
+		// we first assert that the container is running and then the process too.
+		// it's not possible to use `docker top...` command because it's the execsnoop itself who takes some time to start.
+		logs, err := c.SSH(m, fmt.Sprintf("sudo cat $(docker inspect --format='{{.LogPath}}' %s)", containerName))
+		if err != nil {
+			return fmt.Errorf("getting running process: %w", err)
 		}
 
-		// wait for the container and the `execsnoop` command to be correctly started before
-		// generating traffic.
-		if err := util.Retry(10, 2*time.Second, func() error {
-			_ = c.MustSSH(m, "docker ps")
-
-			// we first assert that the container is running and then the process too.
-			// it's not possible to use `docker top...` command because it's the execsnoop itself who takes some time to start.
-			logs, err := c.SSH(m, fmt.Sprintf("sudo cat $(docker inspect --format='{{.LogPath}}' %s)", containerName))
-			if err != nil {
-				return fmt.Errorf("getting running process: %w", err)
-			}
-
-			if len(logs) > 0 {
-				return nil
-			}
-
-			return fmt.Errorf("no logs, the service has not started yet properly")
-		}); err != nil {
-			c.Fatalf("unable to get container ready: %v", err)
+		if len(logs) > 0 {
+			return nil
 		}
 
-		// generate some "traffic"
-		_ = c.MustSSH(m, "docker info")
-		_ = c.MustSSH(m, fmt.Sprintf("docker logs %s", containerName))
-		_ = c.MustSSH(m, fmt.Sprintf("docker top %s", containerName))
+		return fmt.Errorf("no logs, the service has not started yet properly")
+	}); err != nil {
+		c.Fatalf("unable to get container ready: %v", err)
+	}
 
-		plog.Infof("getting logs from %s container", containerName)
+	// generate some "traffic"
+	_ = c.MustSSH(m, "docker info")
+	_ = c.MustSSH(m, fmt.Sprintf("docker logs %s", containerName))
+	_ = c.MustSSH(m, fmt.Sprintf("docker top %s", containerName))
+
+	plog.Infof("getting logs from %s container", containerName)
+	if err := util.Retry(10, 2*time.Second, func() error {
 		logs, err := c.SSH(m, fmt.Sprintf("sudo cat $(docker inspect --format='{{.LogPath}}' %s)", containerName))
 		if err != nil {
 			c.Fatalf("unable to run SSH command: %v", err)
 		}
-
-		l := Log{}
 		dockerLogs := bytes.Split(logs, []byte("\n"))
-		if len(dockerLogs) != 3 {
-			// we have the headers of the table
-			// then 2 lines for docker ps and the torcx call
-			c.Fatalf("docker logs should hold 3 values. Got: %d", len(dockerLogs))
+
+		// we have the headers of the table
+		// then 2 lines for docker ps and the torcx call if torcx is used
+		if len(dockerLogs) < 2 {
+			return fmt.Errorf("Waiting for execsnoop log entries")
 		}
 
-		execFound := false
+		l := Log{}
 		for _, log := range dockerLogs {
-			if len(log) == 0 {
-				continue
-			}
 
 			if err := json.Unmarshal(log, &l); err != nil {
-				c.Fatalf("unable to unmarshal log: %v", err)
+				return fmt.Errorf("Transient error unmarshalling docker log: %v", err)
 			}
-			plog.Infof("handling log %v", l)
-
 			if l.Stream == "stderr" {
-				c.Fatal("stream should not log to 'stderr'")
+				return fmt.Errorf("Transient error: stream should not log to 'stderr'")
 			}
-
 			if strings.Contains(l.Log, "docker info") || strings.Contains(l.Log, "docker top") || strings.Contains(l.Log, "docker logs") {
 				c.Fatal("log should not contain 'docker info' or 'docker top' or 'docker logs'")
 			}
 
 			if strings.Contains(l.Log, "docker ps") {
-				execFound = true
+				return nil
 			}
 		}
-
-		if !execFound {
-			c.Fatal("did not get 'docker ps' in the logs")
-		}
-	})
+		return fmt.Errorf("Waiting for execsnoop log entries")
+	}); err != nil {
+		c.Fatalf("Unable to find 'docker ps' log lines in execsnoop logs: %v", err)
+	}
 }

kola/tests/docker/enable_docker.go

@@ -0,0 +1,83 @@
+// Copyright 2017 CoreOS, Inc.
+// Copyright 2023 The Flatcar Maintainers.
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//     http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+// This test originated as torcx test and is kept to shield against
+// similar activation issues with sysext.
+
+// The test ensures that, given respective user configuration, Docker is started
+// at boot (instead just socket-activated). This is necessary for auto-restarting
+// containers at boot that have been running at shutodwn. That's done by docker,
+// so docker must be running.
+
+package docker
+
+import (
+	"github.com/coreos/go-semver/semver"
+	"github.com/flatcar/mantle/kola/cluster"
+	"github.com/flatcar/mantle/kola/register"
+	"github.com/flatcar/mantle/platform/conf"
+)
+
+func init() {
+	// Regression test for https://github.com/coreos/bugs/issues/2079
+	register.Register(&register.Test{
+		Run:         dockerEnable,
+		ClusterSize: 1,
+		// This test is normally not related to the cloud environment
+		Platforms:  []string{"qemu", "qemu-unpriv"},
+		Name:       "docker.enable-service.torcx",
+		EndVersion: semver.Version{Major: 3745},
+		UserData: conf.Butane(`
+variant: flatcar
+version: 1.0.0
+systemd:
+  units:
+  - name: docker.service
+    enabled: true
+`),
+		Distros: []string{"cl"},
+	})
+
+	register.Register(&register.Test{
+		Run:         dockerEnable,
+		ClusterSize: 1,
+		// This test is normally not related to the cloud environment
+		Platforms:  []string{"qemu", "qemu-unpriv"},
+		Name:       "docker.enable-service.sysext",
+		MinVersion: semver.Version{Major: 3746},
+		UserData: conf.Butane(`
+variant: flatcar
+version: 1.0.0
+systemd:
+  units:
+  - name: docker.service
+    enabled: true
+storage:
+  links:
+  - path: /etc/systemd/system/multi-user.target.wants/docker.service
+    target: /usr/lib/systemd/system/docker.service
+    hard: false
+    overwrite: true
+`),
+		// TODO FIXME: Convert this to a multi-user.target.upholds/docker.service symlink
+		// after we switch to systemd-254.
+		Distros: []string{"cl"},
+	})
+}
+
+func dockerEnable(c cluster.TestCluster) {
+	m := c.Machines()[0]
+        c.AssertCmdOutputContains(m, "systemctl is-enabled docker", "enabled")
+}

kola/tests/docker/torcx_manifest_pkgs.go

@@ -37,7 +37,8 @@ func init() {
 		ExcludePlatforms: []string{"do"},
 		Distros:          []string{"cl"},
 		// This test is normally not related to the cloud environment
-		Platforms: []string{"qemu", "qemu-unpriv"},
+		Platforms:  []string{"qemu", "qemu-unpriv"},
+		EndVersion: semver.Version{Major: 3745},
 		SkipFunc: func(version semver.Version, channel, arch, platform string) bool {
 			// LTS (3033) does not have the network-kargs service pulled in:
 			// https://github.com/flatcar/coreos-overlay/pull/1848/commits/9e04bc12c3c7eb38da05173dc0ff7beaefa13446

kola/tests/sysext/disable_containerd.go

@@ -0,0 +1,47 @@
+// Copyright 2023 The Flatcar Maintainers.
+// SPDX-License-Identifier: Apache-2.0
+
+package sysext
+
+import (
+	"github.com/coreos/go-semver/semver"
+	"github.com/flatcar/mantle/kola/cluster"
+	"github.com/flatcar/mantle/kola/register"
+	"github.com/flatcar/mantle/platform/conf"
+)
+
+func init() {
+	register.Register(&register.Test{
+		Run:         containerdDisable,
+		ClusterSize: 1,
+		Platforms:   []string{"qemu", "qemu-unpriv"},
+		Name:        "sysext.disable-containerd",
+		// Only releases after 3745 ship sysext
+		MinVersion: semver.Version{Major: 3746},
+		// We also disable our vendor docker sysext since it depends on the containerd sysext.
+		UserData: conf.Butane(`
+variant: flatcar
+version: 1.0.0
+storage:
+  links:
+  - path: /etc/extensions/containerd-flatcar.raw
+    target: /dev/null
+    hard: false
+    overwrite: true
+  - path: /etc/extensions/docker-flatcar.raw
+    target: /dev/null
+    hard: false
+    overwrite: true
+`),
+		Distros: []string{"cl"},
+	})
+}
+
+func containerdDisable(c cluster.TestCluster) {
+	m := c.Machines()[0]
+	output := c.MustSSH(m,
+		`test -f /usr/bin/containerd && echo "ERROR" || true`)
+	if string(output) == "ERROR" {
+		c.Errorf("/usr/bin/containerd exists even when sysext is disabled!")
+	}
+}

kola/tests/sysext/disable_docker.go

@@ -0,0 +1,42 @@
+// Copyright 2023 The Flatcar Maintainers.
+// SPDX-License-Identifier: Apache-2.0
+
+package sysext
+
+import (
+	"github.com/coreos/go-semver/semver"
+	"github.com/flatcar/mantle/kola/cluster"
+	"github.com/flatcar/mantle/kola/register"
+	"github.com/flatcar/mantle/platform/conf"
+)
+
+func init() {
+	register.Register(&register.Test{
+		Run:         dockerDisable,
+		ClusterSize: 1,
+		Platforms:   []string{"qemu", "qemu-unpriv"},
+		Name:        "sysext.disable-docker",
+		// Only releases after 3745 ship sysext
+		MinVersion: semver.Version{Major: 3746},
+		UserData: conf.Butane(`
+variant: flatcar
+version: 1.0.0
+storage:
+  links:
+  - path: /etc/extensions/docker-flatcar.raw
+    target: /dev/null
+    hard: false
+    overwrite: true
+`),
+		Distros: []string{"cl"},
+	})
+}
+
+func dockerDisable(c cluster.TestCluster) {
+	m := c.Machines()[0]
+	output := c.MustSSH(m,
+		`test -f /usr/bin/docker && echo "ERROR" || true`)
+	if string(output) == "ERROR" {
+		c.Errorf("/usr/bin/docker exists even when sysext is disabled!")
+	}
+}