tests: run remote worker on tcp and container endpoints

Signed-off-by: CrazyMax <1951866+crazy-max@users.noreply.github.com>

Author: crazy-max

.github/workflows/build.yml

@@ -48,13 +48,14 @@ jobs:
           - v0.11.6
         worker:
           - docker-container
-          - remote
+          - remote+docker-container
+          - remote+tcp
         pkg:
           - ./tests
         include:
           - worker: docker
             pkg: ./tests
-          - worker: docker\+containerd  # same as docker, but with containerd snapshotter
+          - worker: docker+containerd  # same as docker, but with containerd snapshotter
             pkg: ./tests
     steps:
       -
@@ -64,6 +65,18 @@ jobs:
           if [ -n "${{ matrix.buildkit }}" ]; then
             echo "TEST_BUILDKIT_TAG=${{ matrix.buildkit }}" >> $GITHUB_ENV
           fi
+          testFlags="--run=//worker=$(echo "${{ matrix.worker }}" | sed 's/\+/\\+/g')$"
+          case "${{ matrix.worker }}" in
+            docker | docker+containerd | docker@* | docker+containerd@* | remote*)
+              echo "TESTFLAGS=${{ env.TESTFLAGS_DOCKER }} $testFlags" >> $GITHUB_ENV
+              ;;
+            *)
+              echo "TESTFLAGS=${{ env.TESTFLAGS }} $testFlags" >> $GITHUB_ENV
+              ;;
+          esac
+          if [[ "${{ matrix.worker }}" == "docker"* ]]; then
+            echo "TEST_DOCKERD=1" >> $GITHUB_ENV
+          fi
       -
         name: Checkout
         uses: actions/checkout@v4
@@ -92,8 +105,6 @@ jobs:
           ./hack/test
         env:
           TEST_REPORT_SUFFIX: "-${{ env.TESTREPORTS_NAME }}"
-          TEST_DOCKERD: "${{ startsWith(matrix.worker, 'docker') && '1' || '0' }}"
-          TESTFLAGS: "${{ (matrix.worker == 'docker' || matrix.worker == 'docker\\+containerd') && env.TESTFLAGS_DOCKER || env.TESTFLAGS }} --run=//worker=${{ matrix.worker }}$"
           TESTPKGS: "${{ matrix.pkg }}"
       -
         name: Send to Codecov

Dockerfile

@@ -7,15 +7,13 @@ ARG XX_VERSION=1.4.0
 ARG DOCKER_VERSION=25.0.2
 ARG GOTESTSUM_VERSION=v1.9.0
 ARG REGISTRY_VERSION=2.8.0
-ARG BUILDKIT_VERSION=v0.13.0
 ARG UNDOCK_VERSION=0.7.0
 
 FROM --platform=$BUILDPLATFORM tonistiigi/xx:${XX_VERSION} AS xx
 FROM --platform=$BUILDPLATFORM golang:${GO_VERSION}-alpine AS golatest
 FROM moby/moby-bin:$DOCKER_VERSION AS docker-engine
 FROM dockereng/cli-bin:$DOCKER_VERSION AS docker-cli
 FROM registry:$REGISTRY_VERSION AS registry
-FROM moby/buildkit:$BUILDKIT_VERSION AS buildkit
 FROM crazymax/undock:$UNDOCK_VERSION AS undock
 
 FROM golatest AS gobase
@@ -92,8 +90,6 @@ COPY --link --from=gotestsum /out/gotestsum /usr/bin/
 COPY --link --from=registry /bin/registry /usr/bin/
 COPY --link --from=docker-engine / /usr/bin/
 COPY --link --from=docker-cli / /usr/bin/
-COPY --link --from=buildkit /usr/bin/buildkitd /usr/bin/
-COPY --link --from=buildkit /usr/bin/buildctl /usr/bin/
 COPY --link --from=undock /usr/local/bin/undock /usr/bin/
 COPY --link --from=binaries /buildx /usr/bin/
 

docker-bake.hcl

@@ -180,17 +180,13 @@ variable "HTTPS_PROXY" {
 variable "NO_PROXY" {
   default = ""
 }
-variable "TEST_BUILDKIT_TAG" {
-  default = null
-}
 
 target "integration-test-base" {
   inherits = ["_common"]
   args = {
     HTTP_PROXY = HTTP_PROXY
     HTTPS_PROXY = HTTPS_PROXY
     NO_PROXY = NO_PROXY
-    BUILDKIT_VERSION = TEST_BUILDKIT_TAG
   }
   target = "integration-test-base"
   output = ["type=cacheonly"]

tests/workers/docker.go

@@ -24,7 +24,8 @@ func InitDockerWorker() {
 type dockerWorker struct {
 	id                    string
 	containerdSnapshotter bool
-	unsupported           []string
+
+	unsupported []string
 }
 
 func (c dockerWorker) Name() string {

tests/workers/remote.go

@@ -2,24 +2,41 @@ package workers
 
 import (
 	"context"
+	"fmt"
+	"net"
 	"os"
 	"os/exec"
+	"sync"
 
 	"github.com/moby/buildkit/identity"
 	"github.com/moby/buildkit/util/testutil/integration"
-	bkworkers "github.com/moby/buildkit/util/testutil/workers"
 	"github.com/pkg/errors"
 )
 
+var protos = []string{
+	"docker-container",
+	"tcp",
+}
+
 func InitRemoteWorker() {
-	integration.Register(&remoteWorker{
-		id: "remote",
-	})
+	for _, p := range protos {
+		integration.Register(&remoteWorker{
+			id:    "remote+" + p,
+			proto: p,
+		})
+	}
 }
 
 type remoteWorker struct {
-	id          string
+	id    string
+	proto string
+
 	unsupported []string
+
+	docker      integration.Backend
+	dockerClose func() error
+	dockerErr   error
+	dockerOnce  sync.Once
 }
 
 func (w remoteWorker) Name() string {
@@ -35,32 +52,71 @@ func (w remoteWorker) NetNSDetached() bool {
 }
 
 func (w remoteWorker) New(ctx context.Context, cfg *integration.BackendConfig) (b integration.Backend, cl func() error, err error) {
-	oci := bkworkers.OCI{ID: w.id}
-	bk, bkclose, err := oci.New(ctx, cfg)
-	if err != nil {
-		return bk, cl, err
+	w.dockerOnce.Do(func() {
+		w.docker, w.dockerClose, w.dockerErr = dockerWorker{id: w.id}.New(ctx, cfg)
+	})
+	if w.dockerErr != nil {
+		return w.docker, w.dockerClose, w.dockerErr
 	}
 
+	bkCtnName := "buildkit-integration-" + identity.NewID()
 	name := "integration-remote-" + identity.NewID()
+	envs := append(
+		os.Environ(),
+		"BUILDX_CONFIG=/tmp/buildx-"+name,
+		"DOCKER_CONTEXT="+w.docker.DockerAddress(),
+	)
+
+	// random host port for buildkit container
+	l, _ := net.Listen("tcp", ":0") //nolint:gosec
+	_ = l.Close()
+	bkPort := l.Addr().(*net.TCPAddr).Port
+
+	// create buildkit container
+	bkCtnCmd := exec.Command("docker", "run",
+		"-d", "--rm",
+		"--privileged",
+		"-p", fmt.Sprintf("%d:1234", bkPort),
+		"--name="+bkCtnName,
+		"moby/buildkit:buildx-stable-1",
+		"--addr=tcp://0.0.0.0:1234",
+	)
+	bkCtnCmd.Env = envs
+	if out, err := bkCtnCmd.CombinedOutput(); err != nil {
+		return nil, nil, errors.Wrapf(err, "failed to create buildkit container %s: %s", bkCtnName, string(out))
+	}
+
+	// create builder
+	var endpoint string
+	switch w.proto {
+	case "docker-container":
+		endpoint = fmt.Sprintf("docker-container://%s", bkCtnName)
+	case "tcp":
+		endpoint = fmt.Sprintf("tcp://localhost:%d", bkPort)
+	default:
+		return nil, nil, errors.Errorf("unsupported protocol %s", w.proto)
+	}
 	cmd := exec.Command("buildx", "create",
 		"--bootstrap",
 		"--name="+name,
 		"--driver=remote",
-		bk.Address(),
+		endpoint,
 	)
-	cmd.Env = append(os.Environ(), "BUILDX_CONFIG=/tmp/buildx-"+name)
-	if err := cmd.Run(); err != nil {
-		return nil, nil, errors.Wrapf(err, "failed to create buildx instance %s", name)
+	cmd.Env = envs
+	if out, err := cmd.CombinedOutput(); err != nil {
+		return nil, nil, errors.Wrapf(err, "failed to create buildx instance %s: %s", name, string(out))
 	}
 
 	cl = func() error {
-		var err error
-		if err1 := bkclose(); err == nil {
-			err = err1
+		cmd := exec.Command("docker", "container", "rm", "-f", name)
+		cmd.Env = envs
+		if err1 := cmd.Run(); err1 != nil {
+			err = errors.Wrapf(err1, "failed to remove buildkit container %s", bkCtnName)
 		}
-		cmd := exec.Command("buildx", "rm", "-f", name)
-		if err1 := cmd.Run(); err == nil {
-			err = err1
+		cmd = exec.Command("buildx", "rm", "-f", name)
+		cmd.Env = envs
+		if err1 := cmd.Run(); err1 != nil {
+			err = errors.Wrapf(err1, "failed to remove buildx instance %s", name)
 		}
 		return err
 	}