feat: Kurtosis backend can now connect to a remote Docker backend (#285)

## Description:
This PR adds a specific type of backend consisting of both the local
Docker backend and a remote Kurtosis backend (which is effectively
Docker). CLI and Engine are using this specific backend to be able to
offload some containers to the remote backend based on the current
context config

## Is this change user facing?
NO -- since we don't have the possibility to switch backend right now,
this is effectively not exposed
<!-- If yes, please add the "user facing" label to the PR -->
<!-- If yes, don't forget to include docs changes where relevant -->

## References (if applicable):
<!-- Add relevant Github Issues, Discord threads, or other helpful
information. -->

cli/cli/go.mod

@@ -47,7 +47,6 @@ require (
 )
 
 require (
-	github.com/Azure/go-ansiterm v0.0.0-20210617225240-d185dfc1b5a1 // indirect
 	github.com/Microsoft/go-winio v0.4.17 // indirect
 	github.com/chzyer/readline v0.0.0-20180603132655-2972be24d48e // indirect
 	github.com/davecgh/go-spew v1.1.1 // indirect
@@ -86,6 +85,7 @@ require (
 	golang.org/x/mod v0.6.0-dev.0.20220419223038-86c51ed26bb4 // indirect
 	golang.org/x/net v0.7.0 // indirect
 	golang.org/x/oauth2 v0.0.0-20211104180415-d3ed0bb246c8 // indirect
+	golang.org/x/sync v0.0.0-20220722155255-886fb9371eb4 // indirect
 	golang.org/x/sys v0.5.0 // indirect
 	golang.org/x/term v0.5.0 // indirect
 	golang.org/x/text v0.7.0 // indirect

cli/cli/go.sum

@@ -36,8 +36,7 @@ cloud.google.com/go/storage v1.6.0/go.mod h1:N7U0C8pVQ/+NIKOBQyamJIeKQKkZ+mxpohl
 cloud.google.com/go/storage v1.8.0/go.mod h1:Wv1Oy7z6Yz3DshWRJFhqM/UCfaWIRTdp0RXyy7KQOVs=
 cloud.google.com/go/storage v1.10.0/go.mod h1:FLPqc6j+Ki4BU591ie1oL6qBQGu2Bl/tZ9ullr3+Kg0=
 dmitri.shuralyov.com/gpu/mtl v0.0.0-20190408044501-666a987793e9/go.mod h1:H6x//7gZCb22OMCxBHrMx7a5I7Hp++hsVxbQ4BYO7hU=
-github.com/Azure/go-ansiterm v0.0.0-20210617225240-d185dfc1b5a1 h1:UQHMgLO+TxOElx5B5HZ4hJQsoJ/PvUvKRhJHDQXO8P8=
-github.com/Azure/go-ansiterm v0.0.0-20210617225240-d185dfc1b5a1/go.mod h1:xomTg63KZ2rFqZQzSB4Vz2SUXa1BpHTVz9L5PTmPC4E=
+github.com/Azure/go-ansiterm v0.0.0-20230124172434-306776ec8161 h1:L/gRVlceqvL25UVaW/CKtUDjefjrs0SPonmDGUVOYP0=
 github.com/Azure/go-autorest v14.2.0+incompatible/go.mod h1:r+4oMnoxhatjLLJ6zxSWATqVooLgysK6ZNox3g/xq24=
 github.com/Azure/go-autorest/autorest v0.11.18/go.mod h1:dSiJPy22c3u0OtOKDNttNgqpNFY/GeWa7GH/Pz56QRA=
 github.com/Azure/go-autorest/autorest/adal v0.9.13/go.mod h1:W/MM4U6nLxnIskrw4UwWzlHfGjwUS50aOsc/I3yuU8M=
@@ -486,6 +485,8 @@ golang.org/x/sync v0.0.0-20200625203802-6e8e738ad208/go.mod h1:RxMgew5VJxzue5/jJ
 golang.org/x/sync v0.0.0-20201020160332-67f06af15bc9/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM=
 golang.org/x/sync v0.0.0-20201207232520-09787c993a3a/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM=
 golang.org/x/sync v0.0.0-20210220032951-036812b2e83c/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM=
+golang.org/x/sync v0.0.0-20220722155255-886fb9371eb4 h1:uVc8UZUe6tr40fFVnUP5Oj+veunVezqYl9z7DYw9xzw=
+golang.org/x/sync v0.0.0-20220722155255-886fb9371eb4/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM=
 golang.org/x/sys v0.0.0-20180830151530-49385e6e1522/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY=
 golang.org/x/sys v0.0.0-20180909124046-d0be0721c37e/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY=
 golang.org/x/sys v0.0.0-20181122145206-62eef0e2fa9b/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY=
@@ -536,7 +537,6 @@ golang.org/x/sys v0.0.0-20210330210617-4fbd30eecc44/go.mod h1:h1NjWce9XRLGQEsW7w
 golang.org/x/sys v0.0.0-20210423082822-04245dca01da/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
 golang.org/x/sys v0.0.0-20210510120138-977fb7262007/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg=
 golang.org/x/sys v0.0.0-20210615035016-665e8c7367d1/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg=
-golang.org/x/sys v0.0.0-20210616094352-59db8d763f22/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg=
 golang.org/x/sys v0.0.0-20210630005230-0f9fa26af87c/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg=
 golang.org/x/sys v0.0.0-20211025201205-69cdffdb9359/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg=
 golang.org/x/sys v0.0.0-20211216021012-1d35b9e2eb4e/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg=

cli/cli/helpers/engine_manager/engine_existence_guarantor.go

@@ -46,6 +46,8 @@ type engineExistenceGuarantor struct {
 
 	engineServerKurtosisBackendConfigSupplier engine_server_launcher.KurtosisBackendConfigSupplier
 
+	kurtosisRemoteBackendConfigSupplier *engine_server_launcher.KurtosisRemoteBackendConfigSupplier
+
 	engineServerLauncher *engine_server_launcher.EngineServerLauncher
 
 	imageVersionTag string
@@ -70,6 +72,7 @@ func newEngineExistenceGuarantorWithDefaultVersion(
 	kurtosisBackend backend_interface.KurtosisBackend,
 	shouldSendMetrics bool,
 	engineServerKurtosisBackendConfigSupplier engine_server_launcher.KurtosisBackendConfigSupplier,
+	kurtosisRemoteBackendConfigSupplier *engine_server_launcher.KurtosisRemoteBackendConfigSupplier,
 	logLevel logrus.Level,
 	maybeCurrentlyRunningEngineVersionTag string,
 	kurtosisClusterType resolved_config.KurtosisClusterType,
@@ -80,6 +83,7 @@ func newEngineExistenceGuarantorWithDefaultVersion(
 		kurtosisBackend,
 		shouldSendMetrics,
 		engineServerKurtosisBackendConfigSupplier,
+		kurtosisRemoteBackendConfigSupplier,
 		defaultEngineImageVersionTag,
 		logLevel,
 		maybeCurrentlyRunningEngineVersionTag,
@@ -93,6 +97,7 @@ func newEngineExistenceGuarantorWithCustomVersion(
 	kurtosisBackend backend_interface.KurtosisBackend,
 	shouldSendMetrics bool,
 	engineServerKurtosisBackendConfigSupplier engine_server_launcher.KurtosisBackendConfigSupplier,
+	kurtosisRemoteBackendConfigSupplier *engine_server_launcher.KurtosisRemoteBackendConfigSupplier,
 	imageVersionTag string,
 	logLevel logrus.Level,
 	maybeCurrentlyRunningEngineVersionTag string,
@@ -103,6 +108,7 @@ func newEngineExistenceGuarantorWithCustomVersion(
 		preVisitingMaybeHostMachineIpAndPort: preVisitingMaybeHostMachineIpAndPort,
 		kurtosisBackend:                      kurtosisBackend,
 		engineServerKurtosisBackendConfigSupplier: engineServerKurtosisBackendConfigSupplier,
+		kurtosisRemoteBackendConfigSupplier:       kurtosisRemoteBackendConfigSupplier,
 		engineServerLauncher:                      engine_server_launcher.NewEngineServerLauncher(kurtosisBackend),
 		imageVersionTag:                           imageVersionTag,
 		logLevel:                                  logLevel,
@@ -137,6 +143,7 @@ func (guarantor *engineExistenceGuarantor) VisitStopped() error {
 			metricsUserId,
 			guarantor.shouldSendMetrics,
 			guarantor.engineServerKurtosisBackendConfigSupplier,
+			guarantor.kurtosisRemoteBackendConfigSupplier,
 		)
 	} else {
 		_, _, engineLaunchErr = guarantor.engineServerLauncher.LaunchWithCustomVersion(
@@ -148,6 +155,7 @@ func (guarantor *engineExistenceGuarantor) VisitStopped() error {
 			metricsUserId,
 			guarantor.shouldSendMetrics,
 			guarantor.engineServerKurtosisBackendConfigSupplier,
+			guarantor.kurtosisRemoteBackendConfigSupplier,
 		)
 	}
 	if engineLaunchErr != nil {

cli/cli/helpers/engine_manager/engine_manager.go

@@ -29,6 +29,7 @@ type EngineManager struct {
 	kurtosisBackend                           backend_interface.KurtosisBackend
 	shouldSendMetrics                         bool
 	engineServerKurtosisBackendConfigSupplier engine_server_launcher.KurtosisBackendConfigSupplier
+	remoteBackendConfigSupplier               *engine_server_launcher.KurtosisRemoteBackendConfigSupplier
 	clusterConfig                             *resolved_config.KurtosisClusterConfig
 	// Make engine IP, port, and protocol configurable in the future
 }
@@ -71,12 +72,14 @@ func NewEngineManager(ctx context.Context) (*EngineManager, error) {
 		return nil, stacktrace.Propagate(err, "An error occurred getting the Kurtosis backend for cluster '%v'", clusterName)
 	}
 	engineBackendConfigSupplier := clusterConfig.GetEngineBackendConfigSupplier()
+	remoteBackendConfigSupplier := clusterConfig.GetKurtosisRemoteBackendConfigSupplier()
 
 	return &EngineManager{
 		kurtosisBackend:   kurtosisBackend,
 		shouldSendMetrics: kurtosisConfig.GetShouldSendMetrics(),
 		engineServerKurtosisBackendConfigSupplier: engineBackendConfigSupplier,
-		clusterConfig: clusterConfig,
+		remoteBackendConfigSupplier:               remoteBackendConfigSupplier,
+		clusterConfig:                             clusterConfig,
 	}, nil
 }
 
@@ -143,6 +146,7 @@ func (manager *EngineManager) StartEngineIdempotentlyWithDefaultVersion(ctx cont
 		manager.kurtosisBackend,
 		manager.shouldSendMetrics,
 		manager.engineServerKurtosisBackendConfigSupplier,
+		manager.remoteBackendConfigSupplier,
 		logLevel,
 		engineVersion,
 		clusterType,
@@ -170,6 +174,7 @@ func (manager *EngineManager) StartEngineIdempotentlyWithCustomVersion(ctx conte
 		manager.kurtosisBackend,
 		manager.shouldSendMetrics,
 		manager.engineServerKurtosisBackendConfigSupplier,
+		manager.remoteBackendConfigSupplier,
 		engineImageVersionTag,
 		logLevel,
 		engineVersion,

cli/cli/kurtosis_config/resolved_config/kurtosis_cluster_config.go

@@ -4,7 +4,9 @@ import (
 	"context"
 	v2 "github.com/kurtosis-tech/kurtosis/cli/cli/kurtosis_config/overrides_objects/v2"
 	"github.com/kurtosis-tech/kurtosis/container-engine-lib/lib/backend_impls/docker/docker_kurtosis_backend/backend_creator"
+	"github.com/kurtosis-tech/kurtosis/container-engine-lib/lib/backend_impls/remote_context_backend"
 	"github.com/kurtosis-tech/kurtosis/container-engine-lib/lib/backend_interface"
+	"github.com/kurtosis-tech/kurtosis/contexts-config-store/store"
 	"github.com/kurtosis-tech/kurtosis/engine/launcher/engine_server_launcher"
 	"github.com/kurtosis-tech/stacktrace"
 	"strings"
@@ -20,9 +22,10 @@ var dockerBackendApiContainerModeArgs *backend_creator.APIContainerModeArgs = ni
 type kurtosisBackendSupplier func(ctx context.Context) (backend_interface.KurtosisBackend, error)
 
 type KurtosisClusterConfig struct {
-	kurtosisBackendSupplier     kurtosisBackendSupplier
-	engineBackendConfigSupplier engine_server_launcher.KurtosisBackendConfigSupplier
-	clusterType                 KurtosisClusterType
+	kurtosisBackendSupplier             kurtosisBackendSupplier
+	engineBackendConfigSupplier         engine_server_launcher.KurtosisBackendConfigSupplier
+	kurtosisRemoteBackendConfigSupplier *engine_server_launcher.KurtosisRemoteBackendConfigSupplier
+	clusterType                         KurtosisClusterType
 }
 
 func NewKurtosisClusterConfigFromOverrides(clusterId string, overrides *v2.KurtosisClusterConfigV2) (*KurtosisClusterConfig, error) {
@@ -41,15 +44,16 @@ func NewKurtosisClusterConfigFromOverrides(clusterId string, overrides *v2.Kurto
 		)
 	}
 
-	backendSupplier, engineBackendConfigSupplier, err := getSuppliers(clusterId, clusterType, overrides.Config)
+	backendSupplier, engineBackendConfigSupplier, kurtosisRemoteBackendConfigSupplier, err := getSuppliers(clusterId, clusterType, overrides.Config)
 	if err != nil {
 		return nil, stacktrace.Propagate(err, "An error occurred getting the suppliers that cluster '%v' will use", clusterId)
 	}
 
 	return &KurtosisClusterConfig{
-		kurtosisBackendSupplier:     backendSupplier,
-		engineBackendConfigSupplier: engineBackendConfigSupplier,
-		clusterType:                 clusterType,
+		kurtosisBackendSupplier:             backendSupplier,
+		engineBackendConfigSupplier:         engineBackendConfigSupplier,
+		kurtosisRemoteBackendConfigSupplier: kurtosisRemoteBackendConfigSupplier,
+		clusterType:                         clusterType,
 	}, nil
 }
 
@@ -65,6 +69,10 @@ func (clusterConfig *KurtosisClusterConfig) GetEngineBackendConfigSupplier() eng
 	return clusterConfig.engineBackendConfigSupplier
 }
 
+func (clusterConfig *KurtosisClusterConfig) GetKurtosisRemoteBackendConfigSupplier() *engine_server_launcher.KurtosisRemoteBackendConfigSupplier {
+	return clusterConfig.kurtosisRemoteBackendConfigSupplier
+}
+
 func (clusterConfig *KurtosisClusterConfig) GetClusterType() KurtosisClusterType {
 	return clusterConfig.clusterType
 }
@@ -77,21 +85,29 @@ func (clusterConfig *KurtosisClusterConfig) GetClusterType() KurtosisClusterType
 func getSuppliers(clusterId string, clusterType KurtosisClusterType, kubernetesConfig *v2.KubernetesClusterConfigV2) (
 	kurtosisBackendSupplier,
 	engine_server_launcher.KurtosisBackendConfigSupplier,
+	*engine_server_launcher.KurtosisRemoteBackendConfigSupplier,
 	error,
 ) {
+	kurtosisRemoteBackendConfigSupplier := engine_server_launcher.NewKurtosisRemoteBackendConfigSupplier(store.GetContextsConfigStore().GetCurrentContext)
+
 	var backendSupplier kurtosisBackendSupplier
 	var engineConfigSupplier engine_server_launcher.KurtosisBackendConfigSupplier
 	switch clusterType {
 	case KurtosisClusterType_Docker:
 		if kubernetesConfig != nil {
-			return nil, nil, stacktrace.NewError(
+			return nil, nil, nil, stacktrace.NewError(
 				"Cluster '%v' defines cluster config, but config must not be provided when cluster type is '%v'",
 				clusterId,
 				clusterType.String(),
 			)
 		}
+
 		backendSupplier = func(_ context.Context) (backend_interface.KurtosisBackend, error) {
-			backend, err := backend_creator.GetLocalDockerKurtosisBackend(dockerBackendApiContainerModeArgs)
+			kurtosisRemoteBackendConfigMaybe, err := kurtosisRemoteBackendConfigSupplier.GetOptionalRemoteConfig()
+			if err != nil {
+				return nil, stacktrace.Propagate(err, "Error building optional remote Kurtosis backend config")
+			}
+			backend, err := remote_context_backend.GetContextAwareKurtosisBackend(kurtosisRemoteBackendConfigMaybe, dockerBackendApiContainerModeArgs)
 			if err != nil {
 				return nil, stacktrace.Propagate(err, "An error occurred creating the Docker Kurtosis backend")
 			}
@@ -101,14 +117,14 @@ func getSuppliers(clusterId string, clusterType KurtosisClusterType, kubernetesC
 		engineConfigSupplier = engine_server_launcher.NewDockerKurtosisBackendConfigSupplier()
 	case KurtosisClusterType_Kubernetes:
 		if kubernetesConfig == nil {
-			return nil, nil, stacktrace.NewError(
+			return nil, nil, nil, stacktrace.NewError(
 				"Cluster '%v' doesn't define cluster config, but config must be provided when cluster type is '%v'",
 				clusterId,
 				clusterType.String(),
 			)
 		}
 		if kubernetesConfig.KubernetesClusterName == nil {
-			return nil, nil, stacktrace.NewError(
+			return nil, nil, nil, stacktrace.NewError(
 				"Type of cluster '%v' is '%v' but has no Kubernetes cluster name in its config map",
 				clusterId,
 				clusterType,
@@ -119,7 +135,7 @@ func getSuppliers(clusterId string, clusterType KurtosisClusterType, kubernetesC
 		_ = *kubernetesConfig.KubernetesClusterName
 
 		if kubernetesConfig.StorageClass == nil {
-			return nil, nil, stacktrace.NewError(
+			return nil, nil, nil, stacktrace.NewError(
 				"Type of cluster '%v' is '%v' but no storage class was defined in the config",
 				clusterId,
 				clusterType,
@@ -133,6 +149,16 @@ func getSuppliers(clusterId string, clusterType KurtosisClusterType, kubernetesC
 		}
 
 		backendSupplier = func(ctx context.Context) (backend_interface.KurtosisBackend, error) {
+			kurtosisRemoteBackendConfigMaybe, err := kurtosisRemoteBackendConfigSupplier.GetOptionalRemoteConfig()
+			if err != nil {
+				return nil, stacktrace.Propagate(err, "Error building optional remote Kurtosis backend config")
+			}
+			if kurtosisRemoteBackendConfigMaybe != nil {
+				return nil, stacktrace.NewError("Using a Remote Kurtosis Backend isn't allowed with Kubernetes. " +
+					"Either switch to a local only context to use Kubernetes or switch the cluster to Docker to " +
+					"connect to a remote Kurtosis backend")
+			}
+
 			pluginPath := backend_interface.GetPluginPathForCLI(backend_interface.KubernetesPluginName)
 			plugin, err := backend_interface.OpenBackendPlugin(pluginPath)
 			if err != nil {
@@ -158,11 +184,11 @@ func getSuppliers(clusterId string, clusterType KurtosisClusterType, kubernetesC
 		engineConfigSupplier = engine_server_launcher.NewKubernetesKurtosisBackendConfigSupplier(storageClass, enclaveDataVolumeSizeInMb)
 	default:
 		// This should never happen because we enforce this via unit tests
-		return nil, nil, stacktrace.NewError(
+		return nil, nil, nil, stacktrace.NewError(
 			"Cluster '%v' has unrecognized type '%v'; this is a bug in Kurtosis",
 			clusterId,
 			clusterType.String(),
 		)
 	}
-	return backendSupplier, engineConfigSupplier, nil
+	return backendSupplier, engineConfigSupplier, kurtosisRemoteBackendConfigSupplier, nil
 }

container-engine-lib/go.mod

@@ -12,6 +12,7 @@ require (
 	github.com/sirupsen/logrus v1.8.1
 	github.com/stretchr/testify v1.7.4
 	go.etcd.io/bbolt v1.3.6
+	golang.org/x/sync v0.0.0-20210220032951-036812b2e83c
 	gopkg.in/yaml.v3 v3.0.1
 )
 

container-engine-lib/go.sum

@@ -87,6 +87,8 @@ golang.org/x/net v0.7.0/go.mod h1:2Tu9+aMcznHK/AK1HMvgo6xiTLG5rD5rZLDS+rp2Bjs=
 golang.org/x/sync v0.0.0-20190423024810-112230192c58/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM=
 golang.org/x/sync v0.0.0-20190911185100-cd5d95a43a6e/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM=
 golang.org/x/sync v0.0.0-20201020160332-67f06af15bc9/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM=
+golang.org/x/sync v0.0.0-20210220032951-036812b2e83c h1:5KslGYwFpkhGh+Q16bwMP3cOontH8FOep7tGV86Y7SQ=
+golang.org/x/sync v0.0.0-20210220032951-036812b2e83c/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM=
 golang.org/x/sys v0.0.0-20190215142949-d0b11bdaac8a/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY=
 golang.org/x/sys v0.0.0-20190412213103-97732733099d/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
 golang.org/x/sys v0.0.0-20191026070338-33540a1f6037/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=

container-engine-lib/lib/backend_impls/docker/docker_kurtosis_backend/backend_creator/backend_creator.go

@@ -30,11 +30,22 @@ type APIContainerModeArgs struct {
 func GetLocalDockerKurtosisBackend(
 	optionalApiContainerModeArgs *APIContainerModeArgs,
 ) (backend_interface.KurtosisBackend, error) {
-	dockerClient, err := client.NewClientWithOpts(client.FromEnv, client.WithAPIVersionNegotiation())
+	localDockerClient, err := client.NewClientWithOpts(client.FromEnv, client.WithAPIVersionNegotiation())
 	if err != nil {
 		return nil, stacktrace.Propagate(err, "An error occurred creating a Docker client connected to the local environment")
 	}
 
+	localDockerBackend, err := GetDockerKurtosisBackend(localDockerClient, optionalApiContainerModeArgs)
+	if err != nil {
+		return nil, stacktrace.Propagate(err, "Unable to build local Kurtosis Docker backend")
+	}
+	return localDockerBackend, nil
+}
+
+func GetDockerKurtosisBackend(
+	dockerClient *client.Client,
+	optionalApiContainerModeArgs *APIContainerModeArgs,
+) (backend_interface.KurtosisBackend, error) {
 	dockerManager := docker_manager.NewDockerManager(dockerClient)
 
 	// If running within the API container context, detect the network that the API container is running inside