feat: Automatically map all service ports to local ports post Starlar…

…k run and service add (#363)

## Description:
When running on a remote context, all services added by `kurtosis run`
and `kurtosis service add` will have their ephemeral ports automatically
mapped to the user laptop local port

## Is this change user facing?
YES - but it's related to remote contexts
<!-- If yes, please add the "user facing" label to the PR -->
<!-- If yes, don't forget to include docs changes where relevant -->

## References (if applicable):
<!-- Add relevant Github Issues, Discord threads, or other helpful
information. -->

api/golang/engine/lib/kurtosis_context/kurtosis_context.go

@@ -39,7 +39,11 @@ const (
 	validUuidMatchesAllowed = 1
 )
 
-var apiContainerLogLevel = logrus.DebugLevel
+var (
+	apiContainerLogLevel = logrus.DebugLevel
+
+	apicPortTransportProtocol = portal_api.TransportProtocol_TCP
+)
 
 // Docs available at https://docs.kurtosis.com/sdk#kurtosiscontext
 type KurtosisContext struct {
@@ -347,13 +351,13 @@ func newEnclaveContextFromEnclaveInfo(
 	// for remote contexts, we need to tunnel the APIC port to the local machine
 	if portalClient != nil {
 		apicGrpcPort := enclaveInfo.GetApiContainerHostMachineInfo().GetGrpcPortOnHostMachine()
-		forwardApicPortArgs := portal_constructors.NewForwardPortArgs(apicGrpcPort, apicGrpcPort)
+		forwardApicPortArgs := portal_constructors.NewForwardPortArgs(apicGrpcPort, apicGrpcPort, &apicPortTransportProtocol)
 		if _, err := portalClient.ForwardPort(ctx, forwardApicPortArgs); err != nil {
 			return nil, stacktrace.Propagate(err, "Unable to forward remote API container port to the local machine")
 		}
 
 		apicGrpcProxyPort := enclaveInfo.GetApiContainerHostMachineInfo().GetGrpcProxyPortOnHostMachine()
-		forwardApicProxyPortArgs := portal_constructors.NewForwardPortArgs(apicGrpcProxyPort, apicGrpcProxyPort)
+		forwardApicProxyPortArgs := portal_constructors.NewForwardPortArgs(apicGrpcProxyPort, apicGrpcProxyPort, &apicPortTransportProtocol)
 		if _, err := portalClient.ForwardPort(ctx, forwardApicProxyPortArgs); err != nil {
 			return nil, stacktrace.Propagate(err, "Unable to forward remote API container proxy port to the local machine")
 		}

api/golang/go.mod

@@ -7,7 +7,7 @@ replace github.com/kurtosis-tech/kurtosis/contexts-config-store => ../../context
 require (
 	github.com/Masterminds/semver/v3 v3.1.1
 	github.com/go-yaml/yaml v2.1.0+incompatible
-	github.com/kurtosis-tech/kurtosis-portal/api/golang v0.0.0-20230323091236-fbfe0355b588
+	github.com/kurtosis-tech/kurtosis-portal/api/golang v0.0.0-20230328194643-b4dea3081e25
 	github.com/kurtosis-tech/kurtosis/contexts-config-store v0.0.0 // local dependency
 	github.com/kurtosis-tech/stacktrace v0.0.0-20211028211901-1c67a77b5409
 	github.com/mholt/archiver v3.1.1+incompatible

api/golang/go.sum

@@ -57,8 +57,8 @@ github.com/kr/pty v1.1.1/go.mod h1:pFQYn66WHrOpPYNljwOMqo10TkYh1fy3cYio2l3bCsQ=
 github.com/kr/text v0.1.0/go.mod h1:4Jbv+DJW3UT/LiOwJeYQe1efqtUx/iVham/4vfdArNI=
 github.com/kr/text v0.2.0 h1:5Nx0Ya0ZqY2ygV366QzturHI13Jq95ApcVaJBhpS+AY=
 github.com/kr/text v0.2.0/go.mod h1:eLer722TekiGuMkidMxC/pM04lWEeraHUUmBw8l2grE=
-github.com/kurtosis-tech/kurtosis-portal/api/golang v0.0.0-20230323091236-fbfe0355b588 h1:zFUutUImTPe6Wbk3vMnris8QyQa772iQDDsEhGXZIBk=
-github.com/kurtosis-tech/kurtosis-portal/api/golang v0.0.0-20230323091236-fbfe0355b588/go.mod h1:YjVghnKmmELgH8DmIKBFxwArWbtLUYqwnol9DAWnBM8=
+github.com/kurtosis-tech/kurtosis-portal/api/golang v0.0.0-20230328194643-b4dea3081e25 h1:ig5umBAI6smmP/4xPLSL5KSlH9N/bZURzDkJzD8qWb8=
+github.com/kurtosis-tech/kurtosis-portal/api/golang v0.0.0-20230328194643-b4dea3081e25/go.mod h1:YjVghnKmmELgH8DmIKBFxwArWbtLUYqwnol9DAWnBM8=
 github.com/kurtosis-tech/stacktrace v0.0.0-20211028211901-1c67a77b5409 h1:YQTATifMUwZEtZYb0LVA7DK2pj8s71iY8rzweuUQ5+g=
 github.com/kurtosis-tech/stacktrace v0.0.0-20211028211901-1c67a77b5409/go.mod h1:y5weVs5d9wXXHcDA1awRxkIhhHC1xxYJN8a7aXnE6S8=
 github.com/mholt/archiver v3.1.1+incompatible h1:1dCVxuqs0dJseYEhi5pl7MYPH9zDa1wBi7mF09cbNkU=

cli/cli/commands/run/run.go

@@ -6,6 +6,7 @@ import (
 	"fmt"
 	"github.com/kurtosis-tech/kurtosis/api/golang/core/kurtosis_core_rpc_api_bindings"
 	"github.com/kurtosis-tech/kurtosis/api/golang/core/lib/enclaves"
+	"github.com/kurtosis-tech/kurtosis/api/golang/core/lib/services"
 	"github.com/kurtosis-tech/kurtosis/api/golang/engine/kurtosis_engine_rpc_api_bindings"
 	"github.com/kurtosis-tech/kurtosis/api/golang/engine/lib/kurtosis_context"
 	command_args_run "github.com/kurtosis-tech/kurtosis/cli/cli/command_args/run"
@@ -16,6 +17,7 @@ import (
 	"github.com/kurtosis-tech/kurtosis/cli/cli/command_str_consts"
 	"github.com/kurtosis-tech/kurtosis/cli/cli/commands/enclave/inspect"
 	"github.com/kurtosis-tech/kurtosis/cli/cli/helpers/output_printers"
+	"github.com/kurtosis-tech/kurtosis/cli/cli/helpers/portal_manager"
 	"github.com/kurtosis-tech/kurtosis/cli/cli/user_support_constants"
 	"github.com/kurtosis-tech/kurtosis/container-engine-lib/lib/backend_interface"
 	metrics_client "github.com/kurtosis-tech/metrics-library/golang/lib/client"
@@ -62,6 +64,10 @@ const (
 	parallelismFlagKey = "parallelism"
 	defaultParallelism = "4"
 
+	mapPortsFlagKey = "map-ports"
+	// we're mapping ports by default such that remote run and local run gives the exact same state: ports are reachable from local laptop
+	defaultMapPortsFlagKey = "true"
+
 	githubDomainPrefix          = "github.com/"
 	isNewEnclaveFlagWhenCreated = true
 	interruptChanBufferSize     = 5
@@ -73,6 +79,8 @@ const (
 
 	runFailed    = false
 	runSucceeded = true
+
+	portMappingSeparatorForLogs = ", "
 )
 
 var (
@@ -141,6 +149,14 @@ var StarlarkRunCmd = &engine_consuming_kurtosis_command.EngineConsumingKurtosisC
 			Type:    flags.FlagType_Bool,
 			Default: fullUuidFlagKeyDefault,
 		},
+		{
+			Key: mapPortsFlagKey,
+			Usage: "If true then services running remotely will have their ports mapped to the local host, such that " +
+				"they are reachable as if they were running locally. This applies inside a remote context - in a " +
+				"local context, all services are always reachable locally on their ephemeral ports. Default true",
+			Type:    flags.FlagType_Bool,
+			Default: defaultMapPortsFlagKey,
+		},
 	},
 	Args: []*args.ArgConfig{
 		// TODO add a `Usage` description here when ArgConfig supports it
@@ -215,6 +231,11 @@ func run(
 		return stacktrace.Propagate(err, "Expected a value for the '%v' flag but failed to get it", showEnclaveInspectFlagKey)
 	}
 
+	mapPorts, err := flags.GetBool(mapPortsFlagKey)
+	if err != nil {
+		return stacktrace.Propagate(err, "Expected a value for the '%v' flag but failed to get it", mapPortsFlagKey)
+	}
+
 	kurtosisCtx, err := kurtosis_context.NewKurtosisContextFromLocalEngine()
 	if err != nil {
 		return stacktrace.Propagate(err, "An error occurred connecting to the local Kurtosis engine")
@@ -270,35 +291,70 @@ func run(
 		return stacktrace.Propagate(errRunningKurtosis, "An error starting the Kurtosis code execution '%v'", starlarkScriptOrPackagePath)
 	}
 
-	if err := metricsClient.TrackKurtosisRun(starlarkScriptOrPackagePath, isRemotePackage, dryRun, isStandAloneScript); err != nil {
+	if err = metricsClient.TrackKurtosisRun(starlarkScriptOrPackagePath, isRemotePackage, dryRun, isStandAloneScript); err != nil {
 		//We don't want to interrupt users flow if something fails when tracking metrics
 		logrus.Warn("An error occurred tracking kurtosis run event")
 	}
 
 	errRunningKurtosis = readAndPrintResponseLinesUntilClosed(responseLineChan, cancelFunc, verbosity, dryRun)
+	var runStatusForMetrics bool
 	if errRunningKurtosis != nil {
-		servicesInEnclaveForMetrics, servicesInEnclaveForMetricsError := enclaveCtx.GetServices()
-		if servicesInEnclaveForMetricsError != nil {
-			logrus.Warn("Tried getting number of services in the enclave to log metrics but failed")
-		} else {
-			if err := metricsClient.TrackKurtosisRunFinishedEvent(starlarkScriptOrPackagePath, len(servicesInEnclaveForMetrics), runFailed); err != nil {
-				logrus.Warn("An error occurred tracking kurtosis run finished event")
-			}
-		}
-
-		// do not print the go trace in case of APIC failures
-		return nil
+		runStatusForMetrics = runFailed
+	} else {
+		runStatusForMetrics = runSucceeded
 	}
 
-	servicesInEnclaveForMetrics, servicesInEnclaveForMetricsError := enclaveCtx.GetServices()
+	servicesInEnclavePostRun, servicesInEnclaveForMetricsError := enclaveCtx.GetServices()
 	if servicesInEnclaveForMetricsError != nil {
-		logrus.Error("Tried getting number of services in the enclave to log metrics but failed")
+		logrus.Warn("Tried getting number of services in the enclave to log metrics but failed")
 	} else {
-		if err := metricsClient.TrackKurtosisRunFinishedEvent(starlarkScriptOrPackagePath, len(servicesInEnclaveForMetrics), runSucceeded); err != nil {
+		if err = metricsClient.TrackKurtosisRunFinishedEvent(starlarkScriptOrPackagePath, len(servicesInEnclavePostRun), runStatusForMetrics); err != nil {
 			logrus.Warn("An error occurred tracking kurtosis run finished event")
 		}
 	}
 
+	if errRunningKurtosis != nil {
+		// This error thrown by the APIC is not informative right now as it just tells the user to look at errors
+		// in the above log. For this reason we're ignoring it and returning nil. This is exceptional to not clutter
+		// the CLI output. We should still use stacktrace.Propagate for other errors.
+		return nil
+	}
+
+	if servicesInEnclaveForMetricsError != nil {
+		logrus.Warnf("Unable to retrieve the services running inside the enclave so their ports will not be" +
+			" mapped to local ports.")
+		return nil
+	}
+
+	if !mapPorts {
+		logrus.Info("Not mapping service ports locally as requested")
+		return nil
+	}
+	portalManager := portal_manager.NewPortalManager()
+	portsMapping := map[uint16]*services.PortSpec{}
+	for serviceInEnclaveName, servicesInEnclaveUuid := range servicesInEnclavePostRun {
+		serviceCtx, err := enclaveCtx.GetServiceContext(string(servicesInEnclaveUuid))
+		if err != nil {
+			return stacktrace.Propagate(err, "Error getting service object for service '%s' with UUID '%s'", serviceInEnclaveName, servicesInEnclaveUuid)
+		}
+		for _, portSpec := range serviceCtx.GetPublicPorts() {
+			portsMapping[portSpec.GetNumber()] = portSpec
+		}
+	}
+	successfullyMappedPorts, failedPorts, err := portalManager.MapPorts(ctx, portsMapping)
+	if err != nil {
+		var stringifiedPortMapping []string
+		for localPort, remotePort := range failedPorts {
+			stringifiedPortMapping = append(stringifiedPortMapping, fmt.Sprintf("%d:%d", localPort, remotePort.GetNumber()))
+		}
+		// TODO: once we have a manual `kurtosis port map` command, suggest using it here to manually map the failed port
+		logrus.Warnf("The enclave was successfully run but the following port(s) could not be mapped locally: %s. "+
+			"The associated service(s) will not be reachable on the local host",
+			strings.Join(stringifiedPortMapping, portMappingSeparatorForLogs))
+		return nil
+	}
+	logrus.Infof("Successfully mapped %d ports. All services running inside the enclave are reachable locally on"+
+		" their ephemeral port numbers", len(successfullyMappedPorts))
 	return nil
 }
 

cli/cli/commands/service/add/add.go

@@ -13,10 +13,12 @@ import (
 	"github.com/kurtosis-tech/kurtosis/cli/cli/command_framework/lowlevel/flags"
 	"github.com/kurtosis-tech/kurtosis/cli/cli/command_str_consts"
 	"github.com/kurtosis-tech/kurtosis/cli/cli/helpers/output_printers"
+	"github.com/kurtosis-tech/kurtosis/cli/cli/helpers/portal_manager"
 	"github.com/kurtosis-tech/kurtosis/container-engine-lib/lib/backend_interface"
 	"github.com/kurtosis-tech/kurtosis/container-engine-lib/lib/uuid_generator"
 	metrics_client "github.com/kurtosis-tech/metrics-library/golang/lib/client"
 	"github.com/kurtosis-tech/stacktrace"
+	"github.com/sirupsen/logrus"
 	"strconv"
 	"strings"
 )
@@ -36,6 +38,10 @@ const (
 
 	entrypointBinaryFlagKey = "entrypoint"
 
+	mapPortsFlagKey = "map-ports"
+	// we're mapping ports by default such that remote run and local run gives the exact same state: ports are reachable from local laptop
+	defaultMapPortsFlagKey = "true"
+
 	envvarsFlagKey              = "env"
 	envvarKeyValueDelimiter     = "="
 	envvarDeclarationsDelimiter = ","
@@ -70,7 +76,7 @@ const (
 	maxRemainingPortSpecComponents = 2
 
 	emptyApplicationProtocol = ""
-	linkDelimeter            = "://"
+	linkDelimiter            = "://"
 
 	maybeApplicationProtocolSpecForHelp = "MAYBE_APPLICATION_PROTOCOL"
 	transportProtocolSpecForHelp        = "TRANSPORT_PROTOCOL"
@@ -81,6 +87,8 @@ const (
 	fullUuidFlagKeyDefault = "false"
 
 	defaultParallelism = 1
+
+	portMappingSeparatorForLogs = ", "
 )
 
 var (
@@ -189,6 +197,14 @@ var ServiceAddCmd = &engine_consuming_kurtosis_command.EngineConsumingKurtosisCo
 			Type:    flags.FlagType_Bool,
 			Default: fullUuidFlagKeyDefault,
 		},
+		{
+			Key: mapPortsFlagKey,
+			Usage: "If true the service running remotely will have its ports mapped to the local host, such that " +
+				"it is reachable as if it was running locally. This applies inside a remote context - in a " +
+				"local context, the service is always reachable locally on its ephemeral ports. Default true",
+			Type:    flags.FlagType_Bool,
+			Default: defaultMapPortsFlagKey,
+		},
 	},
 	RunFunc: run,
 }
@@ -251,6 +267,11 @@ func run(
 		return stacktrace.Propagate(err, "Expected a value for the '%v' flag but failed to get it", fullUuidsFlagKey)
 	}
 
+	mapPorts, err := flags.GetBool(mapPortsFlagKey)
+	if err != nil {
+		return stacktrace.Propagate(err, "Expected a value for the '%v' flag but failed to get it", mapPortsFlagKey)
+	}
+
 	kurtosisCtx, err := kurtosis_context.NewKurtosisContextFromLocalEngine()
 	if err != nil {
 		return stacktrace.Propagate(err, "An error occurred connecting to the local Kurtosis engine")
@@ -298,6 +319,27 @@ func run(
 	publicPorts := serviceCtx.GetPublicPorts()
 	publicIpAddr := serviceCtx.GetMaybePublicIPAddress()
 
+	// Map the service public ports to their local port
+	if mapPorts {
+		portalManager := portal_manager.NewPortalManager()
+		portsMapping := map[uint16]*services.PortSpec{}
+		for _, portSpec := range serviceCtx.GetPublicPorts() {
+			portsMapping[portSpec.GetNumber()] = portSpec
+		}
+		successfullyMappedPorts, failedPorts, err := portalManager.MapPorts(ctx, portsMapping)
+		if err != nil {
+			var stringifiedPortMapping []string
+			for localPort, remotePort := range failedPorts {
+				stringifiedPortMapping = append(stringifiedPortMapping, fmt.Sprintf("%d:%d", localPort, remotePort.GetNumber()))
+			}
+			// TODO: once we have a manual `kurtosis port map` command, suggest using it here to manually map the failed port
+			logrus.Warnf("The service is running but the following port(s) could not be mapped locally: %s.",
+				strings.Join(stringifiedPortMapping, portMappingSeparatorForLogs))
+		}
+		logrus.Infof("Successfully mapped %d ports. The service is reachable locally on its ephemeral port numbers",
+			len(successfullyMappedPorts))
+	}
+
 	fmt.Printf("Service ID: %v\n", serviceName)
 	if len(privatePorts) > 0 {
 		fmt.Println("Ports Bindings:")
@@ -326,7 +368,7 @@ func run(
 
 		portApplicationProtocolStr := emptyApplicationProtocol
 		if privatePortSpec.GetMaybeApplicationProtocol() != emptyApplicationProtocol {
-			portApplicationProtocolStr = fmt.Sprintf("%v%v", privatePortSpec.GetMaybeApplicationProtocol(), linkDelimeter)
+			portApplicationProtocolStr = fmt.Sprintf("%v%v", privatePortSpec.GetMaybeApplicationProtocol(), linkDelimiter)
 		}
 		portBindingInfo := fmt.Sprintf(
 			"%v/%v -> %v%v:%v",

cli/cli/go.mod

@@ -23,7 +23,6 @@ require (
 	github.com/kurtosis-tech/kurtosis/contexts-config-store v0.0.0 // local dependency
 	github.com/kurtosis-tech/kurtosis/engine/launcher v0.0.0 // local dependency
 	github.com/kurtosis-tech/kurtosis/kurtosis_version v0.0.0 // Local dependency generated during build
-	github.com/kurtosis-tech/kurtosis/lsp v0.0.0
 	github.com/kurtosis-tech/metrics-library/golang v0.0.0-20230221115618-70c305416224
 	github.com/kurtosis-tech/stacktrace v0.0.0-20211028211901-1c67a77b5409
 	github.com/manifoldco/promptui v0.9.0
@@ -44,7 +43,8 @@ require github.com/bazelbuild/buildtools v0.0.0-20221110131218-762712d8ce3f
 require (
 	github.com/briandowns/spinner v1.20.0
 	github.com/fatih/color v1.13.0
-	github.com/kurtosis-tech/kurtosis-portal/api/golang v0.0.0-20230323091236-fbfe0355b588
+	github.com/kurtosis-tech/kurtosis-portal/api/golang v0.0.0-20230328194643-b4dea3081e25
+	github.com/kurtosis-tech/kurtosis/lsp v0.0.0-00010101000000-000000000000
 	github.com/kurtosis-tech/vscode-kurtosis/starlark-lsp v0.0.0-20230324071217-6348e066f3e5
 	github.com/mholt/archiver v3.1.1+incompatible
 	github.com/savioxavier/termlink v1.2.1

cli/cli/go.sum

@@ -291,8 +291,8 @@ github.com/kr/pty v1.1.3/go.mod h1:pFQYn66WHrOpPYNljwOMqo10TkYh1fy3cYio2l3bCsQ=
 github.com/kr/text v0.1.0/go.mod h1:4Jbv+DJW3UT/LiOwJeYQe1efqtUx/iVham/4vfdArNI=
 github.com/kr/text v0.2.0 h1:5Nx0Ya0ZqY2ygV366QzturHI13Jq95ApcVaJBhpS+AY=
 github.com/kr/text v0.2.0/go.mod h1:eLer722TekiGuMkidMxC/pM04lWEeraHUUmBw8l2grE=
-github.com/kurtosis-tech/kurtosis-portal/api/golang v0.0.0-20230323091236-fbfe0355b588 h1:zFUutUImTPe6Wbk3vMnris8QyQa772iQDDsEhGXZIBk=
-github.com/kurtosis-tech/kurtosis-portal/api/golang v0.0.0-20230323091236-fbfe0355b588/go.mod h1:YjVghnKmmELgH8DmIKBFxwArWbtLUYqwnol9DAWnBM8=
+github.com/kurtosis-tech/kurtosis-portal/api/golang v0.0.0-20230328194643-b4dea3081e25 h1:ig5umBAI6smmP/4xPLSL5KSlH9N/bZURzDkJzD8qWb8=
+github.com/kurtosis-tech/kurtosis-portal/api/golang v0.0.0-20230328194643-b4dea3081e25/go.mod h1:YjVghnKmmELgH8DmIKBFxwArWbtLUYqwnol9DAWnBM8=
 github.com/kurtosis-tech/metrics-library/golang v0.0.0-20230221115618-70c305416224 h1:7KI1v88Wq+yrL6Nz71lxhGynouo6vBWrae6IizH7W1c=
 github.com/kurtosis-tech/metrics-library/golang v0.0.0-20230221115618-70c305416224/go.mod h1:tteWV+M47xMHxqCIPQmdmgPW80rhN8YfzrgRRWbQhOw=
 github.com/kurtosis-tech/stacktrace v0.0.0-20211028211901-1c67a77b5409 h1:YQTATifMUwZEtZYb0LVA7DK2pj8s71iY8rzweuUQ5+g=