feat: new feature to restart the API containers after the Kurtosis en…

…gine is started (#2413)

## Description
- New feature to restart the API containers after the Kurtosis engine is
started
- A new `restart-api-containers` flag was added to the engine's `start`
and `restart` commands.

## REMINDER: Tag Reviewers, so they get notified to review

## Is this change user facing?
YES

## References (if applicable)
This is part of the `Upgrade Kurtosi from Cloud UI` project.

cli/cli/command_framework/highlevel/engine_consuming_kurtosis_command/engine_consuming_kurtosis_command.go

@@ -186,7 +186,8 @@ func (cmd *EngineConsumingKurtosisCommand) getSetupFunc() func(context.Context)
 		//  commands only access the Kurtosis APIs, we can remove this.
 		kurtosisBackend := engineManager.GetKurtosisBackend()
 
-		engineClient, closeClientFunc, err := engineManager.StartEngineIdempotentlyWithDefaultVersion(ctx, defaults.DefaultEngineLogLevel, defaults.DefaultEngineEnclavePoolSize, defaults.DefaultGitHubAuthTokenOverride)
+		dontRestartAPIContainers := false
+		engineClient, closeClientFunc, err := engineManager.StartEngineIdempotentlyWithDefaultVersion(ctx, defaults.DefaultEngineLogLevel, defaults.DefaultEngineEnclavePoolSize, defaults.DefaultGitHubAuthTokenOverride, dontRestartAPIContainers)
 		if err != nil {
 			return nil, stacktrace.Propagate(err, "An error occurred creating a new Kurtosis engine client")
 		}

cli/cli/commands/cluster/set/set.go

@@ -123,7 +123,8 @@ func run(ctx context.Context, flags *flags.ParsedFlags, args *args.ParsedArgs) e
 	// we only start in a stopped state, the idempotent visitor gets stuck with engine_manager.EngineStatus_ContainerRunningButServerNotResponding if the gateway isn't running
 	// TODO - fix the idempotent starter longer term
 	if engineStatus == engine_manager.EngineStatus_Stopped {
-		_, engineClientCloseFunc, err := engineManagerNewCluster.StartEngineIdempotentlyWithDefaultVersion(ctx, defaults.DefaultEngineLogLevel, defaults.DefaultEngineEnclavePoolSize, defaults.DefaultGitHubAuthTokenOverride)
+		dontRestartAPIContainers := false
+		_, engineClientCloseFunc, err := engineManagerNewCluster.StartEngineIdempotentlyWithDefaultVersion(ctx, defaults.DefaultEngineLogLevel, defaults.DefaultEngineEnclavePoolSize, defaults.DefaultGitHubAuthTokenOverride, dontRestartAPIContainers)
 		if err != nil {
 			return stacktrace.Propagate(err, "Engine could not be started after cluster was updated. Its status can be retrieved "+
 				"running 'kurtosis %s %s' and it can potentially be started running 'kurtosis %s %s'",

cli/cli/commands/enclave/add/add.go

@@ -115,11 +115,12 @@ func run(
 		return stacktrace.Propagate(err, "An error occurred while getting the enclave name using flag with key '%v'; this is a bug in Kurtosis ", enclaveNameFlagKey)
 	}
 
+	dontRestartAPIContainers := false
 	engineManager, err := engine_manager.NewEngineManager(ctx)
 	if err != nil {
 		return stacktrace.Propagate(err, "An error occurred creating an engine manager.")
 	}
-	engineClient, closeClientFunc, err := engineManager.StartEngineIdempotentlyWithDefaultVersion(ctx, defaults.DefaultEngineLogLevel, defaults.DefaultEngineEnclavePoolSize, defaults.DefaultGitHubAuthTokenOverride)
+	engineClient, closeClientFunc, err := engineManager.StartEngineIdempotentlyWithDefaultVersion(ctx, defaults.DefaultEngineLogLevel, defaults.DefaultEngineEnclavePoolSize, defaults.DefaultGitHubAuthTokenOverride, dontRestartAPIContainers)
 	if err != nil {
 		return stacktrace.Propagate(err, "An error occurred creating a new Kurtosis engine client")
 	}

cli/cli/commands/engine/restart/restart.go

@@ -26,6 +26,9 @@ const (
 
 	defaultEngineVersion                   = ""
 	restartEngineOnSameVersionIfAnyRunning = false
+
+	defaultShouldRestartAPIContainers = "false"
+	restartAPIContainersFlagKey       = "restart-api-containers"
 )
 
 var RestartCmd = &lowlevel.LowlevelKurtosisCommand{
@@ -71,6 +74,13 @@ var RestartCmd = &lowlevel.LowlevelKurtosisCommand{
 			Type:      flags.FlagType_String,
 			Default:   defaults.DefaultGitHubAuthTokenOverride,
 		},
+		{
+			Key:       restartAPIContainersFlagKey,
+			Usage:     "Also restart the current API containers after starting the Kurtosis engine.",
+			Shorthand: "",
+			Type:      flags.FlagType_Bool,
+			Default:   defaultShouldRestartAPIContainers,
+		},
 	},
 	PreValidationAndRunFunc:  nil,
 	RunFunc:                  run,
@@ -127,9 +137,14 @@ func run(_ context.Context, flags *flags.ParsedFlags, _ *args.ParsedArgs) error
 		shouldStartInDebugMode = true
 	}
 
+	shouldRestartAPIContainers, err := flags.GetBool(restartAPIContainersFlagKey)
+	if err != nil {
+		return stacktrace.Propagate(err, "Expected a value for the '%v' flag but failed to get it", restartAPIContainersFlagKey)
+	}
+
 	var engineClientCloseFunc func() error
 	var restartEngineErr error
-	_, engineClientCloseFunc, restartEngineErr = engineManager.RestartEngineIdempotently(ctx, logLevel, engineVersion, restartEngineOnSameVersionIfAnyRunning, enclavePoolSize, shouldStartInDebugMode, githubAuthTokenOverride)
+	_, engineClientCloseFunc, restartEngineErr = engineManager.RestartEngineIdempotently(ctx, logLevel, engineVersion, restartEngineOnSameVersionIfAnyRunning, enclavePoolSize, shouldStartInDebugMode, githubAuthTokenOverride, shouldRestartAPIContainers)
 	if restartEngineErr != nil {
 		return stacktrace.Propagate(restartEngineErr, "An error occurred restarting the Kurtosis engine")
 	}

cli/cli/commands/engine/start/start.go

@@ -27,6 +27,9 @@ const (
 	defaultEngineVersion          = ""
 	kurtosisTechEngineImagePrefix = "kurtosistech/engine"
 	imageVersionDelimiter         = ":"
+
+	defaultShouldRestartAPIContainers = "false"
+	restartAPIContainersFlagKey       = "restart-api-containers"
 )
 
 var StartCmd = &lowlevel.LowlevelKurtosisCommand{
@@ -72,6 +75,13 @@ var StartCmd = &lowlevel.LowlevelKurtosisCommand{
 			Type:      flags.FlagType_String,
 			Default:   defaults.DefaultGitHubAuthTokenOverride,
 		},
+		{
+			Key:       restartAPIContainersFlagKey,
+			Usage:     "Restart the current API containers after starting the Kurtosis engine.",
+			Shorthand: "",
+			Type:      flags.FlagType_Bool,
+			Default:   defaultShouldRestartAPIContainers,
+		},
 	},
 	PreValidationAndRunFunc:  nil,
 	RunFunc:                  run,
@@ -123,16 +133,21 @@ func run(_ context.Context, flags *flags.ParsedFlags, _ *args.ParsedArgs) error
 		return stacktrace.Propagate(err, "Expected a value for the '%v' flag but failed to get it", defaults.DebugModeFlagKey)
 	}
 
+	shouldRestartAPIContainers, err := flags.GetBool(restartAPIContainersFlagKey)
+	if err != nil {
+		return stacktrace.Propagate(err, "Expected a value for the '%v' flag but failed to get it", restartAPIContainersFlagKey)
+	}
+
 	if engineVersion == defaultEngineVersion && isDebugMode {
 		engineDebugVersion := fmt.Sprintf("%s-%s", kurtosis_version.KurtosisVersion, defaults.DefaultKurtosisContainerDebugImageNameSuffix)
 		logrus.Infof("Starting Kurtosis engine in debug mode from image '%v%v%v'...", kurtosisTechEngineImagePrefix, imageVersionDelimiter, engineDebugVersion)
-		_, engineClientCloseFunc, startEngineErr = engineManager.StartEngineIdempotentlyWithCustomVersion(ctx, engineDebugVersion, logLevel, enclavePoolSize, true, githubAuthTokenOverride)
+		_, engineClientCloseFunc, startEngineErr = engineManager.StartEngineIdempotentlyWithCustomVersion(ctx, engineDebugVersion, logLevel, enclavePoolSize, true, githubAuthTokenOverride, shouldRestartAPIContainers)
 	} else if engineVersion == defaultEngineVersion {
 		logrus.Infof("Starting Kurtosis engine from image '%v%v%v'...", kurtosisTechEngineImagePrefix, imageVersionDelimiter, kurtosis_version.KurtosisVersion)
-		_, engineClientCloseFunc, startEngineErr = engineManager.StartEngineIdempotentlyWithDefaultVersion(ctx, logLevel, enclavePoolSize, githubAuthTokenOverride)
+		_, engineClientCloseFunc, startEngineErr = engineManager.StartEngineIdempotentlyWithDefaultVersion(ctx, logLevel, enclavePoolSize, githubAuthTokenOverride, shouldRestartAPIContainers)
 	} else {
 		logrus.Infof("Starting Kurtosis engine from image '%v%v%v'...", kurtosisTechEngineImagePrefix, imageVersionDelimiter, engineVersion)
-		_, engineClientCloseFunc, startEngineErr = engineManager.StartEngineIdempotentlyWithCustomVersion(ctx, engineVersion, logLevel, enclavePoolSize, defaults.DefaultEnableDebugMode, githubAuthTokenOverride)
+		_, engineClientCloseFunc, startEngineErr = engineManager.StartEngineIdempotentlyWithCustomVersion(ctx, engineVersion, logLevel, enclavePoolSize, defaults.DefaultEnableDebugMode, githubAuthTokenOverride, shouldRestartAPIContainers)
 	}
 	if startEngineErr != nil {
 		return stacktrace.Propagate(startEngineErr, "An error occurred starting the Kurtosis engine")

cli/cli/commands/github/login/login.go

@@ -70,7 +70,8 @@ func RestartEngineAfterGitHubAuth(ctx context.Context) error {
 	}
 	var engineClientCloseFunc func() error
 	var restartEngineErr error
-	_, engineClientCloseFunc, restartEngineErr = engineManager.RestartEngineIdempotently(ctx, defaults.DefaultEngineLogLevel, defaultEngineVersion, restartEngineOnSameVersionIfAnyRunning, defaults.DefaultEngineEnclavePoolSize, defaults.DefaultEnableDebugMode, defaults.DefaultGitHubAuthTokenOverride)
+	dontRestartAPIContainers := false
+	_, engineClientCloseFunc, restartEngineErr = engineManager.RestartEngineIdempotently(ctx, defaults.DefaultEngineLogLevel, defaultEngineVersion, restartEngineOnSameVersionIfAnyRunning, defaults.DefaultEngineEnclavePoolSize, defaults.DefaultEnableDebugMode, defaults.DefaultGitHubAuthTokenOverride, dontRestartAPIContainers)
 	if restartEngineErr != nil {
 		return stacktrace.Propagate(restartEngineErr, "An error occurred restarting the Kurtosis engine")
 	}

cli/cli/commands/kurtosis_context/set/set.go

@@ -147,7 +147,8 @@ func SetContext(
 		return stacktrace.Propagate(err, "An error occurred creating an engine manager for the new context.")
 	}
 
-	_, engineClientCloseFunc, startEngineErr := engineManager.StartEngineIdempotentlyWithDefaultVersion(ctx, logrus.InfoLevel, defaults.DefaultEngineEnclavePoolSize, defaults.DefaultGitHubAuthTokenOverride)
+	dontRestartAPIContainers := false
+	_, engineClientCloseFunc, startEngineErr := engineManager.StartEngineIdempotentlyWithDefaultVersion(ctx, logrus.InfoLevel, defaults.DefaultEngineEnclavePoolSize, defaults.DefaultGitHubAuthTokenOverride, dontRestartAPIContainers)
 	if startEngineErr != nil {
 		logrus.Warnf("The context was successfully set to '%s' but Kurtosis failed to start an engine in "+
 			"this new context. A new engine should be started manually with '%s %s %s'. The error was:\n%v",

cli/cli/helpers/engine_manager/engine_existence_guarantor.go

@@ -81,6 +81,9 @@ type engineExistenceGuarantor struct {
 
 	// token with git auth to override existing GitHub auth if there is any
 	githubAuthTokenOverride string
+
+	// To restart the current API containers after the engine has been restarted
+	restartAPIContainers bool
 }
 
 func newEngineExistenceGuarantorWithDefaultVersion(
@@ -98,7 +101,7 @@ func newEngineExistenceGuarantorWithDefaultVersion(
 	allowedCORSOrigins *[]string,
 	shouldRunInDebugMode bool,
 	githubAuthTokenOverride string,
-
+	restartAPIContainers bool,
 ) *engineExistenceGuarantor {
 	return newEngineExistenceGuarantorWithCustomVersion(
 		ctx,
@@ -116,6 +119,7 @@ func newEngineExistenceGuarantorWithDefaultVersion(
 		allowedCORSOrigins,
 		shouldRunInDebugMode,
 		githubAuthTokenOverride,
+		restartAPIContainers,
 	)
 }
 
@@ -135,6 +139,7 @@ func newEngineExistenceGuarantorWithCustomVersion(
 	allowedCORSOrigins *[]string,
 	shouldRunInDebugMode bool,
 	githubAuthTokenOverride string,
+	restartAPIContainers bool,
 ) *engineExistenceGuarantor {
 	return &engineExistenceGuarantor{
 		ctx:                                  ctx,
@@ -154,6 +159,7 @@ func newEngineExistenceGuarantorWithCustomVersion(
 		allowedCORSOrigins:                        allowedCORSOrigins,
 		shouldRunInDebugMode:                      shouldRunInDebugMode,
 		githubAuthTokenOverride:                   githubAuthTokenOverride,
+		restartAPIContainers:                      restartAPIContainers,
 	}
 }
 
@@ -212,6 +218,7 @@ func (guarantor *engineExistenceGuarantor) VisitStopped() error {
 			guarantor.allowedCORSOrigins,
 			guarantor.shouldRunInDebugMode,
 			githubAuthToken,
+			guarantor.restartAPIContainers,
 		)
 	} else {
 		_, _, engineLaunchErr = guarantor.engineServerLauncher.LaunchWithCustomVersion(
@@ -231,6 +238,7 @@ func (guarantor *engineExistenceGuarantor) VisitStopped() error {
 			guarantor.allowedCORSOrigins,
 			guarantor.shouldRunInDebugMode,
 			githubAuthToken,
+			guarantor.restartAPIContainers,
 		)
 	}
 	if engineLaunchErr != nil {

cli/cli/helpers/engine_manager/engine_manager.go

@@ -180,7 +180,9 @@ func (manager *EngineManager) StartEngineIdempotentlyWithDefaultVersion(
 	ctx context.Context,
 	logLevel logrus.Level,
 	poolSize uint8,
-	githubAuthTokenOverride string) (kurtosis_engine_rpc_api_bindings.EngineServiceClient, func() error, error) {
+	githubAuthTokenOverride string,
+	restartAPIContainers bool,
+) (kurtosis_engine_rpc_api_bindings.EngineServiceClient, func() error, error) {
 	status, maybeHostMachinePortBinding, engineVersion, err := manager.GetEngineStatus(ctx)
 	if err != nil {
 		return nil, nil, stacktrace.Propagate(err, "An error occurred retrieving the Kurtosis engine status, which is necessary for creating a connection to the engine")
@@ -202,6 +204,7 @@ func (manager *EngineManager) StartEngineIdempotentlyWithDefaultVersion(
 		manager.allowedCORSOrigins,
 		doNotStartTheEngineInDebugModeForDefaultVersion,
 		githubAuthTokenOverride,
+		restartAPIContainers,
 	)
 	// TODO Need to handle the Kubernetes case, where a gateway needs to be started after the engine is started but
 	//  before we can return an EngineClient
@@ -219,7 +222,9 @@ func (manager *EngineManager) StartEngineIdempotentlyWithCustomVersion(
 	logLevel logrus.Level,
 	poolSize uint8,
 	shouldStartInDebugMode bool,
-	githubAuthTokenOverride string) (kurtosis_engine_rpc_api_bindings.EngineServiceClient, func() error, error) {
+	githubAuthTokenOverride string,
+	restartAPIContainers bool,
+) (kurtosis_engine_rpc_api_bindings.EngineServiceClient, func() error, error) {
 	status, maybeHostMachinePortBinding, engineVersion, err := manager.GetEngineStatus(ctx)
 	if err != nil {
 		return nil, nil, stacktrace.Propagate(err, "An error occurred retrieving the Kurtosis engine status, which is necessary for creating a connection to the engine")
@@ -242,6 +247,7 @@ func (manager *EngineManager) StartEngineIdempotentlyWithCustomVersion(
 		manager.allowedCORSOrigins,
 		shouldStartInDebugMode,
 		githubAuthTokenOverride,
+		restartAPIContainers,
 	)
 	engineClient, engineClientCloseFunc, err := manager.startEngineWithGuarantor(ctx, status, engineGuarantor)
 	if err != nil {
@@ -340,7 +346,9 @@ func (manager *EngineManager) RestartEngineIdempotently(
 	restartEngineOnSameVersionIfAnyRunning bool,
 	poolSize uint8,
 	shouldStartInDebugMode bool,
-	githubAuthTokenOverride string) (kurtosis_engine_rpc_api_bindings.EngineServiceClient, func() error, error) {
+	githubAuthTokenOverride string,
+	shouldRestartAPIContainers bool,
+) (kurtosis_engine_rpc_api_bindings.EngineServiceClient, func() error, error) {
 	var versionOfNewEngine string
 	// We try to do our best to restart an engine on the same version the current on is on
 	_, _, currentEngineVersion, err := manager.GetEngineStatus(ctx)
@@ -365,9 +373,9 @@ func (manager *EngineManager) RestartEngineIdempotently(
 	var engineClientCloseFunc func() error
 	var restartEngineErr error
 	if versionOfNewEngine != defaultEngineVersion {
-		_, engineClientCloseFunc, restartEngineErr = manager.StartEngineIdempotentlyWithCustomVersion(ctx, versionOfNewEngine, logLevel, poolSize, shouldStartInDebugMode, githubAuthTokenOverride)
+		_, engineClientCloseFunc, restartEngineErr = manager.StartEngineIdempotentlyWithCustomVersion(ctx, versionOfNewEngine, logLevel, poolSize, shouldStartInDebugMode, githubAuthTokenOverride, shouldRestartAPIContainers)
 	} else {
-		_, engineClientCloseFunc, restartEngineErr = manager.StartEngineIdempotentlyWithDefaultVersion(ctx, logLevel, poolSize, githubAuthTokenOverride)
+		_, engineClientCloseFunc, restartEngineErr = manager.StartEngineIdempotentlyWithDefaultVersion(ctx, logLevel, poolSize, githubAuthTokenOverride, shouldRestartAPIContainers)
 	}
 	if restartEngineErr != nil {
 		return nil, nil, stacktrace.Propagate(restartEngineErr, "An error occurred starting a new engine")

container-engine-lib/lib/backend_interface/kurtosis_backend.go

@@ -169,7 +169,7 @@ type KurtosisBackend interface {
 		resultErr error,
 	)
 
-	// Stops API containers matching the given filters
+	// Destroy API containers matching the given filters
 	DestroyAPIContainers(
 		ctx context.Context,
 		filters *api_container.APIContainerFilters,

engine/launcher/args/args.go

@@ -61,6 +61,9 @@ type EngineServerArgs struct {
 
 	// List of allowed origins to validate CORS requests on the REST API. If undefined, defaults to '*' (any origin).
 	AllowedCORSOrigins *[]string `json:"allowed_cors_origins,omitempty"`
+
+	// To restart the current API containers after the engine has been restarted
+	RestartAPIContainers bool `json:"restart_api_containers"`
 }
 
 var skipValidation = map[string]bool{
@@ -116,6 +119,7 @@ func NewEngineServerArgs(
 	cloudUserID metrics_client.CloudUserID,
 	cloudInstanceID metrics_client.CloudInstanceID,
 	allowedCORSOrigins *[]string,
+	restartAPIContainers bool,
 ) (*EngineServerArgs, error) {
 	if enclaveEnvVars == "" {
 		enclaveEnvVars = emptyJsonField
@@ -135,6 +139,7 @@ func NewEngineServerArgs(
 		CloudUserID:                 cloudUserID,
 		CloudInstanceID:             cloudInstanceID,
 		AllowedCORSOrigins:          allowedCORSOrigins,
+		RestartAPIContainers:        restartAPIContainers,
 	}
 	if err := result.validate(); err != nil {
 		return nil, stacktrace.Propagate(err, "An error occurred validating engine server args")

engine/launcher/engine_server_launcher/engine_server_launcher.go

@@ -45,7 +45,9 @@ func (launcher *EngineServerLauncher) LaunchWithDefaultVersion(
 	cloudInstanceID metrics_client.CloudInstanceID,
 	allowedCORSOrigins *[]string,
 	shouldStartInDebugMode bool,
-	githubAuthToken string) (
+	githubAuthToken string,
+	restartAPIContainers bool,
+) (
 	resultPublicIpAddr net.IP,
 	resultPublicGrpcPortSpec *port_spec.PortSpec,
 	resultErr error,
@@ -66,7 +68,8 @@ func (launcher *EngineServerLauncher) LaunchWithDefaultVersion(
 		cloudInstanceID,
 		allowedCORSOrigins,
 		shouldStartInDebugMode,
-		githubAuthToken)
+		githubAuthToken,
+		restartAPIContainers)
 	if err != nil {
 		return nil, nil, stacktrace.Propagate(err, "An error occurred launching the engine server container with default version tag '%v'", kurtosis_version.KurtosisVersion)
 	}
@@ -89,7 +92,9 @@ func (launcher *EngineServerLauncher) LaunchWithCustomVersion(
 	cloudInstanceID metrics_client.CloudInstanceID,
 	allowedCORSOrigins *[]string,
 	shouldStartInDebugMode bool,
-	githubAuthToken string) (
+	githubAuthToken string,
+	restartAPIContainers bool,
+) (
 	resultPublicIpAddr net.IP,
 	resultPublicGrpcPortSpec *port_spec.PortSpec,
 	resultErr error,
@@ -111,6 +116,7 @@ func (launcher *EngineServerLauncher) LaunchWithCustomVersion(
 		cloudUserID,
 		cloudInstanceID,
 		allowedCORSOrigins,
+		restartAPIContainers,
 	)
 	if err != nil {
 		return nil, nil, stacktrace.Propagate(err, "An error occurred creating the engine server args")

engine/server/engine/enclave_manager/enclave_creator.go

@@ -100,7 +100,7 @@ func (creator *EnclaveCreator) CreateEnclave(
 		}()
 	}
 
-	apiContainer, err := creator.launchApiContainer(setupCtx,
+	apiContainer, err := creator.LaunchApiContainer(setupCtx,
 		apiContainerImageVersionTag,
 		apiContainerLogLevel,
 		enclaveUuid,
@@ -185,7 +185,7 @@ func (creator *EnclaveCreator) CreateEnclave(
 	return newEnclaveInfo, nil
 }
 
-func (creator *EnclaveCreator) launchApiContainer(
+func (creator *EnclaveCreator) LaunchApiContainer(
 	ctx context.Context,
 	apiContainerImageVersionTag string,
 	logLevel logrus.Level,