feat: Added enclave pool for improving performance on enclave creation (

#787)

## Description:
- Added enclave pool for improving performance on enclave creation
- The `pool-size` new flag was added in the `engine start and engine
restart` commands

## What is pending? 
- reuse idle enclaves when the engine is restarted

## Is this change user-facing?
YES

## References (if applicable):
[Docs are
here](https://www.notion.so/kurtosistech/Enclave-Pool-Project-7273552e53e44b0f931a57568cd170a0?d=2f870450d01d4ad7b8af6d0045602a2a#a20862960dad4868b0b9b6eb320b6178)

---------

Signed-off-by: dependabot[bot] <support@github.com>
Co-authored-by: Guillaume Bouvignies <guillaume.bouvignies@kurtosistech.com>
Co-authored-by: dependabot[bot] <49699333+dependabot[bot]@users.noreply.github.com>
Co-authored-by: kurtosisbot <89932784+kurtosisbot@users.noreply.github.com>
Co-authored-by: kurtosisbot <kurtosisbot@users.noreply.github.com>
Co-authored-by: Gyanendra Mishra <anomaly.the@gmail.com>
Co-authored-by: h4ck3rk3y <h4ck3rk3y@users.noreply.github.com>
Co-authored-by: thylocine33 <137217810+thylocine33@users.noreply.github.com>
Co-authored-by: Karla Forester <karlaforester@Karlas-MacBook-Pro.local>

.circleci/config.yml

@@ -372,7 +372,7 @@ jobs:
           image_name_with_version="${IMAGE_ORG_AND_REPO}:${version_to_publish}"
           echo "Version that will be persisted to workspace: ${version_to_publish}"
           docker save -o << pipeline.parameters.engine-server-image-filename >> "${image_name_with_version}"          
-          
+
 
       - save_cache:
           key: << pipeline.parameters.engine-server-build-cache-key-prefix>>-{{ checksum "engine/server/go.sum" }}
@@ -478,7 +478,7 @@ jobs:
                   touch /tmp/testsuite-failed
                 fi
 
-      # TODO TODO TODO DELETE THIS HACK!!!!! 
+      # TODO TODO TODO DELETE THIS HACK!!!!!
       # As of 2022-11-18, 'enclave dump' in Kubernetes in CircleCI (but not locally) takes 30s to run per enclave, and "suceeds"
       # but without dumping anything! To avoid our CI runs getting an extra 15 minutes added on, we're only dumping enclaves in Docker
       - when:
@@ -490,7 +490,7 @@ jobs:
       # TODO UNCOMMENT THIS WHEN WE FIX THE BUG ABOVE!!!
       # Dump enclave data so we can debug any issues that arise
       # - <<: *run_dump_kurtosis_enclaves
-      
+
       - store_artifacts:
           path: /tmp/enclave-dumps.zip
           destination: enclave-dumps.zip
@@ -547,7 +547,7 @@ jobs:
                    touch /tmp/testsuite-failed
                  fi
 
-      # TODO TODO TODO DELETE THIS HACK!!!!! 
+      # TODO TODO TODO DELETE THIS HACK!!!!!
       # As of 2022-11-18, 'enclave dump' in Kubernetes in CircleCI (but not locally) takes 30s to run per enclave, and "suceeds"
       # but without dumping anything! To avoid our CI runs getting an extra 15 minutes added on, we're only dumping enclaves in Docker
       - when:
@@ -559,7 +559,7 @@ jobs:
       # TODO UNCOMMENT THIS WHEN WE FIX THE BUG ABOVE!!!
       # Dump enclave data so we can debug any issues that arise
       # - <<: *run_dump_kurtosis_enclaves
-      
+
       - store_artifacts:
           path: /tmp/enclave-dumps.zip
           destination: enclave-dumps.zip
@@ -619,7 +619,7 @@ jobs:
       - run:
           name: "Verify only the engine container remains after the clean"
           command: |
-            if ! [ "$(docker container ls -a | tail -n+2 | wc -l)" -eq 1 ]; then
+            if ! [ "$(docker container ls -a | tail -n+2 | wc -l)" -eq 0 ]; then
               docker container ls -a
               false
             fi
@@ -710,7 +710,7 @@ jobs:
 
       # Execute github starlark module
       - run: "${KURTOSIS_BINPATH} run --enclave test-datastore github.com/kurtosis-tech/datastore-army-package '{\"num_datastores\": 2}'"
-        
+
 
       - run: "${KURTOSIS_BINPATH} enclave stop test-enclave"
       - run: "${KURTOSIS_BINPATH} enclave stop test-datastore"
@@ -804,11 +804,11 @@ jobs:
             equal: [ "docker", << parameters.cli-cluster-backend >> ]
           <<: *steps_prepare_testing_docker
 
-    # adding same enclave fails
+      # adding same enclave fails
       - run: "${KURTOSIS_BINPATH} enclave add --name test-enclave"
       - run: "! ${KURTOSIS_BINPATH} service add --name test-encalve"
 
-    # running a broken package fails
+      # running a broken package fails
       - run: "! ${KURTOSIS_BINPATH} run github.com/kurtosis-tech/datastore-army-package@25ecb4c271f870d6fdf8745169a142519e49aab8"
 
   # -- End build jobs ---------------------------------------
@@ -1114,17 +1114,17 @@ workflows:
             - build_files_artifacts_expander
           <<: *filters_ignore_main
 
-#      - test_basic_cli_functionality:
-#          name: "Test Basic CLI Functionality in Kubernetes"
-#          cli-cluster-backend: "minikube"
-#          context:
-#            - docker-user
-#          requires:
-#            - build_cli
-#            - build_api_container_server
-#            - build_engine_server
-#            - build_files_artifacts_expander
-#          <<: *filters_ignore_main
+      #      - test_basic_cli_functionality:
+      #          name: "Test Basic CLI Functionality in Kubernetes"
+      #          cli-cluster-backend: "minikube"
+      #          context:
+      #            - docker-user
+      #          requires:
+      #            - build_cli
+      #            - build_api_container_server
+      #            - build_engine_server
+      #            - build_files_artifacts_expander
+      #          <<: *filters_ignore_main
 
       - test_ci_for_failure:
           name: "Test Basic CLI for failure"

cli/cli/command_framework/highlevel/engine_consuming_kurtosis_command/engine_consuming_kurtosis_command.go

@@ -173,7 +173,7 @@ func (cmd *EngineConsumingKurtosisCommand) getSetupFunc() func(context.Context)
 		//  commands only access the Kurtosis APIs, we can remove this.
 		kurtosisBackend := engineManager.GetKurtosisBackend()
 
-		engineClient, closeClientFunc, err := engineManager.StartEngineIdempotentlyWithDefaultVersion(ctx, defaults.DefaultEngineLogLevel)
+		engineClient, closeClientFunc, err := engineManager.StartEngineIdempotentlyWithDefaultVersion(ctx, defaults.DefaultEngineLogLevel, defaults.DefaultEngineEnclavePoolSize)
 		if err != nil {
 			return nil, stacktrace.Propagate(err, "An error occurred creating a new Kurtosis engine client")
 		}

cli/cli/commands/cluster/set/set.go

@@ -6,6 +6,7 @@ import (
 	"github.com/kurtosis-tech/kurtosis/cli/cli/command_framework/lowlevel/args"
 	"github.com/kurtosis-tech/kurtosis/cli/cli/command_framework/lowlevel/flags"
 	"github.com/kurtosis-tech/kurtosis/cli/cli/command_str_consts"
+	"github.com/kurtosis-tech/kurtosis/cli/cli/defaults"
 	"github.com/kurtosis-tech/kurtosis/cli/cli/helpers/engine_manager"
 	"github.com/kurtosis-tech/kurtosis/cli/cli/kurtosis_cluster_setting"
 	"github.com/kurtosis-tech/kurtosis/cli/cli/kurtosis_config"
@@ -90,7 +91,7 @@ func run(ctx context.Context, flags *flags.ParsedFlags, args *args.ParsedArgs) e
 		return stacktrace.Propagate(err, "An error occurred creating an engine manager.")
 	}
 	// Start the engine inside the new cluster
-	_, engineClientCloseFunc, err := engineManagerNewCluster.StartEngineIdempotentlyWithDefaultVersion(ctx, logrus.InfoLevel)
+	_, engineClientCloseFunc, err := engineManagerNewCluster.StartEngineIdempotentlyWithDefaultVersion(ctx, defaults.DefaultEngineLogLevel, defaults.DefaultEngineEnclavePoolSize)
 	if err != nil {
 		return stacktrace.Propagate(err, "Engine could not be started after cluster was updated. The cluster "+
 			"will be rolled back, but it is possible the engine will remain stopped. Its status can be retrieved "+

cli/cli/commands/enclave/add/add.go

@@ -113,7 +113,7 @@ func run(
 	if err != nil {
 		return stacktrace.Propagate(err, "An error occurred creating an engine manager.")
 	}
-	engineClient, closeClientFunc, err := engineManager.StartEngineIdempotentlyWithDefaultVersion(ctx, defaults.DefaultEngineLogLevel)
+	engineClient, closeClientFunc, err := engineManager.StartEngineIdempotentlyWithDefaultVersion(ctx, defaults.DefaultEngineLogLevel, defaults.DefaultEngineEnclavePoolSize)
 	if err != nil {
 		return stacktrace.Propagate(err, "An error occurred creating a new Kurtosis engine client")
 	}

cli/cli/commands/engine/common/common.go

@@ -0,0 +1,34 @@
+package common
+
+import (
+	"github.com/kurtosis-tech/kurtosis/cli/cli/helpers/kurtosis_config_getter"
+	"github.com/kurtosis-tech/kurtosis/cli/cli/kurtosis_config/resolved_config"
+	"github.com/kurtosis-tech/stacktrace"
+)
+
+func ValidateEnclavePoolSizeFlag(enclavePoolSize uint8) error {
+	if enclavePoolSize > 0 {
+		isEnclavePoolAvailableForCurrentCluster, err := isEnclavePoolAvailableForCurrentClusterType()
+		if err != nil {
+			return stacktrace.Propagate(err, "An error occurred validating if the enclave pool feature is valid for the current cluster type")
+		}
+		if !isEnclavePoolAvailableForCurrentCluster {
+			return stacktrace.NewError("The enclave pool feature is not available for the current cluster type, please switch to the 'Kubernetes' Kurtosis backend type if you want to use it.")
+		}
+	}
+	return nil
+}
+
+func isEnclavePoolAvailableForCurrentClusterType() (bool, error) {
+	clusterConfig, err := kurtosis_config_getter.GetKurtosisClusterConfig()
+	if err != nil {
+		return false, stacktrace.Propagate(err, "An error occurred getting the Kurtosis cluster config")
+	}
+
+	clusterType := clusterConfig.GetClusterType()
+
+	if clusterType == resolved_config.KurtosisClusterType_Kubernetes {
+		return true, nil
+	}
+	return false, nil
+}

cli/cli/commands/engine/restart/restart.go

@@ -4,6 +4,7 @@ import (
 	"context"
 	"fmt"
 	"github.com/kurtosis-tech/kurtosis/cli/cli/command_str_consts"
+	"github.com/kurtosis-tech/kurtosis/cli/cli/commands/engine/common"
 	"github.com/kurtosis-tech/kurtosis/cli/cli/defaults"
 	"github.com/kurtosis-tech/kurtosis/cli/cli/helpers/engine_manager"
 	"github.com/kurtosis-tech/kurtosis/cli/cli/helpers/logrus_log_levels"
@@ -14,15 +15,17 @@ import (
 )
 
 const (
-	engineVersionArg = "version"
-	logLevelArg      = "log-level"
+	engineVersionArg    = "version"
+	logLevelArg         = "log-level"
+	enclavePoolSizeFlag = "enclave-pool-size"
 
 	defaultEngineVersion                   = ""
 	restartEngineOnSameVersionIfAnyRunning = false
 )
 
 var engineVersion string
 var logLevelStr string
+var enclavePoolSize uint8
 
 // RestartCmd Suppressing exhaustruct requirement because this struct has ~40 properties
 // nolint: exhaustruct
@@ -52,11 +55,24 @@ func init() {
 			),
 		),
 	)
+	RestartCmd.Flags().Uint8Var(
+		&enclavePoolSize,
+		enclavePoolSizeFlag,
+		defaults.DefaultEngineEnclavePoolSize,
+		fmt.Sprintf(
+			"The enclave pool size, the default value is '%v' which means it will be disabled. CAUTION: This is only available for Kubernetes, and this command will fail if you want to use it for Docker.",
+			defaults.DefaultEngineEnclavePoolSize,
+		),
+	)
 }
 
 func run(cmd *cobra.Command, args []string) error {
 	ctx := context.Background()
 
+	if err := common.ValidateEnclavePoolSizeFlag(enclavePoolSize); err != nil {
+		return stacktrace.Propagate(err, "An error occurred validating the '%v' flag", enclavePoolSizeFlag)
+	}
+
 	logrus.Infof("Restarting Kurtosis engine...")
 
 	logLevel, err := logrus.ParseLevel(logLevelStr)
@@ -71,7 +87,7 @@ func run(cmd *cobra.Command, args []string) error {
 
 	var engineClientCloseFunc func() error
 	var restartEngineErr error
-	_, engineClientCloseFunc, restartEngineErr = engineManager.RestartEngineIdempotently(ctx, logLevel, engineVersion, restartEngineOnSameVersionIfAnyRunning)
+	_, engineClientCloseFunc, restartEngineErr = engineManager.RestartEngineIdempotently(ctx, logLevel, engineVersion, restartEngineOnSameVersionIfAnyRunning, enclavePoolSize)
 	if restartEngineErr != nil {
 		return stacktrace.Propagate(restartEngineErr, "An error occurred restarting the Kurtosis engine")
 	}

cli/cli/commands/engine/start/start.go

@@ -4,6 +4,7 @@ import (
 	"context"
 	"fmt"
 	"github.com/kurtosis-tech/kurtosis/cli/cli/command_str_consts"
+	"github.com/kurtosis-tech/kurtosis/cli/cli/commands/engine/common"
 	"github.com/kurtosis-tech/kurtosis/cli/cli/defaults"
 	"github.com/kurtosis-tech/kurtosis/cli/cli/helpers/engine_manager"
 	"github.com/kurtosis-tech/kurtosis/cli/cli/helpers/logrus_log_levels"
@@ -15,8 +16,9 @@ import (
 )
 
 const (
-	engineVersionArg = "version"
-	logLevelArg      = "log-level"
+	engineVersionArg    = "version"
+	logLevelArg         = "log-level"
+	enclavePoolSizeFlag = "enclave-pool-size"
 
 	defaultEngineVersion          = ""
 	kurtosisTechEngineImagePrefix = "kurtosistech/engine"
@@ -25,6 +27,7 @@ const (
 
 var engineVersion string
 var logLevelStr string
+var enclavePoolSize uint8
 
 // StartCmd Suppressing exhaustruct requirement because this struct has ~40 properties
 // nolint: exhaustruct
@@ -54,11 +57,24 @@ func init() {
 			),
 		),
 	)
+	StartCmd.Flags().Uint8Var(
+		&enclavePoolSize,
+		enclavePoolSizeFlag,
+		defaults.DefaultEngineEnclavePoolSize,
+		fmt.Sprintf(
+			"The enclave pool size, the default value is '%v' which means it will be disabled. CAUTION: This is only available for Kubernetes, and this command will fail if you want to use it for Docker.",
+			defaults.DefaultEngineEnclavePoolSize,
+		),
+	)
 }
 
 func run(cmd *cobra.Command, args []string) error {
 	ctx := context.Background()
 
+	if err := common.ValidateEnclavePoolSizeFlag(enclavePoolSize); err != nil {
+		return stacktrace.Propagate(err, "An error occurred validating the '%v' flag", enclavePoolSizeFlag)
+	}
+
 	logLevel, err := logrus.ParseLevel(logLevelStr)
 	if err != nil {
 		return stacktrace.Propagate(err, "An error occurred parsing log level string '%v'", logLevelStr)
@@ -73,10 +89,10 @@ func run(cmd *cobra.Command, args []string) error {
 	var startEngineErr error
 	if engineVersion == defaultEngineVersion {
 		logrus.Infof("Starting Kurtosis engine from image '%v%v%v'...", kurtosisTechEngineImagePrefix, imageVersionDelimiter, kurtosis_version.KurtosisVersion)
-		_, engineClientCloseFunc, startEngineErr = engineManager.StartEngineIdempotentlyWithDefaultVersion(ctx, logLevel)
+		_, engineClientCloseFunc, startEngineErr = engineManager.StartEngineIdempotentlyWithDefaultVersion(ctx, logLevel, enclavePoolSize)
 	} else {
 		logrus.Infof("Starting Kurtosis engine from image '%v%v%v'...", kurtosisTechEngineImagePrefix, imageVersionDelimiter, engineVersion)
-		_, engineClientCloseFunc, startEngineErr = engineManager.StartEngineIdempotentlyWithCustomVersion(ctx, engineVersion, logLevel)
+		_, engineClientCloseFunc, startEngineErr = engineManager.StartEngineIdempotentlyWithCustomVersion(ctx, engineVersion, logLevel, enclavePoolSize)
 	}
 	if startEngineErr != nil {
 		return stacktrace.Propagate(startEngineErr, "An error occurred starting the Kurtosis engine")

cli/cli/commands/kurtosis_context/context_switch/switch.go

@@ -9,6 +9,7 @@ import (
 	"github.com/kurtosis-tech/kurtosis/cli/cli/command_framework/lowlevel/args"
 	"github.com/kurtosis-tech/kurtosis/cli/cli/command_framework/lowlevel/flags"
 	"github.com/kurtosis-tech/kurtosis/cli/cli/command_str_consts"
+	"github.com/kurtosis-tech/kurtosis/cli/cli/defaults"
 	"github.com/kurtosis-tech/kurtosis/cli/cli/helpers/engine_manager"
 	"github.com/kurtosis-tech/kurtosis/cli/cli/helpers/portal_manager"
 	"github.com/kurtosis-tech/kurtosis/contexts-config-store/store"
@@ -113,7 +114,7 @@ func run(ctx context.Context, _ *flags.ParsedFlags, args *args.ParsedArgs) error
 	}
 	logrus.Infof("Context switched to '%s', Kurtosis engine will now be restarted", contextIdentifier)
 
-	_, engineClientCloseFunc, restartEngineErr := engineManager.RestartEngineIdempotently(ctx, logrus.InfoLevel, noEngineVersion, restartEngineOnSameVersionIfAnyRunning)
+	_, engineClientCloseFunc, restartEngineErr := engineManager.RestartEngineIdempotently(ctx, logrus.InfoLevel, noEngineVersion, restartEngineOnSameVersionIfAnyRunning, defaults.DefaultEngineEnclavePoolSize)
 	if restartEngineErr != nil {
 		return stacktrace.Propagate(restartEngineErr, "Engine could not be restarted after context was switched. The context"+
 			"will be rolled back, but it is possible the engine will remain stopped. Its status can be retrieved "+

cli/cli/defaults/defaults.go

@@ -14,6 +14,9 @@ const (
 	DefaultAPIContainerVersion = ""
 	// TODO perhaps move this to the metrics library
 	SendMetricsByDefault = true
+
+	// engine-enclave-pool-size = 0 means that enclave pool feat will be disabled
+	DefaultEngineEnclavePoolSize uint8 = 0
 )
 
 var DefaultApiContainerLogLevel = logrus.DebugLevel

cli/cli/helpers/engine_manager/engine_existence_guarantor.go

@@ -64,6 +64,8 @@ type engineExistenceGuarantor struct {
 
 	//TODO This is a temporary hack we should remove it when centralized logs be implemented in the KubernetesBackend
 	kurtosisClusterType resolved_config.KurtosisClusterType
+
+	poolSize uint8
 }
 
 func newEngineExistenceGuarantorWithDefaultVersion(
@@ -76,6 +78,7 @@ func newEngineExistenceGuarantorWithDefaultVersion(
 	logLevel logrus.Level,
 	maybeCurrentlyRunningEngineVersionTag string,
 	kurtosisClusterType resolved_config.KurtosisClusterType,
+	poolSize uint8,
 ) *engineExistenceGuarantor {
 	return newEngineExistenceGuarantorWithCustomVersion(
 		ctx,
@@ -88,6 +91,7 @@ func newEngineExistenceGuarantorWithDefaultVersion(
 		logLevel,
 		maybeCurrentlyRunningEngineVersionTag,
 		kurtosisClusterType,
+		poolSize,
 	)
 }
 
@@ -102,6 +106,7 @@ func newEngineExistenceGuarantorWithCustomVersion(
 	logLevel logrus.Level,
 	maybeCurrentlyRunningEngineVersionTag string,
 	kurtosisClusterType resolved_config.KurtosisClusterType,
+	poolSize uint8,
 ) *engineExistenceGuarantor {
 	return &engineExistenceGuarantor{
 		ctx:                                  ctx,
@@ -116,6 +121,7 @@ func newEngineExistenceGuarantorWithCustomVersion(
 		postVisitingHostMachineIpAndPort:          nil, // Will be filled in upon successful visitation
 		shouldSendMetrics:                         shouldSendMetrics,
 		kurtosisClusterType:                       kurtosisClusterType,
+		poolSize:                                  poolSize,
 	}
 }
 
@@ -143,6 +149,7 @@ func (guarantor *engineExistenceGuarantor) VisitStopped() error {
 			guarantor.shouldSendMetrics,
 			guarantor.engineServerKurtosisBackendConfigSupplier,
 			guarantor.kurtosisRemoteBackendConfigSupplier,
+			guarantor.poolSize,
 		)
 	} else {
 		_, _, engineLaunchErr = guarantor.engineServerLauncher.LaunchWithCustomVersion(
@@ -154,6 +161,7 @@ func (guarantor *engineExistenceGuarantor) VisitStopped() error {
 			guarantor.shouldSendMetrics,
 			guarantor.engineServerKurtosisBackendConfigSupplier,
 			guarantor.kurtosisRemoteBackendConfigSupplier,
+			guarantor.poolSize,
 		)
 	}
 	if engineLaunchErr != nil {