Merge branch 'main' into bgazzard/fix-polka

README.md

@@ -33,7 +33,7 @@ Check out an introductory demo video here:
 
 How do I get going?
 ===================
-To see Kurtosis in action, first install it using the instructions [here](https://docs.kurtosis.com/install) or visit [Kurtosis Cloud][https://cloud.kurtosis.com/] to provision a remote host.
+To see Kurtosis in action, first install it using the instructions [here](https://docs.kurtosis.com/install) or visit [Kurtosis Cloud](https://cloud.kurtosis.com/) to provision a remote host.
 
 Then, run the [Redis voting app Kurtosis package](https://github.com/kurtosis-tech/awesome-kurtosis/tree/main/redis-voting-app):
 

container-engine-lib/lib/backend_impls/docker/docker_kurtosis_backend/user_services_functions/persistent_directories.go

@@ -14,17 +14,17 @@ func getOrCreatePersistentDirectories(
 	ctx context.Context,
 	serviceUuid service.ServiceUUID,
 	objAttrsProvider object_attributes_provider.DockerEnclaveObjectAttributesProvider,
-	serviceMountpointsToPersistentKey map[string]service_directory.DirectoryPersistentKey,
+	serviceMountpointsToPersistentKey map[string]service_directory.PersistentDirectory,
 	dockerManager *docker_manager.DockerManager,
 ) (map[string]string, error) {
 	shouldDeleteVolumes := true
 	volumeNamesToRemoveIfFailure := map[string]bool{}
 	persistentDirectories := map[string]string{}
 
-	for serviceDirPath, persistentKey := range serviceMountpointsToPersistentKey {
-		volumeAttrs, err := objAttrsProvider.ForSinglePersistentDirectoryVolume(serviceUuid, persistentKey)
+	for serviceDirPath, persistentDirectory := range serviceMountpointsToPersistentKey {
+		volumeAttrs, err := objAttrsProvider.ForSinglePersistentDirectoryVolume(serviceUuid, persistentDirectory.PersistentKey)
 		if err != nil {
-			return nil, stacktrace.Propagate(err, "Error creating persistent directory labels for '%s'", persistentKey)
+			return nil, stacktrace.Propagate(err, "Error creating persistent directory labels for '%s'", persistentDirectory.PersistentKey)
 		}
 
 		volumeName := volumeAttrs.GetName().GetString()
@@ -45,11 +45,15 @@ func getOrCreatePersistentDirectories(
 			return nil, stacktrace.NewError("More than one volume with name '%s' exists in docker. This is unexpected", volumeName)
 		}
 
+		// note this doesn't consider persistentDirectory.Size
+		// Docker doesn't support sized volumes - the best you can do is create tmpfs (in memory) persistent volumes
+		// but that would go away if you restart Docker
+		// TODO Make the `lsp` aware of `size`
 		if err = dockerManager.CreateVolume(ctx, volumeName, volumeLabelsStrs); err != nil {
 			return nil, stacktrace.Propagate(
 				err,
 				"An error occurred creating persistent directory volume '%s' for service '%v'",
-				persistentKey,
+				persistentDirectory.PersistentKey,
 				serviceUuid,
 			)
 		}

container-engine-lib/lib/backend_impls/docker/docker_kurtosis_backend/user_services_functions/start_user_services.go

@@ -593,7 +593,7 @@ func createStartServiceOperation(
 				ctx,
 				serviceUUID,
 				enclaveObjAttrsProvider,
-				persistentDirectories.ServiceDirpathToDirectoryPersistentKey,
+				persistentDirectories.ServiceDirpathToPersistentDirectory,
 				dockerManager,
 			)
 			if err != nil {

container-engine-lib/lib/backend_impls/docker/docker_manager/docker_manager.go

@@ -414,19 +414,7 @@ func (manager *DockerManager) CreateVolume(context context.Context, volumeName s
 		Name:              volumeName,
 	}
 
-	/*
-		We don't use the return value of VolumeCreate because there's not much useful information on there - Docker doesn't
-		use UUIDs to identify volumes - only the name - so there's no UUID to retrieve, and the volume's Mountpoint (what you'd
-		think would be the path of the volume on the local machine) isn't useful either because Docker itself runs inside a VM
-		so *this path is only a path inside the Docker VM* (meaning we can't use it to read/write files). AFAICT, the only way
-		to read/write data to a volume is to mount it in a container. ~ ktoday, 2020-07-01
-	*/
-	_, err := manager.dockerClient.VolumeCreate(context, volumeConfig)
-	if err != nil {
-		return stacktrace.Propagate(err, "Could not create Docker volume for test controller")
-	}
-
-	return nil
+	return manager.createPersistentVolumeInternal(context, volumeConfig)
 }
 
 /*
@@ -1374,6 +1362,23 @@ func (manager *DockerManager) GetAvailableCPUAndMemory(ctx context.Context) (com
 //	INSTANCE HELPER FUNCTIONS
 //
 // =================================================================================================================
+func (manager *DockerManager) createPersistentVolumeInternal(context context.Context, volumeConfig volume.CreateOptions) error {
+	/*
+		We don't use the return value of VolumeCreate because there's not much useful information on there - Docker doesn't
+		use UUIDs to identify volumes - only the name - so there's no UUID to retrieve, and the volume's Mountpoint (what you'd
+		think would be the path of the volume on the local machine) isn't useful either because Docker itself runs inside a VM
+		so *this path is only a path inside the Docker VM* (meaning we can't use it to read/write files). AFAICT, the only way
+		to read/write data to a volume is to mount it in a container. ~ ktoday, 2020-07-01
+	*/
+	_, err := manager.dockerClient.VolumeCreate(context, volumeConfig)
+	if err != nil {
+		return stacktrace.Propagate(err, "Could not create Docker volume for test controller")
+	}
+
+	return nil
+
+}
+
 func (manager *DockerManager) isImageAvailableLocally(imageName string) (bool, error) {
 	// Own context for checking if the image is locally available because we do not want to cancel this works in case the main context in the request is cancelled
 	// if the first request fails the image will be ready for following request making the process faster

container-engine-lib/lib/backend_impls/kubernetes/kubernetes_kurtosis_backend/user_services_functions/persistent_directories.go

@@ -73,7 +73,7 @@ func preparePersistentDirectoriesResources(
 	namespace string,
 	serviceUuid service.ServiceUUID,
 	objAttributeProviders object_attributes_provider.KubernetesEnclaveObjectAttributesProvider,
-	serviceMountpointsToPersistentKey map[string]service_directory.DirectoryPersistentKey,
+	serviceMountpointsToPersistentKey map[string]service_directory.PersistentDirectory,
 	kubernetesManager *kubernetes_manager.KubernetesManager,
 ) (map[string]*kubernetesVolumeWithClaim, error) {
 	shouldDeleteVolumesAndClaimsCreated := true
@@ -82,10 +82,10 @@ func preparePersistentDirectoriesResources(
 
 	persistentVolumesAndClaims := map[string]*kubernetesVolumeWithClaim{}
 
-	for dirPath, persistentKey := range serviceMountpointsToPersistentKey {
-		volumeAttrs, err := objAttributeProviders.ForSinglePersistentDirectoryVolume(serviceUuid, persistentKey)
+	for dirPath, persistentDirectory := range serviceMountpointsToPersistentKey {
+		volumeAttrs, err := objAttributeProviders.ForSinglePersistentDirectoryVolume(serviceUuid, persistentDirectory.PersistentKey)
 		if err != nil {
-			return nil, stacktrace.Propagate(err, "An error occurred creating the labels for persist service directory '%s'", persistentKey)
+			return nil, stacktrace.Propagate(err, "An error occurred creating the labels for persist service directory '%s'", persistentDirectory.PersistentKey)
 		}
 
 		volumeName := volumeAttrs.GetName().GetString()
@@ -94,21 +94,23 @@ func preparePersistentDirectoriesResources(
 			volumeLabelsStrs[key.GetString()] = value.GetString()
 		}
 
+		persistentVolumeSize := int64(persistentDirectory.Size)
+
 		var persistentVolume *apiv1.PersistentVolume
 		if persistentVolume, err = kubernetesManager.GetPersistentVolume(ctx, volumeName); err != nil {
-			persistentVolume, err = kubernetesManager.CreatePersistentVolume(ctx, namespace, volumeName, volumeLabelsStrs)
+			persistentVolume, err = kubernetesManager.CreatePersistentVolume(ctx, namespace, volumeName, volumeLabelsStrs, persistentVolumeSize)
 			if err != nil {
-				return nil, stacktrace.Propagate(err, "An error occurred creating the persistent volume for '%s'", persistentKey)
+				return nil, stacktrace.Propagate(err, "An error occurred creating the persistent volume for '%s'", persistentDirectory.PersistentKey)
 			}
 			volumesCreated[persistentVolume.Name] = persistentVolume
 		}
 
 		// For now, we have a 1:1 mapping between volume and volume claims, so it's fine giving it the same name
 		var persistentVolumeClaim *apiv1.PersistentVolumeClaim
 		if persistentVolumeClaim, err = kubernetesManager.GetPersistentVolumeClaim(ctx, namespace, volumeName); err != nil {
-			persistentVolumeClaim, err = kubernetesManager.CreatePersistentVolumeClaim(ctx, namespace, volumeName, volumeLabelsStrs)
+			persistentVolumeClaim, err = kubernetesManager.CreatePersistentVolumeClaim(ctx, namespace, volumeName, volumeLabelsStrs, persistentVolumeSize)
 			if err != nil {
-				return nil, stacktrace.Propagate(err, "An error occurred creating the persistent volume claim for '%s'", persistentKey)
+				return nil, stacktrace.Propagate(err, "An error occurred creating the persistent volume claim for '%s'", persistentDirectory.PersistentKey)
 			}
 			volumeClaimsCreated[persistentVolumeClaim.Name] = persistentVolumeClaim
 		}

container-engine-lib/lib/backend_impls/kubernetes/kubernetes_kurtosis_backend/user_services_functions/start_user_services.go

@@ -346,7 +346,7 @@ func createStartServiceOperation(
 				namespaceName,
 				serviceUuid,
 				enclaveObjAttributesProvider,
-				persistentDirectories.ServiceDirpathToDirectoryPersistentKey,
+				persistentDirectories.ServiceDirpathToPersistentDirectory,
 				kubernetesManager)
 			if err != nil {
 				return nil, stacktrace.Propagate(err, "An error occurred creating the persistent volumes and claims requested for service '%s'", serviceName)

container-engine-lib/lib/backend_impls/kubernetes/kubernetes_manager/kubernetes_manager.go

@@ -75,10 +75,9 @@ const (
 	volumeHostPathRootDirectory = "/kurtosis-persistent-service-data"
 	// TODO: Maybe pipe this to Starlark to let users choose the size of their persistent directories
 	//  The difficulty is that Docker doesn't have such a feature, so we would need somehow to hack it
-	persistentVolumeDefaultSize                          int64 = 500 * 1024 * 1024 // 500Mb
-	waitForPersistentVolumeBoundTimeout                        = 30 * time.Second
-	waitForPersistentVolumeBoundInitialDelayMilliSeconds       = 100
-	waitForPersistentVolumeBoundRetriesDelayMilliSeconds       = 500
+	waitForPersistentVolumeBoundTimeout                  = 30 * time.Second
+	waitForPersistentVolumeBoundInitialDelayMilliSeconds = 100
+	waitForPersistentVolumeBoundRetriesDelayMilliSeconds = 500
 )
 
 // We'll try to use the nicer-to-use shells first before we drop down to the lower shells
@@ -306,7 +305,12 @@ func (manager *KubernetesManager) CreatePersistentVolume(
 	namespace string,
 	volumeName string,
 	labels map[string]string,
+	requiredSize int64,
 ) (*apiv1.PersistentVolume, error) {
+	if requiredSize == 0 {
+		return nil, stacktrace.NewError("Cannot create volume '%v' of size 0; need size greater than 0", volumeName)
+	}
+
 	volumesClient := manager.kubernetesClientSet.CoreV1().PersistentVolumes()
 
 	// Check that there's only one node, otherwise using HostPath volumes will not work.
@@ -356,7 +360,7 @@ func (manager *KubernetesManager) CreatePersistentVolume(
 		},
 		Spec: apiv1.PersistentVolumeSpec{
 			Capacity: apiv1.ResourceList{
-				apiv1.ResourceStorage: *resource.NewQuantity(persistentVolumeDefaultSize, resource.BinarySI),
+				apiv1.ResourceStorage: *resource.NewQuantity(requiredSize, resource.BinarySI),
 			},
 			PersistentVolumeSource: apiv1.PersistentVolumeSource{
 				GCEPersistentDisk:    nil,
@@ -463,7 +467,12 @@ func (manager *KubernetesManager) CreatePersistentVolumeClaim(
 	namespace string,
 	volumeClaimName string,
 	labels map[string]string,
+	requiredSize int64,
 ) (*apiv1.PersistentVolumeClaim, error) {
+	if requiredSize == 0 {
+		return nil, stacktrace.NewError("Cannot create volume '%v' of 0 size; need a value greater than 0", volumeClaimName)
+	}
+
 	volumeClaimsClient := manager.kubernetesClientSet.CoreV1().PersistentVolumeClaims(namespace)
 
 	volumeClaimsDefinition := apiv1.PersistentVolumeClaim{
@@ -500,7 +509,7 @@ func (manager *KubernetesManager) CreatePersistentVolumeClaim(
 				Requests: apiv1.ResourceList{
 					// we give each claim 100% of the corresponding volume. Since we have a 1:1 mapping between volumes
 					// and claims right now, it's the best we can do
-					apiv1.ResourceStorage: *resource.NewQuantity(persistentVolumeDefaultSize, resource.BinarySI),
+					apiv1.ResourceStorage: *resource.NewQuantity(requiredSize, resource.BinarySI),
 				},
 				Claims: nil,
 			},

container-engine-lib/lib/backend_interface/objects/service/service_config_test.go

@@ -79,9 +79,9 @@ func getServiceConfigForTest(t *testing.T, imageName string) *ServiceConfig {
 }
 
 func testPersistentDirectory() *service_directory.PersistentDirectories {
-	persistentDirectoriesMap := map[string]service_directory.DirectoryPersistentKey{
-		"dirpath1": service_directory.DirectoryPersistentKey("dirpath1_persistent_directory_key"),
-		"dirpath2": service_directory.DirectoryPersistentKey("dirpath2_persistent_directory_key"),
+	persistentDirectoriesMap := map[string]service_directory.PersistentDirectory{
+		"dirpath1": {PersistentKey: service_directory.DirectoryPersistentKey("dirpath1_persistent_directory_key"), Size: service_directory.DirectoryPersistentSize(int64(0))},
+		"dirpath2": {PersistentKey: service_directory.DirectoryPersistentKey("dirpath2_persistent_directory_key"), Size: service_directory.DirectoryPersistentSize(int64(0))},
 	}
 
 	return service_directory.NewPersistentDirectories(persistentDirectoriesMap)

container-engine-lib/lib/backend_interface/objects/service_directory/persistent_directories.go

@@ -1,13 +1,19 @@
 package service_directory
 
 type DirectoryPersistentKey string
+type DirectoryPersistentSize int64
+
+type PersistentDirectory struct {
+	PersistentKey DirectoryPersistentKey
+	Size          DirectoryPersistentSize
+}
 
 type PersistentDirectories struct {
-	ServiceDirpathToDirectoryPersistentKey map[string]DirectoryPersistentKey
+	ServiceDirpathToPersistentDirectory map[string]PersistentDirectory
 }
 
-func NewPersistentDirectories(serviceDirpathToDirectoryPersistentKey map[string]DirectoryPersistentKey) *PersistentDirectories {
+func NewPersistentDirectories(persistentDirectories map[string]PersistentDirectory) *PersistentDirectories {
 	return &PersistentDirectories{
-		ServiceDirpathToDirectoryPersistentKey: serviceDirpathToDirectoryPersistentKey,
+		ServiceDirpathToPersistentDirectory: persistentDirectories,
 	}
 }

container-engine-lib/lib/database_accessors/enclave_db/service_registration/repository_test.go

@@ -326,9 +326,9 @@ func getServiceConfigForTest(t *testing.T, imageName string) *service.ServiceCon
 }
 
 func testPersistentDirectory() *service_directory.PersistentDirectories {
-	persistentDirectoriesMap := map[string]service_directory.DirectoryPersistentKey{
-		"dirpath1": service_directory.DirectoryPersistentKey("dirpath1_persistent_directory_key"),
-		"dirpath2": service_directory.DirectoryPersistentKey("dirpath2_persistent_directory_key"),
+	persistentDirectoriesMap := map[string]service_directory.PersistentDirectory{
+		"dirpath1": {PersistentKey: service_directory.DirectoryPersistentKey("dirpath1_persistent_directory_key"), Size: service_directory.DirectoryPersistentSize(int64(0))},
+		"dirpath2": {PersistentKey: service_directory.DirectoryPersistentKey("dirpath2_persistent_directory_key"), Size: service_directory.DirectoryPersistentSize(int64(0))},
 	}
 
 	return service_directory.NewPersistentDirectories(persistentDirectoriesMap)

core/server/api_container/server/startosis_engine/kurtosis_starlark_framework/builtin_argument/validators.go

@@ -36,6 +36,21 @@ func Uint64InRange(value starlark.Value, argNameForLogging string, min uint64, m
 	return nil
 }
 
+func Int64InRange(value starlark.Value, argNameForLogging string, min int64, max int64) *startosis_errors.InterpretationError {
+	valueInt, ok := value.(starlark.Int)
+	if !ok {
+		return startosis_errors.NewInterpretationError("Value for '%s' was expected to be an integer between %d and %d, but it was '%s'", argNameForLogging, min, max, reflect.TypeOf(value))
+	}
+	valueInt64, ok := valueInt.Int64()
+	if !ok {
+		return startosis_errors.NewInterpretationError("Value for '%s' was expected to be an integer between %d and %d, but it was %v", argNameForLogging, min, max, valueInt)
+	}
+	if valueInt64 < min || valueInt64 > max {
+		return startosis_errors.NewInterpretationError("Value for '%s' was expected to be an integer between %d and %d, but it was %v", argNameForLogging, min, max, valueInt64)
+	}
+	return nil
+}
+
 func FloatInRange(value starlark.Value, argNameForLogging string, min float64, max float64) *startosis_errors.InterpretationError {
 	valueStarlarkFloat, ok := value.(starlark.Float)
 	if !ok {

core/server/api_container/server/startosis_engine/kurtosis_starlark_framework/test_engine/directory_file_artifact_framework_test.go

@@ -4,6 +4,7 @@ import (
 	"fmt"
 	"github.com/kurtosis-tech/kurtosis/core/server/api_container/server/startosis_engine/kurtosis_starlark_framework/builtin_argument"
 	"github.com/kurtosis-tech/kurtosis/core/server/api_container/server/startosis_engine/kurtosis_types/directory"
+	"github.com/kurtosis-tech/kurtosis/core/server/api_container/server/startosis_engine/startosis_constants"
 	"github.com/stretchr/testify/require"
 	"testing"
 )
@@ -35,4 +36,8 @@ func (t *directoryFileArtifactTestCase) Assert(typeValue builtin_argument.Kurtos
 	require.Nil(t, err)
 	require.False(t, found)
 	require.Empty(t, persistentKey)
+
+	size, err := directoryStarlark.GetSizeOrDefault()
+	require.Nil(t, err)
+	require.Equal(t, startosis_constants.DefaultPersistentDirectorySize, size)
 }

core/server/api_container/server/startosis_engine/kurtosis_starlark_framework/test_engine/directory_persistent_directory_framework_test.go

@@ -19,7 +19,7 @@ func (suite *KurtosisTypeConstructorTestSuite) TestDirectoryPersistentDirectory(
 }
 
 func (t *directoryPersistentDirectoryTestCase) GetStarlarkCode() string {
-	return fmt.Sprintf("%s(%s=%q)", directory.DirectoryTypeName, directory.PersistentKeyAttr, testPersistentDirectoryKey)
+	return fmt.Sprintf("%s(%s=%q, %s=%d)", directory.DirectoryTypeName, directory.PersistentKeyAttr, testPersistentDirectoryKey, directory.SizeKeyAttr, testPersistentDirectorySize)
 }
 
 func (t *directoryPersistentDirectoryTestCase) Assert(typeValue builtin_argument.KurtosisValueType) {
@@ -35,4 +35,8 @@ func (t *directoryPersistentDirectoryTestCase) Assert(typeValue builtin_argument
 	require.Nil(t, err)
 	require.True(t, found)
 	require.Equal(t, testPersistentDirectoryKey, persistentKey)
+
+	size, err := directoryStarlark.GetSizeOrDefault()
+	require.Nil(t, err)
+	require.Equal(t, testPersistentDirectorySize, size)
 }

core/server/api_container/server/startosis_engine/kurtosis_starlark_framework/test_engine/service_config_full_framework_test.go

@@ -8,6 +8,7 @@ import (
 	"github.com/kurtosis-tech/kurtosis/core/server/api_container/server/startosis_engine/kurtosis_starlark_framework/builtin_argument"
 	"github.com/kurtosis-tech/kurtosis/core/server/api_container/server/startosis_engine/kurtosis_types/directory"
 	"github.com/kurtosis-tech/kurtosis/core/server/api_container/server/startosis_engine/kurtosis_types/service_config"
+	"github.com/kurtosis-tech/kurtosis/core/server/api_container/server/startosis_engine/startosis_constants"
 	"github.com/stretchr/testify/require"
 	"net"
 	"testing"
@@ -88,11 +89,14 @@ func (t *serviceConfigFullTestCase) Assert(typeValue builtin_argument.KurtosisVa
 	require.NotNil(t, serviceConfig.GetFilesArtifactsExpansion())
 	require.Equal(t, expectedFilesArtifactMap, serviceConfig.GetFilesArtifactsExpansion().ServiceDirpathsToArtifactIdentifiers)
 
-	expectedPersistentDirectoryMap := map[string]service_directory.DirectoryPersistentKey{
-		testPersistentDirectoryPath: service_directory.DirectoryPersistentKey(testPersistentDirectoryKey),
+	expectedPersistentDirectoryMap := map[string]service_directory.PersistentDirectory{
+		testPersistentDirectoryPath: {
+			PersistentKey: service_directory.DirectoryPersistentKey(testPersistentDirectoryKey),
+			Size:          service_directory.DirectoryPersistentSize(startosis_constants.DefaultPersistentDirectorySize),
+		},
 	}
 	require.NotNil(t, serviceConfig.GetPersistentDirectories())
-	require.Equal(t, expectedPersistentDirectoryMap, serviceConfig.GetPersistentDirectories().ServiceDirpathToDirectoryPersistentKey)
+	require.Equal(t, expectedPersistentDirectoryMap, serviceConfig.GetPersistentDirectories().ServiceDirpathToPersistentDirectory)
 
 	require.Equal(t, testEntryPointSlice, serviceConfig.GetEntrypointArgs())
 	require.Equal(t, testCmdSlice, serviceConfig.GetCmdArgs())

core/server/api_container/server/startosis_engine/kurtosis_starlark_framework/test_engine/static_constants.go

@@ -48,6 +48,7 @@ var (
 	testFilesArtifactName2      = "file_2"
 	testPersistentDirectoryPath = "path/to/persistent/dir"
 	testPersistentDirectoryKey  = "persistent-dir-test"
+	testPersistentDirectorySize = int64(30)
 
 	testEntryPointSlice = []string{
 		"127.0.0.0",