fix!: change persistent directory name to deterministic value (#2006)

Closes #1998

CHANGELOG.md

@@ -5,7 +5,7 @@
 
 ### Features
 
-* docker compose integration [pt. 1] ([#2001](https://github.com/kurtosis-tech/kurtosis/issues/2001)) ([385833d](https://github.com/kurtosis-tech/kurtosis/commit/385833de9d7620f4c65473adc763bb38df8fb995))
+* docker compose integration([#2001](https://github.com/kurtosis-tech/kurtosis/issues/2001)) ([385833d](https://github.com/kurtosis-tech/kurtosis/commit/385833de9d7620f4c65473adc763bb38df8fb995))
 
 
 ### Bug Fixes

container-engine-lib/lib/backend_impls/docker/object_attributes_provider/enclave_object_attributes_provider.go

@@ -1,8 +1,6 @@
 package object_attributes_provider
 
 import (
-	"crypto/md5"
-	"encoding/hex"
 	"fmt"
 	"net"
 	"strconv"
@@ -26,8 +24,7 @@ const (
 	networkPrefix          = "kt-"
 	apiContainerNamePrefix = "kurtosis-api"
 
-	artifactExpansionVolumeNameFragment    = "files-artifact-expansion"
-	persistentServiceDirectoryNameFragment = "service-persistent-directory"
+	artifactExpansionVolumeNameFragment = "files-artifact-expansion"
 
 	artifactsExpanderContainerNameFragment = "files-artifacts-expander"
 	logsCollectorFragment                  = "kurtosis-logs-collector"
@@ -333,14 +330,8 @@ func (provider *dockerEnclaveObjectAttributesProviderImpl) ForSinglePersistentDi
 		return nil, stacktrace.Propagate(err, "An error occurred generating a UUID for the persistent directory volume for service '%v'", serviceUuidStr)
 	}
 
-	hasher := md5.New()
-	hasher.Write([]byte(serviceUUID))
-	hasher.Write([]byte(persistentKey))
-	persistentKeyHash := hex.EncodeToString(hasher.Sum(nil))
-
 	name, err := provider.getNameForEnclaveObject([]string{
-		persistentServiceDirectoryNameFragment,
-		persistentKeyHash,
+		string(persistentKey),
 	})
 	if err != nil {
 		return nil, stacktrace.Propagate(err, "An error occurred creating the files artifact expansion volume name object using GUID '%v' and service GUID '%v'", guidStr, serviceUuidStr)

container-engine-lib/lib/backend_impls/kubernetes/object_attributes_provider/enclave_object_attributes_provider.go

@@ -25,8 +25,6 @@ const (
 
 	enclaveDataDirFragment = "enclave-data-dir"
 
-	persistentServiceDirectoryNameFragment = "service-persistent-directory"
-
 	traefikIngressRouterEntrypointsValue = "web"
 )
 
@@ -264,7 +262,7 @@ func (provider *kubernetesEnclaveObjectAttributesProviderImpl) ForSinglePersiste
 	//No userServiceService annotations.
 	annotations := map[*kubernetes_annotation_key.KubernetesAnnotationKey]*kubernetes_annotation_value.KubernetesAnnotationValue{}
 
-	name, err := getKubernetesPersistentDirectoryName(persistentKeyHash)
+	name, err := getKubernetesPersistentDirectoryName(string(persistentKey))
 	if err != nil {
 		return nil, stacktrace.Propagate(err, "Failed to create service persistent directory name for hash: '%s'", persistentKeyHash)
 	}
@@ -329,12 +327,11 @@ func getKubernetesObjectName(
 }
 
 func getKubernetesPersistentDirectoryName(
-	persistentServiceDirectoryName string,
+	persistentKey string,
 ) (*kubernetes_object_name.KubernetesObjectName, error) {
 	name, err := getCompositeKubernetesObjectName(
 		[]string{
-			persistentServiceDirectoryNameFragment,
-			persistentServiceDirectoryName,
+			persistentKey,
 		})
 	return name, err
 }

container-engine-lib/lib/backend_interface/objects/service_directory/persistent_directories.go

@@ -1,5 +1,23 @@
 package service_directory
 
+import "regexp"
+
+const (
+	// PersistentKeyRegex implements RFC-1035 for naming persistent directory keys, namely:
+	// * contain at most 63 characters
+	// * contain only lowercase alphanumeric characters or '-'
+	// * start with an alphabetic character
+	// * end with an alphanumeric character
+	// This is in order to stick to the 1035 standard which we enforce for all objects created
+	// https://kubernetes.io/docs/concepts/overview/working-with-objects/names/#dns-label-names
+	PersistentKeyRegex            = "[a-z]([-a-z0-9]{0,61}[a-z0-9])?"
+	WordWrappedPersistentKeyRegex = "^" + PersistentKeyRegex + "$"
+)
+
+var (
+	compiledWordWrappedPersistentKeyRegex = regexp.MustCompile(WordWrappedPersistentKeyRegex)
+)
+
 type DirectoryPersistentKey string
 type DirectoryPersistentSize int64
 
@@ -17,3 +35,7 @@ func NewPersistentDirectories(persistentDirectories map[string]PersistentDirecto
 		ServiceDirpathToPersistentDirectory: persistentDirectories,
 	}
 }
+
+func IsPersistentKeyValid(persistentKey DirectoryPersistentKey) bool {
+	return compiledWordWrappedPersistentKeyRegex.MatchString(string(persistentKey))
+}

core/server/api_container/server/startosis_engine/kurtosis_instruction/add_service/add_service_shared.go

@@ -4,6 +4,7 @@ import (
 	"context"
 	"fmt"
 	"github.com/kurtosis-tech/kurtosis/container-engine-lib/lib/backend_interface/objects/service"
+	"github.com/kurtosis-tech/kurtosis/container-engine-lib/lib/backend_interface/objects/service_directory"
 	"github.com/kurtosis-tech/kurtosis/core/server/api_container/server/service_network"
 	"github.com/kurtosis-tech/kurtosis/core/server/api_container/server/startosis_engine/kurtosis_instruction/shared_helpers"
 	"github.com/kurtosis-tech/kurtosis/core/server/api_container/server/startosis_engine/kurtosis_instruction/shared_helpers/magic_string_helper"
@@ -70,6 +71,18 @@ func validateSingleService(validatorEnvironment *startosis_validator.ValidatorEn
 		return startosis_errors.NewValidationError(invalidServiceNameErrorText(serviceName))
 	}
 
+	if persistentDirectories := serviceConfig.GetPersistentDirectories(); persistentDirectories != nil {
+		for _, directory := range persistentDirectories.ServiceDirpathToPersistentDirectory {
+			if !service_directory.IsPersistentKeyValid(directory.PersistentKey) {
+				return startosis_errors.NewValidationError(invalidPersistentKeyErrorText(directory.PersistentKey))
+			}
+			if validatorEnvironment.DoesPersistentKeyExist(directory.PersistentKey) == startosis_validator.ComponentCreatedOrUpdatedDuringPackageRun {
+				return startosis_errors.NewValidationError("There was an error validating '%s' as persistent key '%s' already exists inside the enclave", serviceName, directory.PersistentKey)
+			}
+			validatorEnvironment.AddPersistentKey(directory.PersistentKey)
+		}
+	}
+
 	if validatorEnvironment.DoesServiceNameExist(serviceName) == startosis_validator.ComponentCreatedOrUpdatedDuringPackageRun {
 		return startosis_errors.NewValidationError("There was an error validating '%s' as service with the name '%s' already exists inside the package. Adding two different services with the same name isn't allowed; we recommend prefixing/suffixing the two service names or using two different names entirely.", AddServiceBuiltinName, serviceName)
 	}
@@ -117,6 +130,16 @@ func invalidServiceNameErrorText(
 	)
 }
 
+func invalidPersistentKeyErrorText(
+	persistentKey service_directory.DirectoryPersistentKey,
+) string {
+	return fmt.Sprintf(
+		"Persistent Key '%v' is invalid as it contains disallowed characters. Persistent Key must adhere to the RFC 1035 standard, specifically implementing this regex and be 1-63 characters long: %s. This means the service name must only contain lowercase alphanumeric characters or '-', and must start with a lowercase alphabet and end with a lowercase alphanumeric character.",
+		persistentKey,
+		service_directory.WordWrappedPersistentKeyRegex,
+	)
+}
+
 func replaceMagicStrings(
 	runtimeValueStore *runtime_value_store.RuntimeValueStore,
 	serviceName service.ServiceName,

core/server/api_container/server/startosis_engine/startosis_validator/validator_environment.go

@@ -5,6 +5,7 @@ import (
 	"github.com/kurtosis-tech/kurtosis/container-engine-lib/lib/backend_interface/objects/image_build_spec"
 	"github.com/kurtosis-tech/kurtosis/container-engine-lib/lib/backend_interface/objects/image_download_mode"
 	"github.com/kurtosis-tech/kurtosis/container-engine-lib/lib/backend_interface/objects/service"
+	"github.com/kurtosis-tech/kurtosis/container-engine-lib/lib/backend_interface/objects/service_directory"
 	"github.com/kurtosis-tech/kurtosis/core/server/api_container/server/startosis_engine/startosis_errors"
 	"github.com/sirupsen/logrus"
 )
@@ -15,6 +16,7 @@ type ValidatorEnvironment struct {
 	imagesToBuild                 map[string]*image_build_spec.ImageBuildSpec
 	serviceNames                  map[service.ServiceName]ComponentExistence
 	artifactNames                 map[string]ComponentExistence
+	persistentKeys                map[service_directory.DirectoryPersistentKey]ComponentExistence
 	serviceNameToPrivatePortIDs   map[service.ServiceName][]string
 	availableCpuInMilliCores      compute_resources.CpuMilliCores
 	availableMemoryInMegaBytes    compute_resources.MemoryInMegaBytes
@@ -42,9 +44,11 @@ func NewValidatorEnvironment(serviceNames map[service.ServiceName]bool, artifact
 		availableCpuInMilliCores:      availableCpuInMilliCores,
 		availableMemoryInMegaBytes:    availableMemoryInMegaBytes,
 		isResourceInformationComplete: isResourceInformationComplete,
-		minMemoryByServiceName:        map[service.ServiceName]compute_resources.MemoryInMegaBytes{},
-		minCPUByServiceName:           map[service.ServiceName]compute_resources.CpuMilliCores{},
-		imageDownloadMode:             imageDownloadMode,
+		// TODO account for idempotent runs on this and make it pre-load the cache whenever we create a NewValidatorEnvironment
+		persistentKeys:         map[service_directory.DirectoryPersistentKey]ComponentExistence{},
+		minMemoryByServiceName: map[service.ServiceName]compute_resources.MemoryInMegaBytes{},
+		minCPUByServiceName:    map[service.ServiceName]compute_resources.CpuMilliCores{},
+		imageDownloadMode:      imageDownloadMode,
 	}
 }
 
@@ -160,3 +164,15 @@ func (environment *ValidatorEnvironment) HasEnoughMemory(memoryToConsume uint64,
 	}
 	return startosis_errors.NewValidationError("service '%v' requires '%v' megabytes of memory but based on our calculation we will only have '%v' megabytes available at the time we start the service", serviceNameForLogging, memoryToConsume, environment.availableMemoryInMegaBytes)
 }
+
+func (environment *ValidatorEnvironment) AddPersistentKey(persistentKey service_directory.DirectoryPersistentKey) {
+	environment.persistentKeys[persistentKey] = ComponentCreatedOrUpdatedDuringPackageRun
+}
+
+func (environment *ValidatorEnvironment) DoesPersistentKeyExist(persistentKey service_directory.DirectoryPersistentKey) ComponentExistence {
+	persistentKeyExistence, found := environment.persistentKeys[persistentKey]
+	if !found {
+		return ComponentNotFound
+	}
+	return persistentKeyExistence
+}

docs/docs/api-reference/starlark-reference/directory.md

@@ -24,7 +24,7 @@ the above example, `files_artifact_1` is a files artifact name. (see [upload_fil
 on how to create file artifacts). 
 
 A persistent directory, as its name indicates, persists over service updates and restarts. It is uniquely identified 
-by its `persistent_key` and the service ID on which it is being used (a persistent directory cannot be shared across
+by its `persistent_key` (a persistent directory cannot be shared across
 multiple services). When it is first created, it will be empty. The service can write anything in it. When the service 
 gets updated, the data in it persists. It is particularly useful for a service's data directory, logs directory, etc.
 

internal_testsuites/golang/testsuite/startosis_persistent_directory_test/startosis_persistent_directory_test.go

@@ -21,7 +21,7 @@ def run(plan):
 		image=IMAGE,
 		files={
 			"/data": Directory(
-				persistent_key="persistent_data",
+				persistent_key="persistent-data",
 			),
 		},
 		min_cpu=%d,