fix: allow persistent directory to be reused across services (#2123)

There are workflows where people want to use prepopulated persistent
volumes in different services. #2121 is an example of this workflow

This PR
- removes the Service UUID label from the PVCs / Persistent Volumes
- Allows PV to be re-used

Note in Kubernetes this will break for multi node clusters and this
reuse of PVs will have to be in conjunction with node affinity selectors

---------

Co-authored-by: Tedi Mitiku <tedi.m52@gmail.com>

container-engine-lib/lib/backend_impls/docker/docker_kurtosis_backend/user_services_functions/persistent_directories.go

@@ -22,7 +22,7 @@ func getOrCreatePersistentDirectories(
 	persistentDirectories := map[string]string{}
 
 	for serviceDirPath, persistentDirectory := range serviceMountpointsToPersistentKey {
-		volumeAttrs, err := objAttrsProvider.ForSinglePersistentDirectoryVolume(serviceUuid, persistentDirectory.PersistentKey)
+		volumeAttrs, err := objAttrsProvider.ForSinglePersistentDirectoryVolume(persistentDirectory.PersistentKey)
 		if err != nil {
 			return nil, stacktrace.Propagate(err, "Error creating persistent directory labels for '%s'", persistentDirectory.PersistentKey)
 		}

container-engine-lib/lib/backend_impls/docker/object_attributes_provider/enclave_object_attributes_provider.go

@@ -54,7 +54,6 @@ type DockerEnclaveObjectAttributesProvider interface {
 		serviceUUID service.ServiceUUID,
 	) (DockerObjectAttributes, error)
 	ForSinglePersistentDirectoryVolume(
-		serviceUUID service.ServiceUUID,
 		persistentKey service_directory.DirectoryPersistentKey,
 	) (DockerObjectAttributes, error)
 	ForLogsCollector(tcpPortId string, tcpPortSpec *port_spec.PortSpec, httpPortId string, httpPortSpec *port_spec.PortSpec) (DockerObjectAttributes, error)
@@ -317,36 +316,28 @@ func (provider *dockerEnclaveObjectAttributesProviderImpl) ForSingleFilesArtifac
 
 // In Docker we get one volume per persistent directory
 func (provider *dockerEnclaveObjectAttributesProviderImpl) ForSinglePersistentDirectoryVolume(
-	serviceUUID service.ServiceUUID,
 	persistentKey service_directory.DirectoryPersistentKey,
 ) (
 	DockerObjectAttributes,
 	error,
 ) {
-	serviceUuidStr := string(serviceUUID)
-
 	guidStr, err := uuid_generator.GenerateUUIDString()
 	if err != nil {
-		return nil, stacktrace.Propagate(err, "An error occurred generating a UUID for the persistent directory volume for service '%v'", serviceUuidStr)
+		return nil, stacktrace.Propagate(err, "An error occurred generating a UUID for the persistent directory volume for persistentKey '%v'", persistentKey)
 	}
 
 	name, err := provider.getNameForEnclaveObject([]string{
 		string(persistentKey),
 	})
 	if err != nil {
-		return nil, stacktrace.Propagate(err, "An error occurred creating the files artifact expansion volume name object using GUID '%v' and service GUID '%v'", guidStr, serviceUuidStr)
+		return nil, stacktrace.Propagate(err, "An error occurred creating the persistent volume name object using GUID '%v' and persistent key '%v'", guidStr, persistentKey)
 	}
 
 	labels, err := provider.getLabelsForEnclaveObjectWithGUID(guidStr)
 	if err != nil {
 		return nil, stacktrace.Propagate(err, "An error occurred getting labels for files artifact expansion volume with UUID '%v'", guidStr)
 	}
 
-	serviceUuidLabelValue, err := docker_label_value.CreateNewDockerLabelValue(serviceUuidStr)
-	if err != nil {
-		return nil, stacktrace.Propagate(err, "An error occurred creating a Docker label value from service GUID string '%v'", serviceUuidStr)
-	}
-	labels[docker_label_key.UserServiceGUIDDockerLabelKey] = serviceUuidLabelValue
 	labels[docker_label_key.VolumeTypeDockerLabelKey] = label_value_consts.PersistentDirectoryVolumeTypeDockerLabelValue
 	// TODO Create a KurtosisResourceDockerLabelKey object, like Kubernetes, and apply the "user-service" label here?
 

container-engine-lib/lib/backend_impls/kubernetes/kubernetes_kurtosis_backend/user_services_functions/persistent_directories.go

@@ -4,7 +4,6 @@ import (
 	"context"
 	"github.com/kurtosis-tech/kurtosis/container-engine-lib/lib/backend_impls/kubernetes/kubernetes_manager"
 	"github.com/kurtosis-tech/kurtosis/container-engine-lib/lib/backend_impls/kubernetes/object_attributes_provider"
-	"github.com/kurtosis-tech/kurtosis/container-engine-lib/lib/backend_interface/objects/service"
 	"github.com/kurtosis-tech/kurtosis/container-engine-lib/lib/backend_interface/objects/service_directory"
 	"github.com/kurtosis-tech/stacktrace"
 	"github.com/sirupsen/logrus"
@@ -69,7 +68,6 @@ func (volumeAndClaim *kubernetesVolumeWithClaim) GetVolumeMount(mountPath string
 func preparePersistentDirectoriesResources(
 	ctx context.Context,
 	namespace string,
-	serviceUuid service.ServiceUUID,
 	objAttributeProviders object_attributes_provider.KubernetesEnclaveObjectAttributesProvider,
 	serviceMountpointsToPersistentKey map[string]service_directory.PersistentDirectory,
 	kubernetesManager *kubernetes_manager.KubernetesManager,
@@ -80,7 +78,7 @@ func preparePersistentDirectoriesResources(
 	persistentVolumesAndClaims := map[string]*kubernetesVolumeWithClaim{}
 
 	for dirPath, persistentDirectory := range serviceMountpointsToPersistentKey {
-		volumeAttrs, err := objAttributeProviders.ForSinglePersistentDirectoryVolume(serviceUuid, persistentDirectory.PersistentKey)
+		volumeAttrs, err := objAttributeProviders.ForSinglePersistentDirectoryVolume(persistentDirectory.PersistentKey)
 		if err != nil {
 			return nil, stacktrace.Propagate(err, "An error occurred creating the labels for persist service directory '%s'", persistentDirectory.PersistentKey)
 		}

container-engine-lib/lib/backend_impls/kubernetes/kubernetes_kurtosis_backend/user_services_functions/start_user_services.go

@@ -356,7 +356,6 @@ func createStartServiceOperation(
 			createVolumesWithClaims, err = preparePersistentDirectoriesResources(
 				ctx,
 				namespaceName,
-				serviceUuid,
 				enclaveObjAttributesProvider,
 				persistentDirectories.ServiceDirpathToPersistentDirectory,
 				kubernetesManager)

container-engine-lib/lib/backend_impls/kubernetes/object_attributes_provider/enclave_object_attributes_provider.go

@@ -43,7 +43,6 @@ type KubernetesEnclaveObjectAttributesProvider interface {
 		userLabels map[string]string,
 	) (KubernetesObjectAttributes, error)
 	ForSinglePersistentDirectoryVolume(
-		serviceUUID service.ServiceUUID,
 		persistentKey service_directory.DirectoryPersistentKey,
 	) (KubernetesObjectAttributes, error)
 	ForUserServiceIngress(
@@ -242,10 +241,9 @@ func (provider *kubernetesEnclaveObjectAttributesProviderImpl) ForEnclaveDataDir
 	return objectAttributes, nil
 }
 
-func (provider *kubernetesEnclaveObjectAttributesProviderImpl) ForSinglePersistentDirectoryVolume(serviceUUID service.ServiceUUID, persistentKey service_directory.DirectoryPersistentKey) (KubernetesObjectAttributes, error) {
+func (provider *kubernetesEnclaveObjectAttributesProviderImpl) ForSinglePersistentDirectoryVolume(persistentKey service_directory.DirectoryPersistentKey) (KubernetesObjectAttributes, error) {
 	hasher := md5.New()
 	hasher.Write([]byte(provider.enclaveId))
-	hasher.Write([]byte(serviceUUID))
 	hasher.Write([]byte(persistentKey))
 	persistentKeyHash := hex.EncodeToString(hasher.Sum(nil))
 

core/server/api_container/server/startosis_engine/kurtosis_instruction/add_service/add_service_shared.go

@@ -76,9 +76,6 @@ func validateSingleService(validatorEnvironment *startosis_validator.ValidatorEn
 			if !service_directory.IsPersistentKeyValid(directory.PersistentKey) {
 				return startosis_errors.NewValidationError(invalidPersistentKeyErrorText(directory.PersistentKey))
 			}
-			if validatorEnvironment.DoesPersistentKeyExist(directory.PersistentKey) == startosis_validator.ComponentCreatedOrUpdatedDuringPackageRun {
-				return startosis_errors.NewValidationError("There was an error validating '%s' as persistent key '%s' already exists inside the enclave", serviceName, directory.PersistentKey)
-			}
 			validatorEnvironment.AddPersistentKey(directory.PersistentKey)
 		}
 	}

core/server/api_container/server/startosis_engine/startosis_validator/validator_environment.go

@@ -173,11 +173,3 @@ func (environment *ValidatorEnvironment) HasEnoughMemory(memoryToConsume uint64,
 func (environment *ValidatorEnvironment) AddPersistentKey(persistentKey service_directory.DirectoryPersistentKey) {
 	environment.persistentKeys[persistentKey] = ComponentCreatedOrUpdatedDuringPackageRun
 }
-
-func (environment *ValidatorEnvironment) DoesPersistentKeyExist(persistentKey service_directory.DirectoryPersistentKey) ComponentExistence {
-	persistentKeyExistence, found := environment.persistentKeys[persistentKey]
-	if !found {
-		return ComponentNotFound
-	}
-	return persistentKeyExistence
-}