diff --git a/CHANGELOG.md b/CHANGELOG.md index 6a0810b1d6..373d90c0bd 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -5,7 +5,7 @@ ### Features -* docker compose integration [pt. 1] ([#2001](https://github.com/kurtosis-tech/kurtosis/issues/2001)) ([385833d](https://github.com/kurtosis-tech/kurtosis/commit/385833de9d7620f4c65473adc763bb38df8fb995)) +* docker compose integration([#2001](https://github.com/kurtosis-tech/kurtosis/issues/2001)) ([385833d](https://github.com/kurtosis-tech/kurtosis/commit/385833de9d7620f4c65473adc763bb38df8fb995)) ### Bug Fixes diff --git a/container-engine-lib/lib/backend_impls/docker/object_attributes_provider/enclave_object_attributes_provider.go b/container-engine-lib/lib/backend_impls/docker/object_attributes_provider/enclave_object_attributes_provider.go index c6840ef219..549dff52cd 100644 --- a/container-engine-lib/lib/backend_impls/docker/object_attributes_provider/enclave_object_attributes_provider.go +++ b/container-engine-lib/lib/backend_impls/docker/object_attributes_provider/enclave_object_attributes_provider.go @@ -1,8 +1,6 @@ package object_attributes_provider import ( - "crypto/md5" - "encoding/hex" "fmt" "net" "strconv" @@ -26,8 +24,7 @@ const ( networkPrefix = "kt-" apiContainerNamePrefix = "kurtosis-api" - artifactExpansionVolumeNameFragment = "files-artifact-expansion" - persistentServiceDirectoryNameFragment = "service-persistent-directory" + artifactExpansionVolumeNameFragment = "files-artifact-expansion" artifactsExpanderContainerNameFragment = "files-artifacts-expander" logsCollectorFragment = "kurtosis-logs-collector" @@ -333,14 +330,8 @@ func (provider *dockerEnclaveObjectAttributesProviderImpl) ForSinglePersistentDi return nil, stacktrace.Propagate(err, "An error occurred generating a UUID for the persistent directory volume for service '%v'", serviceUuidStr) } - hasher := md5.New() - hasher.Write([]byte(serviceUUID)) - hasher.Write([]byte(persistentKey)) - persistentKeyHash := hex.EncodeToString(hasher.Sum(nil)) - name, err := provider.getNameForEnclaveObject([]string{ - persistentServiceDirectoryNameFragment, - persistentKeyHash, + string(persistentKey), }) if err != nil { return nil, stacktrace.Propagate(err, "An error occurred creating the files artifact expansion volume name object using GUID '%v' and service GUID '%v'", guidStr, serviceUuidStr) diff --git a/container-engine-lib/lib/backend_impls/kubernetes/object_attributes_provider/enclave_object_attributes_provider.go b/container-engine-lib/lib/backend_impls/kubernetes/object_attributes_provider/enclave_object_attributes_provider.go index 6ac31c84b8..88c2c4493d 100644 --- a/container-engine-lib/lib/backend_impls/kubernetes/object_attributes_provider/enclave_object_attributes_provider.go +++ b/container-engine-lib/lib/backend_impls/kubernetes/object_attributes_provider/enclave_object_attributes_provider.go @@ -25,8 +25,6 @@ const ( enclaveDataDirFragment = "enclave-data-dir" - persistentServiceDirectoryNameFragment = "service-persistent-directory" - traefikIngressRouterEntrypointsValue = "web" ) @@ -264,7 +262,7 @@ func (provider *kubernetesEnclaveObjectAttributesProviderImpl) ForSinglePersiste //No userServiceService annotations. annotations := map[*kubernetes_annotation_key.KubernetesAnnotationKey]*kubernetes_annotation_value.KubernetesAnnotationValue{} - name, err := getKubernetesPersistentDirectoryName(persistentKeyHash) + name, err := getKubernetesPersistentDirectoryName(string(persistentKey)) if err != nil { return nil, stacktrace.Propagate(err, "Failed to create service persistent directory name for hash: '%s'", persistentKeyHash) } @@ -329,12 +327,11 @@ func getKubernetesObjectName( } func getKubernetesPersistentDirectoryName( - persistentServiceDirectoryName string, + persistentKey string, ) (*kubernetes_object_name.KubernetesObjectName, error) { name, err := getCompositeKubernetesObjectName( []string{ - persistentServiceDirectoryNameFragment, - persistentServiceDirectoryName, + persistentKey, }) return name, err } diff --git a/container-engine-lib/lib/backend_interface/objects/service_directory/persistent_directories.go b/container-engine-lib/lib/backend_interface/objects/service_directory/persistent_directories.go index fc96d20a45..0b195a71b5 100644 --- a/container-engine-lib/lib/backend_interface/objects/service_directory/persistent_directories.go +++ b/container-engine-lib/lib/backend_interface/objects/service_directory/persistent_directories.go @@ -1,5 +1,23 @@ package service_directory +import "regexp" + +const ( + // PersistentKeyRegex implements RFC-1035 for naming persistent directory keys, namely: + // * contain at most 63 characters + // * contain only lowercase alphanumeric characters or '-' + // * start with an alphabetic character + // * end with an alphanumeric character + // This is in order to stick to the 1035 standard which we enforce for all objects created + // https://kubernetes.io/docs/concepts/overview/working-with-objects/names/#dns-label-names + PersistentKeyRegex = "[a-z]([-a-z0-9]{0,61}[a-z0-9])?" + WordWrappedPersistentKeyRegex = "^" + PersistentKeyRegex + "$" +) + +var ( + compiledWordWrappedPersistentKeyRegex = regexp.MustCompile(WordWrappedPersistentKeyRegex) +) + type DirectoryPersistentKey string type DirectoryPersistentSize int64 @@ -17,3 +35,7 @@ func NewPersistentDirectories(persistentDirectories map[string]PersistentDirecto ServiceDirpathToPersistentDirectory: persistentDirectories, } } + +func IsPersistentKeyValid(persistentKey DirectoryPersistentKey) bool { + return compiledWordWrappedPersistentKeyRegex.MatchString(string(persistentKey)) +} diff --git a/core/server/api_container/server/startosis_engine/kurtosis_instruction/add_service/add_service_shared.go b/core/server/api_container/server/startosis_engine/kurtosis_instruction/add_service/add_service_shared.go index dda149f4d2..70d40974ee 100644 --- a/core/server/api_container/server/startosis_engine/kurtosis_instruction/add_service/add_service_shared.go +++ b/core/server/api_container/server/startosis_engine/kurtosis_instruction/add_service/add_service_shared.go @@ -4,6 +4,7 @@ import ( "context" "fmt" "github.com/kurtosis-tech/kurtosis/container-engine-lib/lib/backend_interface/objects/service" + "github.com/kurtosis-tech/kurtosis/container-engine-lib/lib/backend_interface/objects/service_directory" "github.com/kurtosis-tech/kurtosis/core/server/api_container/server/service_network" "github.com/kurtosis-tech/kurtosis/core/server/api_container/server/startosis_engine/kurtosis_instruction/shared_helpers" "github.com/kurtosis-tech/kurtosis/core/server/api_container/server/startosis_engine/kurtosis_instruction/shared_helpers/magic_string_helper" @@ -70,6 +71,18 @@ func validateSingleService(validatorEnvironment *startosis_validator.ValidatorEn return startosis_errors.NewValidationError(invalidServiceNameErrorText(serviceName)) } + if persistentDirectories := serviceConfig.GetPersistentDirectories(); persistentDirectories != nil { + for _, directory := range persistentDirectories.ServiceDirpathToPersistentDirectory { + if !service_directory.IsPersistentKeyValid(directory.PersistentKey) { + return startosis_errors.NewValidationError(invalidPersistentKeyErrorText(directory.PersistentKey)) + } + if validatorEnvironment.DoesPersistentKeyExist(directory.PersistentKey) == startosis_validator.ComponentCreatedOrUpdatedDuringPackageRun { + return startosis_errors.NewValidationError("There was an error validating '%s' as persistent key '%s' already exists inside the enclave", serviceName, directory.PersistentKey) + } + validatorEnvironment.AddPersistentKey(directory.PersistentKey) + } + } + if validatorEnvironment.DoesServiceNameExist(serviceName) == startosis_validator.ComponentCreatedOrUpdatedDuringPackageRun { return startosis_errors.NewValidationError("There was an error validating '%s' as service with the name '%s' already exists inside the package. Adding two different services with the same name isn't allowed; we recommend prefixing/suffixing the two service names or using two different names entirely.", AddServiceBuiltinName, serviceName) } @@ -117,6 +130,16 @@ func invalidServiceNameErrorText( ) } +func invalidPersistentKeyErrorText( + persistentKey service_directory.DirectoryPersistentKey, +) string { + return fmt.Sprintf( + "Persistent Key '%v' is invalid as it contains disallowed characters. Persistent Key must adhere to the RFC 1035 standard, specifically implementing this regex and be 1-63 characters long: %s. This means the service name must only contain lowercase alphanumeric characters or '-', and must start with a lowercase alphabet and end with a lowercase alphanumeric character.", + persistentKey, + service_directory.WordWrappedPersistentKeyRegex, + ) +} + func replaceMagicStrings( runtimeValueStore *runtime_value_store.RuntimeValueStore, serviceName service.ServiceName, diff --git a/core/server/api_container/server/startosis_engine/startosis_validator/validator_environment.go b/core/server/api_container/server/startosis_engine/startosis_validator/validator_environment.go index 2d581810ad..dcea4e0338 100644 --- a/core/server/api_container/server/startosis_engine/startosis_validator/validator_environment.go +++ b/core/server/api_container/server/startosis_engine/startosis_validator/validator_environment.go @@ -5,6 +5,7 @@ import ( "github.com/kurtosis-tech/kurtosis/container-engine-lib/lib/backend_interface/objects/image_build_spec" "github.com/kurtosis-tech/kurtosis/container-engine-lib/lib/backend_interface/objects/image_download_mode" "github.com/kurtosis-tech/kurtosis/container-engine-lib/lib/backend_interface/objects/service" + "github.com/kurtosis-tech/kurtosis/container-engine-lib/lib/backend_interface/objects/service_directory" "github.com/kurtosis-tech/kurtosis/core/server/api_container/server/startosis_engine/startosis_errors" "github.com/sirupsen/logrus" ) @@ -15,6 +16,7 @@ type ValidatorEnvironment struct { imagesToBuild map[string]*image_build_spec.ImageBuildSpec serviceNames map[service.ServiceName]ComponentExistence artifactNames map[string]ComponentExistence + persistentKeys map[service_directory.DirectoryPersistentKey]ComponentExistence serviceNameToPrivatePortIDs map[service.ServiceName][]string availableCpuInMilliCores compute_resources.CpuMilliCores availableMemoryInMegaBytes compute_resources.MemoryInMegaBytes @@ -42,9 +44,11 @@ func NewValidatorEnvironment(serviceNames map[service.ServiceName]bool, artifact availableCpuInMilliCores: availableCpuInMilliCores, availableMemoryInMegaBytes: availableMemoryInMegaBytes, isResourceInformationComplete: isResourceInformationComplete, - minMemoryByServiceName: map[service.ServiceName]compute_resources.MemoryInMegaBytes{}, - minCPUByServiceName: map[service.ServiceName]compute_resources.CpuMilliCores{}, - imageDownloadMode: imageDownloadMode, + // TODO account for idempotent runs on this and make it pre-load the cache whenever we create a NewValidatorEnvironment + persistentKeys: map[service_directory.DirectoryPersistentKey]ComponentExistence{}, + minMemoryByServiceName: map[service.ServiceName]compute_resources.MemoryInMegaBytes{}, + minCPUByServiceName: map[service.ServiceName]compute_resources.CpuMilliCores{}, + imageDownloadMode: imageDownloadMode, } } @@ -160,3 +164,15 @@ func (environment *ValidatorEnvironment) HasEnoughMemory(memoryToConsume uint64, } return startosis_errors.NewValidationError("service '%v' requires '%v' megabytes of memory but based on our calculation we will only have '%v' megabytes available at the time we start the service", serviceNameForLogging, memoryToConsume, environment.availableMemoryInMegaBytes) } + +func (environment *ValidatorEnvironment) AddPersistentKey(persistentKey service_directory.DirectoryPersistentKey) { + environment.persistentKeys[persistentKey] = ComponentCreatedOrUpdatedDuringPackageRun +} + +func (environment *ValidatorEnvironment) DoesPersistentKeyExist(persistentKey service_directory.DirectoryPersistentKey) ComponentExistence { + persistentKeyExistence, found := environment.persistentKeys[persistentKey] + if !found { + return ComponentNotFound + } + return persistentKeyExistence +} diff --git a/docs/docs/api-reference/starlark-reference/directory.md b/docs/docs/api-reference/starlark-reference/directory.md index 716b94e680..d6a717e9f2 100644 --- a/docs/docs/api-reference/starlark-reference/directory.md +++ b/docs/docs/api-reference/starlark-reference/directory.md @@ -24,7 +24,7 @@ the above example, `files_artifact_1` is a files artifact name. (see [upload_fil on how to create file artifacts). A persistent directory, as its name indicates, persists over service updates and restarts. It is uniquely identified -by its `persistent_key` and the service ID on which it is being used (a persistent directory cannot be shared across +by its `persistent_key` (a persistent directory cannot be shared across multiple services). When it is first created, it will be empty. The service can write anything in it. When the service gets updated, the data in it persists. It is particularly useful for a service's data directory, logs directory, etc. diff --git a/internal_testsuites/golang/testsuite/startosis_persistent_directory_test/startosis_persistent_directory_test.go b/internal_testsuites/golang/testsuite/startosis_persistent_directory_test/startosis_persistent_directory_test.go index ea30082164..5376c8f09f 100644 --- a/internal_testsuites/golang/testsuite/startosis_persistent_directory_test/startosis_persistent_directory_test.go +++ b/internal_testsuites/golang/testsuite/startosis_persistent_directory_test/startosis_persistent_directory_test.go @@ -21,7 +21,7 @@ def run(plan): image=IMAGE, files={ "/data": Directory( - persistent_key="persistent_data", + persistent_key="persistent-data", ), }, min_cpu=%d,