diff --git a/container-engine-lib/lib/backend_impls/docker/docker_kurtosis_backend/user_services_functions/start_user_services.go b/container-engine-lib/lib/backend_impls/docker/docker_kurtosis_backend/user_services_functions/start_user_services.go index 1bde0965b0..d229461293 100644 --- a/container-engine-lib/lib/backend_impls/docker/docker_kurtosis_backend/user_services_functions/start_user_services.go +++ b/container-engine-lib/lib/backend_impls/docker/docker_kurtosis_backend/user_services_functions/start_user_services.go @@ -2,6 +2,8 @@ package user_service_functions import ( "context" + "fmt" + "strconv" "strings" "sync" @@ -539,6 +541,7 @@ func createStartServiceOperation( memoryAllocationMegabytes := serviceConfig.GetMemoryAllocationMegabytes() privateIPAddrPlaceholder := serviceConfig.GetPrivateIPAddrPlaceholder() user := serviceConfig.GetUser() + filesToBeMoved := serviceConfig.GetFilesToBeMoved() // We replace the placeholder value with the actual private IP address privateIPAddrStr := privateIpAddr.String() @@ -552,6 +555,17 @@ func createStartServiceOperation( envVars[key] = strings.Replace(envVars[key], privateIPAddrPlaceholder, privateIPAddrStr, unlimitedReplacements) } + // TODO clean this hack up + // this path will only be hit if `files_to_be_moved` is set in Starlark; which is a hidden property + // used by compose transpilation + if len(filesToBeMoved) > 0 { + var err error + cmdArgs, entrypointArgs, err = getUpdatedEntrypointAndCmdFromFilesToBeMoved(ctx, dockerManager, containerImageName, cmdArgs, entrypointArgs, filesToBeMoved) + if err != nil { + return nil, stacktrace.Propagate(err, "an error occurred while handling files for compose") + } + } + volumeMounts := map[string]string{} shouldDeleteVolumes := true if filesArtifactsExpansion != nil { @@ -753,6 +767,50 @@ func createStartServiceOperation( } } +// TODO - clean this up this is super janky, a way to handle compose & volume +func getUpdatedEntrypointAndCmdFromFilesToBeMoved(ctx context.Context, dockerManager *docker_manager.DockerManager, containerImageName string, cmdArgs []string, entrypointArgs []string, filesToBeMoved map[string]string) ([]string, []string, error) { + concatenatedFilesToBeMoved := []string{} + for source, destination := range filesToBeMoved { + // TODO improve this; the first condition handles files the other folders + concatenatedFilesToBeMoved = append(concatenatedFilesToBeMoved, fmt.Sprintf("mv %v %v", source, destination)) + } + + concatenatedFilesToBeMovedAsStr := strings.Join(concatenatedFilesToBeMoved, " && ") + originalEntrypointArgs, originalCmdArgs, err := dockerManager.GetEntryPointAndCommand(ctx, containerImageName) + if err != nil { + return nil, nil, stacktrace.Propagate(err, "an error occurred fetching data about image '%v'", containerImageName) + } + + // we do this replacement as we want to keep the original command args as they are not overwritten + // it might be that none of the two are set + if len(cmdArgs) == 0 && len(originalCmdArgs) > 0 { + cmdArgs = originalCmdArgs + } + if len(entrypointArgs) == 0 && len(originalEntrypointArgs) > 0 { + entrypointArgs = originalEntrypointArgs + } + + entryPointArgsAsStr := quoteAndJoinArgs(entrypointArgs) + cmdArgsAsStr := quoteAndJoinArgs(cmdArgs) + + if len(cmdArgs) > 0 { + if len(entrypointArgs) > 0 { + cmdArgs = []string{"-c", concatenatedFilesToBeMovedAsStr + " && " + entryPointArgsAsStr + " " + cmdArgsAsStr} + } else { + cmdArgs = []string{"-c", concatenatedFilesToBeMovedAsStr + " && " + cmdArgsAsStr} + } + } else { + if len(entrypointArgs) > 0 { + cmdArgs = []string{"-c", concatenatedFilesToBeMovedAsStr + " && " + entryPointArgsAsStr} + } else { + // no entrypoint and no command; this shouldn't really happen + cmdArgs = []string{"-c", concatenatedFilesToBeMovedAsStr} + } + } + entrypointArgs = []string{"/bin/sh"} + return cmdArgs, entrypointArgs, nil +} + // Ensure that provided [privatePorts] and [publicPorts] are one to one by checking: // - There is a matching publicPort for every portID in privatePorts // - There are the same amount of private and public ports @@ -845,3 +903,11 @@ func registerUserServices( return successfulRegistrations, failedRegistrations, nil } + +func quoteAndJoinArgs(args []string) string { + var quotedArgs []string + for _, arg := range args { + quotedArgs = append(quotedArgs, strconv.Quote(arg)) + } + return strings.Join(quotedArgs, " ") +} diff --git a/container-engine-lib/lib/backend_impls/docker/docker_manager/docker_manager.go b/container-engine-lib/lib/backend_impls/docker/docker_manager/docker_manager.go index eee8d00786..390ce1c0d0 100644 --- a/container-engine-lib/lib/backend_impls/docker/docker_manager/docker_manager.go +++ b/container-engine-lib/lib/backend_impls/docker/docker_manager/docker_manager.go @@ -1660,6 +1660,17 @@ func (manager *DockerManager) getImagePlatform(ctx context.Context, imageName st return imageInspect.Architecture, nil } +func (manager *DockerManager) GetEntryPointAndCommand(ctx context.Context, imageName string) ([]string, []string, error) { + imageInspect, _, err := manager.dockerClient.ImageInspectWithRaw(ctx, imageName) + if err != nil { + return nil, nil, stacktrace.Propagate(err, "an error occurred while running image inspect on image '%v'", imageName) + } + if imageInspect.Config == nil { + return nil, nil, stacktrace.NewError("image inspect config was empty, can't geet entrypoint or cmd: %v", imageInspect) + } + return imageInspect.Config.Entrypoint, imageInspect.Config.Cmd, nil +} + /* Creates a Docker-Container-To-Host Port mapping, defining how a Container's JSON RPC and service-specific ports are mapped to the host ports. diff --git a/container-engine-lib/lib/backend_interface/objects/service/service_config.go b/container-engine-lib/lib/backend_interface/objects/service/service_config.go index e2b3d75f41..f3cb33d100 100644 --- a/container-engine-lib/lib/backend_interface/objects/service/service_config.go +++ b/container-engine-lib/lib/backend_interface/objects/service/service_config.go @@ -2,9 +2,8 @@ package service import ( "encoding/json" - "github.com/kurtosis-tech/kurtosis/container-engine-lib/lib/backend_interface/objects/image_download_mode" - "github.com/kurtosis-tech/kurtosis/container-engine-lib/lib/backend_interface/objects/image_build_spec" + "github.com/kurtosis-tech/kurtosis/container-engine-lib/lib/backend_interface/objects/image_download_mode" "github.com/kurtosis-tech/kurtosis/container-engine-lib/lib/backend_interface/objects/image_registry_spec" "github.com/kurtosis-tech/kurtosis/container-engine-lib/lib/backend_interface/objects/nix_build_spec" "github.com/kurtosis-tech/kurtosis/container-engine-lib/lib/backend_interface/objects/port_spec" @@ -70,6 +69,8 @@ type privateServiceConfig struct { NodeSelectors map[string]string ImageDownloadMode image_download_mode.ImageDownloadMode + + FilesToBeMoved map[string]string } func CreateServiceConfig( @@ -123,6 +124,7 @@ func CreateServiceConfig( Tolerations: tolerations, NodeSelectors: nodeSelectors, ImageDownloadMode: imageDownloadMode, + FilesToBeMoved: map[string]string{}, } return &ServiceConfig{internalServiceConfig}, nil } @@ -217,6 +219,14 @@ func (serviceConfig *ServiceConfig) GetNodeSelectors() map[string]string { return serviceConfig.privateServiceConfig.NodeSelectors } +func (serviceConfig *ServiceConfig) SetFilesToBeMoved(filesToBeMoved map[string]string) { + serviceConfig.privateServiceConfig.FilesToBeMoved = filesToBeMoved +} + +func (serviceConfig *ServiceConfig) GetFilesToBeMoved() map[string]string { + return serviceConfig.privateServiceConfig.FilesToBeMoved +} + func (serviceConfig *ServiceConfig) UnmarshalJSON(data []byte) error { // Suppressing exhaustruct requirement because we want an object with zero values diff --git a/core/server/api_container/server/docker_compose_transpiler/docker_compose_transpiler.go b/core/server/api_container/server/docker_compose_transpiler/docker_compose_transpiler.go index 685aa043dd..ab106e3274 100644 --- a/core/server/api_container/server/docker_compose_transpiler/docker_compose_transpiler.go +++ b/core/server/api_container/server/docker_compose_transpiler/docker_compose_transpiler.go @@ -3,9 +3,11 @@ package docker_compose_transpiler import ( "errors" "fmt" + "github.com/hashicorp/go-envparse" "golang.org/x/exp/slices" "os" "path" + "regexp" "sort" "strconv" "strings" @@ -58,13 +60,17 @@ const ( newStarlarkLineFmtStr = " %s\n" - unixHomePathSymbol = "~" - upstreamRelativePathSymbol = ".." - httpProtocol = "http" + + alphanumericCharWithDashesRegexStr = `[^a-z0-9-]` + consecutiveDashesRegexStr = `-+` ) -var possibleHttpPorts = []uint32{8080, 8000, 80, 443} +var ( + alphanumericCharWithDashesRegex = regexp.MustCompile(alphanumericCharWithDashesRegexStr) + consecutiveDashesRegex = regexp.MustCompile(consecutiveDashesRegexStr) + possibleHttpPorts = []uint32{8080, 8000, 80, 443} +) type ComposeService types.ServiceConfig @@ -99,7 +105,7 @@ func TranspileDockerComposePackageToStarlark(packageAbsDirpath string, relativeP envVarsInFile = map[string]string{} } - starlarkScript, err := convertComposeToStarlarkScript(composeBytes, envVarsInFile) + starlarkScript, err := convertComposeToStarlarkScript(composeBytes, envVarsInFile, packageAbsDirpath) if err != nil { return "", stacktrace.Propagate(err, "An error occurred converting Compose file '%v' to a Starlark script.", composeFilename) } @@ -111,13 +117,13 @@ func TranspileDockerComposePackageToStarlark(packageAbsDirpath string, relativeP // Private Helper Functions // // ==================================================================================================== -func convertComposeToStarlarkScript(composeBytes []byte, envVars map[string]string) (string, error) { +func convertComposeToStarlarkScript(composeBytes []byte, envVars map[string]string, packageAbsDirPath string) (string, error) { composeStruct, err := convertComposeBytesToComposeStruct(composeBytes, envVars) if err != nil { return "", stacktrace.Propagate(err, "An error occurred converting compose bytes into a struct.") } - serviceNameToStarlarkServiceConfig, serviceDependencyGraph, perServiceFilesArtifactsToUpload, err := convertComposeServicesToStarlarkInfo(composeStruct.Services) + serviceNameToStarlarkServiceConfig, serviceDependencyGraph, perServiceFilesArtifactsToUpload, err := convertComposeServicesToStarlarkInfo(composeStruct.Services, packageAbsDirPath) if err != nil { return "", stacktrace.Propagate(err, "An error occurred converting compose services to starlark service configs.") } @@ -132,8 +138,6 @@ func convertComposeBytesToComposeStruct(composeBytes []byte, envVars map[string] ConfigFiles: []types.ConfigFile{{ Content: composeBytes, }}, - // TODO: To leverage this env file at the base of the project: need to parse referenced environment variables - // https://docs.docker.com/compose/environment-variables/set-environment-variables/#substitute-with-an-env-file Environment: envVars, } setOptionsFunc := func(options *loader.Options) { @@ -142,7 +146,9 @@ func convertComposeBytesToComposeStruct(composeBytes []byte, envVars map[string] options.ConvertWindowsPaths = shouldConvertWindowsPathsToLinux } compose, err := loader.Load(composeParseConfig, setOptionsFunc) - if err != nil { + // don't err if env file not found, transpiler will handle finding it + // TODO: fork compose loader package and change the env file logic so we don't have to catch this err + if err != nil && !isEnvFileNotFoundErr(err) { return nil, stacktrace.Propagate(err, "An error occurred parsing compose based on provided parsing config and set options function.") } return compose, nil @@ -188,9 +194,8 @@ func createStarlarkScript( return script, nil } -// TODO add support for User here // Turns DockerCompose Service into Kurtosis ServiceConfigs and returns info needed for creating a valid starlark script -func convertComposeServicesToStarlarkInfo(composeServices types.Services) ( +func convertComposeServicesToStarlarkInfo(composeServices types.Services, packageAbsDirPath string) ( map[string]StarlarkServiceConfig, // Map of service names to Kurtosis ServiceConfig's map[string]map[string]bool, // Graph of service dependencies based on depends_on key (determines order in which to add services) map[string]map[string]string, // Map of service names to map of relative paths to files artifacts names that need to get uploaded for the service (determines files artifacts that need to be uploaded) @@ -199,23 +204,29 @@ func convertComposeServicesToStarlarkInfo(composeServices types.Services) ( perServiceDependencies := map[string]map[string]bool{} servicesToFilesArtifactsToUpload := map[string]map[string]string{} + serviceNameToContainerNameMap := map[string]string{} + for _, s := range composeServices { + if s.ContainerName != "" { + serviceNameToContainerNameMap[s.Name] = s.ContainerName + } + } + for _, service := range composeServices { composeService := ComposeService(service) serviceConfigKwargs := []starlark.Tuple{} - // NAME serviceName := composeService.Name - - // IMAGE - if composeService.Image != "" { - serviceConfigKwargs = appendKwarg( - serviceConfigKwargs, - service_config.ImageAttr, - starlark.String(composeService.Image), - ) + // hostname becomes container name if it's set in compose + // in kurtosis service name becomes hostname, so set service name as container name (thus hostname) so other services can comm with it + if containerName, ok := serviceNameToContainerNameMap[serviceName]; ok { + serviceName = containerName } + serviceName = convertToRFC1035(serviceName) - // IMAGE BUILD SPEC + // IMAGE + imageName := composeService.Image + // if build directive used, create an image build spec + // otherwise, use image name if composeService.Build != nil { imageBuildSpec, err := getStarlarkImageBuildSpec(composeService.Build, serviceName) if err != nil { @@ -226,6 +237,12 @@ func convertComposeServicesToStarlarkInfo(composeServices types.Services) ( service_config.ImageAttr, imageBuildSpec, ) + } else { + serviceConfigKwargs = appendKwarg( + serviceConfigKwargs, + service_config.ImageAttr, + starlark.String(imageName), + ) } // PORTS @@ -262,8 +279,8 @@ func convertComposeServicesToStarlarkInfo(composeServices types.Services) ( } // ENV VARS - if composeService.Environment != nil { - envVarsDict, err := getStarlarkEnvVars(composeService.Environment) + if composeService.Environment != nil || composeService.EnvFile != nil { + envVarsDict, err := getStarlarkEnvVars(composeService.Environment, composeService.EnvFile, packageAbsDirPath) if err != nil { return nil, nil, nil, stacktrace.Propagate(err, "An error occurred creating the env vars dict for service '%s'", serviceName) } @@ -276,7 +293,7 @@ func convertComposeServicesToStarlarkInfo(composeServices types.Services) ( // VOLUMES -> FILES ARTIFACTS if composeService.Volumes != nil { - filesDict, artifactsToUpload, err := getStarlarkFilesArtifacts(composeService.Volumes, serviceName) + filesDict, artifactsToUpload, filesToBeMoved, err := getStarlarkFilesArtifacts(composeService.Volumes, serviceName, packageAbsDirPath) if err != nil { return nil, nil, nil, stacktrace.Propagate(err, "An error occurred creating the files dict for service '%s'", serviceName) } @@ -285,6 +302,9 @@ func convertComposeServicesToStarlarkInfo(composeServices types.Services) ( service_config.FilesAttr, filesDict, ) + if filesToBeMoved.Len() > 0 { + serviceConfigKwargs = appendKwarg(serviceConfigKwargs, service_config.FilesToBeMovedAttr, filesToBeMoved) + } servicesToFilesArtifactsToUpload[serviceName] = artifactsToUpload } @@ -307,7 +327,11 @@ func convertComposeServicesToStarlarkInfo(composeServices types.Services) ( // DEPENDS ON dependencyServiceNames := map[string]bool{} for dependencyName := range composeService.DependsOn { - dependencyServiceNames[dependencyName] = true + // do container name switch if it exists + if containerName, ok := serviceNameToContainerNameMap[dependencyName]; ok { + dependencyName = containerName + } + dependencyServiceNames[convertToRFC1035(dependencyName)] = true } perServiceDependencies[serviceName] = dependencyServiceNames @@ -424,27 +448,49 @@ func getStarlarkCommand(composeCommand types.ShellCommand) *starlark.List { return starlark.NewList(commandSLStrs) } -func getStarlarkEnvVars(composeEnvironment types.MappingWithEquals) (*starlark.Dict, error) { +func getStarlarkEnvVars(composeEnvironment types.MappingWithEquals, envFiles types.StringList, packageAbsDirPath string) (*starlark.Dict, error) { // make iteration order of [composeEnvironment] deterministic by getting the keys and sorting them envVarKeys := []string{} for key := range composeEnvironment { envVarKeys = append(envVarKeys, key) } sort.Strings(envVarKeys) - enVarsSLDict := starlark.NewDict(len(composeEnvironment)) + envVarsSLDict := starlark.NewDict(len(composeEnvironment)) for _, key := range envVarKeys { value := composeEnvironment[key] if value == nil { continue } - if err := enVarsSLDict.SetKey( + if err := envVarsSLDict.SetKey( starlark.String(key), starlark.String(*value), ); err != nil { return nil, stacktrace.Propagate(err, "An error occurred setting key '%s' in environment variables Starlark dict.", key) } } - return enVarsSLDict, nil + + // if env file is specified, manually parse the env file at the location it is inside the package on the APIC + for _, envFilePath := range envFiles { + serviceEnvFilePath := path.Join(packageAbsDirPath, envFilePath) + envFileReader, err := os.Open(serviceEnvFilePath) + if err != nil { + return nil, stacktrace.Propagate(err, "An error occurred opening env file at path: %v", serviceEnvFilePath) + } + envVars, err := envparse.Parse(envFileReader) + if err != nil { + return nil, stacktrace.Propagate(err, "An error occurred parsing env file.") + } + for key, value := range envVars { + if err := envVarsSLDict.SetKey( + starlark.String(key), + starlark.String(value), + ); err != nil { + return nil, stacktrace.Propagate(err, "An error occurred setting key '%s' in environment variables Starlark dict.", key) + } + } + } + + return envVarsSLDict, nil } // The 'volumes:' compose key supports named volumes and bind mounts https://docs.docker.com/storage/volumes/ @@ -454,44 +500,36 @@ func getStarlarkEnvVars(composeEnvironment types.MappingWithEquals) (*starlark.D // := create a persistent directory on container at // := create a persistent directory on container at // Named volumes are treated https://docs.docker.com/storage/volumes/ as absolute paths persistence layers, and thus a persistent directory is created -func getStarlarkFilesArtifacts(composeVolumes []types.ServiceVolumeConfig, serviceName string) (starlark.Value, map[string]string, error) { +func getStarlarkFilesArtifacts(composeVolumes []types.ServiceVolumeConfig, serviceName string, packageAbsDirPath string) (starlark.Value, map[string]string, *starlark.Dict, error) { filesArgSLDict := starlark.NewDict(len(composeVolumes)) filesArtifactsToUpload := map[string]string{} + filesToBeMoved := starlark.NewDict(len(composeVolumes)) + for volumeIdx, volume := range composeVolumes { volumeType := volume.Type var shouldPersist bool switch volumeType { + // if an absolute path is specified, assume user wants to use volume as a persistence layer and create a Persistent Directory + // if path is relative, assume it's read only and do an upload files case types.VolumeTypeBind: - // Handle case where home path is reference - if strings.Contains(volume.Source, unixHomePathSymbol) { - return nil, map[string]string{}, stacktrace.NewError( - "Volume path '%v' uses '%v', likely referencing home path on a unix filesystem. Currently, Kurtosis does not support uploading from host filesystem. "+ - "Place the contents of '%v' directory inside the package where the compose yaml exists and update the volume filepath to be a relative path", - volume.Source, unixHomePathSymbol, volume.Source) - } - // Handle case where upstream relative path is reference - if strings.Contains(volume.Source, upstreamRelativePathSymbol) { - return nil, map[string]string{}, stacktrace.NewError( - "Volume path '%v' uses '%v', likely referencing an upstream path on a filesystem. Currently, Kurtosis does not support uploading from host filesystem. "+ - "Place the contents of '%v' directory inside the package where the compose yaml exists and update the volume filepath to be a relative path within the package.", - volume.Source, upstreamRelativePathSymbol, volume.Source) - } - - // Assume that if an absolute path is specified, user wants to use volume as a persistence layer - // Additionally, assume relative paths are read-only - shouldPersist = path.IsAbs(volume.Source) + volumePath := cleanFilePath(volume.Source) + shouldPersist = path.IsAbs(volumePath) + // if named volume is provided, assume user wants to use volume as a persistence layer and create a Persistent Directory case types.VolumeTypeVolume: shouldPersist = true + default: + shouldPersist = false } var filesDictValue starlark.Value + targetDirectoryForFilesArtifact := volume.Target if shouldPersist { persistenceKey := fmt.Sprintf("%s--volume%d", serviceName, volumeIdx) persistentDirectory, err := getStarlarkPersistentDirectory(persistenceKey) if err != nil { - return nil, nil, stacktrace.Propagate(err, "An error occurred creating persistent directory with key '%s' for volume #%d.", persistenceKey, volumeIdx) + return nil, nil, nil, stacktrace.Propagate(err, "An error occurred creating persistent directory with key '%s' for volume #%d.", persistenceKey, volumeIdx) } filesDictValue = persistentDirectory } else { @@ -499,14 +537,28 @@ func getStarlarkFilesArtifacts(composeVolumes []types.ServiceVolumeConfig, servi filesArtifactName := fmt.Sprintf("%s--volume%d", serviceName, volumeIdx) filesArtifactsToUpload[volume.Source] = filesArtifactName filesDictValue = starlark.String(filesArtifactName) - } - if err := filesArgSLDict.SetKey(starlark.String(volume.Target), filesDictValue); err != nil { - return nil, nil, stacktrace.Propagate(err, "An error occurred setting volume mountpoint '%s' in the files Starlark dict.", volume.Target) + // TODO: update files artifact expansion to handle mounting files, not only directories so files_to_be_moved hack can be removed + // if the volume is referencing a file, use files_to_be_moved + maybeFileOrDirVolume, err := os.Stat(path.Join(packageAbsDirPath, volume.Source)) + if err != nil { + return nil, nil, nil, stacktrace.Propagate(err, "An error occurred checking is the volume path existed in the package on disc: %v.", volume.Source) + } + if !maybeFileOrDirVolume.IsDir() { + sourcePathNameEnd := path.Base(volume.Source) + targetDirectoryForFilesArtifact = path.Join("/tmp", filesArtifactName) + targetToMovePath := path.Join(targetDirectoryForFilesArtifact, sourcePathNameEnd) + if err := filesToBeMoved.SetKey(starlark.String(targetToMovePath), starlark.String(volume.Target)); err != nil { + return nil, nil, nil, stacktrace.Propagate(err, "An error occurred setting files to be moved for targetDirectoryForFilesArtifact '%v'", volume.Target) + } + } + } + if err := filesArgSLDict.SetKey(starlark.String(targetDirectoryForFilesArtifact), filesDictValue); err != nil { + return nil, nil, nil, stacktrace.Propagate(err, "An error occurred setting volume mountpoint '%s' in the files Starlark dict.", volume.Target) } } - return filesArgSLDict, filesArtifactsToUpload, nil + return filesArgSLDict, filesArtifactsToUpload, filesToBeMoved, nil } func getStarlarkPersistentDirectory(persistenceKey string) (starlark.Value, error) { @@ -612,3 +664,32 @@ func sortServicesBasedOnDependencies(serviceDependencyGraph map[string]map[strin return sortedServices, nil } + +// Converts a string to a RFC 1035 compliant format +func convertToRFC1035(input string) string { + // Remove leading and trailing spaces + input = strings.TrimSpace(input) + // Convert to lowercase + input = strings.ToLower(input) + // Replace non-alphanumeric characters with dashes + input = alphanumericCharWithDashesRegex.ReplaceAllString(input, "-") + // Remove consecutive dashes + input = consecutiveDashesRegex.ReplaceAllString(input, "-") + // Remove dashes at the beginning or end + input = strings.Trim(input, "-") + return input +} + +// CleanFilePath cleans up a file path by removing '~', '..', and '_' characters +// while retaining absolute paths as absolute and relative paths as relative. +func cleanFilePath(filePath string) string { + filePath = strings.ReplaceAll(filePath, "~", "") + filePath = strings.ReplaceAll(filePath, "..", "") + filePath = strings.ReplaceAll(filePath, "_", "") + return filePath +} + +func isEnvFileNotFoundErr(err error) bool { + // Prob not the best way to check for this error but right now the only time anything's loaded by compose is for env_file + return strings.Contains(err.Error(), "Failed to load") +} diff --git a/core/server/api_container/server/docker_compose_transpiler/docker_compose_transpiler_test.go b/core/server/api_container/server/docker_compose_transpiler/docker_compose_transpiler_test.go index d8c106583a..632463af78 100644 --- a/core/server/api_container/server/docker_compose_transpiler/docker_compose_transpiler_test.go +++ b/core/server/api_container/server/docker_compose_transpiler/docker_compose_transpiler_test.go @@ -3,12 +3,23 @@ package docker_compose_transpiler import ( "fmt" "github.com/stretchr/testify/require" + "os" + "path" "testing" ) -// TODO: Create a test framework like starlark test framework +const ( + testPackageAbsDirPathPattern = "package" + testFilePerms = 0644 +) + +// TODO: Create a test framework like starlark test framework so we updating tests manually with starlark or compose is less annoying func TestMinimalCompose(t *testing.T) { + testPackageAbsDirPath, err := os.MkdirTemp("", testPackageAbsDirPathPattern) + defer os.RemoveAll(testPackageAbsDirPath) + require.Nil(t, err) + composeBytes := []byte(` services: web: @@ -21,12 +32,16 @@ services: plan.add_service(name = "web", config = ServiceConfig(image="app/server", ports={"port0": PortSpec(number=80, transport_protocol="TCP", application_protocol="http", url="http://web:80")}, env_vars={})) ` - result, err := convertComposeToStarlarkScript(composeBytes, map[string]string{}) + result, err := convertComposeToStarlarkScript(composeBytes, map[string]string{}, testPackageAbsDirPath) require.NoError(t, err) require.Equal(t, expectedResult, result) } func TestMinimalComposeWithImageBuildSpec(t *testing.T) { + testPackageAbsDirPath, err := os.MkdirTemp("", testPackageAbsDirPathPattern) + defer os.RemoveAll(testPackageAbsDirPath) + require.Nil(t, err) + composeBytes := []byte(` services: web: @@ -38,12 +53,16 @@ services: plan.add_service(name = "web", config = ServiceConfig(image=ImageBuildSpec(image_name="web%s", build_context_dir="app/server"), ports={"port0": PortSpec(number=80, transport_protocol="TCP", application_protocol="http", url="http://web:80")}, env_vars={})) `, builtImageSuffix) - result, err := convertComposeToStarlarkScript(composeBytes, map[string]string{}) + result, err := convertComposeToStarlarkScript(composeBytes, map[string]string{}, testPackageAbsDirPath) require.NoError(t, err) require.Equal(t, expectedResult, result) } func TestMinimalComposeWithImageBuildSpecAndTarget(t *testing.T) { + testPackageAbsDirPath, err := os.MkdirTemp("", testPackageAbsDirPathPattern) + defer os.RemoveAll(testPackageAbsDirPath) + require.Nil(t, err) + composeBytes := []byte(` services: web: @@ -57,12 +76,44 @@ services: plan.add_service(name = "web", config = ServiceConfig(image=ImageBuildSpec(image_name="web%s", build_context_dir="app", target_stage="builder"), ports={"port0": PortSpec(number=80, transport_protocol="TCP", application_protocol="http", url="http://web:80")}, env_vars={})) `, builtImageSuffix) - result, err := convertComposeToStarlarkScript(composeBytes, map[string]string{}) + result, err := convertComposeToStarlarkScript(composeBytes, map[string]string{}, testPackageAbsDirPath) + require.NoError(t, err) + require.Equal(t, expectedResult, result) +} + +func TestMinimalComposeWithImageBuildSpecAndTargetAndName(t *testing.T) { + testPackageAbsDirPath, err := os.MkdirTemp("", testPackageAbsDirPathPattern) + defer os.RemoveAll(testPackageAbsDirPath) + require.Nil(t, err) + + // ignore the image name, and have kurtosis set it + composeBytes := []byte(` +services: + web: + image: web + build: + context: app + target: builder + ports: + - 80:80 +`) + expectedResult := fmt.Sprintf(`def run(plan): + plan.add_service(name = "web", config = ServiceConfig(image=ImageBuildSpec(image_name="web%s", build_context_dir="app", target_stage="builder"), ports={"port0": PortSpec(number=80, transport_protocol="TCP", application_protocol="http", url="http://web:80")}, env_vars={})) +`, builtImageSuffix) + + result, err := convertComposeToStarlarkScript(composeBytes, map[string]string{}, testPackageAbsDirPath) require.NoError(t, err) require.Equal(t, expectedResult, result) } func TestMinimalComposeWithVolume(t *testing.T) { + testPackageAbsDirPath, err := os.MkdirTemp("", testPackageAbsDirPathPattern) + defer os.RemoveAll(testPackageAbsDirPath) + require.Nil(t, err) + relVolumePath := "./data" + err = os.Mkdir(path.Join(testPackageAbsDirPath, relVolumePath), testFilePerms) + require.Nil(t, err) + composeBytes := []byte(` services: web: @@ -76,15 +127,19 @@ services: `) expectedResult := fmt.Sprintf(`def run(plan): plan.upload_files(src = "./data", name = "web--volume0") - plan.add_service(name = "web", config = ServiceConfig(image=ImageBuildSpec(image_name="web%s", build_context_dir="app", target_stage="builder"), ports={"port0": PortSpec(number=80, transport_protocol="TCP", application_protocol="http", url="http://web:80")}, files={"/data": "web--volume0"}, env_vars={})) + plan.add_service(name = "web", config = ServiceConfig(image=ImageBuildSpec(image_name="web%v", build_context_dir="app", target_stage="builder"), ports={"port0": PortSpec(number=80, transport_protocol="TCP", application_protocol="http", url="http://web:80")}, files={"/data": "web--volume0"}, env_vars={})) `, builtImageSuffix) - result, err := convertComposeToStarlarkScript(composeBytes, map[string]string{}) + result, err := convertComposeToStarlarkScript(composeBytes, map[string]string{}, testPackageAbsDirPath) require.NoError(t, err) require.Equal(t, expectedResult, result) } func TestMinimalComposeWithPersistentVolume(t *testing.T) { + testPackageAbsDirPath, err := os.MkdirTemp("", testPackageAbsDirPathPattern) + defer os.RemoveAll(testPackageAbsDirPath) + require.Nil(t, err) + composeBytes := []byte(` services: web: @@ -100,12 +155,16 @@ services: plan.add_service(name = "web", config = ServiceConfig(image=ImageBuildSpec(image_name="web%s", build_context_dir="app", target_stage="builder"), ports={"port0": PortSpec(number=80, transport_protocol="TCP", application_protocol="http", url="http://web:80")}, files={"/project/node_modules": Directory(persistent_key="web--volume0")}, env_vars={})) `, builtImageSuffix) - result, err := convertComposeToStarlarkScript(composeBytes, map[string]string{}) + result, err := convertComposeToStarlarkScript(composeBytes, map[string]string{}, testPackageAbsDirPath) require.NoError(t, err) require.Equal(t, expectedResult, result) } func TestMinimalComposeWithPersistentVolumeAtProvidedPath(t *testing.T) { + testPackageAbsDirPath, err := os.MkdirTemp("", testPackageAbsDirPathPattern) + defer os.RemoveAll(testPackageAbsDirPath) + require.Nil(t, err) + composeBytes := []byte(` services: web: @@ -121,12 +180,16 @@ services: plan.add_service(name = "web", config = ServiceConfig(image=ImageBuildSpec(image_name="web%s", build_context_dir="app", target_stage="builder"), ports={"port0": PortSpec(number=80, transport_protocol="TCP", application_protocol="http", url="http://web:80")}, files={"/node_modules": Directory(persistent_key="web--volume0")}, env_vars={})) `, builtImageSuffix) - result, err := convertComposeToStarlarkScript(composeBytes, map[string]string{}) + result, err := convertComposeToStarlarkScript(composeBytes, map[string]string{}, testPackageAbsDirPath) require.NoError(t, err) require.Equal(t, expectedResult, result) } func TestMinimalComposeWithPersistentVolumeAtProvidedPathAreUnique(t *testing.T) { + testPackageAbsDirPath, err := os.MkdirTemp("", testPackageAbsDirPathPattern) + defer os.RemoveAll(testPackageAbsDirPath) + require.Nil(t, err) + composeBytes := []byte(` services: web2: @@ -151,13 +214,71 @@ services: plan.add_service(name = "web3", config = ServiceConfig(image=ImageBuildSpec(image_name="web3%s", build_context_dir="app", target_stage="builder"), ports={"port0": PortSpec(number=80, transport_protocol="TCP", application_protocol="http", url="http://web3:80")}, files={"/node_modules": Directory(persistent_key="web3--volume0")}, env_vars={})) `, builtImageSuffix, builtImageSuffix) - result, err := convertComposeToStarlarkScript(composeBytes, map[string]string{}) + result, err := convertComposeToStarlarkScript(composeBytes, map[string]string{}, testPackageAbsDirPath) + require.NoError(t, err) + require.Equal(t, expectedResult, result) +} + +func TestMinimalComposeWithEnvFile(t *testing.T) { + testPackageAbsDirPath, err := os.MkdirTemp("", testPackageAbsDirPathPattern) + defer os.RemoveAll(testPackageAbsDirPath) + require.Nil(t, err) + relEnvFilePath := "./web.env" + err = os.WriteFile(path.Join(testPackageAbsDirPath, relEnvFilePath), []byte("USERNAME=kurtosis"), testFilePerms) + require.Nil(t, err) + + composeBytes := []byte(` +services: + web: + build: + context: app + target: builder + env_file: + - ./web.env + ports: + - 80:80 +`) + expectedResult := fmt.Sprintf(`def run(plan): + plan.add_service(name = "web", config = ServiceConfig(image=ImageBuildSpec(image_name="web%s", build_context_dir="app", target_stage="builder"), ports={"port0": PortSpec(number=80, transport_protocol="TCP", application_protocol="http", url="http://web:80")}, env_vars={"USERNAME": "kurtosis"})) +`, builtImageSuffix) + + result, err := convertComposeToStarlarkScript(composeBytes, map[string]string{}, testPackageAbsDirPath) + require.NoError(t, err) + require.Equal(t, expectedResult, result) +} + +func TestMinimalComposeWithNonRFC1035ServiceName(t *testing.T) { + testPackageAbsDirPath, err := os.MkdirTemp("", testPackageAbsDirPathPattern) + defer os.RemoveAll(testPackageAbsDirPath) + require.Nil(t, err) + + composeBytes := []byte(` +services: + Web_Service: + build: + context: app + target: builder + ports: + - 80:80 +`) + expectedResult := fmt.Sprintf(`def run(plan): + plan.add_service(name = "web-service", config = ServiceConfig(image=ImageBuildSpec(image_name="web-service%s", build_context_dir="app", target_stage="builder"), ports={"port0": PortSpec(number=80, transport_protocol="TCP", application_protocol="http", url="http://web-service:80")}, env_vars={})) +`, builtImageSuffix) + + result, err := convertComposeToStarlarkScript(composeBytes, map[string]string{}, testPackageAbsDirPath) require.NoError(t, err) require.Equal(t, expectedResult, result) } // Tests all supported compose functionalities for a single service func TestFullCompose(t *testing.T) { + testPackageAbsDirPath, err := os.MkdirTemp("", testPackageAbsDirPathPattern) + defer os.RemoveAll(testPackageAbsDirPath) + require.Nil(t, err) + relVolumePath := "./data" + err = os.Mkdir(path.Join(testPackageAbsDirPath, relVolumePath), testFilePerms) + require.Nil(t, err) + composeBytes := []byte(` services: web: @@ -179,15 +300,19 @@ services: `) expectedResult := fmt.Sprintf(`def run(plan): plan.upload_files(src = "./data", name = "web--volume0") - plan.add_service(name = "web", config = ServiceConfig(image=ImageBuildSpec(image_name="web%s", build_context_dir="app", target_stage="builder"), ports={"port0": PortSpec(number=80, transport_protocol="TCP", application_protocol="http", url="http://web:80")}, files={"/data": "web--volume0", "/node_modules": Directory(persistent_key="web--volume1")}, entrypoint=["/bin/echo", "-c", "echo \"Hello\""], cmd=["echo", "Hello,", "World!"], env_vars={"NODE_ENV": "development"})) + plan.add_service(name = "web", config = ServiceConfig(image=ImageBuildSpec(image_name="web%v", build_context_dir="app", target_stage="builder"), ports={"port0": PortSpec(number=80, transport_protocol="TCP", application_protocol="http", url="http://web:80")}, files={"/data": "web--volume0", "/node_modules": Directory(persistent_key="web--volume1")}, entrypoint=["/bin/echo", "-c", "echo \"Hello\""], cmd=["echo", "Hello,", "World!"], env_vars={"NODE_ENV": "development"})) `, builtImageSuffix) - result, err := convertComposeToStarlarkScript(composeBytes, map[string]string{}) + result, err := convertComposeToStarlarkScript(composeBytes, map[string]string{}, testPackageAbsDirPath) require.NoError(t, err) require.Equal(t, expectedResult, result) } func TestMultiServiceCompose(t *testing.T) { + testPackageAbsDirPath, err := os.MkdirTemp("", testPackageAbsDirPathPattern) + defer os.RemoveAll(testPackageAbsDirPath) + require.Nil(t, err) + composeBytes := []byte(` services: redis: @@ -220,7 +345,7 @@ services: plan.add_service(name = "web2", config = ServiceConfig(image=ImageBuildSpec(image_name="web2%s", build_context_dir="./web"), ports={"port0": PortSpec(number=5000, transport_protocol="TCP")}, env_vars={})) `, builtImageSuffix, builtImageSuffix, builtImageSuffix) - result, err := convertComposeToStarlarkScript(composeBytes, map[string]string{}) + result, err := convertComposeToStarlarkScript(composeBytes, map[string]string{}, testPackageAbsDirPath) require.NoError(t, err) require.Equal(t, expectedResult, result) } @@ -297,6 +422,10 @@ func TestSortServiceBasedOnDependenciesWithLinearDependencies(t *testing.T) { } func TestMultiServiceComposeWithDependsOn(t *testing.T) { + testPackageAbsDirPath, err := os.MkdirTemp("", testPackageAbsDirPathPattern) + defer os.RemoveAll(testPackageAbsDirPath) + require.Nil(t, err) + composeBytes := []byte(` services: redis: @@ -334,13 +463,17 @@ services: plan.add_service(name = "nginx", config = ServiceConfig(image=ImageBuildSpec(image_name="nginx%s", build_context_dir="./nginx"), ports={"port0": PortSpec(number=80, transport_protocol="TCP", application_protocol="http", url="http://nginx:80")}, env_vars={})) `, builtImageSuffix, builtImageSuffix, builtImageSuffix) - result, err := convertComposeToStarlarkScript(composeBytes, map[string]string{}) + result, err := convertComposeToStarlarkScript(composeBytes, map[string]string{}, testPackageAbsDirPath) require.NoError(t, err) require.Equal(t, expectedResult, result) } // Test depends on with circular dependency returns error func TestMultiServiceComposeWithCycleInDependsOn(t *testing.T) { + testPackageAbsDirPath, err := os.MkdirTemp("", testPackageAbsDirPathPattern) + defer os.RemoveAll(testPackageAbsDirPath) + require.Nil(t, err) + composeBytes := []byte(` services: redis: @@ -371,7 +504,7 @@ services: - web1 - web2 `) - _, err := convertComposeToStarlarkScript(composeBytes, map[string]string{}) + _, err = convertComposeToStarlarkScript(composeBytes, map[string]string{}, testPackageAbsDirPath) require.Error(t, err) require.ErrorIs(t, CyclicalDependencyError, err) } @@ -383,6 +516,10 @@ services: // ==================================================================================================== // https://github.com/docker/awesome-compose/tree/master/minecraft func TestMinecraftCompose(t *testing.T) { + testPackageAbsDirPath, err := os.MkdirTemp("", testPackageAbsDirPathPattern) + defer os.RemoveAll(testPackageAbsDirPath) + require.Nil(t, err) + composeBytes := []byte(` services: minecraft: @@ -398,14 +535,24 @@ services: volumes: - "~/minecraft_data:/data" `) + expectedResult := `def run(plan): + plan.add_service(name = "minecraft", config = ServiceConfig(image="itzg/minecraft-server", ports={"port0": PortSpec(number=25565, transport_protocol="TCP")}, files={"/data": Directory(persistent_key="minecraft--volume0")}, env_vars={"EULA": "TRUE"}, min_cpu=0, min_memory=0)) +` - // Returns error because '~' indicates the user is trying to reference their home path which is outside the package - _, err := convertComposeToStarlarkScript(composeBytes, map[string]string{}) - require.Error(t, err) + result, err := convertComposeToStarlarkScript(composeBytes, map[string]string{}, testPackageAbsDirPath) + require.NoError(t, err) + require.Equal(t, expectedResult, result) } // https://github.com/docker/awesome-compose/tree/master/angular func TestAngularCompose(t *testing.T) { + testPackageAbsDirPath, err := os.MkdirTemp("", testPackageAbsDirPathPattern) + defer os.RemoveAll(testPackageAbsDirPath) + require.Nil(t, err) + relVolumePath := "./angular" + err = os.Mkdir(path.Join(testPackageAbsDirPath, relVolumePath), testFilePerms) + require.Nil(t, err) + composeBytes := []byte(` services: web: @@ -420,16 +567,29 @@ services: `) expectedResult := fmt.Sprintf(`def run(plan): plan.upload_files(src = "./angular", name = "web--volume0") - plan.add_service(name = "web", config = ServiceConfig(image=ImageBuildSpec(image_name="web%s", build_context_dir="angular", target_stage="builder"), ports={"port0": PortSpec(number=4200, transport_protocol="TCP")}, files={"/project": "web--volume0", "/project/node_modules": Directory(persistent_key="web--volume1")}, env_vars={})) + plan.add_service(name = "web", config = ServiceConfig(image=ImageBuildSpec(image_name="web%v", build_context_dir="angular", target_stage="builder"), ports={"port0": PortSpec(number=4200, transport_protocol="TCP")}, files={"/project": "web--volume0", "/project/node_modules": Directory(persistent_key="web--volume1")}, env_vars={})) `, builtImageSuffix) - result, err := convertComposeToStarlarkScript(composeBytes, map[string]string{}) + result, err := convertComposeToStarlarkScript(composeBytes, map[string]string{}, testPackageAbsDirPath) require.NoError(t, err) require.Equal(t, expectedResult, result) } // From https://github.com/docker/awesome-compose/blob/master/elasticsearch-logstash-kibana func TestElasticSearchLogStashAndKibanaCompose(t *testing.T) { + testPackageAbsDirPath, err := os.MkdirTemp("", testPackageAbsDirPathPattern) + defer os.RemoveAll(testPackageAbsDirPath) + require.Nil(t, err) + logstashDirPath := "./logstash/pipeline" + err = os.MkdirAll(path.Join(testPackageAbsDirPath, logstashDirPath), 0750) + require.Nil(t, err) + nginxConfigRelFilePath := "./logstash/pipeline/logstash-nginx.config" + _, err = os.Create(path.Join(testPackageAbsDirPath, nginxConfigRelFilePath)) + require.Nil(t, err) + nginxLogRelFilePath := "./logstash/nginx.log" + _, err = os.Create(path.Join(testPackageAbsDirPath, nginxLogRelFilePath)) + require.Nil(t, err) + composeBytes := []byte(` services: elasticsearch: @@ -480,28 +640,33 @@ networks: elastic: driver: bridge `) + + // service names are equal to container names + // files_be_moved added to service config to handle mounting files specifically expectedResult := `def run(plan): - plan.add_service(name = "elasticsearch", config = ServiceConfig(image="elasticsearch:7.16.1", ports={"port0": PortSpec(number=9200, transport_protocol="TCP"), "port1": PortSpec(number=9300, transport_protocol="TCP")}, env_vars={"ES_JAVA_OPTS": "-Xms512m -Xmx512m", "discovery.type": "single-node"})) - plan.add_service(name = "kibana", config = ServiceConfig(image="kibana:7.16.1", ports={"port0": PortSpec(number=5601, transport_protocol="TCP")}, env_vars={})) - plan.upload_files(src = "./logstash/nginx.log", name = "logstash--volume1") - plan.upload_files(src = "./logstash/pipeline/logstash-nginx.config", name = "logstash--volume0") - plan.add_service(name = "logstash", config = ServiceConfig(image="logstash:7.16.1", ports={"port0": PortSpec(number=5000, transport_protocol="TCP"), "port1": PortSpec(number=5000, transport_protocol="UDP"), "port2": PortSpec(number=5044, transport_protocol="TCP"), "port3": PortSpec(number=9600, transport_protocol="TCP")}, files={"/home/nginx.log": "logstash--volume1", "/usr/share/logstash/pipeline/logstash-nginx.config": "logstash--volume0"}, cmd=["logstash", "-f", "/usr/share/logstash/pipeline/logstash-nginx.config"], env_vars={"LS_JAVA_OPTS": "-Xms512m -Xmx512m", "discovery.seed_hosts": "logstash"})) + plan.add_service(name = "es", config = ServiceConfig(image="elasticsearch:7.16.1", ports={"port0": PortSpec(number=9200, transport_protocol="TCP"), "port1": PortSpec(number=9300, transport_protocol="TCP")}, env_vars={"ES_JAVA_OPTS": "-Xms512m -Xmx512m", "discovery.type": "single-node"})) + plan.add_service(name = "kib", config = ServiceConfig(image="kibana:7.16.1", ports={"port0": PortSpec(number=5601, transport_protocol="TCP")}, env_vars={})) + plan.upload_files(src = "./logstash/nginx.log", name = "log--volume1") + plan.upload_files(src = "./logstash/pipeline/logstash-nginx.config", name = "log--volume0") + plan.add_service(name = "log", config = ServiceConfig(image="logstash:7.16.1", ports={"port0": PortSpec(number=5000, transport_protocol="TCP"), "port1": PortSpec(number=5000, transport_protocol="UDP"), "port2": PortSpec(number=5044, transport_protocol="TCP"), "port3": PortSpec(number=9600, transport_protocol="TCP")}, files={"/tmp/log--volume0": "log--volume0", "/tmp/log--volume1": "log--volume1"}, cmd=["logstash", "-f", "/usr/share/logstash/pipeline/logstash-nginx.config"], env_vars={"LS_JAVA_OPTS": "-Xms512m -Xmx512m", "discovery.seed_hosts": "logstash"}, files_to_be_moved={"/tmp/log--volume0/logstash-nginx.config": "/usr/share/logstash/pipeline/logstash-nginx.config", "/tmp/log--volume1/nginx.log": "/home/nginx.log"})) ` - - result, err := convertComposeToStarlarkScript(composeBytes, map[string]string{}) + result, err := convertComposeToStarlarkScript(composeBytes, map[string]string{}, testPackageAbsDirPath) require.NoError(t, err) require.Equal(t, expectedResult, result) } // From https://github.com/docker/awesome-compose/blob/master/fastapi/compose.yaml func TestFastApiCompose(t *testing.T) { + testPackageAbsDirPath, err := os.MkdirTemp("", testPackageAbsDirPathPattern) + defer os.RemoveAll(testPackageAbsDirPath) + require.Nil(t, err) + composeBytes := []byte(` services: api: build: context: . target: builder - container_name: fastapi-application environment: PORT: 8000 ports: @@ -512,13 +677,20 @@ services: plan.add_service(name = "api", config = ServiceConfig(image=ImageBuildSpec(image_name="api%v", build_context_dir=".", target_stage="builder"), ports={"port0": PortSpec(number=8000, transport_protocol="TCP", application_protocol="http", url="http://api:8000")}, env_vars={"PORT": "8000"})) `, builtImageSuffix) - result, err := convertComposeToStarlarkScript(composeBytes, map[string]string{}) + result, err := convertComposeToStarlarkScript(composeBytes, map[string]string{}, testPackageAbsDirPath) require.NoError(t, err) require.Equal(t, expectedResult, result) } // From https://github.com/docker/awesome-compose/blob/master/flask-redis/compose.yaml func TestFlaskRedisCompose(t *testing.T) { + testPackageAbsDirPath, err := os.MkdirTemp("", testPackageAbsDirPathPattern) + defer os.RemoveAll(testPackageAbsDirPath) + require.Nil(t, err) + relVolumePath := "./code" + err = os.Mkdir(path.Join(testPackageAbsDirPath, relVolumePath), testFilePerms) + require.Nil(t, err) + composeBytes := []byte(` services: redis: @@ -545,13 +717,17 @@ services: plan.add_service(name = "web", config = ServiceConfig(image=ImageBuildSpec(image_name="web%v", build_context_dir=".", target_stage="builder"), ports={"port0": PortSpec(number=8000, transport_protocol="TCP", application_protocol="http", url="http://web:8000")}, files={"/code": "web--volume0"}, env_vars={})) `, builtImageSuffix) - result, err := convertComposeToStarlarkScript(composeBytes, map[string]string{}) + result, err := convertComposeToStarlarkScript(composeBytes, map[string]string{}, testPackageAbsDirPath) require.NoError(t, err) require.Equal(t, expectedResult, result) } // From https://github.com/docker/awesome-compose/blob/master/nextcloud-redis-mariadb/compose.yaml func TestNextCloudRedisMariaDBCompose(t *testing.T) { + testPackageAbsDirPath, err := os.MkdirTemp("", testPackageAbsDirPathPattern) + defer os.RemoveAll(testPackageAbsDirPath) + require.Nil(t, err) + composeBytes := []byte(` services: nc: @@ -605,7 +781,7 @@ networks: plan.add_service(name = "redis", config = ServiceConfig(image="redis:alpine", env_vars={})) ` - result, err := convertComposeToStarlarkScript(composeBytes, map[string]string{}) + result, err := convertComposeToStarlarkScript(composeBytes, map[string]string{}, testPackageAbsDirPath) require.NoError(t, err) require.Equal(t, expectedResult, result) } @@ -623,5 +799,3 @@ networks: // Tests from other docker composes in the wild // // ==================================================================================================== - -// TODO: Test this docker compose when named volumes are supported https://github.com/OffchainLabs/nitro-testnode/blob/release/docker-compose.yaml diff --git a/core/server/api_container/server/startosis_engine/kurtosis_instruction/add_service/add_service_shared.go b/core/server/api_container/server/startosis_engine/kurtosis_instruction/add_service/add_service_shared.go index 72daad2547..957d1b0fc0 100644 --- a/core/server/api_container/server/startosis_engine/kurtosis_instruction/add_service/add_service_shared.go +++ b/core/server/api_container/server/startosis_engine/kurtosis_instruction/add_service/add_service_shared.go @@ -219,9 +219,11 @@ func replaceMagicStrings( serviceConfig.GetNodeSelectors(), serviceConfig.GetImageDownloadMode(), ) + if err != nil { return "", nil, stacktrace.Propagate(err, "An error occurred creating a service config") } + renderedServiceConfig.SetFilesToBeMoved(serviceConfig.GetFilesToBeMoved()) return service.ServiceName(serviceNameStr), renderedServiceConfig, nil } diff --git a/core/server/api_container/server/startosis_engine/kurtosis_starlark_framework/test_engine/service_config_full_framework_test.go b/core/server/api_container/server/startosis_engine/kurtosis_starlark_framework/test_engine/service_config_full_framework_test.go index ad489f63b8..535fcdfda8 100644 --- a/core/server/api_container/server/startosis_engine/kurtosis_starlark_framework/test_engine/service_config_full_framework_test.go +++ b/core/server/api_container/server/startosis_engine/kurtosis_starlark_framework/test_engine/service_config_full_framework_test.go @@ -39,7 +39,7 @@ func (t *serviceConfigFullTestCase) GetStarlarkCode() string { fileArtifact1 := fmt.Sprintf("%s(%s=[%q])", directory.DirectoryTypeName, directory.ArtifactNamesAttr, testFilesArtifactName1) fileArtifact2 := fmt.Sprintf("%s(%s=[%q])", directory.DirectoryTypeName, directory.ArtifactNamesAttr, testFilesArtifactName2) persistentDirectory := fmt.Sprintf("%s(%s=%q)", directory.DirectoryTypeName, directory.PersistentKeyAttr, testPersistentDirectoryKey) - starlarkCode := fmt.Sprintf("%s(%s=%q, %s=%s, %s=%s, %s=%s, %s=%s, %s=%s, %s=%s, %s=%q, %s=%d, %s=%d, %s=%d, %s=%d, %s=%s, %s=%v, %s=%v)", + starlarkCode := fmt.Sprintf("%s(%s=%q, %s=%s, %s=%s, %s=%s, %s=%s, %s=%s, %s=%s, %s=%q, %s=%d, %s=%d, %s=%d, %s=%d, %s=%s, %s=%v, %s=%v, %s=%v)", service_config.ServiceConfigTypeName, service_config.ImageAttr, testContainerImageName, service_config.PortsAttr, fmt.Sprintf("{%q: PortSpec(number=%d, transport_protocol=%q, application_protocol=%q, wait=%q)}", testPrivatePortId, testPrivatePortNumber, testPrivatePortProtocolStr, testPrivateApplicationProtocol, testWaitConfiguration), @@ -56,7 +56,8 @@ func (t *serviceConfigFullTestCase) GetStarlarkCode() string { service_config.ReadyConditionsAttr, getDefaultReadyConditionsScriptPart(), service_config.LabelsAttr, fmt.Sprintf("{%q: %q, %q: %q}", testServiceConfigLabelsKey1, testServiceConfigLabelsValue1, testServiceConfigLabelsKey2, testServiceConfigLabelsValue2), - service_config.NodeSelectorsAttr, fmt.Sprintf("{%q: %q}", testNodeSelectorKey1, testNodeSelectorValue1)) + service_config.NodeSelectorsAttr, fmt.Sprintf("{%q: %q}", testNodeSelectorKey1, testNodeSelectorValue1), + service_config.FilesToBeMovedAttr, fmt.Sprintf("{%q: %q}", testFilesToBeMoved, testFilesToBeMoved)) return starlarkCode } @@ -112,6 +113,8 @@ func (t *serviceConfigFullTestCase) Assert(typeValue builtin_argument.KurtosisVa require.Equal(t, testEntryPointSlice, serviceConfig.GetEntrypointArgs()) require.Equal(t, testCmdSlice, serviceConfig.GetCmdArgs()) + require.NotEmpty(t, serviceConfig.GetFilesToBeMoved()) + expectedEnvVars := map[string]string{ testEnvVarName1: testEnvVarValue1, testEnvVarName2: testEnvVarValue2, diff --git a/core/server/api_container/server/startosis_engine/kurtosis_starlark_framework/test_engine/static_constants.go b/core/server/api_container/server/startosis_engine/kurtosis_starlark_framework/test_engine/static_constants.go index 3602872ab8..9037358714 100644 --- a/core/server/api_container/server/startosis_engine/kurtosis_starlark_framework/test_engine/static_constants.go +++ b/core/server/api_container/server/startosis_engine/kurtosis_starlark_framework/test_engine/static_constants.go @@ -148,4 +148,6 @@ var ( testTolerationKey = "test-key" testTolerationValue = "test-value" testTolerationSeconds = int64(64) + + testFilesToBeMoved = "test.txt" ) diff --git a/core/server/api_container/server/startosis_engine/kurtosis_types/service_config/service_config.go b/core/server/api_container/server/startosis_engine/kurtosis_types/service_config/service_config.go index aff2818ff5..54145d6145 100644 --- a/core/server/api_container/server/startosis_engine/kurtosis_types/service_config/service_config.go +++ b/core/server/api_container/server/startosis_engine/kurtosis_types/service_config/service_config.go @@ -51,6 +51,7 @@ const ( UserAttr = "user" TolerationsAttr = "tolerations" NodeSelectorsAttr = "node_selectors" + FilesToBeMovedAttr = "files_to_be_moved" DefaultPrivateIPAddrPlaceholder = "KURTOSIS_IP_ADDR_PLACEHOLDER" @@ -212,6 +213,14 @@ func NewServiceConfigType() *kurtosis_type_constructor.KurtosisTypeConstructor { return builtin_argument.StringMappingToString(value, NodeSelectorsAttr) }, }, + { + Name: FilesToBeMovedAttr, + IsOptional: true, + ZeroValueProvider: builtin_argument.ZeroValueProvider[*starlark.Dict], + Validator: func(value starlark.Value) *startosis_errors.InterpretationError { + return builtin_argument.StringMappingToString(value, FilesToBeMovedAttr) + }, + }, }, }, @@ -504,6 +513,18 @@ func (config *ServiceConfig) ToKurtosisType( } } + filesToBeMoved := map[string]string{} + filesToBeMovedStarlark, found, interpretationErr := kurtosis_type_constructor.ExtractAttrValue[*starlark.Dict](config.KurtosisValueTypeDefault, FilesToBeMovedAttr) + if interpretationErr != nil { + return nil, interpretationErr + } + if found && filesToBeMovedStarlark.Len() > 0 { + filesToBeMoved, interpretationErr = kurtosis_types.SafeCastToMapStringString(filesToBeMovedStarlark, FilesToBeMovedAttr) + if interpretationErr != nil { + return nil, interpretationErr + } + } + serviceConfig, err := service.CreateServiceConfig( imageName, maybeImageBuildSpec, @@ -530,6 +551,7 @@ func (config *ServiceConfig) ToKurtosisType( if err != nil { return nil, startosis_errors.WrapWithInterpretationError(err, "An error occurred creating a service config") } + serviceConfig.SetFilesToBeMoved(filesToBeMoved) return serviceConfig, nil } diff --git a/core/server/api_container/server/startosis_engine/startosis_packages/git_package_content_provider/git_package_content_provider.go b/core/server/api_container/server/startosis_engine/startosis_packages/git_package_content_provider/git_package_content_provider.go index 99b2d9796f..800cb8b453 100644 --- a/core/server/api_container/server/startosis_engine/startosis_packages/git_package_content_provider/git_package_content_provider.go +++ b/core/server/api_container/server/startosis_engine/startosis_packages/git_package_content_provider/git_package_content_provider.go @@ -115,12 +115,15 @@ func (provider *GitPackageContentProvider) getOnDiskAbsolutePath(absoluteLocator if err != nil { return "", startosis_errors.WrapWithInterpretationError(err, "An error occurred parsing Git URL for absolute file locator '%s'", repositoryPathURL) } + pathToFileOnDisk := path.Join(provider.repositoriesDir, parsedURL.GetRelativeFilePath()) + pathToPackageOnDisk := path.Join(provider.repositoriesDir, parsedURL.GetRelativeRepoPath()) - if shouldOnlyAcceptsPackageFilePath && parsedURL.GetRelativeFilePath() == "" { - return "", startosis_errors.NewInterpretationError("The path '%v' needs to point to a specific file but it didn't. Users can only read or import specific files and not entire packages.", repositoryPathURL) + // TODO(tedi): see if its safe to adjust ParsedGitURL api instead to prevent having to make this check + // parsedURL.GetRelativeFilePath() is empty when repositoryPathURL does not refer to a specific file + // In this case, assume caller wants to get on base of repository and adjust the pathToFileOnDisk to point to base of repository on disk + if parsedURL.GetRelativeFilePath() == "" { + pathToFileOnDisk = pathToPackageOnDisk } - pathToFileOnDisk := path.Join(provider.repositoriesDir, parsedURL.GetRelativeFilePath()) - packagePath := path.Join(provider.repositoriesDir, parsedURL.GetRelativeRepoPath()) // Return the file path straight if it exists if _, err := os.Stat(pathToFileOnDisk); err == nil { @@ -129,7 +132,7 @@ func (provider *GitPackageContentProvider) getOnDiskAbsolutePath(absoluteLocator // Check if the repo exists // If the repo exists but the `pathToFileOnDisk` doesn't that means there's a mistake in the locator - if _, err := os.Stat(packagePath); err == nil { + if _, err := os.Stat(pathToPackageOnDisk); err == nil { relativeFilePathWithoutPackageName := strings.Replace(parsedURL.GetRelativeFilePath(), parsedURL.GetRelativeRepoPath(), replacedWithEmptyString, onlyOneReplacement) return "", startosis_errors.NewInterpretationError("'%v' doesn't exist in the package '%v'", relativeFilePathWithoutPackageName, parsedURL.GetRelativeRepoPath()) } diff --git a/core/server/api_container/server/startosis_engine/startosis_packages/git_package_content_provider/git_package_content_provider_test.go b/core/server/api_container/server/startosis_engine/startosis_packages/git_package_content_provider/git_package_content_provider_test.go index e5307b8ac2..8d18cee25d 100644 --- a/core/server/api_container/server/startosis_engine/startosis_packages/git_package_content_provider/git_package_content_provider_test.go +++ b/core/server/api_container/server/startosis_engine/startosis_packages/git_package_content_provider/git_package_content_provider_test.go @@ -21,7 +21,6 @@ const ( repositoriesTmpDirRelPath = "tmp-repositories" githubAuthDirRelPath = "github-auth" githubAuthTokenFilename = "token.txt" - genericRepositoriesDirRelPath = "generic-repositories" packageDescriptionForTest = "package description test" localAbsoluteLocatorNotAllowedMsg = "is referencing a file within the same package using absolute import syntax" ) diff --git a/core/server/go.mod b/core/server/go.mod index 5bce7fe833..1772384c81 100644 --- a/core/server/go.mod +++ b/core/server/go.mod @@ -93,6 +93,7 @@ require ( github.com/google/gofuzz v1.2.0 // indirect github.com/gorilla/websocket v1.5.1 // indirect github.com/grpc-ecosystem/go-grpc-middleware v1.3.0 // indirect + github.com/hashicorp/go-envparse v0.1.0 // indirect github.com/imdario/mergo v0.3.16 // indirect github.com/improbable-eng/grpc-web v0.15.0 // indirect github.com/itchyny/timefmt-go v0.1.5 // indirect diff --git a/core/server/go.sum b/core/server/go.sum index ac7b26a93a..14afb6624b 100644 --- a/core/server/go.sum +++ b/core/server/go.sum @@ -1,6 +1,7 @@ cloud.google.com/go v0.26.0/go.mod h1:aQUYkXzVsufM+DwF1aE+0xfcU+56JwCaLick0ClmMTw= cloud.google.com/go v0.34.0/go.mod h1:aQUYkXzVsufM+DwF1aE+0xfcU+56JwCaLick0ClmMTw= cloud.google.com/go v0.65.0 h1:Dg9iHVQfrhq82rUNu9ZxUDrJLaxFUe/HlCVaLyRruq8= +cloud.google.com/go v0.65.0/go.mod h1:O5N8zS7uWy9vkA9vayVHs65eM1ubvY4h553ofrNHObY= cloud.google.com/go/compute v1.20.1 h1:6aKEtlUiwEpJzM001l0yFkpXmUVXaN8W+fbkb2AZNbg= cloud.google.com/go/compute/metadata v0.2.3 h1:mg4jlk7mCAj6xXp9UJ4fjI9VUI5rubuGBW5aJ7UnBMY= cloud.google.com/go/compute/metadata v0.2.3/go.mod h1:VAV5nSsACxMJvgaAuX6Pk2AawlZn8kiOGuCv6gTkwuA= @@ -247,6 +248,7 @@ github.com/google/renameio v0.1.0/go.mod h1:KWCgfxg9yswjAJkECMjeO8J8rahYeXnNhOm4 github.com/google/uuid v1.0.0/go.mod h1:TIyPZe4MgqvfeYDBFedMoGGpEw/LqOeaOT+nhxU+yHo= github.com/google/uuid v1.4.0 h1:MtMxsa51/r9yyhkyLsVeVt0B+BGQZzpQiTQ4eHZ8bc4= github.com/google/uuid v1.4.0/go.mod h1:TIyPZe4MgqvfeYDBFedMoGGpEw/LqOeaOT+nhxU+yHo= +github.com/googleapis/gax-go/v2 v2.0.5/go.mod h1:DWXyrwAJ9X0FpwwEdw+IPEYBICEFu5mhpdKc/us6bOk= github.com/gopherjs/gopherjs v0.0.0-20181017120253-0766667cb4d1/go.mod h1:wJfORRmW1u3UXTncJ5qlYoELFm8eSnnEO6hX4iZ3EWY= github.com/gorilla/context v1.1.1/go.mod h1:kBGZzfjB9CEq2AlWe17Uuf7NDRt0dE0s8S51q0aT7Yg= github.com/gorilla/mux v1.6.2/go.mod h1:1lud6UwP+6orDFRuTfBEV8e9/aOM/c4fVVCaMa2zaAs= @@ -266,6 +268,8 @@ github.com/hashicorp/consul/api v1.3.0/go.mod h1:MmDNSzIMUjNpY/mQ398R4bk2FnqQLoP github.com/hashicorp/consul/sdk v0.3.0/go.mod h1:VKf9jXwCTEY1QZP2MOLRhb5i/I/ssyNV1vwHyQBF0x8= github.com/hashicorp/errwrap v1.0.0/go.mod h1:YH+1FKiLXxHSkmPseP+kNlulaMuP3n2brvKWEqk/Jc4= github.com/hashicorp/go-cleanhttp v0.5.1/go.mod h1:JpRdi6/HCYpAwUzNwuwqhbovhLtngrth3wmdIIUrZ80= +github.com/hashicorp/go-envparse v0.1.0 h1:bE++6bhIsNCPLvgDZkYqo3nA+/PFI51pkrHdmPSDFPY= +github.com/hashicorp/go-envparse v0.1.0/go.mod h1:OHheN1GoygLlAkTlXLXvAdnXdZxy8JUweQ1rAXx1xnc= github.com/hashicorp/go-immutable-radix v1.0.0/go.mod h1:0y9vanUI8NX6FsYoO3zeMjhV/C5i9g4Q3DwcSNZ4P60= github.com/hashicorp/go-msgpack v0.5.3/go.mod h1:ahLV/dePpqEmjfWmKiqvPkv/twdG7iPBM1vqhUKIvfM= github.com/hashicorp/go-multierror v1.0.0/go.mod h1:dHtQlpGsu+cZNNAkkCN/P3hoUDHhCYQXV3UM06sGGrk= @@ -754,6 +758,7 @@ golang.org/x/xerrors v0.0.0-20191011141410-1b5146add898/go.mod h1:I/5z698sn9Ka8T golang.org/x/xerrors v0.0.0-20191204190536-9bdfabe68543/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0= golang.org/x/xerrors v0.0.0-20200804184101-5ec99f83aff1/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0= google.golang.org/api v0.3.1/go.mod h1:6wY9I6uQWHQ8EM57III9mq/AjF+i8G65rmVagqKMtkk= +google.golang.org/api v0.30.0/go.mod h1:QGmEvQ87FHZNiUVJkT14jQNYJ4ZJjdRF23ZXz5138Fc= google.golang.org/appengine v1.1.0/go.mod h1:EbEs0AVv82hx2wNQdGPgUI5lhzA/G0D9YwlJXL52JkM= google.golang.org/appengine v1.2.0/go.mod h1:xpcJRLb0r/rnEns0DIKYYv+WjYCduHsrkT7/EB5XEv4= google.golang.org/appengine v1.4.0/go.mod h1:xpcJRLb0r/rnEns0DIKYYv+WjYCduHsrkT7/EB5XEv4= diff --git a/path-compression/path_compression_test.go b/path-compression/path_compression_test.go index a7b58392a9..36a336ccb8 100644 --- a/path-compression/path_compression_test.go +++ b/path-compression/path_compression_test.go @@ -238,14 +238,10 @@ func TestCompressPath_CheckFileContentsAreTakenIntoAccountForHash(t *testing.T) require.Equal(t, expectedHashHex, hex.EncodeToString(md5)) } -func TestCompressPath_EmptyDirError(t *testing.T) { +func TestCompressPath_EmptyDirDoesNotError(t *testing.T) { dirPath, err := os.MkdirTemp("", "test-dir-*") require.NoError(t, err) - compressedData, size, md5, err := CompressPath(dirPath, false) - require.Error(t, err) - require.Contains(t, err.Error(), "you are trying to compress is empty") - require.Nil(t, compressedData) - require.Equal(t, size, uint64(0)) - require.Nil(t, md5) + _, _, _, err = CompressPath(dirPath, false) + require.NoError(t, err) } diff --git a/path-compression/path_compresssion.go b/path-compression/path_compresssion.go index ae3097dcf5..5235021c43 100644 --- a/path-compression/path_compresssion.go +++ b/path-compression/path_compresssion.go @@ -127,9 +127,6 @@ func listFilesInPathInternal(filesInPath *[]string, path string, recursiveMode b if err != nil { return stacktrace.Propagate(err, "There was an error in getting a list of files in the directory '%s' provided", trimmedPath) } - if topLevel && len(filesInDirectory) == 0 { - return stacktrace.NewError("The directory '%s' you are trying to compress is empty", path) - } for _, fileInDirectory := range filesInDirectory { fileInDirectoryPath := filepath.Join(trimmedPath, fileInDirectory.Name()) *filesInPath = append(*filesInPath, fileInDirectoryPath)