Skip to content

Commit

Permalink
feat: Add volume bind support for kurtosis import (#984)
Browse files Browse the repository at this point in the history
## Description:
<!-- Describe this change, how it works, and the motivation behind it.
-->

## Is this change user facing?
YES
<!-- If yes, please add the "user facing" label to the PR -->
<!-- If yes, don't forget to include docs changes where relevant -->

## References (if applicable):
<!-- Add relevant Github Issues, Discord threads, or other helpful
information. -->
  • Loading branch information
victorcolombo committed Jul 26, 2023
1 parent e6ff482 commit 391c016
Show file tree
Hide file tree
Showing 3 changed files with 58 additions and 20 deletions.
67 changes: 51 additions & 16 deletions cli/cli/commands/import/import.go
Expand Up @@ -7,6 +7,7 @@ import (
"github.com/compose-spec/compose-go/types"
"github.com/joho/godotenv"
"github.com/kurtosis-tech/kurtosis/api/golang/core/kurtosis_core_rpc_api_bindings"
"github.com/kurtosis-tech/kurtosis/api/golang/core/lib/enclaves"
enclave_consts "github.com/kurtosis-tech/kurtosis/api/golang/engine/lib/enclave"
"github.com/kurtosis-tech/kurtosis/api/golang/engine/lib/kurtosis_context"
"github.com/kurtosis-tech/kurtosis/cli/cli/command_framework/highlevel/file_system_path_arg"
Expand All @@ -16,6 +17,7 @@ import (
"github.com/kurtosis-tech/kurtosis/cli/cli/command_str_consts"
"github.com/kurtosis-tech/kurtosis/cli/cli/commands/service/add"
"github.com/kurtosis-tech/kurtosis/cli/cli/helpers/output_printers"
"github.com/kurtosis-tech/kurtosis/name_generator"
"github.com/kurtosis-tech/stacktrace"
"github.com/sirupsen/logrus"
"strings"
Expand Down Expand Up @@ -92,7 +94,7 @@ func run(ctx context.Context, flags *flags.ParsedFlags, args *args.ParsedArgs) e
}
logrus.Infof("Enviroment loaded: %v", dotEnvMap)

script, err := convertComposeFileToStarlark(path, dotEnvMap)
script, artifacts, err := convertComposeFileToStarlark(path, dotEnvMap)
if err != nil {
return stacktrace.Propagate(err, "Failed to convert compose to starlark")
}
Expand All @@ -103,32 +105,62 @@ func run(ctx context.Context, flags *flags.ParsedFlags, args *args.ParsedArgs) e
if err != nil {
return stacktrace.Propagate(err, "Couldn't find enclave name flag '%v'", enclaveNameFlagKey)
}
err = runStarlark(ctx, enclaveName, script)
enclaveCtx, err := createEnclave(ctx, enclaveName)
if err != nil {
return stacktrace.Propagate(err, "Couldn't create enclave")
}
defer output_printers.PrintEnclaveName(enclaveCtx.GetEnclaveName())
err = uploadArtifacts(enclaveCtx, artifacts)
if err != nil {
return stacktrace.Propagate(err, "Failed to upload all required artifacts for execution")
}
err = runStarlark(ctx, enclaveCtx, script)
if err != nil {
return stacktrace.Propagate(err, "Failed to run generated starlark from compose")
}
return nil
}

func convertComposeFileToStarlark(path string, dotEnvMap map[string]string) (string, error) {
func convertComposeFileToStarlark(path string, dotEnvMap map[string]string) (string, map[string]string, error) {
project, err := loader.Load(types.ConfigDetails{ //nolint:exhaustruct
ConfigFiles: []types.ConfigFile{{Filename: path}},
Environment: dotEnvMap,
})
if err != nil {
return "", stacktrace.Propagate(err, "Error parsing docker compose")
return "", nil, stacktrace.Propagate(err, "Error parsing docker compose")
}
script, err := convertComposeProjectToStarlark(project)
script, artifacts, err := convertComposeProjectToStarlark(project)
if err != nil {
return "", stacktrace.Propagate(err, "Error translating docker compose to Starlark")
return "", nil, stacktrace.Propagate(err, "Error translating docker compose to Starlark")
}
return script, nil
return script, artifacts, nil
}

func uploadArtifacts(enclaveCtx *enclaves.EnclaveContext, artifactUploadMap map[string]string) error {
for source, artifactName := range artifactUploadMap {
_, _, err := enclaveCtx.UploadFiles(source, artifactName)
if err != nil {
return stacktrace.Propagate(err, "Failed to upload path '%v' as artifact '%s'", source, artifactName)
}
}
return nil
}

// TODO(victor.colombo): Have a better UX letting people know ports have been remapped
func convertComposeProjectToStarlark(compose *types.Project) (string, error) {
func convertComposeProjectToStarlark(compose *types.Project) (string, map[string]string, error) {
serviceStarlarks := map[string]string{}
requiredFileUploads := map[string]string{}
for _, serviceConfig := range compose.Services {
artifactsPiecesStr := []string{}
for _, volume := range serviceConfig.Volumes {
if volume.Type != types.VolumeTypeBind {
return "", nil, stacktrace.NewError("Volume type '%v' is not supported", volume.Type)
}
if _, ok := requiredFileUploads[volume.Source]; !ok {
requiredFileUploads[volume.Source] = name_generator.GenerateNatureThemeNameForFileArtifacts()
}
artifactsPiecesStr = append(artifactsPiecesStr, fmt.Sprintf("%s:%s", volume.Target, requiredFileUploads[volume.Source]))
}
portPiecesStr := []string{}
for _, port := range serviceConfig.Ports {
portStr := fmt.Sprintf("docker-%s=%d", port.Published, port.Target)
Expand All @@ -151,33 +183,36 @@ func convertComposeProjectToStarlark(compose *types.Project) (string, error) {
serviceConfig.Command,
serviceConfig.Entrypoint,
strings.Join(envvarsPiecesStr, ","),
nonSupportedField,
strings.Join(artifactsPiecesStr, ","),
emptyPrivateIpPlaceholder)
if err != nil {
return "", stacktrace.Propagate(err, "Error getting service config starlark for '%v'", serviceConfig)
return "", nil, stacktrace.Propagate(err, "Error getting service config starlark for '%v'", serviceConfig)
}
serviceStarlarks[serviceConfig.Name] = starlarkConfig
}
script := "def run(plan):\n"
for serviceName, serviceConfig := range serviceStarlarks {
script += fmt.Sprintf("\tplan.add_service(name = '%s', config = %s)\n", serviceName, serviceConfig)
}
return script, nil
return script, requiredFileUploads, nil
}

// TODO(victor.colombo): This should be part of the SDK, since we implement this over and over again
func runStarlark(ctx context.Context, enclaveName string, starlarkScript string) error {
func createEnclave(ctx context.Context, enclaveName string) (*enclaves.EnclaveContext, error) {
kurtosisCtx, err := kurtosis_context.NewKurtosisContextFromLocalEngine()
if err != nil {
return stacktrace.Propagate(err, "An error occurred connecting to the local Kurtosis engine")
return nil, stacktrace.Propagate(err, "An error occurred connecting to the local Kurtosis engine")
}

enclaveCtx, err := kurtosisCtx.CreateEnclave(ctx, enclaveName, false)
if err != nil {
return stacktrace.Propagate(err, "An error occurred creating an enclave '%v'", enclaveName)
return nil, stacktrace.Propagate(err, "An error occurred creating an enclave '%v'", enclaveName)
}
defer output_printers.PrintEnclaveName(enclaveCtx.GetEnclaveName())

return enclaveCtx, nil
}

// TODO(victor.colombo): This should be part of the SDK, since we implement this over and over again
func runStarlark(ctx context.Context, enclaveCtx *enclaves.EnclaveContext, starlarkScript string) error {
starlarkRunResult, err := enclaveCtx.RunStarlarkScriptBlocking(ctx, defaultMainFunction, starlarkScript, noStarlarkParams, false, 1, []kurtosis_core_rpc_api_bindings.KurtosisFeatureFlag{})
if err != nil {
return stacktrace.Propagate(err, "An error has occurred when running Starlark to add service")
Expand Down
@@ -1,21 +1,23 @@
services:
postgres:
container_name: postgres
image: postgres:latest
environment:
- POSTGRES_USER=${POSTGRES_USER}
- POSTGRES_PASSWORD=${POSTGRES_PW}
- POSTGRES_DB=${POSTGRES_DB} #optional (specify default database instead of $POSTGRES_DB)
ports:
- "5432:5432"
restart: always

pgadmin:
container_name: pgadmin
image: dpage/pgadmin4:latest
environment:
- PGADMIN_DEFAULT_EMAIL=${PGADMIN_MAIL}
- PGADMIN_DEFAULT_PASSWORD=${PGADMIN_PW}
ports:
- "5050:80"
restart: always

simple:
image: alpine:latest
entrypoint: ["cat", "/tmp/file.txt"]
volumes:
- "./internal_testsuites/resources/docker_import_cli_test/volume/:/tmp"
@@ -0,0 +1 @@
Hi

0 comments on commit 391c016

Please sign in to comment.