Skip to content

Commit

Permalink
feat: Add environment support for kurtosis import (#982)
Browse files Browse the repository at this point in the history
## Description:
<!-- Describe this change, how it works, and the motivation behind it.
-->
This PR notably does not include support for `env_file`

## Is this change user facing?
YES
<!-- If yes, please add the "user facing" label to the PR -->
<!-- If yes, don't forget to include docs changes where relevant -->

## References (if applicable):
<!-- Add relevant Github Issues, Discord threads, or other helpful
information. -->
Closes #981
  • Loading branch information
victorcolombo committed Jul 26, 2023
1 parent e185fe6 commit 24e71d1
Show file tree
Hide file tree
Showing 6 changed files with 74 additions and 27 deletions.
7 changes: 5 additions & 2 deletions .circleci/config.yml
Expand Up @@ -195,7 +195,10 @@ parameters:
rendertemplate-cli-test-template-relative-path:
type: string
default: "internal_testsuites/resources/render_template_cli_test/template.txt"
dockerimport-cli-test-relative-path:
dockerimport-cli-dotenv-relative-path:
type: string
default: "internal_testsuites/resources/docker_import_cli_test/.env"
dockerimport-cli-dockerfile-relative-path:
type: string
default: "internal_testsuites/resources/docker_import_cli_test/docker-compose.yml"
rendertemplate-cli-test-data-json-relative-path:
Expand Down Expand Up @@ -754,7 +757,7 @@ jobs:
- run: "${KURTOSIS_BINPATH} run --enclave test-datastore github.com/kurtosis-tech/datastore-army-package '{\"num_datastores\": 2}'"

# Execute docker compose import
- run: "${KURTOSIS_BINPATH} import --enclave test-import << pipeline.parameters.dockerimport-cli-test-relative-path >>"
- run: "${KURTOSIS_BINPATH} import --enclave test-import --env << pipeline.parameters.dockerimport-cli-dotenv-relative-path >> << pipeline.parameters.dockerimport-cli-dockerfile-relative-path >>"


- run: "${KURTOSIS_BINPATH} enclave stop test-enclave"
Expand Down
60 changes: 39 additions & 21 deletions cli/cli/commands/import/import.go
Expand Up @@ -5,6 +5,7 @@ import (
"fmt"
"github.com/compose-spec/compose-go/loader"
"github.com/compose-spec/compose-go/types"
"github.com/joho/godotenv"
"github.com/kurtosis-tech/kurtosis/api/golang/core/kurtosis_core_rpc_api_bindings"
enclave_consts "github.com/kurtosis-tech/kurtosis/api/golang/engine/lib/enclave"
"github.com/kurtosis-tech/kurtosis/api/golang/engine/lib/kurtosis_context"
Expand All @@ -17,16 +18,16 @@ import (
"github.com/kurtosis-tech/kurtosis/cli/cli/helpers/output_printers"
"github.com/kurtosis-tech/stacktrace"
"github.com/sirupsen/logrus"
"io"
"os"
"strings"
)

const (
enclaveNameFlagKey = "enclave"
pathArgKey = "file-path"
dotEnvPathFlagKey = "env"
isPathArgOptional = false
defaultPathArg = ""
defaultDotEnvPathFlag = ".env"
emptyPrivateIpPlaceholder = ""
nonSupportedField = ""
defaultMainFunction = ""
Expand All @@ -43,7 +44,7 @@ var ImportCmd = &lowlevel.LowlevelKurtosisCommand{
Flags: []*flags.FlagConfig{
{
Key: enclaveNameFlagKey,
Shorthand: "e",
Shorthand: "n",
Default: autogenerateEnclaveNameKeyword,
Usage: fmt.Sprintf(
"The enclave name to give the new enclave, which must match regex '%v' "+
Expand All @@ -52,6 +53,13 @@ var ImportCmd = &lowlevel.LowlevelKurtosisCommand{
),
Type: flags.FlagType_String,
},
{
Key: dotEnvPathFlagKey,
Shorthand: "e",
Default: defaultDotEnvPathFlag,
Usage: "The .env file path to be loaded into docker compose",
Type: flags.FlagType_String,
},
},
Args: []*args.ArgConfig{
file_system_path_arg.NewFilepathOrDirpathArg(
Expand All @@ -71,19 +79,20 @@ func run(ctx context.Context, flags *flags.ParsedFlags, args *args.ParsedArgs) e
if err != nil {
return stacktrace.Propagate(err, "Path arg '%v' is missing", pathArgKey)
}
file, err := os.Open(path)

dotEnvPath, err := flags.GetString(dotEnvPathFlagKey)
if err != nil {
return stacktrace.Propagate(err, "File on '%v' was not found", path)
return stacktrace.Propagate(err, "Dot env path arg '%v' is missing", dotEnvPath)
}
defer file.Close()

// Read the content of the file into a []byte slice
content, err := io.ReadAll(file)
dotEnvMap, err := godotenv.Read(dotEnvPath)
if err != nil {
return stacktrace.Propagate(err, "Error reading file: %s\n", err)
logrus.Debugf("No dotenv file was found: %v", err)
dotEnvMap = map[string]string{}
}
logrus.Infof("Enviroment loaded: %v", dotEnvMap)

script, err := convertComposeFileToStarlark(content)
script, err := convertComposeFileToStarlark(path, dotEnvMap)
if err != nil {
return stacktrace.Propagate(err, "Failed to convert compose to starlark")
}
Expand All @@ -101,16 +110,10 @@ func run(ctx context.Context, flags *flags.ParsedFlags, args *args.ParsedArgs) e
return nil
}

func convertComposeFileToStarlark(content []byte) (string, error) {
// Load the Compose configuration from the []byte slice
config, err := loader.ParseYAML(content)
if err != nil {
return "", stacktrace.Propagate(err, "Error parsing YAML: %s\n", err)
}

// Convert the generic map to a structured compose.Config
func convertComposeFileToStarlark(path string, dotEnvMap map[string]string) (string, error) {
project, err := loader.Load(types.ConfigDetails{ //nolint:exhaustruct
ConfigFiles: []types.ConfigFile{{Config: config}},
ConfigFiles: []types.ConfigFile{{Filename: path}},
Environment: dotEnvMap,
})
if err != nil {
return "", stacktrace.Propagate(err, "Error parsing docker compose")
Expand All @@ -134,7 +137,22 @@ func convertComposeProjectToStarlark(compose *types.Project) (string, error) {
}
portPiecesStr = append(portPiecesStr, portStr)
}
starlarkConfig, err := add.GetServiceConfigStarlark(serviceConfig.Image, strings.Join(portPiecesStr, ","), serviceConfig.Command, serviceConfig.Entrypoint, nonSupportedField, nonSupportedField, emptyPrivateIpPlaceholder)
envvarsPiecesStr := []string{}
for envKey, envValue := range serviceConfig.Environment {
envValueStr := ""
if envValue != nil {
envValueStr = *envValue
}
envvarsPiecesStr = append(envvarsPiecesStr, fmt.Sprintf("%s=%s", envKey, envValueStr))
}
starlarkConfig, err := add.GetServiceConfigStarlark(
serviceConfig.Image,
strings.Join(portPiecesStr, ","),
serviceConfig.Command,
serviceConfig.Entrypoint,
strings.Join(envvarsPiecesStr, ","),
nonSupportedField,
emptyPrivateIpPlaceholder)
if err != nil {
return "", stacktrace.Propagate(err, "Error getting service config starlark for '%v'", serviceConfig)
}
Expand All @@ -158,7 +176,7 @@ func runStarlark(ctx context.Context, enclaveName string, starlarkScript string)
if err != nil {
return stacktrace.Propagate(err, "An error occurred creating an enclave '%v'", enclaveName)
}
defer output_printers.PrintEnclaveName(enclaveName)
defer output_printers.PrintEnclaveName(enclaveCtx.GetEnclaveName())

starlarkRunResult, err := enclaveCtx.RunStarlarkScriptBlocking(ctx, defaultMainFunction, starlarkScript, noStarlarkParams, false, 1, []kurtosis_core_rpc_api_bindings.KurtosisFeatureFlag{})
if err != nil {
Expand Down
1 change: 1 addition & 0 deletions cli/cli/go.mod
Expand Up @@ -93,6 +93,7 @@ require (
github.com/imdario/mergo v0.3.16 // indirect
github.com/improbable-eng/grpc-web v0.15.0 // indirect
github.com/inconshreveable/mousetrap v1.1.0 // indirect
github.com/joho/godotenv v1.5.1 // indirect
github.com/josharian/intern v1.0.0 // indirect
github.com/json-iterator/go v1.1.12 // indirect
github.com/klauspost/compress v1.16.7 // indirect
Expand Down
5 changes: 5 additions & 0 deletions cli/cli/go.sum
Expand Up @@ -261,6 +261,7 @@ github.com/google/uuid v1.3.0 h1:t6JiXgmwXMjEs8VusXIJk2BXHsn+wx8BZdTaoZ5fu7I=
github.com/google/uuid v1.3.0/go.mod h1:TIyPZe4MgqvfeYDBFedMoGGpEw/LqOeaOT+nhxU+yHo=
github.com/googleapis/gax-go v2.0.0+incompatible/go.mod h1:SFVmujtThgffbyetf+mdk2eWhX2bMyUtNHzFKcPA9HY=
github.com/googleapis/gax-go/v2 v2.0.3/go.mod h1:LLvjysVCY1JZeum8Z6l8qUty8fiNwE08qbEPm1M08qg=
github.com/googleapis/gax-go/v2 v2.0.5/go.mod h1:DWXyrwAJ9X0FpwwEdw+IPEYBICEFu5mhpdKc/us6bOk=
github.com/gopherjs/gopherjs v0.0.0-20181017120253-0766667cb4d1/go.mod h1:wJfORRmW1u3UXTncJ5qlYoELFm8eSnnEO6hX4iZ3EWY=
github.com/gorilla/context v1.1.1/go.mod h1:kBGZzfjB9CEq2AlWe17Uuf7NDRt0dE0s8S51q0aT7Yg=
github.com/gorilla/mux v1.6.2/go.mod h1:1lud6UwP+6orDFRuTfBEV8e9/aOM/c4fVVCaMa2zaAs=
Expand Down Expand Up @@ -310,6 +311,8 @@ github.com/influxdata/influxdb1-client v0.0.0-20191209144304-8bf82d3c094d/go.mod
github.com/jarcoal/httpmock v1.0.4/go.mod h1:ATjnClrvW/3tijVmpL/va5Z3aAyGvqU3gCT8nX0Txik=
github.com/jellevandenhooff/dkim v0.0.0-20150330215556-f50fe3d243e1/go.mod h1:E0B/fFc00Y+Rasa88328GlI/XbtyysCtTHZS8h7IrBU=
github.com/jmespath/go-jmespath v0.0.0-20180206201540-c2b33e8439af/go.mod h1:Nht3zPeWKUH0NzdCt2Blrr5ys8VGpn0CEB0cQHVjt7k=
github.com/joho/godotenv v1.5.1 h1:7eLL/+HRGLY0ldzfGMeQkb7vMd0as4CfYvUVzLqw0N0=
github.com/joho/godotenv v1.5.1/go.mod h1:f4LDr5Voq0i2e/R5DDNOoa2zzDfwtkZa6DnEwAbqwq4=
github.com/jonboulle/clockwork v0.1.0/go.mod h1:Ii8DK3G1RaLaWxj9trq07+26W01tbo22gdxWY5EU2bo=
github.com/josharian/intern v1.0.0 h1:vlS4z54oSdjm0bgjRigI+G1HpF+tI+9rE5LLzOg8HmY=
github.com/josharian/intern v1.0.0/go.mod h1:5DoeVV0s6jJacbCEi61lwdGj/aVlrQvzHFFd8Hwg//Y=
Expand Down Expand Up @@ -644,6 +647,7 @@ go.opencensus.io v0.18.0/go.mod h1:vKdFvxhtzZ9onBp9VKHK8z/sRpBMnKAsufL7wlDrCOA=
go.opencensus.io v0.20.1/go.mod h1:6WKK9ahsWS3RSO+PY9ZHZUfv2irvY6gN279GOPZjmmk=
go.opencensus.io v0.20.2/go.mod h1:6WKK9ahsWS3RSO+PY9ZHZUfv2irvY6gN279GOPZjmmk=
go.opencensus.io v0.22.2/go.mod h1:yxeiOL68Rb0Xd1ddK5vPZ/oVn4vY4Ynel7k9FzqtOIw=
go.opencensus.io v0.22.4/go.mod h1:yxeiOL68Rb0Xd1ddK5vPZ/oVn4vY4Ynel7k9FzqtOIw=
go.starlark.net v0.0.0-20210223155950-e043a3d3c984/go.mod h1:t3mmBBPzAVvK0L0n1drDmrQsJ8FoIx4INCqVMTr/Zo0=
go.uber.org/atomic v1.3.2/go.mod h1:gD2HeocX3+yG+ygLZcrzQJaqmWj9AIm7n08wl/qW/PE=
go.uber.org/atomic v1.4.0/go.mod h1:gD2HeocX3+yG+ygLZcrzQJaqmWj9AIm7n08wl/qW/PE=
Expand Down Expand Up @@ -841,6 +845,7 @@ google.golang.org/api v0.0.0-20180910000450-7ca32eb868bf/go.mod h1:4mhQ8q/RsB7i+
google.golang.org/api v0.0.0-20181030000543-1d582fd0359e/go.mod h1:4mhQ8q/RsB7i+udVvVy5NUi08OU8ZlA0gRVgrF7VFY0=
google.golang.org/api v0.1.0/go.mod h1:UGEZY7KEX120AnNLIHFMKIo4obdJhkp2tPbaPlQx13Y=
google.golang.org/api v0.3.1/go.mod h1:6wY9I6uQWHQ8EM57III9mq/AjF+i8G65rmVagqKMtkk=
google.golang.org/api v0.30.0/go.mod h1:QGmEvQ87FHZNiUVJkT14jQNYJ4ZJjdRF23ZXz5138Fc=
google.golang.org/appengine v1.1.0/go.mod h1:EbEs0AVv82hx2wNQdGPgUI5lhzA/G0D9YwlJXL52JkM=
google.golang.org/appengine v1.2.0/go.mod h1:xpcJRLb0r/rnEns0DIKYYv+WjYCduHsrkT7/EB5XEv4=
google.golang.org/appengine v1.3.0/go.mod h1:xpcJRLb0r/rnEns0DIKYYv+WjYCduHsrkT7/EB5XEv4=
Expand Down
5 changes: 5 additions & 0 deletions internal_testsuites/resources/docker_import_cli_test/.env
@@ -0,0 +1,5 @@
POSTGRES_USER=yourUser
POSTGRES_PW=changeit
POSTGRES_DB=postgres
PGADMIN_MAIL=your@email.com
PGADMIN_PW=changeit
@@ -1,6 +1,21 @@
services:
nginx-a:
image: nginx:latest
container_name: nginx
postgres:
container_name: postgres
image: postgres:latest
environment:
- POSTGRES_USER=${POSTGRES_USER}
- POSTGRES_PASSWORD=${POSTGRES_PW}
- POSTGRES_DB=${POSTGRES_DB} #optional (specify default database instead of $POSTGRES_DB)
ports:
- 123:80
- "5432:5432"
restart: always

pgadmin:
container_name: pgadmin
image: dpage/pgadmin4:latest
environment:
- PGADMIN_DEFAULT_EMAIL=${PGADMIN_MAIL}
- PGADMIN_DEFAULT_PASSWORD=${PGADMIN_PW}
ports:
- "5050:80"
restart: always

0 comments on commit 24e71d1

Please sign in to comment.