From b9b08ff930c7f52b8c11745bdb62963f2887adb4 Mon Sep 17 00:00:00 2001 From: Egor Sklyarov Date: Wed, 12 Jul 2023 17:37:03 +0400 Subject: [PATCH 1/2] Set DSTACK_RUNNER_VERSION default to latest --- cli/dstack/_internal/backend/base/compute.py | 2 +- cli/dstack/version.py | 1 - 2 files changed, 1 insertion(+), 2 deletions(-) diff --git a/cli/dstack/_internal/backend/base/compute.py b/cli/dstack/_internal/backend/base/compute.py index b3574769e..c15de1252 100644 --- a/cli/dstack/_internal/backend/base/compute.py +++ b/cli/dstack/_internal/backend/base/compute.py @@ -129,7 +129,7 @@ def get_dstack_runner() -> str: build = version.__version__ else: # stgn bucket = "dstack-runner-downloads-stgn" - build = version.__version__ or os.environ.get("DSTACK_RUNNER_BUILD", None) + build = version.__version__ or os.environ.get("DSTACK_RUNNER_VERSION", "latest") commands = [ f'sudo curl --output /usr/local/bin/dstack-runner "https://{bucket}.s3.eu-west-1.amazonaws.com/{build}/binaries/dstack-runner-linux-amd64"', diff --git a/cli/dstack/version.py b/cli/dstack/version.py index c0ea4762c..8a621fa00 100644 --- a/cli/dstack/version.py +++ b/cli/dstack/version.py @@ -1,4 +1,3 @@ __version__ = None __is_release__ = False miniforge_image = "0.3" -runner_build = "latest" From c9e31dee2adfb5e1ea099d40c66fd466221cb692 Mon Sep 17 00:00:00 2001 From: Egor Sklyarov Date: Wed, 12 Jul 2023 18:01:55 +0400 Subject: [PATCH 2/2] Pass envs as export commands to allow interpolation --- runner/internal/container/engine.go | 12 ++++++++++-- runner/internal/executor/executor.go | 3 +-- 2 files changed, 11 insertions(+), 4 deletions(-) diff --git a/runner/internal/container/engine.go b/runner/internal/container/engine.go index f1231dd94..9febe9a97 100644 --- a/runner/internal/container/engine.go +++ b/runner/internal/container/engine.go @@ -312,7 +312,6 @@ func (r *Engine) NewBuildSpec(ctx context.Context, job *models.Job, spec *Spec, commands := append([]string{}, job.BuildCommands...) commands = append(commands, job.OptionalBuildCommands...) env := environment.New() - env.AddMapString(job.Environment) env.AddMapString(secrets) buildSpec := &BuildSpec{ @@ -321,7 +320,7 @@ func (r *Engine) NewBuildSpec(ctx context.Context, job *models.Job, spec *Spec, WorkDir: spec.WorkDir, ConfigurationPath: job.ConfigurationPath, ConfigurationType: job.ConfigurationType, - Commands: ShellCommands(commands), + Commands: ShellCommands(InsertEnvs(commands, job.Environment)), Entrypoint: spec.Entrypoint, Env: env.ToSlice(), RegistryAuthBase64: spec.RegistryAuthBase64, @@ -453,6 +452,15 @@ func ShellCommands(commands []string) []string { return []string{sb.String()} } +// InsertEnvs allows interpolation of env variables (e.g. PATH=/foo/bar:$PATH) +func InsertEnvs(commands []string, envs map[string]string) []string { + output := make([]string, 0) + for name, val := range envs { + output = append(output, fmt.Sprintf("export %s=%s", name, val)) + } + return append(output, commands...) +} + func BytesToMiB(bytesCount int64) uint64 { var mib int64 = 1024 * 1024 return uint64(bytesCount / mib) diff --git a/runner/internal/executor/executor.go b/runner/internal/executor/executor.go index b59795726..60af47596 100644 --- a/runner/internal/executor/executor.go +++ b/runner/internal/executor/executor.go @@ -515,7 +515,6 @@ func (ex *Executor) environment(ctx context.Context, includeRun bool) []string { env.AddMapString(job.RunEnvironment) env.AddMapString(cons) } - env.AddMapString(job.Environment) secrets, err := ex.backend.Secrets(ctx) if err != nil { log.Error(ctx, "Fail fetching secrets", "err", err) @@ -629,7 +628,7 @@ func (ex *Executor) newSpec(ctx context.Context, credPath string) (*container.Sp Image: job.Image, RegistryAuthBase64: makeRegistryAuthBase64(username, password), WorkDir: path.Join("/workflow", job.WorkingDir), - Commands: container.ShellCommands(job.Commands), + Commands: container.ShellCommands(container.InsertEnvs(job.Commands, job.Environment)), Entrypoint: job.Entrypoint, Env: ex.environment(ctx, true), Mounts: uniqueMount(bindings),