From 754b6c3e773437ee59b7eadfdcd850c8610ef9eb Mon Sep 17 00:00:00 2001 From: Keming Date: Wed, 4 Jan 2023 09:48:11 +0800 Subject: [PATCH] fix: change to dynamic PATH (#1375) * change to dynamic path Signed-off-by: Keming * rm debug log Signed-off-by: Keming * fix lint Signed-off-by: Keming * Update pkg/lang/ir/v1/julia.go Co-authored-by: Ce Gao Signed-off-by: Keming * address commment Signed-off-by: Keming Signed-off-by: Keming Co-authored-by: Ce Gao --- pkg/lang/ir/types.go | 1 + pkg/lang/ir/v0/compile.go | 8 ++++---- pkg/lang/ir/v0/conda.go | 9 +++++---- pkg/lang/ir/v0/julia.go | 5 ++++- pkg/lang/ir/v0/system.go | 20 +++++++++++--------- pkg/lang/ir/v0/types.go | 1 - pkg/lang/ir/v0/user.go | 3 +++ pkg/lang/ir/v1/compile.go | 5 ++--- pkg/lang/ir/v1/conda.go | 2 ++ pkg/lang/ir/v1/julia.go | 7 +++++-- pkg/lang/ir/v1/system.go | 13 +++++++++---- pkg/lang/ir/v1/types.go | 1 - pkg/lang/ir/v1/user.go | 3 +++ pkg/types/envd.go | 18 +++++------------- 14 files changed, 54 insertions(+), 42 deletions(-) diff --git a/pkg/lang/ir/types.go b/pkg/lang/ir/types.go index 4dcbff62b..aca58219e 100644 --- a/pkg/lang/ir/types.go +++ b/pkg/lang/ir/types.go @@ -24,6 +24,7 @@ type RuntimeGraph struct { RuntimeDaemon [][]string RuntimeInitScript [][]string RuntimeEnviron map[string]string + RuntimeEnvPaths []string RuntimeExpose []ExposeItem } diff --git a/pkg/lang/ir/v0/compile.go b/pkg/lang/ir/v0/compile.go index d7d956507..fdea4af3f 100644 --- a/pkg/lang/ir/v0/compile.go +++ b/pkg/lang/ir/v0/compile.go @@ -41,6 +41,7 @@ func NewGraph() ir.Graph { runtimeGraph := ir.RuntimeGraph{ RuntimeCommands: make(map[string]string), RuntimeEnviron: make(map[string]string), + RuntimeEnvPaths: []string{types.DefaultSystemPath}, } langVersion := languageVersionDefault conda := &ir.CondaConfig{} @@ -60,7 +61,6 @@ func NewGraph() ir.Graph { SystemPackages: []string{}, Exec: []ir.RunBuildCommand{}, UserDirectories: []string{}, - RuntimeEnvPaths: []string{types.DefaultPathEnv()}, Shell: shellBASH, CondaConfig: conda, RuntimeGraph: runtimeGraph, @@ -240,6 +240,7 @@ func (g generalGraph) EnvString() []string { for k, v := range g.RuntimeEnviron { envs = append(envs, fmt.Sprintf("%s=%s", k, v)) } + envs = append(envs, fmt.Sprintf("PATH=%s", strings.Join(g.RuntimeEnvPaths, ":"))) return envs } @@ -247,10 +248,9 @@ func (g generalGraph) GetEnviron() []string { if g.Image != nil { return g.EnvString() } - // Add PATH and LC_ALL. return append(g.EnvString(), - "PATH="+strings.Join(g.RuntimeEnvPaths, ":"), "LC_ALL=en_US.UTF-8", + "LANG=en_US.UTF-8", ) } @@ -287,7 +287,7 @@ func (g *generalGraph) GetEntrypoint(buildContextDir string) ([]string, error) { return []string{"horust"}, nil } -func (g generalGraph) CompileLLB(uid, gid int) (llb.State, error) { +func (g *generalGraph) CompileLLB(uid, gid int) (llb.State, error) { g.uid = uid // TODO(gaocegege): Remove the hack for https://github.com/tensorchord/envd/issues/370 diff --git a/pkg/lang/ir/v0/conda.go b/pkg/lang/ir/v0/conda.go index e967da91d..47352b0c7 100644 --- a/pkg/lang/ir/v0/conda.go +++ b/pkg/lang/ir/v0/conda.go @@ -25,6 +25,7 @@ import ( "github.com/sirupsen/logrus" "github.com/tensorchord/envd/pkg/flag" + "github.com/tensorchord/envd/pkg/types" ) const ( @@ -139,8 +140,8 @@ func (g generalGraph) compileCondaEnvironment(root llb.State) (llb.State, error) return run.Root(), nil } -// nolint:unparam -func (g *generalGraph) installConda(root llb.State) (llb.State, error) { +func (g *generalGraph) installConda(root llb.State) llb.State { + root = g.updateEnvPath(root, types.DefaultCondaPath) // this directory is related to conda envd env meta (used by `conda env config vars set key=value`) g.UserDirectories = append(g.UserDirectories, fmt.Sprintf("%s/envs/envd/conda-meta", condaRootPrefix)) if g.CondaConfig.UseMicroMamba { @@ -149,12 +150,12 @@ func (g *generalGraph) installConda(root llb.State) (llb.State, error) { AddEnv("MAMBA_VERSION", mambaVersionDefault). Run(llb.Shlexf("bash -c '%s'", installMambaBash), llb.WithCustomName("[internal] install micro mamba")) - return run.Root(), nil + return run.Root() } run := root.AddEnv("CONDA_VERSION", condaVersionDefault). File(llb.Mkdir(condaRootPrefix, 0755, llb.WithParents(true)), llb.WithCustomName("[internal] create conda directory")). Run(llb.Shlexf("bash -c '%s'", installCondaBash), llb.WithCustomName("[internal] install conda")) - return run.Root(), nil + return run.Root() } diff --git a/pkg/lang/ir/v0/julia.go b/pkg/lang/ir/v0/julia.go index fdb61b9eb..995d42ec7 100644 --- a/pkg/lang/ir/v0/julia.go +++ b/pkg/lang/ir/v0/julia.go @@ -21,9 +21,12 @@ import ( "github.com/cockroachdb/errors" "github.com/moby/buildkit/client/llb" "github.com/sirupsen/logrus" + + "github.com/tensorchord/envd/pkg/types" ) -func (g generalGraph) compileJulia(baseStage llb.State) (llb.State, error) { +func (g *generalGraph) compileJulia(baseStage llb.State) (llb.State, error) { + baseStage = g.updateEnvPath(baseStage, types.DefaultJuliaPath) if err := g.compileJupyter(); err != nil { return llb.State{}, errors.Wrap(err, "failed to compile jupyter") } diff --git a/pkg/lang/ir/v0/system.go b/pkg/lang/ir/v0/system.go index cfe05528a..98df4da62 100644 --- a/pkg/lang/ir/v0/system.go +++ b/pkg/lang/ir/v0/system.go @@ -54,7 +54,6 @@ func (g generalGraph) compileRun(root llb.State) llb.State { } workingDir := g.getWorkingDir() - stage := root.AddEnv("PATH", types.DefaultPathEnvUnix) for _, execGroup := range g.Exec { var sb strings.Builder sb.WriteString("set -euo pipefail\n") @@ -68,13 +67,13 @@ func (g generalGraph) compileRun(root llb.State) llb.State { // TODO(gaocegege): Maybe we should make it readonly, // but these cases then cannot be supported: // run(commands=["git clone xx.git"]) - run := stage.Dir(workingDir).Run(llb.Shlex(cmdStr)) + run := root.Dir(workingDir).Run(llb.Shlex(cmdStr)) if execGroup.MountHost { run.AddMount(workingDir, llb.Local(flag.FlagBuildContext)) } - stage = run.Root() + root = run.Root() } - return stage + return root } func (g generalGraph) compileCopy(root llb.State) llb.State { @@ -215,10 +214,7 @@ func (g *generalGraph) compileBase() (llb.State, error) { } // Install conda first. - condaStage, err := g.installConda(base) - if err != nil { - return llb.State{}, errors.Wrap(err, "failed to install conda") - } + condaStage := g.installConda(base) supervisor := g.installHorust(condaStage) sshdStage := g.compileSSHD(supervisor) source, err := g.compileExtraSource(sshdStage) @@ -247,7 +243,8 @@ func (g *generalGraph) customBase() (llb.State, error) { // Set the environment variables to RuntimeEnviron to keep it in the resulting image. for _, e := range envs { - kv := strings.Split(e, "=") + // in case the env value also contains `=` + kv := strings.SplitN(e, "=", 2) g.RuntimeEnviron[kv[0]] = kv[1] } return base, nil @@ -287,3 +284,8 @@ func (g generalGraph) compileMountDir(root llb.State) llb.State { } return mount } + +func (g *generalGraph) updateEnvPath(root llb.State, path string) llb.State { + g.RuntimeEnvPaths = append(g.RuntimeEnvPaths, path) + return root.AddEnv("PATH", strings.Join(g.RuntimeEnvPaths, ":")) +} diff --git a/pkg/lang/ir/v0/types.go b/pkg/lang/ir/v0/types.go index eee98fd5d..8b7ce6cfc 100644 --- a/pkg/lang/ir/v0/types.go +++ b/pkg/lang/ir/v0/types.go @@ -54,7 +54,6 @@ type generalGraph struct { VSCodePlugins []vscode.Plugin UserDirectories []string - RuntimeEnvPaths []string Exec []ir.RunBuildCommand Copy []ir.CopyInfo diff --git a/pkg/lang/ir/v0/user.go b/pkg/lang/ir/v0/user.go index 5ef993ddf..4b18451ae 100644 --- a/pkg/lang/ir/v0/user.go +++ b/pkg/lang/ir/v0/user.go @@ -15,6 +15,8 @@ package v0 import ( + "strings" + "github.com/moby/buildkit/client/llb" "github.com/tensorchord/envd/pkg/types" @@ -36,6 +38,7 @@ func (g *generalGraph) compileUserOwn(root llb.State) llb.State { for _, env := range types.BaseEnvironment { user = user.AddEnv(env.Name, env.Value) } + user.AddEnv("PATH", strings.Join(g.RuntimeEnvPaths, ":")) return user } diff --git a/pkg/lang/ir/v1/compile.go b/pkg/lang/ir/v1/compile.go index ef2fcee50..2c903a721 100644 --- a/pkg/lang/ir/v1/compile.go +++ b/pkg/lang/ir/v1/compile.go @@ -41,6 +41,7 @@ func NewGraph() ir.Graph { runtimeGraph := ir.RuntimeGraph{ RuntimeCommands: make(map[string]string), RuntimeEnviron: make(map[string]string), + RuntimeEnvPaths: []string{types.DefaultSystemPath}, } return &generalGraph{ uid: -1, @@ -57,7 +58,6 @@ func NewGraph() ir.Graph { SystemPackages: []string{}, Exec: []ir.RunBuildCommand{}, UserDirectories: []string{}, - RuntimeEnvPaths: []string{types.DefaultPathEnv()}, Shell: shellBASH, RuntimeGraph: runtimeGraph, } @@ -131,9 +131,7 @@ func (g *generalGraph) Compile(ctx context.Context, envName string, pub string) } func (g generalGraph) GetEnviron() []string { - // Add PATH and LC_ALL. return append(g.EnvString(), - "PATH="+strings.Join(g.RuntimeEnvPaths, ":"), "LC_ALL=en_US.UTF-8", "LANG=C.UTF-8", ) @@ -253,6 +251,7 @@ func (g generalGraph) EnvString() []string { for k, v := range g.RuntimeEnviron { envs = append(envs, fmt.Sprintf("%s=%s", k, v)) } + envs = append(envs, fmt.Sprintf("PATH=%s", strings.Join(g.RuntimeEnvPaths, ":"))) return envs } diff --git a/pkg/lang/ir/v1/conda.go b/pkg/lang/ir/v1/conda.go index 1df9d57f6..0a1027d50 100644 --- a/pkg/lang/ir/v1/conda.go +++ b/pkg/lang/ir/v1/conda.go @@ -25,6 +25,7 @@ import ( "github.com/sirupsen/logrus" "github.com/tensorchord/envd/pkg/flag" + "github.com/tensorchord/envd/pkg/types" ) const ( @@ -152,6 +153,7 @@ func (g generalGraph) compileCondaEnvironment(root llb.State) (llb.State, error) } func (g *generalGraph) installConda(root llb.State) llb.State { + root = g.updateEnvPath(root, types.DefaultCondaPath) if g.Dev { // We only create envd user for dev env. // This directory is related to conda envd env meta (used by `conda env config vars set key=value`) diff --git a/pkg/lang/ir/v1/julia.go b/pkg/lang/ir/v1/julia.go index 762231844..5aa51c786 100644 --- a/pkg/lang/ir/v1/julia.go +++ b/pkg/lang/ir/v1/julia.go @@ -21,10 +21,13 @@ import ( "github.com/cockroachdb/errors" "github.com/moby/buildkit/client/llb" "github.com/sirupsen/logrus" + + "github.com/tensorchord/envd/pkg/types" ) -func (g generalGraph) installJulia(root llb.State) (llb.State, error) { - return llb.State{}, errors.New("not implemented") +func (g *generalGraph) installJulia(root llb.State) (llb.State, error) { + root = g.updateEnvPath(root, types.DefaultJuliaPath) + return root, errors.New("not implemented") } func (g generalGraph) installJuliaPackages(root llb.State) llb.State { diff --git a/pkg/lang/ir/v1/system.go b/pkg/lang/ir/v1/system.go index 6d0ea4fd5..70e657d94 100644 --- a/pkg/lang/ir/v1/system.go +++ b/pkg/lang/ir/v1/system.go @@ -143,7 +143,6 @@ func (g generalGraph) compileRun(root llb.State) llb.State { } workingDir := g.getWorkingDir() - stage := root.AddEnv("PATH", types.DefaultPathEnvUnix) for _, execGroup := range g.Exec { var sb strings.Builder sb.WriteString("set -euo pipefail\n") @@ -157,13 +156,13 @@ func (g generalGraph) compileRun(root llb.State) llb.State { // TODO(gaocegege): Maybe we should make it readonly, // but these cases then cannot be supported: // run(commands=["git clone xx.git"]) - run := stage.Dir(workingDir).Run(llb.Shlex(cmdStr)) + run := root.Dir(workingDir).Run(llb.Shlex(cmdStr)) if execGroup.MountHost { run.AddMount(workingDir, llb.Local(flag.FlagBuildContext)) } - stage = run.Root() + root = run.Root() } - return stage + return root } func (g generalGraph) compileCopy(root llb.State) llb.State { @@ -324,6 +323,7 @@ func (g *generalGraph) compileBaseImage() (llb.State, error) { // Set the environment variables to RuntimeEnviron to keep it in the resulting image. for _, e := range envs { + // in case the env value also contains `=` kv := strings.SplitN(e, "=", 2) g.RuntimeEnviron[kv[0]] = kv[1] } @@ -366,3 +366,8 @@ func (g generalGraph) compileMountDir(root llb.State) llb.State { } return mount } + +func (g *generalGraph) updateEnvPath(root llb.State, path string) llb.State { + g.RuntimeEnvPaths = append(g.RuntimeEnvPaths, path) + return root.AddEnv("PATH", strings.Join(g.RuntimeEnvPaths, ":")) +} diff --git a/pkg/lang/ir/v1/types.go b/pkg/lang/ir/v1/types.go index aa1919bbb..3bd89137e 100644 --- a/pkg/lang/ir/v1/types.go +++ b/pkg/lang/ir/v1/types.go @@ -55,7 +55,6 @@ type generalGraph struct { VSCodePlugins []vscode.Plugin UserDirectories []string - RuntimeEnvPaths []string Exec []ir.RunBuildCommand Copy []ir.CopyInfo diff --git a/pkg/lang/ir/v1/user.go b/pkg/lang/ir/v1/user.go index 413fb8d1c..680b79e09 100644 --- a/pkg/lang/ir/v1/user.go +++ b/pkg/lang/ir/v1/user.go @@ -15,6 +15,8 @@ package v1 import ( + "strings" + "github.com/moby/buildkit/client/llb" "github.com/tensorchord/envd/pkg/types" @@ -37,6 +39,7 @@ func (g *generalGraph) compileUserOwn(root llb.State) llb.State { for _, env := range types.BaseEnvironment { user = user.AddEnv(env.Name, env.Value) } + user = user.AddEnv("PATH", strings.Join(g.RuntimeEnvPaths, ":")) return user } diff --git a/pkg/types/envd.go b/pkg/types/envd.go index 4e0efcd09..6c73ecae0 100644 --- a/pkg/types/envd.go +++ b/pkg/types/envd.go @@ -28,14 +28,10 @@ import ( ) const ( - // DefaultPathEnvUnix is unix style list of directories to search for - // executables. Each directory is separated from the next by a colon - // ':' character . - DefaultPathEnvUnix = "/opt/conda/envs/envd/bin:/opt/conda/bin:/home/envd/.local/bin:/usr/local/julia/bin:" + system.DefaultPathEnvUnix - // DefaultPathEnvWindows is windows style list of directories to search for - // executables. Each directory is separated from the next by a colon - // ';' character . - DefaultPathEnvWindows = system.DefaultPathEnvWindows + // environment PATH + DefaultSystemPath = system.DefaultPathEnvUnix + DefaultCondaPath = "/opt/conda/envs/envd/bin:/opt/conda/bin:/home/envd/.local/bin" + DefaultJuliaPath = "/usr/local/julia/bin" // image PythonBaseImage = "ubuntu:20.04" // supervisor @@ -55,7 +51,7 @@ var BaseEnvironment = []struct { Value string }{ {"DEBIAN_FRONTEND", "noninteractive"}, - {"PATH", DefaultPathEnvUnix}, + {"PATH", DefaultSystemPath}, {"LANG", "en_US.UTF-8"}, {"LC_ALL", "en_US.UTF-8"}, } @@ -170,10 +166,6 @@ type AuthConfig struct { JWTToken string `json:"jwt_token,omitempty"` } -func DefaultPathEnv() string { - return DefaultPathEnvUnix -} - func NewImageFromSummary(image types.ImageSummary) (*EnvdImage, error) { img := EnvdImage{ ImageMeta: servertypes.ImageMeta{