From 45d6b9fa45bf4ec952fbeb13a0f09d72ec8eeedf Mon Sep 17 00:00:00 2001 From: Dmitry Meyer Date: Sat, 30 Aug 2025 06:32:27 +0000 Subject: [PATCH 1/4] Add `repos[].path` property - If not set, the legacy `/workflow` path is used (only during the transitional period) - Relative paths are resolved relative to `working_dir` - `~[/path]` is supported, but `~username[/path]` is not, as with files - Available inside the container as `DSTACK_REPO_RER` env variable `working_dir` - Must be absolute - If not set, the image default value is used for tasks and services without `commands` and the legacy `/workflow` path for other configurations (replicating pre-0.19.27 `JobConfigurator` logic in CLI only during the transitional period) - `~[/path]` is supported, but `~username[/path]` is not, as with files runner - `/workflow/.venv` moved to `/dstack/venv` - `/tmp/dstack_profile` moved to `/dstack/profile` - `--working-dir` is deprecated and ignored Closes: https://github.com/dstackai/dstack/issues/2851 --- .../reference/dstack.yml/dev-environment.md | 22 +++ docs/docs/reference/dstack.yml/service.md | 22 +++ docs/docs/reference/dstack.yml/task.md | 22 +++ examples/misc/airflow/dags/dstack_tasks.py | 2 +- runner/cmd/runner/cmd.go | 24 ++-- runner/cmd/runner/main.go | 4 +- runner/cmd/shim/main.go | 4 +- runner/consts/consts.go | 5 + runner/internal/common/utils.go | 54 +++++++ runner/internal/common/utils_test.go | 109 +++++++++++++++ runner/internal/executor/exec.go | 19 --- runner/internal/executor/exec_test.go | 27 ---- runner/internal/executor/executor.go | 85 ++++++++--- runner/internal/executor/executor_test.go | 39 ++++-- runner/internal/executor/files.go | 75 +++------- runner/internal/executor/repo.go | 34 +++-- runner/internal/runner/api/server.go | 4 +- runner/internal/schemas/schemas.go | 1 + runner/internal/shim/runner.go | 1 - .../cli/services/configurators/run.py | 40 +++++- .../_internal/core/backends/base/compute.py | 5 +- .../_internal/core/compatibility/runs.py | 7 +- .../_internal/core/models/configurations.py | 28 +++- src/dstack/_internal/core/models/files.py | 2 +- src/dstack/_internal/core/models/runs.py | 23 ++- src/dstack/_internal/server/schemas/runner.py | 1 + .../services/jobs/configurators/base.py | 35 ++++- .../server/services/jobs/configurators/dev.py | 8 +- .../jobs/configurators/extensions/cursor.py | 8 +- .../jobs/configurators/extensions/vscode.py | 10 +- .../services/jobs/configurators/service.py | 3 - .../services/jobs/configurators/task.py | 3 - src/dstack/_internal/server/testing/common.py | 2 +- src/dstack/_internal/utils/path.py | 9 +- src/dstack/api/_public/runs.py | 15 +- .../core/models/test_configurations.py | 30 ++-- .../_internal/server/routers/test_runs.py | 132 ++++++++++-------- .../services/jobs/configurators/test_task.py | 6 +- 38 files changed, 638 insertions(+), 282 deletions(-) create mode 100644 runner/internal/common/utils.go create mode 100644 runner/internal/common/utils_test.go delete mode 100644 runner/internal/executor/exec.go delete mode 100644 runner/internal/executor/exec_test.go diff --git a/docs/docs/reference/dstack.yml/dev-environment.md b/docs/docs/reference/dstack.yml/dev-environment.md index 1c4926930..364297e43 100644 --- a/docs/docs/reference/dstack.yml/dev-environment.md +++ b/docs/docs/reference/dstack.yml/dev-environment.md @@ -100,6 +100,28 @@ The `dev-environment` configuration type allows running [dev environments](../.. * `volume-name:/container/path` for network volumes * `/instance/path:/container/path` for instance volumes +### `repos[n]` { #_repos data-toc-label="repos" } + +> Currently, a maximum of one repo is supported. + +> Either `local_path` or `url` must be specified. + +#SCHEMA# dstack._internal.core.models.configurations.RepoSpec + overrides: + show_root_heading: false + type: + required: true + +??? info "Short syntax" + + The short syntax for repos is a colon-separated string in the form of `local_path_or_url:path`. + + * `.:/repo` + * `..:repo` + * `~/repos/demo:~/repo` + * `https://github.com/org/repo:~/data/repo` + * `git@github.com:org/repo.git:data/repo` + ### `files[n]` { #_files data-toc-label="files" } #SCHEMA# dstack._internal.core.models.files.FilePathMapping diff --git a/docs/docs/reference/dstack.yml/service.md b/docs/docs/reference/dstack.yml/service.md index 85612d862..8d89b2d57 100644 --- a/docs/docs/reference/dstack.yml/service.md +++ b/docs/docs/reference/dstack.yml/service.md @@ -215,6 +215,28 @@ The `service` configuration type allows running [services](../../concepts/servic * `volume-name:/container/path` for network volumes * `/instance/path:/container/path` for instance volumes +### `repos[n]` { #_repos data-toc-label="repos" } + +> Currently, a maximum of one repo is supported. + +> Either `local_path` or `url` must be specified. + +#SCHEMA# dstack._internal.core.models.configurations.RepoSpec + overrides: + show_root_heading: false + type: + required: true + +??? info "Short syntax" + + The short syntax for repos is a colon-separated string in the form of `local_path_or_url:path`. + + * `.:/repo` + * `..:repo` + * `~/repos/demo:~/repo` + * `https://github.com/org/repo:~/data/repo` + * `git@github.com:org/repo.git:data/repo` + ### `files[n]` { #_files data-toc-label="files" } #SCHEMA# dstack._internal.core.models.files.FilePathMapping diff --git a/docs/docs/reference/dstack.yml/task.md b/docs/docs/reference/dstack.yml/task.md index 6f92b1477..356383319 100644 --- a/docs/docs/reference/dstack.yml/task.md +++ b/docs/docs/reference/dstack.yml/task.md @@ -100,6 +100,28 @@ The `task` configuration type allows running [tasks](../../concepts/tasks.md). * `volume-name:/container/path` for network volumes * `/instance/path:/container/path` for instance volumes +### `repos[n]` { #_repos data-toc-label="repos" } + +> Currently, a maximum of one repo is supported. + +> Either `local_path` or `url` must be specified. + +#SCHEMA# dstack._internal.core.models.configurations.RepoSpec + overrides: + show_root_heading: false + type: + required: true + +??? info "Short syntax" + + The short syntax for repos is a colon-separated string in the form of `local_path_or_url:path`. + + * `.:/repo` + * `..:repo` + * `~/repos/demo:~/repo` + * `https://github.com/org/repo:~/data/repo` + * `git@github.com:org/repo.git:data/repo` + ### `files[n]` { #_files data-toc-label="files" } #SCHEMA# dstack._internal.core.models.files.FilePathMapping diff --git a/examples/misc/airflow/dags/dstack_tasks.py b/examples/misc/airflow/dags/dstack_tasks.py index 30741dbcb..8002e123b 100644 --- a/examples/misc/airflow/dags/dstack_tasks.py +++ b/examples/misc/airflow/dags/dstack_tasks.py @@ -47,7 +47,7 @@ def dstack_cli_apply_venv() -> str: dstack is installed into a separate virtual environment available to Airflow. """ return ( - f"source {DSTACK_VENV_PATH}/bin/activate" + f". {DSTACK_VENV_PATH}/bin/activate" f" && cd {DSTACK_REPO_PATH}" " && dstack apply -y -f task.dstack.yml --repo ." ) diff --git a/runner/cmd/runner/cmd.go b/runner/cmd/runner/cmd.go index 8428d456e..d91b5d8fb 100644 --- a/runner/cmd/runner/cmd.go +++ b/runner/cmd/runner/cmd.go @@ -4,6 +4,7 @@ import ( "log" "os" + "github.com/dstackai/dstack/runner/consts" "github.com/urfave/cli/v2" ) @@ -11,7 +12,8 @@ import ( var Version string func App() { - var paths struct{ tempDir, homeDir, workingDir string } + var tempDir string + var homeDir string var httpPort int var sshPort int var logLevel int @@ -37,36 +39,36 @@ func App() { &cli.PathFlag{ Name: "temp-dir", Usage: "Temporary directory for logs and other files", - Required: true, - Destination: &paths.tempDir, + Value: consts.RunnerTempDir, + Destination: &tempDir, }, &cli.PathFlag{ Name: "home-dir", Usage: "HomeDir directory for credentials and $HOME", - Required: true, - Destination: &paths.homeDir, + Value: consts.RunnerHomeDir, + Destination: &homeDir, }, + // TODO: Not used, left for compatibility with old servers. Remove eventually. &cli.PathFlag{ Name: "working-dir", - Usage: "Base path for the job", - Required: true, - Destination: &paths.workingDir, + Hidden: true, + Destination: nil, }, &cli.IntFlag{ Name: "http-port", Usage: "Set a http port", - Value: 10999, + Value: consts.RunnerHTTPPort, Destination: &httpPort, }, &cli.IntFlag{ Name: "ssh-port", Usage: "Set the ssh port", - Required: true, + Value: consts.RunnerSSHPort, Destination: &sshPort, }, }, Action: func(c *cli.Context) error { - err := start(paths.tempDir, paths.homeDir, paths.workingDir, httpPort, sshPort, logLevel, Version) + err := start(tempDir, homeDir, httpPort, sshPort, logLevel, Version) if err != nil { return cli.Exit(err, 1) } diff --git a/runner/cmd/runner/main.go b/runner/cmd/runner/main.go index 173b12a1d..6ed851568 100644 --- a/runner/cmd/runner/main.go +++ b/runner/cmd/runner/main.go @@ -19,7 +19,7 @@ func main() { App() } -func start(tempDir string, homeDir string, workingDir string, httpPort int, sshPort int, logLevel int, version string) error { +func start(tempDir string, homeDir string, httpPort int, sshPort int, logLevel int, version string) error { if err := os.MkdirAll(tempDir, 0o755); err != nil { return tracerr.Errorf("Failed to create temp directory: %w", err) } @@ -38,7 +38,7 @@ func start(tempDir string, homeDir string, workingDir string, httpPort int, sshP log.DefaultEntry.Logger.SetOutput(io.MultiWriter(os.Stdout, defaultLogFile)) log.DefaultEntry.Logger.SetLevel(logrus.Level(logLevel)) - server, err := api.NewServer(tempDir, homeDir, workingDir, fmt.Sprintf(":%d", httpPort), sshPort, version) + server, err := api.NewServer(tempDir, homeDir, fmt.Sprintf(":%d", httpPort), sshPort, version) if err != nil { return tracerr.Errorf("Failed to create server: %w", err) } diff --git a/runner/cmd/shim/main.go b/runner/cmd/shim/main.go index 3ca57a0ab..3be1b1568 100644 --- a/runner/cmd/shim/main.go +++ b/runner/cmd/shim/main.go @@ -79,14 +79,14 @@ func main() { &cli.IntFlag{ Name: "runner-http-port", Usage: "Set runner's http port", - Value: 10999, + Value: consts.RunnerHTTPPort, Destination: &args.Runner.HTTPPort, EnvVars: []string{"DSTACK_RUNNER_HTTP_PORT"}, }, &cli.IntFlag{ Name: "runner-ssh-port", Usage: "Set runner's ssh port", - Value: 10022, + Value: consts.RunnerSSHPort, Destination: &args.Runner.SSHPort, EnvVars: []string{"DSTACK_RUNNER_SSH_PORT"}, }, diff --git a/runner/consts/consts.go b/runner/consts/consts.go index e1d033e1a..d57faed6c 100644 --- a/runner/consts/consts.go +++ b/runner/consts/consts.go @@ -30,4 +30,9 @@ const ( RunnerWorkingDir = "/workflow" ) +const ( + RunnerHTTPPort = 10999 + RunnerSSHPort = 10022 +) + const ShimLogFileName = "shim.log" diff --git a/runner/internal/common/utils.go b/runner/internal/common/utils.go new file mode 100644 index 000000000..05d6efb63 --- /dev/null +++ b/runner/internal/common/utils.go @@ -0,0 +1,54 @@ +package common + +import ( + "context" + "errors" + "os" + "path" + "slices" + + "github.com/dstackai/dstack/runner/internal/log" +) + +func ExpandPath(pth string, base string, home string) (string, error) { + pth = path.Clean(pth) + if pth == "~" { + return path.Clean(home), nil + } + if len(pth) >= 2 && pth[0] == '~' { + if pth[1] == '/' { + return path.Join(home, pth[2:]), nil + } + return "", errors.New("~username syntax is not supported") + } + if base != "" && !path.IsAbs(pth) { + return path.Join(base, pth), nil + } + return pth, nil +} + +func MkdirAll(ctx context.Context, pth string, uid int, gid int) error { + paths := []string{pth} + for { + pth = path.Dir(pth) + if pth == "/" { + break + } + paths = append(paths, pth) + } + for _, p := range slices.Backward(paths) { + if _, err := os.Stat(p); errors.Is(err, os.ErrNotExist) { + if err := os.Mkdir(p, 0o755); err != nil { + return err + } + if uid != -1 || gid != -1 { + if err := os.Chown(p, uid, gid); err != nil { + log.Warning(ctx, "Failed to chown", "path", p, "err", err) + } + } + } else if err != nil { + return err + } + } + return nil +} diff --git a/runner/internal/common/utils_test.go b/runner/internal/common/utils_test.go new file mode 100644 index 000000000..d1c399700 --- /dev/null +++ b/runner/internal/common/utils_test.go @@ -0,0 +1,109 @@ +package common + +import ( + "testing" + + "github.com/stretchr/testify/require" +) + +func TestExpandPath_NoPath_NoBase(t *testing.T) { + path, err := ExpandPath("", "", "") + require.NoError(t, err) + require.Equal(t, ".", path) +} + +func TestExpandPath_NoPath_RelBase(t *testing.T) { + path, err := ExpandPath("", "repo", "") + require.NoError(t, err) + require.Equal(t, "repo", path) +} + +func TestExpandPath_NoPath_AbsBase(t *testing.T) { + path, err := ExpandPath("", "/repo", "") + require.NoError(t, err) + require.Equal(t, "/repo", path) +} + +func TestExpandtPath_RelPath_NoBase(t *testing.T) { + path, err := ExpandPath("repo", "", "") + require.NoError(t, err) + require.Equal(t, "repo", path) +} + +func TestExpandtPath_RelPath_RelBase(t *testing.T) { + path, err := ExpandPath("repo", "data", "") + require.NoError(t, err) + require.Equal(t, "data/repo", path) +} + +func TestExpandtPath_RelPath_AbsBase(t *testing.T) { + path, err := ExpandPath("repo", "/data", "") + require.NoError(t, err) + require.Equal(t, "/data/repo", path) +} + +func TestExpandtPath_AbsPath_NoBase(t *testing.T) { + path, err := ExpandPath("/repo", "", "") + require.NoError(t, err) + require.Equal(t, "/repo", path) +} + +func TestExpandtPath_AbsPath_RelBase(t *testing.T) { + path, err := ExpandPath("/repo", "data", "") + require.NoError(t, err) + require.Equal(t, "/repo", path) +} + +func TestExpandtPath_AbsPath_AbsBase(t *testing.T) { + path, err := ExpandPath("/repo", "/data", "") + require.NoError(t, err) + require.Equal(t, "/repo", path) +} + +func TestExpandPath_BareTilde_NoHome(t *testing.T) { + path, err := ExpandPath("~", "", "") + require.NoError(t, err) + require.Equal(t, ".", path) +} + +func TestExpandPath_BareTilde_RelHome(t *testing.T) { + path, err := ExpandPath("~", "", "user") + require.NoError(t, err) + require.Equal(t, "user", path) +} + +func TestExpandPath_BareTilde_AbsHome(t *testing.T) { + path, err := ExpandPath("~", "", "/home/user") + require.NoError(t, err) + require.Equal(t, "/home/user", path) +} + +func TestExpandtPath_TildeWithPath_NoHome(t *testing.T) { + path, err := ExpandPath("~/repo", "", "") + require.NoError(t, err) + require.Equal(t, "repo", path) +} + +func TestExpandtPath_TildeWithPath_RelHome(t *testing.T) { + path, err := ExpandPath("~/repo", "", "user") + require.NoError(t, err) + require.Equal(t, "user/repo", path) +} + +func TestExpandtPath_TildeWithPath_AbsHome(t *testing.T) { + path, err := ExpandPath("~/repo", "", "/home/user") + require.NoError(t, err) + require.Equal(t, "/home/user/repo", path) +} + +func TestExpandtPath_ErrorTildeUsernameNotSupported_BareTildeUsername(t *testing.T) { + path, err := ExpandPath("~username", "", "") + require.ErrorContains(t, err, "~username syntax is not supported") + require.Equal(t, "", path) +} + +func TestExpandtPath_ErrorTildeUsernameNotSupported_TildeUsernameWithPath(t *testing.T) { + path, err := ExpandPath("~username/repo", "", "") + require.ErrorContains(t, err, "~username syntax is not supported") + require.Equal(t, "", path) +} diff --git a/runner/internal/executor/exec.go b/runner/internal/executor/exec.go deleted file mode 100644 index b6b75eb2c..000000000 --- a/runner/internal/executor/exec.go +++ /dev/null @@ -1,19 +0,0 @@ -package executor - -import ( - "path/filepath" - "strings" - - "github.com/dstackai/dstack/runner/internal/gerrors" -) - -func joinRelPath(rootDir string, path string) (string, error) { - if filepath.IsAbs(path) { - return "", gerrors.New("path must be relative") - } - targetPath := filepath.Join(rootDir, path) - if !strings.HasPrefix(targetPath, rootDir) { - return "", gerrors.New("path is outside of the root directory") - } - return targetPath, nil -} diff --git a/runner/internal/executor/exec_test.go b/runner/internal/executor/exec_test.go deleted file mode 100644 index 841f4e6b1..000000000 --- a/runner/internal/executor/exec_test.go +++ /dev/null @@ -1,27 +0,0 @@ -package executor - -import ( - "testing" - - "github.com/stretchr/testify/assert" -) - -func TestJoinRelPath(t *testing.T) { - base := "/tmp/repo" - var err error - var res string - - res, err = joinRelPath(base, ".") - assert.NoError(t, err) - assert.Equal(t, "/tmp/repo", res) - - _, err = joinRelPath(base, "..") - assert.Error(t, err) - - res, err = joinRelPath(base, "task") - assert.NoError(t, err) - assert.Equal(t, "/tmp/repo/task", res) - - _, err = joinRelPath(base, "/tmp/repo/task") - assert.Error(t, err) -} diff --git a/runner/internal/executor/executor.go b/runner/internal/executor/executor.go index d2ced5dc3..39999156c 100644 --- a/runner/internal/executor/executor.go +++ b/runner/internal/executor/executor.go @@ -10,6 +10,7 @@ import ( "os" "os/exec" osuser "os/user" + "path" "path/filepath" "runtime" "strconv" @@ -21,6 +22,7 @@ import ( "github.com/creack/pty" "github.com/dstackai/ansistrip" "github.com/dstackai/dstack/runner/consts" + "github.com/dstackai/dstack/runner/internal/common" "github.com/dstackai/dstack/runner/internal/connections" "github.com/dstackai/dstack/runner/internal/gerrors" "github.com/dstackai/dstack/runner/internal/log" @@ -50,7 +52,6 @@ type ConnectionTracker interface { type RunExecutor struct { tempDir string homeDir string - workingDir string archiveDir string sshPort int currentUid uint32 @@ -61,7 +62,12 @@ type RunExecutor struct { clusterInfo schemas.ClusterInfo secrets map[string]string repoCredentials *schemas.RepoCredentials + repoDir string codePath string + jobUid int + jobGid int + jobHomeDir string + jobWorkingDir string mu *sync.RWMutex state string @@ -82,7 +88,7 @@ func (s *stubConnectionTracker) GetNoConnectionsSecs() int64 { return 0 } func (s *stubConnectionTracker) Track(ticker <-chan time.Time) {} func (s *stubConnectionTracker) Stop() {} -func NewRunExecutor(tempDir string, homeDir string, workingDir string, sshPort int) (*RunExecutor, error) { +func NewRunExecutor(tempDir string, homeDir string, sshPort int) (*RunExecutor, error) { mu := &sync.RWMutex{} timestamp := NewMonotonicTimestamp() user, err := osuser.Current() @@ -115,10 +121,11 @@ func NewRunExecutor(tempDir string, homeDir string, workingDir string, sshPort i return &RunExecutor{ tempDir: tempDir, homeDir: homeDir, - workingDir: workingDir, archiveDir: filepath.Join(tempDir, "file_archives"), sshPort: sshPort, currentUid: uid, + jobUid: -1, + jobGid: -1, mu: mu, state: WaitSubmit, @@ -187,12 +194,14 @@ func (ex *RunExecutor) Run(ctx context.Context) (err error) { return gerrors.Wrap(err) } - if err := ex.setupFiles(ctx); err != nil { + ex.setJobCredentials(ctx) + + if err := ex.setJobWorkingDir(ctx); err != nil { ex.SetJobStateWithTerminationReason( ctx, types.JobStateFailed, types.TerminationReasonExecutorError, - fmt.Sprintf("Failed to set up files (%s)", err), + fmt.Sprintf("Failed to set the working dir (%s)", err), ) return gerrors.Wrap(err) } @@ -207,6 +216,16 @@ func (ex *RunExecutor) Run(ctx context.Context) (err error) { return gerrors.Wrap(err) } + if err := ex.setupFiles(ctx); err != nil { + ex.SetJobStateWithTerminationReason( + ctx, + types.JobStateFailed, + types.TerminationReasonExecutorError, + fmt.Sprintf("Failed to set up files (%s)", err), + ) + return gerrors.Wrap(err) + } + cleanupCredentials, err := ex.setupCredentials(ctx) if err != nil { ex.SetJobState(ctx, types.JobStateFailed) @@ -318,6 +337,41 @@ func (ex *RunExecutor) SetRunnerState(state string) { ex.state = state } +func (ex *RunExecutor) setJobCredentials(ctx context.Context) { + if ex.jobSpec.User.Uid != nil { + ex.jobUid = int(*ex.jobSpec.User.Uid) + } + if ex.jobSpec.User.Gid != nil { + ex.jobGid = int(*ex.jobSpec.User.Gid) + } + if ex.jobSpec.User.HomeDir != "" { + ex.jobHomeDir = ex.jobSpec.User.HomeDir + } else { + ex.jobHomeDir = "/" + } + log.Trace(ctx, "Job credentials", "uid", ex.jobUid, "gid", ex.jobGid, "home", ex.jobHomeDir) +} + +func (ex *RunExecutor) setJobWorkingDir(ctx context.Context) error { + var err error + if ex.jobSpec.WorkingDir == nil { + ex.jobWorkingDir, err = os.Getwd() + if err != nil { + return gerrors.Wrap(err) + } + } else { + ex.jobWorkingDir, err = common.ExpandPath(*ex.jobSpec.WorkingDir, "", ex.jobHomeDir) + if err != nil { + return gerrors.Wrap(err) + } + if !path.IsAbs(ex.jobWorkingDir) { + return fmt.Errorf("working_dir must be absolute: %s", ex.jobWorkingDir) + } + } + log.Trace(ctx, "Job working dir", "path", ex.jobWorkingDir) + return nil +} + func (ex *RunExecutor) getRepoData() schemas.RepoData { if ex.jobSpec.RepoData == nil { // jobs submitted before 0.19.17 do not have jobSpec.RepoData @@ -339,6 +393,7 @@ func (ex *RunExecutor) execJob(ctx context.Context, jobLogFile io.Writer) error "DSTACK_JOB_ID": ex.jobSubmission.Id, "DSTACK_RUN_NAME": ex.run.RunSpec.RunName, "DSTACK_REPO_ID": ex.run.RunSpec.RepoId, + "DSTACK_REPO_DIR": ex.repoDir, "DSTACK_NODES_IPS": strings.Join(ex.clusterInfo.JobIPs, "\n"), "DSTACK_MASTER_NODE_IP": ex.clusterInfo.MasterJobIP, "DSTACK_NODE_RANK": strconv.Itoa(node_rank), @@ -364,14 +419,7 @@ func (ex *RunExecutor) execJob(ctx context.Context, jobLogFile io.Writer) error } cmd.WaitDelay = ex.killDelay // kills the process if it doesn't exit in time - cmd.Dir = ex.workingDir - if ex.jobSpec.WorkingDir != nil { - workingDir, err := joinRelPath(ex.workingDir, *ex.jobSpec.WorkingDir) - if err != nil { - return gerrors.Wrap(err) - } - cmd.Dir = workingDir - } + cmd.Dir = ex.jobWorkingDir // User must be already set user := ex.jobSpec.User @@ -402,7 +450,7 @@ func (ex *RunExecutor) execJob(ctx context.Context, jobLogFile io.Writer) error envMap.Update(ex.jobSpec.Env, false) const profilePath = "/etc/profile" - const dstackProfilePath = "/tmp/dstack_profile" + const dstackProfilePath = "/dstack/profile" if err := writeDstackProfile(envMap, dstackProfilePath); err != nil { log.Warning(ctx, "failed to write dstack_profile", "path", dstackProfilePath, "err", err) } else if err := includeDstackProfile(profilePath, dstackProfilePath); err != nil { @@ -797,8 +845,11 @@ func writeMpiHostfile(ctx context.Context, ips []string, gpus_per_node int, path return nil } -func writeDstackProfile(env map[string]string, path string) error { - file, err := os.OpenFile(path, os.O_CREATE|os.O_TRUNC|os.O_WRONLY, 0o644) +func writeDstackProfile(env map[string]string, pth string) error { + if err := os.MkdirAll(path.Dir(pth), 0o755); err != nil { + return err + } + file, err := os.OpenFile(pth, os.O_CREATE|os.O_TRUNC|os.O_WRONLY, 0o644) if err != nil { return err } @@ -813,7 +864,7 @@ func writeDstackProfile(env map[string]string, path string) error { return err } } - if err = os.Chmod(path, 0o644); err != nil { + if err = os.Chmod(pth, 0o644); err != nil { return err } return nil diff --git a/runner/internal/executor/executor_test.go b/runner/internal/executor/executor_test.go index e13184513..351dc4941 100644 --- a/runner/internal/executor/executor_test.go +++ b/runner/internal/executor/executor_test.go @@ -8,6 +8,7 @@ import ( "io" "os" "os/exec" + "path" "path/filepath" "strings" "testing" @@ -18,28 +19,38 @@ import ( "github.com/stretchr/testify/require" ) -func TestExecutor_WorkingDir_Current(t *testing.T) { +func TestExecutor_WorkingDir_Set(t *testing.T) { var b bytes.Buffer ex := makeTestExecutor(t) - workingDir := "." + workingDir := path.Join(t.TempDir(), "path/to/wd") ex.jobSpec.WorkingDir = &workingDir ex.jobSpec.Commands = append(ex.jobSpec.Commands, "pwd") + err := ex.setJobWorkingDir(context.TODO()) + require.NoError(t, err) + require.Equal(t, workingDir, ex.jobWorkingDir) + err = os.MkdirAll(workingDir, 0o755) + require.NoError(t, err) - err := ex.execJob(context.TODO(), io.Writer(&b)) + err = ex.execJob(context.TODO(), io.Writer(&b)) assert.NoError(t, err) // Normalize line endings for cross-platform compatibility. - assert.Equal(t, ex.workingDir+"\n", strings.ReplaceAll(b.String(), "\r\n", "\n")) + assert.Equal(t, workingDir+"\n", strings.ReplaceAll(b.String(), "\r\n", "\n")) } -func TestExecutor_WorkingDir_Nil(t *testing.T) { +func TestExecutor_WorkingDir_NotSet(t *testing.T) { var b bytes.Buffer ex := makeTestExecutor(t) + cwd, err := os.Getwd() + require.NoError(t, err) ex.jobSpec.WorkingDir = nil ex.jobSpec.Commands = append(ex.jobSpec.Commands, "pwd") + err = ex.setJobWorkingDir(context.TODO()) + require.NoError(t, err) + require.Equal(t, cwd, ex.jobWorkingDir) - err := ex.execJob(context.TODO(), io.Writer(&b)) + err = ex.execJob(context.TODO(), io.Writer(&b)) assert.NoError(t, err) - assert.Equal(t, ex.workingDir+"\n", strings.ReplaceAll(b.String(), "\r\n", "\n")) + assert.Equal(t, cwd+"\n", strings.ReplaceAll(b.String(), "\r\n", "\n")) } func TestExecutor_HomeDir(t *testing.T) { @@ -88,7 +99,8 @@ func TestExecutor_SSHCredentials(t *testing.T) { func TestExecutor_LocalRepo(t *testing.T) { var b bytes.Buffer ex := makeTestExecutor(t) - ex.jobSpec.Commands = append(ex.jobSpec.Commands, "cat foo") + cmd := fmt.Sprintf("cat %s/foo", *ex.jobSpec.RepoDir) + ex.jobSpec.Commands = append(ex.jobSpec.Commands, cmd) makeCodeTar(t, ex.codePath) err := ex.setupRepo(context.TODO()) @@ -143,6 +155,8 @@ func TestExecutor_RemoteRepo(t *testing.T) { err := os.WriteFile(ex.codePath, []byte{}, 0o600) // empty diff require.NoError(t, err) + err = ex.setJobWorkingDir(context.TODO()) + require.NoError(t, err) err = ex.setupRepo(context.TODO()) require.NoError(t, err) @@ -157,9 +171,9 @@ func TestExecutor_RemoteRepo(t *testing.T) { func makeTestExecutor(t *testing.T) *RunExecutor { t.Helper() baseDir, err := filepath.EvalSymlinks(t.TempDir()) - workingDir := "." require.NoError(t, err) + repo := filepath.Join(baseDir, "repo") body := schemas.SubmitBody{ Run: schemas.Run{ Id: "12346", @@ -177,7 +191,8 @@ func makeTestExecutor(t *testing.T) *RunExecutor { Commands: []string{"/bin/bash", "-c"}, Env: make(map[string]string), MaxDuration: 0, // no timeout - WorkingDir: &workingDir, + WorkingDir: &repo, + RepoDir: &repo, RepoData: &schemas.RepoData{RepoType: "local"}, }, Secrets: make(map[string]string), @@ -190,9 +205,7 @@ func makeTestExecutor(t *testing.T) *RunExecutor { _ = os.Mkdir(temp, 0o700) home := filepath.Join(baseDir, "home") _ = os.Mkdir(home, 0o700) - repo := filepath.Join(baseDir, "repo") - _ = os.Mkdir(repo, 0o700) - ex, _ := NewRunExecutor(temp, home, repo, 10022) + ex, _ := NewRunExecutor(temp, home, 10022) ex.SetJob(body) ex.SetCodePath(filepath.Join(baseDir, "code")) // note: create file before run return ex diff --git a/runner/internal/executor/files.go b/runner/internal/executor/files.go index 8e82516e7..3179e5290 100644 --- a/runner/internal/executor/files.go +++ b/runner/internal/executor/files.go @@ -2,16 +2,14 @@ package executor import ( "context" - "errors" "fmt" "io" "os" "path" "regexp" - "slices" - "strings" "github.com/codeclysm/extract/v4" + "github.com/dstackai/dstack/runner/internal/common" "github.com/dstackai/dstack/runner/internal/gerrors" "github.com/dstackai/dstack/runner/internal/log" ) @@ -35,24 +33,11 @@ func (ex *RunExecutor) AddFileArchive(id string, src io.Reader) error { } // setupFiles must be called from Run +// ex.jobWorkingDir must be already created func (ex *RunExecutor) setupFiles(ctx context.Context) error { - homeDir := ex.workingDir - uid := -1 - gid := -1 - // User must be already set - if ex.jobSpec.User.HomeDir != "" { - homeDir = ex.jobSpec.User.HomeDir - } - if ex.jobSpec.User.Uid != nil { - uid = int(*ex.jobSpec.User.Uid) - } - if ex.jobSpec.User.Gid != nil { - gid = int(*ex.jobSpec.User.Gid) - } - for _, fa := range ex.jobSpec.FileArchives { archivePath := path.Join(ex.archiveDir, fa.Id) - if err := extractFileArchive(ctx, archivePath, fa.Path, ex.workingDir, uid, gid, homeDir); err != nil { + if err := extractFileArchive(ctx, archivePath, fa.Path, ex.jobWorkingDir, ex.jobUid, ex.jobGid, ex.jobHomeDir); err != nil { return gerrors.Wrap(err) } } @@ -64,24 +49,19 @@ func (ex *RunExecutor) setupFiles(ctx context.Context) error { return nil } -func extractFileArchive(ctx context.Context, archivePath string, targetPath string, targetRoot string, uid int, gid int, homeDir string) error { - log.Trace(ctx, "Extracting file archive", "archive", archivePath, "target", targetPath) +func extractFileArchive(ctx context.Context, archivePath string, destPath string, baseDir string, uid int, gid int, homeDir string) error { + log.Trace(ctx, "Extracting file archive", "archive", archivePath, "dest", destPath, "base", baseDir, "home", homeDir) - targetPath = path.Clean(targetPath) - // `~username[/path/to]` is not supported - if targetPath == "~" { - targetPath = homeDir - } else if rest, found := strings.CutPrefix(targetPath, "~/"); found { - targetPath = path.Join(homeDir, rest) - } else if !path.IsAbs(targetPath) { - targetPath = path.Join(targetRoot, targetPath) + destPath, err := common.ExpandPath(destPath, baseDir, homeDir) + if err != nil { + return gerrors.Wrap(err) } - dir, root := path.Split(targetPath) - if err := mkdirAll(ctx, dir, uid, gid); err != nil { + destBase, destName := path.Split(destPath) + if err := common.MkdirAll(ctx, destBase, uid, gid); err != nil { return gerrors.Wrap(err) } - if err := os.RemoveAll(targetPath); err != nil { - log.Warning(ctx, "Failed to remove", "path", targetPath, "err", err) + if err := os.RemoveAll(destPath); err != nil { + log.Warning(ctx, "Failed to remove", "path", destPath, "err", err) } archive, err := os.Open(archivePath) @@ -91,19 +71,20 @@ func extractFileArchive(ctx context.Context, archivePath string, targetPath stri defer archive.Close() var paths []string - repl := fmt.Sprintf("%s$2", root) + repl := fmt.Sprintf("%s$2", destName) renameAndRemember := func(s string) string { s = renameRegex.ReplaceAllString(s, repl) paths = append(paths, s) return s } - if err := extract.Tar(ctx, archive, dir, renameAndRemember); err != nil { + if err := extract.Tar(ctx, archive, destBase, renameAndRemember); err != nil { return gerrors.Wrap(err) } if uid != -1 || gid != -1 { for _, p := range paths { - if err := os.Chown(path.Join(dir, p), uid, gid); err != nil { + log.Warning(ctx, "path", "path", p) + if err := os.Chown(path.Join(destBase, p), uid, gid); err != nil { log.Warning(ctx, "Failed to chown", "path", p, "err", err) } } @@ -111,27 +92,3 @@ func extractFileArchive(ctx context.Context, archivePath string, targetPath stri return nil } - -func mkdirAll(ctx context.Context, p string, uid int, gid int) error { - var paths []string - for { - p = path.Dir(p) - if p == "/" { - break - } - paths = append(paths, p) - } - for _, p := range slices.Backward(paths) { - if _, err := os.Stat(p); errors.Is(err, os.ErrNotExist) { - if err := os.Mkdir(p, 0o755); err != nil { - return err - } - if err := os.Chown(p, uid, gid); err != nil { - log.Warning(ctx, "Failed to chown", "path", p, "err", err) - } - } else if err != nil { - return err - } - } - return nil -} diff --git a/runner/internal/executor/repo.go b/runner/internal/executor/repo.go index 4e6271f29..ac60405ed 100644 --- a/runner/internal/executor/repo.go +++ b/runner/internal/executor/repo.go @@ -2,19 +2,33 @@ package executor import ( "context" + "errors" "fmt" "os" "os/exec" "path/filepath" "github.com/codeclysm/extract/v4" + "github.com/dstackai/dstack/runner/internal/common" "github.com/dstackai/dstack/runner/internal/gerrors" "github.com/dstackai/dstack/runner/internal/log" "github.com/dstackai/dstack/runner/internal/repo" ) // setupRepo must be called from Run +// TODO: change ownership to uid:gid func (ex *RunExecutor) setupRepo(ctx context.Context) error { + if ex.jobSpec.RepoDir == nil { + return errors.New("repo_dir is not set") + } + + var err error + ex.repoDir, err = common.ExpandPath(*ex.jobSpec.RepoDir, ex.jobWorkingDir, ex.jobHomeDir) + if err != nil { + return gerrors.Wrap(err) + } + log.Trace(ctx, "Job repo dir", "path", ex.repoDir) + shouldCheckout, err := ex.shouldCheckout(ctx) if err != nil { return gerrors.Wrap(err) @@ -64,7 +78,7 @@ func (ex *RunExecutor) prepareGit(ctx context.Context) error { ex.getRepoData().RepoBranch, ex.getRepoData().RepoHash, ex.jobSpec.SingleBranch, - ).WithLocalPath(ex.workingDir) + ).WithLocalPath(ex.repoDir) if ex.repoCredentials != nil { log.Trace(ctx, "Credentials is not empty") switch ex.repoCredentials.GetProtocol() { @@ -102,7 +116,7 @@ func (ex *RunExecutor) prepareGit(ctx context.Context) error { return gerrors.Wrap(err) } if len(repoDiff) > 0 { - if err := repo.ApplyDiff(ctx, ex.workingDir, string(repoDiff)); err != nil { + if err := repo.ApplyDiff(ctx, ex.repoDir, string(repoDiff)); err != nil { return gerrors.Wrap(err) } } @@ -115,8 +129,8 @@ func (ex *RunExecutor) prepareArchive(ctx context.Context) error { return gerrors.Wrap(err) } defer func() { _ = file.Close() }() - log.Trace(ctx, "Extracting code archive", "src", ex.codePath, "dst", ex.workingDir) - if err := extract.Tar(ctx, file, ex.workingDir, nil); err != nil { + log.Trace(ctx, "Extracting code archive", "src", ex.codePath, "dst", ex.repoDir) + if err := extract.Tar(ctx, file, ex.repoDir, nil); err != nil { return gerrors.Wrap(err) } return nil @@ -124,10 +138,10 @@ func (ex *RunExecutor) prepareArchive(ctx context.Context) error { func (ex *RunExecutor) shouldCheckout(ctx context.Context) (bool, error) { log.Trace(ctx, "checking if repo checkout is needed") - info, err := os.Stat(ex.workingDir) + info, err := os.Stat(ex.repoDir) if err != nil { if os.IsNotExist(err) { - if err = os.MkdirAll(ex.workingDir, 0o777); err != nil { + if err = common.MkdirAll(ctx, ex.repoDir, ex.jobUid, ex.jobGid); err != nil { return false, gerrors.Wrap(err) } // No repo dir - created a new one @@ -136,9 +150,9 @@ func (ex *RunExecutor) shouldCheckout(ctx context.Context) (bool, error) { return false, gerrors.Wrap(err) } if !info.IsDir() { - return false, fmt.Errorf("failed to set up repo dir: %s is not a dir", ex.workingDir) + return false, fmt.Errorf("failed to set up repo dir: %s is not a dir", ex.repoDir) } - entries, err := os.ReadDir(ex.workingDir) + entries, err := os.ReadDir(ex.repoDir) if err != nil { return false, gerrors.Wrap(err) } @@ -158,14 +172,14 @@ func (ex *RunExecutor) shouldCheckout(ctx context.Context) (bool, error) { } func (ex *RunExecutor) moveRepoDir(tmpDir string) error { - if err := moveDir(ex.workingDir, tmpDir); err != nil { + if err := moveDir(ex.repoDir, tmpDir); err != nil { return gerrors.Wrap(err) } return nil } func (ex *RunExecutor) restoreRepoDir(tmpDir string) error { - if err := moveDir(tmpDir, ex.workingDir); err != nil { + if err := moveDir(tmpDir, ex.repoDir); err != nil { return gerrors.Wrap(err) } return nil diff --git a/runner/internal/runner/api/server.go b/runner/internal/runner/api/server.go index a204c1c46..968a037d3 100644 --- a/runner/internal/runner/api/server.go +++ b/runner/internal/runner/api/server.go @@ -33,9 +33,9 @@ type Server struct { version string } -func NewServer(tempDir string, homeDir string, workingDir string, address string, sshPort int, version string) (*Server, error) { +func NewServer(tempDir string, homeDir string, address string, sshPort int, version string) (*Server, error) { r := api.NewRouter() - ex, err := executor.NewRunExecutor(tempDir, homeDir, workingDir, sshPort) + ex, err := executor.NewRunExecutor(tempDir, homeDir, sshPort) if err != nil { return nil, err } diff --git a/runner/internal/schemas/schemas.go b/runner/internal/schemas/schemas.go index 8e439dae0..389ed4c9e 100644 --- a/runner/internal/schemas/schemas.go +++ b/runner/internal/schemas/schemas.go @@ -67,6 +67,7 @@ type JobSpec struct { MaxDuration int `json:"max_duration"` SSHKey *SSHKey `json:"ssh_key"` WorkingDir *string `json:"working_dir"` + RepoDir *string `json:"repo_dir"` // `RepoData` is optional for compatibility with jobs submitted before 0.19.17. // Use `RunExecutor.getRepoData()` to get non-nil `RepoData`. // TODO: make required when supporting jobs submitted before 0.19.17 is no longer relevant. diff --git a/runner/internal/shim/runner.go b/runner/internal/shim/runner.go index e044cfcce..209d4b47f 100644 --- a/runner/internal/shim/runner.go +++ b/runner/internal/shim/runner.go @@ -34,7 +34,6 @@ func (c *CLIArgs) getRunnerArgs() []string { "--ssh-port", strconv.Itoa(c.Runner.SSHPort), "--temp-dir", consts.RunnerTempDir, "--home-dir", consts.RunnerHomeDir, - "--working-dir", consts.RunnerWorkingDir, } } diff --git a/src/dstack/_internal/cli/services/configurators/run.py b/src/dstack/_internal/cli/services/configurators/run.py index 4a4a4d453..9974d10e9 100644 --- a/src/dstack/_internal/cli/services/configurators/run.py +++ b/src/dstack/_internal/cli/services/configurators/run.py @@ -2,7 +2,7 @@ import subprocess import sys import time -from pathlib import Path +from pathlib import Path, PurePosixPath from typing import Dict, List, Optional, Set, TypeVar import gpuhunt @@ -33,6 +33,7 @@ ) from dstack._internal.core.models.common import ApplyAction, RegistryAuth from dstack._internal.core.models.configurations import ( + LEGACY_REPO_DIR, AnyRunConfiguration, ApplyConfigurationType, ConfigurationWithPortsParams, @@ -53,6 +54,7 @@ from dstack._internal.utils.interpolator import InterpolatorError, VariablesInterpolator from dstack._internal.utils.logging import get_logger from dstack._internal.utils.nested_list import NestedList, NestedListItem +from dstack._internal.utils.path import is_absolute_posix_path from dstack.api._public.repos import get_ssh_keypair from dstack.api._public.runs import Run from dstack.api.utils import load_profile @@ -89,6 +91,42 @@ def apply_configuration( self.validate_gpu_vendor_and_image(conf) self.validate_cpu_arch_and_image(conf) + working_dir = conf.working_dir + if working_dir is None: + # Use the default working dir for the image for tasks and services if `commands` + # is not set (emulate pre-0.19.27 JobConfigutor logic), otherwise fall back to + # `/workflow`. + if isinstance(conf, DevEnvironmentConfiguration) or conf.commands: + conf.working_dir = LEGACY_REPO_DIR + warn( + "[code]working_dir[/code] is not set." + f" Using legacy working directory [code]{LEGACY_REPO_DIR}[/code]\n\n" + "In future versions the default value will be taken from the image\n" + f"- To keep using legacy working directory, set" + f" [code]working_dir[/code] to [code]{LEGACY_REPO_DIR}[/code]\n" + f"- To prepare for the future default, set [code]working_dir[/code]" + " to the image default working directory (only until the new default" + " takes effect)" + ) + elif not is_absolute_posix_path(working_dir): + legacy_working_dir = PurePosixPath(LEGACY_REPO_DIR) / working_dir + warn( + "[code]working_dir[/code] is relative." + f" Using legacy working directory [code]{legacy_working_dir}[/code]\n\n" + "Future versions will require absolute path\n" + f"To keep using legacy working directory, set" + f" [code]working_dir[/code] to [code]{legacy_working_dir}[/code]\n" + ) + + if conf.repos and conf.repos[0].path is None: + warn( + "[code]repos[0].path[/code] is not set," + f" using legacy repo path [code]{LEGACY_REPO_DIR}[/code]\n\n" + "In future versions [code]path[/code] will be mandatory." + f" To keep using [code]{LEGACY_REPO_DIR}[/code], explicitly set" + f" [code]repos[0].path[/code] to [code]{LEGACY_REPO_DIR}[/code]\n" + ) + config_manager = ConfigManager() repo = self.get_repo(conf, configuration_path, configurator_args, config_manager) self.api.ssh_identity_file = get_ssh_keypair( diff --git a/src/dstack/_internal/core/backends/base/compute.py b/src/dstack/_internal/core/backends/base/compute.py index 9e604286a..63de39b43 100644 --- a/src/dstack/_internal/core/backends/base/compute.py +++ b/src/dstack/_internal/core/backends/base/compute.py @@ -19,7 +19,7 @@ DSTACK_RUNNER_SSH_PORT, DSTACK_SHIM_HTTP_PORT, ) -from dstack._internal.core.models.configurations import DEFAULT_REPO_DIR +from dstack._internal.core.models.configurations import LEGACY_REPO_DIR from dstack._internal.core.models.gateways import ( GatewayComputeConfiguration, GatewayProvisioningData, @@ -773,7 +773,8 @@ def get_docker_commands( f" --ssh-port {DSTACK_RUNNER_SSH_PORT}" " --temp-dir /tmp/runner" " --home-dir /root" - f" --working-dir {DEFAULT_REPO_DIR}" + # TODO: Not used, left for compatibility with old runners. Remove eventually. + f" --working-dir {LEGACY_REPO_DIR}" ), ] diff --git a/src/dstack/_internal/core/compatibility/runs.py b/src/dstack/_internal/core/compatibility/runs.py index cfb809acb..4edd28808 100644 --- a/src/dstack/_internal/core/compatibility/runs.py +++ b/src/dstack/_internal/core/compatibility/runs.py @@ -1,7 +1,7 @@ from typing import Optional from dstack._internal.core.models.common import IncludeExcludeDictType, IncludeExcludeSetType -from dstack._internal.core.models.configurations import ServiceConfiguration +from dstack._internal.core.models.configurations import LEGACY_REPO_DIR, ServiceConfiguration from dstack._internal.core.models.runs import ApplyRunPlanInput, JobSpec, JobSubmission, RunSpec from dstack._internal.server.schemas.runs import GetRunPlanRequest, ListRunsRequest @@ -102,6 +102,9 @@ def get_run_spec_excludes(run_spec: RunSpec) -> IncludeExcludeDictType: configuration = run_spec.configuration profile = run_spec.profile + if run_spec.repo_dir in [None, LEGACY_REPO_DIR]: + spec_excludes["repo_dir"] = True + if configuration.fleets is None: configuration_excludes["fleets"] = True if profile is not None and profile.fleets is None: @@ -163,6 +166,8 @@ def get_job_spec_excludes(job_specs: list[JobSpec]) -> IncludeExcludeDictType: spec_excludes["service_port"] = True if all(not s.probes for s in job_specs): spec_excludes["probes"] = True + if all(s.repo_dir in [None, LEGACY_REPO_DIR] for s in job_specs): + spec_excludes["repo_dir"] = True return spec_excludes diff --git a/src/dstack/_internal/core/models/configurations.py b/src/dstack/_internal/core/models/configurations.py index a72792da2..39baeb6dd 100644 --- a/src/dstack/_internal/core/models/configurations.py +++ b/src/dstack/_internal/core/models/configurations.py @@ -34,7 +34,7 @@ RUN_PRIOTIRY_MIN = 0 RUN_PRIOTIRY_MAX = 100 RUN_PRIORITY_DEFAULT = 0 -DEFAULT_REPO_DIR = "/workflow" +LEGACY_REPO_DIR = "/workflow" MIN_PROBE_TIMEOUT = 1 MIN_PROBE_INTERVAL = 1 DEFAULT_PROBE_URL = "/" @@ -112,8 +112,15 @@ class RepoSpec(CoreModel): Optional[str], Field(description="The commit hash"), ] = None - # Not implemented, has no effect, hidden in the docs - path: str = DEFAULT_REPO_DIR + path: Annotated[ + Optional[str], + Field( + description=( + "The repo path inside the run container. Relative paths are resolved" + f" relative to the working directory. Defaults to `{LEGACY_REPO_DIR}`" + ) + ), + ] = None @classmethod def parse(cls, v: str) -> Self: @@ -149,6 +156,14 @@ def validate_local_path_or_url(cls, values): raise ValueError("Either `local_path` or `url` must be specified") return values + @validator("path") + def validate_path(cls, v: Optional[str]) -> Optional[str]: + if v is None: + return v + if v.startswith("~") and PurePosixPath(v).parts[0] != "~": + raise ValueError("`~username` syntax is not supported") + return v + class ScalingSpec(CoreModel): metric: Annotated[ @@ -380,7 +395,7 @@ class BaseRunConfiguration(CoreModel): Field( description=( "The user inside the container, `user_name_or_id[:group_name_or_id]`" - " (e.g., `ubuntu`, `1000:1000`). Defaults to the default `image` user" + " (e.g., `ubuntu`, `1000:1000`). Defaults to the default user from the `image`" ) ), ] = None @@ -390,9 +405,8 @@ class BaseRunConfiguration(CoreModel): Optional[str], Field( description=( - "The path to the working directory inside the container." - f" It's specified relative to the repository directory (`{DEFAULT_REPO_DIR}`) and should be inside it." - ' Defaults to `"."` ' + "The absolute path to the working directory inside the container." + f" Defaults to `{LEGACY_REPO_DIR}`" ) ), ] = None diff --git a/src/dstack/_internal/core/models/files.py b/src/dstack/_internal/core/models/files.py index 797c64da2..2c82fd53b 100644 --- a/src/dstack/_internal/core/models/files.py +++ b/src/dstack/_internal/core/models/files.py @@ -28,7 +28,7 @@ class FilePathMapping(CoreModel): Field( description=( "The path in the container. Relative paths are resolved relative to" - " the repo directory" + " the working directory" ) ), ] diff --git a/src/dstack/_internal/core/models/runs.py b/src/dstack/_internal/core/models/runs.py index 75f3b6b82..c8085b55b 100644 --- a/src/dstack/_internal/core/models/runs.py +++ b/src/dstack/_internal/core/models/runs.py @@ -10,7 +10,7 @@ from dstack._internal.core.models.common import ApplyAction, CoreModel, NetworkMode, RegistryAuth from dstack._internal.core.models.configurations import ( DEFAULT_PROBE_METHOD, - DEFAULT_REPO_DIR, + LEGACY_REPO_DIR, AnyRunConfiguration, HTTPHeaderSpec, HTTPMethod, @@ -259,6 +259,7 @@ class JobSpec(CoreModel): retry: Optional[Retry] volumes: Optional[List[MountPoint]] = None ssh_key: Optional[JobSSHKey] = None + # `working_dir` is always absolute (if not None) since 0.19.27 working_dir: Optional[str] # `repo_data` is optional for client compatibility with pre-0.19.17 servers and for compatibility # with jobs submitted before 0.19.17. All new jobs are expected to have non-None `repo_data`. @@ -268,6 +269,8 @@ class JobSpec(CoreModel): # submitted before 0.19.17. See `_get_repo_code_hash` on how to get the correct `repo_code_hash` # TODO: drop this comment when supporting jobs submitted before 0.19.17 is no longer relevant. repo_code_hash: Optional[str] = None + # `repo_dir` was added in 0.19.27. Default value is set for backward compatibility + repo_dir: str = LEGACY_REPO_DIR file_archives: list[FileArchiveMapping] = [] # None for non-services and pre-0.19.19 services. See `get_service_port` service_port: Optional[int] = None @@ -409,17 +412,27 @@ class RunSpec(CoreModel): Optional[str], Field(description="The hash of the repo diff. Can be omitted if there is no repo diff."), ] = None + repo_dir: Annotated[ + Optional[str], + Field( + description=( + "The repo path inside the container. Relative paths are resolved" + f" relative to the working directory. Defaults to `{LEGACY_REPO_DIR}`." + ) + ), + ] = None file_archives: Annotated[ list[FileArchiveMapping], - Field(description="The list of file archive ID to container path mappings"), + Field(description="The list of file archive ID to container path mappings."), ] = [] + # Server uses configuration.working_dir instead of this field since 0.19.27, but + # the field still exists for compatibility with older servers working_dir: Annotated[ Optional[str], Field( description=( - "The path to the working directory inside the container." - f" It's specified relative to the repository directory (`{DEFAULT_REPO_DIR}`) and should be inside it." - ' Defaults to `"."`.' + "The absolute path to the working directory inside the container." + " Defaults to the default working directory from the `image`." ) ), ] = None diff --git a/src/dstack/_internal/server/schemas/runner.py b/src/dstack/_internal/server/schemas/runner.py index 6de49f35a..62ec3f6e3 100644 --- a/src/dstack/_internal/server/schemas/runner.py +++ b/src/dstack/_internal/server/schemas/runner.py @@ -78,6 +78,7 @@ class SubmitBody(CoreModel): "max_duration", "ssh_key", "working_dir", + "repo_dir", "repo_data", "file_archives", } diff --git a/src/dstack/_internal/server/services/jobs/configurators/base.py b/src/dstack/_internal/server/services/jobs/configurators/base.py index ee9bb05f6..7eb469865 100644 --- a/src/dstack/_internal/server/services/jobs/configurators/base.py +++ b/src/dstack/_internal/server/services/jobs/configurators/base.py @@ -16,7 +16,7 @@ DEFAULT_PROBE_READY_AFTER, DEFAULT_PROBE_TIMEOUT, DEFAULT_PROBE_URL, - DEFAULT_REPO_DIR, + LEGACY_REPO_DIR, PortMapping, ProbeConfig, PythonVersion, @@ -45,6 +45,14 @@ from dstack._internal.utils import crypto from dstack._internal.utils.common import run_async from dstack._internal.utils.interpolator import InterpolatorError, VariablesInterpolator +from dstack._internal.utils.logging import get_logger +from dstack._internal.utils.path import is_absolute_posix_path + +logger = get_logger(__name__) + + +DSTACK_DIR = "/dstack" +DSTACK_PROFILE_PATH = f"{DSTACK_DIR}/profile" def get_default_python_verison() -> str: @@ -160,6 +168,7 @@ async def _get_job_spec( ssh_key=self._ssh_key(jobs_per_replica), repo_data=self.run_spec.repo_data, repo_code_hash=self.run_spec.repo_code_hash, + repo_dir=self._repo_dir(), file_archives=self.run_spec.file_archives, service_port=self._service_port(), probes=self._probes(), @@ -209,9 +218,9 @@ def _dstack_image_commands(self) -> List[str]: ): return [] return [ - f"uv venv --python {self._python()} --prompt workflow --seed {DEFAULT_REPO_DIR}/.venv > /dev/null 2>&1", - f"echo 'source {DEFAULT_REPO_DIR}/.venv/bin/activate' >> ~/.bashrc", - f"source {DEFAULT_REPO_DIR}/.venv/bin/activate", + f"uv venv -q --prompt $DSTACK_RUN_NAME --seed -p {self._python()} {DSTACK_DIR}/venv", + f"echo '. {DSTACK_DIR}/venv/bin/activate' >> {DSTACK_PROFILE_PATH}", + f". {DSTACK_DIR}/venv/bin/activate", ] def _app_specs(self) -> List[AppSpec]: @@ -290,11 +299,25 @@ def _requirements(self) -> Requirements: def _retry(self) -> Optional[Retry]: return get_retry(self.run_spec.merged_profile) + def _repo_dir(self) -> str: + """ + Returns absolute or relative path + """ + repo_dir = self.run_spec.repo_dir + if repo_dir is None: + return LEGACY_REPO_DIR + return repo_dir + def _working_dir(self) -> Optional[str]: """ - None means default working directory + Returns absolute path or None + None means the default working directory taken from the image """ - return self.run_spec.working_dir + working_dir = self.run_spec.configuration.working_dir + if working_dir is None or is_absolute_posix_path(working_dir): + return working_dir + # Legacy configuration; relative working_dir is deprecated + return str(PurePosixPath(LEGACY_REPO_DIR) / working_dir) def _python(self) -> str: if self.run_spec.configuration.python is not None: diff --git a/src/dstack/_internal/server/services/jobs/configurators/dev.py b/src/dstack/_internal/server/services/jobs/configurators/dev.py index 20aad1f23..3efea3fa2 100644 --- a/src/dstack/_internal/server/services/jobs/configurators/dev.py +++ b/src/dstack/_internal/server/services/jobs/configurators/dev.py @@ -9,8 +9,8 @@ from dstack._internal.server.services.jobs.configurators.extensions.vscode import VSCodeDesktop INSTALL_IPYKERNEL = ( - "(echo pip install ipykernel... && pip install -q --no-cache-dir ipykernel 2> /dev/null) || " - 'echo "no pip, ipykernel was not installed"' + "(echo 'pip install ipykernel...' && pip install -q --no-cache-dir ipykernel 2> /dev/null) || " + "echo 'no pip, ipykernel was not installed'" ) @@ -39,12 +39,12 @@ def _shell_commands(self) -> List[str]: commands = self.ide.get_install_commands() commands.append(INSTALL_IPYKERNEL) commands += self.run_spec.configuration.setup - commands.append("echo ''") + commands.append("echo") commands += self.run_spec.configuration.init commands += self.ide.get_print_readme_commands() commands += [ f"echo 'To connect via SSH, use: `ssh {self.run_spec.run_name}`'", - "echo ''", + "echo", "echo -n 'To exit, press Ctrl+C.'", ] commands += ["tail -f /dev/null"] # idle diff --git a/src/dstack/_internal/server/services/jobs/configurators/extensions/cursor.py b/src/dstack/_internal/server/services/jobs/configurators/extensions/cursor.py index 9c5e68d96..9227fa125 100644 --- a/src/dstack/_internal/server/services/jobs/configurators/extensions/cursor.py +++ b/src/dstack/_internal/server/services/jobs/configurators/extensions/cursor.py @@ -1,7 +1,5 @@ from typing import List, Optional -from dstack._internal.core.models.configurations import DEFAULT_REPO_DIR - class CursorDesktop: def __init__( @@ -38,7 +36,7 @@ def get_install_commands(self) -> List[str]: def get_print_readme_commands(self) -> List[str]: return [ "echo To open in Cursor, use link below:", - "echo ''", - f"echo ' cursor://vscode-remote/ssh-remote+{self.run_name}{DEFAULT_REPO_DIR}'", # TODO use $REPO_DIR - "echo ''", + "echo", + f'echo " vscode://vscode-remote/ssh-remote+{self.run_name}$DSTACK_REPO_DIR"', + "echo", ] diff --git a/src/dstack/_internal/server/services/jobs/configurators/extensions/vscode.py b/src/dstack/_internal/server/services/jobs/configurators/extensions/vscode.py index a10b254d0..f431cf547 100644 --- a/src/dstack/_internal/server/services/jobs/configurators/extensions/vscode.py +++ b/src/dstack/_internal/server/services/jobs/configurators/extensions/vscode.py @@ -1,7 +1,5 @@ from typing import List, Optional -from dstack._internal.core.models.configurations import DEFAULT_REPO_DIR - class VSCodeDesktop: def __init__( @@ -37,8 +35,8 @@ def get_install_commands(self) -> List[str]: def get_print_readme_commands(self) -> List[str]: return [ - "echo To open in VS Code Desktop, use link below:", - "echo ''", - f"echo ' vscode://vscode-remote/ssh-remote+{self.run_name}{DEFAULT_REPO_DIR}'", # TODO use $REPO_DIR - "echo ''", + "echo 'To open in VS Code Desktop, use link below:'", + "echo", + f'echo " vscode://vscode-remote/ssh-remote+{self.run_name}$DSTACK_REPO_DIR"', + "echo", ] diff --git a/src/dstack/_internal/server/services/jobs/configurators/service.py b/src/dstack/_internal/server/services/jobs/configurators/service.py index a00216a6d..be15c4b23 100644 --- a/src/dstack/_internal/server/services/jobs/configurators/service.py +++ b/src/dstack/_internal/server/services/jobs/configurators/service.py @@ -23,6 +23,3 @@ def _spot_policy(self) -> SpotPolicy: def _ports(self) -> List[PortMapping]: return [] - - def _working_dir(self) -> Optional[str]: - return None if not self._shell_commands() else super()._working_dir() diff --git a/src/dstack/_internal/server/services/jobs/configurators/task.py b/src/dstack/_internal/server/services/jobs/configurators/task.py index 6a0da9f00..51c136dfe 100644 --- a/src/dstack/_internal/server/services/jobs/configurators/task.py +++ b/src/dstack/_internal/server/services/jobs/configurators/task.py @@ -37,6 +37,3 @@ def _spot_policy(self) -> SpotPolicy: def _ports(self) -> List[PortMapping]: assert self.run_spec.configuration.type == "task" return self.run_spec.configuration.ports - - def _working_dir(self) -> Optional[str]: - return None if not self._shell_commands() else super()._working_dir() diff --git a/src/dstack/_internal/server/testing/common.py b/src/dstack/_internal/server/testing/common.py index 6745deac7..c99733bb3 100644 --- a/src/dstack/_internal/server/testing/common.py +++ b/src/dstack/_internal/server/testing/common.py @@ -271,7 +271,7 @@ def get_run_spec( repo_id=repo_id, repo_data=LocalRunRepoData(repo_dir="/"), repo_code_hash=None, - working_dir=".", + working_dir=None, configuration_path=configuration_path, configuration=configuration or DevEnvironmentConfiguration(ide="vscode"), profile=profile, diff --git a/src/dstack/_internal/utils/path.py b/src/dstack/_internal/utils/path.py index fc51c488a..18e0b7c81 100644 --- a/src/dstack/_internal/utils/path.py +++ b/src/dstack/_internal/utils/path.py @@ -1,6 +1,6 @@ import os from dataclasses import dataclass -from pathlib import Path, PurePath +from pathlib import Path, PurePath, PurePosixPath from typing import Union PathLike = Union[str, os.PathLike] @@ -48,3 +48,10 @@ def resolve_relative_path(path: PathLike) -> PurePath: return normalize_path(path) except ValueError: raise ValueError("Path is outside of the repo") + + +def is_absolute_posix_path(path: PathLike) -> bool: + # Passing Windows path leads to undefined behavior + if str(path).startswith("~"): + return True + return PurePosixPath(path).is_absolute() diff --git a/src/dstack/api/_public/runs.py b/src/dstack/api/_public/runs.py index 8a87fd879..c6b786337 100644 --- a/src/dstack/api/_public/runs.py +++ b/src/dstack/api/_public/runs.py @@ -433,6 +433,7 @@ def get_run_plan( repo: Optional[Repo] = None, profile: Optional[Profile] = None, configuration_path: Optional[str] = None, + repo_dir: Optional[str] = None, ) -> RunPlan: """ Get a run plan. @@ -443,11 +444,17 @@ def get_run_plan( repo (Union[LocalRepo, RemoteRepo, VirtualRepo, None]): The repo to use for the run. Pass `None` if repo is not needed. profile: The profile to use for the run. - configuration_path: The path to the configuration file. Omit if the configuration is not loaded from a file. + configuration_path: The path to the configuration file. Omit if the configuration + is not loaded from a file. + repo_dir: The path of the cloned repo inside the run container. If not set, + defaults first to the `repos[0].path` property of the configuration (for remote + repos only), then to `/workflow`. Returns: Run plan. """ + # XXX: not using the LEGACY_REPO_DIR const in the docstring above, as the docs generator, + # apparently, doesn't support f-strings (f"""..."""). if repo is None: repo = VirtualRepo() repo_code_hash = None @@ -455,11 +462,17 @@ def get_run_plan( with _prepare_code_file(repo) as (_, repo_code_hash): pass + if repo_dir is None and configuration.repos: + repo_dir = configuration.repos[0].path + run_spec = RunSpec( run_name=configuration.name, repo_id=repo.repo_id, repo_data=repo.run_repo_data, repo_code_hash=repo_code_hash, + repo_dir=repo_dir, + # Server doesn't use this field since 0.19.27, but we still send it for compatibility + # with older servers working_dir=configuration.working_dir, configuration_path=configuration_path, configuration=configuration, diff --git a/src/tests/_internal/core/models/test_configurations.py b/src/tests/_internal/core/models/test_configurations.py index a4fbe8f4d..4a43a8c46 100644 --- a/src/tests/_internal/core/models/test_configurations.py +++ b/src/tests/_internal/core/models/test_configurations.py @@ -4,11 +4,7 @@ from dstack._internal.core.errors import ConfigurationError from dstack._internal.core.models.common import RegistryAuth -from dstack._internal.core.models.configurations import ( - DEFAULT_REPO_DIR, - RepoSpec, - parse_run_configuration, -) +from dstack._internal.core.models.configurations import RepoSpec, parse_run_configuration from dstack._internal.core.models.resources import Range @@ -80,7 +76,7 @@ def test_shell_invalid(self): class TestRepoSpec: @pytest.mark.parametrize("value", [".", "rel/path", "/abs/path/"]) def test_parse_local_path_no_path(self, value: str): - assert RepoSpec.parse(value) == RepoSpec(local_path=value, path=DEFAULT_REPO_DIR) + assert RepoSpec.parse(value) == RepoSpec(local_path=value, path=None) @pytest.mark.parametrize( ["value", "expected_repo_path"], @@ -90,14 +86,14 @@ def test_parse_local_path_with_path(self, value: str, expected_repo_path: str): assert RepoSpec.parse(value) == RepoSpec(local_path=expected_repo_path, path="/repo") def test_parse_windows_abs_local_path_no_path(self): - assert RepoSpec.parse("C:\\repo") == RepoSpec(local_path="C:\\repo", path=DEFAULT_REPO_DIR) + assert RepoSpec.parse("C:\\repo") == RepoSpec(local_path="C:\\repo", path=None) def test_parse_windows_abs_local_path_with_path(self): assert RepoSpec.parse("C:\\repo:/repo") == RepoSpec(local_path="C:\\repo", path="/repo") def test_parse_url_no_path(self): assert RepoSpec.parse("https://example.com/repo.git") == RepoSpec( - url="https://example.com/repo.git", path=DEFAULT_REPO_DIR + url="https://example.com/repo.git", path=None ) def test_parse_url_with_path(self): @@ -107,7 +103,7 @@ def test_parse_url_with_path(self): def test_parse_scp_no_path(self): assert RepoSpec.parse("git@example.com:repo.git") == RepoSpec( - url="git@example.com:repo.git", path=DEFAULT_REPO_DIR + url="git@example.com:repo.git", path=None ) def test_parse_scp_with_path(self): @@ -115,10 +111,26 @@ def test_parse_scp_with_path(self): url="git@example.com:repo.git", path="/repo" ) + @pytest.mark.parametrize("path", ["~", "~/repo"]) + def test_path_tilde(self, path: str): + assert RepoSpec(local_path=".", path=path).path == path + def test_error_invalid_mapping_if_more_than_two_parts(self): with pytest.raises(ValueError, match="Invalid repo"): RepoSpec.parse("./foo:bar:baz") + def test_error_local_path_url_mutually_exclusive(self): + with pytest.raises(ValueError, match="mutually exclusive"): + RepoSpec(local_path=".", url="https://example.com/repo.git") + + def test_error_local_path_or_url_required(self): + with pytest.raises(ValueError, match="must be specified"): + RepoSpec() + + def test_error_path_tilde_username_not_supported(self): + with pytest.raises(ValueError, match="syntax is not supported"): + RepoSpec(local_path=".", path="~alice/repo") + def test_registry_auth_hashable(): """ diff --git a/src/tests/_internal/server/routers/test_runs.py b/src/tests/_internal/server/routers/test_runs.py index 4438b61dc..c28f5458c 100644 --- a/src/tests/_internal/server/routers/test_runs.py +++ b/src/tests/_internal/server/routers/test_runs.py @@ -87,18 +87,21 @@ def get_dev_env_run_plan_dict( "/bin/bash", "-i", "-c", - "start-dockerd && (echo pip install ipykernel... && " - "pip install -q --no-cache-dir " - 'ipykernel 2> /dev/null) || echo "no ' - 'pip, ipykernel was not installed" ' - "&& echo '' && echo To open in VS " - "Code Desktop, use link below: && " - "echo '' && echo ' " - "vscode://vscode-remote/ssh-remote+dry-run/workflow' " - "&& echo '' && echo 'To connect via " - "SSH, use: `ssh dry-run`' && echo '' " - "&& echo -n 'To exit, press Ctrl+C.' " - "&& tail -f /dev/null", + ( + "start-dockerd" + " && (echo 'pip install ipykernel...'" + " && pip install -q --no-cache-dir ipykernel 2> /dev/null)" + " || echo 'no pip, ipykernel was not installed'" + " && echo" + " && echo 'To open in VS Code Desktop, use link below:'" + " && echo" + ' && echo " vscode://vscode-remote/ssh-remote+dry-run$DSTACK_REPO_DIR"' + " && echo" + " && echo 'To connect via SSH, use: `ssh dry-run`'" + " && echo" + " && echo -n 'To exit, press Ctrl+C.'" + " && tail -f /dev/null" + ), ] image_name = "dstackai/dind" else: @@ -106,21 +109,23 @@ def get_dev_env_run_plan_dict( "/bin/bash", "-i", "-c", - "uv venv --python 3.13 --prompt workflow --seed /workflow/.venv > /dev/null 2>&1" - " && echo 'source /workflow/.venv/bin/activate' >> ~/.bashrc" - " && source /workflow/.venv/bin/activate" - " && (echo pip install ipykernel... && " - "pip install -q --no-cache-dir " - 'ipykernel 2> /dev/null) || echo "no ' - 'pip, ipykernel was not installed" ' - "&& echo '' && echo To open in VS " - "Code Desktop, use link below: && " - "echo '' && echo ' " - "vscode://vscode-remote/ssh-remote+dry-run/workflow' " - "&& echo '' && echo 'To connect via " - "SSH, use: `ssh dry-run`' && echo '' " - "&& echo -n 'To exit, press Ctrl+C.' " - "&& tail -f /dev/null", + ( + "uv venv -q --prompt $DSTACK_RUN_NAME --seed -p 3.13 /dstack/venv" + " && echo '. /dstack/venv/bin/activate' >> /dstack/profile" + " && . /dstack/venv/bin/activate" + " && (echo 'pip install ipykernel...'" + " && pip install -q --no-cache-dir ipykernel 2> /dev/null)" + " || echo 'no pip, ipykernel was not installed'" + " && echo" + " && echo 'To open in VS Code Desktop, use link below:'" + " && echo" + ' && echo " vscode://vscode-remote/ssh-remote+dry-run$DSTACK_REPO_DIR"' + " && echo" + " && echo 'To connect via SSH, use: `ssh dry-run`'" + " && echo" + " && echo -n 'To exit, press Ctrl+C.'" + " && tail -f /dev/null" + ), ] image_name = "dstackai/base:0.10-base-ubuntu22.04" @@ -204,9 +209,10 @@ def get_dev_env_run_plan_dict( "repo_code_hash": None, "repo_data": {"repo_dir": "/repo", "repo_type": "local"}, "repo_id": repo_id, + "repo_dir": "~/repo", "run_name": run_name, "ssh_key_pub": "ssh_key", - "working_dir": ".", + "working_dir": None, } return { "project_name": project_name, @@ -247,9 +253,10 @@ def get_dev_env_run_plan_dict( "retry": None, "volumes": volumes, "ssh_key": None, - "working_dir": ".", + "working_dir": None, "repo_code_hash": None, "repo_data": {"repo_dir": "/repo", "repo_type": "local"}, + "repo_dir": "~/repo", "file_archives": [], "service_port": None, "probes": [], @@ -284,18 +291,21 @@ def get_dev_env_run_dict( "/bin/bash", "-i", "-c", - "start-dockerd && (echo pip install ipykernel... && " - "pip install -q --no-cache-dir " - 'ipykernel 2> /dev/null) || echo "no ' - 'pip, ipykernel was not installed" ' - "&& echo '' && echo To open in VS " - "Code Desktop, use link below: && " - "echo '' && echo ' " - "vscode://vscode-remote/ssh-remote+test-run/workflow' " - "&& echo '' && echo 'To connect via " - "SSH, use: `ssh test-run`' && echo '' " - "&& echo -n 'To exit, press Ctrl+C.' " - "&& tail -f /dev/null", + ( + "start-dockerd" + " && (echo 'pip install ipykernel...'" + " && pip install -q --no-cache-dir ipykernel 2> /dev/null)" + " || echo 'no pip, ipykernel was not installed'" + " && echo" + " && echo 'To open in VS Code Desktop, use link below:'" + " && echo" + ' && echo " vscode://vscode-remote/ssh-remote+test-run$DSTACK_REPO_DIR"' + " && echo" + " && echo 'To connect via SSH, use: `ssh test-run`'" + " && echo" + " && echo -n 'To exit, press Ctrl+C.'" + " && tail -f /dev/null" + ), ] image_name = "dstackai/dind" else: @@ -303,21 +313,23 @@ def get_dev_env_run_dict( "/bin/bash", "-i", "-c", - "uv venv --python 3.13 --prompt workflow --seed /workflow/.venv > /dev/null 2>&1" - " && echo 'source /workflow/.venv/bin/activate' >> ~/.bashrc" - " && source /workflow/.venv/bin/activate" - " && (echo pip install ipykernel... && " - "pip install -q --no-cache-dir " - 'ipykernel 2> /dev/null) || echo "no ' - 'pip, ipykernel was not installed" ' - "&& echo '' && echo To open in VS " - "Code Desktop, use link below: && " - "echo '' && echo ' " - "vscode://vscode-remote/ssh-remote+test-run/workflow' " - "&& echo '' && echo 'To connect via " - "SSH, use: `ssh test-run`' && echo '' " - "&& echo -n 'To exit, press Ctrl+C.' " - "&& tail -f /dev/null", + ( + "uv venv -q --prompt $DSTACK_RUN_NAME --seed -p 3.13 /dstack/venv" + " && echo '. /dstack/venv/bin/activate' >> /dstack/profile" + " && . /dstack/venv/bin/activate" + " && (echo 'pip install ipykernel...'" + " && pip install -q --no-cache-dir ipykernel 2> /dev/null)" + " || echo 'no pip, ipykernel was not installed'" + " && echo" + " && echo 'To open in VS Code Desktop, use link below:'" + " && echo" + ' && echo " vscode://vscode-remote/ssh-remote+test-run$DSTACK_REPO_DIR"' + " && echo" + " && echo 'To connect via SSH, use: `ssh test-run`'" + " && echo" + " && echo -n 'To exit, press Ctrl+C.'" + " && tail -f /dev/null" + ), ] image_name = "dstackai/base:0.10-base-ubuntu22.04" @@ -409,9 +421,10 @@ def get_dev_env_run_dict( "repo_code_hash": None, "repo_data": {"repo_dir": "/repo", "repo_type": "local"}, "repo_id": repo_id, + "repo_dir": "~/repo", "run_name": run_name, "ssh_key_pub": "ssh_key", - "working_dir": ".", + "working_dir": None, }, "jobs": [ { @@ -447,9 +460,10 @@ def get_dev_env_run_dict( "retry": None, "volumes": [], "ssh_key": None, - "working_dir": ".", + "working_dir": None, "repo_code_hash": None, "repo_data": {"repo_dir": "/repo", "repo_type": "local"}, + "repo_dir": "~/repo", "file_archives": [], "service_port": None, "probes": [], @@ -526,7 +540,7 @@ def get_service_run_spec( "repo_id": repo_id, "run_name": run_name, "ssh_key_pub": "ssh_key", - "working_dir": ".", + "working_dir": None, } diff --git a/src/tests/_internal/server/services/jobs/configurators/test_task.py b/src/tests/_internal/server/services/jobs/configurators/test_task.py index e954e6a01..8e70b2e0d 100644 --- a/src/tests/_internal/server/services/jobs/configurators/test_task.py +++ b/src/tests/_internal/server/services/jobs/configurators/test_task.py @@ -98,9 +98,9 @@ async def test_with_commands_no_image(self, shell: Optional[str], expected_shell expected_shell, "-i", "-c", - "uv venv --python 3.12 --prompt workflow --seed /workflow/.venv > /dev/null 2>&1" - " && echo 'source /workflow/.venv/bin/activate' >> ~/.bashrc" - " && source /workflow/.venv/bin/activate" + "uv venv -q --prompt $DSTACK_RUN_NAME --seed -p 3.12 /dstack/venv" + " && echo '. /dstack/venv/bin/activate' >> /dstack/profile" + " && . /dstack/venv/bin/activate" " && sleep inf", ] From 6ecc3964483a748e06ccd3f419a72ae1347fa95f Mon Sep 17 00:00:00 2001 From: Dmitry Meyer Date: Sat, 30 Aug 2025 06:53:30 +0000 Subject: [PATCH 2/4] Fix runner test on macOS --- runner/internal/executor/executor_test.go | 7 +++++-- 1 file changed, 5 insertions(+), 2 deletions(-) diff --git a/runner/internal/executor/executor_test.go b/runner/internal/executor/executor_test.go index 351dc4941..410c577ce 100644 --- a/runner/internal/executor/executor_test.go +++ b/runner/internal/executor/executor_test.go @@ -22,10 +22,13 @@ import ( func TestExecutor_WorkingDir_Set(t *testing.T) { var b bytes.Buffer ex := makeTestExecutor(t) - workingDir := path.Join(t.TempDir(), "path/to/wd") + baseDir, err := filepath.EvalSymlinks(t.TempDir()) + require.NoError(t, err) + workingDir := path.Join(baseDir, "path/to/wd") + ex.jobSpec.WorkingDir = &workingDir ex.jobSpec.Commands = append(ex.jobSpec.Commands, "pwd") - err := ex.setJobWorkingDir(context.TODO()) + err = ex.setJobWorkingDir(context.TODO()) require.NoError(t, err) require.Equal(t, workingDir, ex.jobWorkingDir) err = os.MkdirAll(workingDir, 0o755) From 6248b5600b8e3a16f33a0f675ad013c3a0a9fff9 Mon Sep 17 00:00:00 2001 From: Dmitry Meyer Date: Mon, 1 Sep 2025 07:26:51 +0000 Subject: [PATCH 3/4] Update src/dstack/_internal/server/services/jobs/configurators/extensions/cursor.py Co-authored-by: Victor Skvortsov --- .../server/services/jobs/configurators/extensions/cursor.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/dstack/_internal/server/services/jobs/configurators/extensions/cursor.py b/src/dstack/_internal/server/services/jobs/configurators/extensions/cursor.py index 9227fa125..0703cba87 100644 --- a/src/dstack/_internal/server/services/jobs/configurators/extensions/cursor.py +++ b/src/dstack/_internal/server/services/jobs/configurators/extensions/cursor.py @@ -37,6 +37,6 @@ def get_print_readme_commands(self) -> List[str]: return [ "echo To open in Cursor, use link below:", "echo", - f'echo " vscode://vscode-remote/ssh-remote+{self.run_name}$DSTACK_REPO_DIR"', + f'echo " cursor://vscode-remote/ssh-remote+{self.run_name}$DSTACK_REPO_DIR"', "echo", ] From 0319d25b04115eb888e98988e22a84fea896e6fc Mon Sep 17 00:00:00 2001 From: Dmitry Meyer Date: Mon, 1 Sep 2025 08:48:05 +0000 Subject: [PATCH 4/4] Update legacy repo path warning --- src/dstack/_internal/cli/services/configurators/run.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/dstack/_internal/cli/services/configurators/run.py b/src/dstack/_internal/cli/services/configurators/run.py index 9974d10e9..3c7c03be9 100644 --- a/src/dstack/_internal/cli/services/configurators/run.py +++ b/src/dstack/_internal/cli/services/configurators/run.py @@ -122,7 +122,7 @@ def apply_configuration( warn( "[code]repos[0].path[/code] is not set," f" using legacy repo path [code]{LEGACY_REPO_DIR}[/code]\n\n" - "In future versions [code]path[/code] will be mandatory." + "In a future version the default value will be changed." f" To keep using [code]{LEGACY_REPO_DIR}[/code], explicitly set" f" [code]repos[0].path[/code] to [code]{LEGACY_REPO_DIR}[/code]\n" )