From a93771b4fe0c111102684270e8e6ed88c57eca99 Mon Sep 17 00:00:00 2001 From: Rada Kamysheva Date: Tue, 28 Jul 2026 13:05:17 +0000 Subject: [PATCH 01/23] testserver: roll task outcomes up into the run state The fake workspace reported every run as TERMINATED SUCCESS, overwriting the FAILED state it had just recorded for a task it executed locally. A run now reports the terminal state its tasks add up to, so a failing run can be exercised end to end locally. Tasks whose code the fake workspace does not have are left successful. An immutable deployment, for example, uploads the bundle as a snapshot zip that the fake workspace never unpacks, so there is nothing to execute; that gap is in the fake workspace, not in the job under test. Originally reviewed as #6082. --- libs/testserver/jobs.go | 57 ++++++++++++++++++++--------- libs/testserver/jobs_test.go | 71 ++++++++++++++++++++++++++++++++++++ 2 files changed, 111 insertions(+), 17 deletions(-) diff --git a/libs/testserver/jobs.go b/libs/testserver/jobs.go index 9097d43c086..e8d29877e98 100644 --- a/libs/testserver/jobs.go +++ b/libs/testserver/jobs.go @@ -20,6 +20,11 @@ import ( const missingJobGitProviderMessage = "git_source.git_provider must be one of: github,gitlab,bitbucketcloud,gitlabenterpriseedition,bitbucketserver,azuredevopsservices,githubenterprise,awscodecommit" +// errNoCodeInWorkspace marks a task there is nothing to execute for, e.g. +// because an immutable deployment uploaded the code as a snapshot zip this +// server never unpacks. The gap is here, not in the job, so the task succeeds. +var errNoCodeInWorkspace = errors.New("task code is not in the workspace") + // venvPython returns the path to the Python executable in a venv. // On Unix: venv/bin/python // On Windows: venv\Scripts\python.exe @@ -387,12 +392,15 @@ func (s *FakeWorkspace) JobsRunNow(req Request) Response { logs, err = s.executeSparkPythonTask(t) } - if err != nil { + switch { + case errors.Is(err, errNoCodeInWorkspace): + // Nothing ran, so the task keeps its SUCCESS state. + case err != nil: taskRun.State.ResultState = jobs.RunResultStateFailed s.JobRunOutputs[taskRunId] = jobs.RunOutput{ Error: err.Error(), } - } else if logs != "" { + case logs != "": s.JobRunOutputs[taskRunId] = jobs.RunOutput{ Logs: logs, } @@ -599,7 +607,7 @@ func (s *FakeWorkspace) executePythonWheelTask(jobSettings *jobs.JobSettings, ta } data := s.files[whlPath].Data if len(data) == 0 { - return "", fmt.Errorf("wheel file not found in workspace: %s", whlPath) + return "", fmt.Errorf("%w: wheel file not found in workspace: %s", errNoCodeInWorkspace, whlPath) } localPath := filepath.Join(env.dir, filepath.Base(whlPath)) if err := os.WriteFile(localPath, data, 0o644); err != nil { @@ -618,7 +626,7 @@ func (s *FakeWorkspace) executePythonWheelTask(jobSettings *jobs.JobSettings, ta } if len(env.installedLibs) == 0 { - return "", errors.New("no wheel libraries found in task") + return "", fmt.Errorf("%w: no wheel libraries found in task", errNoCodeInWorkspace) } // Run the entry point using runpy with sys.argv[0] set to the package name, @@ -664,7 +672,7 @@ func (s *FakeWorkspace) executeNotebookTask(task jobs.Task, notebookParams map[s notebookData = s.files[notebookPath+".py"].Data } if len(notebookData) == 0 { - return "", fmt.Errorf("notebook not found in workspace: %s (also tried .py)", notebookPath) + return "", fmt.Errorf("%w: notebook not found in workspace: %s (also tried .py)", errNoCodeInWorkspace, notebookPath) } // Create a temporary Python environment for notebook execution @@ -750,7 +758,7 @@ func (s *FakeWorkspace) executeSparkPythonTask(task jobs.Task) (string, error) { pythonData := s.files[pythonPath].Data if len(pythonData) == 0 { - return "", fmt.Errorf("python file not found in workspace: %s", pythonPath) + return "", fmt.Errorf("%w: python file not found in workspace: %s", errNoCodeInWorkspace, pythonPath) } env, cleanup, err := s.getOrCreateClusterEnv(task) @@ -848,6 +856,30 @@ func sparkVersionToPython(task jobs.Task) string { return "3.10" } +// terminateRun completes the run, rolling task outcomes up into the run-level +// state the way the Jobs API does: one failed task fails the whole run. +func terminateRun(run *jobs.Run) { + for i := range run.Tasks { + // Tasks that were never executed (jobs/runs/submit) are still running. + if run.Tasks[i].State.LifeCycleState != jobs.RunLifeCycleStateTerminated { + run.Tasks[i].State.LifeCycleState = jobs.RunLifeCycleStateTerminated + run.Tasks[i].State.ResultState = jobs.RunResultStateSuccess + } + } + + run.State = &jobs.RunState{ + LifeCycleState: jobs.RunLifeCycleStateTerminated, + ResultState: jobs.RunResultStateSuccess, + } + for _, task := range run.Tasks { + if task.State.ResultState != jobs.RunResultStateSuccess { + run.State.ResultState = task.State.ResultState + run.State.StateMessage = fmt.Sprintf("task %s failed", task.TaskKey) + return + } + } +} + func (s *FakeWorkspace) JobsGetRun(req Request) Response { runId := req.URL.Query().Get("run_id") runIdInt, err := strconv.ParseInt(runId, 10, 64) @@ -865,19 +897,10 @@ func (s *FakeWorkspace) JobsGetRun(req Request) Response { return Response{StatusCode: 404} } - // Simulate cloud behavior: first poll returns RUNNING, next returns TERMINATED SUCCESS. + // Simulate cloud behavior: first poll returns RUNNING, next the terminal state. if run.State.LifeCycleState == jobs.RunLifeCycleStateRunning { // Transition stored state to TERMINATED for the next poll. - run.State = &jobs.RunState{ - LifeCycleState: jobs.RunLifeCycleStateTerminated, - ResultState: jobs.RunResultStateSuccess, - } - for i := range run.Tasks { - run.Tasks[i].State = &jobs.RunState{ - LifeCycleState: jobs.RunLifeCycleStateTerminated, - ResultState: jobs.RunResultStateSuccess, - } - } + terminateRun(&run) s.JobRuns[runIdInt] = run // Return RUNNING for this poll (before the transition). diff --git a/libs/testserver/jobs_test.go b/libs/testserver/jobs_test.go index 38e47b68b08..27898c3664f 100644 --- a/libs/testserver/jobs_test.go +++ b/libs/testserver/jobs_test.go @@ -87,6 +87,77 @@ func TestJobsSubmit_RunReachesTerminalStateOnPoll(t *testing.T) { assert.Equal(t, jobs.RunResultStateSuccess, second.State.ResultState) } +func createJob(t *testing.T, workspace *FakeWorkspace, tasks ...jobs.Task) int64 { + t.Helper() + body, err := json.Marshal(jobs.CreateJob{Name: "my-job", Tasks: tasks}) + require.NoError(t, err) + + response := workspace.JobsCreate(Request{Body: body}) + require.Equal(t, 0, response.StatusCode) + return response.Body.(jobs.CreateResponse).JobId +} + +func runNow(t *testing.T, workspace *FakeWorkspace, request jobs.RunNow) Response { + t.Helper() + body, err := json.Marshal(request) + require.NoError(t, err) + return workspace.JobsRunNow(Request{Body: body}) +} + +func terminatedTask(taskKey string, result jobs.RunResultState) jobs.RunTask { + return jobs.RunTask{ + TaskKey: taskKey, + State: &jobs.RunState{ + LifeCycleState: jobs.RunLifeCycleStateTerminated, + ResultState: result, + }, + } +} + +func TestTerminateRun_FailedTaskFailsTheRun(t *testing.T) { + run := jobs.Run{Tasks: []jobs.RunTask{ + terminatedTask("first", jobs.RunResultStateSuccess), + terminatedTask("second", jobs.RunResultStateFailed), + }} + + terminateRun(&run) + + assert.Equal(t, jobs.RunLifeCycleStateTerminated, run.State.LifeCycleState) + assert.Equal(t, jobs.RunResultStateFailed, run.State.ResultState) + assert.Equal(t, "task second failed", run.State.StateMessage) +} + +func TestTerminateRun_CompletesTasksThatAreStillRunning(t *testing.T) { + // jobs/runs/submit records its tasks as running: they are never executed. + run := jobs.Run{Tasks: []jobs.RunTask{ + {TaskKey: "main", State: &jobs.RunState{LifeCycleState: jobs.RunLifeCycleStateRunning}}, + }} + + terminateRun(&run) + + assert.Equal(t, jobs.RunResultStateSuccess, run.State.ResultState) + assert.Empty(t, run.State.StateMessage) + assert.Equal(t, jobs.RunResultStateSuccess, run.Tasks[0].State.ResultState) +} + +// See errNoCodeInWorkspace: a missing notebook is this server's gap, not a +// failure of the job. +func TestJobsGetRun_TaskWithoutCodeDoesNotFailTheRun(t *testing.T) { + workspace := NewFakeWorkspace("http://test", "dbapi123") + jobID := createJob(t, workspace, jobs.Task{ + TaskKey: "main", + NotebookTask: &jobs.NotebookTask{NotebookPath: "/missing-notebook"}, + }) + + response := runNow(t, workspace, jobs.RunNow{JobId: jobID}) + require.Equal(t, 0, response.StatusCode) + runID := response.Body.(jobs.RunNowResponse).RunId + + // The first poll reports RUNNING, the second the terminal state. + require.Equal(t, jobs.RunLifeCycleStateRunning, getRun(t, workspace, runID).State.LifeCycleState) + assert.Equal(t, jobs.RunResultStateSuccess, getRun(t, workspace, runID).State.ResultState) +} + func TestJobsSubmit_RejectsInvalidGitProvider(t *testing.T) { workspace := NewFakeWorkspace("http://test", "dbapi123") From 94bd55675482da44918e15cdca0c3f8b3bdf07c8 Mon Sep 17 00:00:00 2001 From: Rada Kamysheva Date: Tue, 28 Jul 2026 13:05:34 +0000 Subject: [PATCH 02/23] job_runs: wait for run completion in WaitAfterCreate Deploying a job_run triggered the run and moved on, so a resource referencing the run's outcome saw whatever state the run happened to be in. The resource now implements the framework's WaitAfterCreate hook, which blocks until the run is terminal and republishes the settled remote state; only SUCCESS lets the deploy continue. While it waits, the deploy reports the run page URL and each state change, the way `bundle run` does, since a run can take hours. A run that does not succeed fails the deploy with the failed task, the message that task reported, and a link to the run page. Bounded by 24h, matching `bundle run`. The framework saves the run id before the wait, so a run that fails stays recorded and an unchanged config plans no second run; failed_run covers that. run_page_url is now normalized to the path form that also resolves for non-admins. invariant/configs/job_run.yml.tmpl loses its notebook task, which deploying it would now actually run, and is excluded from cloud runs: a real workspace reports a condition-task-only run as SKIPPED, and a task that does succeed would add a multi-minute cluster run to every variant of a suite that asserts plan and state invariants. Still covered locally. --- .../bundles/job-runs-wait-for-completion.md | 1 + .../bundle/invariant/configs/job_run.yml.tmpl | 13 +- acceptance/bundle/invariant/test.toml | 5 + .../resources/job_runs/basic/output.txt | 3 + .../job_runs/failed_run/databricks.yml | 34 ++++ .../resources/job_runs/failed_run/fail.py | 4 + .../job_runs/failed_run/out.test.toml | 3 + .../resources/job_runs/failed_run/output.txt | 57 ++++++ .../resources/job_runs/failed_run/script | 24 +++ .../resources/job_runs/failed_run/test.toml | 7 + .../job_runs/job_parameters/output.txt | 3 + .../resources/job_runs/redeploy/output.txt | 11 +- .../job_runs/wait_output/databricks.yml | 30 +++ .../job_runs/wait_output/out.test.toml | 3 + .../resources/job_runs/wait_output/output.txt | 91 +++++++++ .../resources/job_runs/wait_output/script | 18 ++ .../resources/job_runs/wait_output/test.toml | 4 + bundle/direct/dresources/all_test.go | 6 +- bundle/direct/dresources/job_run.go | 147 ++++++++++++++- bundle/direct/dresources/job_run_test.go | 173 ++++++++++++++++++ bundle/internal/schema/annotations.yml | 2 + bundle/schema/jsonschema.json | 2 +- 22 files changed, 627 insertions(+), 14 deletions(-) create mode 100644 .nextchanges/bundles/job-runs-wait-for-completion.md create mode 100644 acceptance/bundle/resources/job_runs/failed_run/databricks.yml create mode 100644 acceptance/bundle/resources/job_runs/failed_run/fail.py create mode 100644 acceptance/bundle/resources/job_runs/failed_run/out.test.toml create mode 100644 acceptance/bundle/resources/job_runs/failed_run/output.txt create mode 100644 acceptance/bundle/resources/job_runs/failed_run/script create mode 100644 acceptance/bundle/resources/job_runs/failed_run/test.toml create mode 100644 acceptance/bundle/resources/job_runs/wait_output/databricks.yml create mode 100644 acceptance/bundle/resources/job_runs/wait_output/out.test.toml create mode 100644 acceptance/bundle/resources/job_runs/wait_output/output.txt create mode 100644 acceptance/bundle/resources/job_runs/wait_output/script create mode 100644 acceptance/bundle/resources/job_runs/wait_output/test.toml create mode 100644 bundle/direct/dresources/job_run_test.go diff --git a/.nextchanges/bundles/job-runs-wait-for-completion.md b/.nextchanges/bundles/job-runs-wait-for-completion.md new file mode 100644 index 00000000000..e78497e24cb --- /dev/null +++ b/.nextchanges/bundles/job-runs-wait-for-completion.md @@ -0,0 +1 @@ +direct: the experimental `job_runs` resource now waits for the triggered run to finish, and fails the deploy if it does not succeed, so other resources can reference the run's outcome (e.g. `${resources.job_runs.nightly.state.result_state}`). The deploy reports the run page URL and each state change while it waits, and names the failed task and the message it reported when the run does not succeed. diff --git a/acceptance/bundle/invariant/configs/job_run.yml.tmpl b/acceptance/bundle/invariant/configs/job_run.yml.tmpl index a09987da02a..2fab76ff020 100644 --- a/acceptance/bundle/invariant/configs/job_run.yml.tmpl +++ b/acceptance/bundle/invariant/configs/job_run.yml.tmpl @@ -6,14 +6,13 @@ resources: foo: name: test-job-$UNIQUE_NAME tasks: + # Deploying a job_run actually runs the job, so use a condition task, + # which needs no workspace files or compute. - task_key: only_task - notebook_task: - notebook_path: /Shared/notebook - new_cluster: - spark_version: $DEFAULT_SPARK_VERSION - node_type_id: $NODE_TYPE_ID - instance_pool_id: $TEST_INSTANCE_POOL_ID - num_workers: 1 + condition_task: + op: EQUAL_TO + left: "1" + right: "1" job_runs: foo_run: diff --git a/acceptance/bundle/invariant/test.toml b/acceptance/bundle/invariant/test.toml index a53a93b84a6..cd7cb81dc4f 100644 --- a/acceptance/bundle/invariant/test.toml +++ b/acceptance/bundle/invariant/test.toml @@ -84,6 +84,11 @@ no_alert_on_cloud = ["CONFIG_Cloud=true", "INPUT_CONFIG=alert.yml.tmpl"] # so this config is local-only (the mock server stores it verbatim). no_run_job_ref_on_cloud = ["CONFIG_Cloud=true", "INPUT_CONFIG=job_run_job_ref.yml.tmpl"] +# Deploying a job_run waits for the run to succeed, and a real workspace reports +# a run of condition tasks alone as SKIPPED. A task that does succeed would add a +# cluster run to every variant of a suite that asserts plan and state invariants. +no_job_run_on_cloud = ["CONFIG_Cloud=true", "INPUT_CONFIG=job_run.yml.tmpl"] + # Postgres resources only work on AWS no_postgres_project_on_cloud = ["CONFIG_Cloud=true", "INPUT_CONFIG=postgres_project.yml.tmpl"] no_postgres_branch_on_cloud = ["CONFIG_Cloud=true", "INPUT_CONFIG=postgres_branch.yml.tmpl"] diff --git a/acceptance/bundle/resources/job_runs/basic/output.txt b/acceptance/bundle/resources/job_runs/basic/output.txt index 14018b93ec6..9ce260e8c85 100644 --- a/acceptance/bundle/resources/job_runs/basic/output.txt +++ b/acceptance/bundle/resources/job_runs/basic/output.txt @@ -34,6 +34,9 @@ Resources: >>> [CLI] bundle deploy Uploading bundle files to /Workspace/Users/[USERNAME]/.bundle/job-runs-basic/default/files... Deploying resources... +job run [MY_RUN_ID]: Run URL: [DATABRICKS_URL]/jobs/[MY_JOB_ID]/runs/[MY_RUN_ID]?o=[NUMID] +job run [MY_RUN_ID]: [TIMESTAMP] "my-job" RUNNING +job run [MY_RUN_ID]: [TIMESTAMP] "my-job" TERMINATED SUCCESS Updating deployment state... Deployment complete! diff --git a/acceptance/bundle/resources/job_runs/failed_run/databricks.yml b/acceptance/bundle/resources/job_runs/failed_run/databricks.yml new file mode 100644 index 00000000000..8a538daf853 --- /dev/null +++ b/acceptance/bundle/resources/job_runs/failed_run/databricks.yml @@ -0,0 +1,34 @@ +bundle: + name: job-runs-failed-run + +resources: + jobs: + my_job: + name: my-job + tasks: + # The test server runs this locally; the script exits non-zero, which + # fails the task and with it the run. + - task_key: main + spark_python_task: + python_file: ./fail.py + environment_key: default + + environments: + - environment_key: default + spec: + client: "2" + + # Depends on my_run's result_state, so the failing run aborts the deploy + # before this job is created. + downstream_job: + name: downstream-job + tags: + run_result: ${resources.job_runs.my_run.state.result_state} + tasks: + - task_key: main + notebook_task: + notebook_path: /Workspace/test + + job_runs: + my_run: + job_id: ${resources.jobs.my_job.id} diff --git a/acceptance/bundle/resources/job_runs/failed_run/fail.py b/acceptance/bundle/resources/job_runs/failed_run/fail.py new file mode 100644 index 00000000000..3262aa05529 --- /dev/null +++ b/acceptance/bundle/resources/job_runs/failed_run/fail.py @@ -0,0 +1,4 @@ +import sys + +print("intentional failure", file=sys.stderr) +sys.exit(1) diff --git a/acceptance/bundle/resources/job_runs/failed_run/out.test.toml b/acceptance/bundle/resources/job_runs/failed_run/out.test.toml new file mode 100644 index 00000000000..e90b6d5d1ba --- /dev/null +++ b/acceptance/bundle/resources/job_runs/failed_run/out.test.toml @@ -0,0 +1,3 @@ +Local = true +Cloud = false +EnvMatrix.DATABRICKS_BUNDLE_ENGINE = ["direct"] diff --git a/acceptance/bundle/resources/job_runs/failed_run/output.txt b/acceptance/bundle/resources/job_runs/failed_run/output.txt new file mode 100644 index 00000000000..e8a5098a0df --- /dev/null +++ b/acceptance/bundle/resources/job_runs/failed_run/output.txt @@ -0,0 +1,57 @@ + +=== a run that finishes FAILED fails the deploy +>>> [CLI] bundle deploy +Uploading bundle files to /Workspace/Users/[USERNAME]/.bundle/job-runs-failed-run/default/files... +Deploying resources... +job run [MY_RUN_ID]: Run URL: [DATABRICKS_URL]/jobs/[NUMID]/runs/[MY_RUN_ID]?o=[NUMID] +job run [MY_RUN_ID]: [TIMESTAMP] "my-job" RUNNING +job run [MY_RUN_ID]: [TIMESTAMP] "my-job" TERMINATED FAILED task main failed +Error: cannot create resources.job_runs.my_run: waiting after creating id=[MY_RUN_ID]: job run [MY_RUN_ID] did not succeed: FAILED: task main failed +task "main": spark python task execution failed: exit status 1 +intentional failure + +run page: [DATABRICKS_URL]/jobs/[NUMID]/runs/[MY_RUN_ID]?o=[NUMID] + +Error: cannot create resources.jobs.downstream_job: dependency failed: resources.job_runs.my_run + +Updating deployment state... + +=== the failed run stays recorded, so a redeploy starts no new run +>>> read_id.py my_run +[MY_RUN_ID] + +>>> [CLI] bundle plan +create jobs.downstream_job + +Plan: 1 to add, 0 to change, 0 to delete, 2 unchanged + +>>> [CLI] bundle deploy +Uploading bundle files to /Workspace/Users/[USERNAME]/.bundle/job-runs-failed-run/default/files... +Deploying resources... +Updating deployment state... +Deployment complete! + +=== downstream_job resolved its tag from the failed run +>>> jq -r select(.path == "/api/2.2/jobs/create" and .body.name == "downstream-job") | .body.tags.run_result out.requests.txt +FAILED + +=== run-now was issued once +>>> print_requests.py //jobs/run-now +{ + "method": "POST", + "path": "/api/2.2/jobs/run-now", + "body": { + "job_id": [NUMID] + } +} + +>>> [CLI] bundle destroy --auto-approve +The following resources will be deleted: + delete resources.job_runs.my_run + delete resources.jobs.downstream_job + delete resources.jobs.my_job + +All files and directories at the following location will be deleted: /Workspace/Users/[USERNAME]/.bundle/job-runs-failed-run/default + +Deleting files... +Destroy complete! diff --git a/acceptance/bundle/resources/job_runs/failed_run/script b/acceptance/bundle/resources/job_runs/failed_run/script new file mode 100644 index 00000000000..40dba04f8b3 --- /dev/null +++ b/acceptance/bundle/resources/job_runs/failed_run/script @@ -0,0 +1,24 @@ +cleanup() { + trace $CLI bundle destroy --auto-approve + rm -f out.requests.txt +} +trap cleanup EXIT + +# The run finishes FAILED, so the deploy aborts: the error names the failed task +# and the message it reported, and downstream_job is reported as a failed +# dependency because it reads the run's result_state. +title "a run that finishes FAILED fails the deploy" +musterr trace $CLI bundle deploy + +# The framework saves the run id before calling WaitAfterCreate, so a run that +# failed stays recorded and an unchanged config plans no second run. +title "the failed run stays recorded, so a redeploy starts no new run" +trace read_id.py my_run +trace $CLI bundle plan +trace $CLI bundle deploy + +title "downstream_job resolved its tag from the failed run" +trace jq -r 'select(.path == "/api/2.2/jobs/create" and .body.name == "downstream-job") | .body.tags.run_result' out.requests.txt + +title "run-now was issued once" +trace print_requests.py //jobs/run-now diff --git a/acceptance/bundle/resources/job_runs/failed_run/test.toml b/acceptance/bundle/resources/job_runs/failed_run/test.toml new file mode 100644 index 00000000000..03530b96c6f --- /dev/null +++ b/acceptance/bundle/resources/job_runs/failed_run/test.toml @@ -0,0 +1,7 @@ +# job_runs is a direct-engine-only resource; the Terraform provider has no +# equivalent, so restrict the matrix to direct. +EnvMatrix.DATABRICKS_BUNDLE_ENGINE = ["direct"] +RecordRequests = true + +# The deploy fails mid-way, leaving local deployment state behind. +Ignore = [".databricks"] diff --git a/acceptance/bundle/resources/job_runs/job_parameters/output.txt b/acceptance/bundle/resources/job_runs/job_parameters/output.txt index bcf3f21e017..986caf2b754 100644 --- a/acceptance/bundle/resources/job_runs/job_parameters/output.txt +++ b/acceptance/bundle/resources/job_runs/job_parameters/output.txt @@ -3,6 +3,9 @@ >>> [CLI] bundle deploy Uploading bundle files to /Workspace/Users/[USERNAME]/.bundle/job-runs-job-parameters/default/files... Deploying resources... +job run [NUMID]: Run URL: [DATABRICKS_URL]/jobs/[NUMID]/runs/[NUMID]?o=[NUMID] +job run [NUMID]: [TIMESTAMP] "my-job" RUNNING +job run [NUMID]: [TIMESTAMP] "my-job" TERMINATED SUCCESS Updating deployment state... Deployment complete! diff --git a/acceptance/bundle/resources/job_runs/redeploy/output.txt b/acceptance/bundle/resources/job_runs/redeploy/output.txt index 6ec19be8bce..8662b4880f0 100644 --- a/acceptance/bundle/resources/job_runs/redeploy/output.txt +++ b/acceptance/bundle/resources/job_runs/redeploy/output.txt @@ -3,6 +3,9 @@ >>> [CLI] bundle deploy Uploading bundle files to /Workspace/Users/[USERNAME]/.bundle/job-runs-redeploy/default/files... Deploying resources... +job run [NUMID]: Run URL: [DATABRICKS_URL]/jobs/[MY_JOB_ID]/runs/[NUMID]?o=[NUMID] +job run [NUMID]: [TIMESTAMP] "my-job" RUNNING +job run [NUMID]: [TIMESTAMP] "my-job" TERMINATED SUCCESS Updating deployment state... Deployment complete! @@ -64,10 +67,11 @@ Resources: }, "run_id": [NUMID], "run_name": "my-job", - "run_page_url": "[DATABRICKS_URL]/?o=[NUMID]#job/[MY_JOB_ID]/run/[NUMID]", + "run_page_url": "[DATABRICKS_URL]/jobs/[MY_JOB_ID]/runs/[NUMID]?o=[NUMID]", "run_type": "JOB_RUN", "state": { - "life_cycle_state": "RUNNING" + "life_cycle_state": "TERMINATED", + "result_state": "SUCCESS" } }, "changes": { @@ -84,6 +88,9 @@ Resources: >>> [CLI] bundle deploy Uploading bundle files to /Workspace/Users/[USERNAME]/.bundle/job-runs-redeploy/default/files... Deploying resources... +job run [NUMID]: Run URL: [DATABRICKS_URL]/jobs/[MY_JOB_ID]/runs/[NUMID]?o=[NUMID] +job run [NUMID]: [TIMESTAMP] "my-job" RUNNING +job run [NUMID]: [TIMESTAMP] "my-job" TERMINATED SUCCESS Updating deployment state... Deployment complete! diff --git a/acceptance/bundle/resources/job_runs/wait_output/databricks.yml b/acceptance/bundle/resources/job_runs/wait_output/databricks.yml new file mode 100644 index 00000000000..8b9d6691220 --- /dev/null +++ b/acceptance/bundle/resources/job_runs/wait_output/databricks.yml @@ -0,0 +1,30 @@ +bundle: + name: job-runs-wait-output + +resources: + jobs: + my_job: + name: my-job + tasks: + - task_key: main + condition_task: + op: EQUAL_TO + left: "1" + right: "1" + + # Reads the run's output. A job separate from my_job (the run's target), + # since my_run already depends on my_job.id and a reference back would cycle. + downstream_job: + name: downstream-job + tags: + run_result: ${resources.job_runs.my_run.state.result_state} + tasks: + - task_key: main + condition_task: + op: EQUAL_TO + left: "1" + right: "1" + + job_runs: + my_run: + job_id: ${resources.jobs.my_job.id} diff --git a/acceptance/bundle/resources/job_runs/wait_output/out.test.toml b/acceptance/bundle/resources/job_runs/wait_output/out.test.toml new file mode 100644 index 00000000000..e90b6d5d1ba --- /dev/null +++ b/acceptance/bundle/resources/job_runs/wait_output/out.test.toml @@ -0,0 +1,3 @@ +Local = true +Cloud = false +EnvMatrix.DATABRICKS_BUNDLE_ENGINE = ["direct"] diff --git a/acceptance/bundle/resources/job_runs/wait_output/output.txt b/acceptance/bundle/resources/job_runs/wait_output/output.txt new file mode 100644 index 00000000000..bd9def5e990 --- /dev/null +++ b/acceptance/bundle/resources/job_runs/wait_output/output.txt @@ -0,0 +1,91 @@ + +=== deploy waits for the run to finish, then the downstream job reads its result_state +>>> [CLI] bundle deploy +Uploading bundle files to /Workspace/Users/[USERNAME]/.bundle/job-runs-wait-output/default/files... +Deploying resources... +job run [NUMID]: Run URL: [DATABRICKS_URL]/jobs/[NUMID]/runs/[NUMID]?o=[NUMID] +job run [NUMID]: [TIMESTAMP] "my-job" RUNNING +job run [NUMID]: [TIMESTAMP] "my-job" TERMINATED SUCCESS +Updating deployment state... +Deployment complete! + +=== the downstream job was created with the run's result_state resolved into its tag +>>> print_requests.py //jobs/create +{ + "method": "POST", + "path": "/api/2.2/jobs/create", + "body": { + "deployment": { + "kind": "BUNDLE", + "metadata_file_path": "/Workspace/Users/[USERNAME]/.bundle/job-runs-wait-output/default/state/metadata.json" + }, + "edit_mode": "UI_LOCKED", + "format": "MULTI_TASK", + "max_concurrent_runs": 1, + "name": "my-job", + "queue": { + "enabled": true + }, + "tasks": [ + { + "condition_task": { + "left": "1", + "op": "EQUAL_TO", + "right": "1" + }, + "task_key": "main" + } + ] + } +} +{ + "method": "POST", + "path": "/api/2.2/jobs/create", + "body": { + "deployment": { + "kind": "BUNDLE", + "metadata_file_path": "/Workspace/Users/[USERNAME]/.bundle/job-runs-wait-output/default/state/metadata.json" + }, + "edit_mode": "UI_LOCKED", + "format": "MULTI_TASK", + "max_concurrent_runs": 1, + "name": "downstream-job", + "queue": { + "enabled": true + }, + "tags": { + "run_result": "SUCCESS" + }, + "tasks": [ + { + "condition_task": { + "left": "1", + "op": "EQUAL_TO", + "right": "1" + }, + "task_key": "main" + } + ] + } +} + +=== redeploy is a no-op: the resolved result_state tag is stable, not perpetual drift +>>> [CLI] bundle plan -o json +"skip" + +>>> [CLI] bundle deploy +Uploading bundle files to /Workspace/Users/[USERNAME]/.bundle/job-runs-wait-output/default/files... +Deploying resources... +Updating deployment state... +Deployment complete! + +>>> [CLI] bundle destroy --auto-approve +The following resources will be deleted: + delete resources.job_runs.my_run + delete resources.jobs.downstream_job + delete resources.jobs.my_job + +All files and directories at the following location will be deleted: /Workspace/Users/[USERNAME]/.bundle/job-runs-wait-output/default + +Deleting files... +Destroy complete! diff --git a/acceptance/bundle/resources/job_runs/wait_output/script b/acceptance/bundle/resources/job_runs/wait_output/script new file mode 100644 index 00000000000..85812d8c18b --- /dev/null +++ b/acceptance/bundle/resources/job_runs/wait_output/script @@ -0,0 +1,18 @@ +cleanup() { + trace $CLI bundle destroy --auto-approve + rm -f out.requests.txt +} +trap cleanup EXIT + +title "deploy waits for the run to finish, then the downstream job reads its result_state" +trace $CLI bundle deploy + +title "the downstream job was created with the run's result_state resolved into its tag" +# A concrete SUCCESS tag proves the run finished and the wait published its +# output before the downstream job was created. +trace print_requests.py //jobs/create + +title "redeploy is a no-op: the resolved result_state tag is stable, not perpetual drift" +# The resolved tag reads back identically, so the job stays as deployed. +trace $CLI bundle plan -o json | jq '.plan["resources.jobs.downstream_job"].action // "none"' +trace $CLI bundle deploy diff --git a/acceptance/bundle/resources/job_runs/wait_output/test.toml b/acceptance/bundle/resources/job_runs/wait_output/test.toml new file mode 100644 index 00000000000..4b94d8b58e9 --- /dev/null +++ b/acceptance/bundle/resources/job_runs/wait_output/test.toml @@ -0,0 +1,4 @@ +# job_runs is a direct-engine-only resource; the Terraform provider has no +# equivalent, so restrict the matrix to direct. +EnvMatrix.DATABRICKS_BUNDLE_ENGINE = ["direct"] +RecordRequests = true diff --git a/bundle/direct/dresources/all_test.go b/bundle/direct/dresources/all_test.go index 54541d94ba9..89d690dfead 100644 --- a/bundle/direct/dresources/all_test.go +++ b/bundle/direct/dresources/all_test.go @@ -1003,7 +1003,11 @@ func testCRUD(t *testing.T, group string, adapter *Adapter, client *databricks.W remoteStateFromWaitCreate, err := adapter.WaitAfterCreate(ctx, createdID, newState) require.NoError(t, err) if remoteStateFromWaitCreate != nil { - require.Equal(t, remote, remoteStateFromWaitCreate) + // WaitAfterCreate returns the settled state; the read right after DoCreate + // may still be non-terminal, so compare against a fresh read, not that one. + remotePostWaitCreate, err := adapter.DoRead(ctx, createdID) + require.NoError(t, err) + require.Equal(t, remotePostWaitCreate, remoteStateFromWaitCreate) } if adapter.HasDoUpdate() { diff --git a/bundle/direct/dresources/job_run.go b/bundle/direct/dresources/job_run.go index 0a2ae0ea6af..f5b53dafc63 100644 --- a/bundle/direct/dresources/job_run.go +++ b/bundle/direct/dresources/job_run.go @@ -1,16 +1,28 @@ package dresources import ( + "cmp" "context" + "errors" "fmt" "strconv" + "strings" + "time" "github.com/databricks/cli/bundle/config/resources" + "github.com/databricks/cli/bundle/run/progress" + "github.com/databricks/cli/libs/cmdio" + "github.com/databricks/cli/libs/log" + "github.com/databricks/cli/libs/workspaceurls" "github.com/databricks/databricks-sdk-go" "github.com/databricks/databricks-sdk-go/marshal" "github.com/databricks/databricks-sdk-go/service/jobs" ) +// jobRunTimeout bounds the wait for a run to finish, matching `bundle run` +// (jobRunTimeout in bundle/run/job.go). +const jobRunTimeout = 24 * time.Hour + // JobRunState is what we persist for a triggered run: the RunNow request. type JobRunState struct { jobs.RunNow @@ -100,7 +112,7 @@ func makeJobRunRemote(run *jobs.Run) *JobRunRemote { RunId: run.RunId, RunName: run.RunName, State: run.State, - RunPageUrl: run.RunPageUrl, + RunPageUrl: workspaceurls.ModernizeJobRunPageURL(run.RunPageUrl), RunType: run.RunType, } } @@ -138,12 +150,141 @@ func (r *ResourceJobRun) DoCreate(ctx context.Context, config *JobRunState) (str return strconv.FormatInt(wait.RunId, 10), nil, nil } +// WaitAfterCreate blocks until the triggered run finishes, so a resource that +// references this run's output (e.g. state.result_state) is created only once the +// run has produced it. Only a SUCCESS lets the deploy continue. +func (r *ResourceJobRun) WaitAfterCreate(ctx context.Context, id string, _ *JobRunState) (*JobRunRemote, error) { + runID, err := parseRunID(id) + if err != nil { + return nil, err + } + return r.waitForRun(ctx, runID) +} + +// waitForRun blocks until the run reaches a terminal state and returns its +// remote view; only SUCCESS returns a nil error. +func (r *ResourceJobRun) waitForRun(ctx context.Context, runID int64) (*JobRunRemote, error) { + // A run can take hours, so report progress like `bundle run` does. pageURL + // outlives the callback so an abandoned wait can still link the run. + var prevState *jobs.RunState + var pageURL string + run, err := r.client.Jobs.WaitGetRunJobTerminatedOrSkipped(ctx, runID, jobRunTimeout, func(run *jobs.Run) { + pageURL = run.RunPageUrl + prevState = logRunProgress(ctx, run, prevState) + }) + if err != nil { + // The run hit INTERNAL_ERROR, or we gave up on timeout or interrupt while it + // kept going; either way the run id is what makes the error actionable. + return nil, fmt.Errorf("waiting for job run %d: %w%s", runID, err, runPageLine(pageURL)) + } + // FAILED, TIMEDOUT, CANCELED, SUCCESS_WITH_FAILURES and SKIPPED all fail the + // deploy; the waiter already errored on INTERNAL_ERROR and on timeout. + if run.State.ResultState != jobs.RunResultStateSuccess { + return nil, r.runFailedError(ctx, run) + } + return makeJobRunRemote(run), nil +} + +// runFailedError reports why the run did not succeed, naming each failed task +// and the error it reported. +func (r *ResourceJobRun) runFailedError(ctx context.Context, run *jobs.Run) error { + outcome := string(run.State.ResultState) + if outcome == "" { + // A skipped run has no result_state; report the lifecycle state. + outcome = string(run.State.LifeCycleState) + } + var msg strings.Builder + fmt.Fprintf(&msg, "job run %d did not succeed: %s", run.RunId, outcome) + if run.State.StateMessage != "" { + fmt.Fprintf(&msg, ": %s", run.State.StateMessage) + } + for _, task := range run.Tasks { + if taskFailed(task) { + fmt.Fprintf(&msg, "\ntask %q: %s", task.TaskKey, r.taskError(ctx, task)) + } + } + msg.WriteString(runPageLine(run.RunPageUrl)) + return errors.New(msg.String()) +} + +// taskFailed reports whether a task caused the run to fail rather than being a +// casualty of it. Tasks left SKIPPED or UPSTREAM_FAILED by an earlier failure +// add noise without naming the problem. +func taskFailed(task jobs.RunTask) bool { + // State is deprecated in favour of Status, so it may be absent. + if task.State == nil { + return false + } + return task.State.LifeCycleState == jobs.RunLifeCycleStateInternalError || + task.State.ResultState == jobs.RunResultStateFailed || + task.State.ResultState == jobs.RunResultStateTimedout +} + +// taskError returns the message the task reported, from the same place +// `bundle run` reads it. Only called for tasks that taskFailed accepted, so +// State is set. +func (r *ResourceJobRun) taskError(ctx context.Context, task jobs.RunTask) string { + var reported string + output, err := r.client.Jobs.GetRunOutput(ctx, jobs.GetRunOutputRequest{RunId: task.RunId}) + if err != nil { + log.Debugf(ctx, "could not read output of task %s: %v", task.TaskKey, err) + } else { + reported = output.Error + } + // Not every task type reports an error through GetRunOutput, so fall back to + // what the run itself says about the task. + return cmp.Or(reported, task.State.StateMessage, string(task.State.ResultState), string(task.State.LifeCycleState)) +} + +// runPageLine returns a line linking the run page, or an empty string when the +// URL is unknown. +func runPageLine(rawURL string) string { + if rawURL == "" { + return "" + } + return "\nrun page: " + workspaceurls.ModernizeJobRunPageURL(rawURL) +} + +// logRunProgress mirrors `bundle run`'s monitor: the run page URL once, then +// each state change. It returns the state to remember for the next poll. +func logRunProgress(ctx context.Context, run *jobs.Run, prev *jobs.RunState) *jobs.RunState { + if run.State == nil { + return prev + } + if prev != nil && + prev.LifeCycleState == run.State.LifeCycleState && + prev.ResultState == run.State.ResultState { + return prev + } + if prev == nil && run.RunPageUrl != "" { + logRunLine(ctx, run.RunId, "Run URL: "+workspaceurls.ModernizeJobRunPageURL(run.RunPageUrl)) + } + logRunLine(ctx, run.RunId, (&progress.JobProgressEvent{ + Timestamp: time.Now(), + JobId: run.JobId, + RunId: run.RunId, + RunName: run.RunName, + State: *run.State, + }).String()) + return run.State +} + +// logRunLine reports one line about a run to the user and the log. Resources +// deploy concurrently onto one stream, so the user-facing copy names the run it +// describes; the log already carries the resource key via log.WithPrefix. +func logRunLine(ctx context.Context, runID int64, msg string) { + log.Info(ctx, msg) + if cmdio.HasIO(ctx) { + cmdio.LogString(ctx, fmt.Sprintf("job run %d: %s", runID, msg)) + } +} + // DoUpdate is intentionally not implemented: a run can't be modified in place, // so any change recreates it (delete + a fresh RunNow). // DoDelete deletes the run via jobs/runs/delete, on both destroy and the -// recreate path. The API rejects a still-active run; this milestone doesn't -// await completion, so that error surfaces to the user. +// recreate path. The API rejects a still-active run, which WaitAfterCreate +// leaves terminal; that error surfaces for a run whose wait was interrupted. func (r *ResourceJobRun) DoDelete(ctx context.Context, id string, _ *JobRunState) error { runID, err := parseRunID(id) if err != nil { diff --git a/bundle/direct/dresources/job_run_test.go b/bundle/direct/dresources/job_run_test.go new file mode 100644 index 00000000000..0177b077f2a --- /dev/null +++ b/bundle/direct/dresources/job_run_test.go @@ -0,0 +1,173 @@ +package dresources + +import ( + "context" + "sync/atomic" + "testing" + "time" + + "github.com/databricks/cli/libs/testserver" + "github.com/databricks/databricks-sdk-go" + "github.com/databricks/databricks-sdk-go/service/jobs" + "github.com/stretchr/testify/assert" + "github.com/stretchr/testify/require" +) + +// jobRunClientFor returns a client talking to server. Call it after the test +// registers its own handlers: first registration wins, so the defaults added here +// only fill the gaps. +func jobRunClientFor(t *testing.T, server *testserver.Server) *databricks.WorkspaceClient { + t.Helper() + testserver.AddDefaultHandlers(server) + + client, err := databricks.NewWorkspaceClient(&databricks.Config{ + Host: server.URL, + Token: "testtoken", + }) + require.NoError(t, err) + return client +} + +// jobRunServer returns a test server whose runs/get handler is the given one, +// so a wait can be exercised without a real run. +func jobRunServer(t *testing.T, getRun testserver.HandlerFunc) *databricks.WorkspaceClient { + t.Helper() + server := testserver.New(t) + server.Handle("GET", "/api/2.2/jobs/runs/get", getRun) + return jobRunClientFor(t, server) +} + +// jobRunClient returns a client whose GetRun always reports the given run state. +func jobRunClient(t *testing.T, state *jobs.RunState) *databricks.WorkspaceClient { + t.Helper() + return jobRunServer(t, func(req testserver.Request) any { + return jobs.Run{RunId: 123, JobId: 456, State: state} + }) +} + +// waitForTestRun drives the framework hook, so it covers parsing the id the +// framework hands back from DoCreate along with the wait itself. +func waitForTestRun(t *testing.T, ctx context.Context, client *databricks.WorkspaceClient) (*JobRunRemote, error) { + t.Helper() + r := (&ResourceJobRun{}).New(client) + return r.WaitAfterCreate(ctx, "123", &JobRunState{}) +} + +func TestJobRunWaitSucceeds(t *testing.T) { + client := jobRunClient(t, &jobs.RunState{ + LifeCycleState: jobs.RunLifeCycleStateTerminated, + ResultState: jobs.RunResultStateSuccess, + }) + + remote, err := waitForTestRun(t, t.Context(), client) + + require.NoError(t, err) + require.NotNil(t, remote.State) + assert.Equal(t, jobs.RunResultStateSuccess, remote.State.ResultState) +} + +func TestJobRunWaitFailsOnFailedResult(t *testing.T) { + client := jobRunClient(t, &jobs.RunState{ + LifeCycleState: jobs.RunLifeCycleStateTerminated, + ResultState: jobs.RunResultStateFailed, + StateMessage: "task failed", + }) + + _, err := waitForTestRun(t, t.Context(), client) + + // Only SUCCESS completes the deploy; a FAILED result fails it. + require.ErrorContains(t, err, "did not succeed: FAILED: task failed") +} + +func TestJobRunWaitReportsFailedTask(t *testing.T) { + failed := &jobs.RunState{ + LifeCycleState: jobs.RunLifeCycleStateTerminated, + ResultState: jobs.RunResultStateFailed, + } + server := testserver.New(t) + server.Handle("GET", "/api/2.2/jobs/runs/get", func(req testserver.Request) any { + return jobs.Run{ + RunId: 123, + JobId: 456, + State: failed, + Tasks: []jobs.RunTask{ + {TaskKey: "ok", RunId: 998, State: &jobs.RunState{ + LifeCycleState: jobs.RunLifeCycleStateTerminated, + ResultState: jobs.RunResultStateSuccess, + }}, + {TaskKey: "main", RunId: 999, State: failed}, + }, + } + }) + server.Handle("GET", "/api/2.2/jobs/runs/get-output", func(req testserver.Request) any { + return jobs.RunOutput{Error: "notebook not found"} + }) + + _, err := waitForTestRun(t, t.Context(), jobRunClientFor(t, server)) + + // The error names the failing task and the message it reported, and leaves out + // the tasks that did not fail. + require.ErrorContains(t, err, `task "main": notebook not found`) + assert.NotContains(t, err.Error(), `task "ok"`) +} + +func TestJobRunWaitFailsOnSkipped(t *testing.T) { + // A skipped run has no result_state, so the lifecycle state is reported. + client := jobRunClient(t, &jobs.RunState{ + LifeCycleState: jobs.RunLifeCycleStateSkipped, + }) + + _, err := waitForTestRun(t, t.Context(), client) + + require.ErrorContains(t, err, "did not succeed: SKIPPED") +} + +func TestJobRunWaitFailsOnInternalError(t *testing.T) { + client := jobRunClient(t, &jobs.RunState{ + LifeCycleState: jobs.RunLifeCycleStateInternalError, + }) + + _, err := waitForTestRun(t, t.Context(), client) + + // The SDK waiter errors on INTERNAL_ERROR, ahead of the result check, so the + // wrapping is all that names the run. + require.ErrorContains(t, err, "waiting for job run 123") + require.ErrorContains(t, err, "INTERNAL_ERROR") +} + +func TestJobRunWaitAbandonedNamesTheRun(t *testing.T) { + client := jobRunClient(t, &jobs.RunState{LifeCycleState: jobs.RunLifeCycleStateRunning}) + + ctx, cancel := context.WithTimeout(t.Context(), time.Millisecond) + defer cancel() + + _, err := waitForTestRun(t, ctx, client) + + // Giving up on the wait does not stop the run, so the error has to name it. + require.ErrorContains(t, err, "waiting for job run 123") +} + +// Reporting RUNNING for the first two polls exercises the poll loop; the other +// tests stub an already-terminal state. +func TestJobRunWaitPollsUntilTerminal(t *testing.T) { + var gets atomic.Int32 + client := jobRunServer(t, func(req testserver.Request) any { + if gets.Add(1) <= 2 { + return jobs.Run{RunId: 123, JobId: 456, State: &jobs.RunState{ + LifeCycleState: jobs.RunLifeCycleStateRunning, + }} + } + return jobs.Run{RunId: 123, JobId: 456, State: &jobs.RunState{ + LifeCycleState: jobs.RunLifeCycleStateTerminated, + ResultState: jobs.RunResultStateSuccess, + }} + }) + + remote, err := waitForTestRun(t, t.Context(), client) + require.NoError(t, err) + + // SUCCESS is only reachable by polling past the RUNNING reads. + require.NotNil(t, remote.State) + assert.Equal(t, jobs.RunResultStateSuccess, remote.State.ResultState) + assert.GreaterOrEqual(t, gets.Load(), int32(2), "expected the wait to poll more than once") +} diff --git a/bundle/internal/schema/annotations.yml b/bundle/internal/schema/annotations.yml index 4db1d40524d..77edbf0802b 100644 --- a/bundle/internal/schema/annotations.yml +++ b/bundle/internal/schema/annotations.yml @@ -968,6 +968,8 @@ resources: "job_runs": "description": |- The job run definitions for the bundle, where each key is the name of the job run. Each job run triggers a run of an existing job as part of bundle deployment. + + The deployment waits for the run to finish and fails if it does not succeed, so other resources can reference the run's outcome, for example `${resources.job_runs..state.result_state}`. A run that did not succeed is still recorded, so an unchanged configuration is not run again. "$fields": "lifecycle": "description": |- diff --git a/bundle/schema/jsonschema.json b/bundle/schema/jsonschema.json index 9c41d1a0b9b..6f11e620828 100644 --- a/bundle/schema/jsonschema.json +++ b/bundle/schema/jsonschema.json @@ -3270,7 +3270,7 @@ "markdownDescription": "The instance pool definitions for the bundle, where each key is the name of the instance pool. See [instance_pools](https://docs.databricks.com/dev-tools/bundles/resources.html#instance_pools)." }, "job_runs": { - "description": "The job run definitions for the bundle, where each key is the name of the job run. Each job run triggers a run of an existing job as part of bundle deployment.", + "description": "The job run definitions for the bundle, where each key is the name of the job run. Each job run triggers a run of an existing job as part of bundle deployment.\n\nThe deployment waits for the run to finish and fails if it does not succeed, so other resources can reference the run's outcome, for example `${resources.job_runs.\u003cname\u003e.state.result_state}`. A run that did not succeed is still recorded, so an unchanged configuration is not run again.", "$ref": "#/$defs/map/github.com/databricks/cli/bundle/config/resources.JobRun" }, "jobs": { From a65e0ea91676a59b5aced420b1154d53f208bc93 Mon Sep 17 00:00:00 2001 From: Rada Kamysheva Date: Wed, 29 Jul 2026 12:02:50 +0000 Subject: [PATCH 03/23] job_runs: report run progress through a tracker shared with bundle run The wait reimplemented what `bundle run`'s monitor already does: report the run page URL once, then each state change. Both now go through progress.JobStateTracker, which decides what a poll is worth reporting; the two callers keep their own sinks, since a concurrent deploy reports plain prefixed lines where `bundle run` reports progress events. The failure the deploy reports no longer repeats the run id that the framework's wrapper already carries; what the wait adds is the link to a run that outlives it. --- .../bundles/job-runs-wait-for-completion.md | 2 +- .../resources/job_runs/failed_run/output.txt | 2 +- bundle/direct/dresources/job_run.go | 59 +++++++------------ bundle/direct/dresources/job_run_test.go | 23 +++++--- bundle/run/job.go | 27 ++------- bundle/run/progress/job.go | 27 +++++++++ bundle/run/progress/job_test.go | 33 +++++++++++ 7 files changed, 103 insertions(+), 70 deletions(-) diff --git a/.nextchanges/bundles/job-runs-wait-for-completion.md b/.nextchanges/bundles/job-runs-wait-for-completion.md index e78497e24cb..4520d1d9180 100644 --- a/.nextchanges/bundles/job-runs-wait-for-completion.md +++ b/.nextchanges/bundles/job-runs-wait-for-completion.md @@ -1 +1 @@ -direct: the experimental `job_runs` resource now waits for the triggered run to finish, and fails the deploy if it does not succeed, so other resources can reference the run's outcome (e.g. `${resources.job_runs.nightly.state.result_state}`). The deploy reports the run page URL and each state change while it waits, and names the failed task and the message it reported when the run does not succeed. +direct: the experimental `job_runs` resource now waits for the triggered run to finish, so other resources can reference the run's outcome (e.g. `${resources.job_runs.nightly.state.result_state}`). A run that does not succeed fails the deploy, naming the failed task and the message it reported; while waiting, the deploy reports the run page URL and each state change. diff --git a/acceptance/bundle/resources/job_runs/failed_run/output.txt b/acceptance/bundle/resources/job_runs/failed_run/output.txt index e8a5098a0df..78b95c98da1 100644 --- a/acceptance/bundle/resources/job_runs/failed_run/output.txt +++ b/acceptance/bundle/resources/job_runs/failed_run/output.txt @@ -6,7 +6,7 @@ Deploying resources... job run [MY_RUN_ID]: Run URL: [DATABRICKS_URL]/jobs/[NUMID]/runs/[MY_RUN_ID]?o=[NUMID] job run [MY_RUN_ID]: [TIMESTAMP] "my-job" RUNNING job run [MY_RUN_ID]: [TIMESTAMP] "my-job" TERMINATED FAILED task main failed -Error: cannot create resources.job_runs.my_run: waiting after creating id=[MY_RUN_ID]: job run [MY_RUN_ID] did not succeed: FAILED: task main failed +Error: cannot create resources.job_runs.my_run: waiting after creating id=[MY_RUN_ID]: run did not succeed: FAILED: task main failed task "main": spark python task execution failed: exit status 1 intentional failure diff --git a/bundle/direct/dresources/job_run.go b/bundle/direct/dresources/job_run.go index f5b53dafc63..c811af9bf21 100644 --- a/bundle/direct/dresources/job_run.go +++ b/bundle/direct/dresources/job_run.go @@ -150,32 +150,27 @@ func (r *ResourceJobRun) DoCreate(ctx context.Context, config *JobRunState) (str return strconv.FormatInt(wait.RunId, 10), nil, nil } -// WaitAfterCreate blocks until the triggered run finishes, so a resource that -// references this run's output (e.g. state.result_state) is created only once the -// run has produced it. Only a SUCCESS lets the deploy continue. +// WaitAfterCreate blocks until the run finishes, so a resource referencing its +// output (e.g. state.result_state) sees a settled run. Only SUCCESS lets the +// deploy continue. func (r *ResourceJobRun) WaitAfterCreate(ctx context.Context, id string, _ *JobRunState) (*JobRunRemote, error) { runID, err := parseRunID(id) if err != nil { return nil, err } - return r.waitForRun(ctx, runID) -} -// waitForRun blocks until the run reaches a terminal state and returns its -// remote view; only SUCCESS returns a nil error. -func (r *ResourceJobRun) waitForRun(ctx context.Context, runID int64) (*JobRunRemote, error) { // A run can take hours, so report progress like `bundle run` does. pageURL // outlives the callback so an abandoned wait can still link the run. - var prevState *jobs.RunState + var tracker progress.JobStateTracker var pageURL string run, err := r.client.Jobs.WaitGetRunJobTerminatedOrSkipped(ctx, runID, jobRunTimeout, func(run *jobs.Run) { pageURL = run.RunPageUrl - prevState = logRunProgress(ctx, run, prevState) + logRunProgress(ctx, run, &tracker) }) if err != nil { - // The run hit INTERNAL_ERROR, or we gave up on timeout or interrupt while it - // kept going; either way the run id is what makes the error actionable. - return nil, fmt.Errorf("waiting for job run %d: %w%s", runID, err, runPageLine(pageURL)) + // The wait can end with the run still going (timeout, interrupt), so link + // the run page; the framework's wrapper carries the id. + return nil, fmt.Errorf("%w%s", err, runPageLine(pageURL)) } // FAILED, TIMEDOUT, CANCELED, SUCCESS_WITH_FAILURES and SKIPPED all fail the // deploy; the waiter already errored on INTERNAL_ERROR and on timeout. @@ -194,7 +189,8 @@ func (r *ResourceJobRun) runFailedError(ctx context.Context, run *jobs.Run) erro outcome = string(run.State.LifeCycleState) } var msg strings.Builder - fmt.Fprintf(&msg, "job run %d did not succeed: %s", run.RunId, outcome) + // The framework already prefixes the resource key and the run id. + fmt.Fprintf(&msg, "run did not succeed: %s", outcome) if run.State.StateMessage != "" { fmt.Fprintf(&msg, ": %s", run.State.StateMessage) } @@ -208,8 +204,7 @@ func (r *ResourceJobRun) runFailedError(ctx context.Context, run *jobs.Run) erro } // taskFailed reports whether a task caused the run to fail rather than being a -// casualty of it. Tasks left SKIPPED or UPSTREAM_FAILED by an earlier failure -// add noise without naming the problem. +// casualty of it: tasks left SKIPPED or UPSTREAM_FAILED add noise. func taskFailed(task jobs.RunTask) bool { // State is deprecated in favour of Status, so it may be absent. if task.State == nil { @@ -220,9 +215,8 @@ func taskFailed(task jobs.RunTask) bool { task.State.ResultState == jobs.RunResultStateTimedout } -// taskError returns the message the task reported, from the same place -// `bundle run` reads it. Only called for tasks that taskFailed accepted, so -// State is set. +// taskError returns the message the task reported, from the same place `bundle +// run` reads it. Only called for tasks taskFailed accepted, so State is set. func (r *ResourceJobRun) taskError(ctx context.Context, task jobs.RunTask) string { var reported string output, err := r.client.Jobs.GetRunOutput(ctx, jobs.GetRunOutputRequest{RunId: task.RunId}) @@ -245,28 +239,17 @@ func runPageLine(rawURL string) string { return "\nrun page: " + workspaceurls.ModernizeJobRunPageURL(rawURL) } -// logRunProgress mirrors `bundle run`'s monitor: the run page URL once, then -// each state change. It returns the state to remember for the next poll. -func logRunProgress(ctx context.Context, run *jobs.Run, prev *jobs.RunState) *jobs.RunState { - if run.State == nil { - return prev - } - if prev != nil && - prev.LifeCycleState == run.State.LifeCycleState && - prev.ResultState == run.State.ResultState { - return prev +// logRunProgress reports what `bundle run` reports: the run page URL once, then +// each state change. +func logRunProgress(ctx context.Context, run *jobs.Run, tracker *progress.JobStateTracker) { + event, first := tracker.Poll(run) + if event == nil { + return } - if prev == nil && run.RunPageUrl != "" { + if first && run.RunPageUrl != "" { logRunLine(ctx, run.RunId, "Run URL: "+workspaceurls.ModernizeJobRunPageURL(run.RunPageUrl)) } - logRunLine(ctx, run.RunId, (&progress.JobProgressEvent{ - Timestamp: time.Now(), - JobId: run.JobId, - RunId: run.RunId, - RunName: run.RunName, - State: *run.State, - }).String()) - return run.State + logRunLine(ctx, run.RunId, event.String()) } // logRunLine reports one line about a run to the user and the log. Resources diff --git a/bundle/direct/dresources/job_run_test.go b/bundle/direct/dresources/job_run_test.go index 0177b077f2a..e9c29366136 100644 --- a/bundle/direct/dresources/job_run_test.go +++ b/bundle/direct/dresources/job_run_test.go @@ -37,11 +37,18 @@ func jobRunServer(t *testing.T, getRun testserver.HandlerFunc) *databricks.Works return jobRunClientFor(t, server) } +// The Jobs API reports the run page in the legacy fragment form; errors and +// progress lines carry the path form it converts to. +const ( + testRunPageURL = "https://myworkspace.databricks.test/?o=900800700600#job/456/run/123" + testRunPageLink = "run page: https://myworkspace.databricks.test/jobs/456/runs/123?o=900800700600" +) + // jobRunClient returns a client whose GetRun always reports the given run state. func jobRunClient(t *testing.T, state *jobs.RunState) *databricks.WorkspaceClient { t.Helper() return jobRunServer(t, func(req testserver.Request) any { - return jobs.Run{RunId: 123, JobId: 456, State: state} + return jobs.Run{RunId: 123, JobId: 456, State: state, RunPageUrl: testRunPageURL} }) } @@ -129,13 +136,12 @@ func TestJobRunWaitFailsOnInternalError(t *testing.T) { _, err := waitForTestRun(t, t.Context(), client) - // The SDK waiter errors on INTERNAL_ERROR, ahead of the result check, so the - // wrapping is all that names the run. - require.ErrorContains(t, err, "waiting for job run 123") + // The SDK waiter errors on INTERNAL_ERROR, ahead of the result check. require.ErrorContains(t, err, "INTERNAL_ERROR") + require.ErrorContains(t, err, testRunPageLink) } -func TestJobRunWaitAbandonedNamesTheRun(t *testing.T) { +func TestJobRunWaitAbandonedLinksTheRun(t *testing.T) { client := jobRunClient(t, &jobs.RunState{LifeCycleState: jobs.RunLifeCycleStateRunning}) ctx, cancel := context.WithTimeout(t.Context(), time.Millisecond) @@ -143,8 +149,9 @@ func TestJobRunWaitAbandonedNamesTheRun(t *testing.T) { _, err := waitForTestRun(t, ctx, client) - // Giving up on the wait does not stop the run, so the error has to name it. - require.ErrorContains(t, err, "waiting for job run 123") + // Giving up on the wait does not stop the run, so the error links to it. + require.Error(t, err) + require.ErrorContains(t, err, testRunPageLink) } // Reporting RUNNING for the first two polls exercises the poll loop; the other @@ -169,5 +176,5 @@ func TestJobRunWaitPollsUntilTerminal(t *testing.T) { // SUCCESS is only reachable by polling past the RUNNING reads. require.NotNil(t, remote.State) assert.Equal(t, jobs.RunResultStateSuccess, remote.State.ResultState) - assert.GreaterOrEqual(t, gets.Load(), int32(2), "expected the wait to poll more than once") + assert.Equal(t, int32(3), gets.Load(), "expected the wait to poll past both RUNNING reads") } diff --git a/bundle/run/job.go b/bundle/run/job.go index 98f059e5784..38534cf544d 100644 --- a/bundle/run/job.go +++ b/bundle/run/job.go @@ -84,42 +84,25 @@ func (r *jobRunner) logFailedTasks(ctx context.Context, runId int64) { // jobRunMonitor tracks state for a single job run and provides callbacks // for monitoring progress. type jobRunMonitor struct { - ctx context.Context - prevState *jobs.RunState + ctx context.Context + tracker progress.JobStateTracker } // onProgress is the single callback that handles all state tracking and logging. func (m *jobRunMonitor) onProgress(info *jobs.Run) { - state := info.State - if state == nil { + event, first := m.tracker.Poll(info) + if event == nil { return } // First time we see this run. - if m.prevState == nil { + if first { runURL := workspaceurls.ModernizeJobRunPageURL(info.RunPageUrl) log.Infof(m.ctx, "Run available at %s", runURL) cmdio.Log(m.ctx, progress.NewJobRunUrlEvent(runURL)) } - // No state change: do not log. - if m.prevState != nil && - m.prevState.LifeCycleState == state.LifeCycleState && - m.prevState.ResultState == state.ResultState { - return - } - - // Capture current state as previous state for next call. - m.prevState = state - // Log progress event both to the terminal (in place or append), and to the logger. - event := &progress.JobProgressEvent{ - Timestamp: time.Now(), - JobId: info.JobId, - RunId: info.RunId, - RunName: info.RunName, - State: *info.State, - } cmdio.Log(m.ctx, event) log.Info(m.ctx, event.String()) } diff --git a/bundle/run/progress/job.go b/bundle/run/progress/job.go index 6deee451ad6..11183681743 100644 --- a/bundle/run/progress/job.go +++ b/bundle/run/progress/job.go @@ -16,6 +16,33 @@ type JobProgressEvent struct { State jobs.RunState `json:"state"` } +// JobStateTracker turns the polls of a job run into one event per state change, +// for callers that report a run's progress as it goes. +type JobStateTracker struct { + prev *jobs.RunState +} + +// Poll returns the event to report for this poll of run, or nil when the state +// has not changed since the last one. first is true for the state a run is seen +// in initially, where callers also report the run page URL. +func (t *JobStateTracker) Poll(run *jobs.Run) (event *JobProgressEvent, first bool) { + if run.State == nil { + return nil, false + } + first = t.prev == nil + if !first && t.prev.LifeCycleState == run.State.LifeCycleState && t.prev.ResultState == run.State.ResultState { + return nil, false + } + t.prev = run.State + return &JobProgressEvent{ + Timestamp: time.Now(), + JobId: run.JobId, + RunId: run.RunId, + RunName: run.RunName, + State: *run.State, + }, first +} + func (event *JobProgressEvent) String() string { result := strings.Builder{} result.WriteString(event.Timestamp.Format("2006-01-02 15:04:05") + " ") diff --git a/bundle/run/progress/job_test.go b/bundle/run/progress/job_test.go index 31196520305..521c54f357a 100644 --- a/bundle/run/progress/job_test.go +++ b/bundle/run/progress/job_test.go @@ -22,3 +22,36 @@ func TestJobProgressEventString(t *testing.T) { } assert.Equal(t, "-0001-11-30 00:00:00 \"run_name\" TERMINATED SUCCESS state_message", event.String()) } + +func TestJobStateTrackerPoll(t *testing.T) { + running := &jobs.Run{RunId: 456, State: &jobs.RunState{LifeCycleState: jobs.RunLifeCycleStateRunning}} + terminated := &jobs.Run{RunId: 456, State: &jobs.RunState{ + LifeCycleState: jobs.RunLifeCycleStateTerminated, + ResultState: jobs.RunResultStateSuccess, + }} + + var tracker JobStateTracker + + event, first := tracker.Poll(running) + assert.True(t, first) + assert.Equal(t, jobs.RunLifeCycleStateRunning, event.State.LifeCycleState) + + // The same state again is not worth reporting, and is no longer the first one. + event, first = tracker.Poll(running) + assert.Nil(t, event) + assert.False(t, first) + + event, first = tracker.Poll(terminated) + assert.False(t, first) + assert.Equal(t, jobs.RunResultStateSuccess, event.State.ResultState) +} + +func TestJobStateTrackerPollWithoutState(t *testing.T) { + var tracker JobStateTracker + + event, first := tracker.Poll(&jobs.Run{RunId: 456}) + + // A run reported without a state has no progress to report. + assert.Nil(t, event) + assert.False(t, first) +} From 3b908b28601becae6c87f88400e1c370dd5c6639 Mon Sep 17 00:00:00 2001 From: Rada Kamysheva Date: Wed, 29 Jul 2026 12:03:07 +0000 Subject: [PATCH 04/23] acc: run a job_run against a real workspace Excluding job_run.yml.tmpl from cloud left job_runs with no cloud coverage at all: every test under resources/job_runs inherits Cloud=false. The behaviour this milestone adds is the one that depends most on the real Jobs API, so wait_cloud triggers a run for real and reads the outcome back out of the downstream job, the way the vector search exclusion points at a dedicated test. Serverless keeps the run to about a minute, and the deploy's progress stream stays out of the golden: a real run reports an unpredictable number of intermediate states. --- acceptance/bundle/invariant/test.toml | 6 ++- .../job_runs/wait_cloud/databricks.yml.tmpl | 39 +++++++++++++++++++ .../resources/job_runs/wait_cloud/hello.py | 1 + .../job_runs/wait_cloud/out.test.toml | 4 ++ .../resources/job_runs/wait_cloud/output.txt | 22 +++++++++++ .../resources/job_runs/wait_cloud/script | 20 ++++++++++ .../resources/job_runs/wait_cloud/test.toml | 20 ++++++++++ 7 files changed, 110 insertions(+), 2 deletions(-) create mode 100644 acceptance/bundle/resources/job_runs/wait_cloud/databricks.yml.tmpl create mode 100644 acceptance/bundle/resources/job_runs/wait_cloud/hello.py create mode 100644 acceptance/bundle/resources/job_runs/wait_cloud/out.test.toml create mode 100644 acceptance/bundle/resources/job_runs/wait_cloud/output.txt create mode 100644 acceptance/bundle/resources/job_runs/wait_cloud/script create mode 100644 acceptance/bundle/resources/job_runs/wait_cloud/test.toml diff --git a/acceptance/bundle/invariant/test.toml b/acceptance/bundle/invariant/test.toml index cd7cb81dc4f..a2fcd76ac10 100644 --- a/acceptance/bundle/invariant/test.toml +++ b/acceptance/bundle/invariant/test.toml @@ -85,8 +85,10 @@ no_alert_on_cloud = ["CONFIG_Cloud=true", "INPUT_CONFIG=alert.yml.tmpl"] no_run_job_ref_on_cloud = ["CONFIG_Cloud=true", "INPUT_CONFIG=job_run_job_ref.yml.tmpl"] # Deploying a job_run waits for the run to succeed, and a real workspace reports -# a run of condition tasks alone as SKIPPED. A task that does succeed would add a -# cluster run to every variant of a suite that asserts plan and state invariants. +# a run of condition tasks alone as SKIPPED. A task that does succeed would run in +# every variant of a suite that asserts plan and state invariants, so +# resources/job_runs/wait_cloud covers the real run on cloud instead. Still +# exercised locally here. no_job_run_on_cloud = ["CONFIG_Cloud=true", "INPUT_CONFIG=job_run.yml.tmpl"] # Postgres resources only work on AWS diff --git a/acceptance/bundle/resources/job_runs/wait_cloud/databricks.yml.tmpl b/acceptance/bundle/resources/job_runs/wait_cloud/databricks.yml.tmpl new file mode 100644 index 00000000000..c72555f11ae --- /dev/null +++ b/acceptance/bundle/resources/job_runs/wait_cloud/databricks.yml.tmpl @@ -0,0 +1,39 @@ +bundle: + name: job-runs-wait-cloud + +workspace: + root_path: ~/.bundle/$UNIQUE_NAME + +resources: + jobs: + my_job: + name: test-job-$UNIQUE_NAME + tasks: + # Serverless keeps the run to about a minute; a job cluster would take + # several. + - task_key: main + spark_python_task: + python_file: ./hello.py + environment_key: default + + environments: + - environment_key: default + spec: + environment_version: "2" + + # Reads the run's outcome, so the tag it is created with shows whether the + # deploy waited for the run before creating resources that depend on it. + downstream_job: + name: test-downstream-job-$UNIQUE_NAME + tags: + run_result: ${resources.job_runs.my_run.state.result_state} + tasks: + - task_key: main + condition_task: + op: EQUAL_TO + left: "1" + right: "1" + + job_runs: + my_run: + job_id: ${resources.jobs.my_job.id} diff --git a/acceptance/bundle/resources/job_runs/wait_cloud/hello.py b/acceptance/bundle/resources/job_runs/wait_cloud/hello.py new file mode 100644 index 00000000000..93e0cef4a92 --- /dev/null +++ b/acceptance/bundle/resources/job_runs/wait_cloud/hello.py @@ -0,0 +1 @@ +print("hello from a job_run") diff --git a/acceptance/bundle/resources/job_runs/wait_cloud/out.test.toml b/acceptance/bundle/resources/job_runs/wait_cloud/out.test.toml new file mode 100644 index 00000000000..fe4076cdf9b --- /dev/null +++ b/acceptance/bundle/resources/job_runs/wait_cloud/out.test.toml @@ -0,0 +1,4 @@ +Local = true +Cloud = true +RequiresUnityCatalog = true +EnvMatrix.DATABRICKS_BUNDLE_ENGINE = ["direct"] diff --git a/acceptance/bundle/resources/job_runs/wait_cloud/output.txt b/acceptance/bundle/resources/job_runs/wait_cloud/output.txt new file mode 100644 index 00000000000..673337551c3 --- /dev/null +++ b/acceptance/bundle/resources/job_runs/wait_cloud/output.txt @@ -0,0 +1,22 @@ + +=== the deploy waits for the run to finish +>>> grep -c Run URL: deploy.log +1 + +>>> grep -o TERMINATED SUCCESS deploy.log +TERMINATED SUCCESS + +=== the downstream job was created with the run's result_state +>>> [CLI] jobs get [DOWNSTREAM_JOB_ID] -o json +SUCCESS + +>>> [CLI] bundle destroy --auto-approve +The following resources will be deleted: + delete resources.job_runs.my_run + delete resources.jobs.downstream_job + delete resources.jobs.my_job + +All files and directories at the following location will be deleted: /Workspace/Users/[USERNAME]/.bundle/[UNIQUE_NAME] + +Deleting files... +Destroy complete! diff --git a/acceptance/bundle/resources/job_runs/wait_cloud/script b/acceptance/bundle/resources/job_runs/wait_cloud/script new file mode 100644 index 00000000000..937e9e18bd6 --- /dev/null +++ b/acceptance/bundle/resources/job_runs/wait_cloud/script @@ -0,0 +1,20 @@ +envsubst < databricks.yml.tmpl > databricks.yml + +cleanup() { + trace $CLI bundle destroy --auto-approve +} +trap cleanup EXIT + +# A real run reports a variable number of intermediate states (PENDING while +# serverless compute starts), so keep the deploy's progress stream out of the +# golden and assert on the terminal state it waited for. +title "the deploy waits for the run to finish" +if ! $CLI bundle deploy > deploy.log 2>&1; then + cat deploy.log +fi +trace grep -c "Run URL: " deploy.log +trace grep -o "TERMINATED SUCCESS" deploy.log + +title "the downstream job was created with the run's result_state" +downstream_id=$(read_id.py downstream_job) +trace $CLI jobs get $downstream_id -o json | jq -r ".settings.tags.run_result" diff --git a/acceptance/bundle/resources/job_runs/wait_cloud/test.toml b/acceptance/bundle/resources/job_runs/wait_cloud/test.toml new file mode 100644 index 00000000000..63977f7d2e9 --- /dev/null +++ b/acceptance/bundle/resources/job_runs/wait_cloud/test.toml @@ -0,0 +1,20 @@ +# job_runs is a direct-engine-only resource; the Terraform provider has no +# equivalent, so restrict the matrix to direct. +EnvMatrix.DATABRICKS_BUNDLE_ENGINE = ["direct"] + +# The cloud counterpart of wait_output: it runs the job for real, which is what +# invariant/configs/job_run.yml.tmpl no longer does on cloud. Serverless needs +# Unity Catalog. +Cloud = true +RequiresUnityCatalog = true + +# A real workspace is not proxied, so there are no recorded requests to assert +# on; this test reads the deployed job back instead. +RecordRequests = false + +Ignore = [ + "databricks.yml", + "databricks.yml.tmpl", + "hello.py", + "deploy.log", +] From 6d4d73cb7a9c13ac804da5b78c248f2f2acbafe5 Mon Sep 17 00:00:00 2001 From: Rada Kamysheva Date: Wed, 29 Jul 2026 12:24:09 +0000 Subject: [PATCH 05/23] acc: check the resolved job parameters are not drift on cloud The ignore_remote_changes for job_parameters assumes GetRun reports every parameter the job defines, not just the ones the run overrode. Assert that on the run wait_cloud already triggers by overriding one of two parameters and planning after the deploy. --- .../resources/job_runs/wait_cloud/databricks.yml.tmpl | 10 ++++++++++ .../bundle/resources/job_runs/wait_cloud/output.txt | 4 ++++ acceptance/bundle/resources/job_runs/wait_cloud/script | 3 +++ 3 files changed, 17 insertions(+) diff --git a/acceptance/bundle/resources/job_runs/wait_cloud/databricks.yml.tmpl b/acceptance/bundle/resources/job_runs/wait_cloud/databricks.yml.tmpl index c72555f11ae..3457f638246 100644 --- a/acceptance/bundle/resources/job_runs/wait_cloud/databricks.yml.tmpl +++ b/acceptance/bundle/resources/job_runs/wait_cloud/databricks.yml.tmpl @@ -8,6 +8,11 @@ resources: jobs: my_job: name: test-job-$UNIQUE_NAME + parameters: + - name: env + default: dev + - name: region + default: us tasks: # Serverless keeps the run to about a minute; a job cluster would take # several. @@ -37,3 +42,8 @@ resources: job_runs: my_run: job_id: ${resources.jobs.my_job.id} + # Override one of the job's two parameters. GetRun reports the full resolved + # set, including the region default this run does not override, so a real + # workspace is where the ignore_remote_changes for job_parameters can regress. + job_parameters: + env: prod diff --git a/acceptance/bundle/resources/job_runs/wait_cloud/output.txt b/acceptance/bundle/resources/job_runs/wait_cloud/output.txt index 673337551c3..92b6ad11f0a 100644 --- a/acceptance/bundle/resources/job_runs/wait_cloud/output.txt +++ b/acceptance/bundle/resources/job_runs/wait_cloud/output.txt @@ -10,6 +10,10 @@ TERMINATED SUCCESS >>> [CLI] jobs get [DOWNSTREAM_JOB_ID] -o json SUCCESS +=== the parameters the run resolved are not drift +>>> [CLI] bundle plan -o json +skip + >>> [CLI] bundle destroy --auto-approve The following resources will be deleted: delete resources.job_runs.my_run diff --git a/acceptance/bundle/resources/job_runs/wait_cloud/script b/acceptance/bundle/resources/job_runs/wait_cloud/script index 937e9e18bd6..d047e5294dc 100644 --- a/acceptance/bundle/resources/job_runs/wait_cloud/script +++ b/acceptance/bundle/resources/job_runs/wait_cloud/script @@ -18,3 +18,6 @@ trace grep -o "TERMINATED SUCCESS" deploy.log title "the downstream job was created with the run's result_state" downstream_id=$(read_id.py downstream_job) trace $CLI jobs get $downstream_id -o json | jq -r ".settings.tags.run_result" + +title "the parameters the run resolved are not drift" +trace $CLI bundle plan -o json | jq -r '.plan["resources.job_runs.my_run"].action' From bbf19b4d23a8ac86099fd384e996272ddd0d1be9 Mon Sep 17 00:00:00 2001 From: Rada Kamysheva Date: Wed, 29 Jul 2026 12:32:02 +0000 Subject: [PATCH 06/23] job_runs: tighten the comments added by this branch Drop the comments that only restate the code, the duplicated note about the framework prefixing the run id, and the filler in the ones that carry a reason. --- acceptance/bundle/invariant/test.toml | 9 ++++---- .../job_runs/failed_run/databricks.yml | 4 ++-- .../job_runs/wait_cloud/databricks.yml.tmpl | 6 +++--- bundle/direct/dresources/all_test.go | 2 +- bundle/direct/dresources/job_run.go | 21 +++++++------------ bundle/run/progress/job.go | 9 ++++---- 6 files changed, 22 insertions(+), 29 deletions(-) diff --git a/acceptance/bundle/invariant/test.toml b/acceptance/bundle/invariant/test.toml index a2fcd76ac10..8caebbd5770 100644 --- a/acceptance/bundle/invariant/test.toml +++ b/acceptance/bundle/invariant/test.toml @@ -84,11 +84,10 @@ no_alert_on_cloud = ["CONFIG_Cloud=true", "INPUT_CONFIG=alert.yml.tmpl"] # so this config is local-only (the mock server stores it verbatim). no_run_job_ref_on_cloud = ["CONFIG_Cloud=true", "INPUT_CONFIG=job_run_job_ref.yml.tmpl"] -# Deploying a job_run waits for the run to succeed, and a real workspace reports -# a run of condition tasks alone as SKIPPED. A task that does succeed would run in -# every variant of a suite that asserts plan and state invariants, so -# resources/job_runs/wait_cloud covers the real run on cloud instead. Still -# exercised locally here. +# Deploying a job_run waits for the run to succeed, and a real workspace reports a +# run of condition tasks alone as SKIPPED. A task that does succeed would run in +# every variant of this suite, so resources/job_runs/wait_cloud covers the real run +# on cloud instead. Still exercised locally here. no_job_run_on_cloud = ["CONFIG_Cloud=true", "INPUT_CONFIG=job_run.yml.tmpl"] # Postgres resources only work on AWS diff --git a/acceptance/bundle/resources/job_runs/failed_run/databricks.yml b/acceptance/bundle/resources/job_runs/failed_run/databricks.yml index 8a538daf853..756474a90e2 100644 --- a/acceptance/bundle/resources/job_runs/failed_run/databricks.yml +++ b/acceptance/bundle/resources/job_runs/failed_run/databricks.yml @@ -6,8 +6,8 @@ resources: my_job: name: my-job tasks: - # The test server runs this locally; the script exits non-zero, which - # fails the task and with it the run. + # The test server runs this locally: it exits non-zero, failing the task + # and with it the run. - task_key: main spark_python_task: python_file: ./fail.py diff --git a/acceptance/bundle/resources/job_runs/wait_cloud/databricks.yml.tmpl b/acceptance/bundle/resources/job_runs/wait_cloud/databricks.yml.tmpl index 3457f638246..c2e8d3fd919 100644 --- a/acceptance/bundle/resources/job_runs/wait_cloud/databricks.yml.tmpl +++ b/acceptance/bundle/resources/job_runs/wait_cloud/databricks.yml.tmpl @@ -42,8 +42,8 @@ resources: job_runs: my_run: job_id: ${resources.jobs.my_job.id} - # Override one of the job's two parameters. GetRun reports the full resolved - # set, including the region default this run does not override, so a real - # workspace is where the ignore_remote_changes for job_parameters can regress. + # Override one of the job's two parameters: a real GetRun reports the full + # resolved set, including the region default, which ignore_remote_changes + # has to absorb to avoid perpetual drift. job_parameters: env: prod diff --git a/bundle/direct/dresources/all_test.go b/bundle/direct/dresources/all_test.go index 89d690dfead..f00ec21ed93 100644 --- a/bundle/direct/dresources/all_test.go +++ b/bundle/direct/dresources/all_test.go @@ -1004,7 +1004,7 @@ func testCRUD(t *testing.T, group string, adapter *Adapter, client *databricks.W require.NoError(t, err) if remoteStateFromWaitCreate != nil { // WaitAfterCreate returns the settled state; the read right after DoCreate - // may still be non-terminal, so compare against a fresh read, not that one. + // may still be non-terminal, so compare against a fresh read. remotePostWaitCreate, err := adapter.DoRead(ctx, createdID) require.NoError(t, err) require.Equal(t, remotePostWaitCreate, remoteStateFromWaitCreate) diff --git a/bundle/direct/dresources/job_run.go b/bundle/direct/dresources/job_run.go index c811af9bf21..59724d42705 100644 --- a/bundle/direct/dresources/job_run.go +++ b/bundle/direct/dresources/job_run.go @@ -19,8 +19,7 @@ import ( "github.com/databricks/databricks-sdk-go/service/jobs" ) -// jobRunTimeout bounds the wait for a run to finish, matching `bundle run` -// (jobRunTimeout in bundle/run/job.go). +// jobRunTimeout matches the timeout `bundle run` allows a run (bundle/run/job.go). const jobRunTimeout = 24 * time.Hour // JobRunState is what we persist for a triggered run: the RunNow request. @@ -169,7 +168,7 @@ func (r *ResourceJobRun) WaitAfterCreate(ctx context.Context, id string, _ *JobR }) if err != nil { // The wait can end with the run still going (timeout, interrupt), so link - // the run page; the framework's wrapper carries the id. + // the run page. return nil, fmt.Errorf("%w%s", err, runPageLine(pageURL)) } // FAILED, TIMEDOUT, CANCELED, SUCCESS_WITH_FAILURES and SKIPPED all fail the @@ -216,7 +215,7 @@ func taskFailed(task jobs.RunTask) bool { } // taskError returns the message the task reported, from the same place `bundle -// run` reads it. Only called for tasks taskFailed accepted, so State is set. +// run` reads it. Only reached for a task taskFailed accepted, so State is set. func (r *ResourceJobRun) taskError(ctx context.Context, task jobs.RunTask) string { var reported string output, err := r.client.Jobs.GetRunOutput(ctx, jobs.GetRunOutputRequest{RunId: task.RunId}) @@ -230,8 +229,6 @@ func (r *ResourceJobRun) taskError(ctx context.Context, task jobs.RunTask) strin return cmp.Or(reported, task.State.StateMessage, string(task.State.ResultState), string(task.State.LifeCycleState)) } -// runPageLine returns a line linking the run page, or an empty string when the -// URL is unknown. func runPageLine(rawURL string) string { if rawURL == "" { return "" @@ -239,8 +236,7 @@ func runPageLine(rawURL string) string { return "\nrun page: " + workspaceurls.ModernizeJobRunPageURL(rawURL) } -// logRunProgress reports what `bundle run` reports: the run page URL once, then -// each state change. +// logRunProgress mirrors `bundle run`: the run page URL once, then each state change. func logRunProgress(ctx context.Context, run *jobs.Run, tracker *progress.JobStateTracker) { event, first := tracker.Poll(run) if event == nil { @@ -252,9 +248,8 @@ func logRunProgress(ctx context.Context, run *jobs.Run, tracker *progress.JobSta logRunLine(ctx, run.RunId, event.String()) } -// logRunLine reports one line about a run to the user and the log. Resources -// deploy concurrently onto one stream, so the user-facing copy names the run it -// describes; the log already carries the resource key via log.WithPrefix. +// logRunLine names the run in the user-facing copy, since resources deploy +// concurrently onto one stream; the log carries the resource key already. func logRunLine(ctx context.Context, runID int64, msg string) { log.Info(ctx, msg) if cmdio.HasIO(ctx) { @@ -266,8 +261,8 @@ func logRunLine(ctx context.Context, runID int64, msg string) { // so any change recreates it (delete + a fresh RunNow). // DoDelete deletes the run via jobs/runs/delete, on both destroy and the -// recreate path. The API rejects a still-active run, which WaitAfterCreate -// leaves terminal; that error surfaces for a run whose wait was interrupted. +// recreate path. The API rejects a still-active run; WaitAfterCreate leaves it +// terminal, so that error only surfaces when a wait was interrupted. func (r *ResourceJobRun) DoDelete(ctx context.Context, id string, _ *JobRunState) error { runID, err := parseRunID(id) if err != nil { diff --git a/bundle/run/progress/job.go b/bundle/run/progress/job.go index 11183681743..fb29a7fadd6 100644 --- a/bundle/run/progress/job.go +++ b/bundle/run/progress/job.go @@ -16,15 +16,14 @@ type JobProgressEvent struct { State jobs.RunState `json:"state"` } -// JobStateTracker turns the polls of a job run into one event per state change, -// for callers that report a run's progress as it goes. +// JobStateTracker turns the polls of a job run into one event per state change. type JobStateTracker struct { prev *jobs.RunState } -// Poll returns the event to report for this poll of run, or nil when the state -// has not changed since the last one. first is true for the state a run is seen -// in initially, where callers also report the run page URL. +// Poll returns the event to report for this poll, or nil when the state has not +// changed. first is true for the state the run is seen in initially, where +// callers also report the run page URL. func (t *JobStateTracker) Poll(run *jobs.Run) (event *JobProgressEvent, first bool) { if run.State == nil { return nil, false From 3d985cea7142de00ebc28a8631de0e7cb97eb75f Mon Sep 17 00:00:00 2001 From: Rada Kamysheva Date: Wed, 29 Jul 2026 13:29:36 +0000 Subject: [PATCH 07/23] job_runs: report only the run URL and the state the run ends in Reporting every state change made a deploy's output depend on how many states the run passed through, which varies with how long its compute takes to start. That cost the cloud test its coverage: it had to grep the deploy log instead of comparing it. The full state history is still in the log. Also record that a failed run is not run again in the changelog entry. --- .../bundles/job-runs-wait-for-completion.md | 2 +- .../resources/job_runs/basic/output.txt | 1 - .../resources/job_runs/failed_run/output.txt | 1 - .../job_runs/job_parameters/output.txt | 1 - .../resources/job_runs/redeploy/output.txt | 2 -- .../resources/job_runs/wait_cloud/output.txt | 13 +++++--- .../resources/job_runs/wait_cloud/script | 13 +++----- .../resources/job_runs/wait_cloud/test.toml | 7 +++- .../resources/job_runs/wait_output/output.txt | 1 - bundle/direct/dresources/job_run.go | 33 ++++++++++++++----- 10 files changed, 45 insertions(+), 29 deletions(-) diff --git a/.nextchanges/bundles/job-runs-wait-for-completion.md b/.nextchanges/bundles/job-runs-wait-for-completion.md index 4520d1d9180..438070dde4a 100644 --- a/.nextchanges/bundles/job-runs-wait-for-completion.md +++ b/.nextchanges/bundles/job-runs-wait-for-completion.md @@ -1 +1 @@ -direct: the experimental `job_runs` resource now waits for the triggered run to finish, so other resources can reference the run's outcome (e.g. `${resources.job_runs.nightly.state.result_state}`). A run that does not succeed fails the deploy, naming the failed task and the message it reported; while waiting, the deploy reports the run page URL and each state change. +direct: the experimental `job_runs` resource now waits for the triggered run to finish, so other resources can reference the run's outcome (e.g. `${resources.job_runs.nightly.state.result_state}`). The deploy reports the run page URL while it waits, and a run that does not succeed fails the deploy, naming the failed task and the message it reported. A failed run stays recorded, so the next deploy does not run the job again. diff --git a/acceptance/bundle/resources/job_runs/basic/output.txt b/acceptance/bundle/resources/job_runs/basic/output.txt index 9ce260e8c85..67d2a32021b 100644 --- a/acceptance/bundle/resources/job_runs/basic/output.txt +++ b/acceptance/bundle/resources/job_runs/basic/output.txt @@ -35,7 +35,6 @@ Resources: Uploading bundle files to /Workspace/Users/[USERNAME]/.bundle/job-runs-basic/default/files... Deploying resources... job run [MY_RUN_ID]: Run URL: [DATABRICKS_URL]/jobs/[MY_JOB_ID]/runs/[MY_RUN_ID]?o=[NUMID] -job run [MY_RUN_ID]: [TIMESTAMP] "my-job" RUNNING job run [MY_RUN_ID]: [TIMESTAMP] "my-job" TERMINATED SUCCESS Updating deployment state... Deployment complete! diff --git a/acceptance/bundle/resources/job_runs/failed_run/output.txt b/acceptance/bundle/resources/job_runs/failed_run/output.txt index 78b95c98da1..7b0f33c5574 100644 --- a/acceptance/bundle/resources/job_runs/failed_run/output.txt +++ b/acceptance/bundle/resources/job_runs/failed_run/output.txt @@ -4,7 +4,6 @@ Uploading bundle files to /Workspace/Users/[USERNAME]/.bundle/job-runs-failed-run/default/files... Deploying resources... job run [MY_RUN_ID]: Run URL: [DATABRICKS_URL]/jobs/[NUMID]/runs/[MY_RUN_ID]?o=[NUMID] -job run [MY_RUN_ID]: [TIMESTAMP] "my-job" RUNNING job run [MY_RUN_ID]: [TIMESTAMP] "my-job" TERMINATED FAILED task main failed Error: cannot create resources.job_runs.my_run: waiting after creating id=[MY_RUN_ID]: run did not succeed: FAILED: task main failed task "main": spark python task execution failed: exit status 1 diff --git a/acceptance/bundle/resources/job_runs/job_parameters/output.txt b/acceptance/bundle/resources/job_runs/job_parameters/output.txt index 986caf2b754..1f793b6f01b 100644 --- a/acceptance/bundle/resources/job_runs/job_parameters/output.txt +++ b/acceptance/bundle/resources/job_runs/job_parameters/output.txt @@ -4,7 +4,6 @@ Uploading bundle files to /Workspace/Users/[USERNAME]/.bundle/job-runs-job-parameters/default/files... Deploying resources... job run [NUMID]: Run URL: [DATABRICKS_URL]/jobs/[NUMID]/runs/[NUMID]?o=[NUMID] -job run [NUMID]: [TIMESTAMP] "my-job" RUNNING job run [NUMID]: [TIMESTAMP] "my-job" TERMINATED SUCCESS Updating deployment state... Deployment complete! diff --git a/acceptance/bundle/resources/job_runs/redeploy/output.txt b/acceptance/bundle/resources/job_runs/redeploy/output.txt index 8662b4880f0..e75196469df 100644 --- a/acceptance/bundle/resources/job_runs/redeploy/output.txt +++ b/acceptance/bundle/resources/job_runs/redeploy/output.txt @@ -4,7 +4,6 @@ Uploading bundle files to /Workspace/Users/[USERNAME]/.bundle/job-runs-redeploy/default/files... Deploying resources... job run [NUMID]: Run URL: [DATABRICKS_URL]/jobs/[MY_JOB_ID]/runs/[NUMID]?o=[NUMID] -job run [NUMID]: [TIMESTAMP] "my-job" RUNNING job run [NUMID]: [TIMESTAMP] "my-job" TERMINATED SUCCESS Updating deployment state... Deployment complete! @@ -89,7 +88,6 @@ Resources: Uploading bundle files to /Workspace/Users/[USERNAME]/.bundle/job-runs-redeploy/default/files... Deploying resources... job run [NUMID]: Run URL: [DATABRICKS_URL]/jobs/[MY_JOB_ID]/runs/[NUMID]?o=[NUMID] -job run [NUMID]: [TIMESTAMP] "my-job" RUNNING job run [NUMID]: [TIMESTAMP] "my-job" TERMINATED SUCCESS Updating deployment state... Deployment complete! diff --git a/acceptance/bundle/resources/job_runs/wait_cloud/output.txt b/acceptance/bundle/resources/job_runs/wait_cloud/output.txt index 92b6ad11f0a..a38ed72002e 100644 --- a/acceptance/bundle/resources/job_runs/wait_cloud/output.txt +++ b/acceptance/bundle/resources/job_runs/wait_cloud/output.txt @@ -1,10 +1,15 @@ === the deploy waits for the run to finish ->>> grep -c Run URL: deploy.log -1 +>>> [CLI] bundle deploy +Uploading bundle files to /Workspace/Users/[USERNAME]/.bundle/[UNIQUE_NAME]/files... +Deploying resources... +job run [MY_RUN_ID]: Run URL: [RUN_URL] +job run [MY_RUN_ID]: [TIMESTAMP] "test-job-[UNIQUE_NAME]" TERMINATED SUCCESS +Updating deployment state... +Deployment complete! ->>> grep -o TERMINATED SUCCESS deploy.log -TERMINATED SUCCESS +>>> read_id.py my_run +[MY_RUN_ID] === the downstream job was created with the run's result_state >>> [CLI] jobs get [DOWNSTREAM_JOB_ID] -o json diff --git a/acceptance/bundle/resources/job_runs/wait_cloud/script b/acceptance/bundle/resources/job_runs/wait_cloud/script index d047e5294dc..dc58273b5e8 100644 --- a/acceptance/bundle/resources/job_runs/wait_cloud/script +++ b/acceptance/bundle/resources/job_runs/wait_cloud/script @@ -5,15 +5,12 @@ cleanup() { } trap cleanup EXIT -# A real run reports a variable number of intermediate states (PENDING while -# serverless compute starts), so keep the deploy's progress stream out of the -# golden and assert on the terminal state it waited for. title "the deploy waits for the run to finish" -if ! $CLI bundle deploy > deploy.log 2>&1; then - cat deploy.log -fi -trace grep -c "Run URL: " deploy.log -trace grep -o "TERMINATED SUCCESS" deploy.log +trace $CLI bundle deploy + +# Registers the run id as a replacement, so the deploy output above compares the +# same way against a real workspace as against the test server. +trace read_id.py my_run title "the downstream job was created with the run's result_state" downstream_id=$(read_id.py downstream_job) diff --git a/acceptance/bundle/resources/job_runs/wait_cloud/test.toml b/acceptance/bundle/resources/job_runs/wait_cloud/test.toml index 63977f7d2e9..9653ec13c31 100644 --- a/acceptance/bundle/resources/job_runs/wait_cloud/test.toml +++ b/acceptance/bundle/resources/job_runs/wait_cloud/test.toml @@ -16,5 +16,10 @@ Ignore = [ "databricks.yml", "databricks.yml.tmpl", "hello.py", - "deploy.log", ] + +# The host and the workspace selector in the run URL differ per workspace, and the +# URL form itself is covered by libs/workspaceurls; assert only that it is reported. +[[Repls]] +Old = 'Run URL: .*' +New = 'Run URL: [RUN_URL]' diff --git a/acceptance/bundle/resources/job_runs/wait_output/output.txt b/acceptance/bundle/resources/job_runs/wait_output/output.txt index bd9def5e990..5a2bcfa4892 100644 --- a/acceptance/bundle/resources/job_runs/wait_output/output.txt +++ b/acceptance/bundle/resources/job_runs/wait_output/output.txt @@ -4,7 +4,6 @@ Uploading bundle files to /Workspace/Users/[USERNAME]/.bundle/job-runs-wait-output/default/files... Deploying resources... job run [NUMID]: Run URL: [DATABRICKS_URL]/jobs/[NUMID]/runs/[NUMID]?o=[NUMID] -job run [NUMID]: [TIMESTAMP] "my-job" RUNNING job run [NUMID]: [TIMESTAMP] "my-job" TERMINATED SUCCESS Updating deployment state... Deployment complete! diff --git a/bundle/direct/dresources/job_run.go b/bundle/direct/dresources/job_run.go index 59724d42705..7c6a05bb2d4 100644 --- a/bundle/direct/dresources/job_run.go +++ b/bundle/direct/dresources/job_run.go @@ -202,8 +202,8 @@ func (r *ResourceJobRun) runFailedError(ctx context.Context, run *jobs.Run) erro return errors.New(msg.String()) } -// taskFailed reports whether a task caused the run to fail rather than being a -// casualty of it: tasks left SKIPPED or UPSTREAM_FAILED add noise. +// taskFailed reports whether a task is a cause of the run's failure. A task left +// SKIPPED or UPSTREAM_FAILED never ran, so it has no error to report. func taskFailed(task jobs.RunTask) bool { // State is deprecated in favour of Status, so it may be absent. if task.State == nil { @@ -236,22 +236,37 @@ func runPageLine(rawURL string) string { return "\nrun page: " + workspaceurls.ModernizeJobRunPageURL(rawURL) } -// logRunProgress mirrors `bundle run`: the run page URL once, then each state change. +// logRunProgress logs every state change like `bundle run` does, but reports only +// the run page URL and the state the run ends in to the user: how many states a +// run passes through depends on how long its compute takes to start, which would +// make a deploy's output differ between runs of the same bundle. func logRunProgress(ctx context.Context, run *jobs.Run, tracker *progress.JobStateTracker) { event, first := tracker.Poll(run) if event == nil { return } + log.Info(ctx, event.String()) if first && run.RunPageUrl != "" { - logRunLine(ctx, run.RunId, "Run URL: "+workspaceurls.ModernizeJobRunPageURL(run.RunPageUrl)) + line := "Run URL: " + workspaceurls.ModernizeJobRunPageURL(run.RunPageUrl) + log.Info(ctx, line) + reportRunLine(ctx, run.RunId, line) } - logRunLine(ctx, run.RunId, event.String()) + if runIsTerminal(run.State.LifeCycleState) { + reportRunLine(ctx, run.RunId, event.String()) + } +} + +// runIsTerminal reports whether a run is done, i.e. in one of the states the SDK +// waiter stops on. +func runIsTerminal(state jobs.RunLifeCycleState) bool { + return state == jobs.RunLifeCycleStateTerminated || + state == jobs.RunLifeCycleStateSkipped || + state == jobs.RunLifeCycleStateInternalError } -// logRunLine names the run in the user-facing copy, since resources deploy -// concurrently onto one stream; the log carries the resource key already. -func logRunLine(ctx context.Context, runID int64, msg string) { - log.Info(ctx, msg) +// reportRunLine names the run it describes, since resources deploy concurrently +// onto one output stream. +func reportRunLine(ctx context.Context, runID int64, msg string) { if cmdio.HasIO(ctx) { cmdio.LogString(ctx, fmt.Sprintf("job run %d: %s", runID, msg)) } From 117bb91062fc60697592d84d53d4184752a77e7c Mon Sep 17 00:00:00 2001 From: Rada Kamysheva Date: Wed, 29 Jul 2026 13:51:55 +0000 Subject: [PATCH 08/23] acc: run a failing job_run against a real workspace The message the deploy names a failed task with came from an error the test server writes itself, so nothing checked that a real workspace reports one at all. Assert it does, and that we are not falling back to the states the run reports. --- .../job_runs/failed_cloud/databricks.yml.tmpl | 26 +++++++++++++++++++ .../resources/job_runs/failed_cloud/fail.py | 1 + .../job_runs/failed_cloud/out.test.toml | 4 +++ .../job_runs/failed_cloud/output.txt | 20 ++++++++++++++ .../resources/job_runs/failed_cloud/script | 19 ++++++++++++++ .../resources/job_runs/failed_cloud/test.toml | 21 +++++++++++++++ 6 files changed, 91 insertions(+) create mode 100644 acceptance/bundle/resources/job_runs/failed_cloud/databricks.yml.tmpl create mode 100644 acceptance/bundle/resources/job_runs/failed_cloud/fail.py create mode 100644 acceptance/bundle/resources/job_runs/failed_cloud/out.test.toml create mode 100644 acceptance/bundle/resources/job_runs/failed_cloud/output.txt create mode 100644 acceptance/bundle/resources/job_runs/failed_cloud/script create mode 100644 acceptance/bundle/resources/job_runs/failed_cloud/test.toml diff --git a/acceptance/bundle/resources/job_runs/failed_cloud/databricks.yml.tmpl b/acceptance/bundle/resources/job_runs/failed_cloud/databricks.yml.tmpl new file mode 100644 index 00000000000..1393bdaf33e --- /dev/null +++ b/acceptance/bundle/resources/job_runs/failed_cloud/databricks.yml.tmpl @@ -0,0 +1,26 @@ +bundle: + name: job-runs-failed-cloud + +workspace: + root_path: ~/.bundle/$UNIQUE_NAME + +resources: + jobs: + my_job: + name: test-job-$UNIQUE_NAME + tasks: + # Serverless keeps the run to about a minute; a job cluster would take + # several. + - task_key: main + spark_python_task: + python_file: ./fail.py + environment_key: default + + environments: + - environment_key: default + spec: + environment_version: "2" + + job_runs: + my_run: + job_id: ${resources.jobs.my_job.id} diff --git a/acceptance/bundle/resources/job_runs/failed_cloud/fail.py b/acceptance/bundle/resources/job_runs/failed_cloud/fail.py new file mode 100644 index 00000000000..fa56481ece5 --- /dev/null +++ b/acceptance/bundle/resources/job_runs/failed_cloud/fail.py @@ -0,0 +1 @@ +raise RuntimeError("intentional failure") diff --git a/acceptance/bundle/resources/job_runs/failed_cloud/out.test.toml b/acceptance/bundle/resources/job_runs/failed_cloud/out.test.toml new file mode 100644 index 00000000000..fe4076cdf9b --- /dev/null +++ b/acceptance/bundle/resources/job_runs/failed_cloud/out.test.toml @@ -0,0 +1,4 @@ +Local = true +Cloud = true +RequiresUnityCatalog = true +EnvMatrix.DATABRICKS_BUNDLE_ENGINE = ["direct"] diff --git a/acceptance/bundle/resources/job_runs/failed_cloud/output.txt b/acceptance/bundle/resources/job_runs/failed_cloud/output.txt new file mode 100644 index 00000000000..bc4e275e341 --- /dev/null +++ b/acceptance/bundle/resources/job_runs/failed_cloud/output.txt @@ -0,0 +1,20 @@ + +=== a run that fails on a real workspace fails the deploy +>>> contains.py run did not succeed: FAILED run page: http !task "main": FAILED + +>>> grep -cE task "main": .+ deploy.log +1 + +=== the failed run is recorded, so it is destroyed rather than left behind +>>> read_id.py my_run +[MY_RUN_ID] + +>>> [CLI] bundle destroy --auto-approve +The following resources will be deleted: + delete resources.job_runs.my_run + delete resources.jobs.my_job + +All files and directories at the following location will be deleted: /Workspace/Users/[USERNAME]/.bundle/[UNIQUE_NAME] + +Deleting files... +Destroy complete! diff --git a/acceptance/bundle/resources/job_runs/failed_cloud/script b/acceptance/bundle/resources/job_runs/failed_cloud/script new file mode 100644 index 00000000000..662d43fc738 --- /dev/null +++ b/acceptance/bundle/resources/job_runs/failed_cloud/script @@ -0,0 +1,19 @@ +envsubst < databricks.yml.tmpl > databricks.yml + +cleanup() { + trace $CLI bundle destroy --auto-approve +} +trap cleanup EXIT + +# A traceback from a real task is words we do not control, so keep the deploy's +# output out of the golden and assert the parts the CLI is responsible for. +title "a run that fails on a real workspace fails the deploy" +musterr $CLI bundle deploy > deploy.log 2>&1 +trace contains.py 'run did not succeed: FAILED' 'run page: http' '!task "main": FAILED' < deploy.log > /dev/null + +# The message comes from the task, not from our fallback to the states the run +# itself reports, which is what the last assertion above rules out. +trace grep -cE 'task "main": .+' deploy.log + +title "the failed run is recorded, so it is destroyed rather than left behind" +trace read_id.py my_run diff --git a/acceptance/bundle/resources/job_runs/failed_cloud/test.toml b/acceptance/bundle/resources/job_runs/failed_cloud/test.toml new file mode 100644 index 00000000000..dd06a0daee3 --- /dev/null +++ b/acceptance/bundle/resources/job_runs/failed_cloud/test.toml @@ -0,0 +1,21 @@ +# job_runs is a direct-engine-only resource; the Terraform provider has no +# equivalent, so restrict the matrix to direct. +EnvMatrix.DATABRICKS_BUNDLE_ENGINE = ["direct"] + +# The failure counterpart of wait_cloud: only a real workspace shows whether a +# failed task reports a message the deploy can name, since the error the test +# server reports is one it writes itself. Serverless needs Unity Catalog. +Cloud = true +RequiresUnityCatalog = true + +# A real workspace is not proxied, so there are no recorded requests to assert on. +RecordRequests = false + +# The deploy fails mid-way, leaving local deployment state behind. +Ignore = [ + ".databricks", + "databricks.yml", + "databricks.yml.tmpl", + "fail.py", + "deploy.log", +] From 001af99782b5ff7933f458ad742bd2fd29c69258 Mon Sep 17 00:00:00 2001 From: Rada Kamysheva Date: Wed, 29 Jul 2026 13:57:11 +0000 Subject: [PATCH 09/23] job_runs: shorten the comments added by this branch Also fix jobRunServer's doc, which said it returns a server when it returns a client. --- .../bundle/resources/job_runs/failed_cloud/script | 8 ++++---- .../resources/job_runs/failed_cloud/test.toml | 6 +++--- .../bundle/resources/job_runs/wait_cloud/script | 2 +- bundle/direct/dresources/job_run.go | 15 ++++++--------- bundle/direct/dresources/job_run_test.go | 4 ++-- 5 files changed, 16 insertions(+), 19 deletions(-) diff --git a/acceptance/bundle/resources/job_runs/failed_cloud/script b/acceptance/bundle/resources/job_runs/failed_cloud/script index 662d43fc738..ce49c3a9aa2 100644 --- a/acceptance/bundle/resources/job_runs/failed_cloud/script +++ b/acceptance/bundle/resources/job_runs/failed_cloud/script @@ -5,14 +5,14 @@ cleanup() { } trap cleanup EXIT -# A traceback from a real task is words we do not control, so keep the deploy's -# output out of the golden and assert the parts the CLI is responsible for. +# A real task's traceback is text we do not control, so keep the deploy output out +# of the golden and assert the parts the CLI produces. title "a run that fails on a real workspace fails the deploy" musterr $CLI bundle deploy > deploy.log 2>&1 trace contains.py 'run did not succeed: FAILED' 'run page: http' '!task "main": FAILED' < deploy.log > /dev/null -# The message comes from the task, not from our fallback to the states the run -# itself reports, which is what the last assertion above rules out. +# A non-empty message means the task reported one: the negative assertion above +# rules out the fallback to the states the run itself reports. trace grep -cE 'task "main": .+' deploy.log title "the failed run is recorded, so it is destroyed rather than left behind" diff --git a/acceptance/bundle/resources/job_runs/failed_cloud/test.toml b/acceptance/bundle/resources/job_runs/failed_cloud/test.toml index dd06a0daee3..9d396b2cab8 100644 --- a/acceptance/bundle/resources/job_runs/failed_cloud/test.toml +++ b/acceptance/bundle/resources/job_runs/failed_cloud/test.toml @@ -2,9 +2,9 @@ # equivalent, so restrict the matrix to direct. EnvMatrix.DATABRICKS_BUNDLE_ENGINE = ["direct"] -# The failure counterpart of wait_cloud: only a real workspace shows whether a -# failed task reports a message the deploy can name, since the error the test -# server reports is one it writes itself. Serverless needs Unity Catalog. +# The failure counterpart of wait_cloud: the error the test server reports is one it +# writes itself, so only a real workspace shows whether a failed task reports a +# message the deploy can name. Serverless needs Unity Catalog. Cloud = true RequiresUnityCatalog = true diff --git a/acceptance/bundle/resources/job_runs/wait_cloud/script b/acceptance/bundle/resources/job_runs/wait_cloud/script index dc58273b5e8..22879da2149 100644 --- a/acceptance/bundle/resources/job_runs/wait_cloud/script +++ b/acceptance/bundle/resources/job_runs/wait_cloud/script @@ -9,7 +9,7 @@ title "the deploy waits for the run to finish" trace $CLI bundle deploy # Registers the run id as a replacement, so the deploy output above compares the -# same way against a real workspace as against the test server. +# same against a real workspace as against the test server. trace read_id.py my_run title "the downstream job was created with the run's result_state" diff --git a/bundle/direct/dresources/job_run.go b/bundle/direct/dresources/job_run.go index 7c6a05bb2d4..8e95dbdf81d 100644 --- a/bundle/direct/dresources/job_run.go +++ b/bundle/direct/dresources/job_run.go @@ -150,8 +150,7 @@ func (r *ResourceJobRun) DoCreate(ctx context.Context, config *JobRunState) (str } // WaitAfterCreate blocks until the run finishes, so a resource referencing its -// output (e.g. state.result_state) sees a settled run. Only SUCCESS lets the -// deploy continue. +// output (e.g. state.result_state) sees a settled run. Only SUCCESS continues the deploy. func (r *ResourceJobRun) WaitAfterCreate(ctx context.Context, id string, _ *JobRunState) (*JobRunRemote, error) { runID, err := parseRunID(id) if err != nil { @@ -237,9 +236,9 @@ func runPageLine(rawURL string) string { } // logRunProgress logs every state change like `bundle run` does, but reports only -// the run page URL and the state the run ends in to the user: how many states a -// run passes through depends on how long its compute takes to start, which would -// make a deploy's output differ between runs of the same bundle. +// the run page URL and the run's final state to the user: how many states a run +// passes through varies with how long its compute takes to start, so a deploy's +// output would not be reproducible. func logRunProgress(ctx context.Context, run *jobs.Run, tracker *progress.JobStateTracker) { event, first := tracker.Poll(run) if event == nil { @@ -256,16 +255,14 @@ func logRunProgress(ctx context.Context, run *jobs.Run, tracker *progress.JobSta } } -// runIsTerminal reports whether a run is done, i.e. in one of the states the SDK -// waiter stops on. +// runIsTerminal reports whether a run is done, i.e. in a state the SDK waiter stops on. func runIsTerminal(state jobs.RunLifeCycleState) bool { return state == jobs.RunLifeCycleStateTerminated || state == jobs.RunLifeCycleStateSkipped || state == jobs.RunLifeCycleStateInternalError } -// reportRunLine names the run it describes, since resources deploy concurrently -// onto one output stream. +// reportRunLine names the run, since resources deploy concurrently onto one stream. func reportRunLine(ctx context.Context, runID int64, msg string) { if cmdio.HasIO(ctx) { cmdio.LogString(ctx, fmt.Sprintf("job run %d: %s", runID, msg)) diff --git a/bundle/direct/dresources/job_run_test.go b/bundle/direct/dresources/job_run_test.go index e9c29366136..2625ef406c2 100644 --- a/bundle/direct/dresources/job_run_test.go +++ b/bundle/direct/dresources/job_run_test.go @@ -28,8 +28,8 @@ func jobRunClientFor(t *testing.T, server *testserver.Server) *databricks.Worksp return client } -// jobRunServer returns a test server whose runs/get handler is the given one, -// so a wait can be exercised without a real run. +// jobRunServer returns a client whose runs/get is the given handler, so a wait can +// be driven without a real run. func jobRunServer(t *testing.T, getRun testserver.HandlerFunc) *databricks.WorkspaceClient { t.Helper() server := testserver.New(t) From f5954ee4e6aea201598fdcbfbc03c4617b578533 Mon Sep 17 00:00:00 2001 From: Rada Kamysheva Date: Thu, 30 Jul 2026 08:14:24 +0000 Subject: [PATCH 10/23] job_runs: handle a wait the user interrupted Interrupting a deploy mid-wait leaves the run going, and jobs/runs/delete rejects an active run, so destroy failed and the bundle could not be torn down without cancelling the run by hand. Delete now cancels it first, and waits for the cancellation to settle since the API cancels asynchronously. The interrupt itself was reported as a timeout, blaming the 24h bound for something the user did. It now says it was interrupted, and still links the run, whose page URL is pinned to the first poll that reported one. The run left going is what the next deploy reads, and it triggers no second run, so a reference to the outcome resolves to an empty string. Recorded in a test rather than fixed here: stopping a run the user did not ask to stop is a departure from `bundle run`, which leaves interrupted runs alive. --- .../bundles/job-runs-wait-for-completion.md | 2 +- bundle/direct/dresources/job_run.go | 36 +++++++- bundle/direct/dresources/job_run_test.go | 84 ++++++++++++++++++- 3 files changed, 115 insertions(+), 7 deletions(-) diff --git a/.nextchanges/bundles/job-runs-wait-for-completion.md b/.nextchanges/bundles/job-runs-wait-for-completion.md index 438070dde4a..9c7d3941c56 100644 --- a/.nextchanges/bundles/job-runs-wait-for-completion.md +++ b/.nextchanges/bundles/job-runs-wait-for-completion.md @@ -1 +1 @@ -direct: the experimental `job_runs` resource now waits for the triggered run to finish, so other resources can reference the run's outcome (e.g. `${resources.job_runs.nightly.state.result_state}`). The deploy reports the run page URL while it waits, and a run that does not succeed fails the deploy, naming the failed task and the message it reported. A failed run stays recorded, so the next deploy does not run the job again. +direct: the experimental `job_runs` resource now waits for the triggered run to finish, so other resources can reference the run's outcome (e.g. `${resources.job_runs.nightly.state.result_state}`). The deploy reports the run page URL while it waits, and a run that does not succeed fails the deploy, naming the failed task and the message it reported. A failed run stays recorded, so the next deploy does not run the job again. Destroying a run that has not finished, which is what interrupting a deploy mid-wait leaves behind, cancels it first. diff --git a/bundle/direct/dresources/job_run.go b/bundle/direct/dresources/job_run.go index 8e95dbdf81d..e504aafa056 100644 --- a/bundle/direct/dresources/job_run.go +++ b/bundle/direct/dresources/job_run.go @@ -162,12 +162,16 @@ func (r *ResourceJobRun) WaitAfterCreate(ctx context.Context, id string, _ *JobR var tracker progress.JobStateTracker var pageURL string run, err := r.client.Jobs.WaitGetRunJobTerminatedOrSkipped(ctx, runID, jobRunTimeout, func(run *jobs.Run) { - pageURL = run.RunPageUrl + pageURL = cmp.Or(pageURL, run.RunPageUrl) logRunProgress(ctx, run, &tracker) }) if err != nil { // The wait can end with the run still going (timeout, interrupt), so link - // the run page. + // the run page: the next deploy triggers no second run, finished or not. + if ctx.Err() != nil { + // The waiter reports a cancelled context as a timeout. + return nil, fmt.Errorf("interrupted while waiting for the run to finish: %w%s", ctx.Err(), runPageLine(pageURL)) + } return nil, fmt.Errorf("%w%s", err, runPageLine(pageURL)) } // FAILED, TIMEDOUT, CANCELED, SUCCESS_WITH_FAILURES and SKIPPED all fail the @@ -273,16 +277,40 @@ func reportRunLine(ctx context.Context, runID int64, msg string) { // so any change recreates it (delete + a fresh RunNow). // DoDelete deletes the run via jobs/runs/delete, on both destroy and the -// recreate path. The API rejects a still-active run; WaitAfterCreate leaves it -// terminal, so that error only surfaces when a wait was interrupted. +// recreate path. The API rejects a still-active run, which an interrupted wait +// leaves behind, so cancel it first. func (r *ResourceJobRun) DoDelete(ctx context.Context, id string, _ *JobRunState) error { runID, err := parseRunID(id) if err != nil { return err } + remote, err := r.DoRead(ctx, id) + if err != nil { + return err + } + if !runIsTerminal(remote.State.LifeCycleState) { + err = r.cancelRun(ctx, runID) + if err != nil { + return err + } + } return r.client.Jobs.DeleteRunByRunId(ctx, runID) } +// cancelRun cancels a run and waits for it to settle. Cancellation is +// asynchronous, so a delete issued right after would still be rejected. +func (r *ResourceJobRun) cancelRun(ctx context.Context, runID int64) error { + waiter, err := r.client.Jobs.CancelRun(ctx, jobs.CancelRun{RunId: runID}) + if err != nil { + return fmt.Errorf("cancelling run %d before deleting it: %w", runID, err) + } + _, err = waiter.Get() + if err != nil { + return fmt.Errorf("waiting for run %d to be cancelled: %w", runID, err) + } + return nil +} + func parseRunID(id string) (int64, error) { result, err := strconv.ParseInt(id, 10, 64) if err != nil { diff --git a/bundle/direct/dresources/job_run_test.go b/bundle/direct/dresources/job_run_test.go index 2625ef406c2..9dce5dbf2ca 100644 --- a/bundle/direct/dresources/job_run_test.go +++ b/bundle/direct/dresources/job_run_test.go @@ -149,11 +149,25 @@ func TestJobRunWaitAbandonedLinksTheRun(t *testing.T) { _, err := waitForTestRun(t, ctx, client) - // Giving up on the wait does not stop the run, so the error links to it. - require.Error(t, err) + // The run keeps going, so the error links to it and names the interrupt rather + // than the 24h bound. + require.ErrorContains(t, err, "interrupted while waiting for the run to finish") require.ErrorContains(t, err, testRunPageLink) } +// After an abandoned wait the next deploy triggers no second run: it reads this +// one, so a reference to the outcome resolves to an empty string. +func TestJobRunReadOfUnfinishedRunReportsNoResult(t *testing.T) { + client := jobRunClient(t, &jobs.RunState{LifeCycleState: jobs.RunLifeCycleStateRunning}) + + remote, err := (&ResourceJobRun{}).New(client).DoRead(t.Context(), "123") + + require.NoError(t, err) + require.NotNil(t, remote.State) + assert.Equal(t, jobs.RunLifeCycleStateRunning, remote.State.LifeCycleState) + assert.Empty(t, remote.State.ResultState) +} + // Reporting RUNNING for the first two polls exercises the poll loop; the other // tests stub an already-terminal state. func TestJobRunWaitPollsUntilTerminal(t *testing.T) { @@ -178,3 +192,69 @@ func TestJobRunWaitPollsUntilTerminal(t *testing.T) { assert.Equal(t, jobs.RunResultStateSuccess, remote.State.ResultState) assert.Equal(t, int32(3), gets.Load(), "expected the wait to poll past both RUNNING reads") } + +// jobRunDeletion records what the fake workspace saw while a run was deleted. +type jobRunDeletion struct { + cancelled atomic.Bool + settled atomic.Bool + settledAtDelete atomic.Bool +} + +// jobRunDeleteClient returns a client for a run in the given state, whose cancel +// settles one poll late the way the API's asynchronous cancellation does. +func jobRunDeleteClient(t *testing.T, state *jobs.RunState) (*databricks.WorkspaceClient, *jobRunDeletion) { + t.Helper() + var deletion jobRunDeletion + cancelled := &jobs.RunState{ + LifeCycleState: jobs.RunLifeCycleStateTerminated, + ResultState: jobs.RunResultStateCanceled, + } + + server := testserver.New(t) + server.Handle("GET", "/api/2.2/jobs/runs/get", func(req testserver.Request) any { + current := state + switch { + case deletion.settled.Load(): + current = cancelled + case deletion.cancelled.Load(): + // Report the run's old state once more, then settle on the next poll. + deletion.settled.Store(true) + } + return jobs.Run{RunId: 123, JobId: 456, State: current} + }) + server.Handle("POST", "/api/2.2/jobs/runs/cancel", func(req testserver.Request) any { + deletion.cancelled.Store(true) + return testserver.Response{} + }) + server.Handle("POST", "/api/2.2/jobs/runs/delete", func(req testserver.Request) any { + deletion.settledAtDelete.Store(deletion.settled.Load()) + return testserver.Response{} + }) + return jobRunClientFor(t, server), &deletion +} + +func deleteTestRun(t *testing.T, client *databricks.WorkspaceClient) error { + t.Helper() + return (&ResourceJobRun{}).New(client).DoDelete(t.Context(), "123", &JobRunState{}) +} + +func TestJobRunDeleteCancelsUnfinishedRun(t *testing.T) { + // An interrupted wait leaves the run going, and jobs/runs/delete rejects it. + client, deletion := jobRunDeleteClient(t, &jobs.RunState{LifeCycleState: jobs.RunLifeCycleStateRunning}) + + require.NoError(t, deleteTestRun(t, client)) + + assert.True(t, deletion.cancelled.Load(), "expected the run to be cancelled") + assert.True(t, deletion.settledAtDelete.Load(), "expected the delete to wait for the cancellation to settle") +} + +func TestJobRunDeleteLeavesFinishedRunAlone(t *testing.T) { + client, deletion := jobRunDeleteClient(t, &jobs.RunState{ + LifeCycleState: jobs.RunLifeCycleStateTerminated, + ResultState: jobs.RunResultStateSuccess, + }) + + require.NoError(t, deleteTestRun(t, client)) + + assert.False(t, deletion.cancelled.Load(), "a run that already finished has nothing to cancel") +} From 1c84f569ea18a1cadb1e848001c4b9d5d7480240 Mon Sep 17 00:00:00 2001 From: Rada Kamysheva Date: Thu, 30 Jul 2026 11:11:45 +0000 Subject: [PATCH 11/23] job_runs: wait for any terminal state, not the two the SDK stops on A real workspace reports a run whose task failed as INTERNAL_ERROR in the deprecated life_cycle_state, though status.state is TERMINATED with termination code RUN_EXECUTION_ERROR. The SDK waiter halts on INTERNAL_ERROR with an error of its own, so the deploy blamed the run for an internal failure instead of naming the task that failed and the message it reported. The wait now polls for any state runIsTerminal accepts, the definition the delete path already used, and leaves the verdict to the run's result. The Jobs API retries a task that failed and reports it once per attempt, so the same task was named twice over. Only its last attempt is reported now. The fake workspace rolls a failed task up to TERMINATED FAILED, so failed_cloud was the only test that saw either of these; both are now covered by unit tests. --- bundle/direct/dresources/job_run.go | 55 ++++++++++++++++---- bundle/direct/dresources/job_run_test.go | 66 +++++++++++++++++++++++- 2 files changed, 110 insertions(+), 11 deletions(-) diff --git a/bundle/direct/dresources/job_run.go b/bundle/direct/dresources/job_run.go index e504aafa056..5dcbec3c8b8 100644 --- a/bundle/direct/dresources/job_run.go +++ b/bundle/direct/dresources/job_run.go @@ -16,6 +16,7 @@ import ( "github.com/databricks/cli/libs/workspaceurls" "github.com/databricks/databricks-sdk-go" "github.com/databricks/databricks-sdk-go/marshal" + "github.com/databricks/databricks-sdk-go/retries" "github.com/databricks/databricks-sdk-go/service/jobs" ) @@ -158,24 +159,37 @@ func (r *ResourceJobRun) WaitAfterCreate(ctx context.Context, id string, _ *JobR } // A run can take hours, so report progress like `bundle run` does. pageURL - // outlives the callback so an abandoned wait can still link the run. + // outlives the poll so an abandoned wait can still link the run. var tracker progress.JobStateTracker var pageURL string - run, err := r.client.Jobs.WaitGetRunJobTerminatedOrSkipped(ctx, runID, jobRunTimeout, func(run *jobs.Run) { + // Polled here rather than through Jobs.WaitGetRunJobTerminatedOrSkipped: a run + // whose task failed reports the deprecated life_cycle_state as INTERNAL_ERROR + // (status.state is TERMINATED, termination code RUN_EXECUTION_ERROR), and the + // SDK waiter halts on it with an error of its own, which hides the task that + // failed. + run, err := retries.Poll(ctx, jobRunTimeout, func() (*jobs.Run, *retries.Err) { + var req jobs.GetRunRequest + req.RunId = runID + run, err := r.client.Jobs.GetRun(ctx, req) + if err != nil { + return nil, retries.Halt(err) + } pageURL = cmp.Or(pageURL, run.RunPageUrl) logRunProgress(ctx, run, &tracker) + if !runIsTerminal(run.State.LifeCycleState) { + return nil, retries.Continues(run.State.StateMessage) + } + return run, nil }) if err != nil { // The wait can end with the run still going (timeout, interrupt), so link // the run page: the next deploy triggers no second run, finished or not. if ctx.Err() != nil { - // The waiter reports a cancelled context as a timeout. + // A cancelled context is reported as a timeout. return nil, fmt.Errorf("interrupted while waiting for the run to finish: %w%s", ctx.Err(), runPageLine(pageURL)) } return nil, fmt.Errorf("%w%s", err, runPageLine(pageURL)) } - // FAILED, TIMEDOUT, CANCELED, SUCCESS_WITH_FAILURES and SKIPPED all fail the - // deploy; the waiter already errored on INTERNAL_ERROR and on timeout. if run.State.ResultState != jobs.RunResultStateSuccess { return nil, r.runFailedError(ctx, run) } @@ -196,15 +210,38 @@ func (r *ResourceJobRun) runFailedError(ctx context.Context, run *jobs.Run) erro if run.State.StateMessage != "" { fmt.Fprintf(&msg, ": %s", run.State.StateMessage) } - for _, task := range run.Tasks { - if taskFailed(task) { - fmt.Fprintf(&msg, "\ntask %q: %s", task.TaskKey, r.taskError(ctx, task)) - } + for _, task := range lastFailedAttempts(run.Tasks) { + fmt.Fprintf(&msg, "\ntask %q: %s", task.TaskKey, r.taskError(ctx, task)) } msg.WriteString(runPageLine(run.RunPageUrl)) return errors.New(msg.String()) } +// lastFailedAttempts returns the failed tasks in the order the run reports them, +// one per task key: a task the Jobs API retried is reported once per attempt, and +// only its last one says how the run ended up. +func lastFailedAttempts(tasks []jobs.RunTask) []jobs.RunTask { + latest := make(map[string]jobs.RunTask) + var keys []string + for _, task := range tasks { + if !taskFailed(task) { + continue + } + previous, seen := latest[task.TaskKey] + if !seen { + keys = append(keys, task.TaskKey) + } + if !seen || task.AttemptNumber > previous.AttemptNumber { + latest[task.TaskKey] = task + } + } + result := make([]jobs.RunTask, 0, len(keys)) + for _, key := range keys { + result = append(result, latest[key]) + } + return result +} + // taskFailed reports whether a task is a cause of the run's failure. A task left // SKIPPED or UPSTREAM_FAILED never ran, so it has no error to report. func taskFailed(task jobs.RunTask) bool { diff --git a/bundle/direct/dresources/job_run_test.go b/bundle/direct/dresources/job_run_test.go index 9dce5dbf2ca..50580104437 100644 --- a/bundle/direct/dresources/job_run_test.go +++ b/bundle/direct/dresources/job_run_test.go @@ -136,11 +136,73 @@ func TestJobRunWaitFailsOnInternalError(t *testing.T) { _, err := waitForTestRun(t, t.Context(), client) - // The SDK waiter errors on INTERNAL_ERROR, ahead of the result check. - require.ErrorContains(t, err, "INTERNAL_ERROR") + require.ErrorContains(t, err, "run did not succeed: INTERNAL_ERROR") require.ErrorContains(t, err, testRunPageLink) } +// A real workspace reports a run whose task failed as INTERNAL_ERROR in the +// deprecated life_cycle_state, which the SDK waiter halts on with an error of its +// own. The failing task still has to be named. +func TestJobRunWaitReportsFailedTaskOfInternalErrorRun(t *testing.T) { + server := testserver.New(t) + server.Handle("GET", "/api/2.2/jobs/runs/get", func(req testserver.Request) any { + return jobs.Run{ + RunId: 123, + JobId: 456, + RunPageUrl: testRunPageURL, + State: &jobs.RunState{ + LifeCycleState: jobs.RunLifeCycleStateInternalError, + ResultState: jobs.RunResultStateFailed, + StateMessage: "Task main failed with message: Workload failed, see run output for details.", + }, + Tasks: []jobs.RunTask{ + {TaskKey: "main", RunId: 999, State: &jobs.RunState{ + LifeCycleState: jobs.RunLifeCycleStateTerminated, + ResultState: jobs.RunResultStateFailed, + }}, + }, + } + }) + server.Handle("GET", "/api/2.2/jobs/runs/get-output", func(req testserver.Request) any { + return jobs.RunOutput{Error: "RuntimeError: intentional failure"} + }) + + _, err := waitForTestRun(t, t.Context(), jobRunClientFor(t, server)) + + require.ErrorContains(t, err, "run did not succeed: FAILED") + require.ErrorContains(t, err, `task "main": RuntimeError: intentional failure`) + require.ErrorContains(t, err, testRunPageLink) +} + +func TestJobRunWaitReportsOnlyTheLastAttemptOfATask(t *testing.T) { + failed := &jobs.RunState{ + LifeCycleState: jobs.RunLifeCycleStateTerminated, + ResultState: jobs.RunResultStateFailed, + } + server := testserver.New(t) + server.Handle("GET", "/api/2.2/jobs/runs/get", func(req testserver.Request) any { + return jobs.Run{ + RunId: 123, + JobId: 456, + State: failed, + Tasks: []jobs.RunTask{ + {TaskKey: "main", RunId: 998, AttemptNumber: 0, State: failed}, + {TaskKey: "main", RunId: 999, AttemptNumber: 1, State: failed}, + }, + } + }) + server.Handle("GET", "/api/2.2/jobs/runs/get-output", func(req testserver.Request) any { + return jobs.RunOutput{Error: "output of run " + req.URL.Query().Get("run_id")} + }) + + _, err := waitForTestRun(t, t.Context(), jobRunClientFor(t, server)) + + // The Jobs API reports a retried task once per attempt; only the last one says + // how the run ended up. + require.ErrorContains(t, err, `task "main": output of run 999`) + assert.NotContains(t, err.Error(), "output of run 998") +} + func TestJobRunWaitAbandonedLinksTheRun(t *testing.T) { client := jobRunClient(t, &jobs.RunState{LifeCycleState: jobs.RunLifeCycleStateRunning}) From 63f9d1a7e1d64729d971e10bb4e7b8c573ee7122 Mon Sep 17 00:00:00 2001 From: Rada Kamysheva Date: Mon, 3 Aug 2026 10:47:18 +0000 Subject: [PATCH 12/23] job_runs: refuse to resolve a reference to a run that has not finished Interrupting a deploy mid-wait leaves the run going and its id recorded, so the next deploy plans no change for it and serves references to it from the remote state cache. An unfinished run reports an empty result_state, and that was substituted into whatever referenced it, configuring a downstream resource with the outcome of a run that had not reached one. Resources now declare when their outputs are not final yet, through an optional CheckSettled that mirrors IsGone, and reference resolution fails with what the resource reports instead of handing out the zero value. job_runs answers with runIsTerminal, the definition the wait loop already polls for, so the two cannot drift apart, and names the run and links its page. The check sits at reference resolution rather than in DoRead or the planner, so it only fires for a bundle that actually reads the outcome. A deploy that merely carries the run keeps working, as do plan, summary and destroy, while the run finishes on its own; cancelling it would be the departure from `bundle run` that this branch already declined to make. --- bundle/direct/bundle_plan.go | 8 ++++ bundle/direct/bundle_plan_test.go | 53 +++++++++++++++++++++++ bundle/direct/dresources/adapter.go | 30 +++++++++++++ bundle/direct/dresources/job_run.go | 11 +++++ bundle/direct/dresources/job_run_test.go | 54 +++++++++++++++++++++++- 5 files changed, 155 insertions(+), 1 deletion(-) diff --git a/bundle/direct/bundle_plan.go b/bundle/direct/bundle_plan.go index 669c187aac5..8b0ab74b3b0 100644 --- a/bundle/direct/bundle_plan.go +++ b/bundle/direct/bundle_plan.go @@ -805,6 +805,10 @@ func (b *DeploymentBundle) LookupReferencePreDeploy(ctx context.Context, path *s if canReadRemoteCache { remoteState, ok := b.RemoteStateCache.Load(targetResourceKey) if ok { + err = adapter.CheckSettled(remoteState) + if err != nil { + return nil, err + } return structaccess.Get(remoteState, fieldPath) } else { return nil, fmt.Errorf("internal error: no entry in remote state cache for %q (remote-only)", targetResourceKey) @@ -824,6 +828,10 @@ func (b *DeploymentBundle) LookupReferencePreDeploy(ctx context.Context, path *s if canReadRemoteCache { remoteState, ok := b.RemoteStateCache.Load(targetResourceKey) if ok { + err = adapter.CheckSettled(remoteState) + if err != nil { + return nil, err + } return structaccess.Get(remoteState, fieldPath) } else { return nil, fmt.Errorf("internal error: no entry in remote state cache for %q", targetResourceKey) diff --git a/bundle/direct/bundle_plan_test.go b/bundle/direct/bundle_plan_test.go index 37605302874..f0d1fb1b972 100644 --- a/bundle/direct/bundle_plan_test.go +++ b/bundle/direct/bundle_plan_test.go @@ -4,11 +4,14 @@ import ( "bytes" "testing" + "github.com/databricks/cli/bundle/config/resources" "github.com/databricks/cli/bundle/deployplan" "github.com/databricks/cli/bundle/direct/dresources" "github.com/databricks/cli/libs/dyn" "github.com/databricks/cli/libs/dyn/yamlloader" "github.com/databricks/cli/libs/structs/structpath" + "github.com/databricks/cli/libs/structs/structvar" + "github.com/databricks/databricks-sdk-go/service/jobs" "github.com/databricks/databricks-sdk-go/service/pipelines" "github.com/stretchr/testify/assert" "github.com/stretchr/testify/require" @@ -284,3 +287,53 @@ func TestShouldSkipBackendDefault_MapDriftUsesBracketKeys(t *testing.T) { assert.True(t, ok) assert.Equal(t, deployplan.ReasonBackendDefault, reason) } + +const jobRunKey = "resources.job_runs.my_run" + +// planWithSkippedJobRun builds the state a deploy is in when a run recorded by an +// earlier deploy needs no change: the plan skips it, so references to it are served +// from the remote state cache. +func planWithSkippedJobRun(t *testing.T, state *jobs.RunState) *DeploymentBundle { + t.Helper() + + adapters, err := dresources.InitAll(nil) + require.NoError(t, err) + + plan := deployplan.NewPlanDirect() + plan.Plan[jobRunKey] = &deployplan.PlanEntry{ + Action: deployplan.Skip, + NewState: &structvar.StructVarJSON{}, + } + + b := &DeploymentBundle{Adapters: adapters, Plan: plan} + b.StateCache.Store(jobRunKey, structvar.NewStructVar(&resources.JobRun{}, nil)) + b.RemoteStateCache.Store(jobRunKey, &dresources.JobRunRemote{ + RunId: 123, + State: state, + }) + return b +} + +// A run that has not finished reports an empty result_state, which would otherwise +// be substituted into whatever references it. +func TestLookupReferencePreDeploy_UnfinishedJobRun(t *testing.T) { + b := planWithSkippedJobRun(t, &jobs.RunState{ + LifeCycleState: jobs.RunLifeCycleStateRunning, + }) + + _, err := b.LookupReferencePreDeploy(t.Context(), structpath.MustParsePath(jobRunKey+".state.result_state")) + + require.ErrorContains(t, err, "run 123 has not finished (RUNNING)") +} + +func TestLookupReferencePreDeploy_FinishedJobRun(t *testing.T) { + b := planWithSkippedJobRun(t, &jobs.RunState{ + LifeCycleState: jobs.RunLifeCycleStateTerminated, + ResultState: jobs.RunResultStateSuccess, + }) + + value, err := b.LookupReferencePreDeploy(t.Context(), structpath.MustParsePath(jobRunKey+".state.result_state")) + + require.NoError(t, err) + assert.Equal(t, jobs.RunResultStateSuccess, value) +} diff --git a/bundle/direct/dresources/adapter.go b/bundle/direct/dresources/adapter.go index fdaa15bfcea..d51aa5d6451 100644 --- a/bundle/direct/dresources/adapter.go +++ b/bundle/direct/dresources/adapter.go @@ -90,6 +90,14 @@ type IResource interface { // (returned by GET, not 404) that rejects a second DELETE. // Example: func (*ResourceApp) IsGone(remote *AppRemote) bool IsGone(remoteState any) bool + + // [Optional] CheckSettled reports whether the remote's output fields are + // final. Implement it for a resource whose remote can be tracked and + // unchanged while its outputs are still filling in (e.g. a job run that has + // not finished): a reference to such a resource fails with the returned + // error instead of resolving to a zero value. + // Example: func (*ResourceJobRun) CheckSettled(remote *JobRunRemote) error + CheckSettled(remoteState any) error } // Adapter wraps resource implementation, validates signatures and type consistency across methods @@ -111,6 +119,7 @@ type Adapter struct { overrideChangeDesc *calladapt.BoundCaller doResize *calladapt.BoundCaller isGone *calladapt.BoundCaller + checkSettled *calladapt.BoundCaller resourceConfig *ResourceLifecycleConfig generatedResourceConfig *ResourceLifecycleConfig @@ -144,6 +153,7 @@ func NewAdapter(typedNil any, resourceType string, client *databricks.WorkspaceC waitAfterDelete: nil, overrideChangeDesc: nil, isGone: nil, + checkSettled: nil, resourceConfig: GetResourceConfig(resourceType), generatedResourceConfig: GetGeneratedResourceConfig(resourceType), keyedSlices: nil, @@ -245,6 +255,11 @@ func (a *Adapter) initMethods(resource any) error { return err } + a.checkSettled, err = calladapt.PrepareCall(resource, reflect.TypeFor[IResource](), "CheckSettled") + if err != nil { + return err + } + keyedSlicesCall, err := calladapt.PrepareCall(resource, reflect.TypeFor[IResource](), "KeyedSlices") if err != nil { return err @@ -330,6 +345,10 @@ func (a *Adapter) validate() error { validations = append(validations, "IsGone remoteState", a.isGone.InTypes[0], remoteType) } + if a.checkSettled != nil { + validations = append(validations, "CheckSettled remoteState", a.checkSettled.InTypes[0], remoteType) + } + if a.doUpdateWithID != nil { validations = append(validations, "DoUpdateWithID newState", a.doUpdateWithID.InTypes[2], stateType) // DoUpdateWithID must return (string, remoteType, error) @@ -595,6 +614,17 @@ func (a *Adapter) IsGone(remoteState any) bool { return outs[0].(bool) } +// CheckSettled reports whether the remote's output fields are final, so a +// reference to one resolves to the value it will keep. Resources that don't +// implement CheckSettled are always settled. +func (a *Adapter) CheckSettled(remoteState any) error { + if a.checkSettled == nil { + return nil + } + _, err := a.checkSettled.Call(remoteState) + return err +} + // prepareCallRequired prepares a call and ensures the method is found. func prepareCallRequired(resource any, methodName string) (*calladapt.BoundCaller, error) { caller, err := calladapt.PrepareCall(resource, reflect.TypeFor[IResource](), methodName) diff --git a/bundle/direct/dresources/job_run.go b/bundle/direct/dresources/job_run.go index 5dcbec3c8b8..4c6729fd858 100644 --- a/bundle/direct/dresources/job_run.go +++ b/bundle/direct/dresources/job_run.go @@ -140,6 +140,17 @@ func (*ResourceJobRun) RemapState(remote *JobRunRemote) *JobRunState { return &JobRunState{RunNow: remote.RunNow} } +// CheckSettled rejects a run that has not finished. Interrupting a deploy +// mid-wait leaves the run going and its id recorded, so the next deploy plans no +// change for it; without this, a reference to an outcome the run has not reached +// would resolve to an empty string. +func (*ResourceJobRun) CheckSettled(remote *JobRunRemote) error { + if runIsTerminal(remote.State.LifeCycleState) { + return nil + } + return fmt.Errorf("run %d has not finished (%s)%s", remote.RunId, remote.State.LifeCycleState, runPageLine(remote.RunPageUrl)) +} + func (r *ResourceJobRun) DoCreate(ctx context.Context, config *JobRunState) (string, *JobRunRemote, error) { // RunNow returns only the new run id, so we return a nil remote and let the // framework read it back via DoRead. diff --git a/bundle/direct/dresources/job_run_test.go b/bundle/direct/dresources/job_run_test.go index 50580104437..c53b151d365 100644 --- a/bundle/direct/dresources/job_run_test.go +++ b/bundle/direct/dresources/job_run_test.go @@ -218,7 +218,7 @@ func TestJobRunWaitAbandonedLinksTheRun(t *testing.T) { } // After an abandoned wait the next deploy triggers no second run: it reads this -// one, so a reference to the outcome resolves to an empty string. +// one, and the empty outcome is what CheckSettled keeps out of a reference. func TestJobRunReadOfUnfinishedRunReportsNoResult(t *testing.T) { client := jobRunClient(t, &jobs.RunState{LifeCycleState: jobs.RunLifeCycleStateRunning}) @@ -230,6 +230,58 @@ func TestJobRunReadOfUnfinishedRunReportsNoResult(t *testing.T) { assert.Empty(t, remote.State.ResultState) } +func TestJobRunCheckSettled(t *testing.T) { + // Every state runIsTerminal accepts is settled: the run has the outcome it is + // going to keep, whether or not it succeeded. + for _, state := range []jobs.RunLifeCycleState{ + jobs.RunLifeCycleStateTerminated, + jobs.RunLifeCycleStateSkipped, + jobs.RunLifeCycleStateInternalError, + } { + t.Run(string(state), func(t *testing.T) { + remote := &JobRunRemote{RunId: 123, State: &jobs.RunState{LifeCycleState: state}} + + require.NoError(t, (&ResourceJobRun{}).CheckSettled(remote)) + }) + } +} + +func TestJobRunCheckSettledRejectsUnfinishedRun(t *testing.T) { + for _, state := range []jobs.RunLifeCycleState{ + jobs.RunLifeCycleStatePending, + jobs.RunLifeCycleStateRunning, + } { + t.Run(string(state), func(t *testing.T) { + remote := &JobRunRemote{ + RunId: 123, + State: &jobs.RunState{LifeCycleState: state}, + RunPageUrl: testRunPageURL, + } + + err := (&ResourceJobRun{}).CheckSettled(remote) + + // The run has no outcome yet, and the error links the run so the user + // can see what it is still doing. + require.ErrorContains(t, err, "run 123 has not finished ("+string(state)+")") + require.ErrorContains(t, err, testRunPageLink) + }) + } +} + +func TestJobRunCheckSettledIsWiredIntoTheAdapter(t *testing.T) { + adapters, err := InitAll(nil) + require.NoError(t, err) + + running := &JobRunRemote{RunId: 123, State: &jobs.RunState{ + LifeCycleState: jobs.RunLifeCycleStateRunning, + }} + require.ErrorContains(t, adapters["job_runs"].CheckSettled(running), "has not finished") + + // A resource that does not implement CheckSettled is always settled, so the + // remote state is never even looked at. + require.NoError(t, adapters["jobs"].CheckSettled(nil)) +} + // Reporting RUNNING for the first two polls exercises the poll loop; the other // tests stub an already-terminal state. func TestJobRunWaitPollsUntilTerminal(t *testing.T) { From af3da9966ed49081f0c448a329252472b0bb37aa Mon Sep 17 00:00:00 2001 From: Rada Kamysheva Date: Tue, 4 Aug 2026 06:45:17 +0000 Subject: [PATCH 13/23] acc: merge wait_output into wait, which already ran locally too wait_cloud inherited Local=true, so both tests deployed the same job_run against the test server and differed only in the assertions that followed. The merged test keeps wait_output's coverage that the resolved result_state tag is not perpetual drift, and reads the tag back with `jobs get` rather than from recorded requests, so that assertion holds against a real workspace as well. Recording requests stays off: the wait polls GetRun until the run is terminal, so the recorded requests depend on how long the run takes. The _cloud suffix goes with it, since the test was never cloud-only. --- acceptance/bundle/invariant/test.toml | 2 +- .../resources/job_runs/failed_cloud/test.toml | 2 +- .../{wait_cloud => wait}/databricks.yml.tmpl | 4 +- .../job_runs/{wait_cloud => wait}/hello.py | 0 .../{wait_cloud => wait}/out.test.toml | 0 .../job_runs/{wait_cloud => wait}/output.txt | 12 ++- .../job_runs/{wait_cloud => wait}/script | 7 +- .../job_runs/{wait_cloud => wait}/test.toml | 9 +- .../job_runs/wait_output/databricks.yml | 30 ------- .../job_runs/wait_output/out.test.toml | 3 - .../resources/job_runs/wait_output/output.txt | 90 ------------------- .../resources/job_runs/wait_output/script | 18 ---- .../resources/job_runs/wait_output/test.toml | 4 - 13 files changed, 22 insertions(+), 159 deletions(-) rename acceptance/bundle/resources/job_runs/{wait_cloud => wait}/databricks.yml.tmpl (91%) rename acceptance/bundle/resources/job_runs/{wait_cloud => wait}/hello.py (100%) rename acceptance/bundle/resources/job_runs/{wait_cloud => wait}/out.test.toml (100%) rename acceptance/bundle/resources/job_runs/{wait_cloud => wait}/output.txt (71%) rename acceptance/bundle/resources/job_runs/{wait_cloud => wait}/script (59%) rename acceptance/bundle/resources/job_runs/{wait_cloud => wait}/test.toml (63%) delete mode 100644 acceptance/bundle/resources/job_runs/wait_output/databricks.yml delete mode 100644 acceptance/bundle/resources/job_runs/wait_output/out.test.toml delete mode 100644 acceptance/bundle/resources/job_runs/wait_output/output.txt delete mode 100644 acceptance/bundle/resources/job_runs/wait_output/script delete mode 100644 acceptance/bundle/resources/job_runs/wait_output/test.toml diff --git a/acceptance/bundle/invariant/test.toml b/acceptance/bundle/invariant/test.toml index 8caebbd5770..c4c9286cd7a 100644 --- a/acceptance/bundle/invariant/test.toml +++ b/acceptance/bundle/invariant/test.toml @@ -86,7 +86,7 @@ no_run_job_ref_on_cloud = ["CONFIG_Cloud=true", "INPUT_CONFIG=job_run_job_ref.ym # Deploying a job_run waits for the run to succeed, and a real workspace reports a # run of condition tasks alone as SKIPPED. A task that does succeed would run in -# every variant of this suite, so resources/job_runs/wait_cloud covers the real run +# every variant of this suite, so resources/job_runs/wait covers the real run # on cloud instead. Still exercised locally here. no_job_run_on_cloud = ["CONFIG_Cloud=true", "INPUT_CONFIG=job_run.yml.tmpl"] diff --git a/acceptance/bundle/resources/job_runs/failed_cloud/test.toml b/acceptance/bundle/resources/job_runs/failed_cloud/test.toml index 9d396b2cab8..8df99dac130 100644 --- a/acceptance/bundle/resources/job_runs/failed_cloud/test.toml +++ b/acceptance/bundle/resources/job_runs/failed_cloud/test.toml @@ -2,7 +2,7 @@ # equivalent, so restrict the matrix to direct. EnvMatrix.DATABRICKS_BUNDLE_ENGINE = ["direct"] -# The failure counterpart of wait_cloud: the error the test server reports is one it +# The failure counterpart of wait: the error the test server reports is one it # writes itself, so only a real workspace shows whether a failed task reports a # message the deploy can name. Serverless needs Unity Catalog. Cloud = true diff --git a/acceptance/bundle/resources/job_runs/wait_cloud/databricks.yml.tmpl b/acceptance/bundle/resources/job_runs/wait/databricks.yml.tmpl similarity index 91% rename from acceptance/bundle/resources/job_runs/wait_cloud/databricks.yml.tmpl rename to acceptance/bundle/resources/job_runs/wait/databricks.yml.tmpl index c2e8d3fd919..2daebf61240 100644 --- a/acceptance/bundle/resources/job_runs/wait_cloud/databricks.yml.tmpl +++ b/acceptance/bundle/resources/job_runs/wait/databricks.yml.tmpl @@ -1,5 +1,5 @@ bundle: - name: job-runs-wait-cloud + name: job-runs-wait workspace: root_path: ~/.bundle/$UNIQUE_NAME @@ -28,6 +28,8 @@ resources: # Reads the run's outcome, so the tag it is created with shows whether the # deploy waited for the run before creating resources that depend on it. + # Separate from my_job, which my_run already depends on: a reference back + # would cycle. downstream_job: name: test-downstream-job-$UNIQUE_NAME tags: diff --git a/acceptance/bundle/resources/job_runs/wait_cloud/hello.py b/acceptance/bundle/resources/job_runs/wait/hello.py similarity index 100% rename from acceptance/bundle/resources/job_runs/wait_cloud/hello.py rename to acceptance/bundle/resources/job_runs/wait/hello.py diff --git a/acceptance/bundle/resources/job_runs/wait_cloud/out.test.toml b/acceptance/bundle/resources/job_runs/wait/out.test.toml similarity index 100% rename from acceptance/bundle/resources/job_runs/wait_cloud/out.test.toml rename to acceptance/bundle/resources/job_runs/wait/out.test.toml diff --git a/acceptance/bundle/resources/job_runs/wait_cloud/output.txt b/acceptance/bundle/resources/job_runs/wait/output.txt similarity index 71% rename from acceptance/bundle/resources/job_runs/wait_cloud/output.txt rename to acceptance/bundle/resources/job_runs/wait/output.txt index a38ed72002e..c842c9c667e 100644 --- a/acceptance/bundle/resources/job_runs/wait_cloud/output.txt +++ b/acceptance/bundle/resources/job_runs/wait/output.txt @@ -15,9 +15,17 @@ Deployment complete! >>> [CLI] jobs get [DOWNSTREAM_JOB_ID] -o json SUCCESS -=== the parameters the run resolved are not drift +=== nothing the run resolved is drift, so a redeploy starts no new run >>> [CLI] bundle plan -o json -skip +resources.job_runs.my_run skip +resources.jobs.downstream_job skip +resources.jobs.my_job skip + +>>> [CLI] bundle deploy +Uploading bundle files to /Workspace/Users/[USERNAME]/.bundle/[UNIQUE_NAME]/files... +Deploying resources... +Updating deployment state... +Deployment complete! >>> [CLI] bundle destroy --auto-approve The following resources will be deleted: diff --git a/acceptance/bundle/resources/job_runs/wait_cloud/script b/acceptance/bundle/resources/job_runs/wait/script similarity index 59% rename from acceptance/bundle/resources/job_runs/wait_cloud/script rename to acceptance/bundle/resources/job_runs/wait/script index 22879da2149..70cee1c9082 100644 --- a/acceptance/bundle/resources/job_runs/wait_cloud/script +++ b/acceptance/bundle/resources/job_runs/wait/script @@ -8,13 +8,12 @@ trap cleanup EXIT title "the deploy waits for the run to finish" trace $CLI bundle deploy -# Registers the run id as a replacement, so the deploy output above compares the -# same against a real workspace as against the test server. trace read_id.py my_run title "the downstream job was created with the run's result_state" downstream_id=$(read_id.py downstream_job) trace $CLI jobs get $downstream_id -o json | jq -r ".settings.tags.run_result" -title "the parameters the run resolved are not drift" -trace $CLI bundle plan -o json | jq -r '.plan["resources.job_runs.my_run"].action' +title "nothing the run resolved is drift, so a redeploy starts no new run" +trace $CLI bundle plan -o json | jq -r '.plan | to_entries[] | "\(.key) \(.value.action)"' +trace $CLI bundle deploy diff --git a/acceptance/bundle/resources/job_runs/wait_cloud/test.toml b/acceptance/bundle/resources/job_runs/wait/test.toml similarity index 63% rename from acceptance/bundle/resources/job_runs/wait_cloud/test.toml rename to acceptance/bundle/resources/job_runs/wait/test.toml index 9653ec13c31..fe1d7f5eea0 100644 --- a/acceptance/bundle/resources/job_runs/wait_cloud/test.toml +++ b/acceptance/bundle/resources/job_runs/wait/test.toml @@ -2,14 +2,13 @@ # equivalent, so restrict the matrix to direct. EnvMatrix.DATABRICKS_BUNDLE_ENGINE = ["direct"] -# The cloud counterpart of wait_output: it runs the job for real, which is what -# invariant/configs/job_run.yml.tmpl no longer does on cloud. Serverless needs -# Unity Catalog. +# Runs the job for real, which invariant/configs/job_run.yml.tmpl no longer does +# on cloud. Serverless needs Unity Catalog. Cloud = true RequiresUnityCatalog = true -# A real workspace is not proxied, so there are no recorded requests to assert -# on; this test reads the deployed job back instead. +# The wait polls GetRun until the run is terminal, so the recorded requests +# depend on how long the run takes. RecordRequests = false Ignore = [ diff --git a/acceptance/bundle/resources/job_runs/wait_output/databricks.yml b/acceptance/bundle/resources/job_runs/wait_output/databricks.yml deleted file mode 100644 index 8b9d6691220..00000000000 --- a/acceptance/bundle/resources/job_runs/wait_output/databricks.yml +++ /dev/null @@ -1,30 +0,0 @@ -bundle: - name: job-runs-wait-output - -resources: - jobs: - my_job: - name: my-job - tasks: - - task_key: main - condition_task: - op: EQUAL_TO - left: "1" - right: "1" - - # Reads the run's output. A job separate from my_job (the run's target), - # since my_run already depends on my_job.id and a reference back would cycle. - downstream_job: - name: downstream-job - tags: - run_result: ${resources.job_runs.my_run.state.result_state} - tasks: - - task_key: main - condition_task: - op: EQUAL_TO - left: "1" - right: "1" - - job_runs: - my_run: - job_id: ${resources.jobs.my_job.id} diff --git a/acceptance/bundle/resources/job_runs/wait_output/out.test.toml b/acceptance/bundle/resources/job_runs/wait_output/out.test.toml deleted file mode 100644 index e90b6d5d1ba..00000000000 --- a/acceptance/bundle/resources/job_runs/wait_output/out.test.toml +++ /dev/null @@ -1,3 +0,0 @@ -Local = true -Cloud = false -EnvMatrix.DATABRICKS_BUNDLE_ENGINE = ["direct"] diff --git a/acceptance/bundle/resources/job_runs/wait_output/output.txt b/acceptance/bundle/resources/job_runs/wait_output/output.txt deleted file mode 100644 index 5a2bcfa4892..00000000000 --- a/acceptance/bundle/resources/job_runs/wait_output/output.txt +++ /dev/null @@ -1,90 +0,0 @@ - -=== deploy waits for the run to finish, then the downstream job reads its result_state ->>> [CLI] bundle deploy -Uploading bundle files to /Workspace/Users/[USERNAME]/.bundle/job-runs-wait-output/default/files... -Deploying resources... -job run [NUMID]: Run URL: [DATABRICKS_URL]/jobs/[NUMID]/runs/[NUMID]?o=[NUMID] -job run [NUMID]: [TIMESTAMP] "my-job" TERMINATED SUCCESS -Updating deployment state... -Deployment complete! - -=== the downstream job was created with the run's result_state resolved into its tag ->>> print_requests.py //jobs/create -{ - "method": "POST", - "path": "/api/2.2/jobs/create", - "body": { - "deployment": { - "kind": "BUNDLE", - "metadata_file_path": "/Workspace/Users/[USERNAME]/.bundle/job-runs-wait-output/default/state/metadata.json" - }, - "edit_mode": "UI_LOCKED", - "format": "MULTI_TASK", - "max_concurrent_runs": 1, - "name": "my-job", - "queue": { - "enabled": true - }, - "tasks": [ - { - "condition_task": { - "left": "1", - "op": "EQUAL_TO", - "right": "1" - }, - "task_key": "main" - } - ] - } -} -{ - "method": "POST", - "path": "/api/2.2/jobs/create", - "body": { - "deployment": { - "kind": "BUNDLE", - "metadata_file_path": "/Workspace/Users/[USERNAME]/.bundle/job-runs-wait-output/default/state/metadata.json" - }, - "edit_mode": "UI_LOCKED", - "format": "MULTI_TASK", - "max_concurrent_runs": 1, - "name": "downstream-job", - "queue": { - "enabled": true - }, - "tags": { - "run_result": "SUCCESS" - }, - "tasks": [ - { - "condition_task": { - "left": "1", - "op": "EQUAL_TO", - "right": "1" - }, - "task_key": "main" - } - ] - } -} - -=== redeploy is a no-op: the resolved result_state tag is stable, not perpetual drift ->>> [CLI] bundle plan -o json -"skip" - ->>> [CLI] bundle deploy -Uploading bundle files to /Workspace/Users/[USERNAME]/.bundle/job-runs-wait-output/default/files... -Deploying resources... -Updating deployment state... -Deployment complete! - ->>> [CLI] bundle destroy --auto-approve -The following resources will be deleted: - delete resources.job_runs.my_run - delete resources.jobs.downstream_job - delete resources.jobs.my_job - -All files and directories at the following location will be deleted: /Workspace/Users/[USERNAME]/.bundle/job-runs-wait-output/default - -Deleting files... -Destroy complete! diff --git a/acceptance/bundle/resources/job_runs/wait_output/script b/acceptance/bundle/resources/job_runs/wait_output/script deleted file mode 100644 index 85812d8c18b..00000000000 --- a/acceptance/bundle/resources/job_runs/wait_output/script +++ /dev/null @@ -1,18 +0,0 @@ -cleanup() { - trace $CLI bundle destroy --auto-approve - rm -f out.requests.txt -} -trap cleanup EXIT - -title "deploy waits for the run to finish, then the downstream job reads its result_state" -trace $CLI bundle deploy - -title "the downstream job was created with the run's result_state resolved into its tag" -# A concrete SUCCESS tag proves the run finished and the wait published its -# output before the downstream job was created. -trace print_requests.py //jobs/create - -title "redeploy is a no-op: the resolved result_state tag is stable, not perpetual drift" -# The resolved tag reads back identically, so the job stays as deployed. -trace $CLI bundle plan -o json | jq '.plan["resources.jobs.downstream_job"].action // "none"' -trace $CLI bundle deploy diff --git a/acceptance/bundle/resources/job_runs/wait_output/test.toml b/acceptance/bundle/resources/job_runs/wait_output/test.toml deleted file mode 100644 index 4b94d8b58e9..00000000000 --- a/acceptance/bundle/resources/job_runs/wait_output/test.toml +++ /dev/null @@ -1,4 +0,0 @@ -# job_runs is a direct-engine-only resource; the Terraform provider has no -# equivalent, so restrict the matrix to direct. -EnvMatrix.DATABRICKS_BUNDLE_ENGINE = ["direct"] -RecordRequests = true From a397bc01e4b2fe75357ebb39926704ac6b39024d Mon Sep 17 00:00:00 2001 From: Rada Kamysheva Date: Tue, 4 Aug 2026 11:05:24 +0000 Subject: [PATCH 14/23] job_runs: re-run a run that did not succeed, rather than refusing to resolve it CheckSettled kept a reference to an unfinished run from resolving to an empty outcome, but it needed an adapter method of its own and left a run that failed recorded as if it were done. Record the outcome the run has to reach instead: PrepareState always sets result_state to SUCCESS, DoRead reports the outcome the run reached, and the existing drift machinery compares the two. Anything other than SUCCESS is a recreate, so a run that failed, or that an interrupted deploy left going, is triggered again on the next deploy. ignore_remote_changes can no longer be a root rule, since that would suppress the comparison, so it now lists every jobs.RunNow field. TestJobRunIgnoresEveryRequestField keeps the list in step with the SDK. --- .../bundles/job-runs-wait-for-completion.md | 2 +- acceptance/bundle/refschema/out.fields.txt | 1 + .../resources/job_runs/failed_run/output.txt | 33 ++++- .../resources/job_runs/failed_run/script | 20 +-- .../resources/job_runs/redeploy/output.txt | 4 +- bundle/direct/bundle_plan.go | 8 - bundle/direct/bundle_plan_test.go | 140 +++++++++++++----- bundle/direct/dresources/adapter.go | 30 ---- bundle/direct/dresources/all_test.go | 5 + bundle/direct/dresources/job_run.go | 63 ++++---- bundle/direct/dresources/job_run_test.go | 98 ++++++------ bundle/direct/dresources/resources.yml | 39 ++++- bundle/internal/schema/annotations.yml | 2 +- bundle/schema/jsonschema.json | 2 +- 14 files changed, 268 insertions(+), 179 deletions(-) diff --git a/.nextchanges/bundles/job-runs-wait-for-completion.md b/.nextchanges/bundles/job-runs-wait-for-completion.md index 9c7d3941c56..d7052ab4b5a 100644 --- a/.nextchanges/bundles/job-runs-wait-for-completion.md +++ b/.nextchanges/bundles/job-runs-wait-for-completion.md @@ -1 +1 @@ -direct: the experimental `job_runs` resource now waits for the triggered run to finish, so other resources can reference the run's outcome (e.g. `${resources.job_runs.nightly.state.result_state}`). The deploy reports the run page URL while it waits, and a run that does not succeed fails the deploy, naming the failed task and the message it reported. A failed run stays recorded, so the next deploy does not run the job again. Destroying a run that has not finished, which is what interrupting a deploy mid-wait leaves behind, cancels it first. +direct: the experimental `job_runs` resource now waits for the triggered run to finish, so other resources can reference its outcome (e.g. `${resources.job_runs.nightly.state.result_state}`). A run that does not succeed fails the deploy, naming the failed task, and is run again on the next deploy. diff --git a/acceptance/bundle/refschema/out.fields.txt b/acceptance/bundle/refschema/out.fields.txt index ffec0976857..9fb0058ca7b 100644 --- a/acceptance/bundle/refschema/out.fields.txt +++ b/acceptance/bundle/refschema/out.fields.txt @@ -870,6 +870,7 @@ resources.job_runs.*.python_params[*] string ALL resources.job_runs.*.queue *jobs.QueueSettings ALL resources.job_runs.*.queue.enabled bool ALL resources.job_runs.*.resolved_job_id int64 INPUT +resources.job_runs.*.result_state jobs.RunResultState REMOTE STATE resources.job_runs.*.run_id int64 REMOTE resources.job_runs.*.run_name string REMOTE resources.job_runs.*.run_page_url string REMOTE diff --git a/acceptance/bundle/resources/job_runs/failed_run/output.txt b/acceptance/bundle/resources/job_runs/failed_run/output.txt index 7b0f33c5574..3ac4eab736d 100644 --- a/acceptance/bundle/resources/job_runs/failed_run/output.txt +++ b/acceptance/bundle/resources/job_runs/failed_run/output.txt @@ -15,26 +15,37 @@ Error: cannot create resources.jobs.downstream_job: dependency failed: resources Updating deployment state... -=== the failed run stays recorded, so a redeploy starts no new run +=== the failed run is recorded, and not having succeeded is drift >>> read_id.py my_run [MY_RUN_ID] >>> [CLI] bundle plan +recreate job_runs.my_run create jobs.downstream_job -Plan: 1 to add, 0 to change, 0 to delete, 2 unchanged +Plan: 2 to add, 0 to change, 1 to delete, 1 unchanged +=== so a redeploy runs the job again, and fails again >>> [CLI] bundle deploy Uploading bundle files to /Workspace/Users/[USERNAME]/.bundle/job-runs-failed-run/default/files... Deploying resources... +job run [NUMID]: Run URL: [DATABRICKS_URL]/jobs/[NUMID]/runs/[NUMID]?o=[NUMID] +job run [NUMID]: [TIMESTAMP] "my-job" TERMINATED FAILED task main failed +Error: cannot recreate resources.job_runs.my_run: waiting after creating id=[NUMID]: run did not succeed: FAILED: task main failed +task "main": spark python task execution failed: exit status 1 +intentional failure + +run page: [DATABRICKS_URL]/jobs/[NUMID]/runs/[NUMID]?o=[NUMID] + +Error: cannot create resources.jobs.downstream_job: dependency failed: resources.job_runs.my_run + Updating deployment state... -Deployment complete! -=== downstream_job resolved its tag from the failed run ->>> jq -r select(.path == "/api/2.2/jobs/create" and .body.name == "downstream-job") | .body.tags.run_result out.requests.txt -FAILED +=== downstream_job was never created +>>> jq -r select(.path == "/api/2.2/jobs/create") | .body.name out.requests.txt +my-job -=== run-now was issued once +=== run-now was issued once per deploy >>> print_requests.py //jobs/run-now { "method": "POST", @@ -43,11 +54,17 @@ FAILED "job_id": [NUMID] } } +{ + "method": "POST", + "path": "/api/2.2/jobs/run-now", + "body": { + "job_id": [NUMID] + } +} >>> [CLI] bundle destroy --auto-approve The following resources will be deleted: delete resources.job_runs.my_run - delete resources.jobs.downstream_job delete resources.jobs.my_job All files and directories at the following location will be deleted: /Workspace/Users/[USERNAME]/.bundle/job-runs-failed-run/default diff --git a/acceptance/bundle/resources/job_runs/failed_run/script b/acceptance/bundle/resources/job_runs/failed_run/script index 40dba04f8b3..753bdcc0fd3 100644 --- a/acceptance/bundle/resources/job_runs/failed_run/script +++ b/acceptance/bundle/resources/job_runs/failed_run/script @@ -4,21 +4,23 @@ cleanup() { } trap cleanup EXIT -# The run finishes FAILED, so the deploy aborts: the error names the failed task -# and the message it reported, and downstream_job is reported as a failed +# The error names the failed task, and downstream_job is reported as a failed # dependency because it reads the run's result_state. title "a run that finishes FAILED fails the deploy" musterr trace $CLI bundle deploy -# The framework saves the run id before calling WaitAfterCreate, so a run that -# failed stays recorded and an unchanged config plans no second run. -title "the failed run stays recorded, so a redeploy starts no new run" +# The framework saves the run id before calling WaitAfterCreate, so the failed run +# stays recorded, and FAILED against the required SUCCESS is drift. +title "the failed run is recorded, and not having succeeded is drift" trace read_id.py my_run trace $CLI bundle plan -trace $CLI bundle deploy -title "downstream_job resolved its tag from the failed run" -trace jq -r 'select(.path == "/api/2.2/jobs/create" and .body.name == "downstream-job") | .body.tags.run_result' out.requests.txt +title "so a redeploy runs the job again, and fails again" +musterr trace $CLI bundle deploy + +# Only my-job: nothing downstream of a run that never succeeded is created. +title "downstream_job was never created" +trace jq -r 'select(.path == "/api/2.2/jobs/create") | .body.name' out.requests.txt -title "run-now was issued once" +title "run-now was issued once per deploy" trace print_requests.py //jobs/run-now diff --git a/acceptance/bundle/resources/job_runs/redeploy/output.txt b/acceptance/bundle/resources/job_runs/redeploy/output.txt index e75196469df..03c7dcf22ec 100644 --- a/acceptance/bundle/resources/job_runs/redeploy/output.txt +++ b/acceptance/bundle/resources/job_runs/redeploy/output.txt @@ -56,7 +56,8 @@ Resources: "job_id": [MY_JOB_ID], "job_parameters": { "env": "prod" - } + }, + "result_state": "SUCCESS" } }, "remote_state": { @@ -64,6 +65,7 @@ Resources: "job_parameters": { "env": "dev" }, + "result_state": "SUCCESS", "run_id": [NUMID], "run_name": "my-job", "run_page_url": "[DATABRICKS_URL]/jobs/[MY_JOB_ID]/runs/[NUMID]?o=[NUMID]", diff --git a/bundle/direct/bundle_plan.go b/bundle/direct/bundle_plan.go index 8b0ab74b3b0..669c187aac5 100644 --- a/bundle/direct/bundle_plan.go +++ b/bundle/direct/bundle_plan.go @@ -805,10 +805,6 @@ func (b *DeploymentBundle) LookupReferencePreDeploy(ctx context.Context, path *s if canReadRemoteCache { remoteState, ok := b.RemoteStateCache.Load(targetResourceKey) if ok { - err = adapter.CheckSettled(remoteState) - if err != nil { - return nil, err - } return structaccess.Get(remoteState, fieldPath) } else { return nil, fmt.Errorf("internal error: no entry in remote state cache for %q (remote-only)", targetResourceKey) @@ -828,10 +824,6 @@ func (b *DeploymentBundle) LookupReferencePreDeploy(ctx context.Context, path *s if canReadRemoteCache { remoteState, ok := b.RemoteStateCache.Load(targetResourceKey) if ok { - err = adapter.CheckSettled(remoteState) - if err != nil { - return nil, err - } return structaccess.Get(remoteState, fieldPath) } else { return nil, fmt.Errorf("internal error: no entry in remote state cache for %q", targetResourceKey) diff --git a/bundle/direct/bundle_plan_test.go b/bundle/direct/bundle_plan_test.go index f0d1fb1b972..0e93200f2ca 100644 --- a/bundle/direct/bundle_plan_test.go +++ b/bundle/direct/bundle_plan_test.go @@ -290,50 +290,124 @@ func TestShouldSkipBackendDefault_MapDriftUsesBracketKeys(t *testing.T) { const jobRunKey = "resources.job_runs.my_run" -// planWithSkippedJobRun builds the state a deploy is in when a run recorded by an -// earlier deploy needs no change: the plan skips it, so references to it are served -// from the remote state cache. -func planWithSkippedJobRun(t *testing.T, state *jobs.RunState) *DeploymentBundle { - t.Helper() +// The plan skips only a run that reached the required SUCCESS, so a reference +// served from the remote state cache reads a finished run. +func TestLookupReferencePreDeploy_FinishedJobRun(t *testing.T) { + b := bundleWithSkippedJobRun(t, &dresources.JobRunRemote{ + RunId: 123, + ResultState: jobs.RunResultStateSuccess, + State: &jobs.RunState{ + LifeCycleState: jobs.RunLifeCycleStateTerminated, + ResultState: jobs.RunResultStateSuccess, + }, + }) - adapters, err := dresources.InitAll(nil) - require.NoError(t, err) + value, err := b.LookupReferencePreDeploy(t.Context(), structpath.MustParsePath(jobRunKey+".state.result_state")) - plan := deployplan.NewPlanDirect() - plan.Plan[jobRunKey] = &deployplan.PlanEntry{ - Action: deployplan.Skip, - NewState: &structvar.StructVarJSON{}, - } + require.NoError(t, err) + assert.Equal(t, jobs.RunResultStateSuccess, value) +} - b := &DeploymentBundle{Adapters: adapters, Plan: plan} - b.StateCache.Store(jobRunKey, structvar.NewStructVar(&resources.JobRun{}, nil)) - b.RemoteStateCache.Store(jobRunKey, &dresources.JobRunRemote{ - RunId: 123, - State: state, +// A resource reads the run the workspace reports. These paths are absent from +// JobRunState, so they resolve from the remote, and the failure-shaped values +// below come back unrewritten. result_state is the exception: it resolves to the +// outcome PrepareState requires, which TestJobRunOutcomeIsDrift keeps from +// standing in for one the run never reached. +func TestLookupReferencePreDeploy_JobRunReferencesAreRaw(t *testing.T) { + b := bundleWithSkippedJobRun(t, &dresources.JobRunRemote{ + RunId: 123, + RunName: "my-job", + RunPageUrl: "https://myworkspace.databricks.test/jobs/456/runs/123", + ResultState: jobs.RunResultStateFailed, + State: &jobs.RunState{ + LifeCycleState: jobs.RunLifeCycleStateTerminated, + ResultState: jobs.RunResultStateFailed, + StateMessage: "task main failed", + }, }) - return b + + for field, want := range map[string]any{ + "state.result_state": jobs.RunResultStateFailed, + "state.life_cycle_state": jobs.RunLifeCycleStateTerminated, + "state.state_message": "task main failed", + "run_id": int64(123), + "run_name": "my-job", + "run_page_url": "https://myworkspace.databricks.test/jobs/456/runs/123", + } { + t.Run(field, func(t *testing.T) { + value, err := b.LookupReferencePreDeploy(t.Context(), structpath.MustParsePath(jobRunKey+"."+field)) + + require.NoError(t, err) + assert.Equal(t, want, value) + }) + } } -// A run that has not finished reports an empty result_state, which would otherwise -// be substituted into whatever references it. -func TestLookupReferencePreDeploy_UnfinishedJobRun(t *testing.T) { - b := planWithSkippedJobRun(t, &jobs.RunState{ - LifeCycleState: jobs.RunLifeCycleStateRunning, - }) +// References are served from the remote state cache only for a run the plan +// skips, and any outcome other than the required SUCCESS is a recreate. +func TestJobRunOutcomeIsDrift(t *testing.T) { + adapters, err := dresources.InitAll(nil) + require.NoError(t, err) - _, err := b.LookupReferencePreDeploy(t.Context(), structpath.MustParsePath(jobRunKey+".state.result_state")) + for name, remote := range map[string]jobs.RunResultState{ + "FAILED": jobs.RunResultStateFailed, + "CANCELED": jobs.RunResultStateCanceled, + "TIMEDOUT": jobs.RunResultStateTimedout, + // A run that is still going reports no result at all. + "no result yet": "", + } { + t.Run(name, func(t *testing.T) { + // Old == New: a deploy records the outcome it asked for, and only the + // remote says the run missed it. + changes := deployplan.Changes{"result_state": &deployplan.ChangeDesc{ + Old: jobs.RunResultStateSuccess, + New: jobs.RunResultStateSuccess, + Remote: remote, + }} + + err := addPerFieldActions(t.Context(), adapters["job_runs"], changes, nil) - require.ErrorContains(t, err, "run 123 has not finished (RUNNING)") + require.NoError(t, err) + assert.Equal(t, deployplan.Recreate, changes["result_state"].Action) + }) + } } -func TestLookupReferencePreDeploy_FinishedJobRun(t *testing.T) { - b := planWithSkippedJobRun(t, &jobs.RunState{ - LifeCycleState: jobs.RunLifeCycleStateTerminated, - ResultState: jobs.RunResultStateSuccess, - }) +// The run reached the required outcome, so the plan can skip it. +func TestJobRunSucceededOutcomeIsNotDrift(t *testing.T) { + adapters, err := dresources.InitAll(nil) + require.NoError(t, err) - value, err := b.LookupReferencePreDeploy(t.Context(), structpath.MustParsePath(jobRunKey+".state.result_state")) + changes := deployplan.Changes{"result_state": &deployplan.ChangeDesc{ + Old: jobs.RunResultStateSuccess, + New: jobs.RunResultStateSuccess, + Remote: jobs.RunResultStateSuccess, + }} + + err = addPerFieldActions(t.Context(), adapters["job_runs"], changes, nil) require.NoError(t, err) - assert.Equal(t, jobs.RunResultStateSuccess, value) + assert.Equal(t, deployplan.Skip, changes["result_state"].Action) +} + +// bundleWithSkippedJobRun sets up a deploy whose plan skips the run, so +// references to it resolve from the remote state cache. The state cache holds +// PrepareState's output, as a real plan does. +func bundleWithSkippedJobRun(t *testing.T, remote *dresources.JobRunRemote) *DeploymentBundle { + t.Helper() + + adapters, err := dresources.InitAll(nil) + require.NoError(t, err) + + plan := deployplan.NewPlanDirect() + plan.Plan[jobRunKey] = &deployplan.PlanEntry{ + Action: deployplan.Skip, + NewState: &structvar.StructVarJSON{}, + } + + b := &DeploymentBundle{Adapters: adapters, Plan: plan} + state := (&dresources.ResourceJobRun{}).PrepareState(&resources.JobRun{}) + b.StateCache.Store(jobRunKey, structvar.NewStructVar(state, nil)) + b.RemoteStateCache.Store(jobRunKey, remote) + return b } diff --git a/bundle/direct/dresources/adapter.go b/bundle/direct/dresources/adapter.go index d51aa5d6451..fdaa15bfcea 100644 --- a/bundle/direct/dresources/adapter.go +++ b/bundle/direct/dresources/adapter.go @@ -90,14 +90,6 @@ type IResource interface { // (returned by GET, not 404) that rejects a second DELETE. // Example: func (*ResourceApp) IsGone(remote *AppRemote) bool IsGone(remoteState any) bool - - // [Optional] CheckSettled reports whether the remote's output fields are - // final. Implement it for a resource whose remote can be tracked and - // unchanged while its outputs are still filling in (e.g. a job run that has - // not finished): a reference to such a resource fails with the returned - // error instead of resolving to a zero value. - // Example: func (*ResourceJobRun) CheckSettled(remote *JobRunRemote) error - CheckSettled(remoteState any) error } // Adapter wraps resource implementation, validates signatures and type consistency across methods @@ -119,7 +111,6 @@ type Adapter struct { overrideChangeDesc *calladapt.BoundCaller doResize *calladapt.BoundCaller isGone *calladapt.BoundCaller - checkSettled *calladapt.BoundCaller resourceConfig *ResourceLifecycleConfig generatedResourceConfig *ResourceLifecycleConfig @@ -153,7 +144,6 @@ func NewAdapter(typedNil any, resourceType string, client *databricks.WorkspaceC waitAfterDelete: nil, overrideChangeDesc: nil, isGone: nil, - checkSettled: nil, resourceConfig: GetResourceConfig(resourceType), generatedResourceConfig: GetGeneratedResourceConfig(resourceType), keyedSlices: nil, @@ -255,11 +245,6 @@ func (a *Adapter) initMethods(resource any) error { return err } - a.checkSettled, err = calladapt.PrepareCall(resource, reflect.TypeFor[IResource](), "CheckSettled") - if err != nil { - return err - } - keyedSlicesCall, err := calladapt.PrepareCall(resource, reflect.TypeFor[IResource](), "KeyedSlices") if err != nil { return err @@ -345,10 +330,6 @@ func (a *Adapter) validate() error { validations = append(validations, "IsGone remoteState", a.isGone.InTypes[0], remoteType) } - if a.checkSettled != nil { - validations = append(validations, "CheckSettled remoteState", a.checkSettled.InTypes[0], remoteType) - } - if a.doUpdateWithID != nil { validations = append(validations, "DoUpdateWithID newState", a.doUpdateWithID.InTypes[2], stateType) // DoUpdateWithID must return (string, remoteType, error) @@ -614,17 +595,6 @@ func (a *Adapter) IsGone(remoteState any) bool { return outs[0].(bool) } -// CheckSettled reports whether the remote's output fields are final, so a -// reference to one resolves to the value it will keep. Resources that don't -// implement CheckSettled are always settled. -func (a *Adapter) CheckSettled(remoteState any) error { - if a.checkSettled == nil { - return nil - } - _, err := a.checkSettled.Call(remoteState) - return err -} - // prepareCallRequired prepares a call and ensures the method is found. func prepareCallRequired(resource any, methodName string) (*calladapt.BoundCaller, error) { caller, err := calladapt.PrepareCall(resource, reflect.TypeFor[IResource](), methodName) diff --git a/bundle/direct/dresources/all_test.go b/bundle/direct/dresources/all_test.go index f00ec21ed93..6d8ad997c06 100644 --- a/bundle/direct/dresources/all_test.go +++ b/bundle/direct/dresources/all_test.go @@ -1008,6 +1008,11 @@ func testCRUD(t *testing.T, group string, adapter *Adapter, client *databricks.W remotePostWaitCreate, err := adapter.DoRead(ctx, createdID) require.NoError(t, err) require.Equal(t, remotePostWaitCreate, remoteStateFromWaitCreate) + + // The settled state is the one a deploy records, so the field checks below + // run against it: a job run's result_state fills in only once it settles. + remappedState, err = adapter.RemapState(remoteStateFromWaitCreate) + require.NoError(t, err) } if adapter.HasDoUpdate() { diff --git a/bundle/direct/dresources/job_run.go b/bundle/direct/dresources/job_run.go index 4c6729fd858..63af7f47459 100644 --- a/bundle/direct/dresources/job_run.go +++ b/bundle/direct/dresources/job_run.go @@ -23,9 +23,14 @@ import ( // jobRunTimeout matches the timeout `bundle run` allows a run (bundle/run/job.go). const jobRunTimeout = 24 * time.Hour -// JobRunState is what we persist for a triggered run: the RunNow request. +// JobRunState is what we persist for a triggered run: the RunNow request, plus +// the outcome the run is required to reach. type JobRunState struct { jobs.RunNow + + // Always SUCCESS, never read from the config: a run that did not succeed + // differs from the remote and is re-triggered by recreate_on_changes. + ResultState jobs.RunResultState `json:"result_state,omitempty"` } func (s *JobRunState) UnmarshalJSON(b []byte) error { @@ -41,6 +46,10 @@ func (s JobRunState) MarshalJSON() ([]byte, error) { type JobRunRemote struct { jobs.RunNow + // Repeats state.result_state at the path JobRunState uses, so the planner can + // compare the two. See "RemapState is a dumb copy" in README.md. + ResultState jobs.RunResultState `json:"result_state,omitempty"` + RunId int64 `json:"run_id,omitempty"` RunName string `json:"run_name,omitempty"` State *jobs.RunState `json:"state,omitempty"` @@ -70,7 +79,8 @@ func (*ResourceJobRun) New(client *databricks.WorkspaceClient) *ResourceJobRun { func (*ResourceJobRun) PrepareState(input *resources.JobRun) *JobRunState { return &JobRunState{ - RunNow: input.RunNow, + RunNow: input.RunNow, + ResultState: jobs.RunResultStateSuccess, } } @@ -109,17 +119,16 @@ func makeJobRunRemote(run *jobs.Run) *JobRunRemote { Queue: nil, ForceSendFields: nil, }, - RunId: run.RunId, - RunName: run.RunName, - State: run.State, - RunPageUrl: workspaceurls.ModernizeJobRunPageURL(run.RunPageUrl), - RunType: run.RunType, + ResultState: run.State.ResultState, + RunId: run.RunId, + RunName: run.RunName, + State: run.State, + RunPageUrl: workspaceurls.ModernizeJobRunPageURL(run.RunPageUrl), + RunType: run.RunType, } } -// DoRead returns the run as GetRun reports it; a 404 lets the planner -// re-trigger. Root ignore_remote_changes suppresses all remote drift, so a run -// is recreated only on a local config change. +// DoRead returns the run as GetRun reports it; a 404 lets the planner re-trigger. func (r *ResourceJobRun) DoRead(ctx context.Context, id string) (*JobRunRemote, error) { runID, err := parseRunID(id) if err != nil { @@ -135,20 +144,10 @@ func (r *ResourceJobRun) DoRead(ctx context.Context, id string) (*JobRunRemote, return makeJobRunRemote(run), nil } -// RemapState extracts the embedded RunNow as the state used for diffing. +// RemapState extracts the fields used for diffing: the RunNow request and the +// outcome the run reached. func (*ResourceJobRun) RemapState(remote *JobRunRemote) *JobRunState { - return &JobRunState{RunNow: remote.RunNow} -} - -// CheckSettled rejects a run that has not finished. Interrupting a deploy -// mid-wait leaves the run going and its id recorded, so the next deploy plans no -// change for it; without this, a reference to an outcome the run has not reached -// would resolve to an empty string. -func (*ResourceJobRun) CheckSettled(remote *JobRunRemote) error { - if runIsTerminal(remote.State.LifeCycleState) { - return nil - } - return fmt.Errorf("run %d has not finished (%s)%s", remote.RunId, remote.State.LifeCycleState, runPageLine(remote.RunPageUrl)) + return &JobRunState{RunNow: remote.RunNow, ResultState: remote.ResultState} } func (r *ResourceJobRun) DoCreate(ctx context.Context, config *JobRunState) (string, *JobRunRemote, error) { @@ -173,11 +172,9 @@ func (r *ResourceJobRun) WaitAfterCreate(ctx context.Context, id string, _ *JobR // outlives the poll so an abandoned wait can still link the run. var tracker progress.JobStateTracker var pageURL string - // Polled here rather than through Jobs.WaitGetRunJobTerminatedOrSkipped: a run - // whose task failed reports the deprecated life_cycle_state as INTERNAL_ERROR - // (status.state is TERMINATED, termination code RUN_EXECUTION_ERROR), and the - // SDK waiter halts on it with an error of its own, which hides the task that - // failed. + // Polled here rather than through the SDK waiter, which halts with an error of + // its own on the INTERNAL_ERROR a run whose task failed reports, hiding the + // task that failed. run, err := retries.Poll(ctx, jobRunTimeout, func() (*jobs.Run, *retries.Err) { var req jobs.GetRunRequest req.RunId = runID @@ -194,7 +191,7 @@ func (r *ResourceJobRun) WaitAfterCreate(ctx context.Context, id string, _ *JobR }) if err != nil { // The wait can end with the run still going (timeout, interrupt), so link - // the run page: the next deploy triggers no second run, finished or not. + // the run page. if ctx.Err() != nil { // A cancelled context is reported as a timeout. return nil, fmt.Errorf("interrupted while waiting for the run to finish: %w%s", ctx.Err(), runPageLine(pageURL)) @@ -288,9 +285,9 @@ func runPageLine(rawURL string) string { } // logRunProgress logs every state change like `bundle run` does, but reports only -// the run page URL and the run's final state to the user: how many states a run -// passes through varies with how long its compute takes to start, so a deploy's -// output would not be reproducible. +// the run page URL and the final state to the user: how many states a run passes +// through varies with how long its compute takes to start, and a deploy's output +// has to stay reproducible. func logRunProgress(ctx context.Context, run *jobs.Run, tracker *progress.JobStateTracker) { event, first := tracker.Poll(run) if event == nil { @@ -307,7 +304,7 @@ func logRunProgress(ctx context.Context, run *jobs.Run, tracker *progress.JobSta } } -// runIsTerminal reports whether a run is done, i.e. in a state the SDK waiter stops on. +// runIsTerminal reports whether a run has stopped, whatever it stopped as. func runIsTerminal(state jobs.RunLifeCycleState) bool { return state == jobs.RunLifeCycleStateTerminated || state == jobs.RunLifeCycleStateSkipped || diff --git a/bundle/direct/dresources/job_run_test.go b/bundle/direct/dresources/job_run_test.go index c53b151d365..89f0c029f59 100644 --- a/bundle/direct/dresources/job_run_test.go +++ b/bundle/direct/dresources/job_run_test.go @@ -2,10 +2,15 @@ package dresources import ( "context" + "reflect" + "slices" + "strings" "sync/atomic" "testing" "time" + "github.com/databricks/cli/bundle/config/resources" + "github.com/databricks/cli/libs/structs/structpath" "github.com/databricks/cli/libs/testserver" "github.com/databricks/databricks-sdk-go" "github.com/databricks/databricks-sdk-go/service/jobs" @@ -82,7 +87,6 @@ func TestJobRunWaitFailsOnFailedResult(t *testing.T) { _, err := waitForTestRun(t, t.Context(), client) - // Only SUCCESS completes the deploy; a FAILED result fails it. require.ErrorContains(t, err, "did not succeed: FAILED: task failed") } @@ -112,8 +116,6 @@ func TestJobRunWaitReportsFailedTask(t *testing.T) { _, err := waitForTestRun(t, t.Context(), jobRunClientFor(t, server)) - // The error names the failing task and the message it reported, and leaves out - // the tasks that did not fail. require.ErrorContains(t, err, `task "main": notebook not found`) assert.NotContains(t, err.Error(), `task "ok"`) } @@ -141,8 +143,7 @@ func TestJobRunWaitFailsOnInternalError(t *testing.T) { } // A real workspace reports a run whose task failed as INTERNAL_ERROR in the -// deprecated life_cycle_state, which the SDK waiter halts on with an error of its -// own. The failing task still has to be named. +// deprecated life_cycle_state. The failing task still has to be named. func TestJobRunWaitReportsFailedTaskOfInternalErrorRun(t *testing.T) { server := testserver.New(t) server.Handle("GET", "/api/2.2/jobs/runs/get", func(req testserver.Request) any { @@ -211,14 +212,13 @@ func TestJobRunWaitAbandonedLinksTheRun(t *testing.T) { _, err := waitForTestRun(t, ctx, client) - // The run keeps going, so the error links to it and names the interrupt rather - // than the 24h bound. + // The run keeps going, so the error links to it and names the interrupt. require.ErrorContains(t, err, "interrupted while waiting for the run to finish") require.ErrorContains(t, err, testRunPageLink) } -// After an abandoned wait the next deploy triggers no second run: it reads this -// one, and the empty outcome is what CheckSettled keeps out of a reference. +// An abandoned wait leaves the run going with its id recorded, so the next deploy +// reads an empty outcome, which result_state drift catches. func TestJobRunReadOfUnfinishedRunReportsNoResult(t *testing.T) { client := jobRunClient(t, &jobs.RunState{LifeCycleState: jobs.RunLifeCycleStateRunning}) @@ -227,63 +227,61 @@ func TestJobRunReadOfUnfinishedRunReportsNoResult(t *testing.T) { require.NoError(t, err) require.NotNil(t, remote.State) assert.Equal(t, jobs.RunLifeCycleStateRunning, remote.State.LifeCycleState) - assert.Empty(t, remote.State.ResultState) + assert.Empty(t, remote.ResultState) } -func TestJobRunCheckSettled(t *testing.T) { - // Every state runIsTerminal accepts is settled: the run has the outcome it is - // going to keep, whether or not it succeeded. - for _, state := range []jobs.RunLifeCycleState{ - jobs.RunLifeCycleStateTerminated, - jobs.RunLifeCycleStateSkipped, - jobs.RunLifeCycleStateInternalError, - } { - t.Run(string(state), func(t *testing.T) { - remote := &JobRunRemote{RunId: 123, State: &jobs.RunState{LifeCycleState: state}} +// PrepareState records the outcome the run must reach, the same for every run, +// so the planner has something to compare the remote against. +func TestJobRunPrepareStateRequiresSuccess(t *testing.T) { + state := (&ResourceJobRun{}).PrepareState(&resources.JobRun{RunNow: jobs.RunNow{JobId: 456}}) - require.NoError(t, (&ResourceJobRun{}).CheckSettled(remote)) - }) - } + assert.Equal(t, jobs.RunResultStateSuccess, state.ResultState) } -func TestJobRunCheckSettledRejectsUnfinishedRun(t *testing.T) { - for _, state := range []jobs.RunLifeCycleState{ - jobs.RunLifeCycleStatePending, - jobs.RunLifeCycleStateRunning, +// The planner diffs RemapState(remote) against PrepareState(config), so a run +// that did not end in SUCCESS has to surface as a difference on result_state. +func TestJobRunRemapStateCarriesTheOutcome(t *testing.T) { + for _, outcome := range []jobs.RunResultState{ + jobs.RunResultStateSuccess, + jobs.RunResultStateFailed, + // A run still going, and a SKIPPED one, report no result at all. + "", } { - t.Run(string(state), func(t *testing.T) { - remote := &JobRunRemote{ - RunId: 123, - State: &jobs.RunState{LifeCycleState: state}, - RunPageUrl: testRunPageURL, - } - - err := (&ResourceJobRun{}).CheckSettled(remote) - - // The run has no outcome yet, and the error links the run so the user - // can see what it is still doing. - require.ErrorContains(t, err, "run 123 has not finished ("+string(state)+")") - require.ErrorContains(t, err, testRunPageLink) + t.Run(string(outcome), func(t *testing.T) { + remote := &JobRunRemote{RunId: 123, ResultState: outcome} + + state := (&ResourceJobRun{}).RemapState(remote) + + assert.Equal(t, outcome, state.ResultState) }) } } -func TestJobRunCheckSettledIsWiredIntoTheAdapter(t *testing.T) { +// resources.yml ignores remote drift on everything the RunNow request carries, +// since GetRun does not echo it back faithfully, and leaves result_state alone. +func TestJobRunIgnoresEveryRequestField(t *testing.T) { adapters, err := InitAll(nil) require.NoError(t, err) + ignored := adapters["job_runs"].ResourceConfig().IgnoreRemoteChanges - running := &JobRunRemote{RunId: 123, State: &jobs.RunState{ - LifeCycleState: jobs.RunLifeCycleStateRunning, - }} - require.ErrorContains(t, adapters["job_runs"].CheckSettled(running), "has not finished") + for field := range reflect.TypeFor[jobs.RunNow]().Fields() { + name, _, _ := strings.Cut(field.Tag.Get("json"), ",") + if name == "" || name == "-" { + continue + } + assert.True(t, ignoresRemoteChanges(ignored, name), "jobs.RunNow field %q is not in job_runs ignore_remote_changes", name) + } + + assert.False(t, ignoresRemoteChanges(ignored, "result_state"), "result_state must stay comparable against the remote") +} - // A resource that does not implement CheckSettled is always settled, so the - // remote state is never even looked at. - require.NoError(t, adapters["jobs"].CheckSettled(nil)) +// ignoresRemoteChanges reports whether the rules suppress remote drift on field. +func ignoresRemoteChanges(rules []FieldRule, field string) bool { + path := structpath.MustParsePath(field) + return slices.ContainsFunc(rules, func(r FieldRule) bool { return path.HasPatternPrefix(r.Field) }) } -// Reporting RUNNING for the first two polls exercises the poll loop; the other -// tests stub an already-terminal state. +// Reporting RUNNING for the first two polls exercises the poll loop. func TestJobRunWaitPollsUntilTerminal(t *testing.T) { var gets atomic.Int32 client := jobRunServer(t, func(req testserver.Request) any { diff --git a/bundle/direct/dresources/resources.yml b/bundle/direct/dresources/resources.yml index 6061db0fb04..9ccc9468040 100644 --- a/bundle/direct/dresources/resources.yml +++ b/bundle/direct/dresources/resources.yml @@ -139,11 +139,42 @@ resources: - field: job_clusters[*].new_cluster.data_security_mode job_runs: - # A run is immutable and fire-once: re-trigger only on a local config change, - # never on remote drift. Both rules omit `field` to match every field (root; - # see TestFieldRuleOmittedIsRoot). `field: ""` would instead match nothing. + # Every jobs.RunNow field: the request cannot drift, and GetRun does not echo + # it back faithfully (job_parameters come back resolved against the job's + # defaults, request-only fields not at all). TestJobRunIgnoresEveryRequestField + # keeps the list in step with the SDK. result_state is left out on purpose: + # comparing it against the remote re-triggers a run that did not succeed. ignore_remote_changes: - - reason: immutable + - field: dbt_commands + reason: immutable + - field: idempotency_token + reason: immutable + - field: jar_params + reason: immutable + - field: job_id + reason: immutable + - field: job_parameters + reason: immutable + - field: notebook_params + reason: immutable + - field: only + reason: immutable + - field: performance_target + reason: immutable + - field: pipeline_params + reason: immutable + - field: python_named_params + reason: immutable + - field: python_params + reason: immutable + - field: queue + reason: immutable + - field: spark_submit_params + reason: immutable + - field: sql_params + reason: immutable + # A run is immutable and fire-once, so any change recreates it. Omitting + # `field` matches every field (root; see TestFieldRuleOmittedIsRoot). recreate_on_changes: - reason: immutable diff --git a/bundle/internal/schema/annotations.yml b/bundle/internal/schema/annotations.yml index 77edbf0802b..88e4b7db396 100644 --- a/bundle/internal/schema/annotations.yml +++ b/bundle/internal/schema/annotations.yml @@ -969,7 +969,7 @@ resources: "description": |- The job run definitions for the bundle, where each key is the name of the job run. Each job run triggers a run of an existing job as part of bundle deployment. - The deployment waits for the run to finish and fails if it does not succeed, so other resources can reference the run's outcome, for example `${resources.job_runs..state.result_state}`. A run that did not succeed is still recorded, so an unchanged configuration is not run again. + The deployment waits for the run to finish and fails if it does not succeed, so other resources can reference the run's outcome, for example `${resources.job_runs..state.result_state}`. A run that did not succeed is run again on the next deployment. "$fields": "lifecycle": "description": |- diff --git a/bundle/schema/jsonschema.json b/bundle/schema/jsonschema.json index 6f11e620828..1584dc3bcf3 100644 --- a/bundle/schema/jsonschema.json +++ b/bundle/schema/jsonschema.json @@ -3270,7 +3270,7 @@ "markdownDescription": "The instance pool definitions for the bundle, where each key is the name of the instance pool. See [instance_pools](https://docs.databricks.com/dev-tools/bundles/resources.html#instance_pools)." }, "job_runs": { - "description": "The job run definitions for the bundle, where each key is the name of the job run. Each job run triggers a run of an existing job as part of bundle deployment.\n\nThe deployment waits for the run to finish and fails if it does not succeed, so other resources can reference the run's outcome, for example `${resources.job_runs.\u003cname\u003e.state.result_state}`. A run that did not succeed is still recorded, so an unchanged configuration is not run again.", + "description": "The job run definitions for the bundle, where each key is the name of the job run. Each job run triggers a run of an existing job as part of bundle deployment.\n\nThe deployment waits for the run to finish and fails if it does not succeed, so other resources can reference the run's outcome, for example `${resources.job_runs.\u003cname\u003e.state.result_state}`. A run that did not succeed is run again on the next deployment.", "$ref": "#/$defs/map/github.com/databricks/cli/bundle/config/resources.JobRun" }, "jobs": { From c944d7a0f3b5f8d68de8b5786113f9dfe4b53b82 Mon Sep 17 00:00:00 2001 From: Rada Kamysheva Date: Tue, 4 Aug 2026 11:05:40 +0000 Subject: [PATCH 15/23] job_runs: shorten the comments added by this branch Drop the counterfactuals, keeping only what the code does and the reasons a reader cannot get from the code itself. --- acceptance/bundle/invariant/test.toml | 5 ++--- .../job_runs/failed_cloud/databricks.yml.tmpl | 3 +-- .../bundle/resources/job_runs/failed_cloud/test.toml | 5 ++--- .../resources/job_runs/failed_run/databricks.yml | 3 +-- .../resources/job_runs/wait/databricks.yml.tmpl | 12 +++++------- acceptance/bundle/resources/job_runs/wait/test.toml | 7 +++---- bundle/run/progress/job.go | 6 +++--- libs/testserver/jobs.go | 6 +++--- 8 files changed, 20 insertions(+), 27 deletions(-) diff --git a/acceptance/bundle/invariant/test.toml b/acceptance/bundle/invariant/test.toml index c4c9286cd7a..5ba7712ddab 100644 --- a/acceptance/bundle/invariant/test.toml +++ b/acceptance/bundle/invariant/test.toml @@ -85,9 +85,8 @@ no_alert_on_cloud = ["CONFIG_Cloud=true", "INPUT_CONFIG=alert.yml.tmpl"] no_run_job_ref_on_cloud = ["CONFIG_Cloud=true", "INPUT_CONFIG=job_run_job_ref.yml.tmpl"] # Deploying a job_run waits for the run to succeed, and a real workspace reports a -# run of condition tasks alone as SKIPPED. A task that does succeed would run in -# every variant of this suite, so resources/job_runs/wait covers the real run -# on cloud instead. Still exercised locally here. +# run of condition tasks alone as SKIPPED. resources/job_runs/wait covers the real +# run on cloud; this config stays local. no_job_run_on_cloud = ["CONFIG_Cloud=true", "INPUT_CONFIG=job_run.yml.tmpl"] # Postgres resources only work on AWS diff --git a/acceptance/bundle/resources/job_runs/failed_cloud/databricks.yml.tmpl b/acceptance/bundle/resources/job_runs/failed_cloud/databricks.yml.tmpl index 1393bdaf33e..545d42fad87 100644 --- a/acceptance/bundle/resources/job_runs/failed_cloud/databricks.yml.tmpl +++ b/acceptance/bundle/resources/job_runs/failed_cloud/databricks.yml.tmpl @@ -9,8 +9,7 @@ resources: my_job: name: test-job-$UNIQUE_NAME tasks: - # Serverless keeps the run to about a minute; a job cluster would take - # several. + # Serverless keeps the run to about a minute. - task_key: main spark_python_task: python_file: ./fail.py diff --git a/acceptance/bundle/resources/job_runs/failed_cloud/test.toml b/acceptance/bundle/resources/job_runs/failed_cloud/test.toml index 8df99dac130..1a3315a6865 100644 --- a/acceptance/bundle/resources/job_runs/failed_cloud/test.toml +++ b/acceptance/bundle/resources/job_runs/failed_cloud/test.toml @@ -2,9 +2,8 @@ # equivalent, so restrict the matrix to direct. EnvMatrix.DATABRICKS_BUNDLE_ENGINE = ["direct"] -# The failure counterpart of wait: the error the test server reports is one it -# writes itself, so only a real workspace shows whether a failed task reports a -# message the deploy can name. Serverless needs Unity Catalog. +# The failure counterpart of wait: only a real workspace shows whether a failed +# task reports a message the deploy can name. Serverless needs Unity Catalog. Cloud = true RequiresUnityCatalog = true diff --git a/acceptance/bundle/resources/job_runs/failed_run/databricks.yml b/acceptance/bundle/resources/job_runs/failed_run/databricks.yml index 756474a90e2..211b7015d91 100644 --- a/acceptance/bundle/resources/job_runs/failed_run/databricks.yml +++ b/acceptance/bundle/resources/job_runs/failed_run/databricks.yml @@ -6,8 +6,7 @@ resources: my_job: name: my-job tasks: - # The test server runs this locally: it exits non-zero, failing the task - # and with it the run. + # The test server runs this locally; it exits non-zero and fails the run. - task_key: main spark_python_task: python_file: ./fail.py diff --git a/acceptance/bundle/resources/job_runs/wait/databricks.yml.tmpl b/acceptance/bundle/resources/job_runs/wait/databricks.yml.tmpl index 2daebf61240..6d1f58cea51 100644 --- a/acceptance/bundle/resources/job_runs/wait/databricks.yml.tmpl +++ b/acceptance/bundle/resources/job_runs/wait/databricks.yml.tmpl @@ -14,8 +14,7 @@ resources: - name: region default: us tasks: - # Serverless keeps the run to about a minute; a job cluster would take - # several. + # Serverless keeps the run to about a minute. - task_key: main spark_python_task: python_file: ./hello.py @@ -26,10 +25,9 @@ resources: spec: environment_version: "2" - # Reads the run's outcome, so the tag it is created with shows whether the - # deploy waited for the run before creating resources that depend on it. - # Separate from my_job, which my_run already depends on: a reference back - # would cycle. + # Reads the run's outcome, so its tag shows the deploy waited for the run + # before creating what depends on it. Separate from my_job, which my_run + # already depends on, to avoid a cycle. downstream_job: name: test-downstream-job-$UNIQUE_NAME tags: @@ -46,6 +44,6 @@ resources: job_id: ${resources.jobs.my_job.id} # Override one of the job's two parameters: a real GetRun reports the full # resolved set, including the region default, which ignore_remote_changes - # has to absorb to avoid perpetual drift. + # has to absorb. job_parameters: env: prod diff --git a/acceptance/bundle/resources/job_runs/wait/test.toml b/acceptance/bundle/resources/job_runs/wait/test.toml index fe1d7f5eea0..12b0a1b8ed6 100644 --- a/acceptance/bundle/resources/job_runs/wait/test.toml +++ b/acceptance/bundle/resources/job_runs/wait/test.toml @@ -2,8 +2,7 @@ # equivalent, so restrict the matrix to direct. EnvMatrix.DATABRICKS_BUNDLE_ENGINE = ["direct"] -# Runs the job for real, which invariant/configs/job_run.yml.tmpl no longer does -# on cloud. Serverless needs Unity Catalog. +# Runs the job for real on cloud. Serverless needs Unity Catalog. Cloud = true RequiresUnityCatalog = true @@ -17,8 +16,8 @@ Ignore = [ "hello.py", ] -# The host and the workspace selector in the run URL differ per workspace, and the -# URL form itself is covered by libs/workspaceurls; assert only that it is reported. +# The host and the workspace selector in the run URL differ per workspace; the URL +# form itself is covered by libs/workspaceurls. [[Repls]] Old = 'Run URL: .*' New = 'Run URL: [RUN_URL]' diff --git a/bundle/run/progress/job.go b/bundle/run/progress/job.go index fb29a7fadd6..9a849e84918 100644 --- a/bundle/run/progress/job.go +++ b/bundle/run/progress/job.go @@ -21,9 +21,9 @@ type JobStateTracker struct { prev *jobs.RunState } -// Poll returns the event to report for this poll, or nil when the state has not -// changed. first is true for the state the run is seen in initially, where -// callers also report the run page URL. +// Poll returns the event to report, or nil when the state has not changed. first +// is true for the state the run is seen in initially, where callers also report +// the run page URL. func (t *JobStateTracker) Poll(run *jobs.Run) (event *JobProgressEvent, first bool) { if run.State == nil { return nil, false diff --git a/libs/testserver/jobs.go b/libs/testserver/jobs.go index e8d29877e98..c4a1d6b9826 100644 --- a/libs/testserver/jobs.go +++ b/libs/testserver/jobs.go @@ -20,9 +20,9 @@ import ( const missingJobGitProviderMessage = "git_source.git_provider must be one of: github,gitlab,bitbucketcloud,gitlabenterpriseedition,bitbucketserver,azuredevopsservices,githubenterprise,awscodecommit" -// errNoCodeInWorkspace marks a task there is nothing to execute for, e.g. -// because an immutable deployment uploaded the code as a snapshot zip this -// server never unpacks. The gap is here, not in the job, so the task succeeds. +// errNoCodeInWorkspace marks a task there is nothing to execute for, e.g. code +// uploaded as a snapshot zip this server never unpacks. The gap is here, not in +// the job, so the task succeeds. var errNoCodeInWorkspace = errors.New("task code is not in the workspace") // venvPython returns the path to the Python executable in a venv. From 65db56da8117500d8a94edca41042258fc730442 Mon Sep 17 00:00:00 2001 From: Rada Kamysheva Date: Tue, 4 Aug 2026 11:16:06 +0000 Subject: [PATCH 16/23] acc: run the job_run invariant config on cloud too The exclusion was added alongside the wait itself, on the assumption that a real workspace reports a run of condition tasks alone as SKIPPED. That state means something else: a run aborted because a previous run of the same job was already active. Nothing was ever measured against a workspace, so drop the exclusion and let cloud CI say whether the run succeeds. This is the only cloud coverage of destroying and deleting a run the backend has already removed; resources/job_runs/wait covers deploy and redeploy but never re-issues a delete. --- acceptance/bundle/invariant/test.toml | 5 ----- 1 file changed, 5 deletions(-) diff --git a/acceptance/bundle/invariant/test.toml b/acceptance/bundle/invariant/test.toml index 5ba7712ddab..a53a93b84a6 100644 --- a/acceptance/bundle/invariant/test.toml +++ b/acceptance/bundle/invariant/test.toml @@ -84,11 +84,6 @@ no_alert_on_cloud = ["CONFIG_Cloud=true", "INPUT_CONFIG=alert.yml.tmpl"] # so this config is local-only (the mock server stores it verbatim). no_run_job_ref_on_cloud = ["CONFIG_Cloud=true", "INPUT_CONFIG=job_run_job_ref.yml.tmpl"] -# Deploying a job_run waits for the run to succeed, and a real workspace reports a -# run of condition tasks alone as SKIPPED. resources/job_runs/wait covers the real -# run on cloud; this config stays local. -no_job_run_on_cloud = ["CONFIG_Cloud=true", "INPUT_CONFIG=job_run.yml.tmpl"] - # Postgres resources only work on AWS no_postgres_project_on_cloud = ["CONFIG_Cloud=true", "INPUT_CONFIG=postgres_project.yml.tmpl"] no_postgres_branch_on_cloud = ["CONFIG_Cloud=true", "INPUT_CONFIG=postgres_branch.yml.tmpl"] From f89cb7305bf963074afcd752557908881fa58bd1 Mon Sep 17 00:00:00 2001 From: Rada Kamysheva Date: Tue, 4 Aug 2026 13:56:23 +0000 Subject: [PATCH 17/23] testserver: report a failed task the way a real workspace does Measured on serverless against a spark_python_task that raises: the run reports INTERNAL_ERROR in the deprecated life_cycle_state, though status.state is TERMINATED with termination code RUN_EXECUTION_ERROR; both the run and the task carry a generic message pointing at the run output; and jobs/runs/get-output splits the failure into error, the exception, and error_trace, the traceback. The fake rolled a failed task up to TERMINATED, invented "task main failed", and put its own Go error into the error field. The message a deploy names a failed task with was therefore one this server wrote rather than one a workspace reports, which is what resources/job_runs/failed_cloud exists to check. --- .../resources/job_runs/failed_run/output.txt | 16 +++--- libs/testserver/jobs.go | 50 ++++++++++++++++--- libs/testserver/jobs_test.go | 21 +++++++- 3 files changed, 68 insertions(+), 19 deletions(-) diff --git a/acceptance/bundle/resources/job_runs/failed_run/output.txt b/acceptance/bundle/resources/job_runs/failed_run/output.txt index 3ac4eab736d..720d2faa62f 100644 --- a/acceptance/bundle/resources/job_runs/failed_run/output.txt +++ b/acceptance/bundle/resources/job_runs/failed_run/output.txt @@ -4,11 +4,9 @@ Uploading bundle files to /Workspace/Users/[USERNAME]/.bundle/job-runs-failed-run/default/files... Deploying resources... job run [MY_RUN_ID]: Run URL: [DATABRICKS_URL]/jobs/[NUMID]/runs/[MY_RUN_ID]?o=[NUMID] -job run [MY_RUN_ID]: [TIMESTAMP] "my-job" TERMINATED FAILED task main failed -Error: cannot create resources.job_runs.my_run: waiting after creating id=[MY_RUN_ID]: run did not succeed: FAILED: task main failed -task "main": spark python task execution failed: exit status 1 -intentional failure - +job run [MY_RUN_ID]: [TIMESTAMP] "my-job" INTERNAL_ERROR FAILED Task main failed with message: Workload failed, see run output for details. +Error: cannot create resources.job_runs.my_run: waiting after creating id=[MY_RUN_ID]: run did not succeed: FAILED: Task main failed with message: Workload failed, see run output for details. +task "main": intentional failure run page: [DATABRICKS_URL]/jobs/[NUMID]/runs/[MY_RUN_ID]?o=[NUMID] Error: cannot create resources.jobs.downstream_job: dependency failed: resources.job_runs.my_run @@ -30,11 +28,9 @@ Plan: 2 to add, 0 to change, 1 to delete, 1 unchanged Uploading bundle files to /Workspace/Users/[USERNAME]/.bundle/job-runs-failed-run/default/files... Deploying resources... job run [NUMID]: Run URL: [DATABRICKS_URL]/jobs/[NUMID]/runs/[NUMID]?o=[NUMID] -job run [NUMID]: [TIMESTAMP] "my-job" TERMINATED FAILED task main failed -Error: cannot recreate resources.job_runs.my_run: waiting after creating id=[NUMID]: run did not succeed: FAILED: task main failed -task "main": spark python task execution failed: exit status 1 -intentional failure - +job run [NUMID]: [TIMESTAMP] "my-job" INTERNAL_ERROR FAILED Task main failed with message: Workload failed, see run output for details. +Error: cannot recreate resources.job_runs.my_run: waiting after creating id=[NUMID]: run did not succeed: FAILED: Task main failed with message: Workload failed, see run output for details. +task "main": intentional failure run page: [DATABRICKS_URL]/jobs/[NUMID]/runs/[NUMID]?o=[NUMID] Error: cannot create resources.jobs.downstream_job: dependency failed: resources.job_runs.my_run diff --git a/libs/testserver/jobs.go b/libs/testserver/jobs.go index c4a1d6b9826..4c99f7b624f 100644 --- a/libs/testserver/jobs.go +++ b/libs/testserver/jobs.go @@ -25,6 +25,34 @@ const missingJobGitProviderMessage = "git_source.git_provider must be one of: gi // the job, so the task succeeds. var errNoCodeInWorkspace = errors.New("task code is not in the workspace") +// taskFailureMessage is what a real workspace reports for a task that failed: a +// generic pointer at the run output, on both the task and the run. Measured on +// serverless against a spark_python_task that raises. The run-level message +// wraps it as "Task failed with message: ." and ends in a period, +// which the task-level one does not. +const taskFailureMessage = "Workload failed, see run output for details" + +// taskFailure splits a failed task's output the way jobs/runs/get-output does: +// error carries the exception, error_trace the traceback. Reporting the whole +// output as the error instead would leak this server's own wording into the +// message a deploy names the task with. +type taskFailure struct { + message string + trace string +} + +func (e *taskFailure) Error() string { + return e.message +} + +// newTaskFailure takes the exception from the last line of the task's output, +// where a Python traceback ends. +func newTaskFailure(output string) *taskFailure { + trimmed := strings.TrimRight(output, "\r\n") + lastLine := strings.TrimSpace(trimmed[strings.LastIndex(trimmed, "\n")+1:]) + return &taskFailure{message: lastLine, trace: trimmed} +} + // venvPython returns the path to the Python executable in a venv. // On Unix: venv/bin/python // On Windows: venv\Scripts\python.exe @@ -397,9 +425,14 @@ func (s *FakeWorkspace) JobsRunNow(req Request) Response { // Nothing ran, so the task keeps its SUCCESS state. case err != nil: taskRun.State.ResultState = jobs.RunResultStateFailed - s.JobRunOutputs[taskRunId] = jobs.RunOutput{ - Error: err.Error(), + taskRun.State.StateMessage = taskFailureMessage + runOutput := jobs.RunOutput{Error: err.Error()} + // A task this server could not even start (e.g. uv failed) has no + // traceback to report. + if failure, ok := errors.AsType[*taskFailure](err); ok { + runOutput.ErrorTrace = failure.trace } + s.JobRunOutputs[taskRunId] = runOutput case logs != "": s.JobRunOutputs[taskRunId] = jobs.RunOutput{ Logs: logs, @@ -646,7 +679,7 @@ func (s *FakeWorkspace) executePythonWheelTask(jobSettings *jobs.JobSettings, ta output, err := cmd.CombinedOutput() if err != nil { - return string(output), fmt.Errorf("wheel task execution failed: %s\n%s", err, output) + return string(output), newTaskFailure(string(output)) } // Normalize trailing newlines to match cloud behavior (exactly one trailing newline) @@ -734,7 +767,7 @@ func (s *FakeWorkspace) executeNotebookTask(task jobs.Task, notebookParams map[s output, err := cmd.CombinedOutput() if err != nil { - return string(output), fmt.Errorf("notebook task execution failed: %s\n%s", err, output) + return string(output), newTaskFailure(string(output)) } // Normalize trailing newlines to match cloud behavior (exactly one trailing newline) @@ -780,7 +813,7 @@ func (s *FakeWorkspace) executeSparkPythonTask(task jobs.Task) (string, error) { output, err := exec.Command(venvPython(env.venvDir), runArgs...).CombinedOutput() if err != nil { - return string(output), fmt.Errorf("spark python task execution failed: %s\n%s", err, output) + return string(output), newTaskFailure(string(output)) } // Normalize trailing newlines to match cloud behavior (exactly one trailing newline) @@ -857,7 +890,9 @@ func sparkVersionToPython(task jobs.Task) string { } // terminateRun completes the run, rolling task outcomes up into the run-level -// state the way the Jobs API does: one failed task fails the whole run. +// state the way the Jobs API does: one failed task fails the whole run, and the +// run reports INTERNAL_ERROR in the deprecated life_cycle_state even though its +// tasks are TERMINATED (status.state is TERMINATED with RUN_EXECUTION_ERROR). func terminateRun(run *jobs.Run) { for i := range run.Tasks { // Tasks that were never executed (jobs/runs/submit) are still running. @@ -873,8 +908,9 @@ func terminateRun(run *jobs.Run) { } for _, task := range run.Tasks { if task.State.ResultState != jobs.RunResultStateSuccess { + run.State.LifeCycleState = jobs.RunLifeCycleStateInternalError run.State.ResultState = task.State.ResultState - run.State.StateMessage = fmt.Sprintf("task %s failed", task.TaskKey) + run.State.StateMessage = fmt.Sprintf("Task %s failed with message: %s.", task.TaskKey, taskFailureMessage) return } } diff --git a/libs/testserver/jobs_test.go b/libs/testserver/jobs_test.go index 27898c3664f..8a7886ed24b 100644 --- a/libs/testserver/jobs_test.go +++ b/libs/testserver/jobs_test.go @@ -4,6 +4,7 @@ import ( "encoding/json" "net/url" "strconv" + "strings" "testing" "github.com/databricks/databricks-sdk-go/service/jobs" @@ -122,9 +123,25 @@ func TestTerminateRun_FailedTaskFailsTheRun(t *testing.T) { terminateRun(&run) - assert.Equal(t, jobs.RunLifeCycleStateTerminated, run.State.LifeCycleState) + // A failed run reports INTERNAL_ERROR, not TERMINATED, in life_cycle_state. + assert.Equal(t, jobs.RunLifeCycleStateInternalError, run.State.LifeCycleState) assert.Equal(t, jobs.RunResultStateFailed, run.State.ResultState) - assert.Equal(t, "task second failed", run.State.StateMessage) + assert.Equal(t, "Task second failed with message: Workload failed, see run output for details.", run.State.StateMessage) +} + +func TestNewTaskFailure_ReportsTheExceptionAndTheTraceback(t *testing.T) { + output := "Traceback (most recent call last):\n File \"fail.py\", line 1\nRuntimeError: intentional failure\n" + + failure := newTaskFailure(output) + + assert.Equal(t, "RuntimeError: intentional failure", failure.Error()) + assert.Equal(t, strings.TrimRight(output, "\n"), failure.trace) +} + +func TestNewTaskFailure_SingleLineOutputIsTheException(t *testing.T) { + failure := newTaskFailure("RuntimeError: intentional failure\n") + + assert.Equal(t, "RuntimeError: intentional failure", failure.Error()) } func TestTerminateRun_CompletesTasksThatAreStillRunning(t *testing.T) { From b9d2a747c3911eea1b11cc046816f6ea9d385d3a Mon Sep 17 00:00:00 2001 From: Rada Kamysheva Date: Tue, 4 Aug 2026 13:57:56 +0000 Subject: [PATCH 18/23] acc: merge failed_cloud into failed_run, which already ran locally too failed_cloud inherited Local=true, so both tests deployed the same failing job_run against the test server and differed only in the assertions that followed. Now that the fake reports a failed task the way a workspace does, the deploy renders identically in both places and one golden covers both: the run's INTERNAL_ERROR and its generic message are fixed strings, and the message the task reports is the exception fail.py raises. That golden replaces failed_cloud's grep over a deploy.log kept out of it, so the assertions that only held by hand-waving at the output -- the framework prefix, the failed dependency, the order of the run URL and the final state -- are pinned against a real workspace as well. The redeploy moves to failed_redeploy: a second run costs another few minutes on cloud for a recreate the test server already proves. --- .../job_runs/failed_cloud/output.txt | 20 -------- .../resources/job_runs/failed_cloud/script | 19 ------- .../resources/job_runs/failed_cloud/test.toml | 20 -------- .../databricks.yml} | 11 ++-- .../{failed_cloud => failed_redeploy}/fail.py | 0 .../out.test.toml | 3 +- .../job_runs/failed_redeploy/output.txt | 51 +++++++++++++++++++ .../resources/job_runs/failed_redeploy/script | 16 ++++++ .../job_runs/failed_redeploy/test.toml | 12 +++++ .../job_runs/failed_run/databricks.yml | 33 ------------ .../job_runs/failed_run/databricks.yml.tmpl | 40 +++++++++++++++ .../resources/job_runs/failed_run/fail.py | 5 +- .../job_runs/failed_run/out.test.toml | 3 +- .../resources/job_runs/failed_run/output.txt | 47 +++-------------- .../resources/job_runs/failed_run/script | 18 ++----- .../resources/job_runs/failed_run/test.toml | 29 +++++++++-- 16 files changed, 164 insertions(+), 163 deletions(-) delete mode 100644 acceptance/bundle/resources/job_runs/failed_cloud/output.txt delete mode 100644 acceptance/bundle/resources/job_runs/failed_cloud/script delete mode 100644 acceptance/bundle/resources/job_runs/failed_cloud/test.toml rename acceptance/bundle/resources/job_runs/{failed_cloud/databricks.yml.tmpl => failed_redeploy/databricks.yml} (60%) rename acceptance/bundle/resources/job_runs/{failed_cloud => failed_redeploy}/fail.py (100%) rename acceptance/bundle/resources/job_runs/{failed_cloud => failed_redeploy}/out.test.toml (59%) create mode 100644 acceptance/bundle/resources/job_runs/failed_redeploy/output.txt create mode 100644 acceptance/bundle/resources/job_runs/failed_redeploy/script create mode 100644 acceptance/bundle/resources/job_runs/failed_redeploy/test.toml delete mode 100644 acceptance/bundle/resources/job_runs/failed_run/databricks.yml create mode 100644 acceptance/bundle/resources/job_runs/failed_run/databricks.yml.tmpl diff --git a/acceptance/bundle/resources/job_runs/failed_cloud/output.txt b/acceptance/bundle/resources/job_runs/failed_cloud/output.txt deleted file mode 100644 index bc4e275e341..00000000000 --- a/acceptance/bundle/resources/job_runs/failed_cloud/output.txt +++ /dev/null @@ -1,20 +0,0 @@ - -=== a run that fails on a real workspace fails the deploy ->>> contains.py run did not succeed: FAILED run page: http !task "main": FAILED - ->>> grep -cE task "main": .+ deploy.log -1 - -=== the failed run is recorded, so it is destroyed rather than left behind ->>> read_id.py my_run -[MY_RUN_ID] - ->>> [CLI] bundle destroy --auto-approve -The following resources will be deleted: - delete resources.job_runs.my_run - delete resources.jobs.my_job - -All files and directories at the following location will be deleted: /Workspace/Users/[USERNAME]/.bundle/[UNIQUE_NAME] - -Deleting files... -Destroy complete! diff --git a/acceptance/bundle/resources/job_runs/failed_cloud/script b/acceptance/bundle/resources/job_runs/failed_cloud/script deleted file mode 100644 index ce49c3a9aa2..00000000000 --- a/acceptance/bundle/resources/job_runs/failed_cloud/script +++ /dev/null @@ -1,19 +0,0 @@ -envsubst < databricks.yml.tmpl > databricks.yml - -cleanup() { - trace $CLI bundle destroy --auto-approve -} -trap cleanup EXIT - -# A real task's traceback is text we do not control, so keep the deploy output out -# of the golden and assert the parts the CLI produces. -title "a run that fails on a real workspace fails the deploy" -musterr $CLI bundle deploy > deploy.log 2>&1 -trace contains.py 'run did not succeed: FAILED' 'run page: http' '!task "main": FAILED' < deploy.log > /dev/null - -# A non-empty message means the task reported one: the negative assertion above -# rules out the fallback to the states the run itself reports. -trace grep -cE 'task "main": .+' deploy.log - -title "the failed run is recorded, so it is destroyed rather than left behind" -trace read_id.py my_run diff --git a/acceptance/bundle/resources/job_runs/failed_cloud/test.toml b/acceptance/bundle/resources/job_runs/failed_cloud/test.toml deleted file mode 100644 index 1a3315a6865..00000000000 --- a/acceptance/bundle/resources/job_runs/failed_cloud/test.toml +++ /dev/null @@ -1,20 +0,0 @@ -# job_runs is a direct-engine-only resource; the Terraform provider has no -# equivalent, so restrict the matrix to direct. -EnvMatrix.DATABRICKS_BUNDLE_ENGINE = ["direct"] - -# The failure counterpart of wait: only a real workspace shows whether a failed -# task reports a message the deploy can name. Serverless needs Unity Catalog. -Cloud = true -RequiresUnityCatalog = true - -# A real workspace is not proxied, so there are no recorded requests to assert on. -RecordRequests = false - -# The deploy fails mid-way, leaving local deployment state behind. -Ignore = [ - ".databricks", - "databricks.yml", - "databricks.yml.tmpl", - "fail.py", - "deploy.log", -] diff --git a/acceptance/bundle/resources/job_runs/failed_cloud/databricks.yml.tmpl b/acceptance/bundle/resources/job_runs/failed_redeploy/databricks.yml similarity index 60% rename from acceptance/bundle/resources/job_runs/failed_cloud/databricks.yml.tmpl rename to acceptance/bundle/resources/job_runs/failed_redeploy/databricks.yml index 545d42fad87..6364279bdc0 100644 --- a/acceptance/bundle/resources/job_runs/failed_cloud/databricks.yml.tmpl +++ b/acceptance/bundle/resources/job_runs/failed_redeploy/databricks.yml @@ -1,15 +1,12 @@ bundle: - name: job-runs-failed-cloud - -workspace: - root_path: ~/.bundle/$UNIQUE_NAME + name: job-runs-failed-redeploy resources: jobs: my_job: - name: test-job-$UNIQUE_NAME + name: my-job tasks: - # Serverless keeps the run to about a minute. + # The test server runs this locally; the exception fails the run. - task_key: main spark_python_task: python_file: ./fail.py @@ -18,7 +15,7 @@ resources: environments: - environment_key: default spec: - environment_version: "2" + client: "2" job_runs: my_run: diff --git a/acceptance/bundle/resources/job_runs/failed_cloud/fail.py b/acceptance/bundle/resources/job_runs/failed_redeploy/fail.py similarity index 100% rename from acceptance/bundle/resources/job_runs/failed_cloud/fail.py rename to acceptance/bundle/resources/job_runs/failed_redeploy/fail.py diff --git a/acceptance/bundle/resources/job_runs/failed_cloud/out.test.toml b/acceptance/bundle/resources/job_runs/failed_redeploy/out.test.toml similarity index 59% rename from acceptance/bundle/resources/job_runs/failed_cloud/out.test.toml rename to acceptance/bundle/resources/job_runs/failed_redeploy/out.test.toml index fe4076cdf9b..e90b6d5d1ba 100644 --- a/acceptance/bundle/resources/job_runs/failed_cloud/out.test.toml +++ b/acceptance/bundle/resources/job_runs/failed_redeploy/out.test.toml @@ -1,4 +1,3 @@ Local = true -Cloud = true -RequiresUnityCatalog = true +Cloud = false EnvMatrix.DATABRICKS_BUNDLE_ENGINE = ["direct"] diff --git a/acceptance/bundle/resources/job_runs/failed_redeploy/output.txt b/acceptance/bundle/resources/job_runs/failed_redeploy/output.txt new file mode 100644 index 00000000000..efe9f0f28cb --- /dev/null +++ b/acceptance/bundle/resources/job_runs/failed_redeploy/output.txt @@ -0,0 +1,51 @@ + +=== a run that finishes FAILED fails the deploy +>>> [CLI] bundle deploy +Uploading bundle files to /Workspace/Users/[USERNAME]/.bundle/job-runs-failed-redeploy/default/files... +Deploying resources... +job run [NUMID]: Run URL: [DATABRICKS_URL]/jobs/[NUMID]/runs/[NUMID]?o=[NUMID] +job run [NUMID]: [TIMESTAMP] "my-job" INTERNAL_ERROR FAILED Task main failed with message: Workload failed, see run output for details. +Error: cannot create resources.job_runs.my_run: waiting after creating id=[NUMID]: run did not succeed: FAILED: Task main failed with message: Workload failed, see run output for details. +task "main": RuntimeError: intentional failure +run page: [DATABRICKS_URL]/jobs/[NUMID]/runs/[NUMID]?o=[NUMID] + +Updating deployment state... + +=== so a redeploy runs the job again, and fails again +>>> [CLI] bundle deploy +Uploading bundle files to /Workspace/Users/[USERNAME]/.bundle/job-runs-failed-redeploy/default/files... +Deploying resources... +job run [NUMID]: Run URL: [DATABRICKS_URL]/jobs/[NUMID]/runs/[NUMID]?o=[NUMID] +job run [NUMID]: [TIMESTAMP] "my-job" INTERNAL_ERROR FAILED Task main failed with message: Workload failed, see run output for details. +Error: cannot recreate resources.job_runs.my_run: waiting after creating id=[NUMID]: run did not succeed: FAILED: Task main failed with message: Workload failed, see run output for details. +task "main": RuntimeError: intentional failure +run page: [DATABRICKS_URL]/jobs/[NUMID]/runs/[NUMID]?o=[NUMID] + +Updating deployment state... + +=== run-now was issued once per deploy +>>> print_requests.py //jobs/run-now +{ + "method": "POST", + "path": "/api/2.2/jobs/run-now", + "body": { + "job_id": [NUMID] + } +} +{ + "method": "POST", + "path": "/api/2.2/jobs/run-now", + "body": { + "job_id": [NUMID] + } +} + +>>> [CLI] bundle destroy --auto-approve +The following resources will be deleted: + delete resources.job_runs.my_run + delete resources.jobs.my_job + +All files and directories at the following location will be deleted: /Workspace/Users/[USERNAME]/.bundle/job-runs-failed-redeploy/default + +Deleting files... +Destroy complete! diff --git a/acceptance/bundle/resources/job_runs/failed_redeploy/script b/acceptance/bundle/resources/job_runs/failed_redeploy/script new file mode 100644 index 00000000000..7c0d528c060 --- /dev/null +++ b/acceptance/bundle/resources/job_runs/failed_redeploy/script @@ -0,0 +1,16 @@ +cleanup() { + trace $CLI bundle destroy --auto-approve + rm -f out.requests.txt +} +trap cleanup EXIT + +title "a run that finishes FAILED fails the deploy" +musterr trace $CLI bundle deploy + +# A run that never succeeded is drift, so it is recreated rather than left alone: +# a redeploy re-runs the job instead of accepting the recorded failure. +title "so a redeploy runs the job again, and fails again" +musterr trace $CLI bundle deploy + +title "run-now was issued once per deploy" +trace print_requests.py //jobs/run-now diff --git a/acceptance/bundle/resources/job_runs/failed_redeploy/test.toml b/acceptance/bundle/resources/job_runs/failed_redeploy/test.toml new file mode 100644 index 00000000000..dac4ee77a94 --- /dev/null +++ b/acceptance/bundle/resources/job_runs/failed_redeploy/test.toml @@ -0,0 +1,12 @@ +# job_runs is a direct-engine-only resource; the Terraform provider has no +# equivalent, so restrict the matrix to direct. +EnvMatrix.DATABRICKS_BUNDLE_ENGINE = ["direct"] +RecordRequests = true + +# Local only: resources/job_runs/failed_run covers the failing deploy against a +# real workspace, and a second run there costs another few minutes for a recreate +# the test server already proves. +Cloud = false + +# The deploy fails mid-way, leaving local deployment state behind. +Ignore = [".databricks"] diff --git a/acceptance/bundle/resources/job_runs/failed_run/databricks.yml b/acceptance/bundle/resources/job_runs/failed_run/databricks.yml deleted file mode 100644 index 211b7015d91..00000000000 --- a/acceptance/bundle/resources/job_runs/failed_run/databricks.yml +++ /dev/null @@ -1,33 +0,0 @@ -bundle: - name: job-runs-failed-run - -resources: - jobs: - my_job: - name: my-job - tasks: - # The test server runs this locally; it exits non-zero and fails the run. - - task_key: main - spark_python_task: - python_file: ./fail.py - environment_key: default - - environments: - - environment_key: default - spec: - client: "2" - - # Depends on my_run's result_state, so the failing run aborts the deploy - # before this job is created. - downstream_job: - name: downstream-job - tags: - run_result: ${resources.job_runs.my_run.state.result_state} - tasks: - - task_key: main - notebook_task: - notebook_path: /Workspace/test - - job_runs: - my_run: - job_id: ${resources.jobs.my_job.id} diff --git a/acceptance/bundle/resources/job_runs/failed_run/databricks.yml.tmpl b/acceptance/bundle/resources/job_runs/failed_run/databricks.yml.tmpl new file mode 100644 index 00000000000..a2b382f501a --- /dev/null +++ b/acceptance/bundle/resources/job_runs/failed_run/databricks.yml.tmpl @@ -0,0 +1,40 @@ +bundle: + name: job-runs-failed-run + +workspace: + root_path: ~/.bundle/$UNIQUE_NAME + +resources: + jobs: + my_job: + name: test-job-$UNIQUE_NAME + tasks: + # Serverless keeps the run to a few minutes: the Jobs API retries the + # failed task once before giving up on it. + - task_key: main + spark_python_task: + python_file: ./fail.py + environment_key: default + + environments: + - environment_key: default + spec: + environment_version: "2" + + # Reads the run's outcome, so the failing run aborts the deploy before this + # job is created. Separate from my_job, which my_run already depends on, to + # avoid a cycle. + downstream_job: + name: test-downstream-job-$UNIQUE_NAME + tags: + run_result: ${resources.job_runs.my_run.state.result_state} + tasks: + - task_key: main + condition_task: + op: EQUAL_TO + left: "1" + right: "1" + + job_runs: + my_run: + job_id: ${resources.jobs.my_job.id} diff --git a/acceptance/bundle/resources/job_runs/failed_run/fail.py b/acceptance/bundle/resources/job_runs/failed_run/fail.py index 3262aa05529..fa56481ece5 100644 --- a/acceptance/bundle/resources/job_runs/failed_run/fail.py +++ b/acceptance/bundle/resources/job_runs/failed_run/fail.py @@ -1,4 +1 @@ -import sys - -print("intentional failure", file=sys.stderr) -sys.exit(1) +raise RuntimeError("intentional failure") diff --git a/acceptance/bundle/resources/job_runs/failed_run/out.test.toml b/acceptance/bundle/resources/job_runs/failed_run/out.test.toml index e90b6d5d1ba..fe4076cdf9b 100644 --- a/acceptance/bundle/resources/job_runs/failed_run/out.test.toml +++ b/acceptance/bundle/resources/job_runs/failed_run/out.test.toml @@ -1,3 +1,4 @@ Local = true -Cloud = false +Cloud = true +RequiresUnityCatalog = true EnvMatrix.DATABRICKS_BUNDLE_ENGINE = ["direct"] diff --git a/acceptance/bundle/resources/job_runs/failed_run/output.txt b/acceptance/bundle/resources/job_runs/failed_run/output.txt index 720d2faa62f..bc05f225e84 100644 --- a/acceptance/bundle/resources/job_runs/failed_run/output.txt +++ b/acceptance/bundle/resources/job_runs/failed_run/output.txt @@ -1,13 +1,13 @@ === a run that finishes FAILED fails the deploy >>> [CLI] bundle deploy -Uploading bundle files to /Workspace/Users/[USERNAME]/.bundle/job-runs-failed-run/default/files... +Uploading bundle files to /Workspace/Users/[USERNAME]/.bundle/[UNIQUE_NAME]/files... Deploying resources... -job run [MY_RUN_ID]: Run URL: [DATABRICKS_URL]/jobs/[NUMID]/runs/[MY_RUN_ID]?o=[NUMID] -job run [MY_RUN_ID]: [TIMESTAMP] "my-job" INTERNAL_ERROR FAILED Task main failed with message: Workload failed, see run output for details. +job run [MY_RUN_ID]: Run URL: [RUN_URL] +job run [MY_RUN_ID]: [TIMESTAMP] "test-job-[UNIQUE_NAME]" INTERNAL_ERROR FAILED Task main failed with message: Workload failed, see run output for details. Error: cannot create resources.job_runs.my_run: waiting after creating id=[MY_RUN_ID]: run did not succeed: FAILED: Task main failed with message: Workload failed, see run output for details. -task "main": intentional failure -run page: [DATABRICKS_URL]/jobs/[NUMID]/runs/[MY_RUN_ID]?o=[NUMID] +task "main": RuntimeError: intentional failure +run page: [RUN_URL] Error: cannot create resources.jobs.downstream_job: dependency failed: resources.job_runs.my_run @@ -23,47 +23,12 @@ create jobs.downstream_job Plan: 2 to add, 0 to change, 1 to delete, 1 unchanged -=== so a redeploy runs the job again, and fails again ->>> [CLI] bundle deploy -Uploading bundle files to /Workspace/Users/[USERNAME]/.bundle/job-runs-failed-run/default/files... -Deploying resources... -job run [NUMID]: Run URL: [DATABRICKS_URL]/jobs/[NUMID]/runs/[NUMID]?o=[NUMID] -job run [NUMID]: [TIMESTAMP] "my-job" INTERNAL_ERROR FAILED Task main failed with message: Workload failed, see run output for details. -Error: cannot recreate resources.job_runs.my_run: waiting after creating id=[NUMID]: run did not succeed: FAILED: Task main failed with message: Workload failed, see run output for details. -task "main": intentional failure -run page: [DATABRICKS_URL]/jobs/[NUMID]/runs/[NUMID]?o=[NUMID] - -Error: cannot create resources.jobs.downstream_job: dependency failed: resources.job_runs.my_run - -Updating deployment state... - -=== downstream_job was never created ->>> jq -r select(.path == "/api/2.2/jobs/create") | .body.name out.requests.txt -my-job - -=== run-now was issued once per deploy ->>> print_requests.py //jobs/run-now -{ - "method": "POST", - "path": "/api/2.2/jobs/run-now", - "body": { - "job_id": [NUMID] - } -} -{ - "method": "POST", - "path": "/api/2.2/jobs/run-now", - "body": { - "job_id": [NUMID] - } -} - >>> [CLI] bundle destroy --auto-approve The following resources will be deleted: delete resources.job_runs.my_run delete resources.jobs.my_job -All files and directories at the following location will be deleted: /Workspace/Users/[USERNAME]/.bundle/job-runs-failed-run/default +All files and directories at the following location will be deleted: /Workspace/Users/[USERNAME]/.bundle/[UNIQUE_NAME] Deleting files... Destroy complete! diff --git a/acceptance/bundle/resources/job_runs/failed_run/script b/acceptance/bundle/resources/job_runs/failed_run/script index 753bdcc0fd3..27c1c53c2d9 100644 --- a/acceptance/bundle/resources/job_runs/failed_run/script +++ b/acceptance/bundle/resources/job_runs/failed_run/script @@ -1,11 +1,13 @@ +envsubst < databricks.yml.tmpl > databricks.yml + cleanup() { trace $CLI bundle destroy --auto-approve - rm -f out.requests.txt } trap cleanup EXIT -# The error names the failed task, and downstream_job is reported as a failed -# dependency because it reads the run's result_state. +# The error names the failed task and the message the workspace reported for it, +# and downstream_job is reported as a failed dependency because it reads the +# run's result_state. title "a run that finishes FAILED fails the deploy" musterr trace $CLI bundle deploy @@ -14,13 +16,3 @@ musterr trace $CLI bundle deploy title "the failed run is recorded, and not having succeeded is drift" trace read_id.py my_run trace $CLI bundle plan - -title "so a redeploy runs the job again, and fails again" -musterr trace $CLI bundle deploy - -# Only my-job: nothing downstream of a run that never succeeded is created. -title "downstream_job was never created" -trace jq -r 'select(.path == "/api/2.2/jobs/create") | .body.name' out.requests.txt - -title "run-now was issued once per deploy" -trace print_requests.py //jobs/run-now diff --git a/acceptance/bundle/resources/job_runs/failed_run/test.toml b/acceptance/bundle/resources/job_runs/failed_run/test.toml index 03530b96c6f..55abfd2fc83 100644 --- a/acceptance/bundle/resources/job_runs/failed_run/test.toml +++ b/acceptance/bundle/resources/job_runs/failed_run/test.toml @@ -1,7 +1,30 @@ # job_runs is a direct-engine-only resource; the Terraform provider has no # equivalent, so restrict the matrix to direct. EnvMatrix.DATABRICKS_BUNDLE_ENGINE = ["direct"] -RecordRequests = true -# The deploy fails mid-way, leaving local deployment state behind. -Ignore = [".databricks"] +# Runs the failing job for real, so the message the deploy names the task with is +# one a workspace reported rather than one the test server wrote. Serverless needs +# Unity Catalog. +Cloud = true +RequiresUnityCatalog = true + +# This test asserts the deploy's output, not the requests behind it, which +# resources/job_runs/failed_redeploy covers locally. +RecordRequests = false + +# databricks.yml is rendered by the script, and the deploy fails mid-way, leaving +# local deployment state behind. +Ignore = [ + ".databricks", + "databricks.yml", +] + +# The host and the workspace selector in the run URL differ per workspace; the URL +# form itself is covered by libs/workspaceurls. +[[Repls]] +Old = 'Run URL: .*' +New = 'Run URL: [RUN_URL]' + +[[Repls]] +Old = 'run page: .*' +New = 'run page: [RUN_URL]' From 464a0a7a0993a503a199730228d8e2db70453597 Mon Sep 17 00:00:00 2001 From: Rada Kamysheva Date: Tue, 4 Aug 2026 13:58:45 +0000 Subject: [PATCH 19/23] acc: stop ignoring job_run test inputs the comparison never flagged Only the rendered databricks.yml is absent from the test directory and has to be ignored. The template it is rendered from and the Python file the job runs are checked in and identical in the temporary directory, so listing them only hid the fact that they are compared. --- acceptance/bundle/resources/job_runs/wait/test.toml | 7 ++----- 1 file changed, 2 insertions(+), 5 deletions(-) diff --git a/acceptance/bundle/resources/job_runs/wait/test.toml b/acceptance/bundle/resources/job_runs/wait/test.toml index 12b0a1b8ed6..5c84ac24640 100644 --- a/acceptance/bundle/resources/job_runs/wait/test.toml +++ b/acceptance/bundle/resources/job_runs/wait/test.toml @@ -10,11 +10,8 @@ RequiresUnityCatalog = true # depend on how long the run takes. RecordRequests = false -Ignore = [ - "databricks.yml", - "databricks.yml.tmpl", - "hello.py", -] +# databricks.yml is rendered by the script. +Ignore = ["databricks.yml"] # The host and the workspace selector in the run URL differ per workspace; the URL # form itself is covered by libs/workspaceurls. From a0570f581a49ee95c2ee31106bd1a8e26fce923a Mon Sep 17 00:00:00 2001 From: Rada Kamysheva Date: Tue, 4 Aug 2026 14:28:45 +0000 Subject: [PATCH 20/23] acc: merge failed_redeploy into failed_run, so the redeploy runs on cloud failed_redeploy stayed local on the grounds that a second cloud run costs another few minutes for a recreate the test server already proves. Running it against a real workspace is worth that: the recreate deletes the run that failed and triggers a fresh one, and a real Jobs API is what decides both. Request recording goes back on so the assertions failed_redeploy carried survive the merge. The cloud leg records through the proxy, which captures the same bodies the local leg sends, and the delete now names the first run, which is what shows the failed run is discarded rather than left in the workspace. --- .../job_runs/failed_redeploy/databricks.yml | 22 -------- .../job_runs/failed_redeploy/fail.py | 1 - .../job_runs/failed_redeploy/out.test.toml | 3 -- .../job_runs/failed_redeploy/output.txt | 51 ------------------- .../resources/job_runs/failed_redeploy/script | 16 ------ .../job_runs/failed_redeploy/test.toml | 12 ----- .../resources/job_runs/failed_run/output.txt | 40 +++++++++++++++ .../resources/job_runs/failed_run/script | 11 ++++ .../resources/job_runs/failed_run/test.toml | 5 +- 9 files changed, 52 insertions(+), 109 deletions(-) delete mode 100644 acceptance/bundle/resources/job_runs/failed_redeploy/databricks.yml delete mode 100644 acceptance/bundle/resources/job_runs/failed_redeploy/fail.py delete mode 100644 acceptance/bundle/resources/job_runs/failed_redeploy/out.test.toml delete mode 100644 acceptance/bundle/resources/job_runs/failed_redeploy/output.txt delete mode 100644 acceptance/bundle/resources/job_runs/failed_redeploy/script delete mode 100644 acceptance/bundle/resources/job_runs/failed_redeploy/test.toml diff --git a/acceptance/bundle/resources/job_runs/failed_redeploy/databricks.yml b/acceptance/bundle/resources/job_runs/failed_redeploy/databricks.yml deleted file mode 100644 index 6364279bdc0..00000000000 --- a/acceptance/bundle/resources/job_runs/failed_redeploy/databricks.yml +++ /dev/null @@ -1,22 +0,0 @@ -bundle: - name: job-runs-failed-redeploy - -resources: - jobs: - my_job: - name: my-job - tasks: - # The test server runs this locally; the exception fails the run. - - task_key: main - spark_python_task: - python_file: ./fail.py - environment_key: default - - environments: - - environment_key: default - spec: - client: "2" - - job_runs: - my_run: - job_id: ${resources.jobs.my_job.id} diff --git a/acceptance/bundle/resources/job_runs/failed_redeploy/fail.py b/acceptance/bundle/resources/job_runs/failed_redeploy/fail.py deleted file mode 100644 index fa56481ece5..00000000000 --- a/acceptance/bundle/resources/job_runs/failed_redeploy/fail.py +++ /dev/null @@ -1 +0,0 @@ -raise RuntimeError("intentional failure") diff --git a/acceptance/bundle/resources/job_runs/failed_redeploy/out.test.toml b/acceptance/bundle/resources/job_runs/failed_redeploy/out.test.toml deleted file mode 100644 index e90b6d5d1ba..00000000000 --- a/acceptance/bundle/resources/job_runs/failed_redeploy/out.test.toml +++ /dev/null @@ -1,3 +0,0 @@ -Local = true -Cloud = false -EnvMatrix.DATABRICKS_BUNDLE_ENGINE = ["direct"] diff --git a/acceptance/bundle/resources/job_runs/failed_redeploy/output.txt b/acceptance/bundle/resources/job_runs/failed_redeploy/output.txt deleted file mode 100644 index efe9f0f28cb..00000000000 --- a/acceptance/bundle/resources/job_runs/failed_redeploy/output.txt +++ /dev/null @@ -1,51 +0,0 @@ - -=== a run that finishes FAILED fails the deploy ->>> [CLI] bundle deploy -Uploading bundle files to /Workspace/Users/[USERNAME]/.bundle/job-runs-failed-redeploy/default/files... -Deploying resources... -job run [NUMID]: Run URL: [DATABRICKS_URL]/jobs/[NUMID]/runs/[NUMID]?o=[NUMID] -job run [NUMID]: [TIMESTAMP] "my-job" INTERNAL_ERROR FAILED Task main failed with message: Workload failed, see run output for details. -Error: cannot create resources.job_runs.my_run: waiting after creating id=[NUMID]: run did not succeed: FAILED: Task main failed with message: Workload failed, see run output for details. -task "main": RuntimeError: intentional failure -run page: [DATABRICKS_URL]/jobs/[NUMID]/runs/[NUMID]?o=[NUMID] - -Updating deployment state... - -=== so a redeploy runs the job again, and fails again ->>> [CLI] bundle deploy -Uploading bundle files to /Workspace/Users/[USERNAME]/.bundle/job-runs-failed-redeploy/default/files... -Deploying resources... -job run [NUMID]: Run URL: [DATABRICKS_URL]/jobs/[NUMID]/runs/[NUMID]?o=[NUMID] -job run [NUMID]: [TIMESTAMP] "my-job" INTERNAL_ERROR FAILED Task main failed with message: Workload failed, see run output for details. -Error: cannot recreate resources.job_runs.my_run: waiting after creating id=[NUMID]: run did not succeed: FAILED: Task main failed with message: Workload failed, see run output for details. -task "main": RuntimeError: intentional failure -run page: [DATABRICKS_URL]/jobs/[NUMID]/runs/[NUMID]?o=[NUMID] - -Updating deployment state... - -=== run-now was issued once per deploy ->>> print_requests.py //jobs/run-now -{ - "method": "POST", - "path": "/api/2.2/jobs/run-now", - "body": { - "job_id": [NUMID] - } -} -{ - "method": "POST", - "path": "/api/2.2/jobs/run-now", - "body": { - "job_id": [NUMID] - } -} - ->>> [CLI] bundle destroy --auto-approve -The following resources will be deleted: - delete resources.job_runs.my_run - delete resources.jobs.my_job - -All files and directories at the following location will be deleted: /Workspace/Users/[USERNAME]/.bundle/job-runs-failed-redeploy/default - -Deleting files... -Destroy complete! diff --git a/acceptance/bundle/resources/job_runs/failed_redeploy/script b/acceptance/bundle/resources/job_runs/failed_redeploy/script deleted file mode 100644 index 7c0d528c060..00000000000 --- a/acceptance/bundle/resources/job_runs/failed_redeploy/script +++ /dev/null @@ -1,16 +0,0 @@ -cleanup() { - trace $CLI bundle destroy --auto-approve - rm -f out.requests.txt -} -trap cleanup EXIT - -title "a run that finishes FAILED fails the deploy" -musterr trace $CLI bundle deploy - -# A run that never succeeded is drift, so it is recreated rather than left alone: -# a redeploy re-runs the job instead of accepting the recorded failure. -title "so a redeploy runs the job again, and fails again" -musterr trace $CLI bundle deploy - -title "run-now was issued once per deploy" -trace print_requests.py //jobs/run-now diff --git a/acceptance/bundle/resources/job_runs/failed_redeploy/test.toml b/acceptance/bundle/resources/job_runs/failed_redeploy/test.toml deleted file mode 100644 index dac4ee77a94..00000000000 --- a/acceptance/bundle/resources/job_runs/failed_redeploy/test.toml +++ /dev/null @@ -1,12 +0,0 @@ -# job_runs is a direct-engine-only resource; the Terraform provider has no -# equivalent, so restrict the matrix to direct. -EnvMatrix.DATABRICKS_BUNDLE_ENGINE = ["direct"] -RecordRequests = true - -# Local only: resources/job_runs/failed_run covers the failing deploy against a -# real workspace, and a second run there costs another few minutes for a recreate -# the test server already proves. -Cloud = false - -# The deploy fails mid-way, leaving local deployment state behind. -Ignore = [".databricks"] diff --git a/acceptance/bundle/resources/job_runs/failed_run/output.txt b/acceptance/bundle/resources/job_runs/failed_run/output.txt index bc05f225e84..71811715cd5 100644 --- a/acceptance/bundle/resources/job_runs/failed_run/output.txt +++ b/acceptance/bundle/resources/job_runs/failed_run/output.txt @@ -23,6 +23,46 @@ create jobs.downstream_job Plan: 2 to add, 0 to change, 1 to delete, 1 unchanged +=== so a redeploy runs the job again, and fails again +>>> [CLI] bundle deploy +Uploading bundle files to /Workspace/Users/[USERNAME]/.bundle/[UNIQUE_NAME]/files... +Deploying resources... +job run [NUMID]: Run URL: [RUN_URL] +job run [NUMID]: [TIMESTAMP] "test-job-[UNIQUE_NAME]" INTERNAL_ERROR FAILED Task main failed with message: Workload failed, see run output for details. +Error: cannot recreate resources.job_runs.my_run: waiting after creating id=[NUMID]: run did not succeed: FAILED: Task main failed with message: Workload failed, see run output for details. +task "main": RuntimeError: intentional failure +run page: [RUN_URL] + +Error: cannot create resources.jobs.downstream_job: dependency failed: resources.job_runs.my_run + +Updating deployment state... + +=== run-now was issued once per deploy, and the recreate deleted the failed run +>>> print_requests.py --keep //jobs/run-now +{ + "method": "POST", + "path": "/api/2.2/jobs/run-now", + "body": { + "job_id": [NUMID] + } +} +{ + "method": "POST", + "path": "/api/2.2/jobs/run-now", + "body": { + "job_id": [NUMID] + } +} + +>>> print_requests.py //jobs/runs/delete +{ + "method": "POST", + "path": "/api/2.2/jobs/runs/delete", + "body": { + "run_id": [MY_RUN_ID] + } +} + >>> [CLI] bundle destroy --auto-approve The following resources will be deleted: delete resources.job_runs.my_run diff --git a/acceptance/bundle/resources/job_runs/failed_run/script b/acceptance/bundle/resources/job_runs/failed_run/script index 27c1c53c2d9..430d531414c 100644 --- a/acceptance/bundle/resources/job_runs/failed_run/script +++ b/acceptance/bundle/resources/job_runs/failed_run/script @@ -2,6 +2,7 @@ envsubst < databricks.yml.tmpl > databricks.yml cleanup() { trace $CLI bundle destroy --auto-approve + rm -f out.requests.txt } trap cleanup EXIT @@ -16,3 +17,13 @@ musterr trace $CLI bundle deploy title "the failed run is recorded, and not having succeeded is drift" trace read_id.py my_run trace $CLI bundle plan + +# The recreate re-runs the job instead of accepting the recorded failure. +title "so a redeploy runs the job again, and fails again" +musterr trace $CLI bundle deploy + +# [MY_RUN_ID] is the first run, so the delete naming it is what shows the recreate +# discarded the failed run rather than leaving it in the workspace. +title "run-now was issued once per deploy, and the recreate deleted the failed run" +trace print_requests.py --keep //jobs/run-now +trace print_requests.py //jobs/runs/delete diff --git a/acceptance/bundle/resources/job_runs/failed_run/test.toml b/acceptance/bundle/resources/job_runs/failed_run/test.toml index 55abfd2fc83..2f30fb8d8a9 100644 --- a/acceptance/bundle/resources/job_runs/failed_run/test.toml +++ b/acceptance/bundle/resources/job_runs/failed_run/test.toml @@ -1,6 +1,7 @@ # job_runs is a direct-engine-only resource; the Terraform provider has no # equivalent, so restrict the matrix to direct. EnvMatrix.DATABRICKS_BUNDLE_ENGINE = ["direct"] +RecordRequests = true # Runs the failing job for real, so the message the deploy names the task with is # one a workspace reported rather than one the test server wrote. Serverless needs @@ -8,10 +9,6 @@ EnvMatrix.DATABRICKS_BUNDLE_ENGINE = ["direct"] Cloud = true RequiresUnityCatalog = true -# This test asserts the deploy's output, not the requests behind it, which -# resources/job_runs/failed_redeploy covers locally. -RecordRequests = false - # databricks.yml is rendered by the script, and the deploy fails mid-way, leaving # local deployment state behind. Ignore = [ From 60b5e6c3443c8b500c3af523028efb2ca5c1f7c4 Mon Sep 17 00:00:00 2001 From: Rada Kamysheva Date: Tue, 4 Aug 2026 14:37:31 +0000 Subject: [PATCH 21/23] acc: name the job and run ids every job_runs test prints Whether an id rendered as [MY_JOB_ID] or as a bare [NUMID] depended on nothing but whether the script happened to call read_id.py for that resource, so basic named both ids, redeploy named only the job, and job_parameters named neither. Reading three neighbouring tests meant guessing which number was which. Register the ids instead, via the assignment form that adds the replacement without printing it, and let the auto-suffix distinguish the generations of a recreated run. That turns two assertions from unverifiable into checkable: redeploy and failed_run both claimed a redeploy replaces the run, but printed the deleted run and its replacement as the same [NUMID]. They now read [MY_RUN_ID] and [MY_RUN_ID_2]. [NUMID] stays for the ?o= / ?w= workspace selector, which is not a resource id. --- .../resources/job_runs/failed_run/output.txt | 10 +++++----- .../resources/job_runs/failed_run/script | 8 ++++++-- .../job_runs/job_parameters/output.txt | 10 +++++----- .../resources/job_runs/job_parameters/script | 6 ++++++ .../resources/job_runs/redeploy/output.txt | 18 +++++++++--------- .../bundle/resources/job_runs/redeploy/script | 6 ++++++ 6 files changed, 37 insertions(+), 21 deletions(-) diff --git a/acceptance/bundle/resources/job_runs/failed_run/output.txt b/acceptance/bundle/resources/job_runs/failed_run/output.txt index 71811715cd5..894b5c48736 100644 --- a/acceptance/bundle/resources/job_runs/failed_run/output.txt +++ b/acceptance/bundle/resources/job_runs/failed_run/output.txt @@ -27,9 +27,9 @@ Plan: 2 to add, 0 to change, 1 to delete, 1 unchanged >>> [CLI] bundle deploy Uploading bundle files to /Workspace/Users/[USERNAME]/.bundle/[UNIQUE_NAME]/files... Deploying resources... -job run [NUMID]: Run URL: [RUN_URL] -job run [NUMID]: [TIMESTAMP] "test-job-[UNIQUE_NAME]" INTERNAL_ERROR FAILED Task main failed with message: Workload failed, see run output for details. -Error: cannot recreate resources.job_runs.my_run: waiting after creating id=[NUMID]: run did not succeed: FAILED: Task main failed with message: Workload failed, see run output for details. +job run [MY_RUN_ID_2]: Run URL: [RUN_URL] +job run [MY_RUN_ID_2]: [TIMESTAMP] "test-job-[UNIQUE_NAME]" INTERNAL_ERROR FAILED Task main failed with message: Workload failed, see run output for details. +Error: cannot recreate resources.job_runs.my_run: waiting after creating id=[MY_RUN_ID_2]: run did not succeed: FAILED: Task main failed with message: Workload failed, see run output for details. task "main": RuntimeError: intentional failure run page: [RUN_URL] @@ -43,14 +43,14 @@ Updating deployment state... "method": "POST", "path": "/api/2.2/jobs/run-now", "body": { - "job_id": [NUMID] + "job_id": [MY_JOB_ID] } } { "method": "POST", "path": "/api/2.2/jobs/run-now", "body": { - "job_id": [NUMID] + "job_id": [MY_JOB_ID] } } diff --git a/acceptance/bundle/resources/job_runs/failed_run/script b/acceptance/bundle/resources/job_runs/failed_run/script index 430d531414c..7c74b20a637 100644 --- a/acceptance/bundle/resources/job_runs/failed_run/script +++ b/acceptance/bundle/resources/job_runs/failed_run/script @@ -18,12 +18,16 @@ title "the failed run is recorded, and not having succeeded is drift" trace read_id.py my_run trace $CLI bundle plan +read_id.py my_job > /dev/null + # The recreate re-runs the job instead of accepting the recorded failure. title "so a redeploy runs the job again, and fails again" musterr trace $CLI bundle deploy +read_id.py my_run > /dev/null -# [MY_RUN_ID] is the first run, so the delete naming it is what shows the recreate -# discarded the failed run rather than leaving it in the workspace. +# The delete names [MY_RUN_ID], the run that failed first, rather than the +# [MY_RUN_ID_2] that replaced it: the recreate discards the failed run instead of +# leaving it in the workspace. title "run-now was issued once per deploy, and the recreate deleted the failed run" trace print_requests.py --keep //jobs/run-now trace print_requests.py //jobs/runs/delete diff --git a/acceptance/bundle/resources/job_runs/job_parameters/output.txt b/acceptance/bundle/resources/job_runs/job_parameters/output.txt index 1f793b6f01b..7424580640b 100644 --- a/acceptance/bundle/resources/job_runs/job_parameters/output.txt +++ b/acceptance/bundle/resources/job_runs/job_parameters/output.txt @@ -3,8 +3,8 @@ >>> [CLI] bundle deploy Uploading bundle files to /Workspace/Users/[USERNAME]/.bundle/job-runs-job-parameters/default/files... Deploying resources... -job run [NUMID]: Run URL: [DATABRICKS_URL]/jobs/[NUMID]/runs/[NUMID]?o=[NUMID] -job run [NUMID]: [TIMESTAMP] "my-job" TERMINATED SUCCESS +job run [MY_RUN_ID]: Run URL: [DATABRICKS_URL]/jobs/[MY_JOB_ID]/runs/[MY_RUN_ID]?o=[NUMID] +job run [MY_RUN_ID]: [TIMESTAMP] "my-job" TERMINATED SUCCESS Updating deployment state... Deployment complete! @@ -13,7 +13,7 @@ Deployment complete! "method": "POST", "path": "/api/2.2/jobs/run-now", "body": { - "job_id": [NUMID], + "job_id": [MY_JOB_ID], "job_parameters": { "env": "prod" } @@ -34,11 +34,11 @@ Resources: Job Runs: my_run: Name: - URL: [DATABRICKS_URL]/jobs/[NUMID]/runs/[NUMID]?w=[NUMID] + URL: [DATABRICKS_URL]/jobs/[MY_JOB_ID]/runs/[MY_RUN_ID]?w=[NUMID] Jobs: my_job: Name: my-job - URL: [DATABRICKS_URL]/jobs/[NUMID]?w=[NUMID] + URL: [DATABRICKS_URL]/jobs/[MY_JOB_ID]?w=[NUMID] >>> [CLI] bundle destroy --auto-approve The following resources will be deleted: diff --git a/acceptance/bundle/resources/job_runs/job_parameters/script b/acceptance/bundle/resources/job_runs/job_parameters/script index 7128b53ef68..16eedf5d7dc 100644 --- a/acceptance/bundle/resources/job_runs/job_parameters/script +++ b/acceptance/bundle/resources/job_runs/job_parameters/script @@ -6,6 +6,12 @@ trap cleanup EXIT title "deploy triggers the run with only the overridden parameter" trace $CLI bundle deploy + +# Name the ids so the request body and the URLs below say which number is the job +# and which is the run. +read_id.py my_job > /dev/null +read_id.py my_run > /dev/null + trace print_requests.py //jobs/run-now title "plan is stable: the resolved job default does not look like drift" diff --git a/acceptance/bundle/resources/job_runs/redeploy/output.txt b/acceptance/bundle/resources/job_runs/redeploy/output.txt index 03c7dcf22ec..701104e7a36 100644 --- a/acceptance/bundle/resources/job_runs/redeploy/output.txt +++ b/acceptance/bundle/resources/job_runs/redeploy/output.txt @@ -3,8 +3,8 @@ >>> [CLI] bundle deploy Uploading bundle files to /Workspace/Users/[USERNAME]/.bundle/job-runs-redeploy/default/files... Deploying resources... -job run [NUMID]: Run URL: [DATABRICKS_URL]/jobs/[MY_JOB_ID]/runs/[NUMID]?o=[NUMID] -job run [NUMID]: [TIMESTAMP] "my-job" TERMINATED SUCCESS +job run [MY_RUN_ID]: Run URL: [DATABRICKS_URL]/jobs/[MY_JOB_ID]/runs/[MY_RUN_ID]?o=[NUMID] +job run [MY_RUN_ID]: [TIMESTAMP] "my-job" TERMINATED SUCCESS Updating deployment state... Deployment complete! @@ -18,7 +18,7 @@ Resources: Job Runs: my_run: Name: - URL: [DATABRICKS_URL]/jobs/[MY_JOB_ID]/runs/[NUMID]?w=[NUMID] + URL: [DATABRICKS_URL]/jobs/[MY_JOB_ID]/runs/[MY_RUN_ID]?w=[NUMID] Jobs: my_job: Name: my-job @@ -66,9 +66,9 @@ Resources: "env": "dev" }, "result_state": "SUCCESS", - "run_id": [NUMID], + "run_id": [MY_RUN_ID], "run_name": "my-job", - "run_page_url": "[DATABRICKS_URL]/jobs/[MY_JOB_ID]/runs/[NUMID]?o=[NUMID]", + "run_page_url": "[DATABRICKS_URL]/jobs/[MY_JOB_ID]/runs/[MY_RUN_ID]?o=[NUMID]", "run_type": "JOB_RUN", "state": { "life_cycle_state": "TERMINATED", @@ -89,8 +89,8 @@ Resources: >>> [CLI] bundle deploy Uploading bundle files to /Workspace/Users/[USERNAME]/.bundle/job-runs-redeploy/default/files... Deploying resources... -job run [NUMID]: Run URL: [DATABRICKS_URL]/jobs/[MY_JOB_ID]/runs/[NUMID]?o=[NUMID] -job run [NUMID]: [TIMESTAMP] "my-job" TERMINATED SUCCESS +job run [MY_RUN_ID_2]: Run URL: [DATABRICKS_URL]/jobs/[MY_JOB_ID]/runs/[MY_RUN_ID_2]?o=[NUMID] +job run [MY_RUN_ID_2]: [TIMESTAMP] "my-job" TERMINATED SUCCESS Updating deployment state... Deployment complete! @@ -104,7 +104,7 @@ Resources: Job Runs: my_run: Name: - URL: [DATABRICKS_URL]/jobs/[MY_JOB_ID]/runs/[NUMID]?w=[NUMID] + URL: [DATABRICKS_URL]/jobs/[MY_JOB_ID]/runs/[MY_RUN_ID_2]?w=[NUMID] Jobs: my_job: Name: my-job @@ -116,7 +116,7 @@ Resources: "method": "POST", "path": "/api/2.2/jobs/runs/delete", "body": { - "run_id": [NUMID] + "run_id": [MY_RUN_ID] } } diff --git a/acceptance/bundle/resources/job_runs/redeploy/script b/acceptance/bundle/resources/job_runs/redeploy/script index 87d035b9eee..8ceb3a88052 100644 --- a/acceptance/bundle/resources/job_runs/redeploy/script +++ b/acceptance/bundle/resources/job_runs/redeploy/script @@ -8,12 +8,18 @@ title "initial deploy triggers the first run" trace $CLI bundle deploy trace $CLI bundle summary trace read_id.py my_job + +# Name the first run, so the second one below reads as [MY_RUN_ID_2] and the two +# are told apart in the URLs and in the delete request. +read_id.py my_run > /dev/null + trace print_requests.py //jobs/run-now title "change the run configuration and redeploy" trace update_file.py databricks.yml "env: dev" "env: prod" trace $CLI bundle plan -o json | jq '.plan["resources.job_runs.my_run"]' trace $CLI bundle deploy +read_id.py my_run > /dev/null trace $CLI bundle summary title "the config change deleted the previous run and triggered a second, different run" From 7314ae2d86fb00d5ec9b6f3aa4b4f565e8557b11 Mon Sep 17 00:00:00 2001 From: Rada Kamysheva Date: Tue, 4 Aug 2026 15:02:11 +0000 Subject: [PATCH 22/23] dresources: scope the settled-state read in testCRUD to job_runs Comparing WaitAfterCreate's return against a fresh read, and rebasing the field checks onto it, applied to every resource, but job_runs is the only one whose remote state moves while the wait blocks: the testserver settles the others on create, so both forms assert the same thing for them. Keep the original comparison for those and take the new path only for job_runs, so the change stays where it is needed. A resource whose handler later models a create-then-settle transition will fail the plain comparison and needs adding to the exception. --- bundle/direct/dresources/all_test.go | 23 ++++++++++++++--------- 1 file changed, 14 insertions(+), 9 deletions(-) diff --git a/bundle/direct/dresources/all_test.go b/bundle/direct/dresources/all_test.go index 6d8ad997c06..4569c5ea51c 100644 --- a/bundle/direct/dresources/all_test.go +++ b/bundle/direct/dresources/all_test.go @@ -1000,19 +1000,24 @@ func testCRUD(t *testing.T, group string, adapter *Adapter, client *databricks.W "unexpected differences between remappedState and remappedRemoteStateFromCreate") } + // Only a job run's state moves while WaitAfterCreate blocks: it polls a RUNNING + // run until terminal. Everything else settles on create here. + waitSettlesState := group == "job_runs" + remoteStateFromWaitCreate, err := adapter.WaitAfterCreate(ctx, createdID, newState) require.NoError(t, err) if remoteStateFromWaitCreate != nil { - // WaitAfterCreate returns the settled state; the read right after DoCreate - // may still be non-terminal, so compare against a fresh read. - remotePostWaitCreate, err := adapter.DoRead(ctx, createdID) - require.NoError(t, err) - require.Equal(t, remotePostWaitCreate, remoteStateFromWaitCreate) + if waitSettlesState { + // The pre-wait read is stale, and result_state fills in only once settled. + remotePostWaitCreate, err := adapter.DoRead(ctx, createdID) + require.NoError(t, err) + require.Equal(t, remotePostWaitCreate, remoteStateFromWaitCreate) - // The settled state is the one a deploy records, so the field checks below - // run against it: a job run's result_state fills in only once it settles. - remappedState, err = adapter.RemapState(remoteStateFromWaitCreate) - require.NoError(t, err) + remappedState, err = adapter.RemapState(remoteStateFromWaitCreate) + require.NoError(t, err) + } else { + require.Equal(t, remote, remoteStateFromWaitCreate) + } } if adapter.HasDoUpdate() { From 2d11b0ba6c9f2695097f5c271dc18149193803ea Mon Sep 17 00:00:00 2001 From: Rada Kamysheva Date: Tue, 4 Aug 2026 15:09:08 +0000 Subject: [PATCH 23/23] dresources: drop the redundant settled-state assertion for job_runs Comparing WaitAfterCreate's remote against a fresh DoRead held by construction: both build the value with makeJobRunRemote from a GetRun response, and the run is terminal by then, so the two cannot differ. It also did not check the thing worth checking -- an early return from the wait would leave both sides equally non-terminal -- and the drift a lossy recorded state would cause is already covered by resources/job_runs/{basic,wait}. What remains is not an assertion: result_state fills in only once the run settles, so the field checks at the end of testCRUD have to read the settled state rather than the one from just after create. --- bundle/direct/dresources/all_test.go | 9 ++------- 1 file changed, 2 insertions(+), 7 deletions(-) diff --git a/bundle/direct/dresources/all_test.go b/bundle/direct/dresources/all_test.go index 4569c5ea51c..d0e8b02de6d 100644 --- a/bundle/direct/dresources/all_test.go +++ b/bundle/direct/dresources/all_test.go @@ -1000,19 +1000,14 @@ func testCRUD(t *testing.T, group string, adapter *Adapter, client *databricks.W "unexpected differences between remappedState and remappedRemoteStateFromCreate") } - // Only a job run's state moves while WaitAfterCreate blocks: it polls a RUNNING - // run until terminal. Everything else settles on create here. + // Only a job run's state moves while WaitAfterCreate blocks, and its result_state + // fills in only then, so its field checks below need the settled state. waitSettlesState := group == "job_runs" remoteStateFromWaitCreate, err := adapter.WaitAfterCreate(ctx, createdID, newState) require.NoError(t, err) if remoteStateFromWaitCreate != nil { if waitSettlesState { - // The pre-wait read is stale, and result_state fills in only once settled. - remotePostWaitCreate, err := adapter.DoRead(ctx, createdID) - require.NoError(t, err) - require.Equal(t, remotePostWaitCreate, remoteStateFromWaitCreate) - remappedState, err = adapter.RemapState(remoteStateFromWaitCreate) require.NoError(t, err) } else {