Job runs wait for completion

.nextchanges/bundles/job-runs-wait-for-completion.md

@@ -0,0 +1 @@
+direct: the experimental `job_runs` resource now waits for the triggered run to finish, so other resources can reference its outcome (e.g. `${resources.job_runs.nightly.state.result_state}`). A run that does not succeed fails the deploy, naming the failed task, and is run again on the next deploy.

acceptance/bundle/invariant/configs/job_run.yml.tmpl

@@ -6,14 +6,13 @@ resources:
     foo:
       name: test-job-$UNIQUE_NAME
       tasks:
+        # Deploying a job_run actually runs the job, so use a condition task,
+        # which needs no workspace files or compute.
         - task_key: only_task
-          notebook_task:
-            notebook_path: /Shared/notebook
-          new_cluster:
-            spark_version: $DEFAULT_SPARK_VERSION
-            node_type_id: $NODE_TYPE_ID
-            instance_pool_id: $TEST_INSTANCE_POOL_ID
-            num_workers: 1
+          condition_task:
+            op: EQUAL_TO
+            left: "1"
+            right: "1"
 
   job_runs:
     foo_run:

acceptance/bundle/refschema/out.fields.txt

@@ -870,6 +870,7 @@ resources.job_runs.*.python_params[*]	string	ALL
 resources.job_runs.*.queue	*jobs.QueueSettings	ALL
 resources.job_runs.*.queue.enabled	bool	ALL
 resources.job_runs.*.resolved_job_id	int64	INPUT
+resources.job_runs.*.result_state	jobs.RunResultState	REMOTE	STATE
 resources.job_runs.*.run_id	int64	REMOTE
 resources.job_runs.*.run_name	string	REMOTE
 resources.job_runs.*.run_page_url	string	REMOTE

acceptance/bundle/resources/job_runs/basic/output.txt

@@ -34,6 +34,8 @@ Resources:
 >>> [CLI] bundle deploy
 Uploading bundle files to /Workspace/Users/[USERNAME]/.bundle/job-runs-basic/default/files...
 Deploying resources...
+job run [MY_RUN_ID]: Run URL: [DATABRICKS_URL]/jobs/[MY_JOB_ID]/runs/[MY_RUN_ID]?o=[NUMID]
+job run [MY_RUN_ID]: [TIMESTAMP] "my-job" TERMINATED SUCCESS
 Updating deployment state...
 Deployment complete!
 

acceptance/bundle/resources/job_runs/failed_run/databricks.yml.tmpl

@@ -0,0 +1,40 @@
+bundle:
+  name: job-runs-failed-run
+
+workspace:
+  root_path: ~/.bundle/$UNIQUE_NAME
+
+resources:
+  jobs:
+    my_job:
+      name: test-job-$UNIQUE_NAME
+      tasks:
+        # Serverless keeps the run to a few minutes: the Jobs API retries the
+        # failed task once before giving up on it.
+        - task_key: main
+          spark_python_task:
+            python_file: ./fail.py
+          environment_key: default
+
+      environments:
+        - environment_key: default
+          spec:
+            environment_version: "2"
+
+    # Reads the run's outcome, so the failing run aborts the deploy before this
+    # job is created. Separate from my_job, which my_run already depends on, to
+    # avoid a cycle.
+    downstream_job:
+      name: test-downstream-job-$UNIQUE_NAME
+      tags:
+        run_result: ${resources.job_runs.my_run.state.result_state}
+      tasks:
+        - task_key: main
+          condition_task:
+            op: EQUAL_TO
+            left: "1"
+            right: "1"
+
+  job_runs:
+    my_run:
+      job_id: ${resources.jobs.my_job.id}

acceptance/bundle/resources/job_runs/failed_run/fail.py

@@ -0,0 +1 @@
+raise RuntimeError("intentional failure")

acceptance/bundle/resources/job_runs/failed_run/output.txt

@@ -0,0 +1,74 @@
+
+=== a run that finishes FAILED fails the deploy
+>>> [CLI] bundle deploy
+Uploading bundle files to /Workspace/Users/[USERNAME]/.bundle/[UNIQUE_NAME]/files...
+Deploying resources...
+job run [MY_RUN_ID]: Run URL: [RUN_URL]
+job run [MY_RUN_ID]: [TIMESTAMP] "test-job-[UNIQUE_NAME]" INTERNAL_ERROR FAILED Task main failed with message: Workload failed, see run output for details.
+Error: cannot create resources.job_runs.my_run: waiting after creating id=[MY_RUN_ID]: run did not succeed: FAILED: Task main failed with message: Workload failed, see run output for details.
+task "main": RuntimeError: intentional failure
+run page: [RUN_URL]
+
+Error: cannot create resources.jobs.downstream_job: dependency failed: resources.job_runs.my_run
+
+Updating deployment state...
+
+=== the failed run is recorded, and not having succeeded is drift
+>>> read_id.py my_run
+[MY_RUN_ID]
+
+>>> [CLI] bundle plan
+recreate job_runs.my_run
+create jobs.downstream_job
+
+Plan: 2 to add, 0 to change, 1 to delete, 1 unchanged
+
+=== so a redeploy runs the job again, and fails again
+>>> [CLI] bundle deploy
+Uploading bundle files to /Workspace/Users/[USERNAME]/.bundle/[UNIQUE_NAME]/files...
+Deploying resources...
+job run [MY_RUN_ID_2]: Run URL: [RUN_URL]
+job run [MY_RUN_ID_2]: [TIMESTAMP] "test-job-[UNIQUE_NAME]" INTERNAL_ERROR FAILED Task main failed with message: Workload failed, see run output for details.
+Error: cannot recreate resources.job_runs.my_run: waiting after creating id=[MY_RUN_ID_2]: run did not succeed: FAILED: Task main failed with message: Workload failed, see run output for details.
+task "main": RuntimeError: intentional failure
+run page: [RUN_URL]
+
+Error: cannot create resources.jobs.downstream_job: dependency failed: resources.job_runs.my_run
+
+Updating deployment state...
+
+=== run-now was issued once per deploy, and the recreate deleted the failed run
+>>> print_requests.py --keep //jobs/run-now
+{
+  "method": "POST",
+  "path": "/api/2.2/jobs/run-now",
+  "body": {
+    "job_id": [MY_JOB_ID]
+  }
+}
+{
+  "method": "POST",
+  "path": "/api/2.2/jobs/run-now",
+  "body": {
+    "job_id": [MY_JOB_ID]
+  }
+}
+
+>>> print_requests.py //jobs/runs/delete
+{
+  "method": "POST",
+  "path": "/api/2.2/jobs/runs/delete",
+  "body": {
+    "run_id": [MY_RUN_ID]
+  }
+}
+
+>>> [CLI] bundle destroy --auto-approve
+The following resources will be deleted:
+  delete resources.job_runs.my_run
+  delete resources.jobs.my_job
+
+All files and directories at the following location will be deleted: /Workspace/Users/[USERNAME]/.bundle/[UNIQUE_NAME]
+
+Deleting files...
+Destroy complete!

acceptance/bundle/resources/job_runs/failed_run/script

@@ -0,0 +1,33 @@
+envsubst < databricks.yml.tmpl > databricks.yml
+
+cleanup() {
+    trace $CLI bundle destroy --auto-approve
+    rm -f out.requests.txt
+}
+trap cleanup EXIT
+
+# The error names the failed task and the message the workspace reported for it,
+# and downstream_job is reported as a failed dependency because it reads the
+# run's result_state.
+title "a run that finishes FAILED fails the deploy"
+musterr trace $CLI bundle deploy
+
+# The framework saves the run id before calling WaitAfterCreate, so the failed run
+# stays recorded, and FAILED against the required SUCCESS is drift.
+title "the failed run is recorded, and not having succeeded is drift"
+trace read_id.py my_run
+trace $CLI bundle plan
+
+read_id.py my_job > /dev/null
+
+# The recreate re-runs the job instead of accepting the recorded failure.
+title "so a redeploy runs the job again, and fails again"
+musterr trace $CLI bundle deploy
+read_id.py my_run > /dev/null
+
+# The delete names [MY_RUN_ID], the run that failed first, rather than the
+# [MY_RUN_ID_2] that replaced it: the recreate discards the failed run instead of
+# leaving it in the workspace.
+title "run-now was issued once per deploy, and the recreate deleted the failed run"
+trace print_requests.py --keep //jobs/run-now
+trace print_requests.py //jobs/runs/delete

acceptance/bundle/resources/job_runs/failed_run/test.toml

@@ -0,0 +1,27 @@
+# job_runs is a direct-engine-only resource; the Terraform provider has no
+# equivalent, so restrict the matrix to direct.
+EnvMatrix.DATABRICKS_BUNDLE_ENGINE = ["direct"]
+RecordRequests = true
+
+# Runs the failing job for real, so the message the deploy names the task with is
+# one a workspace reported rather than one the test server wrote. Serverless needs
+# Unity Catalog.
+Cloud = true
+RequiresUnityCatalog = true
+
+# databricks.yml is rendered by the script, and the deploy fails mid-way, leaving
+# local deployment state behind.
+Ignore = [
+    ".databricks",
+    "databricks.yml",
+]
+
+# The host and the workspace selector in the run URL differ per workspace; the URL
+# form itself is covered by libs/workspaceurls.
+[[Repls]]
+Old = 'Run URL: .*'
+New = 'Run URL: [RUN_URL]'
+
+[[Repls]]
+Old = 'run page: .*'
+New = 'run page: [RUN_URL]'

acceptance/bundle/resources/job_runs/job_parameters/output.txt

@@ -3,6 +3,8 @@
 >>> [CLI] bundle deploy
 Uploading bundle files to /Workspace/Users/[USERNAME]/.bundle/job-runs-job-parameters/default/files...
 Deploying resources...
+job run [MY_RUN_ID]: Run URL: [DATABRICKS_URL]/jobs/[MY_JOB_ID]/runs/[MY_RUN_ID]?o=[NUMID]
+job run [MY_RUN_ID]: [TIMESTAMP] "my-job" TERMINATED SUCCESS
 Updating deployment state...
 Deployment complete!
 
@@ -11,7 +13,7 @@ Deployment complete!
   "method": "POST",
   "path": "/api/2.2/jobs/run-now",
   "body": {
-    "job_id": [NUMID],
+    "job_id": [MY_JOB_ID],
     "job_parameters": {
       "env": "prod"
     }
@@ -32,11 +34,11 @@ Resources:
   Job Runs:
     my_run:
       Name: 
-      URL:  [DATABRICKS_URL]/jobs/[NUMID]/runs/[NUMID]?w=[NUMID]
+      URL:  [DATABRICKS_URL]/jobs/[MY_JOB_ID]/runs/[MY_RUN_ID]?w=[NUMID]
   Jobs:
     my_job:
       Name: my-job
-      URL:  [DATABRICKS_URL]/jobs/[NUMID]?w=[NUMID]
+      URL:  [DATABRICKS_URL]/jobs/[MY_JOB_ID]?w=[NUMID]
 
 >>> [CLI] bundle destroy --auto-approve
 The following resources will be deleted:

acceptance/bundle/resources/job_runs/job_parameters/script

@@ -6,6 +6,12 @@ trap cleanup EXIT
 
 title "deploy triggers the run with only the overridden parameter"
 trace $CLI bundle deploy
+
+# Name the ids so the request body and the URLs below say which number is the job
+# and which is the run.
+read_id.py my_job > /dev/null
+read_id.py my_run > /dev/null
+
 trace print_requests.py //jobs/run-now
 
 title "plan is stable: the resolved job default does not look like drift"

acceptance/bundle/resources/job_runs/redeploy/output.txt

@@ -3,6 +3,8 @@
 >>> [CLI] bundle deploy
 Uploading bundle files to /Workspace/Users/[USERNAME]/.bundle/job-runs-redeploy/default/files...
 Deploying resources...
+job run [MY_RUN_ID]: Run URL: [DATABRICKS_URL]/jobs/[MY_JOB_ID]/runs/[MY_RUN_ID]?o=[NUMID]
+job run [MY_RUN_ID]: [TIMESTAMP] "my-job" TERMINATED SUCCESS
 Updating deployment state...
 Deployment complete!
 
@@ -16,7 +18,7 @@ Resources:
   Job Runs:
     my_run:
       Name: 
-      URL:  [DATABRICKS_URL]/jobs/[MY_JOB_ID]/runs/[NUMID]?w=[NUMID]
+      URL:  [DATABRICKS_URL]/jobs/[MY_JOB_ID]/runs/[MY_RUN_ID]?w=[NUMID]
   Jobs:
     my_job:
       Name: my-job
@@ -54,20 +56,23 @@ Resources:
       "job_id": [MY_JOB_ID],
       "job_parameters": {
         "env": "prod"
-      }
+      },
+      "result_state": "SUCCESS"
     }
   },
   "remote_state": {
     "job_id": [MY_JOB_ID],
     "job_parameters": {
       "env": "dev"
     },
-    "run_id": [NUMID],
+    "result_state": "SUCCESS",
+    "run_id": [MY_RUN_ID],
     "run_name": "my-job",
-    "run_page_url": "[DATABRICKS_URL]/?o=[NUMID]#job/[MY_JOB_ID]/run/[NUMID]",
+    "run_page_url": "[DATABRICKS_URL]/jobs/[MY_JOB_ID]/runs/[MY_RUN_ID]?o=[NUMID]",
     "run_type": "JOB_RUN",
     "state": {
-      "life_cycle_state": "RUNNING"
+      "life_cycle_state": "TERMINATED",
+      "result_state": "SUCCESS"
     }
   },
   "changes": {
@@ -84,6 +89,8 @@ Resources:
 >>> [CLI] bundle deploy
 Uploading bundle files to /Workspace/Users/[USERNAME]/.bundle/job-runs-redeploy/default/files...
 Deploying resources...
+job run [MY_RUN_ID_2]: Run URL: [DATABRICKS_URL]/jobs/[MY_JOB_ID]/runs/[MY_RUN_ID_2]?o=[NUMID]
+job run [MY_RUN_ID_2]: [TIMESTAMP] "my-job" TERMINATED SUCCESS
 Updating deployment state...
 Deployment complete!
 
@@ -97,7 +104,7 @@ Resources:
   Job Runs:
     my_run:
       Name: 
-      URL:  [DATABRICKS_URL]/jobs/[MY_JOB_ID]/runs/[NUMID]?w=[NUMID]
+      URL:  [DATABRICKS_URL]/jobs/[MY_JOB_ID]/runs/[MY_RUN_ID_2]?w=[NUMID]
   Jobs:
     my_job:
       Name: my-job
@@ -109,7 +116,7 @@ Resources:
   "method": "POST",
   "path": "/api/2.2/jobs/runs/delete",
   "body": {
-    "run_id": [NUMID]
+    "run_id": [MY_RUN_ID]
   }
 }
 

acceptance/bundle/resources/job_runs/redeploy/script

@@ -8,12 +8,18 @@ title "initial deploy triggers the first run"
 trace $CLI bundle deploy
 trace $CLI bundle summary
 trace read_id.py my_job
+
+# Name the first run, so the second one below reads as [MY_RUN_ID_2] and the two
+# are told apart in the URLs and in the delete request.
+read_id.py my_run > /dev/null
+
 trace print_requests.py //jobs/run-now
 
 title "change the run configuration and redeploy"
 trace update_file.py databricks.yml "env: dev" "env: prod"
 trace $CLI bundle plan -o json | jq '.plan["resources.job_runs.my_run"]'
 trace $CLI bundle deploy
+read_id.py my_run > /dev/null
 trace $CLI bundle summary
 
 title "the config change deleted the previous run and triggered a second, different run"