AIR CLI Migration Pt. 2

acceptance/experimental/air/cancel/test.toml

@@ -1,6 +1,3 @@
-# This command does not deploy a bundle, so no engine matrix is needed.
-EnvMatrix.DATABRICKS_BUNDLE_ENGINE = ["direct"]
-
 # The SDK occasionally probes host reachability with a HEAD request; stub it so
 # the test is deterministic.
 [[Server]]

acceptance/experimental/air/get-ai-runtime/test.toml

@@ -1,6 +1,3 @@
-# This command does not deploy a bundle, so no engine matrix is needed.
-EnvMatrix.DATABRICKS_BUNDLE_ENGINE = ["direct"]
-
 # On Windows, Git Bash rewrites the leading-/ workspace paths passed to
 # `workspace mkdirs`/`import` into C:/... paths; disable that conversion.
 [Env]

acceptance/experimental/air/get/test.toml

@@ -1,6 +1,3 @@
-# This command does not deploy a bundle, so no engine matrix is needed.
-EnvMatrix.DATABRICKS_BUNDLE_ENGINE = ["direct"]
-
 # The SDK occasionally probes host reachability with a HEAD request; stub it so
 # the test is deterministic.
 [[Server]]

acceptance/experimental/air/help/output.txt

@@ -47,3 +47,24 @@ Global Flags:
   -o, --output type      output type: text or json (default text)
   -p, --profile string   ~/.databrickscfg profile
   -t, --target string    bundle target to use (if applicable)
+
+=== logs help
+>>> [CLI] experimental air logs --help
+Stream logs from an active run, or fetch logs from a completed run.
+
+Usage:
+  databricks experimental air logs JOB_RUN_ID [flags]
+
+Flags:
+      --download-to string   Download all logs to this directory instead of printing
+  -h, --help                 help for logs
+      --lines int            For completed runs, print the last N lines (default 10000)
+      --minutes int          Fetch only logs from the last N minutes
+      --node int             Fetch logs from this node
+      --retry int            View logs from a specific retry attempt; -1 means latest (default -1)
+
+Global Flags:
+      --debug            enable debug logging
+  -o, --output type      output type: text or json (default text)
+  -p, --profile string   ~/.databrickscfg profile
+  -t, --target string    bundle target to use (if applicable)

acceptance/experimental/air/help/script

@@ -6,3 +6,6 @@ trace $CLI experimental air --help
 
 title "list help"
 trace $CLI experimental air list --help
+
+title "logs help"
+trace $CLI experimental air logs --help

acceptance/experimental/air/list/test.toml

@@ -1,6 +1,3 @@
-# This command does not deploy a bundle, so no engine matrix is needed.
-EnvMatrix.DATABRICKS_BUNDLE_ENGINE = ["direct"]
-
 # Disable the on-disk run cache so --all-status output is deterministic across runs.
 [Env]
 DATABRICKS_CACHE_ENABLED = "false"

acceptance/experimental/air/logs-mlflow-fallback/output.txt

@@ -0,0 +1,8 @@
+
+=== logs falls back to mlflow (no logs)
+>>> [CLI] experimental air logs 123
+No logs available for run 123. Run terminated in state SUCCESS
+
+=== logs falls back to mlflow (json)
+>>> [CLI] experimental air logs 123 -o json
+{"type":"ERROR","ts":"[TIMESTAMP]","node":0,"line":"No logs available for run 123. Run terminated in state SUCCESS"}

acceptance/experimental/air/logs-mlflow-fallback/script

@@ -0,0 +1,9 @@
+# Bricklens is gated off (FEATURE_DISABLED), so the command falls back to the
+# MLflow log path. With no MLflow run id resolvable, the fallback reports no
+# logs and exits non-zero — proving the try/catch routes to MLflow.
+
+title "logs falls back to mlflow (no logs)"
+errcode trace $CLI experimental air logs 123
+
+title "logs falls back to mlflow (json)"
+errcode trace $CLI experimental air logs 123 -o json

acceptance/experimental/air/logs-mlflow-fallback/test.toml

@@ -0,0 +1,32 @@
+# The SDK occasionally probes host reachability with a HEAD request; stub it so
+# the test is deterministic.
+[[Server]]
+Pattern = "HEAD /"
+Response.Body = ''
+
+# A completed run.
+[[Server]]
+Pattern = "GET /api/2.2/jobs/runs/get"
+Response.Body = '''
+{
+  "run_id": 123,
+  "start_time": 1700000000000,
+  "end_time": 1700000012000,
+  "state": {"life_cycle_state": "TERMINATED", "result_state": "SUCCESS"},
+  "tasks": [{"task_key": "train", "run_id": 456, "attempt_number": 0}]
+}
+'''
+
+# Bricklens is gated off by the backend SAFE flag, forcing the MLflow fallback.
+[[Server]]
+Pattern = "GET /api/2.0/ai-training/workflows/by-run-id/123/logs"
+Response.StatusCode = 403
+Response.Body = '''
+{"error_code": "FEATURE_DISABLED", "message": "training log streaming is not enabled"}
+'''
+
+# The MLflow fallback has no run output to resolve an MLflow run id from, so it
+# reports no logs rather than failing — exercising the fallback wiring end to end.
+[[Server]]
+Pattern = "GET /api/2.2/jobs/runs/get-output"
+Response.Body = '{}'

acceptance/experimental/air/logs/output.txt

@@ -0,0 +1,72 @@
+
+=== logs (text, completed run)
+>>> [CLI] experimental air logs 123
+step 1
+step 2
+CUDA out of memory
+
+=== logs (json)
+>>> [CLI] experimental air logs 123 -o json
+{"type":"LOG","ts":"[TIMESTAMP]","node":0,"line":"step 1"}
+{"type":"LOG","ts":"[TIMESTAMP]","node":0,"line":"step 2"}
+{"type":"ALERT","ts":"[TIMESTAMP]","node":0,"line":"CUDA out of memory"}
+{"type":"LOG","ts":"[TIMESTAMP]","node":0,"line":"CUDA out of memory"}
+
+=== logs with --minutes
+>>> [CLI] experimental air logs 123 --minutes 30
+step 1
+step 2
+CUDA out of memory
+
+=== logs with --lines
+>>> [CLI] experimental air logs 123 --lines 1
+CUDA out of memory
+
+=== logs with --lines 0 prints nothing
+>>> [CLI] experimental air logs 123 --lines 0
+No logs available for run 123. Run terminated in state SUCCESS
+
+=== logs from a specific retry
+>>> [CLI] experimental air logs 123 --retry 0
+step 1
+step 2
+CUDA out of memory
+
+=== logs --lines and --minutes are mutually exclusive
+>>> [CLI] experimental air logs 123 --lines 100 --minutes 30
+Error: cannot combine --lines with --minutes: --lines tails by line count, --minutes by time window
+
+Exit code: 1
+
+=== logs --lines and --minutes are mutually exclusive (json)
+>>> [CLI] experimental air logs 123 --lines 100 --minutes 30 -o json
+{
+  "v": 1,
+  "ts": "[TIMESTAMP]",
+  "error": {
+    "code": "INVALID_ARGS",
+    "kind": "PERMANENT",
+    "message": "cannot combine --lines with --minutes: --lines tails by line count, --minutes by time window",
+    "retryable": false
+  }
+}
+
+Exit code: 1
+
+=== invalid run id
+>>> [CLI] experimental air logs notanumber
+Error: invalid JOB_RUN_ID "notanumber": must be a positive integer
+
+Exit code: 1
+
+=== negative node
+>>> [CLI] experimental air logs 123 --node -1
+Error: invalid --node -1: must not be negative
+
+Exit code: 1
+
+=== --download-to not implemented
+>>> [CLI] experimental air logs 123 --download-to /tmp/out
+Error: --download-to is not implemented yet
+
+Exit code: 1

acceptance/experimental/air/logs/script

@@ -0,0 +1,32 @@
+title "logs (text, completed run)"
+trace $CLI experimental air logs 123
+
+title "logs (json)"
+trace $CLI experimental air logs 123 -o json
+
+title "logs with --minutes"
+trace $CLI experimental air logs 123 --minutes 30
+
+title "logs with --lines"
+trace $CLI experimental air logs 123 --lines 1
+
+title "logs with --lines 0 prints nothing"
+trace $CLI experimental air logs 123 --lines 0
+
+title "logs from a specific retry"
+trace $CLI experimental air logs 123 --retry 0
+
+title "logs --lines and --minutes are mutually exclusive"
+errcode trace $CLI experimental air logs 123 --lines 100 --minutes 30
+
+title "logs --lines and --minutes are mutually exclusive (json)"
+errcode trace $CLI experimental air logs 123 --lines 100 --minutes 30 -o json
+
+title "invalid run id"
+errcode trace $CLI experimental air logs notanumber
+
+title "negative node"
+errcode trace $CLI experimental air logs 123 --node -1
+
+title "--download-to not implemented"
+errcode trace $CLI experimental air logs 123 --download-to /tmp/out

acceptance/experimental/air/logs/test.toml

@@ -0,0 +1,30 @@
+# The SDK occasionally probes host reachability with a HEAD request; stub it so
+# the test is deterministic.
+[[Server]]
+Pattern = "HEAD /"
+Response.Body = ''
+
+# A completed run: Bricklens serves its logs via the tail drain.
+[[Server]]
+Pattern = "GET /api/2.2/jobs/runs/get"
+Response.Body = '''
+{
+  "run_id": 123,
+  "start_time": 1700000000000,
+  "end_time": 1700000012000,
+  "state": {"life_cycle_state": "TERMINATED", "result_state": "SUCCESS"},
+  "tasks": [{"task_key": "train", "run_id": 456, "attempt_number": 0}]
+}
+'''
+
+# Bricklens log records, returned newest-first (as the tail fetch requests);
+# printed oldest-first.
+[[Server]]
+Pattern = "GET /api/2.0/ai-training/workflows/by-run-id/123/logs"
+Response.Body = '''
+{"log_records": [
+  {"time_unix_nano": 1700000003000000000, "body": "CUDA out of memory", "node_index": 0},
+  {"time_unix_nano": 1700000002000000000, "body": "step 2", "node_index": 0},
+  {"time_unix_nano": 1700000001000000000, "body": "step 1", "node_index": 0}
+]}
+'''

acceptance/experimental/air/register-image-no-secret-permission/dockercfg/config.json

@@ -0,0 +1,7 @@
+{
+  "auths": {
+    "nvcr.io": {
+      "auth": "Ym9iOnNlY3JldHBhdA=="
+    }
+  }
+}

acceptance/experimental/air/register-image-no-secret-permission/output.txt

@@ -0,0 +1,29 @@
+
+=== private image, but the user cannot create a secret scope to store credentials
+>>> [CLI] experimental air register-image nvcr.io/org/img:1.0
+Error: image "nvcr.io/org/img:1.0" requires credentials, and the credentials found in your local Docker config could not be stored: creating secret scope "docker-credentials-[USERNAME]" was denied (user does not have permission to create secret scopes). Ask a workspace admin for permission to create secret scopes
+
+Host:      [DATABRICKS_URL]
+Auth type: Personal Access Token (pat)
+
+Next steps:
+  - Verify you have the required permissions for this operation
+  - Check your identity: databricks auth describe
+  - Consider setting up a profile: databricks auth login --profile <name>
+
+Exit code: 1
+
+=== same failure as a JSON envelope
+>>> [CLI] experimental air register-image nvcr.io/org/img:1.0 -o json
+{
+  "v": 1,
+  "ts": "[TIMESTAMP]",
+  "error": {
+    "code": "REGISTRATION_FAILED",
+    "kind": "PERMANENT",
+    "message": "image \"nvcr.io/org/img:1.0\" requires credentials, and the credentials found in your local Docker config could not be stored: creating secret scope \"docker-credentials-[USERNAME]\" was denied (user does not have permission to create secret scopes). Ask a workspace admin for permission to create secret scopes",
+    "retryable": false
+  }
+}
+
+Exit code: 1

acceptance/experimental/air/register-image-no-secret-permission/script

@@ -0,0 +1,9 @@
+# Exported rather than passed through trace: $TESTDIR is a backslash path on
+# Windows, so echoing it into the output would not match this golden.
+export DOCKER_CONFIG="$TESTDIR/dockercfg"
+
+title "private image, but the user cannot create a secret scope to store credentials"
+errcode trace $CLI experimental air register-image nvcr.io/org/img:1.0
+
+title "same failure as a JSON envelope"
+errcode trace $CLI experimental air register-image nvcr.io/org/img:1.0 -o json

acceptance/experimental/air/register-image-no-secret-permission/test.toml

@@ -0,0 +1,40 @@
+# A user who cannot create secret scopes: the Docker credentials found locally
+# can't be stored, so registration falls back to anonymous access and the private
+# image is rejected. The error must name the storage failure as the cause rather
+# than telling the user to `docker login` (which they already did successfully).
+
+# The SDK occasionally probes host reachability with a HEAD request; stub it so
+# the test is deterministic.
+[[Server]]
+Pattern = "HEAD /"
+Response.Body = ''
+
+[[Server]]
+Pattern = "GET /api/2.0/ai-compute-manager/images:get"
+Response.StatusCode = 404
+Response.Body = '''
+{"error_code": "NOT_FOUND", "message": "image not registered"}
+'''
+
+# The image is private, so credential discovery proceeds.
+[[Server]]
+Pattern = "GET /api/2.0/ai-compute-manager/images:checkImageAccess"
+Response.Body = '''
+{"publicly_accessible": false}
+'''
+
+# The workspace admin has not granted permission to create secret scopes.
+[[Server]]
+Pattern = "POST /api/2.0/secrets/scopes/create"
+Response.StatusCode = 403
+Response.Body = '''
+{"error_code": "PERMISSION_DENIED", "message": "user does not have permission to create secret scopes"}
+'''
+
+# Without credentials the manager cannot pull the private image.
+[[Server]]
+Pattern = "POST /api/2.0/ai-compute-manager/images"
+Response.StatusCode = 403
+Response.Body = '''
+{"error_code": "PERMISSION_DENIED", "message": "cannot pull nvcr.io/org/img:1.0: unauthorized"}
+'''

acceptance/experimental/air/register-image/dockercfg/config.json

@@ -0,0 +1,7 @@
+{
+  "auths": {
+    "nvcr.io": {
+      "auth": "Ym9iOnNlY3JldHBhdA=="
+    }
+  }
+}