air: add convert-to-dabs (run YAML -> Databricks Asset Bundle)

acceptance/experimental/air/convert-to-dabs/docker.yaml

@@ -0,0 +1,8 @@
+experiment_name: docker-test
+command: python train.py
+compute:
+  accelerator_type: GPU_1xA10
+  num_accelerators: 1
+environment:
+  docker_image:
+    url: myregistry/img:tag

acceptance/experimental/air/convert-to-dabs/output.txt

@@ -0,0 +1,99 @@
+
+=== convert an AIR run YAML into a DABs bundle (in place, next to the source)
+>>> [CLI] experimental air convert-to-dabs train.yaml
+Wrote a Databricks Asset Bundle to .:
+  databricks.yml
+  generated_artifacts/training_config.yaml
+  generated_artifacts/command.sh
+
+To deploy and run this workload as a bundle:
+  1. [CLI] bundle validate
+  2. [CLI] bundle deploy
+  3. [CLI] bundle run torchrun-a10-smoke-test
+
+bundle deploy uploads the code source and launch scripts automatically.
+To see what it deployed and where: [CLI] bundle summary
+
+Unlike `air run` (which submits an ephemeral run), bundle deploy creates a
+persistent job that is not garbage-collected. When you are done, remove the
+job and its uploaded files with:
+  [CLI] bundle destroy
+
+=== emitted databricks.yml (code_source_path points at ./src; no code is copied)
+>>> cat databricks.yml
+bundle:
+  name: torchrun-a10-smoke-test
+sync:
+  paths:
+    - generated_artifacts
+targets:
+  dev:
+    mode: development
+    default: true
+resources:
+  jobs:
+    torchrun-a10-smoke-test:
+      name: torchrun-a10-smoke-test
+      tasks:
+        - task_key: torchrun-a10-smoke-test
+          environment_key: default
+          max_retries: 3
+          ai_runtime_task:
+            experiment: torchrun-a10-smoke-test
+            deployments:
+              - command_path: ./generated_artifacts/command.sh
+                compute:
+                  accelerator_type: GPU_1xA10
+                  accelerator_count: 1
+            code_source_path: ./src
+      environments:
+        - environment_key: default
+          spec:
+            environment_version: "5"
+            dependencies:
+              - numpy
+
+=== the generated command.sh carries the run command
+>>> cat generated_artifacts/command.sh
+torchrun --nproc_per_node=1 train.py
+=== the emitted bundle validates
+>>> [CLI] bundle validate
+Name: torchrun-a10-smoke-test
+Target: dev
+Workspace:
+  User: [USERNAME]
+  Path: /Workspace/Users/[USERNAME]/.bundle/torchrun-a10-smoke-test/dev
+
+Validation OK!
+
+=== re-converting refuses to clobber the generated bundle
+>>> [CLI] experimental air convert-to-dabs train.yaml
+Error: databricks.yml already exists in .; pass --force to overwrite or remove it
+
+Exit code: 1
+
+=== --force overwrites it
+>>> [CLI] experimental air convert-to-dabs train.yaml --force
+Wrote a Databricks Asset Bundle to .:
+  databricks.yml
+  generated_artifacts/training_config.yaml
+  generated_artifacts/command.sh
+
+To deploy and run this workload as a bundle:
+  1. [CLI] bundle validate
+  2. [CLI] bundle deploy
+  3. [CLI] bundle run torchrun-a10-smoke-test
+
+bundle deploy uploads the code source and launch scripts automatically.
+To see what it deployed and where: [CLI] bundle summary
+
+Unlike `air run` (which submits an ephemeral run), bundle deploy creates a
+persistent job that is not garbage-collected. When you are done, remove the
+job and its uploaded files with:
+  [CLI] bundle destroy
+
+=== docker_image is not supported yet
+>>> [CLI] experimental air convert-to-dabs docker.yaml --output-dir generated-docker
+Error: environment.docker_image is not yet supported by convert-to-dabs
+
+Exit code: 1

acceptance/experimental/air/convert-to-dabs/script

@@ -0,0 +1,20 @@
+title "convert an AIR run YAML into a DABs bundle (in place, next to the source)"
+trace $CLI experimental air convert-to-dabs train.yaml
+
+title "emitted databricks.yml (code_source_path points at ./src; no code is copied)"
+trace cat databricks.yml
+
+title "the generated command.sh carries the run command"
+trace cat generated_artifacts/command.sh
+
+title "the emitted bundle validates"
+trace $CLI bundle validate
+
+title "re-converting refuses to clobber the generated bundle"
+errcode trace $CLI experimental air convert-to-dabs train.yaml
+
+title "--force overwrites it"
+trace $CLI experimental air convert-to-dabs train.yaml --force
+
+title "docker_image is not supported yet"
+errcode trace $CLI experimental air convert-to-dabs docker.yaml --output-dir generated-docker

acceptance/experimental/air/convert-to-dabs/src/train.py

@@ -0,0 +1 @@
+print("train")

acceptance/experimental/air/convert-to-dabs/test.toml

@@ -0,0 +1,3 @@
+# convert-to-dabs writes the bundle in place (next to train.yaml + src/). These are
+# generated artifacts, not committed inputs, so exclude them from the repo-diff check.
+Ignore = ["databricks.yml", "generated_artifacts", "generated-docker"]

acceptance/experimental/air/convert-to-dabs/train.yaml

@@ -0,0 +1,13 @@
+experiment_name: torchrun-a10-smoke-test
+command: torchrun --nproc_per_node=1 train.py
+compute:
+  accelerator_type: GPU_1xA10
+  num_accelerators: 1
+environment:
+  version: 5
+  dependencies:
+    - numpy
+code_source:
+  type: snapshot
+  snapshot:
+    root_path: ./src

acceptance/experimental/air/help/output.txt

@@ -10,12 +10,13 @@ Usage:
   databricks experimental air [command]
 
 Available Commands:
-  cancel         Cancel one or more runs
-  get            Show status, configuration, and timing details for a specific run
-  list           List your active runs for the current profile (use --all-status for finished runs)
-  logs           Stream or fetch logs for a run
-  register-image Mirror a Docker image into the workspace registry
-  run            Submit a training workload from a YAML config
+  cancel          Cancel one or more runs
+  convert-to-dabs Convert an AIR run YAML into a Databricks Asset Bundle
+  get             Show status, configuration, and timing details for a specific run
+  list            List your active runs for the current profile (use --all-status for finished runs)
+  logs            Stream or fetch logs for a run
+  register-image  Mirror a Docker image into the workspace registry
+  run             Submit a training workload from a YAML config
 
 Flags:
   -h, --help   help for air

experimental/air/cmd/air.go

@@ -21,6 +21,7 @@ experimental and may change in future versions.`,
 	cmd.AddCommand(newLogsCommand())
 	cmd.AddCommand(newCancelCommand())
 	cmd.AddCommand(newRegisterImageCommand())
+	cmd.AddCommand(newConvertToDabsCommand())
 
 	return cmd
 }

experimental/air/cmd/air_test.go

@@ -14,7 +14,7 @@ func TestNewRegistersAllSubcommands(t *testing.T) {
 		registered[c.Name()] = true
 	}
 
-	want := []string{"run", "get", "list", "logs", "cancel", "register-image"}
+	want := []string{"run", "get", "list", "logs", "cancel", "register-image", "convert-to-dabs"}
 	for _, name := range want {
 		assert.True(t, registered[name], "subcommand %q is not registered", name)
 	}