From 95fab1223a2707336c5fd7f503ba4c00c01dd90f Mon Sep 17 00:00:00 2001 From: Denis Bilenko Date: Thu, 10 Jul 2025 12:49:08 +0200 Subject: [PATCH 1/4] mov resources/jobs to jobs/update --- acceptance/bundle/resources/jobs/{ => update}/databricks.yml | 0 acceptance/bundle/resources/jobs/{ => update}/out.test.toml | 0 acceptance/bundle/resources/jobs/{ => update}/output.txt | 0 acceptance/bundle/resources/jobs/{ => update}/script | 0 4 files changed, 0 insertions(+), 0 deletions(-) rename acceptance/bundle/resources/jobs/{ => update}/databricks.yml (100%) rename acceptance/bundle/resources/jobs/{ => update}/out.test.toml (100%) rename acceptance/bundle/resources/jobs/{ => update}/output.txt (100%) rename acceptance/bundle/resources/jobs/{ => update}/script (100%) diff --git a/acceptance/bundle/resources/jobs/databricks.yml b/acceptance/bundle/resources/jobs/update/databricks.yml similarity index 100% rename from acceptance/bundle/resources/jobs/databricks.yml rename to acceptance/bundle/resources/jobs/update/databricks.yml diff --git a/acceptance/bundle/resources/jobs/out.test.toml b/acceptance/bundle/resources/jobs/update/out.test.toml similarity index 100% rename from acceptance/bundle/resources/jobs/out.test.toml rename to acceptance/bundle/resources/jobs/update/out.test.toml diff --git a/acceptance/bundle/resources/jobs/output.txt b/acceptance/bundle/resources/jobs/update/output.txt similarity index 100% rename from acceptance/bundle/resources/jobs/output.txt rename to acceptance/bundle/resources/jobs/update/output.txt diff --git a/acceptance/bundle/resources/jobs/script b/acceptance/bundle/resources/jobs/update/script similarity index 100% rename from acceptance/bundle/resources/jobs/script rename to acceptance/bundle/resources/jobs/update/script From c3d1a710453c5259a2f8c17c4446b61030961fe4 Mon Sep 17 00:00:00 2001 From: Denis Bilenko Date: Thu, 10 Jul 2025 12:54:57 +0200 Subject: [PATCH 2/4] add test wip --- .../databricks.yml | 18 ++ .../instance_pool_and_node_type/out.test.toml | 5 + .../instance_pool_and_node_type/output.txt | 245 ++++++++++++++++++ .../jobs/instance_pool_and_node_type/script | 7 + .../instance_pool_and_node_type/test.toml | 1 + 5 files changed, 276 insertions(+) create mode 100644 acceptance/bundle/resources/jobs/instance_pool_and_node_type/databricks.yml create mode 100644 acceptance/bundle/resources/jobs/instance_pool_and_node_type/out.test.toml create mode 100644 acceptance/bundle/resources/jobs/instance_pool_and_node_type/output.txt create mode 100644 acceptance/bundle/resources/jobs/instance_pool_and_node_type/script create mode 100644 acceptance/bundle/resources/jobs/instance_pool_and_node_type/test.toml diff --git a/acceptance/bundle/resources/jobs/instance_pool_and_node_type/databricks.yml b/acceptance/bundle/resources/jobs/instance_pool_and_node_type/databricks.yml new file mode 100644 index 0000000000..d4912e2751 --- /dev/null +++ b/acceptance/bundle/resources/jobs/instance_pool_and_node_type/databricks.yml @@ -0,0 +1,18 @@ +resources: + jobs: + some_other_job: + name: "[${bundle.target}] Test Wheel Job $UNIQUE_NAME" + tasks: + - task_key: TestTask + new_cluster: + num_workers: 1 + spark_version: $DEFAULT_SPARK_VERSION + node_type_id: $NODE_TYPE_ID + data_security_mode: USER_ISOLATION + instance_pool_id: $TEST_INSTANCE_POOL_ID + python_wheel_task: + package_name: my_test_code + entry_point: run + parameters: + - "one" + - "two" diff --git a/acceptance/bundle/resources/jobs/instance_pool_and_node_type/out.test.toml b/acceptance/bundle/resources/jobs/instance_pool_and_node_type/out.test.toml new file mode 100644 index 0000000000..8f3575be7b --- /dev/null +++ b/acceptance/bundle/resources/jobs/instance_pool_and_node_type/out.test.toml @@ -0,0 +1,5 @@ +Local = true +Cloud = false + +[EnvMatrix] + DATABRICKS_CLI_DEPLOYMENT = ["terraform", "direct-exp"] diff --git a/acceptance/bundle/resources/jobs/instance_pool_and_node_type/output.txt b/acceptance/bundle/resources/jobs/instance_pool_and_node_type/output.txt new file mode 100644 index 0000000000..acea4db992 --- /dev/null +++ b/acceptance/bundle/resources/jobs/instance_pool_and_node_type/output.txt @@ -0,0 +1,245 @@ + +>>> [CLI] bundle validate -o json +Warn: Processing job! some_other_job &compute.ClusterSpec{ApplyPolicyDefaultValues:false, Autoscale:(*compute.AutoScale)(nil), AutoterminationMinutes:0, AwsAttributes:(*compute.AwsAttributes)(nil), AzureAttributes:(*compute.AzureAttributes)(nil), ClusterLogConf:(*compute.ClusterLogConf)(nil), ClusterName:"", CustomTags:map[string]string(nil), DataSecurityMode:"USER_ISOLATION", DockerImage:(*compute.DockerImage)(nil), DriverInstancePoolId:"", DriverNodeTypeId:"", EnableElasticDisk:false, EnableLocalDiskEncryption:false, GcpAttributes:(*compute.GcpAttributes)(nil), InitScripts:[]compute.InitScriptInfo(nil), InstancePoolId:"$TEST_INSTANCE_POOL_ID", IsSingleNode:false, Kind:"", NodeTypeId:"$NODE_TYPE_ID", NumWorkers:1, PolicyId:"", RemoteDiskThroughput:0, RuntimeEngine:"", SingleUserName:"", SparkConf:map[string]string(nil), SparkEnvVars:map[string]string(nil), SparkVersion:"$DEFAULT_SPARK_VERSION", SshPublicKeys:[]string(nil), TotalInitialRemoteDiskSize:0, UseMlRuntime:false, WorkloadType:(*compute.WorkloadType)(nil), ForceSendFields:[]string(nil)} +Warn: Processed job! some_other_job &compute.ClusterSpec{ApplyPolicyDefaultValues:false, Autoscale:(*compute.AutoScale)(nil), AutoterminationMinutes:0, AwsAttributes:(*compute.AwsAttributes)(nil), AzureAttributes:(*compute.AzureAttributes)(nil), ClusterLogConf:(*compute.ClusterLogConf)(nil), ClusterName:"", CustomTags:map[string]string(nil), DataSecurityMode:"USER_ISOLATION", DockerImage:(*compute.DockerImage)(nil), DriverInstancePoolId:"", DriverNodeTypeId:"", EnableElasticDisk:false, EnableLocalDiskEncryption:false, GcpAttributes:(*compute.GcpAttributes)(nil), InitScripts:[]compute.InitScriptInfo(nil), InstancePoolId:"$TEST_INSTANCE_POOL_ID", IsSingleNode:false, Kind:"", NodeTypeId:"", NumWorkers:1, PolicyId:"", RemoteDiskThroughput:0, RuntimeEngine:"", SingleUserName:"", SparkConf:map[string]string(nil), SparkEnvVars:map[string]string(nil), SparkVersion:"$DEFAULT_SPARK_VERSION", SshPublicKeys:[]string(nil), TotalInitialRemoteDiskSize:0, UseMlRuntime:false, WorkloadType:(*compute.WorkloadType)(nil), ForceSendFields:[]string(nil)} +Warn: READING job! some_other_job &compute.ClusterSpec{ApplyPolicyDefaultValues:false, Autoscale:(*compute.AutoScale)(nil), AutoterminationMinutes:0, AwsAttributes:(*compute.AwsAttributes)(nil), AzureAttributes:(*compute.AzureAttributes)(nil), ClusterLogConf:(*compute.ClusterLogConf)(nil), ClusterName:"", CustomTags:map[string]string(nil), DataSecurityMode:"USER_ISOLATION", DockerImage:(*compute.DockerImage)(nil), DriverInstancePoolId:"", DriverNodeTypeId:"", EnableElasticDisk:false, EnableLocalDiskEncryption:false, GcpAttributes:(*compute.GcpAttributes)(nil), InitScripts:[]compute.InitScriptInfo(nil), InstancePoolId:"$TEST_INSTANCE_POOL_ID", IsSingleNode:false, Kind:"", NodeTypeId:"", NumWorkers:1, PolicyId:"", RemoteDiskThroughput:0, RuntimeEngine:"", SingleUserName:"", SparkConf:map[string]string(nil), SparkEnvVars:map[string]string(nil), SparkVersion:"$DEFAULT_SPARK_VERSION", SshPublicKeys:[]string(nil), TotalInitialRemoteDiskSize:0, UseMlRuntime:false, WorkloadType:(*compute.WorkloadType)(nil), ForceSendFields:[]string(nil)} +Warn: BUNDLE { + "bundle": { + "environment": "default", + "git": { + "bundle_root_path": "." + }, + "name": "test-bundle", + "target": "default" + }, + "resources": { + "jobs": { + "some_other_job": { + "name": "[default] Test Wheel Job $UNIQUE_NAME", + "tasks": [ + { + "new_cluster": { + "data_security_mode": "USER_ISOLATION", + "instance_pool_id": "$TEST_INSTANCE_POOL_ID", + "node_type_id": "$NODE_TYPE_ID", + "num_workers": 1, + "spark_version": "$DEFAULT_SPARK_VERSION" + }, + "python_wheel_task": { + "entry_point": "run", + "package_name": "my_test_code", + "parameters": [ + "one", + "two" + ] + }, + "task_key": "TestTask" + } + ] + } + } + }, + "sync": { + "paths": [ + "." + ] + }, + "targets": null, + "workspace": { + "artifact_path": "/Workspace/Users/[USERNAME]/.bundle/test-bundle/default/artifacts", + "current_user": { + "id": "[USERID]", + "short_name": "[USERNAME]", + "userName": "[USERNAME]" + }, + "file_path": "/Workspace/Users/[USERNAME]/.bundle/test-bundle/default/files", + "resource_path": "/Workspace/Users/[USERNAME]/.bundle/test-bundle/default/resources", + "root_path": "/Workspace/Users/[USERNAME]/.bundle/test-bundle/default", + "state_path": "/Workspace/Users/[USERNAME]/.bundle/test-bundle/default/state" + } +} +[ + { + "new_cluster": { + "data_security_mode": "USER_ISOLATION", + "instance_pool_id": "$TEST_INSTANCE_POOL_ID", + "node_type_id": "", + "num_workers": 1, + "spark_version": "$DEFAULT_SPARK_VERSION" + }, + "python_wheel_task": { + "entry_point": "run", + "package_name": "my_test_code", + "parameters": [ + "one", + "two" + ] + }, + "task_key": "TestTask" + } +] + +>>> [CLI] bundle summary -o json +Warn: Processing job! some_other_job &compute.ClusterSpec{ApplyPolicyDefaultValues:false, Autoscale:(*compute.AutoScale)(nil), AutoterminationMinutes:0, AwsAttributes:(*compute.AwsAttributes)(nil), AzureAttributes:(*compute.AzureAttributes)(nil), ClusterLogConf:(*compute.ClusterLogConf)(nil), ClusterName:"", CustomTags:map[string]string(nil), DataSecurityMode:"USER_ISOLATION", DockerImage:(*compute.DockerImage)(nil), DriverInstancePoolId:"", DriverNodeTypeId:"", EnableElasticDisk:false, EnableLocalDiskEncryption:false, GcpAttributes:(*compute.GcpAttributes)(nil), InitScripts:[]compute.InitScriptInfo(nil), InstancePoolId:"$TEST_INSTANCE_POOL_ID", IsSingleNode:false, Kind:"", NodeTypeId:"$NODE_TYPE_ID", NumWorkers:1, PolicyId:"", RemoteDiskThroughput:0, RuntimeEngine:"", SingleUserName:"", SparkConf:map[string]string(nil), SparkEnvVars:map[string]string(nil), SparkVersion:"$DEFAULT_SPARK_VERSION", SshPublicKeys:[]string(nil), TotalInitialRemoteDiskSize:0, UseMlRuntime:false, WorkloadType:(*compute.WorkloadType)(nil), ForceSendFields:[]string(nil)} +Warn: Processed job! some_other_job &compute.ClusterSpec{ApplyPolicyDefaultValues:false, Autoscale:(*compute.AutoScale)(nil), AutoterminationMinutes:0, AwsAttributes:(*compute.AwsAttributes)(nil), AzureAttributes:(*compute.AzureAttributes)(nil), ClusterLogConf:(*compute.ClusterLogConf)(nil), ClusterName:"", CustomTags:map[string]string(nil), DataSecurityMode:"USER_ISOLATION", DockerImage:(*compute.DockerImage)(nil), DriverInstancePoolId:"", DriverNodeTypeId:"", EnableElasticDisk:false, EnableLocalDiskEncryption:false, GcpAttributes:(*compute.GcpAttributes)(nil), InitScripts:[]compute.InitScriptInfo(nil), InstancePoolId:"$TEST_INSTANCE_POOL_ID", IsSingleNode:false, Kind:"", NodeTypeId:"", NumWorkers:1, PolicyId:"", RemoteDiskThroughput:0, RuntimeEngine:"", SingleUserName:"", SparkConf:map[string]string(nil), SparkEnvVars:map[string]string(nil), SparkVersion:"$DEFAULT_SPARK_VERSION", SshPublicKeys:[]string(nil), TotalInitialRemoteDiskSize:0, UseMlRuntime:false, WorkloadType:(*compute.WorkloadType)(nil), ForceSendFields:[]string(nil)} +Warn: READING job! some_other_job &compute.ClusterSpec{ApplyPolicyDefaultValues:false, Autoscale:(*compute.AutoScale)(nil), AutoterminationMinutes:0, AwsAttributes:(*compute.AwsAttributes)(nil), AzureAttributes:(*compute.AzureAttributes)(nil), ClusterLogConf:(*compute.ClusterLogConf)(nil), ClusterName:"", CustomTags:map[string]string(nil), DataSecurityMode:"USER_ISOLATION", DockerImage:(*compute.DockerImage)(nil), DriverInstancePoolId:"", DriverNodeTypeId:"", EnableElasticDisk:false, EnableLocalDiskEncryption:false, GcpAttributes:(*compute.GcpAttributes)(nil), InitScripts:[]compute.InitScriptInfo(nil), InstancePoolId:"$TEST_INSTANCE_POOL_ID", IsSingleNode:false, Kind:"", NodeTypeId:"", NumWorkers:1, PolicyId:"", RemoteDiskThroughput:0, RuntimeEngine:"", SingleUserName:"", SparkConf:map[string]string(nil), SparkEnvVars:map[string]string(nil), SparkVersion:"$DEFAULT_SPARK_VERSION", SshPublicKeys:[]string(nil), TotalInitialRemoteDiskSize:0, UseMlRuntime:false, WorkloadType:(*compute.WorkloadType)(nil), ForceSendFields:[]string(nil)} +Warn: BUNDLE { + "bundle": { + "environment": "default", + "git": { + "bundle_root_path": "." + }, + "name": "test-bundle", + "target": "default" + }, + "resources": { + "jobs": { + "some_other_job": { + "name": "[default] Test Wheel Job $UNIQUE_NAME", + "tasks": [ + { + "new_cluster": { + "data_security_mode": "USER_ISOLATION", + "instance_pool_id": "$TEST_INSTANCE_POOL_ID", + "node_type_id": "$NODE_TYPE_ID", + "num_workers": 1, + "spark_version": "$DEFAULT_SPARK_VERSION" + }, + "python_wheel_task": { + "entry_point": "run", + "package_name": "my_test_code", + "parameters": [ + "one", + "two" + ] + }, + "task_key": "TestTask" + } + ] + } + } + }, + "sync": { + "paths": [ + "." + ] + }, + "targets": null, + "workspace": { + "artifact_path": "/Workspace/Users/[USERNAME]/.bundle/test-bundle/default/artifacts", + "current_user": { + "id": "[USERID]", + "short_name": "[USERNAME]", + "userName": "[USERNAME]" + }, + "file_path": "/Workspace/Users/[USERNAME]/.bundle/test-bundle/default/files", + "resource_path": "/Workspace/Users/[USERNAME]/.bundle/test-bundle/default/resources", + "root_path": "/Workspace/Users/[USERNAME]/.bundle/test-bundle/default", + "state_path": "/Workspace/Users/[USERNAME]/.bundle/test-bundle/default/state" + } +} +[ + { + "new_cluster": { + "data_security_mode": "USER_ISOLATION", + "instance_pool_id": "$TEST_INSTANCE_POOL_ID", + "node_type_id": "", + "num_workers": 1, + "spark_version": "$DEFAULT_SPARK_VERSION" + }, + "python_wheel_task": { + "entry_point": "run", + "package_name": "my_test_code", + "parameters": [ + "one", + "two" + ] + }, + "task_key": "TestTask" + } +] + +>>> [CLI] bundle deploy +Warn: Processing job! some_other_job &compute.ClusterSpec{ApplyPolicyDefaultValues:false, Autoscale:(*compute.AutoScale)(nil), AutoterminationMinutes:0, AwsAttributes:(*compute.AwsAttributes)(nil), AzureAttributes:(*compute.AzureAttributes)(nil), ClusterLogConf:(*compute.ClusterLogConf)(nil), ClusterName:"", CustomTags:map[string]string(nil), DataSecurityMode:"USER_ISOLATION", DockerImage:(*compute.DockerImage)(nil), DriverInstancePoolId:"", DriverNodeTypeId:"", EnableElasticDisk:false, EnableLocalDiskEncryption:false, GcpAttributes:(*compute.GcpAttributes)(nil), InitScripts:[]compute.InitScriptInfo(nil), InstancePoolId:"$TEST_INSTANCE_POOL_ID", IsSingleNode:false, Kind:"", NodeTypeId:"$NODE_TYPE_ID", NumWorkers:1, PolicyId:"", RemoteDiskThroughput:0, RuntimeEngine:"", SingleUserName:"", SparkConf:map[string]string(nil), SparkEnvVars:map[string]string(nil), SparkVersion:"$DEFAULT_SPARK_VERSION", SshPublicKeys:[]string(nil), TotalInitialRemoteDiskSize:0, UseMlRuntime:false, WorkloadType:(*compute.WorkloadType)(nil), ForceSendFields:[]string(nil)} +Warn: Processed job! some_other_job &compute.ClusterSpec{ApplyPolicyDefaultValues:false, Autoscale:(*compute.AutoScale)(nil), AutoterminationMinutes:0, AwsAttributes:(*compute.AwsAttributes)(nil), AzureAttributes:(*compute.AzureAttributes)(nil), ClusterLogConf:(*compute.ClusterLogConf)(nil), ClusterName:"", CustomTags:map[string]string(nil), DataSecurityMode:"USER_ISOLATION", DockerImage:(*compute.DockerImage)(nil), DriverInstancePoolId:"", DriverNodeTypeId:"", EnableElasticDisk:false, EnableLocalDiskEncryption:false, GcpAttributes:(*compute.GcpAttributes)(nil), InitScripts:[]compute.InitScriptInfo(nil), InstancePoolId:"$TEST_INSTANCE_POOL_ID", IsSingleNode:false, Kind:"", NodeTypeId:"", NumWorkers:1, PolicyId:"", RemoteDiskThroughput:0, RuntimeEngine:"", SingleUserName:"", SparkConf:map[string]string(nil), SparkEnvVars:map[string]string(nil), SparkVersion:"$DEFAULT_SPARK_VERSION", SshPublicKeys:[]string(nil), TotalInitialRemoteDiskSize:0, UseMlRuntime:false, WorkloadType:(*compute.WorkloadType)(nil), ForceSendFields:[]string(nil)} +Warn: READING job! some_other_job &compute.ClusterSpec{ApplyPolicyDefaultValues:false, Autoscale:(*compute.AutoScale)(nil), AutoterminationMinutes:0, AwsAttributes:(*compute.AwsAttributes)(nil), AzureAttributes:(*compute.AzureAttributes)(nil), ClusterLogConf:(*compute.ClusterLogConf)(nil), ClusterName:"", CustomTags:map[string]string(nil), DataSecurityMode:"USER_ISOLATION", DockerImage:(*compute.DockerImage)(nil), DriverInstancePoolId:"", DriverNodeTypeId:"", EnableElasticDisk:false, EnableLocalDiskEncryption:false, GcpAttributes:(*compute.GcpAttributes)(nil), InitScripts:[]compute.InitScriptInfo(nil), InstancePoolId:"$TEST_INSTANCE_POOL_ID", IsSingleNode:false, Kind:"", NodeTypeId:"", NumWorkers:1, PolicyId:"", RemoteDiskThroughput:0, RuntimeEngine:"", SingleUserName:"", SparkConf:map[string]string(nil), SparkEnvVars:map[string]string(nil), SparkVersion:"$DEFAULT_SPARK_VERSION", SshPublicKeys:[]string(nil), TotalInitialRemoteDiskSize:0, UseMlRuntime:false, WorkloadType:(*compute.WorkloadType)(nil), ForceSendFields:[]string(nil)} +Warn: BUNDLE { + "bundle": { + "environment": "default", + "git": { + "bundle_root_path": "." + }, + "name": "test-bundle", + "target": "default" + }, + "resources": { + "jobs": { + "some_other_job": { + "name": "[default] Test Wheel Job $UNIQUE_NAME", + "tasks": [ + { + "new_cluster": { + "data_security_mode": "USER_ISOLATION", + "instance_pool_id": "$TEST_INSTANCE_POOL_ID", + "node_type_id": "$NODE_TYPE_ID", + "num_workers": 1, + "spark_version": "$DEFAULT_SPARK_VERSION" + }, + "python_wheel_task": { + "entry_point": "run", + "package_name": "my_test_code", + "parameters": [ + "one", + "two" + ] + }, + "task_key": "TestTask" + } + ] + } + } + }, + "sync": { + "paths": [ + "." + ] + }, + "targets": null, + "workspace": { + "artifact_path": "/Workspace/Users/[USERNAME]/.bundle/test-bundle/default/artifacts", + "current_user": { + "id": "[USERID]", + "short_name": "[USERNAME]", + "userName": "[USERNAME]" + }, + "file_path": "/Workspace/Users/[USERNAME]/.bundle/test-bundle/default/files", + "resource_path": "/Workspace/Users/[USERNAME]/.bundle/test-bundle/default/resources", + "root_path": "/Workspace/Users/[USERNAME]/.bundle/test-bundle/default", + "state_path": "/Workspace/Users/[USERNAME]/.bundle/test-bundle/default/state" + } +} +Uploading bundle files to /Workspace/Users/[USERNAME]/.bundle/test-bundle/default/files... +Deploying resources... +Updating deployment state... +Deployment complete! + +>>> jq -s .[] | select(.path=="/api/2.2/jobs/create") | .body.tasks out.requests.txt +[ + { + "new_cluster": { + "data_security_mode": "USER_ISOLATION", + "instance_pool_id": "$TEST_INSTANCE_POOL_ID", + "num_workers": 1, + "spark_version": "$DEFAULT_SPARK_VERSION" + }, + "python_wheel_task": { + "entry_point": "run", + "package_name": "my_test_code", + "parameters": [ + "one", + "two" + ] + }, + "task_key": "TestTask" + } +] diff --git a/acceptance/bundle/resources/jobs/instance_pool_and_node_type/script b/acceptance/bundle/resources/jobs/instance_pool_and_node_type/script new file mode 100644 index 0000000000..4f39d94ce7 --- /dev/null +++ b/acceptance/bundle/resources/jobs/instance_pool_and_node_type/script @@ -0,0 +1,7 @@ +trace $CLI bundle validate -o json | jq .resources.jobs.some_other_job.tasks +trace $CLI bundle summary -o json | jq .resources.jobs.some_other_job.tasks + +trace $CLI bundle deploy + +trace jq -s '.[] | select(.path=="/api/2.2/jobs/create") | .body.tasks' out.requests.txt +rm out.requests.txt diff --git a/acceptance/bundle/resources/jobs/instance_pool_and_node_type/test.toml b/acceptance/bundle/resources/jobs/instance_pool_and_node_type/test.toml new file mode 100644 index 0000000000..159efe0269 --- /dev/null +++ b/acceptance/bundle/resources/jobs/instance_pool_and_node_type/test.toml @@ -0,0 +1 @@ +RecordRequests = true From 7c1ea533fb1c87e19fb8fba56c118a5531cb502a Mon Sep 17 00:00:00 2001 From: Denis Bilenko Date: Thu, 10 Jul 2025 13:16:07 +0200 Subject: [PATCH 3/4] update --- .../instance_pool_and_node_type/output.txt | 174 ------------------ .../instance_pool_and_node_type/test.toml | 17 ++ 2 files changed, 17 insertions(+), 174 deletions(-) diff --git a/acceptance/bundle/resources/jobs/instance_pool_and_node_type/output.txt b/acceptance/bundle/resources/jobs/instance_pool_and_node_type/output.txt index acea4db992..d3482bd1b5 100644 --- a/acceptance/bundle/resources/jobs/instance_pool_and_node_type/output.txt +++ b/acceptance/bundle/resources/jobs/instance_pool_and_node_type/output.txt @@ -1,63 +1,5 @@ >>> [CLI] bundle validate -o json -Warn: Processing job! some_other_job &compute.ClusterSpec{ApplyPolicyDefaultValues:false, Autoscale:(*compute.AutoScale)(nil), AutoterminationMinutes:0, AwsAttributes:(*compute.AwsAttributes)(nil), AzureAttributes:(*compute.AzureAttributes)(nil), ClusterLogConf:(*compute.ClusterLogConf)(nil), ClusterName:"", CustomTags:map[string]string(nil), DataSecurityMode:"USER_ISOLATION", DockerImage:(*compute.DockerImage)(nil), DriverInstancePoolId:"", DriverNodeTypeId:"", EnableElasticDisk:false, EnableLocalDiskEncryption:false, GcpAttributes:(*compute.GcpAttributes)(nil), InitScripts:[]compute.InitScriptInfo(nil), InstancePoolId:"$TEST_INSTANCE_POOL_ID", IsSingleNode:false, Kind:"", NodeTypeId:"$NODE_TYPE_ID", NumWorkers:1, PolicyId:"", RemoteDiskThroughput:0, RuntimeEngine:"", SingleUserName:"", SparkConf:map[string]string(nil), SparkEnvVars:map[string]string(nil), SparkVersion:"$DEFAULT_SPARK_VERSION", SshPublicKeys:[]string(nil), TotalInitialRemoteDiskSize:0, UseMlRuntime:false, WorkloadType:(*compute.WorkloadType)(nil), ForceSendFields:[]string(nil)} -Warn: Processed job! some_other_job &compute.ClusterSpec{ApplyPolicyDefaultValues:false, Autoscale:(*compute.AutoScale)(nil), AutoterminationMinutes:0, AwsAttributes:(*compute.AwsAttributes)(nil), AzureAttributes:(*compute.AzureAttributes)(nil), ClusterLogConf:(*compute.ClusterLogConf)(nil), ClusterName:"", CustomTags:map[string]string(nil), DataSecurityMode:"USER_ISOLATION", DockerImage:(*compute.DockerImage)(nil), DriverInstancePoolId:"", DriverNodeTypeId:"", EnableElasticDisk:false, EnableLocalDiskEncryption:false, GcpAttributes:(*compute.GcpAttributes)(nil), InitScripts:[]compute.InitScriptInfo(nil), InstancePoolId:"$TEST_INSTANCE_POOL_ID", IsSingleNode:false, Kind:"", NodeTypeId:"", NumWorkers:1, PolicyId:"", RemoteDiskThroughput:0, RuntimeEngine:"", SingleUserName:"", SparkConf:map[string]string(nil), SparkEnvVars:map[string]string(nil), SparkVersion:"$DEFAULT_SPARK_VERSION", SshPublicKeys:[]string(nil), TotalInitialRemoteDiskSize:0, UseMlRuntime:false, WorkloadType:(*compute.WorkloadType)(nil), ForceSendFields:[]string(nil)} -Warn: READING job! some_other_job &compute.ClusterSpec{ApplyPolicyDefaultValues:false, Autoscale:(*compute.AutoScale)(nil), AutoterminationMinutes:0, AwsAttributes:(*compute.AwsAttributes)(nil), AzureAttributes:(*compute.AzureAttributes)(nil), ClusterLogConf:(*compute.ClusterLogConf)(nil), ClusterName:"", CustomTags:map[string]string(nil), DataSecurityMode:"USER_ISOLATION", DockerImage:(*compute.DockerImage)(nil), DriverInstancePoolId:"", DriverNodeTypeId:"", EnableElasticDisk:false, EnableLocalDiskEncryption:false, GcpAttributes:(*compute.GcpAttributes)(nil), InitScripts:[]compute.InitScriptInfo(nil), InstancePoolId:"$TEST_INSTANCE_POOL_ID", IsSingleNode:false, Kind:"", NodeTypeId:"", NumWorkers:1, PolicyId:"", RemoteDiskThroughput:0, RuntimeEngine:"", SingleUserName:"", SparkConf:map[string]string(nil), SparkEnvVars:map[string]string(nil), SparkVersion:"$DEFAULT_SPARK_VERSION", SshPublicKeys:[]string(nil), TotalInitialRemoteDiskSize:0, UseMlRuntime:false, WorkloadType:(*compute.WorkloadType)(nil), ForceSendFields:[]string(nil)} -Warn: BUNDLE { - "bundle": { - "environment": "default", - "git": { - "bundle_root_path": "." - }, - "name": "test-bundle", - "target": "default" - }, - "resources": { - "jobs": { - "some_other_job": { - "name": "[default] Test Wheel Job $UNIQUE_NAME", - "tasks": [ - { - "new_cluster": { - "data_security_mode": "USER_ISOLATION", - "instance_pool_id": "$TEST_INSTANCE_POOL_ID", - "node_type_id": "$NODE_TYPE_ID", - "num_workers": 1, - "spark_version": "$DEFAULT_SPARK_VERSION" - }, - "python_wheel_task": { - "entry_point": "run", - "package_name": "my_test_code", - "parameters": [ - "one", - "two" - ] - }, - "task_key": "TestTask" - } - ] - } - } - }, - "sync": { - "paths": [ - "." - ] - }, - "targets": null, - "workspace": { - "artifact_path": "/Workspace/Users/[USERNAME]/.bundle/test-bundle/default/artifacts", - "current_user": { - "id": "[USERID]", - "short_name": "[USERNAME]", - "userName": "[USERNAME]" - }, - "file_path": "/Workspace/Users/[USERNAME]/.bundle/test-bundle/default/files", - "resource_path": "/Workspace/Users/[USERNAME]/.bundle/test-bundle/default/resources", - "root_path": "/Workspace/Users/[USERNAME]/.bundle/test-bundle/default", - "state_path": "/Workspace/Users/[USERNAME]/.bundle/test-bundle/default/state" - } -} [ { "new_cluster": { @@ -80,64 +22,6 @@ Warn: BUNDLE { ] >>> [CLI] bundle summary -o json -Warn: Processing job! some_other_job &compute.ClusterSpec{ApplyPolicyDefaultValues:false, Autoscale:(*compute.AutoScale)(nil), AutoterminationMinutes:0, AwsAttributes:(*compute.AwsAttributes)(nil), AzureAttributes:(*compute.AzureAttributes)(nil), ClusterLogConf:(*compute.ClusterLogConf)(nil), ClusterName:"", CustomTags:map[string]string(nil), DataSecurityMode:"USER_ISOLATION", DockerImage:(*compute.DockerImage)(nil), DriverInstancePoolId:"", DriverNodeTypeId:"", EnableElasticDisk:false, EnableLocalDiskEncryption:false, GcpAttributes:(*compute.GcpAttributes)(nil), InitScripts:[]compute.InitScriptInfo(nil), InstancePoolId:"$TEST_INSTANCE_POOL_ID", IsSingleNode:false, Kind:"", NodeTypeId:"$NODE_TYPE_ID", NumWorkers:1, PolicyId:"", RemoteDiskThroughput:0, RuntimeEngine:"", SingleUserName:"", SparkConf:map[string]string(nil), SparkEnvVars:map[string]string(nil), SparkVersion:"$DEFAULT_SPARK_VERSION", SshPublicKeys:[]string(nil), TotalInitialRemoteDiskSize:0, UseMlRuntime:false, WorkloadType:(*compute.WorkloadType)(nil), ForceSendFields:[]string(nil)} -Warn: Processed job! some_other_job &compute.ClusterSpec{ApplyPolicyDefaultValues:false, Autoscale:(*compute.AutoScale)(nil), AutoterminationMinutes:0, AwsAttributes:(*compute.AwsAttributes)(nil), AzureAttributes:(*compute.AzureAttributes)(nil), ClusterLogConf:(*compute.ClusterLogConf)(nil), ClusterName:"", CustomTags:map[string]string(nil), DataSecurityMode:"USER_ISOLATION", DockerImage:(*compute.DockerImage)(nil), DriverInstancePoolId:"", DriverNodeTypeId:"", EnableElasticDisk:false, EnableLocalDiskEncryption:false, GcpAttributes:(*compute.GcpAttributes)(nil), InitScripts:[]compute.InitScriptInfo(nil), InstancePoolId:"$TEST_INSTANCE_POOL_ID", IsSingleNode:false, Kind:"", NodeTypeId:"", NumWorkers:1, PolicyId:"", RemoteDiskThroughput:0, RuntimeEngine:"", SingleUserName:"", SparkConf:map[string]string(nil), SparkEnvVars:map[string]string(nil), SparkVersion:"$DEFAULT_SPARK_VERSION", SshPublicKeys:[]string(nil), TotalInitialRemoteDiskSize:0, UseMlRuntime:false, WorkloadType:(*compute.WorkloadType)(nil), ForceSendFields:[]string(nil)} -Warn: READING job! some_other_job &compute.ClusterSpec{ApplyPolicyDefaultValues:false, Autoscale:(*compute.AutoScale)(nil), AutoterminationMinutes:0, AwsAttributes:(*compute.AwsAttributes)(nil), AzureAttributes:(*compute.AzureAttributes)(nil), ClusterLogConf:(*compute.ClusterLogConf)(nil), ClusterName:"", CustomTags:map[string]string(nil), DataSecurityMode:"USER_ISOLATION", DockerImage:(*compute.DockerImage)(nil), DriverInstancePoolId:"", DriverNodeTypeId:"", EnableElasticDisk:false, EnableLocalDiskEncryption:false, GcpAttributes:(*compute.GcpAttributes)(nil), InitScripts:[]compute.InitScriptInfo(nil), InstancePoolId:"$TEST_INSTANCE_POOL_ID", IsSingleNode:false, Kind:"", NodeTypeId:"", NumWorkers:1, PolicyId:"", RemoteDiskThroughput:0, RuntimeEngine:"", SingleUserName:"", SparkConf:map[string]string(nil), SparkEnvVars:map[string]string(nil), SparkVersion:"$DEFAULT_SPARK_VERSION", SshPublicKeys:[]string(nil), TotalInitialRemoteDiskSize:0, UseMlRuntime:false, WorkloadType:(*compute.WorkloadType)(nil), ForceSendFields:[]string(nil)} -Warn: BUNDLE { - "bundle": { - "environment": "default", - "git": { - "bundle_root_path": "." - }, - "name": "test-bundle", - "target": "default" - }, - "resources": { - "jobs": { - "some_other_job": { - "name": "[default] Test Wheel Job $UNIQUE_NAME", - "tasks": [ - { - "new_cluster": { - "data_security_mode": "USER_ISOLATION", - "instance_pool_id": "$TEST_INSTANCE_POOL_ID", - "node_type_id": "$NODE_TYPE_ID", - "num_workers": 1, - "spark_version": "$DEFAULT_SPARK_VERSION" - }, - "python_wheel_task": { - "entry_point": "run", - "package_name": "my_test_code", - "parameters": [ - "one", - "two" - ] - }, - "task_key": "TestTask" - } - ] - } - } - }, - "sync": { - "paths": [ - "." - ] - }, - "targets": null, - "workspace": { - "artifact_path": "/Workspace/Users/[USERNAME]/.bundle/test-bundle/default/artifacts", - "current_user": { - "id": "[USERID]", - "short_name": "[USERNAME]", - "userName": "[USERNAME]" - }, - "file_path": "/Workspace/Users/[USERNAME]/.bundle/test-bundle/default/files", - "resource_path": "/Workspace/Users/[USERNAME]/.bundle/test-bundle/default/resources", - "root_path": "/Workspace/Users/[USERNAME]/.bundle/test-bundle/default", - "state_path": "/Workspace/Users/[USERNAME]/.bundle/test-bundle/default/state" - } -} [ { "new_cluster": { @@ -160,64 +44,6 @@ Warn: BUNDLE { ] >>> [CLI] bundle deploy -Warn: Processing job! some_other_job &compute.ClusterSpec{ApplyPolicyDefaultValues:false, Autoscale:(*compute.AutoScale)(nil), AutoterminationMinutes:0, AwsAttributes:(*compute.AwsAttributes)(nil), AzureAttributes:(*compute.AzureAttributes)(nil), ClusterLogConf:(*compute.ClusterLogConf)(nil), ClusterName:"", CustomTags:map[string]string(nil), DataSecurityMode:"USER_ISOLATION", DockerImage:(*compute.DockerImage)(nil), DriverInstancePoolId:"", DriverNodeTypeId:"", EnableElasticDisk:false, EnableLocalDiskEncryption:false, GcpAttributes:(*compute.GcpAttributes)(nil), InitScripts:[]compute.InitScriptInfo(nil), InstancePoolId:"$TEST_INSTANCE_POOL_ID", IsSingleNode:false, Kind:"", NodeTypeId:"$NODE_TYPE_ID", NumWorkers:1, PolicyId:"", RemoteDiskThroughput:0, RuntimeEngine:"", SingleUserName:"", SparkConf:map[string]string(nil), SparkEnvVars:map[string]string(nil), SparkVersion:"$DEFAULT_SPARK_VERSION", SshPublicKeys:[]string(nil), TotalInitialRemoteDiskSize:0, UseMlRuntime:false, WorkloadType:(*compute.WorkloadType)(nil), ForceSendFields:[]string(nil)} -Warn: Processed job! some_other_job &compute.ClusterSpec{ApplyPolicyDefaultValues:false, Autoscale:(*compute.AutoScale)(nil), AutoterminationMinutes:0, AwsAttributes:(*compute.AwsAttributes)(nil), AzureAttributes:(*compute.AzureAttributes)(nil), ClusterLogConf:(*compute.ClusterLogConf)(nil), ClusterName:"", CustomTags:map[string]string(nil), DataSecurityMode:"USER_ISOLATION", DockerImage:(*compute.DockerImage)(nil), DriverInstancePoolId:"", DriverNodeTypeId:"", EnableElasticDisk:false, EnableLocalDiskEncryption:false, GcpAttributes:(*compute.GcpAttributes)(nil), InitScripts:[]compute.InitScriptInfo(nil), InstancePoolId:"$TEST_INSTANCE_POOL_ID", IsSingleNode:false, Kind:"", NodeTypeId:"", NumWorkers:1, PolicyId:"", RemoteDiskThroughput:0, RuntimeEngine:"", SingleUserName:"", SparkConf:map[string]string(nil), SparkEnvVars:map[string]string(nil), SparkVersion:"$DEFAULT_SPARK_VERSION", SshPublicKeys:[]string(nil), TotalInitialRemoteDiskSize:0, UseMlRuntime:false, WorkloadType:(*compute.WorkloadType)(nil), ForceSendFields:[]string(nil)} -Warn: READING job! some_other_job &compute.ClusterSpec{ApplyPolicyDefaultValues:false, Autoscale:(*compute.AutoScale)(nil), AutoterminationMinutes:0, AwsAttributes:(*compute.AwsAttributes)(nil), AzureAttributes:(*compute.AzureAttributes)(nil), ClusterLogConf:(*compute.ClusterLogConf)(nil), ClusterName:"", CustomTags:map[string]string(nil), DataSecurityMode:"USER_ISOLATION", DockerImage:(*compute.DockerImage)(nil), DriverInstancePoolId:"", DriverNodeTypeId:"", EnableElasticDisk:false, EnableLocalDiskEncryption:false, GcpAttributes:(*compute.GcpAttributes)(nil), InitScripts:[]compute.InitScriptInfo(nil), InstancePoolId:"$TEST_INSTANCE_POOL_ID", IsSingleNode:false, Kind:"", NodeTypeId:"", NumWorkers:1, PolicyId:"", RemoteDiskThroughput:0, RuntimeEngine:"", SingleUserName:"", SparkConf:map[string]string(nil), SparkEnvVars:map[string]string(nil), SparkVersion:"$DEFAULT_SPARK_VERSION", SshPublicKeys:[]string(nil), TotalInitialRemoteDiskSize:0, UseMlRuntime:false, WorkloadType:(*compute.WorkloadType)(nil), ForceSendFields:[]string(nil)} -Warn: BUNDLE { - "bundle": { - "environment": "default", - "git": { - "bundle_root_path": "." - }, - "name": "test-bundle", - "target": "default" - }, - "resources": { - "jobs": { - "some_other_job": { - "name": "[default] Test Wheel Job $UNIQUE_NAME", - "tasks": [ - { - "new_cluster": { - "data_security_mode": "USER_ISOLATION", - "instance_pool_id": "$TEST_INSTANCE_POOL_ID", - "node_type_id": "$NODE_TYPE_ID", - "num_workers": 1, - "spark_version": "$DEFAULT_SPARK_VERSION" - }, - "python_wheel_task": { - "entry_point": "run", - "package_name": "my_test_code", - "parameters": [ - "one", - "two" - ] - }, - "task_key": "TestTask" - } - ] - } - } - }, - "sync": { - "paths": [ - "." - ] - }, - "targets": null, - "workspace": { - "artifact_path": "/Workspace/Users/[USERNAME]/.bundle/test-bundle/default/artifacts", - "current_user": { - "id": "[USERID]", - "short_name": "[USERNAME]", - "userName": "[USERNAME]" - }, - "file_path": "/Workspace/Users/[USERNAME]/.bundle/test-bundle/default/files", - "resource_path": "/Workspace/Users/[USERNAME]/.bundle/test-bundle/default/resources", - "root_path": "/Workspace/Users/[USERNAME]/.bundle/test-bundle/default", - "state_path": "/Workspace/Users/[USERNAME]/.bundle/test-bundle/default/state" - } -} Uploading bundle files to /Workspace/Users/[USERNAME]/.bundle/test-bundle/default/files... Deploying resources... Updating deployment state... diff --git a/acceptance/bundle/resources/jobs/instance_pool_and_node_type/test.toml b/acceptance/bundle/resources/jobs/instance_pool_and_node_type/test.toml index 159efe0269..94ab1af79a 100644 --- a/acceptance/bundle/resources/jobs/instance_pool_and_node_type/test.toml +++ b/acceptance/bundle/resources/jobs/instance_pool_and_node_type/test.toml @@ -1 +1,18 @@ RecordRequests = true + +# Fails on direct with +# --- FAIL: TestAccept/bundle/resources/jobs/instance_pool_and_node_type (0.00s) +# --- FAIL: TestAccept/bundle/resources/jobs/instance_pool_and_node_type/DATABRICKS_CLI_DEPLOYMENT=direct-exp (1.60s) +# acceptance_test.go:1178: Writing updated bundle config to databricks.yml. BundleConfig sections: default_name +# acceptance_test.go:722: Diff: +# --- bundle/resources/jobs/instance_pool_and_node_type/output.txt +# +++ /var/folders/5y/9kkdnjw91p11vsqwk0cvmk200000gp/T/TestAcceptbundleresourcesjobsinstance_pool_and_node_typeDATABRICKS_CLI_DEPLOYMENT=direct-exp3221363519/001/output.txt +# @@ -55,6 +55,7 @@ +# "new_cluster": { +# "data_security_mode": "USER_ISOLATION", +# "instance_pool_id": "$TEST_INSTANCE_POOL_ID", +# + "node_type_id": "", +# "num_workers": 1, +# "spark_version": "$DEFAULT_SPARK_VERSION" +# }, +EnvMatrix.DATABRICKS_CLI_DEPLOYMENT = ["terraform"] From b4c072606c83977181dda5b94d4d7faaf2e30b5a Mon Sep 17 00:00:00 2001 From: Denis Bilenko Date: Thu, 10 Jul 2025 13:29:47 +0200 Subject: [PATCH 4/4] update out.test.toml --- .../resources/jobs/instance_pool_and_node_type/out.test.toml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/acceptance/bundle/resources/jobs/instance_pool_and_node_type/out.test.toml b/acceptance/bundle/resources/jobs/instance_pool_and_node_type/out.test.toml index 8f3575be7b..387f29e8bb 100644 --- a/acceptance/bundle/resources/jobs/instance_pool_and_node_type/out.test.toml +++ b/acceptance/bundle/resources/jobs/instance_pool_and_node_type/out.test.toml @@ -2,4 +2,4 @@ Local = true Cloud = false [EnvMatrix] - DATABRICKS_CLI_DEPLOYMENT = ["terraform", "direct-exp"] + DATABRICKS_CLI_DEPLOYMENT = ["terraform"]