Drop empty strings on omitempty fields before deploy

.nextchanges/bundles/drop-empty-strings.md

@@ -0,0 +1 @@
+Empty-string values on optional (omitempty) resource fields are now dropped before deployment instead of being sent to the backend. This fixes deploys failing with errors like `'' is not a valid cluster policy ID` when a field such as `policy_id` was set to `""` (often via a variable that resolved to an empty string). The behavior now matches between the terraform and direct engines and is reflected in `bundle validate -o json`.

acceptance/bundle/empty_string_dropped/base.yml

@@ -21,6 +21,9 @@ resources:
   clusters:
     my_cluster:
       num_workers: 1
+      # spark_version is tagged omitempty in the SDK but required by terraform's
+      # schema and the backend, so it must have a real value or the deploy fails.
+      spark_version: 13.3.x-scala2.12
       aws_attributes:
         availability: SPOT
       cluster_log_conf:
@@ -38,6 +41,7 @@ resources:
         - job_cluster_key: main
           new_cluster:
             num_workers: 1
+            spark_version: 13.3.x-scala2.12
             aws_attributes:
               availability: SPOT
             cluster_log_conf:

acceptance/bundle/empty_string_dropped/databricks.yml

@@ -29,7 +29,7 @@ resources:
       num_workers: 1
       policy_id: ''
       single_user_name: ''
-      spark_version: ''
+      spark_version: 13.3.x-scala2.12
   database_instances:
     my_db_instance:
       capacity: CU_1
@@ -67,7 +67,7 @@ resources:
             num_workers: 1
             policy_id: ''
             single_user_name: ''
-            spark_version: ''
+            spark_version: 13.3.x-scala2.12
       max_concurrent_runs: 1
       name: ''
       parent_path: ''

acceptance/bundle/empty_string_dropped/out.empty_dropped_by_both.txt

@@ -1 +1,58 @@
+/apps budget_policy_id
+/apps space
+/apps usage_policy_id
+/database/instances usage_policy_id
+/experiments/create artifact_location
+/mlflow/registered-models description
+/pipelines budget_policy_id
+/pipelines clusters.aws_attributes.instance_profile_arn
+/pipelines clusters.aws_attributes.zone_id
+/pipelines clusters.driver_instance_pool_id
+/pipelines clusters.driver_node_type_id
+/pipelines clusters.instance_pool_id
+/pipelines clusters.node_type_id
+/pipelines clusters.policy_id
+/pipelines libraries.jar
+/pipelines libraries.whl
+/pipelines name
+/pipelines serverless_compute_id
+/pipelines usage_policy_id
+/unity-catalog/models comment
+/unity-catalog/models created_by
+/unity-catalog/models full_name
+/unity-catalog/models metastore_id
+/unity-catalog/models owner
+/unity-catalog/models storage_location
+/unity-catalog/models updated_by
+/unity-catalog/schemas comment
+/unity-catalog/schemas storage_root
+/unity-catalog/volumes comment
+/unity-catalog/volumes storage_location
 /unity-catalog/volumes volume_path
+/warehouses instance_profile_arn
+clusters/create aws_attributes.instance_profile_arn
+clusters/create aws_attributes.zone_id
+clusters/create cluster_log_conf.s3.canned_acl
+clusters/create cluster_log_conf.s3.encryption_type
+clusters/create cluster_log_conf.s3.endpoint
+clusters/create cluster_log_conf.s3.kms_key
+clusters/create cluster_log_conf.s3.region
+clusters/create cluster_name
+clusters/create node_type_id
+clusters/create policy_id
+clusters/create single_user_name
+jobs/create budget_policy_id
+jobs/create description
+jobs/create job_clusters.new_cluster.aws_attributes.instance_profile_arn
+jobs/create job_clusters.new_cluster.aws_attributes.zone_id
+jobs/create job_clusters.new_cluster.cluster_log_conf.s3.canned_acl
+jobs/create job_clusters.new_cluster.cluster_log_conf.s3.encryption_type
+jobs/create job_clusters.new_cluster.cluster_log_conf.s3.endpoint
+jobs/create job_clusters.new_cluster.cluster_log_conf.s3.kms_key
+jobs/create job_clusters.new_cluster.cluster_log_conf.s3.region
+jobs/create job_clusters.new_cluster.cluster_name
+jobs/create job_clusters.new_cluster.node_type_id
+jobs/create job_clusters.new_cluster.policy_id
+jobs/create job_clusters.new_cluster.single_user_name
+jobs/create parent_path
+jobs/create usage_policy_id

acceptance/bundle/empty_string_dropped/out.empty_sent_by_both.txt

@@ -1,6 +1 @@
-/api/2.0/apps budget_policy_id
 /api/2.0/apps description
-/api/2.0/apps space
-/api/2.0/apps usage_policy_id
-/api/2.0/database/instances usage_policy_id
-/api/2.1/clusters/create spark_version

acceptance/bundle/empty_string_dropped/out.empty_sent_by_direct_only.txt

@@ -1,55 +0,0 @@
-/api/2.0/mlflow/experiments/create artifact_location
-/api/2.0/mlflow/registered-models/create description
-/api/2.0/pipelines budget_policy_id
-/api/2.0/pipelines clusters.aws_attributes.instance_profile_arn
-/api/2.0/pipelines clusters.aws_attributes.zone_id
-/api/2.0/pipelines clusters.driver_instance_pool_id
-/api/2.0/pipelines clusters.driver_node_type_id
-/api/2.0/pipelines clusters.instance_pool_id
-/api/2.0/pipelines clusters.node_type_id
-/api/2.0/pipelines clusters.policy_id
-/api/2.0/pipelines libraries.jar
-/api/2.0/pipelines libraries.whl
-/api/2.0/pipelines name
-/api/2.0/pipelines serverless_compute_id
-/api/2.0/pipelines usage_policy_id
-/api/2.0/sql/warehouses instance_profile_arn
-/api/2.1/clusters/create aws_attributes.instance_profile_arn
-/api/2.1/clusters/create aws_attributes.zone_id
-/api/2.1/clusters/create cluster_log_conf.s3.canned_acl
-/api/2.1/clusters/create cluster_log_conf.s3.encryption_type
-/api/2.1/clusters/create cluster_log_conf.s3.endpoint
-/api/2.1/clusters/create cluster_log_conf.s3.kms_key
-/api/2.1/clusters/create cluster_log_conf.s3.region
-/api/2.1/clusters/create cluster_name
-/api/2.1/clusters/create node_type_id
-/api/2.1/clusters/create policy_id
-/api/2.1/clusters/create single_user_name
-/api/2.1/unity-catalog/models comment
-/api/2.1/unity-catalog/models created_by
-/api/2.1/unity-catalog/models full_name
-/api/2.1/unity-catalog/models metastore_id
-/api/2.1/unity-catalog/models owner
-/api/2.1/unity-catalog/models storage_location
-/api/2.1/unity-catalog/models updated_by
-/api/2.1/unity-catalog/schemas comment
-/api/2.1/unity-catalog/schemas storage_root
-/api/2.1/unity-catalog/volumes comment
-/api/2.1/unity-catalog/volumes storage_location
-/api/2.2/jobs/create budget_policy_id
-/api/2.2/jobs/create description
-/api/2.2/jobs/create job_clusters.new_cluster.aws_attributes.instance_profile_arn
-/api/2.2/jobs/create job_clusters.new_cluster.aws_attributes.zone_id
-/api/2.2/jobs/create job_clusters.new_cluster.cluster_log_conf.s3.canned_acl
-/api/2.2/jobs/create job_clusters.new_cluster.cluster_log_conf.s3.encryption_type
-/api/2.2/jobs/create job_clusters.new_cluster.cluster_log_conf.s3.endpoint
-/api/2.2/jobs/create job_clusters.new_cluster.cluster_log_conf.s3.kms_key
-/api/2.2/jobs/create job_clusters.new_cluster.cluster_log_conf.s3.region
-/api/2.2/jobs/create job_clusters.new_cluster.cluster_name
-/api/2.2/jobs/create job_clusters.new_cluster.node_type_id
-/api/2.2/jobs/create job_clusters.new_cluster.policy_id
-/api/2.2/jobs/create job_clusters.new_cluster.single_user_name
-/api/2.2/jobs/create job_clusters.new_cluster.spark_version
-/api/2.2/jobs/create name
-/api/2.2/jobs/create parent_path
-/api/2.2/jobs/create usage_policy_id

acceptance/bundle/empty_string_dropped/out.requests.direct.json

@@ -5,57 +5,43 @@
     "no_compute": "true"
   },
   "body": {
-    "budget_policy_id": "",
     "description": "",
-    "name": "test-app-direct",
-    "space": "",
-    "usage_policy_id": ""
+    "name": "test-app-direct"
   }
 }
 {
   "method": "POST",
   "path": "/api/2.0/database/instances",
   "body": {
     "capacity": "CU_1",
-    "name": "test-db-instance",
-    "usage_policy_id": ""
+    "name": "test-db-instance"
   }
 }
 {
   "method": "POST",
   "path": "/api/2.0/mlflow/experiments/create",
   "body": {
-    "artifact_location": "",
     "name": "/Users/[USERNAME]/test-experiment"
   }
 }
 {
   "method": "POST",
   "path": "/api/2.0/mlflow/registered-models/create",
   "body": {
-    "description": "",
     "name": "test-model"
   }
 }
 {
   "method": "POST",
   "path": "/api/2.0/pipelines",
   "body": {
-    "budget_policy_id": "",
     "channel": "CURRENT",
     "clusters": [
       {
         "aws_attributes": {
-          "availability": "SPOT",
-          "instance_profile_arn": "",
-          "zone_id": ""
+          "availability": "SPOT"
         },
-        "driver_instance_pool_id": "",
-        "driver_node_type_id": "",
-        "instance_pool_id": "",
-        "label": "default",
-        "node_type_id": "",
-        "policy_id": ""
+        "label": "default"
       }
     ],
     "deployment": {
@@ -67,15 +53,10 @@
       {
         "file": {
           "path": "/Workspace/Users/[USERNAME]/.bundle/test-bundle/direct/files/pipeline.py"
-        },
-        "jar": "",
-        "whl": ""
+        }
       }
     ],
-    "name": "",
-    "root_path": "/Workspace/Users/[USERNAME]/.bundle/test-bundle/direct/files",
-    "serverless_compute_id": "",
-    "usage_policy_id": ""
+    "root_path": "/Workspace/Users/[USERNAME]/.bundle/test-bundle/direct/files"
   }
 }
 {
@@ -85,7 +66,6 @@
     "auto_stop_mins": 10,
     "cluster_size": "2X-Small",
     "enable_photon": true,
-    "instance_profile_arn": "",
     "max_num_clusters": 1,
     "min_num_clusters": 1,
     "name": "test-warehouse",
@@ -98,76 +78,52 @@
   "body": {
     "autotermination_minutes": 60,
     "aws_attributes": {
-      "availability": "SPOT",
-      "instance_profile_arn": "",
-      "zone_id": ""
+      "availability": "SPOT"
     },
     "cluster_log_conf": {
       "s3": {
-        "canned_acl": "",
-        "destination": "s3://test-bucket/logs",
-        "encryption_type": "",
-        "endpoint": "",
-        "kms_key": "",
-        "region": ""
+        "destination": "s3://test-bucket/logs"
       }
     },
-    "cluster_name": "",
-    "node_type_id": "",
     "num_workers": 1,
-    "policy_id": "",
-    "single_user_name": "",
-    "spark_version": ""
+    "spark_version": "13.3.x-scala2.12"
   }
 }
 {
   "method": "POST",
   "path": "/api/2.1/unity-catalog/models",
   "body": {
     "catalog_name": "main",
-    "comment": "",
-    "created_by": "",
-    "full_name": "",
-    "metastore_id": "",
     "name": "test-registered-model",
-    "owner": "",
-    "schema_name": "default",
-    "storage_location": "",
-    "updated_by": ""
+    "schema_name": "default"
   }
 }
 {
   "method": "POST",
   "path": "/api/2.1/unity-catalog/schemas",
   "body": {
     "catalog_name": "main",
-    "comment": "",
-    "name": "test-schema",
-    "storage_root": ""
+    "name": "test-schema"
   }
 }
 {
   "method": "POST",
   "path": "/api/2.1/unity-catalog/volumes",
   "body": {
     "catalog_name": "main",
-    "comment": "",
     "name": "test-volume",
     "schema_name": "default",
-    "storage_location": "",
     "volume_type": "MANAGED"
   }
 }
 {
   "method": "POST",
   "path": "/api/2.2/jobs/create",
   "body": {
-    "budget_policy_id": "",
     "deployment": {
       "kind": "BUNDLE",
       "metadata_file_path": "/Workspace/Users/[USERNAME]/.bundle/test-bundle/direct/state/metadata.json"
     },
-    "description": "",
     "edit_mode": "UI_LOCKED",
     "format": "MULTI_TASK",
     "git_source": {
@@ -180,35 +136,21 @@
         "job_cluster_key": "main",
         "new_cluster": {
           "aws_attributes": {
-            "availability": "SPOT",
-            "instance_profile_arn": "",
-            "zone_id": ""
+            "availability": "SPOT"
           },
           "cluster_log_conf": {
             "s3": {
-              "canned_acl": "",
-              "destination": "s3://test-bucket/logs",
-              "encryption_type": "",
-              "endpoint": "",
-              "kms_key": "",
-              "region": ""
+              "destination": "s3://test-bucket/logs"
             }
           },
-          "cluster_name": "",
-          "node_type_id": "",
           "num_workers": 1,
-          "policy_id": "",
-          "single_user_name": "",
-          "spark_version": ""
+          "spark_version": "13.3.x-scala2.12"
         }
       }
     ],
     "max_concurrent_runs": 1,
-    "name": "",
-    "parent_path": "",
     "queue": {
       "enabled": true
-    },
-    "usage_policy_id": ""
+    }
   }
 }