feat: use spetlr-tools for intgration test

spetlr-org · May 12, 2023 · 010aefc · 010aefc
1 parent 9462a43
commit 010aefc
Show file tree

Hide file tree

Showing 6 changed files with 71 additions and 327 deletions.
diff --git a/.github/submit/cluster104.json b/.github/submit/cluster104.json
@@ -0,0 +1,27 @@
+{
+    "spark_version": "10.4.x-scala2.12",
+    "spark_conf": {
+      "spark.databricks.cluster.profile": "singleNode",
+      "spark.master": "local[*, 4]",
+      "spark.databricks.delta.preview.enabled": true,
+      "spark.databricks.io.cache.enabled":true
+    },
+    "azure_attributes": {
+      "availability": "ON_DEMAND_AZURE",
+      "first_on_demand": 1,
+      "spot_bid_max_price": -1
+    },
+    "node_type_id": "Standard_DS3_v2",
+    "custom_tags": {
+      "ResourceClass":"SingleNode"
+    },
+    "spark_env_vars": {
+      "PYSPARK_PYTHON": "/databricks/python3/bin/python3"
+    },
+    "cluster_log_conf": {
+      "dbfs": {
+        "destination": ""
+      }
+    },
+    "num_workers": 0
+  }
diff --git a/.github/submit/cluster91.json b/.github/submit/cluster91.json
@@ -0,0 +1,27 @@
+{
+    "spark_version": "9.1.x-scala2.12",
+    "spark_conf": {
+      "spark.databricks.cluster.profile": "singleNode",
+      "spark.master": "local[*, 4]",
+      "spark.databricks.delta.preview.enabled": true,
+      "spark.databricks.io.cache.enabled":true
+    },
+    "azure_attributes": {
+      "availability": "ON_DEMAND_AZURE",
+      "first_on_demand": 1,
+      "spot_bid_max_price": -1
+    },
+    "node_type_id": "Standard_DS3_v2",
+    "custom_tags": {
+      "ResourceClass":"SingleNode"
+    },
+    "spark_env_vars": {
+      "PYSPARK_PYTHON": "/databricks/python3/bin/python3"
+    },
+    "cluster_log_conf": {
+      "dbfs": {
+        "destination": ""
+      }
+    },
+    "num_workers": 0
+  }
diff --git a/.github/submit/fetch_test_job.ps1 b/.github/submit/fetch_test_job.ps1
diff --git a/.github/submit/main.py b/.github/submit/main.py
diff --git a/.github/submit/submit_test_job.ps1 b/.github/submit/submit_test_job.ps1