Skip to content

Commit

Permalink
Exporter: Remove dead code that reflects old Jobs API 2.0 structures (#…
Browse files Browse the repository at this point in the history 
…2981)

As we already switched to API 2.1, the code that is compatible with Jobs API 2.0 isn't
necessary anymore and could be removed.

Also fixed a number of tests where JSON files weren't read correctly, or JSON structures
were not completely correct.
  • Loading branch information
@alexott
alexott committed Dec 11, 2023
1 parent 58fdc3b commit 1ff9e88
Show file tree
Hide file tree
Showing 11 changed files with 168 additions and 160 deletions.
39 changes: 23 additions & 16 deletions exporter/exporter_test.go
View file
Expand Up @@ -38,22 +38,28 @@ import (

// nolint
func getJSONObject(filename string) any {
data, _ := os.ReadFile(filename)
var obj map[string]any
err := json.Unmarshal(data, &obj)
data, err := os.ReadFile(filename)
if err != nil {
fmt.Printf("[ERROR] error! err=%v\n", err)
panic(err)
}
err = json.Unmarshal(data, &obj)
if err != nil {
fmt.Printf("[ERROR] error! file=%s err=%v\n", filename, err)
fmt.Printf("[ERROR] data=%s\n", string(data))
}
return obj
}

func getJSONArray(filename string) any {
data, _ := os.ReadFile(filename)
data, err := os.ReadFile(filename)
if err != nil {
panic(err)
}
var obj []any
err := json.Unmarshal(data, &obj)
err = json.Unmarshal(data, &obj)
if err != nil {
fmt.Printf("[ERROR] error! err=%v\n", err)
fmt.Printf("[ERROR] error! file=%s err=%v\n", filename, err)
fmt.Printf("[ERROR] data=%s\n", string(data))
}
return obj
Expand Down Expand Up @@ -336,7 +342,7 @@ var emptySqlQueries = qa.HTTPFixture{
var emptySqlAlerts = qa.HTTPFixture{
Method: "GET",
Resource: "/api/2.0/preview/sql/alerts",
Response: map[string]any{},
Response: []sql.AlertEntity{},
ReuseRequest: true,
}

Expand Down Expand Up @@ -840,7 +846,7 @@ func TestImportingClusters(t *testing.T) {
Method: "GET",
Resource: "/api/2.0/permissions/instance-pools/pool1",
ReuseRequest: true,
Response: getJSONObject("test-data/get-job-14-permissions.json"),
Response: getJSONObject("test-data/get-job-permissions-14.json"),
},
{
Method: "GET",
Expand Down Expand Up @@ -925,7 +931,7 @@ func TestImportingJobs_JobList(t *testing.T) {
{
Method: "GET",
Resource: "/api/2.0/permissions/jobs/14",
Response: getJSONObject("test-data/get-job-14-permissions.json"),
Response: getJSONObject("test-data/get-job-permissions-14.json"),
},
{
Method: "GET",
Expand All @@ -949,7 +955,7 @@ func TestImportingJobs_JobList(t *testing.T) {
Method: "GET",
Resource: "/api/2.0/permissions/instance-pools/pool1",
ReuseRequest: true,
Response: getJSONObject("test-data/get-job-14-permissions.json"),
Response: getJSONObject("test-data/get-job-permissions-14.json"),
},
{
Method: "GET",
Expand Down Expand Up @@ -1048,7 +1054,7 @@ func TestImportingJobs_JobList(t *testing.T) {
Method: "GET",
Resource: "/api/2.0/permissions/instance-pools/pool1",
ReuseRequest: true,
Response: getJSONObject("test-data/get-job-14-permissions.json"),
Response: getJSONObject("test-data/get-job-permissions-14.json"),
},
{
Method: "GET",
Expand Down Expand Up @@ -1133,9 +1139,10 @@ func TestImportingJobs_JobListMultiTask(t *testing.T) {
},
},
{
Method: "GET",
Resource: "/api/2.0/permissions/jobs/14",
Response: getJSONObject("test-data/get-job-14-permissions.json"),
Method: "GET",
Resource: "/api/2.0/permissions/jobs/14",
Response: getJSONObject("test-data/get-job-permissions-14.json"),
ReuseRequest: true,
},
{
Method: "GET",
Expand All @@ -1159,7 +1166,7 @@ func TestImportingJobs_JobListMultiTask(t *testing.T) {
Method: "GET",
Resource: "/api/2.0/permissions/instance-pools/pool1",
ReuseRequest: true,
Response: getJSONObject("test-data/get-job-14-permissions.json"),
Response: getJSONObject("test-data/get-job-permissions-14.json"),
},
{
Method: "GET",
Expand Down Expand Up @@ -1298,7 +1305,7 @@ func TestImportingJobs_JobListMultiTask(t *testing.T) {
Method: "GET",
Resource: "/api/2.0/permissions/instance-pools/pool1",
ReuseRequest: true,
Response: getJSONObject("test-data/get-job-14-permissions.json"),
Response: getJSONObject("test-data/get-job-permissions-14.json"),
},
{
Method: "GET",
Expand Down
55 changes: 0 additions & 55 deletions exporter/importables.go
View file
Expand Up @@ -356,22 +356,6 @@ var resourcesMap map[string]importable = map[string]importable{
{Path: "email_notifications.on_start", Resource: "databricks_user", Match: "user_name", MatchType: MatchCaseInsensitive},
{Path: "email_notifications.on_duration_warning_threshold_exceeded", Resource: "databricks_user",
Match: "user_name", MatchType: MatchCaseInsensitive},
{Path: "new_cluster.aws_attributes.instance_profile_arn", Resource: "databricks_instance_profile"},
{Path: "new_cluster.init_scripts.dbfs.destination", Resource: "databricks_dbfs_file", Match: "dbfs_path"},
{Path: "new_cluster.init_scripts.workspace.destination", Resource: "databricks_workspace_file"},
{Path: "new_cluster.instance_pool_id", Resource: "databricks_instance_pool"},
{Path: "new_cluster.driver_instance_pool_id", Resource: "databricks_instance_pool"},
{Path: "new_cluster.policy_id", Resource: "databricks_cluster_policy"},
{Path: "existing_cluster_id", Resource: "databricks_cluster"},
{Path: "library.jar", Resource: "databricks_dbfs_file", Match: "dbfs_path"},
{Path: "library.whl", Resource: "databricks_dbfs_file", Match: "dbfs_path"},
{Path: "library.egg", Resource: "databricks_dbfs_file", Match: "dbfs_path"},
{Path: "spark_python_task.python_file", Resource: "databricks_dbfs_file", Match: "dbfs_path"},
{Path: "spark_python_task.parameters", Resource: "databricks_dbfs_file", Match: "dbfs_path"},
{Path: "spark_jar_task.jar_uri", Resource: "databricks_dbfs_file", Match: "dbfs_path"},
{Path: "notebook_task.notebook_path", Resource: "databricks_notebook"},
{Path: "notebook_task.notebook_path", Resource: "databricks_repo", Match: "path", MatchType: MatchPrefix},
{Path: "pipeline_task.pipeline_id", Resource: "databricks_pipeline"},
{Path: "task.library.jar", Resource: "databricks_dbfs_file", Match: "dbfs_path"},
{Path: "task.library.whl", Resource: "databricks_dbfs_file", Match: "dbfs_path"},
{Path: "task.library.egg", Resource: "databricks_dbfs_file", Match: "dbfs_path"},
Expand Down Expand Up @@ -419,45 +403,6 @@ var resourcesMap map[string]importable = map[string]importable{
Name: "job_" + ic.Importables["databricks_job"].Name(ic, r.Data),
})
}
if job.SparkPythonTask != nil {
ic.emitIfDbfsFile(job.SparkPythonTask.PythonFile)
for _, p := range job.SparkPythonTask.Parameters {
ic.emitIfDbfsFile(p)
}
}
if job.SparkJarTask != nil {
jarURI := job.SparkJarTask.JarURI
if jarURI != "" {
if libs, ok := r.Data.Get("library").(*schema.Set); ok {
// nolint remove legacy jar uri support
r.Data.Set("spark_jar_task", []any{
map[string]any{
"main_class_name": job.SparkJarTask.MainClassName,
"parameters": job.SparkJarTask.Parameters,
},
})
// if variable doesn't contain a sad face, it's a job jar
if !strings.Contains(jarURI, ":/") {
jarURI = fmt.Sprintf("dbfs:/FileStore/job-jars/%s", jarURI)
}
ic.emitIfDbfsFile(jarURI)
libs.Add(map[string]any{
"jar": jarURI,
})
// nolint
r.Data.Set("library", libs)
}
}
}
if job.NotebookTask != nil {
ic.emitNotebookOrRepo(job.NotebookTask.NotebookPath)
}
if job.PipelineTask != nil {
ic.Emit(&resource{
Resource: "databricks_pipeline",
ID: job.PipelineTask.PipelineID,
})
}
// Support for multitask jobs
for _, task := range job.Tasks {
if task.NotebookTask != nil {
Expand Down
27 changes: 27 additions & 0 deletions exporter/test-data/get-cluster-permissions-test2-response.json
View file
@@ -0,0 +1,27 @@
{
"access_control_list": [
{
"all_permissions": [
{
"inherited": false,
"permission_level": "CAN_MANAGE"
}
],
"user_name": "test@test.com"
},
{
"all_permissions": [
{
"inherited": true,
"inherited_from_object": [
"/clusters/"
],
"permission_level": "CAN_MANAGE"
}
],
"group_name": "admins"
}
],
"object_id": "/clusters/test2",
"object_type": "cluster"
}
13 changes: 9 additions & 4 deletions exporter/test-data/get-job-12.json
View file
Expand Up @@ -4,12 +4,17 @@
"job_id": 12,
"settings": {
"email_notifications": {},
"existing_cluster_id": "test2",
"max_concurrent_runs": 1,
"name": "Test",
"notebook_task": {
"notebook_path": "/Users/test@test.com/Test"
},
"tasks": [
{
"task_key": "test",
"existing_cluster_id": "test2",
"notebook_task": {
"notebook_path": "/Users/test@test.com/Test"
}
}
],
"timeout_seconds": 0
}
}
59 changes: 32 additions & 27 deletions exporter/test-data/get-job-14.json
View file
Expand Up @@ -8,35 +8,40 @@
"test@test.com"
]
},
"libraries": [
{
"maven": {
"coordinates": "com.microsoft.azure:azure-eventhubs-spark_2.12:2.3.17"
}
},
{
"pypi": {
"package": "spacy"
}
}
],
"max_concurrent_runs": 1,
"name": "Demo job",
"new_cluster": {
"azure_attributes": {
"availability": "ON_DEMAND_AZURE"
},
"enable_elastic_disk": true,
"node_type_id": "Standard_DS3_v2",
"num_workers": 2,
"spark_conf": {
"spark.databricks.delta.preview.enabled": "true"
},
"spark_version": "7.3.x-scala2.12"
},
"notebook_task": {
"notebook_path": "/Production/MakeFeatures"
},
"tasks": [
{
"task_key": "test",
"libraries": [
{
"maven": {
"coordinates": "com.microsoft.azure:azure-eventhubs-spark_2.12:2.3.17"
}
},
{
"pypi": {
"package": "spacy"
}
}
],
"new_cluster": {
"azure_attributes": {
"availability": "ON_DEMAND_AZURE"
},
"enable_elastic_disk": true,
"node_type_id": "Standard_DS3_v2",
"num_workers": 2,
"spark_conf": {
"spark.databricks.delta.preview.enabled": "true"
},
"spark_version": "7.3.x-scala2.12"
},
"notebook_task": {
"notebook_path": "/Production/MakeFeatures"
}
}
],
"schedule": {
"pause_status": "UNPAUSED",
"quartz_cron_expression": "0 15 22 ? * *",
Expand Down
35 changes: 20 additions & 15 deletions exporter/test-data/get-job-15.json
View file
Expand Up @@ -4,26 +4,31 @@
"job_id": 15,
"settings": {
"email_notifications": {},
"libraries": [
{
"jar": "dbfs:/FileStore/jars/test.jar"
}
],
"max_concurrent_runs": 1,
"max_retries": 1,
"min_retry_interval_millis": 0,
"name": "Scala Project",
"new_cluster": {
"instance_pool_id": "pool1",
"num_workers": 2,
"spark_version": "6.4.x-scala2.11"
},
"retry_on_timeout": false,
"spark_jar_task": {
"jar_uri": "dbfs:/FileStore/jars/test.jar",
"main_class_name": "com.databricks.examples.ProjectDriver",
"run_as_repl": true
},
"tasks": [
{
"task_key": "test",
"libraries": [
{
"jar": "dbfs:/FileStore/jars/test.jar"
}
],
"new_cluster": {
"instance_pool_id": "pool1",
"num_workers": 2,
"spark_version": "6.4.x-scala2.11"
},
"spark_jar_task": {
"jar_uri": "dbfs:/FileStore/jars/test.jar",
"main_class_name": "com.databricks.examples.ProjectDriver",
"run_as_repl": true
}
}
],
"timeout_seconds": 3600
}
}
33 changes: 19 additions & 14 deletions exporter/test-data/get-job-16.json
View file
Expand Up @@ -4,25 +4,30 @@
"job_id": 16,
"settings": {
"email_notifications": {},
"libraries": [
{
"jar": "dbfs:/FileStore/jars/test.jar"
}
],
"max_concurrent_runs": 1,
"max_retries": 1,
"min_retry_interval_millis": 0,
"name": "Python Project",
"new_cluster": {
"instance_pool_id": "pool1",
"num_workers": 2,
"spark_version": "6.4.x-scala2.11"
},
"tasks": [
{
"task_key": "test",
"libraries": [
{
"jar": "dbfs:/FileStore/jars/test.jar"
}
],
"new_cluster": {
"instance_pool_id": "pool1",
"num_workers": 2,
"spark_version": "6.4.x-scala2.11"
},
"spark_python_task": {
"python_file": "dbfs:/FileStore/jars/test.jar",
"parameters": ["param1"]
},
}
],
"retry_on_timeout": false,
"spark_python_task": {
"python_file": "dbfs:/FileStore/jars/test.jar",
"parameters": ["param1"]
},
"timeout_seconds": 3600
}
}

0 comments on commit 1ff9e88

Please sign in to comment.