From e35c1e9700a9e07bc72a3ac35dbf7a8466f8c733 Mon Sep 17 00:00:00 2001 From: Scott Romney Date: Fri, 17 Nov 2023 12:00:29 -0800 Subject: [PATCH 1/2] [CB-2] Fix bug in finding cluster in run --- sync/_databricks.py | 10 +++++----- 1 file changed, 5 insertions(+), 5 deletions(-) diff --git a/sync/_databricks.py b/sync/_databricks.py index 2332f50..de20d6e 100644 --- a/sync/_databricks.py +++ b/sync/_databricks.py @@ -1611,8 +1611,8 @@ def _get_cluster_tasks( job_clusters = {c["job_cluster_key"]: c["new_cluster"] for c in run.get("job_clusters", [])} cluster_id_tasks = defaultdict(list) - cluster_path_ids = defaultdict(list) - cluster_project_paths = defaultdict(list) + cluster_path_ids = defaultdict(set) + cluster_project_paths = defaultdict(set) for task in run["tasks"]: if "cluster_instance" in task and ( @@ -1630,15 +1630,15 @@ def _get_cluster_tasks( if task_cluster: cluster_project_id = task_cluster.get("custom_tags", {}).get("sync:project-id") cluster_id_tasks[cluster_id].append(task) - cluster_path_ids[cluster_path].append(cluster_id) - cluster_project_paths[cluster_project_id].append(cluster_path) + cluster_path_ids[cluster_path].add(cluster_id) + cluster_project_paths[cluster_project_id].add(cluster_path) result_cluster_project_tasks = {} for project_id, cluster_paths in cluster_project_paths.items(): cluster_path_tasks = {} for cluster_path in cluster_paths: if len(cluster_path_ids[cluster_path]) == 1: - cluster_id = cluster_path_ids[cluster_path][0] + cluster_id = cluster_path_ids[cluster_path].pop() cluster_path_tasks[cluster_path] = (cluster_id, cluster_id_tasks[cluster_id]) else: # Maybe this will happen if the same job cluster is used by 2 non adjacent tasks From 3a369d11a1c3a820a0c31bcf9fa066e0d311f31d Mon Sep 17 00:00:00 2001 From: Scott Romney Date: Fri, 17 Nov 2023 12:05:13 -0800 Subject: [PATCH 2/2] [CB-2] Bump version --- sync/__init__.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/sync/__init__.py b/sync/__init__.py index fdab5a5..6920b47 100644 --- a/sync/__init__.py +++ b/sync/__init__.py @@ -1,4 +1,4 @@ """Library for leveraging the power of Sync""" -__version__ = "0.4.12" +__version__ = "0.4.13" TIME_FORMAT = "%Y-%m-%dT%H:%M:%SZ"