From 1ec0ac5207c8af4bebed12fbf9e4c53c6dd22043 Mon Sep 17 00:00:00 2001 From: Li Jiang Date: Tue, 13 Sep 2022 19:40:19 +0800 Subject: [PATCH 1/6] chore: added trident platform --- .../src/main/python/synapse/ml/core/platform/Platform.py | 9 ++++++++- 1 file changed, 8 insertions(+), 1 deletion(-) diff --git a/core/src/main/python/synapse/ml/core/platform/Platform.py b/core/src/main/python/synapse/ml/core/platform/Platform.py index 2636679d73..7efd357bd6 100644 --- a/core/src/main/python/synapse/ml/core/platform/Platform.py +++ b/core/src/main/python/synapse/ml/core/platform/Platform.py @@ -3,6 +3,7 @@ import os +PLATFORM_TRIDENT = "trident" PLATFORM_SYNAPSE = "synapse" PLATFORM_BINDER = "binder" PLATFORM_DATABRICKS = "databricks" @@ -12,7 +13,9 @@ def current_platform(): - if os.environ.get("AZURE_SERVICE", None) == SYNAPSE_PROJECT_NAME: + if "lakehouse" in os.listdir("/"): + return PLATFORM_TRIDENT + elif os.environ.get("AZURE_SERVICE", None) == SYNAPSE_PROJECT_NAME: return PLATFORM_SYNAPSE elif "dbfs" in os.listdir("/"): return PLATFORM_DATABRICKS @@ -22,6 +25,10 @@ def current_platform(): return PLATFORM_UNKNOWN +def running_on_trident(): + return current_platform() is PLATFORM_TRIDENT + + def running_on_synapse(): return current_platform() is PLATFORM_SYNAPSE From b28a3052eae0f5999e50e929b9abf95cc28e9e95 Mon Sep 17 00:00:00 2001 From: Li Jiang Date: Tue, 13 Sep 2022 20:03:45 +0800 Subject: [PATCH 2/6] chore: update naming --- core/src/main/python/synapse/ml/core/platform/Platform.py | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/core/src/main/python/synapse/ml/core/platform/Platform.py b/core/src/main/python/synapse/ml/core/platform/Platform.py index 7efd357bd6..656c831f5b 100644 --- a/core/src/main/python/synapse/ml/core/platform/Platform.py +++ b/core/src/main/python/synapse/ml/core/platform/Platform.py @@ -3,7 +3,7 @@ import os -PLATFORM_TRIDENT = "trident" +PLATFORM_SYNAPSE_INTERNAL = "trident" PLATFORM_SYNAPSE = "synapse" PLATFORM_BINDER = "binder" PLATFORM_DATABRICKS = "databricks" @@ -14,7 +14,7 @@ def current_platform(): if "lakehouse" in os.listdir("/"): - return PLATFORM_TRIDENT + return PLATFORM_SYNAPSE_INTERNAL elif os.environ.get("AZURE_SERVICE", None) == SYNAPSE_PROJECT_NAME: return PLATFORM_SYNAPSE elif "dbfs" in os.listdir("/"): @@ -25,8 +25,8 @@ def current_platform(): return PLATFORM_UNKNOWN -def running_on_trident(): - return current_platform() is PLATFORM_TRIDENT +def running_on_synapse_internal(): + return current_platform() is PLATFORM_SYNAPSE_INTERNAL def running_on_synapse(): From 2e5c0b52bbc3091900747ec73977049f083dfa89 Mon Sep 17 00:00:00 2001 From: Mark Hamilton Date: Wed, 14 Sep 2022 13:07:31 -0400 Subject: [PATCH 3/6] Update core/src/main/python/synapse/ml/core/platform/Platform.py --- core/src/main/python/synapse/ml/core/platform/Platform.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/core/src/main/python/synapse/ml/core/platform/Platform.py b/core/src/main/python/synapse/ml/core/platform/Platform.py index 656c831f5b..6fb4fe4870 100644 --- a/core/src/main/python/synapse/ml/core/platform/Platform.py +++ b/core/src/main/python/synapse/ml/core/platform/Platform.py @@ -3,7 +3,7 @@ import os -PLATFORM_SYNAPSE_INTERNAL = "trident" +PLATFORM_SYNAPSE_INTERNAL = "synapse_internal" PLATFORM_SYNAPSE = "synapse" PLATFORM_BINDER = "binder" PLATFORM_DATABRICKS = "databricks" From a134287de19d7bf54bf0b08aa1dd9717b2b151fd Mon Sep 17 00:00:00 2001 From: Li Jiang Date: Thu, 15 Sep 2022 03:22:27 +0000 Subject: [PATCH 4/6] chore: update detection rules --- .../python/synapse/ml/core/platform/Platform.py | 14 ++++++++++---- 1 file changed, 10 insertions(+), 4 deletions(-) diff --git a/core/src/main/python/synapse/ml/core/platform/Platform.py b/core/src/main/python/synapse/ml/core/platform/Platform.py index 6fb4fe4870..9820dee3c9 100644 --- a/core/src/main/python/synapse/ml/core/platform/Platform.py +++ b/core/src/main/python/synapse/ml/core/platform/Platform.py @@ -13,10 +13,16 @@ def current_platform(): - if "lakehouse" in os.listdir("/"): - return PLATFORM_SYNAPSE_INTERNAL - elif os.environ.get("AZURE_SERVICE", None) == SYNAPSE_PROJECT_NAME: - return PLATFORM_SYNAPSE + if os.environ.get("AZURE_SERVICE", None) == SYNAPSE_PROJECT_NAME: + from pyspark.sql import SparkSession + sc = SparkSession.builder.getOrCreate().sparkContext + cluster_type = sc.getConf().get("spark.cluster.type") + if cluster_type == "synapse": + return PLATFORM_SYNAPSE + elif cluster_type == "trident": + return PLATFORM_SYNAPSE_INTERNAL + else: + return PLATFORM_UNKNOWN elif "dbfs" in os.listdir("/"): return PLATFORM_DATABRICKS elif os.environ.get("BINDER_LAUNCH_HOST", None) is not None: From 2542a32b4033dbc65d7668d5e4c59a4bbd66646f Mon Sep 17 00:00:00 2001 From: Li Jiang Date: Thu, 15 Sep 2022 04:15:48 +0000 Subject: [PATCH 5/6] chore: remove redundant check --- core/src/main/python/synapse/ml/core/platform/Platform.py | 4 +--- 1 file changed, 1 insertion(+), 3 deletions(-) diff --git a/core/src/main/python/synapse/ml/core/platform/Platform.py b/core/src/main/python/synapse/ml/core/platform/Platform.py index 9820dee3c9..ad8c865f12 100644 --- a/core/src/main/python/synapse/ml/core/platform/Platform.py +++ b/core/src/main/python/synapse/ml/core/platform/Platform.py @@ -19,10 +19,8 @@ def current_platform(): cluster_type = sc.getConf().get("spark.cluster.type") if cluster_type == "synapse": return PLATFORM_SYNAPSE - elif cluster_type == "trident": - return PLATFORM_SYNAPSE_INTERNAL else: - return PLATFORM_UNKNOWN + return PLATFORM_SYNAPSE_INTERNAL elif "dbfs" in os.listdir("/"): return PLATFORM_DATABRICKS elif os.environ.get("BINDER_LAUNCH_HOST", None) is not None: From 6782806d23a04200bb2f3afd0bdb30f78ac92cf4 Mon Sep 17 00:00:00 2001 From: Li Jiang Date: Fri, 16 Sep 2022 02:18:14 +0000 Subject: [PATCH 6/6] docs: apply black formatter --- core/src/main/python/synapse/ml/core/platform/Platform.py | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/core/src/main/python/synapse/ml/core/platform/Platform.py b/core/src/main/python/synapse/ml/core/platform/Platform.py index ad8c865f12..48a2f948d2 100644 --- a/core/src/main/python/synapse/ml/core/platform/Platform.py +++ b/core/src/main/python/synapse/ml/core/platform/Platform.py @@ -13,8 +13,9 @@ def current_platform(): - if os.environ.get("AZURE_SERVICE", None) == SYNAPSE_PROJECT_NAME: + if os.environ.get("AZURE_SERVICE", None) == SYNAPSE_PROJECT_NAME: from pyspark.sql import SparkSession + sc = SparkSession.builder.getOrCreate().sparkContext cluster_type = sc.getConf().get("spark.cluster.type") if cluster_type == "synapse":