Skip to content

Spark 3.4 upgrade #3602

New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Closed
wants to merge 5 commits into from
Closed
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
Original file line number Diff line number Diff line change
@@ -32,7 +32,7 @@ jobs:

resources:
instance_type: Standard_E8S_V3
runtime_version: 3.3.0
runtime_version: 3.4.0

count_word:
type: spark
@@ -44,6 +44,6 @@ jobs:

resources:
instance_type: Standard_E8S_V3
runtime_version: 3.3.0
runtime_version: 3.4.0


Original file line number Diff line number Diff line change
@@ -36,7 +36,7 @@ jobs:

resources:
instance_type: Standard_E8S_V3
runtime_version: 3.3.0
runtime_version: 3.4.0

conf:
spark.driver.cores: 2
@@ -71,7 +71,7 @@ jobs:

resources:
instance_type: Standard_E8S_V3
runtime_version: 3.3.0
runtime_version: 3.4.0

conf:
spark.driver.cores: 2
Original file line number Diff line number Diff line change
@@ -22,4 +22,4 @@ jobs:

resources:
instance_type: standard_e8s_v3
runtime_version: "3.3"
runtime_version: "3.4"
Original file line number Diff line number Diff line change
@@ -25,4 +25,4 @@ jobs:

resources:
instance_type: standard_e8s_v3
runtime_version: "3.3"
runtime_version: "3.4"
Original file line number Diff line number Diff line change
@@ -25,4 +25,4 @@ jobs:

resources:
instance_type: standard_e8s_v3
runtime_version: "3.3"
runtime_version: "3.4"
Original file line number Diff line number Diff line change
@@ -31,5 +31,5 @@ args: >-

resources:
instance_type: standard_e4s_v3
runtime_version: "3.3"
runtime_version: "3.4"

Original file line number Diff line number Diff line change
@@ -34,5 +34,5 @@ identity:

resources:
instance_type: standard_e4s_v3
runtime_version: "3.3"
runtime_version: "3.4"

Original file line number Diff line number Diff line change
@@ -34,5 +34,5 @@ identity:

resources:
instance_type: standard_e4s_v3
runtime_version: "3.3"
runtime_version: "3.4"

2 changes: 1 addition & 1 deletion cli/monitoring/advanced-model-monitoring.yaml
Original file line number Diff line number Diff line change
@@ -17,7 +17,7 @@ create_monitor:

compute:
instance_type: standard_e4s_v3
runtime_version: "3.3"
runtime_version: "3.4"

monitoring_target:
ml_task: classification
Original file line number Diff line number Diff line change
@@ -759,7 +759,7 @@
},
{
"cell_type": "code",
"execution_count": 19,
"execution_count": null,
"metadata": {},
"outputs": [
{
@@ -794,7 +794,7 @@
"\n",
"# create the compute\n",
"spark_compute = ServerlessSparkCompute(\n",
" instance_type=\"standard_e4s_v3\", runtime_version=\"3.3\"\n",
" instance_type=\"standard_e4s_v3\", runtime_version=\"3.4\"\n",
")\n",
"\n",
"# specify your online endpoint deployment\n",
@@ -838,7 +838,7 @@
},
{
"cell_type": "code",
"execution_count": 13,
"execution_count": null,
"metadata": {},
"outputs": [
{
@@ -908,7 +908,7 @@
"\n",
"# create your compute\n",
"spark_compute = ServerlessSparkCompute(\n",
" instance_type=\"standard_e4s_v3\", runtime_version=\"3.3\"\n",
" instance_type=\"standard_e4s_v3\", runtime_version=\"3.4\"\n",
")\n",
"\n",
"# specify the online deployment (if you have one)\n",
2 changes: 1 addition & 1 deletion cli/monitoring/custom-monitoring.yaml
Original file line number Diff line number Diff line change
@@ -10,7 +10,7 @@ trigger:
create_monitor:
compute:
instance_type: "standard_e8s_v3"
runtime_version: "3.3"
runtime_version: "3.4"

monitoring_signals:
customSignal:
2 changes: 1 addition & 1 deletion cli/monitoring/generation-safety-quality-monitoring.yaml
Original file line number Diff line number Diff line change
@@ -22,7 +22,7 @@ trigger:
create_monitor:
compute:
instance_type: "standard_e8s_v3"
runtime_version: "3.3"
runtime_version: "3.4"
monitoring_target:
ml_task: classification
endpoint_deployment_id: fake_endpoint_id
2 changes: 1 addition & 1 deletion cli/monitoring/model-monitoring-with-collected-data.yaml
Original file line number Diff line number Diff line change
@@ -16,7 +16,7 @@ trigger:
create_monitor:
compute:
instance_type: standard_e4s_v3
runtime_version: "3.3"
runtime_version: "3.4"
monitoring_target:
ml_task: classification
endpoint_deployment_id: azureml:fraud-detection-endpoint:fraud-detection-deployment
2 changes: 1 addition & 1 deletion cli/monitoring/out-of-box-monitoring.yaml
Original file line number Diff line number Diff line change
@@ -17,7 +17,7 @@ create_monitor:

compute: # specify a spark compute for monitoring job
instance_type: standard_e4s_v3
runtime_version: "3.3"
runtime_version: "3.4"

monitoring_target:
ml_task: classification # model task type: [classification, regression, question_answering]
Original file line number Diff line number Diff line change
@@ -69,7 +69,7 @@
" executor_instances=1,\n",
" resources={\n",
" \"instance_type\": \"Standard_E8S_V3\",\n",
" \"runtime_version\": \"3.3.0\",\n",
" \"runtime_version\": \"3.4.0\",\n",
" },\n",
" conf={\"spark.synapse.library.python.env\": contents},\n",
")\n",
Original file line number Diff line number Diff line change
@@ -69,7 +69,7 @@
" executor_instances=1,\n",
" resources={\n",
" \"instance_type\": \"Standard_E8S_V3\",\n",
" \"runtime_version\": \"3.3.0\",\n",
" \"runtime_version\": \"3.4.0\",\n",
" },\n",
" conf={\"spark.synapse.library.python.env\": contents},\n",
")\n",
Original file line number Diff line number Diff line change
@@ -69,7 +69,7 @@
" executor_instances=1,\n",
" resources={\n",
" \"instance_type\": \"Standard_E8S_V3\",\n",
" \"runtime_version\": \"3.3.0\",\n",
" \"runtime_version\": \"3.4.0\",\n",
" },\n",
" conf={\"spark.synapse.library.python.env\": contents},\n",
")\n",
Original file line number Diff line number Diff line change
@@ -28,7 +28,7 @@ jobs:
observation_data_format: parquet
resources:
instance_type: standard_e4s_v3
runtime_version: "3.3"
runtime_version: "3.4"
outputs:
output_data:
conf:
Original file line number Diff line number Diff line change
@@ -28,7 +28,7 @@ jobs:
observation_data_format: parquet
resources:
instance_type: standard_e4s_v3
runtime_version: "3.3"
runtime_version: "3.4"
outputs:
output_data:
conf:
Original file line number Diff line number Diff line change
@@ -171,7 +171,7 @@
" kmeans_clustering = spark_kmeans(file_input=train_data)\n",
" kmeans_clustering.resources = {\n",
" \"instance_type\": \"Standard_E8S_V3\",\n",
" \"runtime_version\": \"3.3.0\",\n",
" \"runtime_version\": \"3.4.0\",\n",
" }\n",
" kmeans_clustering.outputs.output.mode = InputOutputModes.DIRECT\n",
"\n",
Original file line number Diff line number Diff line change
@@ -601,7 +601,7 @@
"|Property|Description|\n",
"|:-|:-|\n",
"| **instance_type** | A key that defines the compute instance type to be used for the serverless Spark compute. The following instance types are currently supported:<ul><li>`Standard_E4S_V3`</li><li>`Standard_E8S_V3`</li><li>`Standard_E16S_V3`</li><li>`Standard_E32S_V3`</li><li>`Standard_E64S_V3`</li></ul>|\n",
"| **runtime_version** | A key that defines the Spark runtime version. The following Spark runtime versions are currently supported:<ul><li>`3.3.0`</li><li>`3.4.0`</li></ul> |\n",
"| **runtime_version** | A key that defines the Spark runtime version. The following Spark runtime versions are currently supported:<ul><li>`3.4.0`</li></ul> |\n",
"| **driver_cores** | The he number of cores allocated for the Spark driver. |\n",
"| **driver_memory** | The allocated memory for the Spark exedriver, with a size unit suffix `k`, `m`, `g` or `t` (for example, `512m`, `2g`). |\n",
"| **executor_cores** | The number of cores allocated for the Spark executor. |\n",
@@ -639,7 +639,7 @@
"if USE_PARTITIONING_COMPONENT:\n",
" spark_parameters = dict(\n",
" instance_type=\"Standard_E4S_V3\",\n",
" runtime_version=\"3.3.0\",\n",
" runtime_version=\"3.4.0\",\n",
" driver_cores=1,\n",
" driver_memory=\"2g\",\n",
" executor_cores=2,\n",
@@ -704,7 +704,7 @@
"\n",
" partition_step.resources = {\n",
" \"instance_type\": spark_parameters.get(\"instance_type\", \"Standard_E4S_V3\"),\n",
" \"runtime_version\": str(spark_parameters.get(\"runtime_version\", \"3.3.0\")),\n",
" \"runtime_version\": str(spark_parameters.get(\"runtime_version\", \"3.4.0\")),\n",
" }\n",
" partition_step.conf = {\n",
" \"spark.driver.cores\": spark_parameters.get(\"driver_cores\", 1),\n",
Original file line number Diff line number Diff line change
@@ -45,7 +45,7 @@
" executor_instances=2,\n",
" resources={\n",
" \"instance_type\": \"Standard_E8S_V3\",\n",
" \"runtime_version\": \"3.3.0\",\n",
" \"runtime_version\": \"3.4.0\",\n",
" },\n",
")\n",
"\n",
6 changes: 3 additions & 3 deletions sdk/python/jobs/spark/submit_spark_pipeline_jobs.ipynb
Original file line number Diff line number Diff line change
@@ -342,7 +342,7 @@
" spark_step.identity = ManagedIdentityConfiguration()\n",
" spark_step.resources = {\n",
" \"instance_type\": \"Standard_E8S_V3\",\n",
" \"runtime_version\": \"3.3.0\",\n",
" \"runtime_version\": \"3.4.0\",\n",
" }\n",
"\n",
"\n",
@@ -422,7 +422,7 @@
" spark_step.identity = UserIdentityConfiguration()\n",
" spark_step.resources = {\n",
" \"instance_type\": \"Standard_E8S_V3\",\n",
" \"runtime_version\": \"3.3.0\",\n",
" \"runtime_version\": \"3.4.0\",\n",
" }\n",
"\n",
"\n",
@@ -501,7 +501,7 @@
" spark_step.outputs.wrangled_data.mode = InputOutputModes.DIRECT\n",
" spark_step.resources = {\n",
" \"instance_type\": \"Standard_E8S_V3\",\n",
" \"runtime_version\": \"3.3.0\",\n",
" \"runtime_version\": \"3.4.0\",\n",
" }\n",
"\n",
"\n",
6 changes: 3 additions & 3 deletions sdk/python/jobs/spark/submit_spark_standalone_jobs.ipynb
Original file line number Diff line number Diff line change
@@ -267,7 +267,7 @@
" executor_instances=2,\n",
" resources={\n",
" \"instance_type\": \"Standard_E8S_V3\",\n",
" \"runtime_version\": \"3.3.0\",\n",
" \"runtime_version\": \"3.4.0\",\n",
" },\n",
" inputs={\n",
" \"titanic_data\": Input(\n",
@@ -329,7 +329,7 @@
" executor_instances=2,\n",
" resources={\n",
" \"instance_type\": \"Standard_E8S_V3\",\n",
" \"runtime_version\": \"3.3.0\",\n",
" \"runtime_version\": \"3.4.0\",\n",
" },\n",
" inputs={\n",
" \"titanic_data\": Input(\n",
@@ -391,7 +391,7 @@
" executor_instances=2,\n",
" resources={\n",
" \"instance_type\": \"Standard_E8S_V3\",\n",
" \"runtime_version\": \"3.3.0\",\n",
" \"runtime_version\": \"3.4.0\",\n",
" },\n",
" inputs={\n",
" \"titanic_data\": Input(\n",
Original file line number Diff line number Diff line change
@@ -535,7 +535,7 @@
" executor_instances=2,\n",
" resources={\n",
" \"instance_type\": \"Standard_E8S_V3\",\n",
" \"runtime_version\": \"3.3.0\",\n",
" \"runtime_version\": \"3.4.0\",\n",
" },\n",
" inputs={\n",
" \"titanic_data\": Input(\n",
@@ -769,7 +769,7 @@
" executor_instances=2,\n",
" resources={\n",
" \"instance_type\": \"Standard_E8S_V3\",\n",
" \"runtime_version\": \"3.3.0\",\n",
" \"runtime_version\": \"3.4.0\",\n",
" },\n",
" inputs={\n",
" \"titanic_data\": Input(\n",
Loading
Oops, something went wrong.