From 04f3a938895ccc5b142671de0e927bed89b5d8c0 Mon Sep 17 00:00:00 2001 From: Dongjoon Hyun Date: Wed, 1 May 2024 17:50:33 -0700 Subject: [PATCH] [SPARK-48076][K8S] Promote `KubernetesVolumeUtils` to `DeveloperApi` ### What changes were proposed in this pull request? This PR aims to promote `KubernetesVolumeUtils` to `DeveloperApi` from Apache Spark 4.0.0 for Apache Spark Kubernetes Operator. ### Why are the changes needed? This API was added by the following at `Apache Spark 3.0.0` and has been stable. - #22959 Since `Apache Spark Kubernetes Operator` requires this, we had better maintain it as a developer API officially from `Apache Spark 4.0.0`. - https://github.com/apache/spark-kubernetes-operator/pull/10 ### Does this PR introduce _any_ user-facing change? No. ### How was this patch tested? Pass the CIs. ### Was this patch authored or co-authored using generative AI tooling? No. Closes #46326 from dongjoon-hyun/SPARK-48076. Authored-by: Dongjoon Hyun Signed-off-by: Dongjoon Hyun --- .../spark/deploy/k8s/KubernetesVolumeUtils.scala | 11 ++++++++++- 1 file changed, 10 insertions(+), 1 deletion(-) diff --git a/resource-managers/kubernetes/core/src/main/scala/org/apache/spark/deploy/k8s/KubernetesVolumeUtils.scala b/resource-managers/kubernetes/core/src/main/scala/org/apache/spark/deploy/k8s/KubernetesVolumeUtils.scala index baa519658c2e1..ee2108e8234d3 100644 --- a/resource-managers/kubernetes/core/src/main/scala/org/apache/spark/deploy/k8s/KubernetesVolumeUtils.scala +++ b/resource-managers/kubernetes/core/src/main/scala/org/apache/spark/deploy/k8s/KubernetesVolumeUtils.scala @@ -19,9 +19,17 @@ package org.apache.spark.deploy.k8s import java.lang.Long.parseLong import org.apache.spark.SparkConf +import org.apache.spark.annotation.{DeveloperApi, Since, Unstable} import org.apache.spark.deploy.k8s.Config._ -private[spark] object KubernetesVolumeUtils { +/** + * :: DeveloperApi :: + * + * A utility class used for K8s operations internally and Spark K8s operator. + */ +@Unstable +@DeveloperApi +object KubernetesVolumeUtils { /** * Extract Spark volume configuration properties with a given name prefix. * @@ -29,6 +37,7 @@ private[spark] object KubernetesVolumeUtils { * @param prefix the given property name prefix * @return a Map storing with volume name as key and spec as value */ + @Since("3.0.0") def parseVolumesWithPrefix(sparkConf: SparkConf, prefix: String): Seq[KubernetesVolumeSpec] = { val properties = sparkConf.getAllWithPrefix(prefix).toMap