From 1fd826b1956478abf5402969f1dad7a49ef94d8c Mon Sep 17 00:00:00 2001 From: Thomas Graves Date: Tue, 8 Sep 2020 13:18:40 -0500 Subject: [PATCH 1/2] [SPARK-32824] Improve the error message when the user forgets the .amount after a resource like spark.executor.resource.gpu --- .../main/scala/org/apache/spark/resource/ResourceUtils.scala | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/core/src/main/scala/org/apache/spark/resource/ResourceUtils.scala b/core/src/main/scala/org/apache/spark/resource/ResourceUtils.scala index 162f090f011c6..bf6093d963947 100644 --- a/core/src/main/scala/org/apache/spark/resource/ResourceUtils.scala +++ b/core/src/main/scala/org/apache/spark/resource/ResourceUtils.scala @@ -149,6 +149,10 @@ private[spark] object ResourceUtils extends Logging { def listResourceIds(sparkConf: SparkConf, componentName: String): Seq[ResourceID] = { sparkConf.getAllWithPrefix(s"$componentName.$RESOURCE_PREFIX.").map { case (key, _) => + if (key.indexOf('.') < 0) { + throw new SparkException(s"You must specify an amount config for resource: $key " + + s"config: $componentName.$RESOURCE_PREFIX.$key") + } key.substring(0, key.indexOf('.')) }.distinct.map(name => new ResourceID(componentName, name)) } From db665a1e3b71a50953cc6252726205cd54b24f0e Mon Sep 17 00:00:00 2001 From: Thomas Graves Date: Tue, 8 Sep 2020 13:36:15 -0500 Subject: [PATCH 2/2] Change to reuse index --- .../main/scala/org/apache/spark/resource/ResourceUtils.scala | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) diff --git a/core/src/main/scala/org/apache/spark/resource/ResourceUtils.scala b/core/src/main/scala/org/apache/spark/resource/ResourceUtils.scala index bf6093d963947..5a9435653920f 100644 --- a/core/src/main/scala/org/apache/spark/resource/ResourceUtils.scala +++ b/core/src/main/scala/org/apache/spark/resource/ResourceUtils.scala @@ -149,11 +149,12 @@ private[spark] object ResourceUtils extends Logging { def listResourceIds(sparkConf: SparkConf, componentName: String): Seq[ResourceID] = { sparkConf.getAllWithPrefix(s"$componentName.$RESOURCE_PREFIX.").map { case (key, _) => - if (key.indexOf('.') < 0) { + val index = key.indexOf('.') + if (index < 0) { throw new SparkException(s"You must specify an amount config for resource: $key " + s"config: $componentName.$RESOURCE_PREFIX.$key") } - key.substring(0, key.indexOf('.')) + key.substring(0, index) }.distinct.map(name => new ResourceID(componentName, name)) }