From e8634d8f6f8548852a284a32c1b7da24bedd8ff7 Mon Sep 17 00:00:00 2001 From: Thomas Graves Date: Wed, 9 Sep 2020 10:28:40 +0900 Subject: [PATCH] [SPARK-32824][CORE] Improve the error message when the user forgets the .amount in a resource config ### What changes were proposed in this pull request? If the user forgets to specify .amount on a resource config like spark.executor.resource.gpu, the error message thrown is very confusing: ``` ERROR SparkContext: Error initializing SparkContext.java.lang.StringIndexOutOfBoundsException: String index out of range: -1 at java.lang.String.substring(String.java:1967) at ``` This makes it so we have a readable error thrown ### Why are the changes needed? confusing error for users ### Does this PR introduce _any_ user-facing change? just error message ### How was this patch tested? Tested manually on standalone cluster Closes #29685 from tgravescs/SPARK-32824. Authored-by: Thomas Graves Signed-off-by: HyukjinKwon --- .../scala/org/apache/spark/resource/ResourceUtils.scala | 7 ++++++- 1 file changed, 6 insertions(+), 1 deletion(-) diff --git a/core/src/main/scala/org/apache/spark/resource/ResourceUtils.scala b/core/src/main/scala/org/apache/spark/resource/ResourceUtils.scala index 162f090f011c6..5a9435653920f 100644 --- a/core/src/main/scala/org/apache/spark/resource/ResourceUtils.scala +++ b/core/src/main/scala/org/apache/spark/resource/ResourceUtils.scala @@ -149,7 +149,12 @@ private[spark] object ResourceUtils extends Logging { def listResourceIds(sparkConf: SparkConf, componentName: String): Seq[ResourceID] = { sparkConf.getAllWithPrefix(s"$componentName.$RESOURCE_PREFIX.").map { case (key, _) => - key.substring(0, key.indexOf('.')) + val index = key.indexOf('.') + if (index < 0) { + throw new SparkException(s"You must specify an amount config for resource: $key " + + s"config: $componentName.$RESOURCE_PREFIX.$key") + } + key.substring(0, index) }.distinct.map(name => new ResourceID(componentName, name)) }