Skip to content

Commit

Permalink
[SPARK-21418][SQL] NoSuchElementException: None.get in DataSourceScan…
Browse files Browse the repository at this point in the history
…Exec with sun.io.serialization.extendedDebugInfo=true

## What changes were proposed in this pull request?

If no SparkConf is available to Utils.redact, simply don't redact.

## How was this patch tested?

Existing tests

Author: Sean Owen <sowen@cloudera.com>

Closes #19123 from srowen/SPARK-21418.
  • Loading branch information
srowen authored and hvanhovell committed Sep 4, 2017
1 parent 9f30d92 commit ca59445
Show file tree
Hide file tree
Showing 2 changed files with 7 additions and 4 deletions.
9 changes: 6 additions & 3 deletions core/src/main/scala/org/apache/spark/util/Utils.scala
Original file line number Diff line number Diff line change
Expand Up @@ -2639,9 +2639,12 @@ private[spark] object Utils extends Logging {
* Redact the sensitive information in the given string.
*/
def redact(conf: SparkConf, text: String): String = {
if (text == null || text.isEmpty || !conf.contains(STRING_REDACTION_PATTERN)) return text
val regex = conf.get(STRING_REDACTION_PATTERN).get
regex.replaceAllIn(text, REDACTION_REPLACEMENT_TEXT)
if (text == null || text.isEmpty || conf == null || !conf.contains(STRING_REDACTION_PATTERN)) {
text
} else {
val regex = conf.get(STRING_REDACTION_PATTERN).get
regex.replaceAllIn(text, REDACTION_REPLACEMENT_TEXT)
}
}

private def redact(redactionPattern: Regex, kvs: Seq[(String, String)]): Seq[(String, String)] = {
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -69,7 +69,7 @@ trait DataSourceScanExec extends LeafExecNode with CodegenSupport {
* Shorthand for calling redactString() without specifying redacting rules
*/
private def redact(text: String): String = {
Utils.redact(SparkSession.getActiveSession.get.sparkContext.conf, text)
Utils.redact(SparkSession.getActiveSession.map(_.sparkContext.conf).orNull, text)
}
}

Expand Down

0 comments on commit ca59445

Please sign in to comment.