Skip to content

Commit

Permalink
SPARK-35878. endpoint fixup: scalastyle in test suite
Browse files Browse the repository at this point in the history
Change-Id: I80e3160ca85c987862ec5a734774361982876173
  • Loading branch information
steveloughran committed Jun 24, 2021
1 parent bc4a585 commit 4116751
Showing 1 changed file with 2 additions and 4 deletions.
Original file line number Diff line number Diff line change
Expand Up @@ -32,7 +32,7 @@ class SparkHadoopUtilSuite extends SparkFunSuite {
val hadoopConf = new Configuration(false)
sc.set("spark.hadoop.orc.filterPushdown", "true")
new SparkHadoopUtil().appendSparkHadoopConfigs(sc, hadoopConf)
assertConfigValue(hadoopConf, "orc.filterPushdown","true" )
assertConfigValue(hadoopConf, "orc.filterPushdown", "true" )
assertConfigValue(hadoopConf, "fs.s3a.downgrade.syncable.exceptions", "true")
assertConfigValue(hadoopConf, "fs.s3a.endpoint", "s3.amazonaws.com")
}
Expand All @@ -42,7 +42,6 @@ class SparkHadoopUtilSuite extends SparkFunSuite {
* would.
*/
test("appendSparkHadoopConfigs with S3A endpoint set to empty string") {

val sc = new SparkConf()
val hadoopConf = new Configuration(false)
sc.set("spark.hadoop.fs.s3a.endpoint", "")
Expand All @@ -54,10 +53,9 @@ class SparkHadoopUtilSuite extends SparkFunSuite {
* Explicitly set the patched s3a options and verify that they are not overridden.
*/
test("appendSparkHadoopConfigs with S3A options explicitly set") {

val sc = new SparkConf()
val hadoopConf = new Configuration(false)
sc.set("spark.hadoop.fs.s3a.downgrade.syncable.exceptions","false")
sc.set("spark.hadoop.fs.s3a.downgrade.syncable.exceptions", "false")
sc.set("spark.hadoop.fs.s3a.endpoint", "s3-eu-west-1.amazonaws.com")
new SparkHadoopUtil().appendSparkHadoopConfigs(sc, hadoopConf)
assertConfigValue(hadoopConf, "fs.s3a.downgrade.syncable.exceptions", "false")
Expand Down

0 comments on commit 4116751

Please sign in to comment.