Spark 3.5: Support camel case session configs and options

apache · May 10, 2024 · b040b77 · b040b77
1 parent 3c8e046
commit b040b77
Show file tree

Hide file tree

Showing 2 changed files with 54 additions and 0 deletions.
diff --git a/spark/v3.5/spark/src/main/java/org/apache/iceberg/spark/SparkConfParser.java b/spark/v3.5/spark/src/main/java/org/apache/iceberg/spark/SparkConfParser.java
@@ -228,6 +228,11 @@ protected T parse(Function<String, T> conversion, T defaultValue) {
           if (optionValue != null) {
             return conversion.apply(optionValue);
           }
+
+          String sparkOptionValue = options.get(toCamelCase(optionName));
+          if (sparkOptionValue != null) {
+            return conversion.apply(sparkOptionValue);
+          }
         }
       }
 
@@ -236,6 +241,11 @@ protected T parse(Function<String, T> conversion, T defaultValue) {
         if (sessionConfValue != null) {
           return conversion.apply(sessionConfValue);
         }
+
+        String sparkSessionConfValue = sessionConf.get(toCamelCase(sessionConfName), null);
+        if (sparkSessionConfValue != null) {
+          return conversion.apply(sparkSessionConfValue);
+        }
       }
 
       if (tablePropertyName != null) {
@@ -247,5 +257,23 @@ protected T parse(Function<String, T> conversion, T defaultValue) {
 
       return defaultValue;
     }
+
+    private String toCamelCase(String key) {
+      StringBuilder transformedKey = new StringBuilder();
+      boolean capitalizeNext = false;
+
+      for (char character : key.toCharArray()) {
+        if (character == '-') {
+          capitalizeNext = true;
+        } else if (capitalizeNext) {
+          transformedKey.append(Character.toUpperCase(character));
+          capitalizeNext = false;
+        } else {
+          transformedKey.append(character);
+        }
+      }
+
+      return transformedKey.toString();
+    }
   }
 }
diff --git a/spark/v3.5/spark/src/test/java/org/apache/iceberg/spark/TestSparkWriteConf.java b/spark/v3.5/spark/src/test/java/org/apache/iceberg/spark/TestSparkWriteConf.java
@@ -79,6 +79,32 @@ public void after() {
     sql("DROP TABLE IF EXISTS %s", tableName);
   }
 
+  @TestTemplate
+  public void testCamelCaseSparkSessionConf() {
+    Table table = validationCatalog.loadTable(tableIdent);
+    String confName = "spark.sql.iceberg.some-int-conf";
+    String sparkConfName = "spark.sql.iceberg.someIntConf";
+
+    withSQLConf(
+        ImmutableMap.of(sparkConfName, "1"),
+        () -> {
+          SparkConfParser parser = new SparkConfParser(spark, table, ImmutableMap.of());
+          Integer value = parser.intConf().sessionConf(confName).parseOptional();
+          assertThat(value).isEqualTo(1);
+        });
+  }
+
+  @TestTemplate
+  public void testCamelCaseSparkOption() {
+    Table table = validationCatalog.loadTable(tableIdent);
+    String option = "some-int-option";
+    String sparkOption = "someIntOption";
+    Map<String, String> options = ImmutableMap.of(sparkOption, "1");
+    SparkConfParser parser = new SparkConfParser(spark, table, options);
+    Integer value = parser.intConf().option(option).parseOptional();
+    assertThat(value).isEqualTo(1);
+  }
+
   @TestTemplate
   public void testDurationConf() {
     Table table = validationCatalog.loadTable(tableIdent);