Upgrade to Spark 3.5.1, use cloudnativek8s Spark base image, resolve …

Dockerfile

@@ -1,8 +1,7 @@
-ARG SPARK_VERSION=3.5.0
-FROM apache/spark:$SPARK_VERSION
+ARG SPARK_VERSION=3.5.1
+FROM cloudnativek8s/spark:3.5.1-b1.0.18
 
 USER root
-RUN groupadd -g 1001 app && useradd -m -u 1001 -g app app
 RUN mkdir -p /opt/app /opt/DataCaterer/connection /opt/DataCaterer/plan /opt/DataCaterer/execution /opt/DataCaterer/report
 RUN chown -R app:app /opt/app /opt/DataCaterer/connection /opt/DataCaterer/plan /opt/DataCaterer/execution /opt/DataCaterer/report
 COPY --chown=app:app script /opt/app

README.md

@@ -191,3 +191,9 @@ jpackage "@misc/jpackage/jpackage.cfg" "@misc/jpackage/jpackage-windows.cfg"
 # Linux
 jpackage "@misc/jpackage/jpackage.cfg" "@misc/jpackage/jpackage-linux.cfg"
 ```
+
+##### Java 17 VM Options
+
+```shell
+--add-opens=java.base/java.lang=ALL-UNNAMED --add-opens=java.base/java.lang.invoke=ALL-UNNAMED --add-opens=java.base/java.lang.reflect=ALL-UNNAMED --add-opens=java.base/java.io=ALL-UNNAMED --add-opens=java.base/java.net=ALL-UNNAMED --add-opens=java.base/java.nio=ALL-UNNAMED --add-opens=java.base/java.util=ALL-UNNAMED --add-opens=java.base/java.util.concurrent=ALL-UNNAMED --add-opens=java.base/java.util.concurrent.atomic=ALL-UNNAMED --add-opens=java.base/sun.nio.ch=ALL-UNNAMED --add-opens=java.base/sun.nio.cs=ALL-UNNAMED --add-opens=java.base/sun.security.action=ALL-UNNAMED --add-opens=java.base/sun.util.calendar=ALL-UNNAMED --add-opens=java.security.jgss/sun.security.krb5=ALL-UNNAMED
+```

app/build.gradle.kts

@@ -53,9 +53,37 @@ configurations {
 
 dependencies {
     jpackageDep("org.scala-lang:scala-library:$scalaSpecificVersion")
-    jpackageDep("org.apache.spark:spark-sql_$scalaVersion:$sparkVersion")
+    jpackageDep("org.apache.spark:spark-sql_$scalaVersion:$sparkVersion") {
+        exclude(group = "com.google.protobuf")
+        exclude(module = "netty-codec-http")
+        exclude(module = "woodstox-core")
+        exclude(module = "nimbus-jose-jwt")
+        exclude(module = "commons-net")
+        exclude(module = "netty-handler")
+        exclude(module = "json-smart")
+        exclude(module = "avro")
+        exclude(module = "commons-compress")
+        exclude(module = "commons-configuration2")
+        exclude(module = "jetty-http")
+        exclude(module = "ion-jav")
+    }
     jpackageDep(project(":api"))
 
+    // vulnerabilities in Spark
+    basicImpl("com.google.protobuf:protobuf-java:3.21.7")
+    basicImpl("io.netty:netty-codec-http:4.1.109.Final")
+    basicImpl("com.fasterxml.woodstox:woodstox-core:5.4.0")
+    basicImpl("com.nimbusds:nimbus-jose-jwt:9.37.2")
+    basicImpl("commons-net:commons-net:3.9.0")
+    basicImpl("io.netty:netty-handler:4.1.109.Final")
+    basicImpl("net.minidev:json-smart:2.4.9")
+    basicImpl("org.apache.avro:avro:1.11.3")
+    basicImpl("org.apache.commons:commons-compress:1.26.0")
+    basicImpl("org.apache.commons:commons-configuration2:2.10.1")
+    basicImpl("org.codehaus.jettison:jettison:1.5.4")
+    basicImpl("org.eclipse.jetty:jetty-http:9.4.54.v20240208")
+    //basicImpl("software.amazon.ion:ion-java:1.5.1") //should use: basicImpl("com.amazon.ion:ion-java:1.11.8")
+
     // connectors
     // postgres
     basicImpl("org.postgresql:postgresql:42.6.0")

app/src/main/resources/ui/index.js

@@ -25,6 +25,7 @@ import {
     dispatchEvent,
     executePlan,
     getDataConnectionsAndAddToSelect,
+    getOverrideConnectionOptionsAsMap,
     manualContainerDetails,
     wait
 } from "./shared.js";

gradle.properties

@@ -1,7 +1,7 @@
 groupId=io.github.data-catering
-version=0.10.1
+version=0.10.2
 
 scalaVersion=2.12
 scalaSpecificVersion=2.12.15
-sparkVersion=3.5.0
+sparkVersion=3.5.1
 sparkMajorVersion=3.5

local-docker-build.sh

@@ -0,0 +1,20 @@
+#!/usr/bin/env bash
+
+version=$(grep version gradle.properties | cut -d= -f2)
+sparkVersion=$(grep sparkVersion gradle.properties | cut -d= -f2)
+
+echo "Creating API jar"
+gradle clean :api:shadowJar
+
+echo "Creating data caterer jar, version=$version"
+gradle build shadowJar -x test
+build_app=$?
+if [[ "$build_app" -ne 0 ]] ; then
+  echo "Failed to build app, exiting"
+  exit 1
+fi
+
+docker build \
+  --build-arg "APP_VERSION=$version" \
+  --build-arg "SPARK_VERSION=$sparkVersion" \
+  -t datacatering/data-caterer-basic:$version .

misc/vulns/check_vulns.sh

@@ -0,0 +1,7 @@
+#!/usr/bin/env bash
+
+echo "Checking for vulnerabilities in JAR"
+
+trivy rootfs ../../app/build/libs
+
+#gradle dependencyInsight --dependency

script/run-data-caterer.sh

@@ -2,13 +2,20 @@
 DATA_CATERER_MASTER="${DATA_CATERER_MASTER:-local[*]}"
 DEPLOY_MODE="${DEPLOY_MODE:-client}"
 JAVA_OPTS="-Dlog4j.configurationFile=file:///opt/app/log4j2.properties -Djdk.module.illegalAccess=deny"
+JAVA_17_OPTS="--add-opens=java.base/java.lang=ALL-UNNAMED --add-opens=java.base/java.lang.invoke=ALL-UNNAMED --add-opens=java.base/java.lang.reflect=ALL-UNNAMED --add-opens=java.base/java.io=ALL-UNNAMED --add-opens=java.base/java.net=ALL-UNNAMED --add-opens=java.base/java.nio=ALL-UNNAMED --add-opens=java.base/java.util=ALL-UNNAMED --add-opens=java.base/java.util.concurrent=ALL-UNNAMED --add-opens=java.base/java.util.concurrent.atomic=ALL-UNNAMED --add-opens=java.base/sun.nio.ch=ALL-UNNAMED --add-opens=java.base/sun.nio.cs=ALL-UNNAMED --add-opens=java.base/sun.security.action=ALL-UNNAMED --add-opens=java.base/sun.util.calendar=ALL-UNNAMED --add-opens=java.security.jgss/sun.security.krb5=ALL-UNNAMED"
 DRIVER_MEMORY="${DRIVER_MEMORY:-2g}"
 EXECUTOR_MEMORY="${EXECUTOR_MEMORY:-2g}"
 ALL_OPTS="$ADDITIONAL_OPTS --conf \"spark.driver.extraJavaOptions=$JAVA_OPTS\" --conf \"spark.executor.extraJavaOptions=$JAVA_OPTS\""
 
 if [[ "$DEPLOY_MODE" -eq "standalone" ]] ; then
   echo "Running Data Caterer as a standalone application"
-  java -cp "/opt/spark/jars/*:/opt/app/job.jar" io.github.datacatering.datacaterer.core.ui.DataCatererUI
+  CMD=(
+    java
+    "$JAVA_OPTS"
+    "$JAVA_17_OPTS"
+    -cp "/opt/spark/jars/*:/opt/app/job.jar"
+    io.github.datacatering.datacaterer.core.ui.DataCatererUI
+  )
 else
   echo "Running Data Caterer as a Spark job"
   CMD=(
@@ -21,7 +28,6 @@ else
     "$ALL_OPTS"
     file:///opt/app/job.jar
   )
-
-  eval "${CMD[@]}"
 fi
 
+eval "${CMD[@]}"