Skip to content

Commit

Permalink
Merge pull request #20 from data-catering/vulnerabilities-spark-3.5.1
Browse files Browse the repository at this point in the history
Upgrade to Spark 3.5.1, use cloudnativek8s Spark base image, resolve …
  • Loading branch information
pflooky committed May 16, 2024
2 parents 02981ca + 4cb217f commit 1c9833f
Show file tree
Hide file tree
Showing 12 changed files with 76 additions and 9 deletions.
5 changes: 2 additions & 3 deletions Dockerfile
Original file line number Diff line number Diff line change
@@ -1,8 +1,7 @@
ARG SPARK_VERSION=3.5.0
FROM apache/spark:$SPARK_VERSION
ARG SPARK_VERSION=3.5.1
FROM cloudnativek8s/spark:3.5.1-b1.0.18

USER root
RUN groupadd -g 1001 app && useradd -m -u 1001 -g app app
RUN mkdir -p /opt/app /opt/DataCaterer/connection /opt/DataCaterer/plan /opt/DataCaterer/execution /opt/DataCaterer/report
RUN chown -R app:app /opt/app /opt/DataCaterer/connection /opt/DataCaterer/plan /opt/DataCaterer/execution /opt/DataCaterer/report
COPY --chown=app:app script /opt/app
Expand Down
6 changes: 6 additions & 0 deletions README.md
Original file line number Diff line number Diff line change
Expand Up @@ -191,3 +191,9 @@ jpackage "@misc/jpackage/jpackage.cfg" "@misc/jpackage/jpackage-windows.cfg"
# Linux
jpackage "@misc/jpackage/jpackage.cfg" "@misc/jpackage/jpackage-linux.cfg"
```

##### Java 17 VM Options

```shell
--add-opens=java.base/java.lang=ALL-UNNAMED --add-opens=java.base/java.lang.invoke=ALL-UNNAMED --add-opens=java.base/java.lang.reflect=ALL-UNNAMED --add-opens=java.base/java.io=ALL-UNNAMED --add-opens=java.base/java.net=ALL-UNNAMED --add-opens=java.base/java.nio=ALL-UNNAMED --add-opens=java.base/java.util=ALL-UNNAMED --add-opens=java.base/java.util.concurrent=ALL-UNNAMED --add-opens=java.base/java.util.concurrent.atomic=ALL-UNNAMED --add-opens=java.base/sun.nio.ch=ALL-UNNAMED --add-opens=java.base/sun.nio.cs=ALL-UNNAMED --add-opens=java.base/sun.security.action=ALL-UNNAMED --add-opens=java.base/sun.util.calendar=ALL-UNNAMED --add-opens=java.security.jgss/sun.security.krb5=ALL-UNNAMED
```
30 changes: 29 additions & 1 deletion app/build.gradle.kts
Original file line number Diff line number Diff line change
Expand Up @@ -53,9 +53,37 @@ configurations {

dependencies {
jpackageDep("org.scala-lang:scala-library:$scalaSpecificVersion")
jpackageDep("org.apache.spark:spark-sql_$scalaVersion:$sparkVersion")
jpackageDep("org.apache.spark:spark-sql_$scalaVersion:$sparkVersion") {
exclude(group = "com.google.protobuf")
exclude(module = "netty-codec-http")
exclude(module = "woodstox-core")
exclude(module = "nimbus-jose-jwt")
exclude(module = "commons-net")
exclude(module = "netty-handler")
exclude(module = "json-smart")
exclude(module = "avro")
exclude(module = "commons-compress")
exclude(module = "commons-configuration2")
exclude(module = "jetty-http")
exclude(module = "ion-jav")
}
jpackageDep(project(":api"))

// vulnerabilities in Spark
basicImpl("com.google.protobuf:protobuf-java:3.21.7")
basicImpl("io.netty:netty-codec-http:4.1.109.Final")
basicImpl("com.fasterxml.woodstox:woodstox-core:5.4.0")
basicImpl("com.nimbusds:nimbus-jose-jwt:9.37.2")
basicImpl("commons-net:commons-net:3.9.0")
basicImpl("io.netty:netty-handler:4.1.109.Final")
basicImpl("net.minidev:json-smart:2.4.9")
basicImpl("org.apache.avro:avro:1.11.3")
basicImpl("org.apache.commons:commons-compress:1.26.0")
basicImpl("org.apache.commons:commons-configuration2:2.10.1")
basicImpl("org.codehaus.jettison:jettison:1.5.4")
basicImpl("org.eclipse.jetty:jetty-http:9.4.54.v20240208")
//basicImpl("software.amazon.ion:ion-java:1.5.1") //should use: basicImpl("com.amazon.ion:ion-java:1.11.8")

// connectors
// postgres
basicImpl("org.postgresql:postgresql:42.6.0")
Expand Down
1 change: 1 addition & 0 deletions app/src/main/resources/ui/index.js
Original file line number Diff line number Diff line change
Expand Up @@ -25,6 +25,7 @@ import {
dispatchEvent,
executePlan,
getDataConnectionsAndAddToSelect,
getOverrideConnectionOptionsAsMap,
manualContainerDetails,
wait
} from "./shared.js";
Expand Down
File renamed without changes.
4 changes: 2 additions & 2 deletions gradle.properties
Original file line number Diff line number Diff line change
@@ -1,7 +1,7 @@
groupId=io.github.data-catering
version=0.10.1
version=0.10.2

scalaVersion=2.12
scalaSpecificVersion=2.12.15
sparkVersion=3.5.0
sparkVersion=3.5.1
sparkMajorVersion=3.5
20 changes: 20 additions & 0 deletions local-docker-build.sh
Original file line number Diff line number Diff line change
@@ -0,0 +1,20 @@
#!/usr/bin/env bash

version=$(grep version gradle.properties | cut -d= -f2)
sparkVersion=$(grep sparkVersion gradle.properties | cut -d= -f2)

echo "Creating API jar"
gradle clean :api:shadowJar

echo "Creating data caterer jar, version=$version"
gradle build shadowJar -x test
build_app=$?
if [[ "$build_app" -ne 0 ]] ; then
echo "Failed to build app, exiting"
exit 1
fi

docker build \
--build-arg "APP_VERSION=$version" \
--build-arg "SPARK_VERSION=$sparkVersion" \
-t datacatering/data-caterer-basic:$version .
7 changes: 7 additions & 0 deletions misc/vulns/check_vulns.sh
Original file line number Diff line number Diff line change
@@ -0,0 +1,7 @@
#!/usr/bin/env bash

echo "Checking for vulnerabilities in JAR"

trivy rootfs ../../app/build/libs

#gradle dependencyInsight --dependency
12 changes: 9 additions & 3 deletions script/run-data-caterer.sh
Original file line number Diff line number Diff line change
Expand Up @@ -2,13 +2,20 @@
DATA_CATERER_MASTER="${DATA_CATERER_MASTER:-local[*]}"
DEPLOY_MODE="${DEPLOY_MODE:-client}"
JAVA_OPTS="-Dlog4j.configurationFile=file:///opt/app/log4j2.properties -Djdk.module.illegalAccess=deny"
JAVA_17_OPTS="--add-opens=java.base/java.lang=ALL-UNNAMED --add-opens=java.base/java.lang.invoke=ALL-UNNAMED --add-opens=java.base/java.lang.reflect=ALL-UNNAMED --add-opens=java.base/java.io=ALL-UNNAMED --add-opens=java.base/java.net=ALL-UNNAMED --add-opens=java.base/java.nio=ALL-UNNAMED --add-opens=java.base/java.util=ALL-UNNAMED --add-opens=java.base/java.util.concurrent=ALL-UNNAMED --add-opens=java.base/java.util.concurrent.atomic=ALL-UNNAMED --add-opens=java.base/sun.nio.ch=ALL-UNNAMED --add-opens=java.base/sun.nio.cs=ALL-UNNAMED --add-opens=java.base/sun.security.action=ALL-UNNAMED --add-opens=java.base/sun.util.calendar=ALL-UNNAMED --add-opens=java.security.jgss/sun.security.krb5=ALL-UNNAMED"
DRIVER_MEMORY="${DRIVER_MEMORY:-2g}"
EXECUTOR_MEMORY="${EXECUTOR_MEMORY:-2g}"
ALL_OPTS="$ADDITIONAL_OPTS --conf \"spark.driver.extraJavaOptions=$JAVA_OPTS\" --conf \"spark.executor.extraJavaOptions=$JAVA_OPTS\""

if [[ "$DEPLOY_MODE" -eq "standalone" ]] ; then
echo "Running Data Caterer as a standalone application"
java -cp "/opt/spark/jars/*:/opt/app/job.jar" io.github.datacatering.datacaterer.core.ui.DataCatererUI
CMD=(
java
"$JAVA_OPTS"
"$JAVA_17_OPTS"
-cp "/opt/spark/jars/*:/opt/app/job.jar"
io.github.datacatering.datacaterer.core.ui.DataCatererUI
)
else
echo "Running Data Caterer as a Spark job"
CMD=(
Expand All @@ -21,7 +28,6 @@ else
"$ALL_OPTS"
file:///opt/app/job.jar
)

eval "${CMD[@]}"
fi

eval "${CMD[@]}"

0 comments on commit 1c9833f

Please sign in to comment.