Skip to content
New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Jump to bottom

SNOW-801398 Tidy up shaded jar #490

Merged
merged 4 commits into from
May 5, 2023
Merged

SNOW-801398 Tidy up shaded jar #490

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
4 commits
Select commit Hold shift + click to select a range
2eee9b5
SNOW-801398 Tidy up shaded jar
sfc-gh-lsembera May 3, 2023
2516d5c
Exclude snowflake-jdbc from shading
sfc-gh-lsembera May 4, 2023
80cab1c
Fix typo
sfc-gh-lsembera May 4, 2023
268881b
Remove extra space
sfc-gh-lsembera May 5, 2023
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
142 changes: 137 additions & 5 deletions pom.xml
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -153,18 +153,70 @@
<groupId>ch.qos.reload4j</groupId>
<artifactId>reload4j</artifactId>
</exclusion>
<exclusion>
<groupId>com.github.pjfanning</groupId>
<artifactId>jersey-json</artifactId>
</exclusion>
<exclusion>
<groupId>com.sun.jersey</groupId>
<artifactId>jersey-core</artifactId>
</exclusion>
<exclusion>
<groupId>com.sun.jersey</groupId>
<artifactId>jersey-server</artifactId>
</exclusion>
<exclusion>
<groupId>com.sun.jersey</groupId>
<artifactId>jersey-servlet</artifactId>
</exclusion>
<exclusion>
<groupId>dnsjava</groupId>
<artifactId>dnsjava</artifactId>
</exclusion>
<exclusion>
<groupId>javax.activation</groupId>
<artifactId>activation</artifactId>
</exclusion>
<exclusion>
<groupId>javax.servlet</groupId>
<artifactId>javax.servlet-api</artifactId>
</exclusion>
<exclusion>
<groupId>javax.servlet.jsp</groupId>
<artifactId>jsp-api</artifactId>
</exclusion>
<exclusion>
<groupId>org.apache.avro</groupId>
<artifactId>avro</artifactId>
</exclusion>
<exclusion>
<groupId>org.apache.hadoop</groupId>
<artifactId>hadoop-auth</artifactId>
</exclusion>
<exclusion>
<groupId>org.apache.kerby</groupId>
<artifactId>kerb-core</artifactId>
</exclusion>
<exclusion>
<groupId>org.apache.zookeeper</groupId>
<artifactId>zookeeper</artifactId>
</exclusion>
<exclusion>
<groupId>org.eclipse.jetty</groupId>
<artifactId>jetty-server</artifactId>
</exclusion>
<exclusion>
<groupId>org.eclipse.jetty</groupId>
<artifactId>jetty-servlet</artifactId>
</exclusion>
<exclusion>
<groupId>org.eclipse.jetty</groupId>
<artifactId>jetty-util</artifactId>
</exclusion>
<exclusion>
<groupId>org.eclipse.jetty</groupId>
<artifactId>jetty-webapp</artifactId>
</exclusion>
<exclusion>
<groupId>org.slf4j</groupId>
<artifactId>slf4j-log4j12</artifactId>
Expand Down Expand Up @@ -205,11 +257,6 @@
<artifactId>jackson-jaxrs</artifactId>
<version>${codehaus.version}</version>
</dependency>
<dependency>
<groupId>org.codehaus.jackson</groupId>
<artifactId>jackson-mapper-asl</artifactId>
<version>${codehaus.version}</version>
</dependency>
<dependency>
<groupId>org.codehaus.jackson</groupId>
<artifactId>jackson-xc</artifactId>
Expand Down Expand Up @@ -697,8 +744,13 @@
<id>checkShadedContent</id>
<activation>
<os>
<!-- Shaded JAR content checking is a shell script, do not run it on Windows -->
<family>!Windows</family>
</os>
<property>
<!-- Do not check shaded JAR content if shading has been disabled -->
<name>!not-shadeDep</name>
</property>
</activation>
<build>
<plugins>
Expand Down Expand Up @@ -736,6 +788,11 @@
<artifactId>maven-shade-plugin</artifactId>
<version>3.4.1</version>
<configuration>
<artifactSet>
<excludes>
<exclude>net.snowflake:snowflake-jdbc</exclude>
</excludes>
</artifactSet>
<relocations>
<relocation>
<pattern>com.nimbusds</pattern>
Expand All @@ -761,11 +818,85 @@
<pattern>org.apache</pattern>
<shadedPattern>${shadeBase}.apache</shadedPattern>
</relocation>
<relocation>
<pattern>org.xbill</pattern>
<shadedPattern>${shadeBase}.org.xbill</shadedPattern>
</relocation>
<relocation>
<pattern>org.xerial</pattern>
<shadedPattern>${shadeBase}.org.xerial</shadedPattern>
</relocation>
<relocation>
<pattern>io.netty</pattern>
<shadedPattern>${shadeBase}.io.netty</shadedPattern>
</relocation>
<relocation>
<pattern>com.google</pattern>
<shadedPattern>${shadeBase}.com.google</shadedPattern>
</relocation>
<relocation>
<pattern>com.github.benmanes</pattern>
<shadedPattern>${shadeBase}.com.github.benmanes</shadedPattern>
</relocation>
<!-- shaded.parquet contains dependencies shaded in parquet itself. Here we are shading it for the second time -->
<relocation>
<pattern>shaded.parquet</pattern>
<shadedPattern>${shadeBase}.shaded.parquet</shadedPattern>
</relocation>
<relocation>
<pattern>org.codehaus</pattern>
<shadedPattern>${shadeBase}.org.codehaus</shadedPattern>
</relocation>
<relocation>
<pattern>com.jcraft</pattern>
<shadedPattern>${shadeBase}.com.jcraft</shadedPattern>
</relocation>
<relocation>
<pattern>org.eclipse</pattern>
<shadedPattern>${shadeBase}.org.eclipse</shadedPattern>
</relocation>
<relocation>
<pattern>org.checkerframework</pattern>
<shadedPattern>${shadeBase}.org.checkerframework</shadedPattern>
</relocation>
<relocation>
<pattern>com.codahale</pattern>
<shadedPattern>${shadeBase}.com.codahale</shadedPattern>
</relocation>
<relocation>
<pattern>com.ctc</pattern>
<shadedPattern>${shadeBase}.com.ctc</shadedPattern>
</relocation>
<relocation>
<pattern>com.github.luben</pattern>
<shadedPattern>${shadeBase}.com.github.luben</shadedPattern>
</relocation>
<relocation>
<pattern>com.thoughtworks</pattern>
<shadedPattern>${shadeBase}.com.thoughtworks</shadedPattern>
</relocation>
<!-- This belongs to org.apache.arrow:arrow-vector -->
<relocation>
<pattern>codegen</pattern>
<shadedPattern>${shadeBase}.codegen</shadedPattern>
</relocation>
<relocation>
<pattern>javax.annotation</pattern>
<shadedPattern>${shadeBase}.javax.annotation</shadedPattern>
</relocation>
<relocation>
<pattern>javax.activation</pattern>
<shadedPattern>${shadeBase}.javax.activation</shadedPattern>
</relocation>
</relocations>
<filters>
<filter>
<artifact>*:*</artifact>
<excludes>
<exclude>assets/org/apache/commons/math3/exception/util/LocalizedFormats_fr.properties</exclude>
<exclude>about.html</exclude>
<exclude>mozilla/public-suffix-list.txt</exclude>

<exclude>META-INF/LICENSE*</exclude>
<exclude>META-INF/NOTICE*</exclude>
<exclude>META-INF/DEPENDENCIES</exclude>
Expand All @@ -775,6 +906,7 @@
<exclude>META-INF/*.SF</exclude>
<exclude>META-INF/*.DSA</exclude>
<exclude>META-INF/*.RSA</exclude>
<exclude>google/protobuf/**/*.proto</exclude>
</excludes>
</filter>
<filter>
Expand Down
55 changes: 31 additions & 24 deletions scripts/check_content.sh
View file
Open in desktop
Original file line number Diff line number Diff line change
@@ -1,33 +1,40 @@
#!/bin/bash -e

# scripts used to check if all dependency is shaded into snowflake internal path
# Do not add any additional exceptions into this script.
# If this script fails the maven build, re-configure shading relocations in pom.xml.

set -o pipefail

DIR="$( cd "$( dirname "${BASH_SOURCE[0]}" )" >/dev/null && pwd )"
if jar tvf $DIR/../target/snowflake-ingest-sdk.jar | awk '{print $8}' | grep -v -E "^(net|com)/snowflake" \
| grep -v -E "(com|net)/\$" | grep -v -E "^META-INF" | grep -v -E "^mozilla" | grep -v mime.types \
| grep -v project.properties | grep -v -E "javax" | grep -v -E "^org/" | grep -v -E "^com/google" \
| grep -v -E "^com/sun" | grep -v "log4j.properties" | grep -v "git.properties" | grep -v "io/" \
| grep -v "codegen/" | grep -v "com/codahale/" | grep -v "com/ibm/" | grep -v "LICENSE" | grep -v "aix/" \
| grep -v "darwin/" | grep -v "win/" | grep -v "freebsd/" | grep -v "linux/" | grep -v "com/github/" \
| grep -v -E "shaded/" | grep -v "webapps/" | grep -v "microsoft/" | grep -v "com/ctc/" | grep -v "jersey/" \
| grep -v "edu/" | grep -v "com/jcraft/" | grep -v "contribs/" | grep -v "com/zaxxer/" | grep -v "com/squareup/" \
| grep -v "com/thoughtworks/" | grep -v "com/jamesmurty/" | grep -v "net/iharder/" \
| grep -v -E "^core-default.xml" \
| grep -v "yarn-version-info.properties" | grep -v "yarn-version-info.properties" \
| grep -v "about.html" | grep -v "jetty-dir.css" \
| grep -v "krb5_udp-template.conf" | grep -v "krb5-template.conf" \
| grep -v "ccache.txt" | grep -v "keytab.txt" \
| grep -v "common-version-info.properties" | grep -v "LocalizedFormats_fr.properties" \
| grep -v "org.apache.hadoop.application-classloader.properties" | grep -v "assets/" | grep -v "ehcache-core.xsd" \
| grep -v "ehcache-107ext.xsd" | grep -v "parquet.thrift" | grep -v "mapred-default.xml" \
| grep -v "yarn-default.xml" | grep -v ".keep" | grep -v "NOTICE" | grep -v "digesterRules.xml" \
| grep -v "properties.dtd" | grep -v "PropertyList-1.0.dtd" \
| grep -v "about.html" | grep -v "jetty-dir.css" | grep -v "krb5-template.conf" \
| grep -v "krb5_udp-template.conf" | grep -v "ccache.txt" | grep -v "keytab.txt" \
| grep -v "lookup.class" | grep -v "update.class" | grep -v "dig.class" | grep -v "jnamed" \
; then
echo "[ERROR] Ingest SDK jar includes class not under the snowflake namespace"

if jar tvf $DIR/../target/snowflake-ingest-sdk.jar | awk '{print $8}' | \
# Ignore directories
grep -v -E '/$' | \

# Snowflake top-level packages are allowed
grep -v -E "^net/snowflake/ingest" | \

# META-INF is allowed in the shaded JAR
grep -v -E "^META-INF" | \

# Files in the JAR root that are probably required
grep -v mime.types | \
grep -v project.properties | \
grep -v arrow-git.properties | \
grep -v core-default.xml | \
grep -v org.apache.hadoop.application-classloader.properties | \
grep -v common-version-info.properties | \
grep -v PropertyList-1.0.dtd | \
grep -v properties.dtd | \
grep -v parquet.thrift | \

# Native zstd libraries are allowed
grep -v -E '^darwin' | \
grep -v -E '^freebsd' | \
grep -v -E '^linux' | \
grep -v -E '^win' ; then

echo "[ERROR] JDBC jar includes class not under the snowflake namespace"
exit 1
fi
5 changes: 5 additions & 0 deletions scripts/run_gh_actions.sh
View file
Open in desktop
Original file line number Diff line number Diff line change
@@ -1,3 +1,8 @@
#!/bin/bash -e

set -o pipefail

# Build and install shaded JAR first. check_content.sh runs here.
mvn install -DskipTests=true --batch-mode --show-version

PARAMS=()
Expand Down