Skip to content
New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Jump to bottom

[FLINK-2584] Check for unshaded classes in fat jar and shade curator #1076

Closed
wants to merge 1 commit into from
Closed

[FLINK-2584] Check for unshaded classes in fat jar and shade curator #1076

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
1 commit
Select commit
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Jump to
Jump to file
Failed to load files.
Diff view
Diff view
5 changes: 5 additions & 0 deletions flink-contrib/flink-storm-compatibility/flink-storm-compatibility-core/pom.xml
View file
Open in desktop
Expand Up @@ -57,6 +57,11 @@ under the License.
</exclusions>
</dependency>

<dependency>
<groupId>com.google.guava</groupId>
<artifactId>guava</artifactId>
<version>${guava.version}</version>
</dependency>
</dependencies>

<build>
Expand Down
5 changes: 3 additions & 2 deletions flink-runtime/pom.xml
View file
Open in desktop
Expand Up @@ -193,8 +193,9 @@ under the License.
</dependency>

<dependency>
<groupId>org.apache.curator</groupId>
<artifactId>curator-recipes</artifactId>
<groupId>org.apache.flink</groupId>
<artifactId>flink-shaded-curator</artifactId>
<version>${project.version}</version>
</dependency>
</dependencies>

Expand Down
89 changes: 89 additions & 0 deletions flink-shaded-curator/pom.xml
View file
Open in desktop
@@ -0,0 +1,89 @@
<?xml version="1.0" encoding="UTF-8"?>
<!--
Licensed to the Apache Software Foundation (ASF) under one
or more contributor license agreements. See the NOTICE file
distributed with this work for additional information
regarding copyright ownership. The ASF licenses this file
to you under the Apache License, Version 2.0 (the
"License"); you may not use this file except in compliance
with the License. You may obtain a copy of the License at

http://www.apache.org/licenses/LICENSE-2.0

Unless required by applicable law or agreed to in writing,
software distributed under the License is distributed on an
"AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
KIND, either express or implied. See the License for the
specific language governing permissions and limitations
under the License.
-->
<project xmlns="http://maven.apache.org/POM/4.0.0" xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance"
xsi:schemaLocation="http://maven.apache.org/POM/4.0.0 http://maven.apache.org/maven-v4_0_0.xsd">

<modelVersion>4.0.0</modelVersion>

<parent>
<groupId>org.apache.flink</groupId>
<artifactId>flink-parent</artifactId>
<version>0.10-SNAPSHOT</version>
<relativePath>..</relativePath>
</parent>

<artifactId>flink-shaded-curator</artifactId>
<name>flink-shaded-curator</name>

<packaging>jar</packaging>


<dependencies>
<dependency>
<groupId>org.apache.curator</groupId>
<artifactId>curator-recipes</artifactId>
<version>${curatorrecipes.version}</version>
</dependency>
</dependencies>

<build>
<plugins>
<!-- Relocate the Curator's Guava dependency into a different namespace and
put create our own apache curator dependency.
We can easily integrate curator's netty into the jar file.
-->
<plugin>
<groupId>org.apache.maven.plugins</groupId>
<artifactId>maven-shade-plugin</artifactId>
<executions>
<execution>
<id>shade-flink</id> <!-- override inherited execution id -->
<phase>package</phase>
<goals>
<goal>shade</goal>
</goals>
<configuration>
<shadedArtifactAttached>false</shadedArtifactAttached>
<createDependencyReducedPom>true</createDependencyReducedPom>
<dependencyReducedPomLocation>${project.basedir}/target/dependency-reduced-pom.xml</dependencyReducedPomLocation>
<transformers>
<!-- The service transformer is needed to merge META-INF/services files -->
<transformer implementation="org.apache.maven.plugins.shade.resource.ServicesResourceTransformer"/>
<transformer implementation="org.apache.maven.plugins.shade.resource.ApacheNoticeResourceTransformer"/>
</transformers>
<artifactSet>
<includes>
<include>org.apache.curator:*</include>
<include>com.google.guava:*</include>
</includes>
</artifactSet>
<relocations>
<relocation>
<pattern>com.google</pattern>
<shadedPattern>org.apache.curator.shaded.com.google</shadedPattern>
</relocation>
</relocations>
</configuration>
</execution>
</executions>
</plugin>
</plugins>
</build>
</project>
6 changes: 6 additions & 0 deletions ...-staging/flink-streaming/flink-streaming-connectors/flink-connector-elasticsearch/pom.xml
View file
Open in desktop
Expand Up @@ -54,6 +54,12 @@ under the License.
<version>${elasticsearch.version}</version>
</dependency>

<dependency>
<groupId>com.google.guava</groupId>
<artifactId>guava</artifactId>
<version>${guava.version}</version>
</dependency>

<dependency>
<groupId>org.apache.flink</groupId>
<artifactId>flink-streaming-core</artifactId>
Expand Down
6 changes: 6 additions & 0 deletions flink-staging/flink-streaming/flink-streaming-connectors/flink-connector-kafka/pom.xml
View file
Open in desktop
Expand Up @@ -92,6 +92,12 @@ under the License.
</exclusions>
</dependency>

<dependency>
<groupId>com.google.guava</groupId>
<artifactId>guava</artifactId>
<version>${guava.version}</version>
</dependency>

<dependency>
<groupId>org.apache.curator</groupId>
<artifactId>curator-test</artifactId>
Expand Down
7 changes: 1 addition & 6 deletions pom.xml
View file
Open in desktop
Expand Up @@ -51,6 +51,7 @@ under the License.

<modules>
<module>flink-shaded-hadoop</module>
<module>flink-shaded-curator</module>
<module>flink-core</module>
<module>flink-java</module>
<module>flink-scala</module>
Expand Down Expand Up @@ -336,12 +337,6 @@ under the License.
<artifactId>zookeeper</artifactId>
<version>${zookeeper.version}</version>
</dependency>

<dependency>
<groupId>org.apache.curator</groupId>
<artifactId>curator-recipes</artifactId>
<version>${curatorrecipes.version}</version>
</dependency>
</dependencies>
</dependencyManagement>

Expand Down
23 changes: 23 additions & 0 deletions tools/travis_mvn_watchdog.sh
View file
Open in desktop
Expand Up @@ -156,6 +156,27 @@ watchdog () {
done
}

# Check the final fat jar for illegal artifacts
check_shaded_artifacts() {
jar tf build-target/lib/flink-dist-*.jar > allClasses
ASM=`cat allClasses | grep '^org/objectweb/asm/' | wc -l`
if [ $ASM != "0" ]; then
echo "=============================================================================="
echo "Detected $ASM asm dependencies in fat jar"
echo "=============================================================================="
exit 1
fi

GUAVA=`cat allClasses | grep '^com/google/common' | wc -l`
if [ $GUAVA != "0" ]; then
echo "=============================================================================="
echo "Detected $GUAVA guava dependencies in fat jar"
echo "=============================================================================="
exit 1
fi

}

# =============================================================================
# WATCHDOG
# =============================================================================
Expand Down Expand Up @@ -189,6 +210,8 @@ echo "MVN exited with EXIT CODE: ${EXIT_CODE}."
rm $MVN_PID
rm $MVN_EXIT

check_shaded_artifacts

put_yarn_logs_to_artifacts

upload_artifacts_s3
Expand Down