From ee87f7c5852a746bf412eac3d186229a940033a9 Mon Sep 17 00:00:00 2001
From: Chesnay Schepler <chesnay@apache.org>
Date: Wed, 5 May 2021 12:51:09 +0200
Subject: [PATCH 1/4] [FLINK-22560][build] Move generic filters/transformers
 into general shade-plugin configuration

---
 .../flink-connector-cassandra/pom.xml         |  5 +-
 flink-dist/pom.xml                            | 33 +------------
 flink-kubernetes/pom.xml                      |  2 +-
 pom.xml                                       | 48 +++++++++++--------
 4 files changed, 35 insertions(+), 53 deletions(-)

diff --git a/flink-connectors/flink-connector-cassandra/pom.xml b/flink-connectors/flink-connector-cassandra/pom.xml
index 9aa0fc45b63d7..850f38e318105 100644
--- a/flink-connectors/flink-connector-cassandra/pom.xml
+++ b/flink-connectors/flink-connector-cassandra/pom.xml
@@ -63,8 +63,9 @@ under the License.
 						<goals>
 							<goal>shade</goal>
 						</goals>
-						<configuration combine.self="override">
-							<dependencyReducedPomLocation>${project.basedir}/target/dependency-reduced-pom.xml</dependencyReducedPomLocation>
+						<configuration>
+							<!-- This is necessary because we bundle a subset of our dependencies,
+							and transitive dependencies of that subset should still be pulled into the user jar.-->
 							<promoteTransitiveDependencies>true</promoteTransitiveDependencies>
 							<artifactSet>
 								<includes>
diff --git a/flink-dist/pom.xml b/flink-dist/pom.xml
index c0c3bd69a3f50..4a546b3e2b0e0 100644
--- a/flink-dist/pom.xml
+++ b/flink-dist/pom.xml
@@ -679,7 +679,7 @@ under the License.
 						<goals>
 							<goal>shade</goal>
 						</goals>
-						<configuration combine.self="override">
+						<configuration>
 							<createDependencyReducedPom>false</createDependencyReducedPom>
 							<shadedArtifactAttached>false</shadedArtifactAttached>
 							<finalName>${project.artifactId}-${project.version}</finalName>
@@ -688,11 +688,6 @@ under the License.
 								<filter>
 									<artifact>*</artifact>
 									<excludes>
-										<exclude>log4j.properties</exclude>
-										<exclude>log4j-test.properties</exclude>
-										<exclude>META-INF/*.SF</exclude>
-										<exclude>META-INF/*.DSA</exclude>
-										<exclude>META-INF/*.RSA</exclude>
 										<exclude>org/apache/flink/runtime/util/bash/BashJavaUtils.class</exclude>
 									</excludes>
 								</filter>
@@ -709,12 +704,6 @@ under the License.
 								<transformer implementation="org.apache.maven.plugins.shade.resource.AppendingTransformer">
 									<resource>reference.conf</resource>
 								</transformer>
-								<!-- The service transformer is needed to merge META-INF/services files -->
-								<transformer implementation="org.apache.maven.plugins.shade.resource.ServicesResourceTransformer"/>
-								<transformer implementation="org.apache.maven.plugins.shade.resource.ApacheNoticeResourceTransformer">
-									<projectName>Apache Flink</projectName>
-									<encoding>UTF-8</encoding>
-								</transformer>
 							</transformers>
 						</configuration>
 					</execution>
@@ -734,22 +723,11 @@ under the License.
 						<goals>
 							<goal>shade</goal>
 						</goals>
-						<configuration combine.self="override">
+						<configuration>
 							<createDependencyReducedPom>false</createDependencyReducedPom>
 							<shadedArtifactAttached>false</shadedArtifactAttached>
 							<finalName>bash-java-utils</finalName>
 							<filters>
-								<!-- Globally exclude log4j.properties from our JAR files. -->
-								<filter>
-									<artifact>*</artifact>
-									<excludes>
-										<exclude>log4j.properties</exclude>
-										<exclude>log4j-test.properties</exclude>
-										<exclude>META-INF/*.SF</exclude>
-										<exclude>META-INF/*.DSA</exclude>
-										<exclude>META-INF/*.RSA</exclude>
-									</excludes>
-								</filter>
 								<!-- Include only the BashJavaUtils, other required classes should come from the flink-dist-->
 								<filter>
 									<artifact>org.apache.flink:*</artifact>
@@ -770,13 +748,6 @@ under the License.
 									<resource>log4j2.properties</resource>
 									<file>src/main/resources/log4j-bash-utils.properties</file>
 								</transformer>
-								<transformer implementation="org.apache.maven.plugins.shade.resource.AppendingTransformer">
-									<resource>reference.conf</resource>
-								</transformer>
-								<transformer implementation="org.apache.maven.plugins.shade.resource.ApacheNoticeResourceTransformer">
-									<projectName>Apache Flink</projectName>
-									<encoding>UTF-8</encoding>
-								</transformer>
 							</transformers>
 						</configuration>
 					</execution>
diff --git a/flink-kubernetes/pom.xml b/flink-kubernetes/pom.xml
index 3ad188d6e15c0..04df35f75dc5e 100644
--- a/flink-kubernetes/pom.xml
+++ b/flink-kubernetes/pom.xml
@@ -129,7 +129,7 @@ under the License.
 						<goals>
 							<goal>shade</goal>
 						</goals>
-						<configuration combine.children="append">
+						<configuration>
 							<artifactSet>
 								<includes combine.children="append">
 									<include>io.fabric8:kubernetes-client</include>
diff --git a/pom.xml b/pom.xml
index c84a29133c07d..f46d0ac917897 100644
--- a/pom.xml
+++ b/pom.xml
@@ -1726,6 +1726,35 @@ under the License.
 			<plugin>
 				<groupId>org.apache.maven.plugins</groupId>
 				<artifactId>maven-shade-plugin</artifactId>
+				<configuration>
+					<!-- This section contains the core configuration that is applied to every jar that we create.-->
+					<filters combine.children="append">
+						<filter>
+							<artifact>*</artifact>
+							<excludes>
+								<!-- Globally exclude log4j.properties from our JAR files. -->
+								<exclude>log4j.properties</exclude>
+								<exclude>log4j2.properties</exclude>
+								<exclude>log4j-test.properties</exclude>
+								<exclude>log4j2-test.properties</exclude>
+								<!-- Do not copy the signatures in the META-INF folder.
+								Otherwise, this might cause SecurityExceptions when using the JAR. -->
+								<exclude>META-INF/*.SF</exclude>
+								<exclude>META-INF/*.DSA</exclude>
+								<exclude>META-INF/*.RSA</exclude>
+							</excludes>
+						</filter>
+					</filters>
+					<transformers combine.children="append">
+						<!-- The service transformer is needed to merge META-INF/services files -->
+						<transformer implementation="org.apache.maven.plugins.shade.resource.ServicesResourceTransformer"/>
+						<!-- The ApacheNoticeResourceTransformer collects and aggregates NOTICE files -->
+						<transformer implementation="org.apache.maven.plugins.shade.resource.ApacheNoticeResourceTransformer">
+							<projectName>Apache Flink</projectName>
+							<encoding>UTF-8</encoding>
+						</transformer>
+					</transformers>
+				</configuration>
 				<executions>
 					<execution>
 						<id>shade-flink</id>
@@ -1740,16 +1769,6 @@ under the License.
 							<dependencyReducedPomLocation>${project.basedir}/target/dependency-reduced-pom.xml</dependencyReducedPomLocation>
 							<!-- Filters MUST be appended; merging filters does not work properly, see MSHADE-305 -->
 							<filters combine.children="append">
-								<!-- Globally exclude log4j.properties from our JAR files. -->
-								<filter>
-									<artifact>*</artifact>
-									<excludes>
-										<exclude>log4j.properties</exclude>
-										<exclude>log4j2.properties</exclude>
-										<exclude>log4j-test.properties</exclude>
-										<exclude>log4j2-test.properties</exclude>
-									</excludes>
-								</filter>
 								<!-- drop entries into META-INF and NOTICE files for the dummy artifact -->
 								<filter>
 									<artifact>org.apache.flink:force-shading</artifact>
@@ -1776,15 +1795,6 @@ under the License.
 									<include>org.apache.flink:force-shading</include>
 								</includes>
 							</artifactSet>
-							<transformers combine.children="append">
-								<!-- The service transformer is needed to merge META-INF/services files -->
-								<transformer implementation="org.apache.maven.plugins.shade.resource.ServicesResourceTransformer"/>
-								<!-- The ApacheNoticeResourceTransformer collects and aggregates NOTICE files -->
-								<transformer implementation="org.apache.maven.plugins.shade.resource.ApacheNoticeResourceTransformer">
-									<projectName>Apache Flink</projectName>
-									<encoding>UTF-8</encoding>
-								</transformer>
-							</transformers>
 						</configuration>
 					</execution>
 				</executions>

From c161969bbf70d45dc78e2be2e5f5b11921142c82 Mon Sep 17 00:00:00 2001
From: Chesnay Schepler <chesnay@apache.org>
Date: Tue, 4 May 2021 10:41:14 +0200
Subject: [PATCH 2/4] [FLINK-22560][build] Filter maven metadata directory

---
 .../flink-connector-elasticsearch5/pom.xml      | 13 -------------
 flink-connectors/flink-connector-kafka/pom.xml  |  4 ++++
 flink-filesystems/flink-azure-fs-hadoop/pom.xml |  1 -
 .../flink-fs-hadoop-shaded/pom.xml              |  1 -
 flink-filesystems/flink-oss-fs-hadoop/pom.xml   |  1 -
 flink-filesystems/flink-s3-fs-hadoop/pom.xml    |  7 -------
 flink-filesystems/flink-s3-fs-presto/pom.xml    |  7 -------
 flink-kubernetes/pom.xml                        |  1 -
 flink-libraries/flink-gelly-scala/pom.xml       |  4 ++++
 flink-python/pom.xml                            |  1 -
 flink-runtime/pom.xml                           |  6 ------
 .../flink-test-utils/pom.xml                    |  1 -
 pom.xml                                         | 17 +++++++++++++++++
 13 files changed, 25 insertions(+), 39 deletions(-)

diff --git a/flink-connectors/flink-connector-elasticsearch5/pom.xml b/flink-connectors/flink-connector-elasticsearch5/pom.xml
index ef38e69b4e9e5..fdf56b802a9d1 100644
--- a/flink-connectors/flink-connector-elasticsearch5/pom.xml
+++ b/flink-connectors/flink-connector-elasticsearch5/pom.xml
@@ -245,19 +245,6 @@ under the License.
 										<exclude>log4j.properties</exclude>
 										<exclude>config/favicon.ico</exclude>
 										<exclude>mozilla/**</exclude>
-										<exclude>META-INF/maven/com*/**</exclude>
-										<exclude>META-INF/maven/io*/**</exclude>
-										<exclude>META-INF/maven/joda*/**</exclude>
-										<exclude>META-INF/maven/net*/**</exclude>
-										<exclude>META-INF/maven/org.an*/**</exclude>
-										<exclude>META-INF/maven/org.apache.h*/**</exclude>
-										<exclude>META-INF/maven/org.apache.commons/**</exclude>
-										<exclude>META-INF/maven/org.apache.flink/force-shading/**</exclude>
-										<exclude>META-INF/maven/org.apache.logging*/**</exclude>
-										<exclude>META-INF/maven/org.e*/**</exclude>
-										<exclude>META-INF/maven/org.h*/**</exclude>
-										<exclude>META-INF/maven/org.j*/**</exclude>
-										<exclude>META-INF/maven/org.y*/**</exclude>
 									</excludes>
 								</filter>
 								<filter>
diff --git a/flink-connectors/flink-connector-kafka/pom.xml b/flink-connectors/flink-connector-kafka/pom.xml
index e5b65c183dab1..b9b7d9f258bf3 100644
--- a/flink-connectors/flink-connector-kafka/pom.xml
+++ b/flink-connectors/flink-connector-kafka/pom.xml
@@ -247,6 +247,10 @@ under the License.
 							<goal>test-jar-no-fork</goal>
 						</goals>
 						<configuration>
+							<archive>
+								<!-- Globally exclude maven metadata, because it may accidentally bundle files we don't intend to -->
+								<addMavenDescriptor>false</addMavenDescriptor>
+							</archive>
 							<includes>
 								<include>**/KafkaTestEnvironmentImpl*</include>
 								<include>META-INF/LICENSE</include>
diff --git a/flink-filesystems/flink-azure-fs-hadoop/pom.xml b/flink-filesystems/flink-azure-fs-hadoop/pom.xml
index 906f6c3b1488b..b504c9580a341 100644
--- a/flink-filesystems/flink-azure-fs-hadoop/pom.xml
+++ b/flink-filesystems/flink-azure-fs-hadoop/pom.xml
@@ -192,7 +192,6 @@ under the License.
 										<exclude>properties.dtd</exclude>
 										<exclude>PropertyList-1.0.dtd</exclude>
 										<exclude>mozilla/**</exclude>
-										<exclude>META-INF/maven/**</exclude>
 										<exclude>META-INF/LICENSE.txt</exclude>
 										<exclude>META-INF/*.SF</exclude>
 										<exclude>META-INF/*.DSA</exclude>
diff --git a/flink-filesystems/flink-fs-hadoop-shaded/pom.xml b/flink-filesystems/flink-fs-hadoop-shaded/pom.xml
index 4a58e1b23b321..3c6fc6138d59e 100644
--- a/flink-filesystems/flink-fs-hadoop-shaded/pom.xml
+++ b/flink-filesystems/flink-fs-hadoop-shaded/pom.xml
@@ -259,7 +259,6 @@ under the License.
 									<excludes>
 										<exclude>properties.dtd</exclude>
 										<exclude>PropertyList-1.0.dtd</exclude>
-										<exclude>META-INF/maven/**</exclude>
 										<exclude>META-INF/services/javax.xml.stream.*</exclude>
 										<exclude>META-INF/LICENSE.txt</exclude>
 									</excludes>
diff --git a/flink-filesystems/flink-oss-fs-hadoop/pom.xml b/flink-filesystems/flink-oss-fs-hadoop/pom.xml
index e612f04520828..edf289c588d35 100644
--- a/flink-filesystems/flink-oss-fs-hadoop/pom.xml
+++ b/flink-filesystems/flink-oss-fs-hadoop/pom.xml
@@ -154,7 +154,6 @@ under the License.
 										<exclude>.gitkeep</exclude>
 										<exclude>mime.types</exclude>
 										<exclude>mozilla/**</exclude>
-										<exclude>META-INF/maven/**</exclude>
 									</excludes>
 								</filter>
 							</filters>
diff --git a/flink-filesystems/flink-s3-fs-hadoop/pom.xml b/flink-filesystems/flink-s3-fs-hadoop/pom.xml
index 41572d926302e..7dc3b1a9623c7 100644
--- a/flink-filesystems/flink-s3-fs-hadoop/pom.xml
+++ b/flink-filesystems/flink-s3-fs-hadoop/pom.xml
@@ -203,12 +203,6 @@ under the License.
 								</relocation>
 							</relocations>
 							<filters>
-								<filter>
-									<artifact>*</artifact>
-									<excludes>
-										<exclude>META-INF/maven/org.apache.flink/force-shading/**</exclude>
-									</excludes>
-								</filter>
 								<filter>
 									<artifact>*</artifact>
 									<excludes>
@@ -224,7 +218,6 @@ under the License.
 									<excludes>
 										<exclude>properties.dtd</exclude>
 										<exclude>PropertyList-1.0.dtd</exclude>
-										<exclude>META-INF/maven/**</exclude>
 										<exclude>META-INF/services/javax.xml.stream.*</exclude>
 										<exclude>META-INF/LICENSE.txt</exclude>
 									</excludes>
diff --git a/flink-filesystems/flink-s3-fs-presto/pom.xml b/flink-filesystems/flink-s3-fs-presto/pom.xml
index 3f690a3bc13a0..d3bb0508758d0 100644
--- a/flink-filesystems/flink-s3-fs-presto/pom.xml
+++ b/flink-filesystems/flink-s3-fs-presto/pom.xml
@@ -401,19 +401,12 @@ under the License.
 										<exclude>.gitkeep</exclude>
 										<exclude>mime.types</exclude>
 										<exclude>mozilla/**</exclude>
-										<exclude>META-INF/maven/**</exclude>
 										<exclude>META-INF/LICENSE.txt</exclude>
 									</excludes>
 								</filter>
 								<filter>
 									<artifact>*</artifact>
 									<excludes>
-										<exclude>META-INF/maven/org.weakref/**</exclude>
-										<exclude>META-INF/maven/org.hdrhistogram/**</exclude>
-										<exclude>META-INF/maven/joda-time/**</exclude>
-										<exclude>META-INF/maven/io.airlift/**</exclude>
-										<exclude>META-INF/maven/com*/**</exclude>
-										<exclude>META-INF/maven/org.apache.flink/force-shading/**</exclude>
 										<exclude>META-INF/LICENSE.txt</exclude>
 									</excludes>
 								</filter>
diff --git a/flink-kubernetes/pom.xml b/flink-kubernetes/pom.xml
index 04df35f75dc5e..62d74b87a807e 100644
--- a/flink-kubernetes/pom.xml
+++ b/flink-kubernetes/pom.xml
@@ -156,7 +156,6 @@ under the License.
 									<artifact>*:*</artifact>
 									<excludes>
 										<exclude>*.aut</exclude>
-										<exclude>META-INF/maven/**</exclude>
 										<exclude>META-INF/services/*com.fasterxml*</exclude>
 										<exclude>META-INF/proguard/**</exclude>
 										<exclude>OSGI-INF/**</exclude>
diff --git a/flink-libraries/flink-gelly-scala/pom.xml b/flink-libraries/flink-gelly-scala/pom.xml
index 20e0b14c6997c..730a3bd12ca08 100644
--- a/flink-libraries/flink-gelly-scala/pom.xml
+++ b/flink-libraries/flink-gelly-scala/pom.xml
@@ -215,6 +215,10 @@ under the License.
                     <descriptorRefs>
                         <descriptorRef>jar-with-dependencies</descriptorRef>
                     </descriptorRefs>
+                    <archive>
+                        <!-- Globally exclude maven metadata, because it may accidentally bundle files we don't intend to -->
+                        <addMavenDescriptor>false</addMavenDescriptor>
+                    </archive>
                 </configuration>
                 <executions>
                     <execution>
diff --git a/flink-python/pom.xml b/flink-python/pom.xml
index 43079e1509b84..1482e2ac7aec6 100644
--- a/flink-python/pom.xml
+++ b/flink-python/pom.xml
@@ -348,7 +348,6 @@ under the License.
 									<artifact>org.apache.beam:beam-sdks-java-core</artifact>
 									<excludes>
 										<exclude>org/apache/beam/repackaged/core/org/antlr/**</exclude>
-										<exclude>META-INF/maven/org.antlr/**</exclude>
 										<exclude>org/apache/beam/repackaged/core/org/apache/commons/compress/**</exclude>
 										<exclude>org/apache/beam/repackaged/core/org/apache/commons/lang3/**</exclude>
 									</excludes>
diff --git a/flink-runtime/pom.xml b/flink-runtime/pom.xml
index 9aa4246f7c71c..6f5d302e91922 100644
--- a/flink-runtime/pom.xml
+++ b/flink-runtime/pom.xml
@@ -572,12 +572,6 @@ under the License.
 										 	copied into this modules's NOTICE file. -->
 										<exclude>META-INF/NOTICE.txt</exclude>
 									</excludes>
-								</filter>	
-								<filter>
-									<artifact>*</artifact>
-									<excludes>
-										<exclude>META-INF/maven/io.netty/**</exclude>
-									</excludes>
 								</filter>
 							</filters>
 						</configuration>
diff --git a/flink-test-utils-parent/flink-test-utils/pom.xml b/flink-test-utils-parent/flink-test-utils/pom.xml
index 87258624ebc25..9f71e67ef79e1 100644
--- a/flink-test-utils-parent/flink-test-utils/pom.xml
+++ b/flink-test-utils-parent/flink-test-utils/pom.xml
@@ -146,7 +146,6 @@ under the License.
 								<filter>
 									<artifact>io.netty:netty</artifact>
 									<excludes>
-										<exclude>META-INF/maven/io.netty/**</exclude>
 										<!-- Only some of these licenses actually apply to the JAR and have been manually
 										 	placed in this module's resources directory. -->
 										<exclude>META-INF/license/**</exclude>
diff --git a/pom.xml b/pom.xml
index f46d0ac917897..47f76bf2cf355 100644
--- a/pom.xml
+++ b/pom.xml
@@ -1366,6 +1366,8 @@ under the License.
 				<version>2.4</version><!--$NO-MVN-MAN-VER$-->
 				<configuration>
 					<archive>
+						<!-- Globally exclude maven metadata, because it may accidentally bundle files we don't intend to -->
+						<addMavenDescriptor>false</addMavenDescriptor>
 						<manifest>
 							<addDefaultImplementationEntries>true</addDefaultImplementationEntries>
 							<addDefaultSpecificationEntries>true</addDefaultSpecificationEntries>
@@ -1742,6 +1744,21 @@ under the License.
 								<exclude>META-INF/*.SF</exclude>
 								<exclude>META-INF/*.DSA</exclude>
 								<exclude>META-INF/*.RSA</exclude>
+								<!-- META-INF/maven can contain 2 things:
+									- For archetypes, it contains an archetype-metadata.xml.
+									- For other jars, it contains the pom for all dependencies under the respective <groupId>/<artifactId>/ directory.
+
+								 	We want to exclude the poms because they may be under an incompatible license,
+								 	however the archetype metadata is required and we need to keep that around.
+
+								 	This pattern excludes directories under META-INF/maven.
+								 	('?*/**' does not work because '**' also matches zero directories;
+								 	everything that matches '?*' also matches '?*/**')
+
+									The initial '**' allows the pattern to also work for multi-release jars that may contain such entries under
+									'META-INF/versions/11/META-INF/maven/'.
+								 	-->
+								<exclude>**/META-INF/maven/?*/?*/**</exclude>
 							</excludes>
 						</filter>
 					</filters>

From f90acdecfdfcc8a306af832df7736d12a091a00d Mon Sep 17 00:00:00 2001
From: Chesnay Schepler <chesnay@apache.org>
Date: Wed, 5 May 2021 13:18:11 +0200
Subject: [PATCH 3/4] [FLINK-22560][build] Add dedicated name to flink-dist
 shade-plugin execution

---
 flink-dist/pom.xml | 1 +
 1 file changed, 1 insertion(+)

diff --git a/flink-dist/pom.xml b/flink-dist/pom.xml
index 4a546b3e2b0e0..f6e213b50c16f 100644
--- a/flink-dist/pom.xml
+++ b/flink-dist/pom.xml
@@ -675,6 +675,7 @@ under the License.
 				<artifactId>maven-shade-plugin</artifactId>
 				<executions>
 					<execution>
+						<id>shade-dist</id>
 						<phase>package</phase>
 						<goals>
 							<goal>shade</goal>

From 2632d0b01743c70bae508a635d94faab83d23052 Mon Sep 17 00:00:00 2001
From: Chesnay Schepler <chesnay@apache.org>
Date: Tue, 4 May 2021 10:41:46 +0200
Subject: [PATCH 4/4] [FLINK-22555][build][python] Exclude leftover jboss files

---
 flink-python/pom.xml | 1 +
 1 file changed, 1 insertion(+)

diff --git a/flink-python/pom.xml b/flink-python/pom.xml
index 1482e2ac7aec6..9bb15ec3f138a 100644
--- a/flink-python/pom.xml
+++ b/flink-python/pom.xml
@@ -356,6 +356,7 @@ under the License.
 									<artifact>org.apache.beam:beam-vendor-grpc-1_26_0</artifact>
 									<excludes>
 										<exclude>org/apache/beam/vendor/grpc/v1p26p0/org/jboss/**</exclude>
+										<exclude>schema/**</exclude>
 										<exclude>org/apache/beam/vendor/grpc/v1p26p0/org/eclipse/jetty/**</exclude>
 									</excludes>
 								</filter>