From 4a4ed4202eac66bc288c8fcb2107b0608cc1e32f Mon Sep 17 00:00:00 2001
From: Marcelo Vanzin <vanzin@cloudera.com>
Date: Fri, 21 Nov 2014 12:25:15 -0800
Subject: [PATCH 1/4] [SPARK-4809] Rework Guava library shading.

The current way of shading Guava is a little problematic. Code that
depends on "spark-core" does not see the transitive dependency, yet
classes in "spark-core" actually depend on Guava. So it's a little
tricky to run unit tests that use spark-core classes, since you need
a compatible version of Guava in your dependencies when running the
tests. This can become a little tricky, and is kind of a bad user
experience.

This change modifies the way Guava is shaded so that it's applied
uniformly across the Spark build. This means Guava is shaded inside
spark-core itself, so that the dependency issues above are solved.
Aside from that, all Spark sub-modules have their Guava references
relocated, so that they refer to the relocated classes now packaged
inside spark-core. Before, this was only done by the time the assembly
was built, so project that did not end up inside the assembly (such
as streaming backends) could still reference the original location
of Guava classes).

This relocation does not apply to the sub-modules under network/,
though. For those cases, we want to keep the Guava dependency alive,
since we want to use the same Guava as the rest of the Yarn NM
when deploying the auxiliary shuffle service. For this reason, also,
the network/ dependencies are shaded into the spark-core artifact
too, so that the raw Guava dependency doesn't leak.
---
 assembly/pom.xml        | 20 -------------
 core/pom.xml            | 50 +++++++++-----------------------
 examples/pom.xml        | 64 +++++++++++------------------------------
 network/common/pom.xml  | 14 +++++----
 network/shuffle/pom.xml | 10 +++++++
 network/yarn/pom.xml    |  2 ++
 pom.xml                 | 16 ++++++++++-
 streaming/pom.xml       | 18 ++++++++----
 8 files changed, 79 insertions(+), 115 deletions(-)

diff --git a/assembly/pom.xml b/assembly/pom.xml
index 4e2b773e7d2f3..c006d0d55f13b 100644
--- a/assembly/pom.xml
+++ b/assembly/pom.xml
@@ -43,12 +43,6 @@
   </properties>
 
   <dependencies>
-    <!-- Promote Guava to compile scope in this module so it's included while shading. -->
-    <dependency>
-      <groupId>com.google.guava</groupId>
-      <artifactId>guava</artifactId>
-      <scope>compile</scope>
-    </dependency>
     <dependency>
       <groupId>org.apache.spark</groupId>
       <artifactId>spark-core_${scala.binary.version}</artifactId>
@@ -133,20 +127,6 @@
               <goal>shade</goal>
             </goals>
             <configuration>
-              <relocations>
-                <relocation>
-                  <pattern>com.google</pattern>
-                  <shadedPattern>org.spark-project.guava</shadedPattern>
-                  <includes>
-                    <include>com.google.common.**</include>
-                  </includes>
-                  <excludes>
-                    <exclude>com/google/common/base/Absent*</exclude>
-                    <exclude>com/google/common/base/Optional*</exclude>
-                    <exclude>com/google/common/base/Present*</exclude>
-                  </excludes>
-                </relocation>
-              </relocations>
               <transformers>
                 <transformer implementation="org.apache.maven.plugins.shade.resource.ServicesResourceTransformer" />
                 <transformer implementation="org.apache.maven.plugins.shade.resource.AppendingTransformer">
diff --git a/core/pom.xml b/core/pom.xml
index 1feb00b3a7fb8..558b9387cbd37 100644
--- a/core/pom.xml
+++ b/core/pom.xml
@@ -368,42 +368,6 @@
           <verbose>true</verbose>
         </configuration>
       </plugin>
-      <plugin>
-        <groupId>org.apache.maven.plugins</groupId>
-        <artifactId>maven-shade-plugin</artifactId>
-        <executions>
-          <execution>
-            <phase>package</phase>
-            <goals>
-              <goal>shade</goal>
-            </goals>
-            <configuration>
-              <shadedArtifactAttached>false</shadedArtifactAttached>
-              <artifactSet>
-                <includes>
-                  <include>com.google.guava:guava</include>
-                </includes>
-              </artifactSet>
-              <filters>
-                <!-- See comment in the guava dependency declaration above. -->
-                <filter>
-                  <artifact>com.google.guava:guava</artifact>
-                  <includes>
-                    <include>com/google/common/base/Absent*</include>
-                    <include>com/google/common/base/Optional*</include>
-                    <include>com/google/common/base/Present*</include>
-                  </includes>
-                </filter>
-              </filters>
-            </configuration>
-          </execution>
-        </executions>
-      </plugin>
-      <!--
-        Copy guava to the build directory. This is needed to make the SPARK_PREPEND_CLASSES
-        option work in compute-classpath.sh, since it would put the non-shaded Spark classes in
-        the runtime classpath.
-      -->
       <plugin>
         <groupId>org.apache.maven.plugins</groupId>
         <artifactId>maven-dependency-plugin</artifactId>
@@ -426,6 +390,20 @@
           </execution>
         </executions>
       </plugin>
+      <plugin>
+        <groupId>org.apache.maven.plugins</groupId>
+        <artifactId>maven-shade-plugin</artifactId>
+        <configuration>
+          <shadedArtifactAttached>false</shadedArtifactAttached>
+          <artifactSet>
+            <includes>
+              <include>com.google.guava:guava</include>
+              <include>org.apache.spark:spark-network-common_${scala.binary.version}</include>
+              <include>org.apache.spark:spark-network-shuffle_${scala.binary.version}</include>
+            </includes>
+          </artifactSet>
+        </configuration>
+      </plugin>
     </plugins>
 
     <resources>
diff --git a/examples/pom.xml b/examples/pom.xml
index 8713230e1e8ed..983e50b6dd887 100644
--- a/examples/pom.xml
+++ b/examples/pom.xml
@@ -35,12 +35,6 @@
   <url>http://spark.apache.org/</url>
 
   <dependencies>
-    <!-- Promote Guava to compile scope in this module so it's included while shading. -->
-    <dependency>
-      <groupId>com.google.guava</groupId>
-      <artifactId>guava</artifactId>
-      <scope>compile</scope>
-    </dependency>
     <dependency>
       <groupId>org.apache.spark</groupId>
       <artifactId>spark-core_${scala.binary.version}</artifactId>
@@ -322,12 +316,6 @@
             </includes>
           </artifactSet>
           <filters>
-            <filter>
-              <artifact>com.google.guava:guava</artifact>
-              <excludes>
-                <exclude>com/google/common/base/Optional*</exclude>
-              </excludes>
-            </filter>
             <filter>
               <artifact>*:*</artifact>
               <excludes>
@@ -337,42 +325,22 @@
               </excludes>
             </filter>
           </filters>
+          <relocations combine.children="append">
+            <relocation>
+              <pattern>org.apache.commons.math3</pattern>
+              <shadedPattern>org.spark-project.commons.math3</shadedPattern>
+            </relocation>
+          </relocations>
+          <transformers>
+            <transformer implementation="org.apache.maven.plugins.shade.resource.ServicesResourceTransformer" />
+            <transformer implementation="org.apache.maven.plugins.shade.resource.AppendingTransformer">
+              <resource>reference.conf</resource>
+            </transformer>
+            <transformer implementation="org.apache.maven.plugins.shade.resource.DontIncludeResourceTransformer">
+              <resource>log4j.properties</resource>
+            </transformer>
+          </transformers>
         </configuration>
-        <executions>
-          <execution>
-            <phase>package</phase>
-            <goals>
-              <goal>shade</goal>
-            </goals>
-            <configuration>
-              <relocations>
-                <relocation>
-                  <pattern>com.google</pattern>
-                  <shadedPattern>org.spark-project.guava</shadedPattern>
-                  <includes>
-                    <include>com.google.common.**</include>
-                  </includes>
-                  <excludes>
-                    <exclude>com.google.common.base.Optional**</exclude>
-                  </excludes>
-                </relocation>
-                <relocation>
-                  <pattern>org.apache.commons.math3</pattern>
-                  <shadedPattern>org.spark-project.commons.math3</shadedPattern>
-                </relocation>
-              </relocations>
-              <transformers>
-                <transformer implementation="org.apache.maven.plugins.shade.resource.ServicesResourceTransformer" />
-                <transformer implementation="org.apache.maven.plugins.shade.resource.AppendingTransformer">
-                  <resource>reference.conf</resource>
-                </transformer>
-                <transformer implementation="org.apache.maven.plugins.shade.resource.DontIncludeResourceTransformer">
-                  <resource>log4j.properties</resource>
-                </transformer>
-              </transformers>
-            </configuration>
-          </execution>
-        </executions>
       </plugin>
     </plugins>
   </build>
@@ -416,7 +384,7 @@
       </properties>
     </profile>
     <profile>
-      <!-- We add a source directory specific to Scala 2.10 since Kafka 
+      <!-- We add a source directory specific to Scala 2.10 since Kafka
            only works with it -->
       <id>scala-2.10</id>
       <activation>
diff --git a/network/common/pom.xml b/network/common/pom.xml
index baca859fa5011..f1e6a105d727d 100644
--- a/network/common/pom.xml
+++ b/network/common/pom.xml
@@ -92,11 +92,6 @@
         <artifactId>maven-jar-plugin</artifactId>
         <version>2.2</version>
         <executions>
-          <execution>
-            <goals>
-              <goal>test-jar</goal>
-            </goals>
-          </execution>
           <execution>
             <id>test-jar-on-test-compile</id>
             <phase>test-compile</phase>
@@ -106,6 +101,15 @@
           </execution>
         </executions>
       </plugin>
+      <!-- Disable relocation in the shade plugin invocation. -->
+      <plugin>
+        <groupId>org.apache.maven.plugins</groupId>
+        <artifactId>maven-shade-plugin</artifactId>
+        <configuration>
+          <shadeTestJar>true</shadeTestJar>
+          <relocations combine.self="override" />
+        </configuration>
+      </plugin>
     </plugins>
   </build>
 </project>
diff --git a/network/shuffle/pom.xml b/network/shuffle/pom.xml
index 12468567c3aed..f746e96a75a1a 100644
--- a/network/shuffle/pom.xml
+++ b/network/shuffle/pom.xml
@@ -93,5 +93,15 @@
   <build>
     <outputDirectory>target/scala-${scala.binary.version}/classes</outputDirectory>
     <testOutputDirectory>target/scala-${scala.binary.version}/test-classes</testOutputDirectory>
+    <plugins>
+      <!-- Disable relocation in the shade plugin invocation. -->
+      <plugin>
+        <groupId>org.apache.maven.plugins</groupId>
+        <artifactId>maven-shade-plugin</artifactId>
+        <configuration>
+          <relocations combine.self="override" />
+        </configuration>
+      </plugin>
+    </plugins>
   </build>
 </project>
diff --git a/network/yarn/pom.xml b/network/yarn/pom.xml
index acec8f18f2b5c..ba4bbf8c06631 100644
--- a/network/yarn/pom.xml
+++ b/network/yarn/pom.xml
@@ -76,6 +76,8 @@
               </excludes>
             </filter>
           </filters>
+          <!-- Disable relocation declared in root pom. -->
+          <relocations combine.self="override" />
         </configuration>
         <executions>
           <execution>
diff --git a/pom.xml b/pom.xml
index 6c1c1214a7d3e..87bb5c7ea0002 100644
--- a/pom.xml
+++ b/pom.xml
@@ -1094,7 +1094,10 @@
           </execution>
         </executions>
       </plugin>
-      <!-- The shade plug-in is used here to create effective pom's (see SPARK-3812). -->
+      <!--
+        The shade plug-in is used here to create effective pom's (see SPARK-3812), and also
+        remove references from the shaded libraries from artifacts published by Spark.
+      -->
       <plugin>
         <groupId>org.apache.maven.plugins</groupId>
         <artifactId>maven-shade-plugin</artifactId>
@@ -1105,6 +1108,17 @@
               <include>org.spark-project.spark:unused</include>
             </includes>
           </artifactSet>
+          <relocations>
+            <relocation>
+              <pattern>com.google.common</pattern>
+              <shadedPattern>org.spark-project.guava</shadedPattern>
+              <excludes>
+                <exclude>com/google/common/base/Absent*</exclude>
+                <exclude>com/google/common/base/Optional*</exclude>
+                <exclude>com/google/common/base/Present*</exclude>
+              </excludes>
+            </relocation>
+          </relocations>
         </configuration>
         <executions>
           <execution>
diff --git a/streaming/pom.xml b/streaming/pom.xml
index b8b8f2e6cab65..7d75e3ce81585 100644
--- a/streaming/pom.xml
+++ b/streaming/pom.xml
@@ -77,14 +77,14 @@
         <groupId>org.scalatest</groupId>
         <artifactId>scalatest-maven-plugin</artifactId>
       </plugin>
-      
-      <!-- 
-           This plugin forces the generation of jar containing streaming test classes, 
+
+      <!--
+           This plugin forces the generation of jar containing streaming test classes,
            so that the tests classes of external modules can use them. The two execution profiles
            are necessary - first one for 'mvn package', second one for 'mvn test-compile'. Ideally,
-           'mvn compile' should not compile test classes and therefore should not need this. 
+           'mvn compile' should not compile test classes and therefore should not need this.
            However, an open Maven bug (http://jira.codehaus.org/browse/MNG-3559)
-           causes the compilation to fail if streaming test-jar is not generated. Hence, the 
+           causes the compilation to fail if streaming test-jar is not generated. Hence, the
            second execution profile for 'mvn test-compile'.
       -->
       <plugin>
@@ -105,6 +105,14 @@
           </execution>
         </executions>
       </plugin>
+
+      <plugin>
+        <groupId>org.apache.maven.plugins</groupId>
+        <artifactId>maven-shade-plugin</artifactId>
+        <configuration>
+          <shadeTestJar>true</shadeTestJar>
+        </configuration>
+      </plugin>
     </plugins>
   </build>
 </project>

From 107d7dac27ee4c9131588b8afabda3bc64902e80 Mon Sep 17 00:00:00 2001
From: Marcelo Vanzin <vanzin@cloudera.com>
Date: Tue, 6 Jan 2015 19:09:12 -0800
Subject: [PATCH 2/4] Add fix for SPARK-5052 (PR #3874).

---
 pom.xml | 2 ++
 1 file changed, 2 insertions(+)

diff --git a/pom.xml b/pom.xml
index 2ee7dfdf816b0..e9b776d51f863 100644
--- a/pom.xml
+++ b/pom.xml
@@ -1086,8 +1086,10 @@
               <shadedPattern>org.spark-project.guava</shadedPattern>
               <excludes>
                 <exclude>com/google/common/base/Absent*</exclude>
+                <exclude>com/google/common/base/Function</exclude>
                 <exclude>com/google/common/base/Optional*</exclude>
                 <exclude>com/google/common/base/Present*</exclude>
+                <exclude>com/google/common/base/Supplier</exclude>
               </excludes>
             </relocation>
           </relocations>

From b3104fcb12d5d85f7514f575beeb671816b02251 Mon Sep 17 00:00:00 2001
From: Marcelo Vanzin <vanzin@cloudera.com>
Date: Mon, 26 Jan 2015 13:52:04 -0800
Subject: [PATCH 3/4] Add comment.

---
 pom.xml | 4 ++++
 1 file changed, 4 insertions(+)

diff --git a/pom.xml b/pom.xml
index fdf7d60375393..c34a4697e6ca8 100644
--- a/pom.xml
+++ b/pom.xml
@@ -1284,6 +1284,10 @@
               <pattern>com.google.common</pattern>
               <shadedPattern>org.spark-project.guava</shadedPattern>
               <excludes>
+                <!--
+                  These classes cannot be relocated, because the Java API exposes the
+                  "Optional" type; the others are referenced by the Optional class.
+                -->
                 <exclude>com/google/common/base/Absent*</exclude>
                 <exclude>com/google/common/base/Function</exclude>
                 <exclude>com/google/common/base/Optional*</exclude>

From 3c93e42a5e9474b33aa53f7fd6f22998d44a8c52 Mon Sep 17 00:00:00 2001
From: Marcelo Vanzin <vanzin@cloudera.com>
Date: Tue, 27 Jan 2015 17:02:40 -0800
Subject: [PATCH 4/4] Shade Guava in the network-common artifact.

This makes all Spark-generated artifacts use the Guava defined in
the build file at runtime too. The leaked classes ("Optional" and
friends) are now exposed in a few places where they weren't before,
but hopefully that won't cause issues.
---
 core/pom.xml            | 24 ------------------------
 network/common/pom.xml  | 16 ++++++++++++----
 network/shuffle/pom.xml | 11 -----------
 network/yarn/pom.xml    |  2 --
 4 files changed, 12 insertions(+), 41 deletions(-)

diff --git a/core/pom.xml b/core/pom.xml
index 42150ada65952..3c51b2d6b58f9 100644
--- a/core/pom.xml
+++ b/core/pom.xml
@@ -106,16 +106,6 @@
       <groupId>org.eclipse.jetty</groupId>
       <artifactId>jetty-server</artifactId>
     </dependency>
-    <!--
-      Promote Guava to "compile" so that maven-shade-plugin picks it up (for packaging the Optional
-      class exposed in the Java API). The plugin will then remove this dependency from the published
-      pom, so that Guava does not pollute the client's compilation classpath.
-    -->
-    <dependency>
-      <groupId>com.google.guava</groupId>
-      <artifactId>guava</artifactId>
-      <scope>compile</scope>
-    </dependency>
     <dependency>
       <groupId>org.apache.commons</groupId>
       <artifactId>commons-lang3</artifactId>
@@ -372,20 +362,6 @@
           </execution>
         </executions>
       </plugin>
-      <plugin>
-        <groupId>org.apache.maven.plugins</groupId>
-        <artifactId>maven-shade-plugin</artifactId>
-        <configuration>
-          <shadedArtifactAttached>false</shadedArtifactAttached>
-          <artifactSet>
-            <includes>
-              <include>com.google.guava:guava</include>
-              <include>org.apache.spark:spark-network-common_${scala.binary.version}</include>
-              <include>org.apache.spark:spark-network-shuffle_${scala.binary.version}</include>
-            </includes>
-          </artifactSet>
-        </configuration>
-      </plugin>
     </plugins>
 
     <resources>
diff --git a/network/common/pom.xml b/network/common/pom.xml
index 565bf822caac7..5a9bbe105d9f1 100644
--- a/network/common/pom.xml
+++ b/network/common/pom.xml
@@ -48,10 +48,15 @@
       <artifactId>slf4j-api</artifactId>
       <scope>provided</scope>
     </dependency>
+    <!--
+      Promote Guava to "compile" so that maven-shade-plugin picks it up (for packaging the Optional
+      class exposed in the Java API). The plugin will then remove this dependency from the published
+      pom, so that Guava does not pollute the client's compilation classpath.
+    -->
     <dependency>
       <groupId>com.google.guava</groupId>
       <artifactId>guava</artifactId>
-      <scope>provided</scope>
+      <scope>compile</scope>
     </dependency>
 
     <!-- Test dependencies -->
@@ -96,13 +101,16 @@
           </execution>
         </executions>
       </plugin>
-      <!-- Disable relocation in the shade plugin invocation. -->
       <plugin>
         <groupId>org.apache.maven.plugins</groupId>
         <artifactId>maven-shade-plugin</artifactId>
         <configuration>
-          <shadeTestJar>true</shadeTestJar>
-          <relocations combine.self="override" />
+          <shadedArtifactAttached>false</shadedArtifactAttached>
+          <artifactSet>
+            <includes>
+              <include>com.google.guava:guava</include>
+            </includes>
+          </artifactSet>
         </configuration>
       </plugin>
     </plugins>
diff --git a/network/shuffle/pom.xml b/network/shuffle/pom.xml
index 36842ebe5ec6e..c2d0300ecd904 100644
--- a/network/shuffle/pom.xml
+++ b/network/shuffle/pom.xml
@@ -52,7 +52,6 @@
     <dependency>
       <groupId>com.google.guava</groupId>
       <artifactId>guava</artifactId>
-      <scope>provided</scope>
     </dependency>
 
     <!-- Test dependencies -->
@@ -88,15 +87,5 @@
   <build>
     <outputDirectory>target/scala-${scala.binary.version}/classes</outputDirectory>
     <testOutputDirectory>target/scala-${scala.binary.version}/test-classes</testOutputDirectory>
-    <plugins>
-      <!-- Disable relocation in the shade plugin invocation. -->
-      <plugin>
-        <groupId>org.apache.maven.plugins</groupId>
-        <artifactId>maven-shade-plugin</artifactId>
-        <configuration>
-          <relocations combine.self="override" />
-        </configuration>
-      </plugin>
-    </plugins>
   </build>
 </project>
diff --git a/network/yarn/pom.xml b/network/yarn/pom.xml
index ba4bbf8c06631..acec8f18f2b5c 100644
--- a/network/yarn/pom.xml
+++ b/network/yarn/pom.xml
@@ -76,8 +76,6 @@
               </excludes>
             </filter>
           </filters>
-          <!-- Disable relocation declared in root pom. -->
-          <relocations combine.self="override" />
         </configuration>
         <executions>
           <execution>