From fef43700973c00860856f923bf0a2171b5dd8cee Mon Sep 17 00:00:00 2001 From: Marcelo Vanzin Date: Mon, 11 Aug 2014 17:09:21 -0700 Subject: [PATCH] Unfork Optional.java. Just package the class from the guava dependency instead. Note the dependency-fu needed in core/pom.xml and assembly/pom.xml to not expose the guava dependency in the user's compilation classpath. --- assembly/pom.xml | 6 - core/pom.xml | 34 +++ .../java/com/google/common/base/Optional.java | 243 ------------------ project/Relocator.scala | 27 +- project/SparkBuild.scala | 49 +++- 5 files changed, 77 insertions(+), 282 deletions(-) delete mode 100644 core/src/main/java/com/google/common/base/Optional.java diff --git a/assembly/pom.xml b/assembly/pom.xml index 915f5f816102b..89788fd2e8dee 100644 --- a/assembly/pom.xml +++ b/assembly/pom.xml @@ -101,12 +101,6 @@ - - com.google.guava:guava - - com/google/common/base/Optional* - - *:* diff --git a/core/pom.xml b/core/pom.xml index 7c60cf10c3dc2..8cd0b7604f6de 100644 --- a/core/pom.xml +++ b/core/pom.xml @@ -68,9 +68,15 @@ org.eclipse.jetty jetty-server + com.google.guava guava + compile org.apache.commons @@ -322,6 +328,34 @@ + + org.apache.maven.plugins + maven-shade-plugin + + + package + + shade + + + false + + + com.google.guava:guava + + + + + com.google.guava:guava + + com/google/common/base/Optional* + + + + + + + diff --git a/core/src/main/java/com/google/common/base/Optional.java b/core/src/main/java/com/google/common/base/Optional.java deleted file mode 100644 index de3139c3cc9a8..0000000000000 --- a/core/src/main/java/com/google/common/base/Optional.java +++ /dev/null @@ -1,243 +0,0 @@ -/* - * Copyright (C) 2011 The Guava Authors - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -package com.google.common.base; - -import static com.google.common.base.Preconditions.checkNotNull; - -import com.google.common.annotations.Beta; -import com.google.common.annotations.GwtCompatible; - -import java.io.Serializable; -import java.util.Iterator; -import java.util.Set; - -import javax.annotation.Nullable; - -/** - * An immutable object that may contain a non-null reference to another object. Each - * instance of this type either contains a non-null reference, or contains nothing (in - * which case we say that the reference is "absent"); it is never said to "contain {@code - * null}". - * - *

A non-null {@code Optional} reference can be used as a replacement for a nullable - * {@code T} reference. It allows you to represent "a {@code T} that must be present" and - * a "a {@code T} that might be absent" as two distinct types in your program, which can - * aid clarity. - * - *

Some uses of this class include - * - *

    - *
  • As a method return type, as an alternative to returning {@code null} to indicate - * that no value was available - *
  • To distinguish between "unknown" (for example, not present in a map) and "known to - * have no value" (present in the map, with value {@code Optional.absent()}) - *
  • To wrap nullable references for storage in a collection that does not support - * {@code null} (though there are - * - * several other approaches to this that should be considered first) - *
- * - *

A common alternative to using this class is to find or create a suitable - * null object for the - * type in question. - * - *

This class is not intended as a direct analogue of any existing "option" or "maybe" - * construct from other programming environments, though it may bear some similarities. - * - *

See the Guava User Guide article on - * using {@code Optional}. - * - * @param the type of instance that can be contained. {@code Optional} is naturally - * covariant on this type, so it is safe to cast an {@code Optional} to {@code - * Optional} for any supertype {@code S} of {@code T}. - * @author Kurt Alfred Kluever - * @author Kevin Bourrillion - * @since 10.0 - */ -@GwtCompatible(serializable = true) -public abstract class Optional implements Serializable { - /** - * Returns an {@code Optional} instance with no contained reference. - */ - @SuppressWarnings("unchecked") - public static Optional absent() { - return (Optional) Absent.INSTANCE; - } - - /** - * Returns an {@code Optional} instance containing the given non-null reference. - */ - public static Optional of(T reference) { - return new Present(checkNotNull(reference)); - } - - /** - * If {@code nullableReference} is non-null, returns an {@code Optional} instance containing that - * reference; otherwise returns {@link Optional#absent}. - */ - public static Optional fromNullable(@Nullable T nullableReference) { - return (nullableReference == null) - ? Optional.absent() - : new Present(nullableReference); - } - - Optional() {} - - /** - * Returns {@code true} if this holder contains a (non-null) instance. - */ - public abstract boolean isPresent(); - - /** - * Returns the contained instance, which must be present. If the instance might be - * absent, use {@link #or(Object)} or {@link #orNull} instead. - * - * @throws IllegalStateException if the instance is absent ({@link #isPresent} returns - * {@code false}) - */ - public abstract T get(); - - /** - * Returns the contained instance if it is present; {@code defaultValue} otherwise. If - * no default value should be required because the instance is known to be present, use - * {@link #get()} instead. For a default value of {@code null}, use {@link #orNull}. - * - *

Note about generics: The signature {@code public T or(T defaultValue)} is overly - * restrictive. However, the ideal signature, {@code public S or(S)}, is not legal - * Java. As a result, some sensible operations involving subtypes are compile errors: - * 

   {@code
-   *
-   *   Optional optionalInt = getSomeOptionalInt();
-   *   Number value = optionalInt.or(0.5); // error
-   *
-   *   FluentIterable numbers = getSomeNumbers();
-   *   Optional first = numbers.first();
-   *   Number value = first.or(0.5); // error}
- * - * As a workaround, it is always safe to cast an {@code Optional} to {@code - * Optional}. Casting either of the above example {@code Optional} instances to {@code - * Optional} (where {@code Number} is the desired output type) solves the problem: - * 
   {@code
-   *
-   *   Optional optionalInt = (Optional) getSomeOptionalInt();
-   *   Number value = optionalInt.or(0.5); // fine
-   *
-   *   FluentIterable numbers = getSomeNumbers();
-   *   Optional first = (Optional) numbers.first();
-   *   Number value = first.or(0.5); // fine}
- */ - public abstract T or(T defaultValue); - - /** - * Returns this {@code Optional} if it has a value present; {@code secondChoice} - * otherwise. - */ - public abstract Optional or(Optional secondChoice); - - /** - * Returns the contained instance if it is present; {@code supplier.get()} otherwise. If the - * supplier returns {@code null}, a {@link NullPointerException} is thrown. - * - * @throws NullPointerException if the supplier returns {@code null} - */ - @Beta - public abstract T or(Supplier supplier); - - /** - * Returns the contained instance if it is present; {@code null} otherwise. If the - * instance is known to be present, use {@link #get()} instead. - */ - @Nullable - public abstract T orNull(); - - /** - * Returns an immutable singleton {@link Set} whose only element is the contained instance - * if it is present; an empty immutable {@link Set} otherwise. - * - * @since 11.0 - */ - public abstract Set asSet(); - - /** - * If the instance is present, it is transformed with the given {@link Function}; otherwise, - * {@link Optional#absent} is returned. If the function returns {@code null}, a - * {@link NullPointerException} is thrown. - * - * @throws NullPointerException if the function returns {@code null} - * - * @since 12.0 - */ - public abstract Optional transform(Function function); - - /** - * Returns {@code true} if {@code object} is an {@code Optional} instance, and either - * the contained references are {@linkplain Object#equals equal} to each other or both - * are absent. Note that {@code Optional} instances of differing parameterized types can - * be equal. - */ - @Override - public abstract boolean equals(@Nullable Object object); - - /** - * Returns a hash code for this instance. - */ - @Override - public abstract int hashCode(); - - /** - * Returns a string representation for this instance. The form of this string - * representation is unspecified. - */ - @Override - public abstract String toString(); - - /** - * Returns the value of each present instance from the supplied {@code optionals}, in order, - * skipping over occurrences of {@link Optional#absent}. Iterators are unmodifiable and are - * evaluated lazily. - * - * @since 11.0 (generics widened in 13.0) - */ - @Beta - public static Iterable presentInstances( - final Iterable> optionals) { - checkNotNull(optionals); - return new Iterable() { - @Override - public Iterator iterator() { - return new AbstractIterator() { - private final Iterator> iterator = - checkNotNull(optionals.iterator()); - - @Override - protected T computeNext() { - while (iterator.hasNext()) { - Optional optional = iterator.next(); - if (optional.isPresent()) { - return optional.get(); - } - } - return endOfData(); - } - }; - } - }; - } - - private static final long serialVersionUID = 0; -} diff --git a/project/Relocator.scala b/project/Relocator.scala index ed209e904506e..c8c0a333b77b7 100644 --- a/project/Relocator.scala +++ b/project/Relocator.scala @@ -88,18 +88,9 @@ class RelocatorRemapper(relocators: List[Relocator]) extends Remapper { * should be relocated are moved to a new location, and all classes are passed through the * remapper so that references to relocated classes are fixed. * - * Note about `preferLocal`: this is a hack to make sure that we always ship the Spark-compiled - * version of Guava's `Optional` class. Unlike maven, sbt-assembly doesn't seem to allow filtering - * of specific entries in dependencies. It also does not provide information about where does - * a particular file come from. The only hint is that the temp path for the file ends with a "_dir" - * when it comes from a directory, and with a hash when it comes from a jar file. So for classes - * that match a regex in the `preferLocal` list, we choose the first class file in a local - * directory. - * * @param relocators List of relocators to apply to classes being shaded. - * @param preferLocal List of regexes that match classes for which a local version is preferred. */ -class ShadeStrategy(relocators: List[Relocator], preferLocal: List[Regex]) extends MergeStrategy { +class ShadeStrategy(relocators: List[Relocator]) extends MergeStrategy { private val remapper = new RelocatorRemapper(relocators) @@ -111,7 +102,7 @@ class ShadeStrategy(relocators: List[Relocator], preferLocal: List[Regex]) exten (files.head, path) } else { val className = path.substring(0, path.length() - ".class".length()) - (remap(chooseFile(path, files), tempDir), remapper.rename(className) + ".class") + (remap(files.head, tempDir), remapper.rename(className) + ".class") } Right(Seq(file -> newPath)) } @@ -139,18 +130,4 @@ class ShadeStrategy(relocators: List[Relocator], preferLocal: List[Regex]) exten } } - private def chooseFile(path: String, files: Seq[File]): File = { - if (!preferLocal.filter { r => r.pattern.matcher(path).matches() }.isEmpty) { - def isLocal(f: File) = { - val abs = f.getAbsolutePath() - val dir = abs.substring(0, abs.length() - path.length()) - dir.endsWith("_dir") - } - - files.filter(isLocal).orElse(files)(0) - } else { - files.head - } - } - } diff --git a/project/SparkBuild.scala b/project/SparkBuild.scala index 226fd63a20314..42775e8f39336 100644 --- a/project/SparkBuild.scala +++ b/project/SparkBuild.scala @@ -30,10 +30,12 @@ object BuildCommons { private val buildLocation = file(".").getAbsoluteFile.getParentFile - val allProjects@Seq(bagel, catalyst, core, graphx, hive, hiveThriftServer, mllib, repl, spark, + val coreProject@Seq(core) = Seq("core").map(ProjectRef(buildLocation, _)) + + val allProjects@Seq(bagel, catalyst, graphx, hive, hiveThriftServer, mllib, repl, spark, sql, streaming, streamingFlumeSink, streamingFlume, streamingKafka, streamingMqtt, streamingTwitter, streamingZeromq) = - Seq("bagel", "catalyst", "core", "graphx", "hive", "hive-thriftserver", "mllib", "repl", + Seq("bagel", "catalyst", "graphx", "hive", "hive-thriftserver", "mllib", "repl", "spark", "sql", "streaming", "streaming-flume-sink", "streaming-flume", "streaming-kafka", "streaming-mqtt", "streaming-twitter", "streaming-zeromq").map(ProjectRef(buildLocation, _)) @@ -152,15 +154,19 @@ object SparkBuild extends PomBuild { // Note ordering of these settings matter. /* Enable shared settings on all projects */ - (allProjects ++ optionallyEnabledProjects ++ assemblyProjects).foreach(enable(sharedSettings)) + (coreProject ++ allProjects ++ optionallyEnabledProjects ++ assemblyProjects) + .foreach(enable(sharedSettings)) /* Enable tests settings for all projects except examples, assembly and tools */ - (allProjects ++ optionallyEnabledProjects).foreach(enable(TestSettings.settings)) + (coreProject ++ allProjects ++ optionallyEnabledProjects).foreach(enable(TestSettings.settings)) // TODO: Add Sql to mima checks - allProjects.filterNot(x => Seq(spark, sql, hive, hiveThriftServer, catalyst, repl, + (coreProject ++ allProjects).filterNot(x => Seq(spark, sql, hive, hiveThriftServer, catalyst, repl, streamingFlumeSink).contains(x)).foreach(x => enable(MimaBuild.mimaSettings(sparkHome, x))(x)) + /* Set up assembly settings for the core project. */ + coreProject.foreach(enable(CoreAssembly.settings)) + /* Enable Assembly for all assembly projects */ assemblyProjects.foreach(enable(Assembly.settings)) @@ -255,9 +261,7 @@ object Assembly { private val relocators = List( new Relocator("com.google", "org.spark-project.guava", Seq("com\\.google\\.common\\..*".r), Seq("com\\.google\\.common\\.base\\.Optional.*".r))) - private val localFilters = List( - "com/google/common/base/Optional.*".r) - private val shade = new ShadeStrategy(relocators, localFilters) + private val shade = new ShadeStrategy(relocators) lazy val settings = assemblySettings ++ Seq( test in assembly := {}, @@ -278,6 +282,35 @@ object Assembly { } +object CoreAssembly { + import sbtassembly.Plugin._ + import AssemblyKeys._ + + lazy val settings = assemblySettings ++ Seq( + publishArtifact in (Compile, packageBin) := false, + test in assembly := {}, + jarName in assembly <<= (name, scalaVersion, version) map { + _ + "_" + _ + "-" + _ + "-assembly.jar" + }, + excludedJars in assembly := { + val cp = (fullClasspath in assembly).value + cp filter {!_.data.getName.startsWith("guava")} + }, + mergeStrategy in assembly := { + case PathList("com", "google", "common", xs @ _*) => + if (xs.size == 2 && xs(0) == "base" && xs(1).startsWith("Optional")) { + MergeStrategy.first + } else { + MergeStrategy.discard + } + case PathList("META-INF", "maven", "com.google.guava", xs @ _*) => MergeStrategy.discard + case m if m.toLowerCase.endsWith("manifest.mf") => MergeStrategy.discard + case _ => MergeStrategy.first + } + ) ++ addArtifact(Artifact("spark-core"), assembly).settings + +} + object Unidoc { import BuildCommons._