From 14ac009b4a86df86352a40f3e889417bcae22f00 Mon Sep 17 00:00:00 2001 From: Stas Levin Date: Sun, 30 Apr 2017 13:56:40 +0300 Subject: [PATCH] [BEAM-1676] Internalize ClassPath from Guava 21.0 to fix the failures related to having whitespaces in resource names. --- .../main/resources/beam/findbugs-filter.xml | 6 + .../org/apache/beam/sdk/util/ApiSurface.java | 4 +- .../org/apache/beam/sdk/util/ClassPath.java | 544 ++++++++++++++++++ 3 files changed, 551 insertions(+), 3 deletions(-) create mode 100644 sdks/java/core/src/main/java/org/apache/beam/sdk/util/ClassPath.java diff --git a/sdks/java/build-tools/src/main/resources/beam/findbugs-filter.xml b/sdks/java/build-tools/src/main/resources/beam/findbugs-filter.xml index cf7d668540fd..c96257015806 100644 --- a/sdks/java/build-tools/src/main/resources/beam/findbugs-filter.xml +++ b/sdks/java/build-tools/src/main/resources/beam/findbugs-filter.xml @@ -378,6 +378,12 @@ + + + + + + diff --git a/sdks/java/core/src/main/java/org/apache/beam/sdk/util/ApiSurface.java b/sdks/java/core/src/main/java/org/apache/beam/sdk/util/ApiSurface.java index 9530e88d3c47..8c2b988e6b03 100644 --- a/sdks/java/core/src/main/java/org/apache/beam/sdk/util/ApiSurface.java +++ b/sdks/java/core/src/main/java/org/apache/beam/sdk/util/ApiSurface.java @@ -33,8 +33,6 @@ import com.google.common.collect.Multimaps; import com.google.common.collect.Ordering; import com.google.common.collect.Sets; -import com.google.common.reflect.ClassPath; -import com.google.common.reflect.ClassPath.ClassInfo; import com.google.common.reflect.Invokable; import com.google.common.reflect.Parameter; import com.google.common.reflect.TypeToken; @@ -353,7 +351,7 @@ public ApiSurface includingPackage(String packageName, ClassLoader classLoader) ClassPath classPath = ClassPath.from(classLoader); Set> newRootClasses = Sets.newHashSet(); - for (ClassInfo classInfo : classPath.getTopLevelClassesRecursive(packageName)) { + for (ClassPath.ClassInfo classInfo : classPath.getTopLevelClassesRecursive(packageName)) { Class clazz = classInfo.load(); if (exposed(clazz.getModifiers())) { newRootClasses.add(clazz); diff --git a/sdks/java/core/src/main/java/org/apache/beam/sdk/util/ClassPath.java b/sdks/java/core/src/main/java/org/apache/beam/sdk/util/ClassPath.java new file mode 100644 index 000000000000..35632ed063bf --- /dev/null +++ b/sdks/java/core/src/main/java/org/apache/beam/sdk/util/ClassPath.java @@ -0,0 +1,544 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.beam.sdk.util; + +import static com.google.common.base.Preconditions.checkArgument; +import static com.google.common.base.Preconditions.checkNotNull; +import com.google.common.annotations.Beta; +import com.google.common.annotations.VisibleForTesting; +import com.google.common.base.CharMatcher; +import com.google.common.base.Predicate; +import com.google.common.base.Splitter; +import com.google.common.collect.FluentIterable; +import com.google.common.collect.ImmutableMap; +import com.google.common.collect.ImmutableSet; +import com.google.common.collect.Maps; +import com.google.common.collect.MultimapBuilder; +import com.google.common.collect.SetMultimap; +import com.google.common.collect.Sets; +import com.google.common.io.ByteSource; +import com.google.common.io.CharSource; +import com.google.common.io.Resources; +import com.google.common.reflect.Reflection; +import java.io.File; +import java.io.IOException; +import java.net.MalformedURLException; +import java.net.URISyntaxException; +import java.net.URL; +import java.net.URLClassLoader; +import java.nio.charset.Charset; +import java.util.Enumeration; +import java.util.LinkedHashMap; +import java.util.Map; +import java.util.NoSuchElementException; +import java.util.Set; +import java.util.jar.Attributes; +import java.util.jar.JarEntry; +import java.util.jar.JarFile; +import java.util.jar.Manifest; +import javax.annotation.Nullable; +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; + +/** + * Scans the source of a {@link ClassLoader} and finds all loadable classes and resources. + * + *

Warning: Currently only {@link URLClassLoader} and only {@code file://} urls are + * supported. + *

+ * + *

Based on Ben Yu's implementation in + * Guava. + *

+ * + *

Note: Internalised here to avoid a forced upgrade to + * Guava 21.0 which requires + * Java 8. + *

+ */ +@Beta +final class ClassPath { + + private static final Logger logger = LoggerFactory.getLogger(ClassPath.class.getName()); + + private static final Predicate IS_TOP_LEVEL = + new Predicate() { + + @Override + public boolean apply(ClassInfo info) { + return info != null && info.className.indexOf('$') == -1; + } + }; + + /** Separator for the Class-Path manifest attribute value in jar files. */ + private static final Splitter CLASS_PATH_ATTRIBUTE_SEPARATOR = + Splitter.on(" ").omitEmptyStrings(); + + private static final String CLASS_FILE_NAME_EXTENSION = ".class"; + + private final ImmutableSet resources; + + private ClassPath(ImmutableSet resources) { + this.resources = resources; + } + + /** + * Returns a {@code ClassPath} representing all classes and resources loadable from {@code + * classloader} and its parent class loaders. + * + *

Warning: Currently only {@link URLClassLoader} and only {@code file://} urls are + * supported. + * + * @throws IOException if the attempt to read class path resources (jar files or directories) + * failed. + */ + public static ClassPath from(ClassLoader classloader) throws IOException { + DefaultScanner scanner = new DefaultScanner(); + scanner.scan(classloader); + return new ClassPath(scanner.getResources()); + } + + /** + * Returns all resources loadable from the current class path, including the class files of all + * loadable classes but excluding the "META-INF/MANIFEST.MF" file. + */ + public ImmutableSet getResources() { + return resources; + } + + /** + * Returns all classes loadable from the current class path. + * + * @since 16.0 + */ + public ImmutableSet getAllClasses() { + return FluentIterable.from(resources).filter(ClassInfo.class).toSet(); + } + + /** Returns all top level classes loadable from the current class path. */ + public ImmutableSet getTopLevelClasses() { + return FluentIterable.from(resources).filter(ClassInfo.class).filter(IS_TOP_LEVEL).toSet(); + } + + /** Returns all top level classes whose package name is {@code packageName}. */ + public ImmutableSet getTopLevelClasses(String packageName) { + checkNotNull(packageName); + ImmutableSet.Builder builder = ImmutableSet.builder(); + for (ClassInfo classInfo : getTopLevelClasses()) { + if (classInfo.getPackageName().equals(packageName)) { + builder.add(classInfo); + } + } + return builder.build(); + } + + /** + * Returns all top level classes whose package name is {@code packageName} or starts with + * {@code packageName} followed by a '.'. + */ + public ImmutableSet getTopLevelClassesRecursive(String packageName) { + checkNotNull(packageName); + String packagePrefix = packageName + '.'; + ImmutableSet.Builder builder = ImmutableSet.builder(); + for (ClassInfo classInfo : getTopLevelClasses()) { + if (classInfo.getName().startsWith(packagePrefix)) { + builder.add(classInfo); + } + } + return builder.build(); + } + + /** + * Represents a class path resource that can be either a class file or any other resource file + * loadable from the class path. + * + * @since 14.0 + */ + @Beta + public static class ResourceInfo { + + private final String resourceName; + + final ClassLoader loader; + + static ResourceInfo of(String resourceName, ClassLoader loader) { + if (resourceName.endsWith(CLASS_FILE_NAME_EXTENSION)) { + return new ClassInfo(resourceName, loader); + } else { + return new ResourceInfo(resourceName, loader); + } + } + + ResourceInfo(String resourceName, ClassLoader loader) { + this.resourceName = checkNotNull(resourceName); + this.loader = checkNotNull(loader); + } + + /** + * Returns the url identifying the resource. + * + *

See {@link ClassLoader#getResource} + * + * @throws NoSuchElementException if the resource cannot be loaded through the class loader, + * despite physically existing in the class path. + */ + public final URL url() { + URL url = loader.getResource(resourceName); + if (url == null) { + throw new NoSuchElementException(resourceName); + } + return url; + } + + /** + * Returns a {@link ByteSource} view of the resource from which its bytes can be read. + * + * @throws NoSuchElementException if the resource cannot be loaded through the class loader, + * despite physically existing in the class path. + * @since 20.0 + */ + public final ByteSource asByteSource() { + return Resources.asByteSource(url()); + } + + /** + * Returns a {@link CharSource} view of the resource from which its bytes can be read as + * characters decoded with the given {@code charset}. + * + * @throws NoSuchElementException if the resource cannot be loaded through the class loader, + * despite physically existing in the class path. + * @since 20.0 + */ + public final CharSource asCharSource(Charset charset) { + return Resources.asCharSource(url(), charset); + } + + /** Returns the fully qualified name of the resource. Such as "com/mycomp/foo/bar.txt". */ + public final String getResourceName() { + return resourceName; + } + + @Override + public int hashCode() { + return resourceName.hashCode(); + } + + @Override + public boolean equals(Object obj) { + if (obj instanceof ResourceInfo) { + ResourceInfo that = (ResourceInfo) obj; + return resourceName.equals(that.resourceName) && loader == that.loader; + } + return false; + } + + // Do not change this arbitrarily. We rely on it for sorting ResourceInfo. + @Override + public String toString() { + return resourceName; + } + } + + /** + * Represents a class that can be loaded through {@link #load}. + * + * @since 14.0 + */ + @Beta + static final class ClassInfo extends ResourceInfo { + + private final String className; + + ClassInfo(String resourceName, ClassLoader loader) { + super(resourceName, loader); + this.className = getClassName(resourceName); + } + + /** + * Returns the package name of the class, without attempting to load the class. + * + *

Behaves identically to {@link Package#getName()} but does not require the class (or + * package) to be loaded. + */ + public String getPackageName() { + return Reflection.getPackageName(className); + } + + /** + * Returns the simple name of the underlying class as given in the source code. + * + *

Behaves identically to {@link Class#getSimpleName()} but does not require the class + * to be + * loaded. + */ + public String getSimpleName() { + int lastDollarSign = className.lastIndexOf('$'); + if (lastDollarSign != -1) { + String innerClassName = className.substring(lastDollarSign + 1); + // local and anonymous classes are prefixed with number (1,2,3...), anonymous classes are + // entirely numeric whereas local classes have the user supplied name as a suffix + return CharMatcher.digit().trimLeadingFrom(innerClassName); + } + String packageName = getPackageName(); + if (packageName.isEmpty()) { + return className; + } + + // Since this is a top level class, its simple name is always the part after package name. + return className.substring(packageName.length() + 1); + } + + /** + * Returns the fully qualified name of the class. + * + *

Behaves identically to {@link Class#getName()} but does not require the class to be + * loaded. + */ + public String getName() { + return className; + } + + /** + * Loads (but doesn't link or initialize) the class. + * + * @throws LinkageError when there were errors in loading classes that this class depends on. + * For example, {@link NoClassDefFoundError}. + */ + public Class load() { + try { + return loader.loadClass(className); + } catch (ClassNotFoundException e) { + // Shouldn't happen, since the class name is read from the class path. + throw new IllegalStateException(e); + } + } + + @Override + public String toString() { + return className; + } + } + + /** + * Abstract class that scans through the class path represented by a {@link ClassLoader} and + * calls + * {@link #scanDirectory} and {@link #scanJarFile} for directories and jar files on the class + * path + * respectively. + */ + abstract static class Scanner { + + // We only scan each file once independent of the classloader that resource might be + // associated + // with. + private final Set scannedUris = Sets.newHashSet(); + + public final void scan(ClassLoader classloader) throws IOException { + for (Map.Entry entry : getClassPathEntries(classloader).entrySet()) { + scan(entry.getKey(), entry.getValue()); + } + } + + /** Called when a directory is scanned for resource files. */ + protected abstract void scanDirectory(ClassLoader loader, File directory) throws IOException; + + /** Called when a jar file is scanned for resource entries. */ + protected abstract void scanJarFile(ClassLoader loader, JarFile file) throws IOException; + + @VisibleForTesting + final void scan(File file, ClassLoader classloader) throws IOException { + if (scannedUris.add(file.getCanonicalFile())) { + scanFrom(file, classloader); + } + } + + private void scanFrom(File file, ClassLoader classloader) throws IOException { + try { + if (!file.exists()) { + return; + } + } catch (SecurityException e) { + logger.warn("Cannot access " + file + ": " + e); + return; + } + if (file.isDirectory()) { + scanDirectory(classloader, file); + } else { + scanJar(file, classloader); + } + } + + private void scanJar(File file, ClassLoader classloader) throws IOException { + JarFile jarFile; + try { + jarFile = new JarFile(file); + } catch (IOException e) { + // Not a jar file + return; + } + try { + for (File path : getClassPathFromManifest(file, jarFile.getManifest())) { + scan(path, classloader); + } + scanJarFile(classloader, jarFile); + } finally { + try { + jarFile.close(); + } catch (IOException ignored) { + } + } + } + + /** + * Returns the class path URIs specified by the {@code Class-Path} manifest attribute, + * according + * to + * + * JAR File Specification. If {@code manifest} is null, it means the jar file has no + * manifest, and an empty set will be returned. + */ + @VisibleForTesting + static ImmutableSet getClassPathFromManifest(File jarFile, + @Nullable Manifest manifest) { + if (manifest == null) { + return ImmutableSet.of(); + } + ImmutableSet.Builder builder = ImmutableSet.builder(); + String classpathAttribute = + manifest.getMainAttributes().getValue(Attributes.Name.CLASS_PATH.toString()); + if (classpathAttribute != null) { + for (String path : CLASS_PATH_ATTRIBUTE_SEPARATOR.split(classpathAttribute)) { + URL url; + try { + url = getClassPathEntry(jarFile, path); + } catch (MalformedURLException e) { + // Ignore bad entry + logger.warn("Invalid Class-Path entry: " + path); + continue; + } + if (url.getProtocol().equals("file")) { + builder.add(toFile(url)); + } + } + } + return builder.build(); + } + + @VisibleForTesting + static ImmutableMap getClassPathEntries(ClassLoader classloader) { + LinkedHashMap entries = Maps.newLinkedHashMap(); + // Search parent first, since it's the order ClassLoader#loadClass() uses. + ClassLoader parent = classloader.getParent(); + if (parent != null) { + entries.putAll(getClassPathEntries(parent)); + } + if (classloader instanceof URLClassLoader) { + URLClassLoader urlClassLoader = (URLClassLoader) classloader; + for (URL entry : urlClassLoader.getURLs()) { + if (entry.getProtocol().equals("file")) { + File file = toFile(entry); + if (!entries.containsKey(file)) { + entries.put(file, classloader); + } + } + } + } + return ImmutableMap.copyOf(entries); + } + + /** + * Returns the absolute uri of the Class-Path entry value as specified in + * + * JAR File Specification. Even though the specification only talks about relative urls, + * absolute urls are actually supported too (for example, in Maven surefire plugin). + */ + @VisibleForTesting + static URL getClassPathEntry(File jarFile, String path) throws MalformedURLException { + return new URL(jarFile.toURI().toURL(), path); + } + } + + @VisibleForTesting + static final class DefaultScanner extends Scanner { + + private final SetMultimap resources = + MultimapBuilder.hashKeys().linkedHashSetValues().build(); + + ImmutableSet getResources() { + ImmutableSet.Builder builder = ImmutableSet.builder(); + for (Map.Entry entry : resources.entries()) { + builder.add(ResourceInfo.of(entry.getValue(), entry.getKey())); + } + return builder.build(); + } + + @Override + protected void scanJarFile(ClassLoader classloader, JarFile file) { + Enumeration entries = file.entries(); + while (entries.hasMoreElements()) { + JarEntry entry = entries.nextElement(); + if (entry.isDirectory() || entry.getName().equals(JarFile.MANIFEST_NAME)) { + continue; + } + resources.get(classloader).add(entry.getName()); + } + } + + @Override + protected void scanDirectory(ClassLoader classloader, File directory) throws IOException { + scanDirectory(directory, classloader, ""); + } + + private void scanDirectory(File directory, ClassLoader classloader, String packagePrefix) + throws IOException { + File[] files = directory.listFiles(); + if (files == null) { + logger.warn("Cannot read directory " + directory); + // IO error, just skip the directory + return; + } + for (File f : files) { + String name = f.getName(); + if (f.isDirectory()) { + scanDirectory(f, classloader, packagePrefix + name + "/"); + } else { + String resourceName = packagePrefix + name; + if (!resourceName.equals(JarFile.MANIFEST_NAME)) { + resources.get(classloader).add(resourceName); + } + } + } + } + } + + @VisibleForTesting + static String getClassName(String filename) { + int classNameEnd = filename.length() - CLASS_FILE_NAME_EXTENSION.length(); + return filename.substring(0, classNameEnd).replace('/', '.'); + } + + @VisibleForTesting + static File toFile(URL url) { + checkArgument(url.getProtocol().equals("file")); + try { + return new File(url.toURI()); // Accepts escaped characters like %20. + } catch (URISyntaxException e) { // URL.toURI() doesn't escape chars. + return new File(url.getPath()); // Accepts non-escaped chars like space. + } + } +}