From f14000bcee275988170b6d3b0178bfd9a8e2ba9a Mon Sep 17 00:00:00 2001 From: Terence Yim Date: Fri, 31 Mar 2017 15:39:20 -0700 Subject: [PATCH] (TWILL-228) Remove the optimization when building application jar - The optimization can leading skipping certain jars in the application jar --- .../apache/twill/yarn/YarnTwillPreparer.java | 23 +++---------------- .../twill/yarn/YarnTwillRunnerService.java | 21 +---------------- 2 files changed, 4 insertions(+), 40 deletions(-) diff --git a/twill-yarn/src/main/java/org/apache/twill/yarn/YarnTwillPreparer.java b/twill-yarn/src/main/java/org/apache/twill/yarn/YarnTwillPreparer.java index 4846fe37..c8abf4fb 100644 --- a/twill-yarn/src/main/java/org/apache/twill/yarn/YarnTwillPreparer.java +++ b/twill-yarn/src/main/java/org/apache/twill/yarn/YarnTwillPreparer.java @@ -146,7 +146,6 @@ public String apply(Class cls) { private final Credentials credentials; private final Map> logLevels = Maps.newHashMap(); private final LocationCache locationCache; - private final Set twillClassPaths; private String schedulerQueue; private String extraOptions; private JvmOptions.DebugOptions debugOptions = JvmOptions.DebugOptions.NO_DEBUG; @@ -154,8 +153,8 @@ public String apply(Class cls) { private final Map maxRetries = Maps.newHashMap(); YarnTwillPreparer(Configuration config, TwillSpecification twillSpec, RunId runId, - String zkConnectString, Location appLocation, Set twillClassPaths, - String extraOptions, LocationCache locationCache, YarnTwillControllerFactory controllerFactory) { + String zkConnectString, Location appLocation, String extraOptions, + LocationCache locationCache, YarnTwillControllerFactory controllerFactory) { this.config = config; this.twillSpec = twillSpec; this.runId = runId; @@ -166,7 +165,6 @@ public String apply(Class cls) { this.extraOptions = extraOptions; this.classAcceptor = new ClassAcceptor(); this.locationCache = locationCache; - this.twillClassPaths = twillClassPaths; } private void confirmRunnableName(String runnableName) { @@ -372,7 +370,7 @@ public ProcessController call() throws Exception { createLauncherJar(localFiles); createTwillJar(createBundler(classAcceptor), yarnAppClient, localFiles); - createApplicationJar(createApplicationJarBundler(classAcceptor), localFiles); + createApplicationJar(createBundler(classAcceptor), localFiles); createResourcesJar(createBundler(classAcceptor), localFiles); Path runtimeConfigDir = Files.createTempDirectory(getLocalStagingDir().toPath(), @@ -835,19 +833,4 @@ private ClassLoader getClassLoader() { private ApplicationBundler createBundler(ClassAcceptor classAcceptor) { return new ApplicationBundler(classAcceptor).setTempDir(getLocalStagingDir()); } - - /** - * Creates a {@link ApplicationBundler} for building application jar. The bundler will include classes - * accepted by the given {@link ClassAcceptor}, as long as it is not a twill class. - */ - private ApplicationBundler createApplicationJarBundler(final ClassAcceptor classAcceptor) { - // Accept classes based on the classAcceptor and also excluding all twill classes as they are already - // in the twill.jar - return createBundler(new ClassAcceptor() { - @Override - public boolean accept(String className, URL classUrl, URL classPathUrl) { - return !twillClassPaths.contains(classPathUrl) && classAcceptor.accept(className, classUrl, classPathUrl); - } - }); - } } diff --git a/twill-yarn/src/main/java/org/apache/twill/yarn/YarnTwillRunnerService.java b/twill-yarn/src/main/java/org/apache/twill/yarn/YarnTwillRunnerService.java index 269ffdf9..20627e27 100644 --- a/twill-yarn/src/main/java/org/apache/twill/yarn/YarnTwillRunnerService.java +++ b/twill-yarn/src/main/java/org/apache/twill/yarn/YarnTwillRunnerService.java @@ -44,7 +44,6 @@ import org.apache.hadoop.security.Credentials; import org.apache.hadoop.security.UserGroupInformation; import org.apache.hadoop.yarn.conf.YarnConfiguration; -import org.apache.twill.api.ClassAcceptor; import org.apache.twill.api.Configs; import org.apache.twill.api.LocalFile; import org.apache.twill.api.ResourceSpecification; @@ -73,7 +72,6 @@ import org.apache.twill.internal.io.BasicLocationCache; import org.apache.twill.internal.io.LocationCache; import org.apache.twill.internal.io.NoCachingLocationCache; -import org.apache.twill.internal.utils.Dependencies; import org.apache.twill.internal.yarn.VersionDetectYarnAppClientFactory; import org.apache.twill.internal.yarn.YarnAppClient; import org.apache.twill.internal.yarn.YarnApplicationReport; @@ -96,7 +94,6 @@ import java.io.DataOutputStream; import java.io.FileNotFoundException; import java.io.IOException; -import java.net.URL; import java.util.HashSet; import java.util.Iterator; import java.util.Map; @@ -134,7 +131,6 @@ public TwillController apply(YarnTwillController controller) { private final ZKClientService zkClientService; private final LocationFactory locationFactory; private final Table controllers; - private final Set twillClassPaths; // A Guava service to help the state transition. private final Service serviceDelegate; private LocationCache locationCache; @@ -169,7 +165,6 @@ public YarnTwillRunnerService(YarnConfiguration config, String zkConnect, Locati this.locationFactory = locationFactory; this.zkClientService = getZKClientService(zkConnect); this.controllers = HashBasedTable.create(); - this.twillClassPaths = new HashSet<>(); this.serviceDelegate = new AbstractIdleService() { @Override protected void startUp() throws Exception { @@ -308,8 +303,7 @@ public TwillPreparer prepare(TwillApplication application) { Configuration config = new Configuration(yarnConfig); return new YarnTwillPreparer(config, twillSpec, runId, zkClientService.getConnectString(), - appLocation, twillClassPaths, jvmOptions, - locationCache, new YarnTwillControllerFactory() { + appLocation, jvmOptions, locationCache, new YarnTwillControllerFactory() { @Override public YarnTwillController create(RunId runId, boolean logCollectionEnabled, Iterable logHandlers, Callable> startUp, @@ -355,19 +349,6 @@ public Iterable lookupLive() { private void startUp() throws Exception { zkClientService.startAndWait(); - // Find all the classpaths for Twill classes. It is used for class filtering when building application jar - // in the YarnTwillPreparer - Dependencies.findClassDependencies(getClass().getClassLoader(), new ClassAcceptor() { - @Override - public boolean accept(String className, URL classUrl, URL classPathUrl) { - if (!className.startsWith("org.apache.twill.")) { - return false; - } - twillClassPaths.add(classPathUrl); - return true; - } - }, getClass().getName()); - // Create the root node, so that the namespace root would get created if it is missing // If the exception is caused by node exists, then it's ok. Otherwise propagate the exception. ZKOperations.ignoreError(zkClientService.create("/", null, CreateMode.PERSISTENT),