From f52af6ee5799aee6badff40440361b657202e03a Mon Sep 17 00:00:00 2001 From: Terence Yim Date: Tue, 28 Feb 2017 18:03:45 -0800 Subject: [PATCH] (TWILL-207) Only use list of class names as the cache name - Also some indentation changes. --- .../apache/twill/yarn/YarnTwillPreparer.java | 123 +++++++++--------- 1 file changed, 63 insertions(+), 60 deletions(-) diff --git a/twill-yarn/src/main/java/org/apache/twill/yarn/YarnTwillPreparer.java b/twill-yarn/src/main/java/org/apache/twill/yarn/YarnTwillPreparer.java index 07594e38..6fbbf930 100644 --- a/twill-yarn/src/main/java/org/apache/twill/yarn/YarnTwillPreparer.java +++ b/twill-yarn/src/main/java/org/apache/twill/yarn/YarnTwillPreparer.java @@ -186,7 +186,7 @@ public String apply(Class cls) { private void confirmRunnableName(String runnableName) { Preconditions.checkNotNull(runnableName); Preconditions.checkArgument(twillSpec.getRunnables().containsKey(runnableName), - "Runnable %s is not defined in the application.", runnableName); + "Runnable %s is not defined in the application.", runnableName); } @Override @@ -369,57 +369,59 @@ public TwillController start(long timeout, TimeUnit timeoutUnit) { final ApplicationMasterInfo appMasterInfo = launcher.getContainerInfo(); Callable> submitTask = new Callable>() { - @Override - public ProcessController call() throws Exception { - - // Local files needed by AM - Map localFiles = Maps.newHashMap(); - - createLauncherJar(localFiles); - createTwillJar(createBundler(classAcceptor), localFiles); - createApplicationJar(createApplicationJarBundler(classAcceptor), localFiles); - createResourcesJar(createBundler(classAcceptor), localFiles); - - Path runtimeConfigDir = Files.createTempDirectory(localStagingDir.toPath(), - Constants.Files.RUNTIME_CONFIG_JAR); - try { - saveSpecification(twillSpec, runtimeConfigDir.resolve(Constants.Files.TWILL_SPEC)); - saveLogback(runtimeConfigDir.resolve(Constants.Files.LOGBACK_TEMPLATE)); - saveClassPaths(runtimeConfigDir); - saveJvmOptions(runtimeConfigDir.resolve(Constants.Files.JVM_OPTIONS)); - saveArguments(new Arguments(arguments, runnableArgs), runtimeConfigDir.resolve(Constants.Files.ARGUMENTS)); - saveEnvironments(runtimeConfigDir.resolve(Constants.Files.ENVIRONMENTS)); - createRuntimeConfigJar(runtimeConfigDir, localFiles); - } finally { - Paths.deleteRecursively(runtimeConfigDir); - } + @Override + public ProcessController call() throws Exception { + + // Local files needed by AM + Map localFiles = Maps.newHashMap(); + + createLauncherJar(localFiles); + createTwillJar(createBundler(classAcceptor), localFiles); + createApplicationJar(createApplicationJarBundler(classAcceptor), localFiles); + createResourcesJar(createBundler(classAcceptor), localFiles); + + Path runtimeConfigDir = Files.createTempDirectory(localStagingDir.toPath(), + Constants.Files.RUNTIME_CONFIG_JAR); + try { + saveSpecification(twillSpec, runtimeConfigDir.resolve(Constants.Files.TWILL_SPEC)); + saveLogback(runtimeConfigDir.resolve(Constants.Files.LOGBACK_TEMPLATE)); + saveClassPaths(runtimeConfigDir); + saveJvmOptions(runtimeConfigDir.resolve(Constants.Files.JVM_OPTIONS)); + saveArguments(new Arguments(arguments, runnableArgs), + runtimeConfigDir.resolve(Constants.Files.ARGUMENTS)); + saveEnvironments(runtimeConfigDir.resolve(Constants.Files.ENVIRONMENTS)); + createRuntimeConfigJar(runtimeConfigDir, localFiles); + } finally { + Paths.deleteRecursively(runtimeConfigDir); + } - createLocalizeFilesJson(localFiles); - - LOG.debug("Submit AM container spec: {}", appMasterInfo); - // java -Djava.io.tmpdir=tmp -cp launcher.jar:$HADOOP_CONF_DIR -XmxMemory - // org.apache.twill.internal.TwillLauncher - // appMaster.jar - // org.apache.twill.internal.appmaster.ApplicationMasterMain - // false - - int memory = Resources.computeMaxHeapSize(appMasterInfo.getMemoryMB(), - Constants.APP_MASTER_RESERVED_MEMORY_MB, Constants.HEAP_MIN_RATIO); - return launcher.prepareLaunch(ImmutableMap.of(), localFiles.values(), credentials) - .addCommand( - "$JAVA_HOME/bin/java", - "-Djava.io.tmpdir=tmp", - "-Dyarn.appId=$" + EnvKeys.YARN_APP_ID_STR, - "-Dtwill.app=$" + Constants.TWILL_APP_NAME, - "-cp", Constants.Files.LAUNCHER_JAR + ":$HADOOP_CONF_DIR", - "-Xmx" + memory + "m", - extraOptions == null ? "" : extraOptions, - TwillLauncher.class.getName(), - ApplicationMasterMain.class.getName(), - Boolean.FALSE.toString()) - .launch(); - } - }; + createLocalizeFilesJson(localFiles); + + LOG.debug("Submit AM container spec: {}", appMasterInfo); + // java -Djava.io.tmpdir=tmp -cp launcher.jar:$HADOOP_CONF_DIR -XmxMemory + // org.apache.twill.internal.TwillLauncher + // appMaster.jar + // org.apache.twill.internal.appmaster.ApplicationMasterMain + // false + + int memory = Resources.computeMaxHeapSize(appMasterInfo.getMemoryMB(), + Constants.APP_MASTER_RESERVED_MEMORY_MB, + Constants.HEAP_MIN_RATIO); + return launcher.prepareLaunch(ImmutableMap.of(), localFiles.values(), credentials) + .addCommand( + "$JAVA_HOME/bin/java", + "-Djava.io.tmpdir=tmp", + "-Dyarn.appId=$" + EnvKeys.YARN_APP_ID_STR, + "-Dtwill.app=$" + Constants.TWILL_APP_NAME, + "-cp", Constants.Files.LAUNCHER_JAR + ":$HADOOP_CONF_DIR", + "-Xmx" + memory + "m", + extraOptions == null ? "" : extraOptions, + TwillLauncher.class.getName(), + ApplicationMasterMain.class.getName(), + Boolean.FALSE.toString()) + .launch(); + } + }; YarnTwillController controller = controllerFactory.create(runId, logHandlers, submitTask, timeout, timeoutUnit); controller.start(); @@ -487,7 +489,7 @@ private void createTwillJar(final ApplicationBundler bundler, Map runtimeSpec = Maps.transformEntries( spec.getRunnables(), new Maps.EntryTransformer() { - @Override - public RuntimeSpecification transformEntry(String key, RuntimeSpecification value) { - return new DefaultRuntimeSpecification(value.getName(), value.getRunnableSpecification(), - value.getResourceSpecification(), runnableLocalFiles.get(key)); - } - }); + @Override + public RuntimeSpecification transformEntry(String key, RuntimeSpecification value) { + return new DefaultRuntimeSpecification(value.getName(), value.getRunnableSpecification(), + value.getResourceSpecification(), runnableLocalFiles.get(key)); + } + }); // Serialize into a local temp file. LOG.debug("Creating {}", targetFile); @@ -661,7 +664,7 @@ private void createLauncherJar(Map localFiles) throws URISynt @Override public void load(String name, Location targetLocation) throws IOException { // Create a jar file with the TwillLauncher and FindFreePort and dependent classes inside. - try (final JarOutputStream jarOut = new JarOutputStream(targetLocation.getOutputStream())) { + try (JarOutputStream jarOut = new JarOutputStream(targetLocation.getOutputStream())) { ClassLoader classLoader = Thread.currentThread().getContextClassLoader(); if (classLoader == null) { classLoader = getClass().getClassLoader();