(TWILL-254) Update to use ContainerId.fromString

This closes #65 on Github. Signed-off-by: Terence Yim <chtyim@apache.org>
apache · Feb 6, 2018 · d6095d4 · d6095d4
1 parent 55f6d6f
commit d6095d4
Show file tree

Hide file tree

Showing 8 changed files with 132 additions and 12 deletions.
diff --git a/pom.xml b/pom.xml
@@ -171,7 +171,6 @@
         <junit.version>4.11</junit.version>
         <jopt-simple.version>3.2</jopt-simple.version>
         <commons-compress.version>1.5</commons-compress.version>
-        <hadoop.version>[2.0.2-alpha,2.3.0]</hadoop.version>
         <hadoop20.output.dir>target/hadoop20-classes</hadoop20.output.dir>
     </properties>
 
@@ -538,9 +537,6 @@
             <properties>
                 <hadoop.version>2.3.0</hadoop.version>
             </properties>
-            <activation>
-                <activeByDefault>true</activeByDefault>
-            </activation>
             <build>
                 <plugins>
                     <plugin>
@@ -722,6 +718,53 @@
                 </plugins>
             </build>
         </profile>
+        <profile>
+            <id>hadoop-2.6</id>
+            <properties>
+                <hadoop.version>2.6.5</hadoop.version>
+            </properties>
+            <activation>
+                <activeByDefault>true</activeByDefault>
+            </activation>
+            <build>
+                <plugins>
+                    <plugin>
+                        <groupId>org.codehaus.mojo</groupId>
+                        <artifactId>build-helper-maven-plugin</artifactId>
+                        <version>1.8</version>
+                        <executions>
+                            <execution>
+                                <id>add-source</id>
+                                <phase>generate-sources</phase>
+                                <goals>
+                                    <goal>add-source</goal>
+                                </goals>
+                                <configuration>
+                                    <sources>
+                                        <source>src/main/hadoop21</source>
+                                        <source>src/main/hadoop22</source>
+                                        <source>src/main/hadoop23</source>
+                                        <source>src/main/hadoop26</source>
+                                    </sources>
+                                </configuration>
+                            </execution>
+                            <execution>
+                                <id>add-source-2.0</id>
+                                <phase>prepare-package</phase>
+                                <goals>
+                                    <goal>add-source</goal>
+                                </goals>
+                                <configuration>
+                                    <sources>
+                                        <source>src/main/hadoop20</source>
+                                    </sources>
+                                </configuration>
+                            </execution>
+                        </executions>
+                    </plugin>
+                </plugins>
+            </build>
+        </profile>
         <profile>
             <id>java8-test</id>
             <modules>

diff --git a/twill-yarn/src/main/hadoop20/org/apache/twill/internal/yarn/Hadoop20YarnAMClient.java b/twill-yarn/src/main/hadoop20/org/apache/twill/internal/yarn/Hadoop20YarnAMClient.java
@@ -26,11 +26,13 @@
 import org.apache.hadoop.yarn.api.ApplicationConstants;
 import org.apache.hadoop.yarn.api.protocolrecords.RegisterApplicationMasterResponse;
 import org.apache.hadoop.yarn.api.records.Container;
+import org.apache.hadoop.yarn.api.records.ContainerId;
 import org.apache.hadoop.yarn.api.records.ContainerStatus;
 import org.apache.hadoop.yarn.api.records.FinalApplicationStatus;
 import org.apache.hadoop.yarn.api.records.Priority;
 import org.apache.hadoop.yarn.api.records.Resource;
 import org.apache.hadoop.yarn.ipc.YarnRPC;
+import org.apache.hadoop.yarn.util.ConverterUtils;
 import org.apache.twill.internal.appmaster.RunnableProcessLauncher;
 import org.apache.twill.internal.yarn.ports.AMRMClient;
 import org.apache.twill.internal.yarn.ports.AMRMClientImpl;
@@ -71,6 +73,11 @@ public Hadoop20YarnAMClient(Configuration conf) {
     this.nmClient = new Hadoop20YarnNMClient(YarnRPC.create(conf), conf);
   }
 
+  @Override
+  protected ContainerId containerIdLookup(String containerIdStr) {
+    return (ConverterUtils.toContainerId(containerIdStr));
+  }
+
   @Override
   protected void startUp() throws Exception {
     Preconditions.checkNotNull(trackerAddr, "Tracker address not set.");

diff --git a/twill-yarn/src/main/hadoop21/org/apache/twill/internal/yarn/Hadoop21YarnAMClient.java b/twill-yarn/src/main/hadoop21/org/apache/twill/internal/yarn/Hadoop21YarnAMClient.java
@@ -27,11 +27,13 @@
 import org.apache.hadoop.yarn.api.protocolrecords.AllocateResponse;
 import org.apache.hadoop.yarn.api.protocolrecords.RegisterApplicationMasterResponse;
 import org.apache.hadoop.yarn.api.records.Container;
+import org.apache.hadoop.yarn.api.records.ContainerId;
 import org.apache.hadoop.yarn.api.records.ContainerStatus;
 import org.apache.hadoop.yarn.api.records.FinalApplicationStatus;
 import org.apache.hadoop.yarn.api.records.Priority;
 import org.apache.hadoop.yarn.api.records.Resource;
 import org.apache.hadoop.yarn.client.api.AMRMClient;
+import org.apache.hadoop.yarn.util.ConverterUtils;
 import org.apache.twill.internal.appmaster.RunnableProcessLauncher;
 import org.slf4j.Logger;
 import org.slf4j.LoggerFactory;
@@ -57,6 +59,11 @@ public YarnContainerStatus apply(ContainerStatus status) {
     };
   }
 
+  @Override
+  protected ContainerId containerIdLookup(String containerIdStr) {
+    return (ConverterUtils.toContainerId(containerIdStr));
+  }
+
   protected final AMRMClient<AMRMClient.ContainerRequest> amrmClient;
   protected final Hadoop21YarnNMClient nmClient;
   protected Resource maxCapability;

diff --git a/twill-yarn/src/main/hadoop22/org/apache/twill/internal/yarn/Hadoop22YarnAMClient.java b/twill-yarn/src/main/hadoop22/org/apache/twill/internal/yarn/Hadoop22YarnAMClient.java
@@ -26,7 +26,7 @@
 /**
  * Wrapper class for AMRMClient for Hadoop version 2.2 or greater.
  */
-public final class Hadoop22YarnAMClient extends Hadoop21YarnAMClient {
+public class Hadoop22YarnAMClient extends Hadoop21YarnAMClient {
 
   private static final Logger LOG = LoggerFactory.getLogger(Hadoop22YarnAMClient.class);
 

diff --git a/twill-yarn/src/main/hadoop26/org/apache/twill/internal/yarn/Hadoop26YarnAMClient.java b/twill-yarn/src/main/hadoop26/org/apache/twill/internal/yarn/Hadoop26YarnAMClient.java
@@ -0,0 +1,42 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.twill.internal.yarn;
+
+import org.apache.hadoop.conf.Configuration;
+import org.apache.hadoop.yarn.api.records.ContainerId;
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
+
+import java.util.List;
+
+/**
+ * Wrapper class for AMRMClient for Hadoop version 2.6 or greater.
+ */
+public final class Hadoop26YarnAMClient extends Hadoop22YarnAMClient {
+
+  private static final Logger LOG = LoggerFactory.getLogger(Hadoop26YarnAMClient.class);
+
+  public Hadoop26YarnAMClient(Configuration conf) {
+    super(conf);
+  }
+
+  @Override
+  protected final ContainerId containerIdLookup(String containerIdStr) {
+    return (ContainerId.fromString(containerIdStr));
+  }
+}
diff --git a/twill-yarn/src/main/java/org/apache/twill/internal/yarn/AbstractYarnAMClient.java b/twill-yarn/src/main/java/org/apache/twill/internal/yarn/AbstractYarnAMClient.java
@@ -27,7 +27,6 @@
 import org.apache.hadoop.yarn.api.records.ContainerId;
 import org.apache.hadoop.yarn.api.records.Priority;
 import org.apache.hadoop.yarn.api.records.Resource;
-import org.apache.hadoop.yarn.util.ConverterUtils;
 import org.apache.twill.internal.ProcessLauncher;
 import org.apache.twill.internal.appmaster.RunnableProcessLauncher;
 import org.slf4j.Logger;
@@ -80,7 +79,7 @@ protected AbstractYarnAMClient(String containerIdEnvName) {
     String masterContainerId = System.getenv().get(containerIdEnvName);
     Preconditions.checkArgument(masterContainerId != null,
                                 "Missing %s from environment", containerIdEnvName);
-    this.containerId = ConverterUtils.toContainerId(masterContainerId);
+    this.containerId = containerIdLookup(masterContainerId);
     this.inflightRequests = ArrayListMultimap.create();
     this.pendingRequests = ArrayListMultimap.create();
     this.pendingRemoves = Lists.newLinkedList();
@@ -89,7 +88,6 @@ protected AbstractYarnAMClient(String containerIdEnvName) {
     this.blacklistedResources = Lists.newArrayList();
   }
 
-
   @Override
   public final ContainerId getContainerId() {
     return containerId;
@@ -226,6 +224,14 @@ protected boolean recordUnsupportedFeature(String unsupportedFeature) {
     return true;
   }
 
+  /**
+   * Returns the ContainerId given a container ID string
+   *
+   * @param containerIdStr the container ID string to lookup
+   * @return A {@link ContainerId} instance representing the result.
+   */
+  protected abstract ContainerId containerIdLookup(String containerIdStr);
+
   /**
    * Adjusts the given resource capability to fit in the cluster limit.
    *

diff --git a/...l-yarn/src/main/java/org/apache/twill/internal/yarn/VersionDetectYarnAMClientFactory.java b/...l-yarn/src/main/java/org/apache/twill/internal/yarn/VersionDetectYarnAMClientFactory.java
@@ -48,11 +48,17 @@ public YarnAMClient create() {
           clzName = getClass().getPackage().getName() + ".Hadoop21YarnAMClient";
           clz = (Class<YarnAMClient>) Class.forName(clzName);
           break;
-        default:
-          // Uses hadoop-2.2 or above class
+        case HADOOP_22:
+        case HADOOP_23:
+          // Uses hadoop-2.2 class
           clzName = getClass().getPackage().getName() + ".Hadoop22YarnAMClient";
           clz = (Class<YarnAMClient>) Class.forName(clzName);
           break;
+        default:
+          // Uses hadoop-2.6 or above class
+          clzName = getClass().getPackage().getName() + ".Hadoop26YarnAMClient";
+          clz = (Class<YarnAMClient>) Class.forName(clzName);
+          break;
       }
 
       return clz.getConstructor(Configuration.class).newInstance(conf);

diff --git a/twill-yarn/src/main/java/org/apache/twill/internal/yarn/YarnUtils.java b/twill-yarn/src/main/java/org/apache/twill/internal/yarn/YarnUtils.java
@@ -68,7 +68,8 @@ public enum HadoopVersions {
     HADOOP_20,
     HADOOP_21,
     HADOOP_22,
-    HADOOP_23
+    HADOOP_23,
+    HADOOP_26
   }
 
   private static final Logger LOG = LoggerFactory.getLogger(YarnUtils.class);
@@ -263,7 +264,15 @@ public static HadoopVersions getHadoopVersion() {
         Class.forName("org.apache.hadoop.yarn.client.cli.LogsCLI");
         try {
           Class.forName("org.apache.hadoop.yarn.conf.HAUtil");
-          HADOOP_VERSION.set(HadoopVersions.HADOOP_23);
+          try {
+            Class[] args = new Class[1];
+            args[0] = String.class;
+            // see if we have a org.apache.hadoop.yarn.api.records.ContainerId.fromString() method
+            Class.forName("org.apache.hadoop.yarn.api.records.ContainerId").getMethod("fromString", args);
+            HADOOP_VERSION.set(HadoopVersions.HADOOP_26);
+          } catch (NoSuchMethodException e) {
+            HADOOP_VERSION.set(HadoopVersions.HADOOP_23);
+          }
         } catch (ClassNotFoundException e) {
           HADOOP_VERSION.set(HadoopVersions.HADOOP_22);
         }