YARN-6757. Refactor the usage of yarn.nodemanager.linux-container-exe…

…cutor.cgroups.mount-path

(Contributed by Miklos Szegedi via Daniel Templeton)

hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/resources/yarn-default.xml

@@ -134,7 +134,7 @@
 
   <property>
       <description>
-        This configures the HTTP endpoint for Yarn Daemons.The following
+        This configures the HTTP endpoint for YARN Daemons.The following
         values are supported:
         - HTTP_ONLY : Service is provided only on http
         - HTTPS_ONLY : Service is provided only on https
@@ -1063,14 +1063,14 @@
       DeletionService will delete the application's localized file directory
       and log directory.
 
-      To diagnose Yarn application problems, set this property's value large
+      To diagnose YARN application problems, set this property's value large
       enough (for example, to 600 = 10 minutes) to permit examination of these
       directories. After changing the property's value, you must restart the 
       nodemanager in order for it to have an effect.
 
-      The roots of Yarn applications' work directories is configurable with
+      The roots of YARN applications' work directories is configurable with
       the yarn.nodemanager.local-dirs property (see below), and the roots
-      of the Yarn applications' log directories is configurable with the 
+      of the YARN applications' log directories is configurable with the
       yarn.nodemanager.log-dirs property (see also below).
     </description>
     <name>yarn.nodemanager.delete.debug-delay-sec</name>
@@ -1510,28 +1510,45 @@
   <property>
     <description>The cgroups hierarchy under which to place YARN proccesses (cannot contain commas).
     If yarn.nodemanager.linux-container-executor.cgroups.mount is false
-    (that is, if cgroups have been pre-configured) and the Yarn user has write
+    (that is, if cgroups have been pre-configured) and the YARN user has write
     access to the parent directory, then the directory will be created.
-    If the directory already exists, the administrator has to give Yarn
+    If the directory already exists, the administrator has to give YARN
     write permissions to it recursively.
-    Only used when the LCE resources handler is set to the CgroupsLCEResourcesHandler.</description>
+    This property only applies when the LCE resources handler is set to
+    CgroupsLCEResourcesHandler.</description>
     <name>yarn.nodemanager.linux-container-executor.cgroups.hierarchy</name>
     <value>/hadoop-yarn</value>
   </property>
 
   <property>
     <description>Whether the LCE should attempt to mount cgroups if not found.
-    Only used when the LCE resources handler is set to the CgroupsLCEResourcesHandler.</description>
+    This property only applies when the LCE resources handler is set to
+    CgroupsLCEResourcesHandler.
+    </description>
     <name>yarn.nodemanager.linux-container-executor.cgroups.mount</name>
     <value>false</value>
   </property>
 
   <property>
-    <description>Where the LCE should attempt to mount cgroups if not found. Common locations
-    include /sys/fs/cgroup and /cgroup; the default location can vary depending on the Linux
-    distribution in use. This path must exist before the NodeManager is launched.
-    Only used when the LCE resources handler is set to the CgroupsLCEResourcesHandler, and
-    yarn.nodemanager.linux-container-executor.cgroups.mount is true.</description>
+    <description>This property sets the path from which YARN will read the
+    CGroups configuration. YARN has built-in functionality to discover the
+    system CGroup mount paths, so use this property only if YARN's automatic
+    mount path discovery does not work.
+
+    The path specified by this property must exist before the NodeManager is
+    launched.
+    If yarn.nodemanager.linux-container-executor.cgroups.mount is set to true,
+    YARN will first try to mount the CGroups at the specified path before
+    reading them.
+    If yarn.nodemanager.linux-container-executor.cgroups.mount is set to
+    false, YARN will read the CGroups at the specified path.
+    If this property is empty, YARN tries to detect the CGroups location.
+
+    Please refer to NodeManagerCgroups.html in the documentation for further
+    details.
+    This property only applies when the LCE resources handler is set to
+    CgroupsLCEResourcesHandler.
+    </description>
     <name>yarn.nodemanager.linux-container-executor.cgroups.mount-path</name>
   </property>
 

hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/linux/resources/CGroupsHandler.java

@@ -23,6 +23,9 @@
 import org.apache.hadoop.classification.InterfaceAudience;
 import org.apache.hadoop.classification.InterfaceStability;
 
+import java.util.HashSet;
+import java.util.Set;
+
 /**
  * Provides CGroups functionality. Implementations are expected to be
  * thread-safe
@@ -54,6 +57,18 @@ enum CGroupController {
     String getName() {
       return name;
     }
+
+    /**
+     * Get the list of valid cgroup names.
+     * @return The set of cgroup name strings
+     */
+    public static Set<String> getValidCGroups() {
+      HashSet<String> validCgroups = new HashSet<>();
+      for (CGroupController controller : CGroupController.values()) {
+        validCgroups.add(controller.getName());
+      }
+      return validCgroups;
+    }
   }
 
   String CGROUP_FILE_TASKS = "tasks";

hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/linux/resources/CGroupsHandlerImpl.java

@@ -83,7 +83,7 @@ class CGroupsHandlerImpl implements CGroupsHandler {
    * @param mtab mount file location
    * @throws ResourceHandlerException if initialization failed
    */
-  public CGroupsHandlerImpl(Configuration conf, PrivilegedOperationExecutor
+  CGroupsHandlerImpl(Configuration conf, PrivilegedOperationExecutor
       privilegedOperationExecutor, String mtab)
       throws ResourceHandlerException {
     this.cGroupPrefix = conf.get(YarnConfiguration.
@@ -115,7 +115,7 @@ public CGroupsHandlerImpl(Configuration conf, PrivilegedOperationExecutor
    *                                    PrivilegedContainerOperations
    * @throws ResourceHandlerException if initialization failed
    */
-  public CGroupsHandlerImpl(Configuration conf, PrivilegedOperationExecutor
+  CGroupsHandlerImpl(Configuration conf, PrivilegedOperationExecutor
       privilegedOperationExecutor) throws ResourceHandlerException {
     this(conf, privilegedOperationExecutor, MTAB_FILE);
   }
@@ -142,11 +142,18 @@ private void initializeControllerPaths() throws ResourceHandlerException {
     // the same hierarchy will be mounted at each mount point with the same
     // subsystem set.
 
-    Map<String, Set<String>> newMtab;
+    Map<String, Set<String>> newMtab = null;
     Map<CGroupController, String> cPaths;
     try {
-      // parse mtab
-      newMtab = parseMtab(mtabFile);
+      if (this.cGroupMountPath != null && !this.enableCGroupMount) {
+        newMtab = ResourceHandlerModule.
+            parseConfiguredCGroupPath(this.cGroupMountPath);
+      }
+
+      if (newMtab == null) {
+        // parse mtab
+        newMtab = parseMtab(mtabFile);
+      }
 
       // find cgroup controller paths
       cPaths = initializeControllerPathsFromMtab(newMtab);
@@ -203,10 +210,8 @@ static Map<String, Set<String>> parseMtab(String mtab)
       throws IOException {
     Map<String, Set<String>> ret = new HashMap<>();
     BufferedReader in = null;
-    HashSet<String> validCgroups = new HashSet<>();
-    for (CGroupController controller : CGroupController.values()) {
-      validCgroups.add(controller.getName());
-    }
+    Set<String> validCgroups =
+        CGroupsHandler.CGroupController.getValidCGroups();
 
     try {
       FileInputStream fis = new FileInputStream(new File(mtab));
@@ -487,7 +492,8 @@ private void logLineFromTasksFile(File cgf) {
       try (BufferedReader inl =
           new BufferedReader(new InputStreamReader(new FileInputStream(cgf
               + "/tasks"), "UTF-8"))) {
-        if ((str = inl.readLine()) != null) {
+        str = inl.readLine();
+        if (str != null) {
           LOG.debug("First line in cgroup tasks file: " + cgf + " " + str);
         }
       } catch (IOException e) {

hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/linux/resources/ResourceHandlerModule.java

@@ -31,6 +31,13 @@
 import org.apache.hadoop.yarn.server.nodemanager.util.CgroupsLCEResourcesHandler;
 import org.apache.hadoop.yarn.server.nodemanager.util.DefaultLCEResourcesHandler;
 
+import java.io.File;
+import java.io.IOException;
+import java.util.Set;
+import java.util.HashSet;
+import java.util.Map;
+import java.util.HashMap;
+import java.util.Arrays;
 import java.util.ArrayList;
 import java.util.List;
 
@@ -113,8 +120,8 @@ private static CGroupsCpuResourceHandlerImpl getCGroupsCpuResourceHandler(
   }
 
   private static TrafficControlBandwidthHandlerImpl
-    getTrafficControlBandwidthHandler(Configuration conf)
-      throws ResourceHandlerException {
+      getTrafficControlBandwidthHandler(Configuration conf)
+        throws ResourceHandlerException {
     if (conf.getBoolean(YarnConfiguration.NM_NETWORK_RESOURCE_ENABLED,
         YarnConfiguration.DEFAULT_NM_NETWORK_RESOURCE_ENABLED)) {
       if (trafficControlBandwidthHandler == null) {
@@ -137,8 +144,8 @@ private static CGroupsCpuResourceHandlerImpl getCGroupsCpuResourceHandler(
   }
 
   public static OutboundBandwidthResourceHandler
-    getOutboundBandwidthResourceHandler(Configuration conf)
-      throws ResourceHandlerException {
+      getOutboundBandwidthResourceHandler(Configuration conf)
+        throws ResourceHandlerException {
     return getTrafficControlBandwidthHandler(conf);
   }
 
@@ -176,7 +183,7 @@ public static MemoryResourceHandler getMemoryResourceHandler(
   }
 
   private static CGroupsMemoryResourceHandlerImpl
-    getCgroupsMemoryResourceHandler(
+      getCgroupsMemoryResourceHandler(
       Configuration conf) throws ResourceHandlerException {
     if (cGroupsMemoryResourceHandler == null) {
       synchronized (MemoryResourceHandler.class) {
@@ -229,4 +236,45 @@ public static ResourceHandlerChain getConfiguredResourceHandlerChain(
   static void nullifyResourceHandlerChain() throws ResourceHandlerException {
     resourceHandlerChain = null;
   }
+
+  /**
+   * If a cgroup mount directory is specified, it returns cgroup directories
+   * with valid names.
+   * The requirement is that each hierarchy has to be named with the comma
+   * separated names of subsystems supported.
+   * For example: /sys/fs/cgroup/cpu,cpuacct
+   * @param cgroupMountPath Root cgroup mount path (/sys/fs/cgroup in the
+   *                        example above)
+   * @return A path to cgroup subsystem set mapping in the same format as
+   *         {@link CGroupsHandlerImpl#parseMtab(String)}
+   * @throws IOException if the specified directory cannot be listed
+   */
+  public static Map<String, Set<String>> parseConfiguredCGroupPath(
+      String cgroupMountPath) throws IOException {
+    File cgroupDir = new File(cgroupMountPath);
+    File[] list = cgroupDir.listFiles();
+    if (list == null) {
+      throw new IOException("Empty cgroup mount directory specified: " +
+          cgroupMountPath);
+    }
+
+    Map<String, Set<String>> pathSubsystemMappings = new HashMap<>();
+    Set<String> validCGroups =
+        CGroupsHandler.CGroupController.getValidCGroups();
+    for (File candidate: list) {
+      Set<String> cgroupList =
+          new HashSet<>(Arrays.asList(candidate.getName().split(",")));
+      // Collect the valid subsystem names
+      cgroupList.retainAll(validCGroups);
+      if (!cgroupList.isEmpty()) {
+        if (candidate.isDirectory() && candidate.canWrite()) {
+          pathSubsystemMappings.put(candidate.getAbsolutePath(), cgroupList);
+        } else {
+          LOG.warn("The following cgroup is not a directory or it is not"
+              + " writable" + candidate.getAbsolutePath());
+        }
+      }
+    }
+    return pathSubsystemMappings;
+  }
 }