From 78a3a5c06e03a74dd0e4150c9017560c188b0c6f Mon Sep 17 00:00:00 2001 From: Miklos Szegedi Date: Fri, 2 Jun 2017 14:55:26 -0700 Subject: [PATCH 1/8] YARN-6668 Add new CGroupsResourceCalculator --- .../util/ResourceCalculatorProcessTree.java | 2 +- .../linux/resources/CGroupsHandler.java | 9 +- .../linux/resources/CGroupsHandlerImpl.java | 2 +- .../resources/CGroupsResourceCalculator.java | 292 ++++++++++++++++++ .../monitor/ContainersMonitorImpl.java | 32 +- .../TestCGroupsResourceCalculator.java | 167 ++++++++++ 6 files changed, 498 insertions(+), 6 deletions(-) create mode 100644 hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/linux/resources/CGroupsResourceCalculator.java create mode 100644 hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/test/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/linux/resources/TestCGroupsResourceCalculator.java diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/util/ResourceCalculatorProcessTree.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/util/ResourceCalculatorProcessTree.java index 7e5cf55164847..adbba84767bf8 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/util/ResourceCalculatorProcessTree.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/util/ResourceCalculatorProcessTree.java @@ -37,7 +37,7 @@ @Public @Evolving public abstract class ResourceCalculatorProcessTree extends Configured { - static final Log LOG = LogFactory + static protected final Log LOG = LogFactory .getLog(ResourceCalculatorProcessTree.class); public static final int UNAVAILABLE = -1; diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/linux/resources/CGroupsHandler.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/linux/resources/CGroupsHandler.java index 8fc35a8232d65..f093c2b98130e 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/linux/resources/CGroupsHandler.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/linux/resources/CGroupsHandler.java @@ -51,7 +51,7 @@ enum CGroupController { this.name = name; } - String getName() { + public String getName() { return name; } } @@ -97,6 +97,13 @@ String createCGroup(CGroupController controller, String cGroupId) void deleteCGroup(CGroupController controller, String cGroupId) throws ResourceHandlerException; + /** + * Gets the absolute path to the specified cgroup controller. + * @param controller - controller type for the cgroup + * @return the root of the controller. + */ + String getControllerPath(CGroupController controller); + /** * Gets the relative path for the cgroup, independent of a controller, for a * given cgroup id. diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/linux/resources/CGroupsHandlerImpl.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/linux/resources/CGroupsHandlerImpl.java index 85b01cd1e834a..f2c85dfbe5d1c 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/linux/resources/CGroupsHandlerImpl.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/linux/resources/CGroupsHandlerImpl.java @@ -124,7 +124,7 @@ private void init() throws ResourceHandlerException { initializeControllerPaths(); } - private String getControllerPath(CGroupController controller) { + public String getControllerPath(CGroupController controller) { try { rwLock.readLock().lock(); return controllerPaths.get(controller); diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/linux/resources/CGroupsResourceCalculator.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/linux/resources/CGroupsResourceCalculator.java new file mode 100644 index 0000000000000..466f7676e0839 --- /dev/null +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/linux/resources/CGroupsResourceCalculator.java @@ -0,0 +1,292 @@ +/** + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.hadoop.yarn.server.nodemanager.containermanager.linux.resources; + +import org.apache.hadoop.util.CpuTimeTracker; +import org.apache.hadoop.util.Shell; +import org.apache.hadoop.util.SysInfoLinux; +import org.apache.hadoop.yarn.exceptions.YarnException; +import org.apache.hadoop.yarn.util.Clock; +import org.apache.hadoop.yarn.util.ResourceCalculatorProcessTree; +import org.apache.hadoop.yarn.util.SystemClock; + +import java.io.BufferedReader; +import java.io.File; +import java.io.FileInputStream; +import java.io.FileNotFoundException; +import java.io.IOException; +import java.io.InputStreamReader; +import java.math.BigInteger; +import java.nio.charset.Charset; +import java.util.function.Function; +import java.util.regex.Matcher; +import java.util.regex.Pattern; + +/** + * A cgroups file-system based Resource calculator without the process tree + * features. + */ + +public class CGroupsResourceCalculator extends ResourceCalculatorProcessTree { + enum Result { + Continue, + Exit + } + private static final String PROCFS = "/proc"; + static final String CGROUP = "cgroup"; + static final String CPU_STAT = "cpuacct.stat"; + static final String MEM_STAT = "memory.usage_in_bytes"; + static final String MEMSW_STAT = "memory.memsw.usage_in_bytes"; + private static final String USER = "user "; + private static final String SYSTEM = "system "; + + private static final Pattern CGROUP_FILE_FORMAT = Pattern.compile( + "^(\\d+):([^:]+):/(.*)$"); + private final String procfsDir; + private CGroupsHandler cGroupsHandler; + + private String pid; + private File cpuStat; + private File memStat; + private File memswStat; + + private final long jiffyLengthMs; + private BigInteger processTotalJiffies = BigInteger.ZERO; + private final CpuTimeTracker cpuTimeTracker; + private Clock clock; + + private long mem = UNAVAILABLE; + private long memSw = UNAVAILABLE; + + public CGroupsResourceCalculator(String pid) throws YarnException { + this(pid, PROCFS, ResourceHandlerModule.getCGroupsHandler(), + SystemClock.getInstance()); + } + + CGroupsResourceCalculator(String pid, String procfsDir, + CGroupsHandler cGroupsHandler, Clock clock) + throws YarnException { + super(pid); + this.procfsDir = procfsDir; + this.cGroupsHandler = cGroupsHandler; + this.pid = pid; + this.cpuTimeTracker = + new CpuTimeTracker(SysInfoLinux.JIFFY_LENGTH_IN_MILLIS); + this.clock = clock; + this.jiffyLengthMs = (clock == SystemClock.getInstance()) ? + SysInfoLinux.JIFFY_LENGTH_IN_MILLIS : 10; + setCGroupFilePaths(); + } + + @Override + public float getCpuUsagePercent() { + readTotalProcessJiffies(); + cpuTimeTracker.updateElapsedJiffies( + processTotalJiffies, + clock.getTime()); + return cpuTimeTracker.getCpuTrackerUsagePercent(); + } + + @Override + public long getCumulativeCpuTime() { + if (jiffyLengthMs < 0) { + return UNAVAILABLE; + } + readTotalProcessJiffies(); + return + processTotalJiffies.longValue() * jiffyLengthMs; + } + + @Override + public long getRssMemorySize(int olderThanAge) { + if (olderThanAge > 0) { + return 0; + } + try { + processFile(memStat, (String line) -> { + mem = Long.parseLong(line); + return Result.Exit; + }); + return mem; + } catch (YarnException e) { + LOG.warn("Failed to parse cgroups " + memswStat, e); + } + return UNAVAILABLE; + } + + @Override + public long getVirtualMemorySize(int olderThanAge) { + if (olderThanAge > 0) { + return 0; + } + try { + processFile(memswStat, (String line) -> { + memSw = Long.parseLong(line); + return Result.Exit; + }); + return memSw; + } catch (YarnException e) { + LOG.warn("Failed to parse cgroups " + memswStat, e); + } + return UNAVAILABLE; + } + + @Override + public void updateProcessTree() { + } + + @Override + public String getProcessTreeDump() { + // We do not have a process tree in cgroups return just the pid for tracking + return pid; + } + + @Override + public boolean checkPidPgrpidForMatch() { + // We do not have a process tree in cgroups returning default ok + return true; + } + + /** + * Checks if the CGroupsResourceCalculator is available on this system. + * + * @return true if CGroupsResourceCalculator is available. False otherwise. + */ + public static boolean isAvailable() { + try { + if (!Shell.LINUX) { + LOG.info("CGroupsResourceCalculator currently is supported only on " + + "Linux."); + return false; + } + if (ResourceHandlerModule.getCGroupsHandler() == null) { + LOG.info("CGroupsResourceCalculator requires enabling CGroups"); + return false; + } + } catch (SecurityException se) { + LOG.warn("Failed to get Operating System name. " + se); + return false; + } + return true; + } + + private void readTotalProcessJiffies() { + try { + final BigInteger[] totalCPUTimeJiffies = new BigInteger[1]; + totalCPUTimeJiffies[0] = BigInteger.ZERO; + processFile(cpuStat, (String line) -> { + if (line.startsWith(USER)) { + totalCPUTimeJiffies[0] = totalCPUTimeJiffies[0].add( + new BigInteger(line.substring(USER.length()))); + } + if (line.startsWith(SYSTEM)) { + totalCPUTimeJiffies[0] = totalCPUTimeJiffies[0].add( + new BigInteger(line.substring(SYSTEM.length()))); + } + return Result.Continue; + }); + processTotalJiffies = totalCPUTimeJiffies[0]; + } catch (YarnException e) { + LOG.warn("Failed to parse " + pid, e); + } + } + + private String getCGroupRelativePathForPid( + CGroupsHandler.CGroupController controller) + throws YarnException { + File pidCgroupFile = new File(new File(procfsDir, pid), CGROUP); + String[] result = new String[1]; + processFile(pidCgroupFile, (String line)->{ + Matcher m = CGROUP_FILE_FORMAT.matcher(line); + boolean mat = m.find(); + if (mat) { + if (m.group(2).contains(controller.getName())) { + result[0] = m.group(3); + return Result.Exit; + } + } else { + LOG.warn( + "Unexpected: cgroup file is not in the expected format" + + " for process with pid " + pid); + } + return Result.Continue; + }); + if (result[0] == null) { + throw new YarnException(controller.getName() + "CGroup for " + pid + + " not found " + pidCgroupFile); + } + return result[0]; + } + + private void processFile(File file, Function processLine) + throws YarnException { + // Read "procfsDir//stat" file - typically /proc//stat + BufferedReader in = null; + InputStreamReader fReader = null; + try { + fReader = new InputStreamReader( + new FileInputStream( + file), Charset.forName("UTF-8")); + in = new BufferedReader(fReader); + } catch (FileNotFoundException f) { + throw new YarnException("The process vanished in the interim " + pid, f); + } + + try { + String str; + while ((str = in.readLine()) != null) { + Result result = processLine.apply(str); + if (result == Result.Exit) { + return; + } + } + } catch (IOException io) { + throw new YarnException("Error reading the stream " + io, io); + } finally { + // Close the streams + try { + fReader.close(); + try { + in.close(); + } catch (IOException i) { + LOG.warn("Error closing the stream " + in, i); + } + } catch (IOException i) { + LOG.warn("Error closing the stream " + fReader, i); + } + } + } + + private void setCGroupFilePaths() throws YarnException { + if (cGroupsHandler == null) { + throw new YarnException("CGroups handler is not initialized"); + } + File cpuDir = new File( + cGroupsHandler.getControllerPath( + CGroupsHandler.CGroupController.CPUACCT), + getCGroupRelativePathForPid(CGroupsHandler.CGroupController.CPUACCT)); + File memDir = new File( + cGroupsHandler.getControllerPath( + CGroupsHandler.CGroupController.MEMORY), + getCGroupRelativePathForPid(CGroupsHandler.CGroupController.MEMORY)); + cpuStat = new File(cpuDir, CPU_STAT); + memStat = new File(memDir, MEM_STAT); + memswStat = new File(memDir, MEMSW_STAT); + } +} diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/monitor/ContainersMonitorImpl.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/monitor/ContainersMonitorImpl.java index 55aac265d0e3f..9387793f7041f 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/monitor/ContainersMonitorImpl.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/monitor/ContainersMonitorImpl.java @@ -33,6 +33,7 @@ import org.apache.hadoop.yarn.conf.YarnConfiguration; import org.apache.hadoop.yarn.event.AsyncDispatcher; import org.apache.hadoop.yarn.event.Dispatcher; +import org.apache.hadoop.yarn.exceptions.YarnException; import org.apache.hadoop.yarn.server.api.records.OverAllocationInfo; import org.apache.hadoop.yarn.server.api.records.ResourceThresholds; import org.apache.hadoop.yarn.server.nodemanager.ContainerExecutor; @@ -41,6 +42,7 @@ import org.apache.hadoop.yarn.server.nodemanager.containermanager.container.Container; import org.apache.hadoop.yarn.server.nodemanager.containermanager.container.ContainerImpl; import org.apache.hadoop.yarn.server.nodemanager.containermanager.container.ContainerKillEvent; +import org.apache.hadoop.yarn.server.nodemanager.containermanager.linux.resources.CGroupsResourceCalculator; import org.apache.hadoop.yarn.server.nodemanager.timelineservice.NMTimelinePublisher; import org.apache.hadoop.yarn.server.nodemanager.util.NodeManagerHardwareUtils; import org.apache.hadoop.yarn.util.ResourceCalculatorPlugin; @@ -552,9 +554,8 @@ private void initializeProcessTrees( LOG.debug("Tracking ProcessTree " + pId + " for the first time"); } ResourceCalculatorProcessTree pt = - ResourceCalculatorProcessTree. - getResourceCalculatorProcessTree( - pId, processTreeClass, conf); + getResourceCalculatorProcessTree( + pId); ptInfo.setPid(pId); ptInfo.setProcessTree(pt); @@ -587,6 +588,31 @@ private void initializeProcessTrees( // End of initializing any uninitialized processTrees } + /** + * Get the best process tree calculator. + * @param pId container process id + * @return process tree calculator + */ + private ResourceCalculatorProcessTree + getResourceCalculatorProcessTree(String pId) { + ResourceCalculatorProcessTree pt = null; + + // CGroups is best in perforance, so try to use it, if it is enabled + if (processTreeClass == null && + CGroupsResourceCalculator.isAvailable()) { + try { + pt = new CGroupsResourceCalculator(pId); + } catch (YarnException e) { + LOG.info("CGroupsResourceCalculator cannot be created", e); + } + } + + pt = ResourceCalculatorProcessTree. + getResourceCalculatorProcessTree( + pId, processTreeClass, conf); + return pt; + } + /** * Record usage metrics. * @param containerId container id diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/test/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/linux/resources/TestCGroupsResourceCalculator.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/test/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/linux/resources/TestCGroupsResourceCalculator.java new file mode 100644 index 0000000000000..bb3010ff86a4e --- /dev/null +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/test/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/linux/resources/TestCGroupsResourceCalculator.java @@ -0,0 +1,167 @@ +package org.apache.hadoop.yarn.server.nodemanager.containermanager.linux.resources; + +import org.apache.commons.io.FileUtils; +import org.apache.hadoop.yarn.exceptions.YarnException; +import org.apache.hadoop.yarn.util.ControlledClock; +import org.apache.hadoop.yarn.util.ResourceCalculatorProcessTree; +import org.junit.Assert; +import org.junit.Test; + +import java.io.File; +import java.io.FileNotFoundException; + +import static org.mockito.Mockito.*; + +public class TestCGroupsResourceCalculator { + + ControlledClock clock = new ControlledClock(); + CGroupsHandler cGroupsHandler = mock(CGroupsHandler.class); + + @Test + public void testNoPid() throws Exception { + try { + CGroupsResourceCalculator calculator = + new CGroupsResourceCalculator( + "1234", ".", cGroupsHandler, clock); + } catch (YarnException e) { + Assert.assertTrue("Missing file should be caught", + e.getCause() instanceof FileNotFoundException); + } + } + + @Test + public void testNoMemoryCGgroupMount() throws Exception { + File procfs = new File("/tmp/" + this.getClass().getName() + "/1234"); + Assert.assertTrue("Setup error", procfs.mkdirs()); + try { + FileUtils.writeStringToFile( + new File(procfs, CGroupsResourceCalculator.CGROUP), + "7:devices:/yarn\n" + + "6:cpuacct,cpu:/yarn\n" + + "5:pids:/yarn\n"); + try { + CGroupsResourceCalculator calculator = + new CGroupsResourceCalculator( + "1234", "/tmp/" + this.getClass().getName(), + cGroupsHandler, clock); + } catch (YarnException e) { + Assert.assertTrue("Missing file should be caught", + e.getMessage().startsWith("memoryCGroup")); + } + } finally { + FileUtils.deleteDirectory(procfs); + } + } + + @Test + public void testNoCGgroup() throws Exception { + File procfs = new File("/tmp/" + this.getClass().getName() + "/1234"); + Assert.assertTrue("Setup error", procfs.mkdirs()); + try { + FileUtils.writeStringToFile( + new File(procfs, CGroupsResourceCalculator.CGROUP), + "7:devices:/yarn\n" + + "6:cpuacct,cpu:/yarn\n" + + "5:pids:/yarn\n" + + "4:memory:/yarn\n"); + try { + CGroupsResourceCalculator calculator = + new CGroupsResourceCalculator( + "1234", "/tmp/" + this.getClass().getName(), + cGroupsHandler, clock); + Assert.assertEquals("cgroups should be missing", + (long)ResourceCalculatorProcessTree.UNAVAILABLE, + calculator.getRssMemorySize(0)); + } catch (YarnException e) { + Assert.assertTrue("Missing file should be caught", + e.getMessage().startsWith("memoryCGroup")); + } + } finally { + FileUtils.deleteDirectory(procfs); + } + } + + @Test + public void testCPUParsing() throws Exception { + File cgcpuacctDir = + new File("/tmp/" + this.getClass().getName() + "/cgcpuacct/yarn"); + File procfs = new File("/tmp/" + this.getClass().getName() + "/1234"); + when(cGroupsHandler.getControllerPath( + CGroupsHandler.CGroupController.CPUACCT)). + thenReturn(cgcpuacctDir.getParent()); + Assert.assertTrue("Setup error", procfs.mkdirs()); + Assert.assertTrue("Setup error", cgcpuacctDir.mkdirs()); + try { + FileUtils.writeStringToFile( + new File(procfs, CGroupsResourceCalculator.CGROUP), + "7:devices:/yarn\n" + + "6:cpuacct,cpu:/yarn\n" + + "5:pids:/yarn\n" + + "4:memory:/yarn\n"); + FileUtils.writeStringToFile( + new File(cgcpuacctDir, CGroupsResourceCalculator.CPU_STAT), + "Can you handle this?\n" + + "user 5415\n" + + "system 3632"); + CGroupsResourceCalculator calculator = + new CGroupsResourceCalculator( + "1234", "/tmp/" + this.getClass().getName(), + cGroupsHandler, clock); + Assert.assertEquals("Incorrect CPU usage", + 90470, + calculator.getCumulativeCpuTime()); + } finally { + FileUtils.deleteDirectory(procfs); + FileUtils.deleteDirectory(cgcpuacctDir); + } + } + + @Test + public void testMemoryParsing() throws Exception { + File cgcpuacctDir = + new File("/tmp/" + this.getClass().getName() + "/cgcpuacct/yarn"); + File cgmemoryDir = + new File("/tmp/" + this.getClass().getName() + "/memory/yarn"); + File procfs = new File("/tmp/" + this.getClass().getName() + "/1234"); + when(cGroupsHandler.getControllerPath( + CGroupsHandler.CGroupController.MEMORY)). + thenReturn(cgmemoryDir.getParent()); + Assert.assertTrue("Setup error", procfs.mkdirs()); + Assert.assertTrue("Setup error", cgcpuacctDir.mkdirs()); + Assert.assertTrue("Setup error", cgmemoryDir.mkdirs()); + try { + FileUtils.writeStringToFile( + new File(procfs, CGroupsResourceCalculator.CGROUP), + "6:cpuacct,cpu:/yarn\n" + + "4:memory:/yarn\n"); + FileUtils.writeStringToFile( + new File(cgmemoryDir, CGroupsResourceCalculator.MEM_STAT), + "418496512\n"); + + CGroupsResourceCalculator calculator = + new CGroupsResourceCalculator( + "1234", "/tmp/" + this.getClass().getName(), + cGroupsHandler, clock); + + // Test the case where memsw is not available (Ubuntu) + Assert.assertEquals("Incorrect memory usage", + 418496512, + calculator.getRssMemorySize()); + Assert.assertEquals("Incorrect swap usage", + (long)ResourceCalculatorProcessTree.UNAVAILABLE, + calculator.getVirtualMemorySize()); + + // Test the case where memsw is available + FileUtils.writeStringToFile( + new File(cgmemoryDir, CGroupsResourceCalculator.MEMSW_STAT), + "418496513\n"); + Assert.assertEquals("Incorrect swap usage", + 418496513, + calculator.getVirtualMemorySize()); + } finally { + FileUtils.deleteDirectory(procfs); + FileUtils.deleteDirectory(cgcpuacctDir); + FileUtils.deleteDirectory(cgmemoryDir); + } + } +} From 043926ca77a21ea429b547c53bde6cce9b8a5c1a Mon Sep 17 00:00:00 2001 From: Miklos Szegedi Date: Tue, 6 Jun 2017 13:27:59 -0700 Subject: [PATCH 2/8] YARN-6668 Add new CGroupsResourceCalculator --- .../resources/CGroupsResourceCalculator.java | 80 ++++++++++------ .../monitor/ContainersMonitorImpl.java | 8 +- .../TestCGroupsResourceCalculator.java | 91 ++++++++++--------- 3 files changed, 102 insertions(+), 77 deletions(-) diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/linux/resources/CGroupsResourceCalculator.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/linux/resources/CGroupsResourceCalculator.java index 466f7676e0839..5b0400a6f51eb 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/linux/resources/CGroupsResourceCalculator.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/linux/resources/CGroupsResourceCalculator.java @@ -42,7 +42,6 @@ * A cgroups file-system based Resource calculator without the process tree * features. */ - public class CGroupsResourceCalculator extends ResourceCalculatorProcessTree { enum Result { Continue, @@ -71,8 +70,8 @@ enum Result { private final CpuTimeTracker cpuTimeTracker; private Clock clock; - private long mem = UNAVAILABLE; - private long memSw = UNAVAILABLE; + private final static Object LOCK = new Object(); + private static boolean firstError = true; public CGroupsResourceCalculator(String pid) throws YarnException { this(pid, PROCFS, ResourceHandlerModule.getCGroupsHandler(), @@ -115,36 +114,18 @@ public long getCumulativeCpuTime() { @Override public long getRssMemorySize(int olderThanAge) { - if (olderThanAge > 0) { + if (olderThanAge > 1) { return 0; } - try { - processFile(memStat, (String line) -> { - mem = Long.parseLong(line); - return Result.Exit; - }); - return mem; - } catch (YarnException e) { - LOG.warn("Failed to parse cgroups " + memswStat, e); - } - return UNAVAILABLE; + return getMemorySize(memStat); } @Override public long getVirtualMemorySize(int olderThanAge) { - if (olderThanAge > 0) { + if (olderThanAge > 1) { return 0; } - try { - processFile(memswStat, (String line) -> { - memSw = Long.parseLong(line); - return Result.Exit; - }); - return memSw; - } catch (YarnException e) { - LOG.warn("Failed to parse cgroups " + memswStat, e); - } - return UNAVAILABLE; + return getMemorySize(memswStat); } @Override @@ -186,6 +167,25 @@ public static boolean isAvailable() { return true; } + private long getMemorySize(File cgroupUsageFile) { + long[] mem = new long[1]; + try { + processFile(cgroupUsageFile, (String line) -> { + mem[0] = Long.parseLong(line); + return Result.Exit; + }); + return mem[0]; + } catch (YarnException e) { + synchronized (LOCK) { + if (firstError) { + LOG.warn("Failed to parse cgroups " + memswStat, e); + firstError = false; + } + } + } + return UNAVAILABLE; + } + private void readTotalProcessJiffies() { try { final BigInteger[] totalCPUTimeJiffies = new BigInteger[1]; @@ -203,7 +203,12 @@ private void readTotalProcessJiffies() { }); processTotalJiffies = totalCPUTimeJiffies[0]; } catch (YarnException e) { - LOG.warn("Failed to parse " + pid, e); + synchronized (LOCK) { + if (firstError) { + LOG.warn("Failed to parse " + pid, e); + firstError = false; + } + } } } @@ -217,7 +222,22 @@ private String getCGroupRelativePathForPid( boolean mat = m.find(); if (mat) { if (m.group(2).contains(controller.getName())) { - result[0] = m.group(3); + // Instead of returning the full path we compose it + // based on the last item as the container id + // This helps to avoid confusion within a privileged Docker container + // where the path is referred in /proc//cgroup as + // /docker//hadoop-yarn/ + // but it is /hadoop-yarn/ in the cgroups hierarchy + String cgroupPath = m.group(3); + String cgroup = + new File(cgroupPath).toPath().getFileName().toString(); + + if (cgroup!=null && !cgroup.isEmpty()) { + result[0] = cGroupsHandler.getRelativePathForCGroup(cgroup); + } else { + LOG.warn("Invalid cgroup path " + cgroupPath + + " for " + pidCgroupFile); + } return Result.Exit; } } else { @@ -228,7 +248,7 @@ private String getCGroupRelativePathForPid( return Result.Continue; }); if (result[0] == null) { - throw new YarnException(controller.getName() + "CGroup for " + pid + + throw new YarnException(controller.getName() + " CGroup for pid " + pid + " not found " + pidCgroupFile); } return result[0]; @@ -237,8 +257,8 @@ private String getCGroupRelativePathForPid( private void processFile(File file, Function processLine) throws YarnException { // Read "procfsDir//stat" file - typically /proc//stat - BufferedReader in = null; - InputStreamReader fReader = null; + BufferedReader in; + InputStreamReader fReader; try { fReader = new InputStreamReader( new FileInputStream( diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/monitor/ContainersMonitorImpl.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/monitor/ContainersMonitorImpl.java index 9387793f7041f..159e08ac2a859 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/monitor/ContainersMonitorImpl.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/monitor/ContainersMonitorImpl.java @@ -607,9 +607,11 @@ private void initializeProcessTrees( } } - pt = ResourceCalculatorProcessTree. - getResourceCalculatorProcessTree( - pId, processTreeClass, conf); + if (pt == null) { + pt = ResourceCalculatorProcessTree. + getResourceCalculatorProcessTree( + pId, processTreeClass, conf); + } return pt; } diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/test/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/linux/resources/TestCGroupsResourceCalculator.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/test/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/linux/resources/TestCGroupsResourceCalculator.java index bb3010ff86a4e..4525b2e016bf2 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/test/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/linux/resources/TestCGroupsResourceCalculator.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/test/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/linux/resources/TestCGroupsResourceCalculator.java @@ -17,6 +17,10 @@ public class TestCGroupsResourceCalculator { ControlledClock clock = new ControlledClock(); CGroupsHandler cGroupsHandler = mock(CGroupsHandler.class); + public TestCGroupsResourceCalculator() { + when(cGroupsHandler.getRelativePathForCGroup(any())).thenReturn("/yarn/container_1"); + } + @Test public void testNoPid() throws Exception { try { @@ -36,9 +40,9 @@ public void testNoMemoryCGgroupMount() throws Exception { try { FileUtils.writeStringToFile( new File(procfs, CGroupsResourceCalculator.CGROUP), - "7:devices:/yarn\n" + - "6:cpuacct,cpu:/yarn\n" + - "5:pids:/yarn\n"); + "7:devices:/yarn/container_1\n" + + "6:cpuacct,cpu:/yarn/container_1\n" + + "5:pids:/yarn/container_1\n"); try { CGroupsResourceCalculator calculator = new CGroupsResourceCalculator( @@ -46,10 +50,10 @@ public void testNoMemoryCGgroupMount() throws Exception { cGroupsHandler, clock); } catch (YarnException e) { Assert.assertTrue("Missing file should be caught", - e.getMessage().startsWith("memoryCGroup")); + e.getMessage().startsWith("memory CGroup")); } } finally { - FileUtils.deleteDirectory(procfs); + FileUtils.deleteDirectory(new File("/tmp/" + this.getClass().getName())); } } @@ -60,46 +64,44 @@ public void testNoCGgroup() throws Exception { try { FileUtils.writeStringToFile( new File(procfs, CGroupsResourceCalculator.CGROUP), - "7:devices:/yarn\n" + - "6:cpuacct,cpu:/yarn\n" + - "5:pids:/yarn\n" + - "4:memory:/yarn\n"); - try { - CGroupsResourceCalculator calculator = - new CGroupsResourceCalculator( - "1234", "/tmp/" + this.getClass().getName(), - cGroupsHandler, clock); - Assert.assertEquals("cgroups should be missing", - (long)ResourceCalculatorProcessTree.UNAVAILABLE, - calculator.getRssMemorySize(0)); - } catch (YarnException e) { - Assert.assertTrue("Missing file should be caught", - e.getMessage().startsWith("memoryCGroup")); - } + "7:devices:/yarn/container_1\n" + + "6:cpuacct,cpu:/yarn/container_1\n" + + "5:pids:/yarn/container_1\n" + + "4:memory:/yarn/container_1\n"); + + CGroupsResourceCalculator calculator = + new CGroupsResourceCalculator( + "1234", "/tmp/" + this.getClass().getName(), + cGroupsHandler, clock); + Assert.assertEquals("cgroups should be missing", + (long)ResourceCalculatorProcessTree.UNAVAILABLE, + calculator.getRssMemorySize(0)); } finally { - FileUtils.deleteDirectory(procfs); + FileUtils.deleteDirectory(new File("/tmp/" + this.getClass().getName())); } } @Test public void testCPUParsing() throws Exception { File cgcpuacctDir = - new File("/tmp/" + this.getClass().getName() + "/cgcpuacct/yarn"); + new File("/tmp/" + this.getClass().getName() + "/cgcpuacct"); + File cgcpuacctContainerDir = + new File(cgcpuacctDir, "/yarn/container_1"); File procfs = new File("/tmp/" + this.getClass().getName() + "/1234"); when(cGroupsHandler.getControllerPath( CGroupsHandler.CGroupController.CPUACCT)). - thenReturn(cgcpuacctDir.getParent()); + thenReturn(cgcpuacctDir.getAbsolutePath()); Assert.assertTrue("Setup error", procfs.mkdirs()); - Assert.assertTrue("Setup error", cgcpuacctDir.mkdirs()); + Assert.assertTrue("Setup error", cgcpuacctContainerDir.mkdirs()); try { FileUtils.writeStringToFile( new File(procfs, CGroupsResourceCalculator.CGROUP), - "7:devices:/yarn\n" + - "6:cpuacct,cpu:/yarn\n" + - "5:pids:/yarn\n" + - "4:memory:/yarn\n"); + "7:devices:/yarn/container_1\n" + + "6:cpuacct,cpu:/yarn/container_1\n" + + "5:pids:/yarn/container_1\n" + + "4:memory:/yarn/container_1\n"); FileUtils.writeStringToFile( - new File(cgcpuacctDir, CGroupsResourceCalculator.CPU_STAT), + new File(cgcpuacctContainerDir, CGroupsResourceCalculator.CPU_STAT), "Can you handle this?\n" + "user 5415\n" + "system 3632"); @@ -111,31 +113,34 @@ public void testCPUParsing() throws Exception { 90470, calculator.getCumulativeCpuTime()); } finally { - FileUtils.deleteDirectory(procfs); - FileUtils.deleteDirectory(cgcpuacctDir); + FileUtils.deleteDirectory(new File("/tmp/" + this.getClass().getName())); } } @Test public void testMemoryParsing() throws Exception { File cgcpuacctDir = - new File("/tmp/" + this.getClass().getName() + "/cgcpuacct/yarn"); + new File("/tmp/" + this.getClass().getName() + "/cgcpuacct"); + File cgcpuacctContainerDir = + new File(cgcpuacctDir, "/yarn/container_1"); File cgmemoryDir = - new File("/tmp/" + this.getClass().getName() + "/memory/yarn"); + new File("/tmp/" + this.getClass().getName() + "/memory"); + File cgMemoryContainerDir = + new File(cgmemoryDir, "/yarn/container_1"); File procfs = new File("/tmp/" + this.getClass().getName() + "/1234"); when(cGroupsHandler.getControllerPath( CGroupsHandler.CGroupController.MEMORY)). - thenReturn(cgmemoryDir.getParent()); + thenReturn(cgmemoryDir.getAbsolutePath()); Assert.assertTrue("Setup error", procfs.mkdirs()); - Assert.assertTrue("Setup error", cgcpuacctDir.mkdirs()); - Assert.assertTrue("Setup error", cgmemoryDir.mkdirs()); + Assert.assertTrue("Setup error", cgcpuacctContainerDir.mkdirs()); + Assert.assertTrue("Setup error", cgMemoryContainerDir.mkdirs()); try { FileUtils.writeStringToFile( new File(procfs, CGroupsResourceCalculator.CGROUP), - "6:cpuacct,cpu:/yarn\n" + - "4:memory:/yarn\n"); + "6:cpuacct,cpu:/yarn/container_1\n" + + "4:memory:/yarn/container_1\n"); FileUtils.writeStringToFile( - new File(cgmemoryDir, CGroupsResourceCalculator.MEM_STAT), + new File(cgMemoryContainerDir, CGroupsResourceCalculator.MEM_STAT), "418496512\n"); CGroupsResourceCalculator calculator = @@ -153,15 +158,13 @@ public void testMemoryParsing() throws Exception { // Test the case where memsw is available FileUtils.writeStringToFile( - new File(cgmemoryDir, CGroupsResourceCalculator.MEMSW_STAT), + new File(cgMemoryContainerDir, CGroupsResourceCalculator.MEMSW_STAT), "418496513\n"); Assert.assertEquals("Incorrect swap usage", 418496513, calculator.getVirtualMemorySize()); } finally { - FileUtils.deleteDirectory(procfs); - FileUtils.deleteDirectory(cgcpuacctDir); - FileUtils.deleteDirectory(cgmemoryDir); + FileUtils.deleteDirectory(new File("/tmp/" + this.getClass().getName())); } } } From 701b411f9ce8d2d489925853afc07d6153ca0a74 Mon Sep 17 00:00:00 2001 From: Miklos Szegedi Date: Mon, 19 Jun 2017 10:36:52 -0700 Subject: [PATCH 3/8] YARN-6668 Cgroups based resource calculator --- .../util/ResourceCalculatorProcessTree.java | 2 +- .../resources/CGroupsResourceCalculator.java | 38 ++++++++- .../TestCGroupsResourceCalculator.java | 85 ++++++++++++++++++- 3 files changed, 119 insertions(+), 6 deletions(-) diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/util/ResourceCalculatorProcessTree.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/util/ResourceCalculatorProcessTree.java index adbba84767bf8..2512a77214624 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/util/ResourceCalculatorProcessTree.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/util/ResourceCalculatorProcessTree.java @@ -37,7 +37,7 @@ @Public @Evolving public abstract class ResourceCalculatorProcessTree extends Configured { - static protected final Log LOG = LogFactory + protected static final Log LOG = LogFactory .getLog(ResourceCalculatorProcessTree.class); public static final int UNAVAILABLE = -1; diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/linux/resources/CGroupsResourceCalculator.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/linux/resources/CGroupsResourceCalculator.java index 5b0400a6f51eb..a8df076340a81 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/linux/resources/CGroupsResourceCalculator.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/linux/resources/CGroupsResourceCalculator.java @@ -18,6 +18,7 @@ package org.apache.hadoop.yarn.server.nodemanager.containermanager.linux.resources; +import com.google.common.annotations.VisibleForTesting; import org.apache.hadoop.util.CpuTimeTracker; import org.apache.hadoop.util.Shell; import org.apache.hadoop.util.SysInfoLinux; @@ -73,11 +74,34 @@ enum Result { private final static Object LOCK = new Object(); private static boolean firstError = true; + /** + * Create resource calculator for all Yarn containers. + * @throws YarnException Could not access cgroups + */ + public CGroupsResourceCalculator() throws YarnException { + this(null, PROCFS, ResourceHandlerModule.getCGroupsHandler(), + SystemClock.getInstance()); + } + + /** + * Create resource calculator for the container that has the specified pid. + * @param pid A pid from the cgroup or null for all containers + * @throws YarnException Could not access cgroups + */ public CGroupsResourceCalculator(String pid) throws YarnException { this(pid, PROCFS, ResourceHandlerModule.getCGroupsHandler(), SystemClock.getInstance()); } + /** + * Create resource calculator for testing. + * @param pid A pid from the cgroup or null for all containers + * @param procfsDir Path to /proc or a mock /proc directory + * @param cGroupsHandler Initialized cgroups handler object + * @param clock A clock object + * @throws YarnException YarnException Could not access cgroups + */ + @VisibleForTesting CGroupsResourceCalculator(String pid, String procfsDir, CGroupsHandler cGroupsHandler, Clock clock) throws YarnException { @@ -212,6 +236,16 @@ private void readTotalProcessJiffies() { } } + private String getCGroupRelativePath( + CGroupsHandler.CGroupController controller) + throws YarnException { + if (pid == null) { + return cGroupsHandler.getRelativePathForCGroup(""); + } else { + return getCGroupRelativePathForPid(controller); + } + } + private String getCGroupRelativePathForPid( CGroupsHandler.CGroupController controller) throws YarnException { @@ -300,11 +334,11 @@ private void setCGroupFilePaths() throws YarnException { File cpuDir = new File( cGroupsHandler.getControllerPath( CGroupsHandler.CGroupController.CPUACCT), - getCGroupRelativePathForPid(CGroupsHandler.CGroupController.CPUACCT)); + getCGroupRelativePath(CGroupsHandler.CGroupController.CPUACCT)); File memDir = new File( cGroupsHandler.getControllerPath( CGroupsHandler.CGroupController.MEMORY), - getCGroupRelativePathForPid(CGroupsHandler.CGroupController.MEMORY)); + getCGroupRelativePath(CGroupsHandler.CGroupController.MEMORY)); cpuStat = new File(cpuDir, CPU_STAT); memStat = new File(memDir, MEM_STAT); memswStat = new File(memDir, MEMSW_STAT); diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/test/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/linux/resources/TestCGroupsResourceCalculator.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/test/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/linux/resources/TestCGroupsResourceCalculator.java index 4525b2e016bf2..1762cb9e6c386 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/test/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/linux/resources/TestCGroupsResourceCalculator.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/test/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/linux/resources/TestCGroupsResourceCalculator.java @@ -12,13 +12,18 @@ import static org.mockito.Mockito.*; +/** + * Unit test for CGroupsResourceCalculator. + */ public class TestCGroupsResourceCalculator { - ControlledClock clock = new ControlledClock(); - CGroupsHandler cGroupsHandler = mock(CGroupsHandler.class); + private ControlledClock clock = new ControlledClock(); + private CGroupsHandler cGroupsHandler = mock(CGroupsHandler.class); public TestCGroupsResourceCalculator() { - when(cGroupsHandler.getRelativePathForCGroup(any())).thenReturn("/yarn/container_1"); + when(cGroupsHandler.getRelativePathForCGroup("container_1")) + .thenReturn("/yarn/container_1"); + when(cGroupsHandler.getRelativePathForCGroup("")).thenReturn("/yarn/"); } @Test @@ -167,4 +172,78 @@ public void testMemoryParsing() throws Exception { FileUtils.deleteDirectory(new File("/tmp/" + this.getClass().getName())); } } + + @Test + public void testCPUParsingRoot() throws Exception { + File cgcpuacctDir = + new File("/tmp/" + this.getClass().getName() + "/cgcpuacct"); + File cgcpuacctRootDir = + new File(cgcpuacctDir, "/yarn"); + when(cGroupsHandler.getControllerPath( + CGroupsHandler.CGroupController.CPUACCT)). + thenReturn(cgcpuacctDir.getAbsolutePath()); + Assert.assertTrue("Setup error", cgcpuacctRootDir.mkdirs()); + try { + FileUtils.writeStringToFile( + new File(cgcpuacctRootDir, CGroupsResourceCalculator.CPU_STAT), + "user 5415\n" + + "system 3632"); + CGroupsResourceCalculator calculator = + new CGroupsResourceCalculator( + null, "/tmp/" + this.getClass().getName(), + cGroupsHandler, clock); + Assert.assertEquals("Incorrect CPU usage", + 90470, + calculator.getCumulativeCpuTime()); + } finally { + FileUtils.deleteDirectory(new File("/tmp/" + this.getClass().getName())); + } + } + + @Test + public void testMemoryParsingRoot() throws Exception { + File cgcpuacctDir = + new File("/tmp/" + this.getClass().getName() + "/cgcpuacct"); + File cgcpuacctRootDir = + new File(cgcpuacctDir, "/yarn"); + File cgmemoryDir = + new File("/tmp/" + this.getClass().getName() + "/memory"); + File cgMemoryRootDir = + new File(cgmemoryDir, "/yarn"); + File procfs = new File("/tmp/" + this.getClass().getName() + "/1234"); + when(cGroupsHandler.getControllerPath( + CGroupsHandler.CGroupController.MEMORY)). + thenReturn(cgmemoryDir.getAbsolutePath()); + Assert.assertTrue("Setup error", procfs.mkdirs()); + Assert.assertTrue("Setup error", cgcpuacctRootDir.mkdirs()); + Assert.assertTrue("Setup error", cgMemoryRootDir.mkdirs()); + try { + FileUtils.writeStringToFile( + new File(cgMemoryRootDir, CGroupsResourceCalculator.MEM_STAT), + "418496512\n"); + + CGroupsResourceCalculator calculator = + new CGroupsResourceCalculator( + null, "/tmp/" + this.getClass().getName(), + cGroupsHandler, clock); + + // Test the case where memsw is not available (Ubuntu) + Assert.assertEquals("Incorrect memory usage", + 418496512, + calculator.getRssMemorySize()); + Assert.assertEquals("Incorrect swap usage", + (long)ResourceCalculatorProcessTree.UNAVAILABLE, + calculator.getVirtualMemorySize()); + + // Test the case where memsw is available + FileUtils.writeStringToFile( + new File(cgMemoryRootDir, CGroupsResourceCalculator.MEMSW_STAT), + "418496513\n"); + Assert.assertEquals("Incorrect swap usage", + 418496513, + calculator.getVirtualMemorySize()); + } finally { + FileUtils.deleteDirectory(new File("/tmp/" + this.getClass().getName())); + } + } } From a9021baa30f78b7cd3bbe73720cba016eac48bd8 Mon Sep 17 00:00:00 2001 From: Miklos Szegedi Date: Fri, 23 Jun 2017 17:14:02 -0700 Subject: [PATCH 4/8] YARN-6668 Fix license --- .../TestCGroupsResourceCalculator.java | 18 ++++++++++++++++++ 1 file changed, 18 insertions(+) diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/test/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/linux/resources/TestCGroupsResourceCalculator.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/test/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/linux/resources/TestCGroupsResourceCalculator.java index 1762cb9e6c386..5a10e1517b322 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/test/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/linux/resources/TestCGroupsResourceCalculator.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/test/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/linux/resources/TestCGroupsResourceCalculator.java @@ -1,3 +1,21 @@ +/** + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + package org.apache.hadoop.yarn.server.nodemanager.containermanager.linux.resources; import org.apache.commons.io.FileUtils; From ad9f6526db36a6c9aaf5e831e62a1ee44abcb0e1 Mon Sep 17 00:00:00 2001 From: Miklos Szegedi Date: Thu, 3 Aug 2017 12:45:25 -0700 Subject: [PATCH 5/8] YARN-6668 Fix comments --- .../resources/CGroupsResourceCalculator.java | 80 +++++++++---------- .../resources/ResourceHandlerModule.java | 36 +++++++-- .../resources/TestResourceHandlerModule.java | 8 +- 3 files changed, 70 insertions(+), 54 deletions(-) diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/linux/resources/CGroupsResourceCalculator.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/linux/resources/CGroupsResourceCalculator.java index a8df076340a81..9dbd68c3704c4 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/linux/resources/CGroupsResourceCalculator.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/linux/resources/CGroupsResourceCalculator.java @@ -19,6 +19,8 @@ package org.apache.hadoop.yarn.server.nodemanager.containermanager.linux.resources; import com.google.common.annotations.VisibleForTesting; +import org.apache.commons.logging.Log; +import org.apache.commons.logging.LogFactory; import org.apache.hadoop.util.CpuTimeTracker; import org.apache.hadoop.util.Shell; import org.apache.hadoop.util.SysInfoLinux; @@ -48,6 +50,8 @@ enum Result { Continue, Exit } + protected static final Log LOG = LogFactory + .getLog(CGroupsResourceCalculator.class); private static final String PROCFS = "/proc"; static final String CGROUP = "cgroup"; static final String CPU_STAT = "cpuacct.stat"; @@ -67,7 +71,6 @@ enum Result { private File memswStat; private final long jiffyLengthMs; - private BigInteger processTotalJiffies = BigInteger.ZERO; private final CpuTimeTracker cpuTimeTracker; private Clock clock; @@ -119,10 +122,13 @@ public CGroupsResourceCalculator(String pid) throws YarnException { @Override public float getCpuUsagePercent() { - readTotalProcessJiffies(); - cpuTimeTracker.updateElapsedJiffies( - processTotalJiffies, - clock.getTime()); + try { + cpuTimeTracker.updateElapsedJiffies( + readTotalProcessJiffies(), + clock.getTime()); + } catch (YarnException e) { + LOG.debug(e.getMessage()); + } return cpuTimeTracker.getCpuTrackerUsagePercent(); } @@ -131,9 +137,11 @@ public long getCumulativeCpuTime() { if (jiffyLengthMs < 0) { return UNAVAILABLE; } - readTotalProcessJiffies(); - return - processTotalJiffies.longValue() * jiffyLengthMs; + try { + return readTotalProcessJiffies().longValue() * jiffyLengthMs; + } catch (YarnException e) { + return UNAVAILABLE; + } } @Override @@ -180,8 +188,11 @@ public static boolean isAvailable() { + "Linux."); return false; } - if (ResourceHandlerModule.getCGroupsHandler() == null) { - LOG.info("CGroupsResourceCalculator requires enabling CGroups"); + if (ResourceHandlerModule.getCGroupsHandler() == null || + ResourceHandlerModule.getCpuResourceHandler() == null || + ResourceHandlerModule.getMemoryResourceHandler() == null) { + LOG.info("CGroupsResourceCalculator requires enabling CGroups" + + "cpu and memory"); return false; } } catch (SecurityException se) { @@ -210,7 +221,7 @@ private long getMemorySize(File cgroupUsageFile) { return UNAVAILABLE; } - private void readTotalProcessJiffies() { + private BigInteger readTotalProcessJiffies() throws YarnException{ try { final BigInteger[] totalCPUTimeJiffies = new BigInteger[1]; totalCPUTimeJiffies[0] = BigInteger.ZERO; @@ -225,7 +236,7 @@ private void readTotalProcessJiffies() { } return Result.Continue; }); - processTotalJiffies = totalCPUTimeJiffies[0]; + return totalCPUTimeJiffies[0]; } catch (YarnException e) { synchronized (LOCK) { if (firstError) { @@ -233,6 +244,7 @@ private void readTotalProcessJiffies() { firstError = false; } } + throw new YarnException("Cannot read process jiffies", e); } } @@ -291,39 +303,23 @@ private String getCGroupRelativePathForPid( private void processFile(File file, Function processLine) throws YarnException { // Read "procfsDir//stat" file - typically /proc//stat - BufferedReader in; - InputStreamReader fReader; - try { - fReader = new InputStreamReader( - new FileInputStream( - file), Charset.forName("UTF-8")); - in = new BufferedReader(fReader); - } catch (FileNotFoundException f) { - throw new YarnException("The process vanished in the interim " + pid, f); - } - - try { - String str; - while ((str = in.readLine()) != null) { - Result result = processLine.apply(str); - if (result == Result.Exit) { - return; - } - } - } catch (IOException io) { - throw new YarnException("Error reading the stream " + io, io); - } finally { - // Close the streams - try { - fReader.close(); + try (InputStreamReader fReader = new InputStreamReader( + new FileInputStream(file), Charset.forName("UTF-8"))) { + try (BufferedReader in = new BufferedReader(fReader)) { try { - in.close(); - } catch (IOException i) { - LOG.warn("Error closing the stream " + in, i); + String str; + while ((str = in.readLine()) != null) { + Result result = processLine.apply(str); + if (result == Result.Exit) { + return; + } + } + } catch (IOException io) { + throw new YarnException("Error reading the stream " + io, io); } - } catch (IOException i) { - LOG.warn("Error closing the stream " + fReader, i); } + } catch (IOException f) { + throw new YarnException("The process vanished in the interim " + pid, f); } } diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/linux/resources/ResourceHandlerModule.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/linux/resources/ResourceHandlerModule.java index 7fc04bdb41efa..69f8324a61804 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/linux/resources/ResourceHandlerModule.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/linux/resources/ResourceHandlerModule.java @@ -87,7 +87,27 @@ public static CGroupsHandler getCGroupsHandler() { return cGroupsHandler; } - private static CGroupsCpuResourceHandlerImpl getCGroupsCpuResourceHandler( + public static OutboundBandwidthResourceHandler + getNetworkResourceHandler() { + return trafficControlBandwidthHandler; + } + + public static DiskResourceHandler + getDiskResourceHandler() { + return cGroupsBlkioResourceHandler; + } + + public static MemoryResourceHandler + getMemoryResourceHandler() { + return cGroupsMemoryResourceHandler; + } + + public static CpuResourceHandler + getCpuResourceHandler() { + return cGroupsCpuResourceHandler; + } + + private static CGroupsCpuResourceHandlerImpl initCGroupsCpuResourceHandler( Configuration conf) throws ResourceHandlerException { boolean cgroupsCpuEnabled = conf.getBoolean(YarnConfiguration.NM_CPU_RESOURCE_ENABLED, @@ -137,12 +157,12 @@ private static CGroupsCpuResourceHandlerImpl getCGroupsCpuResourceHandler( } public static OutboundBandwidthResourceHandler - getOutboundBandwidthResourceHandler(Configuration conf) + initOutboundBandwidthResourceHandler(Configuration conf) throws ResourceHandlerException { return getTrafficControlBandwidthHandler(conf); } - public static DiskResourceHandler getDiskResourceHandler(Configuration conf) + public static DiskResourceHandler initDiskResourceHandler(Configuration conf) throws ResourceHandlerException { if (conf.getBoolean(YarnConfiguration.NM_DISK_RESOURCE_ENABLED, YarnConfiguration.DEFAULT_NM_DISK_RESOURCE_ENABLED)) { @@ -166,7 +186,7 @@ private static CGroupsBlkioResourceHandlerImpl getCgroupsBlkioResourceHandler( return cGroupsBlkioResourceHandler; } - public static MemoryResourceHandler getMemoryResourceHandler( + public static MemoryResourceHandler initMemoryResourceHandler( Configuration conf) throws ResourceHandlerException { if (conf.getBoolean(YarnConfiguration.NM_MEMORY_RESOURCE_ENABLED, YarnConfiguration.DEFAULT_NM_MEMORY_RESOURCE_ENABLED)) { @@ -201,10 +221,10 @@ private static void initializeConfiguredResourceHandlerChain( Configuration conf) throws ResourceHandlerException { ArrayList handlerList = new ArrayList<>(); - addHandlerIfNotNull(handlerList, getOutboundBandwidthResourceHandler(conf)); - addHandlerIfNotNull(handlerList, getDiskResourceHandler(conf)); - addHandlerIfNotNull(handlerList, getMemoryResourceHandler(conf)); - addHandlerIfNotNull(handlerList, getCGroupsCpuResourceHandler(conf)); + addHandlerIfNotNull(handlerList, initOutboundBandwidthResourceHandler(conf)); + addHandlerIfNotNull(handlerList, initDiskResourceHandler(conf)); + addHandlerIfNotNull(handlerList, initMemoryResourceHandler(conf)); + addHandlerIfNotNull(handlerList, initCGroupsCpuResourceHandler(conf)); resourceHandlerChain = new ResourceHandlerChain(handlerList); } diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/test/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/linux/resources/TestResourceHandlerModule.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/test/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/linux/resources/TestResourceHandlerModule.java index 9da37dd294f81..c9a7ad2ebf717 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/test/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/linux/resources/TestResourceHandlerModule.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/test/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/linux/resources/TestResourceHandlerModule.java @@ -52,12 +52,12 @@ public void testOutboundBandwidthHandler() { //This resourceHandler should be non-null only if network as a resource //is explicitly enabled OutboundBandwidthResourceHandler resourceHandler = ResourceHandlerModule - .getOutboundBandwidthResourceHandler(emptyConf); + .initOutboundBandwidthResourceHandler(emptyConf); Assert.assertNull(resourceHandler); //When network as a resource is enabled this should be non-null resourceHandler = ResourceHandlerModule - .getOutboundBandwidthResourceHandler(networkEnabledConf); + .initOutboundBandwidthResourceHandler(networkEnabledConf); Assert.assertNotNull(resourceHandler); //Ensure that outbound bandwidth resource handler is present in the chain @@ -78,13 +78,13 @@ public void testOutboundBandwidthHandler() { public void testDiskResourceHandler() throws Exception { DiskResourceHandler handler = - ResourceHandlerModule.getDiskResourceHandler(emptyConf); + ResourceHandlerModule.initDiskResourceHandler(emptyConf); Assert.assertNull(handler); Configuration diskConf = new YarnConfiguration(); diskConf.setBoolean(YarnConfiguration.NM_DISK_RESOURCE_ENABLED, true); - handler = ResourceHandlerModule.getDiskResourceHandler(diskConf); + handler = ResourceHandlerModule.initDiskResourceHandler(diskConf); Assert.assertNotNull(handler); ResourceHandlerChain resourceHandlerChain = From 4878b6631d012c8803aa80042ed910c493d7169c Mon Sep 17 00:00:00 2001 From: Miklos Szegedi Date: Mon, 7 Aug 2017 15:45:19 -0700 Subject: [PATCH 6/8] YARN-6668 --- .../util/ResourceCalculatorProcessTree.java | 2 +- .../linux/resources/CGroupsHandlerImpl.java | 1 + .../resources/CGroupsResourceCalculator.java | 8 +-- .../monitor/ContainersMonitorImpl.java | 2 +- .../TestCGroupsResourceCalculator.java | 67 +++++++++---------- 5 files changed, 39 insertions(+), 41 deletions(-) diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/util/ResourceCalculatorProcessTree.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/util/ResourceCalculatorProcessTree.java index 2512a77214624..7e5cf55164847 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/util/ResourceCalculatorProcessTree.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/util/ResourceCalculatorProcessTree.java @@ -37,7 +37,7 @@ @Public @Evolving public abstract class ResourceCalculatorProcessTree extends Configured { - protected static final Log LOG = LogFactory + static final Log LOG = LogFactory .getLog(ResourceCalculatorProcessTree.class); public static final int UNAVAILABLE = -1; diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/linux/resources/CGroupsHandlerImpl.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/linux/resources/CGroupsHandlerImpl.java index f2c85dfbe5d1c..debd2f729c732 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/linux/resources/CGroupsHandlerImpl.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/linux/resources/CGroupsHandlerImpl.java @@ -124,6 +124,7 @@ private void init() throws ResourceHandlerException { initializeControllerPaths(); } + @Override public String getControllerPath(CGroupController controller) { try { rwLock.readLock().lock(); diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/linux/resources/CGroupsResourceCalculator.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/linux/resources/CGroupsResourceCalculator.java index 9dbd68c3704c4..6cb05fd0c3c1e 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/linux/resources/CGroupsResourceCalculator.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/linux/resources/CGroupsResourceCalculator.java @@ -32,7 +32,6 @@ import java.io.BufferedReader; import java.io.File; import java.io.FileInputStream; -import java.io.FileNotFoundException; import java.io.IOException; import java.io.InputStreamReader; import java.math.BigInteger; @@ -112,11 +111,11 @@ public CGroupsResourceCalculator(String pid) throws YarnException { this.procfsDir = procfsDir; this.cGroupsHandler = cGroupsHandler; this.pid = pid; + this.jiffyLengthMs = (clock == SystemClock.getInstance()) ? + SysInfoLinux.JIFFY_LENGTH_IN_MILLIS : 10; this.cpuTimeTracker = - new CpuTimeTracker(SysInfoLinux.JIFFY_LENGTH_IN_MILLIS); + new CpuTimeTracker(this.jiffyLengthMs); this.clock = clock; - this.jiffyLengthMs = (clock == SystemClock.getInstance()) ? - SysInfoLinux.JIFFY_LENGTH_IN_MILLIS : 10; setCGroupFilePaths(); } @@ -178,6 +177,7 @@ public boolean checkPidPgrpidForMatch() { /** * Checks if the CGroupsResourceCalculator is available on this system. + * This assumes that Linux container executor is already initialized. * * @return true if CGroupsResourceCalculator is available. False otherwise. */ diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/monitor/ContainersMonitorImpl.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/monitor/ContainersMonitorImpl.java index 159e08ac2a859..0ab61f97d89a4 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/monitor/ContainersMonitorImpl.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/monitor/ContainersMonitorImpl.java @@ -597,7 +597,7 @@ private void initializeProcessTrees( getResourceCalculatorProcessTree(String pId) { ResourceCalculatorProcessTree pt = null; - // CGroups is best in perforance, so try to use it, if it is enabled + // CGroups is best in performance, so try to use it, if it is enabled if (processTreeClass == null && CGroupsResourceCalculator.isAvailable()) { try { diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/test/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/linux/resources/TestCGroupsResourceCalculator.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/test/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/linux/resources/TestCGroupsResourceCalculator.java index 5a10e1517b322..0d1092e3d6d32 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/test/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/linux/resources/TestCGroupsResourceCalculator.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/test/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/linux/resources/TestCGroupsResourceCalculator.java @@ -23,7 +23,9 @@ import org.apache.hadoop.yarn.util.ControlledClock; import org.apache.hadoop.yarn.util.ResourceCalculatorProcessTree; import org.junit.Assert; +import org.junit.Rule; import org.junit.Test; +import org.junit.rules.ExpectedException; import java.io.File; import java.io.FileNotFoundException; @@ -35,8 +37,11 @@ */ public class TestCGroupsResourceCalculator { + @Rule + public ExpectedException thrown= ExpectedException.none(); private ControlledClock clock = new ControlledClock(); private CGroupsHandler cGroupsHandler = mock(CGroupsHandler.class); + private String basePath = "/tmp/" + this.getClass().getName(); public TestCGroupsResourceCalculator() { when(cGroupsHandler.getRelativePathForCGroup("container_1")) @@ -45,20 +50,17 @@ public TestCGroupsResourceCalculator() { } @Test - public void testNoPid() throws Exception { - try { - CGroupsResourceCalculator calculator = - new CGroupsResourceCalculator( - "1234", ".", cGroupsHandler, clock); - } catch (YarnException e) { - Assert.assertTrue("Missing file should be caught", - e.getCause() instanceof FileNotFoundException); - } + public void testPidNotFound() throws Exception { + thrown.expect(YarnException.class); + CGroupsResourceCalculator calculator = + new CGroupsResourceCalculator( + "1234", ".", cGroupsHandler, clock); } @Test public void testNoMemoryCGgroupMount() throws Exception { - File procfs = new File("/tmp/" + this.getClass().getName() + "/1234"); + thrown.expect(YarnException.class); + File procfs = new File(basePath + "/1234"); Assert.assertTrue("Setup error", procfs.mkdirs()); try { FileUtils.writeStringToFile( @@ -66,23 +68,18 @@ public void testNoMemoryCGgroupMount() throws Exception { "7:devices:/yarn/container_1\n" + "6:cpuacct,cpu:/yarn/container_1\n" + "5:pids:/yarn/container_1\n"); - try { - CGroupsResourceCalculator calculator = - new CGroupsResourceCalculator( - "1234", "/tmp/" + this.getClass().getName(), - cGroupsHandler, clock); - } catch (YarnException e) { - Assert.assertTrue("Missing file should be caught", - e.getMessage().startsWith("memory CGroup")); - } + CGroupsResourceCalculator calculator = + new CGroupsResourceCalculator( + "1234", basePath, + cGroupsHandler, clock); } finally { FileUtils.deleteDirectory(new File("/tmp/" + this.getClass().getName())); } } @Test - public void testNoCGgroup() throws Exception { - File procfs = new File("/tmp/" + this.getClass().getName() + "/1234"); + public void testCGgroupNotFound() throws Exception { + File procfs = new File(basePath + "/1234"); Assert.assertTrue("Setup error", procfs.mkdirs()); try { FileUtils.writeStringToFile( @@ -94,7 +91,7 @@ public void testNoCGgroup() throws Exception { CGroupsResourceCalculator calculator = new CGroupsResourceCalculator( - "1234", "/tmp/" + this.getClass().getName(), + "1234", basePath, cGroupsHandler, clock); Assert.assertEquals("cgroups should be missing", (long)ResourceCalculatorProcessTree.UNAVAILABLE, @@ -107,10 +104,10 @@ public void testNoCGgroup() throws Exception { @Test public void testCPUParsing() throws Exception { File cgcpuacctDir = - new File("/tmp/" + this.getClass().getName() + "/cgcpuacct"); + new File(basePath + "/cgcpuacct"); File cgcpuacctContainerDir = new File(cgcpuacctDir, "/yarn/container_1"); - File procfs = new File("/tmp/" + this.getClass().getName() + "/1234"); + File procfs = new File(basePath + "/1234"); when(cGroupsHandler.getControllerPath( CGroupsHandler.CGroupController.CPUACCT)). thenReturn(cgcpuacctDir.getAbsolutePath()); @@ -130,7 +127,7 @@ public void testCPUParsing() throws Exception { "system 3632"); CGroupsResourceCalculator calculator = new CGroupsResourceCalculator( - "1234", "/tmp/" + this.getClass().getName(), + "1234", basePath, cGroupsHandler, clock); Assert.assertEquals("Incorrect CPU usage", 90470, @@ -143,14 +140,14 @@ public void testCPUParsing() throws Exception { @Test public void testMemoryParsing() throws Exception { File cgcpuacctDir = - new File("/tmp/" + this.getClass().getName() + "/cgcpuacct"); + new File(basePath + "/cgcpuacct"); File cgcpuacctContainerDir = new File(cgcpuacctDir, "/yarn/container_1"); File cgmemoryDir = - new File("/tmp/" + this.getClass().getName() + "/memory"); + new File(basePath + "/memory"); File cgMemoryContainerDir = new File(cgmemoryDir, "/yarn/container_1"); - File procfs = new File("/tmp/" + this.getClass().getName() + "/1234"); + File procfs = new File(basePath + "/1234"); when(cGroupsHandler.getControllerPath( CGroupsHandler.CGroupController.MEMORY)). thenReturn(cgmemoryDir.getAbsolutePath()); @@ -168,7 +165,7 @@ public void testMemoryParsing() throws Exception { CGroupsResourceCalculator calculator = new CGroupsResourceCalculator( - "1234", "/tmp/" + this.getClass().getName(), + "1234", basePath, cGroupsHandler, clock); // Test the case where memsw is not available (Ubuntu) @@ -194,7 +191,7 @@ public void testMemoryParsing() throws Exception { @Test public void testCPUParsingRoot() throws Exception { File cgcpuacctDir = - new File("/tmp/" + this.getClass().getName() + "/cgcpuacct"); + new File(basePath + "/cgcpuacct"); File cgcpuacctRootDir = new File(cgcpuacctDir, "/yarn"); when(cGroupsHandler.getControllerPath( @@ -208,7 +205,7 @@ public void testCPUParsingRoot() throws Exception { "system 3632"); CGroupsResourceCalculator calculator = new CGroupsResourceCalculator( - null, "/tmp/" + this.getClass().getName(), + null, basePath, cGroupsHandler, clock); Assert.assertEquals("Incorrect CPU usage", 90470, @@ -221,14 +218,14 @@ public void testCPUParsingRoot() throws Exception { @Test public void testMemoryParsingRoot() throws Exception { File cgcpuacctDir = - new File("/tmp/" + this.getClass().getName() + "/cgcpuacct"); + new File(basePath + "/cgcpuacct"); File cgcpuacctRootDir = new File(cgcpuacctDir, "/yarn"); File cgmemoryDir = - new File("/tmp/" + this.getClass().getName() + "/memory"); + new File(basePath + "/memory"); File cgMemoryRootDir = new File(cgmemoryDir, "/yarn"); - File procfs = new File("/tmp/" + this.getClass().getName() + "/1234"); + File procfs = new File(basePath + "/1234"); when(cGroupsHandler.getControllerPath( CGroupsHandler.CGroupController.MEMORY)). thenReturn(cgmemoryDir.getAbsolutePath()); @@ -242,7 +239,7 @@ public void testMemoryParsingRoot() throws Exception { CGroupsResourceCalculator calculator = new CGroupsResourceCalculator( - null, "/tmp/" + this.getClass().getName(), + null, basePath, cGroupsHandler, clock); // Test the case where memsw is not available (Ubuntu) From 16b2407910a52978cccc3f79639f54db02b98210 Mon Sep 17 00:00:00 2001 From: Miklos Szegedi Date: Mon, 7 Aug 2017 17:34:35 -0700 Subject: [PATCH 7/8] YARN-6668 --- .../resources/ResourceHandlerModule.java | 18 +++++---- .../monitor/ContainersMonitorImpl.java | 6 +-- .../TestCGroupsResourceCalculator.java | 22 +++++----- .../resources/TestResourceHandlerModule.java | 40 ++++++++++--------- 4 files changed, 46 insertions(+), 40 deletions(-) diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/linux/resources/ResourceHandlerModule.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/linux/resources/ResourceHandlerModule.java index 69f8324a61804..0115b7ed77721 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/linux/resources/ResourceHandlerModule.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/linux/resources/ResourceHandlerModule.java @@ -133,7 +133,7 @@ private static CGroupsCpuResourceHandlerImpl initCGroupsCpuResourceHandler( } private static TrafficControlBandwidthHandlerImpl - getTrafficControlBandwidthHandler(Configuration conf) + getTrafficControlBandwidthHandler(Configuration conf) throws ResourceHandlerException { if (conf.getBoolean(YarnConfiguration.NM_NETWORK_RESOURCE_ENABLED, YarnConfiguration.DEFAULT_NM_NETWORK_RESOURCE_ENABLED)) { @@ -157,7 +157,7 @@ private static CGroupsCpuResourceHandlerImpl initCGroupsCpuResourceHandler( } public static OutboundBandwidthResourceHandler - initOutboundBandwidthResourceHandler(Configuration conf) + initOutboundBandwidthResourceHandler(Configuration conf) throws ResourceHandlerException { return getTrafficControlBandwidthHandler(conf); } @@ -196,7 +196,7 @@ public static MemoryResourceHandler initMemoryResourceHandler( } private static CGroupsMemoryResourceHandlerImpl - getCgroupsMemoryResourceHandler( + getCgroupsMemoryResourceHandler( Configuration conf) throws ResourceHandlerException { if (cGroupsMemoryResourceHandler == null) { synchronized (MemoryResourceHandler.class) { @@ -221,10 +221,14 @@ private static void initializeConfiguredResourceHandlerChain( Configuration conf) throws ResourceHandlerException { ArrayList handlerList = new ArrayList<>(); - addHandlerIfNotNull(handlerList, initOutboundBandwidthResourceHandler(conf)); - addHandlerIfNotNull(handlerList, initDiskResourceHandler(conf)); - addHandlerIfNotNull(handlerList, initMemoryResourceHandler(conf)); - addHandlerIfNotNull(handlerList, initCGroupsCpuResourceHandler(conf)); + addHandlerIfNotNull(handlerList, + initOutboundBandwidthResourceHandler(conf)); + addHandlerIfNotNull(handlerList, + initDiskResourceHandler(conf)); + addHandlerIfNotNull(handlerList, + initMemoryResourceHandler(conf)); + addHandlerIfNotNull(handlerList, + initCGroupsCpuResourceHandler(conf)); resourceHandlerChain = new ResourceHandlerChain(handlerList); } diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/monitor/ContainersMonitorImpl.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/monitor/ContainersMonitorImpl.java index 0ab61f97d89a4..d651c848e65b4 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/monitor/ContainersMonitorImpl.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/monitor/ContainersMonitorImpl.java @@ -225,8 +225,8 @@ private boolean isContainerMonitorEnabled() { YarnConfiguration.DEFAULT_NM_CONTAINER_MONITOR_ENABLED); } - private void initializeOverAllocation(Configuration conf) { - float overAllocationTreshold = conf.getFloat( + private void initializeOverAllocation(Configuration confParam) { + float overAllocationTreshold = confParam.getFloat( YarnConfiguration.NM_OVERALLOCATION_ALLOCATION_THRESHOLD, YarnConfiguration.DEFAULT_NM_OVERALLOCATION_ALLOCATION_THRESHOLD); overAllocationTreshold = Math.min(overAllocationTreshold, @@ -238,7 +238,7 @@ private void initializeOverAllocation(Configuration conf) { OverAllocationInfo.newInstance( ResourceThresholds.newInstance(overAllocationTreshold))); - float preemptionThreshold = conf.getFloat( + float preemptionThreshold = confParam.getFloat( YarnConfiguration.NM_OVERALLOCATION_PREEMPTION_THRESHOLD, YarnConfiguration.DEFAULT_NM_OVERALLOCATION_PREEMPTION_THRESHOLD); diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/test/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/linux/resources/TestCGroupsResourceCalculator.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/test/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/linux/resources/TestCGroupsResourceCalculator.java index 0d1092e3d6d32..d966e3d14e217 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/test/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/linux/resources/TestCGroupsResourceCalculator.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/test/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/linux/resources/TestCGroupsResourceCalculator.java @@ -37,8 +37,6 @@ */ public class TestCGroupsResourceCalculator { - @Rule - public ExpectedException thrown= ExpectedException.none(); private ControlledClock clock = new ControlledClock(); private CGroupsHandler cGroupsHandler = mock(CGroupsHandler.class); private String basePath = "/tmp/" + this.getClass().getName(); @@ -49,17 +47,16 @@ public TestCGroupsResourceCalculator() { when(cGroupsHandler.getRelativePathForCGroup("")).thenReturn("/yarn/"); } - @Test + @Test(expected = YarnException.class) public void testPidNotFound() throws Exception { - thrown.expect(YarnException.class); CGroupsResourceCalculator calculator = new CGroupsResourceCalculator( "1234", ".", cGroupsHandler, clock); + Assert.assertEquals("Expected exception", null, calculator); } - @Test + @Test(expected = YarnException.class) public void testNoMemoryCGgroupMount() throws Exception { - thrown.expect(YarnException.class); File procfs = new File(basePath + "/1234"); Assert.assertTrue("Setup error", procfs.mkdirs()); try { @@ -72,8 +69,9 @@ public void testNoMemoryCGgroupMount() throws Exception { new CGroupsResourceCalculator( "1234", basePath, cGroupsHandler, clock); + Assert.assertEquals("Expected exception", null, calculator); } finally { - FileUtils.deleteDirectory(new File("/tmp/" + this.getClass().getName())); + FileUtils.deleteDirectory(new File(basePath)); } } @@ -97,7 +95,7 @@ public void testCGgroupNotFound() throws Exception { (long)ResourceCalculatorProcessTree.UNAVAILABLE, calculator.getRssMemorySize(0)); } finally { - FileUtils.deleteDirectory(new File("/tmp/" + this.getClass().getName())); + FileUtils.deleteDirectory(new File(basePath)); } } @@ -133,7 +131,7 @@ public void testCPUParsing() throws Exception { 90470, calculator.getCumulativeCpuTime()); } finally { - FileUtils.deleteDirectory(new File("/tmp/" + this.getClass().getName())); + FileUtils.deleteDirectory(new File(basePath)); } } @@ -184,7 +182,7 @@ public void testMemoryParsing() throws Exception { 418496513, calculator.getVirtualMemorySize()); } finally { - FileUtils.deleteDirectory(new File("/tmp/" + this.getClass().getName())); + FileUtils.deleteDirectory(new File(basePath)); } } @@ -211,7 +209,7 @@ public void testCPUParsingRoot() throws Exception { 90470, calculator.getCumulativeCpuTime()); } finally { - FileUtils.deleteDirectory(new File("/tmp/" + this.getClass().getName())); + FileUtils.deleteDirectory(new File(basePath)); } } @@ -258,7 +256,7 @@ public void testMemoryParsingRoot() throws Exception { 418496513, calculator.getVirtualMemorySize()); } finally { - FileUtils.deleteDirectory(new File("/tmp/" + this.getClass().getName())); + FileUtils.deleteDirectory(new File(basePath)); } } } diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/test/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/linux/resources/TestResourceHandlerModule.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/test/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/linux/resources/TestResourceHandlerModule.java index c9a7ad2ebf717..c670a1fac0569 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/test/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/linux/resources/TestResourceHandlerModule.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/test/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/linux/resources/TestResourceHandlerModule.java @@ -20,8 +20,6 @@ package org.apache.hadoop.yarn.server.nodemanager.containermanager.linux.resources; -import org.apache.commons.logging.Log; -import org.apache.commons.logging.LogFactory; import org.apache.hadoop.conf.Configuration; import org.apache.hadoop.yarn.conf.YarnConfiguration; import org.junit.Assert; @@ -31,10 +29,8 @@ import java.util.List; public class TestResourceHandlerModule { - private static final Log LOG = LogFactory. - getLog(TestResourceHandlerModule.class); - Configuration emptyConf; - Configuration networkEnabledConf; + private Configuration emptyConf; + private Configuration networkEnabledConf; @Before public void setup() throws Exception { @@ -63,12 +59,16 @@ public void testOutboundBandwidthHandler() { //Ensure that outbound bandwidth resource handler is present in the chain ResourceHandlerChain resourceHandlerChain = ResourceHandlerModule .getConfiguredResourceHandlerChain(networkEnabledConf); - List resourceHandlers = resourceHandlerChain - .getResourceHandlerList(); - //Exactly one resource handler in chain - Assert.assertEquals(resourceHandlers.size(), 1); - //Same instance is expected to be in the chain. - Assert.assertTrue(resourceHandlers.get(0) == resourceHandler); + if (resourceHandlerChain != null) { + List resourceHandlers = resourceHandlerChain + .getResourceHandlerList(); + //Exactly one resource handler in chain + Assert.assertEquals(resourceHandlers.size(), 1); + //Same instance is expected to be in the chain. + Assert.assertTrue(resourceHandlers.get(0) == resourceHandler); + } else { + Assert.fail("Null returned"); + } } catch (ResourceHandlerException e) { Assert.fail("Unexpected ResourceHandlerException: " + e); } @@ -89,11 +89,15 @@ public void testDiskResourceHandler() throws Exception { ResourceHandlerChain resourceHandlerChain = ResourceHandlerModule.getConfiguredResourceHandlerChain(diskConf); - List resourceHandlers = - resourceHandlerChain.getResourceHandlerList(); - // Exactly one resource handler in chain - Assert.assertEquals(resourceHandlers.size(), 1); - // Same instance is expected to be in the chain. - Assert.assertTrue(resourceHandlers.get(0) == handler); + if (resourceHandlerChain != null) { + List resourceHandlers = + resourceHandlerChain.getResourceHandlerList(); + // Exactly one resource handler in chain + Assert.assertEquals(resourceHandlers.size(), 1); + // Same instance is expected to be in the chain. + Assert.assertTrue(resourceHandlers.get(0) == handler); + } else { + Assert.fail("Null returned"); + } } } \ No newline at end of file From dbd07e20a4b5902593b6e407bf9f4fe78d94b68a Mon Sep 17 00:00:00 2001 From: Miklos Szegedi Date: Tue, 8 Aug 2017 11:47:24 -0700 Subject: [PATCH 8/8] YARN-6668 --- .../linux/resources/TestCGroupsResourceCalculator.java | 3 --- 1 file changed, 3 deletions(-) diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/test/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/linux/resources/TestCGroupsResourceCalculator.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/test/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/linux/resources/TestCGroupsResourceCalculator.java index d966e3d14e217..4d31ae8c25353 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/test/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/linux/resources/TestCGroupsResourceCalculator.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/test/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/linux/resources/TestCGroupsResourceCalculator.java @@ -23,12 +23,9 @@ import org.apache.hadoop.yarn.util.ControlledClock; import org.apache.hadoop.yarn.util.ResourceCalculatorProcessTree; import org.junit.Assert; -import org.junit.Rule; import org.junit.Test; -import org.junit.rules.ExpectedException; import java.io.File; -import java.io.FileNotFoundException; import static org.mockito.Mockito.*;