From b562f0e00016cd059d0e176128876e1338864fd8 Mon Sep 17 00:00:00 2001 From: George Tavares Date: Mon, 1 Jun 2020 13:18:17 -0300 Subject: [PATCH 01/11] Add cgroup support (#1197) --- .../agent/metrics/builtin/SystemMetrics.java | 134 +++++++++++++++++- .../metrics/builtin/SystemMetricsTest.java | 16 ++- .../test/resources/proc/memory.limit_in_bytes | 1 + .../proc/memory.limit_in_bytes-limited | 1 + .../src/test/resources/proc/memory.stat | 40 ++++++ .../test/resources/proc/memory.usage_in_bytes | 1 + 6 files changed, 186 insertions(+), 7 deletions(-) create mode 100644 apm-agent-core/src/test/resources/proc/memory.limit_in_bytes create mode 100644 apm-agent-core/src/test/resources/proc/memory.limit_in_bytes-limited create mode 100644 apm-agent-core/src/test/resources/proc/memory.stat create mode 100644 apm-agent-core/src/test/resources/proc/memory.usage_in_bytes diff --git a/apm-agent-core/src/main/java/co/elastic/apm/agent/metrics/builtin/SystemMetrics.java b/apm-agent-core/src/main/java/co/elastic/apm/agent/metrics/builtin/SystemMetrics.java index 75fed22d13..bbc58afa38 100644 --- a/apm-agent-core/src/main/java/co/elastic/apm/agent/metrics/builtin/SystemMetrics.java +++ b/apm-agent-core/src/main/java/co/elastic/apm/agent/metrics/builtin/SystemMetrics.java @@ -44,6 +44,7 @@ import java.util.HashMap; import java.util.List; import java.util.Map; +import java.util.concurrent.atomic.AtomicBoolean; import static co.elastic.apm.agent.matcher.WildcardMatcher.caseSensitiveMatcher; @@ -63,6 +64,14 @@ public class SystemMetrics extends AbstractLifecycleListener { private final OperatingSystemMXBean operatingSystemBean; + private static String CGROUP1_MAX_MEMORY = "/sys/fs/cgroup/memory/memory.limit_in_bytes"; + private static String CGROUP1_USED_MEMORY = "/sys/fs/cgroup/memory/memory.usage_in_bytes"; + private static String CGROUP1_STAT_MEMORY = "/sys/fs/cgroup/memory/memory.stat"; + private static String CGROUP2_MAX_MEMORY = "memory.max"; + private static String CGROUP2_USED_MEMORY = "memory.current"; + private static String CGROUP2_STAT_MEMORY = "memory.stat"; + private static long UNLIMITED = 0x7FFFFFFFFFFFF000L; + @Nullable private final Method systemCpuUsage; @@ -78,12 +87,16 @@ public class SystemMetrics extends AbstractLifecycleListener { @Nullable private final Method virtualProcessMemory; private final File memInfoFile; + private final CgroupFiles cgroup1Files; + private final CgroupFiles cgroup2Files; public SystemMetrics() { - this(new File("/proc/meminfo")); + this(new File("/proc/meminfo"), + new CgroupFiles(CGROUP1_MAX_MEMORY, CGROUP1_USED_MEMORY, CGROUP1_STAT_MEMORY), + new Cgroup2Files(CGROUP2_MAX_MEMORY, CGROUP2_USED_MEMORY, CGROUP2_STAT_MEMORY)); } - SystemMetrics(File memInfoFile) { + SystemMetrics(File memInfoFile, CgroupFiles cgroup1Files, CgroupFiles cgroup2Files) { this.operatingSystemBean = ManagementFactory.getOperatingSystemMXBean(); this.systemCpuUsage = JmxUtils.getOperatingSystemMBeanMethod(operatingSystemBean, "getSystemCpuLoad"); this.processCpuUsage = JmxUtils.getOperatingSystemMBeanMethod(operatingSystemBean, "getProcessCpuLoad"); @@ -91,6 +104,8 @@ public SystemMetrics() { this.totalMemory = JmxUtils.getOperatingSystemMBeanMethod(operatingSystemBean, "getTotalPhysicalMemorySize"); this.virtualProcessMemory = JmxUtils.getOperatingSystemMBeanMethod(operatingSystemBean, "getCommittedVirtualMemorySize"); this.memInfoFile = memInfoFile; + this.cgroup1Files = cgroup1Files; + this.cgroup2Files = cgroup2Files;; } @Override @@ -114,7 +129,67 @@ public double get() { } }); - if (memInfoFile.canRead()) { + AtomicBoolean usingCgroups = new AtomicBoolean(false); + for(CgroupFiles cgroupFiles_ : new CgroupFiles[]{cgroup1Files, cgroup2Files} ) { + final CgroupFiles cgroupFiles = cgroupFiles_; + if (!usingCgroups.get() && cgroupFiles.getMaxMemory().canRead() && cgroupFiles.getUsedMemory().canRead() && cgroupFiles.getStatMemory().canRead()) { + try(BufferedReader fileReader = new BufferedReader(new FileReader(cgroupFiles.getMaxMemory()))) { + String memMaxLine = fileReader.readLine(); + if ("max".equalsIgnoreCase(memMaxLine)) continue; // Cgroup2 use a string to disabled limits + long memMax = Long.parseLong(memMaxLine); + if (memMax < UNLIMITED) { // Cgroup1 use a contant to disabled limits + usingCgroups.set(true); + metricRegistry.addUnlessNan("system.memory.actual.free", Labels.EMPTY, new DoubleSupplier() { + @Override + public double get() { + try(BufferedReader fileReaderMemoryMax = new BufferedReader(new FileReader(cgroupFiles.getMaxMemory())); + BufferedReader fileReaderMemoryUsed = new BufferedReader(new FileReader(cgroupFiles.getUsedMemory())) + ) { + long memMax = Long.parseLong(fileReaderMemoryMax.readLine()); + long memUsed = Long.parseLong(fileReaderMemoryUsed.readLine()); + return memMax - memUsed; + } catch (Exception ignored) { + return Double.NaN; + } + } + }); + metricRegistry.addUnlessNan("system.memory.total", Labels.EMPTY, new DoubleSupplier() { + @Override + public double get() { + try(BufferedReader fileReaderMemoryMax = new BufferedReader(new FileReader(cgroupFiles.getMaxMemory()))) { + long memMax = Long.parseLong(fileReaderMemoryMax.readLine()); + return memMax; + } catch (Exception ignored) { + return Double.NaN; + } + } + }); + metricRegistry.addUnlessNan("system.process.memory.rss.bytes", Labels.EMPTY, new DoubleSupplier() { + final List relevantLines = Arrays.asList(caseSensitiveMatcher("total_rss *"), caseSensitiveMatcher("anon *")); + + @Override + public double get() { + try(BufferedReader fileReaderStatFile = new BufferedReader(new FileReader(cgroupFiles.getStatMemory()))) { + long sum = 0; + for (String statLine = fileReaderStatFile.readLine(); statLine != null && !statLine.isEmpty(); statLine = fileReaderStatFile.readLine()) { + if (WildcardMatcher.isAnyMatch(relevantLines, statLine)) { + final String[] statLineSplit = StringUtils.split(statLine, ' '); + sum += Long.parseLong(statLineSplit[1]); + } + } + return sum == 0 ? Double.NaN: sum; + } catch (Exception ignored) { + return Double.NaN; + } + } + }); + } + } catch (Exception ignored) { + } + } + }; + + if (!usingCgroups.get() && memInfoFile.canRead()) { metricRegistry.addUnlessNan("system.memory.actual.free", Labels.EMPTY, new DoubleSupplier() { final List relevantLines = Arrays.asList( caseSensitiveMatcher("MemAvailable:*kB"), @@ -191,4 +266,57 @@ private double invoke(@Nullable Method method) { return Double.NaN; } } + + public static class CgroupFiles { + protected File maxMemory; + protected File usedMemory; + protected File statMemory; + + protected CgroupFiles() { + + } + + public CgroupFiles(String maxMemory, String usedMemory, String statMemory) { + this.maxMemory = new File(maxMemory); + this.usedMemory = new File(usedMemory); + this.statMemory = new File(statMemory); + } + + public CgroupFiles(File maxMemory, File usedMemory, File statMemory) { + this.maxMemory = maxMemory; + this.usedMemory = usedMemory; + this.statMemory = statMemory; + } + + public File getMaxMemory() { + return maxMemory; + } + + public File getUsedMemory() { + return usedMemory; + } + + public File getStatMemory() { + return statMemory; + } + } + public static class Cgroup2Files extends CgroupFiles { + public Cgroup2Files(String maxMemory, String usedMemory, String statMemory) { + try(BufferedReader fileReader = new BufferedReader(new FileReader("/proc/self/cgroup"))) { + for (String cgroupLine = fileReader.readLine(); cgroupLine != null && !cgroupLine.isEmpty(); cgroupLine = fileReader.readLine()) { + if (cgroupLine.startsWith("0:")) { + final String[] cgroupSplit = StringUtils.split(cgroupLine, ':'); + this.maxMemory = new File("/sys/fs/cgroup" + cgroupSplit[cgroupSplit.length - 1] + "/" + maxMemory); + this.usedMemory = new File("/sys/fs/cgroup" + cgroupSplit[cgroupSplit.length - 1] + "/" + usedMemory); + this.statMemory = new File("/sys/fs/cgroup" + cgroupSplit[cgroupSplit.length - 1] + "/" + statMemory); + } + } + } + catch (Exception ignored) { + this.maxMemory = new File(maxMemory); + this.usedMemory = new File(usedMemory); + this.statMemory = new File(statMemory); + } + } + } } diff --git a/apm-agent-core/src/test/java/co/elastic/apm/agent/metrics/builtin/SystemMetricsTest.java b/apm-agent-core/src/test/java/co/elastic/apm/agent/metrics/builtin/SystemMetricsTest.java index 212f6491de..8cf467f7cc 100644 --- a/apm-agent-core/src/test/java/co/elastic/apm/agent/metrics/builtin/SystemMetricsTest.java +++ b/apm-agent-core/src/test/java/co/elastic/apm/agent/metrics/builtin/SystemMetricsTest.java @@ -59,14 +59,22 @@ void testSystemMetrics() throws InterruptedException { @ParameterizedTest @CsvSource({ - "/proc/meminfo, 6235127808", - "/proc/meminfo-3.14, 556630016" + "/proc/meminfo, 6235127808, /proc/memory.limit_in_bytes , /proc/memory.usage_in_bytes, /proc/memory.stat, 0", + "/proc/meminfo-3.14, 556630016, /proc/memory.limit_in_bytes , /proc/memory.usage_in_bytes, /proc/memory.stat, 0", + "/proc/meminfo, 7000000000, /proc/memory.limit_in_bytes-limited, /proc/memory.usage_in_bytes, /proc/memory.stat, 778842112" }) - void testFreeMemoryMeminfo(String file, long value) throws Exception { - SystemMetrics systemMetrics = new SystemMetrics(new File(getClass().getResource(file).toURI())); + void testFreeMemoryMeminfo(String file, long value, String cgroupLimit, String cgroupUsage, String cgroupStat, long rss) throws Exception { + SystemMetrics.CgroupFiles cgroupFiles = new SystemMetrics.CgroupFiles(new File(getClass().getResource(cgroupLimit).toURI()), + new File(getClass().getResource(cgroupUsage).toURI()), + new File(getClass().getResource(cgroupStat).toURI()) ); + SystemMetrics systemMetrics = new SystemMetrics(new File(getClass().getResource(file).toURI()), cgroupFiles, cgroupFiles); systemMetrics.bindTo(metricRegistry); + assertThat(metricRegistry.getGaugeValue("system.memory.actual.free", Labels.EMPTY)).isEqualTo(value); assertThat(metricRegistry.getGaugeValue("system.memory.total", Labels.EMPTY)).isEqualTo(7964778496L); + if (rss > 0) { + assertThat(metricRegistry.getGaugeValue("system.process.memory.rss.bytes", Labels.EMPTY)).isEqualTo(rss); + } } private void consumeCpu() { diff --git a/apm-agent-core/src/test/resources/proc/memory.limit_in_bytes b/apm-agent-core/src/test/resources/proc/memory.limit_in_bytes new file mode 100644 index 0000000000..564113cfaf --- /dev/null +++ b/apm-agent-core/src/test/resources/proc/memory.limit_in_bytes @@ -0,0 +1 @@ +9223372036854771712 diff --git a/apm-agent-core/src/test/resources/proc/memory.limit_in_bytes-limited b/apm-agent-core/src/test/resources/proc/memory.limit_in_bytes-limited new file mode 100644 index 0000000000..1df8dd444d --- /dev/null +++ b/apm-agent-core/src/test/resources/proc/memory.limit_in_bytes-limited @@ -0,0 +1 @@ +7964778496 diff --git a/apm-agent-core/src/test/resources/proc/memory.stat b/apm-agent-core/src/test/resources/proc/memory.stat new file mode 100644 index 0000000000..bf248be317 --- /dev/null +++ b/apm-agent-core/src/test/resources/proc/memory.stat @@ -0,0 +1,40 @@ +cache 10407936 +rss 778842112 +rss_huge 0 +shmem 0 +mapped_file 0 +dirty 0 +writeback 0 +swap 0 +pgpgin 234465 +pgpgout 41732 +pgfault 233838 +pgmajfault 0 +inactive_anon 0 +active_anon 778702848 +inactive_file 10407936 +active_file 0 +unevictable 0 +hierarchical_memory_limit 1073741824 +hierarchical_memsw_limit 2147483648 +total_cache 10407936 +total_rss 778842112 +total_rss_huge 0 +total_shmem 0 +total_mapped_file 0 +total_dirty 0 +total_writeback 0 +total_swap 0 +total_pgpgin 234465 +total_pgpgout 41732 +total_pgfault 233838 +total_pgmajfault 0 +total_inactive_anon 0 +total_active_anon 778702848 +total_inactive_file 10407936 +total_active_file 0 +total_unevictable 0 +recent_rotated_anon 231947 +recent_rotated_file 2 +recent_scanned_anon 231947 +recent_scanned_file 2622 diff --git a/apm-agent-core/src/test/resources/proc/memory.usage_in_bytes b/apm-agent-core/src/test/resources/proc/memory.usage_in_bytes new file mode 100644 index 0000000000..237642783d --- /dev/null +++ b/apm-agent-core/src/test/resources/proc/memory.usage_in_bytes @@ -0,0 +1 @@ +964778496 From 0e16ff0cf27fce8d1e4d5398705830e51aff0564 Mon Sep 17 00:00:00 2001 From: George Tavares Date: Tue, 2 Jun 2020 10:15:54 -0300 Subject: [PATCH 02/11] Fix Suggestion --- .../agent/metrics/builtin/SystemMetrics.java | 186 ++++++++---------- .../metrics/builtin/SystemMetricsTest.java | 20 +- apm-agent-core/src/test/resources/proc/cgroup | 2 + .../src/test/resources/proc/cgroup2 | 2 + .../memory/memory.limit_in_bytes} | 0 .../memory}/memory.usage_in_bytes | 0 .../src/test/resources/proc/memory.stat | 40 ---- .../proc/sys_cgroup2/slice/memory.current | 1 + .../proc/sys_cgroup2/slice/memory.max | 1 + .../memory}/memory.limit_in_bytes | 0 .../unlimited/memory/memory.usage_in_bytes | 1 + 11 files changed, 95 insertions(+), 158 deletions(-) create mode 100644 apm-agent-core/src/test/resources/proc/cgroup create mode 100644 apm-agent-core/src/test/resources/proc/cgroup2 rename apm-agent-core/src/test/resources/proc/{memory.limit_in_bytes-limited => limited/memory/memory.limit_in_bytes} (100%) rename apm-agent-core/src/test/resources/proc/{ => limited/memory}/memory.usage_in_bytes (100%) delete mode 100644 apm-agent-core/src/test/resources/proc/memory.stat create mode 100644 apm-agent-core/src/test/resources/proc/sys_cgroup2/slice/memory.current create mode 100644 apm-agent-core/src/test/resources/proc/sys_cgroup2/slice/memory.max rename apm-agent-core/src/test/resources/proc/{ => unlimited/memory}/memory.limit_in_bytes (100%) create mode 100644 apm-agent-core/src/test/resources/proc/unlimited/memory/memory.usage_in_bytes diff --git a/apm-agent-core/src/main/java/co/elastic/apm/agent/metrics/builtin/SystemMetrics.java b/apm-agent-core/src/main/java/co/elastic/apm/agent/metrics/builtin/SystemMetrics.java index bbc58afa38..da60f8e485 100644 --- a/apm-agent-core/src/main/java/co/elastic/apm/agent/metrics/builtin/SystemMetrics.java +++ b/apm-agent-core/src/main/java/co/elastic/apm/agent/metrics/builtin/SystemMetrics.java @@ -44,7 +44,6 @@ import java.util.HashMap; import java.util.List; import java.util.Map; -import java.util.concurrent.atomic.AtomicBoolean; import static co.elastic.apm.agent.matcher.WildcardMatcher.caseSensitiveMatcher; @@ -62,14 +61,14 @@ */ public class SystemMetrics extends AbstractLifecycleListener { + public static final String PROC_SELF_CGROUP = "/proc/self/cgroup"; + public static final String SYS_FS_CGROUP = "/sys/fs/cgroup"; private final OperatingSystemMXBean operatingSystemBean; - private static String CGROUP1_MAX_MEMORY = "/sys/fs/cgroup/memory/memory.limit_in_bytes"; - private static String CGROUP1_USED_MEMORY = "/sys/fs/cgroup/memory/memory.usage_in_bytes"; - private static String CGROUP1_STAT_MEMORY = "/sys/fs/cgroup/memory/memory.stat"; + private static String CGROUP1_MAX_MEMORY = "memory/memory.limit_in_bytes"; + private static String CGROUP1_USED_MEMORY = "memory/memory.usage_in_bytes"; private static String CGROUP2_MAX_MEMORY = "memory.max"; private static String CGROUP2_USED_MEMORY = "memory.current"; - private static String CGROUP2_STAT_MEMORY = "memory.stat"; private static long UNLIMITED = 0x7FFFFFFFFFFFF000L; @Nullable @@ -87,16 +86,13 @@ public class SystemMetrics extends AbstractLifecycleListener { @Nullable private final Method virtualProcessMemory; private final File memInfoFile; - private final CgroupFiles cgroup1Files; - private final CgroupFiles cgroup2Files; + private final CgroupFiles cgroupFiles; public SystemMetrics() { - this(new File("/proc/meminfo"), - new CgroupFiles(CGROUP1_MAX_MEMORY, CGROUP1_USED_MEMORY, CGROUP1_STAT_MEMORY), - new Cgroup2Files(CGROUP2_MAX_MEMORY, CGROUP2_USED_MEMORY, CGROUP2_STAT_MEMORY)); + this(new File("/proc/meminfo"), new File(PROC_SELF_CGROUP), new File(SYS_FS_CGROUP)); } - SystemMetrics(File memInfoFile, CgroupFiles cgroup1Files, CgroupFiles cgroup2Files) { + SystemMetrics(File memInfoFile, File procSelfCgroup, File sysFsCgroup) { this.operatingSystemBean = ManagementFactory.getOperatingSystemMXBean(); this.systemCpuUsage = JmxUtils.getOperatingSystemMBeanMethod(operatingSystemBean, "getSystemCpuLoad"); this.processCpuUsage = JmxUtils.getOperatingSystemMBeanMethod(operatingSystemBean, "getProcessCpuLoad"); @@ -104,8 +100,51 @@ public SystemMetrics() { this.totalMemory = JmxUtils.getOperatingSystemMBeanMethod(operatingSystemBean, "getTotalPhysicalMemorySize"); this.virtualProcessMemory = JmxUtils.getOperatingSystemMBeanMethod(operatingSystemBean, "getCommittedVirtualMemorySize"); this.memInfoFile = memInfoFile; - this.cgroup1Files = cgroup1Files; - this.cgroup2Files = cgroup2Files;; + + cgroupFiles = verifyCgroupEnabled(procSelfCgroup, sysFsCgroup); + } + + public CgroupFiles verifyCgroupEnabled(File procSelfCgroup, File sysFsCgroup) { + try(BufferedReader fileReader = new BufferedReader(new FileReader(procSelfCgroup))) { + String lineCgroup = null; + for (String cgroupLine = fileReader.readLine(); cgroupLine != null && !cgroupLine.isEmpty(); cgroupLine = fileReader.readLine()) { + if (lineCgroup==null && cgroupLine.startsWith("0:")) { + lineCgroup = cgroupLine; + } + if (cgroupLine.startsWith("9:memory")) { + lineCgroup = cgroupLine; + } + } + if ( lineCgroup!=null ) { + final String[] cgroupSplit = StringUtils.split(lineCgroup, ':'); + // Checking cgroup2 + File maxMemory = new File(sysFsCgroup, cgroupSplit[cgroupSplit.length - 1] + "/" + CGROUP2_MAX_MEMORY); + if (maxMemory.canRead()) { + try(BufferedReader fileReaderMem = new BufferedReader(new FileReader(maxMemory))) { + String memMaxLine = fileReaderMem.readLine(); + if (!"max".equalsIgnoreCase(memMaxLine)) { + return new CgroupFiles(maxMemory, + new File(sysFsCgroup, cgroupSplit[cgroupSplit.length - 1] + "/" + CGROUP2_USED_MEMORY)); + } + } + } + // Checking cgroup1 + maxMemory = new File(sysFsCgroup, CGROUP1_MAX_MEMORY); + if (maxMemory.canRead()) { + try(BufferedReader fileReaderMem = new BufferedReader(new FileReader(maxMemory))) { + String memMaxLine = fileReaderMem.readLine(); + long memMax = Long.parseLong(memMaxLine); + if (memMax < UNLIMITED) { // Cgroup1 use a contant to disabled limits + return new CgroupFiles(maxMemory, + new File(sysFsCgroup, CGROUP1_USED_MEMORY)); + } + } + } + } + } + catch (Exception e) { + } + return null; } @Override @@ -129,67 +168,35 @@ public double get() { } }); - AtomicBoolean usingCgroups = new AtomicBoolean(false); - for(CgroupFiles cgroupFiles_ : new CgroupFiles[]{cgroup1Files, cgroup2Files} ) { - final CgroupFiles cgroupFiles = cgroupFiles_; - if (!usingCgroups.get() && cgroupFiles.getMaxMemory().canRead() && cgroupFiles.getUsedMemory().canRead() && cgroupFiles.getStatMemory().canRead()) { - try(BufferedReader fileReader = new BufferedReader(new FileReader(cgroupFiles.getMaxMemory()))) { - String memMaxLine = fileReader.readLine(); - if ("max".equalsIgnoreCase(memMaxLine)) continue; // Cgroup2 use a string to disabled limits - long memMax = Long.parseLong(memMaxLine); - if (memMax < UNLIMITED) { // Cgroup1 use a contant to disabled limits - usingCgroups.set(true); - metricRegistry.addUnlessNan("system.memory.actual.free", Labels.EMPTY, new DoubleSupplier() { - @Override - public double get() { - try(BufferedReader fileReaderMemoryMax = new BufferedReader(new FileReader(cgroupFiles.getMaxMemory())); - BufferedReader fileReaderMemoryUsed = new BufferedReader(new FileReader(cgroupFiles.getUsedMemory())) - ) { - long memMax = Long.parseLong(fileReaderMemoryMax.readLine()); - long memUsed = Long.parseLong(fileReaderMemoryUsed.readLine()); - return memMax - memUsed; - } catch (Exception ignored) { - return Double.NaN; - } - } - }); - metricRegistry.addUnlessNan("system.memory.total", Labels.EMPTY, new DoubleSupplier() { - @Override - public double get() { - try(BufferedReader fileReaderMemoryMax = new BufferedReader(new FileReader(cgroupFiles.getMaxMemory()))) { - long memMax = Long.parseLong(fileReaderMemoryMax.readLine()); - return memMax; - } catch (Exception ignored) { - return Double.NaN; - } - } - }); - metricRegistry.addUnlessNan("system.process.memory.rss.bytes", Labels.EMPTY, new DoubleSupplier() { - final List relevantLines = Arrays.asList(caseSensitiveMatcher("total_rss *"), caseSensitiveMatcher("anon *")); - - @Override - public double get() { - try(BufferedReader fileReaderStatFile = new BufferedReader(new FileReader(cgroupFiles.getStatMemory()))) { - long sum = 0; - for (String statLine = fileReaderStatFile.readLine(); statLine != null && !statLine.isEmpty(); statLine = fileReaderStatFile.readLine()) { - if (WildcardMatcher.isAnyMatch(relevantLines, statLine)) { - final String[] statLineSplit = StringUtils.split(statLine, ' '); - sum += Long.parseLong(statLineSplit[1]); - } - } - return sum == 0 ? Double.NaN: sum; - } catch (Exception ignored) { - return Double.NaN; - } - } - }); + if (cgroupFiles != null) { + metricRegistry.addUnlessNan("system.memory.actual.free", Labels.EMPTY, new DoubleSupplier() { + @Override + public double get() { + try(BufferedReader fileReaderMemoryMax = new BufferedReader(new FileReader(cgroupFiles.getMaxMemory())); + BufferedReader fileReaderMemoryUsed = new BufferedReader(new FileReader(cgroupFiles.getUsedMemory())) + ) { + long memMax = Long.parseLong(fileReaderMemoryMax.readLine()); + long memUsed = Long.parseLong(fileReaderMemoryUsed.readLine()); + return memMax - memUsed; + } catch (Exception ignored) { + return Double.NaN; } - } catch (Exception ignored) { } - } - }; + }); + metricRegistry.addUnlessNan("system.memory.total", Labels.EMPTY, new DoubleSupplier() { + @Override + public double get() { + try(BufferedReader fileReaderMemoryMax = new BufferedReader(new FileReader(cgroupFiles.getMaxMemory()))) { + long memMax = Long.parseLong(fileReaderMemoryMax.readLine()); + return memMax; + } catch (Exception ignored) { + return Double.NaN; + } + } + }); + } - if (!usingCgroups.get() && memInfoFile.canRead()) { + if (cgroupFiles == null && memInfoFile.canRead()) { metricRegistry.addUnlessNan("system.memory.actual.free", Labels.EMPTY, new DoubleSupplier() { final List relevantLines = Arrays.asList( caseSensitiveMatcher("MemAvailable:*kB"), @@ -270,22 +277,10 @@ private double invoke(@Nullable Method method) { public static class CgroupFiles { protected File maxMemory; protected File usedMemory; - protected File statMemory; - - protected CgroupFiles() { - } - - public CgroupFiles(String maxMemory, String usedMemory, String statMemory) { - this.maxMemory = new File(maxMemory); - this.usedMemory = new File(usedMemory); - this.statMemory = new File(statMemory); - } - - public CgroupFiles(File maxMemory, File usedMemory, File statMemory) { + public CgroupFiles(File maxMemory, File usedMemory) { this.maxMemory = maxMemory; this.usedMemory = usedMemory; - this.statMemory = statMemory; } public File getMaxMemory() { @@ -295,28 +290,5 @@ public File getMaxMemory() { public File getUsedMemory() { return usedMemory; } - - public File getStatMemory() { - return statMemory; - } - } - public static class Cgroup2Files extends CgroupFiles { - public Cgroup2Files(String maxMemory, String usedMemory, String statMemory) { - try(BufferedReader fileReader = new BufferedReader(new FileReader("/proc/self/cgroup"))) { - for (String cgroupLine = fileReader.readLine(); cgroupLine != null && !cgroupLine.isEmpty(); cgroupLine = fileReader.readLine()) { - if (cgroupLine.startsWith("0:")) { - final String[] cgroupSplit = StringUtils.split(cgroupLine, ':'); - this.maxMemory = new File("/sys/fs/cgroup" + cgroupSplit[cgroupSplit.length - 1] + "/" + maxMemory); - this.usedMemory = new File("/sys/fs/cgroup" + cgroupSplit[cgroupSplit.length - 1] + "/" + usedMemory); - this.statMemory = new File("/sys/fs/cgroup" + cgroupSplit[cgroupSplit.length - 1] + "/" + statMemory); - } - } - } - catch (Exception ignored) { - this.maxMemory = new File(maxMemory); - this.usedMemory = new File(usedMemory); - this.statMemory = new File(statMemory); - } - } } } diff --git a/apm-agent-core/src/test/java/co/elastic/apm/agent/metrics/builtin/SystemMetricsTest.java b/apm-agent-core/src/test/java/co/elastic/apm/agent/metrics/builtin/SystemMetricsTest.java index 8cf467f7cc..6ed9dfcbe7 100644 --- a/apm-agent-core/src/test/java/co/elastic/apm/agent/metrics/builtin/SystemMetricsTest.java +++ b/apm-agent-core/src/test/java/co/elastic/apm/agent/metrics/builtin/SystemMetricsTest.java @@ -59,22 +59,20 @@ void testSystemMetrics() throws InterruptedException { @ParameterizedTest @CsvSource({ - "/proc/meminfo, 6235127808, /proc/memory.limit_in_bytes , /proc/memory.usage_in_bytes, /proc/memory.stat, 0", - "/proc/meminfo-3.14, 556630016, /proc/memory.limit_in_bytes , /proc/memory.usage_in_bytes, /proc/memory.stat, 0", - "/proc/meminfo, 7000000000, /proc/memory.limit_in_bytes-limited, /proc/memory.usage_in_bytes, /proc/memory.stat, 778842112" + "/proc/meminfo, 6235127808, /proc/cgroup, /proc/unlimited", + "/proc/meminfo-3.14, 556630016, /proc/cgroup, /proc/unlimited", + "/proc/meminfo, 7000000000, /proc/cgroup, /proc/limited", + "/proc/meminfo, 7000000000, /proc/cgroup2, /proc/sys_cgroup2" }) - void testFreeMemoryMeminfo(String file, long value, String cgroupLimit, String cgroupUsage, String cgroupStat, long rss) throws Exception { - SystemMetrics.CgroupFiles cgroupFiles = new SystemMetrics.CgroupFiles(new File(getClass().getResource(cgroupLimit).toURI()), - new File(getClass().getResource(cgroupUsage).toURI()), - new File(getClass().getResource(cgroupStat).toURI()) ); - SystemMetrics systemMetrics = new SystemMetrics(new File(getClass().getResource(file).toURI()), cgroupFiles, cgroupFiles); + void testFreeMemoryMeminfo(String file, long value, String selfCGroup, String sysFsGroup) throws Exception { + SystemMetrics systemMetrics = new SystemMetrics(new File(getClass().getResource(file).toURI()), + new File(getClass().getResource(selfCGroup).toURI()), + new File(getClass().getResource(sysFsGroup).toURI()) + ); systemMetrics.bindTo(metricRegistry); assertThat(metricRegistry.getGaugeValue("system.memory.actual.free", Labels.EMPTY)).isEqualTo(value); assertThat(metricRegistry.getGaugeValue("system.memory.total", Labels.EMPTY)).isEqualTo(7964778496L); - if (rss > 0) { - assertThat(metricRegistry.getGaugeValue("system.process.memory.rss.bytes", Labels.EMPTY)).isEqualTo(rss); - } } private void consumeCpu() { diff --git a/apm-agent-core/src/test/resources/proc/cgroup b/apm-agent-core/src/test/resources/proc/cgroup new file mode 100644 index 0000000000..eb1f1ab1b2 --- /dev/null +++ b/apm-agent-core/src/test/resources/proc/cgroup @@ -0,0 +1,2 @@ +9:memory:/ +0::/ diff --git a/apm-agent-core/src/test/resources/proc/cgroup2 b/apm-agent-core/src/test/resources/proc/cgroup2 new file mode 100644 index 0000000000..5ecc3b9ec9 --- /dev/null +++ b/apm-agent-core/src/test/resources/proc/cgroup2 @@ -0,0 +1,2 @@ +9:memory:/slice +0::/slice diff --git a/apm-agent-core/src/test/resources/proc/memory.limit_in_bytes-limited b/apm-agent-core/src/test/resources/proc/limited/memory/memory.limit_in_bytes similarity index 100% rename from apm-agent-core/src/test/resources/proc/memory.limit_in_bytes-limited rename to apm-agent-core/src/test/resources/proc/limited/memory/memory.limit_in_bytes diff --git a/apm-agent-core/src/test/resources/proc/memory.usage_in_bytes b/apm-agent-core/src/test/resources/proc/limited/memory/memory.usage_in_bytes similarity index 100% rename from apm-agent-core/src/test/resources/proc/memory.usage_in_bytes rename to apm-agent-core/src/test/resources/proc/limited/memory/memory.usage_in_bytes diff --git a/apm-agent-core/src/test/resources/proc/memory.stat b/apm-agent-core/src/test/resources/proc/memory.stat deleted file mode 100644 index bf248be317..0000000000 --- a/apm-agent-core/src/test/resources/proc/memory.stat +++ /dev/null @@ -1,40 +0,0 @@ -cache 10407936 -rss 778842112 -rss_huge 0 -shmem 0 -mapped_file 0 -dirty 0 -writeback 0 -swap 0 -pgpgin 234465 -pgpgout 41732 -pgfault 233838 -pgmajfault 0 -inactive_anon 0 -active_anon 778702848 -inactive_file 10407936 -active_file 0 -unevictable 0 -hierarchical_memory_limit 1073741824 -hierarchical_memsw_limit 2147483648 -total_cache 10407936 -total_rss 778842112 -total_rss_huge 0 -total_shmem 0 -total_mapped_file 0 -total_dirty 0 -total_writeback 0 -total_swap 0 -total_pgpgin 234465 -total_pgpgout 41732 -total_pgfault 233838 -total_pgmajfault 0 -total_inactive_anon 0 -total_active_anon 778702848 -total_inactive_file 10407936 -total_active_file 0 -total_unevictable 0 -recent_rotated_anon 231947 -recent_rotated_file 2 -recent_scanned_anon 231947 -recent_scanned_file 2622 diff --git a/apm-agent-core/src/test/resources/proc/sys_cgroup2/slice/memory.current b/apm-agent-core/src/test/resources/proc/sys_cgroup2/slice/memory.current new file mode 100644 index 0000000000..237642783d --- /dev/null +++ b/apm-agent-core/src/test/resources/proc/sys_cgroup2/slice/memory.current @@ -0,0 +1 @@ +964778496 diff --git a/apm-agent-core/src/test/resources/proc/sys_cgroup2/slice/memory.max b/apm-agent-core/src/test/resources/proc/sys_cgroup2/slice/memory.max new file mode 100644 index 0000000000..1df8dd444d --- /dev/null +++ b/apm-agent-core/src/test/resources/proc/sys_cgroup2/slice/memory.max @@ -0,0 +1 @@ +7964778496 diff --git a/apm-agent-core/src/test/resources/proc/memory.limit_in_bytes b/apm-agent-core/src/test/resources/proc/unlimited/memory/memory.limit_in_bytes similarity index 100% rename from apm-agent-core/src/test/resources/proc/memory.limit_in_bytes rename to apm-agent-core/src/test/resources/proc/unlimited/memory/memory.limit_in_bytes diff --git a/apm-agent-core/src/test/resources/proc/unlimited/memory/memory.usage_in_bytes b/apm-agent-core/src/test/resources/proc/unlimited/memory/memory.usage_in_bytes new file mode 100644 index 0000000000..237642783d --- /dev/null +++ b/apm-agent-core/src/test/resources/proc/unlimited/memory/memory.usage_in_bytes @@ -0,0 +1 @@ +964778496 From e0daa4776b68eb9bce402cdd072ca856254ad74c Mon Sep 17 00:00:00 2001 From: George Tavares Date: Wed, 3 Jun 2020 08:54:35 -0300 Subject: [PATCH 03/11] More fixes --- .../agent/metrics/builtin/SystemMetrics.java | 70 ++++++++++--------- 1 file changed, 37 insertions(+), 33 deletions(-) diff --git a/apm-agent-core/src/main/java/co/elastic/apm/agent/metrics/builtin/SystemMetrics.java b/apm-agent-core/src/main/java/co/elastic/apm/agent/metrics/builtin/SystemMetrics.java index da60f8e485..cae2a03b15 100644 --- a/apm-agent-core/src/main/java/co/elastic/apm/agent/metrics/builtin/SystemMetrics.java +++ b/apm-agent-core/src/main/java/co/elastic/apm/agent/metrics/builtin/SystemMetrics.java @@ -44,6 +44,8 @@ import java.util.HashMap; import java.util.List; import java.util.Map; +import java.util.regex.Matcher; +import java.util.regex.Pattern; import static co.elastic.apm.agent.matcher.WildcardMatcher.caseSensitiveMatcher; @@ -71,6 +73,8 @@ public class SystemMetrics extends AbstractLifecycleListener { private static String CGROUP2_USED_MEMORY = "memory.current"; private static long UNLIMITED = 0x7FFFFFFFFFFFF000L; + private Pattern MEMORY_CGROUP = Pattern.compile("^\\d+\\:memory\\:.*"); + @Nullable private final Method systemCpuUsage; @@ -105,45 +109,46 @@ public SystemMetrics() { } public CgroupFiles verifyCgroupEnabled(File procSelfCgroup, File sysFsCgroup) { - try(BufferedReader fileReader = new BufferedReader(new FileReader(procSelfCgroup))) { - String lineCgroup = null; - for (String cgroupLine = fileReader.readLine(); cgroupLine != null && !cgroupLine.isEmpty(); cgroupLine = fileReader.readLine()) { - if (lineCgroup==null && cgroupLine.startsWith("0:")) { - lineCgroup = cgroupLine; - } - if (cgroupLine.startsWith("9:memory")) { - lineCgroup = cgroupLine; + if (procSelfCgroup.canRead()) { + try(BufferedReader fileReader = new BufferedReader(new FileReader(procSelfCgroup))) { + String lineCgroup = null; + for (String cgroupLine = fileReader.readLine(); cgroupLine != null && !cgroupLine.isEmpty(); cgroupLine = fileReader.readLine()) { + if (lineCgroup == null && cgroupLine.startsWith("0:")) { + lineCgroup = cgroupLine; + } + if (MEMORY_CGROUP.matcher(cgroupLine).matches()) { + lineCgroup = cgroupLine; + } } - } - if ( lineCgroup!=null ) { - final String[] cgroupSplit = StringUtils.split(lineCgroup, ':'); - // Checking cgroup2 - File maxMemory = new File(sysFsCgroup, cgroupSplit[cgroupSplit.length - 1] + "/" + CGROUP2_MAX_MEMORY); - if (maxMemory.canRead()) { - try(BufferedReader fileReaderMem = new BufferedReader(new FileReader(maxMemory))) { - String memMaxLine = fileReaderMem.readLine(); - if (!"max".equalsIgnoreCase(memMaxLine)) { - return new CgroupFiles(maxMemory, - new File(sysFsCgroup, cgroupSplit[cgroupSplit.length - 1] + "/" + CGROUP2_USED_MEMORY)); + if (lineCgroup != null) { + final String[] cgroupSplit = StringUtils.split(lineCgroup, ':'); + // Checking cgroup2 + File maxMemory = new File(sysFsCgroup, cgroupSplit[cgroupSplit.length - 1] + "/" + CGROUP2_MAX_MEMORY); + if (maxMemory.canRead()) { + try(BufferedReader fileReaderMem = new BufferedReader(new FileReader(maxMemory))) { + String memMaxLine = fileReaderMem.readLine(); + if (!"max".equalsIgnoreCase(memMaxLine)) { + return new CgroupFiles(maxMemory, + new File(sysFsCgroup, cgroupSplit[cgroupSplit.length - 1] + "/" + CGROUP2_USED_MEMORY)); + } } } - } - // Checking cgroup1 - maxMemory = new File(sysFsCgroup, CGROUP1_MAX_MEMORY); - if (maxMemory.canRead()) { - try(BufferedReader fileReaderMem = new BufferedReader(new FileReader(maxMemory))) { - String memMaxLine = fileReaderMem.readLine(); - long memMax = Long.parseLong(memMaxLine); - if (memMax < UNLIMITED) { // Cgroup1 use a contant to disabled limits - return new CgroupFiles(maxMemory, - new File(sysFsCgroup, CGROUP1_USED_MEMORY)); + // Checking cgroup1 + maxMemory = new File(sysFsCgroup, CGROUP1_MAX_MEMORY); + if (maxMemory.canRead()) { + try(BufferedReader fileReaderMem = new BufferedReader(new FileReader(maxMemory))) { + String memMaxLine = fileReaderMem.readLine(); + long memMax = Long.parseLong(memMaxLine); + if (memMax < UNLIMITED) { // Cgroup1 use a contant to disabled limits + return new CgroupFiles(maxMemory, + new File(sysFsCgroup, CGROUP1_USED_MEMORY)); + } } } } + } catch (Exception e) { } } - catch (Exception e) { - } return null; } @@ -195,8 +200,7 @@ public double get() { } }); } - - if (cgroupFiles == null && memInfoFile.canRead()) { + else if (memInfoFile.canRead()) { metricRegistry.addUnlessNan("system.memory.actual.free", Labels.EMPTY, new DoubleSupplier() { final List relevantLines = Arrays.asList( caseSensitiveMatcher("MemAvailable:*kB"), From 1b58678825dbe1742fc48c3b2941852c356516e8 Mon Sep 17 00:00:00 2001 From: George Tavares Date: Thu, 4 Jun 2020 10:19:17 -0300 Subject: [PATCH 04/11] cgroup mountpoint discover --- .../agent/metrics/builtin/SystemMetrics.java | 89 +++++++++++++------ .../metrics/builtin/SystemMetricsTest.java | 21 ++++- 2 files changed, 78 insertions(+), 32 deletions(-) diff --git a/apm-agent-core/src/main/java/co/elastic/apm/agent/metrics/builtin/SystemMetrics.java b/apm-agent-core/src/main/java/co/elastic/apm/agent/metrics/builtin/SystemMetrics.java index cae2a03b15..999128f988 100644 --- a/apm-agent-core/src/main/java/co/elastic/apm/agent/metrics/builtin/SystemMetrics.java +++ b/apm-agent-core/src/main/java/co/elastic/apm/agent/metrics/builtin/SystemMetrics.java @@ -37,6 +37,7 @@ import java.io.BufferedReader; import java.io.File; import java.io.FileReader; +import java.io.IOException; import java.lang.management.ManagementFactory; import java.lang.management.OperatingSystemMXBean; import java.lang.reflect.Method; @@ -62,18 +63,20 @@ * under Apache License 2.0 */ public class SystemMetrics extends AbstractLifecycleListener { - public static final String PROC_SELF_CGROUP = "/proc/self/cgroup"; public static final String SYS_FS_CGROUP = "/sys/fs/cgroup"; + public static final String PROC_SELF_MOUNTINFO = "/proc/self/mountinfo"; private final OperatingSystemMXBean operatingSystemBean; - private static String CGROUP1_MAX_MEMORY = "memory/memory.limit_in_bytes"; - private static String CGROUP1_USED_MEMORY = "memory/memory.usage_in_bytes"; + private static String CGROUP1_MAX_MEMORY = "memory.limit_in_bytes"; + private static String CGROUP1_USED_MEMORY = "memory.usage_in_bytes"; private static String CGROUP2_MAX_MEMORY = "memory.max"; private static String CGROUP2_USED_MEMORY = "memory.current"; private static long UNLIMITED = 0x7FFFFFFFFFFFF000L; private Pattern MEMORY_CGROUP = Pattern.compile("^\\d+\\:memory\\:.*"); + private Pattern CGROUP1_MOUNT_POINT = Pattern.compile("^\\d+? \\d+? .+? .+? (.*?) .*cgroup.*cgroup.*memory.*"); + private Pattern CGROUP2_MOUNT_POINT = Pattern.compile("^\\d+? \\d+? .+? .+? (.*?) .*cgroup2.*cgroup.*"); @Nullable private final Method systemCpuUsage; @@ -90,13 +93,14 @@ public class SystemMetrics extends AbstractLifecycleListener { @Nullable private final Method virtualProcessMemory; private final File memInfoFile; + @Nullable private final CgroupFiles cgroupFiles; public SystemMetrics() { - this(new File("/proc/meminfo"), new File(PROC_SELF_CGROUP), new File(SYS_FS_CGROUP)); + this(new File("/proc/meminfo"), new File(PROC_SELF_CGROUP), new File(PROC_SELF_MOUNTINFO)); } - SystemMetrics(File memInfoFile, File procSelfCgroup, File sysFsCgroup) { + SystemMetrics(File memInfoFile, File procSelfCgroup, File mountInfo) { this.operatingSystemBean = ManagementFactory.getOperatingSystemMXBean(); this.systemCpuUsage = JmxUtils.getOperatingSystemMBeanMethod(operatingSystemBean, "getSystemCpuLoad"); this.processCpuUsage = JmxUtils.getOperatingSystemMBeanMethod(operatingSystemBean, "getProcessCpuLoad"); @@ -105,11 +109,11 @@ public SystemMetrics() { this.virtualProcessMemory = JmxUtils.getOperatingSystemMBeanMethod(operatingSystemBean, "getCommittedVirtualMemorySize"); this.memInfoFile = memInfoFile; - cgroupFiles = verifyCgroupEnabled(procSelfCgroup, sysFsCgroup); + cgroupFiles = verifyCgroupEnabled(procSelfCgroup, mountInfo); } - public CgroupFiles verifyCgroupEnabled(File procSelfCgroup, File sysFsCgroup) { - if (procSelfCgroup.canRead()) { + public CgroupFiles verifyCgroupEnabled(File procSelfCgroup, File mountInfo) { + if (procSelfCgroup.canRead() && mountInfo.canRead()) { try(BufferedReader fileReader = new BufferedReader(new FileReader(procSelfCgroup))) { String lineCgroup = null; for (String cgroupLine = fileReader.readLine(); cgroupLine != null && !cgroupLine.isEmpty(); cgroupLine = fileReader.readLine()) { @@ -121,30 +125,27 @@ public CgroupFiles verifyCgroupEnabled(File procSelfCgroup, File sysFsCgroup) { } } if (lineCgroup != null) { - final String[] cgroupSplit = StringUtils.split(lineCgroup, ':'); - // Checking cgroup2 - File maxMemory = new File(sysFsCgroup, cgroupSplit[cgroupSplit.length - 1] + "/" + CGROUP2_MAX_MEMORY); - if (maxMemory.canRead()) { - try(BufferedReader fileReaderMem = new BufferedReader(new FileReader(maxMemory))) { - String memMaxLine = fileReaderMem.readLine(); - if (!"max".equalsIgnoreCase(memMaxLine)) { - return new CgroupFiles(maxMemory, - new File(sysFsCgroup, cgroupSplit[cgroupSplit.length - 1] + "/" + CGROUP2_USED_MEMORY)); + CgroupFiles cgroupFilesTest = null; + try(BufferedReader fileMountInfoReader = new BufferedReader(new FileReader(mountInfo))) { + for (String mountLine = fileMountInfoReader.readLine(); mountLine != null && !mountLine.isEmpty(); mountLine = fileMountInfoReader.readLine()) { + Matcher matcher = CGROUP2_MOUNT_POINT.matcher(mountLine); + if (matcher.matches()) { + cgroupFilesTest = verifyCgroup2Available(lineCgroup, new File(matcher.group(1))); + if (cgroupFilesTest != null) return cgroupFilesTest; } - } - } - // Checking cgroup1 - maxMemory = new File(sysFsCgroup, CGROUP1_MAX_MEMORY); - if (maxMemory.canRead()) { - try(BufferedReader fileReaderMem = new BufferedReader(new FileReader(maxMemory))) { - String memMaxLine = fileReaderMem.readLine(); - long memMax = Long.parseLong(memMaxLine); - if (memMax < UNLIMITED) { // Cgroup1 use a contant to disabled limits - return new CgroupFiles(maxMemory, - new File(sysFsCgroup, CGROUP1_USED_MEMORY)); + matcher = CGROUP1_MOUNT_POINT.matcher(mountLine); + if (matcher.matches()) { + cgroupFilesTest = verifyCgroup1Available(new File(matcher.group(1))); + if (cgroupFilesTest != null) return cgroupFilesTest; } } } + // Fall back to /sys/fs/cgroup if not found on mountinfo + cgroupFilesTest = verifyCgroup2Available(lineCgroup, new File(SYS_FS_CGROUP)); + if (cgroupFilesTest != null) return cgroupFilesTest; + cgroupFilesTest = verifyCgroup1Available( new File(SYS_FS_CGROUP + File.pathSeparator + "memory")); + if (cgroupFilesTest != null) return cgroupFilesTest; + } } catch (Exception e) { } @@ -152,6 +153,38 @@ public CgroupFiles verifyCgroupEnabled(File procSelfCgroup, File sysFsCgroup) { return null; } + private CgroupFiles verifyCgroup2Available(String lineCgroup, File mountDiscovered) throws IOException { + final String[] cgroupSplit = StringUtils.split(lineCgroup, ':'); + // Checking cgroup2 + File maxMemory = new File(mountDiscovered, cgroupSplit[cgroupSplit.length - 1] + "/" + CGROUP2_MAX_MEMORY); + if (maxMemory.canRead()) { + try(BufferedReader fileReaderMem = new BufferedReader(new FileReader(maxMemory))) { + String memMaxLine = fileReaderMem.readLine(); + if (!"max".equalsIgnoreCase(memMaxLine)) { + return new CgroupFiles(maxMemory, + new File(mountDiscovered, cgroupSplit[cgroupSplit.length - 1] + "/" + CGROUP2_USED_MEMORY)); + } + } + } + return null; + } + + private CgroupFiles verifyCgroup1Available(File mountDiscovered) throws IOException { + // Checking cgroup1 + File maxMemory = new File(mountDiscovered, CGROUP1_MAX_MEMORY); + if (maxMemory.canRead()) { + try(BufferedReader fileReaderMem = new BufferedReader(new FileReader(maxMemory))) { + String memMaxLine = fileReaderMem.readLine(); + long memMax = Long.parseLong(memMaxLine); + if (memMax < UNLIMITED) { // Cgroup1 use a contant to disabled limits + return new CgroupFiles(maxMemory, + new File(mountDiscovered, CGROUP1_USED_MEMORY)); + } + } + } + return null; + } + @Override public void start(ElasticApmTracer tracer) { bindTo(tracer.getMetricRegistry()); diff --git a/apm-agent-core/src/test/java/co/elastic/apm/agent/metrics/builtin/SystemMetricsTest.java b/apm-agent-core/src/test/java/co/elastic/apm/agent/metrics/builtin/SystemMetricsTest.java index 6ed9dfcbe7..c72f34e6a3 100644 --- a/apm-agent-core/src/test/java/co/elastic/apm/agent/metrics/builtin/SystemMetricsTest.java +++ b/apm-agent-core/src/test/java/co/elastic/apm/agent/metrics/builtin/SystemMetricsTest.java @@ -34,6 +34,7 @@ import org.junit.jupiter.params.provider.CsvSource; import java.io.File; +import java.io.FileWriter; import static org.assertj.core.api.Assertions.assertThat; import static org.mockito.Mockito.mock; @@ -59,15 +60,27 @@ void testSystemMetrics() throws InterruptedException { @ParameterizedTest @CsvSource({ - "/proc/meminfo, 6235127808, /proc/cgroup, /proc/unlimited", - "/proc/meminfo-3.14, 556630016, /proc/cgroup, /proc/unlimited", - "/proc/meminfo, 7000000000, /proc/cgroup, /proc/limited", + "/proc/meminfo, 6235127808, /proc/cgroup, /proc/unlimited/memory", + "/proc/meminfo-3.14, 556630016, /proc/cgroup, /proc/unlimited/memory", + "/proc/meminfo, 7000000000, /proc/cgroup, /proc/limited/memory", "/proc/meminfo, 7000000000, /proc/cgroup2, /proc/sys_cgroup2" }) void testFreeMemoryMeminfo(String file, long value, String selfCGroup, String sysFsGroup) throws Exception { + File mountInfo = new File(getClass().getResource(sysFsGroup).toURI()); + File fileTmp = File.createTempFile("temp", null); + fileTmp.deleteOnExit(); + FileWriter fw = new FileWriter(fileTmp); + if ("/proc/sys_cgroup2".equalsIgnoreCase(sysFsGroup)) { + fw.write("30 23 0:26 / " + mountInfo.getAbsolutePath() + " rw,nosuid,nodev,noexec,relatime shared:4 - cgroup2 cgroup rw,seclabel\n"); + } + else { + fw.write("39 30 0:35 / " + mountInfo.getAbsolutePath() + " rw,nosuid,nodev,noexec,relatime shared:10 - cgroup cgroup rw,seclabel,memory\n"); + } + fw.close(); + SystemMetrics systemMetrics = new SystemMetrics(new File(getClass().getResource(file).toURI()), new File(getClass().getResource(selfCGroup).toURI()), - new File(getClass().getResource(sysFsGroup).toURI()) + fileTmp ); systemMetrics.bindTo(metricRegistry); From cbbcb7c3aae0ddd5182020c5e1ad77b666512275 Mon Sep 17 00:00:00 2001 From: George Tavares Date: Sat, 11 Jul 2020 10:24:09 -0300 Subject: [PATCH 05/11] Fix variable names --- .../agent/metrics/builtin/SystemMetrics.java | 86 +++++++++++++++---- .../metrics/builtin/SystemMetricsTest.java | 62 +++++++++++-- .../resources/proc/limited/memory/memory.stat | 40 +++++++++ .../proc/sys_cgroup2/slice/memory.stat | 40 +++++++++ .../proc/unlimited/memory/memory.stat | 40 +++++++++ 5 files changed, 243 insertions(+), 25 deletions(-) create mode 100644 apm-agent-core/src/test/resources/proc/limited/memory/memory.stat create mode 100644 apm-agent-core/src/test/resources/proc/sys_cgroup2/slice/memory.stat create mode 100644 apm-agent-core/src/test/resources/proc/unlimited/memory/memory.stat diff --git a/apm-agent-core/src/main/java/co/elastic/apm/agent/metrics/builtin/SystemMetrics.java b/apm-agent-core/src/main/java/co/elastic/apm/agent/metrics/builtin/SystemMetrics.java index 999128f988..86236d8269 100644 --- a/apm-agent-core/src/main/java/co/elastic/apm/agent/metrics/builtin/SystemMetrics.java +++ b/apm-agent-core/src/main/java/co/elastic/apm/agent/metrics/builtin/SystemMetrics.java @@ -68,14 +68,18 @@ public class SystemMetrics extends AbstractLifecycleListener { public static final String PROC_SELF_MOUNTINFO = "/proc/self/mountinfo"; private final OperatingSystemMXBean operatingSystemBean; + final private List inactiveMemoryRelevantLines = Arrays.asList(caseSensitiveMatcher("inactive_file *")); + private static String CGROUP1_MAX_MEMORY = "memory.limit_in_bytes"; private static String CGROUP1_USED_MEMORY = "memory.usage_in_bytes"; + private static String CGROUP1_STAT_MEMORY = "memory.stat"; private static String CGROUP2_MAX_MEMORY = "memory.max"; private static String CGROUP2_USED_MEMORY = "memory.current"; + private static String CGROUP2_STAT_MEMORY = "memory.stat"; private static long UNLIMITED = 0x7FFFFFFFFFFFF000L; private Pattern MEMORY_CGROUP = Pattern.compile("^\\d+\\:memory\\:.*"); - private Pattern CGROUP1_MOUNT_POINT = Pattern.compile("^\\d+? \\d+? .+? .+? (.*?) .*cgroup.*cgroup.*memory.*"); + private Pattern CGROUP1_MOUNT_POINT = Pattern.compile("^\\d+? \\d+? .+? .+? (.*?) .*cgroup.*memory.*"); private Pattern CGROUP2_MOUNT_POINT = Pattern.compile("^\\d+? \\d+? .+? .+? (.*?) .*cgroup2.*cgroup.*"); @Nullable @@ -128,14 +132,14 @@ public CgroupFiles verifyCgroupEnabled(File procSelfCgroup, File mountInfo) { CgroupFiles cgroupFilesTest = null; try(BufferedReader fileMountInfoReader = new BufferedReader(new FileReader(mountInfo))) { for (String mountLine = fileMountInfoReader.readLine(); mountLine != null && !mountLine.isEmpty(); mountLine = fileMountInfoReader.readLine()) { - Matcher matcher = CGROUP2_MOUNT_POINT.matcher(mountLine); - if (matcher.matches()) { - cgroupFilesTest = verifyCgroup2Available(lineCgroup, new File(matcher.group(1))); + String foundRegex = applyCgroup2Regex(mountLine); + if (foundRegex != null) { + cgroupFilesTest = verifyCgroup2Available(lineCgroup, new File(foundRegex)); if (cgroupFilesTest != null) return cgroupFilesTest; } - matcher = CGROUP1_MOUNT_POINT.matcher(mountLine); - if (matcher.matches()) { - cgroupFilesTest = verifyCgroup1Available(new File(matcher.group(1))); + foundRegex = applyCgroup1Regex(mountLine); + if (foundRegex != null) { + cgroupFilesTest = verifyCgroup1Available(new File(foundRegex)); if (cgroupFilesTest != null) return cgroupFilesTest; } } @@ -152,7 +156,20 @@ public CgroupFiles verifyCgroupEnabled(File procSelfCgroup, File mountInfo) { } return null; } - + String applyCgroup1Regex(String mountLine) { + Matcher matcher = CGROUP1_MOUNT_POINT.matcher(mountLine); + if (matcher.matches()) { + return matcher.group(1); + } + return null; + } + String applyCgroup2Regex(String mountLine) { + Matcher matcher = CGROUP2_MOUNT_POINT.matcher(mountLine); + if (matcher.matches()) { + return matcher.group(1); + } + return null; + } private CgroupFiles verifyCgroup2Available(String lineCgroup, File mountDiscovered) throws IOException { final String[] cgroupSplit = StringUtils.split(lineCgroup, ':'); // Checking cgroup2 @@ -162,7 +179,8 @@ private CgroupFiles verifyCgroup2Available(String lineCgroup, File mountDiscover String memMaxLine = fileReaderMem.readLine(); if (!"max".equalsIgnoreCase(memMaxLine)) { return new CgroupFiles(maxMemory, - new File(mountDiscovered, cgroupSplit[cgroupSplit.length - 1] + "/" + CGROUP2_USED_MEMORY)); + new File(mountDiscovered, cgroupSplit[cgroupSplit.length - 1] + "/" + CGROUP2_USED_MEMORY), + new File(mountDiscovered, cgroupSplit[cgroupSplit.length - 1] + "/" + CGROUP2_STAT_MEMORY)); } } } @@ -178,7 +196,8 @@ private CgroupFiles verifyCgroup1Available(File mountDiscovered) throws IOExcept long memMax = Long.parseLong(memMaxLine); if (memMax < UNLIMITED) { // Cgroup1 use a contant to disabled limits return new CgroupFiles(maxMemory, - new File(mountDiscovered, CGROUP1_USED_MEMORY)); + new File(mountDiscovered, CGROUP1_USED_MEMORY), + new File(mountDiscovered, CGROUP1_STAT_MEMORY)); } } } @@ -190,6 +209,21 @@ public void start(ElasticApmTracer tracer) { bindTo(tracer.getMetricRegistry()); } + private double getInactiveMemory() { + try(BufferedReader fileReaderStatFile = new BufferedReader(new FileReader(cgroupFiles.getStatMemory()))) { + long sum = 0; + for (String statLine = fileReaderStatFile.readLine(); statLine != null && !statLine.isEmpty(); statLine = fileReaderStatFile.readLine()) { + if (WildcardMatcher.isAnyMatch(inactiveMemoryRelevantLines, statLine)) { + final String[] statLineSplit = StringUtils.split(statLine, ' '); + sum += Long.parseLong(statLineSplit[1]); + } + } + return sum; + } catch (Exception ignored) { + return Double.NaN; + } + } + void bindTo(MetricRegistry metricRegistry) { // J9 always returns -1 on the first call metricRegistry.addUnlessNan("system.cpu.total.norm.pct", Labels.EMPTY, new DoubleSupplier() { @@ -207,21 +241,31 @@ public double get() { }); if (cgroupFiles != null) { - metricRegistry.addUnlessNan("system.memory.actual.free", Labels.EMPTY, new DoubleSupplier() { + metricRegistry.addUnlessNan("system.process.cgroup.memory.stats.inactive_file.bytes", Labels.EMPTY, new DoubleSupplier() { + @Override public double get() { - try(BufferedReader fileReaderMemoryMax = new BufferedReader(new FileReader(cgroupFiles.getMaxMemory())); - BufferedReader fileReaderMemoryUsed = new BufferedReader(new FileReader(cgroupFiles.getUsedMemory())) + return getInactiveMemory(); + } + + }); + metricRegistry.addUnlessNan("system.process.cgroup.memory.mem.usage.bytes", Labels.EMPTY, new DoubleSupplier() { + @Override + public double get() { + try(BufferedReader fileReaderMemoryUsed = new BufferedReader(new FileReader(cgroupFiles.getUsedMemory())) ) { - long memMax = Long.parseLong(fileReaderMemoryMax.readLine()); long memUsed = Long.parseLong(fileReaderMemoryUsed.readLine()); - return memMax - memUsed; + Double inactiveMemory = getInactiveMemory(); + if (!inactiveMemory.equals(Double.NaN)) { + memUsed -= inactiveMemory; + } + return memUsed; } catch (Exception ignored) { return Double.NaN; } } }); - metricRegistry.addUnlessNan("system.memory.total", Labels.EMPTY, new DoubleSupplier() { + metricRegistry.addUnlessNan("system.process.cgroup.memory.mem.limit.bytes", Labels.EMPTY, new DoubleSupplier() { @Override public double get() { try(BufferedReader fileReaderMemoryMax = new BufferedReader(new FileReader(cgroupFiles.getMaxMemory()))) { @@ -233,7 +277,7 @@ public double get() { } }); } - else if (memInfoFile.canRead()) { + if (memInfoFile.canRead()) { metricRegistry.addUnlessNan("system.memory.actual.free", Labels.EMPTY, new DoubleSupplier() { final List relevantLines = Arrays.asList( caseSensitiveMatcher("MemAvailable:*kB"), @@ -314,10 +358,12 @@ private double invoke(@Nullable Method method) { public static class CgroupFiles { protected File maxMemory; protected File usedMemory; + protected File statMemory; - public CgroupFiles(File maxMemory, File usedMemory) { + public CgroupFiles(File maxMemory, File usedMemory, File statMemory) { this.maxMemory = maxMemory; this.usedMemory = usedMemory; + this.statMemory = statMemory; } public File getMaxMemory() { @@ -327,5 +373,9 @@ public File getMaxMemory() { public File getUsedMemory() { return usedMemory; } + + public File getStatMemory() { + return statMemory; + } } } diff --git a/apm-agent-core/src/test/java/co/elastic/apm/agent/metrics/builtin/SystemMetricsTest.java b/apm-agent-core/src/test/java/co/elastic/apm/agent/metrics/builtin/SystemMetricsTest.java index c72f34e6a3..6153fb047b 100644 --- a/apm-agent-core/src/test/java/co/elastic/apm/agent/metrics/builtin/SystemMetricsTest.java +++ b/apm-agent-core/src/test/java/co/elastic/apm/agent/metrics/builtin/SystemMetricsTest.java @@ -32,9 +32,12 @@ import org.junit.jupiter.api.condition.OS; import org.junit.jupiter.params.ParameterizedTest; import org.junit.jupiter.params.provider.CsvSource; +import org.junit.jupiter.params.provider.ValueSource; import java.io.File; import java.io.FileWriter; +import java.io.IOException; +import java.net.URISyntaxException; import static org.assertj.core.api.Assertions.assertThat; import static org.mockito.Mockito.mock; @@ -60,12 +63,37 @@ void testSystemMetrics() throws InterruptedException { @ParameterizedTest @CsvSource({ - "/proc/meminfo, 6235127808, /proc/cgroup, /proc/unlimited/memory", - "/proc/meminfo-3.14, 556630016, /proc/cgroup, /proc/unlimited/memory", - "/proc/meminfo, 7000000000, /proc/cgroup, /proc/limited/memory", - "/proc/meminfo, 7000000000, /proc/cgroup2, /proc/sys_cgroup2" + "/proc/meminfo, 6235127808", + "/proc/meminfo-3.14, 556630016", }) - void testFreeMemoryMeminfo(String file, long value, String selfCGroup, String sysFsGroup) throws Exception { + void testFreeMemoryMeminfo(String file, long value) throws Exception { + SystemMetrics systemMetrics = createUnlimitedSystemMetrics(file); + systemMetrics.bindTo(metricRegistry); + + assertThat(metricRegistry.getGaugeValue("system.memory.actual.free", Labels.EMPTY)).isEqualTo(value); + assertThat(metricRegistry.getGaugeValue("system.memory.total", Labels.EMPTY)).isEqualTo(7964778496L); + } + + private SystemMetrics createUnlimitedSystemMetrics(String file) throws URISyntaxException, IOException { + File mountInfo = new File(getClass().getResource("/proc/cgroup").toURI()); + File fileTmp = File.createTempFile("temp", null); + fileTmp.deleteOnExit(); + FileWriter fw = new FileWriter(fileTmp); + fw.write("39 30 0:35 / " + mountInfo.getAbsolutePath() + " rw,nosuid,nodev,noexec,relatime shared:10 - cgroup cgroup rw,seclabel,memory\n"); + fw.close(); + + return new SystemMetrics(new File(getClass().getResource(file).toURI()), + new File(getClass().getResource("/proc/unlimited/memory").toURI()), + fileTmp + ); + } + + @ParameterizedTest + @CsvSource({ + "/proc/meminfo, 954370560, /proc/cgroup, /proc/limited/memory", + "/proc/meminfo, 954370560, /proc/cgroup2, /proc/sys_cgroup2" + }) + void testFreeCgroupMemoryMeminfo(String file, long value, String selfCGroup, String sysFsGroup) throws Exception { File mountInfo = new File(getClass().getResource(sysFsGroup).toURI()); File fileTmp = File.createTempFile("temp", null); fileTmp.deleteOnExit(); @@ -84,8 +112,28 @@ void testFreeMemoryMeminfo(String file, long value, String selfCGroup, String sy ); systemMetrics.bindTo(metricRegistry); - assertThat(metricRegistry.getGaugeValue("system.memory.actual.free", Labels.EMPTY)).isEqualTo(value); - assertThat(metricRegistry.getGaugeValue("system.memory.total", Labels.EMPTY)).isEqualTo(7964778496L); + assertThat(metricRegistry.getGaugeValue("system.process.cgroup.memory.mem.usage.bytes", Labels.EMPTY)).isEqualTo(value); + assertThat(metricRegistry.getGaugeValue("system.process.cgroup.memory.mem.limit.bytes", Labels.EMPTY)).isEqualTo(7964778496L); + assertThat(metricRegistry.getGaugeValue("system.process.cgroup.memory.stats.inactive_file.bytes", Labels.EMPTY)).isEqualTo(10407936L); + } + @ParameterizedTest + @ValueSource(strings ={ + "39 30 0:36 / /sys/fs/cgroup/memory rw,nosuid,nodev,noexec,relatime shared:10 - cgroup cgroup rw,seclabel,memory|/sys/fs/cgroup/memory", + }) + void testCgroup1Regex(String testString) throws Exception { + String [] split = testString.split("\\|"); + SystemMetrics systemMetrics = createUnlimitedSystemMetrics("/proc/meminfo"); + assertThat(systemMetrics.applyCgroup1Regex(split[0])).isEqualTo(split[1]); + } + + @ParameterizedTest + @ValueSource(strings ={ + "39 30 0:36 / /sys/fs/cgroup/memory rw,nosuid,nodev,noexec,relatime shared:4 - cgroup2 cgroup rw,seclabel|/sys/fs/cgroup/memory", + }) + void testCgroup2Regex(String testString) throws Exception { + String [] split = testString.split("\\|"); + SystemMetrics systemMetrics = createUnlimitedSystemMetrics("/proc/meminfo"); + assertThat(systemMetrics.applyCgroup2Regex(split[0])).isEqualTo(split[1]); } private void consumeCpu() { diff --git a/apm-agent-core/src/test/resources/proc/limited/memory/memory.stat b/apm-agent-core/src/test/resources/proc/limited/memory/memory.stat new file mode 100644 index 0000000000..bf248be317 --- /dev/null +++ b/apm-agent-core/src/test/resources/proc/limited/memory/memory.stat @@ -0,0 +1,40 @@ +cache 10407936 +rss 778842112 +rss_huge 0 +shmem 0 +mapped_file 0 +dirty 0 +writeback 0 +swap 0 +pgpgin 234465 +pgpgout 41732 +pgfault 233838 +pgmajfault 0 +inactive_anon 0 +active_anon 778702848 +inactive_file 10407936 +active_file 0 +unevictable 0 +hierarchical_memory_limit 1073741824 +hierarchical_memsw_limit 2147483648 +total_cache 10407936 +total_rss 778842112 +total_rss_huge 0 +total_shmem 0 +total_mapped_file 0 +total_dirty 0 +total_writeback 0 +total_swap 0 +total_pgpgin 234465 +total_pgpgout 41732 +total_pgfault 233838 +total_pgmajfault 0 +total_inactive_anon 0 +total_active_anon 778702848 +total_inactive_file 10407936 +total_active_file 0 +total_unevictable 0 +recent_rotated_anon 231947 +recent_rotated_file 2 +recent_scanned_anon 231947 +recent_scanned_file 2622 diff --git a/apm-agent-core/src/test/resources/proc/sys_cgroup2/slice/memory.stat b/apm-agent-core/src/test/resources/proc/sys_cgroup2/slice/memory.stat new file mode 100644 index 0000000000..bf248be317 --- /dev/null +++ b/apm-agent-core/src/test/resources/proc/sys_cgroup2/slice/memory.stat @@ -0,0 +1,40 @@ +cache 10407936 +rss 778842112 +rss_huge 0 +shmem 0 +mapped_file 0 +dirty 0 +writeback 0 +swap 0 +pgpgin 234465 +pgpgout 41732 +pgfault 233838 +pgmajfault 0 +inactive_anon 0 +active_anon 778702848 +inactive_file 10407936 +active_file 0 +unevictable 0 +hierarchical_memory_limit 1073741824 +hierarchical_memsw_limit 2147483648 +total_cache 10407936 +total_rss 778842112 +total_rss_huge 0 +total_shmem 0 +total_mapped_file 0 +total_dirty 0 +total_writeback 0 +total_swap 0 +total_pgpgin 234465 +total_pgpgout 41732 +total_pgfault 233838 +total_pgmajfault 0 +total_inactive_anon 0 +total_active_anon 778702848 +total_inactive_file 10407936 +total_active_file 0 +total_unevictable 0 +recent_rotated_anon 231947 +recent_rotated_file 2 +recent_scanned_anon 231947 +recent_scanned_file 2622 diff --git a/apm-agent-core/src/test/resources/proc/unlimited/memory/memory.stat b/apm-agent-core/src/test/resources/proc/unlimited/memory/memory.stat new file mode 100644 index 0000000000..bf248be317 --- /dev/null +++ b/apm-agent-core/src/test/resources/proc/unlimited/memory/memory.stat @@ -0,0 +1,40 @@ +cache 10407936 +rss 778842112 +rss_huge 0 +shmem 0 +mapped_file 0 +dirty 0 +writeback 0 +swap 0 +pgpgin 234465 +pgpgout 41732 +pgfault 233838 +pgmajfault 0 +inactive_anon 0 +active_anon 778702848 +inactive_file 10407936 +active_file 0 +unevictable 0 +hierarchical_memory_limit 1073741824 +hierarchical_memsw_limit 2147483648 +total_cache 10407936 +total_rss 778842112 +total_rss_huge 0 +total_shmem 0 +total_mapped_file 0 +total_dirty 0 +total_writeback 0 +total_swap 0 +total_pgpgin 234465 +total_pgpgout 41732 +total_pgfault 233838 +total_pgmajfault 0 +total_inactive_anon 0 +total_active_anon 778702848 +total_inactive_file 10407936 +total_active_file 0 +total_unevictable 0 +recent_rotated_anon 231947 +recent_rotated_file 2 +recent_scanned_anon 231947 +recent_scanned_file 2622 From 44d63f7849cd234d63fd8f15f29a58d05c242c18 Mon Sep 17 00:00:00 2001 From: eyalkoren <41850454+eyalkoren@users.noreply.github.com> Date: Tue, 14 Jul 2020 12:01:09 +0300 Subject: [PATCH 06/11] Suggested changes for cgroup metrics --- .../agent/metrics/builtin/SystemMetrics.java | 299 ++++++++++-------- .../metrics/builtin/SystemMetricsTest.java | 25 +- 2 files changed, 190 insertions(+), 134 deletions(-) diff --git a/apm-agent-core/src/main/java/co/elastic/apm/agent/metrics/builtin/SystemMetrics.java b/apm-agent-core/src/main/java/co/elastic/apm/agent/metrics/builtin/SystemMetrics.java index 86236d8269..161deaba93 100644 --- a/apm-agent-core/src/main/java/co/elastic/apm/agent/metrics/builtin/SystemMetrics.java +++ b/apm-agent-core/src/main/java/co/elastic/apm/agent/metrics/builtin/SystemMetrics.java @@ -31,6 +31,8 @@ import co.elastic.apm.agent.metrics.Labels; import co.elastic.apm.agent.metrics.MetricRegistry; import co.elastic.apm.agent.util.JmxUtils; +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; import org.stagemonitor.util.StringUtils; import javax.annotation.Nullable; @@ -63,24 +65,24 @@ * under Apache License 2.0 */ public class SystemMetrics extends AbstractLifecycleListener { + public static final String PROC_SELF_CGROUP = "/proc/self/cgroup"; - public static final String SYS_FS_CGROUP = "/sys/fs/cgroup"; public static final String PROC_SELF_MOUNTINFO = "/proc/self/mountinfo"; - private final OperatingSystemMXBean operatingSystemBean; + public static final String DEFAULT_SYS_FS_CGROUP = "/sys/fs/cgroup"; + + static final String CGROUP1_MAX_MEMORY = "memory.limit_in_bytes"; + static final String CGROUP1_USED_MEMORY = "memory.usage_in_bytes"; + static final String CGROUP2_MAX_MEMORY = "memory.max"; + static final String CGROUP2_USED_MEMORY = "memory.current"; + static final String CGROUP_MEMORY_STAT = "memory.stat"; - final private List inactiveMemoryRelevantLines = Arrays.asList(caseSensitiveMatcher("inactive_file *")); + static final Pattern MEMORY_CGROUP = Pattern.compile("^\\d+\\:memory\\:.*"); + static final Pattern CGROUP1_MOUNT_POINT = Pattern.compile("^\\d+? \\d+? .+? .+? (.*?) .*cgroup.*memory.*"); + static final Pattern CGROUP2_MOUNT_POINT = Pattern.compile("^\\d+? \\d+? .+? .+? (.*?) .*cgroup2.*cgroup.*"); - private static String CGROUP1_MAX_MEMORY = "memory.limit_in_bytes"; - private static String CGROUP1_USED_MEMORY = "memory.usage_in_bytes"; - private static String CGROUP1_STAT_MEMORY = "memory.stat"; - private static String CGROUP2_MAX_MEMORY = "memory.max"; - private static String CGROUP2_USED_MEMORY = "memory.current"; - private static String CGROUP2_STAT_MEMORY = "memory.stat"; - private static long UNLIMITED = 0x7FFFFFFFFFFFF000L; + private static final Logger logger = LoggerFactory.getLogger(SystemMetrics.class); - private Pattern MEMORY_CGROUP = Pattern.compile("^\\d+\\:memory\\:.*"); - private Pattern CGROUP1_MOUNT_POINT = Pattern.compile("^\\d+? \\d+? .+? .+? (.*?) .*cgroup.*memory.*"); - private Pattern CGROUP2_MOUNT_POINT = Pattern.compile("^\\d+? \\d+? .+? .+? (.*?) .*cgroup2.*cgroup.*"); + private final OperatingSystemMXBean operatingSystemBean; @Nullable private final Method systemCpuUsage; @@ -113,93 +115,136 @@ public SystemMetrics() { this.virtualProcessMemory = JmxUtils.getOperatingSystemMBeanMethod(operatingSystemBean, "getCommittedVirtualMemorySize"); this.memInfoFile = memInfoFile; - cgroupFiles = verifyCgroupEnabled(procSelfCgroup, mountInfo); + cgroupFiles = findCgroupFiles(procSelfCgroup, mountInfo); } - public CgroupFiles verifyCgroupEnabled(File procSelfCgroup, File mountInfo) { - if (procSelfCgroup.canRead() && mountInfo.canRead()) { - try(BufferedReader fileReader = new BufferedReader(new FileReader(procSelfCgroup))) { - String lineCgroup = null; - for (String cgroupLine = fileReader.readLine(); cgroupLine != null && !cgroupLine.isEmpty(); cgroupLine = fileReader.readLine()) { - if (lineCgroup == null && cgroupLine.startsWith("0:")) { - lineCgroup = cgroupLine; + /** + * Implementing the cgroup metrics spec - https://github.com/elastic/apm/blob/master/docs/agents/agent-development.md#cgroup-metrics + * + * @param procSelfCgroup /proc/self/cgroup file + * @param mountInfo /proc/self/mountinfo file + * @return a holder for the memory cgroup files if found or {@code null} if not found + */ + @Nullable + public CgroupFiles findCgroupFiles(File procSelfCgroup, File mountInfo) { + if (procSelfCgroup.canRead()) { + String cgroupLine = null; + try (BufferedReader fileReader = new BufferedReader(new FileReader(procSelfCgroup))) { + String currentLine = fileReader.readLine(); + while (currentLine != null) { + if (cgroupLine == null && currentLine.startsWith("0:")) { + cgroupLine = currentLine; } - if (MEMORY_CGROUP.matcher(cgroupLine).matches()) { - lineCgroup = cgroupLine; + if (MEMORY_CGROUP.matcher(currentLine).matches()) { + cgroupLine = currentLine; + break; } + currentLine = fileReader.readLine(); } - if (lineCgroup != null) { - CgroupFiles cgroupFilesTest = null; - try(BufferedReader fileMountInfoReader = new BufferedReader(new FileReader(mountInfo))) { - for (String mountLine = fileMountInfoReader.readLine(); mountLine != null && !mountLine.isEmpty(); mountLine = fileMountInfoReader.readLine()) { - String foundRegex = applyCgroup2Regex(mountLine); - if (foundRegex != null) { - cgroupFilesTest = verifyCgroup2Available(lineCgroup, new File(foundRegex)); - if (cgroupFilesTest != null) return cgroupFilesTest; - } - foundRegex = applyCgroup1Regex(mountLine); - if (foundRegex != null) { - cgroupFilesTest = verifyCgroup1Available(new File(foundRegex)); - if (cgroupFilesTest != null) return cgroupFilesTest; + + if (cgroupLine != null) { + CgroupFiles cgroupFiles; + + // Try to discover the cgroup fs path from the mountinfo file + if (mountInfo.canRead()) { + String mountLine = null; + try (BufferedReader fileMountInfoReader = new BufferedReader(new FileReader(mountInfo))) { + mountLine = fileMountInfoReader.readLine(); + while (mountLine != null) { + // cgroup v2 + String rootCgroupFsPath = applyCgroupRegex(CGROUP2_MOUNT_POINT, mountLine); + if (rootCgroupFsPath != null) { + cgroupFiles = createCgroup2Files(cgroupLine, new File(rootCgroupFsPath)); + if (cgroupFiles != null) { + return cgroupFiles; + } + } + + // cgroup v1 + String memoryMountPath = applyCgroupRegex(CGROUP1_MOUNT_POINT, mountLine); + if (memoryMountPath != null) { + cgroupFiles = createCgroup1Files( + new File(memoryMountPath) + ); + if (cgroupFiles != null) { + return cgroupFiles; + } + } + + mountLine = fileMountInfoReader.readLine(); } + } catch (Exception e) { + logger.info("Failed to discover memory mount files path based on mountinfo line '{}'.", mountLine); } + } else { + logger.info("Failed to find/read /proc/self/mountinfo file. Looking for memory files in /sys/fs/cgroup."); } - // Fall back to /sys/fs/cgroup if not found on mountinfo - cgroupFilesTest = verifyCgroup2Available(lineCgroup, new File(SYS_FS_CGROUP)); - if (cgroupFilesTest != null) return cgroupFilesTest; - cgroupFilesTest = verifyCgroup1Available( new File(SYS_FS_CGROUP + File.pathSeparator + "memory")); - if (cgroupFilesTest != null) return cgroupFilesTest; + // Failed to auto-discover the cgroup fs path from mountinfo, fall back to /sys/fs/cgroup + // cgroup v2 + cgroupFiles = createCgroup2Files(cgroupLine, new File(DEFAULT_SYS_FS_CGROUP)); + if (cgroupFiles != null) { + return cgroupFiles; + } + // cgroup v2 + cgroupFiles = createCgroup1Files(new File(DEFAULT_SYS_FS_CGROUP + File.pathSeparator + "memory")); + if (cgroupFiles != null) { + return cgroupFiles; + } + } else { + logger.warn("No /proc/self/cgroup file line matched the tested patterns. Cgroup metrics will not be reported."); } } catch (Exception e) { + logger.error("Failed to discover memory mount files path based on cgroup line '" + cgroupLine + + "'. Cgroup metrics will not be reported,", e); } + } else { + logger.debug("Cannot find/read /proc/self/cgroup file. Cgroup metrics will not be reported."); } return null; } - String applyCgroup1Regex(String mountLine) { - Matcher matcher = CGROUP1_MOUNT_POINT.matcher(mountLine); - if (matcher.matches()) { - return matcher.group(1); - } - return null; - } - String applyCgroup2Regex(String mountLine) { - Matcher matcher = CGROUP2_MOUNT_POINT.matcher(mountLine); + + @Nullable + String applyCgroupRegex(Pattern regex, String mountLine) { + Matcher matcher = regex.matcher(mountLine); if (matcher.matches()) { return matcher.group(1); } return null; } - private CgroupFiles verifyCgroup2Available(String lineCgroup, File mountDiscovered) throws IOException { - final String[] cgroupSplit = StringUtils.split(lineCgroup, ':'); - // Checking cgroup2 - File maxMemory = new File(mountDiscovered, cgroupSplit[cgroupSplit.length - 1] + "/" + CGROUP2_MAX_MEMORY); - if (maxMemory.canRead()) { - try(BufferedReader fileReaderMem = new BufferedReader(new FileReader(maxMemory))) { - String memMaxLine = fileReaderMem.readLine(); - if (!"max".equalsIgnoreCase(memMaxLine)) { - return new CgroupFiles(maxMemory, - new File(mountDiscovered, cgroupSplit[cgroupSplit.length - 1] + "/" + CGROUP2_USED_MEMORY), - new File(mountDiscovered, cgroupSplit[cgroupSplit.length - 1] + "/" + CGROUP2_STAT_MEMORY)); + + @Nullable + private CgroupFiles createCgroup2Files(String cgroupLine, File rootCgroupFsPath) throws IOException { + final String[] cgroupLineParts = StringUtils.split(cgroupLine, ':'); + String sliceSubdir = cgroupLineParts[cgroupLineParts.length - 1]; + File maxMemoryFile = new File(rootCgroupFsPath, sliceSubdir + File.separatorChar + CGROUP2_MAX_MEMORY); + if (maxMemoryFile.canRead()) { + try (BufferedReader maxFileReader = new BufferedReader(new FileReader(maxMemoryFile))) { + String memMaxLine = maxFileReader.readLine(); + if ("max".equalsIgnoreCase(memMaxLine)) { + // Make sure we don't send the max metric when cgroup is not bound to a memory limit + maxMemoryFile = null; } } + return new CgroupFiles( + maxMemoryFile, + new File(rootCgroupFsPath, sliceSubdir + File.separator + CGROUP2_USED_MEMORY), + new File(rootCgroupFsPath, sliceSubdir + File.separator + CGROUP_MEMORY_STAT) + ); } return null; } - private CgroupFiles verifyCgroup1Available(File mountDiscovered) throws IOException { - // Checking cgroup1 - File maxMemory = new File(mountDiscovered, CGROUP1_MAX_MEMORY); - if (maxMemory.canRead()) { - try(BufferedReader fileReaderMem = new BufferedReader(new FileReader(maxMemory))) { - String memMaxLine = fileReaderMem.readLine(); - long memMax = Long.parseLong(memMaxLine); - if (memMax < UNLIMITED) { // Cgroup1 use a contant to disabled limits - return new CgroupFiles(maxMemory, - new File(mountDiscovered, CGROUP1_USED_MEMORY), - new File(mountDiscovered, CGROUP1_STAT_MEMORY)); - } - } + @Nullable + private CgroupFiles createCgroup1Files(File memeoryMountPath) { + File maxMemoryFile = new File(memeoryMountPath, SystemMetrics.CGROUP1_MAX_MEMORY); + if (maxMemoryFile.canRead()) { + // No need for special treatment for the special "unlimited" value (0x7ffffffffffff000) - omitted by the UI + return new CgroupFiles( + maxMemoryFile, + new File(memeoryMountPath, SystemMetrics.CGROUP1_USED_MEMORY), + new File(memeoryMountPath, SystemMetrics.CGROUP_MEMORY_STAT) + ); } return null; } @@ -209,21 +254,6 @@ public void start(ElasticApmTracer tracer) { bindTo(tracer.getMetricRegistry()); } - private double getInactiveMemory() { - try(BufferedReader fileReaderStatFile = new BufferedReader(new FileReader(cgroupFiles.getStatMemory()))) { - long sum = 0; - for (String statLine = fileReaderStatFile.readLine(); statLine != null && !statLine.isEmpty(); statLine = fileReaderStatFile.readLine()) { - if (WildcardMatcher.isAnyMatch(inactiveMemoryRelevantLines, statLine)) { - final String[] statLineSplit = StringUtils.split(statLine, ' '); - sum += Long.parseLong(statLineSplit[1]); - } - } - return sum; - } catch (Exception ignored) { - return Double.NaN; - } - } - void bindTo(MetricRegistry metricRegistry) { // J9 always returns -1 on the first call metricRegistry.addUnlessNan("system.cpu.total.norm.pct", Labels.EMPTY, new DoubleSupplier() { @@ -242,41 +272,59 @@ public double get() { if (cgroupFiles != null) { metricRegistry.addUnlessNan("system.process.cgroup.memory.stats.inactive_file.bytes", Labels.EMPTY, new DoubleSupplier() { - - @Override - public double get() { - return getInactiveMemory(); - } - - }); - metricRegistry.addUnlessNan("system.process.cgroup.memory.mem.usage.bytes", Labels.EMPTY, new DoubleSupplier() { @Override public double get() { - try(BufferedReader fileReaderMemoryUsed = new BufferedReader(new FileReader(cgroupFiles.getUsedMemory())) - ) { - long memUsed = Long.parseLong(fileReaderMemoryUsed.readLine()); - Double inactiveMemory = getInactiveMemory(); - if (!inactiveMemory.equals(Double.NaN)) { - memUsed -= inactiveMemory; + try (BufferedReader fileReaderStatFile = new BufferedReader(new FileReader(cgroupFiles.getStatMemoryFile()))) { + String statLine = fileReaderStatFile.readLine(); + String inactiveBytes = null; + while (statLine != null) { + final String[] statLineSplit = StringUtils.split(statLine, ' '); + if (statLineSplit.length > 1) { + if ("total_inactive_file".equals(statLineSplit[0])) { + inactiveBytes = statLineSplit[1]; + break; + } else if ("inactive_file".equals(statLineSplit[0])) { + inactiveBytes = statLineSplit[1]; + } + } + statLine = fileReaderStatFile.readLine(); } - return memUsed; - } catch (Exception ignored) { + return inactiveBytes != null ? Long.parseLong(inactiveBytes) : Double.NaN; + } catch (Exception e) { + logger.debug("Failed to read " + cgroupFiles.getStatMemoryFile().getAbsolutePath() + " file", e); return Double.NaN; } } }); - metricRegistry.addUnlessNan("system.process.cgroup.memory.mem.limit.bytes", Labels.EMPTY, new DoubleSupplier() { + + metricRegistry.addUnlessNan("system.process.cgroup.memory.mem.usage.bytes", Labels.EMPTY, new DoubleSupplier() { @Override public double get() { - try(BufferedReader fileReaderMemoryMax = new BufferedReader(new FileReader(cgroupFiles.getMaxMemory()))) { - long memMax = Long.parseLong(fileReaderMemoryMax.readLine()); - return memMax; - } catch (Exception ignored) { + try (BufferedReader fileReaderMemoryUsed = new BufferedReader(new FileReader(cgroupFiles.getUsedMemoryFile()))) { + return Long.parseLong(fileReaderMemoryUsed.readLine()); + } catch (Exception e) { + logger.debug("Failed to read " + cgroupFiles.getUsedMemoryFile().getAbsolutePath() + " file", e); return Double.NaN; } } }); + + final File maxMemoryFile = cgroupFiles.getMaxMemoryFile(); + if (maxMemoryFile != null) { + metricRegistry.addUnlessNan("system.process.cgroup.memory.mem.limit.bytes", Labels.EMPTY, new DoubleSupplier() { + @Override + public double get() { + try (BufferedReader fileReaderMemoryMax = new BufferedReader(new FileReader(maxMemoryFile))) { + return Long.parseLong(fileReaderMemoryMax.readLine()); + } catch (Exception e) { + logger.debug("Failed to read " + maxMemoryFile + " file", e); + return Double.NaN; + } + } + }); + } } + if (memInfoFile.canRead()) { metricRegistry.addUnlessNan("system.memory.actual.free", Labels.EMPTY, new DoubleSupplier() { final List relevantLines = Arrays.asList( @@ -356,26 +404,29 @@ private double invoke(@Nullable Method method) { } public static class CgroupFiles { - protected File maxMemory; - protected File usedMemory; - protected File statMemory; - - public CgroupFiles(File maxMemory, File usedMemory, File statMemory) { - this.maxMemory = maxMemory; - this.usedMemory = usedMemory; - this.statMemory = statMemory; + + @Nullable // may be null if memory mount is found for the cgroup, but memory is unlimited + protected File maxMemoryFile; + protected File usedMemoryFile; + protected File statMemoryFile; + + public CgroupFiles(@Nullable File maxMemoryFile, File usedMemoryFile, File statMemoryFile) { + this.maxMemoryFile = maxMemoryFile; + this.usedMemoryFile = usedMemoryFile; + this.statMemoryFile = statMemoryFile; } - public File getMaxMemory() { - return maxMemory; + @Nullable + public File getMaxMemoryFile() { + return maxMemoryFile; } - public File getUsedMemory() { - return usedMemory; + public File getUsedMemoryFile() { + return usedMemoryFile; } - public File getStatMemory() { - return statMemory; + public File getStatMemoryFile() { + return statMemoryFile; } } } diff --git a/apm-agent-core/src/test/java/co/elastic/apm/agent/metrics/builtin/SystemMetricsTest.java b/apm-agent-core/src/test/java/co/elastic/apm/agent/metrics/builtin/SystemMetricsTest.java index 6153fb047b..eb9c1115b5 100644 --- a/apm-agent-core/src/test/java/co/elastic/apm/agent/metrics/builtin/SystemMetricsTest.java +++ b/apm-agent-core/src/test/java/co/elastic/apm/agent/metrics/builtin/SystemMetricsTest.java @@ -64,7 +64,7 @@ void testSystemMetrics() throws InterruptedException { @ParameterizedTest @CsvSource({ "/proc/meminfo, 6235127808", - "/proc/meminfo-3.14, 556630016", + "/proc/meminfo-3.14, 556630016" }) void testFreeMemoryMeminfo(String file, long value) throws Exception { SystemMetrics systemMetrics = createUnlimitedSystemMetrics(file); @@ -72,26 +72,28 @@ void testFreeMemoryMeminfo(String file, long value) throws Exception { assertThat(metricRegistry.getGaugeValue("system.memory.actual.free", Labels.EMPTY)).isEqualTo(value); assertThat(metricRegistry.getGaugeValue("system.memory.total", Labels.EMPTY)).isEqualTo(7964778496L); + + assertThat(metricRegistry.getGaugeValue("system.process.cgroup.memory.mem.limit.bytes", Labels.EMPTY)).isEqualTo(9223372036854771712L); } - private SystemMetrics createUnlimitedSystemMetrics(String file) throws URISyntaxException, IOException { - File mountInfo = new File(getClass().getResource("/proc/cgroup").toURI()); + private SystemMetrics createUnlimitedSystemMetrics(String memInfoFile) throws URISyntaxException, IOException { + File mountInfo = new File(getClass().getResource("/proc/unlimited/memory").toURI()); File fileTmp = File.createTempFile("temp", null); fileTmp.deleteOnExit(); FileWriter fw = new FileWriter(fileTmp); fw.write("39 30 0:35 / " + mountInfo.getAbsolutePath() + " rw,nosuid,nodev,noexec,relatime shared:10 - cgroup cgroup rw,seclabel,memory\n"); fw.close(); - return new SystemMetrics(new File(getClass().getResource(file).toURI()), - new File(getClass().getResource("/proc/unlimited/memory").toURI()), + return new SystemMetrics(new File(getClass().getResource(memInfoFile).toURI()), + new File(getClass().getResource("/proc/cgroup").toURI()), fileTmp ); } @ParameterizedTest @CsvSource({ - "/proc/meminfo, 954370560, /proc/cgroup, /proc/limited/memory", - "/proc/meminfo, 954370560, /proc/cgroup2, /proc/sys_cgroup2" + "/proc/meminfo, 964778496, /proc/cgroup, /proc/limited/memory", + "/proc/meminfo, 964778496, /proc/cgroup2, /proc/sys_cgroup2" }) void testFreeCgroupMemoryMeminfo(String file, long value, String selfCGroup, String sysFsGroup) throws Exception { File mountInfo = new File(getClass().getResource(sysFsGroup).toURI()); @@ -121,9 +123,12 @@ void testFreeCgroupMemoryMeminfo(String file, long value, String selfCGroup, Str "39 30 0:36 / /sys/fs/cgroup/memory rw,nosuid,nodev,noexec,relatime shared:10 - cgroup cgroup rw,seclabel,memory|/sys/fs/cgroup/memory", }) void testCgroup1Regex(String testString) throws Exception { - String [] split = testString.split("\\|"); + String[] split = testString.split("\\|"); SystemMetrics systemMetrics = createUnlimitedSystemMetrics("/proc/meminfo"); - assertThat(systemMetrics.applyCgroup1Regex(split[0])).isEqualTo(split[1]); + assertThat(systemMetrics.applyCgroupRegex(SystemMetrics.CGROUP1_MOUNT_POINT, split[0])).isEqualTo(split[1]); + + systemMetrics.bindTo(metricRegistry); + assertThat(metricRegistry.getGaugeValue("system.process.cgroup.memory.mem.limit.bytes", Labels.EMPTY)).isEqualTo(9223372036854771712L); } @ParameterizedTest @@ -133,7 +138,7 @@ void testCgroup1Regex(String testString) throws Exception { void testCgroup2Regex(String testString) throws Exception { String [] split = testString.split("\\|"); SystemMetrics systemMetrics = createUnlimitedSystemMetrics("/proc/meminfo"); - assertThat(systemMetrics.applyCgroup2Regex(split[0])).isEqualTo(split[1]); + assertThat(systemMetrics.applyCgroupRegex(SystemMetrics.CGROUP2_MOUNT_POINT, split[0])).isEqualTo(split[1]); } private void consumeCpu() { From 44d0561a75813275064de1915edaabca54e7bfe1 Mon Sep 17 00:00:00 2001 From: George Tavares Date: Wed, 15 Jul 2020 20:57:52 -0300 Subject: [PATCH 07/11] Split CGroupFile --- .../agent/metrics/builtin/CGroupMetrics.java | 307 ++++++++++++++++++ .../agent/metrics/builtin/SystemMetrics.java | 242 +------------- ...lastic.apm.agent.context.LifecycleListener | 1 + .../metrics/builtin/CGroupMetricsTest.java | 131 ++++++++ .../metrics/builtin/SystemMetricsTest.java | 74 +---- .../slice/memory.current | 1 + .../sys_cgroup2_unlimited/slice/memory.max | 1 + .../sys_cgroup2_unlimited/slice/memory.stat | 40 +++ 8 files changed, 484 insertions(+), 313 deletions(-) create mode 100644 apm-agent-core/src/main/java/co/elastic/apm/agent/metrics/builtin/CGroupMetrics.java create mode 100644 apm-agent-core/src/test/java/co/elastic/apm/agent/metrics/builtin/CGroupMetricsTest.java create mode 100644 apm-agent-core/src/test/resources/proc/sys_cgroup2_unlimited/slice/memory.current create mode 100644 apm-agent-core/src/test/resources/proc/sys_cgroup2_unlimited/slice/memory.max create mode 100644 apm-agent-core/src/test/resources/proc/sys_cgroup2_unlimited/slice/memory.stat diff --git a/apm-agent-core/src/main/java/co/elastic/apm/agent/metrics/builtin/CGroupMetrics.java b/apm-agent-core/src/main/java/co/elastic/apm/agent/metrics/builtin/CGroupMetrics.java new file mode 100644 index 0000000000..b2a11b310f --- /dev/null +++ b/apm-agent-core/src/main/java/co/elastic/apm/agent/metrics/builtin/CGroupMetrics.java @@ -0,0 +1,307 @@ +/*- + * #%L + * Elastic APM Java agent + * %% + * Copyright (C) 2018 - 2020 Elastic and contributors + * %% + * Licensed to Elasticsearch B.V. under one or more contributor + * license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright + * ownership. Elasticsearch B.V. licenses this file to you under + * the Apache License, Version 2.0 (the "License"); you may + * not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + * #L% + */ +package co.elastic.apm.agent.metrics.builtin; + +import co.elastic.apm.agent.context.AbstractLifecycleListener; +import co.elastic.apm.agent.impl.ElasticApmTracer; +import co.elastic.apm.agent.matcher.WildcardMatcher; +import co.elastic.apm.agent.metrics.DoubleSupplier; +import co.elastic.apm.agent.metrics.Labels; +import co.elastic.apm.agent.metrics.MetricRegistry; +import co.elastic.apm.agent.util.JmxUtils; +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; +import org.stagemonitor.util.StringUtils; + +import javax.annotation.Nullable; +import java.io.BufferedReader; +import java.io.File; +import java.io.FileReader; +import java.io.IOException; +import java.lang.management.ManagementFactory; +import java.lang.management.OperatingSystemMXBean; +import java.lang.reflect.Method; +import java.util.Arrays; +import java.util.HashMap; +import java.util.List; +import java.util.Map; +import java.util.regex.Matcher; +import java.util.regex.Pattern; + +import static co.elastic.apm.agent.matcher.WildcardMatcher.caseSensitiveMatcher; + +/** + * Record metrics related to the CGroup Usage. + */ +public class CGroupMetrics extends AbstractLifecycleListener { + + public static final String PROC_SELF_CGROUP = "/proc/self/cgroup"; + public static final String PROC_SELF_MOUNTINFO = "/proc/self/mountinfo"; + public static final String DEFAULT_SYS_FS_CGROUP = "/sys/fs/cgroup"; + + static final String CGROUP1_MAX_MEMORY = "memory.limit_in_bytes"; + static final String CGROUP1_USED_MEMORY = "memory.usage_in_bytes"; + static final String CGROUP2_MAX_MEMORY = "memory.max"; + static final String CGROUP2_USED_MEMORY = "memory.current"; + static final String CGROUP_MEMORY_STAT = "memory.stat"; + + static final Pattern MEMORY_CGROUP = Pattern.compile("^\\d+\\:memory\\:.*"); + static final Pattern CGROUP1_MOUNT_POINT = Pattern.compile("^\\d+? \\d+? .+? .+? (.*?) .*cgroup.*memory.*"); + static final Pattern CGROUP2_MOUNT_POINT = Pattern.compile("^\\d+? \\d+? .+? .+? (.*?) .*cgroup2.*cgroup.*"); + + private static final Logger logger = LoggerFactory.getLogger(CGroupMetrics.class); + + + @Nullable + private final CgroupFiles cgroupFiles; + + public CGroupMetrics() { + this(new File(PROC_SELF_CGROUP), new File(PROC_SELF_MOUNTINFO)); + } + + CGroupMetrics( File procSelfCgroup, File mountInfo) { + cgroupFiles = findCgroupFiles(procSelfCgroup, mountInfo); + } + + /** + * Implementing the cgroup metrics spec - https://github.com/elastic/apm/blob/master/docs/agents/agent-development.md#cgroup-metrics + * + * @param procSelfCgroup /proc/self/cgroup file + * @param mountInfo /proc/self/mountinfo file + * @return a holder for the memory cgroup files if found or {@code null} if not found + */ + @Nullable + public CgroupFiles findCgroupFiles(File procSelfCgroup, File mountInfo) { + if (procSelfCgroup.canRead()) { + String cgroupLine = null; + try (BufferedReader fileReader = new BufferedReader(new FileReader(procSelfCgroup))) { + String currentLine = fileReader.readLine(); + while (currentLine != null) { + if (cgroupLine == null && currentLine.startsWith("0:")) { + cgroupLine = currentLine; + } + if (MEMORY_CGROUP.matcher(currentLine).matches()) { + cgroupLine = currentLine; + break; + } + currentLine = fileReader.readLine(); + } + + if (cgroupLine != null) { + CgroupFiles cgroupFiles; + + // Try to discover the cgroup fs path from the mountinfo file + if (mountInfo.canRead()) { + String mountLine = null; + try (BufferedReader fileMountInfoReader = new BufferedReader(new FileReader(mountInfo))) { + mountLine = fileMountInfoReader.readLine(); + while (mountLine != null) { + // cgroup v2 + String rootCgroupFsPath = applyCgroupRegex(CGROUP2_MOUNT_POINT, mountLine); + if (rootCgroupFsPath != null) { + cgroupFiles = createCgroup2Files(cgroupLine, new File(rootCgroupFsPath)); + if (cgroupFiles != null) { + return cgroupFiles; + } + } + + // cgroup v1 + String memoryMountPath = applyCgroupRegex(CGROUP1_MOUNT_POINT, mountLine); + if (memoryMountPath != null) { + cgroupFiles = createCgroup1Files( + new File(memoryMountPath) + ); + if (cgroupFiles != null) { + return cgroupFiles; + } + } + + mountLine = fileMountInfoReader.readLine(); + } + } catch (Exception e) { + logger.info("Failed to discover memory mount files path based on mountinfo line '{}'.", mountLine); + } + } else { + logger.info("Failed to find/read /proc/self/mountinfo file. Looking for memory files in /sys/fs/cgroup."); + } + + // Failed to auto-discover the cgroup fs path from mountinfo, fall back to /sys/fs/cgroup + // cgroup v2 + cgroupFiles = createCgroup2Files(cgroupLine, new File(DEFAULT_SYS_FS_CGROUP)); + if (cgroupFiles != null) { + return cgroupFiles; + } + // cgroup v1 + cgroupFiles = createCgroup1Files(new File(DEFAULT_SYS_FS_CGROUP + File.pathSeparator + "memory")); + if (cgroupFiles != null) { + return cgroupFiles; + } + } else { + logger.warn("No /proc/self/cgroup file line matched the tested patterns. Cgroup metrics will not be reported."); + } + } catch (Exception e) { + logger.error("Failed to discover memory mount files path based on cgroup line '" + cgroupLine + + "'. Cgroup metrics will not be reported,", e); + } + } else { + logger.debug("Cannot find/read /proc/self/cgroup file. Cgroup metrics will not be reported."); + } + return null; + } + + @Nullable + String applyCgroupRegex(Pattern regex, String mountLine) { + Matcher matcher = regex.matcher(mountLine); + if (matcher.matches()) { + return matcher.group(1); + } + return null; + } + + @Nullable + private CgroupFiles createCgroup2Files(String cgroupLine, File rootCgroupFsPath) throws IOException { + final String[] cgroupLineParts = StringUtils.split(cgroupLine, ':'); + String sliceSubdir = cgroupLineParts[cgroupLineParts.length - 1]; + File maxMemoryFile = new File(rootCgroupFsPath, sliceSubdir + File.separatorChar + CGROUP2_MAX_MEMORY); + if (maxMemoryFile.canRead()) { + try (BufferedReader maxFileReader = new BufferedReader(new FileReader(maxMemoryFile))) { + String memMaxLine = maxFileReader.readLine(); + if ("max".equalsIgnoreCase(memMaxLine)) { + // Make sure we don't send the max metric when cgroup is not bound to a memory limit + maxMemoryFile = null; + } + } + return new CgroupFiles( + maxMemoryFile, + new File(rootCgroupFsPath, sliceSubdir + File.separator + CGROUP2_USED_MEMORY), + new File(rootCgroupFsPath, sliceSubdir + File.separator + CGROUP_MEMORY_STAT) + ); + } + return null; + } + + @Nullable + private CgroupFiles createCgroup1Files(File memoryMountPath) { + File maxMemoryFile = new File(memoryMountPath, CGroupMetrics.CGROUP1_MAX_MEMORY); + if (maxMemoryFile.canRead()) { + // No need for special treatment for the special "unlimited" value (0x7ffffffffffff000) - omitted by the UI + return new CgroupFiles( + maxMemoryFile, + new File(memoryMountPath, CGroupMetrics.CGROUP1_USED_MEMORY), + new File(memoryMountPath, CGroupMetrics.CGROUP_MEMORY_STAT) + ); + } + return null; + } + + @Override + public void start(ElasticApmTracer tracer) { + bindTo(tracer.getMetricRegistry()); + } + + void bindTo(MetricRegistry metricRegistry) { + if (cgroupFiles != null) { + metricRegistry.addUnlessNan("system.process.cgroup.memory.stats.inactive_file.bytes", Labels.EMPTY, new DoubleSupplier() { + @Override + public double get() { + try (BufferedReader fileReaderStatFile = new BufferedReader(new FileReader(cgroupFiles.getStatMemoryFile()))) { + String statLine = fileReaderStatFile.readLine(); + String inactiveBytes = null; + while (statLine != null) { + final String[] statLineSplit = StringUtils.split(statLine, ' '); + if (statLineSplit.length > 1) { + if ("total_inactive_file".equals(statLineSplit[0])) { + inactiveBytes = statLineSplit[1]; + break; + } else if ("inactive_file".equals(statLineSplit[0])) { + inactiveBytes = statLineSplit[1]; + } + } + statLine = fileReaderStatFile.readLine(); + } + return inactiveBytes != null ? Long.parseLong(inactiveBytes) : Double.NaN; + } catch (Exception e) { + logger.debug("Failed to read " + cgroupFiles.getStatMemoryFile().getAbsolutePath() + " file", e); + return Double.NaN; + } + } + }); + + metricRegistry.addUnlessNan("system.process.cgroup.memory.mem.usage.bytes", Labels.EMPTY, new DoubleSupplier() { + @Override + public double get() { + try (BufferedReader fileReaderMemoryUsed = new BufferedReader(new FileReader(cgroupFiles.getUsedMemoryFile()))) { + return Long.parseLong(fileReaderMemoryUsed.readLine()); + } catch (Exception e) { + logger.debug("Failed to read " + cgroupFiles.getUsedMemoryFile().getAbsolutePath() + " file", e); + return Double.NaN; + } + } + }); + + final File maxMemoryFile = cgroupFiles.getMaxMemoryFile(); + if (maxMemoryFile != null) { + metricRegistry.addUnlessNan("system.process.cgroup.memory.mem.limit.bytes", Labels.EMPTY, new DoubleSupplier() { + @Override + public double get() { + try (BufferedReader fileReaderMemoryMax = new BufferedReader(new FileReader(maxMemoryFile))) { + return Long.parseLong(fileReaderMemoryMax.readLine()); + } catch (Exception e) { + logger.debug("Failed to read " + maxMemoryFile + " file", e); + return Double.NaN; + } + } + }); + } + } + } + + public static class CgroupFiles { + + @Nullable // may be null if memory mount is found for the cgroup, but memory is unlimited + protected File maxMemoryFile; + protected File usedMemoryFile; + protected File statMemoryFile; + + public CgroupFiles(@Nullable File maxMemoryFile, File usedMemoryFile, File statMemoryFile) { + this.maxMemoryFile = maxMemoryFile; + this.usedMemoryFile = usedMemoryFile; + this.statMemoryFile = statMemoryFile; + } + + @Nullable + public File getMaxMemoryFile() { + return maxMemoryFile; + } + + public File getUsedMemoryFile() { + return usedMemoryFile; + } + + public File getStatMemoryFile() { + return statMemoryFile; + } + } +} diff --git a/apm-agent-core/src/main/java/co/elastic/apm/agent/metrics/builtin/SystemMetrics.java b/apm-agent-core/src/main/java/co/elastic/apm/agent/metrics/builtin/SystemMetrics.java index 161deaba93..75fed22d13 100644 --- a/apm-agent-core/src/main/java/co/elastic/apm/agent/metrics/builtin/SystemMetrics.java +++ b/apm-agent-core/src/main/java/co/elastic/apm/agent/metrics/builtin/SystemMetrics.java @@ -31,15 +31,12 @@ import co.elastic.apm.agent.metrics.Labels; import co.elastic.apm.agent.metrics.MetricRegistry; import co.elastic.apm.agent.util.JmxUtils; -import org.slf4j.Logger; -import org.slf4j.LoggerFactory; import org.stagemonitor.util.StringUtils; import javax.annotation.Nullable; import java.io.BufferedReader; import java.io.File; import java.io.FileReader; -import java.io.IOException; import java.lang.management.ManagementFactory; import java.lang.management.OperatingSystemMXBean; import java.lang.reflect.Method; @@ -47,8 +44,6 @@ import java.util.HashMap; import java.util.List; import java.util.Map; -import java.util.regex.Matcher; -import java.util.regex.Pattern; import static co.elastic.apm.agent.matcher.WildcardMatcher.caseSensitiveMatcher; @@ -66,22 +61,6 @@ */ public class SystemMetrics extends AbstractLifecycleListener { - public static final String PROC_SELF_CGROUP = "/proc/self/cgroup"; - public static final String PROC_SELF_MOUNTINFO = "/proc/self/mountinfo"; - public static final String DEFAULT_SYS_FS_CGROUP = "/sys/fs/cgroup"; - - static final String CGROUP1_MAX_MEMORY = "memory.limit_in_bytes"; - static final String CGROUP1_USED_MEMORY = "memory.usage_in_bytes"; - static final String CGROUP2_MAX_MEMORY = "memory.max"; - static final String CGROUP2_USED_MEMORY = "memory.current"; - static final String CGROUP_MEMORY_STAT = "memory.stat"; - - static final Pattern MEMORY_CGROUP = Pattern.compile("^\\d+\\:memory\\:.*"); - static final Pattern CGROUP1_MOUNT_POINT = Pattern.compile("^\\d+? \\d+? .+? .+? (.*?) .*cgroup.*memory.*"); - static final Pattern CGROUP2_MOUNT_POINT = Pattern.compile("^\\d+? \\d+? .+? .+? (.*?) .*cgroup2.*cgroup.*"); - - private static final Logger logger = LoggerFactory.getLogger(SystemMetrics.class); - private final OperatingSystemMXBean operatingSystemBean; @Nullable @@ -99,14 +78,12 @@ public class SystemMetrics extends AbstractLifecycleListener { @Nullable private final Method virtualProcessMemory; private final File memInfoFile; - @Nullable - private final CgroupFiles cgroupFiles; public SystemMetrics() { - this(new File("/proc/meminfo"), new File(PROC_SELF_CGROUP), new File(PROC_SELF_MOUNTINFO)); + this(new File("/proc/meminfo")); } - SystemMetrics(File memInfoFile, File procSelfCgroup, File mountInfo) { + SystemMetrics(File memInfoFile) { this.operatingSystemBean = ManagementFactory.getOperatingSystemMXBean(); this.systemCpuUsage = JmxUtils.getOperatingSystemMBeanMethod(operatingSystemBean, "getSystemCpuLoad"); this.processCpuUsage = JmxUtils.getOperatingSystemMBeanMethod(operatingSystemBean, "getProcessCpuLoad"); @@ -114,139 +91,6 @@ public SystemMetrics() { this.totalMemory = JmxUtils.getOperatingSystemMBeanMethod(operatingSystemBean, "getTotalPhysicalMemorySize"); this.virtualProcessMemory = JmxUtils.getOperatingSystemMBeanMethod(operatingSystemBean, "getCommittedVirtualMemorySize"); this.memInfoFile = memInfoFile; - - cgroupFiles = findCgroupFiles(procSelfCgroup, mountInfo); - } - - /** - * Implementing the cgroup metrics spec - https://github.com/elastic/apm/blob/master/docs/agents/agent-development.md#cgroup-metrics - * - * @param procSelfCgroup /proc/self/cgroup file - * @param mountInfo /proc/self/mountinfo file - * @return a holder for the memory cgroup files if found or {@code null} if not found - */ - @Nullable - public CgroupFiles findCgroupFiles(File procSelfCgroup, File mountInfo) { - if (procSelfCgroup.canRead()) { - String cgroupLine = null; - try (BufferedReader fileReader = new BufferedReader(new FileReader(procSelfCgroup))) { - String currentLine = fileReader.readLine(); - while (currentLine != null) { - if (cgroupLine == null && currentLine.startsWith("0:")) { - cgroupLine = currentLine; - } - if (MEMORY_CGROUP.matcher(currentLine).matches()) { - cgroupLine = currentLine; - break; - } - currentLine = fileReader.readLine(); - } - - if (cgroupLine != null) { - CgroupFiles cgroupFiles; - - // Try to discover the cgroup fs path from the mountinfo file - if (mountInfo.canRead()) { - String mountLine = null; - try (BufferedReader fileMountInfoReader = new BufferedReader(new FileReader(mountInfo))) { - mountLine = fileMountInfoReader.readLine(); - while (mountLine != null) { - // cgroup v2 - String rootCgroupFsPath = applyCgroupRegex(CGROUP2_MOUNT_POINT, mountLine); - if (rootCgroupFsPath != null) { - cgroupFiles = createCgroup2Files(cgroupLine, new File(rootCgroupFsPath)); - if (cgroupFiles != null) { - return cgroupFiles; - } - } - - // cgroup v1 - String memoryMountPath = applyCgroupRegex(CGROUP1_MOUNT_POINT, mountLine); - if (memoryMountPath != null) { - cgroupFiles = createCgroup1Files( - new File(memoryMountPath) - ); - if (cgroupFiles != null) { - return cgroupFiles; - } - } - - mountLine = fileMountInfoReader.readLine(); - } - } catch (Exception e) { - logger.info("Failed to discover memory mount files path based on mountinfo line '{}'.", mountLine); - } - } else { - logger.info("Failed to find/read /proc/self/mountinfo file. Looking for memory files in /sys/fs/cgroup."); - } - - // Failed to auto-discover the cgroup fs path from mountinfo, fall back to /sys/fs/cgroup - // cgroup v2 - cgroupFiles = createCgroup2Files(cgroupLine, new File(DEFAULT_SYS_FS_CGROUP)); - if (cgroupFiles != null) { - return cgroupFiles; - } - // cgroup v2 - cgroupFiles = createCgroup1Files(new File(DEFAULT_SYS_FS_CGROUP + File.pathSeparator + "memory")); - if (cgroupFiles != null) { - return cgroupFiles; - } - } else { - logger.warn("No /proc/self/cgroup file line matched the tested patterns. Cgroup metrics will not be reported."); - } - } catch (Exception e) { - logger.error("Failed to discover memory mount files path based on cgroup line '" + cgroupLine + - "'. Cgroup metrics will not be reported,", e); - } - } else { - logger.debug("Cannot find/read /proc/self/cgroup file. Cgroup metrics will not be reported."); - } - return null; - } - - @Nullable - String applyCgroupRegex(Pattern regex, String mountLine) { - Matcher matcher = regex.matcher(mountLine); - if (matcher.matches()) { - return matcher.group(1); - } - return null; - } - - @Nullable - private CgroupFiles createCgroup2Files(String cgroupLine, File rootCgroupFsPath) throws IOException { - final String[] cgroupLineParts = StringUtils.split(cgroupLine, ':'); - String sliceSubdir = cgroupLineParts[cgroupLineParts.length - 1]; - File maxMemoryFile = new File(rootCgroupFsPath, sliceSubdir + File.separatorChar + CGROUP2_MAX_MEMORY); - if (maxMemoryFile.canRead()) { - try (BufferedReader maxFileReader = new BufferedReader(new FileReader(maxMemoryFile))) { - String memMaxLine = maxFileReader.readLine(); - if ("max".equalsIgnoreCase(memMaxLine)) { - // Make sure we don't send the max metric when cgroup is not bound to a memory limit - maxMemoryFile = null; - } - } - return new CgroupFiles( - maxMemoryFile, - new File(rootCgroupFsPath, sliceSubdir + File.separator + CGROUP2_USED_MEMORY), - new File(rootCgroupFsPath, sliceSubdir + File.separator + CGROUP_MEMORY_STAT) - ); - } - return null; - } - - @Nullable - private CgroupFiles createCgroup1Files(File memeoryMountPath) { - File maxMemoryFile = new File(memeoryMountPath, SystemMetrics.CGROUP1_MAX_MEMORY); - if (maxMemoryFile.canRead()) { - // No need for special treatment for the special "unlimited" value (0x7ffffffffffff000) - omitted by the UI - return new CgroupFiles( - maxMemoryFile, - new File(memeoryMountPath, SystemMetrics.CGROUP1_USED_MEMORY), - new File(memeoryMountPath, SystemMetrics.CGROUP_MEMORY_STAT) - ); - } - return null; } @Override @@ -270,61 +114,6 @@ public double get() { } }); - if (cgroupFiles != null) { - metricRegistry.addUnlessNan("system.process.cgroup.memory.stats.inactive_file.bytes", Labels.EMPTY, new DoubleSupplier() { - @Override - public double get() { - try (BufferedReader fileReaderStatFile = new BufferedReader(new FileReader(cgroupFiles.getStatMemoryFile()))) { - String statLine = fileReaderStatFile.readLine(); - String inactiveBytes = null; - while (statLine != null) { - final String[] statLineSplit = StringUtils.split(statLine, ' '); - if (statLineSplit.length > 1) { - if ("total_inactive_file".equals(statLineSplit[0])) { - inactiveBytes = statLineSplit[1]; - break; - } else if ("inactive_file".equals(statLineSplit[0])) { - inactiveBytes = statLineSplit[1]; - } - } - statLine = fileReaderStatFile.readLine(); - } - return inactiveBytes != null ? Long.parseLong(inactiveBytes) : Double.NaN; - } catch (Exception e) { - logger.debug("Failed to read " + cgroupFiles.getStatMemoryFile().getAbsolutePath() + " file", e); - return Double.NaN; - } - } - }); - - metricRegistry.addUnlessNan("system.process.cgroup.memory.mem.usage.bytes", Labels.EMPTY, new DoubleSupplier() { - @Override - public double get() { - try (BufferedReader fileReaderMemoryUsed = new BufferedReader(new FileReader(cgroupFiles.getUsedMemoryFile()))) { - return Long.parseLong(fileReaderMemoryUsed.readLine()); - } catch (Exception e) { - logger.debug("Failed to read " + cgroupFiles.getUsedMemoryFile().getAbsolutePath() + " file", e); - return Double.NaN; - } - } - }); - - final File maxMemoryFile = cgroupFiles.getMaxMemoryFile(); - if (maxMemoryFile != null) { - metricRegistry.addUnlessNan("system.process.cgroup.memory.mem.limit.bytes", Labels.EMPTY, new DoubleSupplier() { - @Override - public double get() { - try (BufferedReader fileReaderMemoryMax = new BufferedReader(new FileReader(maxMemoryFile))) { - return Long.parseLong(fileReaderMemoryMax.readLine()); - } catch (Exception e) { - logger.debug("Failed to read " + maxMemoryFile + " file", e); - return Double.NaN; - } - } - }); - } - } - if (memInfoFile.canRead()) { metricRegistry.addUnlessNan("system.memory.actual.free", Labels.EMPTY, new DoubleSupplier() { final List relevantLines = Arrays.asList( @@ -402,31 +191,4 @@ private double invoke(@Nullable Method method) { return Double.NaN; } } - - public static class CgroupFiles { - - @Nullable // may be null if memory mount is found for the cgroup, but memory is unlimited - protected File maxMemoryFile; - protected File usedMemoryFile; - protected File statMemoryFile; - - public CgroupFiles(@Nullable File maxMemoryFile, File usedMemoryFile, File statMemoryFile) { - this.maxMemoryFile = maxMemoryFile; - this.usedMemoryFile = usedMemoryFile; - this.statMemoryFile = statMemoryFile; - } - - @Nullable - public File getMaxMemoryFile() { - return maxMemoryFile; - } - - public File getUsedMemoryFile() { - return usedMemoryFile; - } - - public File getStatMemoryFile() { - return statMemoryFile; - } - } } diff --git a/apm-agent-core/src/main/resources/META-INF/services/co.elastic.apm.agent.context.LifecycleListener b/apm-agent-core/src/main/resources/META-INF/services/co.elastic.apm.agent.context.LifecycleListener index 2148b1ead3..dd72076c26 100644 --- a/apm-agent-core/src/main/resources/META-INF/services/co.elastic.apm.agent.context.LifecycleListener +++ b/apm-agent-core/src/main/resources/META-INF/services/co.elastic.apm.agent.context.LifecycleListener @@ -3,6 +3,7 @@ co.elastic.apm.agent.bci.OsgiBootDelegationEnabler co.elastic.apm.agent.bci.MatcherTimerLifecycleListener co.elastic.apm.agent.metrics.builtin.JvmMemoryMetrics co.elastic.apm.agent.metrics.builtin.SystemMetrics +co.elastic.apm.agent.metrics.builtin.CGroupMetrics co.elastic.apm.agent.metrics.builtin.JvmGcMetrics co.elastic.apm.agent.metrics.builtin.ThreadMetrics co.elastic.apm.agent.impl.circuitbreaker.CircuitBreaker diff --git a/apm-agent-core/src/test/java/co/elastic/apm/agent/metrics/builtin/CGroupMetricsTest.java b/apm-agent-core/src/test/java/co/elastic/apm/agent/metrics/builtin/CGroupMetricsTest.java new file mode 100644 index 0000000000..95fd2f5b83 --- /dev/null +++ b/apm-agent-core/src/test/java/co/elastic/apm/agent/metrics/builtin/CGroupMetricsTest.java @@ -0,0 +1,131 @@ +/*- + * #%L + * Elastic APM Java agent + * %% + * Copyright (C) 2018 - 2020 Elastic and contributors + * %% + * Licensed to Elasticsearch B.V. under one or more contributor + * license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright + * ownership. Elasticsearch B.V. licenses this file to you under + * the Apache License, Version 2.0 (the "License"); you may + * not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + * #L% + */ +package co.elastic.apm.agent.metrics.builtin; + +import co.elastic.apm.agent.metrics.Labels; +import co.elastic.apm.agent.metrics.MetricRegistry; +import co.elastic.apm.agent.report.ReporterConfiguration; +import org.junit.jupiter.api.Test; +import org.junit.jupiter.params.ParameterizedTest; +import org.junit.jupiter.params.provider.CsvSource; +import org.junit.jupiter.params.provider.ValueSource; + +import java.io.File; +import java.io.FileWriter; +import java.io.IOException; +import java.net.URISyntaxException; + +import static org.assertj.core.api.Assertions.assertThat; +import static org.mockito.Mockito.mock; + +class CGroupMetricsTest { + + private MetricRegistry metricRegistry = new MetricRegistry(mock(ReporterConfiguration.class)); + + private CGroupMetrics createUnlimitedSystemMetrics() throws URISyntaxException, IOException { + File mountInfo = new File(getClass().getResource("/proc/unlimited/memory").toURI()); + File fileTmp = File.createTempFile("temp", null); + fileTmp.deleteOnExit(); + FileWriter fw = new FileWriter(fileTmp); + fw.write("39 30 0:35 / " + mountInfo.getAbsolutePath() + " rw,nosuid,nodev,noexec,relatime shared:10 - cgroup cgroup rw,seclabel,memory\n"); + fw.close(); + + return new CGroupMetrics(new File(getClass().getResource("/proc/cgroup").toURI()), + fileTmp + ); + } + + @ParameterizedTest + @CsvSource({ + "964778496, /proc/cgroup, /proc/limited/memory, 7964778496", + "964778496, /proc/cgroup2, /proc/sys_cgroup2, 7964778496", + "964778496, /proc/cgroup2, /proc/sys_cgroup2_unlimited, NaN" + }) + void testFreeCgroupMemory(long value, String selfCGroup, String sysFsGroup, String memLimit) throws Exception { + File mountInfo = new File(getClass().getResource(sysFsGroup).toURI()); + File fileTmp = File.createTempFile("temp", null); + fileTmp.deleteOnExit(); + FileWriter fw = new FileWriter(fileTmp); + if (sysFsGroup.startsWith("/proc/sys_cgroup2")) { + fw.write("30 23 0:26 / " + mountInfo.getAbsolutePath() + " rw,nosuid,nodev,noexec,relatime shared:4 - cgroup2 cgroup rw,seclabel\n"); + } + else { + fw.write("39 30 0:35 / " + mountInfo.getAbsolutePath() + " rw,nosuid,nodev,noexec,relatime shared:10 - cgroup cgroup rw,seclabel,memory\n"); + } + fw.close(); + + CGroupMetrics cgroupMetrics = new CGroupMetrics(new File(getClass().getResource(selfCGroup).toURI()), + fileTmp + ); + cgroupMetrics.bindTo(metricRegistry); + + assertThat(metricRegistry.getGaugeValue("system.process.cgroup.memory.mem.usage.bytes", Labels.EMPTY)).isEqualTo(value); + assertThat(metricRegistry.getGaugeValue("system.process.cgroup.memory.mem.limit.bytes", Labels.EMPTY)).isEqualTo(Double.valueOf(memLimit)); + assertThat(metricRegistry.getGaugeValue("system.process.cgroup.memory.stats.inactive_file.bytes", Labels.EMPTY)).isEqualTo(10407936L); + } + @ParameterizedTest + @ValueSource(strings ={ + "39 30 0:36 / /sys/fs/cgroup/memory rw,nosuid,nodev,noexec,relatime shared:10 - cgroup cgroup rw,seclabel,memory|/sys/fs/cgroup/memory", + }) + void testCgroup1Regex(String testString) throws Exception { + String[] split = testString.split("\\|"); + CGroupMetrics cgroupMetrics = createUnlimitedSystemMetrics(); + assertThat(cgroupMetrics.applyCgroupRegex(CGroupMetrics.CGROUP1_MOUNT_POINT, split[0])).isEqualTo(split[1]); + + cgroupMetrics.bindTo(metricRegistry); + assertThat(metricRegistry.getGaugeValue("system.process.cgroup.memory.mem.limit.bytes", Labels.EMPTY)).isEqualTo(9223372036854771712L); + } + + @ParameterizedTest + @ValueSource(strings ={ + "39 30 0:36 / /sys/fs/cgroup/memory rw,nosuid,nodev,noexec,relatime shared:4 - cgroup2 cgroup rw,seclabel|/sys/fs/cgroup/memory", + }) + void testCgroup2Regex(String testString) throws Exception { + String [] split = testString.split("\\|"); + CGroupMetrics cgroupMetrics = createUnlimitedSystemMetrics(); + assertThat(cgroupMetrics.applyCgroupRegex(CGroupMetrics.CGROUP2_MOUNT_POINT, split[0])).isEqualTo(split[1]); + } + + @Test + void testUnlimitedCgroup1() throws Exception { + CGroupMetrics cgroupMetrics = createUnlimitedSystemMetrics(); + cgroupMetrics.bindTo(metricRegistry); + + assertThat(metricRegistry.getGaugeValue("system.process.cgroup.memory.mem.limit.bytes", Labels.EMPTY)).isEqualTo(9223372036854771712L); + assertThat(metricRegistry.getGaugeValue("system.process.cgroup.memory.mem.usage.bytes", Labels.EMPTY)).isEqualTo(964778496); + assertThat(metricRegistry.getGaugeValue("system.process.cgroup.memory.stats.inactive_file.bytes", Labels.EMPTY)).isEqualTo(10407936L); + } + + @Test + void testUnlimitedCgroup2() throws Exception { + CGroupMetrics cgroupMetrics = createUnlimitedSystemMetrics(); + cgroupMetrics.bindTo(metricRegistry); + + assertThat(metricRegistry.getGaugeValue("system.process.cgroup.memory.mem.limit.bytes", Labels.EMPTY)).isEqualTo(9223372036854771712L); + assertThat(metricRegistry.getGaugeValue("system.process.cgroup.memory.mem.usage.bytes", Labels.EMPTY)).isEqualTo(964778496); + assertThat(metricRegistry.getGaugeValue("system.process.cgroup.memory.stats.inactive_file.bytes", Labels.EMPTY)).isEqualTo(10407936L); + } + +} diff --git a/apm-agent-core/src/test/java/co/elastic/apm/agent/metrics/builtin/SystemMetricsTest.java b/apm-agent-core/src/test/java/co/elastic/apm/agent/metrics/builtin/SystemMetricsTest.java index eb9c1115b5..212f6491de 100644 --- a/apm-agent-core/src/test/java/co/elastic/apm/agent/metrics/builtin/SystemMetricsTest.java +++ b/apm-agent-core/src/test/java/co/elastic/apm/agent/metrics/builtin/SystemMetricsTest.java @@ -32,12 +32,8 @@ import org.junit.jupiter.api.condition.OS; import org.junit.jupiter.params.ParameterizedTest; import org.junit.jupiter.params.provider.CsvSource; -import org.junit.jupiter.params.provider.ValueSource; import java.io.File; -import java.io.FileWriter; -import java.io.IOException; -import java.net.URISyntaxException; import static org.assertj.core.api.Assertions.assertThat; import static org.mockito.Mockito.mock; @@ -67,78 +63,10 @@ void testSystemMetrics() throws InterruptedException { "/proc/meminfo-3.14, 556630016" }) void testFreeMemoryMeminfo(String file, long value) throws Exception { - SystemMetrics systemMetrics = createUnlimitedSystemMetrics(file); + SystemMetrics systemMetrics = new SystemMetrics(new File(getClass().getResource(file).toURI())); systemMetrics.bindTo(metricRegistry); - assertThat(metricRegistry.getGaugeValue("system.memory.actual.free", Labels.EMPTY)).isEqualTo(value); assertThat(metricRegistry.getGaugeValue("system.memory.total", Labels.EMPTY)).isEqualTo(7964778496L); - - assertThat(metricRegistry.getGaugeValue("system.process.cgroup.memory.mem.limit.bytes", Labels.EMPTY)).isEqualTo(9223372036854771712L); - } - - private SystemMetrics createUnlimitedSystemMetrics(String memInfoFile) throws URISyntaxException, IOException { - File mountInfo = new File(getClass().getResource("/proc/unlimited/memory").toURI()); - File fileTmp = File.createTempFile("temp", null); - fileTmp.deleteOnExit(); - FileWriter fw = new FileWriter(fileTmp); - fw.write("39 30 0:35 / " + mountInfo.getAbsolutePath() + " rw,nosuid,nodev,noexec,relatime shared:10 - cgroup cgroup rw,seclabel,memory\n"); - fw.close(); - - return new SystemMetrics(new File(getClass().getResource(memInfoFile).toURI()), - new File(getClass().getResource("/proc/cgroup").toURI()), - fileTmp - ); - } - - @ParameterizedTest - @CsvSource({ - "/proc/meminfo, 964778496, /proc/cgroup, /proc/limited/memory", - "/proc/meminfo, 964778496, /proc/cgroup2, /proc/sys_cgroup2" - }) - void testFreeCgroupMemoryMeminfo(String file, long value, String selfCGroup, String sysFsGroup) throws Exception { - File mountInfo = new File(getClass().getResource(sysFsGroup).toURI()); - File fileTmp = File.createTempFile("temp", null); - fileTmp.deleteOnExit(); - FileWriter fw = new FileWriter(fileTmp); - if ("/proc/sys_cgroup2".equalsIgnoreCase(sysFsGroup)) { - fw.write("30 23 0:26 / " + mountInfo.getAbsolutePath() + " rw,nosuid,nodev,noexec,relatime shared:4 - cgroup2 cgroup rw,seclabel\n"); - } - else { - fw.write("39 30 0:35 / " + mountInfo.getAbsolutePath() + " rw,nosuid,nodev,noexec,relatime shared:10 - cgroup cgroup rw,seclabel,memory\n"); - } - fw.close(); - - SystemMetrics systemMetrics = new SystemMetrics(new File(getClass().getResource(file).toURI()), - new File(getClass().getResource(selfCGroup).toURI()), - fileTmp - ); - systemMetrics.bindTo(metricRegistry); - - assertThat(metricRegistry.getGaugeValue("system.process.cgroup.memory.mem.usage.bytes", Labels.EMPTY)).isEqualTo(value); - assertThat(metricRegistry.getGaugeValue("system.process.cgroup.memory.mem.limit.bytes", Labels.EMPTY)).isEqualTo(7964778496L); - assertThat(metricRegistry.getGaugeValue("system.process.cgroup.memory.stats.inactive_file.bytes", Labels.EMPTY)).isEqualTo(10407936L); - } - @ParameterizedTest - @ValueSource(strings ={ - "39 30 0:36 / /sys/fs/cgroup/memory rw,nosuid,nodev,noexec,relatime shared:10 - cgroup cgroup rw,seclabel,memory|/sys/fs/cgroup/memory", - }) - void testCgroup1Regex(String testString) throws Exception { - String[] split = testString.split("\\|"); - SystemMetrics systemMetrics = createUnlimitedSystemMetrics("/proc/meminfo"); - assertThat(systemMetrics.applyCgroupRegex(SystemMetrics.CGROUP1_MOUNT_POINT, split[0])).isEqualTo(split[1]); - - systemMetrics.bindTo(metricRegistry); - assertThat(metricRegistry.getGaugeValue("system.process.cgroup.memory.mem.limit.bytes", Labels.EMPTY)).isEqualTo(9223372036854771712L); - } - - @ParameterizedTest - @ValueSource(strings ={ - "39 30 0:36 / /sys/fs/cgroup/memory rw,nosuid,nodev,noexec,relatime shared:4 - cgroup2 cgroup rw,seclabel|/sys/fs/cgroup/memory", - }) - void testCgroup2Regex(String testString) throws Exception { - String [] split = testString.split("\\|"); - SystemMetrics systemMetrics = createUnlimitedSystemMetrics("/proc/meminfo"); - assertThat(systemMetrics.applyCgroupRegex(SystemMetrics.CGROUP2_MOUNT_POINT, split[0])).isEqualTo(split[1]); } private void consumeCpu() { diff --git a/apm-agent-core/src/test/resources/proc/sys_cgroup2_unlimited/slice/memory.current b/apm-agent-core/src/test/resources/proc/sys_cgroup2_unlimited/slice/memory.current new file mode 100644 index 0000000000..237642783d --- /dev/null +++ b/apm-agent-core/src/test/resources/proc/sys_cgroup2_unlimited/slice/memory.current @@ -0,0 +1 @@ +964778496 diff --git a/apm-agent-core/src/test/resources/proc/sys_cgroup2_unlimited/slice/memory.max b/apm-agent-core/src/test/resources/proc/sys_cgroup2_unlimited/slice/memory.max new file mode 100644 index 0000000000..355295a05a --- /dev/null +++ b/apm-agent-core/src/test/resources/proc/sys_cgroup2_unlimited/slice/memory.max @@ -0,0 +1 @@ +max diff --git a/apm-agent-core/src/test/resources/proc/sys_cgroup2_unlimited/slice/memory.stat b/apm-agent-core/src/test/resources/proc/sys_cgroup2_unlimited/slice/memory.stat new file mode 100644 index 0000000000..bf248be317 --- /dev/null +++ b/apm-agent-core/src/test/resources/proc/sys_cgroup2_unlimited/slice/memory.stat @@ -0,0 +1,40 @@ +cache 10407936 +rss 778842112 +rss_huge 0 +shmem 0 +mapped_file 0 +dirty 0 +writeback 0 +swap 0 +pgpgin 234465 +pgpgout 41732 +pgfault 233838 +pgmajfault 0 +inactive_anon 0 +active_anon 778702848 +inactive_file 10407936 +active_file 0 +unevictable 0 +hierarchical_memory_limit 1073741824 +hierarchical_memsw_limit 2147483648 +total_cache 10407936 +total_rss 778842112 +total_rss_huge 0 +total_shmem 0 +total_mapped_file 0 +total_dirty 0 +total_writeback 0 +total_swap 0 +total_pgpgin 234465 +total_pgpgout 41732 +total_pgfault 233838 +total_pgmajfault 0 +total_inactive_anon 0 +total_active_anon 778702848 +total_inactive_file 10407936 +total_active_file 0 +total_unevictable 0 +recent_rotated_anon 231947 +recent_rotated_file 2 +recent_scanned_anon 231947 +recent_scanned_file 2622 From e343b2d19d30bf477c0ab91797df6c3ea819f9f6 Mon Sep 17 00:00:00 2001 From: George Tavares Date: Thu, 16 Jul 2020 07:57:40 -0300 Subject: [PATCH 08/11] Increase test coverage --- .../metrics/builtin/CGroupMetricsTest.java | 3 +- .../src/test/resources/proc/cgroup2 | 2 +- .../src/test/resources/proc/cgroup2_only_0 | 1 + .../test/resources/proc/cgroup2_only_memory | 1 + .../sys_cgroup2_unlimited/slice/memory.stat | 2 +- .../slice/memory.current | 1 + .../slice/memory.max | 1 + .../slice/memory.stat | 40 +++++++++++++++++++ 8 files changed, 48 insertions(+), 3 deletions(-) create mode 100644 apm-agent-core/src/test/resources/proc/cgroup2_only_0 create mode 100644 apm-agent-core/src/test/resources/proc/cgroup2_only_memory create mode 100644 apm-agent-core/src/test/resources/proc/sys_cgroup2_unlimited_stat_different_order/slice/memory.current create mode 100644 apm-agent-core/src/test/resources/proc/sys_cgroup2_unlimited_stat_different_order/slice/memory.max create mode 100644 apm-agent-core/src/test/resources/proc/sys_cgroup2_unlimited_stat_different_order/slice/memory.stat diff --git a/apm-agent-core/src/test/java/co/elastic/apm/agent/metrics/builtin/CGroupMetricsTest.java b/apm-agent-core/src/test/java/co/elastic/apm/agent/metrics/builtin/CGroupMetricsTest.java index 95fd2f5b83..6367df066d 100644 --- a/apm-agent-core/src/test/java/co/elastic/apm/agent/metrics/builtin/CGroupMetricsTest.java +++ b/apm-agent-core/src/test/java/co/elastic/apm/agent/metrics/builtin/CGroupMetricsTest.java @@ -61,7 +61,8 @@ private CGroupMetrics createUnlimitedSystemMetrics() throws URISyntaxException, @CsvSource({ "964778496, /proc/cgroup, /proc/limited/memory, 7964778496", "964778496, /proc/cgroup2, /proc/sys_cgroup2, 7964778496", - "964778496, /proc/cgroup2, /proc/sys_cgroup2_unlimited, NaN" + "964778496, /proc/cgroup2_only_0, /proc/sys_cgroup2_unlimited, NaN", // stat have different values to inactive_file and total_inactive_file + "964778496, /proc/cgroup2_only_memory, /proc/sys_cgroup2_unlimited_stat_different_order, NaN" // stat have different values to inactive_file and total_inactive_file different order }) void testFreeCgroupMemory(long value, String selfCGroup, String sysFsGroup, String memLimit) throws Exception { File mountInfo = new File(getClass().getResource(sysFsGroup).toURI()); diff --git a/apm-agent-core/src/test/resources/proc/cgroup2 b/apm-agent-core/src/test/resources/proc/cgroup2 index 5ecc3b9ec9..c4e95eb787 100644 --- a/apm-agent-core/src/test/resources/proc/cgroup2 +++ b/apm-agent-core/src/test/resources/proc/cgroup2 @@ -1,2 +1,2 @@ +0::/slicewrongdir 9:memory:/slice -0::/slice diff --git a/apm-agent-core/src/test/resources/proc/cgroup2_only_0 b/apm-agent-core/src/test/resources/proc/cgroup2_only_0 new file mode 100644 index 0000000000..93fe96524c --- /dev/null +++ b/apm-agent-core/src/test/resources/proc/cgroup2_only_0 @@ -0,0 +1 @@ +0::/slice diff --git a/apm-agent-core/src/test/resources/proc/cgroup2_only_memory b/apm-agent-core/src/test/resources/proc/cgroup2_only_memory new file mode 100644 index 0000000000..95e703b606 --- /dev/null +++ b/apm-agent-core/src/test/resources/proc/cgroup2_only_memory @@ -0,0 +1 @@ +9:memory:/slice diff --git a/apm-agent-core/src/test/resources/proc/sys_cgroup2_unlimited/slice/memory.stat b/apm-agent-core/src/test/resources/proc/sys_cgroup2_unlimited/slice/memory.stat index bf248be317..6339ecd453 100644 --- a/apm-agent-core/src/test/resources/proc/sys_cgroup2_unlimited/slice/memory.stat +++ b/apm-agent-core/src/test/resources/proc/sys_cgroup2_unlimited/slice/memory.stat @@ -12,7 +12,7 @@ pgfault 233838 pgmajfault 0 inactive_anon 0 active_anon 778702848 -inactive_file 10407936 +inactive_file 500 active_file 0 unevictable 0 hierarchical_memory_limit 1073741824 diff --git a/apm-agent-core/src/test/resources/proc/sys_cgroup2_unlimited_stat_different_order/slice/memory.current b/apm-agent-core/src/test/resources/proc/sys_cgroup2_unlimited_stat_different_order/slice/memory.current new file mode 100644 index 0000000000..237642783d --- /dev/null +++ b/apm-agent-core/src/test/resources/proc/sys_cgroup2_unlimited_stat_different_order/slice/memory.current @@ -0,0 +1 @@ +964778496 diff --git a/apm-agent-core/src/test/resources/proc/sys_cgroup2_unlimited_stat_different_order/slice/memory.max b/apm-agent-core/src/test/resources/proc/sys_cgroup2_unlimited_stat_different_order/slice/memory.max new file mode 100644 index 0000000000..355295a05a --- /dev/null +++ b/apm-agent-core/src/test/resources/proc/sys_cgroup2_unlimited_stat_different_order/slice/memory.max @@ -0,0 +1 @@ +max diff --git a/apm-agent-core/src/test/resources/proc/sys_cgroup2_unlimited_stat_different_order/slice/memory.stat b/apm-agent-core/src/test/resources/proc/sys_cgroup2_unlimited_stat_different_order/slice/memory.stat new file mode 100644 index 0000000000..af871fa058 --- /dev/null +++ b/apm-agent-core/src/test/resources/proc/sys_cgroup2_unlimited_stat_different_order/slice/memory.stat @@ -0,0 +1,40 @@ +cache 10407936 +rss 778842112 +rss_huge 0 +shmem 0 +mapped_file 0 +dirty 0 +writeback 0 +swap 0 +pgpgin 234465 +pgpgout 41732 +pgfault 233838 +pgmajfault 0 +total_inactive_file 10407936 +inactive_anon 0 +active_anon 778702848 +inactive_file 500 +active_file 0 +unevictable 0 +hierarchical_memory_limit 1073741824 +hierarchical_memsw_limit 2147483648 +total_cache 10407936 +total_rss 778842112 +total_rss_huge 0 +total_shmem 0 +total_mapped_file 0 +total_dirty 0 +total_writeback 0 +total_swap 0 +total_pgpgin 234465 +total_pgpgout 41732 +total_pgfault 233838 +total_pgmajfault 0 +total_inactive_anon 0 +total_active_anon 778702848 +total_active_file 0 +total_unevictable 0 +recent_rotated_anon 231947 +recent_rotated_file 2 +recent_scanned_anon 231947 +recent_scanned_file 2622 From 7827295524d9c17ba61ce0fc203dd46e4c2fe5ac Mon Sep 17 00:00:00 2001 From: George Tavares Date: Thu, 30 Jul 2020 18:29:45 -0300 Subject: [PATCH 09/11] Cgroup unlimited memory check and documentation --- CHANGELOG.asciidoc | 1 + .../agent/metrics/builtin/CGroupMetrics.java | 25 +++++++++----- .../metrics/builtin/CGroupMetricsTest.java | 6 ++-- docs/metrics.asciidoc | 33 +++++++++++++++++++ 4 files changed, 53 insertions(+), 12 deletions(-) diff --git a/CHANGELOG.asciidoc b/CHANGELOG.asciidoc index 6a7fe94f65..e355287f9d 100644 --- a/CHANGELOG.asciidoc +++ b/CHANGELOG.asciidoc @@ -28,6 +28,7 @@ endif::[] [float] ===== Features +* The agent now collects cgroup memory metrics (see <>) * Experimental support for runtime attachment now also for OSGi containers, JBoss, and WildFly * New mitigation of OSGi bootdelegation errors (`NoClassDefFoundError`). You can remove any `org.osgi.framework.bootdelegation` related configuration. diff --git a/apm-agent-core/src/main/java/co/elastic/apm/agent/metrics/builtin/CGroupMetrics.java b/apm-agent-core/src/main/java/co/elastic/apm/agent/metrics/builtin/CGroupMetrics.java index b2a11b310f..c365c77391 100644 --- a/apm-agent-core/src/main/java/co/elastic/apm/agent/metrics/builtin/CGroupMetrics.java +++ b/apm-agent-core/src/main/java/co/elastic/apm/agent/metrics/builtin/CGroupMetrics.java @@ -66,6 +66,8 @@ public class CGroupMetrics extends AbstractLifecycleListener { static final String CGROUP2_MAX_MEMORY = "memory.max"; static final String CGROUP2_USED_MEMORY = "memory.current"; static final String CGROUP_MEMORY_STAT = "memory.stat"; + static final String CGROUP1_UNLIMITED = "9223372036854771712"; + static final String CGROUP2_UNLIMITED = "max"; static final Pattern MEMORY_CGROUP = Pattern.compile("^\\d+\\:memory\\:.*"); static final Pattern CGROUP1_MOUNT_POINT = Pattern.compile("^\\d+? \\d+? .+? .+? (.*?) .*cgroup.*memory.*"); @@ -186,13 +188,7 @@ private CgroupFiles createCgroup2Files(String cgroupLine, File rootCgroupFsPath) String sliceSubdir = cgroupLineParts[cgroupLineParts.length - 1]; File maxMemoryFile = new File(rootCgroupFsPath, sliceSubdir + File.separatorChar + CGROUP2_MAX_MEMORY); if (maxMemoryFile.canRead()) { - try (BufferedReader maxFileReader = new BufferedReader(new FileReader(maxMemoryFile))) { - String memMaxLine = maxFileReader.readLine(); - if ("max".equalsIgnoreCase(memMaxLine)) { - // Make sure we don't send the max metric when cgroup is not bound to a memory limit - maxMemoryFile = null; - } - } + maxMemoryFile = checkUnlimitedMemory(maxMemoryFile, CGROUP2_UNLIMITED); return new CgroupFiles( maxMemoryFile, new File(rootCgroupFsPath, sliceSubdir + File.separator + CGROUP2_USED_MEMORY), @@ -203,10 +199,10 @@ private CgroupFiles createCgroup2Files(String cgroupLine, File rootCgroupFsPath) } @Nullable - private CgroupFiles createCgroup1Files(File memoryMountPath) { + private CgroupFiles createCgroup1Files(File memoryMountPath) throws IOException { File maxMemoryFile = new File(memoryMountPath, CGroupMetrics.CGROUP1_MAX_MEMORY); if (maxMemoryFile.canRead()) { - // No need for special treatment for the special "unlimited" value (0x7ffffffffffff000) - omitted by the UI + maxMemoryFile = checkUnlimitedMemory(maxMemoryFile, CGROUP1_UNLIMITED); return new CgroupFiles( maxMemoryFile, new File(memoryMountPath, CGroupMetrics.CGROUP1_USED_MEMORY), @@ -216,6 +212,17 @@ private CgroupFiles createCgroup1Files(File memoryMountPath) { return null; } + private File checkUnlimitedMemory(File maxMemoryFile, String cgroupUnlimitedConstant) throws IOException { + try(BufferedReader maxFileReader = new BufferedReader(new FileReader(maxMemoryFile))) { + String memMaxLine = maxFileReader.readLine(); + if (cgroupUnlimitedConstant.equalsIgnoreCase(memMaxLine)) { + // Make sure we don't send the max metric when cgroup is not bound to a memory limit + maxMemoryFile = null; + } + } + return maxMemoryFile; + } + @Override public void start(ElasticApmTracer tracer) { bindTo(tracer.getMetricRegistry()); diff --git a/apm-agent-core/src/test/java/co/elastic/apm/agent/metrics/builtin/CGroupMetricsTest.java b/apm-agent-core/src/test/java/co/elastic/apm/agent/metrics/builtin/CGroupMetricsTest.java index 6367df066d..ff63950f63 100644 --- a/apm-agent-core/src/test/java/co/elastic/apm/agent/metrics/builtin/CGroupMetricsTest.java +++ b/apm-agent-core/src/test/java/co/elastic/apm/agent/metrics/builtin/CGroupMetricsTest.java @@ -96,7 +96,7 @@ void testCgroup1Regex(String testString) throws Exception { assertThat(cgroupMetrics.applyCgroupRegex(CGroupMetrics.CGROUP1_MOUNT_POINT, split[0])).isEqualTo(split[1]); cgroupMetrics.bindTo(metricRegistry); - assertThat(metricRegistry.getGaugeValue("system.process.cgroup.memory.mem.limit.bytes", Labels.EMPTY)).isEqualTo(9223372036854771712L); + assertThat(metricRegistry.getGaugeValue("system.process.cgroup.memory.mem.limit.bytes", Labels.EMPTY)).isEqualTo(Double.NaN); } @ParameterizedTest @@ -114,7 +114,7 @@ void testUnlimitedCgroup1() throws Exception { CGroupMetrics cgroupMetrics = createUnlimitedSystemMetrics(); cgroupMetrics.bindTo(metricRegistry); - assertThat(metricRegistry.getGaugeValue("system.process.cgroup.memory.mem.limit.bytes", Labels.EMPTY)).isEqualTo(9223372036854771712L); + assertThat(metricRegistry.getGaugeValue("system.process.cgroup.memory.mem.limit.bytes", Labels.EMPTY)).isEqualTo(Double.NaN); assertThat(metricRegistry.getGaugeValue("system.process.cgroup.memory.mem.usage.bytes", Labels.EMPTY)).isEqualTo(964778496); assertThat(metricRegistry.getGaugeValue("system.process.cgroup.memory.stats.inactive_file.bytes", Labels.EMPTY)).isEqualTo(10407936L); } @@ -124,7 +124,7 @@ void testUnlimitedCgroup2() throws Exception { CGroupMetrics cgroupMetrics = createUnlimitedSystemMetrics(); cgroupMetrics.bindTo(metricRegistry); - assertThat(metricRegistry.getGaugeValue("system.process.cgroup.memory.mem.limit.bytes", Labels.EMPTY)).isEqualTo(9223372036854771712L); + assertThat(metricRegistry.getGaugeValue("system.process.cgroup.memory.mem.limit.bytes", Labels.EMPTY)).isEqualTo(Double.NaN); assertThat(metricRegistry.getGaugeValue("system.process.cgroup.memory.mem.usage.bytes", Labels.EMPTY)).isEqualTo(964778496); assertThat(metricRegistry.getGaugeValue("system.process.cgroup.memory.stats.inactive_file.bytes", Labels.EMPTY)).isEqualTo(10407936L); } diff --git a/docs/metrics.asciidoc b/docs/metrics.asciidoc index bb107b296c..7e228bf797 100644 --- a/docs/metrics.asciidoc +++ b/docs/metrics.asciidoc @@ -277,3 +277,36 @@ You can filter and group by these dimensions: * `span.subtype`: The sub-type of the span, for example `mysql` (optional) -- + + +*`system.process.cgroup.memory.mem.limit.bytes`*:: ++ +-- +type: long + +format: bytes + +Total memory reserved in current cgroup slice. +-- + + +*`system.process.cgroup.memory.mem.usage.bytes`*:: ++ +-- +type: long + +format: bytes + +Memory usage in current cgroup slice. +-- + + +*`system.process.cgroup.memory.stats.inactive_file.bytes`*:: ++ +-- +type: long + +format: bytes + +Inactive memory in current cgroup slice. +-- From 51e7aba721da7c6c2a45ea74eeadbac248f56d76 Mon Sep 17 00:00:00 2001 From: eyalkoren <41850454+eyalkoren@users.noreply.github.com> Date: Mon, 3 Aug 2020 15:55:45 +0300 Subject: [PATCH 10/11] Small fixes --- .../agent/metrics/builtin/CGroupMetrics.java | 14 +--- docs/metrics.asciidoc | 74 ++++++++++--------- 2 files changed, 42 insertions(+), 46 deletions(-) diff --git a/apm-agent-core/src/main/java/co/elastic/apm/agent/metrics/builtin/CGroupMetrics.java b/apm-agent-core/src/main/java/co/elastic/apm/agent/metrics/builtin/CGroupMetrics.java index c365c77391..6ad7d7e653 100644 --- a/apm-agent-core/src/main/java/co/elastic/apm/agent/metrics/builtin/CGroupMetrics.java +++ b/apm-agent-core/src/main/java/co/elastic/apm/agent/metrics/builtin/CGroupMetrics.java @@ -26,11 +26,9 @@ import co.elastic.apm.agent.context.AbstractLifecycleListener; import co.elastic.apm.agent.impl.ElasticApmTracer; -import co.elastic.apm.agent.matcher.WildcardMatcher; import co.elastic.apm.agent.metrics.DoubleSupplier; import co.elastic.apm.agent.metrics.Labels; import co.elastic.apm.agent.metrics.MetricRegistry; -import co.elastic.apm.agent.util.JmxUtils; import org.slf4j.Logger; import org.slf4j.LoggerFactory; import org.stagemonitor.util.StringUtils; @@ -40,18 +38,9 @@ import java.io.File; import java.io.FileReader; import java.io.IOException; -import java.lang.management.ManagementFactory; -import java.lang.management.OperatingSystemMXBean; -import java.lang.reflect.Method; -import java.util.Arrays; -import java.util.HashMap; -import java.util.List; -import java.util.Map; import java.util.regex.Matcher; import java.util.regex.Pattern; -import static co.elastic.apm.agent.matcher.WildcardMatcher.caseSensitiveMatcher; - /** * Record metrics related to the CGroup Usage. */ @@ -83,7 +72,7 @@ public CGroupMetrics() { this(new File(PROC_SELF_CGROUP), new File(PROC_SELF_MOUNTINFO)); } - CGroupMetrics( File procSelfCgroup, File mountInfo) { + CGroupMetrics(File procSelfCgroup, File mountInfo) { cgroupFiles = findCgroupFiles(procSelfCgroup, mountInfo); } @@ -212,6 +201,7 @@ private CgroupFiles createCgroup1Files(File memoryMountPath) throws IOException return null; } + @Nullable private File checkUnlimitedMemory(File maxMemoryFile, String cgroupUnlimitedConstant) throws IOException { try(BufferedReader maxFileReader = new BufferedReader(new FileReader(maxMemoryFile))) { String memMaxLine = maxFileReader.readLine(); diff --git a/docs/metrics.asciidoc b/docs/metrics.asciidoc index 7e228bf797..5b5555b099 100644 --- a/docs/metrics.asciidoc +++ b/docs/metrics.asciidoc @@ -21,6 +21,7 @@ Starting in Java agent version 1.11.0, it is possible to manually configure a un When multiple JVMs are running on the same host and report data for the same service, this configuration is required in order to be able to view metrics at the JVM level. * <> +* <> * <> * <> @@ -28,7 +29,7 @@ When multiple JVMs are running on the same host and report data for the same ser [[metrics-system]] === System metrics -As of version 6.6, these metrics will be visualized in the APM app. +Host metrics. As of version 6.6, these metrics will be visualized in the APM app. For more system metrics, consider installing {metricbeat-ref}/index.html[metricbeat] on your hosts. @@ -90,6 +91,44 @@ format: bytes The total virtual memory the process has. -- +[float] +[[metrics-cgroup]] +=== cgroup metrics (added in 1.18.0) + +Linux's cgroup metrics. + +*`system.process.cgroup.memory.mem.limit.bytes`*:: ++ +-- +type: long + +format: bytes + +Memory limit for current cgroup slice. +-- + + +*`system.process.cgroup.memory.mem.usage.bytes`*:: ++ +-- +type: long + +format: bytes + +Memory usage in current cgroup slice. +-- + + +*`system.process.cgroup.memory.stats.inactive_file.bytes`*:: ++ +-- +type: long + +format: bytes + +Inactive memory in current cgroup slice. +-- + [float] [[metrics-jvm]] === JVM Metrics @@ -277,36 +316,3 @@ You can filter and group by these dimensions: * `span.subtype`: The sub-type of the span, for example `mysql` (optional) -- - - -*`system.process.cgroup.memory.mem.limit.bytes`*:: -+ --- -type: long - -format: bytes - -Total memory reserved in current cgroup slice. --- - - -*`system.process.cgroup.memory.mem.usage.bytes`*:: -+ --- -type: long - -format: bytes - -Memory usage in current cgroup slice. --- - - -*`system.process.cgroup.memory.stats.inactive_file.bytes`*:: -+ --- -type: long - -format: bytes - -Inactive memory in current cgroup slice. --- From 1a4e2dbdb17fd85f6983332d8e1a86f6ddaa7283 Mon Sep 17 00:00:00 2001 From: Sylvain Juge Date: Tue, 18 Aug 2020 12:26:11 +0200 Subject: [PATCH 11/11] minor style changes --- .../agent/metrics/builtin/CGroupMetrics.java | 170 +++++++++--------- .../metrics/builtin/CGroupMetricsTest.java | 2 +- 2 files changed, 88 insertions(+), 84 deletions(-) diff --git a/apm-agent-core/src/main/java/co/elastic/apm/agent/metrics/builtin/CGroupMetrics.java b/apm-agent-core/src/main/java/co/elastic/apm/agent/metrics/builtin/CGroupMetrics.java index 6ad7d7e653..3f410503eb 100644 --- a/apm-agent-core/src/main/java/co/elastic/apm/agent/metrics/builtin/CGroupMetrics.java +++ b/apm-agent-core/src/main/java/co/elastic/apm/agent/metrics/builtin/CGroupMetrics.java @@ -43,22 +43,24 @@ /** * Record metrics related to the CGroup Usage. + *

+ * Implements the cgroup metrics spec - https://github.com/elastic/apm/blob/master/docs/agents/agent-development.md#cgroup-metrics */ public class CGroupMetrics extends AbstractLifecycleListener { - public static final String PROC_SELF_CGROUP = "/proc/self/cgroup"; - public static final String PROC_SELF_MOUNTINFO = "/proc/self/mountinfo"; - public static final String DEFAULT_SYS_FS_CGROUP = "/sys/fs/cgroup"; + private static final String PROC_SELF_CGROUP = "/proc/self/cgroup"; + private static final String PROC_SELF_MOUNTINFO = "/proc/self/mountinfo"; + private static final String DEFAULT_SYS_FS_CGROUP = "/sys/fs/cgroup"; - static final String CGROUP1_MAX_MEMORY = "memory.limit_in_bytes"; - static final String CGROUP1_USED_MEMORY = "memory.usage_in_bytes"; - static final String CGROUP2_MAX_MEMORY = "memory.max"; - static final String CGROUP2_USED_MEMORY = "memory.current"; - static final String CGROUP_MEMORY_STAT = "memory.stat"; - static final String CGROUP1_UNLIMITED = "9223372036854771712"; - static final String CGROUP2_UNLIMITED = "max"; + private static final String CGROUP1_MAX_MEMORY = "memory.limit_in_bytes"; + private static final String CGROUP1_USED_MEMORY = "memory.usage_in_bytes"; + private static final String CGROUP2_MAX_MEMORY = "memory.max"; + private static final String CGROUP2_USED_MEMORY = "memory.current"; + private static final String CGROUP_MEMORY_STAT = "memory.stat"; + private static final String CGROUP1_UNLIMITED = "9223372036854771712"; + private static final String CGROUP2_UNLIMITED = "max"; - static final Pattern MEMORY_CGROUP = Pattern.compile("^\\d+\\:memory\\:.*"); + static final Pattern MEMORY_CGROUP = Pattern.compile("^\\d+:memory:.*"); static final Pattern CGROUP1_MOUNT_POINT = Pattern.compile("^\\d+? \\d+? .+? .+? (.*?) .*cgroup.*memory.*"); static final Pattern CGROUP2_MOUNT_POINT = Pattern.compile("^\\d+? \\d+? .+? .+? (.*?) .*cgroup2.*cgroup.*"); @@ -77,88 +79,90 @@ public CGroupMetrics() { } /** - * Implementing the cgroup metrics spec - https://github.com/elastic/apm/blob/master/docs/agents/agent-development.md#cgroup-metrics + * Finds cgroup files (if any) * * @param procSelfCgroup /proc/self/cgroup file * @param mountInfo /proc/self/mountinfo file * @return a holder for the memory cgroup files if found or {@code null} if not found */ @Nullable - public CgroupFiles findCgroupFiles(File procSelfCgroup, File mountInfo) { - if (procSelfCgroup.canRead()) { - String cgroupLine = null; - try (BufferedReader fileReader = new BufferedReader(new FileReader(procSelfCgroup))) { - String currentLine = fileReader.readLine(); - while (currentLine != null) { - if (cgroupLine == null && currentLine.startsWith("0:")) { - cgroupLine = currentLine; - } - if (MEMORY_CGROUP.matcher(currentLine).matches()) { - cgroupLine = currentLine; - break; - } - currentLine = fileReader.readLine(); + private CgroupFiles findCgroupFiles(File procSelfCgroup, File mountInfo) { + if (!procSelfCgroup.canRead()) { + logger.debug("Cannot find/read /proc/self/cgroup file. Cgroup metrics will not be reported."); + return null; + } + + String cgroupLine = null; + try (BufferedReader fileReader = new BufferedReader(new FileReader(procSelfCgroup))) { + String currentLine = fileReader.readLine(); + while (currentLine != null) { + if (cgroupLine == null && currentLine.startsWith("0:")) { + cgroupLine = currentLine; + } + if (MEMORY_CGROUP.matcher(currentLine).matches()) { + cgroupLine = currentLine; + break; } + currentLine = fileReader.readLine(); + } - if (cgroupLine != null) { - CgroupFiles cgroupFiles; - - // Try to discover the cgroup fs path from the mountinfo file - if (mountInfo.canRead()) { - String mountLine = null; - try (BufferedReader fileMountInfoReader = new BufferedReader(new FileReader(mountInfo))) { - mountLine = fileMountInfoReader.readLine(); - while (mountLine != null) { - // cgroup v2 - String rootCgroupFsPath = applyCgroupRegex(CGROUP2_MOUNT_POINT, mountLine); - if (rootCgroupFsPath != null) { - cgroupFiles = createCgroup2Files(cgroupLine, new File(rootCgroupFsPath)); - if (cgroupFiles != null) { - return cgroupFiles; - } - } + if (cgroupLine == null) { + logger.warn("No /proc/self/cgroup file line matched the tested patterns. Cgroup metrics will not be reported."); + return null; + } - // cgroup v1 - String memoryMountPath = applyCgroupRegex(CGROUP1_MOUNT_POINT, mountLine); - if (memoryMountPath != null) { - cgroupFiles = createCgroup1Files( - new File(memoryMountPath) - ); - if (cgroupFiles != null) { - return cgroupFiles; - } - } + CgroupFiles cgroupFiles; - mountLine = fileMountInfoReader.readLine(); + // Try to discover the cgroup fs path from the mountinfo file + if (mountInfo.canRead()) { + String mountLine = null; + try (BufferedReader fileMountInfoReader = new BufferedReader(new FileReader(mountInfo))) { + mountLine = fileMountInfoReader.readLine(); + while (mountLine != null) { + // cgroup v2 + String rootCgroupFsPath = applyCgroupRegex(CGROUP2_MOUNT_POINT, mountLine); + if (rootCgroupFsPath != null) { + cgroupFiles = createCgroup2Files(cgroupLine, new File(rootCgroupFsPath)); + if (cgroupFiles != null) { + return cgroupFiles; } - } catch (Exception e) { - logger.info("Failed to discover memory mount files path based on mountinfo line '{}'.", mountLine); } - } else { - logger.info("Failed to find/read /proc/self/mountinfo file. Looking for memory files in /sys/fs/cgroup."); - } - // Failed to auto-discover the cgroup fs path from mountinfo, fall back to /sys/fs/cgroup - // cgroup v2 - cgroupFiles = createCgroup2Files(cgroupLine, new File(DEFAULT_SYS_FS_CGROUP)); - if (cgroupFiles != null) { - return cgroupFiles; - } - // cgroup v1 - cgroupFiles = createCgroup1Files(new File(DEFAULT_SYS_FS_CGROUP + File.pathSeparator + "memory")); - if (cgroupFiles != null) { - return cgroupFiles; + // cgroup v1 + String memoryMountPath = applyCgroupRegex(CGROUP1_MOUNT_POINT, mountLine); + if (memoryMountPath != null) { + cgroupFiles = createCgroup1Files(new File(memoryMountPath)); + if (cgroupFiles != null) { + return cgroupFiles; + } + } + + mountLine = fileMountInfoReader.readLine(); } - } else { - logger.warn("No /proc/self/cgroup file line matched the tested patterns. Cgroup metrics will not be reported."); + } catch (Exception e) { + logger.info("Failed to discover memory mount files path based on mountinfo line '{}'.", mountLine); } - } catch (Exception e) { - logger.error("Failed to discover memory mount files path based on cgroup line '" + cgroupLine + - "'. Cgroup metrics will not be reported,", e); + } else { + logger.info("Failed to find/read /proc/self/mountinfo file. Looking for memory files in /sys/fs/cgroup."); } - } else { - logger.debug("Cannot find/read /proc/self/cgroup file. Cgroup metrics will not be reported."); + + // Failed to auto-discover the cgroup fs path from mountinfo, fall back to /sys/fs/cgroup + // cgroup v2 + cgroupFiles = createCgroup2Files(cgroupLine, new File(DEFAULT_SYS_FS_CGROUP)); + if (cgroupFiles != null) { + return cgroupFiles; + } + // cgroup v1 + cgroupFiles = createCgroup1Files(new File(DEFAULT_SYS_FS_CGROUP + File.pathSeparator + "memory")); + if (cgroupFiles != null) { + return cgroupFiles; + } + + } catch (Exception e) { + logger.error("Failed to discover memory mount files path based on cgroup line '" + cgroupLine + + "'. Cgroup metrics will not be reported", e); } + return null; } @@ -177,7 +181,7 @@ private CgroupFiles createCgroup2Files(String cgroupLine, File rootCgroupFsPath) String sliceSubdir = cgroupLineParts[cgroupLineParts.length - 1]; File maxMemoryFile = new File(rootCgroupFsPath, sliceSubdir + File.separatorChar + CGROUP2_MAX_MEMORY); if (maxMemoryFile.canRead()) { - maxMemoryFile = checkUnlimitedMemory(maxMemoryFile, CGROUP2_UNLIMITED); + maxMemoryFile = getMaxMemoryFile(maxMemoryFile, CGROUP2_UNLIMITED); return new CgroupFiles( maxMemoryFile, new File(rootCgroupFsPath, sliceSubdir + File.separator + CGROUP2_USED_MEMORY), @@ -191,7 +195,7 @@ private CgroupFiles createCgroup2Files(String cgroupLine, File rootCgroupFsPath) private CgroupFiles createCgroup1Files(File memoryMountPath) throws IOException { File maxMemoryFile = new File(memoryMountPath, CGroupMetrics.CGROUP1_MAX_MEMORY); if (maxMemoryFile.canRead()) { - maxMemoryFile = checkUnlimitedMemory(maxMemoryFile, CGROUP1_UNLIMITED); + maxMemoryFile = getMaxMemoryFile(maxMemoryFile, CGROUP1_UNLIMITED); return new CgroupFiles( maxMemoryFile, new File(memoryMountPath, CGroupMetrics.CGROUP1_USED_MEMORY), @@ -202,7 +206,7 @@ private CgroupFiles createCgroup1Files(File memoryMountPath) throws IOException } @Nullable - private File checkUnlimitedMemory(File maxMemoryFile, String cgroupUnlimitedConstant) throws IOException { + private File getMaxMemoryFile(File maxMemoryFile, String cgroupUnlimitedConstant) throws IOException { try(BufferedReader maxFileReader = new BufferedReader(new FileReader(maxMemoryFile))) { String memMaxLine = maxFileReader.readLine(); if (cgroupUnlimitedConstant.equalsIgnoreCase(memMaxLine)) { @@ -275,12 +279,12 @@ public double get() { } } - public static class CgroupFiles { + private static class CgroupFiles { @Nullable // may be null if memory mount is found for the cgroup, but memory is unlimited - protected File maxMemoryFile; - protected File usedMemoryFile; - protected File statMemoryFile; + private final File maxMemoryFile; + private final File usedMemoryFile; + private final File statMemoryFile; public CgroupFiles(@Nullable File maxMemoryFile, File usedMemoryFile, File statMemoryFile) { this.maxMemoryFile = maxMemoryFile; diff --git a/apm-agent-core/src/test/java/co/elastic/apm/agent/metrics/builtin/CGroupMetricsTest.java b/apm-agent-core/src/test/java/co/elastic/apm/agent/metrics/builtin/CGroupMetricsTest.java index ff63950f63..929f08af5e 100644 --- a/apm-agent-core/src/test/java/co/elastic/apm/agent/metrics/builtin/CGroupMetricsTest.java +++ b/apm-agent-core/src/test/java/co/elastic/apm/agent/metrics/builtin/CGroupMetricsTest.java @@ -42,7 +42,7 @@ class CGroupMetricsTest { - private MetricRegistry metricRegistry = new MetricRegistry(mock(ReporterConfiguration.class)); + private final MetricRegistry metricRegistry = new MetricRegistry(mock(ReporterConfiguration.class)); private CGroupMetrics createUnlimitedSystemMetrics() throws URISyntaxException, IOException { File mountInfo = new File(getClass().getResource("/proc/unlimited/memory").toURI());