diff --git a/CHANGELOG.asciidoc b/CHANGELOG.asciidoc index 6a7fe94f65..e355287f9d 100644 --- a/CHANGELOG.asciidoc +++ b/CHANGELOG.asciidoc @@ -28,6 +28,7 @@ endif::[] [float] ===== Features +* The agent now collects cgroup memory metrics (see <>) * Experimental support for runtime attachment now also for OSGi containers, JBoss, and WildFly * New mitigation of OSGi bootdelegation errors (`NoClassDefFoundError`). You can remove any `org.osgi.framework.bootdelegation` related configuration. diff --git a/apm-agent-core/src/main/java/co/elastic/apm/agent/metrics/builtin/CGroupMetrics.java b/apm-agent-core/src/main/java/co/elastic/apm/agent/metrics/builtin/CGroupMetrics.java new file mode 100644 index 0000000000..3f410503eb --- /dev/null +++ b/apm-agent-core/src/main/java/co/elastic/apm/agent/metrics/builtin/CGroupMetrics.java @@ -0,0 +1,308 @@ +/*- + * #%L + * Elastic APM Java agent + * %% + * Copyright (C) 2018 - 2020 Elastic and contributors + * %% + * Licensed to Elasticsearch B.V. under one or more contributor + * license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright + * ownership. Elasticsearch B.V. licenses this file to you under + * the Apache License, Version 2.0 (the "License"); you may + * not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + * #L% + */ +package co.elastic.apm.agent.metrics.builtin; + +import co.elastic.apm.agent.context.AbstractLifecycleListener; +import co.elastic.apm.agent.impl.ElasticApmTracer; +import co.elastic.apm.agent.metrics.DoubleSupplier; +import co.elastic.apm.agent.metrics.Labels; +import co.elastic.apm.agent.metrics.MetricRegistry; +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; +import org.stagemonitor.util.StringUtils; + +import javax.annotation.Nullable; +import java.io.BufferedReader; +import java.io.File; +import java.io.FileReader; +import java.io.IOException; +import java.util.regex.Matcher; +import java.util.regex.Pattern; + +/** + * Record metrics related to the CGroup Usage. + *

+ * Implements the cgroup metrics spec - https://github.com/elastic/apm/blob/master/docs/agents/agent-development.md#cgroup-metrics + */ +public class CGroupMetrics extends AbstractLifecycleListener { + + private static final String PROC_SELF_CGROUP = "/proc/self/cgroup"; + private static final String PROC_SELF_MOUNTINFO = "/proc/self/mountinfo"; + private static final String DEFAULT_SYS_FS_CGROUP = "/sys/fs/cgroup"; + + private static final String CGROUP1_MAX_MEMORY = "memory.limit_in_bytes"; + private static final String CGROUP1_USED_MEMORY = "memory.usage_in_bytes"; + private static final String CGROUP2_MAX_MEMORY = "memory.max"; + private static final String CGROUP2_USED_MEMORY = "memory.current"; + private static final String CGROUP_MEMORY_STAT = "memory.stat"; + private static final String CGROUP1_UNLIMITED = "9223372036854771712"; + private static final String CGROUP2_UNLIMITED = "max"; + + static final Pattern MEMORY_CGROUP = Pattern.compile("^\\d+:memory:.*"); + static final Pattern CGROUP1_MOUNT_POINT = Pattern.compile("^\\d+? \\d+? .+? .+? (.*?) .*cgroup.*memory.*"); + static final Pattern CGROUP2_MOUNT_POINT = Pattern.compile("^\\d+? \\d+? .+? .+? (.*?) .*cgroup2.*cgroup.*"); + + private static final Logger logger = LoggerFactory.getLogger(CGroupMetrics.class); + + + @Nullable + private final CgroupFiles cgroupFiles; + + public CGroupMetrics() { + this(new File(PROC_SELF_CGROUP), new File(PROC_SELF_MOUNTINFO)); + } + + CGroupMetrics(File procSelfCgroup, File mountInfo) { + cgroupFiles = findCgroupFiles(procSelfCgroup, mountInfo); + } + + /** + * Finds cgroup files (if any) + * + * @param procSelfCgroup /proc/self/cgroup file + * @param mountInfo /proc/self/mountinfo file + * @return a holder for the memory cgroup files if found or {@code null} if not found + */ + @Nullable + private CgroupFiles findCgroupFiles(File procSelfCgroup, File mountInfo) { + if (!procSelfCgroup.canRead()) { + logger.debug("Cannot find/read /proc/self/cgroup file. Cgroup metrics will not be reported."); + return null; + } + + String cgroupLine = null; + try (BufferedReader fileReader = new BufferedReader(new FileReader(procSelfCgroup))) { + String currentLine = fileReader.readLine(); + while (currentLine != null) { + if (cgroupLine == null && currentLine.startsWith("0:")) { + cgroupLine = currentLine; + } + if (MEMORY_CGROUP.matcher(currentLine).matches()) { + cgroupLine = currentLine; + break; + } + currentLine = fileReader.readLine(); + } + + if (cgroupLine == null) { + logger.warn("No /proc/self/cgroup file line matched the tested patterns. Cgroup metrics will not be reported."); + return null; + } + + CgroupFiles cgroupFiles; + + // Try to discover the cgroup fs path from the mountinfo file + if (mountInfo.canRead()) { + String mountLine = null; + try (BufferedReader fileMountInfoReader = new BufferedReader(new FileReader(mountInfo))) { + mountLine = fileMountInfoReader.readLine(); + while (mountLine != null) { + // cgroup v2 + String rootCgroupFsPath = applyCgroupRegex(CGROUP2_MOUNT_POINT, mountLine); + if (rootCgroupFsPath != null) { + cgroupFiles = createCgroup2Files(cgroupLine, new File(rootCgroupFsPath)); + if (cgroupFiles != null) { + return cgroupFiles; + } + } + + // cgroup v1 + String memoryMountPath = applyCgroupRegex(CGROUP1_MOUNT_POINT, mountLine); + if (memoryMountPath != null) { + cgroupFiles = createCgroup1Files(new File(memoryMountPath)); + if (cgroupFiles != null) { + return cgroupFiles; + } + } + + mountLine = fileMountInfoReader.readLine(); + } + } catch (Exception e) { + logger.info("Failed to discover memory mount files path based on mountinfo line '{}'.", mountLine); + } + } else { + logger.info("Failed to find/read /proc/self/mountinfo file. Looking for memory files in /sys/fs/cgroup."); + } + + // Failed to auto-discover the cgroup fs path from mountinfo, fall back to /sys/fs/cgroup + // cgroup v2 + cgroupFiles = createCgroup2Files(cgroupLine, new File(DEFAULT_SYS_FS_CGROUP)); + if (cgroupFiles != null) { + return cgroupFiles; + } + // cgroup v1 + cgroupFiles = createCgroup1Files(new File(DEFAULT_SYS_FS_CGROUP + File.pathSeparator + "memory")); + if (cgroupFiles != null) { + return cgroupFiles; + } + + } catch (Exception e) { + logger.error("Failed to discover memory mount files path based on cgroup line '" + cgroupLine + + "'. Cgroup metrics will not be reported", e); + } + + return null; + } + + @Nullable + String applyCgroupRegex(Pattern regex, String mountLine) { + Matcher matcher = regex.matcher(mountLine); + if (matcher.matches()) { + return matcher.group(1); + } + return null; + } + + @Nullable + private CgroupFiles createCgroup2Files(String cgroupLine, File rootCgroupFsPath) throws IOException { + final String[] cgroupLineParts = StringUtils.split(cgroupLine, ':'); + String sliceSubdir = cgroupLineParts[cgroupLineParts.length - 1]; + File maxMemoryFile = new File(rootCgroupFsPath, sliceSubdir + File.separatorChar + CGROUP2_MAX_MEMORY); + if (maxMemoryFile.canRead()) { + maxMemoryFile = getMaxMemoryFile(maxMemoryFile, CGROUP2_UNLIMITED); + return new CgroupFiles( + maxMemoryFile, + new File(rootCgroupFsPath, sliceSubdir + File.separator + CGROUP2_USED_MEMORY), + new File(rootCgroupFsPath, sliceSubdir + File.separator + CGROUP_MEMORY_STAT) + ); + } + return null; + } + + @Nullable + private CgroupFiles createCgroup1Files(File memoryMountPath) throws IOException { + File maxMemoryFile = new File(memoryMountPath, CGroupMetrics.CGROUP1_MAX_MEMORY); + if (maxMemoryFile.canRead()) { + maxMemoryFile = getMaxMemoryFile(maxMemoryFile, CGROUP1_UNLIMITED); + return new CgroupFiles( + maxMemoryFile, + new File(memoryMountPath, CGroupMetrics.CGROUP1_USED_MEMORY), + new File(memoryMountPath, CGroupMetrics.CGROUP_MEMORY_STAT) + ); + } + return null; + } + + @Nullable + private File getMaxMemoryFile(File maxMemoryFile, String cgroupUnlimitedConstant) throws IOException { + try(BufferedReader maxFileReader = new BufferedReader(new FileReader(maxMemoryFile))) { + String memMaxLine = maxFileReader.readLine(); + if (cgroupUnlimitedConstant.equalsIgnoreCase(memMaxLine)) { + // Make sure we don't send the max metric when cgroup is not bound to a memory limit + maxMemoryFile = null; + } + } + return maxMemoryFile; + } + + @Override + public void start(ElasticApmTracer tracer) { + bindTo(tracer.getMetricRegistry()); + } + + void bindTo(MetricRegistry metricRegistry) { + if (cgroupFiles != null) { + metricRegistry.addUnlessNan("system.process.cgroup.memory.stats.inactive_file.bytes", Labels.EMPTY, new DoubleSupplier() { + @Override + public double get() { + try (BufferedReader fileReaderStatFile = new BufferedReader(new FileReader(cgroupFiles.getStatMemoryFile()))) { + String statLine = fileReaderStatFile.readLine(); + String inactiveBytes = null; + while (statLine != null) { + final String[] statLineSplit = StringUtils.split(statLine, ' '); + if (statLineSplit.length > 1) { + if ("total_inactive_file".equals(statLineSplit[0])) { + inactiveBytes = statLineSplit[1]; + break; + } else if ("inactive_file".equals(statLineSplit[0])) { + inactiveBytes = statLineSplit[1]; + } + } + statLine = fileReaderStatFile.readLine(); + } + return inactiveBytes != null ? Long.parseLong(inactiveBytes) : Double.NaN; + } catch (Exception e) { + logger.debug("Failed to read " + cgroupFiles.getStatMemoryFile().getAbsolutePath() + " file", e); + return Double.NaN; + } + } + }); + + metricRegistry.addUnlessNan("system.process.cgroup.memory.mem.usage.bytes", Labels.EMPTY, new DoubleSupplier() { + @Override + public double get() { + try (BufferedReader fileReaderMemoryUsed = new BufferedReader(new FileReader(cgroupFiles.getUsedMemoryFile()))) { + return Long.parseLong(fileReaderMemoryUsed.readLine()); + } catch (Exception e) { + logger.debug("Failed to read " + cgroupFiles.getUsedMemoryFile().getAbsolutePath() + " file", e); + return Double.NaN; + } + } + }); + + final File maxMemoryFile = cgroupFiles.getMaxMemoryFile(); + if (maxMemoryFile != null) { + metricRegistry.addUnlessNan("system.process.cgroup.memory.mem.limit.bytes", Labels.EMPTY, new DoubleSupplier() { + @Override + public double get() { + try (BufferedReader fileReaderMemoryMax = new BufferedReader(new FileReader(maxMemoryFile))) { + return Long.parseLong(fileReaderMemoryMax.readLine()); + } catch (Exception e) { + logger.debug("Failed to read " + maxMemoryFile + " file", e); + return Double.NaN; + } + } + }); + } + } + } + + private static class CgroupFiles { + + @Nullable // may be null if memory mount is found for the cgroup, but memory is unlimited + private final File maxMemoryFile; + private final File usedMemoryFile; + private final File statMemoryFile; + + public CgroupFiles(@Nullable File maxMemoryFile, File usedMemoryFile, File statMemoryFile) { + this.maxMemoryFile = maxMemoryFile; + this.usedMemoryFile = usedMemoryFile; + this.statMemoryFile = statMemoryFile; + } + + @Nullable + public File getMaxMemoryFile() { + return maxMemoryFile; + } + + public File getUsedMemoryFile() { + return usedMemoryFile; + } + + public File getStatMemoryFile() { + return statMemoryFile; + } + } +} diff --git a/apm-agent-core/src/main/resources/META-INF/services/co.elastic.apm.agent.context.LifecycleListener b/apm-agent-core/src/main/resources/META-INF/services/co.elastic.apm.agent.context.LifecycleListener index fbe189ad8f..34569638d0 100644 --- a/apm-agent-core/src/main/resources/META-INF/services/co.elastic.apm.agent.context.LifecycleListener +++ b/apm-agent-core/src/main/resources/META-INF/services/co.elastic.apm.agent.context.LifecycleListener @@ -2,6 +2,7 @@ co.elastic.apm.agent.configuration.StartupInfo co.elastic.apm.agent.bci.MatcherTimerLifecycleListener co.elastic.apm.agent.metrics.builtin.JvmMemoryMetrics co.elastic.apm.agent.metrics.builtin.SystemMetrics +co.elastic.apm.agent.metrics.builtin.CGroupMetrics co.elastic.apm.agent.metrics.builtin.JvmGcMetrics co.elastic.apm.agent.metrics.builtin.ThreadMetrics co.elastic.apm.agent.impl.circuitbreaker.CircuitBreaker diff --git a/apm-agent-core/src/test/java/co/elastic/apm/agent/metrics/builtin/CGroupMetricsTest.java b/apm-agent-core/src/test/java/co/elastic/apm/agent/metrics/builtin/CGroupMetricsTest.java new file mode 100644 index 0000000000..929f08af5e --- /dev/null +++ b/apm-agent-core/src/test/java/co/elastic/apm/agent/metrics/builtin/CGroupMetricsTest.java @@ -0,0 +1,132 @@ +/*- + * #%L + * Elastic APM Java agent + * %% + * Copyright (C) 2018 - 2020 Elastic and contributors + * %% + * Licensed to Elasticsearch B.V. under one or more contributor + * license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright + * ownership. Elasticsearch B.V. licenses this file to you under + * the Apache License, Version 2.0 (the "License"); you may + * not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + * #L% + */ +package co.elastic.apm.agent.metrics.builtin; + +import co.elastic.apm.agent.metrics.Labels; +import co.elastic.apm.agent.metrics.MetricRegistry; +import co.elastic.apm.agent.report.ReporterConfiguration; +import org.junit.jupiter.api.Test; +import org.junit.jupiter.params.ParameterizedTest; +import org.junit.jupiter.params.provider.CsvSource; +import org.junit.jupiter.params.provider.ValueSource; + +import java.io.File; +import java.io.FileWriter; +import java.io.IOException; +import java.net.URISyntaxException; + +import static org.assertj.core.api.Assertions.assertThat; +import static org.mockito.Mockito.mock; + +class CGroupMetricsTest { + + private final MetricRegistry metricRegistry = new MetricRegistry(mock(ReporterConfiguration.class)); + + private CGroupMetrics createUnlimitedSystemMetrics() throws URISyntaxException, IOException { + File mountInfo = new File(getClass().getResource("/proc/unlimited/memory").toURI()); + File fileTmp = File.createTempFile("temp", null); + fileTmp.deleteOnExit(); + FileWriter fw = new FileWriter(fileTmp); + fw.write("39 30 0:35 / " + mountInfo.getAbsolutePath() + " rw,nosuid,nodev,noexec,relatime shared:10 - cgroup cgroup rw,seclabel,memory\n"); + fw.close(); + + return new CGroupMetrics(new File(getClass().getResource("/proc/cgroup").toURI()), + fileTmp + ); + } + + @ParameterizedTest + @CsvSource({ + "964778496, /proc/cgroup, /proc/limited/memory, 7964778496", + "964778496, /proc/cgroup2, /proc/sys_cgroup2, 7964778496", + "964778496, /proc/cgroup2_only_0, /proc/sys_cgroup2_unlimited, NaN", // stat have different values to inactive_file and total_inactive_file + "964778496, /proc/cgroup2_only_memory, /proc/sys_cgroup2_unlimited_stat_different_order, NaN" // stat have different values to inactive_file and total_inactive_file different order + }) + void testFreeCgroupMemory(long value, String selfCGroup, String sysFsGroup, String memLimit) throws Exception { + File mountInfo = new File(getClass().getResource(sysFsGroup).toURI()); + File fileTmp = File.createTempFile("temp", null); + fileTmp.deleteOnExit(); + FileWriter fw = new FileWriter(fileTmp); + if (sysFsGroup.startsWith("/proc/sys_cgroup2")) { + fw.write("30 23 0:26 / " + mountInfo.getAbsolutePath() + " rw,nosuid,nodev,noexec,relatime shared:4 - cgroup2 cgroup rw,seclabel\n"); + } + else { + fw.write("39 30 0:35 / " + mountInfo.getAbsolutePath() + " rw,nosuid,nodev,noexec,relatime shared:10 - cgroup cgroup rw,seclabel,memory\n"); + } + fw.close(); + + CGroupMetrics cgroupMetrics = new CGroupMetrics(new File(getClass().getResource(selfCGroup).toURI()), + fileTmp + ); + cgroupMetrics.bindTo(metricRegistry); + + assertThat(metricRegistry.getGaugeValue("system.process.cgroup.memory.mem.usage.bytes", Labels.EMPTY)).isEqualTo(value); + assertThat(metricRegistry.getGaugeValue("system.process.cgroup.memory.mem.limit.bytes", Labels.EMPTY)).isEqualTo(Double.valueOf(memLimit)); + assertThat(metricRegistry.getGaugeValue("system.process.cgroup.memory.stats.inactive_file.bytes", Labels.EMPTY)).isEqualTo(10407936L); + } + @ParameterizedTest + @ValueSource(strings ={ + "39 30 0:36 / /sys/fs/cgroup/memory rw,nosuid,nodev,noexec,relatime shared:10 - cgroup cgroup rw,seclabel,memory|/sys/fs/cgroup/memory", + }) + void testCgroup1Regex(String testString) throws Exception { + String[] split = testString.split("\\|"); + CGroupMetrics cgroupMetrics = createUnlimitedSystemMetrics(); + assertThat(cgroupMetrics.applyCgroupRegex(CGroupMetrics.CGROUP1_MOUNT_POINT, split[0])).isEqualTo(split[1]); + + cgroupMetrics.bindTo(metricRegistry); + assertThat(metricRegistry.getGaugeValue("system.process.cgroup.memory.mem.limit.bytes", Labels.EMPTY)).isEqualTo(Double.NaN); + } + + @ParameterizedTest + @ValueSource(strings ={ + "39 30 0:36 / /sys/fs/cgroup/memory rw,nosuid,nodev,noexec,relatime shared:4 - cgroup2 cgroup rw,seclabel|/sys/fs/cgroup/memory", + }) + void testCgroup2Regex(String testString) throws Exception { + String [] split = testString.split("\\|"); + CGroupMetrics cgroupMetrics = createUnlimitedSystemMetrics(); + assertThat(cgroupMetrics.applyCgroupRegex(CGroupMetrics.CGROUP2_MOUNT_POINT, split[0])).isEqualTo(split[1]); + } + + @Test + void testUnlimitedCgroup1() throws Exception { + CGroupMetrics cgroupMetrics = createUnlimitedSystemMetrics(); + cgroupMetrics.bindTo(metricRegistry); + + assertThat(metricRegistry.getGaugeValue("system.process.cgroup.memory.mem.limit.bytes", Labels.EMPTY)).isEqualTo(Double.NaN); + assertThat(metricRegistry.getGaugeValue("system.process.cgroup.memory.mem.usage.bytes", Labels.EMPTY)).isEqualTo(964778496); + assertThat(metricRegistry.getGaugeValue("system.process.cgroup.memory.stats.inactive_file.bytes", Labels.EMPTY)).isEqualTo(10407936L); + } + + @Test + void testUnlimitedCgroup2() throws Exception { + CGroupMetrics cgroupMetrics = createUnlimitedSystemMetrics(); + cgroupMetrics.bindTo(metricRegistry); + + assertThat(metricRegistry.getGaugeValue("system.process.cgroup.memory.mem.limit.bytes", Labels.EMPTY)).isEqualTo(Double.NaN); + assertThat(metricRegistry.getGaugeValue("system.process.cgroup.memory.mem.usage.bytes", Labels.EMPTY)).isEqualTo(964778496); + assertThat(metricRegistry.getGaugeValue("system.process.cgroup.memory.stats.inactive_file.bytes", Labels.EMPTY)).isEqualTo(10407936L); + } + +} diff --git a/apm-agent-core/src/test/resources/proc/cgroup b/apm-agent-core/src/test/resources/proc/cgroup new file mode 100644 index 0000000000..eb1f1ab1b2 --- /dev/null +++ b/apm-agent-core/src/test/resources/proc/cgroup @@ -0,0 +1,2 @@ +9:memory:/ +0::/ diff --git a/apm-agent-core/src/test/resources/proc/cgroup2 b/apm-agent-core/src/test/resources/proc/cgroup2 new file mode 100644 index 0000000000..c4e95eb787 --- /dev/null +++ b/apm-agent-core/src/test/resources/proc/cgroup2 @@ -0,0 +1,2 @@ +0::/slicewrongdir +9:memory:/slice diff --git a/apm-agent-core/src/test/resources/proc/cgroup2_only_0 b/apm-agent-core/src/test/resources/proc/cgroup2_only_0 new file mode 100644 index 0000000000..93fe96524c --- /dev/null +++ b/apm-agent-core/src/test/resources/proc/cgroup2_only_0 @@ -0,0 +1 @@ +0::/slice diff --git a/apm-agent-core/src/test/resources/proc/cgroup2_only_memory b/apm-agent-core/src/test/resources/proc/cgroup2_only_memory new file mode 100644 index 0000000000..95e703b606 --- /dev/null +++ b/apm-agent-core/src/test/resources/proc/cgroup2_only_memory @@ -0,0 +1 @@ +9:memory:/slice diff --git a/apm-agent-core/src/test/resources/proc/limited/memory/memory.limit_in_bytes b/apm-agent-core/src/test/resources/proc/limited/memory/memory.limit_in_bytes new file mode 100644 index 0000000000..1df8dd444d --- /dev/null +++ b/apm-agent-core/src/test/resources/proc/limited/memory/memory.limit_in_bytes @@ -0,0 +1 @@ +7964778496 diff --git a/apm-agent-core/src/test/resources/proc/limited/memory/memory.stat b/apm-agent-core/src/test/resources/proc/limited/memory/memory.stat new file mode 100644 index 0000000000..bf248be317 --- /dev/null +++ b/apm-agent-core/src/test/resources/proc/limited/memory/memory.stat @@ -0,0 +1,40 @@ +cache 10407936 +rss 778842112 +rss_huge 0 +shmem 0 +mapped_file 0 +dirty 0 +writeback 0 +swap 0 +pgpgin 234465 +pgpgout 41732 +pgfault 233838 +pgmajfault 0 +inactive_anon 0 +active_anon 778702848 +inactive_file 10407936 +active_file 0 +unevictable 0 +hierarchical_memory_limit 1073741824 +hierarchical_memsw_limit 2147483648 +total_cache 10407936 +total_rss 778842112 +total_rss_huge 0 +total_shmem 0 +total_mapped_file 0 +total_dirty 0 +total_writeback 0 +total_swap 0 +total_pgpgin 234465 +total_pgpgout 41732 +total_pgfault 233838 +total_pgmajfault 0 +total_inactive_anon 0 +total_active_anon 778702848 +total_inactive_file 10407936 +total_active_file 0 +total_unevictable 0 +recent_rotated_anon 231947 +recent_rotated_file 2 +recent_scanned_anon 231947 +recent_scanned_file 2622 diff --git a/apm-agent-core/src/test/resources/proc/limited/memory/memory.usage_in_bytes b/apm-agent-core/src/test/resources/proc/limited/memory/memory.usage_in_bytes new file mode 100644 index 0000000000..237642783d --- /dev/null +++ b/apm-agent-core/src/test/resources/proc/limited/memory/memory.usage_in_bytes @@ -0,0 +1 @@ +964778496 diff --git a/apm-agent-core/src/test/resources/proc/sys_cgroup2/slice/memory.current b/apm-agent-core/src/test/resources/proc/sys_cgroup2/slice/memory.current new file mode 100644 index 0000000000..237642783d --- /dev/null +++ b/apm-agent-core/src/test/resources/proc/sys_cgroup2/slice/memory.current @@ -0,0 +1 @@ +964778496 diff --git a/apm-agent-core/src/test/resources/proc/sys_cgroup2/slice/memory.max b/apm-agent-core/src/test/resources/proc/sys_cgroup2/slice/memory.max new file mode 100644 index 0000000000..1df8dd444d --- /dev/null +++ b/apm-agent-core/src/test/resources/proc/sys_cgroup2/slice/memory.max @@ -0,0 +1 @@ +7964778496 diff --git a/apm-agent-core/src/test/resources/proc/sys_cgroup2/slice/memory.stat b/apm-agent-core/src/test/resources/proc/sys_cgroup2/slice/memory.stat new file mode 100644 index 0000000000..bf248be317 --- /dev/null +++ b/apm-agent-core/src/test/resources/proc/sys_cgroup2/slice/memory.stat @@ -0,0 +1,40 @@ +cache 10407936 +rss 778842112 +rss_huge 0 +shmem 0 +mapped_file 0 +dirty 0 +writeback 0 +swap 0 +pgpgin 234465 +pgpgout 41732 +pgfault 233838 +pgmajfault 0 +inactive_anon 0 +active_anon 778702848 +inactive_file 10407936 +active_file 0 +unevictable 0 +hierarchical_memory_limit 1073741824 +hierarchical_memsw_limit 2147483648 +total_cache 10407936 +total_rss 778842112 +total_rss_huge 0 +total_shmem 0 +total_mapped_file 0 +total_dirty 0 +total_writeback 0 +total_swap 0 +total_pgpgin 234465 +total_pgpgout 41732 +total_pgfault 233838 +total_pgmajfault 0 +total_inactive_anon 0 +total_active_anon 778702848 +total_inactive_file 10407936 +total_active_file 0 +total_unevictable 0 +recent_rotated_anon 231947 +recent_rotated_file 2 +recent_scanned_anon 231947 +recent_scanned_file 2622 diff --git a/apm-agent-core/src/test/resources/proc/sys_cgroup2_unlimited/slice/memory.current b/apm-agent-core/src/test/resources/proc/sys_cgroup2_unlimited/slice/memory.current new file mode 100644 index 0000000000..237642783d --- /dev/null +++ b/apm-agent-core/src/test/resources/proc/sys_cgroup2_unlimited/slice/memory.current @@ -0,0 +1 @@ +964778496 diff --git a/apm-agent-core/src/test/resources/proc/sys_cgroup2_unlimited/slice/memory.max b/apm-agent-core/src/test/resources/proc/sys_cgroup2_unlimited/slice/memory.max new file mode 100644 index 0000000000..355295a05a --- /dev/null +++ b/apm-agent-core/src/test/resources/proc/sys_cgroup2_unlimited/slice/memory.max @@ -0,0 +1 @@ +max diff --git a/apm-agent-core/src/test/resources/proc/sys_cgroup2_unlimited/slice/memory.stat b/apm-agent-core/src/test/resources/proc/sys_cgroup2_unlimited/slice/memory.stat new file mode 100644 index 0000000000..6339ecd453 --- /dev/null +++ b/apm-agent-core/src/test/resources/proc/sys_cgroup2_unlimited/slice/memory.stat @@ -0,0 +1,40 @@ +cache 10407936 +rss 778842112 +rss_huge 0 +shmem 0 +mapped_file 0 +dirty 0 +writeback 0 +swap 0 +pgpgin 234465 +pgpgout 41732 +pgfault 233838 +pgmajfault 0 +inactive_anon 0 +active_anon 778702848 +inactive_file 500 +active_file 0 +unevictable 0 +hierarchical_memory_limit 1073741824 +hierarchical_memsw_limit 2147483648 +total_cache 10407936 +total_rss 778842112 +total_rss_huge 0 +total_shmem 0 +total_mapped_file 0 +total_dirty 0 +total_writeback 0 +total_swap 0 +total_pgpgin 234465 +total_pgpgout 41732 +total_pgfault 233838 +total_pgmajfault 0 +total_inactive_anon 0 +total_active_anon 778702848 +total_inactive_file 10407936 +total_active_file 0 +total_unevictable 0 +recent_rotated_anon 231947 +recent_rotated_file 2 +recent_scanned_anon 231947 +recent_scanned_file 2622 diff --git a/apm-agent-core/src/test/resources/proc/sys_cgroup2_unlimited_stat_different_order/slice/memory.current b/apm-agent-core/src/test/resources/proc/sys_cgroup2_unlimited_stat_different_order/slice/memory.current new file mode 100644 index 0000000000..237642783d --- /dev/null +++ b/apm-agent-core/src/test/resources/proc/sys_cgroup2_unlimited_stat_different_order/slice/memory.current @@ -0,0 +1 @@ +964778496 diff --git a/apm-agent-core/src/test/resources/proc/sys_cgroup2_unlimited_stat_different_order/slice/memory.max b/apm-agent-core/src/test/resources/proc/sys_cgroup2_unlimited_stat_different_order/slice/memory.max new file mode 100644 index 0000000000..355295a05a --- /dev/null +++ b/apm-agent-core/src/test/resources/proc/sys_cgroup2_unlimited_stat_different_order/slice/memory.max @@ -0,0 +1 @@ +max diff --git a/apm-agent-core/src/test/resources/proc/sys_cgroup2_unlimited_stat_different_order/slice/memory.stat b/apm-agent-core/src/test/resources/proc/sys_cgroup2_unlimited_stat_different_order/slice/memory.stat new file mode 100644 index 0000000000..af871fa058 --- /dev/null +++ b/apm-agent-core/src/test/resources/proc/sys_cgroup2_unlimited_stat_different_order/slice/memory.stat @@ -0,0 +1,40 @@ +cache 10407936 +rss 778842112 +rss_huge 0 +shmem 0 +mapped_file 0 +dirty 0 +writeback 0 +swap 0 +pgpgin 234465 +pgpgout 41732 +pgfault 233838 +pgmajfault 0 +total_inactive_file 10407936 +inactive_anon 0 +active_anon 778702848 +inactive_file 500 +active_file 0 +unevictable 0 +hierarchical_memory_limit 1073741824 +hierarchical_memsw_limit 2147483648 +total_cache 10407936 +total_rss 778842112 +total_rss_huge 0 +total_shmem 0 +total_mapped_file 0 +total_dirty 0 +total_writeback 0 +total_swap 0 +total_pgpgin 234465 +total_pgpgout 41732 +total_pgfault 233838 +total_pgmajfault 0 +total_inactive_anon 0 +total_active_anon 778702848 +total_active_file 0 +total_unevictable 0 +recent_rotated_anon 231947 +recent_rotated_file 2 +recent_scanned_anon 231947 +recent_scanned_file 2622 diff --git a/apm-agent-core/src/test/resources/proc/unlimited/memory/memory.limit_in_bytes b/apm-agent-core/src/test/resources/proc/unlimited/memory/memory.limit_in_bytes new file mode 100644 index 0000000000..564113cfaf --- /dev/null +++ b/apm-agent-core/src/test/resources/proc/unlimited/memory/memory.limit_in_bytes @@ -0,0 +1 @@ +9223372036854771712 diff --git a/apm-agent-core/src/test/resources/proc/unlimited/memory/memory.stat b/apm-agent-core/src/test/resources/proc/unlimited/memory/memory.stat new file mode 100644 index 0000000000..bf248be317 --- /dev/null +++ b/apm-agent-core/src/test/resources/proc/unlimited/memory/memory.stat @@ -0,0 +1,40 @@ +cache 10407936 +rss 778842112 +rss_huge 0 +shmem 0 +mapped_file 0 +dirty 0 +writeback 0 +swap 0 +pgpgin 234465 +pgpgout 41732 +pgfault 233838 +pgmajfault 0 +inactive_anon 0 +active_anon 778702848 +inactive_file 10407936 +active_file 0 +unevictable 0 +hierarchical_memory_limit 1073741824 +hierarchical_memsw_limit 2147483648 +total_cache 10407936 +total_rss 778842112 +total_rss_huge 0 +total_shmem 0 +total_mapped_file 0 +total_dirty 0 +total_writeback 0 +total_swap 0 +total_pgpgin 234465 +total_pgpgout 41732 +total_pgfault 233838 +total_pgmajfault 0 +total_inactive_anon 0 +total_active_anon 778702848 +total_inactive_file 10407936 +total_active_file 0 +total_unevictable 0 +recent_rotated_anon 231947 +recent_rotated_file 2 +recent_scanned_anon 231947 +recent_scanned_file 2622 diff --git a/apm-agent-core/src/test/resources/proc/unlimited/memory/memory.usage_in_bytes b/apm-agent-core/src/test/resources/proc/unlimited/memory/memory.usage_in_bytes new file mode 100644 index 0000000000..237642783d --- /dev/null +++ b/apm-agent-core/src/test/resources/proc/unlimited/memory/memory.usage_in_bytes @@ -0,0 +1 @@ +964778496 diff --git a/docs/metrics.asciidoc b/docs/metrics.asciidoc index bb107b296c..5b5555b099 100644 --- a/docs/metrics.asciidoc +++ b/docs/metrics.asciidoc @@ -21,6 +21,7 @@ Starting in Java agent version 1.11.0, it is possible to manually configure a un When multiple JVMs are running on the same host and report data for the same service, this configuration is required in order to be able to view metrics at the JVM level. * <> +* <> * <> * <> @@ -28,7 +29,7 @@ When multiple JVMs are running on the same host and report data for the same ser [[metrics-system]] === System metrics -As of version 6.6, these metrics will be visualized in the APM app. +Host metrics. As of version 6.6, these metrics will be visualized in the APM app. For more system metrics, consider installing {metricbeat-ref}/index.html[metricbeat] on your hosts. @@ -90,6 +91,44 @@ format: bytes The total virtual memory the process has. -- +[float] +[[metrics-cgroup]] +=== cgroup metrics (added in 1.18.0) + +Linux's cgroup metrics. + +*`system.process.cgroup.memory.mem.limit.bytes`*:: ++ +-- +type: long + +format: bytes + +Memory limit for current cgroup slice. +-- + + +*`system.process.cgroup.memory.mem.usage.bytes`*:: ++ +-- +type: long + +format: bytes + +Memory usage in current cgroup slice. +-- + + +*`system.process.cgroup.memory.stats.inactive_file.bytes`*:: ++ +-- +type: long + +format: bytes + +Inactive memory in current cgroup slice. +-- + [float] [[metrics-jvm]] === JVM Metrics