Skip to content

Commit 6d83482

Browse files
jerboaajerboaa
committed
8293540: [Metrics] Incorrectly detected resource limits with additional cgroup fs mounts
Reviewed-by: iklam
1 parent 6974978 commit 6d83482

File tree

6 files changed

+158
-38
lines changed

6 files changed

+158
-38
lines changed

‎src/java.base/linux/classes/jdk/internal/platform/CgroupSubsystemFactory.java‎

Lines changed: 19 additions & 25 deletions
Original file line numberDiff line numberDiff line change
@@ -318,30 +318,11 @@ private static boolean amendCgroupInfos(String mntInfoLine,
318318
case MEMORY_CTRL: // fall-through
319319
case CPU_CTRL:
320320
case CPUACCT_CTRL:
321+
case CPUSET_CTRL:
321322
case PIDS_CTRL:
322323
case BLKIO_CTRL: {
323324
CgroupInfo info = infos.get(controllerName);
324-
assert info.getMountPoint() == null;
325-
assert info.getMountRoot() == null;
326-
info.setMountPoint(mountPath);
327-
info.setMountRoot(mountRoot);
328-
cgroupv1ControllerFound = true;
329-
break;
330-
}
331-
case CPUSET_CTRL: {
332-
CgroupInfo info = infos.get(controllerName);
333-
if (info.getMountPoint() != null) {
334-
// On some systems duplicate cpuset controllers get mounted in addition to
335-
// the main cgroup controllers most likely under /sys/fs/cgroup. In that
336-
// case pick the one under /sys/fs/cgroup and discard others.
337-
if (!info.getMountPoint().startsWith("/sys/fs/cgroup")) {
338-
info.setMountPoint(mountPath);
339-
info.setMountRoot(mountRoot);
340-
}
341-
} else {
342-
info.setMountPoint(mountPath);
343-
info.setMountRoot(mountRoot);
344-
}
325+
setMountPoints(info, mountPath, mountRoot);
345326
cgroupv1ControllerFound = true;
346327
break;
347328
}
@@ -355,10 +336,7 @@ private static boolean amendCgroupInfos(String mntInfoLine,
355336
// All controllers have the same mount point and root mount
356337
// for unified hierarchy.
357338
for (CgroupInfo info: infos.values()) {
358-
assert info.getMountPoint() == null;
359-
assert info.getMountRoot() == null;
360-
info.setMountPoint(mountPath);
361-
info.setMountRoot(mountRoot);
339+
setMountPoints(info, mountPath, mountRoot);
362340
}
363341
}
364342
cgroupv2ControllerFound = true;
@@ -367,6 +345,22 @@ private static boolean amendCgroupInfos(String mntInfoLine,
367345
return cgroupv1ControllerFound || cgroupv2ControllerFound;
368346
}
369347

348+
private static void setMountPoints(CgroupInfo info, String mountPath, String mountRoot) {
349+
if (info.getMountPoint() != null) {
350+
// On some systems duplicate controllers get mounted in addition to
351+
// the main cgroup controllers (which are under /sys/fs/cgroup). In that
352+
// case pick the main one and discard others as the limits
353+
// are associated with the ones in /sys/fs/cgroup.
354+
if (!info.getMountPoint().startsWith("/sys/fs/cgroup")) {
355+
info.setMountPoint(mountPath);
356+
info.setMountRoot(mountRoot);
357+
}
358+
} else {
359+
info.setMountPoint(mountPath);
360+
info.setMountRoot(mountRoot);
361+
}
362+
}
363+
370364
public static final class CgroupTypeResult {
371365
private final boolean isCgroupV2;
372366
private final boolean anyControllersEnabled;

‎test/hotspot/jtreg/containers/docker/TestMemoryAwareness.java‎

Lines changed: 15 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -87,6 +87,11 @@ public static void main(String[] args) throws Exception {
8787
"1G", Integer.toString(((int) Math.pow(2, 20)) * 1024),
8888
"1500M", Integer.toString(((int) Math.pow(2, 20)) * (1500 - 1024))
8989
);
90+
testOperatingSystemMXBeanAwareness(
91+
"100M", Integer.toString(((int) Math.pow(2, 20)) * 100),
92+
"200M", Integer.toString(((int) Math.pow(2, 20)) * (200 - 100)),
93+
true /* additional cgroup fs mounts */
94+
);
9095
final String hostMaxMem = getHostMaxMemory();
9196
testOperatingSystemMXBeanIgnoresMemLimitExceedingPhysicalMemory(hostMaxMem);
9297
testMetricsIgnoresMemLimitExceedingPhysicalMemory(hostMaxMem);
@@ -170,6 +175,12 @@ private static void testOOM(String dockerMemLimit, int sizeToAllocInMb) throws E
170175

171176
private static void testOperatingSystemMXBeanAwareness(String memoryAllocation, String expectedMemory,
172177
String swapAllocation, String expectedSwap) throws Exception {
178+
testOperatingSystemMXBeanAwareness(memoryAllocation, expectedMemory, swapAllocation, expectedSwap, false);
179+
}
180+
181+
private static void testOperatingSystemMXBeanAwareness(String memoryAllocation, String expectedMemory,
182+
String swapAllocation, String expectedSwap, boolean addCgroupMounts) throws Exception {
183+
173184
Common.logNewTestCase("Check OperatingSystemMXBean");
174185

175186
DockerRunOptions opts = Common.newOpts(imageName, "CheckOperatingSystemMXBean")
@@ -181,6 +192,10 @@ private static void testOperatingSystemMXBeanAwareness(String memoryAllocation,
181192
// diagnostics
182193
.addJavaOpts("--add-exports")
183194
.addJavaOpts("java.base/jdk.internal.platform=ALL-UNNAMED");
195+
if (addCgroupMounts) {
196+
// Extra cgroup mount should be ignored by product code
197+
opts.addDockerOpts("--volume", "/sys/fs/cgroup:/cgroup-in:ro");
198+
}
184199

185200
OutputAnalyzer out = DockerTestUtils.dockerRunJava(opts);
186201
out.shouldHaveExitValue(0)

‎test/jdk/jdk/internal/platform/cgroup/TestCgroupSubsystemFactory.java‎

Lines changed: 24 additions & 13 deletions
Original file line numberDiff line numberDiff line change
@@ -51,7 +51,7 @@
5151

5252
/*
5353
* @test
54-
* @bug 8287107 8287073
54+
* @bug 8287107 8287073 8293540
5555
* @key cgroups
5656
* @requires os.family == "linux"
5757
* @modules java.base/jdk.internal.platform
@@ -72,8 +72,8 @@ public class TestCgroupSubsystemFactory {
7272
private Path cgroupv1CgInfoNonZeroHierarchy;
7373
private Path cgroupv1MntInfoNonZeroHierarchy;
7474
private Path cgroupv1MntInfoSystemdOnly;
75-
private Path cgroupv1MntInfoDoubleCpusets;
76-
private Path cgroupv1MntInfoDoubleCpusets2;
75+
private Path cgroupv1MntInfoDoubleControllers;
76+
private Path cgroupv1MntInfoDoubleControllers2;
7777
private Path cgroupv1MntInfoColonsHierarchy;
7878
private Path cgroupv1SelfCgroup;
7979
private Path cgroupv1SelfColons;
@@ -194,9 +194,13 @@ public class TestCgroupSubsystemFactory {
194194
private String mntInfoCgroupsV1SystemdOnly =
195195
"35 26 0:26 / /sys/fs/cgroup/systemd rw,nosuid,nodev,noexec,relatime - cgroup systemd rw,name=systemd\n" +
196196
"26 18 0:19 / /sys/fs/cgroup rw,relatime - tmpfs none rw,size=4k,mode=755\n";
197-
private String mntInfoCgroupv1MoreCpusetLine = "121 32 0:37 / /cpuset rw,relatime shared:69 - cgroup none rw,cpuset\n";
198-
private String mntInfoCgroupsV1DoubleCpuset = mntInfoHybrid + mntInfoCgroupv1MoreCpusetLine;
199-
private String mntInfoCgroupsV1DoubleCpuset2 = mntInfoCgroupv1MoreCpusetLine + mntInfoHybrid;
197+
private String mntInfoCgroupv1MoreControllers = "121 32 0:37 / /cpuset rw,relatime shared:69 - cgroup none rw,cpuset\n" +
198+
"35 30 0:31 / /cgroup-in/memory rw,nosuid,nodev,noexec,relatime shared:7 - cgroup none rw,seclabel,memory\n" +
199+
"36 30 0:32 / /cgroup-in/pids rw,nosuid,nodev,noexec,relatime shared:8 - cgroup none rw,seclabel,pids\n" +
200+
"40 30 0:36 / /cgroup-in/cpu,cpuacct rw,nosuid,nodev,noexec,relatime shared:12 - cgroup none rw,seclabel,cpu,cpuacct\n" +
201+
"40 30 0:36 / /cgroup-in/blkio rw,nosuid,nodev,noexec,relatime shared:12 - cgroup none rw,seclabel,blkio\n";
202+
private String mntInfoCgroupsV1DoubleControllers = mntInfoHybrid + mntInfoCgroupv1MoreControllers;
203+
private String mntInfoCgroupsV1DoubleControllers2 = mntInfoCgroupv1MoreControllers + mntInfoHybrid;
200204
private String cgroupv1SelfCgroupContent = "11:memory:/user.slice/user-1000.slice/user@1000.service\n" +
201205
"10:hugetlb:/\n" +
202206
"9:cpuset:/\n" +
@@ -275,11 +279,11 @@ public void setup() {
275279
cgroupv1MntInfoSystemdOnly = Paths.get(existingDirectory.toString(), "mountinfo_cgroupv1_systemd_only");
276280
Files.writeString(cgroupv1MntInfoSystemdOnly, mntInfoCgroupsV1SystemdOnly);
277281

278-
cgroupv1MntInfoDoubleCpusets = Paths.get(existingDirectory.toString(), "mountinfo_cgroupv1_double_cpuset");
279-
Files.writeString(cgroupv1MntInfoDoubleCpusets, mntInfoCgroupsV1DoubleCpuset);
282+
cgroupv1MntInfoDoubleControllers = Paths.get(existingDirectory.toString(), "mountinfo_cgroupv1_double_controllers");
283+
Files.writeString(cgroupv1MntInfoDoubleControllers, mntInfoCgroupsV1DoubleControllers);
280284

281-
cgroupv1MntInfoDoubleCpusets2 = Paths.get(existingDirectory.toString(), "mountinfo_cgroupv1_double_cpuset2");
282-
Files.writeString(cgroupv1MntInfoDoubleCpusets2, mntInfoCgroupsV1DoubleCpuset2);
285+
cgroupv1MntInfoDoubleControllers2 = Paths.get(existingDirectory.toString(), "mountinfo_cgroupv1_double_controllers2");
286+
Files.writeString(cgroupv1MntInfoDoubleControllers2, mntInfoCgroupsV1DoubleControllers2);
283287

284288
cgroupv1CgroupsJoinControllers = Paths.get(existingDirectory.toString(), "cgroups_cgv1_join_controllers");
285289
Files.writeString(cgroupv1CgroupsJoinControllers, cgroupsNonZeroJoinControllers);
@@ -390,11 +394,11 @@ public void testCgroupv1SystemdOnly() throws IOException {
390394

391395
@Test
392396
public void testCgroupv1MultipleCpusetMounts() throws IOException {
393-
doMultipleCpusetMountsTest(cgroupv1MntInfoDoubleCpusets);
394-
doMultipleCpusetMountsTest(cgroupv1MntInfoDoubleCpusets2);
397+
doMultipleMountsTest(cgroupv1MntInfoDoubleControllers);
398+
doMultipleMountsTest(cgroupv1MntInfoDoubleControllers2);
395399
}
396400

397-
private void doMultipleCpusetMountsTest(Path info) throws IOException {
401+
private void doMultipleMountsTest(Path info) throws IOException {
398402
String cgroups = cgroupv1CgInfoNonZeroHierarchy.toString();
399403
String mountInfo = info.toString();
400404
String selfCgroup = cgroupv1SelfCgroup.toString();
@@ -406,6 +410,13 @@ private void doMultipleCpusetMountsTest(Path info) throws IOException {
406410
CgroupInfo cpuSetInfo = res.getInfos().get("cpuset");
407411
assertEquals("/sys/fs/cgroup/cpuset", cpuSetInfo.getMountPoint());
408412
assertEquals("/", cpuSetInfo.getMountRoot());
413+
// Ensure controllers at /sys/fs/cgroup will be used
414+
String[] ctrlNames = new String[] { "memory", "cpu", "cpuacct", "blkio", "pids" };
415+
for (int i = 0; i < ctrlNames.length; i++) {
416+
CgroupInfo cinfo = res.getInfos().get(ctrlNames[i]);
417+
assertTrue(cinfo.getMountPoint().startsWith("/sys/fs/cgroup/"));
418+
assertEquals("/", cinfo.getMountRoot());
419+
}
409420
}
410421

411422
@Test

‎test/jdk/jdk/internal/platform/docker/TestDockerBasic.java‎

Lines changed: 72 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,72 @@
1+
/*
2+
* Copyright (c) 2022, Red Hat, Inc.
3+
* DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER.
4+
*
5+
* This code is free software; you can redistribute it and/or modify it
6+
* under the terms of the GNU General Public License version 2 only, as
7+
* published by the Free Software Foundation.
8+
*
9+
* This code is distributed in the hope that it will be useful, but WITHOUT
10+
* ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
11+
* FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License
12+
* version 2 for more details (a copy is included in the LICENSE file that
13+
* accompanied this code).
14+
*
15+
* You should have received a copy of the GNU General Public License version
16+
* 2 along with this work; if not, write to the Free Software Foundation,
17+
* Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301 USA.
18+
*
19+
* Please contact Oracle, 500 Oracle Parkway, Redwood Shores, CA 94065 USA
20+
* or visit www.oracle.com if you need additional information or have any
21+
* questions.
22+
*/
23+
24+
/*
25+
* @test
26+
* @bug 8293540
27+
* @summary Verify that -XshowSettings:system works
28+
* @requires docker.support
29+
* @library /test/lib
30+
* @run main/timeout=360 TestDockerBasic
31+
*/
32+
33+
import jdk.test.lib.Utils;
34+
import jdk.test.lib.containers.docker.Common;
35+
import jdk.test.lib.containers.docker.DockerRunOptions;
36+
import jdk.test.lib.containers.docker.DockerTestUtils;
37+
38+
public class TestDockerBasic {
39+
private static final String imageName = Common.imageName("javaDockerBasic");
40+
41+
public static void main(String[] args) throws Exception {
42+
if (!DockerTestUtils.canTestDocker()) {
43+
return;
44+
}
45+
46+
DockerTestUtils.buildJdkContainerImage(imageName);
47+
48+
try {
49+
testXshowSettingsSystem(true);
50+
testXshowSettingsSystem(false);
51+
} finally {
52+
DockerTestUtils.removeDockerImage(imageName);
53+
}
54+
}
55+
56+
private static void testXshowSettingsSystem(boolean addCgroupMounts) throws Exception {
57+
String testMsg = (addCgroupMounts ? " with " : " without ") + " additional cgroup FS mounts in /cgroup-in";
58+
Common.logNewTestCase("Test TestDockerBasic " + testMsg);
59+
DockerRunOptions opts =
60+
new DockerRunOptions(imageName, "/jdk/bin/java", "-version");
61+
opts.addJavaOpts("-esa");
62+
opts.addJavaOpts("-XshowSettings:system");
63+
opts.addDockerOpts("--memory", "300m");
64+
if (addCgroupMounts) {
65+
// Extra cgroup mount should be ignored by product code
66+
opts.addDockerOpts("--volume", "/sys/fs/cgroup:/cgroup-in:ro");
67+
}
68+
DockerTestUtils.dockerRunJava(opts).shouldHaveExitValue(0)
69+
.shouldNotContain("AssertionError")
70+
.shouldContain("Memory Limit: 300.00M");
71+
}
72+
}

‎test/jdk/jdk/internal/platform/docker/TestDockerCpuMetrics.java‎

Lines changed: 18 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -65,11 +65,13 @@ public static void main(String[] args) throws Exception {
6565
testCpuSet((((numCpus - 1) / 2) + 1) + "-" + (numCpus - 1));
6666
}
6767
testCpuSet(IntStream.range(0, numCpus).mapToObj(a -> Integer.toString(a)).collect(Collectors.joining(",")));
68+
testCpuSet("0", true /* additional cgroup fs mount */);
6869

6970
testCpuQuota(50 * 1000, 100 * 1000);
7071
testCpuQuota(100 * 1000, 100 * 1000);
7172
testCpuQuota(150 * 1000, 100 * 1000);
7273
testCpuQuota(400 * 1000, 100 * 1000);
74+
testCpuQuota(200 * 1000, 100 * 1000, true /* additional cgroup fs mount */);
7375

7476
testCpuShares(256);
7577
testCpuShares(2048);
@@ -108,10 +110,18 @@ private static void testCpuSetMems(String value) throws Exception {
108110
}
109111

110112
private static void testCpuSet(String value) throws Exception {
113+
testCpuSet(value, false);
114+
}
115+
116+
private static void testCpuSet(String value, boolean addCgroupMount) throws Exception {
111117
Common.logNewTestCase("testCpuSet, value = " + value);
112118
DockerRunOptions opts =
113119
new DockerRunOptions(imageName, "/jdk/bin/java", "MetricsCpuTester");
114120
opts.addDockerOpts("--volume", Utils.TEST_CLASSES + ":/test-classes/");
121+
if (addCgroupMount) {
122+
// Extra cgroup mount should be ignored by product code
123+
opts.addDockerOpts("--volume", "/sys/fs/cgroup:/cgroup-in:ro");
124+
}
115125
opts.addJavaOpts("-cp", "/test-classes/");
116126
opts.addJavaOpts("--add-exports", "java.base/jdk.internal.platform=ALL-UNNAMED");
117127
opts.addClassOptions("cpusets", value);
@@ -120,10 +130,18 @@ private static void testCpuSet(String value) throws Exception {
120130
}
121131

122132
private static void testCpuQuota(long quota, long period) throws Exception {
133+
testCpuQuota(quota, period, false);
134+
}
135+
136+
private static void testCpuQuota(long quota, long period, boolean addCgroupMount) throws Exception {
123137
Common.logNewTestCase("testCpuQuota, quota = " + quota + ", period = " + period);
124138
DockerRunOptions opts =
125139
new DockerRunOptions(imageName, "/jdk/bin/java", "MetricsCpuTester");
126140
opts.addDockerOpts("--volume", Utils.TEST_CLASSES + ":/test-classes/");
141+
if (addCgroupMount) {
142+
// Extra cgroup mount should be ignored by product code
143+
opts.addDockerOpts("--volume", "/sys/fs/cgroup:/cgroup-in:ro");
144+
}
127145
opts.addDockerOpts("--cpu-period=" + period).addDockerOpts("--cpu-quota=" + quota);
128146
opts.addJavaOpts("-cp", "/test-classes/").addJavaOpts("--add-exports", "java.base/jdk.internal.platform=ALL-UNNAMED");
129147
opts.addClassOptions("cpuquota", quota + "", period + "");

‎test/jdk/jdk/internal/platform/docker/TestDockerMemoryMetrics.java‎

Lines changed: 10 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -56,6 +56,8 @@ public static void main(String[] args) throws Exception {
5656
try {
5757
testMemoryLimit("200m");
5858
testMemoryLimit("1g");
59+
// Memory limit test with additional cgroup fs mounted
60+
testMemoryLimit("500m", true /* cgroup fs mount */);
5961

6062
testMemoryAndSwapLimit("200m", "1g");
6163
testMemoryAndSwapLimit("100m", "200m");
@@ -85,6 +87,10 @@ public static void main(String[] args) throws Exception {
8587
}
8688

8789
private static void testMemoryLimit(String value) throws Exception {
90+
testMemoryLimit(value, false);
91+
}
92+
93+
private static void testMemoryLimit(String value, boolean addCgroupMount) throws Exception {
8894
Common.logNewTestCase("testMemoryLimit, value = " + value);
8995
DockerRunOptions opts =
9096
new DockerRunOptions(imageName, "/jdk/bin/java", "MetricsMemoryTester");
@@ -93,6 +99,10 @@ private static void testMemoryLimit(String value) throws Exception {
9399
.addJavaOpts("-cp", "/test-classes/")
94100
.addJavaOpts("--add-exports", "java.base/jdk.internal.platform=ALL-UNNAMED")
95101
.addClassOptions("memory", value);
102+
if (addCgroupMount) {
103+
// Extra cgroup mount should be ignored by product code
104+
opts.addDockerOpts("--volume", "/sys/fs/cgroup:/cgroup-in:ro");
105+
}
96106
DockerTestUtils.dockerRunJava(opts).shouldHaveExitValue(0).shouldContain("TEST PASSED!!!");
97107
}
98108

0 commit comments

Comments
 (0)