Skip to content

Commit ab2953c

Browse files
fmeumcopybara-github
fmeum
authored and
copybara-github
committed
Use Runtime#availableProcessors on Linux
As of JDK 11, `Runtime#availableProcessors` on Linux is aware of cgroup resource limits and thus behaves as expected in containers, whereas `/proc/cpuinfo` still reports the number of logical processors available to the host. Unfortunately, the JVM-internal `os::total_memory` function, which similarly takes cgroup limits into account when computing the available RAM, does not seem to be accessible from Java. https://hg.openjdk.java.net/jdk/hs/rev/7f22774a5f42#l6.178 Fixes #5042 Closes #16512. RELNOTES[INC]: Bazel's local CPU resource on Linux is now container aware. Use `--local_cpu_resources`, `--loading_phase_threads` or `--jobs` to override. PiperOrigin-RevId: 515032721 Change-Id: I103ee25920d6908545ce6bf03d9c42c604c8589b
1 parent e29e2fb commit ab2953c

File tree

2 files changed

+3
-225
lines changed

2 files changed

+3
-225
lines changed

‎src/main/java/com/google/devtools/build/lib/actions/LocalHostResourceManagerLinux.java‎

Lines changed: 3 additions & 37 deletions
Original file line numberDiff line numberDiff line change
@@ -14,27 +14,21 @@
1414

1515
package com.google.devtools.build.lib.actions;
1616

17-
import com.google.common.base.Splitter;
18-
import com.google.common.io.Files;
1917
import com.google.devtools.build.lib.unix.ProcMeminfoParser;
20-
import java.io.File;
2118
import java.io.IOException;
22-
import java.nio.charset.Charset;
2319
import javax.annotation.Nullable;
2420

2521
/**
2622
* This class estimates the local host's resource capacity for Linux.
2723
*/
2824
public class LocalHostResourceManagerLinux {
29-
private static String cpuInfoContent = null;
3025

31-
private static final Splitter NEWLINE_SPLITTER = Splitter.on('\n').omitEmptyStrings();
32-
private static final String CPU_INFO_FILE = "/proc/cpuinfo";
3326
private static final String MEM_INFO_FILE = "/proc/meminfo";
3427

3528
private static int getLogicalCpuCount() throws IOException {
36-
String content = getCpuInfoContent();
37-
return getLogicalCpuCountHelper(content);
29+
// As of JDK 11, availableProcessors is aware of cgroups as commonly used by containers.
30+
// https://hg.openjdk.java.net/jdk/hs/rev/7f22774a5f42#l6.178
31+
return Runtime.getRuntime().availableProcessors();
3832
}
3933

4034
private static double getMemoryInMb() throws IOException {
@@ -56,34 +50,6 @@ public static ResourceSet getLocalHostResources() {
5650
}
5751
}
5852

59-
private static String getCpuInfoContent() throws IOException {
60-
if (cpuInfoContent == null) {
61-
cpuInfoContent = readContent(CPU_INFO_FILE);
62-
}
63-
return cpuInfoContent;
64-
}
65-
66-
private static String readContent(String filename) throws IOException {
67-
return Files.asCharSource(new File(filename), Charset.defaultCharset()).read();
68-
}
69-
70-
/**
71-
* For testing purposes only. Do not use it.
72-
*/
73-
public static int getLogicalCpuCountHelper(String content) throws IOException {
74-
int count = 0;
75-
Iterable<String> lines = NEWLINE_SPLITTER.split(content);
76-
for (String line : lines) {
77-
if (line.startsWith("processor")) {
78-
count++;
79-
}
80-
}
81-
if (count == 0) {
82-
throw new IllegalArgumentException("Can't get logical CPU count");
83-
}
84-
return count;
85-
}
86-
8753
public static double getMemoryInMbHelper(String memInfoFileName) throws IOException {
8854
ProcMeminfoParser memInfo = new ProcMeminfoParser(memInfoFileName);
8955
double ramMb = ProcMeminfoParser.kbToMb(memInfo.getTotalKb());

‎src/test/java/com/google/devtools/build/lib/actions/LocalHostResourceManagerLinuxTest.java‎

Lines changed: 0 additions & 188 deletions
Original file line numberDiff line numberDiff line change
@@ -31,29 +31,6 @@ public class LocalHostResourceManagerLinuxTest {
3131

3232
@Test
3333
public void testNonHyperthreadedMachine() throws Exception {
34-
String cpuinfoContent =
35-
StringUtilities.joinLines(
36-
"processor\t: 0",
37-
"vendor_id\t: GenuineIntel",
38-
"cpu family\t: 15",
39-
"model\t\t: 4",
40-
"model name\t: Intel(R) Pentium(R) 4 CPU 3.40GHz",
41-
"stepping\t: 10",
42-
"cpu MHz\t\t: 3400.000",
43-
"cache size\t: 2048 KB",
44-
"fpu\t\t: yes",
45-
"fpu_exception\t: yes",
46-
"cpuid level\t: 5",
47-
"wp\t\t: yes",
48-
"flags\t\t: fpu vme de pse tsc msr pae mce cx8 apic sep mtrr pge mca "
49-
+ "cmov pat pse36 clflush dts acpi mmx fxsr sse sse2 ss ht tm "
50-
+ "syscall nx lm constant_tsc up pni monitor ds_cpl est cid cx16 "
51-
+ "xtpr lahf_lm",
52-
"bogomips\t: 6803.83",
53-
"clflush size\t: 64",
54-
"cache_alignment\t: 128",
55-
"address sizes\t: 36 bits physical, 48 bits virtual",
56-
"power management:");
5734
String meminfoContent =
5835
StringUtilities.joinLines(
5936
"MemTotal: 3091732 kB",
@@ -87,7 +64,6 @@ public void testNonHyperthreadedMachine() throws Exception {
8764
"HugePages_Rsvd: 0",
8865
"Hugepagesize: 2048 kB");
8966
String meminfoFile = scratch.file("test_meminfo_nonht", meminfoContent).getPathString();
90-
assertThat(LocalHostResourceManagerLinux.getLogicalCpuCountHelper(cpuinfoContent)).isEqualTo(1);
9167
// +/- 0.1MB
9268
assertThat(LocalHostResourceManagerLinux.getMemoryInMbHelper(meminfoFile))
9369
.isWithin(0.1)
@@ -96,58 +72,6 @@ public void testNonHyperthreadedMachine() throws Exception {
9672

9773
@Test
9874
public void testHyperthreadedMachine() throws Exception {
99-
String cpuinfoContent =
100-
StringUtilities.joinLines(
101-
"processor\t: 0",
102-
"vendor_id\t: GenuineIntel",
103-
"cpu family\t: 15",
104-
"model\t\t: 4",
105-
"model name\t: Intel(R) Pentium(R) 4 CPU 3.40GHz",
106-
"stepping\t: 1",
107-
"cpu MHz\t\t: 3400.245",
108-
"cache size\t: 1024 KB",
109-
"physical id\t: 0",
110-
"siblings\t: 2",
111-
"core id\t\t: 0",
112-
"cpu cores\t: 1",
113-
"fpu\t\t: yes",
114-
"fpu_exception\t: yes",
115-
"cpuid level\t: 5",
116-
"wp\t\t: yes",
117-
"flags\t\t: fpu vme de pse tsc msr pae mce cx8 apic sep mtrr pge "
118-
+ "mca cmov pat pse36 clflush dts acpi mmx fxsr sse sse2 ss ht tm "
119-
+ "syscall lm constant_tsc pni monitor ds_cpl cid cx16 xtpr",
120-
"bogomips\t: 6806.31",
121-
"clflush size\t: 64",
122-
"cache_alignment\t: 128",
123-
"address sizes\t: 36 bits physical, 48 bits virtual",
124-
"power management:",
125-
"",
126-
"processor\t: 1",
127-
"vendor_id\t: GenuineIntel",
128-
"cpu family\t: 15",
129-
"model\t\t: 4",
130-
"model name\t: Intel(R) Pentium(R) 4 CPU 3.40GHz",
131-
"stepping\t: 1",
132-
"cpu MHz\t\t: 3400.245",
133-
"cache size\t: 1024 KB",
134-
"physical id\t: 0",
135-
"siblings\t: 2",
136-
"core id\t\t: 0",
137-
"cpu cores\t: 1",
138-
"fpu\t\t: yes",
139-
"fpu_exception\t: yes",
140-
"cpuid level\t: 5",
141-
"wp\t\t: yes",
142-
"flags\t\t: fpu vme de pse tsc msr pae mce cx8 apic sep mtrr pge "
143-
+ "mca cmov pat pse36 clflush dts acpi mmx fxsr sse sse2 ss ht tm "
144-
+ "syscall lm constant_tsc pni monitor ds_cpl cid cx16 xtpr",
145-
"bogomips\t: 6800.76",
146-
"clflush size\t: 64",
147-
"cache_alignment\t: 128",
148-
"address sizes\t: 36 bits physical, 48 bits virtual",
149-
"power management:",
150-
"");
15175
String meminfoContent =
15276
StringUtilities.joinLines(
15377
"MemTotal: 3092004 kB",
@@ -181,7 +105,6 @@ public void testHyperthreadedMachine() throws Exception {
181105
"HugePages_Rsvd: 0",
182106
"Hugepagesize: 2048 kB");
183107
String meminfoFile = scratch.file("test_meminfo_ht", meminfoContent).getPathString();
184-
assertThat(LocalHostResourceManagerLinux.getLogicalCpuCountHelper(cpuinfoContent)).isEqualTo(2);
185108
// +/- 0.1MB
186109
assertThat(LocalHostResourceManagerLinux.getMemoryInMbHelper(meminfoFile))
187110
.isWithin(0.1)
@@ -190,116 +113,6 @@ public void testHyperthreadedMachine() throws Exception {
190113

191114
@Test
192115
public void testAMDMachine() throws Exception {
193-
String cpuinfoContent =
194-
StringUtilities.joinLines(
195-
"processor\t: 0",
196-
"vendor_id\t: AuthenticAMD",
197-
"cpu family\t: 15",
198-
"model\t\t: 65",
199-
"model name\t: Dual-Core AMD Opteron(tm) Processor 8214 HE",
200-
"stepping\t: 2",
201-
"cpu MHz\t\t: 2200.000",
202-
"cache size\t: 1024 KB",
203-
"physical id\t: 0",
204-
"siblings\t: 2",
205-
"core id\t\t: 0",
206-
"cpu cores\t: 2",
207-
"fpu\t\t: yes",
208-
"fpu_exception\t: yes",
209-
"cpuid level\t: 1",
210-
"wp\t\t: yes",
211-
"flags\t\t: fpu vme de pse tsc msr pae mce cx8 apic sep mtrr "
212-
+ "pge mca cmov pat pse36 clflush mmx fxsr sse sse2 ht syscall "
213-
+ "nx mmxext fxsr_opt rdtscp lm 3dnowext 3dnow pni cx16 lahf_lm "
214-
+ "cmp_legacy svm cr8_legacy",
215-
"bogomips\t: 4425.84",
216-
"TLB size\t: 1024 4K pages",
217-
"clflush size\t: 64",
218-
"cache_alignment\t: 64",
219-
"address sizes\t: 40 bits physical, 48 bits virtual",
220-
"power management: ts fid vid ttp tm stc",
221-
"",
222-
"processor\t: 1",
223-
"vendor_id\t: AuthenticAMD",
224-
"cpu family\t: 15",
225-
"model\t\t: 65",
226-
"model name\t: Dual-Core AMD Opteron(tm) Processor 8214 HE",
227-
"stepping\t: 2",
228-
"cpu MHz\t\t: 2200.000",
229-
"cache size\t: 1024 KB",
230-
"physical id\t: 0",
231-
"siblings\t: 2",
232-
"core id\t\t: 1",
233-
"cpu cores\t: 2",
234-
"fpu\t\t: yes",
235-
"fpu_exception\t: yes",
236-
"cpuid level\t: 1",
237-
"wp\t\t: yes",
238-
"flags\t\t: fpu vme de pse tsc msr pae mce cx8 apic sep mtrr "
239-
+ "pge mca cmov pat pse36 clflush mmx fxsr sse sse2 ht syscall "
240-
+ "nx mmxext fxsr_opt rdtscp lm 3dnowext 3dnow pni cx16 lahf_lm "
241-
+ "cmp_legacy svm cr8_legacy",
242-
"bogomips\t: 4460.61",
243-
"TLB size\t: 1024 4K pages",
244-
"clflush size\t: 64",
245-
"cache_alignment\t: 64",
246-
"address sizes\t: 40 bits physical, 48 bits virtual",
247-
"power management: ts fid vid ttp tm stc",
248-
"",
249-
"processor\t: 2",
250-
"vendor_id\t: AuthenticAMD",
251-
"cpu family\t: 15",
252-
"model\t\t: 65",
253-
"model name\t: Dual-Core AMD Opteron(tm) Processor 8214 HE",
254-
"stepping\t: 2",
255-
"cpu MHz\t\t: 2200.000",
256-
"cache size\t: 1024 KB",
257-
"physical id\t: 1",
258-
"siblings\t: 2",
259-
"core id\t\t: 0",
260-
"cpu cores\t: 2",
261-
"fpu\t\t: yes",
262-
"fpu_exception\t: yes",
263-
"cpuid level\t: 1",
264-
"wp\t\t: yes",
265-
"flags\t\t: fpu vme de pse tsc msr pae mce cx8 apic sep mtrr "
266-
+ "pge mca cmov pat pse36 clflush mmx fxsr sse sse2 ht syscall "
267-
+ "nx mmxext fxsr_opt rdtscp lm 3dnowext 3dnow pni cx16 lahf_lm "
268-
+ "cmp_legacy svm cr8_legacy",
269-
"bogomips\t: 4420.45",
270-
"TLB size\t: 1024 4K pages",
271-
"clflush size\t: 64",
272-
"cache_alignment\t: 64",
273-
"address sizes\t: 40 bits physical, 48 bits virtual",
274-
"power management: ts fid vid ttp tm stc",
275-
"",
276-
"processor\t: 3",
277-
"vendor_id\t: AuthenticAMD",
278-
"cpu family\t: 15",
279-
"model\t\t: 65",
280-
"model name\t: Dual-Core AMD Opteron(tm) Processor 8214 HE",
281-
"stepping\t: 2",
282-
"cpu MHz\t\t: 2200.000",
283-
"cache size\t: 1024 KB",
284-
"physical id\t: 1",
285-
"siblings\t: 2",
286-
"core id\t\t: 1",
287-
"cpu cores\t: 2",
288-
"fpu\t\t: yes",
289-
"fpu_exception\t: yes",
290-
"cpuid level\t: 1",
291-
"wp\t\t: yes",
292-
"flags\t\t: fpu vme de pse tsc msr pae mce cx8 apic sep mtrr "
293-
+ "pge mca cmov pat pse36 clflush mmx fxsr sse sse2 ht syscall "
294-
+ "nx mmxext fxsr_opt rdtscp lm 3dnowext 3dnow pni cx16 lahf_lm "
295-
+ "cmp_legacy svm cr8_legacy",
296-
"bogomips\t: 4460.39",
297-
"TLB size\t: 1024 4K pages",
298-
"clflush size\t: 64",
299-
"cache_alignment\t: 64",
300-
"address sizes\t: 40 bits physical, 48 bits virtual",
301-
"power management: ts fid vid ttp tm stc",
302-
"");
303116
String meminfoContent =
304117
StringUtilities.joinLines(
305118
"MemTotal: 8223956 kB",
@@ -333,7 +146,6 @@ public void testAMDMachine() throws Exception {
333146
"HugePages_Rsvd: 0",
334147
"Hugepagesize: 2048 kB");
335148
String meminfoFile = scratch.file("test_meminfo_amd", meminfoContent).getPathString();
336-
assertThat(LocalHostResourceManagerLinux.getLogicalCpuCountHelper(cpuinfoContent)).isEqualTo(4);
337149
// +/- 0.1MB
338150
assertThat(LocalHostResourceManagerLinux.getMemoryInMbHelper(meminfoFile))
339151
.isWithin(0.1)

0 commit comments

Comments
 (0)