From 16850bc5638b66d98ef1cd3b4db395619d399a5d Mon Sep 17 00:00:00 2001 From: Mark Vieira Date: Mon, 28 Mar 2022 11:31:58 -0700 Subject: [PATCH] Fix calculation of max parallel forks for test execution (#85360) (#85403) --- .../conventions/info/ParallelDetector.java | 14 +++-- .../internal/info/GlobalBuildInfoPlugin.java | 56 ------------------- 2 files changed, 10 insertions(+), 60 deletions(-) diff --git a/build-conventions/src/main/java/org/elasticsearch/gradle/internal/conventions/info/ParallelDetector.java b/build-conventions/src/main/java/org/elasticsearch/gradle/internal/conventions/info/ParallelDetector.java index 53b48c0c58935..9512d38e9bf96 100644 --- a/build-conventions/src/main/java/org/elasticsearch/gradle/internal/conventions/info/ParallelDetector.java +++ b/build-conventions/src/main/java/org/elasticsearch/gradle/internal/conventions/info/ParallelDetector.java @@ -62,19 +62,25 @@ public static int findDefaultParallel(Project project) { } else if (isMac(project.getProviders())) { // Ask macOS to count physical CPUs for us ByteArrayOutputStream stdout = new ByteArrayOutputStream(); + + // On Apple silicon, we only want to use the performance cores + String query = project.getProviders().systemProperty("os.arch").getOrElse("").equals("aarch64") + ? "hw.perflevel0.physicalcpu" + : "hw.physicalcpu"; + project.exec(spec -> { spec.setExecutable("sysctl"); - spec.args("-n", "hw.physicalcpu"); + spec.args("-n", query); spec.setStandardOutput(stdout); }); _defaultParallel = Integer.parseInt(stdout.toString().trim()); + } else { + _defaultParallel = Runtime.getRuntime().availableProcessors() / 2; } - - _defaultParallel = Runtime.getRuntime().availableProcessors() / 2; } - return _defaultParallel; + return Math.min(_defaultParallel, project.getGradle().getStartParameter().getMaxWorkerCount()); } private static boolean isMac(ProviderFactory providers) { diff --git a/build-tools-internal/src/main/java/org/elasticsearch/gradle/internal/info/GlobalBuildInfoPlugin.java b/build-tools-internal/src/main/java/org/elasticsearch/gradle/internal/info/GlobalBuildInfoPlugin.java index be44f003bf036..06e85abd4289f 100644 --- a/build-tools-internal/src/main/java/org/elasticsearch/gradle/internal/info/GlobalBuildInfoPlugin.java +++ b/build-tools-internal/src/main/java/org/elasticsearch/gradle/internal/info/GlobalBuildInfoPlugin.java @@ -8,7 +8,6 @@ package org.elasticsearch.gradle.internal.info; import org.apache.commons.io.IOUtils; -import org.elasticsearch.gradle.OS; import org.elasticsearch.gradle.internal.BwcVersions; import org.elasticsearch.gradle.internal.conventions.info.GitInfo; import org.elasticsearch.gradle.internal.conventions.info.ParallelDetector; @@ -30,10 +29,8 @@ import org.gradle.util.GradleVersion; import java.io.BufferedReader; -import java.io.ByteArrayOutputStream; import java.io.File; import java.io.FileInputStream; -import java.io.FileReader; import java.io.IOException; import java.io.InputStreamReader; import java.io.UncheckedIOException; @@ -41,10 +38,8 @@ import java.time.ZoneOffset; import java.time.ZonedDateTime; import java.util.Arrays; -import java.util.HashMap; import java.util.List; import java.util.Locale; -import java.util.Map; import java.util.Random; import java.util.concurrent.atomic.AtomicReference; import java.util.stream.Collectors; @@ -55,7 +50,6 @@ public class GlobalBuildInfoPlugin implements Plugin { private static final Logger LOGGER = Logging.getLogger(GlobalBuildInfoPlugin.class); private static final String DEFAULT_VERSION_JAVA_FILE_PATH = "server/src/main/java/org/elasticsearch/Version.java"; - private static Integer _defaultParallel = null; private final JavaInstallationRegistry javaInstallationRegistry; private final JvmMetadataDetector metadataDetector; @@ -305,56 +299,6 @@ private static String getJavaHomeEnvVarName(String version) { return "JAVA" + version + "_HOME"; } - private static int findDefaultParallel(Project project) { - // Since it costs IO to compute this, and is done at configuration time we want to cache this if possible - // It's safe to store this in a static variable since it's just a primitive so leaking memory isn't an issue - if (_defaultParallel == null) { - File cpuInfoFile = new File("/proc/cpuinfo"); - if (cpuInfoFile.exists()) { - // Count physical cores on any Linux distro ( don't count hyper-threading ) - Map socketToCore = new HashMap<>(); - String currentID = ""; - - try (BufferedReader reader = new BufferedReader(new FileReader(cpuInfoFile))) { - for (String line = reader.readLine(); line != null; line = reader.readLine()) { - if (line.contains(":")) { - List parts = Arrays.stream(line.split(":", 2)).map(String::trim).collect(Collectors.toList()); - String name = parts.get(0); - String value = parts.get(1); - // the ID of the CPU socket - if (name.equals("physical id")) { - currentID = value; - } - // Number of cores not including hyper-threading - if (name.equals("cpu cores")) { - assert currentID.isEmpty() == false; - socketToCore.put("currentID", Integer.valueOf(value)); - currentID = ""; - } - } - } - } catch (IOException e) { - throw new UncheckedIOException(e); - } - _defaultParallel = socketToCore.values().stream().mapToInt(i -> i).sum(); - } else if (OS.current() == OS.MAC) { - // Ask macOS to count physical CPUs for us - ByteArrayOutputStream stdout = new ByteArrayOutputStream(); - project.exec(spec -> { - spec.setExecutable("sysctl"); - spec.args("-n", "hw.physicalcpu"); - spec.setStandardOutput(stdout); - }); - - _defaultParallel = Integer.parseInt(stdout.toString().trim()); - } - - _defaultParallel = Runtime.getRuntime().availableProcessors() / 2; - } - - return _defaultParallel; - } - public static String getResourceContents(String resourcePath) { try ( BufferedReader reader = new BufferedReader(new InputStreamReader(GlobalBuildInfoPlugin.class.getResourceAsStream(resourcePath)))