Skip to content

Commit

Permalink
[8.6] Improve efficiency of nested Gradle builds (#92889)
Browse files Browse the repository at this point in the history
  • Loading branch information
@mark-vieira
mark-vieira committed Jan 12, 2023
1 parent f0714b9 commit 487d741
Show file tree
Hide file tree
Showing 5 changed files with 52 additions and 30 deletions.
8 changes: 7 additions & 1 deletion .ci/packer_cache.sh
View file
Original file line number Diff line number Diff line change
Expand Up @@ -43,4 +43,10 @@ fi
## Gradle is able to resolve dependencies resolved with earlier gradle versions
## therefore we run main _AFTER_ we run 6.8 which uses an earlier gradle version
export JAVA_HOME="${HOME}"/.java/${ES_BUILD_JAVA}
./gradlew --parallel clean -s resolveAllDependencies -Dorg.gradle.warning.mode=none
./gradlew --parallel clean -s resolveAllDependencies -Dorg.gradle.warning.mode=none -Drecurse.bwc=true

## Copy all dependencies into a "read-only" location to be used by nested Gradle builds
mkdir -p ${HOME}/gradle_ro_cache
cp -R ${HOME}/.gradle/caches/modules-2 ${HOME}/gradle_ro_cache
rm ${HOME}/gradle_ro_cache/modules-2/gc.properties
rm ${HOME}/gradle_ro_cache/modules-2/*.lock
4 changes: 4 additions & 0 deletions build-tools-internal/build.gradle
View file
Original file line number Diff line number Diff line change
Expand Up @@ -336,6 +336,10 @@ tasks.register("bootstrapPerformanceTests", Copy) {
branchWrapper:"${-> gradle.gradleVersion}".toString()])
}

tasks.named("jar") {
exclude("classpath.index")
}

def resolveMainWrapperVersion() {
new URL("https://raw.githubusercontent.com/elastic/elasticsearch/main/build-tools-internal/src/main/resources/minimumGradleVersion").text.trim()
}
Expand Down
31 changes: 24 additions & 7 deletions build-tools-internal/src/main/java/org/elasticsearch/gradle/internal/BwcSetupExtension.java
View file
Original file line number Diff line number Diff line change
Expand Up @@ -12,6 +12,7 @@
import org.apache.tools.ant.taskdefs.condition.Os;
import org.elasticsearch.gradle.LoggedExec;
import org.elasticsearch.gradle.Version;
import org.elasticsearch.gradle.internal.info.BuildParams;
import org.gradle.api.Action;
import org.gradle.api.GradleException;
import org.gradle.api.Project;
Expand All @@ -37,45 +38,58 @@ public class BwcSetupExtension {
private static final Version BUILD_TOOL_MINIMUM_VERSION = Version.fromString("7.14.0");
private final Project project;
private final Provider<BwcVersions.UnreleasedVersionInfo> unreleasedVersionInfo;
private final Provider<InternalDistributionBwcSetupPlugin.BwcTaskThrottle> bwcTaskThrottleProvider;

private Provider<File> checkoutDir;

public BwcSetupExtension(
Project project,
Provider<BwcVersions.UnreleasedVersionInfo> unreleasedVersionInfo,
Provider<InternalDistributionBwcSetupPlugin.BwcTaskThrottle> bwcTaskThrottleProvider,
Provider<File> checkoutDir
) {
this.project = project;
this.unreleasedVersionInfo = unreleasedVersionInfo;
this.bwcTaskThrottleProvider = bwcTaskThrottleProvider;
this.checkoutDir = checkoutDir;
}

TaskProvider<LoggedExec> bwcTask(String name, Action<LoggedExec> configuration) {
return createRunBwcGradleTask(project, name, configuration);
return bwcTask(name, configuration, true);
}

private TaskProvider<LoggedExec> createRunBwcGradleTask(Project project, String name, Action<LoggedExec> configAction) {
TaskProvider<LoggedExec> bwcTask(String name, Action<LoggedExec> configuration, boolean useUniqueUserHome) {
return createRunBwcGradleTask(project, name, configuration, useUniqueUserHome);
}

private TaskProvider<LoggedExec> createRunBwcGradleTask(
Project project,
String name,
Action<LoggedExec> configAction,
boolean useUniqueUserHome
) {
return project.getTasks().register(name, LoggedExec.class, loggedExec -> {
loggedExec.dependsOn("checkoutBwcBranch");
loggedExec.usesService(bwcTaskThrottleProvider);
loggedExec.getWorkingDir().set(checkoutDir.get());

loggedExec.getEnvironment().put("JAVA_HOME", unreleasedVersionInfo.zip(checkoutDir, (version, checkoutDir) -> {
String minimumCompilerVersion = readFromFile(new File(checkoutDir, minimumCompilerVersionPath(version.version())));
return getJavaHome(Integer.parseInt(minimumCompilerVersion));
}));

if (BuildParams.isCi() && Os.isFamily(Os.FAMILY_WINDOWS) == false) {
loggedExec.getEnvironment().put("GRADLE_RO_DEP_CACHE", System.getProperty("user.home") + "/gradle_ro_cache");
}

if (Os.isFamily(Os.FAMILY_WINDOWS)) {
loggedExec.getExecutable().set("cmd");
loggedExec.args("/C", "call", new File(checkoutDir.get(), "gradlew").toString());
} else {
loggedExec.getExecutable().set(new File(checkoutDir.get(), "gradlew").toString());
}

loggedExec.args("-g", project.getGradle().getGradleUserHomeDir().toString());
if (useUniqueUserHome) {
loggedExec.dependsOn("setupGradleUserHome");
loggedExec.args("-g", project.getGradle().getGradleUserHomeDir().getAbsolutePath() + "-" + project.getName());
}

if (project.getGradle().getStartParameter().isOffline()) {
loggedExec.args("--offline");
}
Expand All @@ -101,6 +115,9 @@ private TaskProvider<LoggedExec> createRunBwcGradleTask(Project project, String
if (project.getGradle().getStartParameter().isParallelProjectExecutionEnabled()) {
loggedExec.args("--parallel");
}
for (File initScript : project.getGradle().getStartParameter().getInitScripts()) {
loggedExec.args("-I", initScript.getAbsolutePath());
}
loggedExec.getIndentingConsoleOutput().set(unreleasedVersionInfo.map(v -> v.version().toString()));
configAction.execute(loggedExec);
});
Expand Down
33 changes: 14 additions & 19 deletions ...l/src/main/java/org/elasticsearch/gradle/internal/InternalDistributionBwcSetupPlugin.java
View file
Original file line number Diff line number Diff line change
Expand Up @@ -18,8 +18,7 @@
import org.gradle.api.Task;
import org.gradle.api.provider.Provider;
import org.gradle.api.provider.ProviderFactory;
import org.gradle.api.services.BuildService;
import org.gradle.api.services.BuildServiceParameters;
import org.gradle.api.tasks.Copy;
import org.gradle.api.tasks.TaskProvider;
import org.gradle.language.base.plugins.LifecycleBasePlugin;

Expand All @@ -44,7 +43,6 @@
*/
public class InternalDistributionBwcSetupPlugin implements Plugin<Project> {

private static final String BWC_TASK_THROTTLE_SERVICE = "bwcTaskThrottle";
private ProviderFactory providerFactory;

@Inject
Expand All @@ -55,26 +53,19 @@ public InternalDistributionBwcSetupPlugin(ProviderFactory providerFactory) {
@Override
public void apply(Project project) {
project.getRootProject().getPluginManager().apply(GlobalBuildInfoPlugin.class);
Provider<BwcTaskThrottle> bwcTaskThrottleProvider = project.getGradle()
.getSharedServices()
.registerIfAbsent(BWC_TASK_THROTTLE_SERVICE, BwcTaskThrottle.class, spec -> spec.getMaxParallelUsages().set(1));
BuildParams.getBwcVersions()
.forPreviousUnreleased(
(BwcVersions.UnreleasedVersionInfo unreleasedVersion) -> {
configureBwcProject(project.project(unreleasedVersion.gradleProjectPath()), unreleasedVersion, bwcTaskThrottleProvider);
configureBwcProject(project.project(unreleasedVersion.gradleProjectPath()), unreleasedVersion);
}
);
}

private void configureBwcProject(
Project project,
BwcVersions.UnreleasedVersionInfo versionInfo,
Provider<BwcTaskThrottle> bwcTaskThrottleProvider
) {
private void configureBwcProject(Project project, BwcVersions.UnreleasedVersionInfo versionInfo) {
Provider<BwcVersions.UnreleasedVersionInfo> versionInfoProvider = providerFactory.provider(() -> versionInfo);
Provider<File> checkoutDir = versionInfoProvider.map(info -> new File(project.getBuildDir(), "bwc/checkout-" + info.branch()));
BwcSetupExtension bwcSetupExtension = project.getExtensions()
.create("bwcSetup", BwcSetupExtension.class, project, versionInfoProvider, bwcTaskThrottleProvider, checkoutDir);
.create("bwcSetup", BwcSetupExtension.class, project, versionInfoProvider, checkoutDir);
BwcGitExtension gitExtension = project.getPlugins().apply(InternalBwcGitPlugin.class).getGitExtension();
Provider<Version> bwcVersion = versionInfoProvider.map(info -> info.version());
gitExtension.setBwcVersion(versionInfoProvider.map(info -> info.version()));
Expand All @@ -87,6 +78,15 @@ private void configureBwcProject(
TaskProvider<Task> buildBwcTaskProvider = project.getTasks().register("buildBwc");
List<DistributionProject> distributionProjects = resolveArchiveProjects(checkoutDir.get(), bwcVersion.get());

// Setup gradle user home directory
project.getTasks().register("setupGradleUserHome", Copy.class, copy -> {
copy.into(project.getGradle().getGradleUserHomeDir().getAbsolutePath() + "-" + project.getName());
copy.from(project.getGradle().getGradleUserHomeDir().getAbsolutePath(), copySpec -> {
copySpec.include("gradle.properties");
copySpec.include("init.d/*");
});
});

for (DistributionProject distributionProject : distributionProjects) {
createBuildBwcTask(
bwcSetupExtension,
Expand Down Expand Up @@ -237,11 +237,7 @@ static void createBuildBwcTask(
} else {
c.getOutputs().files(expectedOutputFile);
}
c.getOutputs().cacheIf("BWC distribution caching is disabled on 'main' branch", task -> {
String gitBranch = System.getenv("GIT_BRANCH");
return BuildParams.isCi()
&& (gitBranch == null || gitBranch.endsWith("master") == false || gitBranch.endsWith("main") == false);
});
c.getOutputs().doNotCacheIf("BWC distribution caching is disabled for local builds", task -> BuildParams.isCi() == false);
c.getArgs().add(projectPath.replace('/', ':') + ":" + assembleTaskName);
if (project.getGradle().getStartParameter().isBuildCacheEnabled()) {
c.getArgs().add("--build-cache");
Expand Down Expand Up @@ -323,5 +319,4 @@ private static class DistributionProjectArtifact {
}
}

public abstract class BwcTaskThrottle implements BuildService<BuildServiceParameters.None> {}
}
6 changes: 3 additions & 3 deletions distribution/bwc/build.gradle
View file
Original file line number Diff line number Diff line change
Expand Up @@ -15,10 +15,10 @@ import org.elasticsearch.gradle.internal.info.BuildParams
BuildParams.getBwcVersions().forPreviousUnreleased { unreleasedVersion ->
project(unreleasedVersion.gradleProjectPath) {
Version currentVersion = Version.fromString(version)
TaskProvider<Task> resolveAllBwcDepsTaskProvider = bwcSetup.bwcTask("resolveAllBwcDependencies") {
TaskProvider<Task> resolveAllBwcDepsTaskProvider = bwcSetup.bwcTask("resolveAllBwcDependencies", {
t -> t.args("resolveAllDependencies", "-Dorg.gradle.warning.mode=none")
}
if (currentVersion.getMinor() == 0 && currentVersion.getRevision() == 0) {
}, false)
if (Boolean.getBoolean("recurse.bwc")) {
// We only want to resolve dependencies for live versions of main, without cascading this to older versions
tasks.named("resolveAllDependencies").configure {
dependsOn(resolveAllBwcDepsTaskProvider)
Expand Down

0 comments on commit 487d741

Please sign in to comment.