From 854f1a3c88cf412a169dbd5723aaa400007a001b Mon Sep 17 00:00:00 2001 From: Ryan Ernst Date: Tue, 9 Sep 2025 16:18:08 -0700 Subject: [PATCH] Add initial transport version generation task (#134333) This commit adds a new generateInitialTransportVersion task to gradle. This task will be called by release automation to ensure each new release version of Elasticsearch has a unique transport version id. --- .../AbstractTransportVersionFuncTest.groovy | 18 +++- ...rateInitialTransportVersionFuncTest.groovy | 93 +++++++++++++++++++ .../TransportVersionGenerationFuncTest.groovy | 2 +- .../GenerateInitialTransportVersionTask.java | 71 ++++++++++++++ .../TransportVersionResourcesPlugin.java | 20 +++- .../TransportVersionResourcesService.java | 10 ++ .../fixtures/AbstractGradleFuncTest.groovy | 3 +- 7 files changed, 209 insertions(+), 8 deletions(-) create mode 100644 build-tools-internal/src/integTest/groovy/org/elasticsearch/gradle/internal/transport/GenerateInitialTransportVersionFuncTest.groovy create mode 100644 build-tools-internal/src/main/java/org/elasticsearch/gradle/internal/transport/GenerateInitialTransportVersionTask.java diff --git a/build-tools-internal/src/integTest/groovy/org/elasticsearch/gradle/internal/transport/AbstractTransportVersionFuncTest.groovy b/build-tools-internal/src/integTest/groovy/org/elasticsearch/gradle/internal/transport/AbstractTransportVersionFuncTest.groovy index edfca7e6918b5..f672d3fdf060e 100644 --- a/build-tools-internal/src/integTest/groovy/org/elasticsearch/gradle/internal/transport/AbstractTransportVersionFuncTest.groovy +++ b/build-tools-internal/src/integTest/groovy/org/elasticsearch/gradle/internal/transport/AbstractTransportVersionFuncTest.groovy @@ -93,10 +93,21 @@ class AbstractTransportVersionFuncTest extends AbstractGradleFuncTest { assert file("myserver/src/main/resources/transport/definitions/referable/${name}.csv").exists() == false } + void assertUnreferableDefinition(String name, String content) { + File definitionFile = file("myserver/src/main/resources/transport/definitions/unreferable/${name}.csv") + assert definitionFile.exists() + assert definitionFile.text.strip() == content + } + void assertUpperBound(String name, String content) { assert file("myserver/src/main/resources/transport/upper_bounds/${name}.csv").text.strip() == content } + void assertNoChanges() { + String output = execute("git diff") + assert output.strip().isEmpty() : "Expected no local git changes, but found:${System.lineSeparator()}${output}" + } + def setup() { configurationCacheCompatible = false internalBuild() @@ -104,6 +115,7 @@ class AbstractTransportVersionFuncTest extends AbstractGradleFuncTest { include ':myserver' include ':myplugin' """ + versionPropertiesFile.text = versionPropertiesFile.text.replace("9.1.0", "9.2.0") file("myserver/build.gradle") << """ apply plugin: 'java-library' @@ -116,10 +128,12 @@ class AbstractTransportVersionFuncTest extends AbstractGradleFuncTest { """ referableTransportVersion("existing_91", "8012000") referableTransportVersion("existing_92", "8123000,8012001") - unreferableTransportVersion("initial_9_0_0", "8000000") + unreferableTransportVersion("initial_9.0.0", "8000000") + unreferableTransportVersion("initial_8.19.7", "7123001") transportVersionUpperBound("9.2", "existing_92", "8123000") transportVersionUpperBound("9.1", "existing_92", "8012001") - transportVersionUpperBound("9.0", "initial_9_0_0", "8000000") + transportVersionUpperBound("9.0", "initial_9.0.0", "8000000") + transportVersionUpperBound("8.19", "initial_8.19.7", "7123001") // a mock version of TransportVersion, just here so we can compile Dummy.java et al javaSource("myserver", "org.elasticsearch", "TransportVersion", "", """ public static TransportVersion fromName(String name) { diff --git a/build-tools-internal/src/integTest/groovy/org/elasticsearch/gradle/internal/transport/GenerateInitialTransportVersionFuncTest.groovy b/build-tools-internal/src/integTest/groovy/org/elasticsearch/gradle/internal/transport/GenerateInitialTransportVersionFuncTest.groovy new file mode 100644 index 0000000000000..30582d9f7c8d4 --- /dev/null +++ b/build-tools-internal/src/integTest/groovy/org/elasticsearch/gradle/internal/transport/GenerateInitialTransportVersionFuncTest.groovy @@ -0,0 +1,93 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the "Elastic License + * 2.0", the "GNU Affero General Public License v3.0 only", and the "Server Side + * Public License v 1"; you may not use this file except in compliance with, at + * your election, the "Elastic License 2.0", the "GNU Affero General Public + * License v3.0 only", or the "Server Side Public License, v 1". + */ + +package org.elasticsearch.gradle.internal.transport + +import org.gradle.testkit.runner.BuildResult +import org.gradle.testkit.runner.TaskOutcome + +class GenerateInitialTransportVersionFuncTest extends AbstractTransportVersionFuncTest { + def runGenerateAndValidateTask(String... additionalArgs) { + List args = new ArrayList<>() + args.add(":myserver:validateTransportVersionResources") + args.add(":myserver:generateInitialTransportVersion") + args.addAll(additionalArgs); + return gradleRunner(args.toArray()) + } + + def runGenerateTask(String... additionalArgs) { + List args = new ArrayList<>() + args.add(":myserver:generateInitialTransportVersion") + args.addAll(additionalArgs); + return gradleRunner(args.toArray()) + } + + void assertGenerateSuccess(BuildResult result) { + assert result.task(":myserver:generateInitialTransportVersion").outcome == TaskOutcome.SUCCESS + } + + void assertGenerateFailure(BuildResult result, String expectedOutput) { + assert result.task(":myserver:generateInitialTransportVersion").outcome == TaskOutcome.FAILED + assertOutputContains(result.output, expectedOutput) + } + + void assertValidateSuccess(BuildResult result) { + assert result.task(":myserver:validateTransportVersionResources").outcome == TaskOutcome.SUCCESS + } + + void assertGenerateAndValidateSuccess(BuildResult result) { + assertGenerateSuccess(result) + assertValidateSuccess(result) + } + + def "setup is valid"() { + when: + def result = runGenerateAndValidateTask("--release-version", "9.0.0").build() + + then: + assertGenerateAndValidateSuccess(result) + // should have been idempotent, nothing actually changed + assertNoChanges(); + } + + def "new minor also creates next upper bound"() { + given: + // version properties will be updated by release automation before running initial version generation + versionPropertiesFile.text = versionPropertiesFile.text.replace("9.2.0", "9.3.0") + + when: + System.out.println("Running generation initial task") + def result = runGenerateAndValidateTask("--release-version", "9.2.0").build() + System.out.println("Done running generation task") + + then: + assertGenerateAndValidateSuccess(result) + assertUnreferableDefinition("initial_9.2.0", "8124000") + assertUpperBound("9.2", "initial_9.2.0,8124000") + assertUpperBound("9.3", "initial_9.2.0,8124000") + } + + def "patch updates existing upper bound"() { + when: + def result = runGenerateAndValidateTask("--release-version", "9.1.2").build() + + then: + assertGenerateAndValidateSuccess(result) + assertUnreferableDefinition("initial_9.1.2", "8012002") + assertUpperBound("9.1", "initial_9.1.2,8012002") + } + + def "cannot create upper bound file for patch"() { + when: + def result = runGenerateTask("--release-version", "9.3.7").buildAndFail() + + then: + assertGenerateFailure(result, "Missing upper bound 9.3 for release version 9.3.7") + } +} diff --git a/build-tools-internal/src/integTest/groovy/org/elasticsearch/gradle/internal/transport/TransportVersionGenerationFuncTest.groovy b/build-tools-internal/src/integTest/groovy/org/elasticsearch/gradle/internal/transport/TransportVersionGenerationFuncTest.groovy index 49f849a9996d3..21a8b6a95e105 100644 --- a/build-tools-internal/src/integTest/groovy/org/elasticsearch/gradle/internal/transport/TransportVersionGenerationFuncTest.groovy +++ b/build-tools-internal/src/integTest/groovy/org/elasticsearch/gradle/internal/transport/TransportVersionGenerationFuncTest.groovy @@ -412,7 +412,7 @@ class TransportVersionGenerationFuncTest extends AbstractTransportVersionFuncTes def result = runGenerateTask("--backport-branches=9.1,8.13,7.17,6.0").buildAndFail() then: - assertGenerateFailure(result, "Missing upper bounds files for branches [6.0, 7.17, 8.13], known branches are [9.0, 9.1, 9.2]") + assertGenerateFailure(result, "Missing upper bounds files for branches [6.0, 7.17, 8.13], known branches are [8.19, 9.0, 9.1, 9.2]") } def "name can be found from committed definition"() { diff --git a/build-tools-internal/src/main/java/org/elasticsearch/gradle/internal/transport/GenerateInitialTransportVersionTask.java b/build-tools-internal/src/main/java/org/elasticsearch/gradle/internal/transport/GenerateInitialTransportVersionTask.java new file mode 100644 index 0000000000000..a10c3fabb7ba9 --- /dev/null +++ b/build-tools-internal/src/main/java/org/elasticsearch/gradle/internal/transport/GenerateInitialTransportVersionTask.java @@ -0,0 +1,71 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the "Elastic License + * 2.0", the "GNU Affero General Public License v3.0 only", and the "Server Side + * Public License v 1"; you may not use this file except in compliance with, at + * your election, the "Elastic License 2.0", the "GNU Affero General Public + * License v3.0 only", or the "Server Side Public License, v 1". + */ + +package org.elasticsearch.gradle.internal.transport; + +import org.elasticsearch.gradle.Version; +import org.gradle.api.DefaultTask; +import org.gradle.api.provider.Property; +import org.gradle.api.services.ServiceReference; +import org.gradle.api.tasks.Input; +import org.gradle.api.tasks.TaskAction; +import org.gradle.api.tasks.options.Option; + +import java.io.IOException; +import java.util.List; + +public abstract class GenerateInitialTransportVersionTask extends DefaultTask { + + @ServiceReference("transportVersionResources") + abstract Property getResourceService(); + + @Input + @Option(option = "release-version", description = "The Elasticsearch release version this transport version will be associated with") + public abstract Property getReleaseVersion(); + + @Input + abstract Property getCurrentVersion(); + + @TaskAction + public void run() throws IOException { + Version releaseVersion = Version.fromString(getReleaseVersion().get()); + String upperBoundName = getUpperBoundName(releaseVersion); + TransportVersionResourcesService resources = getResourceService().get(); + TransportVersionUpperBound upstreamUpperBound = resources.getUpperBoundFromUpstream(upperBoundName); + String initialDefinitionName = "initial_" + releaseVersion; + TransportVersionDefinition existingDefinition = resources.getUnreferableDefinitionFromUpstream(initialDefinitionName); + + if (existingDefinition != null) { + // this initial version has already been created upstream + return; + } + + if (upstreamUpperBound == null) { + throw new RuntimeException("Missing upper bound " + upperBoundName + " for release version " + releaseVersion); + } + // minors increment by 1000 to create a unique base, patches increment by 1 as other patches do + int increment = releaseVersion.getRevision() == 0 ? 1000 : 1; + var id = TransportVersionId.fromInt(upstreamUpperBound.definitionId().complete() + increment); + var definition = new TransportVersionDefinition(initialDefinitionName, List.of(id)); + resources.writeUnreferableDefinition(definition); + var newUpperBound = new TransportVersionUpperBound(upperBoundName, initialDefinitionName, id); + resources.writeUpperBound(newUpperBound); + + if (releaseVersion.getRevision() == 0) { + Version currentVersion = getCurrentVersion().get(); + String currentUpperBoundName = getUpperBoundName(currentVersion); + var currentUpperBound = new TransportVersionUpperBound(currentUpperBoundName, initialDefinitionName, id); + resources.writeUpperBound(currentUpperBound); + } + } + + private String getUpperBoundName(Version version) { + return version.getMajor() + "." + version.getMinor(); + } +} diff --git a/build-tools-internal/src/main/java/org/elasticsearch/gradle/internal/transport/TransportVersionResourcesPlugin.java b/build-tools-internal/src/main/java/org/elasticsearch/gradle/internal/transport/TransportVersionResourcesPlugin.java index 931680eb96f5d..899c3ba0ddc0d 100644 --- a/build-tools-internal/src/main/java/org/elasticsearch/gradle/internal/transport/TransportVersionResourcesPlugin.java +++ b/build-tools-internal/src/main/java/org/elasticsearch/gradle/internal/transport/TransportVersionResourcesPlugin.java @@ -10,8 +10,8 @@ package org.elasticsearch.gradle.internal.transport; import org.elasticsearch.gradle.Version; -import org.elasticsearch.gradle.VersionProperties; import org.elasticsearch.gradle.internal.ProjectSubscribeServicePlugin; +import org.elasticsearch.gradle.internal.conventions.VersionPropertiesPlugin; import org.gradle.api.Plugin; import org.gradle.api.Project; import org.gradle.api.file.Directory; @@ -20,6 +20,7 @@ import org.gradle.language.base.plugins.LifecycleBasePlugin; import java.util.Map; +import java.util.Properties; public class TransportVersionResourcesPlugin implements Plugin { @@ -28,7 +29,12 @@ public class TransportVersionResourcesPlugin implements Plugin { @Override public void apply(Project project) { project.getPluginManager().apply(LifecycleBasePlugin.class); + project.getPluginManager().apply(VersionPropertiesPlugin.class); var psService = project.getPlugins().apply(ProjectSubscribeServicePlugin.class).getService(); + + Properties versions = (Properties) project.getExtensions().getByName(VersionPropertiesPlugin.VERSIONS_EXT); + Version currentVersion = Version.fromString(versions.getProperty("elasticsearch")); + var resourceRoot = getResourceRoot(project); String taskGroup = "Transport Versions"; @@ -79,11 +85,17 @@ public void apply(Project project) { t.setDescription("(Re)generates a transport version definition file"); t.getReferencesFiles().setFrom(tvReferencesConfig); t.getIncrement().convention(1000); - Version esVersion = VersionProperties.getElasticsearchVersion(); - t.getCurrentUpperBoundName().convention(esVersion.getMajor() + "." + esVersion.getMinor()); + t.getCurrentUpperBoundName().convention(currentVersion.getMajor() + "." + currentVersion.getMinor()); }); - validateTask.configure(t -> t.mustRunAfter(generateDefinitionsTask)); + + var generateInitialTask = project.getTasks() + .register("generateInitialTransportVersion", GenerateInitialTransportVersionTask.class, t -> { + t.setGroup(taskGroup); + t.setDescription("(Re)generates an initial transport version for an Elasticsearch release version"); + t.getCurrentVersion().set(currentVersion); + }); + validateTask.configure(t -> t.mustRunAfter(generateInitialTask)); } private static String getResourceRoot(Project project) { diff --git a/build-tools-internal/src/main/java/org/elasticsearch/gradle/internal/transport/TransportVersionResourcesService.java b/build-tools-internal/src/main/java/org/elasticsearch/gradle/internal/transport/TransportVersionResourcesService.java index 85590cb3cb196..7df2931c93717 100644 --- a/build-tools-internal/src/main/java/org/elasticsearch/gradle/internal/transport/TransportVersionResourcesService.java +++ b/build-tools-internal/src/main/java/org/elasticsearch/gradle/internal/transport/TransportVersionResourcesService.java @@ -174,6 +174,16 @@ Path getUnreferableDefinitionRepositoryPath(TransportVersionDefinition definitio return rootDir.relativize(transportResourcesDir.resolve(getUnreferableDefinitionRelativePath(definition.name()))); } + void writeUnreferableDefinition(TransportVersionDefinition definition) throws IOException { + Path path = transportResourcesDir.resolve(getUnreferableDefinitionRelativePath(definition.name())); + logger.debug("Writing unreferable definition [" + definition + "] to [" + path + "]"); + Files.writeString( + path, + definition.ids().stream().map(Object::toString).collect(Collectors.joining(",")) + "\n", + StandardCharsets.UTF_8 + ); + } + /** Read all upper bound files and return them mapped by their release name */ Map getUpperBounds() throws IOException { Map upperBounds = new HashMap<>(); diff --git a/build-tools/src/testFixtures/groovy/org/elasticsearch/gradle/fixtures/AbstractGradleFuncTest.groovy b/build-tools/src/testFixtures/groovy/org/elasticsearch/gradle/fixtures/AbstractGradleFuncTest.groovy index df14753375c64..553d5ea7d8d40 100644 --- a/build-tools/src/testFixtures/groovy/org/elasticsearch/gradle/fixtures/AbstractGradleFuncTest.groovy +++ b/build-tools/src/testFixtures/groovy/org/elasticsearch/gradle/fixtures/AbstractGradleFuncTest.groovy @@ -211,7 +211,7 @@ abstract class AbstractGradleFuncTest extends Specification { """ } - void execute(String command, File workingDir = testProjectDir.root) { + String execute(String command, File workingDir = testProjectDir.root) { def proc = command.execute(Collections.emptyList(), workingDir) proc.waitFor() if (proc.exitValue()) { @@ -221,6 +221,7 @@ abstract class AbstractGradleFuncTest extends Specification { """ throw new RuntimeException(msg) } + return proc.inputStream.text } File dir(String path) {