Skip to content

Commit

Permalink
Merge branch 'main' into esql/nested-aggs
Browse files Browse the repository at this point in the history
  • Loading branch information
costin committed Feb 6, 2024
2 parents feeedfd + af163b2 commit afc410b
Show file tree
Hide file tree
Showing 604 changed files with 19,452 additions and 5,187 deletions.
2 changes: 1 addition & 1 deletion .buildkite/pull-requests.json
Original file line number Diff line number Diff line change
Expand Up @@ -11,7 +11,7 @@
"set_commit_status": false,
"build_on_commit": true,
"build_on_comment": true,
"trigger_comment_regex": "(run\\W+elasticsearch-ci.+)|(^\\s*(buildkite\\s*)?test\\s+this(\\s+please)?)",
"trigger_comment_regex": "(run\\W+elasticsearch-ci.+)|(^\\s*((buildkite|@elastic(search)?machine)\\s*)?test\\s+this(\\s+please)?)",
"cancel_intermediate_builds": true,
"cancel_intermediate_builds_on_comment": false
},
Expand Down
108 changes: 108 additions & 0 deletions .buildkite/scripts/pull-request/__snapshots__/pipeline.test.ts.snap
Original file line number Diff line number Diff line change
Expand Up @@ -201,3 +201,111 @@ exports[`generatePipelines should generate correct pipeline when using a trigger
},
]
`;

exports[`generatePipelines should generate correct pipelines with a non-docs change and @elasticmachine 1`] = `
[
{
"name": "bwc-snapshots",
"pipeline": {
"steps": [
{
"group": "bwc-snapshots",
"steps": [
{
"agents": {
"buildDirectory": "/dev/shm/bk",
"image": "family/elasticsearch-ubuntu-2004",
"machineType": "custom-32-98304",
"provider": "gcp",
},
"command": ".ci/scripts/run-gradle.sh -Dignore.tests.seed v{{matrix.BWC_VERSION}}#bwcTest",
"env": {
"BWC_VERSION": "{{matrix.BWC_VERSION}}",
},
"label": "{{matrix.BWC_VERSION}} / bwc-snapshots",
"matrix": {
"setup": {
"BWC_VERSION": [
"7.17.14",
"8.10.3",
"8.11.0",
],
},
},
"timeout_in_minutes": 300,
},
],
},
],
},
},
{
"name": "using-defaults",
"pipeline": {
"env": {
"CUSTOM_ENV_VAR": "value",
},
"steps": [
{
"command": "echo 'hello world'",
"label": "test-step",
},
],
},
},
]
`;

exports[`generatePipelines should generate correct pipelines with a non-docs change and @elasticsearchmachine 1`] = `
[
{
"name": "bwc-snapshots",
"pipeline": {
"steps": [
{
"group": "bwc-snapshots",
"steps": [
{
"agents": {
"buildDirectory": "/dev/shm/bk",
"image": "family/elasticsearch-ubuntu-2004",
"machineType": "custom-32-98304",
"provider": "gcp",
},
"command": ".ci/scripts/run-gradle.sh -Dignore.tests.seed v{{matrix.BWC_VERSION}}#bwcTest",
"env": {
"BWC_VERSION": "{{matrix.BWC_VERSION}}",
},
"label": "{{matrix.BWC_VERSION}} / bwc-snapshots",
"matrix": {
"setup": {
"BWC_VERSION": [
"7.17.14",
"8.10.3",
"8.11.0",
],
},
},
"timeout_in_minutes": 300,
},
],
},
],
},
},
{
"name": "using-defaults",
"pipeline": {
"env": {
"CUSTOM_ENV_VAR": "value",
},
"steps": [
{
"command": "echo 'hello world'",
"label": "test-step",
},
],
},
},
]
`;
20 changes: 18 additions & 2 deletions .buildkite/scripts/pull-request/pipeline.test.ts
Original file line number Diff line number Diff line change
Expand Up @@ -13,11 +13,11 @@ describe("generatePipelines", () => {
});

// Helper for testing pipeline generations that should be the same when using the overall ci trigger comment "buildkite test this"
const testWithTriggerCheck = (directory: string, changedFiles?: string[]) => {
const testWithTriggerCheck = (directory: string, changedFiles?: string[], comment = "buildkite test this") => {
const pipelines = generatePipelines(directory, changedFiles);
expect(pipelines).toMatchSnapshot();

process.env["GITHUB_PR_TRIGGER_COMMENT"] = "buildkite test this";
process.env["GITHUB_PR_TRIGGER_COMMENT"] = comment;
const pipelinesWithTriggerComment = generatePipelines(directory, changedFiles);
expect(pipelinesWithTriggerComment).toEqual(pipelines);
};
Expand All @@ -42,4 +42,20 @@ describe("generatePipelines", () => {
const pipelines = generatePipelines(`${import.meta.dir}/mocks/pipelines`, ["build.gradle"]);
expect(pipelines).toMatchSnapshot();
});

test("should generate correct pipelines with a non-docs change and @elasticmachine", () => {
testWithTriggerCheck(
`${import.meta.dir}/mocks/pipelines`,
["build.gradle", "docs/README.asciidoc"],
"@elasticmachine test this please"
);
});

test("should generate correct pipelines with a non-docs change and @elasticsearchmachine", () => {
testWithTriggerCheck(
`${import.meta.dir}/mocks/pipelines`,
["build.gradle", "docs/README.asciidoc"],
"@elasticsearchmachine test this please"
);
});
});
4 changes: 3 additions & 1 deletion .buildkite/scripts/pull-request/pipeline.ts
Original file line number Diff line number Diff line change
Expand Up @@ -148,7 +148,9 @@ export const generatePipelines = (
// However, if we're using the overall CI trigger "[buildkite] test this [please]", we should use the regular filters above
if (
process.env["GITHUB_PR_TRIGGER_COMMENT"] &&
!process.env["GITHUB_PR_TRIGGER_COMMENT"].match(/^\s*(buildkite\s*)?test\s+this(\s+please)?/i)
!process.env["GITHUB_PR_TRIGGER_COMMENT"].match(
/^\s*((@elastic(search)?machine|buildkite)\s*)?test\s+this(\s+please)?/i
)
) {
filters = [triggerCommentCheck];
}
Expand Down
4 changes: 4 additions & 0 deletions build-tools-internal/build.gradle
Original file line number Diff line number Diff line change
Expand Up @@ -143,6 +143,10 @@ gradlePlugin {
id = 'elasticsearch.mrjar'
implementationClass = 'org.elasticsearch.gradle.internal.MrjarPlugin'
}
embeddedProvider {
id = 'elasticsearch.embedded-providers'
implementationClass = 'org.elasticsearch.gradle.internal.EmbeddedProviderPlugin'
}
releaseTools {
id = 'elasticsearch.release-tools'
implementationClass = 'org.elasticsearch.gradle.internal.release.ReleaseToolsPlugin'
Expand Down
Original file line number Diff line number Diff line change
@@ -0,0 +1,62 @@
/*
* Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
* or more contributor license agreements. Licensed under the Elastic License
* 2.0 and the Server Side Public License, v 1; you may not use this file except
* in compliance with, at your election, the Elastic License 2.0 or the Server
* Side Public License, v 1.
*/

package org.elasticsearch.gradle.internal;

import org.gradle.api.Project;
import org.gradle.api.artifacts.Configuration;
import org.gradle.api.file.Directory;
import org.gradle.api.provider.Provider;
import org.gradle.api.tasks.SourceSet;
import org.gradle.api.tasks.Sync;

import static org.elasticsearch.gradle.internal.conventions.GUtils.capitalize;
import static org.elasticsearch.gradle.util.GradleUtils.getJavaSourceSets;
import static org.gradle.api.artifacts.type.ArtifactTypeDefinition.ARTIFACT_TYPE_ATTRIBUTE;
import static org.gradle.api.artifacts.type.ArtifactTypeDefinition.DIRECTORY_TYPE;

public class EmbeddedProviderExtension {

private final Project project;

public EmbeddedProviderExtension(Project project) {
this.project = project;
}

void impl(String implName, Project implProject) {
String projectName = implProject.getName();
String capitalName = capitalize(projectName);

Configuration implConfig = project.getConfigurations().detachedConfiguration(project.getDependencies().create(implProject));
implConfig.attributes(attrs -> {
attrs.attribute(ARTIFACT_TYPE_ATTRIBUTE, DIRECTORY_TYPE);
attrs.attribute(EmbeddedProviderPlugin.IMPL_ATTR, true);
});

String manifestTaskName = "generate" + capitalName + "ProviderManifest";
Provider<Directory> generatedResourcesDir = project.getLayout().getBuildDirectory().dir("generated-resources");
var generateProviderManifest = project.getTasks().register(manifestTaskName, GenerateProviderManifest.class);
generateProviderManifest.configure(t -> {
t.getManifestFile().set(generatedResourcesDir.map(d -> d.file("LISTING.TXT")));
t.getProviderImplClasspath().from(implConfig);
});

String implTaskName = "generate" + capitalName + "ProviderImpl";
var generateProviderImpl = project.getTasks().register(implTaskName, Sync.class);
generateProviderImpl.configure(t -> {
t.into(generatedResourcesDir);
t.into("IMPL-JARS/" + implName, spec -> {
spec.from(implConfig);
spec.from(generateProviderManifest);
});
});

var mainSourceSet = getJavaSourceSets(project).findByName(SourceSet.MAIN_SOURCE_SET_NAME);
mainSourceSet.getOutput().dir(generateProviderImpl);
}
}
Original file line number Diff line number Diff line change
@@ -0,0 +1,34 @@
/*
* Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
* or more contributor license agreements. Licensed under the Elastic License
* 2.0 and the Server Side Public License, v 1; you may not use this file except
* in compliance with, at your election, the Elastic License 2.0 or the Server
* Side Public License, v 1.
*/

package org.elasticsearch.gradle.internal;

import org.elasticsearch.gradle.transform.UnzipTransform;
import org.gradle.api.Plugin;
import org.gradle.api.Project;
import org.gradle.api.attributes.Attribute;

import static org.gradle.api.artifacts.type.ArtifactTypeDefinition.ARTIFACT_TYPE_ATTRIBUTE;
import static org.gradle.api.artifacts.type.ArtifactTypeDefinition.DIRECTORY_TYPE;
import static org.gradle.api.artifacts.type.ArtifactTypeDefinition.JAR_TYPE;

public class EmbeddedProviderPlugin implements Plugin<Project> {
static final Attribute<Boolean> IMPL_ATTR = Attribute.of("is.impl", Boolean.class);

@Override
public void apply(Project project) {

project.getDependencies().registerTransform(UnzipTransform.class, transformSpec -> {
transformSpec.getFrom().attribute(ARTIFACT_TYPE_ATTRIBUTE, JAR_TYPE).attribute(IMPL_ATTR, true);
transformSpec.getTo().attribute(ARTIFACT_TYPE_ATTRIBUTE, DIRECTORY_TYPE).attribute(IMPL_ATTR, true);
transformSpec.parameters(parameters -> parameters.getIncludeArtifactName().set(true));
});

project.getExtensions().create("embeddedProviders", EmbeddedProviderExtension.class, project);
}
}
Original file line number Diff line number Diff line change
Expand Up @@ -98,6 +98,7 @@ public abstract class CheckForbiddenApisTask extends DefaultTask implements Patt
private File resourcesDir;

private boolean ignoreFailures = false;
private boolean ignoreMissingClasses = false;

@Input
@Optional
Expand Down Expand Up @@ -250,6 +251,15 @@ public void setIgnoreFailures(boolean ignoreFailures) {
this.ignoreFailures = ignoreFailures;
}

@Input
public boolean getIgnoreMissingClasses() {
return ignoreMissingClasses;
}

public void setIgnoreMissingClasses(boolean ignoreMissingClasses) {
this.ignoreMissingClasses = ignoreMissingClasses;
}

/**
* The default compiler target version used to expand references to bundled JDK signatures.
* E.g., if you use "jdk-deprecated", it will expand to this version.
Expand Down Expand Up @@ -378,6 +388,7 @@ public void checkForbidden() {
parameters.getSignatures().set(getSignatures());
parameters.getTargetCompatibility().set(getTargetCompatibility());
parameters.getIgnoreFailures().set(getIgnoreFailures());
parameters.getIgnoreMissingClasses().set(getIgnoreMissingClasses());
parameters.getSuccessMarker().set(getSuccessMarker());
parameters.getSignaturesFiles().from(getSignaturesFiles());
});
Expand Down Expand Up @@ -514,7 +525,9 @@ private URLClassLoader createClassLoader(FileCollection classpath, FileCollectio
@NotNull
private Checker createChecker(URLClassLoader urlLoader) {
final EnumSet<Checker.Option> options = EnumSet.noneOf(Checker.Option.class);
options.add(FAIL_ON_MISSING_CLASSES);
if (getParameters().getIgnoreMissingClasses().get() == false) {
options.add(FAIL_ON_MISSING_CLASSES);
}
if (getParameters().getIgnoreFailures().get() == false) {
options.add(FAIL_ON_VIOLATION);
}
Expand Down Expand Up @@ -573,6 +586,8 @@ interface Parameters extends WorkParameters {

Property<Boolean> getIgnoreFailures();

Property<Boolean> getIgnoreMissingClasses();

ListProperty<String> getSignatures();

}
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -101,7 +101,7 @@ private AdoptiumVersionInfo toVersionInfo(JsonNode node) {
private URI resolveDownloadURI(AdoptiumVersionRequest request, AdoptiumVersionInfo versionInfo) {
return URI.create(
"https://api.adoptium.net/v3/binary/version/jdk-"
+ versionInfo.openjdkVersion
+ versionInfo.semver
+ "/"
+ request.platform
+ "/"
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -59,6 +59,7 @@
"Infra/Scripting",
"Infra/Settings",
"Infra/Transport API",
"Infra/Metrics",
"Ingest",
"Ingest Node",
"Java High Level REST Client",
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -42,7 +42,7 @@ class AdoptiumJdkToolchainResolverSpec extends AbstractToolchainResolverSpec {
1,
1,
"" + languageVersion.asInt() + ".1.1.1+37",
0, "" + languageVersion.asInt() + ".1.1.1"
0, "" + languageVersion.asInt() + ".1.1.1+37.1"
)))

}
Expand All @@ -52,22 +52,22 @@ class AdoptiumJdkToolchainResolverSpec extends AbstractToolchainResolverSpec {
@Override
def supportedRequests() {
return [
[19, ADOPTIUM, MAC_OS, X86_64, "https://api.adoptium.net/v3/binary/version/jdk-19.1.1.1+37/mac/x64/jdk/hotspot/normal/eclipse?project=jdk"],
[19, ADOPTIUM, LINUX, X86_64, "https://api.adoptium.net/v3/binary/version/jdk-19.1.1.1+37/linux/x64/jdk/hotspot/normal/eclipse?project=jdk"],
[19, ADOPTIUM, WINDOWS, X86_64, "https://api.adoptium.net/v3/binary/version/jdk-19.1.1.1+37/windows/x64/jdk/hotspot/normal/eclipse?project=jdk"],
[19, ADOPTIUM, MAC_OS, AARCH64, "https://api.adoptium.net/v3/binary/version/jdk-19.1.1.1+37/mac/aarch64/jdk/hotspot/normal/eclipse?project=jdk"],
[19, ADOPTIUM, LINUX, AARCH64, "https://api.adoptium.net/v3/binary/version/jdk-19.1.1.1+37/linux/aarch64/jdk/hotspot/normal/eclipse?project=jdk"],
[19, ADOPTIUM, MAC_OS, X86_64, "https://api.adoptium.net/v3/binary/version/jdk-19.1.1.1+37.1/mac/x64/jdk/hotspot/normal/eclipse?project=jdk"],
[19, ADOPTIUM, LINUX, X86_64, "https://api.adoptium.net/v3/binary/version/jdk-19.1.1.1+37.1/linux/x64/jdk/hotspot/normal/eclipse?project=jdk"],
[19, ADOPTIUM, WINDOWS, X86_64, "https://api.adoptium.net/v3/binary/version/jdk-19.1.1.1+37.1/windows/x64/jdk/hotspot/normal/eclipse?project=jdk"],
[19, ADOPTIUM, MAC_OS, AARCH64, "https://api.adoptium.net/v3/binary/version/jdk-19.1.1.1+37.1/mac/aarch64/jdk/hotspot/normal/eclipse?project=jdk"],
[19, ADOPTIUM, LINUX, AARCH64, "https://api.adoptium.net/v3/binary/version/jdk-19.1.1.1+37.1/linux/aarch64/jdk/hotspot/normal/eclipse?project=jdk"],

[18, ADOPTIUM, MAC_OS, X86_64, "https://api.adoptium.net/v3/binary/version/jdk-18.1.1.1+37/mac/x64/jdk/hotspot/normal/eclipse?project=jdk"],
[18, ADOPTIUM, LINUX, X86_64, "https://api.adoptium.net/v3/binary/version/jdk-18.1.1.1+37/linux/x64/jdk/hotspot/normal/eclipse?project=jdk"],
[18, ADOPTIUM, WINDOWS, X86_64, "https://api.adoptium.net/v3/binary/version/jdk-18.1.1.1+37/windows/x64/jdk/hotspot/normal/eclipse?project=jdk"],
[18, ADOPTIUM, MAC_OS, AARCH64, "https://api.adoptium.net/v3/binary/version/jdk-18.1.1.1+37/mac/aarch64/jdk/hotspot/normal/eclipse?project=jdk"],
[18, ADOPTIUM, LINUX, AARCH64, "https://api.adoptium.net/v3/binary/version/jdk-18.1.1.1+37/linux/aarch64/jdk/hotspot/normal/eclipse?project=jdk"],
[17, ADOPTIUM, MAC_OS, X86_64, "https://api.adoptium.net/v3/binary/version/jdk-17.1.1.1+37/mac/x64/jdk/hotspot/normal/eclipse?project=jdk"],
[17, ADOPTIUM, LINUX, X86_64, "https://api.adoptium.net/v3/binary/version/jdk-17.1.1.1+37/linux/x64/jdk/hotspot/normal/eclipse?project=jdk"],
[17, ADOPTIUM, WINDOWS, X86_64, "https://api.adoptium.net/v3/binary/version/jdk-17.1.1.1+37/windows/x64/jdk/hotspot/normal/eclipse?project=jdk"],
[17, ADOPTIUM, MAC_OS, AARCH64, "https://api.adoptium.net/v3/binary/version/jdk-17.1.1.1+37/mac/aarch64/jdk/hotspot/normal/eclipse?project=jdk"],
[17, ADOPTIUM, LINUX, AARCH64, "https://api.adoptium.net/v3/binary/version/jdk-17.1.1.1+37/linux/aarch64/jdk/hotspot/normal/eclipse?project=jdk"]
[18, ADOPTIUM, MAC_OS, X86_64, "https://api.adoptium.net/v3/binary/version/jdk-18.1.1.1+37.1/mac/x64/jdk/hotspot/normal/eclipse?project=jdk"],
[18, ADOPTIUM, LINUX, X86_64, "https://api.adoptium.net/v3/binary/version/jdk-18.1.1.1+37.1/linux/x64/jdk/hotspot/normal/eclipse?project=jdk"],
[18, ADOPTIUM, WINDOWS, X86_64, "https://api.adoptium.net/v3/binary/version/jdk-18.1.1.1+37.1/windows/x64/jdk/hotspot/normal/eclipse?project=jdk"],
[18, ADOPTIUM, MAC_OS, AARCH64, "https://api.adoptium.net/v3/binary/version/jdk-18.1.1.1+37.1/mac/aarch64/jdk/hotspot/normal/eclipse?project=jdk"],
[18, ADOPTIUM, LINUX, AARCH64, "https://api.adoptium.net/v3/binary/version/jdk-18.1.1.1+37.1/linux/aarch64/jdk/hotspot/normal/eclipse?project=jdk"],
[17, ADOPTIUM, MAC_OS, X86_64, "https://api.adoptium.net/v3/binary/version/jdk-17.1.1.1+37.1/mac/x64/jdk/hotspot/normal/eclipse?project=jdk"],
[17, ADOPTIUM, LINUX, X86_64, "https://api.adoptium.net/v3/binary/version/jdk-17.1.1.1+37.1/linux/x64/jdk/hotspot/normal/eclipse?project=jdk"],
[17, ADOPTIUM, WINDOWS, X86_64, "https://api.adoptium.net/v3/binary/version/jdk-17.1.1.1+37.1/windows/x64/jdk/hotspot/normal/eclipse?project=jdk"],
[17, ADOPTIUM, MAC_OS, AARCH64, "https://api.adoptium.net/v3/binary/version/jdk-17.1.1.1+37.1/mac/aarch64/jdk/hotspot/normal/eclipse?project=jdk"],
[17, ADOPTIUM, LINUX, AARCH64, "https://api.adoptium.net/v3/binary/version/jdk-17.1.1.1+37.1/linux/aarch64/jdk/hotspot/normal/eclipse?project=jdk"]
]
}

Expand Down
Loading

0 comments on commit afc410b

Please sign in to comment.