Skip to content

Commit

Permalink
Merge branch 'main' into 2024/05/31/update-complete-marker-on-progress
Browse files Browse the repository at this point in the history
  • Loading branch information
@tlrx
tlrx committed Jun 5, 2024
2 parents c12407d + 32dfd18 commit a4835cc
Show file tree
Hide file tree
Showing 367 changed files with 6,405 additions and 3,541 deletions.
16 changes: 8 additions & 8 deletions benchmarks/src/main/java/org/elasticsearch/benchmark/compute/operator/EvalBenchmark.java
View file
Original file line number Diff line number Diff line change
Expand Up @@ -26,7 +26,7 @@
import org.elasticsearch.xpack.esql.core.expression.Literal;
import org.elasticsearch.xpack.esql.core.expression.predicate.regex.RLikePattern;
import org.elasticsearch.xpack.esql.core.tree.Source;
import org.elasticsearch.xpack.esql.core.type.DataTypes;
import org.elasticsearch.xpack.esql.core.type.DataType;
import org.elasticsearch.xpack.esql.core.type.EsField;
import org.elasticsearch.xpack.esql.evaluator.EvalMapper;
import org.elasticsearch.xpack.esql.expression.function.scalar.date.DateTrunc;
Expand Down Expand Up @@ -101,25 +101,25 @@ private static EvalOperator.ExpressionEvaluator evaluator(String operation) {
case "add" -> {
FieldAttribute longField = longField();
yield EvalMapper.toEvaluator(
new Add(Source.EMPTY, longField, new Literal(Source.EMPTY, 1L, DataTypes.LONG)),
new Add(Source.EMPTY, longField, new Literal(Source.EMPTY, 1L, DataType.LONG)),
layout(longField)
).get(driverContext);
}
case "date_trunc" -> {
FieldAttribute timestamp = new FieldAttribute(
Source.EMPTY,
"timestamp",
new EsField("timestamp", DataTypes.DATETIME, Map.of(), true)
new EsField("timestamp", DataType.DATETIME, Map.of(), true)
);
yield EvalMapper.toEvaluator(
new DateTrunc(Source.EMPTY, new Literal(Source.EMPTY, Duration.ofHours(24), DataTypes.TIME_DURATION), timestamp),
new DateTrunc(Source.EMPTY, new Literal(Source.EMPTY, Duration.ofHours(24), DataType.TIME_DURATION), timestamp),
layout(timestamp)
).get(driverContext);
}
case "equal_to_const" -> {
FieldAttribute longField = longField();
yield EvalMapper.toEvaluator(
new Equals(Source.EMPTY, longField, new Literal(Source.EMPTY, 100_000L, DataTypes.LONG)),
new Equals(Source.EMPTY, longField, new Literal(Source.EMPTY, 100_000L, DataType.LONG)),
layout(longField)
).get(driverContext);
}
Expand Down Expand Up @@ -147,15 +147,15 @@ private static EvalOperator.ExpressionEvaluator evaluator(String operation) {
}

private static FieldAttribute longField() {
return new FieldAttribute(Source.EMPTY, "long", new EsField("long", DataTypes.LONG, Map.of(), true));
return new FieldAttribute(Source.EMPTY, "long", new EsField("long", DataType.LONG, Map.of(), true));
}

private static FieldAttribute intField() {
return new FieldAttribute(Source.EMPTY, "int", new EsField("int", DataTypes.INTEGER, Map.of(), true));
return new FieldAttribute(Source.EMPTY, "int", new EsField("int", DataType.INTEGER, Map.of(), true));
}

private static FieldAttribute keywordField() {
return new FieldAttribute(Source.EMPTY, "keyword", new EsField("keyword", DataTypes.KEYWORD, Map.of(), true));
return new FieldAttribute(Source.EMPTY, "keyword", new EsField("keyword", DataType.KEYWORD, Map.of(), true));
}

private static Layout layout(FieldAttribute... fields) {
Expand Down
30 changes: 30 additions & 0 deletions ...internal/src/main/java/org/elasticsearch/gradle/internal/docker/DockerSupportService.java
View file
Original file line number Diff line number Diff line change
Expand Up @@ -7,6 +7,8 @@
*/
package org.elasticsearch.gradle.internal.docker;

import com.avast.gradle.dockercompose.ServiceInfo;

import org.elasticsearch.gradle.Architecture;
import org.elasticsearch.gradle.OS;
import org.elasticsearch.gradle.Version;
Expand Down Expand Up @@ -56,6 +58,9 @@ public abstract class DockerSupportService implements BuildService<DockerSupport

private final ProviderFactory providerFactory;
private DockerAvailability dockerAvailability;
private Map<String, ServiceInfo> serviceInfos;
private Map<String, Map<Integer, Integer>> tcpPorts;
private Map<String, Map<Integer, Integer>> udpPorts;

@Inject
public DockerSupportService(ProviderFactory providerFactory) {
Expand Down Expand Up @@ -145,6 +150,10 @@ public DockerAvailability getDockerAvailability() {
return this.dockerAvailability;
}

public boolean isArchitectureSupported(Architecture architecture) {
return getDockerAvailability().supportedArchitectures().contains(architecture);
}

private DockerResult runCommand(List args, DockerValueSource.OutputFilter outputFilter) {
return providerFactory.of(DockerValueSource.class, params -> {
params.getParameters().getArgs().addAll(args);
Expand Down Expand Up @@ -329,6 +338,27 @@ private void throwDockerRequiredException(final String message, Exception e) {
);
}

public void storeInfo(Map<String, ServiceInfo> servicesInfos) {
tcpPorts = servicesInfos.entrySet()
.stream()
.collect(Collectors.toMap(entry -> entry.getKey(), entry -> entry.getValue().getTcpPorts()));
udpPorts = servicesInfos.entrySet()
.stream()
.collect(Collectors.toMap(entry -> entry.getKey(), entry -> entry.getValue().getUdpPorts()));
}

public Map<String, Map<Integer, Integer>> getTcpPorts() {
return tcpPorts;
}

public Map<String, Map<Integer, Integer>> getUdpPorts() {
return udpPorts;
}

public void setServiceInfos(Map<String, ServiceInfo> serviceInfos) {
this.serviceInfos = serviceInfos;
}

/**
* An immutable class that represents the results of a Docker search from {@link #getDockerAvailability()}}.
*/
Expand Down
55 changes: 27 additions & 28 deletions ...rnal/src/main/java/org/elasticsearch/gradle/internal/testfixtures/TestFixturesPlugin.java
View file
Original file line number Diff line number Diff line change
Expand Up @@ -9,7 +9,6 @@

import com.avast.gradle.dockercompose.ComposeExtension;
import com.avast.gradle.dockercompose.DockerComposePlugin;
import com.avast.gradle.dockercompose.ServiceInfo;
import com.avast.gradle.dockercompose.tasks.ComposeBuild;
import com.avast.gradle.dockercompose.tasks.ComposeDown;
import com.avast.gradle.dockercompose.tasks.ComposePull;
Expand Down Expand Up @@ -106,12 +105,6 @@ public void apply(Project project) {
.register("postProcessFixture", TestFixtureTask.class, task -> {
task.getFixturesDir().set(testFixturesDir);
task.dependsOn(buildFixture);
configureServiceInfoForTask(
task,
project,
false,
(name, port) -> task.getExtensions().getByType(ExtraPropertiesExtension.class).set(name, port)
);
});

maybeSkipTask(dockerSupport, preProcessFixture);
Expand All @@ -131,12 +124,20 @@ public void apply(Project project) {
return composePath != null ? composePath : "/usr/bin/docker-compose";
}));

tasks.named("composeUp").configure(t -> {
tasks.withType(ComposeUp.class).named("composeUp").configure(t -> {
// Avoid running docker-compose tasks in parallel in CI due to some issues on certain Linux distributions
if (BuildParams.isCi()) {
t.usesService(dockerComposeThrottle);
t.usesService(dockerSupport);
}
t.mustRunAfter(preProcessFixture);
t.doLast(new Action<Task>() {
@Override
public void execute(Task task) {
dockerSupport.get().storeInfo(t.getServicesInfos());
}
});

});
tasks.named("composePull").configure(t -> t.mustRunAfter(preProcessFixture));
tasks.named("composeDown").configure(t -> t.doLast(t2 -> getFileSystemOperations().delete(d -> d.delete(testFixturesDir))));
Expand All @@ -153,14 +154,9 @@ public void apply(Project project) {
tasks.withType(Test.class).configureEach(testTask -> {
testTask.dependsOn(postProcessFixture);
testTask.finalizedBy(tasks.named("composeDown"));
configureServiceInfoForTask(
testTask,
project,
true,
(name, host) -> testTask.getExtensions()
.getByType(SystemPropertyCommandLineArgumentProvider.class)
.systemProperty(name, host)
);
SystemPropertyCommandLineArgumentProvider sysArgumentsProvider = testTask.getExtensions()
.getByType(SystemPropertyCommandLineArgumentProvider.class);
configureServiceInfoForTask(testTask, dockerSupport, (name, host) -> sysArgumentsProvider.systemProperty(name, host));
});
}

Expand All @@ -184,31 +180,34 @@ private void maybeSkipTask(Provider<DockerSupportService> dockerSupport, Task ta

private void configureServiceInfoForTask(
Task task,
Project fixtureProject,
boolean enableFilter,
Provider<DockerSupportService> dockerSupportServiceProvider,
BiConsumer<String, Integer> consumer
) {
// Configure ports for the tests as system properties.
// We only know these at execution time so we need to do it in doFirst
task.usesService(dockerSupportServiceProvider);
task.doFirst(new Action<Task>() {
@Override
public void execute(Task theTask) {
fixtureProject.getExtensions().getByType(ComposeExtension.class).getServicesInfos().entrySet().stream().forEach(entry -> {
dockerSupportServiceProvider.get().getTcpPorts().entrySet().stream().forEach(entry -> {
String service = entry.getKey();
ServiceInfo infos = entry.getValue();
infos.getTcpPorts().forEach((container, host) -> {
String name = "test.fixtures." + service + ".tcp." + container;
theTask.getLogger().info("port mapping property: {}={}", name, host);
consumer.accept(name, host);
entry.getValue().entrySet().stream().forEach(portMapping -> {
String name = "test.fixtures." + service + ".tcp." + portMapping.getKey();
theTask.getLogger().info("port mapping property: {}={}", name, portMapping.getValue());
consumer.accept(name, portMapping.getValue());
});
infos.getUdpPorts().forEach((container, host) -> {
String name = "test.fixtures." + service + ".udp." + container;
theTask.getLogger().info("port mapping property: {}={}", name, host);
consumer.accept(name, host);
});
dockerSupportServiceProvider.get().getUdpPorts().entrySet().stream().forEach(entry -> {
String service = entry.getKey();
entry.getValue().entrySet().stream().forEach(portMapping -> {
String name = "test.fixtures." + service + ".udp." + portMapping.getKey();
theTask.getLogger().info("port mapping property: {}={}", name, portMapping.getValue());
consumer.accept(name, portMapping.getValue());
});
});
}
});

}

@SuppressWarnings("unchecked")
Expand Down
Loading

0 comments on commit a4835cc

Please sign in to comment.