Skip to content

Commit

Permalink
Merge remote-tracking branch 'es/6.x' into ccr-6.x
Browse files Browse the repository at this point in the history
* es/6.x: (28 commits)
  TEST: Unmute testPrimaryRelocationWhileIndexing
  Never leave stale delete tombstones in version map (#29619)
  Do not serialize common stats flags using ordinal (#29600)
  Remove stale comment from JVM stats (#29625)
  TEST: Mute testPrimaryRelocationWhileIndexing
  Remove 7.0.0 from 6.x changelog (#29621)
  Add support to match_phrase query for zero_terms_query. (#29598)
  Account translog location to ram usage in version map
  Avoid side-effect in VersionMap when assertion enabled (#29585)
  Build: Move java home checks to pre-execution phase (#29548)
  Add tests for ranking evaluation with aliases (#29452)
  [Test] Fix assertion in SearchDocumentationIT
  Deprecate use of `htmlStrip` as name for HtmlStripCharFilter (#27429)
  test: Assert deprecated http.enebled setting warning
  Update plan for the removal of mapping types. (#29586)
  [Docs] Add rankEval method for Jva HL client
  Make ranking evaluation details accessible for client
  Rename the bulk thread pool to write thread pool (#29593)
  [Test] Minor changes to rank_eval tests (#29577)
  test: Assert deprecated http.enebled setting warning
  ...
  • Loading branch information
martijnvg committed Apr 20, 2018
2 parents 64ea8a5 + 12aebd4 commit 1f8ea58
Show file tree
Hide file tree
Showing 75 changed files with 1,826 additions and 388 deletions.
Original file line number Diff line number Diff line change
Expand Up @@ -38,6 +38,7 @@ import org.gradle.api.artifacts.ModuleVersionIdentifier
import org.gradle.api.artifacts.ProjectDependency
import org.gradle.api.artifacts.ResolvedArtifact
import org.gradle.api.artifacts.dsl.RepositoryHandler
import org.gradle.api.execution.TaskExecutionGraph
import org.gradle.api.plugins.JavaPlugin
import org.gradle.api.publish.maven.MavenPublication
import org.gradle.api.publish.maven.plugins.MavenPublishPlugin
Expand Down Expand Up @@ -221,21 +222,34 @@ class BuildPlugin implements Plugin<Project> {
return System.getenv('JAVA' + version + '_HOME')
}

/**
* Get Java home for the project for the specified version. If the specified version is not configured, an exception with the specified
* message is thrown.
*
* @param project the project
* @param version the version of Java home to obtain
* @param message the exception message if Java home for the specified version is not configured
* @return Java home for the specified version
* @throws GradleException if Java home for the specified version is not configured
*/
static String getJavaHome(final Project project, final int version, final String message) {
if (project.javaVersions.get(version) == null) {
throw new GradleException(message)
/** Add a check before gradle execution phase which ensures java home for the given java version is set. */
static void requireJavaHome(Task task, int version) {
Project rootProject = task.project.rootProject // use root project for global accounting
if (rootProject.hasProperty('requiredJavaVersions') == false) {
rootProject.rootProject.ext.requiredJavaVersions = [:].withDefault{key -> return []}
rootProject.gradle.taskGraph.whenReady { TaskExecutionGraph taskGraph ->
List<String> messages = []
for (entry in rootProject.requiredJavaVersions) {
if (rootProject.javaVersions.get(entry.key) != null) {
continue
}
List<String> tasks = entry.value.findAll { taskGraph.hasTask(it) }.collect { " ${it.path}" }
if (tasks.isEmpty() == false) {
messages.add("JAVA${entry.key}_HOME required to run tasks:\n${tasks.join('\n')}")
}
}
if (messages.isEmpty() == false) {
throw new GradleException(messages.join('\n'))
}
}
}
return project.javaVersions.get(version)
rootProject.requiredJavaVersions.get(version).add(task)
}

/** A convenience method for getting java home for a version of java and requiring that version for the given task to execute */
static String getJavaHome(final Task task, final int version) {
requireJavaHome(task, version)
return task.project.javaVersions.get(version)
}

private static String findRuntimeJavaHome(final String compilerJavaHome) {
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -20,6 +20,7 @@ package org.elasticsearch.gradle.test

import org.apache.tools.ant.DefaultLogger
import org.apache.tools.ant.taskdefs.condition.Os
import org.elasticsearch.gradle.BuildPlugin
import org.elasticsearch.gradle.LoggedExec
import org.elasticsearch.gradle.Version
import org.elasticsearch.gradle.VersionProperties
Expand Down Expand Up @@ -611,6 +612,9 @@ class ClusterFormationTasks {
}

Task start = project.tasks.create(name: name, type: DefaultTask, dependsOn: setup)
if (node.javaVersion != null) {
BuildPlugin.requireJavaHome(start, node.javaVersion)
}
start.doLast(elasticsearchRunner)
return start
}
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -36,6 +36,9 @@ import static org.elasticsearch.gradle.BuildPlugin.getJavaHome
* A container for the files and configuration associated with a single node in a test cluster.
*/
class NodeInfo {
/** Gradle project this node is part of */
Project project

/** common configuration for all nodes, including this one */
ClusterConfiguration config

Expand Down Expand Up @@ -84,6 +87,9 @@ class NodeInfo {
/** directory to install plugins from */
File pluginsTmpDir

/** Major version of java this node runs with, or {@code null} if using the runtime java version */
Integer javaVersion

/** environment variables to start the node with */
Map<String, String> env

Expand All @@ -109,6 +115,7 @@ class NodeInfo {
NodeInfo(ClusterConfiguration config, int nodeNum, Project project, String prefix, Version nodeVersion, File sharedDir) {
this.config = config
this.nodeNum = nodeNum
this.project = project
this.sharedDir = sharedDir
if (config.clusterName != null) {
clusterName = config.clusterName
Expand Down Expand Up @@ -165,12 +172,11 @@ class NodeInfo {
args.add("${esScript}")
}


if (nodeVersion.before("6.2.0")) {
env = ['JAVA_HOME': "${-> getJavaHome(project, 8, "JAVA8_HOME must be set to run BWC tests against [" + nodeVersion + "]")}"]
javaVersion = 8
} else if (nodeVersion.onOrAfter("6.2.0") && nodeVersion.before("6.3.0")) {
env = ['JAVA_HOME': "${-> getJavaHome(project, 9, "JAVA9_HOME must be set to run BWC tests against [" + nodeVersion + "]")}"]
} else {
env = ['JAVA_HOME': (String) project.runtimeJavaHome]
javaVersion = 9
}

args.addAll("-E", "node.portsfile=true")
Expand All @@ -182,7 +188,7 @@ class NodeInfo {
// in the cluster-specific options
esJavaOpts = String.join(" ", "-ea", "-esa", esJavaOpts)
}
env.put('ES_JAVA_OPTS', esJavaOpts)
env = ['ES_JAVA_OPTS': esJavaOpts]
for (Map.Entry<String, String> property : System.properties.entrySet()) {
if (property.key.startsWith('tests.es.')) {
args.add("-E")
Expand Down Expand Up @@ -253,13 +259,19 @@ class NodeInfo {
return Native.toString(shortPath).substring(4)
}

/** Return the java home used by this node. */
String getJavaHome() {
return javaVersion == null ? project.runtimeJavaHome : project.javaVersions.get(javaVersion)
}

/** Returns debug string for the command that started this node. */
String getCommandString() {
String esCommandString = "\nNode ${nodeNum} configuration:\n"
esCommandString += "|-----------------------------------------\n"
esCommandString += "| cwd: ${cwd}\n"
esCommandString += "| command: ${executable} ${args.join(' ')}\n"
esCommandString += '| environment:\n'
esCommandString += "| JAVA_HOME: ${javaHome}\n"
env.each { k, v -> esCommandString += "| ${k}: ${v}\n" }
if (config.daemonize) {
esCommandString += "|\n| [${wrapperScript.name}]\n"
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -44,6 +44,16 @@
import org.elasticsearch.index.query.MatchQueryBuilder;
import org.elasticsearch.index.query.QueryBuilder;
import org.elasticsearch.index.query.QueryBuilders;
import org.elasticsearch.index.rankeval.EvalQueryQuality;
import org.elasticsearch.index.rankeval.EvaluationMetric;
import org.elasticsearch.index.rankeval.MetricDetail;
import org.elasticsearch.index.rankeval.PrecisionAtK;
import org.elasticsearch.index.rankeval.RankEvalRequest;
import org.elasticsearch.index.rankeval.RankEvalResponse;
import org.elasticsearch.index.rankeval.RankEvalSpec;
import org.elasticsearch.index.rankeval.RatedDocument;
import org.elasticsearch.index.rankeval.RatedRequest;
import org.elasticsearch.index.rankeval.RatedSearchHit;
import org.elasticsearch.rest.RestStatus;
import org.elasticsearch.search.Scroll;
import org.elasticsearch.search.SearchHit;
Expand Down Expand Up @@ -74,6 +84,7 @@
import org.elasticsearch.search.suggest.term.TermSuggestion;

import java.io.IOException;
import java.util.ArrayList;
import java.util.Arrays;
import java.util.Collections;
import java.util.List;
Expand Down Expand Up @@ -688,6 +699,77 @@ public void onFailure(Exception e) {
}
}

public void testRankEval() throws Exception {
indexSearchTestData();
RestHighLevelClient client = highLevelClient();
{
// tag::rank-eval-request-basic
EvaluationMetric metric = new PrecisionAtK(); // <1>
List<RatedDocument> ratedDocs = new ArrayList<>();
ratedDocs.add(new RatedDocument("posts", "1", 1)); // <2>
SearchSourceBuilder searchQuery = new SearchSourceBuilder();
searchQuery.query(QueryBuilders.matchQuery("user", "kimchy"));// <3>
RatedRequest ratedRequest = // <4>
new RatedRequest("kimchy_query", ratedDocs, searchQuery);
List<RatedRequest> ratedRequests = Arrays.asList(ratedRequest);
RankEvalSpec specification =
new RankEvalSpec(ratedRequests, metric); // <5>
RankEvalRequest request = // <6>
new RankEvalRequest(specification, new String[] { "posts" });
// end::rank-eval-request-basic

// tag::rank-eval-execute
RankEvalResponse response = client.rankEval(request);
// end::rank-eval-execute

// tag::rank-eval-response
double evaluationResult = response.getEvaluationResult(); // <1>
assertEquals(1.0 / 3.0, evaluationResult, 0.0);
Map<String, EvalQueryQuality> partialResults =
response.getPartialResults();
EvalQueryQuality evalQuality =
partialResults.get("kimchy_query"); // <2>
assertEquals("kimchy_query", evalQuality.getId());
double qualityLevel = evalQuality.getQualityLevel(); // <3>
assertEquals(1.0 / 3.0, qualityLevel, 0.0);
List<RatedSearchHit> hitsAndRatings = evalQuality.getHitsAndRatings();
RatedSearchHit ratedSearchHit = hitsAndRatings.get(0);
assertEquals("2", ratedSearchHit.getSearchHit().getId()); // <4>
assertFalse(ratedSearchHit.getRating().isPresent()); // <5>
MetricDetail metricDetails = evalQuality.getMetricDetails();
String metricName = metricDetails.getMetricName();
assertEquals(PrecisionAtK.NAME, metricName); // <6>
PrecisionAtK.Detail detail = (PrecisionAtK.Detail) metricDetails;
assertEquals(1, detail.getRelevantRetrieved()); // <7>
assertEquals(3, detail.getRetrieved());
// end::rank-eval-response

// tag::rank-eval-execute-listener
ActionListener<RankEvalResponse> listener = new ActionListener<RankEvalResponse>() {
@Override
public void onResponse(RankEvalResponse response) {
// <1>
}

@Override
public void onFailure(Exception e) {
// <2>
}
};
// end::rank-eval-execute-listener

// Replace the empty listener by a blocking listener in test
final CountDownLatch latch = new CountDownLatch(1);
listener = new LatchedActionListener<>(listener, latch);

// tag::rank-eval-execute-async
client.rankEvalAsync(request, listener); // <1>
// end::rank-eval-execute-async

assertTrue(latch.await(30L, TimeUnit.SECONDS));
}
}

public void testMultiSearch() throws Exception {
indexSearchTestData();
RestHighLevelClient client = highLevelClient();
Expand Down
4 changes: 2 additions & 2 deletions distribution/bwc/build.gradle
Original file line number Diff line number Diff line change
Expand Up @@ -147,9 +147,9 @@ subprojects {
workingDir = checkoutDir
if (["5.6", "6.0", "6.1"].contains(bwcBranch)) {
// we are building branches that are officially built with JDK 8, push JAVA8_HOME to JAVA_HOME for these builds
environment('JAVA_HOME', "${-> getJavaHome(project, 8, "JAVA8_HOME is required to build BWC versions for BWC branch [" + bwcBranch + "]")}")
environment('JAVA_HOME', getJavaHome(it, 8))
} else if ("6.2".equals(bwcBranch)) {
environment('JAVA_HOME', "${-> getJavaHome(project, 9, "JAVA9_HOME is required to build BWC versions for BWC branch [" + bwcBranch + "]")}")
environment('JAVA_HOME', getJavaHome(it, 9))
} else {
environment('JAVA_HOME', project.compilerJavaHome)
}
Expand Down
20 changes: 1 addition & 19 deletions docs/CHANGELOG.asciidoc
Original file line number Diff line number Diff line change
Expand Up @@ -5,27 +5,9 @@

= Elasticsearch Release Notes

== Elasticsearch 7.0.0

=== Breaking Changes

=== Breaking Java Changes

=== Deprecations

=== New Features

=== Enhancements

=== Bug Fixes

=== Regressions

=== Known Issues

== Elasticsearch version 6.3.0

=== New Features
=== New Features

=== Enhancements

Expand Down
89 changes: 89 additions & 0 deletions docs/java-rest/high-level/search/rank-eval.asciidoc
Original file line number Diff line number Diff line change
@@ -0,0 +1,89 @@
[[java-rest-high-rank-eval]]
=== Ranking Evaluation API

The `rankEval` method allows to evaluate the quality of ranked search
results over a set of search request. Given sets of manually rated
documents for each search request, ranking evaluation performs a
<<java-rest-high-multi-search,multi search>> request and calculates
information retrieval metrics like _mean reciprocal rank_, _precision_
or _discounted cumulative gain_ on the returned results.

[[java-rest-high-rank-eval-request]]
==== Ranking Evaluation Request

In order to build a `RankEvalRequest`, you first need to create an
evaluation specification (`RankEvalSpec`). This specification requires
to define the evaluation metric that is going to be calculated, as well
as a list of rated documents per search requests. Creating the ranking
evaluation request then takes the specification and a list of target
indices as arguments:

["source","java",subs="attributes,callouts,macros"]
--------------------------------------------------
include-tagged::{doc-tests}/SearchDocumentationIT.java[rank-eval-request-basic]
--------------------------------------------------
<1> Define the metric used in the evaluation
<2> Add rated documents, specified by index name, id and rating
<3> Create the search query to evaluate
<4> Combine the three former parts into a `RatedRequest`
<5> Create the ranking evaluation specification
<6> Create the ranking evaluation request

[[java-rest-high-rank-eval-sync]]
==== Synchronous Execution

The `rankEval` method executes `RankEvalRequest`s synchronously:

["source","java",subs="attributes,callouts,macros"]
--------------------------------------------------
include-tagged::{doc-tests}/SearchDocumentationIT.java[rank-eval-execute]
--------------------------------------------------

[[java-rest-high-rank-eval-async]]
==== Asynchronous Execution

The `rankEvalAsync` method executes `RankEvalRequest`s asynchronously,
calling the provided `ActionListener` when the response is ready.

["source","java",subs="attributes,callouts,macros"]
--------------------------------------------------
include-tagged::{doc-tests}/SearchDocumentationIT.java[rank-eval-execute-async]
--------------------------------------------------
<1> The `RankEvalRequest` to execute and the `ActionListener` to use when
the execution completes

The asynchronous method does not block and returns immediately. Once it is
completed the `ActionListener` is called back using the `onResponse` method
if the execution successfully completed or using the `onFailure` method if
it failed.

A typical listener for `RankEvalResponse` looks like:

["source","java",subs="attributes,callouts,macros"]
--------------------------------------------------
include-tagged::{doc-tests}/SearchDocumentationIT.java[rank-eval-execute-listener]
--------------------------------------------------
<1> Called when the execution is successfully completed.
<2> Called when the whole `RankEvalRequest` fails.

==== RankEvalResponse

The `RankEvalResponse` that is returned by executing the request
contains information about the overall evaluation score, the
scores of each individual search request in the set of queries and
detailed information about search hits and details about the metric
calculation per partial result.

["source","java",subs="attributes,callouts,macros"]
--------------------------------------------------
include-tagged::{doc-tests}/SearchDocumentationIT.java[rank-eval-response]
--------------------------------------------------
<1> The overall evaluation result
<2> Partial results that are keyed by their query id
<3> The metric score for each partial result
<4> Rated search hits contain a fully fledged `SearchHit`
<5> Rated search hits also contain an `Optional<Interger>` rating that
is not present if the document did not get a rating in the request
<6> Metric details are named after the metric used in the request
<7> After casting to the metric used in the request, the
metric details offers insight into parts of the metric calculation
2 changes: 2 additions & 0 deletions docs/java-rest/high-level/supported-apis.asciidoc
Original file line number Diff line number Diff line change
Expand Up @@ -32,10 +32,12 @@ The Java High Level REST Client supports the following Search APIs:
* <<java-rest-high-search-scroll>>
* <<java-rest-high-clear-scroll>>
* <<java-rest-high-multi-search>>
* <<java-rest-high-rank-eval>>

include::search/search.asciidoc[]
include::search/scroll.asciidoc[]
include::search/multi-search.asciidoc[]
include::search/rank-eval.asciidoc[]

== Miscellaneous APIs

Expand Down
Loading

0 comments on commit 1f8ea58

Please sign in to comment.