Skip to content
New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Jump to bottom

Add tests for ranking evaluation with aliases #29452

Merged
merged 4 commits into from
Apr 19, 2018
Merged

Add tests for ranking evaluation with aliases #29452

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
4 commits
Select commit Hold shift + click to select a range
c0f9210
Add tests with aliases to rank-eval
Apr 10, 2018
54cff6a
Merge branch 'master' into test-29011
Apr 19, 2018
8b823da
Correcting test
Apr 19, 2018
6294ff9
Merge branch 'master' into test-29011
Apr 19, 2018
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
50 changes: 29 additions & 21 deletions modules/rank-eval/src/test/java/org/elasticsearch/index/rankeval/RankEvalRequestIT.java
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -20,6 +20,7 @@
package org.elasticsearch.index.rankeval;

import org.elasticsearch.ElasticsearchException;
import org.elasticsearch.action.admin.indices.alias.IndicesAliasesRequest.AliasActions;
import org.elasticsearch.action.search.SearchRequest;
import org.elasticsearch.action.support.IndicesOptions;
import org.elasticsearch.index.IndexNotFoundException;
Expand All @@ -40,10 +41,13 @@
import java.util.Set;

import static org.elasticsearch.index.rankeval.EvaluationMetric.filterUnknownDocuments;
import static org.elasticsearch.test.hamcrest.ElasticsearchAssertions.assertAcked;
import static org.hamcrest.Matchers.instanceOf;

public class RankEvalRequestIT extends ESIntegTestCase {

private static final String TEST_INDEX = "test";
private static final String INDEX_ALIAS = "alias0";
private static final int RELEVANT_RATING_1 = 1;

@Override
Expand All @@ -58,20 +62,23 @@ protected Collection<Class<? extends Plugin>> nodePlugins() {

@Before
public void setup() {
createIndex("test");
createIndex(TEST_INDEX);
ensureGreen();

client().prepareIndex("test", "testtype").setId("1")
client().prepareIndex(TEST_INDEX, "testtype").setId("1")
.setSource("text", "berlin", "title", "Berlin, Germany", "population", 3670622).get();
client().prepareIndex("test", "testtype").setId("2").setSource("text", "amsterdam", "population", 851573).get();
client().prepareIndex("test", "testtype").setId("3").setSource("text", "amsterdam", "population", 851573).get();
client().prepareIndex("test", "testtype").setId("4").setSource("text", "amsterdam", "population", 851573).get();
client().prepareIndex("test", "testtype").setId("5").setSource("text", "amsterdam", "population", 851573).get();
client().prepareIndex("test", "testtype").setId("6").setSource("text", "amsterdam", "population", 851573).get();
client().prepareIndex(TEST_INDEX, "testtype").setId("2").setSource("text", "amsterdam", "population", 851573).get();
client().prepareIndex(TEST_INDEX, "testtype").setId("3").setSource("text", "amsterdam", "population", 851573).get();
client().prepareIndex(TEST_INDEX, "testtype").setId("4").setSource("text", "amsterdam", "population", 851573).get();
client().prepareIndex(TEST_INDEX, "testtype").setId("5").setSource("text", "amsterdam", "population", 851573).get();
client().prepareIndex(TEST_INDEX, "testtype").setId("6").setSource("text", "amsterdam", "population", 851573).get();

// add another index for testing closed indices etc...
client().prepareIndex("test2", "testtype").setId("7").setSource("text", "amsterdam", "population", 851573).get();
refresh();

// set up an alias that can also be used in tests
assertAcked(client().admin().indices().prepareAliases().addAliasAction(AliasActions.add().index(TEST_INDEX).alias(INDEX_ALIAS)));
}

/**
Expand Down Expand Up @@ -101,7 +108,8 @@ public void testPrecisionAtRequest() {
RankEvalAction.INSTANCE, new RankEvalRequest());
builder.setRankEvalSpec(task);

RankEvalResponse response = client().execute(RankEvalAction.INSTANCE, builder.request().indices("test"))
String indexToUse = randomBoolean() ? TEST_INDEX : INDEX_ALIAS;
RankEvalResponse response = client().execute(RankEvalAction.INSTANCE, builder.request().indices(indexToUse))
.actionGet();
// the expected Prec@ for the first query is 4/6 and the expected Prec@ for the
// second is 1/6, divided by 2 to get the average
Expand Down Expand Up @@ -143,7 +151,7 @@ public void testPrecisionAtRequest() {
metric = new PrecisionAtK(1, false, 3);
task = new RankEvalSpec(specifications, metric);

builder = new RankEvalRequestBuilder(client(), RankEvalAction.INSTANCE, new RankEvalRequest(task, new String[] { "test" }));
builder = new RankEvalRequestBuilder(client(), RankEvalAction.INSTANCE, new RankEvalRequest(task, new String[] { TEST_INDEX }));

response = client().execute(RankEvalAction.INSTANCE, builder.request()).actionGet();
// if we look only at top 3 documente, the expected P@3 for the first query is
Expand All @@ -163,19 +171,19 @@ public void testDCGRequest() {

List<RatedRequest> specifications = new ArrayList<>();
List<RatedDocument> ratedDocs = Arrays.asList(
new RatedDocument("test", "1", 3),
new RatedDocument("test", "2", 2),
new RatedDocument("test", "3", 3),
new RatedDocument("test", "4", 0),
new RatedDocument("test", "5", 1),
new RatedDocument("test", "6", 2));
new RatedDocument(TEST_INDEX, "1", 3),
new RatedDocument(TEST_INDEX, "2", 2),
new RatedDocument(TEST_INDEX, "3", 3),
new RatedDocument(TEST_INDEX, "4", 0),
new RatedDocument(TEST_INDEX, "5", 1),
new RatedDocument(TEST_INDEX, "6", 2));
specifications.add(new RatedRequest("amsterdam_query", ratedDocs, testQuery));

DiscountedCumulativeGain metric = new DiscountedCumulativeGain(false, null, 10);
RankEvalSpec task = new RankEvalSpec(specifications, metric);

RankEvalRequestBuilder builder = new RankEvalRequestBuilder(client(), RankEvalAction.INSTANCE,
new RankEvalRequest(task, new String[] { "test" }));
new RankEvalRequest(task, new String[] { TEST_INDEX }));

RankEvalResponse response = client().execute(RankEvalAction.INSTANCE, builder.request()).actionGet();
assertEquals(DiscountedCumulativeGainTests.EXPECTED_DCG, response.getEvaluationResult(), 10E-14);
Expand All @@ -184,7 +192,7 @@ public void testDCGRequest() {
metric = new DiscountedCumulativeGain(false, null, 3);
task = new RankEvalSpec(specifications, metric);

builder = new RankEvalRequestBuilder(client(), RankEvalAction.INSTANCE, new RankEvalRequest(task, new String[] { "test" }));
builder = new RankEvalRequestBuilder(client(), RankEvalAction.INSTANCE, new RankEvalRequest(task, new String[] { TEST_INDEX }));

response = client().execute(RankEvalAction.INSTANCE, builder.request()).actionGet();
assertEquals(12.39278926071437, response.getEvaluationResult(), 10E-14);
Expand All @@ -203,7 +211,7 @@ public void testMRRRequest() {
RankEvalSpec task = new RankEvalSpec(specifications, metric);

RankEvalRequestBuilder builder = new RankEvalRequestBuilder(client(), RankEvalAction.INSTANCE,
new RankEvalRequest(task, new String[] { "test" }));
new RankEvalRequest(task, new String[] { TEST_INDEX }));

RankEvalResponse response = client().execute(RankEvalAction.INSTANCE, builder.request()).actionGet();
// the expected reciprocal rank for the amsterdam_query is 1/5
Expand All @@ -216,7 +224,7 @@ public void testMRRRequest() {
metric = new MeanReciprocalRank(1, 3);
task = new RankEvalSpec(specifications, metric);

builder = new RankEvalRequestBuilder(client(), RankEvalAction.INSTANCE, new RankEvalRequest(task, new String[] { "test" }));
builder = new RankEvalRequestBuilder(client(), RankEvalAction.INSTANCE, new RankEvalRequest(task, new String[] { TEST_INDEX }));

response = client().execute(RankEvalAction.INSTANCE, builder.request()).actionGet();
// limiting to top 3 results, the amsterdam_query has no relevant document in it
Expand Down Expand Up @@ -247,7 +255,7 @@ public void testBadQuery() {
RankEvalSpec task = new RankEvalSpec(specifications, new PrecisionAtK());

RankEvalRequestBuilder builder = new RankEvalRequestBuilder(client(), RankEvalAction.INSTANCE,
new RankEvalRequest(task, new String[] { "test" }));
new RankEvalRequest(task, new String[] { TEST_INDEX }));
builder.setRankEvalSpec(task);

RankEvalResponse response = client().execute(RankEvalAction.INSTANCE, builder.request()).actionGet();
Expand All @@ -267,7 +275,7 @@ public void testIndicesOptions() {
specifications.add(new RatedRequest("amsterdam_query", relevantDocs, amsterdamQuery));
RankEvalSpec task = new RankEvalSpec(specifications, new PrecisionAtK());

RankEvalRequest request = new RankEvalRequest(task, new String[] { "test", "test2" });
RankEvalRequest request = new RankEvalRequest(task, new String[] { TEST_INDEX, "test2" });
request.setRankEvalSpec(task);

RankEvalResponse response = client().execute(RankEvalAction.INSTANCE, request).actionGet();
Expand Down
90 changes: 44 additions & 46 deletions modules/rank-eval/src/test/resources/rest-api-spec/test/rank_eval/10_basic.yml
View file
Open in desktop
Original file line number Diff line number Diff line change
@@ -1,10 +1,4 @@
---
"Response format":

- skip:
version: " - 6.2.99"
reason: response format was updated in 6.3

setup:
- do:
indices.create:
index: foo
Expand Down Expand Up @@ -43,8 +37,21 @@
- do:
indices.refresh: {}

- do:
indices.put_alias:
index: foo
name: alias

---
"Response format":

- skip:
version: " - 6.2.99"
reason: response format was updated in 6.3

- do:
rank_eval:
index: foo,
body: {
"requests" : [
{
Expand Down Expand Up @@ -84,52 +91,43 @@
- match: { details.berlin_query.hits.0.hit._id: "doc1" }
- match: { details.berlin_query.hits.0.rating: 1}
- match: { details.berlin_query.hits.1.hit._id: "doc4" }
- is_false: details.berlin_query.hits.1.rating
- is_false: details.berlin_query.hits.1.rating

---
"Mean Reciprocal Rank":

- skip:
version: " - 6.2.99"
reason: response format was updated in 6.3
"Alias resolution":

- do:
indices.create:
index: foo
body:
settings:
index:
number_of_shards: 1
- do:
index:
index: foo
type: bar
id: doc1
body: { "text": "berlin" }
rank_eval:
index: alias
body: {
"requests" : [
{
"id": "amsterdam_query",
"request": { "query": { "match" : {"text" : "amsterdam" }}},
"ratings": [
{"_index": "foo", "_id": "doc1", "rating": 0},
{"_index": "foo", "_id": "doc2", "rating": 1},
{"_index": "foo", "_id": "doc3", "rating": 1}]
},
{
"id" : "berlin_query",
"request": { "query": { "match" : { "text" : "berlin" } }, "size" : 10 },
"ratings": [{"_index": "foo", "_id": "doc1", "rating": 1}]
}
],
"metric" : { "precision": { "ignore_unlabeled" : true }}
}

- do:
index:
index: foo
type: bar
id: doc2
body: { "text": "amsterdam" }
- match: { quality_level: 1}
- match: { details.amsterdam_query.quality_level: 1.0}
- match: { details.berlin_query.quality_level: 1.0}

- do:
index:
index: foo
type: bar
id: doc3
body: { "text": "amsterdam" }

- do:
index:
index: foo
type: bar
id: doc4
body: { "text": "something about amsterdam and berlin" }
---
"Mean Reciprocal Rank":

- do:
indices.refresh: {}
- skip:
version: " - 6.2.99"
reason: response format was updated in 6.3

- do:
rank_eval:
Expand Down