Skip to content

Commit

Permalink
Merge pull request #19283 from cbuescher/rank-add-restparsing
Browse files Browse the repository at this point in the history 
Query evaluation: Adding rest layer parsing and response rendering
  • Loading branch information
Isabel Drost-Fromm committed Jul 21, 2016
2 parents 00a11b7 + b730494 commit 2b3abad
Show file tree
Hide file tree
Showing 19 changed files with 808 additions and 474 deletions.
10 changes: 5 additions & 5 deletions modules/rank-eval/src/main/java/org/elasticsearch/index/rankeval/EvalQueryQuality.java
View file
Expand Up @@ -21,25 +21,25 @@

import java.util.Collection;

/** Returned for each search intent and search specification combination. Summarises the document ids found that were not
* annotated and the average precision of result sets in each particular combination based on the annotations given.
/** Returned for each search specification. Summarizes the measured quality metric for this search request
* and adds the document ids found that were in the search result but not annotated in the original request.
* */
public class EvalQueryQuality {
private double qualityLevel;

private Collection<String> unknownDocs;

public EvalQueryQuality (double qualityLevel, Collection<String> unknownDocs) {
this.qualityLevel = qualityLevel;
this.unknownDocs = unknownDocs;
}

public Collection<String> getUnknownDocs() {
return unknownDocs;
}

public double getQualityLevel() {
return qualityLevel;
return qualityLevel;
}

}
28 changes: 0 additions & 28 deletions modules/rank-eval/src/main/java/org/elasticsearch/index/rankeval/Evaluator.java
View file

This file was deleted.

66 changes: 38 additions & 28 deletions modules/rank-eval/src/main/java/org/elasticsearch/index/rankeval/PrecisionAtN.java
View file
Expand Up @@ -19,28 +19,31 @@

package org.elasticsearch.index.rankeval;

import org.elasticsearch.common.ParseField;
import org.elasticsearch.common.ParseFieldMatcherSupplier;
import org.elasticsearch.common.io.stream.StreamInput;
import org.elasticsearch.common.io.stream.StreamOutput;
import org.elasticsearch.common.xcontent.ConstructingObjectParser;
import org.elasticsearch.common.xcontent.XContentParser;
import org.elasticsearch.search.SearchHit;

import java.io.IOException;
import java.util.ArrayList;
import java.util.Collection;
import java.util.Map;
import java.util.Map.Entry;
import java.util.List;

import javax.naming.directory.SearchResult;

/**
* Evaluate Precision at N, N being the number of search results to consider for precision calculation.
*
*
* Documents of unkonwn quality are ignored in the precision at n computation and returned by document id.
* */
public class PrecisionAtN implements RankedListQualityMetric {
public class PrecisionAtN extends RankedListQualityMetric {

/** Number of results to check against a given set of relevant results. */
private int n;

public static final String NAME = "precisionatn";

public PrecisionAtN(StreamInput in) throws IOException {
Expand All @@ -63,7 +66,7 @@ public String getWriteableName() {
public PrecisionAtN() {
this.n = 10;
}

/**
* @param n number of top results to check against a given set of relevant results.
* */
Expand All @@ -78,24 +81,31 @@ public int getN() {
return n;
}

private static final ParseField SIZE_FIELD = new ParseField("size");
private static final ConstructingObjectParser<PrecisionAtN, ParseFieldMatcherSupplier> PARSER = new ConstructingObjectParser<>(
"precision_at", a -> new PrecisionAtN((Integer) a[0]));

static {
PARSER.declareInt(ConstructingObjectParser.constructorArg(), SIZE_FIELD);
}

public static PrecisionAtN fromXContent(XContentParser parser, ParseFieldMatcherSupplier matcher) {
return PARSER.apply(parser, matcher);
}

/** Compute precisionAtN based on provided relevant document IDs.
* @return precision at n for above {@link SearchResult} list.
**/
@Override
public EvalQueryQuality evaluate(SearchHit[] hits, RatedQuery intent) {
Map<String, Integer> ratedDocIds = intent.getRatedDocuments();

Collection<String> relevantDocIds = new ArrayList<>();
for (Entry<String, Integer> entry : ratedDocIds.entrySet()) {
if (Rating.RELEVANT.equals(RatingMapping.mapTo(entry.getValue()))) {
relevantDocIds.add(entry.getKey());
}
}

Collection<String> irrelevantDocIds = new ArrayList<>();
for (Entry<String, Integer> entry : ratedDocIds.entrySet()) {
if (Rating.IRRELEVANT.equals(RatingMapping.mapTo(entry.getValue()))) {
irrelevantDocIds.add(entry.getKey());
public EvalQueryQuality evaluate(SearchHit[] hits, List<RatedDocument> ratedDocs) {

Collection<String> relevantDocIds = new ArrayList<>();
Collection<String> irrelevantDocIds = new ArrayList<>();
for (RatedDocument doc : ratedDocs) {
if (Rating.RELEVANT.equals(RatingMapping.mapTo(doc.getRating()))) {
relevantDocIds.add(doc.getDocID());
} else if (Rating.IRRELEVANT.equals(RatingMapping.mapTo(doc.getRating()))) {
irrelevantDocIds.add(doc.getDocID());
}
}

Expand All @@ -117,24 +127,24 @@ public EvalQueryQuality evaluate(SearchHit[] hits, RatedQuery intent) {

return new EvalQueryQuality(precision, unknownDocIds);
}

public enum Rating {
RELEVANT, IRRELEVANT;
IRRELEVANT, RELEVANT;
}

/**
* Needed to get the enum accross serialisation boundaries.
* */
public static class RatingMapping {
public static Integer mapFrom(Rating rating) {
if (Rating.RELEVANT.equals(rating)) {
return 0;
return 1;
}
return 1;
return 0;
}

public static Rating mapTo(Integer rating) {
if (rating == 0) {
if (rating == 1) {
return Rating.RELEVANT;
}
return Rating.IRRELEVANT;
Expand Down
96 changes: 87 additions & 9 deletions modules/rank-eval/src/main/java/org/elasticsearch/index/rankeval/QuerySpec.java
View file
Expand Up @@ -19,9 +19,13 @@

package org.elasticsearch.index.rankeval;

import org.elasticsearch.common.ParseField;
import org.elasticsearch.common.ParsingException;
import org.elasticsearch.common.io.stream.StreamInput;
import org.elasticsearch.common.io.stream.StreamOutput;
import org.elasticsearch.common.io.stream.Writeable;
import org.elasticsearch.common.xcontent.ObjectParser;
import org.elasticsearch.common.xcontent.XContentParser;
import org.elasticsearch.search.builder.SearchSourceBuilder;

import java.io.IOException;
Expand All @@ -32,25 +36,33 @@
* Defines a QA specification: All end user supplied query intents will be mapped to the search request specified in this search request
* template and executed against the targetIndex given. Any filters that should be applied in the target system can be specified as well.
*
* The resulting document lists can then be compared against what was specified in the set of rated documents as part of a QAQuery.
* The resulting document lists can then be compared against what was specified in the set of rated documents as part of a QAQuery.
* */
public class QuerySpec implements Writeable {

private int specId = 0;
private String specId;
private SearchSourceBuilder testRequest;
private List<String> indices = new ArrayList<>();
private List<String> types = new ArrayList<>();

public QuerySpec(
int specId, SearchSourceBuilder testRequest, List<String> indices, List<String> types) {
/** Collection of rated queries for this query QA specification.*/
private List<RatedDocument> ratedDocs = new ArrayList<>();

public QuerySpec() {
// ctor that doesn't require all args to be present immediatly is easier to use with ObjectParser
// TODO decide if we can require only id as mandatory, set default values for the rest?
}

public QuerySpec(String specId, SearchSourceBuilder testRequest, List<String> indices, List<String> types,
List<RatedDocument> ratedDocs) {
this.specId = specId;
this.testRequest = testRequest;
this.indices = indices;
this.types = types;
this.ratedDocs = ratedDocs;
}

public QuerySpec(StreamInput in) throws IOException {
this.specId = in.readInt();
this.specId = in.readString();
testRequest = new SearchSourceBuilder(in);
int indicesSize = in.readInt();
indices = new ArrayList<String>(indicesSize);
Expand All @@ -62,11 +74,16 @@ public QuerySpec(StreamInput in) throws IOException {
for (int i = 0; i < typesSize; i++) {
this.types.add(in.readString());
}
int intentSize = in.readInt();
ratedDocs = new ArrayList<>(intentSize);
for (int i = 0; i < intentSize; i++) {
ratedDocs.add(new RatedDocument(in));
}
}

@Override
public void writeTo(StreamOutput out) throws IOException {
out.writeInt(specId);
out.writeString(specId);
testRequest.writeTo(out);
out.writeInt(indices.size());
for (String index : indices) {
Expand All @@ -76,6 +93,10 @@ public void writeTo(StreamOutput out) throws IOException {
for (String type : types) {
out.writeString(type);
}
out.writeInt(ratedDocs.size());
for (RatedDocument ratedDoc : ratedDocs) {
ratedDoc.writeTo(out);
}
}

public SearchSourceBuilder getTestRequest() {
Expand Down Expand Up @@ -103,12 +124,69 @@ public void setTypes(List<String> types) {
}

/** Returns a user supplied spec id for easier referencing. */
public int getSpecId() {
public String getSpecId() {
return specId;
}

/** Sets a user supplied spec id for easier referencing. */
public void setSpecId(int specId) {
public void setSpecId(String specId) {
this.specId = specId;
}

/** Returns a list of rated documents to evaluate. */
public List<RatedDocument> getRatedDocs() {
return ratedDocs;
}

/** Set a list of rated documents for this query. */
public void setRatedDocs(List<RatedDocument> ratedDocs) {
this.ratedDocs = ratedDocs;
}

private static final ParseField ID_FIELD = new ParseField("id");
private static final ParseField REQUEST_FIELD = new ParseField("request");
private static final ParseField RATINGS_FIELD = new ParseField("ratings");
private static final ObjectParser<QuerySpec, RankEvalContext> PARSER = new ObjectParser<>("requests", QuerySpec::new);

static {
PARSER.declareString(QuerySpec::setSpecId, ID_FIELD);
PARSER.declareObject(QuerySpec::setTestRequest, (p, c) -> {
try {
return SearchSourceBuilder.fromXContent(c.getParseContext(), c.getAggs(), c.getSuggesters());
} catch (IOException ex) {
throw new ParsingException(p.getTokenLocation(), "error parsing request", ex);
}
} , REQUEST_FIELD);
PARSER.declareObjectArray(QuerySpec::setRatedDocs, (p, c) -> {
try {
return RatedDocument.fromXContent(p);
} catch (IOException ex) {
throw new ParsingException(p.getTokenLocation(), "error parsing ratings", ex);
}
} , RATINGS_FIELD);
}

/**
* Parses {@link QuerySpec} from rest representation:
*
* Example:
* {
* "id": "coffee_query",
* "request": {
* "query": {
* "bool": {
* "must": [
* {"match": {"beverage": "coffee"}},
* {"term": {"browser": {"value": "safari"}}},
* {"term": {"time_of_day": {"value": "morning","boost": 2}}},
* {"term": {"ip_location": {"value": "ams","boost": 10}}}]}
* },
* "size": 10
* },
* "ratings": [{ "1": 1 }, { "2": 0 }, { "3": 1 } ]
* }
*/
public static QuerySpec fromXContent(XContentParser parser, RankEvalContext context) throws IOException {
return PARSER.parse(parser, context);
}
}

0 comments on commit 2b3abad

Please sign in to comment.