Skip to content

Commit

Permalink
Dense vector field type minor fixes (#62631)
Browse files Browse the repository at this point in the history 
The dense vector field is not aggregatable although it produces fielddata through its BinaryDocValuesField. It should pass up hasDocValues set to true to its parent class in its constructor, and return isAggregatable false. Same for the sparse vector field (only in 7.x).

This may not have consequences today, but it will be important once we try to share the same exists query implementation throughout all of the mappers with #57607.
  • Loading branch information
@javanna
javanna committed Sep 22, 2020
1 parent 593511e commit 9ae2971
Show file tree
Hide file tree
Showing 15 changed files with 168 additions and 67 deletions.
2 changes: 1 addition & 1 deletion ...mapper-extras/src/test/java/org/elasticsearch/index/mapper/RankFeatureFieldTypeTests.java
View file
Original file line number Diff line number Diff line change
Expand Up @@ -23,7 +23,7 @@

public class RankFeatureFieldTypeTests extends FieldTypeTestCase {

public void testIsAggregatable() {
public void testIsNotAggregatable() {
MappedFieldType fieldType = new RankFeatureFieldMapper.RankFeatureFieldType("field", Collections.emptyMap(), true);
assertFalse(fieldType.isAggregatable());
}
Expand Down
2 changes: 1 addition & 1 deletion ...apper-extras/src/test/java/org/elasticsearch/index/mapper/RankFeaturesFieldTypeTests.java
View file
Original file line number Diff line number Diff line change
Expand Up @@ -23,7 +23,7 @@

public class RankFeaturesFieldTypeTests extends FieldTypeTestCase {

public void testIsAggregatable() {
public void testIsNotAggregatable() {
MappedFieldType fieldType = new RankFeaturesFieldMapper.RankFeaturesFieldType("field", Collections.emptyMap());
assertFalse(fieldType.isAggregatable());
}
Expand Down
1 change: 0 additions & 1 deletion ...mapper-extras/src/test/java/org/elasticsearch/index/mapper/ScaledFloatFieldTypeTests.java
View file
Original file line number Diff line number Diff line change
Expand Up @@ -40,7 +40,6 @@

public class ScaledFloatFieldTypeTests extends FieldTypeTestCase {


public void testTermQuery() {
ScaledFloatFieldMapper.ScaledFloatFieldType ft
= new ScaledFloatFieldMapper.ScaledFloatFieldType("scaled_float", 0.1 + randomDouble() * 100);
Expand Down
2 changes: 1 addition & 1 deletion ...er-extras/src/test/java/org/elasticsearch/index/mapper/SearchAsYouTypeFieldTypeTests.java
View file
Original file line number Diff line number Diff line change
Expand Up @@ -49,7 +49,7 @@ public class SearchAsYouTypeFieldTypeTests extends FieldTypeTestCase {
UNSEARCHABLE.freeze();
}

protected SearchAsYouTypeFieldType createFieldType() {
private static SearchAsYouTypeFieldType createFieldType() {
final SearchAsYouTypeFieldType fieldType = new SearchAsYouTypeFieldType(NAME, Defaults.FIELD_TYPE, null,
Lucene.STANDARD_ANALYZER, Lucene.STANDARD_ANALYZER, Collections.emptyMap());
fieldType.setPrefixField(new PrefixFieldType(NAME, TextSearchInfo.SIMPLE_MATCH_ONLY, Defaults.MIN_GRAM, Defaults.MAX_GRAM));
Expand Down
23 changes: 13 additions & 10 deletions ...ns/analysis-icu/src/test/java/org/elasticsearch/index/mapper/CollationFieldTypeTests.java
View file
Original file line number Diff line number Diff line change
Expand Up @@ -42,8 +42,12 @@ public class CollationFieldTypeTests extends FieldTypeTestCase{

private static final Collator DEFAULT_COLLATOR = Collator.getInstance(ULocale.ROOT).freeze();

private static CollationFieldType createFieldType() {
return new CollationFieldType("field", DEFAULT_COLLATOR);
}

public void testIsFieldWithinQuery() throws IOException {
CollationFieldType ft = new CollationFieldType("field", DEFAULT_COLLATOR);
CollationFieldType ft = createFieldType();
// current impl ignores args and shourd always return INTERSECTS
assertEquals(Relation.INTERSECTS, ft.isFieldWithinQuery(null,
RandomStrings.randomAsciiOfLengthBetween(random(), 0, 5),
Expand Down Expand Up @@ -89,38 +93,37 @@ public void testTermsQuery() {
}

public void testRegexpQuery() {
MappedFieldType ft = new CollationFieldType("field", DEFAULT_COLLATOR);
MappedFieldType ft = createFieldType();
UnsupportedOperationException e = expectThrows(UnsupportedOperationException.class,
() -> ft.regexpQuery("foo.*", 0, 0, 10, null, randomMockShardContext()));
assertEquals("[regexp] queries are not supported on [icu_collation_keyword] fields.", e.getMessage());
}

public void testFuzzyQuery() {
MappedFieldType ft = new CollationFieldType("field", DEFAULT_COLLATOR);
MappedFieldType ft = createFieldType();
UnsupportedOperationException e = expectThrows(UnsupportedOperationException.class,
() -> ft.fuzzyQuery("foo", Fuzziness.fromEdits(2), 1, 50, true, randomMockShardContext()));
assertEquals("[fuzzy] queries are not supported on [icu_collation_keyword] fields.", e.getMessage());
}

public void testPrefixQuery() {
MappedFieldType ft = new CollationFieldType("field", DEFAULT_COLLATOR);
MappedFieldType ft = createFieldType();
UnsupportedOperationException e = expectThrows(UnsupportedOperationException.class,
() -> ft.prefixQuery("prefix", null, randomMockShardContext()));
assertEquals("[prefix] queries are not supported on [icu_collation_keyword] fields.", e.getMessage());
}

public void testWildcardQuery() {
MappedFieldType ft = new CollationFieldType("field", DEFAULT_COLLATOR);
MappedFieldType ft = createFieldType();
UnsupportedOperationException e = expectThrows(UnsupportedOperationException.class,
() -> ft.wildcardQuery("foo*", null, randomMockShardContext()));
assertEquals("[wildcard] queries are not supported on [icu_collation_keyword] fields.", e.getMessage());
}

public void testRangeQuery() {
Collator collator = DEFAULT_COLLATOR;
MappedFieldType ft = new CollationFieldType("field", collator);
RawCollationKey aKey = collator.getRawCollationKey("a", null);
RawCollationKey bKey = collator.getRawCollationKey("b", null);
MappedFieldType ft = createFieldType();
RawCollationKey aKey = DEFAULT_COLLATOR.getRawCollationKey("a", null);
RawCollationKey bKey = DEFAULT_COLLATOR.getRawCollationKey("b", null);

TermRangeQuery expected = new TermRangeQuery("field", new BytesRef(aKey.bytes, 0, aKey.size),
new BytesRef(bKey.bytes, 0, bKey.size), false, false);
Expand All @@ -132,7 +135,7 @@ public void testRangeQuery() {
assertEquals("[range] queries on [text] or [keyword] fields cannot be executed when " +
"'search.allow_expensive_queries' is set to false.", ee.getMessage());

MappedFieldType unsearchable = new CollationFieldType("field", false, true, collator, Collections.emptyMap());
MappedFieldType unsearchable = new CollationFieldType("field", false, true, DEFAULT_COLLATOR, Collections.emptyMap());
IllegalArgumentException e = expectThrows(IllegalArgumentException.class,
() -> unsearchable.rangeQuery("a", "b", false, false, null, null, null, MOCK_QSC));
assertEquals("Cannot search on field [field] since it is not indexed.", e.getMessage());
Expand Down
1 change: 0 additions & 1 deletion server/src/test/java/org/elasticsearch/index/mapper/NumberFieldTypeTests.java
View file
Original file line number Diff line number Diff line change
Expand Up @@ -20,7 +20,6 @@
package org.elasticsearch.index.mapper;

import com.carrotsearch.randomizedtesting.generators.RandomPicks;

import org.apache.lucene.document.Document;
import org.apache.lucene.document.DoublePoint;
import org.apache.lucene.document.FloatPoint;
Expand Down
77 changes: 38 additions & 39 deletions server/src/test/java/org/elasticsearch/index/mapper/RangeFieldTypeTests.java
View file
Original file line number Diff line number Diff line change
Expand Up @@ -54,7 +54,6 @@

public class RangeFieldTypeTests extends FieldTypeTestCase {
RangeType type;
protected static String FIELDNAME = "field";
protected static int DISTANCE = 10;
private static long nowInMillis;

Expand All @@ -64,16 +63,16 @@ public void setupProperties() {
nowInMillis = randomNonNegativeLong();
}

protected RangeFieldType createDefaultFieldType(String name) {
private RangeFieldType createDefaultFieldType() {
if (type == RangeType.DATE) {
return new RangeFieldType(name, true, true, RangeFieldMapper.Defaults.DATE_FORMATTER, Collections.emptyMap());
return new RangeFieldType("field", true, true, RangeFieldMapper.Defaults.DATE_FORMATTER, Collections.emptyMap());
}
return new RangeFieldType(name, type, true, true, Collections.emptyMap());
return new RangeFieldType("field", type, true, true, Collections.emptyMap());
}

public void testRangeQuery() throws Exception {
QueryShardContext context = createContext();
RangeFieldType ft = createDefaultFieldType(FIELDNAME);
RangeFieldType ft = createDefaultFieldType();

ShapeRelation relation = randomFrom(ShapeRelation.values());
boolean includeLower = randomBoolean();
Expand All @@ -95,10 +94,10 @@ public void testRangeQuery() throws Exception {
public void testRangeQueryIntersectsAdjacentValues() throws Exception {
QueryShardContext context = createContext();
ShapeRelation relation = randomFrom(ShapeRelation.values());
RangeFieldType ft = createDefaultFieldType(FIELDNAME);
RangeFieldType ft = createDefaultFieldType();

Object from = null;
Object to = null;
Object from;
Object to;
switch (type) {
case LONG: {
long fromValue = randomLong();
Expand Down Expand Up @@ -152,7 +151,7 @@ public void testRangeQueryIntersectsAdjacentValues() throws Exception {
*/
public void testFromLargerToErrors() throws Exception {
QueryShardContext context = createContext();
RangeFieldType ft = createDefaultFieldType(FIELDNAME);
RangeFieldType ft = createDefaultFieldType();

final Object from;
final Object to;
Expand Down Expand Up @@ -217,7 +216,7 @@ private QueryShardContext createContext() {
public void testDateRangeQueryUsingMappingFormat() {
QueryShardContext context = createContext();
RangeFieldType strict
= new RangeFieldType(FIELDNAME, true, false, RangeFieldMapper.Defaults.DATE_FORMATTER, Collections.emptyMap());
= new RangeFieldType("field", true, false, RangeFieldMapper.Defaults.DATE_FORMATTER, Collections.emptyMap());
// don't use DISJOINT here because it doesn't work on date fields which we want to compare bounds with
ShapeRelation relation = randomValueOtherThan(ShapeRelation.DISJOINT,() -> randomFrom(ShapeRelation.values()));

Expand All @@ -236,13 +235,13 @@ public void testDateRangeQueryUsingMappingFormat() {
assertEquals(1465975790000L, formatter.parseMillis(from));
assertEquals(1466062190000L, formatter.parseMillis(to));

RangeFieldType fieldType = new RangeFieldType(FIELDNAME, true, true, formatter, Collections.emptyMap());
RangeFieldType fieldType = new RangeFieldType("field", true, true, formatter, Collections.emptyMap());
final Query query = fieldType.rangeQuery(from, to, true, true, relation, null, fieldType.dateMathParser(), context);
assertEquals("field:<ranges:[1465975790000 : 1466062190999]>", query.toString());

// compare lower and upper bounds with what we would get on a `date` field
DateFieldType dateFieldType
= new DateFieldType(FIELDNAME, true, true, formatter, DateFieldMapper.Resolution.MILLISECONDS, Collections.emptyMap());
= new DateFieldType("field", true, true, formatter, DateFieldMapper.Resolution.MILLISECONDS, Collections.emptyMap());
final Query queryOnDateField = dateFieldType.rangeQuery(from, to, true, true, relation, null, fieldType.dateMathParser(), context);
assertEquals("field:[1465975790000 TO 1466062190999]", queryOnDateField.toString());
}
Expand All @@ -259,7 +258,7 @@ public void testDateVsDateRangeBounds() {
long lower = randomLongBetween(formatter.parseMillis("2000-01-01T00:00"), formatter.parseMillis("2010-01-01T00:00"));
long upper = randomLongBetween(formatter.parseMillis("2011-01-01T00:00"), formatter.parseMillis("2020-01-01T00:00"));

RangeFieldType fieldType = new RangeFieldType(FIELDNAME, true, false, formatter, Collections.emptyMap());
RangeFieldType fieldType = new RangeFieldType("field", true, false, formatter, Collections.emptyMap());
String lowerAsString = formatter.formatMillis(lower);
String upperAsString = formatter.formatMillis(upper);
// also add date math rounding to days occasionally
Expand All @@ -286,7 +285,7 @@ public void testDateVsDateRangeBounds() {
}

// check that using this bounds we get similar query when constructing equivalent query on date_range field
Query range = LongRange.newIntersectsQuery(FIELDNAME, new long[] { lowerBoundLong }, new long[] { upperBoundLong });
Query range = LongRange.newIntersectsQuery("field", new long[] { lowerBoundLong }, new long[] { upperBoundLong });
assertEquals(range, query);
}

Expand All @@ -313,16 +312,16 @@ private Query getDateRangeQuery(ShapeRelation relation, DateTime from, DateTime
Query indexQuery;
BinaryDocValuesRangeQuery.QueryType queryType;
if (relation == ShapeRelation.WITHIN) {
indexQuery = LongRange.newWithinQuery(FIELDNAME, lower, upper);
indexQuery = LongRange.newWithinQuery("field", lower, upper);
queryType = BinaryDocValuesRangeQuery.QueryType.WITHIN;
} else if (relation == ShapeRelation.CONTAINS) {
indexQuery = LongRange.newContainsQuery(FIELDNAME, lower, upper);
indexQuery = LongRange.newContainsQuery("field", lower, upper);
queryType = BinaryDocValuesRangeQuery.QueryType.CONTAINS;
} else {
indexQuery = LongRange.newIntersectsQuery(FIELDNAME, lower, upper);
indexQuery = LongRange.newIntersectsQuery("field", lower, upper);
queryType = BinaryDocValuesRangeQuery.QueryType.INTERSECTS;
}
Query dvQuery = RangeType.DATE.dvRangeQuery(FIELDNAME, queryType, from.getMillis(),
Query dvQuery = RangeType.DATE.dvRangeQuery("field", queryType, from.getMillis(),
to.getMillis(), includeLower, includeUpper);
return new IndexOrDocValuesQuery(indexQuery, dvQuery);
}
Expand All @@ -333,16 +332,16 @@ private Query getIntRangeQuery(ShapeRelation relation, int from, int to, boolean
Query indexQuery;
BinaryDocValuesRangeQuery.QueryType queryType;
if (relation == ShapeRelation.WITHIN) {
indexQuery = IntRange.newWithinQuery(FIELDNAME, lower, upper);
indexQuery = IntRange.newWithinQuery("field", lower, upper);
queryType = BinaryDocValuesRangeQuery.QueryType.WITHIN;
} else if (relation == ShapeRelation.CONTAINS) {
indexQuery = IntRange.newContainsQuery(FIELDNAME, lower, upper);
indexQuery = IntRange.newContainsQuery("field", lower, upper);
queryType = BinaryDocValuesRangeQuery.QueryType.CONTAINS;
} else {
indexQuery = IntRange.newIntersectsQuery(FIELDNAME, lower, upper);
indexQuery = IntRange.newIntersectsQuery("field", lower, upper);
queryType = BinaryDocValuesRangeQuery.QueryType.INTERSECTS;
}
Query dvQuery = RangeType.INTEGER.dvRangeQuery(FIELDNAME, queryType, from, to,
Query dvQuery = RangeType.INTEGER.dvRangeQuery("field", queryType, from, to,
includeLower, includeUpper);
return new IndexOrDocValuesQuery(indexQuery, dvQuery);
}
Expand All @@ -353,16 +352,16 @@ private Query getLongRangeQuery(ShapeRelation relation, long from, long to, bool
Query indexQuery;
BinaryDocValuesRangeQuery.QueryType queryType;
if (relation == ShapeRelation.WITHIN) {
indexQuery = LongRange.newWithinQuery(FIELDNAME, lower, upper);
indexQuery = LongRange.newWithinQuery("field", lower, upper);
queryType = BinaryDocValuesRangeQuery.QueryType.WITHIN;
} else if (relation == ShapeRelation.CONTAINS) {
indexQuery = LongRange.newContainsQuery(FIELDNAME, lower, upper);
indexQuery = LongRange.newContainsQuery("field", lower, upper);
queryType = BinaryDocValuesRangeQuery.QueryType.CONTAINS;
} else {
indexQuery = LongRange.newIntersectsQuery(FIELDNAME, lower, upper);
indexQuery = LongRange.newIntersectsQuery("field", lower, upper);
queryType = BinaryDocValuesRangeQuery.QueryType.INTERSECTS;
}
Query dvQuery = RangeType.LONG.dvRangeQuery(FIELDNAME, queryType, from, to,
Query dvQuery = RangeType.LONG.dvRangeQuery("field", queryType, from, to,
includeLower, includeUpper);
return new IndexOrDocValuesQuery(indexQuery, dvQuery);
}
Expand All @@ -373,16 +372,16 @@ private Query getFloatRangeQuery(ShapeRelation relation, float from, float to, b
Query indexQuery;
BinaryDocValuesRangeQuery.QueryType queryType;
if (relation == ShapeRelation.WITHIN) {
indexQuery = FloatRange.newWithinQuery(FIELDNAME, lower, upper);
indexQuery = FloatRange.newWithinQuery("field", lower, upper);
queryType = BinaryDocValuesRangeQuery.QueryType.WITHIN;
} else if (relation == ShapeRelation.CONTAINS) {
indexQuery = FloatRange.newContainsQuery(FIELDNAME, lower, upper);
indexQuery = FloatRange.newContainsQuery("field", lower, upper);
queryType = BinaryDocValuesRangeQuery.QueryType.CONTAINS;
} else {
indexQuery = FloatRange.newIntersectsQuery(FIELDNAME, lower, upper);
indexQuery = FloatRange.newIntersectsQuery("field", lower, upper);
queryType = BinaryDocValuesRangeQuery.QueryType.INTERSECTS;
}
Query dvQuery = RangeType.FLOAT.dvRangeQuery(FIELDNAME, queryType, from, to,
Query dvQuery = RangeType.FLOAT.dvRangeQuery("field", queryType, from, to,
includeLower, includeUpper);
return new IndexOrDocValuesQuery(indexQuery, dvQuery);
}
Expand All @@ -394,16 +393,16 @@ private Query getDoubleRangeQuery(ShapeRelation relation, double from, double to
Query indexQuery;
BinaryDocValuesRangeQuery.QueryType queryType;
if (relation == ShapeRelation.WITHIN) {
indexQuery = DoubleRange.newWithinQuery(FIELDNAME, lower, upper);
indexQuery = DoubleRange.newWithinQuery("field", lower, upper);
queryType = BinaryDocValuesRangeQuery.QueryType.WITHIN;
} else if (relation == ShapeRelation.CONTAINS) {
indexQuery = DoubleRange.newContainsQuery(FIELDNAME, lower, upper);
indexQuery = DoubleRange.newContainsQuery("field", lower, upper);
queryType = BinaryDocValuesRangeQuery.QueryType.CONTAINS;
} else {
indexQuery = DoubleRange.newIntersectsQuery(FIELDNAME, lower, upper);
indexQuery = DoubleRange.newIntersectsQuery("field", lower, upper);
queryType = BinaryDocValuesRangeQuery.QueryType.INTERSECTS;
}
Query dvQuery = RangeType.DOUBLE.dvRangeQuery(FIELDNAME, queryType, from, to,
Query dvQuery = RangeType.DOUBLE.dvRangeQuery("field", queryType, from, to,
includeLower, includeUpper);
return new IndexOrDocValuesQuery(indexQuery, dvQuery);
}
Expand All @@ -415,16 +414,16 @@ private Query getInetAddressRangeQuery(ShapeRelation relation, InetAddress from,
Query indexQuery;
BinaryDocValuesRangeQuery.QueryType queryType;
if (relation == ShapeRelation.WITHIN) {
indexQuery = InetAddressRange.newWithinQuery(FIELDNAME, lower, upper);
indexQuery = InetAddressRange.newWithinQuery("field", lower, upper);
queryType = BinaryDocValuesRangeQuery.QueryType.WITHIN;
} else if (relation == ShapeRelation.CONTAINS) {
indexQuery = InetAddressRange.newContainsQuery(FIELDNAME, lower, upper);
indexQuery = InetAddressRange.newContainsQuery("field", lower, upper);
queryType = BinaryDocValuesRangeQuery.QueryType.CONTAINS;
} else {
indexQuery = InetAddressRange.newIntersectsQuery(FIELDNAME, lower, upper);
indexQuery = InetAddressRange.newIntersectsQuery("field", lower, upper);
queryType = BinaryDocValuesRangeQuery.QueryType.INTERSECTS;
}
Query dvQuery = RangeType.IP.dvRangeQuery(FIELDNAME, queryType, from, to,
Query dvQuery = RangeType.IP.dvRangeQuery("field", queryType, from, to,
includeLower, includeUpper);
return new IndexOrDocValuesQuery(indexQuery, dvQuery);
}
Expand Down Expand Up @@ -472,7 +471,7 @@ public void testParseIp() {
public void testTermQuery() throws Exception {
// See https://github.com/elastic/elasticsearch/issues/25950
QueryShardContext context = createContext();
RangeFieldType ft = createDefaultFieldType(FIELDNAME);
RangeFieldType ft = createDefaultFieldType();

Object value = nextFrom();
ShapeRelation relation = ShapeRelation.INTERSECTS;
Expand Down

0 comments on commit 9ae2971

Please sign in to comment.