Skip to content

Commit

Permalink
QL: constant_keyword support (#53241) (#53602)
Browse files Browse the repository at this point in the history 
(cherry picked from commit d6cd4ce)
  • Loading branch information
@astefan
astefan committed Mar 16, 2020
1 parent dc2edc9 commit 91ca9c5
Show file tree
Hide file tree
Showing 27 changed files with 497 additions and 127 deletions.
31 changes: 16 additions & 15 deletions docs/reference/sql/language/data-types.asciidoc
View file
Original file line number Diff line number Diff line change
Expand Up @@ -13,21 +13,22 @@ s|SQL precision

4+h| Core types

| <<null-value, `null`>> | null | NULL | 0
| <<boolean, `boolean`>> | boolean | BOOLEAN | 1
| <<number, `byte`>> | byte | TINYINT | 3
| <<number, `short`>> | short | SMALLINT | 5
| <<number, `integer`>> | integer | INTEGER | 10
| <<number, `long`>> | long | BIGINT | 19
| <<number, `double`>> | double | DOUBLE | 15
| <<number, `float`>> | float | REAL | 7
| <<number, `half_float`>> | half_float | FLOAT | 3
| <<number, `scaled_float`>> | scaled_float | DOUBLE | 15
| <<keyword, `keyword`>> | keyword | VARCHAR | 32,766
| <<text, `text`>> | text | VARCHAR | 2,147,483,647
| <<binary, `binary`>> | binary | VARBINARY | 2,147,483,647
| <<date, `date`>> | datetime | TIMESTAMP | 29
| <<ip, `ip`>> | ip | VARCHAR | 39
| <<null-value, `null`>> | null | NULL | 0
| <<boolean, `boolean`>> | boolean | BOOLEAN | 1
| <<number, `byte`>> | byte | TINYINT | 3
| <<number, `short`>> | short | SMALLINT | 5
| <<number, `integer`>> | integer | INTEGER | 10
| <<number, `long`>> | long | BIGINT | 19
| <<number, `double`>> | double | DOUBLE | 15
| <<number, `float`>> | float | REAL | 7
| <<number, `half_float`>> | half_float | FLOAT | 3
| <<number, `scaled_float`>> | scaled_float | DOUBLE | 15
| <<keyword, `keyword`>> | keyword | VARCHAR | 32,766
| <<constant-keyword, `constant_keyword`>> | constant_keyword| VARCHAR | 32,766
| <<text, `text`>> | text | VARCHAR | 2,147,483,647
| <<binary, `binary`>> | binary | VARBINARY | 2,147,483,647
| <<date, `date`>> | datetime | TIMESTAMP | 29
| <<ip, `ip`>> | ip | VARCHAR | 39

4+h| Complex types

Expand Down
2 changes: 2 additions & 0 deletions ...java/org/elasticsearch/xpack/ql/execution/search/extractor/AbstractFieldHitExtractor.java
View file
Original file line number Diff line number Diff line change
Expand Up @@ -28,6 +28,7 @@
import java.util.Objects;
import java.util.StringJoiner;

import static org.elasticsearch.xpack.ql.type.DataTypes.CONSTANT_KEYWORD;
import static org.elasticsearch.xpack.ql.type.DataTypes.DATETIME;
import static org.elasticsearch.xpack.ql.type.DataTypes.KEYWORD;
import static org.elasticsearch.xpack.ql.type.DataTypes.SCALED_FLOAT;
Expand Down Expand Up @@ -213,6 +214,7 @@ protected Object unwrapMultiValue(Object values) {
protected boolean isFromDocValuesOnly(DataType dataType) {
return dataType == KEYWORD // because of ignore_above.
|| dataType == DATETIME
|| dataType == CONSTANT_KEYWORD // because a non-existent value is considered the constant value itself
|| dataType == SCALED_FLOAT; // because of scaling_factor
}

Expand Down
21 changes: 18 additions & 3 deletions x-pack/plugin/ql/src/main/java/org/elasticsearch/xpack/ql/index/IndexResolver.java
View file
Original file line number Diff line number Diff line change
Expand Up @@ -25,6 +25,7 @@
import org.elasticsearch.index.IndexNotFoundException;
import org.elasticsearch.index.IndexSettings;
import org.elasticsearch.xpack.ql.QlIllegalArgumentException;
import org.elasticsearch.xpack.ql.type.ConstantKeywordEsField;
import org.elasticsearch.xpack.ql.type.DataType;
import org.elasticsearch.xpack.ql.type.DataTypeRegistry;
import org.elasticsearch.xpack.ql.type.DateEsField;
Expand Down Expand Up @@ -60,6 +61,7 @@
import static java.util.Collections.emptyMap;
import static java.util.Collections.emptySet;
import static org.elasticsearch.action.ActionListener.wrap;
import static org.elasticsearch.xpack.ql.type.DataTypes.CONSTANT_KEYWORD;
import static org.elasticsearch.xpack.ql.type.DataTypes.DATETIME;
import static org.elasticsearch.xpack.ql.type.DataTypes.KEYWORD;
import static org.elasticsearch.xpack.ql.type.DataTypes.OBJECT;
Expand Down Expand Up @@ -298,8 +300,13 @@ public static IndexResolution mergedMappings(DataTypeRegistry typeRegistry, Stri
StringBuilder errorMessage = new StringBuilder();

boolean hasUnmapped = types.containsKey(UNMAPPED);
// a keyword field and a constant_keyword field with the same name in two different indices are considered "compatible"
// since a common use case of constant_keyword field involves two indices with a field having the same name: one being
// a keyword, the other being a constant_keyword
boolean hasCompatibleKeywords = types.containsKey(KEYWORD.esType()) && types.containsKey(CONSTANT_KEYWORD.esType());
int allowedTypesCount = (hasUnmapped ? 2 : 1) + (hasCompatibleKeywords ? 1 : 0);

if (types.size() > (hasUnmapped ? 2 : 1)) {
if (types.size() > allowedTypesCount) {
// build the error message
// and create a MultiTypeField

Expand Down Expand Up @@ -344,6 +351,11 @@ public static IndexResolution mergedMappings(DataTypeRegistry typeRegistry, Stri
}
}

// if there are both a keyword and a constant_keyword type for this field, only keep the keyword as a common compatible type
if (hasCompatibleKeywords) {
types.remove(CONSTANT_KEYWORD.esType());
}

// everything checks
return null;
});
Expand Down Expand Up @@ -435,6 +447,9 @@ private static EsField createField(DataTypeRegistry typeRegistry, String fieldNa
if (esType == DATETIME) {
return new DateEsField(fieldName, props, isAggregateable);
}
if (esType == CONSTANT_KEYWORD) {
return new ConstantKeywordEsField(fieldName);
}
if (esType == UNSUPPORTED) {
return new UnsupportedEsField(fieldName, typeName, null, props);
}
Expand Down Expand Up @@ -501,14 +516,14 @@ private static List<EsIndex> buildIndices(DataTypeRegistry typeRegistry, String[

for (Entry<String, Map<String, FieldCapabilities>> entry : sortedFields) {
String fieldName = entry.getKey();
Map<String, FieldCapabilities> types = entry.getValue();

// ignore size added by the mapper plugin
if (FIELD_NAMES_BLACKLIST.contains(fieldName)) {
continue;
}

// apply verification
Map<String, FieldCapabilities> types = new LinkedHashMap<>(entry.getValue());
// apply verification and possibly remove the "duplicate" CONSTANT_KEYWORD field type
final InvalidMappedField invalidField = validityVerifier.apply(fieldName, types);

// filter meta fields and unmapped
Expand Down
22 changes: 22 additions & 0 deletions x-pack/plugin/ql/src/main/java/org/elasticsearch/xpack/ql/type/ConstantKeywordEsField.java
View file
Original file line number Diff line number Diff line change
@@ -0,0 +1,22 @@
/*
* Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
* or more contributor license agreements. Licensed under the Elastic License;
* you may not use this file except in compliance with the Elastic License.
*/

package org.elasticsearch.xpack.ql.type;

import java.util.Collections;

import static org.elasticsearch.xpack.ql.type.DataTypes.CONSTANT_KEYWORD;

/**
* SQL-related information about an index field with a constant_keyword type
*/
public class ConstantKeywordEsField extends KeywordEsField {

public ConstantKeywordEsField(String name) {
super(name, CONSTANT_KEYWORD, Collections.emptyMap(), true, Short.MAX_VALUE, false, false);
}

}
8 changes: 6 additions & 2 deletions x-pack/plugin/ql/src/main/java/org/elasticsearch/xpack/ql/type/DataTypeConverter.java
View file
Original file line number Diff line number Diff line change
Expand Up @@ -21,6 +21,7 @@

import static org.elasticsearch.xpack.ql.type.DataTypes.BOOLEAN;
import static org.elasticsearch.xpack.ql.type.DataTypes.BYTE;
import static org.elasticsearch.xpack.ql.type.DataTypes.CONSTANT_KEYWORD;
import static org.elasticsearch.xpack.ql.type.DataTypes.DATETIME;
import static org.elasticsearch.xpack.ql.type.DataTypes.DOUBLE;
import static org.elasticsearch.xpack.ql.type.DataTypes.FLOAT;
Expand Down Expand Up @@ -59,9 +60,12 @@ public static DataType commonType(DataType left, DataType right) {
return left;
}
if (isString(left) && isString(right)) {
if (left == TEXT) {
if (left == TEXT || right == TEXT) {
return TEXT;
}
if (left == KEYWORD) {
return KEYWORD;
}
return right;
}
if (left.isNumeric() && right.isNumeric()) {
Expand Down Expand Up @@ -120,7 +124,7 @@ public static Converter converterFor(DataType from, DataType to) {
return DefaultConverter.TO_NULL;
}
// proper converters
if (to == KEYWORD || to == TEXT) {
if (to == KEYWORD || to == TEXT || to == CONSTANT_KEYWORD) {
return conversionToString(from);
}
if (to == LONG) {
Expand Down
38 changes: 20 additions & 18 deletions x-pack/plugin/ql/src/main/java/org/elasticsearch/xpack/ql/type/DataTypes.java
View file
Original file line number Diff line number Diff line change
Expand Up @@ -20,33 +20,34 @@
public final class DataTypes {

// @formatter:off
public static final DataType UNSUPPORTED = new DataType("UNSUPPORTED", null, 0, false, false, false);
public static final DataType UNSUPPORTED = new DataType("UNSUPPORTED", null, 0, false, false, false);

public static final DataType NULL = new DataType("null", 0, false, false, false);
public static final DataType NULL = new DataType("null", 0, false, false, false);

public static final DataType BOOLEAN = new DataType("boolean", 1, false, false, false);
public static final DataType BOOLEAN = new DataType("boolean", 1, false, false, false);
// integer numeric
public static final DataType BYTE = new DataType("byte", Byte.BYTES, true, false, true);
public static final DataType SHORT = new DataType("short", Short.BYTES, true, false, true);
public static final DataType INTEGER = new DataType("integer", Integer.BYTES, true, false, true);
public static final DataType LONG = new DataType("long", Long.BYTES, true, false, true);
public static final DataType BYTE = new DataType("byte", Byte.BYTES, true, false, true);
public static final DataType SHORT = new DataType("short", Short.BYTES, true, false, true);
public static final DataType INTEGER = new DataType("integer", Integer.BYTES, true, false, true);
public static final DataType LONG = new DataType("long", Long.BYTES, true, false, true);
// decimal numeric
public static final DataType DOUBLE = new DataType("double", Double.BYTES, false, true, true);
public static final DataType FLOAT = new DataType("float", Float.BYTES, false, true, true);
public static final DataType HALF_FLOAT = new DataType("half_float", Float.BYTES, false, true, true);
public static final DataType SCALED_FLOAT = new DataType("scaled_float", Long.BYTES, false, true, true);
public static final DataType DOUBLE = new DataType("double", Double.BYTES, false, true, true);
public static final DataType FLOAT = new DataType("float", Float.BYTES, false, true, true);
public static final DataType HALF_FLOAT = new DataType("half_float", Float.BYTES, false, true, true);
public static final DataType SCALED_FLOAT = new DataType("scaled_float", Long.BYTES, false, true, true);
// string
public static final DataType KEYWORD = new DataType("keyword", Integer.MAX_VALUE, false, false, true);
public static final DataType TEXT = new DataType("text", Integer.MAX_VALUE, false, false, false);
public static final DataType KEYWORD = new DataType("keyword", Integer.MAX_VALUE, false, false, true);
public static final DataType TEXT = new DataType("text", Integer.MAX_VALUE, false, false, false);
public static final DataType CONSTANT_KEYWORD = new DataType("constant_keyword", Integer.MAX_VALUE, false, false, true);
// date
public static final DataType DATETIME = new DataType("DATETIME", "date", Long.BYTES, false, false, true);
public static final DataType DATETIME = new DataType("DATETIME", "date", Long.BYTES, false, false, true);
// ip
public static final DataType IP = new DataType("ip", 45, false, false, true);
// binary
public static final DataType BINARY = new DataType("binary", Integer.MAX_VALUE, false, false, true);
public static final DataType BINARY = new DataType("binary", Integer.MAX_VALUE, false, false, true);
// complex types
public static final DataType OBJECT = new DataType("object", 0, false, false, false);
public static final DataType NESTED = new DataType("nested", 0, false, false, false);
public static final DataType OBJECT = new DataType("object", 0, false, false, false);
public static final DataType NESTED = new DataType("nested", 0, false, false, false);
//@formatter:on

private static final Collection<DataType> TYPES = unmodifiableList(Arrays.asList(
Expand All @@ -63,6 +64,7 @@ public final class DataTypes {
SCALED_FLOAT,
KEYWORD,
TEXT,
CONSTANT_KEYWORD,
DATETIME,
IP,
BINARY,
Expand Down Expand Up @@ -134,7 +136,7 @@ public static boolean isUnsupported(DataType from) {
}

public static boolean isString(DataType t) {
return t == KEYWORD || t == TEXT;
return t == KEYWORD || t == TEXT || t == CONSTANT_KEYWORD;
}

public static boolean isPrimitive(DataType t) {
Expand Down
7 changes: 6 additions & 1 deletion x-pack/plugin/ql/src/main/java/org/elasticsearch/xpack/ql/type/KeywordEsField.java
View file
Original file line number Diff line number Diff line change
Expand Up @@ -29,7 +29,12 @@ public KeywordEsField(String name, Map<String, EsField> properties, boolean hasD

public KeywordEsField(String name, Map<String, EsField> properties, boolean hasDocValues, int precision,
boolean normalized, boolean isAlias) {
super(name, KEYWORD, properties, hasDocValues, isAlias);
this(name, KEYWORD, properties, hasDocValues, precision, normalized, isAlias);
}

protected KeywordEsField(String name, DataType esDataType, Map<String, EsField> properties, boolean hasDocValues, int precision,
boolean normalized, boolean isAlias) {
super(name, esDataType, properties, hasDocValues, isAlias);
this.precision = precision;
this.normalized = normalized;
}
Expand Down
3 changes: 2 additions & 1 deletion x-pack/plugin/ql/src/main/java/org/elasticsearch/xpack/ql/type/TextEsField.java
View file
Original file line number Diff line number Diff line change
Expand Up @@ -11,6 +11,7 @@
import java.util.Map;
import java.util.function.Function;

import static org.elasticsearch.xpack.ql.type.DataTypes.CONSTANT_KEYWORD;
import static org.elasticsearch.xpack.ql.type.DataTypes.KEYWORD;
import static org.elasticsearch.xpack.ql.type.DataTypes.TEXT;

Expand Down Expand Up @@ -44,7 +45,7 @@ public Exact getExactInfo() {
private Tuple<EsField, String> findExact() {
EsField field = null;
for (EsField property : getProperties().values()) {
if (property.getDataType() == KEYWORD && property.getExactInfo().hasExact()) {
if ((property.getDataType() == KEYWORD || property.getDataType() == CONSTANT_KEYWORD) && property.getExactInfo().hasExact()) {
if (field != null) {
return new Tuple<>(null, "Multiple exact keyword candidates available for [" + getName() +
"]; specify which one to use");
Expand Down
3 changes: 3 additions & 0 deletions x-pack/plugin/ql/src/main/java/org/elasticsearch/xpack/ql/type/Types.java
View file
Original file line number Diff line number Diff line change
Expand Up @@ -14,6 +14,7 @@
import java.util.Map.Entry;

import static java.util.Collections.emptyMap;
import static org.elasticsearch.xpack.ql.type.DataTypes.CONSTANT_KEYWORD;
import static org.elasticsearch.xpack.ql.type.DataTypes.DATETIME;
import static org.elasticsearch.xpack.ql.type.DataTypes.KEYWORD;
import static org.elasticsearch.xpack.ql.type.DataTypes.NESTED;
Expand Down Expand Up @@ -89,6 +90,8 @@ private static void walkMapping(DataTypeRegistry typeRegistry, String name, Obje
int length = intSetting(content.get("ignore_above"), Short.MAX_VALUE);
boolean normalized = Strings.hasText(textSetting(content.get("normalizer"), null));
field = new KeywordEsField(name, properties, docValues, length, normalized);
} else if (esDataType == CONSTANT_KEYWORD) {
field = new ConstantKeywordEsField(name);
} else if (esDataType == DATETIME) {
field = new DateEsField(name, properties, docValues);
} else if (esDataType == UNSUPPORTED) {
Expand Down
7 changes: 7 additions & 0 deletions x-pack/plugin/ql/src/test/java/org/elasticsearch/xpack/ql/type/DataTypeConversionTests.java
View file
Original file line number Diff line number Diff line change
Expand Up @@ -17,6 +17,7 @@
import static org.elasticsearch.xpack.ql.type.DataTypeConverter.converterFor;
import static org.elasticsearch.xpack.ql.type.DataTypes.BOOLEAN;
import static org.elasticsearch.xpack.ql.type.DataTypes.BYTE;
import static org.elasticsearch.xpack.ql.type.DataTypes.CONSTANT_KEYWORD;
import static org.elasticsearch.xpack.ql.type.DataTypes.DATETIME;
import static org.elasticsearch.xpack.ql.type.DataTypes.DOUBLE;
import static org.elasticsearch.xpack.ql.type.DataTypes.FLOAT;
Expand Down Expand Up @@ -360,6 +361,7 @@ public void testCommonType() {
assertEquals(BOOLEAN, commonType(BOOLEAN, BOOLEAN));
assertEquals(NULL, commonType(NULL, NULL));
assertEquals(INTEGER, commonType(INTEGER, KEYWORD));
assertEquals(DOUBLE, commonType(DOUBLE, CONSTANT_KEYWORD));
assertEquals(LONG, commonType(TEXT, LONG));
assertEquals(SHORT, commonType(SHORT, BYTE));
assertEquals(FLOAT, commonType(BYTE, FLOAT));
Expand All @@ -369,6 +371,11 @@ public void testCommonType() {
// strings
assertEquals(TEXT, commonType(TEXT, KEYWORD));
assertEquals(TEXT, commonType(KEYWORD, TEXT));
assertEquals(TEXT, commonType(TEXT, CONSTANT_KEYWORD));
assertEquals(TEXT, commonType(CONSTANT_KEYWORD, TEXT));
assertEquals(KEYWORD, commonType(KEYWORD, CONSTANT_KEYWORD));
assertEquals(KEYWORD, commonType(CONSTANT_KEYWORD, KEYWORD));
assertEquals(CONSTANT_KEYWORD, commonType(CONSTANT_KEYWORD, CONSTANT_KEYWORD));
}

public void testEsDataTypes() {
Expand Down
Loading

0 comments on commit 91ca9c5

Please sign in to comment.