diff --git a/modules/lang-painless/src/main/resources/org/elasticsearch/painless/org.elasticsearch.script.fields.txt b/modules/lang-painless/src/main/resources/org/elasticsearch/painless/org.elasticsearch.script.fields.txt index b21437b14a884..328a751217d2b 100644 --- a/modules/lang-painless/src/main/resources/org/elasticsearch/painless/org.elasticsearch.script.fields.txt +++ b/modules/lang-painless/src/main/resources/org/elasticsearch/painless/org.elasticsearch.script.fields.txt @@ -10,10 +10,18 @@ # API class org.elasticsearch.script.Field { + org.elasticsearch.script.Converter BigInteger + org.elasticsearch.script.Converter Long String getName() boolean isEmpty() List getValues() def getValue(def) + double getDouble(double) + long getLong(long) + org.elasticsearch.script.Field as(org.elasticsearch.script.Converter) +} + +class org.elasticsearch.script.Converter { } class org.elasticsearch.script.DocBasedScript { diff --git a/modules/lang-painless/src/yamlRestTest/resources/rest-api-spec/test/painless/40_fields_api.yml b/modules/lang-painless/src/yamlRestTest/resources/rest-api-spec/test/painless/40_fields_api.yml index 61749d8333cd4..df571349ecb97 100644 --- a/modules/lang-painless/src/yamlRestTest/resources/rest-api-spec/test/painless/40_fields_api.yml +++ b/modules/lang-painless/src/yamlRestTest/resources/rest-api-spec/test/painless/40_fields_api.yml @@ -158,3 +158,403 @@ setup: script: "field('sval.keyword').getValue('b') == 'a'" - match: { hits.total: 1 } - match: { hits.hits.0._id: d3 } + +--- +"missing field": + - do: + indices.create: + index: test + body: + settings: + number_of_shards: 2 + mappings: + properties: + dval: + type: double + - do: + index: + index: test + id: d1 + body: {"foo": 25, "dval": 1.0 } + - do: + index: + index: test + id: d2 + body: {"foo": 9223372036854775807, "dval": 2.0 } + - do: + index: + index: test + id: d3 + body: { "bar": "abc", "dval": 3.0 } + - do: + indices.refresh: {} + - do: + search: + rest_total_hits_as_int: true + index: test + body: + query: { "match_all": {} } + sort: [ { dval: asc } ] + script_fields: + missing_field: + script: + source: "field('foo').as(Field.BigInteger).getValue(BigInteger.TEN).add(BigInteger.ONE)" + - match: { hits.total: 3 } + - match: { hits.hits.0._id: d1 } + - match: { hits.hits.0.fields.missing_field.0: 26 } + - match: { hits.hits.1._id: d2 } + - match: { hits.hits.1.fields.missing_field.0: 9223372036854775808 } + - match: { hits.hits.2._id: d3 } + - match: { hits.hits.2.fields.missing_field.0: 11 } + - do: + search: + rest_total_hits_as_int: true + index: test + body: + query: { "match_all": {} } + sort: [ { dval: asc } ] + script_fields: + missing_field: + script: + source: "field('foo').as(Field.BigInteger).as(Field.Long).getValue(10L) + 2L" + - match: { hits.total: 3 } + - match: { hits.hits.0._id: d1 } + - match: { hits.hits.0.fields.missing_field.0: 27 } + - match: { hits.hits.1._id: d2 } + - match: { hits.hits.1.fields.missing_field.0: -9223372036854775809 } + - match: { hits.hits.2._id: d3 } + - match: { hits.hits.2.fields.missing_field.0: 12 } + +--- +"date to long": + - do: + indices.create: + index: test + body: + settings: + number_of_shards: 2 + mappings: + properties: + dval: + type: double + dtnanos: + type: date_nanos + dt: + type: date + - do: + index: + index: test + id: d1 + body: { "dval": 101, "dt": "2015-01-01T12:10:40Z" } + - do: + index: + index: test + id: d2 + body: { "dval": 202, "dt": "2015-01-01" } + - do: + index: + index: test + id: d3 + body: { "dval": 303, "dt": 1420072496000 } + - do: + index: + index: test + id: d4 + body: { "dval": 404, "dtnanos": "2015-01-01T12:10:30.123456789Z"} + - do: + index: + index: test + id: d5 + body: { "dval": 505, "dtnanos": "2261-04-11T23:47:33.54775807Z" } + - do: + index: + index: test + id: d6 + body: { "dval": 606 } + - do: + indices.refresh: {} + - do: + search: + rest_total_hits_as_int: true + index: test + body: + query: { "match_all": {} } + sort: [ { dval: asc } ] + script_fields: + dt_bigint: + script: + source: "field('dt').as(Field.BigInteger).getValue(BigInteger.TEN).add(BigInteger.ONE)" + - match: { hits.total: 6 } + - match: { hits.hits.0._id: d1 } + - match: { hits.hits.0.fields.dt_bigint.0: 1420114240001 } + - match: { hits.hits.1._id: d2 } + - match: { hits.hits.1.fields.dt_bigint.0: 1420070400001 } + - match: { hits.hits.2._id: d3 } + - match: { hits.hits.2.fields.dt_bigint.0: 1420072496001 } + - match: { hits.hits.3._id: d4 } + - match: { hits.hits.3.fields.dt_bigint.0: 11 } + - match: { hits.hits.4._id: d5 } + - match: { hits.hits.4.fields.dt_bigint.0: 11 } + - match: { hits.hits.5._id: d6 } + - match: { hits.hits.5.fields.dt_bigint.0: 11 } + - do: + search: + rest_total_hits_as_int: true + index: test + body: + query: { "match_all": {} } + sort: [ { dval: asc } ] + script_fields: + dt_long: + script: + source: "field('dt').as(Field.Long).getValue(10L) + 2L" + - match: { hits.total: 6 } + - match: { hits.hits.0._id: d1 } + - match: { hits.hits.0.fields.dt_long.0: 1420114240002 } + - match: { hits.hits.1._id: d2 } + - match: { hits.hits.1.fields.dt_long.0: 1420070400002 } + - match: { hits.hits.2._id: d3 } + - match: { hits.hits.2.fields.dt_long.0: 1420072496002 } + - match: { hits.hits.3._id: d4 } + - match: { hits.hits.3.fields.dt_long.0: 12 } + - match: { hits.hits.4._id: d5 } + - match: { hits.hits.4.fields.dt_long.0: 12 } + - match: { hits.hits.5._id: d6 } + - match: { hits.hits.5.fields.dt_long.0: 12 } + - do: + search: + rest_total_hits_as_int: true + index: test + body: + query: { "match_all": {} } + sort: [ { dval: asc } ] + script_fields: + dtnan_bigint: + script: + source: "field('dtnanos').as(Field.BigInteger).getValue(BigInteger.TEN).add(BigInteger.ONE)" + - match: { hits.total: 6 } + - match: { hits.hits.0._id: d1 } + - match: { hits.hits.0.fields.dtnan_bigint.0: 11 } + - match: { hits.hits.1._id: d2 } + - match: { hits.hits.1.fields.dtnan_bigint.0: 11 } + - match: { hits.hits.2._id: d3 } + - match: { hits.hits.2.fields.dtnan_bigint.0: 11 } + - match: { hits.hits.3._id: d4 } + - match: { hits.hits.3.fields.dtnan_bigint.0: 1420114230123456790 } + - match: { hits.hits.4._id: d5 } + - match: { hits.hits.4.fields.dtnan_bigint.0: 9191836053547758071 } + - match: { hits.hits.5._id: d6 } + - match: { hits.hits.5.fields.dtnan_bigint.0: 11 } + - do: + search: + rest_total_hits_as_int: true + index: test + body: + query: { "match_all": {} } + sort: [ { dval: asc } ] + script_fields: + dtnan_long: + script: + source: "field('dtnanos').as(Field.Long).getValue(field('dt').as(Field.Long).getValue(0L) * 1000) + 2L" + - match: { hits.total: 6 } + - match: { hits.hits.0._id: d1 } + - match: { hits.hits.0.fields.dtnan_long.0: 1420114240000002 } + - match: { hits.hits.1._id: d2 } + - match: { hits.hits.1.fields.dtnan_long.0: 1420070400000002 } + - match: { hits.hits.2._id: d3 } + - match: { hits.hits.2.fields.dtnan_long.0: 1420072496000002 } + - match: { hits.hits.3._id: d4 } + - match: { hits.hits.3.fields.dtnan_long.0: 1420114230123456791 } + - match: { hits.hits.4._id: d5 } + - match: { hits.hits.4.fields.dtnan_long.0: 9191836053547758072 } + - match: { hits.hits.5._id: d6 } + - match: { hits.hits.5.fields.dtnan_long.0: 2 } + +--- +"boolean to long and bigint": + - do: + indices.create: + index: test + body: + settings: + number_of_shards: 2 + mappings: + properties: + dval: + type: double + bool: + type: boolean + - do: + index: + index: test + id: d1 + body: { "dval": 101, "bool": true } + - do: + index: + index: test + id: d2 + body: { "dval": 202, "bool": false } + - do: + index: + index: test + id: d3 + body: { "dval": 303 } + - do: + indices.refresh: {} + - do: + search: + rest_total_hits_as_int: true + index: test + body: + query: { "match_all": {} } + sort: [ { dval: asc } ] + script_fields: + bool: + script: + source: "field('bool').as(Field.Long).getValue(-1)" + - match: { hits.total: 3 } + - match: { hits.hits.0._id: d1 } + - match: { hits.hits.0.fields.bool.0: 1 } + - match: { hits.hits.1._id: d2 } + - match: { hits.hits.1.fields.bool.0: 0 } + - match: { hits.hits.2._id: d3 } + - match: { hits.hits.2.fields.bool.0: -1 } + - do: + search: + rest_total_hits_as_int: true + index: test + body: + query: { "match_all": {} } + sort: [ { dval: asc } ] + script_fields: + bool: + script: + source: "field('bool').as(Field.BigInteger).getValue(-1)" + - match: { hits.total: 3 } + - match: { hits.hits.0._id: d1 } + - match: { hits.hits.0.fields.bool.0: 1 } + - match: { hits.hits.1._id: d2 } + - match: { hits.hits.1.fields.bool.0: 0 } + - match: { hits.hits.2._id: d3 } + - match: { hits.hits.2.fields.bool.0: -1 } +--- +"string to long and bigint": + - do: + indices.create: + index: test + body: + settings: + number_of_shards: 2 + mappings: + properties: + dval: + type: double + str: + type: keyword + - do: + index: + index: test + id: d1 + body: { "dval": 101, "str": "18446744073709551618" } + - do: + index: + index: test + id: d2 + body: { "dval": 202, "str": "9223372036854775809" } + - do: + index: + index: test + id: d3 + body: { "dval": 303, "str": "-1" } + - do: + index: + index: test + id: d4 + body: { "dval": 404 } + - do: + index: + index: test + id: d5 + body: { "dval": 505, "str": "unparseable number" } + - do: + index: + index: test + id: d6 + body: { "dval": 606, "str": "10E40" } + - do: + index: + index: test + id: d7 + body: { "dval": 707, "str": "18446744073709551618.555" } + - do: + indices.refresh: {} + - do: + search: + rest_total_hits_as_int: true + index: test + body: + query: { "match_all": {} } + sort: [ { dval: asc } ] + script_fields: + str: + script: + source: "field('str').as(Field.Long).getValue(-20)" + - match: { hits.total: 7 } + - match: { hits.hits.0._id: d1 } + - match: { hits.hits.0.fields.str.0: -20 } + - match: { hits.hits.1._id: d2 } + - match: { hits.hits.1.fields.str.0: -20 } + - match: { hits.hits.2._id: d3 } + - match: { hits.hits.2.fields.str.0: -1 } + - match: { hits.hits.3._id: d4 } + - match: { hits.hits.3.fields.str.0: -20 } + - match: { hits.hits.4._id: d5 } + - match: { hits.hits.4.fields.str.0: -20 } + - do: + search: + rest_total_hits_as_int: true + index: test + body: + query: { "match_all": {} } + sort: [ { dval: asc } ] + script_fields: + str: + script: + source: "field('str').getLong(-40)" + - match: { hits.total: 7 } + - match: { hits.hits.0._id: d1 } + - match: { hits.hits.0.fields.str.0: -40 } + - match: { hits.hits.1._id: d2 } + - match: { hits.hits.1.fields.str.0: -40 } + - match: { hits.hits.2._id: d3 } + - match: { hits.hits.2.fields.str.0: -1 } + - match: { hits.hits.3._id: d4 } + - match: { hits.hits.3.fields.str.0: -40 } + - match: { hits.hits.4._id: d5 } + - match: { hits.hits.4.fields.str.0: -40 } + - do: + search: + rest_total_hits_as_int: true + index: test + body: + query: { "match_all": {} } + sort: [ { dval: asc } ] + script_fields: + str: + script: + source: "field('str').as(Field.BigInteger).getValue(BigInteger.TEN)" + - match: { hits.total: 7 } + - match: { hits.hits.0._id: d1 } + - match: { hits.hits.0.fields.str.0: 18446744073709551618 } + - match: { hits.hits.1._id: d2 } + - match: { hits.hits.1.fields.str.0: 9223372036854775809 } + - match: { hits.hits.2._id: d3 } + - match: { hits.hits.2.fields.str.0: -1 } + - match: { hits.hits.3._id: d4 } + - match: { hits.hits.3.fields.str.0: 10 } + - match: { hits.hits.4._id: d5 } + - match: { hits.hits.4.fields.str.0: 10 } + - match: { hits.hits.5._id: d6 } + - match: { hits.hits.5.fields.str.0: 100000000000000000000000000000000000000000 } + - match: { hits.hits.6._id: d7 } + - match: { hits.hits.6.fields.str.0: 18446744073709551618 } diff --git a/server/src/main/java/org/elasticsearch/index/fielddata/ScriptDocValues.java b/server/src/main/java/org/elasticsearch/index/fielddata/ScriptDocValues.java index 86d007072197e..dff93fa777402 100644 --- a/server/src/main/java/org/elasticsearch/index/fielddata/ScriptDocValues.java +++ b/server/src/main/java/org/elasticsearch/index/fielddata/ScriptDocValues.java @@ -17,14 +17,19 @@ import org.elasticsearch.common.geo.GeoUtils; import org.elasticsearch.common.time.DateUtils; import org.elasticsearch.geometry.utils.Geohash; +import org.elasticsearch.script.Field; +import org.elasticsearch.script.FieldValues; +import org.elasticsearch.script.InvalidConversion; import org.elasticsearch.script.JodaCompatibleZonedDateTime; import java.io.IOException; import java.time.Instant; import java.time.ZoneOffset; +import java.time.temporal.ChronoUnit; import java.util.AbstractList; import java.util.Arrays; import java.util.Comparator; +import java.util.List; import java.util.function.UnaryOperator; /** @@ -35,7 +40,7 @@ * return as a single {@link ScriptDocValues} instance can be reused to return * values form multiple documents. */ -public abstract class ScriptDocValues extends AbstractList { +public abstract class ScriptDocValues extends AbstractList implements FieldValues { /** * Set the current doc ID. @@ -68,6 +73,31 @@ public final void sort(Comparator c) { throw new UnsupportedOperationException("doc values are unmodifiable"); } + public abstract Field toField(String fieldName); + + public List getValues() { + return this; + } + + public T getNonPrimitiveValue() { + return get(0); + } + + public long getLongValue() { + throw new InvalidConversion(this.getClass(), long.class); + } + + public double getDoubleValue() { + throw new InvalidConversion(this.getClass(), double.class); + } + + protected void throwIfEmpty() { + if (size() == 0) { + throw new IllegalStateException("A document doesn't have a value for a field! " + + "Use doc[].size()==0 to check if a document is missing a field!"); + } + } + public static final class Longs extends ScriptDocValues { private final SortedNumericDocValues in; private long[] values = new long[0]; @@ -107,10 +137,7 @@ public long getValue() { @Override public Long get(int index) { - if (count == 0) { - throw new IllegalStateException("A document doesn't have a value for a field! " + - "Use doc[].size()==0 to check if a document is missing a field!"); - } + throwIfEmpty(); return values[index]; } @@ -118,6 +145,23 @@ public Long get(int index) { public int size() { return count; } + + @Override + public long getLongValue() { + throwIfEmpty(); + return values[0]; + } + + @Override + public double getDoubleValue() { + throwIfEmpty(); + return values[0]; + } + + @Override + public Field toField(String fieldName) { + return new Field.LongField(fieldName, this); + } } public static final class Dates extends ScriptDocValues { @@ -192,6 +236,29 @@ void refreshArray() throws IOException { } } } + + @Override + public long getLongValue() { + throwIfEmpty(); + Instant dt = dates[0].toInstant(); + if (isNanos) { + return ChronoUnit.NANOS.between(java.time.Instant.EPOCH, dt); + } + return dt.toEpochMilli(); + } + + @Override + public double getDoubleValue() { + return getLongValue(); + } + + @Override + public Field toField(String fieldName) { + if (isNanos) { + return new Field.DateNanosField(fieldName, this); + } + return new Field.DateMillisField(fieldName, this); + } } public static final class Doubles extends ScriptDocValues { @@ -246,6 +313,22 @@ public Double get(int index) { public int size() { return count; } + + @Override + public long getLongValue() { + return (long) getDoubleValue(); + } + + @Override + public double getDoubleValue() { + throwIfEmpty(); + return values[0]; + } + + @Override + public Field toField(String fieldName) { + return new Field.DoubleField(fieldName, this); + } } public abstract static class Geometry extends ScriptDocValues { @@ -436,6 +519,11 @@ public double getMercatorHeight() { public GeoBoundingBox getBoundingBox() { return size() == 0 ? null : boundingBox; } + + @Override + public Field toField(String fieldName) { + return new Field.GeoPointField(fieldName, this); + } } public static final class Booleans extends ScriptDocValues { @@ -496,6 +584,22 @@ private static boolean[] grow(boolean[] array, int minSize) { return array; } + @Override + public long getLongValue() { + throwIfEmpty(); + return values[0] ? 1L : 0L; + } + + @Override + public double getDoubleValue() { + throwIfEmpty(); + return values[0] ? 1.0D : 0.0D; + } + + @Override + public Field toField(String fieldName) { + return new Field.BooleanField(fieldName, this); + } } abstract static class BinaryScriptDocValues extends ScriptDocValues { @@ -568,6 +672,21 @@ protected String bytesToString(BytesRef bytes) { public final String getValue() { return get(0); } + + @Override + public long getLongValue() { + return Long.parseLong(get(0)); + } + + @Override + public double getDoubleValue() { + return Double.parseDouble(get(0)); + } + + @Override + public Field toField(String fieldName) { + return new Field.StringField(fieldName, this); + } } public static final class BytesRefs extends BinaryScriptDocValues { @@ -594,5 +713,9 @@ public BytesRef getValue() { return get(0); } + @Override + public Field toField(String fieldName) { + return new Field.BytesRefField(fieldName, this); + } } } diff --git a/server/src/main/java/org/elasticsearch/index/mapper/IpFieldMapper.java b/server/src/main/java/org/elasticsearch/index/mapper/IpFieldMapper.java index 15a7a1c22c1ab..a95556a13bf87 100644 --- a/server/src/main/java/org/elasticsearch/index/mapper/IpFieldMapper.java +++ b/server/src/main/java/org/elasticsearch/index/mapper/IpFieldMapper.java @@ -355,6 +355,11 @@ public String get(int index) { public int size() { return count; } + + @Override + public org.elasticsearch.script.Field toField(String fieldName) { + return new org.elasticsearch.script.Field.IpField(fieldName, this); + } } @Override diff --git a/server/src/main/java/org/elasticsearch/script/Converter.java b/server/src/main/java/org/elasticsearch/script/Converter.java new file mode 100644 index 0000000000000..f903ebfbedb76 --- /dev/null +++ b/server/src/main/java/org/elasticsearch/script/Converter.java @@ -0,0 +1,31 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License + * 2.0 and the Server Side Public License, v 1; you may not use this file except + * in compliance with, at your election, the Elastic License 2.0 or the Server + * Side Public License, v 1. + */ + +package org.elasticsearch.script; + +/** + * Converts between one scripting {@link Field} type and another, {@code FC}, with a different underlying + * value type, {@code TC}. + */ +public interface Converter> { + /** + * Convert {@code sourceField} to a new field-type. Conversions come from user scripts so {@code covert} may + * be called on a {@link Field}'s own type. + */ + FC convert(Field sourceField); + + /** + * The destination {@link Field} class. + */ + Class getFieldClass(); + + /** + * The target value type. + */ + Class getTargetClass(); +} diff --git a/server/src/main/java/org/elasticsearch/script/Converters.java b/server/src/main/java/org/elasticsearch/script/Converters.java new file mode 100644 index 0000000000000..fa96c4dc5b827 --- /dev/null +++ b/server/src/main/java/org/elasticsearch/script/Converters.java @@ -0,0 +1,328 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License + * 2.0 and the Server Side Public License, v 1; you may not use this file except + * in compliance with, at your election, the Elastic License 2.0 or the Server + * Side Public License, v 1. + */ + +package org.elasticsearch.script; + +import java.math.BigDecimal; +import java.math.BigInteger; + +import java.time.Instant; +import java.time.temporal.ChronoUnit; +import java.util.List; +import java.util.stream.Collectors; + +import static org.elasticsearch.script.Field.BigIntegerField; +import static org.elasticsearch.script.Field.BooleanField; +import static org.elasticsearch.script.Field.DoubleField; +import static org.elasticsearch.script.Field.DateMillisField; +import static org.elasticsearch.script.Field.DateNanosField; +import static org.elasticsearch.script.Field.LongField; +import static org.elasticsearch.script.Field.StringField; + +/** + * {@link Converters} for scripting fields. These constants are exposed as static fields on {@link Field} to + * allow a user to convert via {@link Field#as(Converter)}. + */ +public class Converters { + /** + * Convert to a {@link BigIntegerField} from Long, Double or String Fields. + * Longs and Doubles are wrapped as BigIntegers. + * Strings are parsed as either Longs or Doubles and wrapped in a BigInteger. + */ + static final Converter BIGINTEGER; + + /** + * Convert to a {@link LongField} from Double, String, DateMillis, DateNanos, BigInteger or Boolean Fields. + * Double is cast to a Long. + * String is parsed as a Long. + * DateMillis is milliseconds since epoch. + * DateNanos is nanoseconds since epoch. + * {@link BigInteger#longValue()} is used for the BigInteger conversion. + * Boolean is {@code 1L} if {@code true}, {@code 0L} if {@code false}. + */ + static final Converter LONG; + + static { + BIGINTEGER = new Converter<>() { + @Override + public BigIntegerField convert(Field sourceField) { + if (sourceField instanceof LongField) { + return LongToBigInteger((LongField) sourceField); + } + + if (sourceField instanceof DoubleField) { + return DoubleToBigInteger((DoubleField) sourceField); + } + + if (sourceField instanceof StringField) { + return StringToBigInteger((StringField) sourceField); + } + + if (sourceField instanceof DateMillisField) { + return LongToBigInteger(DateMillisToLong((DateMillisField) sourceField)); + } + + if (sourceField instanceof DateNanosField) { + return LongToBigInteger(DateNanosToLong((DateNanosField) sourceField)); + } + + if (sourceField instanceof BooleanField) { + return LongToBigInteger(BooleanToLong((BooleanField) sourceField)); + } + + throw new InvalidConversion(sourceField.getClass(), getFieldClass()); + } + + @Override + public Class getFieldClass() { + return BigIntegerField.class; + } + + @Override + public Class getTargetClass() { + return BigInteger.class; + } + }; + + LONG = new Converter<>() { + @Override + public LongField convert(Field sourceField) { + if (sourceField instanceof DoubleField) { + return DoubleToLong((DoubleField) sourceField); + } + + if (sourceField instanceof StringField) { + return StringToLong((StringField) sourceField); + } + + if (sourceField instanceof DateMillisField) { + return DateMillisToLong((DateMillisField) sourceField); + } + + if (sourceField instanceof DateNanosField) { + return DateNanosToLong((DateNanosField) sourceField); + } + + if (sourceField instanceof BigIntegerField) { + return BigIntegerToLong((BigIntegerField) sourceField); + } + + if (sourceField instanceof BooleanField) { + return BooleanToLong((BooleanField) sourceField); + } + + throw new InvalidConversion(sourceField.getClass(), getFieldClass()); + } + + @Override + public Class getFieldClass() { + return LongField.class; + } + + @Override + public Class getTargetClass() { + return Long.class; + } + }; + } + + // No instances, please + private Converters() {} + + static BigIntegerField LongToBigInteger(LongField sourceField) { + FieldValues fv = sourceField.getFieldValues(); + return new BigIntegerField(sourceField.getName(), new DelegatingFieldValues<>(fv) { + @Override + public List getValues() { + return values.getValues().stream().map(BigInteger::valueOf).collect(Collectors.toList()); + } + + @Override + public BigInteger getNonPrimitiveValue() { + return BigInteger.valueOf(values.getLongValue()); + } + }); + } + + static BigIntegerField DoubleToBigInteger(DoubleField sourceField) { + FieldValues fv = sourceField.getFieldValues(); + return new BigIntegerField(sourceField.getName(), new DelegatingFieldValues<>(fv) { + @Override + public List getValues() { + return values.getValues().stream().map( + dbl -> BigInteger.valueOf(dbl.longValue()) + ).collect(Collectors.toList()); + } + + @Override + public BigInteger getNonPrimitiveValue() { + return BigInteger.valueOf(values.getLongValue()); + } + }); + } + + static BigIntegerField StringToBigInteger(StringField sourceField) { + FieldValues fv = sourceField.getFieldValues(); + return new BigIntegerField(sourceField.getName(), new DelegatingFieldValues(fv) { + protected BigInteger parseNumber(String str) { + try { + return new BigInteger(str); + } catch (NumberFormatException e) { + return new BigDecimal(str).toBigInteger(); + } + } + + @Override + public List getValues() { + // TODO(stu): this may throw + return values.getValues().stream().map(this::parseNumber).collect(Collectors.toList()); + } + + @Override + public BigInteger getNonPrimitiveValue() { + return parseNumber(values.getNonPrimitiveValue()); + } + }); + } + + static LongField BigIntegerToLong(BigIntegerField sourceField) { + FieldValues fv = sourceField.getFieldValues(); + return new LongField(sourceField.getName(), new DelegatingFieldValues(fv) { + @Override + public List getValues() { + return values.getValues().stream().map(BigInteger::longValue).collect(Collectors.toList()); + } + + @Override + public Long getNonPrimitiveValue() { + return values.getLongValue(); + } + }); + } + + static LongField BooleanToLong(BooleanField sourceField) { + FieldValues fv = sourceField.getFieldValues(); + return new LongField(sourceField.getName(), new DelegatingFieldValues(fv) { + @Override + public List getValues() { + return values.getValues().stream().map(bool -> bool ? 1L : 0L).collect(Collectors.toList()); + } + + @Override + public Long getNonPrimitiveValue() { + return getLongValue(); + } + }); + } + + static LongField DateMillisToLong(DateMillisField sourceField) { + FieldValues fv = sourceField.getFieldValues(); + return new LongField(sourceField.getName(), new DelegatingFieldValues(fv) { + @Override + public List getValues() { + return values.getValues().stream().map(dt -> dt.toInstant().toEpochMilli()).collect(Collectors.toList()); + } + + @Override + public Long getNonPrimitiveValue() { + return values.getNonPrimitiveValue().toInstant().toEpochMilli(); + } + }); + } + + static LongField DateNanosToLong(DateNanosField sourceField) { + FieldValues fv = sourceField.getFieldValues(); + return new LongField(sourceField.getName(), new DelegatingFieldValues(fv) { + protected long nanoLong(JodaCompatibleZonedDateTime dt) { + return ChronoUnit.NANOS.between(Instant.EPOCH, dt.toInstant()); + } + + @Override + public List getValues() { + return values.getValues().stream().map(this::nanoLong).collect(Collectors.toList()); + } + + @Override + public Long getNonPrimitiveValue() { + return ChronoUnit.NANOS.between(Instant.EPOCH, values.getNonPrimitiveValue().toInstant()); + } + }); + } + + static LongField DoubleToLong(DoubleField sourceField) { + FieldValues fv = sourceField.getFieldValues(); + return new LongField(sourceField.getName(), new DelegatingFieldValues(fv) { + @Override + public List getValues() { + return values.getValues().stream().map(Double::longValue).collect(Collectors.toList()); + } + + @Override + public Long getNonPrimitiveValue() { + return values.getLongValue(); + } + }); + } + + static LongField StringToLong(StringField sourceField) { + FieldValues fv = sourceField.getFieldValues(); + return new LongField(sourceField.getName(), new DelegatingFieldValues(fv) { + @Override + public List getValues() { + return values.getValues().stream().map(Long::parseLong).collect(Collectors.toList()); + } + + @Override + public Long getNonPrimitiveValue() { + return Long.parseLong(values.getNonPrimitiveValue()); + } + + @Override + public long getLongValue() { + return Long.parseLong(values.getNonPrimitiveValue()); + } + + @Override + public double getDoubleValue() { + return getLongValue(); + } + }); + } + + /** + * Helper for creating {@link Converter} classes which delegates all un-overridden methods to the underlying + * {@link FieldValues}. + */ + public abstract static class DelegatingFieldValues implements FieldValues { + protected FieldValues values; + + public DelegatingFieldValues(FieldValues values) { + this.values = values; + } + + @Override + public boolean isEmpty() { + return values.isEmpty(); + } + + @Override + public int size() { + return values.size(); + } + + @Override + public long getLongValue() { + return values.getLongValue(); + } + + @Override + public double getDoubleValue() { + return values.getDoubleValue(); + } + } +} diff --git a/server/src/main/java/org/elasticsearch/script/DocValuesDocReader.java b/server/src/main/java/org/elasticsearch/script/DocValuesDocReader.java index a10e8219f3d3d..80a6aed9364f3 100644 --- a/server/src/main/java/org/elasticsearch/script/DocValuesDocReader.java +++ b/server/src/main/java/org/elasticsearch/script/DocValuesDocReader.java @@ -16,6 +16,9 @@ import java.util.Map; import java.util.stream.Stream; +/** + * Provide access to DocValues for script {@code field} api and {@code doc} API. + */ public class DocValuesDocReader implements DocReader, LeafReaderContextSupplier { /** A leaf lookup for the bound segment this proxy will operate on. */ protected LeafSearchLookup leafLookup; @@ -23,7 +26,10 @@ public class DocValuesDocReader implements DocReader, LeafReaderContextSupplier // provide access to the leaf context reader for expressions protected final LeafReaderContext leafReaderContext; + protected final SearchLookup lookup; + public DocValuesDocReader(SearchLookup lookup, LeafReaderContext leafContext) { + this.lookup = lookup; this.leafReaderContext = leafContext; this.leafLookup = lookup.getLeafSearchLookup(leafReaderContext); } @@ -33,15 +39,15 @@ public Field field(String fieldName) { Map> doc = leafLookup.doc(); if (doc.containsKey(fieldName) == false) { - return new EmptyField(fieldName); + return new EmptyField<>(fieldName); } - return new DocValuesField<>(fieldName, doc.get(fieldName)); + return doc.get(fieldName).toField(fieldName); } @Override public Stream> fields(String fieldGlob) { - return Stream.empty(); + throw new UnsupportedOperationException("not implemented"); } @Override diff --git a/server/src/main/java/org/elasticsearch/script/DocValuesField.java b/server/src/main/java/org/elasticsearch/script/DocValuesField.java deleted file mode 100644 index 7b94988871e12..0000000000000 --- a/server/src/main/java/org/elasticsearch/script/DocValuesField.java +++ /dev/null @@ -1,46 +0,0 @@ -/* - * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one - * or more contributor license agreements. Licensed under the Elastic License - * 2.0 and the Server Side Public License, v 1; you may not use this file except - * in compliance with, at your election, the Elastic License 2.0 or the Server - * Side Public License, v 1. - */ - -package org.elasticsearch.script; - -import org.elasticsearch.index.fielddata.ScriptDocValues; - -import java.util.List; - -public class DocValuesField implements Field { - protected final String name; - protected final ScriptDocValues scriptDocValues; - - public DocValuesField(String name, ScriptDocValues scriptDocValues) { - this.name = name; - this.scriptDocValues = scriptDocValues; - } - - @Override - public T getValue(T defaultValue) { - if (scriptDocValues.isEmpty()) { - return defaultValue; - } - return scriptDocValues.get(0); - } - - @Override - public String getName() { - return name; - } - - @Override - public boolean isEmpty() { - return scriptDocValues.isEmpty(); - } - - @Override - public List getValues() { - return scriptDocValues; - } -} diff --git a/server/src/main/java/org/elasticsearch/script/EmptyField.java b/server/src/main/java/org/elasticsearch/script/EmptyField.java index c28b8b602f76a..c4e389c38cf7f 100644 --- a/server/src/main/java/org/elasticsearch/script/EmptyField.java +++ b/server/src/main/java/org/elasticsearch/script/EmptyField.java @@ -11,30 +11,42 @@ import java.util.Collections; import java.util.List; -public class EmptyField implements Field { - protected final String name; - +/** + * Script field with no mapping, always returns {@code defaultValue}. + */ +public class EmptyField extends Field { public EmptyField(String name) { - this.name = name; + super(name, null); } @Override - public String getName() { - return name; + public boolean isEmpty() { + return true; } @Override - public boolean isEmpty() { - return true; + public List getValues() { + return Collections.emptyList(); + } + + @Override + public > Field convert(Converter converter) { + // new object created to ensure EmptyField + return new EmptyField<>(name); } @Override - public Object getValue(Object defaultValue) { + public T getValue(T defaultValue) { return defaultValue; } @Override - public List getValues() { - return Collections.emptyList(); + public double getDouble(double defaultValue) { + return defaultValue; + } + + @Override + public long getLong(long defaultValue) { + return defaultValue; } } diff --git a/server/src/main/java/org/elasticsearch/script/Field.java b/server/src/main/java/org/elasticsearch/script/Field.java index 2d501c4bea9ef..fb643e9c79b9d 100644 --- a/server/src/main/java/org/elasticsearch/script/Field.java +++ b/server/src/main/java/org/elasticsearch/script/Field.java @@ -9,6 +9,10 @@ package org.elasticsearch.script; +import org.apache.lucene.util.BytesRef; +import org.elasticsearch.common.geo.GeoPoint; + +import java.math.BigInteger; import java.util.List; /** @@ -19,17 +23,173 @@ * * {@code isEmpty() == true} * * {@code getValues().equals(Collections.emptyList())} * * {@code getValue(defaultValue) == defaultValue} - * @param */ -public interface Field { - String getName(); +public abstract class Field { + public static final Converter BigInteger = Converters.BIGINTEGER; + public static final Converter Long = Converters.LONG; + + protected final String name; + protected final FieldValues values; + + public Field(String name, FieldValues values) { + this.name = name; + this.values = values; + } + + public String getName() { + return name; + } + + /** + * Does the field have any values? An unmapped field may have values from source + */ + public boolean isEmpty() { + return values.isEmpty(); + } + + /** + * Get all values of a multivalued field. If {@code isEmpty()} this returns an empty list + */ + public List getValues() { + return values.getValues(); + } - /** Does the field have any values? An unmapped field may have values from source */ - boolean isEmpty(); + /** + * Convert this {@code Field} into another {@code Field} type using a {@link Converter} of the target type. + * + * As this is called from user scripts, {@code as} may be called to convert a field into its same type, if + * so {@code this} is cast via converters {@link Converter#getFieldClass()}. + * + */ + public final > Field as(Converter converter) { + if (converter.getFieldClass().isInstance(this)) { + return converter.getFieldClass().cast(this); + } - /** Get all values of a multivalued field. If {@code isEmpty()} this returns an empty list */ - List getValues(); + return convert(converter); + } + + /** + * Extensions outside the core package should override this method to implement their own conversions, calling + * the superclass of this method as a fallback. + */ + protected > Field convert(Converter converter) { + return converter.convert(this); + } + + /** + * Provide {@link Converter}s access to the underlying {@link FieldValues}, should not be exposed to scripts + */ + public FieldValues getFieldValues() { + return values; + } /** Get the first value of a field, if {@code isEmpty()} return defaultValue instead */ - T getValue(T defaultValue); + public T getValue(T defaultValue) { + if (isEmpty()) { + return defaultValue; + } + try { + return values.getNonPrimitiveValue(); + } catch (RuntimeException err) { + return defaultValue; + } + } + + /** + * Get the underlying value as a {@code double} unless {@link #isEmpty()}, in which case return {@code defaultValue}. + * May throw {@link InvalidConversion} if the underlying value is not representable as a {@code double}. + */ + public double getDouble(double defaultValue) { + if (isEmpty()) { + return defaultValue; + } + try { + return values.getDoubleValue(); + } catch (RuntimeException err) { + return defaultValue; + } + } + + + /** + * Get the underlying value as a {@code long} unless {@link #isEmpty()}, in which case return {@code defaultValue}. + * May throw {@link InvalidConversion} if the underlying value is not representable as a {@code long}. + */ + public long getLong(long defaultValue) { + if (isEmpty()) { + return defaultValue; + } + try { + return values.getLongValue(); + } catch (RuntimeException err) { + return defaultValue; + } + } + + public static class BooleanField extends Field { + public BooleanField(String name, FieldValues values) { + super(name, values); + } + } + + public static class DoubleField extends Field { + public DoubleField(String name, FieldValues values) { + super(name, values); + } + } + + public static class LongField extends Field { + public LongField(String name, FieldValues values) { + super(name, values); + } + } + + public static class DateNanosField extends Field { + public DateNanosField(String name, FieldValues values) { + super(name, values); + } + } + + public static class DateMillisField extends Field { + public DateMillisField(String name, FieldValues values) { + super(name, values); + } + } + + public static class GeoPointField extends Field { + public GeoPointField(String name, FieldValues values) { + super(name, values); + } + } + + public static class StringField extends Field { + public StringField(String name, FieldValues values) { + super(name, values); + } + } + + public static class BytesRefField extends Field { + public BytesRefField(String name, FieldValues values) { + super(name, values); + } + } + + public static class BigIntegerField extends Field { + public BigIntegerField(String name, FieldValues values) { + super(name, values); + } + } + + public static class VersionField extends Field { + public VersionField(String name, FieldValues values) { + super(name, values); + } + } + + public static class IpField extends Field { + public IpField(String name, FieldValues values) { + super(name, values); + } + } } diff --git a/server/src/main/java/org/elasticsearch/script/FieldValues.java b/server/src/main/java/org/elasticsearch/script/FieldValues.java new file mode 100644 index 0000000000000..9c5242e293cad --- /dev/null +++ b/server/src/main/java/org/elasticsearch/script/FieldValues.java @@ -0,0 +1,36 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License + * 2.0 and the Server Side Public License, v 1; you may not use this file except + * in compliance with, at your election, the Elastic License 2.0 or the Server + * Side Public License, v 1. + */ + +package org.elasticsearch.script; + +import java.util.List; + +/** + * The underlying contents of a scripting Field. Implementations include + * {@link org.elasticsearch.index.fielddata.ScriptDocValues}, {@code _source}, runtime fields, or + * the converted form of the these values from a {@link Converter}. + */ +public interface FieldValues { + /** Are there any values? */ + boolean isEmpty(); + + /** How many values? */ + int size(); + + /** All underlying values. Note this boxes primitives */ + List getValues(); + + /** The first value as a subclass of {@link java.lang.Object}. Boxes primitives */ + T getNonPrimitiveValue(); + + /** The first value as a primitive long. For performance reasons, implementations should avoid intermediate boxings if possible */ + long getLongValue(); + + /** The first value as a primitive double. For performance reasons, implementations should avoid intermediate boxings if possible */ + double getDoubleValue(); +} diff --git a/server/src/main/java/org/elasticsearch/script/InvalidConversion.java b/server/src/main/java/org/elasticsearch/script/InvalidConversion.java new file mode 100644 index 0000000000000..31fd77d3b30ab --- /dev/null +++ b/server/src/main/java/org/elasticsearch/script/InvalidConversion.java @@ -0,0 +1,31 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License + * 2.0 and the Server Side Public License, v 1; you may not use this file except + * in compliance with, at your election, the Elastic License 2.0 or the Server + * Side Public License, v 1. + */ + +package org.elasticsearch.script; + +/** + * A failed conversion of a script {@link Field}. Thrown by {@link Converter}s and + * {@link FieldValues#getDoubleValue()}/{@link FieldValues#getLongValue()}. + */ +public class InvalidConversion extends RuntimeException { + /** Source {@link Field} or underlying value */ + protected final Class from; + + /** Destination class or {@link Converter} */ + protected final Class converter; + + public InvalidConversion(Class from, Class converter) { + this.from = from; + this.converter = converter; + } + + @Override + public String getMessage() { + return "Cannot convert from [" + from.getSimpleName() + "] using converter [" + converter.getSimpleName() + "]"; + } +} diff --git a/server/src/test/java/org/elasticsearch/index/query/SearchExecutionContextTests.java b/server/src/test/java/org/elasticsearch/index/query/SearchExecutionContextTests.java index dcaf285833182..b1146eb2e8b35 100644 --- a/server/src/test/java/org/elasticsearch/index/query/SearchExecutionContextTests.java +++ b/server/src/test/java/org/elasticsearch/index/query/SearchExecutionContextTests.java @@ -487,7 +487,7 @@ public LeafFieldData load(LeafReaderContext context) { return new LeafFieldData() { @Override public ScriptDocValues getScriptValues() { - return new ScriptDocValues<>() { + return new ScriptDocValues() { String value; @Override @@ -509,6 +509,11 @@ public void setNextDocId(int docId) { leafLookup.setDocument(docId); value = runtimeDocValues.apply(leafLookup, docId); } + + @Override + public org.elasticsearch.script.Field toField(String fieldName) { + return new org.elasticsearch.script.Field.StringField(fieldName, this); + } }; } diff --git a/x-pack/plugin/mapper-unsigned-long/src/main/java/org/elasticsearch/xpack/unsignedlong/DocValuesWhitelistExtension.java b/x-pack/plugin/mapper-unsigned-long/src/main/java/org/elasticsearch/xpack/unsignedlong/DocValuesWhitelistExtension.java index 219149453d64b..2aa29efaaa5b9 100644 --- a/x-pack/plugin/mapper-unsigned-long/src/main/java/org/elasticsearch/xpack/unsignedlong/DocValuesWhitelistExtension.java +++ b/x-pack/plugin/mapper-unsigned-long/src/main/java/org/elasticsearch/xpack/unsignedlong/DocValuesWhitelistExtension.java @@ -9,17 +9,43 @@ import org.elasticsearch.painless.spi.PainlessExtension; import org.elasticsearch.painless.spi.Whitelist; +import org.elasticsearch.painless.spi.WhitelistLoader; +import org.elasticsearch.script.AggregationScript; +import org.elasticsearch.script.BucketAggregationSelectorScript; +import org.elasticsearch.script.FieldScript; +import org.elasticsearch.script.FilterScript; +import org.elasticsearch.script.NumberSortScript; +import org.elasticsearch.script.ScoreScript; import org.elasticsearch.script.ScriptContext; +import org.elasticsearch.script.StringSortScript; -import java.util.Collections; import java.util.List; import java.util.Map; +import static java.util.Collections.singletonList; + public class DocValuesWhitelistExtension implements PainlessExtension { + private static final Whitelist WHITELIST = WhitelistLoader.loadFromResourceFiles(DocValuesWhitelistExtension.class, "whitelist.txt"); + @Override public Map, List> getContextWhitelists() { - // TODO: support unsigned_long in scripts - return Collections.emptyMap(); + List whitelist = singletonList(WHITELIST); + return Map.of( + FieldScript.CONTEXT, + whitelist, + ScoreScript.CONTEXT, + whitelist, + FilterScript.CONTEXT, + whitelist, + AggregationScript.CONTEXT, + whitelist, + NumberSortScript.CONTEXT, + whitelist, + StringSortScript.CONTEXT, + whitelist, + BucketAggregationSelectorScript.CONTEXT, + whitelist + ); } } diff --git a/x-pack/plugin/mapper-unsigned-long/src/main/java/org/elasticsearch/xpack/unsignedlong/UnsignedLongField.java b/x-pack/plugin/mapper-unsigned-long/src/main/java/org/elasticsearch/xpack/unsignedlong/UnsignedLongField.java new file mode 100644 index 0000000000000..0df34a32b7448 --- /dev/null +++ b/x-pack/plugin/mapper-unsigned-long/src/main/java/org/elasticsearch/xpack/unsignedlong/UnsignedLongField.java @@ -0,0 +1,55 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License + * 2.0; you may not use this file except in compliance with the Elastic License + * 2.0. + */ + +package org.elasticsearch.xpack.unsignedlong; + +import org.elasticsearch.script.Converter; +import org.elasticsearch.script.Converters; +import org.elasticsearch.script.Field; +import org.elasticsearch.script.FieldValues; + +import java.math.BigInteger; +import java.util.List; +import java.util.stream.Collectors; + +import static org.elasticsearch.xpack.unsignedlong.UnsignedLongFieldMapper.BIGINTEGER_2_64_MINUS_ONE; + +public class UnsignedLongField extends Field.LongField { + public UnsignedLongField(String name, FieldValues values) { + super(name, values); + } + + // UnsignedLongFields must define their own conversions as they are in x-pack + @Override + public > Field convert(Converter converter) { + if (converter.getTargetClass() == BigInteger.class) { + BigIntegerField bigIntegerField = UnsignedLongToBigInteger(this); + return converter.getFieldClass().cast(bigIntegerField); + } + + return super.as(converter); + } + + static BigIntegerField UnsignedLongToBigInteger(UnsignedLongField sourceField) { + FieldValues fv = sourceField.getFieldValues(); + return new BigIntegerField(sourceField.getName(), new Converters.DelegatingFieldValues(fv) { + protected BigInteger toBigInteger(long formatted) { + return java.math.BigInteger.valueOf(formatted).and(BIGINTEGER_2_64_MINUS_ONE); + } + + @Override + public List getValues() { + return values.getValues().stream().map(this::toBigInteger).collect(Collectors.toList()); + } + + @Override + public BigInteger getNonPrimitiveValue() { + return toBigInteger(values.getLongValue()); + } + }); + } +} diff --git a/x-pack/plugin/mapper-unsigned-long/src/main/java/org/elasticsearch/xpack/unsignedlong/UnsignedLongLeafFieldData.java b/x-pack/plugin/mapper-unsigned-long/src/main/java/org/elasticsearch/xpack/unsignedlong/UnsignedLongLeafFieldData.java index 0121cd62cc035..c898d2e147c9e 100644 --- a/x-pack/plugin/mapper-unsigned-long/src/main/java/org/elasticsearch/xpack/unsignedlong/UnsignedLongLeafFieldData.java +++ b/x-pack/plugin/mapper-unsigned-long/src/main/java/org/elasticsearch/xpack/unsignedlong/UnsignedLongLeafFieldData.java @@ -74,8 +74,7 @@ public int docValueCount() { @Override public ScriptDocValues getScriptValues() { - // TODO: add support for scripts - throw new UnsupportedOperationException("Using unsigned_long in scripts is currently not supported!"); + return new UnsignedLongScriptDocValues(getLongValues()); } @Override diff --git a/x-pack/plugin/mapper-unsigned-long/src/main/java/org/elasticsearch/xpack/unsignedlong/UnsignedLongScriptDocValues.java b/x-pack/plugin/mapper-unsigned-long/src/main/java/org/elasticsearch/xpack/unsignedlong/UnsignedLongScriptDocValues.java index f5b8025cfc974..42aac528bcfb1 100644 --- a/x-pack/plugin/mapper-unsigned-long/src/main/java/org/elasticsearch/xpack/unsignedlong/UnsignedLongScriptDocValues.java +++ b/x-pack/plugin/mapper-unsigned-long/src/main/java/org/elasticsearch/xpack/unsignedlong/UnsignedLongScriptDocValues.java @@ -10,11 +10,13 @@ import org.apache.lucene.index.SortedNumericDocValues; import org.apache.lucene.util.ArrayUtil; import org.elasticsearch.index.fielddata.ScriptDocValues; -import org.elasticsearch.search.DocValueFormat; +import org.elasticsearch.script.Field; import java.io.IOException; -public class UnsignedLongScriptDocValues extends ScriptDocValues { +import static org.elasticsearch.search.DocValueFormat.MASK_2_63; + +public class UnsignedLongScriptDocValues extends ScriptDocValues { private final SortedNumericDocValues in; private long[] values = new long[0]; private int count; @@ -47,22 +49,50 @@ protected void resize(int newSize) { values = ArrayUtil.grow(values, count); } - public Number getValue() { - return get(0); + public long getValue() { + throwIfEmpty(); + return format(0); } @Override - public Number get(int index) { - if (count == 0) { - throw new IllegalStateException( - "A document doesn't have a value for a field! Use doc[].size()==0 to check if a document is missing a field!" - ); - } - return (Number) DocValueFormat.UNSIGNED_LONG_SHIFTED.format(values[index]); + public Long get(int index) { + throwIfEmpty(); + return format(index); + } + + /** + * Applies the formatting from {@link org.elasticsearch.search.DocValueFormat.UnsignedLongShiftedDocValueFormat#format(long)} so + * that the underlying value can be treated as a primitive long as that method returns either a {@code long} or a {@code BigInteger}. + */ + protected long format(int index) { + return shiftedLong(values[index]); + } + + // Package private for use in UnsignedLongField + static long shiftedLong(long unshifted) { + return unshifted ^ MASK_2_63; } @Override public int size() { return count; } + + @Override + public long getLongValue() { + throwIfEmpty(); + return format(0); + } + + @Override + public double getDoubleValue() { + throwIfEmpty(); + return format(0); + } + + @Override + public Field toField(String fieldName) { + return new UnsignedLongField(fieldName, this); + } + } diff --git a/x-pack/plugin/mapper-unsigned-long/src/main/resources/org/elasticsearch/xpack/unsignedlong/whitelist.txt b/x-pack/plugin/mapper-unsigned-long/src/main/resources/org/elasticsearch/xpack/unsignedlong/whitelist.txt index f0cd5d92b823e..35699791a613f 100644 --- a/x-pack/plugin/mapper-unsigned-long/src/main/resources/org/elasticsearch/xpack/unsignedlong/whitelist.txt +++ b/x-pack/plugin/mapper-unsigned-long/src/main/resources/org/elasticsearch/xpack/unsignedlong/whitelist.txt @@ -1,11 +1,12 @@ # # Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one # or more contributor license agreements. Licensed under the Elastic License -# 2.0; you may not use this file except in compliance with the Elastic License -# 2.0. +# 2.0 and the Server Side Public License, v 1; you may not use this file except +# in compliance with, at your election, the Elastic License 2.0 or the Server +# Side Public License, v 1. # class org.elasticsearch.xpack.unsignedlong.UnsignedLongScriptDocValues { - Number get(int) - Number getValue() + Long get(int) + long getValue() } diff --git a/x-pack/plugin/mapper-unsigned-long/src/test/java/org/elasticsearch/xpack/unsignedlong/UnsignedLongFieldTests.java b/x-pack/plugin/mapper-unsigned-long/src/test/java/org/elasticsearch/xpack/unsignedlong/UnsignedLongFieldTests.java new file mode 100644 index 0000000000000..d9c0087abbc15 --- /dev/null +++ b/x-pack/plugin/mapper-unsigned-long/src/test/java/org/elasticsearch/xpack/unsignedlong/UnsignedLongFieldTests.java @@ -0,0 +1,67 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License + * 2.0; you may not use this file except in compliance with the Elastic License + * 2.0. + */ + +package org.elasticsearch.xpack.unsignedlong; + +import org.elasticsearch.script.Field; +import org.elasticsearch.script.FieldValues; +import org.elasticsearch.test.ESTestCase; + +import java.math.BigInteger; +import java.util.Arrays; +import java.util.List; +import java.util.stream.Collectors; + +public class UnsignedLongFieldTests extends ESTestCase { + static final BigInteger[] VALUES = { + BigInteger.valueOf(0L), + new BigInteger("18446744073709551615"), // 2^64 - 1 + new BigInteger("9223372036854775807"), // 2^63 - 1 + new BigInteger("9223372036854775808"), // 2^63 + new BigInteger("9223372036854775809"), // 2^63 + 1 + }; + + UnsignedLongField FIELD = new UnsignedLongField("test", new FieldValues() { + protected long[] values = Arrays.stream(UnsignedLongFieldTests.VALUES).mapToLong(BigInteger::longValue).toArray(); + + @Override + public boolean isEmpty() { + return values.length == 0; + } + + @Override + public int size() { + return values.length; + } + + @Override + public List getValues() { + return Arrays.stream(values).boxed().collect(Collectors.toList()); + } + + @Override + public Long getNonPrimitiveValue() { + return UnsignedLongScriptDocValues.shiftedLong(values[0]); + } + + @Override + public long getLongValue() { + return UnsignedLongScriptDocValues.shiftedLong(values[0]); + } + + @Override + public double getDoubleValue() { + return UnsignedLongScriptDocValues.shiftedLong(values[0]); + } + }); + + public void testLongValues() { + long[] expected = { 0L, -1L, 9223372036854775807L, -9223372036854775808L, -9223372036854775807L }; + List asLong = Arrays.stream(expected).boxed().collect(Collectors.toList()); + assertEquals(asLong, FIELD.convert(Field.Long).getValues()); + } +} diff --git a/x-pack/plugin/mapper-version/src/main/java/org/elasticsearch/xpack/versionfield/VersionScriptDocValues.java b/x-pack/plugin/mapper-version/src/main/java/org/elasticsearch/xpack/versionfield/VersionScriptDocValues.java index 25cd66a0eb212..1bd624160610f 100644 --- a/x-pack/plugin/mapper-version/src/main/java/org/elasticsearch/xpack/versionfield/VersionScriptDocValues.java +++ b/x-pack/plugin/mapper-version/src/main/java/org/elasticsearch/xpack/versionfield/VersionScriptDocValues.java @@ -10,6 +10,7 @@ import org.apache.lucene.index.SortedSetDocValues; import org.apache.lucene.util.ArrayUtil; import org.elasticsearch.index.fielddata.ScriptDocValues; +import org.elasticsearch.script.Field; import java.io.IOException; @@ -56,4 +57,9 @@ public String get(int index) { public int size() { return count; } + + @Override + public Field toField(String fieldName) { + return new Field.VersionField(fieldName, this); + } } diff --git a/x-pack/plugin/spatial/src/main/java/org/elasticsearch/xpack/spatial/index/fielddata/plain/AbstractAtomicGeoShapeShapeFieldData.java b/x-pack/plugin/spatial/src/main/java/org/elasticsearch/xpack/spatial/index/fielddata/plain/AbstractAtomicGeoShapeShapeFieldData.java index 62b0abedd788d..c00997ab75245 100644 --- a/x-pack/plugin/spatial/src/main/java/org/elasticsearch/xpack/spatial/index/fielddata/plain/AbstractAtomicGeoShapeShapeFieldData.java +++ b/x-pack/plugin/spatial/src/main/java/org/elasticsearch/xpack/spatial/index/fielddata/plain/AbstractAtomicGeoShapeShapeFieldData.java @@ -12,6 +12,8 @@ import org.elasticsearch.common.geo.GeoPoint; import org.elasticsearch.index.fielddata.ScriptDocValues; import org.elasticsearch.index.fielddata.SortedBinaryDocValues; +import org.elasticsearch.script.Field; +import org.elasticsearch.script.FieldValues; import org.elasticsearch.xpack.spatial.index.fielddata.GeoShapeValues; import org.elasticsearch.xpack.spatial.index.fielddata.LeafGeoShapeFieldData; @@ -115,5 +117,16 @@ public GeoShapeValues.GeoShapeValue get(int index) { public int size() { return value == null ? 0 : 1; } + + @Override + public Field toField(String fieldName) { + return new GeoShapeField(fieldName, this); + } + } + + public static class GeoShapeField extends Field { + public GeoShapeField(String name, FieldValues values) { + super(name, values); + } } } diff --git a/x-pack/plugin/src/yamlRestTest/resources/rest-api-spec/test/unsigned_long/50_script_values.yml b/x-pack/plugin/src/yamlRestTest/resources/rest-api-spec/test/unsigned_long/50_script_values.yml index 56929f56f4742..aacf317294580 100644 --- a/x-pack/plugin/src/yamlRestTest/resources/rest-api-spec/test/unsigned_long/50_script_values.yml +++ b/x-pack/plugin/src/yamlRestTest/resources/rest-api-spec/test/unsigned_long/50_script_values.yml @@ -1,8 +1,8 @@ setup: - skip: - version: "all" - reason: "AwaitsFix https://github.com/elastic/elasticsearch/issues/64361" + version: " - 7.14.99" + reason: "unsigned long script fields api was added in 7.15.0" - do: indices.create: @@ -25,12 +25,42 @@ setup: { "index": {"_id" : "3"} } { "ul": 9223372036854775808 } { "index": {"_id" : "4"} } - { "ul": 18446744073709551614 } + { "ul": 18446744073709551613 } { "index": {"_id" : "5"} } { "ul": 18446744073709551615 } --- -"Scripted fields values return BigInteger or Long": +"Scripted fields values return Long": + - do: + search: + index: test1 + body: + sort: [ { ul: desc } ] + script_fields: + scripted_ul: + script: + source: "field('ul').getValue(1000L)" + + - match: { hits.hits.0.fields.scripted_ul.0: -1 } + - match: { hits.hits.1.fields.scripted_ul.0: -3 } + - match: { hits.hits.2.fields.scripted_ul.0: -9223372036854775808 } + - match: { hits.hits.3.fields.scripted_ul.0: 9223372036854775807 } + - match: { hits.hits.4.fields.scripted_ul.0: 0 } + - do: + search: + index: test1 + body: + sort: [ { ul: desc } ] + script_fields: + scripted_ul: + script: + source: "field('ul').getLong(1000L)" + + - match: { hits.hits.0.fields.scripted_ul.0: -1 } + - match: { hits.hits.1.fields.scripted_ul.0: -3 } + - match: { hits.hits.2.fields.scripted_ul.0: -9223372036854775808 } + - match: { hits.hits.3.fields.scripted_ul.0: 9223372036854775807 } + - match: { hits.hits.4.fields.scripted_ul.0: 0 } - do: search: index: test1 @@ -41,9 +71,9 @@ setup: script: source: "doc['ul'].value" - - match: { hits.hits.0.fields.scripted_ul.0: 18446744073709551615 } - - match: { hits.hits.1.fields.scripted_ul.0: 18446744073709551614 } - - match: { hits.hits.2.fields.scripted_ul.0: 9223372036854775808 } + - match: { hits.hits.0.fields.scripted_ul.0: -1 } + - match: { hits.hits.1.fields.scripted_ul.0: -3 } + - match: { hits.hits.2.fields.scripted_ul.0: -9223372036854775808 } - match: { hits.hits.3.fields.scripted_ul.0: 9223372036854775807 } - match: { hits.hits.4.fields.scripted_ul.0: 0 } @@ -58,13 +88,48 @@ setup: order: desc type: number script: - source: "doc['ul'].value" + source: "field('ul').as(Field.BigInteger).getValue(BigInteger.valueOf(Long.parseUnsignedLong('18446744073709551614'))).doubleValue()" - match: { hits.hits.0.sort: [1.8446744073709552E19] } - match: { hits.hits.1.sort: [1.8446744073709552E19] } - match: { hits.hits.2.sort: [9.223372036854776E18] } - match: { hits.hits.3.sort: [9.223372036854776E18] } - match: { hits.hits.4.sort: [0.0] } + - do: + search: + index: test1 + body: + sort: + _script: + order: desc + type: number + script: + source: "doc['ul'].value" + + + - match: { hits.hits.0.sort: [9.223372036854776E18] } + - match: { hits.hits.1.sort: [0.0] } + - match: { hits.hits.2.sort: [-1.0] } + - match: { hits.hits.3.sort: [-3.0] } + - match: { hits.hits.4.sort: [-9.223372036854776E18] } +--- +"Scripted sort values via doc": + - do: + search: + index: test1 + body: + sort: + _script: + order: desc + type: number + script: + source: "doc['ul'].value" + + - match: { hits.hits.0.sort: [9.223372036854776E18] } + - match: { hits.hits.1.sort: [0.0] } + - match: { hits.hits.2.sort: [-1.0] } + - match: { hits.hits.3.sort: [-3.0] } + - match: { hits.hits.4.sort: [-9.223372036854776E18] } --- "Script query": @@ -77,12 +142,37 @@ setup: filter: script: script: - source: "doc['ul'].value.doubleValue() > 10E18" + source: "field('ul').as(Field.BigInteger).getValue(BigInteger.valueOf(Long.parseUnsignedLong('18446744073709551614'))).doubleValue() > 10E18" sort: [ { ul: asc } ] - match: { hits.total.value: 2 } - match: { hits.hits.0._id: "4" } - match: { hits.hits.1._id: "5" } + - do: + search: + index: test1 + body: + size: 0 + query: + bool: + filter: + script: + script: + source: "field('ul').isEmpty() == false" + - match: { hits.total.value: 5 } + - do: + search: + index: test1 + body: + query: + bool: + filter: + script: + script: + source: "doc['ul'].value.doubleValue() > 9E18" + - match: { hits.total.value: 1 } + - match: { hits.hits.0._id: "2" } + - do: search: index: test1 @@ -106,6 +196,17 @@ setup: script_score: query: {match_all: {}} script: - source: "doc['ul'].value" + source: "field('ul').as(Field.BigInteger).getValue(BigInteger.valueOf(Long.parseUnsignedLong('18446744073709551614'))).doubleValue()" + + - match: { hits.total.value: 5 } + - do: + search: + index: test1 + body: + query: + script_score: + query: {match_all: {}} + script: + source: "Math.abs(doc['ul'].value)" - match: { hits.total.value: 5 } diff --git a/x-pack/plugin/vectors/src/main/java/org/elasticsearch/xpack/vectors/query/DenseVectorScriptDocValues.java b/x-pack/plugin/vectors/src/main/java/org/elasticsearch/xpack/vectors/query/DenseVectorScriptDocValues.java index 18e2e80a090bc..1f3d27c0f7312 100644 --- a/x-pack/plugin/vectors/src/main/java/org/elasticsearch/xpack/vectors/query/DenseVectorScriptDocValues.java +++ b/x-pack/plugin/vectors/src/main/java/org/elasticsearch/xpack/vectors/query/DenseVectorScriptDocValues.java @@ -12,6 +12,7 @@ import org.apache.lucene.util.BytesRef; import org.elasticsearch.Version; import org.elasticsearch.index.fielddata.ScriptDocValues; +import org.elasticsearch.script.Field; import org.elasticsearch.xpack.vectors.mapper.VectorEncoderDecoder; import java.io.IOException; @@ -80,4 +81,14 @@ public int size() { return 1; } } + + @Override + public BytesRef getNonPrimitiveValue() { + return value; + } + + @Override + public Field toField(String fieldName) { + throw new IllegalStateException("not implemented"); + } }