From e8663a95897434255c9c00684bf2bdb2f872aba6 Mon Sep 17 00:00:00 2001 From: Stuart Tettemer Date: Fri, 13 Aug 2021 18:07:16 -0500 Subject: [PATCH 01/14] Script: ulong via fields API Exposes unsigned long via the fields API. Unsigned longs default to java signed longs. That means the upper range appears negative. Consumers should use `Long.compareUnsigned(long, long)` `Long.divideUnsigned(long, long)` and `Long.remainderUnsigned(long, long)` to correctly work with values known to be unsigned long. Alternatively, users may treat the unsigned long type as `BigInteger` using the field API, `field('ul').as(Field.BigInteger).getValue(BigInteger.ZERO)`. ``` field('ul').as(Field.BigInteger).getValue(BigInteger.valueOf(1000)) field('ul').getValue(1000L) ``` This change also implements the beginning of the converters for the fields API. The following conversions have been added: ``` ulong <-> BigInteger long <-> BigInteger double -> BigInteger String (parsed as long or double) -> BigInteger double -> long String (parsed as long or double) -> long Date (epoch milliseconds) -> long Nano Date (epoch nanoseconds) -> long boolean (1L for true, 0L for false) -> long ``` Fixes: #64361 --- .../org.elasticsearch.script.fields.txt | 8 + .../index/fielddata/ScriptDocValues.java | 133 ++++++++- .../index/mapper/IpFieldMapper.java | 5 + .../org/elasticsearch/script/Converter.java | 15 + .../org/elasticsearch/script/Converters.java | 273 ++++++++++++++++++ .../script/DelegatingFieldValues.java | 37 +++ .../script/DenseVectorField.java | 12 + .../script/DenseVectorScriptValue.java | 28 ++ .../script/DocValuesDocReader.java | 5 +- .../elasticsearch/script/DocValuesField.java | 46 --- .../org/elasticsearch/script/EmptyField.java | 31 +- .../java/org/elasticsearch/script/Field.java | 136 ++++++++- .../org/elasticsearch/script/FieldValues.java | 24 ++ .../script/InvalidConversion.java | 24 ++ .../query/SearchExecutionContextTests.java | 7 +- .../xpack/unsignedlong/UnsignedLongField.java | 95 ++++++ .../UnsignedLongLeafFieldData.java | 3 +- .../UnsignedLongScriptDocValues.java | 43 ++- .../versionfield/VersionScriptDocValues.java | 6 + .../AbstractAtomicGeoShapeShapeFieldData.java | 13 + .../test/unsigned_long/50_script_values.yml | 24 +- .../query/DenseVectorScriptDocValues.java | 11 + 22 files changed, 883 insertions(+), 96 deletions(-) create mode 100644 server/src/main/java/org/elasticsearch/script/Converter.java create mode 100644 server/src/main/java/org/elasticsearch/script/Converters.java create mode 100644 server/src/main/java/org/elasticsearch/script/DelegatingFieldValues.java create mode 100644 server/src/main/java/org/elasticsearch/script/DenseVectorField.java create mode 100644 server/src/main/java/org/elasticsearch/script/DenseVectorScriptValue.java delete mode 100644 server/src/main/java/org/elasticsearch/script/DocValuesField.java create mode 100644 server/src/main/java/org/elasticsearch/script/FieldValues.java create mode 100644 server/src/main/java/org/elasticsearch/script/InvalidConversion.java create mode 100644 x-pack/plugin/mapper-unsigned-long/src/main/java/org/elasticsearch/xpack/unsignedlong/UnsignedLongField.java diff --git a/modules/lang-painless/src/main/resources/org/elasticsearch/painless/org.elasticsearch.script.fields.txt b/modules/lang-painless/src/main/resources/org/elasticsearch/painless/org.elasticsearch.script.fields.txt index b21437b14a884..328a751217d2b 100644 --- a/modules/lang-painless/src/main/resources/org/elasticsearch/painless/org.elasticsearch.script.fields.txt +++ b/modules/lang-painless/src/main/resources/org/elasticsearch/painless/org.elasticsearch.script.fields.txt @@ -10,10 +10,18 @@ # API class org.elasticsearch.script.Field { + org.elasticsearch.script.Converter BigInteger + org.elasticsearch.script.Converter Long String getName() boolean isEmpty() List getValues() def getValue(def) + double getDouble(double) + long getLong(long) + org.elasticsearch.script.Field as(org.elasticsearch.script.Converter) +} + +class org.elasticsearch.script.Converter { } class org.elasticsearch.script.DocBasedScript { diff --git a/server/src/main/java/org/elasticsearch/index/fielddata/ScriptDocValues.java b/server/src/main/java/org/elasticsearch/index/fielddata/ScriptDocValues.java index 86d007072197e..bd0eafae0a6f5 100644 --- a/server/src/main/java/org/elasticsearch/index/fielddata/ScriptDocValues.java +++ b/server/src/main/java/org/elasticsearch/index/fielddata/ScriptDocValues.java @@ -17,6 +17,9 @@ import org.elasticsearch.common.geo.GeoUtils; import org.elasticsearch.common.time.DateUtils; import org.elasticsearch.geometry.utils.Geohash; +import org.elasticsearch.script.Field; +import org.elasticsearch.script.FieldValues; +import org.elasticsearch.script.InvalidConversion; import org.elasticsearch.script.JodaCompatibleZonedDateTime; import java.io.IOException; @@ -25,6 +28,8 @@ import java.util.AbstractList; import java.util.Arrays; import java.util.Comparator; +import java.util.List; +import java.util.concurrent.TimeUnit; import java.util.function.UnaryOperator; /** @@ -35,7 +40,7 @@ * return as a single {@link ScriptDocValues} instance can be reused to return * values form multiple documents. */ -public abstract class ScriptDocValues extends AbstractList { +public abstract class ScriptDocValues extends AbstractList implements FieldValues { /** * Set the current doc ID. @@ -68,6 +73,31 @@ public final void sort(Comparator c) { throw new UnsupportedOperationException("doc values are unmodifiable"); } + public abstract Field toField(String fieldName); + + public List getValues() { + return this; + } + + public T getNonPrimitiveValue() { + return get(0); + } + + public long getLongValue() { + throw new InvalidConversion(this.getClass(), long.class); + } + + public double getDoubleValue() { + throw new InvalidConversion(this.getClass(), double.class); + } + + protected void throwIfEmpty() { + if (size() == 0) { + throw new IllegalStateException("A document doesn't have a value for a field! " + + "Use doc[].size()==0 to check if a document is missing a field!"); + } + } + public static final class Longs extends ScriptDocValues { private final SortedNumericDocValues in; private long[] values = new long[0]; @@ -107,10 +137,7 @@ public long getValue() { @Override public Long get(int index) { - if (count == 0) { - throw new IllegalStateException("A document doesn't have a value for a field! " + - "Use doc[].size()==0 to check if a document is missing a field!"); - } + throwIfEmpty(); return values[index]; } @@ -118,6 +145,23 @@ public Long get(int index) { public int size() { return count; } + + @Override + public long getLongValue() { + throwIfEmpty(); + return values[0]; + } + + @Override + public double getDoubleValue() { + throwIfEmpty(); + return values[0]; + } + + @Override + public Field toField(String fieldName) { + return new Field.LongField(fieldName, this); + } } public static final class Dates extends ScriptDocValues { @@ -192,6 +236,29 @@ void refreshArray() throws IOException { } } } + + @Override + public long getLongValue() { + throwIfEmpty(); + Instant dt = dates[0].toInstant(); + if (isNanos) { + return TimeUnit.SECONDS.toNanos(dt.getEpochSecond()) + dt.getNano(); + } + return dt.toEpochMilli(); + } + + @Override + public double getDoubleValue() { + return getLongValue(); + } + + @Override + public Field toField(String fieldName) { + if (isNanos) { + return new Field.DateNanosField(fieldName, this); + } + return new Field.DateMillisField(fieldName, this); + } } public static final class Doubles extends ScriptDocValues { @@ -246,6 +313,22 @@ public Double get(int index) { public int size() { return count; } + + @Override + public long getLongValue() { + return (long) getDoubleValue(); + } + + @Override + public double getDoubleValue() { + throwIfEmpty(); + return values[0]; + } + + @Override + public Field toField(String fieldName) { + return new Field.DoubleField(fieldName, this); + } } public abstract static class Geometry extends ScriptDocValues { @@ -436,6 +519,11 @@ public double getMercatorHeight() { public GeoBoundingBox getBoundingBox() { return size() == 0 ? null : boundingBox; } + + @Override + public Field toField(String fieldName) { + return new Field.GeoPointField(fieldName, this); + } } public static final class Booleans extends ScriptDocValues { @@ -496,6 +584,22 @@ private static boolean[] grow(boolean[] array, int minSize) { return array; } + @Override + public long getLongValue() { + throwIfEmpty(); + return values[0] ? 1L : 0L; + } + + @Override + public double getDoubleValue() { + throwIfEmpty(); + return values[0] ? 1.0D : 0.0D; + } + + @Override + public Field toField(String fieldName) { + return new Field.BooleanField(fieldName, this); + } } abstract static class BinaryScriptDocValues extends ScriptDocValues { @@ -568,6 +672,21 @@ protected String bytesToString(BytesRef bytes) { public final String getValue() { return get(0); } + + @Override + public long getLongValue() { + return Long.parseLong(get(0)); + } + + @Override + public double getDoubleValue() { + return Double.parseDouble(get(0)); + } + + @Override + public Field toField(String fieldName) { + return new Field.StringField(fieldName, this); + } } public static final class BytesRefs extends BinaryScriptDocValues { @@ -594,5 +713,9 @@ public BytesRef getValue() { return get(0); } + @Override + public Field toField(String fieldName) { + return new Field.BytesRefField(fieldName, this); + } } } diff --git a/server/src/main/java/org/elasticsearch/index/mapper/IpFieldMapper.java b/server/src/main/java/org/elasticsearch/index/mapper/IpFieldMapper.java index 15a7a1c22c1ab..a95556a13bf87 100644 --- a/server/src/main/java/org/elasticsearch/index/mapper/IpFieldMapper.java +++ b/server/src/main/java/org/elasticsearch/index/mapper/IpFieldMapper.java @@ -355,6 +355,11 @@ public String get(int index) { public int size() { return count; } + + @Override + public org.elasticsearch.script.Field toField(String fieldName) { + return new org.elasticsearch.script.Field.IpField(fieldName, this); + } } @Override diff --git a/server/src/main/java/org/elasticsearch/script/Converter.java b/server/src/main/java/org/elasticsearch/script/Converter.java new file mode 100644 index 0000000000000..fa9c6a0f37f7a --- /dev/null +++ b/server/src/main/java/org/elasticsearch/script/Converter.java @@ -0,0 +1,15 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License + * 2.0 and the Server Side Public License, v 1; you may not use this file except + * in compliance with, at your election, the Elastic License 2.0 or the Server + * Side Public License, v 1. + */ + +package org.elasticsearch.script; + +public interface Converter> { + CF convert(Field sourceField); + Class getFieldClass(); + Class getTargetClass(); +} diff --git a/server/src/main/java/org/elasticsearch/script/Converters.java b/server/src/main/java/org/elasticsearch/script/Converters.java new file mode 100644 index 0000000000000..61418d41a116a --- /dev/null +++ b/server/src/main/java/org/elasticsearch/script/Converters.java @@ -0,0 +1,273 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License + * 2.0 and the Server Side Public License, v 1; you may not use this file except + * in compliance with, at your election, the Elastic License 2.0 or the Server + * Side Public License, v 1. + */ + +package org.elasticsearch.script; + +import java.math.BigInteger; +import java.time.temporal.ChronoUnit; +import java.util.List; +import java.util.stream.Collectors; + +import static org.elasticsearch.script.Field.BigIntegerField; +import static org.elasticsearch.script.Field.BooleanField; +import static org.elasticsearch.script.Field.DoubleField; +import static org.elasticsearch.script.Field.DateMillisField; +import static org.elasticsearch.script.Field.DateNanosField; +import static org.elasticsearch.script.Field.LongField; +import static org.elasticsearch.script.Field.StringField; + +public class Converters { + public static final Converter BIGINTEGER; + public static final Converter LONG; + + static { + BIGINTEGER = new Converter<>() { + @Override + public BigIntegerField convert(Field sourceField) { + if (sourceField instanceof LongField) { + return LongToBigInteger((LongField) sourceField); + } + + if (sourceField instanceof DoubleField) { + return DoubleToBigInteger((DoubleField) sourceField); + } + + if (sourceField instanceof StringField) { + return StringToBigInteger((StringField) sourceField); + } + + throw new InvalidConversion(sourceField.getClass(), getFieldClass()); + } + + @Override + public Class getFieldClass() { + return BigIntegerField.class; + } + + @Override + public Class getTargetClass() { + return java.math.BigInteger.class; + } + }; + + LONG = new Converter<>() { + @Override + public LongField convert(Field sourceField) { + if (sourceField instanceof DoubleField) { + return DoubleToLong((DoubleField) sourceField); + } + + if (sourceField instanceof StringField) { + return StringToLong((StringField) sourceField); + } + + if (sourceField instanceof DateMillisField) { + return DateMillisToLong((DateMillisField) sourceField); + } + + if (sourceField instanceof DateNanosField) { + return DateNanosToLong((DateNanosField) sourceField); + } + + if (sourceField instanceof BigIntegerField) { + return BigIntegerToLong((BigIntegerField) sourceField); + } + + if (sourceField instanceof BooleanField) { + return BooleanToLong((BooleanField) sourceField); + } + + throw new InvalidConversion(sourceField.getClass(), getFieldClass()); + } + + @Override + public Class getFieldClass() { + return LongField.class; + } + + @Override + public Class getTargetClass() { + return Long.class; + } + }; + } + + public static BigIntegerField LongToBigInteger(LongField sourceField) { + FieldValues fv = sourceField.getFieldValues(); + return new BigIntegerField(sourceField.getName(), new DelegatingFieldValues<>(fv) { + @Override + public List getValues() { + return values.getValues().stream().map(java.math.BigInteger::valueOf).collect(Collectors.toList()); + } + + @Override + public java.math.BigInteger getNonPrimitiveValue() { + return java.math.BigInteger.valueOf(values.getLongValue()); + } + }); + } + + public static BigIntegerField DoubleToBigInteger(DoubleField sourceField) { + FieldValues fv = sourceField.getFieldValues(); + return new BigIntegerField(sourceField.getName(), new DelegatingFieldValues<>(fv) { + @Override + public List getValues() { + return values.getValues().stream().map( + dbl -> java.math.BigInteger.valueOf(dbl.longValue()) + ).collect(Collectors.toList()); + } + + @Override + public java.math.BigInteger getNonPrimitiveValue() { + return java.math.BigInteger.valueOf(values.getLongValue()); + } + }); + } + + public static BigIntegerField StringToBigInteger(StringField sourceField) { + FieldValues fv = sourceField.getFieldValues(); + return new BigIntegerField(sourceField.getName(), new DelegatingFieldValues(fv) { + protected long parseNumber(String str) { + try { + return Long.parseLong(str); + } catch (NumberFormatException err) { + return (long) Double.parseDouble(str); + } + } + + @Override + public List getValues() { + return values.getValues().stream().map( + str -> java.math.BigInteger.valueOf(parseNumber(str)) + ).collect(Collectors.toList()); + } + + @Override + public java.math.BigInteger getNonPrimitiveValue() { + return java.math.BigInteger.valueOf(values.getLongValue()); + } + }); + } + + public static LongField BigIntegerToLong(BigIntegerField sourceField) { + FieldValues fv = sourceField.getFieldValues(); + return new LongField(sourceField.getName(), new DelegatingFieldValues(fv) { + @Override + public List getValues() { + return values.getValues().stream().map(BigInteger::longValue).collect(Collectors.toList()); + } + + @Override + public Long getNonPrimitiveValue() { + return values.getLongValue(); + } + }); + } + + public static LongField BooleanToLong(BooleanField sourceField) { + FieldValues fv = sourceField.getFieldValues(); + return new LongField(sourceField.getName(), new DelegatingFieldValues(fv) { + @Override + public List getValues() { + return values.getValues().stream().map(bool -> bool ? 1L : 0L).collect(Collectors.toList()); + } + + @Override + public Long getNonPrimitiveValue() { + return getLongValue(); + } + }); + } + + public static LongField DateMillisToLong(DateMillisField sourceField) { + FieldValues fv = sourceField.getFieldValues(); + return new LongField(sourceField.getName(), new DelegatingFieldValues(fv) { + @Override + public List getValues() { + return values.getValues().stream().map(dt -> dt.toInstant().toEpochMilli()).collect(Collectors.toList()); + } + + @Override + public Long getNonPrimitiveValue() { + return values.getNonPrimitiveValue().toInstant().toEpochMilli(); + } + }); + } + + public static LongField DateNanosToLong(DateNanosField sourceField) { + FieldValues fv = sourceField.getFieldValues(); + return new LongField(sourceField.getName(), new DelegatingFieldValues(fv) { + protected long nanoLong(JodaCompatibleZonedDateTime dt) { + return ChronoUnit.NANOS.between(java.time.Instant.EPOCH, dt.toInstant()); + } + + @Override + public List getValues() { + return values.getValues().stream().map(this::nanoLong).collect(Collectors.toList()); + } + + @Override + public Long getNonPrimitiveValue() { + return ChronoUnit.NANOS.between(java.time.Instant.EPOCH, values.getNonPrimitiveValue().toInstant()); + } + }); + } + + public static LongField DoubleToLong(DoubleField sourceField) { + FieldValues fv = sourceField.getFieldValues(); + return new LongField(sourceField.getName(), new DelegatingFieldValues(fv) { + @Override + public List getValues() { + return values.getValues().stream().map(Double::longValue).collect(Collectors.toList()); + } + + @Override + public Long getNonPrimitiveValue() { + return values.getLongValue(); + } + }); + } + + public static LongField StringToLong(StringField sourceField) { + FieldValues fv = sourceField.getFieldValues(); + return new LongField(sourceField.getName(), new DelegatingFieldValues(fv) { + protected long parseNumber(String str) { + try { + return Long.parseLong(str); + } catch (NumberFormatException err) { + return (long) Double.parseDouble(str); + } + } + + @Override + public List getValues() { + return values.getValues().stream().map(this::parseNumber).collect(Collectors.toList()); + } + + @Override + public Long getNonPrimitiveValue() { + return parseNumber(values.getNonPrimitiveValue()); + } + + @Override + public long getLongValue() { + return parseNumber(values.getNonPrimitiveValue()); + } + + @Override + public double getDoubleValue() { + String str = values.getNonPrimitiveValue(); + try { + return Double.parseDouble(str); + } catch (NumberFormatException err) { + return Long.parseLong(str); + } + } + }); + } +} diff --git a/server/src/main/java/org/elasticsearch/script/DelegatingFieldValues.java b/server/src/main/java/org/elasticsearch/script/DelegatingFieldValues.java new file mode 100644 index 0000000000000..aec6948fd88da --- /dev/null +++ b/server/src/main/java/org/elasticsearch/script/DelegatingFieldValues.java @@ -0,0 +1,37 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License + * 2.0 and the Server Side Public License, v 1; you may not use this file except + * in compliance with, at your election, the Elastic License 2.0 or the Server + * Side Public License, v 1. + */ + +package org.elasticsearch.script; + +public abstract class DelegatingFieldValues implements FieldValues { + protected FieldValues values; + + public DelegatingFieldValues(FieldValues values) { + this.values = values; + } + + @Override + public boolean isEmpty() { + return values.isEmpty(); + } + + @Override + public int size() { + return values.size(); + } + + @Override + public long getLongValue() { + return values.getLongValue(); + } + + @Override + public double getDoubleValue() { + return values.getDoubleValue(); + } +} diff --git a/server/src/main/java/org/elasticsearch/script/DenseVectorField.java b/server/src/main/java/org/elasticsearch/script/DenseVectorField.java new file mode 100644 index 0000000000000..db4815a13160d --- /dev/null +++ b/server/src/main/java/org/elasticsearch/script/DenseVectorField.java @@ -0,0 +1,12 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License + * 2.0 and the Server Side Public License, v 1; you may not use this file except + * in compliance with, at your election, the Elastic License 2.0 or the Server + * Side Public License, v 1. + */ + +package org.elasticsearch.script; + +public class DenseVectorField { +} diff --git a/server/src/main/java/org/elasticsearch/script/DenseVectorScriptValue.java b/server/src/main/java/org/elasticsearch/script/DenseVectorScriptValue.java new file mode 100644 index 0000000000000..999431d3262c1 --- /dev/null +++ b/server/src/main/java/org/elasticsearch/script/DenseVectorScriptValue.java @@ -0,0 +1,28 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License + * 2.0 and the Server Side Public License, v 1; you may not use this file except + * in compliance with, at your election, the Elastic License 2.0 or the Server + * Side Public License, v 1. + */ + +package org.elasticsearch.script; + +import org.apache.lucene.index.BinaryDocValues; +import org.apache.lucene.util.BytesRef; +import org.elasticsearch.Version; + +public class DenseVectorScriptValue { + private final Version indexVersion; + private final int dims; + private final float[] vector; + private BytesRef value; + + public DenseVectorScriptValue(BinaryDocValues in, Version indexVersion, int dims) { + this.indexVersion = indexVersion; + this.dims = dims; + this.vector = new float[dims]; + } + + +} diff --git a/server/src/main/java/org/elasticsearch/script/DocValuesDocReader.java b/server/src/main/java/org/elasticsearch/script/DocValuesDocReader.java index a10e8219f3d3d..4dec3ee74c9b0 100644 --- a/server/src/main/java/org/elasticsearch/script/DocValuesDocReader.java +++ b/server/src/main/java/org/elasticsearch/script/DocValuesDocReader.java @@ -23,7 +23,10 @@ public class DocValuesDocReader implements DocReader, LeafReaderContextSupplier // provide access to the leaf context reader for expressions protected final LeafReaderContext leafReaderContext; + protected final SearchLookup lookup; + public DocValuesDocReader(SearchLookup lookup, LeafReaderContext leafContext) { + this.lookup = lookup; this.leafReaderContext = leafContext; this.leafLookup = lookup.getLeafSearchLookup(leafReaderContext); } @@ -35,7 +38,7 @@ public Field field(String fieldName) { if (doc.containsKey(fieldName) == false) { return new EmptyField(fieldName); } - return new DocValuesField<>(fieldName, doc.get(fieldName)); + return doc.get(fieldName).toField(fieldName); } diff --git a/server/src/main/java/org/elasticsearch/script/DocValuesField.java b/server/src/main/java/org/elasticsearch/script/DocValuesField.java deleted file mode 100644 index 7b94988871e12..0000000000000 --- a/server/src/main/java/org/elasticsearch/script/DocValuesField.java +++ /dev/null @@ -1,46 +0,0 @@ -/* - * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one - * or more contributor license agreements. Licensed under the Elastic License - * 2.0 and the Server Side Public License, v 1; you may not use this file except - * in compliance with, at your election, the Elastic License 2.0 or the Server - * Side Public License, v 1. - */ - -package org.elasticsearch.script; - -import org.elasticsearch.index.fielddata.ScriptDocValues; - -import java.util.List; - -public class DocValuesField implements Field { - protected final String name; - protected final ScriptDocValues scriptDocValues; - - public DocValuesField(String name, ScriptDocValues scriptDocValues) { - this.name = name; - this.scriptDocValues = scriptDocValues; - } - - @Override - public T getValue(T defaultValue) { - if (scriptDocValues.isEmpty()) { - return defaultValue; - } - return scriptDocValues.get(0); - } - - @Override - public String getName() { - return name; - } - - @Override - public boolean isEmpty() { - return scriptDocValues.isEmpty(); - } - - @Override - public List getValues() { - return scriptDocValues; - } -} diff --git a/server/src/main/java/org/elasticsearch/script/EmptyField.java b/server/src/main/java/org/elasticsearch/script/EmptyField.java index c28b8b602f76a..ab24ebcf66b27 100644 --- a/server/src/main/java/org/elasticsearch/script/EmptyField.java +++ b/server/src/main/java/org/elasticsearch/script/EmptyField.java @@ -11,30 +11,39 @@ import java.util.Collections; import java.util.List; -public class EmptyField implements Field { - protected final String name; - +public class EmptyField extends Field { public EmptyField(String name) { - this.name = name; + super(name, null); } @Override - public String getName() { - return name; + public boolean isEmpty() { + return true; } @Override - public boolean isEmpty() { - return true; + public List getValues() { + return Collections.emptyList(); } @Override - public Object getValue(Object defaultValue) { + public > Field as(Converter converter) { + // TODO(stu): test + return converter.getFieldClass().cast(this); + } + + @Override + public T getValue(T defaultValue) { return defaultValue; } @Override - public List getValues() { - return Collections.emptyList(); + public double getDouble(double defaultValue) { + return defaultValue; + } + + @Override + public long getLong(long defaultValue) { + return defaultValue; } } diff --git a/server/src/main/java/org/elasticsearch/script/Field.java b/server/src/main/java/org/elasticsearch/script/Field.java index 2d501c4bea9ef..c0fb55f69762c 100644 --- a/server/src/main/java/org/elasticsearch/script/Field.java +++ b/server/src/main/java/org/elasticsearch/script/Field.java @@ -9,6 +9,10 @@ package org.elasticsearch.script; +import org.apache.lucene.util.BytesRef; +import org.elasticsearch.common.geo.GeoPoint; + +import java.math.BigInteger; import java.util.List; /** @@ -21,15 +25,133 @@ * * {@code getValue(defaultValue) == defaultValue} * @param */ -public interface Field { - String getName(); +public abstract class Field { + public static final Converter BigInteger = Converters.BIGINTEGER; + public static final Converter Long = Converters.LONG; + + protected final String name; + protected final FieldValues values; + + public Field(String name, FieldValues values) { + this.name = name; + this.values = values; + } + + public String getName() { + return name; + } + + /** + * Does the field have any values? An unmapped field may have values from source + */ + public boolean isEmpty() { + return values.isEmpty(); + } - /** Does the field have any values? An unmapped field may have values from source */ - boolean isEmpty(); + /** + * Get all values of a multivalued field. If {@code isEmpty()} this returns an empty list + */ + public List getValues() { + return values.getValues(); + } - /** Get all values of a multivalued field. If {@code isEmpty()} this returns an empty list */ - List getValues(); + public > Field as(Converter converter) { + if (converter.getFieldClass().isInstance(this)) { + return converter.getFieldClass().cast(this); + } + + return converter.convert(this); + } + + public FieldValues getFieldValues() { + return values; + } /** Get the first value of a field, if {@code isEmpty()} return defaultValue instead */ - T getValue(T defaultValue); + public T getValue(T defaultValue) { + if (isEmpty()) { + return defaultValue; + } + return values.getNonPrimitiveValue(); + } + + public double getDouble(double defaultValue) { + if (isEmpty()) { + return defaultValue; + } + return values.getDoubleValue(); + } + + public long getLong(long defaultValue) { + if (isEmpty()) { + return defaultValue; + } + return values.getLongValue(); + } + + public static class BooleanField extends Field { + public BooleanField(String name, FieldValues values) { + super(name, values); + } + } + + public static class DoubleField extends Field { + public DoubleField(String name, FieldValues values) { + super(name, values); + } + } + + public static class LongField extends Field { + public LongField(String name, FieldValues values) { + super(name, values); + } + } + + public static class DateNanosField extends Field { + public DateNanosField(String name, FieldValues values) { + super(name, values); + } + } + + public static class DateMillisField extends Field { + public DateMillisField(String name, FieldValues values) { + super(name, values); + } + } + + public static class GeoPointField extends Field { + public GeoPointField(String name, FieldValues values) { + super(name, values); + } + } + + public static class StringField extends Field { + public StringField(String name, FieldValues values) { + super(name, values); + } + } + + public static class BytesRefField extends Field { + public BytesRefField(String name, FieldValues values) { + super(name, values); + } + } + + public static class BigIntegerField extends Field { + public BigIntegerField(String name, FieldValues values) { + super(name, values); + } + } + + public static class VersionField extends Field { + public VersionField(String name, FieldValues values) { + super(name, values); + } + } + + public static class IpField extends Field { + public IpField(String name, FieldValues values) { + super(name, values); + } + } } diff --git a/server/src/main/java/org/elasticsearch/script/FieldValues.java b/server/src/main/java/org/elasticsearch/script/FieldValues.java new file mode 100644 index 0000000000000..d87fa0da937c3 --- /dev/null +++ b/server/src/main/java/org/elasticsearch/script/FieldValues.java @@ -0,0 +1,24 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License + * 2.0 and the Server Side Public License, v 1; you may not use this file except + * in compliance with, at your election, the Elastic License 2.0 or the Server + * Side Public License, v 1. + */ + +package org.elasticsearch.script; + +import java.util.List; + +public interface FieldValues { + boolean isEmpty(); + int size(); + + List getValues(); + + T getNonPrimitiveValue(); + long getLongValue(); + double getDoubleValue(); +} + +// TODO DelegatingFieldValues diff --git a/server/src/main/java/org/elasticsearch/script/InvalidConversion.java b/server/src/main/java/org/elasticsearch/script/InvalidConversion.java new file mode 100644 index 0000000000000..5c6b768bad245 --- /dev/null +++ b/server/src/main/java/org/elasticsearch/script/InvalidConversion.java @@ -0,0 +1,24 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License + * 2.0 and the Server Side Public License, v 1; you may not use this file except + * in compliance with, at your election, the Elastic License 2.0 or the Server + * Side Public License, v 1. + */ + +package org.elasticsearch.script; + +public class InvalidConversion extends RuntimeException { + protected final Class from; + protected final Class converter; + + public InvalidConversion(Class from, Class converter) { + this.from = from; + this.converter = converter; + } + + @Override + public String getMessage() { + return "Cannot convert from [" + from.getSimpleName() + "] using converter [" + converter.getSimpleName() + "]"; + } +} diff --git a/server/src/test/java/org/elasticsearch/index/query/SearchExecutionContextTests.java b/server/src/test/java/org/elasticsearch/index/query/SearchExecutionContextTests.java index dcaf285833182..b1146eb2e8b35 100644 --- a/server/src/test/java/org/elasticsearch/index/query/SearchExecutionContextTests.java +++ b/server/src/test/java/org/elasticsearch/index/query/SearchExecutionContextTests.java @@ -487,7 +487,7 @@ public LeafFieldData load(LeafReaderContext context) { return new LeafFieldData() { @Override public ScriptDocValues getScriptValues() { - return new ScriptDocValues<>() { + return new ScriptDocValues() { String value; @Override @@ -509,6 +509,11 @@ public void setNextDocId(int docId) { leafLookup.setDocument(docId); value = runtimeDocValues.apply(leafLookup, docId); } + + @Override + public org.elasticsearch.script.Field toField(String fieldName) { + return new org.elasticsearch.script.Field.StringField(fieldName, this); + } }; } diff --git a/x-pack/plugin/mapper-unsigned-long/src/main/java/org/elasticsearch/xpack/unsignedlong/UnsignedLongField.java b/x-pack/plugin/mapper-unsigned-long/src/main/java/org/elasticsearch/xpack/unsignedlong/UnsignedLongField.java new file mode 100644 index 0000000000000..960b90395f992 --- /dev/null +++ b/x-pack/plugin/mapper-unsigned-long/src/main/java/org/elasticsearch/xpack/unsignedlong/UnsignedLongField.java @@ -0,0 +1,95 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License + * 2.0; you may not use this file except in compliance with the Elastic License + * 2.0. + */ + +package org.elasticsearch.xpack.unsignedlong; + +import org.elasticsearch.script.Converter; +import org.elasticsearch.script.DelegatingFieldValues; +import org.elasticsearch.script.Field; +import org.elasticsearch.script.FieldValues; +import org.elasticsearch.script.InvalidConversion; + +import java.math.BigInteger; +import java.util.List; +import java.util.stream.Collectors; + +import static org.elasticsearch.xpack.unsignedlong.UnsignedLongFieldMapper.BIGINTEGER_2_64_MINUS_ONE; + +public class UnsignedLongField extends Field.LongField { + public UnsignedLongField(String name, FieldValues values) { + super(name, values); + } + + @Override + public > Field as(Converter converter) { + if (converter.getFieldClass().isInstance(this)) { + return converter.getFieldClass().cast(this); + } + + if (converter.getTargetClass() == BigInteger.class) { + BigIntegerField bigIntegerField = UnsignedLongToBigInteger(this); + return converter.getFieldClass().cast(bigIntegerField); + } + + return super.as(converter); + } + + public static class UnsignedLongConverter implements Converter { + @Override + public UnsignedLongField convert(Field sourceField) { + if (sourceField instanceof BigIntegerField) { + return BigIntegerToUnsignedLong((BigIntegerField) sourceField); + } + + throw new InvalidConversion(sourceField.getClass(), getFieldClass()); + } + + @Override + public Class getFieldClass() { + return UnsignedLongField.class; + } + + @Override + public Class getTargetClass() { + return Long.class; + } + } + + public static BigIntegerField UnsignedLongToBigInteger(UnsignedLongField sourceField) { + FieldValues fv = sourceField.getFieldValues(); + return new BigIntegerField(sourceField.getName(), new DelegatingFieldValues(fv) { + protected BigInteger toBigInteger(long formatted) { + return java.math.BigInteger.valueOf(formatted).and(BIGINTEGER_2_64_MINUS_ONE); + } + + @Override + public List getValues() { + return values.getValues().stream().map(this::toBigInteger).collect(Collectors.toList()); + } + + @Override + public BigInteger getNonPrimitiveValue() { + return toBigInteger(values.getLongValue()); + } + }); + } + + public static UnsignedLongField BigIntegerToUnsignedLong(BigIntegerField sourceField) { + FieldValues fv = sourceField.getFieldValues(); + return new UnsignedLongField(sourceField.getName(), new DelegatingFieldValues<>(fv) { + @Override + public List getValues() { + return values.getValues().stream().map(java.math.BigInteger::longValue).collect(Collectors.toList()); + } + + @Override + public Long getNonPrimitiveValue() { + return values.getLongValue(); + } + }); + } +} diff --git a/x-pack/plugin/mapper-unsigned-long/src/main/java/org/elasticsearch/xpack/unsignedlong/UnsignedLongLeafFieldData.java b/x-pack/plugin/mapper-unsigned-long/src/main/java/org/elasticsearch/xpack/unsignedlong/UnsignedLongLeafFieldData.java index 0121cd62cc035..c898d2e147c9e 100644 --- a/x-pack/plugin/mapper-unsigned-long/src/main/java/org/elasticsearch/xpack/unsignedlong/UnsignedLongLeafFieldData.java +++ b/x-pack/plugin/mapper-unsigned-long/src/main/java/org/elasticsearch/xpack/unsignedlong/UnsignedLongLeafFieldData.java @@ -74,8 +74,7 @@ public int docValueCount() { @Override public ScriptDocValues getScriptValues() { - // TODO: add support for scripts - throw new UnsupportedOperationException("Using unsigned_long in scripts is currently not supported!"); + return new UnsignedLongScriptDocValues(getLongValues()); } @Override diff --git a/x-pack/plugin/mapper-unsigned-long/src/main/java/org/elasticsearch/xpack/unsignedlong/UnsignedLongScriptDocValues.java b/x-pack/plugin/mapper-unsigned-long/src/main/java/org/elasticsearch/xpack/unsignedlong/UnsignedLongScriptDocValues.java index f5b8025cfc974..bb97776273c96 100644 --- a/x-pack/plugin/mapper-unsigned-long/src/main/java/org/elasticsearch/xpack/unsignedlong/UnsignedLongScriptDocValues.java +++ b/x-pack/plugin/mapper-unsigned-long/src/main/java/org/elasticsearch/xpack/unsignedlong/UnsignedLongScriptDocValues.java @@ -10,11 +10,13 @@ import org.apache.lucene.index.SortedNumericDocValues; import org.apache.lucene.util.ArrayUtil; import org.elasticsearch.index.fielddata.ScriptDocValues; -import org.elasticsearch.search.DocValueFormat; +import org.elasticsearch.script.Field; import java.io.IOException; -public class UnsignedLongScriptDocValues extends ScriptDocValues { +import static org.elasticsearch.search.DocValueFormat.MASK_2_63; + +public class UnsignedLongScriptDocValues extends ScriptDocValues { private final SortedNumericDocValues in; private long[] values = new long[0]; private int count; @@ -47,22 +49,41 @@ protected void resize(int newSize) { values = ArrayUtil.grow(values, count); } - public Number getValue() { - return get(0); + public long getValue() { + throwIfEmpty(); + return format(0); } @Override - public Number get(int index) { - if (count == 0) { - throw new IllegalStateException( - "A document doesn't have a value for a field! Use doc[].size()==0 to check if a document is missing a field!" - ); - } - return (Number) DocValueFormat.UNSIGNED_LONG_SHIFTED.format(values[index]); + public Long get(int index) { + throwIfEmpty(); + return format(index); + } + + protected long format(int index) { + return values[index] ^ MASK_2_63; } @Override public int size() { return count; } + + @Override + public long getLongValue() { + throwIfEmpty(); + return format(0); + } + + @Override + public double getDoubleValue() { + throwIfEmpty(); + return format(0); + } + + @Override + public Field toField(String fieldName) { + return new UnsignedLongField(fieldName, this); + } + } diff --git a/x-pack/plugin/mapper-version/src/main/java/org/elasticsearch/xpack/versionfield/VersionScriptDocValues.java b/x-pack/plugin/mapper-version/src/main/java/org/elasticsearch/xpack/versionfield/VersionScriptDocValues.java index 25cd66a0eb212..1bd624160610f 100644 --- a/x-pack/plugin/mapper-version/src/main/java/org/elasticsearch/xpack/versionfield/VersionScriptDocValues.java +++ b/x-pack/plugin/mapper-version/src/main/java/org/elasticsearch/xpack/versionfield/VersionScriptDocValues.java @@ -10,6 +10,7 @@ import org.apache.lucene.index.SortedSetDocValues; import org.apache.lucene.util.ArrayUtil; import org.elasticsearch.index.fielddata.ScriptDocValues; +import org.elasticsearch.script.Field; import java.io.IOException; @@ -56,4 +57,9 @@ public String get(int index) { public int size() { return count; } + + @Override + public Field toField(String fieldName) { + return new Field.VersionField(fieldName, this); + } } diff --git a/x-pack/plugin/spatial/src/main/java/org/elasticsearch/xpack/spatial/index/fielddata/plain/AbstractAtomicGeoShapeShapeFieldData.java b/x-pack/plugin/spatial/src/main/java/org/elasticsearch/xpack/spatial/index/fielddata/plain/AbstractAtomicGeoShapeShapeFieldData.java index 62b0abedd788d..c00997ab75245 100644 --- a/x-pack/plugin/spatial/src/main/java/org/elasticsearch/xpack/spatial/index/fielddata/plain/AbstractAtomicGeoShapeShapeFieldData.java +++ b/x-pack/plugin/spatial/src/main/java/org/elasticsearch/xpack/spatial/index/fielddata/plain/AbstractAtomicGeoShapeShapeFieldData.java @@ -12,6 +12,8 @@ import org.elasticsearch.common.geo.GeoPoint; import org.elasticsearch.index.fielddata.ScriptDocValues; import org.elasticsearch.index.fielddata.SortedBinaryDocValues; +import org.elasticsearch.script.Field; +import org.elasticsearch.script.FieldValues; import org.elasticsearch.xpack.spatial.index.fielddata.GeoShapeValues; import org.elasticsearch.xpack.spatial.index.fielddata.LeafGeoShapeFieldData; @@ -115,5 +117,16 @@ public GeoShapeValues.GeoShapeValue get(int index) { public int size() { return value == null ? 0 : 1; } + + @Override + public Field toField(String fieldName) { + return new GeoShapeField(fieldName, this); + } + } + + public static class GeoShapeField extends Field { + public GeoShapeField(String name, FieldValues values) { + super(name, values); + } } } diff --git a/x-pack/plugin/src/yamlRestTest/resources/rest-api-spec/test/unsigned_long/50_script_values.yml b/x-pack/plugin/src/yamlRestTest/resources/rest-api-spec/test/unsigned_long/50_script_values.yml index 56929f56f4742..36390b5004ad1 100644 --- a/x-pack/plugin/src/yamlRestTest/resources/rest-api-spec/test/unsigned_long/50_script_values.yml +++ b/x-pack/plugin/src/yamlRestTest/resources/rest-api-spec/test/unsigned_long/50_script_values.yml @@ -1,8 +1,8 @@ setup: - skip: - version: "all" - reason: "AwaitsFix https://github.com/elastic/elasticsearch/issues/64361" + version: " - 7.14.99" + reason: "unsigned long script fields api was added in 7.15.0" - do: indices.create: @@ -25,12 +25,12 @@ setup: { "index": {"_id" : "3"} } { "ul": 9223372036854775808 } { "index": {"_id" : "4"} } - { "ul": 18446744073709551614 } + { "ul": 18446744073709551613 } { "index": {"_id" : "5"} } { "ul": 18446744073709551615 } --- -"Scripted fields values return BigInteger or Long": +"Scripted fields values return Long": - do: search: index: test1 @@ -39,11 +39,11 @@ setup: script_fields: scripted_ul: script: - source: "doc['ul'].value" + source: "field('ul').getValue(1000L)" - - match: { hits.hits.0.fields.scripted_ul.0: 18446744073709551615 } - - match: { hits.hits.1.fields.scripted_ul.0: 18446744073709551614 } - - match: { hits.hits.2.fields.scripted_ul.0: 9223372036854775808 } + - match: { hits.hits.0.fields.scripted_ul.0: -1 } + - match: { hits.hits.1.fields.scripted_ul.0: -3 } + - match: { hits.hits.2.fields.scripted_ul.0: -9223372036854775808 } - match: { hits.hits.3.fields.scripted_ul.0: 9223372036854775807 } - match: { hits.hits.4.fields.scripted_ul.0: 0 } @@ -58,7 +58,7 @@ setup: order: desc type: number script: - source: "doc['ul'].value" + source: "field('ul').as(Field.BigInteger).getValue(BigInteger.valueOf(Long.parseUnsignedLong('18446744073709551614'))).doubleValue()" - match: { hits.hits.0.sort: [1.8446744073709552E19] } - match: { hits.hits.1.sort: [1.8446744073709552E19] } @@ -77,7 +77,7 @@ setup: filter: script: script: - source: "doc['ul'].value.doubleValue() > 10E18" + source: "field('ul').as(Field.BigInteger).getValue(BigInteger.valueOf(Long.parseUnsignedLong('18446744073709551614'))).doubleValue() > 10E18" sort: [ { ul: asc } ] - match: { hits.total.value: 2 } - match: { hits.hits.0._id: "4" } @@ -93,7 +93,7 @@ setup: filter: script: script: - source: "doc['ul'].size() > 0" + source: "field('ul').isEmpty() == false" - match: { hits.total.value: 5 } --- @@ -106,6 +106,6 @@ setup: script_score: query: {match_all: {}} script: - source: "doc['ul'].value" + source: "field('ul').as(Field.BigInteger).getValue(BigInteger.valueOf(Long.parseUnsignedLong('18446744073709551614'))).doubleValue()" - match: { hits.total.value: 5 } diff --git a/x-pack/plugin/vectors/src/main/java/org/elasticsearch/xpack/vectors/query/DenseVectorScriptDocValues.java b/x-pack/plugin/vectors/src/main/java/org/elasticsearch/xpack/vectors/query/DenseVectorScriptDocValues.java index 18e2e80a090bc..1f3d27c0f7312 100644 --- a/x-pack/plugin/vectors/src/main/java/org/elasticsearch/xpack/vectors/query/DenseVectorScriptDocValues.java +++ b/x-pack/plugin/vectors/src/main/java/org/elasticsearch/xpack/vectors/query/DenseVectorScriptDocValues.java @@ -12,6 +12,7 @@ import org.apache.lucene.util.BytesRef; import org.elasticsearch.Version; import org.elasticsearch.index.fielddata.ScriptDocValues; +import org.elasticsearch.script.Field; import org.elasticsearch.xpack.vectors.mapper.VectorEncoderDecoder; import java.io.IOException; @@ -80,4 +81,14 @@ public int size() { return 1; } } + + @Override + public BytesRef getNonPrimitiveValue() { + return value; + } + + @Override + public Field toField(String fieldName) { + throw new IllegalStateException("not implemented"); + } } From 7adc6d4c95328148bf22e3790e600c4bc843c4ad Mon Sep 17 00:00:00 2001 From: Stuart Tettemer Date: Fri, 13 Aug 2021 18:11:03 -0500 Subject: [PATCH 02/14] Script: dense vector not implemented yet --- .../script/DenseVectorField.java | 12 -------- .../script/DenseVectorScriptValue.java | 28 ------------------- .../org/elasticsearch/script/FieldValues.java | 2 -- 3 files changed, 42 deletions(-) delete mode 100644 server/src/main/java/org/elasticsearch/script/DenseVectorField.java delete mode 100644 server/src/main/java/org/elasticsearch/script/DenseVectorScriptValue.java diff --git a/server/src/main/java/org/elasticsearch/script/DenseVectorField.java b/server/src/main/java/org/elasticsearch/script/DenseVectorField.java deleted file mode 100644 index db4815a13160d..0000000000000 --- a/server/src/main/java/org/elasticsearch/script/DenseVectorField.java +++ /dev/null @@ -1,12 +0,0 @@ -/* - * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one - * or more contributor license agreements. Licensed under the Elastic License - * 2.0 and the Server Side Public License, v 1; you may not use this file except - * in compliance with, at your election, the Elastic License 2.0 or the Server - * Side Public License, v 1. - */ - -package org.elasticsearch.script; - -public class DenseVectorField { -} diff --git a/server/src/main/java/org/elasticsearch/script/DenseVectorScriptValue.java b/server/src/main/java/org/elasticsearch/script/DenseVectorScriptValue.java deleted file mode 100644 index 999431d3262c1..0000000000000 --- a/server/src/main/java/org/elasticsearch/script/DenseVectorScriptValue.java +++ /dev/null @@ -1,28 +0,0 @@ -/* - * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one - * or more contributor license agreements. Licensed under the Elastic License - * 2.0 and the Server Side Public License, v 1; you may not use this file except - * in compliance with, at your election, the Elastic License 2.0 or the Server - * Side Public License, v 1. - */ - -package org.elasticsearch.script; - -import org.apache.lucene.index.BinaryDocValues; -import org.apache.lucene.util.BytesRef; -import org.elasticsearch.Version; - -public class DenseVectorScriptValue { - private final Version indexVersion; - private final int dims; - private final float[] vector; - private BytesRef value; - - public DenseVectorScriptValue(BinaryDocValues in, Version indexVersion, int dims) { - this.indexVersion = indexVersion; - this.dims = dims; - this.vector = new float[dims]; - } - - -} diff --git a/server/src/main/java/org/elasticsearch/script/FieldValues.java b/server/src/main/java/org/elasticsearch/script/FieldValues.java index d87fa0da937c3..ab1a38991f673 100644 --- a/server/src/main/java/org/elasticsearch/script/FieldValues.java +++ b/server/src/main/java/org/elasticsearch/script/FieldValues.java @@ -20,5 +20,3 @@ public interface FieldValues { long getLongValue(); double getDoubleValue(); } - -// TODO DelegatingFieldValues From 60ebf613d83ee0f81e810f6dc6aee5d99c48cb8d Mon Sep 17 00:00:00 2001 From: Stuart Tettemer Date: Mon, 16 Aug 2021 10:14:54 -0500 Subject: [PATCH 03/14] javadoc --- .../org/elasticsearch/script/Converter.java | 16 ++++++++++++++ .../org/elasticsearch/script/Converters.java | 19 +++++++++++++++++ .../script/DelegatingFieldValues.java | 4 ++++ .../script/DocValuesDocReader.java | 5 ++++- .../org/elasticsearch/script/EmptyField.java | 3 +++ .../java/org/elasticsearch/script/Field.java | 21 +++++++++++++++++++ .../org/elasticsearch/script/FieldValues.java | 14 +++++++++++++ .../script/InvalidConversion.java | 7 +++++++ .../xpack/unsignedlong/UnsignedLongField.java | 1 + .../UnsignedLongScriptDocValues.java | 4 ++++ 10 files changed, 93 insertions(+), 1 deletion(-) diff --git a/server/src/main/java/org/elasticsearch/script/Converter.java b/server/src/main/java/org/elasticsearch/script/Converter.java index fa9c6a0f37f7a..b87e4a57a2ac9 100644 --- a/server/src/main/java/org/elasticsearch/script/Converter.java +++ b/server/src/main/java/org/elasticsearch/script/Converter.java @@ -8,8 +8,24 @@ package org.elasticsearch.script; +/** + * Converts between one scripting {@link Field} type and another, {@code CF}, with a different underlying + * value type, {@code CT}. + */ public interface Converter> { + /** + * Convert {@code sourceField} to a new field-type. Conversions come from user scripts so {@code covert} may + * be called on a {@link Field}'s own type. + */ CF convert(Field sourceField); + + /** + * The destination {@link Field} class. + */ Class getFieldClass(); + + /** + * The target value type. + */ Class getTargetClass(); } diff --git a/server/src/main/java/org/elasticsearch/script/Converters.java b/server/src/main/java/org/elasticsearch/script/Converters.java index 61418d41a116a..346b5d3175a8a 100644 --- a/server/src/main/java/org/elasticsearch/script/Converters.java +++ b/server/src/main/java/org/elasticsearch/script/Converters.java @@ -21,8 +21,27 @@ import static org.elasticsearch.script.Field.LongField; import static org.elasticsearch.script.Field.StringField; +/** + * {@link Converters} for scripting fields. These constants are exposed as static fields on {@link Field} to + * allow a user to convert via {@link Field#as(Converter)}. + */ public class Converters { + /** + * Convert to a {@link BigIntegerField} from Long, Double or String Fields. + * Longs and Doubles are wrapped as BigIntegers. + * Strings are parsed as either Longs or Doubles and wrapped in a BigInteger. + */ public static final Converter BIGINTEGER; + + /** + * Convert to a {@link LongField} from Double, String, DateMillis, DateNanos, BigInteger or Boolean Fields. + * Double is cast to a Long. + * String is parsed as a Long. + * DateMillis is milliseconds since epoch. + * DateNanos is nanoseconds since epoch. + * {@link BigInteger#longValue()} is used for the BigInteger conversion. + * Boolean is {@code 1L} if {@code true}, {@code 0L} if {@code false}. + */ public static final Converter LONG; static { diff --git a/server/src/main/java/org/elasticsearch/script/DelegatingFieldValues.java b/server/src/main/java/org/elasticsearch/script/DelegatingFieldValues.java index aec6948fd88da..761d56bac45d0 100644 --- a/server/src/main/java/org/elasticsearch/script/DelegatingFieldValues.java +++ b/server/src/main/java/org/elasticsearch/script/DelegatingFieldValues.java @@ -8,6 +8,10 @@ package org.elasticsearch.script; +/** + * Helper for creating {@link Converter} classes which delegates all un-overridden methods to the underlying + * {@link FieldValues}. + */ public abstract class DelegatingFieldValues implements FieldValues { protected FieldValues values; diff --git a/server/src/main/java/org/elasticsearch/script/DocValuesDocReader.java b/server/src/main/java/org/elasticsearch/script/DocValuesDocReader.java index 4dec3ee74c9b0..1f8bbbcbfb8d9 100644 --- a/server/src/main/java/org/elasticsearch/script/DocValuesDocReader.java +++ b/server/src/main/java/org/elasticsearch/script/DocValuesDocReader.java @@ -16,6 +16,9 @@ import java.util.Map; import java.util.stream.Stream; +/** + * Provide access to DocValues for script {@code field} api and {@code doc} API. + */ public class DocValuesDocReader implements DocReader, LeafReaderContextSupplier { /** A leaf lookup for the bound segment this proxy will operate on. */ protected LeafSearchLookup leafLookup; @@ -44,7 +47,7 @@ public Field field(String fieldName) { @Override public Stream> fields(String fieldGlob) { - return Stream.empty(); + throw new UnsupportedOperationException("not implemented"); } @Override diff --git a/server/src/main/java/org/elasticsearch/script/EmptyField.java b/server/src/main/java/org/elasticsearch/script/EmptyField.java index ab24ebcf66b27..93fa24327a233 100644 --- a/server/src/main/java/org/elasticsearch/script/EmptyField.java +++ b/server/src/main/java/org/elasticsearch/script/EmptyField.java @@ -11,6 +11,9 @@ import java.util.Collections; import java.util.List; +/** + * Script field with no mapping, always returns {@code defaultValue}. + */ public class EmptyField extends Field { public EmptyField(String name) { super(name, null); diff --git a/server/src/main/java/org/elasticsearch/script/Field.java b/server/src/main/java/org/elasticsearch/script/Field.java index c0fb55f69762c..8d0a659a80ff5 100644 --- a/server/src/main/java/org/elasticsearch/script/Field.java +++ b/server/src/main/java/org/elasticsearch/script/Field.java @@ -55,6 +55,15 @@ public List getValues() { return values.getValues(); } + /** + * Convert this {@code Field} into another {@code Field} type using a {@link Converter} of the target type. + * + * As this is called from user scripts, {@code as} may be called to convert a field into it's same type, if + * so {@code this} is cast via that converters {@link Converter#getFieldClass()}. + * + * Extensions outside the core package should override this method to implement their own conversions, calling + * this method as a fallback. + */ public > Field as(Converter converter) { if (converter.getFieldClass().isInstance(this)) { return converter.getFieldClass().cast(this); @@ -63,6 +72,9 @@ public > Field as(Converter converter) { return converter.convert(this); } + /** + * Provide {@link Converter}s access to the underlying {@link FieldValues}, should not be exposed to scripts + */ public FieldValues getFieldValues() { return values; } @@ -75,6 +87,10 @@ public T getValue(T defaultValue) { return values.getNonPrimitiveValue(); } + /** + * Get the underlying value as a {@code double} unless {@link #isEmpty()}, in which case return {@code defaultValue}. + * May throw {@link InvalidConversion} if the underlying value is not representable as a {@code double}. + */ public double getDouble(double defaultValue) { if (isEmpty()) { return defaultValue; @@ -82,6 +98,11 @@ public double getDouble(double defaultValue) { return values.getDoubleValue(); } + + /** + * Get the underlying value as a {@code long} unless {@link #isEmpty()}, in which case return {@code defaultValue}. + * May throw {@link InvalidConversion} if the underlying value is not representable as a {@code long}. + */ public long getLong(long defaultValue) { if (isEmpty()) { return defaultValue; diff --git a/server/src/main/java/org/elasticsearch/script/FieldValues.java b/server/src/main/java/org/elasticsearch/script/FieldValues.java index ab1a38991f673..9c5242e293cad 100644 --- a/server/src/main/java/org/elasticsearch/script/FieldValues.java +++ b/server/src/main/java/org/elasticsearch/script/FieldValues.java @@ -10,13 +10,27 @@ import java.util.List; +/** + * The underlying contents of a scripting Field. Implementations include + * {@link org.elasticsearch.index.fielddata.ScriptDocValues}, {@code _source}, runtime fields, or + * the converted form of the these values from a {@link Converter}. + */ public interface FieldValues { + /** Are there any values? */ boolean isEmpty(); + + /** How many values? */ int size(); + /** All underlying values. Note this boxes primitives */ List getValues(); + /** The first value as a subclass of {@link java.lang.Object}. Boxes primitives */ T getNonPrimitiveValue(); + + /** The first value as a primitive long. For performance reasons, implementations should avoid intermediate boxings if possible */ long getLongValue(); + + /** The first value as a primitive double. For performance reasons, implementations should avoid intermediate boxings if possible */ double getDoubleValue(); } diff --git a/server/src/main/java/org/elasticsearch/script/InvalidConversion.java b/server/src/main/java/org/elasticsearch/script/InvalidConversion.java index 5c6b768bad245..31fd77d3b30ab 100644 --- a/server/src/main/java/org/elasticsearch/script/InvalidConversion.java +++ b/server/src/main/java/org/elasticsearch/script/InvalidConversion.java @@ -8,8 +8,15 @@ package org.elasticsearch.script; +/** + * A failed conversion of a script {@link Field}. Thrown by {@link Converter}s and + * {@link FieldValues#getDoubleValue()}/{@link FieldValues#getLongValue()}. + */ public class InvalidConversion extends RuntimeException { + /** Source {@link Field} or underlying value */ protected final Class from; + + /** Destination class or {@link Converter} */ protected final Class converter; public InvalidConversion(Class from, Class converter) { diff --git a/x-pack/plugin/mapper-unsigned-long/src/main/java/org/elasticsearch/xpack/unsignedlong/UnsignedLongField.java b/x-pack/plugin/mapper-unsigned-long/src/main/java/org/elasticsearch/xpack/unsignedlong/UnsignedLongField.java index 960b90395f992..87663ec3efbe5 100644 --- a/x-pack/plugin/mapper-unsigned-long/src/main/java/org/elasticsearch/xpack/unsignedlong/UnsignedLongField.java +++ b/x-pack/plugin/mapper-unsigned-long/src/main/java/org/elasticsearch/xpack/unsignedlong/UnsignedLongField.java @@ -24,6 +24,7 @@ public UnsignedLongField(String name, FieldValues values) { super(name, values); } + // UnsignedLongFields must define their own conversions as they are in x-pack @Override public > Field as(Converter converter) { if (converter.getFieldClass().isInstance(this)) { diff --git a/x-pack/plugin/mapper-unsigned-long/src/main/java/org/elasticsearch/xpack/unsignedlong/UnsignedLongScriptDocValues.java b/x-pack/plugin/mapper-unsigned-long/src/main/java/org/elasticsearch/xpack/unsignedlong/UnsignedLongScriptDocValues.java index bb97776273c96..8d2fdd95721d0 100644 --- a/x-pack/plugin/mapper-unsigned-long/src/main/java/org/elasticsearch/xpack/unsignedlong/UnsignedLongScriptDocValues.java +++ b/x-pack/plugin/mapper-unsigned-long/src/main/java/org/elasticsearch/xpack/unsignedlong/UnsignedLongScriptDocValues.java @@ -60,6 +60,10 @@ public Long get(int index) { return format(index); } + /** + * Applies the formatting from {@link org.elasticsearch.search.DocValueFormat.UnsignedLongShiftedDocValueFormat#format(long)} so + * that the underlying value can be treated as a primitive long as that method returns either a {@code long} or a {@code BigInteger}. + */ protected long format(int index) { return values[index] ^ MASK_2_63; } From 29dad53a635f5af832c8dd8205e93fc80e671636 Mon Sep 17 00:00:00 2001 From: Stuart Tettemer Date: Mon, 16 Aug 2021 11:40:16 -0500 Subject: [PATCH 04/14] don't cast EmptyField, create one --- .../test/painless/40_fields_api.yml | 66 +++++++++++++++++++ .../org/elasticsearch/script/EmptyField.java | 4 +- 2 files changed, 68 insertions(+), 2 deletions(-) diff --git a/modules/lang-painless/src/yamlRestTest/resources/rest-api-spec/test/painless/40_fields_api.yml b/modules/lang-painless/src/yamlRestTest/resources/rest-api-spec/test/painless/40_fields_api.yml index 61749d8333cd4..099d78b82c7f2 100644 --- a/modules/lang-painless/src/yamlRestTest/resources/rest-api-spec/test/painless/40_fields_api.yml +++ b/modules/lang-painless/src/yamlRestTest/resources/rest-api-spec/test/painless/40_fields_api.yml @@ -158,3 +158,69 @@ setup: script: "field('sval.keyword').getValue('b') == 'a'" - match: { hits.total: 1 } - match: { hits.hits.0._id: d3 } + +--- +"missing field": + - do: + indices.create: + index: test + body: + settings: + number_of_shards: 2 + mappings: + properties: + dval: + type: double + - do: + index: + index: test + id: d1 + body: {"foo": 25, "dval": 1.0 } + - do: + index: + index: test + id: d2 + body: {"foo": 9223372036854775807, "dval": 2.0 } + - do: + index: + index: test + id: d3 + body: { "bar": "abc", "dval": 3.0 } + - do: + indices.refresh: {} + - do: + search: + rest_total_hits_as_int: true + index: test + body: + query: { "match_all": {} } + sort: [ { dval: asc } ] + script_fields: + missing_field: + script: + source: "field('foo').as(Field.BigInteger).getValue(BigInteger.TEN).add(BigInteger.ONE)" + - match: { hits.total: 3 } + - match: { hits.hits.0._id: d1 } + - match: { hits.hits.0.fields.missing_field.0: 26 } + - match: { hits.hits.1._id: d2 } + - match: { hits.hits.1.fields.missing_field.0: 9223372036854775808 } + - match: { hits.hits.2._id: d3 } + - match: { hits.hits.2.fields.missing_field.0: 11 } + - do: + search: + rest_total_hits_as_int: true + index: test + body: + query: { "match_all": {} } + sort: [ { dval: asc } ] + script_fields: + missing_field: + script: + source: "field('foo').as(Field.BigInteger).as(Field.Long).getValue(10L) + 2L" + - match: { hits.total: 3 } + - match: { hits.hits.0._id: d1 } + - match: { hits.hits.0.fields.missing_field.0: 27 } + - match: { hits.hits.1._id: d2 } + - match: { hits.hits.1.fields.missing_field.0: -9223372036854775809 } + - match: { hits.hits.2._id: d3 } + - match: { hits.hits.2.fields.missing_field.0: 12 } diff --git a/server/src/main/java/org/elasticsearch/script/EmptyField.java b/server/src/main/java/org/elasticsearch/script/EmptyField.java index 93fa24327a233..a77a84e195024 100644 --- a/server/src/main/java/org/elasticsearch/script/EmptyField.java +++ b/server/src/main/java/org/elasticsearch/script/EmptyField.java @@ -31,8 +31,8 @@ public List getValues() { @Override public > Field as(Converter converter) { - // TODO(stu): test - return converter.getFieldClass().cast(this); + // new object created to ensure EmptyField + return new EmptyField<>(name); } @Override From 4904abc1e0a82acc786a496baa5197e3f004ba56 Mon Sep 17 00:00:00 2001 From: Stuart Tettemer Date: Mon, 16 Aug 2021 13:49:00 -0500 Subject: [PATCH 05/14] doc['ul'].value syntax --- .../test/painless/40_fields_api.yml | 80 ++++++++++++++ .../org/elasticsearch/script/Converters.java | 35 ++++++ .../script/DelegatingFieldValues.java | 41 ------- .../script/DocValuesDocReader.java | 2 +- .../java/org/elasticsearch/script/Field.java | 1 - .../DocValuesWhitelistExtension.java | 32 +++++- .../xpack/unsignedlong/UnsignedLongField.java | 28 +---- .../UnsignedLongScriptDocValues.java | 7 +- .../xpack/unsignedlong/whitelist.txt | 9 +- .../test/unsigned_long/50_script_values.yml | 101 ++++++++++++++++++ 10 files changed, 260 insertions(+), 76 deletions(-) delete mode 100644 server/src/main/java/org/elasticsearch/script/DelegatingFieldValues.java diff --git a/modules/lang-painless/src/yamlRestTest/resources/rest-api-spec/test/painless/40_fields_api.yml b/modules/lang-painless/src/yamlRestTest/resources/rest-api-spec/test/painless/40_fields_api.yml index 099d78b82c7f2..346c1ebfbbd4a 100644 --- a/modules/lang-painless/src/yamlRestTest/resources/rest-api-spec/test/painless/40_fields_api.yml +++ b/modules/lang-painless/src/yamlRestTest/resources/rest-api-spec/test/painless/40_fields_api.yml @@ -224,3 +224,83 @@ setup: - match: { hits.hits.1.fields.missing_field.0: -9223372036854775809 } - match: { hits.hits.2._id: d3 } - match: { hits.hits.2.fields.missing_field.0: 12 } +--- +"date to long": + - do: + indices.create: + index: test + body: + settings: + number_of_shards: 2 + mappings: + properties: + dval: + type: double + - do: + index: + index: test + id: d1 + body: { "dval": 101, "dt": "2015-01-01T12:10:30Z" } + - do: + index: + index: test + id: d2 + body: { "dval": 202, "dt": 1420070400001 } + - do: + index: + index: test + id: d3 + body: { "dval": 303, "dt": "2015-01-01" } + - do: + index: + index: test + id: d4 + body: { "dval": 404, "dtnanos": "2015-01-01T12:10:30.123456789Z"} + - do: + index: + index: test + id: d5 + body: { "dval": 505, "dtnanos": 1420070412345 } + - do: + index: + index: test + id: d6 + body:{ "dval": 606 } + - do: + indices.refresh: {} + - do: + search: + rest_total_hits_as_int: true + index: test + body: + query: { "match_all": {} } + sort: [ { dval: asc } ] + script_fields: + missing_field: + script: + source: "field('foo').as(Field.BigInteger).getValue(BigInteger.TEN).add(BigInteger.ONE)" + - match: { hits.total: 3 } + - match: { hits.hits.0._id: d1 } + - match: { hits.hits.0.fields.missing_field.0: 26 } + - match: { hits.hits.1._id: d2 } + - match: { hits.hits.1.fields.missing_field.0: 9223372036854775808 } + - match: { hits.hits.2._id: d3 } + - match: { hits.hits.2.fields.missing_field.0: 11 } + - do: + search: + rest_total_hits_as_int: true + index: test + body: + query: { "match_all": {} } + sort: [ { dval: asc } ] + script_fields: + missing_field: + script: + source: "field('foo').as(Field.BigInteger).as(Field.Long).getValue(10L) + 2L" + - match: { hits.total: 3 } + - match: { hits.hits.0._id: d1 } + - match: { hits.hits.0.fields.missing_field.0: 27 } + - match: { hits.hits.1._id: d2 } + - match: { hits.hits.1.fields.missing_field.0: -9223372036854775809 } + - match: { hits.hits.2._id: d3 } + - match: { hits.hits.2.fields.missing_field.0: 12 } diff --git a/server/src/main/java/org/elasticsearch/script/Converters.java b/server/src/main/java/org/elasticsearch/script/Converters.java index 346b5d3175a8a..c370567412312 100644 --- a/server/src/main/java/org/elasticsearch/script/Converters.java +++ b/server/src/main/java/org/elasticsearch/script/Converters.java @@ -116,6 +116,9 @@ public Class getTargetClass() { }; } + // No instances, please + private Converters() {} + public static BigIntegerField LongToBigInteger(LongField sourceField) { FieldValues fv = sourceField.getFieldValues(); return new BigIntegerField(sourceField.getName(), new DelegatingFieldValues<>(fv) { @@ -289,4 +292,36 @@ public double getDoubleValue() { } }); } + + /** + * Helper for creating {@link Converter} classes which delegates all un-overridden methods to the underlying + * {@link FieldValues}. + */ + public abstract static class DelegatingFieldValues implements FieldValues { + protected FieldValues values; + + public DelegatingFieldValues(FieldValues values) { + this.values = values; + } + + @Override + public boolean isEmpty() { + return values.isEmpty(); + } + + @Override + public int size() { + return values.size(); + } + + @Override + public long getLongValue() { + return values.getLongValue(); + } + + @Override + public double getDoubleValue() { + return values.getDoubleValue(); + } + } } diff --git a/server/src/main/java/org/elasticsearch/script/DelegatingFieldValues.java b/server/src/main/java/org/elasticsearch/script/DelegatingFieldValues.java deleted file mode 100644 index 761d56bac45d0..0000000000000 --- a/server/src/main/java/org/elasticsearch/script/DelegatingFieldValues.java +++ /dev/null @@ -1,41 +0,0 @@ -/* - * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one - * or more contributor license agreements. Licensed under the Elastic License - * 2.0 and the Server Side Public License, v 1; you may not use this file except - * in compliance with, at your election, the Elastic License 2.0 or the Server - * Side Public License, v 1. - */ - -package org.elasticsearch.script; - -/** - * Helper for creating {@link Converter} classes which delegates all un-overridden methods to the underlying - * {@link FieldValues}. - */ -public abstract class DelegatingFieldValues implements FieldValues { - protected FieldValues values; - - public DelegatingFieldValues(FieldValues values) { - this.values = values; - } - - @Override - public boolean isEmpty() { - return values.isEmpty(); - } - - @Override - public int size() { - return values.size(); - } - - @Override - public long getLongValue() { - return values.getLongValue(); - } - - @Override - public double getDoubleValue() { - return values.getDoubleValue(); - } -} diff --git a/server/src/main/java/org/elasticsearch/script/DocValuesDocReader.java b/server/src/main/java/org/elasticsearch/script/DocValuesDocReader.java index 1f8bbbcbfb8d9..80a6aed9364f3 100644 --- a/server/src/main/java/org/elasticsearch/script/DocValuesDocReader.java +++ b/server/src/main/java/org/elasticsearch/script/DocValuesDocReader.java @@ -39,7 +39,7 @@ public Field field(String fieldName) { Map> doc = leafLookup.doc(); if (doc.containsKey(fieldName) == false) { - return new EmptyField(fieldName); + return new EmptyField<>(fieldName); } return doc.get(fieldName).toField(fieldName); } diff --git a/server/src/main/java/org/elasticsearch/script/Field.java b/server/src/main/java/org/elasticsearch/script/Field.java index 8d0a659a80ff5..6e8babbc4cca8 100644 --- a/server/src/main/java/org/elasticsearch/script/Field.java +++ b/server/src/main/java/org/elasticsearch/script/Field.java @@ -23,7 +23,6 @@ * * {@code isEmpty() == true} * * {@code getValues().equals(Collections.emptyList())} * * {@code getValue(defaultValue) == defaultValue} - * @param */ public abstract class Field { public static final Converter BigInteger = Converters.BIGINTEGER; diff --git a/x-pack/plugin/mapper-unsigned-long/src/main/java/org/elasticsearch/xpack/unsignedlong/DocValuesWhitelistExtension.java b/x-pack/plugin/mapper-unsigned-long/src/main/java/org/elasticsearch/xpack/unsignedlong/DocValuesWhitelistExtension.java index 219149453d64b..2aa29efaaa5b9 100644 --- a/x-pack/plugin/mapper-unsigned-long/src/main/java/org/elasticsearch/xpack/unsignedlong/DocValuesWhitelistExtension.java +++ b/x-pack/plugin/mapper-unsigned-long/src/main/java/org/elasticsearch/xpack/unsignedlong/DocValuesWhitelistExtension.java @@ -9,17 +9,43 @@ import org.elasticsearch.painless.spi.PainlessExtension; import org.elasticsearch.painless.spi.Whitelist; +import org.elasticsearch.painless.spi.WhitelistLoader; +import org.elasticsearch.script.AggregationScript; +import org.elasticsearch.script.BucketAggregationSelectorScript; +import org.elasticsearch.script.FieldScript; +import org.elasticsearch.script.FilterScript; +import org.elasticsearch.script.NumberSortScript; +import org.elasticsearch.script.ScoreScript; import org.elasticsearch.script.ScriptContext; +import org.elasticsearch.script.StringSortScript; -import java.util.Collections; import java.util.List; import java.util.Map; +import static java.util.Collections.singletonList; + public class DocValuesWhitelistExtension implements PainlessExtension { + private static final Whitelist WHITELIST = WhitelistLoader.loadFromResourceFiles(DocValuesWhitelistExtension.class, "whitelist.txt"); + @Override public Map, List> getContextWhitelists() { - // TODO: support unsigned_long in scripts - return Collections.emptyMap(); + List whitelist = singletonList(WHITELIST); + return Map.of( + FieldScript.CONTEXT, + whitelist, + ScoreScript.CONTEXT, + whitelist, + FilterScript.CONTEXT, + whitelist, + AggregationScript.CONTEXT, + whitelist, + NumberSortScript.CONTEXT, + whitelist, + StringSortScript.CONTEXT, + whitelist, + BucketAggregationSelectorScript.CONTEXT, + whitelist + ); } } diff --git a/x-pack/plugin/mapper-unsigned-long/src/main/java/org/elasticsearch/xpack/unsignedlong/UnsignedLongField.java b/x-pack/plugin/mapper-unsigned-long/src/main/java/org/elasticsearch/xpack/unsignedlong/UnsignedLongField.java index 87663ec3efbe5..69b55bf752230 100644 --- a/x-pack/plugin/mapper-unsigned-long/src/main/java/org/elasticsearch/xpack/unsignedlong/UnsignedLongField.java +++ b/x-pack/plugin/mapper-unsigned-long/src/main/java/org/elasticsearch/xpack/unsignedlong/UnsignedLongField.java @@ -8,10 +8,9 @@ package org.elasticsearch.xpack.unsignedlong; import org.elasticsearch.script.Converter; -import org.elasticsearch.script.DelegatingFieldValues; +import org.elasticsearch.script.Converters; import org.elasticsearch.script.Field; import org.elasticsearch.script.FieldValues; -import org.elasticsearch.script.InvalidConversion; import java.math.BigInteger; import java.util.List; @@ -39,30 +38,9 @@ public > Field as(Converter converter) { return super.as(converter); } - public static class UnsignedLongConverter implements Converter { - @Override - public UnsignedLongField convert(Field sourceField) { - if (sourceField instanceof BigIntegerField) { - return BigIntegerToUnsignedLong((BigIntegerField) sourceField); - } - - throw new InvalidConversion(sourceField.getClass(), getFieldClass()); - } - - @Override - public Class getFieldClass() { - return UnsignedLongField.class; - } - - @Override - public Class getTargetClass() { - return Long.class; - } - } - public static BigIntegerField UnsignedLongToBigInteger(UnsignedLongField sourceField) { FieldValues fv = sourceField.getFieldValues(); - return new BigIntegerField(sourceField.getName(), new DelegatingFieldValues(fv) { + return new BigIntegerField(sourceField.getName(), new Converters.DelegatingFieldValues(fv) { protected BigInteger toBigInteger(long formatted) { return java.math.BigInteger.valueOf(formatted).and(BIGINTEGER_2_64_MINUS_ONE); } @@ -81,7 +59,7 @@ public BigInteger getNonPrimitiveValue() { public static UnsignedLongField BigIntegerToUnsignedLong(BigIntegerField sourceField) { FieldValues fv = sourceField.getFieldValues(); - return new UnsignedLongField(sourceField.getName(), new DelegatingFieldValues<>(fv) { + return new UnsignedLongField(sourceField.getName(), new Converters.DelegatingFieldValues<>(fv) { @Override public List getValues() { return values.getValues().stream().map(java.math.BigInteger::longValue).collect(Collectors.toList()); diff --git a/x-pack/plugin/mapper-unsigned-long/src/main/java/org/elasticsearch/xpack/unsignedlong/UnsignedLongScriptDocValues.java b/x-pack/plugin/mapper-unsigned-long/src/main/java/org/elasticsearch/xpack/unsignedlong/UnsignedLongScriptDocValues.java index 8d2fdd95721d0..42aac528bcfb1 100644 --- a/x-pack/plugin/mapper-unsigned-long/src/main/java/org/elasticsearch/xpack/unsignedlong/UnsignedLongScriptDocValues.java +++ b/x-pack/plugin/mapper-unsigned-long/src/main/java/org/elasticsearch/xpack/unsignedlong/UnsignedLongScriptDocValues.java @@ -65,7 +65,12 @@ public Long get(int index) { * that the underlying value can be treated as a primitive long as that method returns either a {@code long} or a {@code BigInteger}. */ protected long format(int index) { - return values[index] ^ MASK_2_63; + return shiftedLong(values[index]); + } + + // Package private for use in UnsignedLongField + static long shiftedLong(long unshifted) { + return unshifted ^ MASK_2_63; } @Override diff --git a/x-pack/plugin/mapper-unsigned-long/src/main/resources/org/elasticsearch/xpack/unsignedlong/whitelist.txt b/x-pack/plugin/mapper-unsigned-long/src/main/resources/org/elasticsearch/xpack/unsignedlong/whitelist.txt index f0cd5d92b823e..35699791a613f 100644 --- a/x-pack/plugin/mapper-unsigned-long/src/main/resources/org/elasticsearch/xpack/unsignedlong/whitelist.txt +++ b/x-pack/plugin/mapper-unsigned-long/src/main/resources/org/elasticsearch/xpack/unsignedlong/whitelist.txt @@ -1,11 +1,12 @@ # # Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one # or more contributor license agreements. Licensed under the Elastic License -# 2.0; you may not use this file except in compliance with the Elastic License -# 2.0. +# 2.0 and the Server Side Public License, v 1; you may not use this file except +# in compliance with, at your election, the Elastic License 2.0 or the Server +# Side Public License, v 1. # class org.elasticsearch.xpack.unsignedlong.UnsignedLongScriptDocValues { - Number get(int) - Number getValue() + Long get(int) + long getValue() } diff --git a/x-pack/plugin/src/yamlRestTest/resources/rest-api-spec/test/unsigned_long/50_script_values.yml b/x-pack/plugin/src/yamlRestTest/resources/rest-api-spec/test/unsigned_long/50_script_values.yml index 36390b5004ad1..aacf317294580 100644 --- a/x-pack/plugin/src/yamlRestTest/resources/rest-api-spec/test/unsigned_long/50_script_values.yml +++ b/x-pack/plugin/src/yamlRestTest/resources/rest-api-spec/test/unsigned_long/50_script_values.yml @@ -46,6 +46,36 @@ setup: - match: { hits.hits.2.fields.scripted_ul.0: -9223372036854775808 } - match: { hits.hits.3.fields.scripted_ul.0: 9223372036854775807 } - match: { hits.hits.4.fields.scripted_ul.0: 0 } + - do: + search: + index: test1 + body: + sort: [ { ul: desc } ] + script_fields: + scripted_ul: + script: + source: "field('ul').getLong(1000L)" + + - match: { hits.hits.0.fields.scripted_ul.0: -1 } + - match: { hits.hits.1.fields.scripted_ul.0: -3 } + - match: { hits.hits.2.fields.scripted_ul.0: -9223372036854775808 } + - match: { hits.hits.3.fields.scripted_ul.0: 9223372036854775807 } + - match: { hits.hits.4.fields.scripted_ul.0: 0 } + - do: + search: + index: test1 + body: + sort: [ { ul: desc } ] + script_fields: + scripted_ul: + script: + source: "doc['ul'].value" + + - match: { hits.hits.0.fields.scripted_ul.0: -1 } + - match: { hits.hits.1.fields.scripted_ul.0: -3 } + - match: { hits.hits.2.fields.scripted_ul.0: -9223372036854775808 } + - match: { hits.hits.3.fields.scripted_ul.0: 9223372036854775807 } + - match: { hits.hits.4.fields.scripted_ul.0: 0 } --- "Scripted sort values": @@ -65,6 +95,41 @@ setup: - match: { hits.hits.2.sort: [9.223372036854776E18] } - match: { hits.hits.3.sort: [9.223372036854776E18] } - match: { hits.hits.4.sort: [0.0] } + - do: + search: + index: test1 + body: + sort: + _script: + order: desc + type: number + script: + source: "doc['ul'].value" + + + - match: { hits.hits.0.sort: [9.223372036854776E18] } + - match: { hits.hits.1.sort: [0.0] } + - match: { hits.hits.2.sort: [-1.0] } + - match: { hits.hits.3.sort: [-3.0] } + - match: { hits.hits.4.sort: [-9.223372036854776E18] } +--- +"Scripted sort values via doc": + - do: + search: + index: test1 + body: + sort: + _script: + order: desc + type: number + script: + source: "doc['ul'].value" + + - match: { hits.hits.0.sort: [9.223372036854776E18] } + - match: { hits.hits.1.sort: [0.0] } + - match: { hits.hits.2.sort: [-1.0] } + - match: { hits.hits.3.sort: [-3.0] } + - match: { hits.hits.4.sort: [-9.223372036854776E18] } --- "Script query": @@ -95,6 +160,31 @@ setup: script: source: "field('ul').isEmpty() == false" - match: { hits.total.value: 5 } + - do: + search: + index: test1 + body: + query: + bool: + filter: + script: + script: + source: "doc['ul'].value.doubleValue() > 9E18" + - match: { hits.total.value: 1 } + - match: { hits.hits.0._id: "2" } + + - do: + search: + index: test1 + body: + size: 0 + query: + bool: + filter: + script: + script: + source: "doc['ul'].size() > 0" + - match: { hits.total.value: 5 } --- "script_score query": @@ -109,3 +199,14 @@ setup: source: "field('ul').as(Field.BigInteger).getValue(BigInteger.valueOf(Long.parseUnsignedLong('18446744073709551614'))).doubleValue()" - match: { hits.total.value: 5 } + - do: + search: + index: test1 + body: + query: + script_score: + query: {match_all: {}} + script: + source: "Math.abs(doc['ul'].value)" + + - match: { hits.total.value: 5 } From aa52c0051c2e1a7687a8c9843918eb9b3a5954be Mon Sep 17 00:00:00 2001 From: Stuart Tettemer Date: Mon, 16 Aug 2021 14:17:30 -0500 Subject: [PATCH 06/14] remove date tests --- .../test/painless/40_fields_api.yml | 80 ------------------- 1 file changed, 80 deletions(-) diff --git a/modules/lang-painless/src/yamlRestTest/resources/rest-api-spec/test/painless/40_fields_api.yml b/modules/lang-painless/src/yamlRestTest/resources/rest-api-spec/test/painless/40_fields_api.yml index 346c1ebfbbd4a..099d78b82c7f2 100644 --- a/modules/lang-painless/src/yamlRestTest/resources/rest-api-spec/test/painless/40_fields_api.yml +++ b/modules/lang-painless/src/yamlRestTest/resources/rest-api-spec/test/painless/40_fields_api.yml @@ -224,83 +224,3 @@ setup: - match: { hits.hits.1.fields.missing_field.0: -9223372036854775809 } - match: { hits.hits.2._id: d3 } - match: { hits.hits.2.fields.missing_field.0: 12 } ---- -"date to long": - - do: - indices.create: - index: test - body: - settings: - number_of_shards: 2 - mappings: - properties: - dval: - type: double - - do: - index: - index: test - id: d1 - body: { "dval": 101, "dt": "2015-01-01T12:10:30Z" } - - do: - index: - index: test - id: d2 - body: { "dval": 202, "dt": 1420070400001 } - - do: - index: - index: test - id: d3 - body: { "dval": 303, "dt": "2015-01-01" } - - do: - index: - index: test - id: d4 - body: { "dval": 404, "dtnanos": "2015-01-01T12:10:30.123456789Z"} - - do: - index: - index: test - id: d5 - body: { "dval": 505, "dtnanos": 1420070412345 } - - do: - index: - index: test - id: d6 - body:{ "dval": 606 } - - do: - indices.refresh: {} - - do: - search: - rest_total_hits_as_int: true - index: test - body: - query: { "match_all": {} } - sort: [ { dval: asc } ] - script_fields: - missing_field: - script: - source: "field('foo').as(Field.BigInteger).getValue(BigInteger.TEN).add(BigInteger.ONE)" - - match: { hits.total: 3 } - - match: { hits.hits.0._id: d1 } - - match: { hits.hits.0.fields.missing_field.0: 26 } - - match: { hits.hits.1._id: d2 } - - match: { hits.hits.1.fields.missing_field.0: 9223372036854775808 } - - match: { hits.hits.2._id: d3 } - - match: { hits.hits.2.fields.missing_field.0: 11 } - - do: - search: - rest_total_hits_as_int: true - index: test - body: - query: { "match_all": {} } - sort: [ { dval: asc } ] - script_fields: - missing_field: - script: - source: "field('foo').as(Field.BigInteger).as(Field.Long).getValue(10L) + 2L" - - match: { hits.total: 3 } - - match: { hits.hits.0._id: d1 } - - match: { hits.hits.0.fields.missing_field.0: 27 } - - match: { hits.hits.1._id: d2 } - - match: { hits.hits.1.fields.missing_field.0: -9223372036854775809 } - - match: { hits.hits.2._id: d3 } - - match: { hits.hits.2.fields.missing_field.0: 12 } From 2b1dc8a754d0ac94656be9415e6a9c370c4b9f02 Mon Sep 17 00:00:00 2001 From: Stuart Tettemer Date: Mon, 16 Aug 2021 18:20:27 -0500 Subject: [PATCH 07/14] date milli and nano tests --- .../test/painless/40_fields_api.yml | 144 ++++++++++++++++++ .../index/fielddata/ScriptDocValues.java | 3 +- .../org/elasticsearch/script/Converters.java | 12 ++ 3 files changed, 158 insertions(+), 1 deletion(-) diff --git a/modules/lang-painless/src/yamlRestTest/resources/rest-api-spec/test/painless/40_fields_api.yml b/modules/lang-painless/src/yamlRestTest/resources/rest-api-spec/test/painless/40_fields_api.yml index 099d78b82c7f2..cffd38f59a385 100644 --- a/modules/lang-painless/src/yamlRestTest/resources/rest-api-spec/test/painless/40_fields_api.yml +++ b/modules/lang-painless/src/yamlRestTest/resources/rest-api-spec/test/painless/40_fields_api.yml @@ -224,3 +224,147 @@ setup: - match: { hits.hits.1.fields.missing_field.0: -9223372036854775809 } - match: { hits.hits.2._id: d3 } - match: { hits.hits.2.fields.missing_field.0: 12 } +--- +"date to long": + - do: + indices.create: + index: test + body: + settings: + number_of_shards: 2 + mappings: + properties: + dval: + type: double + dtnanos: + type: date_nanos + dt: + type: date + - do: + index: + index: test + id: d1 + body: { "dval": 101, "dt": "2015-01-01T12:10:40Z" } + - do: + index: + index: test + id: d2 + body: { "dval": 202, "dt": "2015-01-01" } + - do: + index: + index: test + id: d3 + body: { "dval": 303, "dt": 1420072496000 } + - do: + index: + index: test + id: d4 + body: { "dval": 404, "dtnanos": "2015-01-01T12:10:30.123456789Z"} + - do: + index: + index: test + id: d5 + body: { "dval": 505, "dtnanos": "2261-04-11T23:47:33.54775807Z" } + - do: + index: + index: test + id: d6 + body: { "dval": 606 } + - do: + indices.refresh: {} + - do: + search: + rest_total_hits_as_int: true + index: test + body: + query: { "match_all": {} } + sort: [ { dval: asc } ] + script_fields: + dt_bigint: + script: + source: "field('dt').as(Field.BigInteger).getValue(BigInteger.TEN).add(BigInteger.ONE)" + - match: { hits.total: 6 } + - match: { hits.hits.0._id: d1 } + - match: { hits.hits.0.fields.dt_bigint.0: 1420114240001 } + - match: { hits.hits.1._id: d2 } + - match: { hits.hits.1.fields.dt_bigint.0: 1420070400001 } + - match: { hits.hits.2._id: d3 } + - match: { hits.hits.2.fields.dt_bigint.0: 1420072496001 } + - match: { hits.hits.3._id: d4 } + - match: { hits.hits.3.fields.dt_bigint.0: 11 } + - match: { hits.hits.4._id: d5 } + - match: { hits.hits.4.fields.dt_bigint.0: 11 } + - match: { hits.hits.5._id: d6 } + - match: { hits.hits.5.fields.dt_bigint.0: 11 } + - do: + search: + rest_total_hits_as_int: true + index: test + body: + query: { "match_all": {} } + sort: [ { dval: asc } ] + script_fields: + dt_long: + script: + source: "field('dt').as(Field.Long).getValue(10L) + 2L" + - match: { hits.total: 6 } + - match: { hits.hits.0._id: d1 } + - match: { hits.hits.0.fields.dt_long.0: 1420114240002 } + - match: { hits.hits.1._id: d2 } + - match: { hits.hits.1.fields.dt_long.0: 1420070400002 } + - match: { hits.hits.2._id: d3 } + - match: { hits.hits.2.fields.dt_long.0: 1420072496002 } + - match: { hits.hits.3._id: d4 } + - match: { hits.hits.3.fields.dt_long.0: 12 } + - match: { hits.hits.4._id: d5 } + - match: { hits.hits.4.fields.dt_long.0: 12 } + - match: { hits.hits.5._id: d6 } + - match: { hits.hits.5.fields.dt_long.0: 12 } + - do: + search: + rest_total_hits_as_int: true + index: test + body: + query: { "match_all": {} } + sort: [ { dval: asc } ] + script_fields: + dtnan_bigint: + script: + source: "field('dtnanos').as(Field.BigInteger).getValue(BigInteger.TEN).add(BigInteger.ONE)" + - match: { hits.total: 6 } + - match: { hits.hits.0._id: d1 } + - match: { hits.hits.0.fields.dtnan_bigint.0: 11 } + - match: { hits.hits.1._id: d2 } + - match: { hits.hits.1.fields.dtnan_bigint.0: 11 } + - match: { hits.hits.2._id: d3 } + - match: { hits.hits.2.fields.dtnan_bigint.0: 11 } + - match: { hits.hits.3._id: d4 } + - match: { hits.hits.3.fields.dtnan_bigint.0: 1420114230123456790 } + - match: { hits.hits.4._id: d5 } + - match: { hits.hits.4.fields.dtnan_bigint.0: 9191836053547758071 } + - match: { hits.hits.5._id: d6 } + - match: { hits.hits.5.fields.dtnan_bigint.0: 11 } + - do: + search: + rest_total_hits_as_int: true + index: test + body: + query: { "match_all": {} } + sort: [ { dval: asc } ] + script_fields: + dtnan_long: + script: + source: "field('dtnanos').as(Field.Long).getValue(field('dt').as(Field.Long).getValue(0L) * 1000) + 2L" + - match: { hits.total: 6 } + - match: { hits.hits.0._id: d1 } + - match: { hits.hits.0.fields.dtnan_long.0: 1420114240000002 } + - match: { hits.hits.1._id: d2 } + - match: { hits.hits.1.fields.dtnan_long.0: 1420070400000002 } + - match: { hits.hits.2._id: d3 } + - match: { hits.hits.2.fields.dtnan_long.0: 1420072496000002 } + - match: { hits.hits.3._id: d4 } + - match: { hits.hits.3.fields.dtnan_long.0: 1420114230123456791 } + - match: { hits.hits.4._id: d5 } + - match: { hits.hits.4.fields.dtnan_long.0: 9191836053547758072 } + - match: { hits.hits.5._id: d6 } + - match: { hits.hits.5.fields.dtnan_long.0: 2 } diff --git a/server/src/main/java/org/elasticsearch/index/fielddata/ScriptDocValues.java b/server/src/main/java/org/elasticsearch/index/fielddata/ScriptDocValues.java index bd0eafae0a6f5..37b1efac753fa 100644 --- a/server/src/main/java/org/elasticsearch/index/fielddata/ScriptDocValues.java +++ b/server/src/main/java/org/elasticsearch/index/fielddata/ScriptDocValues.java @@ -25,6 +25,7 @@ import java.io.IOException; import java.time.Instant; import java.time.ZoneOffset; +import java.time.temporal.ChronoUnit; import java.util.AbstractList; import java.util.Arrays; import java.util.Comparator; @@ -242,7 +243,7 @@ public long getLongValue() { throwIfEmpty(); Instant dt = dates[0].toInstant(); if (isNanos) { - return TimeUnit.SECONDS.toNanos(dt.getEpochSecond()) + dt.getNano(); + return ChronoUnit.NANOS.between(java.time.Instant.EPOCH, dt); } return dt.toEpochMilli(); } diff --git a/server/src/main/java/org/elasticsearch/script/Converters.java b/server/src/main/java/org/elasticsearch/script/Converters.java index c370567412312..e22e8484a339c 100644 --- a/server/src/main/java/org/elasticsearch/script/Converters.java +++ b/server/src/main/java/org/elasticsearch/script/Converters.java @@ -60,6 +60,18 @@ public BigIntegerField convert(Field sourceField) { return StringToBigInteger((StringField) sourceField); } + if (sourceField instanceof DateMillisField) { + return LongToBigInteger(DateMillisToLong((DateMillisField) sourceField)); + } + + if (sourceField instanceof DateNanosField) { + return LongToBigInteger(DateNanosToLong((DateNanosField) sourceField)); + } + + if (sourceField instanceof BooleanField) { + return LongToBigInteger(BooleanToLong((BooleanField) sourceField)); + } + throw new InvalidConversion(sourceField.getClass(), getFieldClass()); } From 69ec48d48f1c8a5a6270d27957863f9ff91952b4 Mon Sep 17 00:00:00 2001 From: Stuart Tettemer Date: Mon, 16 Aug 2021 18:26:25 -0500 Subject: [PATCH 08/14] remove unused TimeUnit --- .../java/org/elasticsearch/index/fielddata/ScriptDocValues.java | 1 - 1 file changed, 1 deletion(-) diff --git a/server/src/main/java/org/elasticsearch/index/fielddata/ScriptDocValues.java b/server/src/main/java/org/elasticsearch/index/fielddata/ScriptDocValues.java index 37b1efac753fa..dff93fa777402 100644 --- a/server/src/main/java/org/elasticsearch/index/fielddata/ScriptDocValues.java +++ b/server/src/main/java/org/elasticsearch/index/fielddata/ScriptDocValues.java @@ -30,7 +30,6 @@ import java.util.Arrays; import java.util.Comparator; import java.util.List; -import java.util.concurrent.TimeUnit; import java.util.function.UnaryOperator; /** From b5a29654214d37b3b1231b3d06a70b7c4a4764fc Mon Sep 17 00:00:00 2001 From: Stuart Tettemer Date: Mon, 16 Aug 2021 19:38:15 -0500 Subject: [PATCH 09/14] lenient big integer parsing --- .../test/painless/40_fields_api.yml | 190 ++++++++++++++++++ .../index/fielddata/ScriptDocValues.java | 5 +- .../org/elasticsearch/script/Converters.java | 35 ++-- .../java/org/elasticsearch/script/Field.java | 18 +- 4 files changed, 231 insertions(+), 17 deletions(-) diff --git a/modules/lang-painless/src/yamlRestTest/resources/rest-api-spec/test/painless/40_fields_api.yml b/modules/lang-painless/src/yamlRestTest/resources/rest-api-spec/test/painless/40_fields_api.yml index cffd38f59a385..e70b9f26ed0c4 100644 --- a/modules/lang-painless/src/yamlRestTest/resources/rest-api-spec/test/painless/40_fields_api.yml +++ b/modules/lang-painless/src/yamlRestTest/resources/rest-api-spec/test/painless/40_fields_api.yml @@ -224,6 +224,7 @@ setup: - match: { hits.hits.1.fields.missing_field.0: -9223372036854775809 } - match: { hits.hits.2._id: d3 } - match: { hits.hits.2.fields.missing_field.0: 12 } + --- "date to long": - do: @@ -368,3 +369,192 @@ setup: - match: { hits.hits.4.fields.dtnan_long.0: 9191836053547758072 } - match: { hits.hits.5._id: d6 } - match: { hits.hits.5.fields.dtnan_long.0: 2 } + +--- +"boolean to long and bigint": + - do: + indices.create: + index: test + body: + settings: + number_of_shards: 2 + mappings: + properties: + dval: + type: double + bool: + type: boolean + - do: + index: + index: test + id: d1 + body: { "dval": 101, "bool": true } + - do: + index: + index: test + id: d2 + body: { "dval": 202, "bool": false } + - do: + index: + index: test + id: d3 + body: { "dval": 303 } + - do: + indices.refresh: {} + - do: + search: + rest_total_hits_as_int: true + index: test + body: + query: { "match_all": {} } + sort: [ { dval: asc } ] + script_fields: + bool: + script: + source: "field('bool').as(Field.Long).getValue(-1)" + - match: { hits.total: 3 } + - match: { hits.hits.0._id: d1 } + - match: { hits.hits.0.fields.bool.0: 1 } + - match: { hits.hits.1._id: d2 } + - match: { hits.hits.1.fields.bool.0: 0 } + - match: { hits.hits.2._id: d3 } + - match: { hits.hits.2.fields.bool.0: -1 } + - do: + search: + rest_total_hits_as_int: true + index: test + body: + query: { "match_all": {} } + sort: [ { dval: asc } ] + script_fields: + bool: + script: + source: "field('bool').as(Field.BigInteger).getValue(-1)" + - match: { hits.total: 3 } + - match: { hits.hits.0._id: d1 } + - match: { hits.hits.0.fields.bool.0: 1 } + - match: { hits.hits.1._id: d2 } + - match: { hits.hits.1.fields.bool.0: 0 } + - match: { hits.hits.2._id: d3 } + - match: { hits.hits.2.fields.bool.0: -1 } +--- +"string to long and bigint": + - do: + indices.create: + index: test + body: + settings: + number_of_shards: 2 + mappings: + properties: + dval: + type: double + str: + type: keyword + - do: + index: + index: test + id: d1 + body: { "dval": 101, "str": "18446744073709551618" } + - do: + index: + index: test + id: d2 + body: { "dval": 202, "str": "9223372036854775809" } + - do: + index: + index: test + id: d3 + body: { "dval": 303, "str": " '-1' " } + - do: + index: + index: test + id: d4 + body: { "dval": 404 } + - do: + index: + index: test + id: d5 + body: { "dval": 505, "str": "unparseable number" } + - do: + index: + index: test + id: d6 + body: { "dval": 606, "str": "10E40" } + - do: + index: + index: test + id: d7 + body: { "dval": 707, "str": "18446744073709551618.555" } + - do: + indices.refresh: {} + - do: + search: + rest_total_hits_as_int: true + index: test + body: + query: { "match_all": {} } + sort: [ { dval: asc } ] + script_fields: + str: + script: + source: "field('str').as(Field.Long).getValue(-20)" + - match: { hits.total: 7 } + - match: { hits.hits.0._id: d1 } + - match: { hits.hits.0.fields.str.0: 9223372036854775807 } + - match: { hits.hits.1._id: d2 } + - match: { hits.hits.1.fields.str.0: 9223372036854775807 } + - match: { hits.hits.2._id: d3 } + - match: { hits.hits.2.fields.str.0: -1 } + - match: { hits.hits.3._id: d4 } + - match: { hits.hits.3.fields.str.0: -20 } + - match: { hits.hits.4._id: d5 } + - match: { hits.hits.4.fields.str.0: -20 } + - do: + search: + rest_total_hits_as_int: true + index: test + body: + query: { "match_all": {} } + sort: [ { dval: asc } ] + script_fields: + str: + script: + source: "field('str').getLong(-40)" + - match: { hits.total: 7 } + - match: { hits.hits.0._id: d1 } + - match: { hits.hits.0.fields.str.0: -40 } + - match: { hits.hits.1._id: d2 } + - match: { hits.hits.1.fields.str.0: -40 } + - match: { hits.hits.2._id: d3 } + - match: { hits.hits.2.fields.str.0: -1 } + - match: { hits.hits.3._id: d4 } + - match: { hits.hits.3.fields.str.0: -40 } + - match: { hits.hits.4._id: d5 } + - match: { hits.hits.4.fields.str.0: -40 } + - do: + search: + rest_total_hits_as_int: true + index: test + body: + query: { "match_all": {} } + sort: [ { dval: asc } ] + script_fields: + str: + script: + source: "field('str').as(Field.BigInteger).getValue(BigInteger.TEN)" + - match: { hits.total: 7 } + - match: { hits.hits.0._id: d1 } + - match: { hits.hits.0.fields.str.0: 18446744073709551618 } + - match: { hits.hits.1._id: d2 } + - match: { hits.hits.1.fields.str.0: 9223372036854775809 } + - match: { hits.hits.2._id: d3 } + - match: { hits.hits.2.fields.str.0: -1 } + - match: { hits.hits.3._id: d4 } + - match: { hits.hits.3.fields.str.0: 10 } + - match: { hits.hits.4._id: d5 } + - match: { hits.hits.4.fields.str.0: 10 } + - match: { hits.hits.5._id: d6 } + - match: { hits.hits.5.fields.str.0: 10 } + - match: { hits.hits.6._id: d7 } + - match: { hits.hits.6.fields.str.0: 18446744073709551618 } diff --git a/server/src/main/java/org/elasticsearch/index/fielddata/ScriptDocValues.java b/server/src/main/java/org/elasticsearch/index/fielddata/ScriptDocValues.java index dff93fa777402..a8073a87837e0 100644 --- a/server/src/main/java/org/elasticsearch/index/fielddata/ScriptDocValues.java +++ b/server/src/main/java/org/elasticsearch/index/fielddata/ScriptDocValues.java @@ -17,6 +17,7 @@ import org.elasticsearch.common.geo.GeoUtils; import org.elasticsearch.common.time.DateUtils; import org.elasticsearch.geometry.utils.Geohash; +import org.elasticsearch.script.Converters; import org.elasticsearch.script.Field; import org.elasticsearch.script.FieldValues; import org.elasticsearch.script.InvalidConversion; @@ -675,12 +676,12 @@ public final String getValue() { @Override public long getLongValue() { - return Long.parseLong(get(0)); + return Long.parseLong(Converters.trimNumber(get(0))); } @Override public double getDoubleValue() { - return Double.parseDouble(get(0)); + return Double.parseDouble(Converters.trimNumber(get(0))); } @Override diff --git a/server/src/main/java/org/elasticsearch/script/Converters.java b/server/src/main/java/org/elasticsearch/script/Converters.java index e22e8484a339c..04ba0e934666d 100644 --- a/server/src/main/java/org/elasticsearch/script/Converters.java +++ b/server/src/main/java/org/elasticsearch/script/Converters.java @@ -166,24 +166,25 @@ public java.math.BigInteger getNonPrimitiveValue() { public static BigIntegerField StringToBigInteger(StringField sourceField) { FieldValues fv = sourceField.getFieldValues(); return new BigIntegerField(sourceField.getName(), new DelegatingFieldValues(fv) { - protected long parseNumber(String str) { - try { - return Long.parseLong(str); - } catch (NumberFormatException err) { - return (long) Double.parseDouble(str); + protected BigInteger parseNumber(String str) { + String trimmed = trimNumber(str); + int decimal = trimmed.indexOf("."); + if (decimal >= 0) { + trimmed = trimmed.substring(0, decimal); } + // TODO(stu): should we use Double.valueOf if this throws, Double.valueOf accepts many more formats + return new BigInteger(trimmed); } @Override public List getValues() { - return values.getValues().stream().map( - str -> java.math.BigInteger.valueOf(parseNumber(str)) - ).collect(Collectors.toList()); + // TODO(stu): this may throw + return values.getValues().stream().map(this::parseNumber).collect(Collectors.toList()); } @Override public java.math.BigInteger getNonPrimitiveValue() { - return java.math.BigInteger.valueOf(values.getLongValue()); + return parseNumber(values.getNonPrimitiveValue()); } }); } @@ -271,10 +272,11 @@ public static LongField StringToLong(StringField sourceField) { FieldValues fv = sourceField.getFieldValues(); return new LongField(sourceField.getName(), new DelegatingFieldValues(fv) { protected long parseNumber(String str) { + String trimmed = trimNumber(str); try { - return Long.parseLong(str); + return Long.parseLong(trimmed); } catch (NumberFormatException err) { - return (long) Double.parseDouble(str); + return (long) Double.parseDouble(trimmed); } } @@ -295,7 +297,7 @@ public long getLongValue() { @Override public double getDoubleValue() { - String str = values.getNonPrimitiveValue(); + String str = trimNumber(values.getNonPrimitiveValue()); try { return Double.parseDouble(str); } catch (NumberFormatException err) { @@ -305,6 +307,15 @@ public double getDoubleValue() { }); } + /** + * Trim common leading and trailing values for strings, double quote, single quote, tab, new line and space. + * + * This avoids NumberFormatException when attempting to parse numbers from Strings for these common cases. + */ + public static String trimNumber(String maybeNumber) { + return maybeNumber.replaceAll("^[\"' \t\n]*|[\"' \t\n]*$", ""); + } + /** * Helper for creating {@link Converter} classes which delegates all un-overridden methods to the underlying * {@link FieldValues}. diff --git a/server/src/main/java/org/elasticsearch/script/Field.java b/server/src/main/java/org/elasticsearch/script/Field.java index 6e8babbc4cca8..43ecd1170a67e 100644 --- a/server/src/main/java/org/elasticsearch/script/Field.java +++ b/server/src/main/java/org/elasticsearch/script/Field.java @@ -83,7 +83,11 @@ public T getValue(T defaultValue) { if (isEmpty()) { return defaultValue; } - return values.getNonPrimitiveValue(); + try { + return values.getNonPrimitiveValue(); + } catch (RuntimeException err) { + return defaultValue; + } } /** @@ -94,7 +98,11 @@ public double getDouble(double defaultValue) { if (isEmpty()) { return defaultValue; } - return values.getDoubleValue(); + try { + return values.getDoubleValue(); + } catch (RuntimeException err) { + return defaultValue; + } } @@ -106,7 +114,11 @@ public long getLong(long defaultValue) { if (isEmpty()) { return defaultValue; } - return values.getLongValue(); + try { + return values.getLongValue(); + } catch (RuntimeException err) { + return defaultValue; + } } public static class BooleanField extends Field { From aef03a1e6bf47009abbcacde4ed01fa97c3802ed Mon Sep 17 00:00:00 2001 From: Stuart Tettemer Date: Mon, 16 Aug 2021 19:49:03 -0500 Subject: [PATCH 10/14] revert number parsing fanciness --- .../test/painless/40_fields_api.yml | 8 ++-- .../index/fielddata/ScriptDocValues.java | 5 +-- .../org/elasticsearch/script/Converters.java | 37 +++---------------- 3 files changed, 11 insertions(+), 39 deletions(-) diff --git a/modules/lang-painless/src/yamlRestTest/resources/rest-api-spec/test/painless/40_fields_api.yml b/modules/lang-painless/src/yamlRestTest/resources/rest-api-spec/test/painless/40_fields_api.yml index e70b9f26ed0c4..60e0b33a05575 100644 --- a/modules/lang-painless/src/yamlRestTest/resources/rest-api-spec/test/painless/40_fields_api.yml +++ b/modules/lang-painless/src/yamlRestTest/resources/rest-api-spec/test/painless/40_fields_api.yml @@ -465,7 +465,7 @@ setup: index: index: test id: d3 - body: { "dval": 303, "str": " '-1' " } + body: { "dval": 303, "str": "-1" } - do: index: index: test @@ -501,9 +501,9 @@ setup: source: "field('str').as(Field.Long).getValue(-20)" - match: { hits.total: 7 } - match: { hits.hits.0._id: d1 } - - match: { hits.hits.0.fields.str.0: 9223372036854775807 } + - match: { hits.hits.0.fields.str.0: -20 } - match: { hits.hits.1._id: d2 } - - match: { hits.hits.1.fields.str.0: 9223372036854775807 } + - match: { hits.hits.1.fields.str.0: -20 } - match: { hits.hits.2._id: d3 } - match: { hits.hits.2.fields.str.0: -1 } - match: { hits.hits.3._id: d4 } @@ -557,4 +557,4 @@ setup: - match: { hits.hits.5._id: d6 } - match: { hits.hits.5.fields.str.0: 10 } - match: { hits.hits.6._id: d7 } - - match: { hits.hits.6.fields.str.0: 18446744073709551618 } + - match: { hits.hits.6.fields.str.0: 10 } diff --git a/server/src/main/java/org/elasticsearch/index/fielddata/ScriptDocValues.java b/server/src/main/java/org/elasticsearch/index/fielddata/ScriptDocValues.java index a8073a87837e0..dff93fa777402 100644 --- a/server/src/main/java/org/elasticsearch/index/fielddata/ScriptDocValues.java +++ b/server/src/main/java/org/elasticsearch/index/fielddata/ScriptDocValues.java @@ -17,7 +17,6 @@ import org.elasticsearch.common.geo.GeoUtils; import org.elasticsearch.common.time.DateUtils; import org.elasticsearch.geometry.utils.Geohash; -import org.elasticsearch.script.Converters; import org.elasticsearch.script.Field; import org.elasticsearch.script.FieldValues; import org.elasticsearch.script.InvalidConversion; @@ -676,12 +675,12 @@ public final String getValue() { @Override public long getLongValue() { - return Long.parseLong(Converters.trimNumber(get(0))); + return Long.parseLong(get(0)); } @Override public double getDoubleValue() { - return Double.parseDouble(Converters.trimNumber(get(0))); + return Double.parseDouble(get(0)); } @Override diff --git a/server/src/main/java/org/elasticsearch/script/Converters.java b/server/src/main/java/org/elasticsearch/script/Converters.java index 04ba0e934666d..5411cbb3dc5fc 100644 --- a/server/src/main/java/org/elasticsearch/script/Converters.java +++ b/server/src/main/java/org/elasticsearch/script/Converters.java @@ -166,25 +166,16 @@ public java.math.BigInteger getNonPrimitiveValue() { public static BigIntegerField StringToBigInteger(StringField sourceField) { FieldValues fv = sourceField.getFieldValues(); return new BigIntegerField(sourceField.getName(), new DelegatingFieldValues(fv) { - protected BigInteger parseNumber(String str) { - String trimmed = trimNumber(str); - int decimal = trimmed.indexOf("."); - if (decimal >= 0) { - trimmed = trimmed.substring(0, decimal); - } - // TODO(stu): should we use Double.valueOf if this throws, Double.valueOf accepts many more formats - return new BigInteger(trimmed); - } @Override public List getValues() { // TODO(stu): this may throw - return values.getValues().stream().map(this::parseNumber).collect(Collectors.toList()); + return values.getValues().stream().map(BigInteger::new).collect(Collectors.toList()); } @Override public java.math.BigInteger getNonPrimitiveValue() { - return parseNumber(values.getNonPrimitiveValue()); + return new BigInteger(values.getNonPrimitiveValue()); } }); } @@ -272,12 +263,7 @@ public static LongField StringToLong(StringField sourceField) { FieldValues fv = sourceField.getFieldValues(); return new LongField(sourceField.getName(), new DelegatingFieldValues(fv) { protected long parseNumber(String str) { - String trimmed = trimNumber(str); - try { - return Long.parseLong(trimmed); - } catch (NumberFormatException err) { - return (long) Double.parseDouble(trimmed); - } + return Long.parseLong(str); } @Override @@ -297,25 +283,12 @@ public long getLongValue() { @Override public double getDoubleValue() { - String str = trimNumber(values.getNonPrimitiveValue()); - try { - return Double.parseDouble(str); - } catch (NumberFormatException err) { - return Long.parseLong(str); - } + String str = values.getNonPrimitiveValue(); + return Double.parseDouble(str); } }); } - /** - * Trim common leading and trailing values for strings, double quote, single quote, tab, new line and space. - * - * This avoids NumberFormatException when attempting to parse numbers from Strings for these common cases. - */ - public static String trimNumber(String maybeNumber) { - return maybeNumber.replaceAll("^[\"' \t\n]*|[\"' \t\n]*$", ""); - } - /** * Helper for creating {@link Converter} classes which delegates all un-overridden methods to the underlying * {@link FieldValues}. From e60fe364bcf02095a33f1bdcd0147ff4d7732c72 Mon Sep 17 00:00:00 2001 From: Stuart Tettemer Date: Tue, 17 Aug 2021 07:58:20 -0500 Subject: [PATCH 11/14] pr comments --- .../test/painless/40_fields_api.yml | 4 +- .../org/elasticsearch/script/Converter.java | 12 +-- .../org/elasticsearch/script/Converters.java | 77 ++++++++++--------- .../org/elasticsearch/script/EmptyField.java | 2 +- .../java/org/elasticsearch/script/Field.java | 16 ++-- .../xpack/unsignedlong/UnsignedLongField.java | 23 +----- 6 files changed, 63 insertions(+), 71 deletions(-) diff --git a/modules/lang-painless/src/yamlRestTest/resources/rest-api-spec/test/painless/40_fields_api.yml b/modules/lang-painless/src/yamlRestTest/resources/rest-api-spec/test/painless/40_fields_api.yml index 60e0b33a05575..df571349ecb97 100644 --- a/modules/lang-painless/src/yamlRestTest/resources/rest-api-spec/test/painless/40_fields_api.yml +++ b/modules/lang-painless/src/yamlRestTest/resources/rest-api-spec/test/painless/40_fields_api.yml @@ -555,6 +555,6 @@ setup: - match: { hits.hits.4._id: d5 } - match: { hits.hits.4.fields.str.0: 10 } - match: { hits.hits.5._id: d6 } - - match: { hits.hits.5.fields.str.0: 10 } + - match: { hits.hits.5.fields.str.0: 100000000000000000000000000000000000000000 } - match: { hits.hits.6._id: d7 } - - match: { hits.hits.6.fields.str.0: 10 } + - match: { hits.hits.6.fields.str.0: 18446744073709551618 } diff --git a/server/src/main/java/org/elasticsearch/script/Converter.java b/server/src/main/java/org/elasticsearch/script/Converter.java index b87e4a57a2ac9..f903ebfbedb76 100644 --- a/server/src/main/java/org/elasticsearch/script/Converter.java +++ b/server/src/main/java/org/elasticsearch/script/Converter.java @@ -9,23 +9,23 @@ package org.elasticsearch.script; /** - * Converts between one scripting {@link Field} type and another, {@code CF}, with a different underlying - * value type, {@code CT}. + * Converts between one scripting {@link Field} type and another, {@code FC}, with a different underlying + * value type, {@code TC}. */ -public interface Converter> { +public interface Converter> { /** * Convert {@code sourceField} to a new field-type. Conversions come from user scripts so {@code covert} may * be called on a {@link Field}'s own type. */ - CF convert(Field sourceField); + FC convert(Field sourceField); /** * The destination {@link Field} class. */ - Class getFieldClass(); + Class getFieldClass(); /** * The target value type. */ - Class getTargetClass(); + Class getTargetClass(); } diff --git a/server/src/main/java/org/elasticsearch/script/Converters.java b/server/src/main/java/org/elasticsearch/script/Converters.java index 5411cbb3dc5fc..fa96c4dc5b827 100644 --- a/server/src/main/java/org/elasticsearch/script/Converters.java +++ b/server/src/main/java/org/elasticsearch/script/Converters.java @@ -8,7 +8,10 @@ package org.elasticsearch.script; +import java.math.BigDecimal; import java.math.BigInteger; + +import java.time.Instant; import java.time.temporal.ChronoUnit; import java.util.List; import java.util.stream.Collectors; @@ -31,7 +34,7 @@ public class Converters { * Longs and Doubles are wrapped as BigIntegers. * Strings are parsed as either Longs or Doubles and wrapped in a BigInteger. */ - public static final Converter BIGINTEGER; + static final Converter BIGINTEGER; /** * Convert to a {@link LongField} from Double, String, DateMillis, DateNanos, BigInteger or Boolean Fields. @@ -42,7 +45,7 @@ public class Converters { * {@link BigInteger#longValue()} is used for the BigInteger conversion. * Boolean is {@code 1L} if {@code true}, {@code 0L} if {@code false}. */ - public static final Converter LONG; + static final Converter LONG; static { BIGINTEGER = new Converter<>() { @@ -81,8 +84,8 @@ public Class getFieldClass() { } @Override - public Class getTargetClass() { - return java.math.BigInteger.class; + public Class getTargetClass() { + return BigInteger.class; } }; @@ -131,56 +134,63 @@ public Class getTargetClass() { // No instances, please private Converters() {} - public static BigIntegerField LongToBigInteger(LongField sourceField) { + static BigIntegerField LongToBigInteger(LongField sourceField) { FieldValues fv = sourceField.getFieldValues(); return new BigIntegerField(sourceField.getName(), new DelegatingFieldValues<>(fv) { @Override - public List getValues() { - return values.getValues().stream().map(java.math.BigInteger::valueOf).collect(Collectors.toList()); + public List getValues() { + return values.getValues().stream().map(BigInteger::valueOf).collect(Collectors.toList()); } @Override - public java.math.BigInteger getNonPrimitiveValue() { - return java.math.BigInteger.valueOf(values.getLongValue()); + public BigInteger getNonPrimitiveValue() { + return BigInteger.valueOf(values.getLongValue()); } }); } - public static BigIntegerField DoubleToBigInteger(DoubleField sourceField) { + static BigIntegerField DoubleToBigInteger(DoubleField sourceField) { FieldValues fv = sourceField.getFieldValues(); return new BigIntegerField(sourceField.getName(), new DelegatingFieldValues<>(fv) { @Override - public List getValues() { + public List getValues() { return values.getValues().stream().map( - dbl -> java.math.BigInteger.valueOf(dbl.longValue()) + dbl -> BigInteger.valueOf(dbl.longValue()) ).collect(Collectors.toList()); } @Override - public java.math.BigInteger getNonPrimitiveValue() { - return java.math.BigInteger.valueOf(values.getLongValue()); + public BigInteger getNonPrimitiveValue() { + return BigInteger.valueOf(values.getLongValue()); } }); } - public static BigIntegerField StringToBigInteger(StringField sourceField) { + static BigIntegerField StringToBigInteger(StringField sourceField) { FieldValues fv = sourceField.getFieldValues(); return new BigIntegerField(sourceField.getName(), new DelegatingFieldValues(fv) { + protected BigInteger parseNumber(String str) { + try { + return new BigInteger(str); + } catch (NumberFormatException e) { + return new BigDecimal(str).toBigInteger(); + } + } @Override - public List getValues() { + public List getValues() { // TODO(stu): this may throw - return values.getValues().stream().map(BigInteger::new).collect(Collectors.toList()); + return values.getValues().stream().map(this::parseNumber).collect(Collectors.toList()); } @Override - public java.math.BigInteger getNonPrimitiveValue() { - return new BigInteger(values.getNonPrimitiveValue()); + public BigInteger getNonPrimitiveValue() { + return parseNumber(values.getNonPrimitiveValue()); } }); } - public static LongField BigIntegerToLong(BigIntegerField sourceField) { + static LongField BigIntegerToLong(BigIntegerField sourceField) { FieldValues fv = sourceField.getFieldValues(); return new LongField(sourceField.getName(), new DelegatingFieldValues(fv) { @Override @@ -195,7 +205,7 @@ public Long getNonPrimitiveValue() { }); } - public static LongField BooleanToLong(BooleanField sourceField) { + static LongField BooleanToLong(BooleanField sourceField) { FieldValues fv = sourceField.getFieldValues(); return new LongField(sourceField.getName(), new DelegatingFieldValues(fv) { @Override @@ -210,7 +220,7 @@ public Long getNonPrimitiveValue() { }); } - public static LongField DateMillisToLong(DateMillisField sourceField) { + static LongField DateMillisToLong(DateMillisField sourceField) { FieldValues fv = sourceField.getFieldValues(); return new LongField(sourceField.getName(), new DelegatingFieldValues(fv) { @Override @@ -225,11 +235,11 @@ public Long getNonPrimitiveValue() { }); } - public static LongField DateNanosToLong(DateNanosField sourceField) { + static LongField DateNanosToLong(DateNanosField sourceField) { FieldValues fv = sourceField.getFieldValues(); return new LongField(sourceField.getName(), new DelegatingFieldValues(fv) { protected long nanoLong(JodaCompatibleZonedDateTime dt) { - return ChronoUnit.NANOS.between(java.time.Instant.EPOCH, dt.toInstant()); + return ChronoUnit.NANOS.between(Instant.EPOCH, dt.toInstant()); } @Override @@ -239,12 +249,12 @@ public List getValues() { @Override public Long getNonPrimitiveValue() { - return ChronoUnit.NANOS.between(java.time.Instant.EPOCH, values.getNonPrimitiveValue().toInstant()); + return ChronoUnit.NANOS.between(Instant.EPOCH, values.getNonPrimitiveValue().toInstant()); } }); } - public static LongField DoubleToLong(DoubleField sourceField) { + static LongField DoubleToLong(DoubleField sourceField) { FieldValues fv = sourceField.getFieldValues(); return new LongField(sourceField.getName(), new DelegatingFieldValues(fv) { @Override @@ -259,32 +269,27 @@ public Long getNonPrimitiveValue() { }); } - public static LongField StringToLong(StringField sourceField) { + static LongField StringToLong(StringField sourceField) { FieldValues fv = sourceField.getFieldValues(); return new LongField(sourceField.getName(), new DelegatingFieldValues(fv) { - protected long parseNumber(String str) { - return Long.parseLong(str); - } - @Override public List getValues() { - return values.getValues().stream().map(this::parseNumber).collect(Collectors.toList()); + return values.getValues().stream().map(Long::parseLong).collect(Collectors.toList()); } @Override public Long getNonPrimitiveValue() { - return parseNumber(values.getNonPrimitiveValue()); + return Long.parseLong(values.getNonPrimitiveValue()); } @Override public long getLongValue() { - return parseNumber(values.getNonPrimitiveValue()); + return Long.parseLong(values.getNonPrimitiveValue()); } @Override public double getDoubleValue() { - String str = values.getNonPrimitiveValue(); - return Double.parseDouble(str); + return getLongValue(); } }); } diff --git a/server/src/main/java/org/elasticsearch/script/EmptyField.java b/server/src/main/java/org/elasticsearch/script/EmptyField.java index a77a84e195024..c4e389c38cf7f 100644 --- a/server/src/main/java/org/elasticsearch/script/EmptyField.java +++ b/server/src/main/java/org/elasticsearch/script/EmptyField.java @@ -30,7 +30,7 @@ public List getValues() { } @Override - public > Field as(Converter converter) { + public > Field convert(Converter converter) { // new object created to ensure EmptyField return new EmptyField<>(name); } diff --git a/server/src/main/java/org/elasticsearch/script/Field.java b/server/src/main/java/org/elasticsearch/script/Field.java index 43ecd1170a67e..fb643e9c79b9d 100644 --- a/server/src/main/java/org/elasticsearch/script/Field.java +++ b/server/src/main/java/org/elasticsearch/script/Field.java @@ -57,17 +57,23 @@ public List getValues() { /** * Convert this {@code Field} into another {@code Field} type using a {@link Converter} of the target type. * - * As this is called from user scripts, {@code as} may be called to convert a field into it's same type, if - * so {@code this} is cast via that converters {@link Converter#getFieldClass()}. + * As this is called from user scripts, {@code as} may be called to convert a field into its same type, if + * so {@code this} is cast via converters {@link Converter#getFieldClass()}. * - * Extensions outside the core package should override this method to implement their own conversions, calling - * this method as a fallback. */ - public > Field as(Converter converter) { + public final > Field as(Converter converter) { if (converter.getFieldClass().isInstance(this)) { return converter.getFieldClass().cast(this); } + return convert(converter); + } + + /** + * Extensions outside the core package should override this method to implement their own conversions, calling + * the superclass of this method as a fallback. + */ + protected > Field convert(Converter converter) { return converter.convert(this); } diff --git a/x-pack/plugin/mapper-unsigned-long/src/main/java/org/elasticsearch/xpack/unsignedlong/UnsignedLongField.java b/x-pack/plugin/mapper-unsigned-long/src/main/java/org/elasticsearch/xpack/unsignedlong/UnsignedLongField.java index 69b55bf752230..0df34a32b7448 100644 --- a/x-pack/plugin/mapper-unsigned-long/src/main/java/org/elasticsearch/xpack/unsignedlong/UnsignedLongField.java +++ b/x-pack/plugin/mapper-unsigned-long/src/main/java/org/elasticsearch/xpack/unsignedlong/UnsignedLongField.java @@ -25,11 +25,7 @@ public UnsignedLongField(String name, FieldValues values) { // UnsignedLongFields must define their own conversions as they are in x-pack @Override - public > Field as(Converter converter) { - if (converter.getFieldClass().isInstance(this)) { - return converter.getFieldClass().cast(this); - } - + public > Field convert(Converter converter) { if (converter.getTargetClass() == BigInteger.class) { BigIntegerField bigIntegerField = UnsignedLongToBigInteger(this); return converter.getFieldClass().cast(bigIntegerField); @@ -38,7 +34,7 @@ public > Field as(Converter converter) { return super.as(converter); } - public static BigIntegerField UnsignedLongToBigInteger(UnsignedLongField sourceField) { + static BigIntegerField UnsignedLongToBigInteger(UnsignedLongField sourceField) { FieldValues fv = sourceField.getFieldValues(); return new BigIntegerField(sourceField.getName(), new Converters.DelegatingFieldValues(fv) { protected BigInteger toBigInteger(long formatted) { @@ -56,19 +52,4 @@ public BigInteger getNonPrimitiveValue() { } }); } - - public static UnsignedLongField BigIntegerToUnsignedLong(BigIntegerField sourceField) { - FieldValues fv = sourceField.getFieldValues(); - return new UnsignedLongField(sourceField.getName(), new Converters.DelegatingFieldValues<>(fv) { - @Override - public List getValues() { - return values.getValues().stream().map(java.math.BigInteger::longValue).collect(Collectors.toList()); - } - - @Override - public Long getNonPrimitiveValue() { - return values.getLongValue(); - } - }); - } } From 5d82d1a8c5018fd897718171cf786b450b52757f Mon Sep 17 00:00:00 2001 From: Stuart Tettemer Date: Tue, 17 Aug 2021 08:42:57 -0500 Subject: [PATCH 12/14] minimal UnsignedLongField.as(Field.Long) test --- .../unsignedlong/UnsignedLongFieldTests.java | 68 +++++++++++++++++++ 1 file changed, 68 insertions(+) create mode 100644 x-pack/plugin/mapper-unsigned-long/src/test/java/org/elasticsearch/xpack/unsignedlong/UnsignedLongFieldTests.java diff --git a/x-pack/plugin/mapper-unsigned-long/src/test/java/org/elasticsearch/xpack/unsignedlong/UnsignedLongFieldTests.java b/x-pack/plugin/mapper-unsigned-long/src/test/java/org/elasticsearch/xpack/unsignedlong/UnsignedLongFieldTests.java new file mode 100644 index 0000000000000..dc3f487f2f5b8 --- /dev/null +++ b/x-pack/plugin/mapper-unsigned-long/src/test/java/org/elasticsearch/xpack/unsignedlong/UnsignedLongFieldTests.java @@ -0,0 +1,68 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License + * 2.0; you may not use this file except in compliance with the Elastic License + * 2.0. + */ + +package org.elasticsearch.xpack.unsignedlong; + +import org.elasticsearch.script.Field; +import org.elasticsearch.script.FieldValues; +import org.elasticsearch.test.ESTestCase; + +import java.math.BigInteger; +import java.util.ArrayList; +import java.util.Arrays; +import java.util.List; +import java.util.stream.Collectors; + +public class UnsignedLongFieldTests extends ESTestCase { + static final BigInteger[] VALUES = { + BigInteger.valueOf(0L), + new BigInteger("18446744073709551615"), // 2^64 - 1 + new BigInteger("9223372036854775807"), // 2^63 - 1 + new BigInteger("9223372036854775808"), // 2^63 + new BigInteger("9223372036854775809"), // 2^63 + 1 + }; + + UnsignedLongField FIELD = new UnsignedLongField("test", new FieldValues() { + protected long[] values = Arrays.stream(UnsignedLongFieldTests.VALUES).mapToLong(BigInteger::longValue).toArray(); + + @Override + public boolean isEmpty() { + return values.length == 0; + } + + @Override + public int size() { + return values.length; + } + + @Override + public List getValues() { + return Arrays.stream(values).boxed().collect(Collectors.toList()); + } + + @Override + public Long getNonPrimitiveValue() { + return UnsignedLongScriptDocValues.shiftedLong(values[0]); + } + + @Override + public long getLongValue() { + return UnsignedLongScriptDocValues.shiftedLong(values[0]); + } + + @Override + public double getDoubleValue() { + return UnsignedLongScriptDocValues.shiftedLong(values[0]); + } + }); + + public void testLongValues() { + long[] expected = {0L, -1L, 9223372036854775807L, -9223372036854775808L, -9223372036854775807L}; + List asLong = Arrays.stream(expected).boxed().collect(Collectors.toList()); + assertEquals(asLong, FIELD.convert(Field.Long).getValues()); + } +} From 51532210d63109fd95da079be8534cb0e07f410b Mon Sep 17 00:00:00 2001 From: Stuart Tettemer Date: Tue, 17 Aug 2021 08:43:50 -0500 Subject: [PATCH 13/14] unused import --- .../elasticsearch/xpack/unsignedlong/UnsignedLongFieldTests.java | 1 - 1 file changed, 1 deletion(-) diff --git a/x-pack/plugin/mapper-unsigned-long/src/test/java/org/elasticsearch/xpack/unsignedlong/UnsignedLongFieldTests.java b/x-pack/plugin/mapper-unsigned-long/src/test/java/org/elasticsearch/xpack/unsignedlong/UnsignedLongFieldTests.java index dc3f487f2f5b8..17b83c111956a 100644 --- a/x-pack/plugin/mapper-unsigned-long/src/test/java/org/elasticsearch/xpack/unsignedlong/UnsignedLongFieldTests.java +++ b/x-pack/plugin/mapper-unsigned-long/src/test/java/org/elasticsearch/xpack/unsignedlong/UnsignedLongFieldTests.java @@ -12,7 +12,6 @@ import org.elasticsearch.test.ESTestCase; import java.math.BigInteger; -import java.util.ArrayList; import java.util.Arrays; import java.util.List; import java.util.stream.Collectors; From 369e58b6de57bf4248e9739d351d402f7271a64b Mon Sep 17 00:00:00 2001 From: Stuart Tettemer Date: Tue, 17 Aug 2021 08:50:34 -0500 Subject: [PATCH 14/14] format --- .../xpack/unsignedlong/UnsignedLongFieldTests.java | 12 ++++++------ 1 file changed, 6 insertions(+), 6 deletions(-) diff --git a/x-pack/plugin/mapper-unsigned-long/src/test/java/org/elasticsearch/xpack/unsignedlong/UnsignedLongFieldTests.java b/x-pack/plugin/mapper-unsigned-long/src/test/java/org/elasticsearch/xpack/unsignedlong/UnsignedLongFieldTests.java index 17b83c111956a..d9c0087abbc15 100644 --- a/x-pack/plugin/mapper-unsigned-long/src/test/java/org/elasticsearch/xpack/unsignedlong/UnsignedLongFieldTests.java +++ b/x-pack/plugin/mapper-unsigned-long/src/test/java/org/elasticsearch/xpack/unsignedlong/UnsignedLongFieldTests.java @@ -18,11 +18,11 @@ public class UnsignedLongFieldTests extends ESTestCase { static final BigInteger[] VALUES = { - BigInteger.valueOf(0L), - new BigInteger("18446744073709551615"), // 2^64 - 1 - new BigInteger("9223372036854775807"), // 2^63 - 1 - new BigInteger("9223372036854775808"), // 2^63 - new BigInteger("9223372036854775809"), // 2^63 + 1 + BigInteger.valueOf(0L), + new BigInteger("18446744073709551615"), // 2^64 - 1 + new BigInteger("9223372036854775807"), // 2^63 - 1 + new BigInteger("9223372036854775808"), // 2^63 + new BigInteger("9223372036854775809"), // 2^63 + 1 }; UnsignedLongField FIELD = new UnsignedLongField("test", new FieldValues() { @@ -60,7 +60,7 @@ public double getDoubleValue() { }); public void testLongValues() { - long[] expected = {0L, -1L, 9223372036854775807L, -9223372036854775808L, -9223372036854775807L}; + long[] expected = { 0L, -1L, 9223372036854775807L, -9223372036854775808L, -9223372036854775807L }; List asLong = Arrays.stream(expected).boxed().collect(Collectors.toList()); assertEquals(asLong, FIELD.convert(Field.Long).getValues()); }