Script: fields API for x-pack version, doc version, seq no, mumur3 (#…

…81476) Adds scripting fields API support the rest of the long fields: * [`_version`](https://www.elastic.co/guide/en/elasticsearch/reference/current/docs-index_.html#index-versioning) - `VersionDocValuesField` * [`_seq_no`](https://www.elastic.co/guide/en/elasticsearch/reference/master/optimistic-concurrency-control.html) - `SeqNoDocValuesField` * [`murmur3`](https://www.elastic.co/guide/en/elasticsearch/plugins/current/mapper-murmur3-usage.html) - `Murmur3DocValueField` * Added Painless support to the murmur3 mapper plugin. All `SortedNumericDocValues` that are interpreted as longs are now subclasses of `AbstractLongDocValuesField`, including murmur, doc version and seq no above as well as `LongDocValuesField` and `UnsignedLongDocValuesField` Also adds: * [x-pack's version](https://www.elastic.co/guide/en/elasticsearch/reference/master/version.html) - `VersionStringDocValuesField` * Created new `Version` value type as a location for future helpers for comparing versions. * Implements `toString` for the expected representation of the version * Implements `asString(String)` and `asString(int, String)`, `asStrings()` converters on field. Refs: #79105
elastic · Jan 5, 2022 · c937a09 · c937a09
1 parent e5efade
commit c937a09
Show file tree

Hide file tree

Showing 28 changed files with 691 additions and 346 deletions.
diff --git a/docs/changelog/81476.yaml b/docs/changelog/81476.yaml
@@ -0,0 +1,5 @@
+pr: 81476
+summary: "Script: fields API for x-pack version, doc version, seq no, mumur3"
+area: Infra/Scripting
+type: enhancement
+issues: []
diff --git a/...ainless/src/main/resources/org/elasticsearch/painless/org.elasticsearch.script.fields.txt b/...ainless/src/main/resources/org/elasticsearch/painless/org.elasticsearch.script.fields.txt
@@ -100,3 +100,16 @@ class org.elasticsearch.script.field.IpDocValuesField @dynamic_type {
   String asString(String)
   String asString(int, String)
 }
+
+class org.elasticsearch.script.field.AbstractLongDocValuesField @dynamic_type {
+  long get(long)
+  long get(int, long)
+}
+
+# subclass of AbstractLongDocValuesField
+class org.elasticsearch.script.field.SeqNoDocValuesField @dynamic_type {
+}
+
+# subclass of AbstractLongDocValuesField
+class org.elasticsearch.script.field.VersionDocValuesField @dynamic_type {
+}
diff --git a/...-painless/src/yamlRestTest/resources/rest-api-spec/test/painless/50_script_doc_values.yml b/...-painless/src/yamlRestTest/resources/rest-api-spec/test/painless/50_script_doc_values.yml
@@ -1446,3 +1446,81 @@ setup:
               script:
                 source: "int value = field('dne').get(1, 1); value"
   - match: { hits.hits.0.fields.field.0: 1 }
+
+---
+"version and sequence number":
+  - do:
+      indices.create:
+        index: versiontest
+        body:
+          settings:
+            number_of_shards: 1
+          mappings:
+            properties:
+              keyword:
+                type: keyword
+
+  - do:
+      index:
+        index: test
+        id: 3000
+        version: 50
+        version_type: external
+        body:
+          keyword: "3k"
+
+  - do:
+      indices.refresh: {}
+
+  - do:
+      search:
+        rest_total_hits_as_int: true
+        body:
+          query: { term: { _id: 3000 } }
+          script_fields:
+            ver:
+              script:
+                source: "field('_version').get(10000)"
+            seq:
+              script:
+                source: "field('_seq_no').get(10000)"
+  - match: { hits.hits.0.fields.ver.0: 50 }
+  - match: { hits.hits.0.fields.seq.0: 3 }
+
+  - do:
+      index:
+        index: test
+        id: 3000
+        version: 60
+        version_type: external
+        body:
+          keyword: "3k+1"
+  - do:
+      indices.refresh: {}
+
+  - do:
+      catch:      conflict
+      index:
+        index: test
+        id: 3000
+        version: 55
+        version_type: external
+        body:
+          keyword: "3k+2"
+  - do:
+      indices.refresh: {}
+
+  - do:
+      search:
+        rest_total_hits_as_int: true
+        body:
+          query: { term: { _id: 3000 } }
+          script_fields:
+            ver:
+              script:
+                source: "field('_version').get(10000)"
+            seq:
+              script:
+                source: "field('_seq_no').get(10000)"
+  - match: { hits.hits.0.fields.ver.0: 60 }
+  - match: { hits.hits.0.fields.seq.0: 4 }
diff --git a/plugins/mapper-murmur3/build.gradle b/plugins/mapper-murmur3/build.gradle
@@ -11,10 +11,21 @@ apply plugin: 'elasticsearch.yaml-rest-compat-test'
 esplugin {
   description 'The Mapper Murmur3 plugin allows to compute hashes of a field\'s values at index-time and to store them in the index.'
   classname 'org.elasticsearch.plugin.mapper.MapperMurmur3Plugin'
+  extendedPlugins = ['lang-painless']
+}
+
+dependencies {
+  compileOnly project(':modules:lang-painless:spi')
+  testImplementation project(':modules:lang-painless')
 }
 
 restResources {
   restApi {
     include '_common', 'indices', 'index', 'search'
   }
 }
+
+testClusters.configureEach {
+  testDistribution = 'DEFAULT'
+  setting 'xpack.security.enabled', 'false'
+}
diff --git a/...pper-murmur3/src/main/java/org/elasticsearch/index/mapper/murmur3/Murmur3FieldMapper.java b/...pper-murmur3/src/main/java/org/elasticsearch/index/mapper/murmur3/Murmur3FieldMapper.java
@@ -12,14 +12,11 @@
 import org.apache.lucene.document.SortedNumericDocValuesField;
 import org.apache.lucene.document.StoredField;
 import org.apache.lucene.index.IndexOptions;
-import org.apache.lucene.index.SortedNumericDocValues;
 import org.apache.lucene.search.Query;
 import org.apache.lucene.util.BytesRef;
 import org.elasticsearch.common.hash.MurmurHash3;
 import org.elasticsearch.index.fielddata.IndexFieldData;
 import org.elasticsearch.index.fielddata.IndexNumericFieldData.NumericType;
-import org.elasticsearch.index.fielddata.ScriptDocValues.Longs;
-import org.elasticsearch.index.fielddata.ScriptDocValues.LongsSupplier;
 import org.elasticsearch.index.fielddata.plain.SortedNumericIndexFieldData;
 import org.elasticsearch.index.mapper.DocumentParserContext;
 import org.elasticsearch.index.mapper.FieldMapper;
@@ -29,8 +26,7 @@
 import org.elasticsearch.index.mapper.TextSearchInfo;
 import org.elasticsearch.index.mapper.ValueFetcher;
 import org.elasticsearch.index.query.SearchExecutionContext;
-import org.elasticsearch.script.field.DelegateDocValuesField;
-import org.elasticsearch.script.field.ToScriptField;
+import org.elasticsearch.script.field.murmur3.Murmur3DocValueField;
 import org.elasticsearch.search.lookup.SearchLookup;
 
 import java.io.IOException;
@@ -84,11 +80,6 @@ public Murmur3FieldMapper build(MapperBuilderContext context) {
     // this only exists so a check can be done to match the field type to using murmur3 hashing...
     public static class Murmur3FieldType extends MappedFieldType {
 
-        public static final ToScriptField<SortedNumericDocValues> TO_SCRIPT_FIELD = (dv, n) -> new DelegateDocValuesField(
-            new Longs(new LongsSupplier(dv)),
-            n
-        );
-
         private Murmur3FieldType(String name, boolean isStored, Map<String, String> meta) {
             super(name, false, isStored, true, TextSearchInfo.NONE, meta);
         }
@@ -101,7 +92,7 @@ public String typeName() {
         @Override
         public IndexFieldData.Builder fielddataBuilder(String fullyQualifiedIndexName, Supplier<SearchLookup> searchLookup) {
             failIfNoDocValues();
-            return new SortedNumericIndexFieldData.Builder(name(), NumericType.LONG, TO_SCRIPT_FIELD);
+            return new SortedNumericIndexFieldData.Builder(name(), NumericType.LONG, Murmur3DocValueField::new);
         }
 
         @Override

diff --git a/...er-murmur3/src/main/java/org/elasticsearch/script/field/murmur3/Murmur3DocValueField.java b/...er-murmur3/src/main/java/org/elasticsearch/script/field/murmur3/Murmur3DocValueField.java
@@ -0,0 +1,19 @@
+/*
+ * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
+ * or more contributor license agreements. Licensed under the Elastic License
+ * 2.0 and the Server Side Public License, v 1; you may not use this file except
+ * in compliance with, at your election, the Elastic License 2.0 or the Server
+ * Side Public License, v 1.
+ */
+
+package org.elasticsearch.script.field.murmur3;
+
+import org.apache.lucene.index.SortedNumericDocValues;
+import org.elasticsearch.script.field.AbstractLongDocValuesField;
+
+public class Murmur3DocValueField extends AbstractLongDocValuesField {
+
+    public Murmur3DocValueField(SortedNumericDocValues input, String name) {
+        super(input, name);
+    }
+}
diff --git a/...urmur3/src/main/java/org/elasticsearch/script/field/murmur3/Murmur3PainlessExtension.java b/...urmur3/src/main/java/org/elasticsearch/script/field/murmur3/Murmur3PainlessExtension.java
@@ -0,0 +1,39 @@
+/*
+ * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
+ * or more contributor license agreements. Licensed under the Elastic License
+ * 2.0 and the Server Side Public License, v 1; you may not use this file except
+ * in compliance with, at your election, the Elastic License 2.0 or the Server
+ * Side Public License, v 1.
+ */
+
+package org.elasticsearch.script.field.murmur3;
+
+import org.elasticsearch.painless.spi.PainlessExtension;
+import org.elasticsearch.painless.spi.Whitelist;
+import org.elasticsearch.painless.spi.WhitelistLoader;
+import org.elasticsearch.script.ScriptContext;
+
+import java.util.HashMap;
+import java.util.List;
+import java.util.Map;
+
+import static java.util.Collections.singletonList;
+import static org.elasticsearch.script.ScriptModule.CORE_CONTEXTS;
+
+public class Murmur3PainlessExtension implements PainlessExtension {
+
+    private static final Whitelist WHITELIST = WhitelistLoader.loadFromResourceFiles(
+        Murmur3PainlessExtension.class,
+        "org.elasticsearch.field.murmur3.txt"
+    );
+
+    @Override
+    public Map<ScriptContext<?>, List<Whitelist>> getContextWhitelists() {
+        List<Whitelist> whitelist = singletonList(WHITELIST);
+        Map<ScriptContext<?>, List<Whitelist>> contextWhitelists = new HashMap<>(CORE_CONTEXTS.size());
+        for (ScriptContext<?> scriptContext : CORE_CONTEXTS.values()) {
+            contextWhitelists.put(scriptContext, whitelist);
+        }
+        return contextWhitelists;
+    }
+}
diff --git a/...ur3/src/main/resources/META-INF/services/org.elasticsearch.painless.spi.PainlessExtension b/...ur3/src/main/resources/META-INF/services/org.elasticsearch.painless.spi.PainlessExtension
@@ -0,0 +1 @@
+org.elasticsearch.script.field.murmur3.Murmur3PainlessExtension
diff --git a/...main/resources/org/elasticsearch/script/field/murmur3/org.elasticsearch.field.murmur3.txt b/...main/resources/org/elasticsearch/script/field/murmur3/org.elasticsearch.field.murmur3.txt
@@ -0,0 +1,11 @@
+#
+# Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
+# or more contributor license agreements. Licensed under the Elastic License
+# 2.0 and the Server Side Public License, v 1; you may not use this file except
+# in compliance with, at your election, the Elastic License 2.0 or the Server
+# Side Public License, v 1.
+#
+
+# subclass of AbstractLongDocValuesField
+class org.elasticsearch.script.field.murmur3.Murmur3DocValueField @dynamic_type {
+}
diff --git a/.../mapper-murmur3/src/yamlRestTest/resources/rest-api-spec/test/mapper_murmur3/10_basic.yml b/.../mapper-murmur3/src/yamlRestTest/resources/rest-api-spec/test/mapper_murmur3/10_basic.yml
@@ -1,9 +1,7 @@
 # Integration tests for Mapper Murmur3 components
 #
 
----
-"Mapper Murmur3":
-
+setup:
     - do:
         indices.create:
             index: test
@@ -20,6 +18,8 @@
     - do:
         indices.refresh: {}
 
+---
+"Mapper Murmur3":
     - do:
         search:
             rest_total_hits_as_int: true
@@ -60,3 +60,27 @@
             body: { "aggs": { "foo_count": { "cardinality": { "field": "foo.hash" } } } }
 
     - match: { aggregations.foo_count.value: 3 }
+
+---
+"Mumur3 script fields api":
+
+  - do:
+      index:
+        index: test
+        id: 1
+        body: { "foo": "foo" }
+
+  - do:
+      indices.refresh: {}
+
+  - do:
+      search:
+        index: test
+        body:
+          sort: [ { foo.hash: desc } ]
+          script_fields:
+            field:
+              script:
+                source: "field('foo.hash').get(10L)"
+
+  - match: { hits.hits.0.fields.field.0: -2129773440516405919 }
diff --git a/server/src/main/java/org/elasticsearch/index/fielddata/ScriptDocValues.java b/server/src/main/java/org/elasticsearch/index/fielddata/ScriptDocValues.java
@@ -8,7 +8,6 @@
 
 package org.elasticsearch.index.fielddata;
 
-import org.apache.lucene.index.SortedNumericDocValues;
 import org.apache.lucene.util.ArrayUtil;
 import org.apache.lucene.util.BytesRef;
 import org.apache.lucene.util.BytesRefBuilder;
@@ -94,48 +93,6 @@ protected void throwIfEmpty() {
         }
     }
 
-    public static class LongsSupplier implements Supplier<Long> {
-
-        private final SortedNumericDocValues in;
-        private long[] values = new long[0];
-        private int count;
-
-        public LongsSupplier(SortedNumericDocValues in) {
-            this.in = in;
-        }
-
-        @Override
-        public void setNextDocId(int docId) throws IOException {
-            if (in.advanceExact(docId)) {
-                resize(in.docValueCount());
-                for (int i = 0; i < count; i++) {
-                    values[i] = in.nextValue();
-                }
-            } else {
-                resize(0);
-            }
-        }
-
-        /**
-         * Set the {@link #size()} and ensure that the {@link #values} array can
-         * store at least that many entries.
-         */
-        private void resize(int newSize) {
-            count = newSize;
-            values = ArrayUtil.grow(values, count);
-        }
-
-        @Override
-        public Long getInternal(int index) {
-            return values[index];
-        }
-
-        @Override
-        public int size() {
-            return count;
-        }
-    }
-
     public static class Longs extends ScriptDocValues<Long> {
 
         public Longs(Supplier<Long> supplier) {

diff --git a/server/src/main/java/org/elasticsearch/index/mapper/SeqNoFieldMapper.java b/server/src/main/java/org/elasticsearch/index/mapper/SeqNoFieldMapper.java
@@ -17,12 +17,10 @@
 import org.elasticsearch.core.Nullable;
 import org.elasticsearch.index.fielddata.IndexFieldData;
 import org.elasticsearch.index.fielddata.IndexNumericFieldData.NumericType;
-import org.elasticsearch.index.fielddata.ScriptDocValues.Longs;
-import org.elasticsearch.index.fielddata.ScriptDocValues.LongsSupplier;
 import org.elasticsearch.index.fielddata.plain.SortedNumericIndexFieldData;
 import org.elasticsearch.index.query.SearchExecutionContext;
 import org.elasticsearch.index.seqno.SequenceNumbers;
-import org.elasticsearch.script.field.DelegateDocValuesField;
+import org.elasticsearch.script.field.SeqNoDocValuesField;
 import org.elasticsearch.search.lookup.SearchLookup;
 
 import java.io.IOException;
@@ -186,11 +184,7 @@ public Query rangeQuery(
         @Override
         public IndexFieldData.Builder fielddataBuilder(String fullyQualifiedIndexName, Supplier<SearchLookup> searchLookup) {
             failIfNoDocValues();
-            return new SortedNumericIndexFieldData.Builder(
-                name(),
-                NumericType.LONG,
-                (dv, n) -> new DelegateDocValuesField(new Longs(new LongsSupplier(dv)), n)
-            );
+            return new SortedNumericIndexFieldData.Builder(name(), NumericType.LONG, SeqNoDocValuesField::new);
         }
     }