Add source-only fields (#64981)

This commit adds the ability to specify a runtime field that fetches values
directly from its corresponding position in _source.

Source-only fields are defined in the runtime mappings by adding a field
with no 'script' parameter. Fields that look into objects can be defined
using the usual dot notation.

"runtime" : {
    "field" : { 
        "type" : "keyword"
    },
    "object.field" : {
        "type" : "integer"
    }
}

server/src/main/java/org/elasticsearch/index/mapper/FieldMapper.java

@@ -485,7 +485,7 @@ protected interface MergeValidator<T> {
     /**
      * Check on whether or not a parameter should be serialized
      */
-    protected interface SerializerCheck<T> {
+    public interface SerializerCheck<T> {
         /**
          * Check on whether or not a parameter should be serialized
          * @param includeDefaults   if defaults have been requested

server/src/test/java/org/elasticsearch/common/xcontent/support/XContentMapValuesTests.java

@@ -233,6 +233,28 @@ public void testExtractRawValue() throws Exception {
             map = parser.map();
         }
         assertThat(XContentMapValues.extractRawValues("path1.xxx.path2.yyy.test", map).get(0).toString(), equalTo("value"));
+
+        builder = XContentFactory.jsonBuilder().startObject()
+            .startObject("path1").startArray("path2")
+            .startArray()
+            .startObject().startObject("path3").field("field", "value1").endObject().endObject()
+            .startObject().startObject("path3").field("field", "value2").endObject().endObject()
+            .endArray()
+            .endArray()
+            .endObject().endObject();
+
+        try (XContentParser parser = createParser(JsonXContent.jsonXContent, Strings.toString(builder))) {
+            map = parser.map();
+        }
+        assertThat(XContentMapValues.extractRawValues("path1.path2.path3.field", map), contains("value1", "value2"));
+
+        builder = XContentFactory.jsonBuilder().startObject()
+            .startObject("path1").array("path2", 9, true, "manglewurzle").endObject().endObject();
+        try (XContentParser parser = createParser(JsonXContent.jsonXContent, Strings.toString(builder))) {
+            map = parser.map();
+        }
+        assertThat(XContentMapValues.extractRawValues("path1.path2", map), contains(9, true, "manglewurzle"));
+        assertThat(XContentMapValues.extractRawValues("path1.path2.path3", map), hasSize(0));
     }
 
     public void testExtractRawValueLeafOnly() throws IOException {

x-pack/plugin/runtime-fields/src/main/java/org/elasticsearch/xpack/runtimefields/mapper/AbstractFieldScript.java

@@ -7,6 +7,7 @@
 package org.elasticsearch.xpack.runtimefields.mapper;
 
 import org.apache.lucene.index.LeafReaderContext;
+import org.elasticsearch.common.xcontent.support.XContentMapValues;
 import org.elasticsearch.index.fielddata.ScriptDocValues;
 import org.elasticsearch.script.AggregationScript;
 import org.elasticsearch.script.DynamicMap;
@@ -17,6 +18,7 @@
 import org.elasticsearch.search.lookup.SourceLookup;
 
 import java.util.HashMap;
+import java.util.List;
 import java.util.Locale;
 import java.util.Map;
 import java.util.function.Function;
@@ -94,6 +96,10 @@ public final Map<String, ScriptDocValues<?>> getDoc() {
         return leafSearchLookup.doc();
     }
 
+    protected final List<Object> extractFromSource(String path) {
+        return XContentMapValues.extractRawValues(path, leafSearchLookup.source().loadSourceIfNeeded());
+    }
+
     /**
      * Check if the we can add another value to the list of values.
      * @param currentSize the current size of the list

x-pack/plugin/runtime-fields/src/main/java/org/elasticsearch/xpack/runtimefields/mapper/AbstractScriptFieldType.java

@@ -50,7 +50,7 @@ abstract class AbstractScriptFieldType<LeafFactory> extends RuntimeFieldType {
     private final CheckedBiConsumer<XContentBuilder, Boolean, IOException> toXContent;
 
     AbstractScriptFieldType(String name, TriFunction<String, Map<String, Object>, SearchLookup, LeafFactory> factory, Builder builder) {
-        this(name, factory, builder.script.getValue(), builder.meta.getValue(), builder::toXContent);
+        this(name, factory, builder.getScript(), builder.meta.getValue(), builder::toXContent);
     }
 
     AbstractScriptFieldType(
@@ -206,6 +206,10 @@ protected final void doXContentBody(XContentBuilder builder, boolean includeDefa
         toXContent.accept(builder, includeDefaults);
     }
 
+    // Placeholder Script for source-only fields
+    // TODO rework things so that we don't need this
+    private static final Script DEFAULT_SCRIPT = new Script("");
+
     /**
      *  For runtime fields the {@link RuntimeFieldType.Parser} returns directly the {@link MappedFieldType}.
      *  Internally we still create a {@link Builder} so we reuse the {@link FieldMapper.Parameter} infrastructure,
@@ -222,11 +226,7 @@ abstract static class Builder extends FieldMapper.Builder {
             () -> null,
             Builder::parseScript,
             initializerNotSupported()
-        ).setValidator(script -> {
-            if (script == null) {
-                throw new IllegalArgumentException("script must be specified for runtime field [" + name + "]");
-            }
-        });
+        ).setSerializerCheck((id, ic, v) -> ic);
 
         Builder(String name) {
             super(name);
@@ -239,6 +239,13 @@ protected List<FieldMapper.Parameter<?>> getParameters() {
 
         protected abstract AbstractScriptFieldType<?> buildFieldType();
 
+        protected final Script getScript() {
+            if (script.get() == null) {
+                return DEFAULT_SCRIPT;
+            }
+            return script.get();
+        }
+
         @Override
         public FieldMapper.Builder init(FieldMapper initializer) {
             throw new UnsupportedOperationException();

x-pack/plugin/runtime-fields/src/main/java/org/elasticsearch/xpack/runtimefields/mapper/BooleanFieldScript.java

@@ -7,6 +7,7 @@
 package org.elasticsearch.xpack.runtimefields.mapper;
 
 import org.apache.lucene.index.LeafReaderContext;
+import org.elasticsearch.common.Booleans;
 import org.elasticsearch.painless.spi.Whitelist;
 import org.elasticsearch.painless.spi.WhitelistLoader;
 import org.elasticsearch.script.ScriptContext;
@@ -17,6 +18,7 @@
 import java.util.Map;
 
 public abstract class BooleanFieldScript extends AbstractFieldScript {
+
     public static final ScriptContext<Factory> CONTEXT = newContext("boolean_script_field", Factory.class);
 
     static List<Whitelist> whitelist() {
@@ -34,6 +36,28 @@ public interface LeafFactory {
         BooleanFieldScript newInstance(LeafReaderContext ctx);
     }
 
+    public static final Factory PARSE_FROM_SOURCE = (field, params, lookup) -> (LeafFactory) ctx -> new BooleanFieldScript(
+        field,
+        params,
+        lookup,
+        ctx
+    ) {
+        @Override
+        public void execute() {
+            for (Object v : extractFromSource(field)) {
+                if (v instanceof Boolean) {
+                    emit((Boolean) v);
+                } else if (v instanceof String) {
+                    try {
+                        emit(Booleans.parseBoolean((String) v));
+                    } catch (IllegalArgumentException e) {
+                        // ignore
+                    }
+                }
+            }
+        }
+    };
+
     private int trues;
     private int falses;
 

x-pack/plugin/runtime-fields/src/main/java/org/elasticsearch/xpack/runtimefields/mapper/BooleanScriptFieldType.java

@@ -36,6 +36,9 @@ public final class BooleanScriptFieldType extends AbstractScriptFieldType<Boolea
     public static final RuntimeFieldType.Parser PARSER = new RuntimeFieldTypeParser((name, parserContext) -> new Builder(name) {
         @Override
         protected AbstractScriptFieldType<?> buildFieldType() {
+            if (script.get() == null) {
+                return new BooleanScriptFieldType(name, BooleanFieldScript.PARSE_FROM_SOURCE, this);
+            }
             BooleanFieldScript.Factory factory = parserContext.scriptService().compile(script.getValue(), BooleanFieldScript.CONTEXT);
             return new BooleanScriptFieldType(name, factory, this);
         }

x-pack/plugin/runtime-fields/src/main/java/org/elasticsearch/xpack/runtimefields/mapper/DateFieldScript.java

@@ -35,6 +35,27 @@ public interface LeafFactory {
         DateFieldScript newInstance(LeafReaderContext ctx);
     }
 
+    public static final Factory PARSE_FROM_SOURCE = (field, params, lookup, formatter) -> (LeafFactory) ctx -> new DateFieldScript(
+        field,
+        params,
+        lookup,
+        formatter,
+        ctx
+    ) {
+        @Override
+        public void execute() {
+            for (Object v : extractFromSource(field)) {
+                if (v instanceof String) {
+                    try {
+                        emit(formatter.parseMillis((String) v));
+                    } catch (Exception e) {
+                        // ignore
+                    }
+                }
+            }
+        }
+    };
+
     private final DateFormatter formatter;
 
     public DateFieldScript(

x-pack/plugin/runtime-fields/src/main/java/org/elasticsearch/xpack/runtimefields/mapper/DateScriptFieldType.java

@@ -79,10 +79,13 @@ protected List<FieldMapper.Parameter<?>> getParameters() {
 
         @Override
         protected AbstractScriptFieldType<?> buildFieldType() {
-            DateFieldScript.Factory factory = parserContext.scriptService().compile(script.getValue(), DateFieldScript.CONTEXT);
             String pattern = format.getValue() == null ? DateFieldMapper.DEFAULT_DATE_TIME_FORMATTER.pattern() : format.getValue();
             Locale locale = this.locale.getValue() == null ? Locale.ROOT : this.locale.getValue();
             DateFormatter dateTimeFormatter = DateFormatter.forPattern(pattern).withLocale(locale);
+            if (script.get() == null) {
+                return new DateScriptFieldType(name, DateFieldScript.PARSE_FROM_SOURCE, dateTimeFormatter, this);
+            }
+            DateFieldScript.Factory factory = parserContext.scriptService().compile(script.getValue(), DateFieldScript.CONTEXT);
             return new DateScriptFieldType(name, factory, dateTimeFormatter, this);
         }
     });

x-pack/plugin/runtime-fields/src/main/java/org/elasticsearch/xpack/runtimefields/mapper/DoubleFieldScript.java

@@ -35,6 +35,28 @@ public interface LeafFactory {
         DoubleFieldScript newInstance(LeafReaderContext ctx);
     }
 
+    public static final Factory PARSE_FROM_SOURCE = (field, params, lookup) -> (LeafFactory) ctx -> new DoubleFieldScript(
+        field,
+        params,
+        lookup,
+        ctx
+    ) {
+        @Override
+        public void execute() {
+            for (Object v : extractFromSource(field)) {
+                if (v instanceof Number) {
+                    emit(((Number) v).doubleValue());
+                } else if (v instanceof String) {
+                    try {
+                        emit(Double.parseDouble((String) v));
+                    } catch (NumberFormatException e) {
+                        // ignore
+                    }
+                }
+            }
+        }
+    };
+
     private double[] values = new double[1];
     private int count;
 

x-pack/plugin/runtime-fields/src/main/java/org/elasticsearch/xpack/runtimefields/mapper/DoubleScriptFieldType.java

@@ -36,6 +36,9 @@ public final class DoubleScriptFieldType extends AbstractScriptFieldType<DoubleF
     public static final RuntimeFieldType.Parser PARSER = new RuntimeFieldTypeParser((name, parserContext) -> new Builder(name) {
         @Override
         protected AbstractScriptFieldType<?> buildFieldType() {
+            if (script.get() == null) {
+                return new DoubleScriptFieldType(name, DoubleFieldScript.PARSE_FROM_SOURCE, this);
+            }
             DoubleFieldScript.Factory factory = parserContext.scriptService().compile(script.getValue(), DoubleFieldScript.CONTEXT);
             return new DoubleScriptFieldType(name, factory, this);
         }

x-pack/plugin/runtime-fields/src/main/java/org/elasticsearch/xpack/runtimefields/mapper/IpFieldScript.java

@@ -56,6 +56,26 @@ public interface LeafFactory {
         IpFieldScript newInstance(LeafReaderContext ctx);
     }
 
+    public static final Factory PARSE_FROM_SOURCE = (field, params, lookup) -> (LeafFactory) ctx -> new IpFieldScript(
+        field,
+        params,
+        lookup,
+        ctx
+    ) {
+        @Override
+        public void execute() {
+            for (Object v : extractFromSource(field)) {
+                if (v instanceof String) {
+                    try {
+                        emit((String) v);
+                    } catch (Exception e) {
+                        // ignore parsing exceptions
+                    }
+                }
+            }
+        }
+    };
+
     private BytesRef[] values = new BytesRef[1];
     private int count;
 
@@ -97,7 +117,10 @@ protected final void emit(String v) {
         if (values.length < count + 1) {
             values = ArrayUtil.grow(values, count + 1);
         }
-        values[count++] = new BytesRef(InetAddressPoint.encode(InetAddresses.forString(v)));
+        BytesRef encoded = new BytesRef(InetAddressPoint.encode(InetAddresses.forString(v)));
+        // encode the address and increment the count on separate lines, to ensure that
+        // we don't increment if the address is badly formed
+        values[count++] = encoded;
     }
 
     public static class Emit {

x-pack/plugin/runtime-fields/src/main/java/org/elasticsearch/xpack/runtimefields/mapper/IpScriptFieldType.java

@@ -45,6 +45,9 @@ public final class IpScriptFieldType extends AbstractScriptFieldType<IpFieldScri
     public static final RuntimeFieldType.Parser PARSER = new RuntimeFieldTypeParser((name, parserContext) -> new Builder(name) {
         @Override
         protected AbstractScriptFieldType<?> buildFieldType() {
+            if (script.get() == null) {
+                return new IpScriptFieldType(name, IpFieldScript.PARSE_FROM_SOURCE, this);
+            }
             IpFieldScript.Factory factory = parserContext.scriptService().compile(script.getValue(), IpFieldScript.CONTEXT);
             return new IpScriptFieldType(name, factory, this);
         }

x-pack/plugin/runtime-fields/src/main/java/org/elasticsearch/xpack/runtimefields/mapper/KeywordScriptFieldType.java

@@ -44,6 +44,9 @@ public final class KeywordScriptFieldType extends AbstractScriptFieldType<String
     public static final RuntimeFieldType.Parser PARSER = new RuntimeFieldTypeParser((name, parserContext) -> new Builder(name) {
         @Override
         protected AbstractScriptFieldType<?> buildFieldType() {
+            if (script.get() == null) {
+                return new KeywordScriptFieldType(name, StringFieldScript.PARSE_FROM_SOURCE, this);
+            }
             StringFieldScript.Factory factory = parserContext.scriptService().compile(script.getValue(), StringFieldScript.CONTEXT);
             return new KeywordScriptFieldType(name, factory, this);
         }

x-pack/plugin/runtime-fields/src/main/java/org/elasticsearch/xpack/runtimefields/mapper/LongFieldScript.java

@@ -34,6 +34,28 @@ public interface LeafFactory {
         LongFieldScript newInstance(LeafReaderContext ctx);
     }
 
+    public static final Factory PARSE_FROM_SOURCE = (field, params, lookup) -> (LeafFactory) ctx -> new LongFieldScript(
+        field,
+        params,
+        lookup,
+        ctx
+    ) {
+        @Override
+        public void execute() {
+            for (Object v : extractFromSource(field)) {
+                if (v instanceof Number) {
+                    emit(((Number) v).longValue());
+                } else if (v instanceof String) {
+                    try {
+                        emit(Long.parseLong((String) v));
+                    } catch (NumberFormatException e) {
+                        // ignore
+                    }
+                }
+            }
+        }
+    };
+
     public LongFieldScript(String fieldName, Map<String, Object> params, SearchLookup searchLookup, LeafReaderContext ctx) {
         super(fieldName, params, searchLookup, ctx);
     }

x-pack/plugin/runtime-fields/src/main/java/org/elasticsearch/xpack/runtimefields/mapper/LongScriptFieldType.java

@@ -36,6 +36,9 @@ public final class LongScriptFieldType extends AbstractScriptFieldType<LongField
     public static final RuntimeFieldType.Parser PARSER = new RuntimeFieldTypeParser((name, parserContext) -> new Builder(name) {
         @Override
         protected AbstractScriptFieldType<?> buildFieldType() {
+            if (script.get() == null) {
+                return new LongScriptFieldType(name, LongFieldScript.PARSE_FROM_SOURCE, this);
+            }
             LongFieldScript.Factory factory = parserContext.scriptService().compile(script.getValue(), LongFieldScript.CONTEXT);
             return new LongScriptFieldType(name, factory, this);
         }

x-pack/plugin/runtime-fields/src/main/java/org/elasticsearch/xpack/runtimefields/mapper/StringFieldScript.java

@@ -41,6 +41,22 @@ public interface LeafFactory {
         StringFieldScript newInstance(LeafReaderContext ctx);
     }
 
+    public static final Factory PARSE_FROM_SOURCE = (field, params, lookup) -> (LeafFactory) ctx -> new StringFieldScript(
+        field,
+        params,
+        lookup,
+        ctx
+    ) {
+        @Override
+        public void execute() {
+            for (Object v : extractFromSource(field)) {
+                if (v != null) {
+                    emit(v.toString());
+                }
+            }
+        }
+    };
+
     private final List<String> results = new ArrayList<>();
     private long chars;
 

x-pack/plugin/runtime-fields/src/test/java/org/elasticsearch/xpack/runtimefields/mapper/AbstractScriptFieldTypeTestCase.java

@@ -102,12 +102,6 @@ public final void testMinimalMappingToMaximal() throws IOException {
         assertEquals(Strings.toString(orig), Strings.toString(parsedFromOrig));
     }
 
-    public void testScriptIsRequired() throws Exception {
-        XContentBuilder mapping = runtimeFieldMapping(b -> { b.field("type", typeName()); });
-        MapperParsingException exception = expectThrows(MapperParsingException.class, () -> createMapperService(mapping));
-        assertEquals("Failed to parse mapping: script must be specified for runtime field [field]", exception.getMessage());
-    }
-
     public void testCopyToIsNotSupported() throws IOException {
         XContentBuilder mapping = runtimeFieldMapping(b -> {
             minimalMapping(b);