Support malformed numbers in synthetic _source (#90428)

This adds support for `ignore_malformed` to numeric fields other than `scaled_float` in synthetic `_source`. Their values are saved to a stored field and loaded to render the `_source`.
elastic · Oct 4, 2022 · bc49392 · bc49392
1 parent ff242a1
commit bc49392
Show file tree

Hide file tree

Showing 50 changed files with 344 additions and 247 deletions.
diff --git a/docs/changelog/90428.yaml b/docs/changelog/90428.yaml
@@ -0,0 +1,5 @@
+pr: 90428
+summary: Support malformed numbers in synthetic `_source`
+area: TSDB
+type: enhancement
+issues: []
diff --git a/...-geo/src/test/java/org/elasticsearch/legacygeo/mapper/LegacyGeoShapeFieldMapperTests.java b/...-geo/src/test/java/org/elasticsearch/legacygeo/mapper/LegacyGeoShapeFieldMapperTests.java
@@ -663,7 +663,7 @@ protected Object generateRandomInputValue(MappedFieldType ft) {
     }
 
     @Override
-    protected SyntheticSourceSupport syntheticSourceSupport() {
+    protected SyntheticSourceSupport syntheticSourceSupport(boolean ignoreMalformed) {
         throw new AssumptionViolatedException("not supported");
     }
 

diff --git a/...er-extras/src/main/java/org/elasticsearch/index/mapper/extras/ScaledFloatFieldMapper.java b/...er-extras/src/main/java/org/elasticsearch/index/mapper/extras/ScaledFloatFieldMapper.java
@@ -709,7 +709,7 @@ public SourceLoader.SyntheticFieldLoader syntheticFieldLoader() {
                 "field [" + name() + "] of type [" + typeName() + "] doesn't support synthetic source because it declares copy_to"
             );
         }
-        return new SortedNumericDocValuesSyntheticFieldLoader(name(), simpleName()) {
+        return new SortedNumericDocValuesSyntheticFieldLoader(name(), simpleName(), ignoreMalformed.value()) {
             @Override
             protected void writeValue(XContentBuilder b, long value) throws IOException {
                 b.value(decodeForSyntheticSource(value, scalingFactor));

diff --git a/...as/src/test/java/org/elasticsearch/index/mapper/extras/MatchOnlyTextFieldMapperTests.java b/...as/src/test/java/org/elasticsearch/index/mapper/extras/MatchOnlyTextFieldMapperTests.java
@@ -213,7 +213,8 @@ protected boolean supportsIgnoreMalformed() {
     }
 
     @Override
-    protected SyntheticSourceSupport syntheticSourceSupport() {
+    protected SyntheticSourceSupport syntheticSourceSupport(boolean ignoreMalformed) {
+        assertFalse("match_only_text doesn't support ignoreMalformed", ignoreMalformed);
         return new MatchOnlyTextSyntheticSourceSupport();
     }
 

diff --git a/...tras/src/test/java/org/elasticsearch/index/mapper/extras/RankFeatureFieldMapperTests.java b/...tras/src/test/java/org/elasticsearch/index/mapper/extras/RankFeatureFieldMapperTests.java
@@ -165,7 +165,7 @@ protected Object generateRandomInputValue(MappedFieldType ft) {
     }
 
     @Override
-    protected SyntheticSourceSupport syntheticSourceSupport() {
+    protected SyntheticSourceSupport syntheticSourceSupport(boolean ignoreMalformed) {
         throw new AssumptionViolatedException("not supported");
     }
 

diff --git a/...ras/src/test/java/org/elasticsearch/index/mapper/extras/RankFeaturesFieldMapperTests.java b/...ras/src/test/java/org/elasticsearch/index/mapper/extras/RankFeaturesFieldMapperTests.java
@@ -184,7 +184,7 @@ protected boolean allowsNullValues() {
     }
 
     @Override
-    protected SyntheticSourceSupport syntheticSourceSupport() {
+    protected SyntheticSourceSupport syntheticSourceSupport(boolean syntheticSource) {
         throw new AssumptionViolatedException("not supported");
     }
 

diff --git a/...tras/src/test/java/org/elasticsearch/index/mapper/extras/ScaledFloatFieldMapperTests.java b/...tras/src/test/java/org/elasticsearch/index/mapper/extras/ScaledFloatFieldMapperTests.java
@@ -337,75 +337,76 @@ protected Object generateRandomInputValue(MappedFieldType ft) {
     }
 
     @Override
-    protected SyntheticSourceSupport syntheticSourceSupport() {
-        return new SyntheticSourceSupport() {
-            private final double scalingFactor = randomDoubleBetween(0, Double.MAX_VALUE, false);
-            private final Double nullValue = usually() ? null : round(randomValue());
-
-            @Override
-            public SyntheticSourceExample example(int maxValues) {
-                if (randomBoolean()) {
-                    Tuple<Double, Double> v = generateValue();
-                    return new SyntheticSourceExample(v.v1(), v.v2(), this::mapping);
-                }
-                List<Tuple<Double, Double>> values = randomList(1, maxValues, this::generateValue);
-                List<Double> in = values.stream().map(Tuple::v1).toList();
-                List<Double> outList = values.stream().map(Tuple::v2).sorted().toList();
-                Object out = outList.size() == 1 ? outList.get(0) : outList;
-                return new SyntheticSourceExample(in, out, this::mapping);
-            }
+    protected SyntheticSourceSupport syntheticSourceSupport(boolean ignoreMalformed) {
+        assumeFalse("scaled_float doesn't support ignore_malformed with synthetic _source", ignoreMalformed);
+        return new ScaledFloatSyntheticSourceSupport();
+    }
 
-            private Tuple<Double, Double> generateValue() {
-                if (nullValue != null && randomBoolean()) {
-                    return Tuple.tuple(null, nullValue);
-                }
-                double d = randomValue();
-                return Tuple.tuple(d, round(d));
+    private static class ScaledFloatSyntheticSourceSupport implements SyntheticSourceSupport {
+        private final double scalingFactor = randomDoubleBetween(0, Double.MAX_VALUE, false);
+        private final Double nullValue = usually() ? null : round(randomValue());
+
+        @Override
+        public SyntheticSourceExample example(int maxValues) {
+            if (randomBoolean()) {
+                Tuple<Double, Double> v = generateValue();
+                return new SyntheticSourceExample(v.v1(), v.v2(), this::mapping);
             }
+            List<Tuple<Double, Double>> values = randomList(1, maxValues, this::generateValue);
+            List<Double> in = values.stream().map(Tuple::v1).toList();
+            List<Double> outList = values.stream().map(Tuple::v2).sorted().toList();
+            Object out = outList.size() == 1 ? outList.get(0) : outList;
+            return new SyntheticSourceExample(in, out, this::mapping);
+        }
 
-            private double round(double d) {
-                long encoded = Math.round(d * scalingFactor);
-                double decoded = encoded / scalingFactor;
-                long reencoded = Math.round(decoded * scalingFactor);
-                if (encoded != reencoded) {
-                    if (encoded > reencoded) {
-                        return decoded + Math.ulp(decoded);
-                    }
-                    return decoded - Math.ulp(decoded);
-                }
-                return decoded;
+        private Tuple<Double, Double> generateValue() {
+            if (nullValue != null && randomBoolean()) {
+                return Tuple.tuple(null, nullValue);
             }
+            double d = randomValue();
+            return Tuple.tuple(d, round(d));
+        }
 
-            private void mapping(XContentBuilder b) throws IOException {
-                b.field("type", "scaled_float");
-                b.field("scaling_factor", scalingFactor);
-                if (nullValue != null) {
-                    b.field("null_value", nullValue);
-                }
-                if (rarely()) {
-                    b.field("index", false);
-                }
-                if (rarely()) {
-                    b.field("store", false);
+        private double round(double d) {
+            long encoded = Math.round(d * scalingFactor);
+            double decoded = encoded / scalingFactor;
+            long reencoded = Math.round(decoded * scalingFactor);
+            if (encoded != reencoded) {
+                if (encoded > reencoded) {
+                    return decoded + Math.ulp(decoded);
                 }
+                return decoded - Math.ulp(decoded);
             }
+            return decoded;
+        }
 
-            @Override
-            public List<SyntheticSourceInvalidExample> invalidExample() throws IOException {
-                return List.of(
-                    new SyntheticSourceInvalidExample(
-                        equalTo("field [field] of type [scaled_float] doesn't support synthetic source because it doesn't have doc values"),
-                        b -> b.field("type", "scaled_float").field("scaling_factor", 10).field("doc_values", false)
-                    ),
-                    new SyntheticSourceInvalidExample(
-                        equalTo(
-                            "field [field] of type [scaled_float] doesn't support synthetic source because it ignores malformed numbers"
-                        ),
-                        b -> b.field("type", "scaled_float").field("scaling_factor", 10).field("ignore_malformed", true)
-                    )
-                );
+        private void mapping(XContentBuilder b) throws IOException {
+            b.field("type", "scaled_float");
+            b.field("scaling_factor", scalingFactor);
+            if (nullValue != null) {
+                b.field("null_value", nullValue);
             }
-        };
+            if (rarely()) {
+                b.field("index", false);
+            }
+            if (rarely()) {
+                b.field("store", false);
+            }
+        }
+
+        @Override
+        public List<SyntheticSourceInvalidExample> invalidExample() throws IOException {
+            return List.of(
+                new SyntheticSourceInvalidExample(
+                    equalTo("field [field] of type [scaled_float] doesn't support synthetic source because it doesn't have doc values"),
+                    b -> b.field("type", "scaled_float").field("scaling_factor", 10).field("doc_values", false)
+                ),
+                new SyntheticSourceInvalidExample(
+                    equalTo("field [field] of type [scaled_float] doesn't support synthetic source because it ignores malformed numbers"),
+                    b -> b.field("type", "scaled_float").field("scaling_factor", 10).field("ignore_malformed", true)
+                )
+            );
+        }
     }
 
     @Override
@@ -504,7 +505,7 @@ private double encodeDecode(double value, double scalingFactor) {
         return ScaledFloatFieldMapper.decodeForSyntheticSource(ScaledFloatFieldMapper.encode(value, scalingFactor), scalingFactor);
     }
 
-    private double randomValue() {
+    private static double randomValue() {
         return randomBoolean() ? randomDoubleBetween(-Double.MAX_VALUE, Double.MAX_VALUE, true) : randomFloat();
     }
 }
diff --git a/.../src/test/java/org/elasticsearch/index/mapper/extras/SearchAsYouTypeFieldMapperTests.java b/.../src/test/java/org/elasticsearch/index/mapper/extras/SearchAsYouTypeFieldMapperTests.java
@@ -801,7 +801,7 @@ protected boolean supportsIgnoreMalformed() {
     }
 
     @Override
-    protected SyntheticSourceSupport syntheticSourceSupport() {
+    protected SyntheticSourceSupport syntheticSourceSupport(boolean syntheticSource) {
         throw new AssumptionViolatedException("not supported");
     }
 

diff --git a/...xtras/src/test/java/org/elasticsearch/index/mapper/extras/TokenCountFieldMapperTests.java b/...xtras/src/test/java/org/elasticsearch/index/mapper/extras/TokenCountFieldMapperTests.java
@@ -195,7 +195,7 @@ protected boolean supportsIgnoreMalformed() {
     }
 
     @Override
-    protected SyntheticSourceSupport syntheticSourceSupport() {
+    protected SyntheticSourceSupport syntheticSourceSupport(boolean ignoreMalformed) {
         throw new AssumptionViolatedException("not supported");
     }
 

diff --git a/.../test/java/org/elasticsearch/plugin/analysis/icu/ICUCollationKeywordFieldMapperTests.java b/.../test/java/org/elasticsearch/plugin/analysis/icu/ICUCollationKeywordFieldMapperTests.java
@@ -308,7 +308,7 @@ protected boolean supportsIgnoreMalformed() {
     }
 
     @Override
-    protected SyntheticSourceSupport syntheticSourceSupport() {
+    protected SyntheticSourceSupport syntheticSourceSupport(boolean ignoreMalformed) {
         throw new AssumptionViolatedException("not supported");
     }
 

diff --git a/...test/java/org/elasticsearch/index/mapper/annotatedtext/AnnotatedTextFieldMapperTests.java b/...test/java/org/elasticsearch/index/mapper/annotatedtext/AnnotatedTextFieldMapperTests.java
@@ -595,7 +595,7 @@ protected boolean supportsIgnoreMalformed() {
     }
 
     @Override
-    protected SyntheticSourceSupport syntheticSourceSupport() {
+    protected SyntheticSourceSupport syntheticSourceSupport(boolean ignoreMalformed) {
         throw new AssumptionViolatedException("not supported");
     }
 

diff --git a/...murmur3/src/test/java/org/elasticsearch/index/mapper/murmur3/Murmur3FieldMapperTests.java b/...murmur3/src/test/java/org/elasticsearch/index/mapper/murmur3/Murmur3FieldMapperTests.java
@@ -70,7 +70,7 @@ protected boolean supportsIgnoreMalformed() {
     }
 
     @Override
-    protected SyntheticSourceSupport syntheticSourceSupport() {
+    protected SyntheticSourceSupport syntheticSourceSupport(boolean ignoreMalformed) {
         throw new AssumptionViolatedException("not supported");
     }
 

diff --git a/server/src/main/java/org/elasticsearch/index/mapper/BooleanFieldMapper.java b/server/src/main/java/org/elasticsearch/index/mapper/BooleanFieldMapper.java
@@ -466,7 +466,7 @@ public SourceLoader.SyntheticFieldLoader syntheticFieldLoader() {
                 "field [" + name() + "] of type [" + typeName() + "] doesn't support synthetic source because it declares copy_to"
             );
         }
-        return new SortedNumericDocValuesSyntheticFieldLoader(name(), simpleName()) {
+        return new SortedNumericDocValuesSyntheticFieldLoader(name(), simpleName(), false) {
             @Override
             protected void writeValue(XContentBuilder b, long value) throws IOException {
                 b.value(value == 1);

diff --git a/server/src/main/java/org/elasticsearch/index/mapper/DateFieldMapper.java b/server/src/main/java/org/elasticsearch/index/mapper/DateFieldMapper.java
@@ -955,7 +955,7 @@ public SourceLoader.SyntheticFieldLoader syntheticFieldLoader() {
                 "field [" + name() + "] of type [" + typeName() + "] doesn't support synthetic source because it declares copy_to"
             );
         }
-        return new SortedNumericDocValuesSyntheticFieldLoader(name(), simpleName()) {
+        return new SortedNumericDocValuesSyntheticFieldLoader(name(), simpleName(), ignoreMalformed) {
             @Override
             protected void writeValue(XContentBuilder b, long value) throws IOException {
                 b.value(fieldType().format(value, fieldType().dateTimeFormatter()));

diff --git a/server/src/main/java/org/elasticsearch/index/mapper/GeoPointFieldMapper.java b/server/src/main/java/org/elasticsearch/index/mapper/GeoPointFieldMapper.java
@@ -497,7 +497,7 @@ public SourceLoader.SyntheticFieldLoader syntheticFieldLoader() {
                 "field [" + name() + "] of type [" + typeName() + "] doesn't support synthetic source because it declares copy_to"
             );
         }
-        return new SortedNumericDocValuesSyntheticFieldLoader(name(), simpleName()) {
+        return new SortedNumericDocValuesSyntheticFieldLoader(name(), simpleName(), ignoreMalformed()) {
             final GeoPoint point = new GeoPoint();
 
             @Override

diff --git a/server/src/main/java/org/elasticsearch/index/mapper/NumberFieldMapper.java b/server/src/main/java/org/elasticsearch/index/mapper/NumberFieldMapper.java
@@ -399,8 +399,8 @@ private static void validateParsed(float value) {
             }
 
             @Override
-            SourceLoader.SyntheticFieldLoader syntheticFieldLoader(String fieldName, String fieldSimpleName) {
-                return new SortedNumericDocValuesSyntheticFieldLoader(fieldName, fieldSimpleName) {
+            SourceLoader.SyntheticFieldLoader syntheticFieldLoader(String fieldName, String fieldSimpleName, boolean ignoreMalformed) {
+                return new SortedNumericDocValuesSyntheticFieldLoader(fieldName, fieldSimpleName, ignoreMalformed) {
                     @Override
                     protected void writeValue(XContentBuilder b, long value) throws IOException {
                         b.value(HalfFloatPoint.sortableShortToHalfFloat((short) value));
@@ -549,8 +549,8 @@ private static void validateParsed(float value) {
             }
 
             @Override
-            SourceLoader.SyntheticFieldLoader syntheticFieldLoader(String fieldName, String fieldSimpleName) {
-                return new SortedNumericDocValuesSyntheticFieldLoader(fieldName, fieldSimpleName) {
+            SourceLoader.SyntheticFieldLoader syntheticFieldLoader(String fieldName, String fieldSimpleName, boolean ignoreMalformed) {
+                return new SortedNumericDocValuesSyntheticFieldLoader(fieldName, fieldSimpleName, ignoreMalformed) {
                     @Override
                     protected void writeValue(XContentBuilder b, long value) throws IOException {
                         b.value(NumericUtils.sortableIntToFloat((int) value));
@@ -677,8 +677,8 @@ private static void validateParsed(double value) {
             }
 
             @Override
-            SourceLoader.SyntheticFieldLoader syntheticFieldLoader(String fieldName, String fieldSimpleName) {
-                return new SortedNumericDocValuesSyntheticFieldLoader(fieldName, fieldSimpleName) {
+            SourceLoader.SyntheticFieldLoader syntheticFieldLoader(String fieldName, String fieldSimpleName, boolean ignoreMalformed) {
+                return new SortedNumericDocValuesSyntheticFieldLoader(fieldName, fieldSimpleName, ignoreMalformed) {
                     @Override
                     protected void writeValue(XContentBuilder b, long value) throws IOException {
                         b.value(NumericUtils.sortableLongToDouble(value));
@@ -774,8 +774,8 @@ public IndexFieldData.Builder getValueFetcherFieldDataBuilder(
             }
 
             @Override
-            SourceLoader.SyntheticFieldLoader syntheticFieldLoader(String fieldName, String fieldSimpleName) {
-                return NumberType.syntheticLongFieldLoader(fieldName, fieldSimpleName);
+            SourceLoader.SyntheticFieldLoader syntheticFieldLoader(String fieldName, String fieldSimpleName, boolean ignoreMalformed) {
+                return NumberType.syntheticLongFieldLoader(fieldName, fieldSimpleName, ignoreMalformed);
             }
         },
         SHORT("short", NumericType.SHORT) {
@@ -862,8 +862,8 @@ public IndexFieldData.Builder getValueFetcherFieldDataBuilder(
             }
 
             @Override
-            SourceLoader.SyntheticFieldLoader syntheticFieldLoader(String fieldName, String fieldSimpleName) {
-                return NumberType.syntheticLongFieldLoader(fieldName, fieldSimpleName);
+            SourceLoader.SyntheticFieldLoader syntheticFieldLoader(String fieldName, String fieldSimpleName, boolean ignoreMalformed) {
+                return NumberType.syntheticLongFieldLoader(fieldName, fieldSimpleName, ignoreMalformed);
             }
         },
         INTEGER("integer", NumericType.INT) {
@@ -1017,8 +1017,8 @@ public IndexFieldData.Builder getValueFetcherFieldDataBuilder(
             }
 
             @Override
-            SourceLoader.SyntheticFieldLoader syntheticFieldLoader(String fieldName, String fieldSimpleName) {
-                return NumberType.syntheticLongFieldLoader(fieldName, fieldSimpleName);
+            SourceLoader.SyntheticFieldLoader syntheticFieldLoader(String fieldName, String fieldSimpleName, boolean ignoreMalformed) {
+                return NumberType.syntheticLongFieldLoader(fieldName, fieldSimpleName, ignoreMalformed);
             }
         },
         LONG("long", NumericType.LONG) {
@@ -1142,8 +1142,8 @@ public IndexFieldData.Builder getValueFetcherFieldDataBuilder(
             }
 
             @Override
-            SourceLoader.SyntheticFieldLoader syntheticFieldLoader(String fieldName, String fieldSimpleName) {
-                return syntheticLongFieldLoader(fieldName, fieldSimpleName);
+            SourceLoader.SyntheticFieldLoader syntheticFieldLoader(String fieldName, String fieldSimpleName, boolean ignoreMalformed) {
+                return syntheticLongFieldLoader(fieldName, fieldSimpleName, ignoreMalformed);
             }
         };
 
@@ -1382,10 +1382,14 @@ public double reduceToStoredPrecision(double value) {
             return ((Number) value).doubleValue();
         }
 
-        abstract SourceLoader.SyntheticFieldLoader syntheticFieldLoader(String fieldName, String fieldSimpleName);
+        abstract SourceLoader.SyntheticFieldLoader syntheticFieldLoader(String fieldName, String fieldSimpleName, boolean ignoreMalformed);
 
-        private static SourceLoader.SyntheticFieldLoader syntheticLongFieldLoader(String fieldName, String fieldSimpleName) {
-            return new SortedNumericDocValuesSyntheticFieldLoader(fieldName, fieldSimpleName) {
+        private static SourceLoader.SyntheticFieldLoader syntheticLongFieldLoader(
+            String fieldName,
+            String fieldSimpleName,
+            boolean ignoreMalformed
+        ) {
+            return new SortedNumericDocValuesSyntheticFieldLoader(fieldName, fieldSimpleName, ignoreMalformed) {
                 @Override
                 protected void writeValue(XContentBuilder b, long value) throws IOException {
                     b.value(value);
@@ -1670,6 +1674,10 @@ protected void parseCreateField(DocumentParserContext context) throws IOExceptio
         } catch (IllegalArgumentException e) {
             if (ignoreMalformed.value() && context.parser().currentToken().isValue()) {
                 context.addIgnoredField(mappedFieldType.name());
+                if (context.isSyntheticSource()) {
+                    // Save a copy of the field so synthetic source can load it
+                    context.doc().add(IgnoreMalformedStoredValues.storedField(name(), context.parser()));
+                }
                 return;
             } else {
                 throw e;
@@ -1757,17 +1765,12 @@ public SourceLoader.SyntheticFieldLoader syntheticFieldLoader() {
                 "field [" + name() + "] of type [" + typeName() + "] doesn't support synthetic source because it doesn't have doc values"
             );
         }
-        if (ignoreMalformed.value()) {
-            throw new IllegalArgumentException(
-                "field [" + name() + "] of type [" + typeName() + "] doesn't support synthetic source because it ignores malformed numbers"
-            );
-        }
         if (copyTo.copyToFields().isEmpty() != true) {
             throw new IllegalArgumentException(
                 "field [" + name() + "] of type [" + typeName() + "] doesn't support synthetic source because it declares copy_to"
             );
         }
-        return type.syntheticFieldLoader(name(), simpleName());
+        return type.syntheticFieldLoader(name(), simpleName(), ignoreMalformed.value());
     }
 
     // For testing only: