From 16cb2eeb423a57c79b81786284cdb82a42ae416c Mon Sep 17 00:00:00 2001
From: Kostas Krikellas
Date: Wed, 17 Apr 2024 16:48:06 +0300
Subject: [PATCH 01/19] Add ignored field values to synthetic source
---
.../indices.create/20_synthetic_source.yml | 151 ++++++++
.../index/mapper/DocumentParserContext.java | 30 ++
.../index/mapper/FieldDataParseHelper.java | 357 ++++++++++++++++++
.../mapper/IgnoreMalformedStoredValues.java | 141 +------
.../mapper/IgnoredValuesFieldMapper.java | 169 +++++++++
.../index/mapper/ObjectMapper.java | 21 +-
.../index/mapper/SourceFieldMapper.java | 8 +
.../elasticsearch/indices/IndicesModule.java | 2 +
.../mapper/DocCountFieldMapperTests.java | 2 -
.../index/mapper/DocumentMapperTests.java | 2 +
.../mapper/FieldDataParseHelperTests.java | 107 ++++++
.../mapper/IgnoredValuedFieldMapperTests.java | 148 ++++++++
.../indices/IndicesModuleTests.java | 2 +
13 files changed, 998 insertions(+), 142 deletions(-)
create mode 100644 server/src/main/java/org/elasticsearch/index/mapper/FieldDataParseHelper.java
create mode 100644 server/src/main/java/org/elasticsearch/index/mapper/IgnoredValuesFieldMapper.java
create mode 100644 server/src/test/java/org/elasticsearch/index/mapper/FieldDataParseHelperTests.java
create mode 100644 server/src/test/java/org/elasticsearch/index/mapper/IgnoredValuedFieldMapperTests.java
diff --git a/rest-api-spec/src/yamlRestTest/resources/rest-api-spec/test/indices.create/20_synthetic_source.yml b/rest-api-spec/src/yamlRestTest/resources/rest-api-spec/test/indices.create/20_synthetic_source.yml
index 62a4e240a5b5d..9c9675d41febe 100644
--- a/rest-api-spec/src/yamlRestTest/resources/rest-api-spec/test/indices.create/20_synthetic_source.yml
+++ b/rest-api-spec/src/yamlRestTest/resources/rest-api-spec/test/indices.create/20_synthetic_source.yml
@@ -36,3 +36,154 @@ nested is disabled:
properties:
foo:
type: keyword
+
+---
+object with unmapped fields:
+ - skip:
+ version: " - 8.13.99"
+ reason: introduced in 8.14.0
+
+ - do:
+ indices.create:
+ index: test
+ body:
+ settings:
+ index:
+ mapping:
+ total_fields:
+ ignore_dynamic_beyond_limit: true
+ limit: 1
+
+ mappings:
+ _source:
+ mode: synthetic
+ properties:
+ name:
+ type: keyword
+
+ - do:
+ bulk:
+ index: test
+ refresh: true
+ body:
+ - '{ "create": { } }'
+ - '{ "name": "aaaa", "some_string": "AaAa", "some_int": 1000, "some_double": 123.456789, "some_bool": true, "a.very.deeply.nested.field": "AAAA" }'
+ - '{ "create": { } }'
+ - '{ "name": "bbbb", "some_string": "BbBb", "some_int": 2000, "some_double": 321.987654, "some_bool": false, "a.very.deeply.nested.field": "BBBB" }'
+
+ - do:
+ search:
+ index: test
+
+ - match: { hits.total.value: 2 }
+ - match: { hits.hits.0._source.name: aaaa }
+ - match: { hits.hits.0._source.some_string: AaAa }
+ - match: { hits.hits.0._source.some_int: 1000 }
+ - match: { hits.hits.0._source.some_double: 123.456789 }
+ - match: { hits.hits.0._source.a.very.deeply.nested.field: AAAA }
+ - match: { hits.hits.0._source.some_bool: true }
+ - match: { hits.hits.1._source.name: bbbb }
+ - match: { hits.hits.1._source.some_string: BbBb }
+ - match: { hits.hits.1._source.some_int: 2000 }
+ - match: { hits.hits.1._source.some_double: 321.987654 }
+ - match: { hits.hits.1._source.a.very.deeply.nested.field: BBBB }
+
+
+---
+nested object with unmapped fields:
+ - skip:
+ version: " - 8.13.99"
+ reason: introduced in 8.14.0
+
+ - do:
+ indices.create:
+ index: test
+ body:
+ settings:
+ index:
+ mapping:
+ total_fields:
+ ignore_dynamic_beyond_limit: true
+ limit: 3
+
+ mappings:
+ _source:
+ mode: synthetic
+ properties:
+ path:
+ properties:
+ to:
+ properties:
+ name:
+ type: keyword
+
+ - do:
+ bulk:
+ index: test
+ refresh: true
+ body:
+ - '{ "create": { } }'
+ - '{ "path.to.name": "aaaa", "path.to.surname": "AaAa", "path.some.other.name": "AaAaAa" }'
+ - '{ "create": { } }'
+ - '{ "path.to.name": "bbbb", "path.to.surname": "BbBb", "path.some.other.name": "BbBbBb" }'
+
+ - do:
+ search:
+ index: test
+
+ - match: { hits.total.value: 2 }
+ - match: { hits.hits.0._source.path.to.name: aaaa }
+ - match: { hits.hits.0._source.path.to.surname: AaAa }
+ - match: { hits.hits.0._source.path.some.other.name: AaAaAa }
+ - match: { hits.hits.1._source.path.to.name: bbbb }
+ - match: { hits.hits.1._source.path.to.surname: BbBb }
+ - match: { hits.hits.1._source.path.some.other.name: BbBbBb }
+
+
+---
+empty object with unmapped fields:
+ - skip:
+ version: " - 8.13.99"
+ reason: introduced in 8.14.0
+
+ - do:
+ indices.create:
+ index: test
+ body:
+ settings:
+ index:
+ mapping:
+ total_fields:
+ ignore_dynamic_beyond_limit: true
+ limit: 3
+
+ mappings:
+ _source:
+ mode: synthetic
+ properties:
+ path:
+ properties:
+ to:
+ properties:
+ name:
+ type: keyword
+
+ - do:
+ bulk:
+ index: test
+ refresh: true
+ body:
+ - '{ "create": { } }'
+ - '{ "path.to.surname": "AaAa", "path.some.other.name": "AaAaAa" }'
+ - '{ "create": { } }'
+ - '{ "path.to.surname": "BbBb", "path.some.other.name": "BbBbBb" }'
+
+ - do:
+ search:
+ index: test
+
+ - match: { hits.total.value: 2 }
+ - match: { hits.hits.0._source.path.to.surname: AaAa }
+ - match: { hits.hits.0._source.path.some.other.name: AaAaAa }
+ - match: { hits.hits.1._source.path.to.surname: BbBb }
+ - match: { hits.hits.1._source.path.some.other.name: BbBbBb }
diff --git a/server/src/main/java/org/elasticsearch/index/mapper/DocumentParserContext.java b/server/src/main/java/org/elasticsearch/index/mapper/DocumentParserContext.java
index a42477bed2146..ad6014eb521bd 100644
--- a/server/src/main/java/org/elasticsearch/index/mapper/DocumentParserContext.java
+++ b/server/src/main/java/org/elasticsearch/index/mapper/DocumentParserContext.java
@@ -104,6 +104,7 @@ public int get() {
private final MappingParserContext mappingParserContext;
private final SourceToParse sourceToParse;
private final Set ignoredFields;
+ private final List ignoredFieldValues;
private final Map> dynamicMappers;
private final DynamicMapperSize dynamicMappersSize;
private final Map dynamicObjectMappers;
@@ -122,6 +123,7 @@ private DocumentParserContext(
MappingParserContext mappingParserContext,
SourceToParse sourceToParse,
Set ignoreFields,
+ List ignoredFieldValues,
Map> dynamicMappers,
Map dynamicObjectMappers,
Map> dynamicRuntimeFields,
@@ -139,6 +141,7 @@ private DocumentParserContext(
this.mappingParserContext = mappingParserContext;
this.sourceToParse = sourceToParse;
this.ignoredFields = ignoreFields;
+ this.ignoredFieldValues = ignoredFieldValues;
this.dynamicMappers = dynamicMappers;
this.dynamicObjectMappers = dynamicObjectMappers;
this.dynamicRuntimeFields = dynamicRuntimeFields;
@@ -159,6 +162,7 @@ private DocumentParserContext(ObjectMapper parent, ObjectMapper.Dynamic dynamic,
in.mappingParserContext,
in.sourceToParse,
in.ignoredFields,
+ in.ignoredFieldValues,
in.dynamicMappers,
in.dynamicObjectMappers,
in.dynamicRuntimeFields,
@@ -186,6 +190,7 @@ protected DocumentParserContext(
mappingParserContext,
source,
new HashSet<>(),
+ new ArrayList<>(),
new HashMap<>(),
new HashMap<>(),
new HashMap<>(),
@@ -251,6 +256,20 @@ public final Collection getIgnoredFields() {
return Collections.unmodifiableCollection(ignoredFields);
}
+ /**
+ * Add the given ignored values to the corresponding list.
+ */
+ public final void addIgnoredValues(IgnoredValuesFieldMapper.Values values) {
+ ignoredFieldValues.add(values);
+ }
+
+ /**
+ * Return the collection of values for fields that have been ignored so far.
+ */
+ public final Collection getIgnoredFieldValues() {
+ return Collections.unmodifiableCollection(ignoredFieldValues);
+ }
+
/**
* Add the given {@code field} to the _field_names field
*
@@ -345,6 +364,17 @@ public final boolean addDynamicMapper(Mapper mapper) {
int additionalFieldsToAdd = getNewFieldsSize() + mapperSize;
if (indexSettings().isIgnoreDynamicFieldsBeyondLimit()) {
if (mappingLookup.exceedsLimit(indexSettings().getMappingTotalFieldsLimit(), additionalFieldsToAdd)) {
+ if (indexSettings().getMode().isSyntheticSourceEnabled() || SourceFieldMapper.isSynthetic(mappingLookup)) {
+ try {
+ int parentOffset = parent() instanceof RootObjectMapper ? 0 : parent().fullPath().length() + 1;
+ addIgnoredValues(
+ new IgnoredValuesFieldMapper.Values(mapper.name(), parentOffset, FieldDataParseHelper.encodeToken(parser()))
+ );
+ } catch (IOException e) {
+ // A parsing failure for ignored field values should be *very* rare.
+ // It is not considered important enough to drop the document, ignore it silently for now.
+ }
+ }
addIgnoredField(mapper.name());
return false;
}
diff --git a/server/src/main/java/org/elasticsearch/index/mapper/FieldDataParseHelper.java b/server/src/main/java/org/elasticsearch/index/mapper/FieldDataParseHelper.java
new file mode 100644
index 0000000000000..1478f905165d5
--- /dev/null
+++ b/server/src/main/java/org/elasticsearch/index/mapper/FieldDataParseHelper.java
@@ -0,0 +1,357 @@
+/*
+ * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
+ * or more contributor license agreements. Licensed under the Elastic License
+ * 2.0 and the Server Side Public License, v 1; you may not use this file except
+ * in compliance with, at your election, the Elastic License 2.0 or the Server
+ * Side Public License, v 1.
+ */
+
+package org.elasticsearch.index.mapper;
+
+import org.apache.lucene.document.StoredField;
+import org.apache.lucene.util.BytesRef;
+import org.apache.lucene.util.BytesRefIterator;
+import org.elasticsearch.common.bytes.BytesReference;
+import org.elasticsearch.common.util.ByteUtils;
+import org.elasticsearch.core.CheckedFunction;
+import org.elasticsearch.xcontent.XContentBuilder;
+import org.elasticsearch.xcontent.XContentParser;
+import org.elasticsearch.xcontent.XContentParserConfiguration;
+import org.elasticsearch.xcontent.XContentType;
+
+import java.io.IOException;
+import java.math.BigDecimal;
+import java.math.BigInteger;
+import java.nio.charset.StandardCharsets;
+
+/**
+ * Helper class for processing field data of any type, as provided by the {@link XContentParser}.
+ */
+public final class FieldDataParseHelper {
+ /**
+ * Build a {@link StoredField} for the value on which the parser is
+ * currently positioned.
+ *
+ * We try to use {@link StoredField}'s native types for fields where
+ * possible but we have to preserve more type information than
+ * stored fields support, so we encode all of those into stored fields'
+ * {@code byte[]} type and then encode type information in the first byte.
+ *
- * We try to use {@link StoredField}'s native types for fields where
- * possible but we have to preserve more type information than
- * stored fields support, so we encode all of those into stored fields'
- * {@code byte[]} type and then encode type information in the first byte.
- *
- */
- public static StoredField storedField(String fieldName, XContentParser parser) throws IOException {
- String name = name(fieldName);
- return switch (parser.currentToken()) {
- case VALUE_STRING -> new StoredField(name, parser.text());
- case VALUE_NUMBER -> switch (parser.numberType()) {
- case INT -> new StoredField(name, parser.intValue());
- case LONG -> new StoredField(name, parser.longValue());
- case DOUBLE -> new StoredField(name, parser.doubleValue());
- case FLOAT -> new StoredField(name, parser.floatValue());
- case BIG_INTEGER -> new StoredField(name, encode((BigInteger) parser.numberValue()));
- case BIG_DECIMAL -> new StoredField(name, encode((BigDecimal) parser.numberValue()));
- };
- case VALUE_BOOLEAN -> new StoredField(name, new byte[] { parser.booleanValue() ? (byte) 't' : (byte) 'f' });
- case VALUE_EMBEDDED_OBJECT -> new StoredField(name, encode(parser.binaryValue()));
- case START_OBJECT, START_ARRAY -> {
- try (XContentBuilder builder = XContentBuilder.builder(parser.contentType().xContent())) {
- builder.copyCurrentStructure(parser);
- yield new StoredField(name, encode(builder));
- }
- }
- default -> throw new IllegalArgumentException("synthetic _source doesn't support malformed objects");
- };
+ public static StoredField storedField(String name, XContentParser parser) throws IOException {
+ return FieldDataParseHelper.storedField(name(name), parser);
}
/**
@@ -136,114 +99,16 @@ public int count() {
public void write(XContentBuilder b) throws IOException {
for (Object v : values) {
if (v instanceof BytesRef r) {
- decodeAndWrite(b, r);
+ FieldDataParseHelper.decodeAndWrite(b, r);
} else {
b.value(v);
}
}
values = emptyList();
}
-
- private static void decodeAndWrite(XContentBuilder b, BytesRef r) throws IOException {
- switch (r.bytes[r.offset]) {
- case 'b':
- b.value(r.bytes, r.offset + 1, r.length - 1);
- return;
- case 'c':
- decodeAndWriteXContent(b, XContentType.CBOR, r);
- return;
- case 'd':
- if (r.length < 5) {
- throw new IllegalArgumentException("Can't decode " + r);
- }
- int scale = ByteUtils.readIntLE(r.bytes, r.offset + 1);
- b.value(new BigDecimal(new BigInteger(r.bytes, r.offset + 5, r.length - 5), scale));
- return;
- case 'f':
- if (r.length != 1) {
- throw new IllegalArgumentException("Can't decode " + r);
- }
- b.value(false);
- return;
- case 'i':
- b.value(new BigInteger(r.bytes, r.offset + 1, r.length - 1));
- return;
- case 'j':
- decodeAndWriteXContent(b, XContentType.JSON, r);
- return;
- case 's':
- decodeAndWriteXContent(b, XContentType.SMILE, r);
- return;
- case 't':
- if (r.length != 1) {
- throw new IllegalArgumentException("Can't decode " + r);
- }
- b.value(true);
- return;
- case 'y':
- decodeAndWriteXContent(b, XContentType.YAML, r);
- return;
- default:
- throw new IllegalArgumentException("Can't decode " + r);
- }
- }
-
- private static void decodeAndWriteXContent(XContentBuilder b, XContentType type, BytesRef r) throws IOException {
- try (
- XContentParser parser = type.xContent().createParser(XContentParserConfiguration.EMPTY, r.bytes, r.offset + 1, r.length - 1)
- ) {
- b.copyCurrentStructure(parser);
- }
- }
}
private static String name(String fieldName) {
return fieldName + "._ignore_malformed";
}
-
- private static byte[] encode(BigInteger n) {
- byte[] twosCompliment = n.toByteArray();
- byte[] encoded = new byte[1 + twosCompliment.length];
- encoded[0] = 'i';
- System.arraycopy(twosCompliment, 0, encoded, 1, twosCompliment.length);
- return encoded;
- }
-
- private static byte[] encode(BigDecimal n) {
- byte[] twosCompliment = n.unscaledValue().toByteArray();
- byte[] encoded = new byte[5 + twosCompliment.length];
- encoded[0] = 'd';
- ByteUtils.writeIntLE(n.scale(), encoded, 1);
- System.arraycopy(twosCompliment, 0, encoded, 5, twosCompliment.length);
- return encoded;
- }
-
- private static byte[] encode(byte[] b) {
- byte[] encoded = new byte[1 + b.length];
- encoded[0] = 'b';
- System.arraycopy(b, 0, encoded, 1, b.length);
- return encoded;
- }
-
- private static byte[] encode(XContentBuilder builder) throws IOException {
- BytesReference b = BytesReference.bytes(builder);
- byte[] encoded = new byte[1 + b.length()];
- encoded[0] = switch (builder.contentType()) {
- case JSON -> 'j';
- case SMILE -> 's';
- case YAML -> 'y';
- case CBOR -> 'c';
- default -> throw new IllegalArgumentException("unsupported type " + builder.contentType());
- };
-
- int position = 1;
- BytesRefIterator itr = b.iterator();
- BytesRef ref;
- while ((ref = itr.next()) != null) {
- System.arraycopy(ref.bytes, ref.offset, encoded, position, ref.length);
- position += ref.length;
- }
- assert position == encoded.length;
- return encoded;
- }
}
diff --git a/server/src/main/java/org/elasticsearch/index/mapper/IgnoredValuesFieldMapper.java b/server/src/main/java/org/elasticsearch/index/mapper/IgnoredValuesFieldMapper.java
new file mode 100644
index 0000000000000..e85a853e00946
--- /dev/null
+++ b/server/src/main/java/org/elasticsearch/index/mapper/IgnoredValuesFieldMapper.java
@@ -0,0 +1,169 @@
+/*
+ * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
+ * or more contributor license agreements. Licensed under the Elastic License
+ * 2.0 and the Server Side Public License, v 1; you may not use this file except
+ * in compliance with, at your election, the Elastic License 2.0 or the Server
+ * Side Public License, v 1.
+ */
+
+package org.elasticsearch.index.mapper;
+
+import org.apache.lucene.document.Field;
+import org.apache.lucene.document.StringField;
+import org.apache.lucene.index.LeafReader;
+import org.apache.lucene.util.BytesRef;
+import org.elasticsearch.common.util.ByteUtils;
+import org.elasticsearch.index.query.SearchExecutionContext;
+import org.elasticsearch.xcontent.XContentBuilder;
+
+import java.io.IOException;
+import java.nio.charset.StandardCharsets;
+import java.util.ArrayList;
+import java.util.Base64;
+import java.util.Collections;
+import java.util.HashMap;
+import java.util.List;
+import java.util.Map;
+import java.util.stream.Stream;
+
+/**
+ * Mapper for the {@code _ignored_values} field.
+ *
+ * A field mapper that records fields that have been ignored, along with their values. It's intended for use
+ * in indexes with synthetic source to reconstruct the latter, taking into account fields that got ignored during
+ * indexing.
+ *
+ * This overlaps with {@link IgnoredFieldMapper} that tracks just the ignored field names. It's worth evaluating
+ * if we can replace it for all use cases to avoid duplication, assuming that the storage tradeoff is favorable.
+ */
+public class IgnoredValuesFieldMapper extends MetadataFieldMapper {
+
+ private static final int OFFSET_IN_NAME = 100_000;
+
+ public static final String NAME = "_ignored_values";
+
+ public static final IgnoredValuesFieldMapper INSTANCE = new IgnoredValuesFieldMapper();
+
+ public static final TypeParser PARSER = new FixedTypeParser(context -> INSTANCE);
+
+ public record Values(String name, int parentOffset, BytesRef value) {}
+
+ static final class IgnoredValuesFieldMapperType extends StringFieldType {
+
+ private static final IgnoredValuesFieldMapperType INSTANCE = new IgnoredValuesFieldMapperType();
+
+ private IgnoredValuesFieldMapperType() {
+ super(NAME, false, true, false, TextSearchInfo.NONE, Collections.emptyMap());
+ }
+
+ @Override
+ public String typeName() {
+ return NAME;
+ }
+
+ @Override
+ public ValueFetcher valueFetcher(SearchExecutionContext context, String format) {
+ return new StoredValueFetcher(context.lookup(), NAME);
+ }
+ }
+
+ private IgnoredValuesFieldMapper() {
+ super(IgnoredValuesFieldMapperType.INSTANCE);
+ }
+
+ @Override
+ protected String contentType() {
+ return NAME;
+ }
+
+ @Override
+ public void postParse(DocumentParserContext context) {
+ for (Values values : context.getIgnoredFieldValues()) {
+ context.doc().add(new StringField(NAME, encode(values), Field.Store.YES));
+ }
+ }
+
+ static String encode(Values values) {
+ assert values.parentOffset < OFFSET_IN_NAME;
+ assert values.parentOffset * (long) OFFSET_IN_NAME < Integer.MAX_VALUE;
+
+ byte[] nameBytes = values.name.getBytes(StandardCharsets.UTF_8);
+ byte[] bytes = new byte[4 + nameBytes.length + values.value.length];
+ ByteUtils.writeIntLE(values.name.length() + OFFSET_IN_NAME * values.parentOffset, bytes, 0);
+ System.arraycopy(nameBytes, 0, bytes, 4, nameBytes.length);
+ System.arraycopy(values.value.bytes, values.value.offset, bytes, 4 + nameBytes.length, values.value.length);
+ return Base64.getUrlEncoder().withoutPadding().encodeToString(bytes);
+ }
+
+ static Values decode(Object field) {
+ byte[] bytes = Base64.getUrlDecoder().decode((String) field);
+ int encodedSize = ByteUtils.readIntLE(bytes, 0);
+ int nameSize = encodedSize % OFFSET_IN_NAME;
+ int parentOffset = encodedSize / OFFSET_IN_NAME;
+ String name = new String(bytes, 4, nameSize, StandardCharsets.UTF_8);
+ BytesRef value = new BytesRef(bytes, 4 + nameSize, bytes.length - nameSize - 4);
+ return new Values(name, parentOffset, value);
+ }
+
+ @Override
+ public SourceLoader.SyntheticFieldLoader syntheticFieldLoader() {
+ return new SyntheticLoader();
+ }
+
+ public static class SyntheticLoader implements SourceLoader.SyntheticFieldLoader {
+ private List
*/
- public static StoredField storedField(String name, XContentParser parser) throws IOException {
+ static StoredField storedField(String name, XContentParser parser) throws IOException {
return (StoredField) processToken(parser, typeUtils -> typeUtils.buildStoredField(name, parser));
}
@@ -47,7 +47,7 @@ public static StoredField storedField(String name, XContentParser parser) throws
* Build a {@link BytesRef} wrapping a byte array containing an encoded form
* the value on which the parser is currently positioned.
*/
- public static BytesRef encodeToken(XContentParser parser) throws IOException {
+ static BytesRef encodeToken(XContentParser parser) throws IOException {
return new BytesRef((byte[]) processToken(parser, (typeUtils) -> typeUtils.encode(parser)));
}
@@ -56,18 +56,20 @@ public static BytesRef encodeToken(XContentParser parser) throws IOException {
* passed build. The assumption is that the passed value has encoded using the function
* {@link #encodeToken(XContentParser)} above.
*/
- public static void decodeAndWrite(XContentBuilder b, BytesRef r) throws IOException {
+ static void decodeAndWrite(XContentBuilder b, BytesRef r) throws IOException {
switch ((char) r.bytes[r.offset]) {
- case BINARY_VALUE -> TypeUtils.EMBEDDED_OBJECT.decodeAndWrite(b, r);
- case CBOR_OBJECT_VALUE, JSON_OBJECT_VALUE, YAML_OBJECT_VALUE, SMILE_OBJECT_VALUE -> TypeUtils.START.decodeAndWrite(b, r);
- case BIG_DECIMAL_VALUE -> TypeUtils.BIG_DECIMAL.decodeAndWrite(b, r);
- case FALSE_VALUE, TRUE_VALUE -> TypeUtils.BOOLEAN.decodeAndWrite(b, r);
- case BIG_INTEGER_VALUE -> TypeUtils.BIG_INTEGER.decodeAndWrite(b, r);
- case STRING_VALUE -> TypeUtils.STRING.decodeAndWrite(b, r);
- case INTEGER_VALUE -> TypeUtils.INTEGER.decodeAndWrite(b, r);
- case LONG_VALUE -> TypeUtils.LONG.decodeAndWrite(b, r);
- case DOUBLE_VALUE -> TypeUtils.DOUBLE.decodeAndWrite(b, r);
- case FLOAT_VALUE -> TypeUtils.FLOAT.decodeAndWrite(b, r);
+ case BINARY_ENCODING -> TypeUtils.EMBEDDED_OBJECT.decodeAndWrite(b, r);
+ case CBOR_OBJECT_ENCODING, JSON_OBJECT_ENCODING, YAML_OBJECT_ENCODING, SMILE_OBJECT_ENCODING -> {
+ TypeUtils.START.decodeAndWrite(b, r);
+ }
+ case BIG_DECIMAL_ENCODING -> TypeUtils.BIG_DECIMAL.decodeAndWrite(b, r);
+ case FALSE_ENCODING, TRUE_ENCODING -> TypeUtils.BOOLEAN.decodeAndWrite(b, r);
+ case BIG_INTEGER_ENCODING -> TypeUtils.BIG_INTEGER.decodeAndWrite(b, r);
+ case STRING_ENCODING -> TypeUtils.STRING.decodeAndWrite(b, r);
+ case INTEGER_ENCODING -> TypeUtils.INTEGER.decodeAndWrite(b, r);
+ case LONG_ENCODING -> TypeUtils.LONG.decodeAndWrite(b, r);
+ case DOUBLE_ENCODING -> TypeUtils.DOUBLE.decodeAndWrite(b, r);
+ case FLOAT_ENCODING -> TypeUtils.FLOAT.decodeAndWrite(b, r);
default -> throw new IllegalArgumentException("Can't decode " + r);
}
}
@@ -90,23 +92,23 @@ private static Object processToken(XContentParser parser, CheckedFunction decodeAndWriteXContent(b, XContentType.CBOR, r);
- case JSON_OBJECT_VALUE -> decodeAndWriteXContent(b, XContentType.JSON, r);
- case SMILE_OBJECT_VALUE -> decodeAndWriteXContent(b, XContentType.SMILE, r);
- case YAML_OBJECT_VALUE -> decodeAndWriteXContent(b, XContentType.YAML, r);
+ case CBOR_OBJECT_ENCODING -> decodeAndWriteXContent(b, XContentType.CBOR, r);
+ case JSON_OBJECT_ENCODING -> decodeAndWriteXContent(b, XContentType.JSON, r);
+ case SMILE_OBJECT_ENCODING -> decodeAndWriteXContent(b, XContentType.SMILE, r);
+ case YAML_OBJECT_ENCODING -> decodeAndWriteXContent(b, XContentType.YAML, r);
default -> throw new IllegalArgumentException("Can't decode " + r);
}
}
@@ -368,10 +370,10 @@ static byte[] encode(XContentBuilder builder) throws IOException {
BytesReference b = BytesReference.bytes(builder);
byte[] encoded = new byte[1 + b.length()];
encoded[0] = switch (builder.contentType()) {
- case JSON -> JSON_OBJECT_VALUE;
- case SMILE -> SMILE_OBJECT_VALUE;
- case YAML -> YAML_OBJECT_VALUE;
- case CBOR -> CBOR_OBJECT_VALUE;
+ case JSON -> JSON_OBJECT_ENCODING;
+ case SMILE -> SMILE_OBJECT_ENCODING;
+ case YAML -> YAML_OBJECT_ENCODING;
+ case CBOR -> CBOR_OBJECT_ENCODING;
default -> throw new IllegalArgumentException("unsupported type " + builder.contentType());
};
diff --git a/server/src/main/java/org/elasticsearch/indices/IndicesModule.java b/server/src/main/java/org/elasticsearch/indices/IndicesModule.java
index 2e409995277eb..17e0105d59d8c 100644
--- a/server/src/main/java/org/elasticsearch/indices/IndicesModule.java
+++ b/server/src/main/java/org/elasticsearch/indices/IndicesModule.java
@@ -39,7 +39,7 @@
import org.elasticsearch.index.mapper.GeoPointScriptFieldType;
import org.elasticsearch.index.mapper.IdFieldMapper;
import org.elasticsearch.index.mapper.IgnoredFieldMapper;
-import org.elasticsearch.index.mapper.IgnoredValuesFieldMapper;
+import org.elasticsearch.index.mapper.IgnoredSourceFieldMapper;
import org.elasticsearch.index.mapper.IndexFieldMapper;
import org.elasticsearch.index.mapper.IpFieldMapper;
import org.elasticsearch.index.mapper.IpScriptFieldType;
@@ -259,7 +259,7 @@ private static Map initBuiltInMetadataMa
builtInMetadataMappers.put(TimeSeriesRoutingHashFieldMapper.NAME, TimeSeriesRoutingHashFieldMapper.PARSER);
builtInMetadataMappers.put(IndexFieldMapper.NAME, IndexFieldMapper.PARSER);
builtInMetadataMappers.put(SourceFieldMapper.NAME, SourceFieldMapper.PARSER);
- builtInMetadataMappers.put(IgnoredValuesFieldMapper.NAME, IgnoredValuesFieldMapper.PARSER);
+ builtInMetadataMappers.put(IgnoredSourceFieldMapper.NAME, IgnoredSourceFieldMapper.PARSER);
builtInMetadataMappers.put(NestedPathFieldMapper.NAME, NestedPathFieldMapper.PARSER);
builtInMetadataMappers.put(VersionFieldMapper.NAME, VersionFieldMapper.PARSER);
builtInMetadataMappers.put(SeqNoFieldMapper.NAME, SeqNoFieldMapper.PARSER);
diff --git a/server/src/test/java/org/elasticsearch/index/mapper/DocumentMapperTests.java b/server/src/test/java/org/elasticsearch/index/mapper/DocumentMapperTests.java
index d707a17e82337..c210fb0654683 100644
--- a/server/src/test/java/org/elasticsearch/index/mapper/DocumentMapperTests.java
+++ b/server/src/test/java/org/elasticsearch/index/mapper/DocumentMapperTests.java
@@ -319,7 +319,7 @@ public void testEmptyDocumentMapper() {
.item(DocCountFieldMapper.class)
.item(FieldNamesFieldMapper.class)
.item(IgnoredFieldMapper.class)
- .item(IgnoredValuesFieldMapper.class)
+ .item(IgnoredSourceFieldMapper.class)
.item(IndexFieldMapper.class)
.item(NestedPathFieldMapper.class)
.item(ProvidedIdFieldMapper.class)
@@ -337,7 +337,7 @@ public void testEmptyDocumentMapper() {
.item(FieldNamesFieldMapper.CONTENT_TYPE)
.item(IdFieldMapper.CONTENT_TYPE)
.item(IgnoredFieldMapper.CONTENT_TYPE)
- .item(IgnoredValuesFieldMapper.NAME)
+ .item(IgnoredSourceFieldMapper.NAME)
.item(IndexFieldMapper.CONTENT_TYPE)
.item(NestedPathFieldMapper.NAME)
.item(RoutingFieldMapper.CONTENT_TYPE)
diff --git a/server/src/test/java/org/elasticsearch/index/mapper/IgnoredValuedFieldMapperTests.java b/server/src/test/java/org/elasticsearch/index/mapper/IgnoredSourceFieldMapperTests.java
similarity index 99%
rename from server/src/test/java/org/elasticsearch/index/mapper/IgnoredValuedFieldMapperTests.java
rename to server/src/test/java/org/elasticsearch/index/mapper/IgnoredSourceFieldMapperTests.java
index 710fb9f2298ea..a21c3993d4f2b 100644
--- a/server/src/test/java/org/elasticsearch/index/mapper/IgnoredValuedFieldMapperTests.java
+++ b/server/src/test/java/org/elasticsearch/index/mapper/IgnoredSourceFieldMapperTests.java
@@ -17,7 +17,7 @@
import java.math.BigInteger;
import java.util.Base64;
-public class IgnoredValuedFieldMapperTests extends MapperServiceTestCase {
+public class IgnoredSourceFieldMapperTests extends MapperServiceTestCase {
private String getSyntheticSource(CheckedConsumer build) throws IOException {
DocumentMapper documentMapper = createMapperService(
diff --git a/server/src/test/java/org/elasticsearch/index/mapper/FieldDataParseHelperTests.java b/server/src/test/java/org/elasticsearch/index/mapper/XContentDataHelperTests.java
similarity index 94%
rename from server/src/test/java/org/elasticsearch/index/mapper/FieldDataParseHelperTests.java
rename to server/src/test/java/org/elasticsearch/index/mapper/XContentDataHelperTests.java
index 3833c3ccbe063..cd142dfd393f7 100644
--- a/server/src/test/java/org/elasticsearch/index/mapper/FieldDataParseHelperTests.java
+++ b/server/src/test/java/org/elasticsearch/index/mapper/XContentDataHelperTests.java
@@ -23,7 +23,7 @@
import static org.hamcrest.Matchers.equalTo;
-public class FieldDataParseHelperTests extends ESTestCase {
+public class XContentDataHelperTests extends ESTestCase {
private String encodeAndDecode(String value) throws IOException {
XContentParser p = createParser(JsonXContent.jsonXContent, "{ \"foo\": " + value + " }");
@@ -34,7 +34,7 @@ private String encodeAndDecode(String value) throws IOException {
XContentBuilder builder = XContentFactory.jsonBuilder();
builder.humanReadable(true);
- FieldDataParseHelper.decodeAndWrite(builder, FieldDataParseHelper.encodeToken(p));
+ XContentDataHelper.decodeAndWrite(builder, XContentDataHelper.encodeToken(p));
return Strings.toString(builder);
}
@@ -94,7 +94,7 @@ public void testObject() throws IOException {
assertThat(p.nextToken(), equalTo(XContentParser.Token.START_OBJECT));
XContentBuilder builder = XContentFactory.jsonBuilder();
builder.humanReadable(true);
- FieldDataParseHelper.decodeAndWrite(builder, FieldDataParseHelper.encodeToken(p));
+ XContentDataHelper.decodeAndWrite(builder, XContentDataHelper.encodeToken(p));
assertEquals(object, Strings.toString(builder));
}
diff --git a/server/src/test/java/org/elasticsearch/indices/IndicesModuleTests.java b/server/src/test/java/org/elasticsearch/indices/IndicesModuleTests.java
index c93b3dfb31bcd..c173a22dcdf57 100644
--- a/server/src/test/java/org/elasticsearch/indices/IndicesModuleTests.java
+++ b/server/src/test/java/org/elasticsearch/indices/IndicesModuleTests.java
@@ -15,7 +15,7 @@
import org.elasticsearch.index.mapper.FieldNamesFieldMapper;
import org.elasticsearch.index.mapper.IdFieldMapper;
import org.elasticsearch.index.mapper.IgnoredFieldMapper;
-import org.elasticsearch.index.mapper.IgnoredValuesFieldMapper;
+import org.elasticsearch.index.mapper.IgnoredSourceFieldMapper;
import org.elasticsearch.index.mapper.IndexFieldMapper;
import org.elasticsearch.index.mapper.KeywordFieldMapper;
import org.elasticsearch.index.mapper.Mapper;
@@ -86,7 +86,7 @@ public Map getMetadataMappers() {
TimeSeriesRoutingHashFieldMapper.NAME,
IndexFieldMapper.NAME,
SourceFieldMapper.NAME,
- IgnoredValuesFieldMapper.NAME,
+ IgnoredSourceFieldMapper.NAME,
NestedPathFieldMapper.NAME,
VersionFieldMapper.NAME,
SeqNoFieldMapper.NAME,
From ace0559842f14ad70722823fd89ee171d76645c2 Mon Sep 17 00:00:00 2001
From: Kostas Krikellas
Date: Fri, 26 Apr 2024 12:02:25 +0300
Subject: [PATCH 18/19] rename _ignored_values to _ignored_source
---
.../index/mapper/IgnoredSourceFieldMapper.java | 7 ++++---
.../index/mapper/MapperFeatures.java | 2 +-
.../index/mapper/DocCountFieldMapperTests.java | 4 ++--
.../index/mapper/XContentDataHelperTests.java | 17 -----------------
4 files changed, 7 insertions(+), 23 deletions(-)
diff --git a/server/src/main/java/org/elasticsearch/index/mapper/IgnoredSourceFieldMapper.java b/server/src/main/java/org/elasticsearch/index/mapper/IgnoredSourceFieldMapper.java
index 6f6b782680ff6..1daa7d1d674e3 100644
--- a/server/src/main/java/org/elasticsearch/index/mapper/IgnoredSourceFieldMapper.java
+++ b/server/src/main/java/org/elasticsearch/index/mapper/IgnoredSourceFieldMapper.java
@@ -18,7 +18,8 @@
import java.util.Collections;
/**
- * Mapper for the {@code _ignored_values} field.
+
+ * Mapper for the {@code _ignored_source} field.
*
* A field mapper that records fields that have been ignored, along with their values. It's intended for use
* in indexes with synthetic source to reconstruct the latter, taking into account fields that got ignored during
@@ -35,13 +36,13 @@ public class IgnoredSourceFieldMapper extends MetadataFieldMapper {
// (N % PARENT_OFFSET_IN_NAME_OFFSET)
private static final int PARENT_OFFSET_IN_NAME_OFFSET = 1 << 16;
- public static final String NAME = "_ignored_values";
+ public static final String NAME = "_ignored_source";
public static final IgnoredSourceFieldMapper INSTANCE = new IgnoredSourceFieldMapper();
public static final TypeParser PARSER = new FixedTypeParser(context -> INSTANCE);
- static final NodeFeature TRACK_IGNORED_VALUES = new NodeFeature("mapper.track_ignored_values");
+ static final NodeFeature TRACK_IGNORED_SOURCE = new NodeFeature("mapper.track_ignored_source");
/*
* Container for the ignored field data:
diff --git a/server/src/main/java/org/elasticsearch/index/mapper/MapperFeatures.java b/server/src/main/java/org/elasticsearch/index/mapper/MapperFeatures.java
index 5b0823b9473ad..dc189aecab01c 100644
--- a/server/src/main/java/org/elasticsearch/index/mapper/MapperFeatures.java
+++ b/server/src/main/java/org/elasticsearch/index/mapper/MapperFeatures.java
@@ -19,6 +19,6 @@
public class MapperFeatures implements FeatureSpecification {
@Override
public Set getFeatures() {
- return Set.of(IgnoredSourceFieldMapper.TRACK_IGNORED_VALUES);
+ return Set.of(IgnoredSourceFieldMapper.TRACK_IGNORED_SOURCE);
}
}
diff --git a/server/src/test/java/org/elasticsearch/index/mapper/DocCountFieldMapperTests.java b/server/src/test/java/org/elasticsearch/index/mapper/DocCountFieldMapperTests.java
index 61259e62d6de5..06e70e84bbb67 100644
--- a/server/src/test/java/org/elasticsearch/index/mapper/DocCountFieldMapperTests.java
+++ b/server/src/test/java/org/elasticsearch/index/mapper/DocCountFieldMapperTests.java
@@ -98,7 +98,7 @@ public void testSyntheticSourceMany() throws IOException {
}
}, reader -> {
SourceLoader loader = mapper.mappingLookup().newSourceLoader();
- assertThat(loader.requiredStoredFields(), Matchers.contains("_ignored_values"));
+ assertThat(loader.requiredStoredFields(), Matchers.contains("_ignored_source"));
for (LeafReaderContext leaf : reader.leaves()) {
int[] docIds = IntStream.range(0, leaf.reader().maxDoc()).toArray();
SourceLoader.Leaf sourceLoaderLeaf = loader.leaf(leaf.reader(), docIds);
@@ -130,7 +130,7 @@ public void testSyntheticSourceManyDoNotHave() throws IOException {
}
}, reader -> {
SourceLoader loader = mapper.mappingLookup().newSourceLoader();
- assertThat(loader.requiredStoredFields(), Matchers.contains("_ignored_values"));
+ assertThat(loader.requiredStoredFields(), Matchers.contains("_ignored_source"));
for (LeafReaderContext leaf : reader.leaves()) {
int[] docIds = IntStream.range(0, leaf.reader().maxDoc()).toArray();
SourceLoader.Leaf sourceLoaderLeaf = loader.leaf(leaf.reader(), docIds);
diff --git a/server/src/test/java/org/elasticsearch/index/mapper/XContentDataHelperTests.java b/server/src/test/java/org/elasticsearch/index/mapper/XContentDataHelperTests.java
index cd142dfd393f7..06db79c3f9fb0 100644
--- a/server/src/test/java/org/elasticsearch/index/mapper/XContentDataHelperTests.java
+++ b/server/src/test/java/org/elasticsearch/index/mapper/XContentDataHelperTests.java
@@ -9,7 +9,6 @@
package org.elasticsearch.index.mapper;
import org.elasticsearch.common.Strings;
-import org.elasticsearch.common.bytes.BytesReference;
import org.elasticsearch.test.ESTestCase;
import org.elasticsearch.xcontent.XContentBuilder;
import org.elasticsearch.xcontent.XContentFactory;
@@ -17,7 +16,6 @@
import org.elasticsearch.xcontent.json.JsonXContent;
import java.io.IOException;
-import java.math.BigDecimal;
import java.math.BigInteger;
import java.util.List;
@@ -73,21 +71,6 @@ public void testBigInteger() throws IOException {
assertEquals(i, new BigInteger(encodeAndDecode(i.toString()), 10));
}
- public void testBigDecimal() throws IOException {
- BigDecimal d = new BigDecimal(randomBigInteger(), randomInt());
- XContentBuilder builder = XContentFactory.jsonBuilder();
- builder.startObject().field("foo", d).endObject();
- XContentParser p = JsonXContent.jsonXContent.createParser(parserConfig(), BytesReference.bytes(builder).streamInput());
- assertThat(p.nextToken(), equalTo(XContentParser.Token.START_OBJECT));
- assertThat(p.nextToken(), equalTo(XContentParser.Token.FIELD_NAME));
- assertThat(p.currentName(), equalTo("foo"));
- p.nextToken();
- assertEquals(p.numberType(), XContentParser.NumberType.DOUBLE);
-
- // FieldDataParseHelper.decodeAndWrite(builder, FieldDataParseHelper.encodeToken(p));
- // assertEquals(d, new BigDecimal(Strings.toString(builder)));
- }
-
public void testObject() throws IOException {
String object = "{\"name\":\"foo\"}";
XContentParser p = createParser(JsonXContent.jsonXContent, object);
From 3cc62dccaa866d7b9f7d133f2d779bbb759ef17c Mon Sep 17 00:00:00 2001
From: Kostas Krikellas
Date: Fri, 26 Apr 2024 13:01:20 +0300
Subject: [PATCH 19/19] rename _ignored_values to _ignored_source
---
.../test/indices.create/20_synthetic_source.yml | 6 +++---
.../rest-api-spec/test/nodes.stats/11_indices_metrics.yml | 6 +++---
2 files changed, 6 insertions(+), 6 deletions(-)
diff --git a/rest-api-spec/src/yamlRestTest/resources/rest-api-spec/test/indices.create/20_synthetic_source.yml b/rest-api-spec/src/yamlRestTest/resources/rest-api-spec/test/indices.create/20_synthetic_source.yml
index 0376604d4c43c..39787366c0cc9 100644
--- a/rest-api-spec/src/yamlRestTest/resources/rest-api-spec/test/indices.create/20_synthetic_source.yml
+++ b/rest-api-spec/src/yamlRestTest/resources/rest-api-spec/test/indices.create/20_synthetic_source.yml
@@ -40,7 +40,7 @@ nested is disabled:
---
object with unmapped fields:
- requires:
- cluster_features: ["mapper.track_ignored_values"]
+ cluster_features: ["mapper.track_ignored_source"]
reason: requires tracking ignored values
- do:
@@ -92,7 +92,7 @@ object with unmapped fields:
---
nested object with unmapped fields:
- requires:
- cluster_features: ["mapper.track_ignored_values"]
+ cluster_features: ["mapper.track_ignored_source"]
reason: requires tracking ignored values
- do:
@@ -143,7 +143,7 @@ nested object with unmapped fields:
---
empty object with unmapped fields:
- requires:
- cluster_features: ["mapper.track_ignored_values"]
+ cluster_features: ["mapper.track_ignored_source"]
reason: requires tracking ignored values
- do:
diff --git a/rest-api-spec/src/yamlRestTest/resources/rest-api-spec/test/nodes.stats/11_indices_metrics.yml b/rest-api-spec/src/yamlRestTest/resources/rest-api-spec/test/nodes.stats/11_indices_metrics.yml
index 1238bc210fc14..ac0f8aec4f3d0 100644
--- a/rest-api-spec/src/yamlRestTest/resources/rest-api-spec/test/nodes.stats/11_indices_metrics.yml
+++ b/rest-api-spec/src/yamlRestTest/resources/rest-api-spec/test/nodes.stats/11_indices_metrics.yml
@@ -417,8 +417,8 @@
- requires:
test_runner_features: [arbitrary_key]
- cluster_features: ["mapper.track_ignored_values"]
- reason: "_ignored_values added to mappings"
+ cluster_features: ["mapper.track_ignored_source"]
+ reason: "_ignored_source added to mappings"
- do:
indices.create:
@@ -476,7 +476,7 @@
# 4. _field_names
# 5. _id
# 6. _ignored
- # 7. _ignored_values
+ # 7. _ignored_source
# 8. _index
# 9. _nested_path
# 10. _routing