From 16cb2eeb423a57c79b81786284cdb82a42ae416c Mon Sep 17 00:00:00 2001 From: Kostas Krikellas Date: Wed, 17 Apr 2024 16:48:06 +0300 Subject: [PATCH 01/19] Add ignored field values to synthetic source --- .../indices.create/20_synthetic_source.yml | 151 ++++++++ .../index/mapper/DocumentParserContext.java | 30 ++ .../index/mapper/FieldDataParseHelper.java | 357 ++++++++++++++++++ .../mapper/IgnoreMalformedStoredValues.java | 141 +------ .../mapper/IgnoredValuesFieldMapper.java | 169 +++++++++ .../index/mapper/ObjectMapper.java | 21 +- .../index/mapper/SourceFieldMapper.java | 8 + .../elasticsearch/indices/IndicesModule.java | 2 + .../mapper/DocCountFieldMapperTests.java | 2 - .../index/mapper/DocumentMapperTests.java | 2 + .../mapper/FieldDataParseHelperTests.java | 107 ++++++ .../mapper/IgnoredValuedFieldMapperTests.java | 148 ++++++++ .../indices/IndicesModuleTests.java | 2 + 13 files changed, 998 insertions(+), 142 deletions(-) create mode 100644 server/src/main/java/org/elasticsearch/index/mapper/FieldDataParseHelper.java create mode 100644 server/src/main/java/org/elasticsearch/index/mapper/IgnoredValuesFieldMapper.java create mode 100644 server/src/test/java/org/elasticsearch/index/mapper/FieldDataParseHelperTests.java create mode 100644 server/src/test/java/org/elasticsearch/index/mapper/IgnoredValuedFieldMapperTests.java diff --git a/rest-api-spec/src/yamlRestTest/resources/rest-api-spec/test/indices.create/20_synthetic_source.yml b/rest-api-spec/src/yamlRestTest/resources/rest-api-spec/test/indices.create/20_synthetic_source.yml index 62a4e240a5b5d..9c9675d41febe 100644 --- a/rest-api-spec/src/yamlRestTest/resources/rest-api-spec/test/indices.create/20_synthetic_source.yml +++ b/rest-api-spec/src/yamlRestTest/resources/rest-api-spec/test/indices.create/20_synthetic_source.yml @@ -36,3 +36,154 @@ nested is disabled: properties: foo: type: keyword + +--- +object with unmapped fields: + - skip: + version: " - 8.13.99" + reason: introduced in 8.14.0 + + - do: + indices.create: + index: test + body: + settings: + index: + mapping: + total_fields: + ignore_dynamic_beyond_limit: true + limit: 1 + + mappings: + _source: + mode: synthetic + properties: + name: + type: keyword + + - do: + bulk: + index: test + refresh: true + body: + - '{ "create": { } }' + - '{ "name": "aaaa", "some_string": "AaAa", "some_int": 1000, "some_double": 123.456789, "some_bool": true, "a.very.deeply.nested.field": "AAAA" }' + - '{ "create": { } }' + - '{ "name": "bbbb", "some_string": "BbBb", "some_int": 2000, "some_double": 321.987654, "some_bool": false, "a.very.deeply.nested.field": "BBBB" }' + + - do: + search: + index: test + + - match: { hits.total.value: 2 } + - match: { hits.hits.0._source.name: aaaa } + - match: { hits.hits.0._source.some_string: AaAa } + - match: { hits.hits.0._source.some_int: 1000 } + - match: { hits.hits.0._source.some_double: 123.456789 } + - match: { hits.hits.0._source.a.very.deeply.nested.field: AAAA } + - match: { hits.hits.0._source.some_bool: true } + - match: { hits.hits.1._source.name: bbbb } + - match: { hits.hits.1._source.some_string: BbBb } + - match: { hits.hits.1._source.some_int: 2000 } + - match: { hits.hits.1._source.some_double: 321.987654 } + - match: { hits.hits.1._source.a.very.deeply.nested.field: BBBB } + + +--- +nested object with unmapped fields: + - skip: + version: " - 8.13.99" + reason: introduced in 8.14.0 + + - do: + indices.create: + index: test + body: + settings: + index: + mapping: + total_fields: + ignore_dynamic_beyond_limit: true + limit: 3 + + mappings: + _source: + mode: synthetic + properties: + path: + properties: + to: + properties: + name: + type: keyword + + - do: + bulk: + index: test + refresh: true + body: + - '{ "create": { } }' + - '{ "path.to.name": "aaaa", "path.to.surname": "AaAa", "path.some.other.name": "AaAaAa" }' + - '{ "create": { } }' + - '{ "path.to.name": "bbbb", "path.to.surname": "BbBb", "path.some.other.name": "BbBbBb" }' + + - do: + search: + index: test + + - match: { hits.total.value: 2 } + - match: { hits.hits.0._source.path.to.name: aaaa } + - match: { hits.hits.0._source.path.to.surname: AaAa } + - match: { hits.hits.0._source.path.some.other.name: AaAaAa } + - match: { hits.hits.1._source.path.to.name: bbbb } + - match: { hits.hits.1._source.path.to.surname: BbBb } + - match: { hits.hits.1._source.path.some.other.name: BbBbBb } + + +--- +empty object with unmapped fields: + - skip: + version: " - 8.13.99" + reason: introduced in 8.14.0 + + - do: + indices.create: + index: test + body: + settings: + index: + mapping: + total_fields: + ignore_dynamic_beyond_limit: true + limit: 3 + + mappings: + _source: + mode: synthetic + properties: + path: + properties: + to: + properties: + name: + type: keyword + + - do: + bulk: + index: test + refresh: true + body: + - '{ "create": { } }' + - '{ "path.to.surname": "AaAa", "path.some.other.name": "AaAaAa" }' + - '{ "create": { } }' + - '{ "path.to.surname": "BbBb", "path.some.other.name": "BbBbBb" }' + + - do: + search: + index: test + + - match: { hits.total.value: 2 } + - match: { hits.hits.0._source.path.to.surname: AaAa } + - match: { hits.hits.0._source.path.some.other.name: AaAaAa } + - match: { hits.hits.1._source.path.to.surname: BbBb } + - match: { hits.hits.1._source.path.some.other.name: BbBbBb } diff --git a/server/src/main/java/org/elasticsearch/index/mapper/DocumentParserContext.java b/server/src/main/java/org/elasticsearch/index/mapper/DocumentParserContext.java index a42477bed2146..ad6014eb521bd 100644 --- a/server/src/main/java/org/elasticsearch/index/mapper/DocumentParserContext.java +++ b/server/src/main/java/org/elasticsearch/index/mapper/DocumentParserContext.java @@ -104,6 +104,7 @@ public int get() { private final MappingParserContext mappingParserContext; private final SourceToParse sourceToParse; private final Set ignoredFields; + private final List ignoredFieldValues; private final Map> dynamicMappers; private final DynamicMapperSize dynamicMappersSize; private final Map dynamicObjectMappers; @@ -122,6 +123,7 @@ private DocumentParserContext( MappingParserContext mappingParserContext, SourceToParse sourceToParse, Set ignoreFields, + List ignoredFieldValues, Map> dynamicMappers, Map dynamicObjectMappers, Map> dynamicRuntimeFields, @@ -139,6 +141,7 @@ private DocumentParserContext( this.mappingParserContext = mappingParserContext; this.sourceToParse = sourceToParse; this.ignoredFields = ignoreFields; + this.ignoredFieldValues = ignoredFieldValues; this.dynamicMappers = dynamicMappers; this.dynamicObjectMappers = dynamicObjectMappers; this.dynamicRuntimeFields = dynamicRuntimeFields; @@ -159,6 +162,7 @@ private DocumentParserContext(ObjectMapper parent, ObjectMapper.Dynamic dynamic, in.mappingParserContext, in.sourceToParse, in.ignoredFields, + in.ignoredFieldValues, in.dynamicMappers, in.dynamicObjectMappers, in.dynamicRuntimeFields, @@ -186,6 +190,7 @@ protected DocumentParserContext( mappingParserContext, source, new HashSet<>(), + new ArrayList<>(), new HashMap<>(), new HashMap<>(), new HashMap<>(), @@ -251,6 +256,20 @@ public final Collection getIgnoredFields() { return Collections.unmodifiableCollection(ignoredFields); } + /** + * Add the given ignored values to the corresponding list. + */ + public final void addIgnoredValues(IgnoredValuesFieldMapper.Values values) { + ignoredFieldValues.add(values); + } + + /** + * Return the collection of values for fields that have been ignored so far. + */ + public final Collection getIgnoredFieldValues() { + return Collections.unmodifiableCollection(ignoredFieldValues); + } + /** * Add the given {@code field} to the _field_names field * @@ -345,6 +364,17 @@ public final boolean addDynamicMapper(Mapper mapper) { int additionalFieldsToAdd = getNewFieldsSize() + mapperSize; if (indexSettings().isIgnoreDynamicFieldsBeyondLimit()) { if (mappingLookup.exceedsLimit(indexSettings().getMappingTotalFieldsLimit(), additionalFieldsToAdd)) { + if (indexSettings().getMode().isSyntheticSourceEnabled() || SourceFieldMapper.isSynthetic(mappingLookup)) { + try { + int parentOffset = parent() instanceof RootObjectMapper ? 0 : parent().fullPath().length() + 1; + addIgnoredValues( + new IgnoredValuesFieldMapper.Values(mapper.name(), parentOffset, FieldDataParseHelper.encodeToken(parser())) + ); + } catch (IOException e) { + // A parsing failure for ignored field values should be *very* rare. + // It is not considered important enough to drop the document, ignore it silently for now. + } + } addIgnoredField(mapper.name()); return false; } diff --git a/server/src/main/java/org/elasticsearch/index/mapper/FieldDataParseHelper.java b/server/src/main/java/org/elasticsearch/index/mapper/FieldDataParseHelper.java new file mode 100644 index 0000000000000..1478f905165d5 --- /dev/null +++ b/server/src/main/java/org/elasticsearch/index/mapper/FieldDataParseHelper.java @@ -0,0 +1,357 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License + * 2.0 and the Server Side Public License, v 1; you may not use this file except + * in compliance with, at your election, the Elastic License 2.0 or the Server + * Side Public License, v 1. + */ + +package org.elasticsearch.index.mapper; + +import org.apache.lucene.document.StoredField; +import org.apache.lucene.util.BytesRef; +import org.apache.lucene.util.BytesRefIterator; +import org.elasticsearch.common.bytes.BytesReference; +import org.elasticsearch.common.util.ByteUtils; +import org.elasticsearch.core.CheckedFunction; +import org.elasticsearch.xcontent.XContentBuilder; +import org.elasticsearch.xcontent.XContentParser; +import org.elasticsearch.xcontent.XContentParserConfiguration; +import org.elasticsearch.xcontent.XContentType; + +import java.io.IOException; +import java.math.BigDecimal; +import java.math.BigInteger; +import java.nio.charset.StandardCharsets; + +/** + * Helper class for processing field data of any type, as provided by the {@link XContentParser}. + */ +public final class FieldDataParseHelper { + /** + * Build a {@link StoredField} for the value on which the parser is + * currently positioned. + *

+ * We try to use {@link StoredField}'s native types for fields where + * possible but we have to preserve more type information than + * stored fields support, so we encode all of those into stored fields' + * {@code byte[]} type and then encode type information in the first byte. + *

+ */ + public static StoredField storedField(String name, XContentParser parser) throws IOException { + return (StoredField) processToken(parser, typeUtils -> typeUtils.buildStoredField(name, parser)); + } + + /** + * Build a {@link BytesRef} wrapping a byte array containing an encoded form + * the value on which the parser is currently positioned. + */ + public static BytesRef encodeToken(XContentParser parser) throws IOException { + return new BytesRef((byte[]) processToken(parser, (typeUtils) -> typeUtils.encode(parser))); + } + + /** + * Decode the value in the passed {@link BytesRef} and add it as a value to the + * passed build. The assumption is that the passed value has encoded using the function + * {@link #encodeToken(XContentParser)} above. + */ + public static void decodeAndWrite(XContentBuilder b, BytesRef r) throws IOException { + switch (r.bytes[r.offset]) { + case 'b' -> TypeUtils.EMBEDDED_OBJECT.decodeAndWrite(b, r); + case 'c', 'j', 's', 'y' -> TypeUtils.START.decodeAndWrite(b, r); + case 'd' -> TypeUtils.BIG_DECIMAL.decodeAndWrite(b, r); + case 'f', 't' -> TypeUtils.BOOLEAN.decodeAndWrite(b, r); + case 'i' -> TypeUtils.BIG_INTEGER.decodeAndWrite(b, r); + case 'S' -> TypeUtils.STRING.decodeAndWrite(b, r); + case 'I' -> TypeUtils.INTEGER.decodeAndWrite(b, r); + case 'L' -> TypeUtils.LONG.decodeAndWrite(b, r); + case 'D' -> TypeUtils.DOUBLE.decodeAndWrite(b, r); + case 'F' -> TypeUtils.FLOAT.decodeAndWrite(b, r); + default -> throw new IllegalArgumentException("Can't decode " + r); + } + } + + private static Object processToken(XContentParser parser, CheckedFunction visitor) throws IOException { + return switch (parser.currentToken()) { + case VALUE_STRING -> visitor.apply(TypeUtils.STRING); + case VALUE_NUMBER -> switch (parser.numberType()) { + case INT -> visitor.apply(TypeUtils.INTEGER); + case LONG -> visitor.apply(TypeUtils.LONG); + case DOUBLE -> visitor.apply(TypeUtils.DOUBLE); + case FLOAT -> visitor.apply(TypeUtils.FLOAT); + case BIG_INTEGER -> visitor.apply(TypeUtils.BIG_INTEGER); + case BIG_DECIMAL -> visitor.apply(TypeUtils.BIG_DECIMAL); + }; + case VALUE_BOOLEAN -> visitor.apply(TypeUtils.BOOLEAN); + case VALUE_EMBEDDED_OBJECT -> visitor.apply(TypeUtils.EMBEDDED_OBJECT); + case START_OBJECT, START_ARRAY -> visitor.apply(TypeUtils.START); + default -> throw new IllegalArgumentException("synthetic _source doesn't support malformed objects"); + }; + } + + private enum TypeUtils { + STRING('S') { + @Override + StoredField buildStoredField(String name, XContentParser parser) throws IOException { + return new StoredField(name, parser.text()); + } + + @Override + byte[] encode(XContentParser parser) throws IOException { + byte[] text = parser.text().getBytes(StandardCharsets.UTF_8); + byte[] bytes = new byte[text.length + 1]; + bytes[0] = getEncoding(); + System.arraycopy(text, 0, bytes, 1, text.length); + return bytes; + } + + @Override + void decodeAndWrite(XContentBuilder b, BytesRef r) throws IOException { + b.value(new BytesRef(r.bytes, r.offset + 1, r.length - 1).utf8ToString()); + } + }, + INTEGER('I') { + @Override + StoredField buildStoredField(String name, XContentParser parser) throws IOException { + return new StoredField(name, parser.intValue()); + } + + @Override + byte[] encode(XContentParser parser) throws IOException { + byte[] bytes = new byte[5]; + bytes[0] = getEncoding(); + ByteUtils.writeIntLE(parser.intValue(), bytes, 1); + return bytes; + } + + @Override + void decodeAndWrite(XContentBuilder b, BytesRef r) throws IOException { + b.value(ByteUtils.readIntLE(r.bytes, 1 + r.offset)); + } + }, + LONG('L') { + @Override + StoredField buildStoredField(String name, XContentParser parser) throws IOException { + return new StoredField(name, parser.longValue()); + } + + @Override + byte[] encode(XContentParser parser) throws IOException { + byte[] bytes = new byte[9]; + bytes[0] = getEncoding(); + ByteUtils.writeLongLE(parser.longValue(), bytes, 1); + return bytes; + } + + @Override + void decodeAndWrite(XContentBuilder b, BytesRef r) throws IOException { + b.value(ByteUtils.readLongLE(r.bytes, 1 + r.offset)); + } + }, + DOUBLE('D') { + @Override + StoredField buildStoredField(String name, XContentParser parser) throws IOException { + return new StoredField(name, parser.doubleValue()); + } + + @Override + byte[] encode(XContentParser parser) throws IOException { + byte[] bytes = new byte[9]; + bytes[0] = getEncoding(); + ByteUtils.writeDoubleLE(parser.doubleValue(), bytes, 1); + return bytes; + } + + @Override + void decodeAndWrite(XContentBuilder b, BytesRef r) throws IOException { + b.value(ByteUtils.readDoubleLE(r.bytes, 1 + r.offset)); + } + }, + FLOAT('F') { + @Override + StoredField buildStoredField(String name, XContentParser parser) throws IOException { + return new StoredField(name, parser.floatValue()); + } + + @Override + byte[] encode(XContentParser parser) throws IOException { + byte[] bytes = new byte[5]; + bytes[0] = getEncoding(); + ByteUtils.writeFloatLE(parser.floatValue(), bytes, 1); + return bytes; + } + + @Override + void decodeAndWrite(XContentBuilder b, BytesRef r) throws IOException { + b.value(ByteUtils.readFloatLE(r.bytes, 1 + r.offset)); + } + }, + BIG_INTEGER('i') { + @Override + StoredField buildStoredField(String name, XContentParser parser) throws IOException { + return new StoredField(name, encode(parser)); + } + + @Override + byte[] encode(XContentParser parser) throws IOException { + return encode((BigInteger) parser.numberValue(), getEncoding()); + } + + @Override + void decodeAndWrite(XContentBuilder b, BytesRef r) throws IOException { + b.value(new BigInteger(r.bytes, r.offset + 1, r.length - 1)); + } + }, + BIG_DECIMAL('d') { + @Override + StoredField buildStoredField(String name, XContentParser parser) throws IOException { + return new StoredField(name, encode(parser)); + } + + @Override + byte[] encode(XContentParser parser) throws IOException { + return encode((BigDecimal) parser.numberValue(), getEncoding()); + } + + @Override + void decodeAndWrite(XContentBuilder b, BytesRef r) throws IOException { + if (r.length < 5) { + throw new IllegalArgumentException("Can't decode " + r); + } + int scale = ByteUtils.readIntLE(r.bytes, r.offset + 1); + b.value(new BigDecimal(new BigInteger(r.bytes, r.offset + 5, r.length - 5), scale)); + } + }, + BOOLEAN('B') { + @Override + StoredField buildStoredField(String name, XContentParser parser) throws IOException { + return new StoredField(name, encode(parser)); + } + + @Override + byte[] encode(XContentParser parser) throws IOException { + return new byte[] { parser.booleanValue() ? (byte) 't' : (byte) 'f' }; + } + + @Override + void decodeAndWrite(XContentBuilder b, BytesRef r) throws IOException { + if (r.length != 1) { + throw new IllegalArgumentException("Can't decode " + r); + } + assert r.bytes[r.offset] == 't' || r.bytes[r.offset] == 'f' : r.bytes[r.offset]; + b.value(r.bytes[r.offset] == 't'); + } + }, + EMBEDDED_OBJECT('b') { + @Override + StoredField buildStoredField(String name, XContentParser parser) throws IOException { + return new StoredField(name, encode(parser.binaryValue())); + } + + @Override + byte[] encode(XContentParser parser) throws IOException { + return encode(parser.binaryValue()); + } + + @Override + void decodeAndWrite(XContentBuilder b, BytesRef r) throws IOException { + b.value(r.bytes, r.offset + 1, r.length - 1); + } + }, + START('O') { + @Override + StoredField buildStoredField(String name, XContentParser parser) throws IOException { + return new StoredField(name, encode(parser)); + } + + @Override + byte[] encode(XContentParser parser) throws IOException { + try (XContentBuilder builder = XContentBuilder.builder(parser.contentType().xContent())) { + builder.copyCurrentStructure(parser); + return encode(builder); + } + } + + @Override + void decodeAndWrite(XContentBuilder b, BytesRef r) throws IOException { + switch (r.bytes[r.offset]) { + case 'c' -> decodeAndWriteXContent(b, XContentType.CBOR, r); + case 'j' -> decodeAndWriteXContent(b, XContentType.JSON, r); + case 's' -> decodeAndWriteXContent(b, XContentType.SMILE, r); + case 'y' -> decodeAndWriteXContent(b, XContentType.YAML, r); + default -> throw new IllegalArgumentException("Can't decode " + r); + } + } + }; + + TypeUtils(char encoding) { + this.encoding = encoding; + } + + byte getEncoding() { + return (byte) encoding; + } + + final char encoding; + + abstract StoredField buildStoredField(String name, XContentParser parser) throws IOException; + + abstract byte[] encode(XContentParser parser) throws IOException; + + abstract void decodeAndWrite(XContentBuilder b, BytesRef r) throws IOException; + + static byte[] encode(BigInteger n, Byte encoding) throws IOException { + byte[] twosCompliment = n.toByteArray(); + byte[] encoded = new byte[1 + twosCompliment.length]; + encoded[0] = encoding; + System.arraycopy(twosCompliment, 0, encoded, 1, twosCompliment.length); + return encoded; + } + + static byte[] encode(BigDecimal n, Byte encoding) { + byte[] twosCompliment = n.unscaledValue().toByteArray(); + byte[] encoded = new byte[5 + twosCompliment.length]; + encoded[0] = 'd'; + ByteUtils.writeIntLE(n.scale(), encoded, 1); + System.arraycopy(twosCompliment, 0, encoded, 5, twosCompliment.length); + return encoded; + } + + static byte[] encode(byte[] b) { + byte[] encoded = new byte[1 + b.length]; + encoded[0] = 'b'; + System.arraycopy(b, 0, encoded, 1, b.length); + return encoded; + } + + static byte[] encode(XContentBuilder builder) throws IOException { + BytesReference b = BytesReference.bytes(builder); + byte[] encoded = new byte[1 + b.length()]; + encoded[0] = switch (builder.contentType()) { + case JSON -> 'j'; + case SMILE -> 's'; + case YAML -> 'y'; + case CBOR -> 'c'; + default -> throw new IllegalArgumentException("unsupported type " + builder.contentType()); + }; + + int position = 1; + BytesRefIterator itr = b.iterator(); + BytesRef ref; + while ((ref = itr.next()) != null) { + System.arraycopy(ref.bytes, ref.offset, encoded, position, ref.length); + position += ref.length; + } + assert position == encoded.length; + return encoded; + } + + static void decodeAndWriteXContent(XContentBuilder b, XContentType type, BytesRef r) throws IOException { + try ( + XContentParser parser = type.xContent().createParser(XContentParserConfiguration.EMPTY, r.bytes, r.offset + 1, r.length - 1) + ) { + b.copyCurrentStructure(parser); + } + } + } +} diff --git a/server/src/main/java/org/elasticsearch/index/mapper/IgnoreMalformedStoredValues.java b/server/src/main/java/org/elasticsearch/index/mapper/IgnoreMalformedStoredValues.java index b7990648539c1..5c2c230dfb4c9 100644 --- a/server/src/main/java/org/elasticsearch/index/mapper/IgnoreMalformedStoredValues.java +++ b/server/src/main/java/org/elasticsearch/index/mapper/IgnoreMalformedStoredValues.java @@ -10,17 +10,10 @@ import org.apache.lucene.document.StoredField; import org.apache.lucene.util.BytesRef; -import org.apache.lucene.util.BytesRefIterator; -import org.elasticsearch.common.bytes.BytesReference; -import org.elasticsearch.common.util.ByteUtils; import org.elasticsearch.xcontent.XContentBuilder; import org.elasticsearch.xcontent.XContentParser; -import org.elasticsearch.xcontent.XContentParserConfiguration; -import org.elasticsearch.xcontent.XContentType; import java.io.IOException; -import java.math.BigDecimal; -import java.math.BigInteger; import java.util.List; import java.util.Map; import java.util.stream.Stream; @@ -32,38 +25,8 @@ * {@code _source}. */ public abstract class IgnoreMalformedStoredValues { - /** - * Build a {@link StoredField} for the value on which the parser is - * currently positioned. - *

- * We try to use {@link StoredField}'s native types for fields where - * possible but we have to preserve more type information than - * stored fields support, so we encode all of those into stored fields' - * {@code byte[]} type and then encode type information in the first byte. - *

- */ - public static StoredField storedField(String fieldName, XContentParser parser) throws IOException { - String name = name(fieldName); - return switch (parser.currentToken()) { - case VALUE_STRING -> new StoredField(name, parser.text()); - case VALUE_NUMBER -> switch (parser.numberType()) { - case INT -> new StoredField(name, parser.intValue()); - case LONG -> new StoredField(name, parser.longValue()); - case DOUBLE -> new StoredField(name, parser.doubleValue()); - case FLOAT -> new StoredField(name, parser.floatValue()); - case BIG_INTEGER -> new StoredField(name, encode((BigInteger) parser.numberValue())); - case BIG_DECIMAL -> new StoredField(name, encode((BigDecimal) parser.numberValue())); - }; - case VALUE_BOOLEAN -> new StoredField(name, new byte[] { parser.booleanValue() ? (byte) 't' : (byte) 'f' }); - case VALUE_EMBEDDED_OBJECT -> new StoredField(name, encode(parser.binaryValue())); - case START_OBJECT, START_ARRAY -> { - try (XContentBuilder builder = XContentBuilder.builder(parser.contentType().xContent())) { - builder.copyCurrentStructure(parser); - yield new StoredField(name, encode(builder)); - } - } - default -> throw new IllegalArgumentException("synthetic _source doesn't support malformed objects"); - }; + public static StoredField storedField(String name, XContentParser parser) throws IOException { + return FieldDataParseHelper.storedField(name(name), parser); } /** @@ -136,114 +99,16 @@ public int count() { public void write(XContentBuilder b) throws IOException { for (Object v : values) { if (v instanceof BytesRef r) { - decodeAndWrite(b, r); + FieldDataParseHelper.decodeAndWrite(b, r); } else { b.value(v); } } values = emptyList(); } - - private static void decodeAndWrite(XContentBuilder b, BytesRef r) throws IOException { - switch (r.bytes[r.offset]) { - case 'b': - b.value(r.bytes, r.offset + 1, r.length - 1); - return; - case 'c': - decodeAndWriteXContent(b, XContentType.CBOR, r); - return; - case 'd': - if (r.length < 5) { - throw new IllegalArgumentException("Can't decode " + r); - } - int scale = ByteUtils.readIntLE(r.bytes, r.offset + 1); - b.value(new BigDecimal(new BigInteger(r.bytes, r.offset + 5, r.length - 5), scale)); - return; - case 'f': - if (r.length != 1) { - throw new IllegalArgumentException("Can't decode " + r); - } - b.value(false); - return; - case 'i': - b.value(new BigInteger(r.bytes, r.offset + 1, r.length - 1)); - return; - case 'j': - decodeAndWriteXContent(b, XContentType.JSON, r); - return; - case 's': - decodeAndWriteXContent(b, XContentType.SMILE, r); - return; - case 't': - if (r.length != 1) { - throw new IllegalArgumentException("Can't decode " + r); - } - b.value(true); - return; - case 'y': - decodeAndWriteXContent(b, XContentType.YAML, r); - return; - default: - throw new IllegalArgumentException("Can't decode " + r); - } - } - - private static void decodeAndWriteXContent(XContentBuilder b, XContentType type, BytesRef r) throws IOException { - try ( - XContentParser parser = type.xContent().createParser(XContentParserConfiguration.EMPTY, r.bytes, r.offset + 1, r.length - 1) - ) { - b.copyCurrentStructure(parser); - } - } } private static String name(String fieldName) { return fieldName + "._ignore_malformed"; } - - private static byte[] encode(BigInteger n) { - byte[] twosCompliment = n.toByteArray(); - byte[] encoded = new byte[1 + twosCompliment.length]; - encoded[0] = 'i'; - System.arraycopy(twosCompliment, 0, encoded, 1, twosCompliment.length); - return encoded; - } - - private static byte[] encode(BigDecimal n) { - byte[] twosCompliment = n.unscaledValue().toByteArray(); - byte[] encoded = new byte[5 + twosCompliment.length]; - encoded[0] = 'd'; - ByteUtils.writeIntLE(n.scale(), encoded, 1); - System.arraycopy(twosCompliment, 0, encoded, 5, twosCompliment.length); - return encoded; - } - - private static byte[] encode(byte[] b) { - byte[] encoded = new byte[1 + b.length]; - encoded[0] = 'b'; - System.arraycopy(b, 0, encoded, 1, b.length); - return encoded; - } - - private static byte[] encode(XContentBuilder builder) throws IOException { - BytesReference b = BytesReference.bytes(builder); - byte[] encoded = new byte[1 + b.length()]; - encoded[0] = switch (builder.contentType()) { - case JSON -> 'j'; - case SMILE -> 's'; - case YAML -> 'y'; - case CBOR -> 'c'; - default -> throw new IllegalArgumentException("unsupported type " + builder.contentType()); - }; - - int position = 1; - BytesRefIterator itr = b.iterator(); - BytesRef ref; - while ((ref = itr.next()) != null) { - System.arraycopy(ref.bytes, ref.offset, encoded, position, ref.length); - position += ref.length; - } - assert position == encoded.length; - return encoded; - } } diff --git a/server/src/main/java/org/elasticsearch/index/mapper/IgnoredValuesFieldMapper.java b/server/src/main/java/org/elasticsearch/index/mapper/IgnoredValuesFieldMapper.java new file mode 100644 index 0000000000000..e85a853e00946 --- /dev/null +++ b/server/src/main/java/org/elasticsearch/index/mapper/IgnoredValuesFieldMapper.java @@ -0,0 +1,169 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License + * 2.0 and the Server Side Public License, v 1; you may not use this file except + * in compliance with, at your election, the Elastic License 2.0 or the Server + * Side Public License, v 1. + */ + +package org.elasticsearch.index.mapper; + +import org.apache.lucene.document.Field; +import org.apache.lucene.document.StringField; +import org.apache.lucene.index.LeafReader; +import org.apache.lucene.util.BytesRef; +import org.elasticsearch.common.util.ByteUtils; +import org.elasticsearch.index.query.SearchExecutionContext; +import org.elasticsearch.xcontent.XContentBuilder; + +import java.io.IOException; +import java.nio.charset.StandardCharsets; +import java.util.ArrayList; +import java.util.Base64; +import java.util.Collections; +import java.util.HashMap; +import java.util.List; +import java.util.Map; +import java.util.stream.Stream; + +/** + * Mapper for the {@code _ignored_values} field. + * + * A field mapper that records fields that have been ignored, along with their values. It's intended for use + * in indexes with synthetic source to reconstruct the latter, taking into account fields that got ignored during + * indexing. + * + * This overlaps with {@link IgnoredFieldMapper} that tracks just the ignored field names. It's worth evaluating + * if we can replace it for all use cases to avoid duplication, assuming that the storage tradeoff is favorable. + */ +public class IgnoredValuesFieldMapper extends MetadataFieldMapper { + + private static final int OFFSET_IN_NAME = 100_000; + + public static final String NAME = "_ignored_values"; + + public static final IgnoredValuesFieldMapper INSTANCE = new IgnoredValuesFieldMapper(); + + public static final TypeParser PARSER = new FixedTypeParser(context -> INSTANCE); + + public record Values(String name, int parentOffset, BytesRef value) {} + + static final class IgnoredValuesFieldMapperType extends StringFieldType { + + private static final IgnoredValuesFieldMapperType INSTANCE = new IgnoredValuesFieldMapperType(); + + private IgnoredValuesFieldMapperType() { + super(NAME, false, true, false, TextSearchInfo.NONE, Collections.emptyMap()); + } + + @Override + public String typeName() { + return NAME; + } + + @Override + public ValueFetcher valueFetcher(SearchExecutionContext context, String format) { + return new StoredValueFetcher(context.lookup(), NAME); + } + } + + private IgnoredValuesFieldMapper() { + super(IgnoredValuesFieldMapperType.INSTANCE); + } + + @Override + protected String contentType() { + return NAME; + } + + @Override + public void postParse(DocumentParserContext context) { + for (Values values : context.getIgnoredFieldValues()) { + context.doc().add(new StringField(NAME, encode(values), Field.Store.YES)); + } + } + + static String encode(Values values) { + assert values.parentOffset < OFFSET_IN_NAME; + assert values.parentOffset * (long) OFFSET_IN_NAME < Integer.MAX_VALUE; + + byte[] nameBytes = values.name.getBytes(StandardCharsets.UTF_8); + byte[] bytes = new byte[4 + nameBytes.length + values.value.length]; + ByteUtils.writeIntLE(values.name.length() + OFFSET_IN_NAME * values.parentOffset, bytes, 0); + System.arraycopy(nameBytes, 0, bytes, 4, nameBytes.length); + System.arraycopy(values.value.bytes, values.value.offset, bytes, 4 + nameBytes.length, values.value.length); + return Base64.getUrlEncoder().withoutPadding().encodeToString(bytes); + } + + static Values decode(Object field) { + byte[] bytes = Base64.getUrlDecoder().decode((String) field); + int encodedSize = ByteUtils.readIntLE(bytes, 0); + int nameSize = encodedSize % OFFSET_IN_NAME; + int parentOffset = encodedSize / OFFSET_IN_NAME; + String name = new String(bytes, 4, nameSize, StandardCharsets.UTF_8); + BytesRef value = new BytesRef(bytes, 4 + nameSize, bytes.length - nameSize - 4); + return new Values(name, parentOffset, value); + } + + @Override + public SourceLoader.SyntheticFieldLoader syntheticFieldLoader() { + return new SyntheticLoader(); + } + + public static class SyntheticLoader implements SourceLoader.SyntheticFieldLoader { + private List values = List.of(); + + // Maps the names of existing objects to lists of ignored fields they contain. + private Map> objectsWithIgnoredFields = null; + + @Override + public Stream> storedFieldLoaders() { + return Stream.of(Map.entry(NAME, values -> this.values = values)); + } + + @Override + public SourceLoader.SyntheticFieldLoader.DocValuesLoader docValuesLoader(LeafReader leafReader, int[] docIdsInLeaf) + throws IOException { + return null; + } + + @Override + public boolean hasValue() { + return false; + } + + @Override + public void write(XContentBuilder b) throws IOException {} + + public void trackObjectsWithIgnoredFields() { + if (values == null || values.isEmpty()) { + values = null; + return; + } + objectsWithIgnoredFields = new HashMap<>(); + for (Object value : values) { + Values parsedValues = decode(value); + objectsWithIgnoredFields.computeIfAbsent( + // _doc corresponds to the root object + (parsedValues.parentOffset == 0) ? "_doc" : parsedValues.name.substring(0, parsedValues.parentOffset - 1), + k -> new ArrayList<>() + ).add(parsedValues); + } + values = null; + } + + public void write(XContentBuilder b, String prefix) throws IOException { + var matches = objectsWithIgnoredFields.get(prefix); + if (matches != null) { + for (var values : matches) { + b.field(values.parentOffset > 0 ? values.name.substring(values.parentOffset) : values.name); + FieldDataParseHelper.decodeAndWrite(b, values.value); + } + } + } + + public boolean containsIgnoredFields(String prefix) { + return objectsWithIgnoredFields.containsKey(prefix); + } + }; +} diff --git a/server/src/main/java/org/elasticsearch/index/mapper/ObjectMapper.java b/server/src/main/java/org/elasticsearch/index/mapper/ObjectMapper.java index ba396e9a72d30..d1850c220c915 100644 --- a/server/src/main/java/org/elasticsearch/index/mapper/ObjectMapper.java +++ b/server/src/main/java/org/elasticsearch/index/mapper/ObjectMapper.java @@ -784,15 +784,32 @@ public boolean hasValue() { @Override public void write(XContentBuilder b) throws IOException { - if (hasValue == false) { + IgnoredValuesFieldMapper.SyntheticLoader ignoredFieldLoader = null; + for (SourceLoader.SyntheticFieldLoader field : fields) { + if (field instanceof IgnoredValuesFieldMapper.SyntheticLoader loader) { + ignoredFieldLoader = loader; + loader.trackObjectsWithIgnoredFields(); + break; + } + } + write(b, ignoredFieldLoader); + } + + public void write(XContentBuilder b, IgnoredValuesFieldMapper.SyntheticLoader ignoredFieldLoader) throws IOException { + if (hasValue == false && ignoredFieldLoader.containsIgnoredFields(name()) == false) { return; } startSyntheticField(b); for (SourceLoader.SyntheticFieldLoader field : fields) { - if (field.hasValue()) { + if (field instanceof ObjectMapper.SyntheticSourceFieldLoader objectLoader) { + objectLoader.write(b, ignoredFieldLoader); + } else if (field.hasValue()) { field.write(b); } } + if (ignoredFieldLoader != null) { + ignoredFieldLoader.write(b, name()); + } b.endObject(); hasValue = false; } diff --git a/server/src/main/java/org/elasticsearch/index/mapper/SourceFieldMapper.java b/server/src/main/java/org/elasticsearch/index/mapper/SourceFieldMapper.java index 233faf462400b..103a6a8e69328 100644 --- a/server/src/main/java/org/elasticsearch/index/mapper/SourceFieldMapper.java +++ b/server/src/main/java/org/elasticsearch/index/mapper/SourceFieldMapper.java @@ -96,6 +96,14 @@ private static SourceFieldMapper toType(FieldMapper in) { return (SourceFieldMapper) in; } + public static boolean isSynthetic(MappingLookup mappingLookup) { + Mapper sourceMapper = mappingLookup.getMapper(SourceFieldMapper.NAME); + if (sourceMapper instanceof SourceFieldMapper sourceFieldMapper) { + return sourceFieldMapper.isSynthetic(); + } + return false; + } + public static class Builder extends MetadataFieldMapper.Builder { private final Parameter> enabled = Parameter.explicitBoolParam("enabled", false, m -> toType(m).enabled, true) diff --git a/server/src/main/java/org/elasticsearch/indices/IndicesModule.java b/server/src/main/java/org/elasticsearch/indices/IndicesModule.java index b17777fc5a91e..2e409995277eb 100644 --- a/server/src/main/java/org/elasticsearch/indices/IndicesModule.java +++ b/server/src/main/java/org/elasticsearch/indices/IndicesModule.java @@ -39,6 +39,7 @@ import org.elasticsearch.index.mapper.GeoPointScriptFieldType; import org.elasticsearch.index.mapper.IdFieldMapper; import org.elasticsearch.index.mapper.IgnoredFieldMapper; +import org.elasticsearch.index.mapper.IgnoredValuesFieldMapper; import org.elasticsearch.index.mapper.IndexFieldMapper; import org.elasticsearch.index.mapper.IpFieldMapper; import org.elasticsearch.index.mapper.IpScriptFieldType; @@ -258,6 +259,7 @@ private static Map initBuiltInMetadataMa builtInMetadataMappers.put(TimeSeriesRoutingHashFieldMapper.NAME, TimeSeriesRoutingHashFieldMapper.PARSER); builtInMetadataMappers.put(IndexFieldMapper.NAME, IndexFieldMapper.PARSER); builtInMetadataMappers.put(SourceFieldMapper.NAME, SourceFieldMapper.PARSER); + builtInMetadataMappers.put(IgnoredValuesFieldMapper.NAME, IgnoredValuesFieldMapper.PARSER); builtInMetadataMappers.put(NestedPathFieldMapper.NAME, NestedPathFieldMapper.PARSER); builtInMetadataMappers.put(VersionFieldMapper.NAME, VersionFieldMapper.PARSER); builtInMetadataMappers.put(SeqNoFieldMapper.NAME, SeqNoFieldMapper.PARSER); diff --git a/server/src/test/java/org/elasticsearch/index/mapper/DocCountFieldMapperTests.java b/server/src/test/java/org/elasticsearch/index/mapper/DocCountFieldMapperTests.java index c1fd872e89f45..013c3dae95cab 100644 --- a/server/src/test/java/org/elasticsearch/index/mapper/DocCountFieldMapperTests.java +++ b/server/src/test/java/org/elasticsearch/index/mapper/DocCountFieldMapperTests.java @@ -97,7 +97,6 @@ public void testSyntheticSourceMany() throws IOException { } }, reader -> { SourceLoader loader = mapper.mappingLookup().newSourceLoader(); - assertTrue(loader.requiredStoredFields().isEmpty()); for (LeafReaderContext leaf : reader.leaves()) { int[] docIds = IntStream.range(0, leaf.reader().maxDoc()).toArray(); SourceLoader.Leaf sourceLoaderLeaf = loader.leaf(leaf.reader(), docIds); @@ -129,7 +128,6 @@ public void testSyntheticSourceManyDoNotHave() throws IOException { } }, reader -> { SourceLoader loader = mapper.mappingLookup().newSourceLoader(); - assertTrue(loader.requiredStoredFields().isEmpty()); for (LeafReaderContext leaf : reader.leaves()) { int[] docIds = IntStream.range(0, leaf.reader().maxDoc()).toArray(); SourceLoader.Leaf sourceLoaderLeaf = loader.leaf(leaf.reader(), docIds); diff --git a/server/src/test/java/org/elasticsearch/index/mapper/DocumentMapperTests.java b/server/src/test/java/org/elasticsearch/index/mapper/DocumentMapperTests.java index 486b33d9b155a..d707a17e82337 100644 --- a/server/src/test/java/org/elasticsearch/index/mapper/DocumentMapperTests.java +++ b/server/src/test/java/org/elasticsearch/index/mapper/DocumentMapperTests.java @@ -319,6 +319,7 @@ public void testEmptyDocumentMapper() { .item(DocCountFieldMapper.class) .item(FieldNamesFieldMapper.class) .item(IgnoredFieldMapper.class) + .item(IgnoredValuesFieldMapper.class) .item(IndexFieldMapper.class) .item(NestedPathFieldMapper.class) .item(ProvidedIdFieldMapper.class) @@ -336,6 +337,7 @@ public void testEmptyDocumentMapper() { .item(FieldNamesFieldMapper.CONTENT_TYPE) .item(IdFieldMapper.CONTENT_TYPE) .item(IgnoredFieldMapper.CONTENT_TYPE) + .item(IgnoredValuesFieldMapper.NAME) .item(IndexFieldMapper.CONTENT_TYPE) .item(NestedPathFieldMapper.NAME) .item(RoutingFieldMapper.CONTENT_TYPE) diff --git a/server/src/test/java/org/elasticsearch/index/mapper/FieldDataParseHelperTests.java b/server/src/test/java/org/elasticsearch/index/mapper/FieldDataParseHelperTests.java new file mode 100644 index 0000000000000..3833c3ccbe063 --- /dev/null +++ b/server/src/test/java/org/elasticsearch/index/mapper/FieldDataParseHelperTests.java @@ -0,0 +1,107 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License + * 2.0 and the Server Side Public License, v 1; you may not use this file except + * in compliance with, at your election, the Elastic License 2.0 or the Server + * Side Public License, v 1. + */ + +package org.elasticsearch.index.mapper; + +import org.elasticsearch.common.Strings; +import org.elasticsearch.common.bytes.BytesReference; +import org.elasticsearch.test.ESTestCase; +import org.elasticsearch.xcontent.XContentBuilder; +import org.elasticsearch.xcontent.XContentFactory; +import org.elasticsearch.xcontent.XContentParser; +import org.elasticsearch.xcontent.json.JsonXContent; + +import java.io.IOException; +import java.math.BigDecimal; +import java.math.BigInteger; +import java.util.List; + +import static org.hamcrest.Matchers.equalTo; + +public class FieldDataParseHelperTests extends ESTestCase { + + private String encodeAndDecode(String value) throws IOException { + XContentParser p = createParser(JsonXContent.jsonXContent, "{ \"foo\": " + value + " }"); + assertThat(p.nextToken(), equalTo(XContentParser.Token.START_OBJECT)); + assertThat(p.nextToken(), equalTo(XContentParser.Token.FIELD_NAME)); + assertThat(p.currentName(), equalTo("foo")); + p.nextToken(); + + XContentBuilder builder = XContentFactory.jsonBuilder(); + builder.humanReadable(true); + FieldDataParseHelper.decodeAndWrite(builder, FieldDataParseHelper.encodeToken(p)); + return Strings.toString(builder); + } + + public void testBoolean() throws IOException { + boolean b = randomBoolean(); + assertEquals(b, Boolean.parseBoolean(encodeAndDecode(Boolean.toString(b)))); + } + + public void testString() throws IOException { + String s = "\"" + randomAlphaOfLength(5) + "\""; + assertEquals(s, encodeAndDecode(s)); + } + + public void testInt() throws IOException { + int i = randomInt(); + assertEquals(i, Integer.parseInt(encodeAndDecode(Integer.toString(i)))); + } + + public void testLong() throws IOException { + long l = randomLong(); + assertEquals(l, Long.parseLong(encodeAndDecode(Long.toString(l)))); + } + + public void testFloat() throws IOException { + float f = randomFloat(); + assertEquals(0, Float.compare(f, Float.parseFloat(encodeAndDecode(Float.toString(f))))); + } + + public void testDouble() throws IOException { + double d = randomDouble(); + assertEquals(0, Double.compare(d, Double.parseDouble(encodeAndDecode(Double.toString(d))))); + } + + public void testBigInteger() throws IOException { + BigInteger i = randomBigInteger(); + assertEquals(i, new BigInteger(encodeAndDecode(i.toString()), 10)); + } + + public void testBigDecimal() throws IOException { + BigDecimal d = new BigDecimal(randomBigInteger(), randomInt()); + XContentBuilder builder = XContentFactory.jsonBuilder(); + builder.startObject().field("foo", d).endObject(); + XContentParser p = JsonXContent.jsonXContent.createParser(parserConfig(), BytesReference.bytes(builder).streamInput()); + assertThat(p.nextToken(), equalTo(XContentParser.Token.START_OBJECT)); + assertThat(p.nextToken(), equalTo(XContentParser.Token.FIELD_NAME)); + assertThat(p.currentName(), equalTo("foo")); + p.nextToken(); + assertEquals(p.numberType(), XContentParser.NumberType.DOUBLE); + + // FieldDataParseHelper.decodeAndWrite(builder, FieldDataParseHelper.encodeToken(p)); + // assertEquals(d, new BigDecimal(Strings.toString(builder))); + } + + public void testObject() throws IOException { + String object = "{\"name\":\"foo\"}"; + XContentParser p = createParser(JsonXContent.jsonXContent, object); + assertThat(p.nextToken(), equalTo(XContentParser.Token.START_OBJECT)); + XContentBuilder builder = XContentFactory.jsonBuilder(); + builder.humanReadable(true); + FieldDataParseHelper.decodeAndWrite(builder, FieldDataParseHelper.encodeToken(p)); + assertEquals(object, Strings.toString(builder)); + } + + public void testArrayInt() throws IOException { + String values = "[" + + String.join(",", List.of(Integer.toString(randomInt()), Integer.toString(randomInt()), Integer.toString(randomInt()))) + + "]"; + assertEquals(values, encodeAndDecode(values)); + } +} diff --git a/server/src/test/java/org/elasticsearch/index/mapper/IgnoredValuedFieldMapperTests.java b/server/src/test/java/org/elasticsearch/index/mapper/IgnoredValuedFieldMapperTests.java new file mode 100644 index 0000000000000..710fb9f2298ea --- /dev/null +++ b/server/src/test/java/org/elasticsearch/index/mapper/IgnoredValuedFieldMapperTests.java @@ -0,0 +1,148 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License + * 2.0 and the Server Side Public License, v 1; you may not use this file except + * in compliance with, at your election, the Elastic License 2.0 or the Server + * Side Public License, v 1. + */ + +package org.elasticsearch.index.mapper; + +import org.elasticsearch.common.settings.Settings; +import org.elasticsearch.core.CheckedConsumer; +import org.elasticsearch.xcontent.XContentBuilder; +import org.hamcrest.Matchers; + +import java.io.IOException; +import java.math.BigInteger; +import java.util.Base64; + +public class IgnoredValuedFieldMapperTests extends MapperServiceTestCase { + + private String getSyntheticSource(CheckedConsumer build) throws IOException { + DocumentMapper documentMapper = createMapperService( + Settings.builder() + .put("index.mapping.total_fields.limit", 2) + .put("index.mapping.total_fields.ignore_dynamic_beyond_limit", true) + .build(), + syntheticSourceMapping(b -> { + b.startObject("foo").field("type", "keyword").endObject(); + b.startObject("bar").field("type", "object").endObject(); + }) + ).documentMapper(); + return syntheticSource(documentMapper, build); + } + + public void testIgnoredBoolean() throws IOException { + boolean value = randomBoolean(); + assertEquals("{\"my_value\":" + value + "}", getSyntheticSource(b -> b.field("my_value", value))); + } + + public void testIgnoredString() throws IOException { + String value = randomAlphaOfLength(5); + assertEquals("{\"my_value\":\"" + value + "\"}", getSyntheticSource(b -> b.field("my_value", value))); + } + + public void testIgnoredInt() throws IOException { + int value = randomInt(); + assertEquals("{\"my_value\":" + value + "}", getSyntheticSource(b -> b.field("my_value", value))); + } + + public void testIgnoredLong() throws IOException { + long value = randomLong(); + assertEquals("{\"my_value\":" + value + "}", getSyntheticSource(b -> b.field("my_value", value))); + } + + public void testIgnoredFloat() throws IOException { + float value = randomFloat(); + assertEquals("{\"my_value\":" + value + "}", getSyntheticSource(b -> b.field("my_value", value))); + } + + public void testIgnoredDouble() throws IOException { + double value = randomDouble(); + assertEquals("{\"my_value\":" + value + "}", getSyntheticSource(b -> b.field("my_value", value))); + } + + public void testIgnoredBigInteger() throws IOException { + BigInteger value = randomBigInteger(); + assertEquals("{\"my_value\":" + value + "}", getSyntheticSource(b -> b.field("my_value", value))); + } + + public void testIgnoredBytes() throws IOException { + byte[] value = randomByteArrayOfLength(10); + assertEquals( + "{\"my_value\":\"" + Base64.getEncoder().encodeToString(value) + "\"}", + getSyntheticSource(b -> b.field("my_value", value)) + ); + } + + public void testIgnoredObjectBoolean() throws IOException { + boolean value = randomBoolean(); + assertEquals("{\"my_value\":" + value + "}", getSyntheticSource(b -> b.field("my_value", value))); + } + + public void testMultipleIgnoredFieldsRootObject() throws IOException { + boolean booleanValue = randomBoolean(); + int intValue = randomInt(); + String stringValue = randomAlphaOfLength(20); + String syntheticSource = getSyntheticSource(b -> { + b.field("boolean_value", booleanValue); + b.field("int_value", intValue); + b.field("string_value", stringValue); + }); + assertThat(syntheticSource, Matchers.containsString("\"boolean_value\":" + booleanValue)); + assertThat(syntheticSource, Matchers.containsString("\"int_value\":" + intValue)); + assertThat(syntheticSource, Matchers.containsString("\"string_value\":\"" + stringValue + "\"")); + } + + public void testMultipleIgnoredFieldsSameObject() throws IOException { + boolean booleanValue = randomBoolean(); + int intValue = randomInt(); + String stringValue = randomAlphaOfLength(20); + String syntheticSource = getSyntheticSource(b -> { + b.startObject("bar"); + { + b.field("boolean_value", booleanValue); + b.field("int_value", intValue); + b.field("string_value", stringValue); + } + b.endObject(); + }); + assertThat(syntheticSource, Matchers.containsString("{\"bar\":{")); + assertThat(syntheticSource, Matchers.containsString("\"boolean_value\":" + booleanValue)); + assertThat(syntheticSource, Matchers.containsString("\"int_value\":" + intValue)); + assertThat(syntheticSource, Matchers.containsString("\"string_value\":\"" + stringValue + "\"")); + } + + public void testMultipleIgnoredFieldsManyObjects() throws IOException { + boolean booleanValue = randomBoolean(); + int intValue = randomInt(); + String stringValue = randomAlphaOfLength(20); + String syntheticSource = getSyntheticSource(b -> { + b.field("boolean_value", booleanValue); + b.startObject("path"); + { + b.startObject("to"); + { + b.field("int_value", intValue); + b.startObject("some"); + { + b.startObject("deeply"); + { + b.startObject("nested"); + b.field("string_value", stringValue); + b.endObject(); + } + b.endObject(); + } + b.endObject(); + } + b.endObject(); + } + b.endObject(); + }); + assertThat(syntheticSource, Matchers.containsString("\"boolean_value\":" + booleanValue)); + assertThat(syntheticSource, Matchers.containsString("\"path\":{\"to\":{\"int_value\":" + intValue)); + assertThat(syntheticSource, Matchers.containsString("\"some\":{\"deeply\":{\"nested\":{\"string_value\":\"" + stringValue + "\"")); + } +} diff --git a/server/src/test/java/org/elasticsearch/indices/IndicesModuleTests.java b/server/src/test/java/org/elasticsearch/indices/IndicesModuleTests.java index 0216bad7cf7a3..c93b3dfb31bcd 100644 --- a/server/src/test/java/org/elasticsearch/indices/IndicesModuleTests.java +++ b/server/src/test/java/org/elasticsearch/indices/IndicesModuleTests.java @@ -15,6 +15,7 @@ import org.elasticsearch.index.mapper.FieldNamesFieldMapper; import org.elasticsearch.index.mapper.IdFieldMapper; import org.elasticsearch.index.mapper.IgnoredFieldMapper; +import org.elasticsearch.index.mapper.IgnoredValuesFieldMapper; import org.elasticsearch.index.mapper.IndexFieldMapper; import org.elasticsearch.index.mapper.KeywordFieldMapper; import org.elasticsearch.index.mapper.Mapper; @@ -85,6 +86,7 @@ public Map getMetadataMappers() { TimeSeriesRoutingHashFieldMapper.NAME, IndexFieldMapper.NAME, SourceFieldMapper.NAME, + IgnoredValuesFieldMapper.NAME, NestedPathFieldMapper.NAME, VersionFieldMapper.NAME, SeqNoFieldMapper.NAME, From 5e246389588cc15c949d52cbd93470b05d6588c4 Mon Sep 17 00:00:00 2001 From: Kostas Krikellas <131142368+kkrik-es@users.noreply.github.com> Date: Wed, 17 Apr 2024 17:02:51 +0300 Subject: [PATCH 02/19] Update docs/changelog/107567.yaml --- docs/changelog/107567.yaml | 5 +++++ 1 file changed, 5 insertions(+) create mode 100644 docs/changelog/107567.yaml diff --git a/docs/changelog/107567.yaml b/docs/changelog/107567.yaml new file mode 100644 index 0000000000000..558b5b570b1fb --- /dev/null +++ b/docs/changelog/107567.yaml @@ -0,0 +1,5 @@ +pr: 107567 +summary: Add ignored field values to synthetic source +area: Mapping +type: enhancement +issues: [] From b4a74c36cfae2ac281c93bdb6a4e323cc4f9ca9a Mon Sep 17 00:00:00 2001 From: Kostas Krikellas Date: Wed, 17 Apr 2024 17:45:39 +0300 Subject: [PATCH 03/19] initialize map --- .../elasticsearch/index/mapper/IgnoredValuesFieldMapper.java | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/server/src/main/java/org/elasticsearch/index/mapper/IgnoredValuesFieldMapper.java b/server/src/main/java/org/elasticsearch/index/mapper/IgnoredValuesFieldMapper.java index e85a853e00946..76f562cb5d011 100644 --- a/server/src/main/java/org/elasticsearch/index/mapper/IgnoredValuesFieldMapper.java +++ b/server/src/main/java/org/elasticsearch/index/mapper/IgnoredValuesFieldMapper.java @@ -114,7 +114,7 @@ public static class SyntheticLoader implements SourceLoader.SyntheticFieldLoader private List values = List.of(); // Maps the names of existing objects to lists of ignored fields they contain. - private Map> objectsWithIgnoredFields = null; + private Map> objectsWithIgnoredFields = Map.of(); @Override public Stream> storedFieldLoaders() { From 9e43764c6781f8822321d7dd484786fb12fa4196 Mon Sep 17 00:00:00 2001 From: Kostas Krikellas Date: Wed, 17 Apr 2024 18:35:57 +0300 Subject: [PATCH 04/19] yaml fix --- .../test/nodes.stats/11_indices_metrics.yml | 47 ++++++++++--------- 1 file changed, 24 insertions(+), 23 deletions(-) diff --git a/rest-api-spec/src/yamlRestTest/resources/rest-api-spec/test/nodes.stats/11_indices_metrics.yml b/rest-api-spec/src/yamlRestTest/resources/rest-api-spec/test/nodes.stats/11_indices_metrics.yml index 1a7da98af9129..574761643498e 100644 --- a/rest-api-spec/src/yamlRestTest/resources/rest-api-spec/test/nodes.stats/11_indices_metrics.yml +++ b/rest-api-spec/src/yamlRestTest/resources/rest-api-spec/test/nodes.stats/11_indices_metrics.yml @@ -476,35 +476,36 @@ # 4. _field_names # 5. _id # 6. _ignored - # 7. _index - # 8. _nested_path - # 9. _routing - # 10. _seq_no - # 11. _source - # 12. _tier - # 13. _version - # 14. @timestamp - # 15. authors.age - # 16. authors.company - # 17. authors.company.keyword - # 18. authors.name.last_name - # 19. authors.name.first_name - # 20. authors.name.full_name - # 21. link - # 22. title - # 23. url + # 7. _ignored_values + # 8. _index + # 9. _nested_path + # 10. _routing + # 11. _seq_no + # 12. _source + # 13. _tier + # 14. _version + # 15. @timestamp + # 16. authors.age + # 17. authors.company + # 18. authors.company.keyword + # 19. authors.name.last_name + # 20. authors.name.first_name + # 21. authors.name.full_name + # 22. link + # 23. title + # 24. url # Object mappers: - # 24. authors - # 25. authors.name + # 25. authors + # 26. authors.name # Runtime field mappers: - # 26. a_source_field + # 27. a_source_field - - gte: { nodes.$node_id.indices.mappings.total_count: 26 } + - gte: { nodes.$node_id.indices.mappings.total_count: 27 } - is_true: nodes.$node_id.indices.mappings.total_estimated_overhead - gte: { nodes.$node_id.indices.mappings.total_estimated_overhead_in_bytes: 26624 } - - match: { nodes.$node_id.indices.indices.index1.mappings.total_count: 26 } + - match: { nodes.$node_id.indices.indices.index1.mappings.total_count: 27 } - is_true: nodes.$node_id.indices.indices.index1.mappings.total_estimated_overhead - - match: { nodes.$node_id.indices.indices.index1.mappings.total_estimated_overhead_in_bytes: 26624 } + - match: { nodes.$node_id.indices.indices.index1.mappings.total_estimated_overhead_in_bytes: 27648 } --- "indices mappings does not exist in shards level": From 528986ec6e7e9e9badb596daff5433ff8a4cfe6b Mon Sep 17 00:00:00 2001 From: Kostas Krikellas Date: Thu, 18 Apr 2024 11:22:56 +0300 Subject: [PATCH 05/19] add node feature --- .../indices.create/20_synthetic_source.yml | 18 +++++++------- server/src/main/java/module-info.java | 1 + .../mapper/IgnoredValuesFieldMapper.java | 10 ++++++++ .../index/mapper/MapperFeatures.java | 24 +++++++++++++++++++ ...lasticsearch.features.FeatureSpecification | 1 + 5 files changed, 45 insertions(+), 9 deletions(-) create mode 100644 server/src/main/java/org/elasticsearch/index/mapper/MapperFeatures.java diff --git a/rest-api-spec/src/yamlRestTest/resources/rest-api-spec/test/indices.create/20_synthetic_source.yml b/rest-api-spec/src/yamlRestTest/resources/rest-api-spec/test/indices.create/20_synthetic_source.yml index 9c9675d41febe..0376604d4c43c 100644 --- a/rest-api-spec/src/yamlRestTest/resources/rest-api-spec/test/indices.create/20_synthetic_source.yml +++ b/rest-api-spec/src/yamlRestTest/resources/rest-api-spec/test/indices.create/20_synthetic_source.yml @@ -39,9 +39,9 @@ nested is disabled: --- object with unmapped fields: - - skip: - version: " - 8.13.99" - reason: introduced in 8.14.0 + - requires: + cluster_features: ["mapper.track_ignored_values"] + reason: requires tracking ignored values - do: indices.create: @@ -91,9 +91,9 @@ object with unmapped fields: --- nested object with unmapped fields: - - skip: - version: " - 8.13.99" - reason: introduced in 8.14.0 + - requires: + cluster_features: ["mapper.track_ignored_values"] + reason: requires tracking ignored values - do: indices.create: @@ -142,9 +142,9 @@ nested object with unmapped fields: --- empty object with unmapped fields: - - skip: - version: " - 8.13.99" - reason: introduced in 8.14.0 + - requires: + cluster_features: ["mapper.track_ignored_values"] + reason: requires tracking ignored values - do: indices.create: diff --git a/server/src/main/java/module-info.java b/server/src/main/java/module-info.java index 6e01cb01626ae..5276b72070ae0 100644 --- a/server/src/main/java/module-info.java +++ b/server/src/main/java/module-info.java @@ -423,6 +423,7 @@ org.elasticsearch.cluster.metadata.MetadataFeatures, org.elasticsearch.rest.RestFeatures, org.elasticsearch.indices.IndicesFeatures, + org.elasticsearch.index.mapper.MapperFeatures, org.elasticsearch.search.retriever.RetrieversFeatures; uses org.elasticsearch.plugins.internal.SettingsExtension; diff --git a/server/src/main/java/org/elasticsearch/index/mapper/IgnoredValuesFieldMapper.java b/server/src/main/java/org/elasticsearch/index/mapper/IgnoredValuesFieldMapper.java index 76f562cb5d011..f30e0eda6484e 100644 --- a/server/src/main/java/org/elasticsearch/index/mapper/IgnoredValuesFieldMapper.java +++ b/server/src/main/java/org/elasticsearch/index/mapper/IgnoredValuesFieldMapper.java @@ -13,6 +13,7 @@ import org.apache.lucene.index.LeafReader; import org.apache.lucene.util.BytesRef; import org.elasticsearch.common.util.ByteUtils; +import org.elasticsearch.features.NodeFeature; import org.elasticsearch.index.query.SearchExecutionContext; import org.elasticsearch.xcontent.XContentBuilder; @@ -46,6 +47,15 @@ public class IgnoredValuesFieldMapper extends MetadataFieldMapper { public static final TypeParser PARSER = new FixedTypeParser(context -> INSTANCE); + static final NodeFeature TRACK_IGNORED_VALUES = new NodeFeature("mapper.track_ignored_values"); + + /* + * Container for the ignored field data: + * - the full name + * - the offset in the full name indicating the end of the substring matching + * the full name of the parent field + * - the value, encoded as a byte array + */ public record Values(String name, int parentOffset, BytesRef value) {} static final class IgnoredValuesFieldMapperType extends StringFieldType { diff --git a/server/src/main/java/org/elasticsearch/index/mapper/MapperFeatures.java b/server/src/main/java/org/elasticsearch/index/mapper/MapperFeatures.java new file mode 100644 index 0000000000000..50ad63efb5241 --- /dev/null +++ b/server/src/main/java/org/elasticsearch/index/mapper/MapperFeatures.java @@ -0,0 +1,24 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License + * 2.0 and the Server Side Public License, v 1; you may not use this file except + * in compliance with, at your election, the Elastic License 2.0 or the Server + * Side Public License, v 1. + */ + +package org.elasticsearch.index.mapper; + +import org.elasticsearch.features.FeatureSpecification; +import org.elasticsearch.features.NodeFeature; + +import java.util.Set; + +/** + * Spec for mapper-related features. + */ +public class MapperFeatures implements FeatureSpecification { + @Override + public Set getFeatures() { + return Set.of(IgnoredValuesFieldMapper.TRACK_IGNORED_VALUES); + } +} diff --git a/server/src/main/resources/META-INF/services/org.elasticsearch.features.FeatureSpecification b/server/src/main/resources/META-INF/services/org.elasticsearch.features.FeatureSpecification index 71b6aacd56ea7..aa6c60383d0e5 100644 --- a/server/src/main/resources/META-INF/services/org.elasticsearch.features.FeatureSpecification +++ b/server/src/main/resources/META-INF/services/org.elasticsearch.features.FeatureSpecification @@ -12,4 +12,5 @@ org.elasticsearch.cluster.service.TransportFeatures org.elasticsearch.cluster.metadata.MetadataFeatures org.elasticsearch.rest.RestFeatures org.elasticsearch.indices.IndicesFeatures +org.elasticsearch.index.mapper.MapperFeatures org.elasticsearch.search.retriever.RetrieversFeatures From 6dd4080758fa0c7fa990bbf31be59170c81768cb Mon Sep 17 00:00:00 2001 From: Kostas Krikellas Date: Thu, 18 Apr 2024 12:28:31 +0300 Subject: [PATCH 06/19] add comments --- .../index/mapper/IgnoredValuesFieldMapper.java | 8 +++++++- 1 file changed, 7 insertions(+), 1 deletion(-) diff --git a/server/src/main/java/org/elasticsearch/index/mapper/IgnoredValuesFieldMapper.java b/server/src/main/java/org/elasticsearch/index/mapper/IgnoredValuesFieldMapper.java index f30e0eda6484e..ce8316b4214af 100644 --- a/server/src/main/java/org/elasticsearch/index/mapper/IgnoredValuesFieldMapper.java +++ b/server/src/main/java/org/elasticsearch/index/mapper/IgnoredValuesFieldMapper.java @@ -121,6 +121,7 @@ public SourceLoader.SyntheticFieldLoader syntheticFieldLoader() { } public static class SyntheticLoader implements SourceLoader.SyntheticFieldLoader { + // Contains stored field values, i.e. encoded ignored field names and values. private List values = List.of(); // Maps the names of existing objects to lists of ignored fields they contain. @@ -143,7 +144,11 @@ public boolean hasValue() { } @Override - public void write(XContentBuilder b) throws IOException {} + public void write(XContentBuilder b) throws IOException { + // This mapper doesn't contribute to source directly as it has no access to the + // object structure. Instead, it's accessed by object mappers to decode + // and write its fields within the appropriate object. + } public void trackObjectsWithIgnoredFields() { if (values == null || values.isEmpty()) { @@ -162,6 +167,7 @@ public void trackObjectsWithIgnoredFields() { values = null; } + // This is expected to be called by object mappers, to add their ignored fields as part of the source. public void write(XContentBuilder b, String prefix) throws IOException { var matches = objectsWithIgnoredFields.get(prefix); if (matches != null) { From 3bd67c17972337ac81c89c3ce33b8394a6b3ff64 Mon Sep 17 00:00:00 2001 From: Kostas Krikellas Date: Mon, 22 Apr 2024 14:20:25 +0300 Subject: [PATCH 07/19] small fixes --- .../index/mapper/IgnoredValuesFieldMapper.java | 16 ++++++++++------ 1 file changed, 10 insertions(+), 6 deletions(-) diff --git a/server/src/main/java/org/elasticsearch/index/mapper/IgnoredValuesFieldMapper.java b/server/src/main/java/org/elasticsearch/index/mapper/IgnoredValuesFieldMapper.java index ce8316b4214af..0972f6edcbab7 100644 --- a/server/src/main/java/org/elasticsearch/index/mapper/IgnoredValuesFieldMapper.java +++ b/server/src/main/java/org/elasticsearch/index/mapper/IgnoredValuesFieldMapper.java @@ -39,7 +39,11 @@ */ public class IgnoredValuesFieldMapper extends MetadataFieldMapper { - private static final int OFFSET_IN_NAME = 100_000; + // This factor is used to combine two offsets within the same integer: + // - the offset of the end of the parent field within the field name (N / PARENT_OFFSET_IN_NAME_OFFSET) + // - the offset of the field value within the encoding string containing the offset (first 4 bytes), the field name and value + // (N % PARENT_OFFSET_IN_NAME_OFFSET) + private static final int PARENT_OFFSET_IN_NAME_OFFSET = 1 << 16; public static final String NAME = "_ignored_values"; @@ -94,12 +98,12 @@ public void postParse(DocumentParserContext context) { } static String encode(Values values) { - assert values.parentOffset < OFFSET_IN_NAME; - assert values.parentOffset * (long) OFFSET_IN_NAME < Integer.MAX_VALUE; + assert values.parentOffset < PARENT_OFFSET_IN_NAME_OFFSET; + assert values.parentOffset * (long) PARENT_OFFSET_IN_NAME_OFFSET < Integer.MAX_VALUE; byte[] nameBytes = values.name.getBytes(StandardCharsets.UTF_8); byte[] bytes = new byte[4 + nameBytes.length + values.value.length]; - ByteUtils.writeIntLE(values.name.length() + OFFSET_IN_NAME * values.parentOffset, bytes, 0); + ByteUtils.writeIntLE(values.name.length() + PARENT_OFFSET_IN_NAME_OFFSET * values.parentOffset, bytes, 0); System.arraycopy(nameBytes, 0, bytes, 4, nameBytes.length); System.arraycopy(values.value.bytes, values.value.offset, bytes, 4 + nameBytes.length, values.value.length); return Base64.getUrlEncoder().withoutPadding().encodeToString(bytes); @@ -108,8 +112,8 @@ static String encode(Values values) { static Values decode(Object field) { byte[] bytes = Base64.getUrlDecoder().decode((String) field); int encodedSize = ByteUtils.readIntLE(bytes, 0); - int nameSize = encodedSize % OFFSET_IN_NAME; - int parentOffset = encodedSize / OFFSET_IN_NAME; + int nameSize = encodedSize % PARENT_OFFSET_IN_NAME_OFFSET; + int parentOffset = encodedSize / PARENT_OFFSET_IN_NAME_OFFSET; String name = new String(bytes, 4, nameSize, StandardCharsets.UTF_8); BytesRef value = new BytesRef(bytes, 4 + nameSize, bytes.length - nameSize - 4); return new Values(name, parentOffset, value); From d3caea11cfc4d237642c0855b9847d85a34ac5f0 Mon Sep 17 00:00:00 2001 From: Kostas Krikellas Date: Mon, 22 Apr 2024 14:58:00 +0300 Subject: [PATCH 08/19] missing cluster feature in yaml --- .../rest-api-spec/test/nodes.stats/11_indices_metrics.yml | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/rest-api-spec/src/yamlRestTest/resources/rest-api-spec/test/nodes.stats/11_indices_metrics.yml b/rest-api-spec/src/yamlRestTest/resources/rest-api-spec/test/nodes.stats/11_indices_metrics.yml index 574761643498e..1238bc210fc14 100644 --- a/rest-api-spec/src/yamlRestTest/resources/rest-api-spec/test/nodes.stats/11_indices_metrics.yml +++ b/rest-api-spec/src/yamlRestTest/resources/rest-api-spec/test/nodes.stats/11_indices_metrics.yml @@ -417,8 +417,8 @@ - requires: test_runner_features: [arbitrary_key] - cluster_features: ["gte_v8.5.0"] - reason: "mappings added in version 8.5.0" + cluster_features: ["mapper.track_ignored_values"] + reason: "_ignored_values added to mappings" - do: indices.create: From d74b7fa42498532bd41fb071251647789d75130b Mon Sep 17 00:00:00 2001 From: Kostas Krikellas Date: Tue, 23 Apr 2024 10:54:54 +0300 Subject: [PATCH 09/19] constants for chars, stored fields --- .../index/mapper/FieldDataParseHelper.java | 114 ++++++++++++------ .../mapper/IgnoredValuesFieldMapper.java | 5 +- 2 files changed, 79 insertions(+), 40 deletions(-) diff --git a/server/src/main/java/org/elasticsearch/index/mapper/FieldDataParseHelper.java b/server/src/main/java/org/elasticsearch/index/mapper/FieldDataParseHelper.java index 1478f905165d5..0fc82e2a37bc7 100644 --- a/server/src/main/java/org/elasticsearch/index/mapper/FieldDataParseHelper.java +++ b/server/src/main/java/org/elasticsearch/index/mapper/FieldDataParseHelper.java @@ -23,6 +23,7 @@ import java.math.BigDecimal; import java.math.BigInteger; import java.nio.charset.StandardCharsets; +import java.util.Arrays; /** * Helper class for processing field data of any type, as provided by the {@link XContentParser}. @@ -56,17 +57,17 @@ public static BytesRef encodeToken(XContentParser parser) throws IOException { * {@link #encodeToken(XContentParser)} above. */ public static void decodeAndWrite(XContentBuilder b, BytesRef r) throws IOException { - switch (r.bytes[r.offset]) { - case 'b' -> TypeUtils.EMBEDDED_OBJECT.decodeAndWrite(b, r); - case 'c', 'j', 's', 'y' -> TypeUtils.START.decodeAndWrite(b, r); - case 'd' -> TypeUtils.BIG_DECIMAL.decodeAndWrite(b, r); - case 'f', 't' -> TypeUtils.BOOLEAN.decodeAndWrite(b, r); - case 'i' -> TypeUtils.BIG_INTEGER.decodeAndWrite(b, r); - case 'S' -> TypeUtils.STRING.decodeAndWrite(b, r); - case 'I' -> TypeUtils.INTEGER.decodeAndWrite(b, r); - case 'L' -> TypeUtils.LONG.decodeAndWrite(b, r); - case 'D' -> TypeUtils.DOUBLE.decodeAndWrite(b, r); - case 'F' -> TypeUtils.FLOAT.decodeAndWrite(b, r); + switch ((char) r.bytes[r.offset]) { + case BINARY_VALUE -> TypeUtils.EMBEDDED_OBJECT.decodeAndWrite(b, r); + case CBOR_OBJECT_VALUE, JSON_OBJECT_VALUE, YAML_OBJECT_VALUE, SMILE_OBJECT_VALUE -> TypeUtils.START.decodeAndWrite(b, r); + case BIG_DECIMAL_VALUE -> TypeUtils.BIG_DECIMAL.decodeAndWrite(b, r); + case FALSE_VALUE, TRUE_VALUE -> TypeUtils.BOOLEAN.decodeAndWrite(b, r); + case BIG_INTEGER_VALUE -> TypeUtils.BIG_INTEGER.decodeAndWrite(b, r); + case STRING_VALUE -> TypeUtils.STRING.decodeAndWrite(b, r); + case INTEGER_VALUE -> TypeUtils.INTEGER.decodeAndWrite(b, r); + case LONG_VALUE -> TypeUtils.LONG.decodeAndWrite(b, r); + case DOUBLE_VALUE -> TypeUtils.DOUBLE.decodeAndWrite(b, r); + case FLOAT_VALUE -> TypeUtils.FLOAT.decodeAndWrite(b, r); default -> throw new IllegalArgumentException("Can't decode " + r); } } @@ -89,8 +90,23 @@ private static Object processToken(XContentParser parser, CheckedFunction decodeAndWriteXContent(b, XContentType.CBOR, r); - case 'j' -> decodeAndWriteXContent(b, XContentType.JSON, r); - case 's' -> decodeAndWriteXContent(b, XContentType.SMILE, r); - case 'y' -> decodeAndWriteXContent(b, XContentType.YAML, r); + switch ((char) r.bytes[r.offset]) { + case CBOR_OBJECT_VALUE -> decodeAndWriteXContent(b, XContentType.CBOR, r); + case JSON_OBJECT_VALUE -> decodeAndWriteXContent(b, XContentType.JSON, r); + case SMILE_OBJECT_VALUE -> decodeAndWriteXContent(b, XContentType.SMILE, r); + case YAML_OBJECT_VALUE -> decodeAndWriteXContent(b, XContentType.YAML, r); default -> throw new IllegalArgumentException("Can't decode " + r); } } }; TypeUtils(char encoding) { + this.encoding = new Character[] { encoding }; + } + + TypeUtils(Character[] encoding) { this.encoding = encoding; } byte getEncoding() { - return (byte) encoding; + assert encoding.length == 1; + return (byte) encoding[0].charValue(); + } + + void assertValidEncoding(byte[] encodedValue) { + assert Arrays.asList(encoding).contains((char) encodedValue[0]); } - final char encoding; + final Character[] encoding; abstract StoredField buildStoredField(String name, XContentParser parser) throws IOException; @@ -328,10 +368,10 @@ static byte[] encode(XContentBuilder builder) throws IOException { BytesReference b = BytesReference.bytes(builder); byte[] encoded = new byte[1 + b.length()]; encoded[0] = switch (builder.contentType()) { - case JSON -> 'j'; - case SMILE -> 's'; - case YAML -> 'y'; - case CBOR -> 'c'; + case JSON -> JSON_OBJECT_VALUE; + case SMILE -> SMILE_OBJECT_VALUE; + case YAML -> YAML_OBJECT_VALUE; + case CBOR -> CBOR_OBJECT_VALUE; default -> throw new IllegalArgumentException("unsupported type " + builder.contentType()); }; diff --git a/server/src/main/java/org/elasticsearch/index/mapper/IgnoredValuesFieldMapper.java b/server/src/main/java/org/elasticsearch/index/mapper/IgnoredValuesFieldMapper.java index 0972f6edcbab7..0523d474511d6 100644 --- a/server/src/main/java/org/elasticsearch/index/mapper/IgnoredValuesFieldMapper.java +++ b/server/src/main/java/org/elasticsearch/index/mapper/IgnoredValuesFieldMapper.java @@ -8,8 +8,7 @@ package org.elasticsearch.index.mapper; -import org.apache.lucene.document.Field; -import org.apache.lucene.document.StringField; +import org.apache.lucene.document.StoredField; import org.apache.lucene.index.LeafReader; import org.apache.lucene.util.BytesRef; import org.elasticsearch.common.util.ByteUtils; @@ -93,7 +92,7 @@ protected String contentType() { @Override public void postParse(DocumentParserContext context) { for (Values values : context.getIgnoredFieldValues()) { - context.doc().add(new StringField(NAME, encode(values), Field.Store.YES)); + context.doc().add(new StoredField(NAME, encode(values))); } } From 4c4f0a7a95ecde0b70e7d70a71f97347b0e2999d Mon Sep 17 00:00:00 2001 From: Kostas Krikellas Date: Tue, 23 Apr 2024 13:16:55 +0300 Subject: [PATCH 10/19] remove duplicate method --- .../elasticsearch/index/mapper/DocumentParserContext.java | 2 +- .../org/elasticsearch/index/mapper/SourceFieldMapper.java | 8 -------- 2 files changed, 1 insertion(+), 9 deletions(-) diff --git a/server/src/main/java/org/elasticsearch/index/mapper/DocumentParserContext.java b/server/src/main/java/org/elasticsearch/index/mapper/DocumentParserContext.java index ad6014eb521bd..1883a82a77b75 100644 --- a/server/src/main/java/org/elasticsearch/index/mapper/DocumentParserContext.java +++ b/server/src/main/java/org/elasticsearch/index/mapper/DocumentParserContext.java @@ -364,7 +364,7 @@ public final boolean addDynamicMapper(Mapper mapper) { int additionalFieldsToAdd = getNewFieldsSize() + mapperSize; if (indexSettings().isIgnoreDynamicFieldsBeyondLimit()) { if (mappingLookup.exceedsLimit(indexSettings().getMappingTotalFieldsLimit(), additionalFieldsToAdd)) { - if (indexSettings().getMode().isSyntheticSourceEnabled() || SourceFieldMapper.isSynthetic(mappingLookup)) { + if (indexSettings().getMode().isSyntheticSourceEnabled() || mappingLookup.isSourceSynthetic()) { try { int parentOffset = parent() instanceof RootObjectMapper ? 0 : parent().fullPath().length() + 1; addIgnoredValues( diff --git a/server/src/main/java/org/elasticsearch/index/mapper/SourceFieldMapper.java b/server/src/main/java/org/elasticsearch/index/mapper/SourceFieldMapper.java index 103a6a8e69328..233faf462400b 100644 --- a/server/src/main/java/org/elasticsearch/index/mapper/SourceFieldMapper.java +++ b/server/src/main/java/org/elasticsearch/index/mapper/SourceFieldMapper.java @@ -96,14 +96,6 @@ private static SourceFieldMapper toType(FieldMapper in) { return (SourceFieldMapper) in; } - public static boolean isSynthetic(MappingLookup mappingLookup) { - Mapper sourceMapper = mappingLookup.getMapper(SourceFieldMapper.NAME); - if (sourceMapper instanceof SourceFieldMapper sourceFieldMapper) { - return sourceFieldMapper.isSynthetic(); - } - return false; - } - public static class Builder extends MetadataFieldMapper.Builder { private final Parameter> enabled = Parameter.explicitBoolParam("enabled", false, m -> toType(m).enabled, true) From a210b7a8da747ca832abd43ab77d3f54595d924a Mon Sep 17 00:00:00 2001 From: Kostas Krikellas Date: Tue, 23 Apr 2024 13:25:57 +0300 Subject: [PATCH 11/19] throw exception on parse failure --- .../org/elasticsearch/index/mapper/DocumentParserContext.java | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/server/src/main/java/org/elasticsearch/index/mapper/DocumentParserContext.java b/server/src/main/java/org/elasticsearch/index/mapper/DocumentParserContext.java index 1883a82a77b75..2bb49bc00a554 100644 --- a/server/src/main/java/org/elasticsearch/index/mapper/DocumentParserContext.java +++ b/server/src/main/java/org/elasticsearch/index/mapper/DocumentParserContext.java @@ -371,8 +371,7 @@ public final boolean addDynamicMapper(Mapper mapper) { new IgnoredValuesFieldMapper.Values(mapper.name(), parentOffset, FieldDataParseHelper.encodeToken(parser())) ); } catch (IOException e) { - // A parsing failure for ignored field values should be *very* rare. - // It is not considered important enough to drop the document, ignore it silently for now. + throw new IllegalArgumentException("failed to parse field [" + mapper.name() + " ]", e); } } addIgnoredField(mapper.name()); From 09f2f776ee7236267cf8cd2a680843af17af0bc9 Mon Sep 17 00:00:00 2001 From: Kostas Krikellas Date: Wed, 24 Apr 2024 09:26:55 +0300 Subject: [PATCH 12/19] remove Base64 encoding --- .../index/mapper/IgnoredValuesFieldMapper.java | 7 +++---- 1 file changed, 3 insertions(+), 4 deletions(-) diff --git a/server/src/main/java/org/elasticsearch/index/mapper/IgnoredValuesFieldMapper.java b/server/src/main/java/org/elasticsearch/index/mapper/IgnoredValuesFieldMapper.java index 0523d474511d6..7e0def5760bf2 100644 --- a/server/src/main/java/org/elasticsearch/index/mapper/IgnoredValuesFieldMapper.java +++ b/server/src/main/java/org/elasticsearch/index/mapper/IgnoredValuesFieldMapper.java @@ -19,7 +19,6 @@ import java.io.IOException; import java.nio.charset.StandardCharsets; import java.util.ArrayList; -import java.util.Base64; import java.util.Collections; import java.util.HashMap; import java.util.List; @@ -96,7 +95,7 @@ public void postParse(DocumentParserContext context) { } } - static String encode(Values values) { + static byte[] encode(Values values) { assert values.parentOffset < PARENT_OFFSET_IN_NAME_OFFSET; assert values.parentOffset * (long) PARENT_OFFSET_IN_NAME_OFFSET < Integer.MAX_VALUE; @@ -105,11 +104,11 @@ static String encode(Values values) { ByteUtils.writeIntLE(values.name.length() + PARENT_OFFSET_IN_NAME_OFFSET * values.parentOffset, bytes, 0); System.arraycopy(nameBytes, 0, bytes, 4, nameBytes.length); System.arraycopy(values.value.bytes, values.value.offset, bytes, 4 + nameBytes.length, values.value.length); - return Base64.getUrlEncoder().withoutPadding().encodeToString(bytes); + return bytes; } static Values decode(Object field) { - byte[] bytes = Base64.getUrlDecoder().decode((String) field); + byte[] bytes = ((BytesRef) field).bytes; int encodedSize = ByteUtils.readIntLE(bytes, 0); int nameSize = encodedSize % PARENT_OFFSET_IN_NAME_OFFSET; int parentOffset = encodedSize / PARENT_OFFSET_IN_NAME_OFFSET; From 6200b2360a05f4525b6b106c6a581feddeb0c671 Mon Sep 17 00:00:00 2001 From: Kostas Krikellas Date: Wed, 24 Apr 2024 17:02:52 +0300 Subject: [PATCH 13/19] add assert on IgnoredValuesFieldMapper::write --- .../org/elasticsearch/index/mapper/IgnoredValuesFieldMapper.java | 1 + 1 file changed, 1 insertion(+) diff --git a/server/src/main/java/org/elasticsearch/index/mapper/IgnoredValuesFieldMapper.java b/server/src/main/java/org/elasticsearch/index/mapper/IgnoredValuesFieldMapper.java index 7e0def5760bf2..331897abf2a30 100644 --- a/server/src/main/java/org/elasticsearch/index/mapper/IgnoredValuesFieldMapper.java +++ b/server/src/main/java/org/elasticsearch/index/mapper/IgnoredValuesFieldMapper.java @@ -150,6 +150,7 @@ public void write(XContentBuilder b) throws IOException { // This mapper doesn't contribute to source directly as it has no access to the // object structure. Instead, it's accessed by object mappers to decode // and write its fields within the appropriate object. + assert false : "IgnoredValuesFieldMapper::write should never be called"; } public void trackObjectsWithIgnoredFields() { From 5389e5835ecb9ef30defb3a022e285b235dcfea8 Mon Sep 17 00:00:00 2001 From: Kostas Krikellas Date: Thu, 25 Apr 2024 14:18:19 +0300 Subject: [PATCH 14/19] changes from review --- .../index/mapper/IgnoredValuesFieldMapper.java | 9 ++++++--- .../index/mapper/DocCountFieldMapperTests.java | 3 +++ 2 files changed, 9 insertions(+), 3 deletions(-) diff --git a/server/src/main/java/org/elasticsearch/index/mapper/IgnoredValuesFieldMapper.java b/server/src/main/java/org/elasticsearch/index/mapper/IgnoredValuesFieldMapper.java index 331897abf2a30..1cd79b261e04e 100644 --- a/server/src/main/java/org/elasticsearch/index/mapper/IgnoredValuesFieldMapper.java +++ b/server/src/main/java/org/elasticsearch/index/mapper/IgnoredValuesFieldMapper.java @@ -90,6 +90,10 @@ protected String contentType() { @Override public void postParse(DocumentParserContext context) { + // Ignored values are only expected in synthetic mode. + assert context.getIgnoredFieldValues().isEmpty() + || context.indexSettings().getMode().isSyntheticSourceEnabled() + || context.mappingLookup().isSourceSynthetic(); for (Values values : context.getIgnoredFieldValues()) { context.doc().add(new StoredField(NAME, encode(values))); } @@ -124,7 +128,7 @@ public SourceLoader.SyntheticFieldLoader syntheticFieldLoader() { public static class SyntheticLoader implements SourceLoader.SyntheticFieldLoader { // Contains stored field values, i.e. encoded ignored field names and values. - private List values = List.of(); + private List values = null; // Maps the names of existing objects to lists of ignored fields they contain. private Map> objectsWithIgnoredFields = Map.of(); @@ -154,8 +158,7 @@ public void write(XContentBuilder b) throws IOException { } public void trackObjectsWithIgnoredFields() { - if (values == null || values.isEmpty()) { - values = null; + if (values == null) { return; } objectsWithIgnoredFields = new HashMap<>(); diff --git a/server/src/test/java/org/elasticsearch/index/mapper/DocCountFieldMapperTests.java b/server/src/test/java/org/elasticsearch/index/mapper/DocCountFieldMapperTests.java index 013c3dae95cab..61259e62d6de5 100644 --- a/server/src/test/java/org/elasticsearch/index/mapper/DocCountFieldMapperTests.java +++ b/server/src/test/java/org/elasticsearch/index/mapper/DocCountFieldMapperTests.java @@ -13,6 +13,7 @@ import org.elasticsearch.index.fieldvisitor.StoredFieldLoader; import org.elasticsearch.xcontent.XContentParserConfiguration; import org.elasticsearch.xcontent.json.JsonXContent; +import org.hamcrest.Matchers; import java.io.IOException; import java.util.List; @@ -97,6 +98,7 @@ public void testSyntheticSourceMany() throws IOException { } }, reader -> { SourceLoader loader = mapper.mappingLookup().newSourceLoader(); + assertThat(loader.requiredStoredFields(), Matchers.contains("_ignored_values")); for (LeafReaderContext leaf : reader.leaves()) { int[] docIds = IntStream.range(0, leaf.reader().maxDoc()).toArray(); SourceLoader.Leaf sourceLoaderLeaf = loader.leaf(leaf.reader(), docIds); @@ -128,6 +130,7 @@ public void testSyntheticSourceManyDoNotHave() throws IOException { } }, reader -> { SourceLoader loader = mapper.mappingLookup().newSourceLoader(); + assertThat(loader.requiredStoredFields(), Matchers.contains("_ignored_values")); for (LeafReaderContext leaf : reader.leaves()) { int[] docIds = IntStream.range(0, leaf.reader().maxDoc()).toArray(); SourceLoader.Leaf sourceLoaderLeaf = loader.leaf(leaf.reader(), docIds); From e2f1f69ed1b39196f9e03a43f9c33ac3b664cd76 Mon Sep 17 00:00:00 2001 From: Kostas Krikellas Date: Fri, 26 Apr 2024 09:52:43 +0300 Subject: [PATCH 15/19] simplify logic --- .../index/mapper/DocumentParserContext.java | 16 ++-- .../mapper/IgnoredValuesFieldMapper.java | 93 ++++--------------- .../index/mapper/ObjectMapper.java | 39 ++++---- .../index/mapper/SourceLoader.java | 17 ++++ 4 files changed, 63 insertions(+), 102 deletions(-) diff --git a/server/src/main/java/org/elasticsearch/index/mapper/DocumentParserContext.java b/server/src/main/java/org/elasticsearch/index/mapper/DocumentParserContext.java index 2bb49bc00a554..7005c8f05114a 100644 --- a/server/src/main/java/org/elasticsearch/index/mapper/DocumentParserContext.java +++ b/server/src/main/java/org/elasticsearch/index/mapper/DocumentParserContext.java @@ -104,7 +104,7 @@ public int get() { private final MappingParserContext mappingParserContext; private final SourceToParse sourceToParse; private final Set ignoredFields; - private final List ignoredFieldValues; + private final List ignoredFieldValues; private final Map> dynamicMappers; private final DynamicMapperSize dynamicMappersSize; private final Map dynamicObjectMappers; @@ -123,7 +123,7 @@ private DocumentParserContext( MappingParserContext mappingParserContext, SourceToParse sourceToParse, Set ignoreFields, - List ignoredFieldValues, + List ignoredFieldValues, Map> dynamicMappers, Map dynamicObjectMappers, Map> dynamicRuntimeFields, @@ -259,14 +259,14 @@ public final Collection getIgnoredFields() { /** * Add the given ignored values to the corresponding list. */ - public final void addIgnoredValues(IgnoredValuesFieldMapper.Values values) { + public final void addIgnoredField(IgnoredValuesFieldMapper.NameValue values) { ignoredFieldValues.add(values); } /** * Return the collection of values for fields that have been ignored so far. */ - public final Collection getIgnoredFieldValues() { + public final Collection getIgnoredFieldValues() { return Collections.unmodifiableCollection(ignoredFieldValues); } @@ -367,8 +367,12 @@ public final boolean addDynamicMapper(Mapper mapper) { if (indexSettings().getMode().isSyntheticSourceEnabled() || mappingLookup.isSourceSynthetic()) { try { int parentOffset = parent() instanceof RootObjectMapper ? 0 : parent().fullPath().length() + 1; - addIgnoredValues( - new IgnoredValuesFieldMapper.Values(mapper.name(), parentOffset, FieldDataParseHelper.encodeToken(parser())) + addIgnoredField( + new IgnoredValuesFieldMapper.NameValue( + mapper.name(), + parentOffset, + FieldDataParseHelper.encodeToken(parser()) + ) ); } catch (IOException e) { throw new IllegalArgumentException("failed to parse field [" + mapper.name() + " ]", e); diff --git a/server/src/main/java/org/elasticsearch/index/mapper/IgnoredValuesFieldMapper.java b/server/src/main/java/org/elasticsearch/index/mapper/IgnoredValuesFieldMapper.java index 1cd79b261e04e..6c615c37a01dc 100644 --- a/server/src/main/java/org/elasticsearch/index/mapper/IgnoredValuesFieldMapper.java +++ b/server/src/main/java/org/elasticsearch/index/mapper/IgnoredValuesFieldMapper.java @@ -9,21 +9,13 @@ package org.elasticsearch.index.mapper; import org.apache.lucene.document.StoredField; -import org.apache.lucene.index.LeafReader; import org.apache.lucene.util.BytesRef; import org.elasticsearch.common.util.ByteUtils; import org.elasticsearch.features.NodeFeature; import org.elasticsearch.index.query.SearchExecutionContext; -import org.elasticsearch.xcontent.XContentBuilder; -import java.io.IOException; import java.nio.charset.StandardCharsets; -import java.util.ArrayList; import java.util.Collections; -import java.util.HashMap; -import java.util.List; -import java.util.Map; -import java.util.stream.Stream; /** * Mapper for the {@code _ignored_values} field. @@ -58,7 +50,16 @@ public class IgnoredValuesFieldMapper extends MetadataFieldMapper { * the full name of the parent field * - the value, encoded as a byte array */ - public record Values(String name, int parentOffset, BytesRef value) {} + public record NameValue(String name, int parentOffset, BytesRef value) { + String getParentFieldName() { + // _doc corresponds to the root object + return (parentOffset == 0) ? "_doc" : name.substring(0, parentOffset - 1); + } + + String getFieldName() { + return parentOffset() == 0 ? name() : name().substring(parentOffset()); + } + } static final class IgnoredValuesFieldMapperType extends StringFieldType { @@ -94,12 +95,12 @@ public void postParse(DocumentParserContext context) { assert context.getIgnoredFieldValues().isEmpty() || context.indexSettings().getMode().isSyntheticSourceEnabled() || context.mappingLookup().isSourceSynthetic(); - for (Values values : context.getIgnoredFieldValues()) { - context.doc().add(new StoredField(NAME, encode(values))); + for (NameValue nameValue : context.getIgnoredFieldValues()) { + context.doc().add(new StoredField(NAME, encode(nameValue))); } } - static byte[] encode(Values values) { + static byte[] encode(NameValue values) { assert values.parentOffset < PARENT_OFFSET_IN_NAME_OFFSET; assert values.parentOffset * (long) PARENT_OFFSET_IN_NAME_OFFSET < Integer.MAX_VALUE; @@ -111,81 +112,19 @@ static byte[] encode(Values values) { return bytes; } - static Values decode(Object field) { + static NameValue decode(Object field) { byte[] bytes = ((BytesRef) field).bytes; int encodedSize = ByteUtils.readIntLE(bytes, 0); int nameSize = encodedSize % PARENT_OFFSET_IN_NAME_OFFSET; int parentOffset = encodedSize / PARENT_OFFSET_IN_NAME_OFFSET; String name = new String(bytes, 4, nameSize, StandardCharsets.UTF_8); BytesRef value = new BytesRef(bytes, 4 + nameSize, bytes.length - nameSize - 4); - return new Values(name, parentOffset, value); + return new NameValue(name, parentOffset, value); } @Override public SourceLoader.SyntheticFieldLoader syntheticFieldLoader() { - return new SyntheticLoader(); + return SourceLoader.SyntheticFieldLoader.NOTHING; } - public static class SyntheticLoader implements SourceLoader.SyntheticFieldLoader { - // Contains stored field values, i.e. encoded ignored field names and values. - private List values = null; - - // Maps the names of existing objects to lists of ignored fields they contain. - private Map> objectsWithIgnoredFields = Map.of(); - - @Override - public Stream> storedFieldLoaders() { - return Stream.of(Map.entry(NAME, values -> this.values = values)); - } - - @Override - public SourceLoader.SyntheticFieldLoader.DocValuesLoader docValuesLoader(LeafReader leafReader, int[] docIdsInLeaf) - throws IOException { - return null; - } - - @Override - public boolean hasValue() { - return false; - } - - @Override - public void write(XContentBuilder b) throws IOException { - // This mapper doesn't contribute to source directly as it has no access to the - // object structure. Instead, it's accessed by object mappers to decode - // and write its fields within the appropriate object. - assert false : "IgnoredValuesFieldMapper::write should never be called"; - } - - public void trackObjectsWithIgnoredFields() { - if (values == null) { - return; - } - objectsWithIgnoredFields = new HashMap<>(); - for (Object value : values) { - Values parsedValues = decode(value); - objectsWithIgnoredFields.computeIfAbsent( - // _doc corresponds to the root object - (parsedValues.parentOffset == 0) ? "_doc" : parsedValues.name.substring(0, parsedValues.parentOffset - 1), - k -> new ArrayList<>() - ).add(parsedValues); - } - values = null; - } - - // This is expected to be called by object mappers, to add their ignored fields as part of the source. - public void write(XContentBuilder b, String prefix) throws IOException { - var matches = objectsWithIgnoredFields.get(prefix); - if (matches != null) { - for (var values : matches) { - b.field(values.parentOffset > 0 ? values.name.substring(values.parentOffset) : values.name); - FieldDataParseHelper.decodeAndWrite(b, values.value); - } - } - } - - public boolean containsIgnoredFields(String prefix) { - return objectsWithIgnoredFields.containsKey(prefix); - } - }; } diff --git a/server/src/main/java/org/elasticsearch/index/mapper/ObjectMapper.java b/server/src/main/java/org/elasticsearch/index/mapper/ObjectMapper.java index d1850c220c915..6ae801781fc3c 100644 --- a/server/src/main/java/org/elasticsearch/index/mapper/ObjectMapper.java +++ b/server/src/main/java/org/elasticsearch/index/mapper/ObjectMapper.java @@ -730,6 +730,7 @@ public SourceLoader.SyntheticFieldLoader syntheticFieldLoader() { private class SyntheticSourceFieldLoader implements SourceLoader.SyntheticFieldLoader { private final List fields; private boolean hasValue; + private List ignoredValues; private SyntheticSourceFieldLoader(List fields) { this.fields = fields; @@ -784,34 +785,34 @@ public boolean hasValue() { @Override public void write(XContentBuilder b) throws IOException { - IgnoredValuesFieldMapper.SyntheticLoader ignoredFieldLoader = null; - for (SourceLoader.SyntheticFieldLoader field : fields) { - if (field instanceof IgnoredValuesFieldMapper.SyntheticLoader loader) { - ignoredFieldLoader = loader; - loader.trackObjectsWithIgnoredFields(); - break; - } - } - write(b, ignoredFieldLoader); - } - - public void write(XContentBuilder b, IgnoredValuesFieldMapper.SyntheticLoader ignoredFieldLoader) throws IOException { - if (hasValue == false && ignoredFieldLoader.containsIgnoredFields(name()) == false) { + if (hasValue == false) { return; } startSyntheticField(b); for (SourceLoader.SyntheticFieldLoader field : fields) { - if (field instanceof ObjectMapper.SyntheticSourceFieldLoader objectLoader) { - objectLoader.write(b, ignoredFieldLoader); - } else if (field.hasValue()) { + if (field.hasValue()) { field.write(b); } } - if (ignoredFieldLoader != null) { - ignoredFieldLoader.write(b, name()); + hasValue = false; + if (ignoredValues != null) { + for (IgnoredValuesFieldMapper.NameValue ignored : ignoredValues) { + b.field(ignored.getFieldName()); + FieldDataParseHelper.decodeAndWrite(b, ignored.value()); + } + ignoredValues = null; } b.endObject(); - hasValue = false; + } + + @Override + public boolean setIgnoredValues(Map> objectsWithIgnoredFields) { + ignoredValues = objectsWithIgnoredFields.get(name()); + hasValue |= ignoredValues != null; + for (SourceLoader.SyntheticFieldLoader loader : fields) { + hasValue |= loader.setIgnoredValues(objectsWithIgnoredFields); + } + return this.ignoredValues != null; } } diff --git a/server/src/main/java/org/elasticsearch/index/mapper/SourceLoader.java b/server/src/main/java/org/elasticsearch/index/mapper/SourceLoader.java index c07821f3c9ae7..48769b206a95e 100644 --- a/server/src/main/java/org/elasticsearch/index/mapper/SourceLoader.java +++ b/server/src/main/java/org/elasticsearch/index/mapper/SourceLoader.java @@ -17,6 +17,8 @@ import java.io.ByteArrayOutputStream; import java.io.IOException; +import java.util.ArrayList; +import java.util.HashMap; import java.util.List; import java.util.Map; import java.util.Set; @@ -89,6 +91,7 @@ public Synthetic(Mapping mapping) { .storedFieldLoaders() .map(Map.Entry::getKey) .collect(Collectors.toSet()); + this.requiredStoredFields.add(IgnoredValuesFieldMapper.NAME); } @Override @@ -122,12 +125,22 @@ private SyntheticLeaf(SyntheticFieldLoader loader, SyntheticFieldLoader.DocValue @Override public Source source(LeafStoredFieldLoader storedFieldLoader, int docId) throws IOException { + // Maps the names of existing objects to lists of ignored fields they contain. + Map> objectsWithIgnoredFields = new HashMap<>(); + for (Map.Entry> e : storedFieldLoader.storedFields().entrySet()) { SyntheticFieldLoader.StoredFieldLoader loader = storedFieldLoaders.get(e.getKey()); if (loader != null) { loader.load(e.getValue()); } + if (IgnoredValuesFieldMapper.NAME.equals(e.getKey())) { + for (Object value : e.getValue()) { + IgnoredValuesFieldMapper.NameValue nameValue = IgnoredValuesFieldMapper.decode(value); + objectsWithIgnoredFields.computeIfAbsent(nameValue.getParentFieldName(), k -> new ArrayList<>()).add(nameValue); + } + } } + loader.setIgnoredValues(objectsWithIgnoredFields); if (docValuesLoader != null) { docValuesLoader.advanceToDoc(docId); } @@ -224,6 +237,10 @@ public void write(XContentBuilder b) {} */ void write(XContentBuilder b) throws IOException; + default boolean setIgnoredValues(Map> objectsWithIgnoredFields) { + return false; + } + /** * Sync for stored field values. */ From ff8896c04400577c21c4ecb741b834342635aa1f Mon Sep 17 00:00:00 2001 From: Kostas Krikellas Date: Fri, 26 Apr 2024 09:58:45 +0300 Subject: [PATCH 16/19] add comment --- .../elasticsearch/index/mapper/IgnoredValuesFieldMapper.java | 2 ++ 1 file changed, 2 insertions(+) diff --git a/server/src/main/java/org/elasticsearch/index/mapper/IgnoredValuesFieldMapper.java b/server/src/main/java/org/elasticsearch/index/mapper/IgnoredValuesFieldMapper.java index 6c615c37a01dc..957b98d2bb2f9 100644 --- a/server/src/main/java/org/elasticsearch/index/mapper/IgnoredValuesFieldMapper.java +++ b/server/src/main/java/org/elasticsearch/index/mapper/IgnoredValuesFieldMapper.java @@ -122,6 +122,8 @@ static NameValue decode(Object field) { return new NameValue(name, parentOffset, value); } + // This mapper doesn't contribute to source directly as it has no access to the object structure. Instead, its contents + // are loaded by SourceLoader and passed to object mappers that, in turn, write their ignore fields at the appropriate level. @Override public SourceLoader.SyntheticFieldLoader syntheticFieldLoader() { return SourceLoader.SyntheticFieldLoader.NOTHING; From cb85495ebc888a43335d85130654ecdc398182c0 Mon Sep 17 00:00:00 2001 From: Kostas Krikellas Date: Fri, 26 Apr 2024 11:26:30 +0300 Subject: [PATCH 17/19] rename classes --- .../index/mapper/DocumentParserContext.java | 12 +-- .../mapper/IgnoreMalformedStoredValues.java | 4 +- ...per.java => IgnoredSourceFieldMapper.java} | 6 +- .../index/mapper/MapperFeatures.java | 2 +- .../index/mapper/ObjectMapper.java | 8 +- .../index/mapper/SourceLoader.java | 10 +- ...rseHelper.java => XContentDataHelper.java} | 94 ++++++++++--------- .../elasticsearch/indices/IndicesModule.java | 4 +- .../index/mapper/DocumentMapperTests.java | 4 +- ...ava => IgnoredSourceFieldMapperTests.java} | 2 +- ...ests.java => XContentDataHelperTests.java} | 6 +- .../indices/IndicesModuleTests.java | 4 +- 12 files changed, 79 insertions(+), 77 deletions(-) rename server/src/main/java/org/elasticsearch/index/mapper/{IgnoredValuesFieldMapper.java => IgnoredSourceFieldMapper.java} (96%) rename server/src/main/java/org/elasticsearch/index/mapper/{FieldDataParseHelper.java => XContentDataHelper.java} (82%) rename server/src/test/java/org/elasticsearch/index/mapper/{IgnoredValuedFieldMapperTests.java => IgnoredSourceFieldMapperTests.java} (99%) rename server/src/test/java/org/elasticsearch/index/mapper/{FieldDataParseHelperTests.java => XContentDataHelperTests.java} (94%) diff --git a/server/src/main/java/org/elasticsearch/index/mapper/DocumentParserContext.java b/server/src/main/java/org/elasticsearch/index/mapper/DocumentParserContext.java index 7005c8f05114a..de1266ae3a7ee 100644 --- a/server/src/main/java/org/elasticsearch/index/mapper/DocumentParserContext.java +++ b/server/src/main/java/org/elasticsearch/index/mapper/DocumentParserContext.java @@ -104,7 +104,7 @@ public int get() { private final MappingParserContext mappingParserContext; private final SourceToParse sourceToParse; private final Set ignoredFields; - private final List ignoredFieldValues; + private final List ignoredFieldValues; private final Map> dynamicMappers; private final DynamicMapperSize dynamicMappersSize; private final Map dynamicObjectMappers; @@ -123,7 +123,7 @@ private DocumentParserContext( MappingParserContext mappingParserContext, SourceToParse sourceToParse, Set ignoreFields, - List ignoredFieldValues, + List ignoredFieldValues, Map> dynamicMappers, Map dynamicObjectMappers, Map> dynamicRuntimeFields, @@ -259,14 +259,14 @@ public final Collection getIgnoredFields() { /** * Add the given ignored values to the corresponding list. */ - public final void addIgnoredField(IgnoredValuesFieldMapper.NameValue values) { + public final void addIgnoredField(IgnoredSourceFieldMapper.NameValue values) { ignoredFieldValues.add(values); } /** * Return the collection of values for fields that have been ignored so far. */ - public final Collection getIgnoredFieldValues() { + public final Collection getIgnoredFieldValues() { return Collections.unmodifiableCollection(ignoredFieldValues); } @@ -368,10 +368,10 @@ public final boolean addDynamicMapper(Mapper mapper) { try { int parentOffset = parent() instanceof RootObjectMapper ? 0 : parent().fullPath().length() + 1; addIgnoredField( - new IgnoredValuesFieldMapper.NameValue( + new IgnoredSourceFieldMapper.NameValue( mapper.name(), parentOffset, - FieldDataParseHelper.encodeToken(parser()) + XContentDataHelper.encodeToken(parser()) ) ); } catch (IOException e) { diff --git a/server/src/main/java/org/elasticsearch/index/mapper/IgnoreMalformedStoredValues.java b/server/src/main/java/org/elasticsearch/index/mapper/IgnoreMalformedStoredValues.java index 5c2c230dfb4c9..52f4048e9b230 100644 --- a/server/src/main/java/org/elasticsearch/index/mapper/IgnoreMalformedStoredValues.java +++ b/server/src/main/java/org/elasticsearch/index/mapper/IgnoreMalformedStoredValues.java @@ -26,7 +26,7 @@ */ public abstract class IgnoreMalformedStoredValues { public static StoredField storedField(String name, XContentParser parser) throws IOException { - return FieldDataParseHelper.storedField(name(name), parser); + return XContentDataHelper.storedField(name(name), parser); } /** @@ -99,7 +99,7 @@ public int count() { public void write(XContentBuilder b) throws IOException { for (Object v : values) { if (v instanceof BytesRef r) { - FieldDataParseHelper.decodeAndWrite(b, r); + XContentDataHelper.decodeAndWrite(b, r); } else { b.value(v); } diff --git a/server/src/main/java/org/elasticsearch/index/mapper/IgnoredValuesFieldMapper.java b/server/src/main/java/org/elasticsearch/index/mapper/IgnoredSourceFieldMapper.java similarity index 96% rename from server/src/main/java/org/elasticsearch/index/mapper/IgnoredValuesFieldMapper.java rename to server/src/main/java/org/elasticsearch/index/mapper/IgnoredSourceFieldMapper.java index 957b98d2bb2f9..6f6b782680ff6 100644 --- a/server/src/main/java/org/elasticsearch/index/mapper/IgnoredValuesFieldMapper.java +++ b/server/src/main/java/org/elasticsearch/index/mapper/IgnoredSourceFieldMapper.java @@ -27,7 +27,7 @@ * This overlaps with {@link IgnoredFieldMapper} that tracks just the ignored field names. It's worth evaluating * if we can replace it for all use cases to avoid duplication, assuming that the storage tradeoff is favorable. */ -public class IgnoredValuesFieldMapper extends MetadataFieldMapper { +public class IgnoredSourceFieldMapper extends MetadataFieldMapper { // This factor is used to combine two offsets within the same integer: // - the offset of the end of the parent field within the field name (N / PARENT_OFFSET_IN_NAME_OFFSET) @@ -37,7 +37,7 @@ public class IgnoredValuesFieldMapper extends MetadataFieldMapper { public static final String NAME = "_ignored_values"; - public static final IgnoredValuesFieldMapper INSTANCE = new IgnoredValuesFieldMapper(); + public static final IgnoredSourceFieldMapper INSTANCE = new IgnoredSourceFieldMapper(); public static final TypeParser PARSER = new FixedTypeParser(context -> INSTANCE); @@ -80,7 +80,7 @@ public ValueFetcher valueFetcher(SearchExecutionContext context, String format) } } - private IgnoredValuesFieldMapper() { + private IgnoredSourceFieldMapper() { super(IgnoredValuesFieldMapperType.INSTANCE); } diff --git a/server/src/main/java/org/elasticsearch/index/mapper/MapperFeatures.java b/server/src/main/java/org/elasticsearch/index/mapper/MapperFeatures.java index 50ad63efb5241..5b0823b9473ad 100644 --- a/server/src/main/java/org/elasticsearch/index/mapper/MapperFeatures.java +++ b/server/src/main/java/org/elasticsearch/index/mapper/MapperFeatures.java @@ -19,6 +19,6 @@ public class MapperFeatures implements FeatureSpecification { @Override public Set getFeatures() { - return Set.of(IgnoredValuesFieldMapper.TRACK_IGNORED_VALUES); + return Set.of(IgnoredSourceFieldMapper.TRACK_IGNORED_VALUES); } } diff --git a/server/src/main/java/org/elasticsearch/index/mapper/ObjectMapper.java b/server/src/main/java/org/elasticsearch/index/mapper/ObjectMapper.java index 6ae801781fc3c..dca6af2489910 100644 --- a/server/src/main/java/org/elasticsearch/index/mapper/ObjectMapper.java +++ b/server/src/main/java/org/elasticsearch/index/mapper/ObjectMapper.java @@ -730,7 +730,7 @@ public SourceLoader.SyntheticFieldLoader syntheticFieldLoader() { private class SyntheticSourceFieldLoader implements SourceLoader.SyntheticFieldLoader { private final List fields; private boolean hasValue; - private List ignoredValues; + private List ignoredValues; private SyntheticSourceFieldLoader(List fields) { this.fields = fields; @@ -796,9 +796,9 @@ public void write(XContentBuilder b) throws IOException { } hasValue = false; if (ignoredValues != null) { - for (IgnoredValuesFieldMapper.NameValue ignored : ignoredValues) { + for (IgnoredSourceFieldMapper.NameValue ignored : ignoredValues) { b.field(ignored.getFieldName()); - FieldDataParseHelper.decodeAndWrite(b, ignored.value()); + XContentDataHelper.decodeAndWrite(b, ignored.value()); } ignoredValues = null; } @@ -806,7 +806,7 @@ public void write(XContentBuilder b) throws IOException { } @Override - public boolean setIgnoredValues(Map> objectsWithIgnoredFields) { + public boolean setIgnoredValues(Map> objectsWithIgnoredFields) { ignoredValues = objectsWithIgnoredFields.get(name()); hasValue |= ignoredValues != null; for (SourceLoader.SyntheticFieldLoader loader : fields) { diff --git a/server/src/main/java/org/elasticsearch/index/mapper/SourceLoader.java b/server/src/main/java/org/elasticsearch/index/mapper/SourceLoader.java index 48769b206a95e..f37f494cb8865 100644 --- a/server/src/main/java/org/elasticsearch/index/mapper/SourceLoader.java +++ b/server/src/main/java/org/elasticsearch/index/mapper/SourceLoader.java @@ -91,7 +91,7 @@ public Synthetic(Mapping mapping) { .storedFieldLoaders() .map(Map.Entry::getKey) .collect(Collectors.toSet()); - this.requiredStoredFields.add(IgnoredValuesFieldMapper.NAME); + this.requiredStoredFields.add(IgnoredSourceFieldMapper.NAME); } @Override @@ -126,16 +126,16 @@ private SyntheticLeaf(SyntheticFieldLoader loader, SyntheticFieldLoader.DocValue @Override public Source source(LeafStoredFieldLoader storedFieldLoader, int docId) throws IOException { // Maps the names of existing objects to lists of ignored fields they contain. - Map> objectsWithIgnoredFields = new HashMap<>(); + Map> objectsWithIgnoredFields = new HashMap<>(); for (Map.Entry> e : storedFieldLoader.storedFields().entrySet()) { SyntheticFieldLoader.StoredFieldLoader loader = storedFieldLoaders.get(e.getKey()); if (loader != null) { loader.load(e.getValue()); } - if (IgnoredValuesFieldMapper.NAME.equals(e.getKey())) { + if (IgnoredSourceFieldMapper.NAME.equals(e.getKey())) { for (Object value : e.getValue()) { - IgnoredValuesFieldMapper.NameValue nameValue = IgnoredValuesFieldMapper.decode(value); + IgnoredSourceFieldMapper.NameValue nameValue = IgnoredSourceFieldMapper.decode(value); objectsWithIgnoredFields.computeIfAbsent(nameValue.getParentFieldName(), k -> new ArrayList<>()).add(nameValue); } } @@ -237,7 +237,7 @@ public void write(XContentBuilder b) {} */ void write(XContentBuilder b) throws IOException; - default boolean setIgnoredValues(Map> objectsWithIgnoredFields) { + default boolean setIgnoredValues(Map> objectsWithIgnoredFields) { return false; } diff --git a/server/src/main/java/org/elasticsearch/index/mapper/FieldDataParseHelper.java b/server/src/main/java/org/elasticsearch/index/mapper/XContentDataHelper.java similarity index 82% rename from server/src/main/java/org/elasticsearch/index/mapper/FieldDataParseHelper.java rename to server/src/main/java/org/elasticsearch/index/mapper/XContentDataHelper.java index 0fc82e2a37bc7..c41fbd5057227 100644 --- a/server/src/main/java/org/elasticsearch/index/mapper/FieldDataParseHelper.java +++ b/server/src/main/java/org/elasticsearch/index/mapper/XContentDataHelper.java @@ -28,7 +28,7 @@ /** * Helper class for processing field data of any type, as provided by the {@link XContentParser}. */ -public final class FieldDataParseHelper { +final class XContentDataHelper { /** * Build a {@link StoredField} for the value on which the parser is * currently positioned. @@ -39,7 +39,7 @@ public final class FieldDataParseHelper { * {@code byte[]} type and then encode type information in the first byte. *

*/ - public static StoredField storedField(String name, XContentParser parser) throws IOException { + static StoredField storedField(String name, XContentParser parser) throws IOException { return (StoredField) processToken(parser, typeUtils -> typeUtils.buildStoredField(name, parser)); } @@ -47,7 +47,7 @@ public static StoredField storedField(String name, XContentParser parser) throws * Build a {@link BytesRef} wrapping a byte array containing an encoded form * the value on which the parser is currently positioned. */ - public static BytesRef encodeToken(XContentParser parser) throws IOException { + static BytesRef encodeToken(XContentParser parser) throws IOException { return new BytesRef((byte[]) processToken(parser, (typeUtils) -> typeUtils.encode(parser))); } @@ -56,18 +56,20 @@ public static BytesRef encodeToken(XContentParser parser) throws IOException { * passed build. The assumption is that the passed value has encoded using the function * {@link #encodeToken(XContentParser)} above. */ - public static void decodeAndWrite(XContentBuilder b, BytesRef r) throws IOException { + static void decodeAndWrite(XContentBuilder b, BytesRef r) throws IOException { switch ((char) r.bytes[r.offset]) { - case BINARY_VALUE -> TypeUtils.EMBEDDED_OBJECT.decodeAndWrite(b, r); - case CBOR_OBJECT_VALUE, JSON_OBJECT_VALUE, YAML_OBJECT_VALUE, SMILE_OBJECT_VALUE -> TypeUtils.START.decodeAndWrite(b, r); - case BIG_DECIMAL_VALUE -> TypeUtils.BIG_DECIMAL.decodeAndWrite(b, r); - case FALSE_VALUE, TRUE_VALUE -> TypeUtils.BOOLEAN.decodeAndWrite(b, r); - case BIG_INTEGER_VALUE -> TypeUtils.BIG_INTEGER.decodeAndWrite(b, r); - case STRING_VALUE -> TypeUtils.STRING.decodeAndWrite(b, r); - case INTEGER_VALUE -> TypeUtils.INTEGER.decodeAndWrite(b, r); - case LONG_VALUE -> TypeUtils.LONG.decodeAndWrite(b, r); - case DOUBLE_VALUE -> TypeUtils.DOUBLE.decodeAndWrite(b, r); - case FLOAT_VALUE -> TypeUtils.FLOAT.decodeAndWrite(b, r); + case BINARY_ENCODING -> TypeUtils.EMBEDDED_OBJECT.decodeAndWrite(b, r); + case CBOR_OBJECT_ENCODING, JSON_OBJECT_ENCODING, YAML_OBJECT_ENCODING, SMILE_OBJECT_ENCODING -> { + TypeUtils.START.decodeAndWrite(b, r); + } + case BIG_DECIMAL_ENCODING -> TypeUtils.BIG_DECIMAL.decodeAndWrite(b, r); + case FALSE_ENCODING, TRUE_ENCODING -> TypeUtils.BOOLEAN.decodeAndWrite(b, r); + case BIG_INTEGER_ENCODING -> TypeUtils.BIG_INTEGER.decodeAndWrite(b, r); + case STRING_ENCODING -> TypeUtils.STRING.decodeAndWrite(b, r); + case INTEGER_ENCODING -> TypeUtils.INTEGER.decodeAndWrite(b, r); + case LONG_ENCODING -> TypeUtils.LONG.decodeAndWrite(b, r); + case DOUBLE_ENCODING -> TypeUtils.DOUBLE.decodeAndWrite(b, r); + case FLOAT_ENCODING -> TypeUtils.FLOAT.decodeAndWrite(b, r); default -> throw new IllegalArgumentException("Can't decode " + r); } } @@ -90,23 +92,23 @@ private static Object processToken(XContentParser parser, CheckedFunction decodeAndWriteXContent(b, XContentType.CBOR, r); - case JSON_OBJECT_VALUE -> decodeAndWriteXContent(b, XContentType.JSON, r); - case SMILE_OBJECT_VALUE -> decodeAndWriteXContent(b, XContentType.SMILE, r); - case YAML_OBJECT_VALUE -> decodeAndWriteXContent(b, XContentType.YAML, r); + case CBOR_OBJECT_ENCODING -> decodeAndWriteXContent(b, XContentType.CBOR, r); + case JSON_OBJECT_ENCODING -> decodeAndWriteXContent(b, XContentType.JSON, r); + case SMILE_OBJECT_ENCODING -> decodeAndWriteXContent(b, XContentType.SMILE, r); + case YAML_OBJECT_ENCODING -> decodeAndWriteXContent(b, XContentType.YAML, r); default -> throw new IllegalArgumentException("Can't decode " + r); } } @@ -368,10 +370,10 @@ static byte[] encode(XContentBuilder builder) throws IOException { BytesReference b = BytesReference.bytes(builder); byte[] encoded = new byte[1 + b.length()]; encoded[0] = switch (builder.contentType()) { - case JSON -> JSON_OBJECT_VALUE; - case SMILE -> SMILE_OBJECT_VALUE; - case YAML -> YAML_OBJECT_VALUE; - case CBOR -> CBOR_OBJECT_VALUE; + case JSON -> JSON_OBJECT_ENCODING; + case SMILE -> SMILE_OBJECT_ENCODING; + case YAML -> YAML_OBJECT_ENCODING; + case CBOR -> CBOR_OBJECT_ENCODING; default -> throw new IllegalArgumentException("unsupported type " + builder.contentType()); }; diff --git a/server/src/main/java/org/elasticsearch/indices/IndicesModule.java b/server/src/main/java/org/elasticsearch/indices/IndicesModule.java index 2e409995277eb..17e0105d59d8c 100644 --- a/server/src/main/java/org/elasticsearch/indices/IndicesModule.java +++ b/server/src/main/java/org/elasticsearch/indices/IndicesModule.java @@ -39,7 +39,7 @@ import org.elasticsearch.index.mapper.GeoPointScriptFieldType; import org.elasticsearch.index.mapper.IdFieldMapper; import org.elasticsearch.index.mapper.IgnoredFieldMapper; -import org.elasticsearch.index.mapper.IgnoredValuesFieldMapper; +import org.elasticsearch.index.mapper.IgnoredSourceFieldMapper; import org.elasticsearch.index.mapper.IndexFieldMapper; import org.elasticsearch.index.mapper.IpFieldMapper; import org.elasticsearch.index.mapper.IpScriptFieldType; @@ -259,7 +259,7 @@ private static Map initBuiltInMetadataMa builtInMetadataMappers.put(TimeSeriesRoutingHashFieldMapper.NAME, TimeSeriesRoutingHashFieldMapper.PARSER); builtInMetadataMappers.put(IndexFieldMapper.NAME, IndexFieldMapper.PARSER); builtInMetadataMappers.put(SourceFieldMapper.NAME, SourceFieldMapper.PARSER); - builtInMetadataMappers.put(IgnoredValuesFieldMapper.NAME, IgnoredValuesFieldMapper.PARSER); + builtInMetadataMappers.put(IgnoredSourceFieldMapper.NAME, IgnoredSourceFieldMapper.PARSER); builtInMetadataMappers.put(NestedPathFieldMapper.NAME, NestedPathFieldMapper.PARSER); builtInMetadataMappers.put(VersionFieldMapper.NAME, VersionFieldMapper.PARSER); builtInMetadataMappers.put(SeqNoFieldMapper.NAME, SeqNoFieldMapper.PARSER); diff --git a/server/src/test/java/org/elasticsearch/index/mapper/DocumentMapperTests.java b/server/src/test/java/org/elasticsearch/index/mapper/DocumentMapperTests.java index d707a17e82337..c210fb0654683 100644 --- a/server/src/test/java/org/elasticsearch/index/mapper/DocumentMapperTests.java +++ b/server/src/test/java/org/elasticsearch/index/mapper/DocumentMapperTests.java @@ -319,7 +319,7 @@ public void testEmptyDocumentMapper() { .item(DocCountFieldMapper.class) .item(FieldNamesFieldMapper.class) .item(IgnoredFieldMapper.class) - .item(IgnoredValuesFieldMapper.class) + .item(IgnoredSourceFieldMapper.class) .item(IndexFieldMapper.class) .item(NestedPathFieldMapper.class) .item(ProvidedIdFieldMapper.class) @@ -337,7 +337,7 @@ public void testEmptyDocumentMapper() { .item(FieldNamesFieldMapper.CONTENT_TYPE) .item(IdFieldMapper.CONTENT_TYPE) .item(IgnoredFieldMapper.CONTENT_TYPE) - .item(IgnoredValuesFieldMapper.NAME) + .item(IgnoredSourceFieldMapper.NAME) .item(IndexFieldMapper.CONTENT_TYPE) .item(NestedPathFieldMapper.NAME) .item(RoutingFieldMapper.CONTENT_TYPE) diff --git a/server/src/test/java/org/elasticsearch/index/mapper/IgnoredValuedFieldMapperTests.java b/server/src/test/java/org/elasticsearch/index/mapper/IgnoredSourceFieldMapperTests.java similarity index 99% rename from server/src/test/java/org/elasticsearch/index/mapper/IgnoredValuedFieldMapperTests.java rename to server/src/test/java/org/elasticsearch/index/mapper/IgnoredSourceFieldMapperTests.java index 710fb9f2298ea..a21c3993d4f2b 100644 --- a/server/src/test/java/org/elasticsearch/index/mapper/IgnoredValuedFieldMapperTests.java +++ b/server/src/test/java/org/elasticsearch/index/mapper/IgnoredSourceFieldMapperTests.java @@ -17,7 +17,7 @@ import java.math.BigInteger; import java.util.Base64; -public class IgnoredValuedFieldMapperTests extends MapperServiceTestCase { +public class IgnoredSourceFieldMapperTests extends MapperServiceTestCase { private String getSyntheticSource(CheckedConsumer build) throws IOException { DocumentMapper documentMapper = createMapperService( diff --git a/server/src/test/java/org/elasticsearch/index/mapper/FieldDataParseHelperTests.java b/server/src/test/java/org/elasticsearch/index/mapper/XContentDataHelperTests.java similarity index 94% rename from server/src/test/java/org/elasticsearch/index/mapper/FieldDataParseHelperTests.java rename to server/src/test/java/org/elasticsearch/index/mapper/XContentDataHelperTests.java index 3833c3ccbe063..cd142dfd393f7 100644 --- a/server/src/test/java/org/elasticsearch/index/mapper/FieldDataParseHelperTests.java +++ b/server/src/test/java/org/elasticsearch/index/mapper/XContentDataHelperTests.java @@ -23,7 +23,7 @@ import static org.hamcrest.Matchers.equalTo; -public class FieldDataParseHelperTests extends ESTestCase { +public class XContentDataHelperTests extends ESTestCase { private String encodeAndDecode(String value) throws IOException { XContentParser p = createParser(JsonXContent.jsonXContent, "{ \"foo\": " + value + " }"); @@ -34,7 +34,7 @@ private String encodeAndDecode(String value) throws IOException { XContentBuilder builder = XContentFactory.jsonBuilder(); builder.humanReadable(true); - FieldDataParseHelper.decodeAndWrite(builder, FieldDataParseHelper.encodeToken(p)); + XContentDataHelper.decodeAndWrite(builder, XContentDataHelper.encodeToken(p)); return Strings.toString(builder); } @@ -94,7 +94,7 @@ public void testObject() throws IOException { assertThat(p.nextToken(), equalTo(XContentParser.Token.START_OBJECT)); XContentBuilder builder = XContentFactory.jsonBuilder(); builder.humanReadable(true); - FieldDataParseHelper.decodeAndWrite(builder, FieldDataParseHelper.encodeToken(p)); + XContentDataHelper.decodeAndWrite(builder, XContentDataHelper.encodeToken(p)); assertEquals(object, Strings.toString(builder)); } diff --git a/server/src/test/java/org/elasticsearch/indices/IndicesModuleTests.java b/server/src/test/java/org/elasticsearch/indices/IndicesModuleTests.java index c93b3dfb31bcd..c173a22dcdf57 100644 --- a/server/src/test/java/org/elasticsearch/indices/IndicesModuleTests.java +++ b/server/src/test/java/org/elasticsearch/indices/IndicesModuleTests.java @@ -15,7 +15,7 @@ import org.elasticsearch.index.mapper.FieldNamesFieldMapper; import org.elasticsearch.index.mapper.IdFieldMapper; import org.elasticsearch.index.mapper.IgnoredFieldMapper; -import org.elasticsearch.index.mapper.IgnoredValuesFieldMapper; +import org.elasticsearch.index.mapper.IgnoredSourceFieldMapper; import org.elasticsearch.index.mapper.IndexFieldMapper; import org.elasticsearch.index.mapper.KeywordFieldMapper; import org.elasticsearch.index.mapper.Mapper; @@ -86,7 +86,7 @@ public Map getMetadataMappers() { TimeSeriesRoutingHashFieldMapper.NAME, IndexFieldMapper.NAME, SourceFieldMapper.NAME, - IgnoredValuesFieldMapper.NAME, + IgnoredSourceFieldMapper.NAME, NestedPathFieldMapper.NAME, VersionFieldMapper.NAME, SeqNoFieldMapper.NAME, From ace0559842f14ad70722823fd89ee171d76645c2 Mon Sep 17 00:00:00 2001 From: Kostas Krikellas Date: Fri, 26 Apr 2024 12:02:25 +0300 Subject: [PATCH 18/19] rename _ignored_values to _ignored_source --- .../index/mapper/IgnoredSourceFieldMapper.java | 7 ++++--- .../index/mapper/MapperFeatures.java | 2 +- .../index/mapper/DocCountFieldMapperTests.java | 4 ++-- .../index/mapper/XContentDataHelperTests.java | 17 ----------------- 4 files changed, 7 insertions(+), 23 deletions(-) diff --git a/server/src/main/java/org/elasticsearch/index/mapper/IgnoredSourceFieldMapper.java b/server/src/main/java/org/elasticsearch/index/mapper/IgnoredSourceFieldMapper.java index 6f6b782680ff6..1daa7d1d674e3 100644 --- a/server/src/main/java/org/elasticsearch/index/mapper/IgnoredSourceFieldMapper.java +++ b/server/src/main/java/org/elasticsearch/index/mapper/IgnoredSourceFieldMapper.java @@ -18,7 +18,8 @@ import java.util.Collections; /** - * Mapper for the {@code _ignored_values} field. + + * Mapper for the {@code _ignored_source} field. * * A field mapper that records fields that have been ignored, along with their values. It's intended for use * in indexes with synthetic source to reconstruct the latter, taking into account fields that got ignored during @@ -35,13 +36,13 @@ public class IgnoredSourceFieldMapper extends MetadataFieldMapper { // (N % PARENT_OFFSET_IN_NAME_OFFSET) private static final int PARENT_OFFSET_IN_NAME_OFFSET = 1 << 16; - public static final String NAME = "_ignored_values"; + public static final String NAME = "_ignored_source"; public static final IgnoredSourceFieldMapper INSTANCE = new IgnoredSourceFieldMapper(); public static final TypeParser PARSER = new FixedTypeParser(context -> INSTANCE); - static final NodeFeature TRACK_IGNORED_VALUES = new NodeFeature("mapper.track_ignored_values"); + static final NodeFeature TRACK_IGNORED_SOURCE = new NodeFeature("mapper.track_ignored_source"); /* * Container for the ignored field data: diff --git a/server/src/main/java/org/elasticsearch/index/mapper/MapperFeatures.java b/server/src/main/java/org/elasticsearch/index/mapper/MapperFeatures.java index 5b0823b9473ad..dc189aecab01c 100644 --- a/server/src/main/java/org/elasticsearch/index/mapper/MapperFeatures.java +++ b/server/src/main/java/org/elasticsearch/index/mapper/MapperFeatures.java @@ -19,6 +19,6 @@ public class MapperFeatures implements FeatureSpecification { @Override public Set getFeatures() { - return Set.of(IgnoredSourceFieldMapper.TRACK_IGNORED_VALUES); + return Set.of(IgnoredSourceFieldMapper.TRACK_IGNORED_SOURCE); } } diff --git a/server/src/test/java/org/elasticsearch/index/mapper/DocCountFieldMapperTests.java b/server/src/test/java/org/elasticsearch/index/mapper/DocCountFieldMapperTests.java index 61259e62d6de5..06e70e84bbb67 100644 --- a/server/src/test/java/org/elasticsearch/index/mapper/DocCountFieldMapperTests.java +++ b/server/src/test/java/org/elasticsearch/index/mapper/DocCountFieldMapperTests.java @@ -98,7 +98,7 @@ public void testSyntheticSourceMany() throws IOException { } }, reader -> { SourceLoader loader = mapper.mappingLookup().newSourceLoader(); - assertThat(loader.requiredStoredFields(), Matchers.contains("_ignored_values")); + assertThat(loader.requiredStoredFields(), Matchers.contains("_ignored_source")); for (LeafReaderContext leaf : reader.leaves()) { int[] docIds = IntStream.range(0, leaf.reader().maxDoc()).toArray(); SourceLoader.Leaf sourceLoaderLeaf = loader.leaf(leaf.reader(), docIds); @@ -130,7 +130,7 @@ public void testSyntheticSourceManyDoNotHave() throws IOException { } }, reader -> { SourceLoader loader = mapper.mappingLookup().newSourceLoader(); - assertThat(loader.requiredStoredFields(), Matchers.contains("_ignored_values")); + assertThat(loader.requiredStoredFields(), Matchers.contains("_ignored_source")); for (LeafReaderContext leaf : reader.leaves()) { int[] docIds = IntStream.range(0, leaf.reader().maxDoc()).toArray(); SourceLoader.Leaf sourceLoaderLeaf = loader.leaf(leaf.reader(), docIds); diff --git a/server/src/test/java/org/elasticsearch/index/mapper/XContentDataHelperTests.java b/server/src/test/java/org/elasticsearch/index/mapper/XContentDataHelperTests.java index cd142dfd393f7..06db79c3f9fb0 100644 --- a/server/src/test/java/org/elasticsearch/index/mapper/XContentDataHelperTests.java +++ b/server/src/test/java/org/elasticsearch/index/mapper/XContentDataHelperTests.java @@ -9,7 +9,6 @@ package org.elasticsearch.index.mapper; import org.elasticsearch.common.Strings; -import org.elasticsearch.common.bytes.BytesReference; import org.elasticsearch.test.ESTestCase; import org.elasticsearch.xcontent.XContentBuilder; import org.elasticsearch.xcontent.XContentFactory; @@ -17,7 +16,6 @@ import org.elasticsearch.xcontent.json.JsonXContent; import java.io.IOException; -import java.math.BigDecimal; import java.math.BigInteger; import java.util.List; @@ -73,21 +71,6 @@ public void testBigInteger() throws IOException { assertEquals(i, new BigInteger(encodeAndDecode(i.toString()), 10)); } - public void testBigDecimal() throws IOException { - BigDecimal d = new BigDecimal(randomBigInteger(), randomInt()); - XContentBuilder builder = XContentFactory.jsonBuilder(); - builder.startObject().field("foo", d).endObject(); - XContentParser p = JsonXContent.jsonXContent.createParser(parserConfig(), BytesReference.bytes(builder).streamInput()); - assertThat(p.nextToken(), equalTo(XContentParser.Token.START_OBJECT)); - assertThat(p.nextToken(), equalTo(XContentParser.Token.FIELD_NAME)); - assertThat(p.currentName(), equalTo("foo")); - p.nextToken(); - assertEquals(p.numberType(), XContentParser.NumberType.DOUBLE); - - // FieldDataParseHelper.decodeAndWrite(builder, FieldDataParseHelper.encodeToken(p)); - // assertEquals(d, new BigDecimal(Strings.toString(builder))); - } - public void testObject() throws IOException { String object = "{\"name\":\"foo\"}"; XContentParser p = createParser(JsonXContent.jsonXContent, object); From 3cc62dccaa866d7b9f7d133f2d779bbb759ef17c Mon Sep 17 00:00:00 2001 From: Kostas Krikellas Date: Fri, 26 Apr 2024 13:01:20 +0300 Subject: [PATCH 19/19] rename _ignored_values to _ignored_source --- .../test/indices.create/20_synthetic_source.yml | 6 +++--- .../rest-api-spec/test/nodes.stats/11_indices_metrics.yml | 6 +++--- 2 files changed, 6 insertions(+), 6 deletions(-) diff --git a/rest-api-spec/src/yamlRestTest/resources/rest-api-spec/test/indices.create/20_synthetic_source.yml b/rest-api-spec/src/yamlRestTest/resources/rest-api-spec/test/indices.create/20_synthetic_source.yml index 0376604d4c43c..39787366c0cc9 100644 --- a/rest-api-spec/src/yamlRestTest/resources/rest-api-spec/test/indices.create/20_synthetic_source.yml +++ b/rest-api-spec/src/yamlRestTest/resources/rest-api-spec/test/indices.create/20_synthetic_source.yml @@ -40,7 +40,7 @@ nested is disabled: --- object with unmapped fields: - requires: - cluster_features: ["mapper.track_ignored_values"] + cluster_features: ["mapper.track_ignored_source"] reason: requires tracking ignored values - do: @@ -92,7 +92,7 @@ object with unmapped fields: --- nested object with unmapped fields: - requires: - cluster_features: ["mapper.track_ignored_values"] + cluster_features: ["mapper.track_ignored_source"] reason: requires tracking ignored values - do: @@ -143,7 +143,7 @@ nested object with unmapped fields: --- empty object with unmapped fields: - requires: - cluster_features: ["mapper.track_ignored_values"] + cluster_features: ["mapper.track_ignored_source"] reason: requires tracking ignored values - do: diff --git a/rest-api-spec/src/yamlRestTest/resources/rest-api-spec/test/nodes.stats/11_indices_metrics.yml b/rest-api-spec/src/yamlRestTest/resources/rest-api-spec/test/nodes.stats/11_indices_metrics.yml index 1238bc210fc14..ac0f8aec4f3d0 100644 --- a/rest-api-spec/src/yamlRestTest/resources/rest-api-spec/test/nodes.stats/11_indices_metrics.yml +++ b/rest-api-spec/src/yamlRestTest/resources/rest-api-spec/test/nodes.stats/11_indices_metrics.yml @@ -417,8 +417,8 @@ - requires: test_runner_features: [arbitrary_key] - cluster_features: ["mapper.track_ignored_values"] - reason: "_ignored_values added to mappings" + cluster_features: ["mapper.track_ignored_source"] + reason: "_ignored_source added to mappings" - do: indices.create: @@ -476,7 +476,7 @@ # 4. _field_names # 5. _id # 6. _ignored - # 7. _ignored_values + # 7. _ignored_source # 8. _index # 9. _nested_path # 10. _routing