Skip to content

Commit

Permalink
more plumbing for nonTextFieldsSharedInvertedIndex setting
Browse files Browse the repository at this point in the history
  • Loading branch information
martijnvg committed Jan 25, 2024
1 parent d21c334 commit c74d5a2
Show file tree
Hide file tree
Showing 65 changed files with 158 additions and 132 deletions.
Original file line number Diff line number Diff line change
Expand Up @@ -182,8 +182,8 @@ private static BlockLoader blockLoader(String name) {
Lucene.KEYWORD_ANALYZER,
Lucene.KEYWORD_ANALYZER,
new KeywordFieldMapper.Builder(name, IndexVersion.current()).docValues(ft.docValuesType() != DocValuesType.NONE),
syntheticSource
).blockLoader(new MappedFieldType.BlockLoaderContext() {
syntheticSource,
false).blockLoader(new MappedFieldType.BlockLoaderContext() {
@Override
public String indexName() {
return "benchmark";
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -180,7 +180,7 @@ private List<String> findRoutingPaths(String indexName, Settings allSettings, Li
for (Iterator<String> iterator = template.pathMatch().iterator(); iterator.hasNext();) {
var mapper = parserContext.typeParser(mappingSnippetType)
.parse(iterator.next(), mappingSnippet, parserContext)
.build(MapperBuilderContext.root(false, false));
.build(MapperBuilderContext.root(false, false, false));
extractPath(routingPaths, mapper);
if (iterator.hasNext()) {
// Since FieldMapper.parse modifies the Map passed in (removing entries for "type"), that means
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -221,7 +221,7 @@ public void setup() throws Exception {
ScriptCompiler.NONE,
false,
IndexVersion.current()
).build(MapperBuilderContext.root(false, false));
).build(MapperBuilderContext.root(false, false, false));
RootObjectMapper.Builder root = new RootObjectMapper.Builder("_doc", ObjectMapper.Defaults.SUBOBJECTS);
root.add(
new DateFieldMapper.Builder(
Expand All @@ -235,7 +235,7 @@ public void setup() throws Exception {
);
MetadataFieldMapper dtfm = DataStreamTestHelper.getDataStreamTimestampFieldMapper();
Mapping mapping = new Mapping(
root.build(MapperBuilderContext.root(false, false)),
root.build(MapperBuilderContext.root(false, false, false)),
new MetadataFieldMapper[] { dtfm },
Collections.emptyMap()
);
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -389,7 +389,7 @@ public void testParse3DPolygon() throws IOException, ParseException {
Polygon expected = GEOMETRY_FACTORY.createPolygon(shell, null);
final IndexVersion version = IndexVersionUtils.randomPreviousCompatibleVersion(random(), IndexVersions.V_8_0_0);
final LegacyGeoShapeFieldMapper mapperBuilder = new LegacyGeoShapeFieldMapper.Builder("test", version, false, true).build(
MapperBuilderContext.root(false, false)
MapperBuilderContext.root(false, false, false)
);
try (XContentParser parser = createParser(polygonGeoJson)) {
parser.nextToken();
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -294,7 +294,7 @@ public void testParseMixedDimensionPolyWithHole() throws IOException, ParseExcep
parser.nextToken();

final LegacyGeoShapeFieldMapper mapperBuilder = new LegacyGeoShapeFieldMapper.Builder("test", IndexVersion.current(), false, true)
.build(MapperBuilderContext.root(false, false));
.build(MapperBuilderContext.root(false, false, false));

// test store z disabled
ElasticsearchException e = expectThrows(ElasticsearchException.class, () -> ShapeParser.parse(parser, mapperBuilder));
Expand Down Expand Up @@ -326,7 +326,7 @@ public void testParseMixedDimensionPolyWithHoleStoredZ() throws IOException {

final IndexVersion version = IndexVersionUtils.randomPreviousCompatibleVersion(random(), IndexVersions.V_8_0_0);
final LegacyGeoShapeFieldMapper mapperBuilder = new LegacyGeoShapeFieldMapper.Builder("test", version, false, true).build(
MapperBuilderContext.root(false, false)
MapperBuilderContext.root(false, false, false)
);

// test store z disabled
Expand All @@ -350,7 +350,7 @@ public void testParsePolyWithStoredZ() throws IOException {

final IndexVersion version = IndexVersionUtils.randomPreviousCompatibleVersion(random(), IndexVersions.V_8_0_0);
final LegacyGeoShapeFieldMapper mapperBuilder = new LegacyGeoShapeFieldMapper.Builder("test", version, false, true).build(
MapperBuilderContext.root(false, false)
MapperBuilderContext.root(false, false, false)
);

ShapeBuilder<?, ?, ?> shapeBuilder = ShapeParser.parse(parser, mapperBuilder);
Expand All @@ -367,7 +367,7 @@ public void testParseOpenPolygon() throws IOException {
final IndexVersion version = IndexVersionUtils.randomPreviousCompatibleVersion(random(), IndexVersions.V_8_0_0);
final LegacyGeoShapeFieldMapper defaultMapperBuilder = new LegacyGeoShapeFieldMapper.Builder("test", version, false, true).coerce(
false
).build(MapperBuilderContext.root(false, false));
).build(MapperBuilderContext.root(false, false, false));
ElasticsearchParseException exception = expectThrows(
ElasticsearchParseException.class,
() -> ShapeParser.parse(parser, defaultMapperBuilder)
Expand All @@ -379,7 +379,7 @@ public void testParseOpenPolygon() throws IOException {
IndexVersion.current(),
false,
true
).coerce(true).build(MapperBuilderContext.root(false, false));
).coerce(true).build(MapperBuilderContext.root(false, false, false));
ShapeBuilder<?, ?, ?> shapeBuilder = ShapeParser.parse(parser, coercingMapperBuilder);
assertNotNull(shapeBuilder);
assertEquals("polygon ((100.0 5.0, 100.0 10.0, 90.0 10.0, 90.0 5.0, 100.0 5.0))", shapeBuilder.toWKT());
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -38,7 +38,7 @@ public void testSetStrategyName() {
public void testFetchSourceValue() throws IOException {
IndexVersion version = IndexVersionUtils.randomPreviousCompatibleVersion(random(), IndexVersions.V_8_0_0);
MappedFieldType mapper = new LegacyGeoShapeFieldMapper.Builder("field", version, false, true).build(
MapperBuilderContext.root(false, false)
MapperBuilderContext.root(false, false, false)
).fieldType();

Map<String, Object> jsonLineString = Map.of("type", "LineString", "coordinates", List.of(List.of(42.0, 27.1), List.of(30.0, 50.0)));
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -25,7 +25,7 @@ public void testIsNotAggregatable() {

public void testFetchSourceValue() throws IOException {
MappedFieldType mapper = new RankFeatureFieldMapper.Builder("field").nullValue(2.0f)
.build(MapperBuilderContext.root(false, false))
.build(MapperBuilderContext.root(false, false, false))
.fieldType();

assertEquals(List.of(3.14f), fetchSourceValue(mapper, 3.14));
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -220,15 +220,15 @@ public void testFieldData() throws IOException {

public void testFetchSourceValue() throws IOException {
MappedFieldType mapper = new ScaledFloatFieldMapper.Builder("field", false, false, null).scalingFactor(100)
.build(MapperBuilderContext.root(false, false))
.build(MapperBuilderContext.root(false, false, false))
.fieldType();
assertEquals(List.of(3.14), fetchSourceValue(mapper, 3.1415926));
assertEquals(List.of(3.14), fetchSourceValue(mapper, "3.1415"));
assertEquals(List.of(), fetchSourceValue(mapper, ""));

MappedFieldType nullValueMapper = new ScaledFloatFieldMapper.Builder("field", false, false, null).scalingFactor(100)
.nullValue(2.71)
.build(MapperBuilderContext.root(false, false))
.build(MapperBuilderContext.root(false, false, false))
.fieldType();
assertEquals(List.of(2.71), fetchSourceValue(nullValueMapper, ""));
}
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -300,7 +300,7 @@ static MappedFieldType[] withJoinFields(MappedFieldType... fieldTypes) {

int i = fieldTypes.length;
result[i++] = new ParentJoinFieldMapper.Builder("join_field").addRelation(PARENT_TYPE, Collections.singleton(CHILD_TYPE))
.build(MapperBuilderContext.root(false, false))
.build(MapperBuilderContext.root(false, false, false))
.fieldType();
result[i++] = new ParentIdFieldMapper.ParentIdFieldType("join_field#" + PARENT_TYPE, false);
assert i == result.length;
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -19,7 +19,7 @@
public class JoinFieldTypeTests extends FieldTypeTestCase {

public void testFetchSourceValue() throws IOException {
MappedFieldType fieldType = new ParentJoinFieldMapper.Builder("field").build(MapperBuilderContext.root(false, false)).fieldType();
MappedFieldType fieldType = new ParentJoinFieldMapper.Builder("field").build(MapperBuilderContext.root(false, false, false)).fieldType();

Map<String, String> parentValue = Map.of("relation", "parent");
assertEquals(List.of(parentValue), fetchSourceValue(fieldType, parentValue));
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -63,7 +63,7 @@ public void testStoringQueryBuilders() throws IOException {
IndexWriterConfig config = new IndexWriterConfig(new WhitespaceAnalyzer());
config.setMergePolicy(NoMergePolicy.INSTANCE);
BinaryFieldMapper fieldMapper = PercolatorFieldMapper.Builder.createQueryBuilderFieldBuilder(
MapperBuilderContext.root(false, false)
MapperBuilderContext.root(false, false, false)
);
MappedFieldType.FielddataOperation fielddataOperation = MappedFieldType.FielddataOperation.SEARCH;

Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -19,19 +19,19 @@ public class ICUCollationKeywordFieldTypeTests extends FieldTypeTestCase {
public void testFetchSourceValue() throws IOException {

ICUCollationKeywordFieldMapper mapper = new ICUCollationKeywordFieldMapper.Builder("field").build(
MapperBuilderContext.root(false, false)
MapperBuilderContext.root(false, false, false)
);
assertEquals(List.of("42"), fetchSourceValue(mapper.fieldType(), 42L));
assertEquals(List.of("true"), fetchSourceValue(mapper.fieldType(), true));

ICUCollationKeywordFieldMapper ignoreAboveMapper = new ICUCollationKeywordFieldMapper.Builder("field").ignoreAbove(4)
.build(MapperBuilderContext.root(false, false));
.build(MapperBuilderContext.root(false, false, false));
assertEquals(List.of(), fetchSourceValue(ignoreAboveMapper.fieldType(), "value"));
assertEquals(List.of("42"), fetchSourceValue(ignoreAboveMapper.fieldType(), 42L));
assertEquals(List.of("true"), fetchSourceValue(ignoreAboveMapper.fieldType(), true));

ICUCollationKeywordFieldMapper nullValueMapper = new ICUCollationKeywordFieldMapper.Builder("field").nullValue("NULL")
.build(MapperBuilderContext.root(false, false));
.build(MapperBuilderContext.root(false, false, false));
assertEquals(List.of("NULL"), fetchSourceValue(nullValueMapper.fieldType(), null));
}
}
Original file line number Diff line number Diff line change
Expand Up @@ -30,7 +30,7 @@ public void testIntervals() throws IOException {

public void testFetchSourceValue() throws IOException {
MappedFieldType fieldType = new AnnotatedTextFieldMapper.Builder("field", IndexVersion.current(), createDefaultIndexAnalyzers())
.build(MapperBuilderContext.root(false, false))
.build(MapperBuilderContext.root(false, false, false))
.fieldType();

assertEquals(List.of("value"), fetchSourceValue(fieldType, "value"));
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -28,7 +28,7 @@ public class DocumentMapper {
*/
public static DocumentMapper createEmpty(MapperService mapperService) {
RootObjectMapper root = new RootObjectMapper.Builder(MapperService.SINGLE_MAPPING_NAME, ObjectMapper.Defaults.SUBOBJECTS).build(
MapperBuilderContext.root(false, false)
MapperBuilderContext.root(false, false, false)
);
MetadataFieldMapper[] metadata = mapperService.getMetadataMappers().values().toArray(new MetadataFieldMapper[0]);
Mapping mapping = new Mapping(root, metadata, null);
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -256,7 +256,10 @@ static Mapping createDynamicUpdate(DocumentParserContext context) {
for (RuntimeField runtimeField : context.getDynamicRuntimeFields()) {
rootBuilder.addRuntimeField(runtimeField);
}
RootObjectMapper root = rootBuilder.build(MapperBuilderContext.root(context.mappingLookup().isSourceSynthetic(), false));
boolean nonTextFieldsSharedInvertedIndex = context.indexSettings().isMappingNonTextFieldsSharedInvertedIndex();
RootObjectMapper root = rootBuilder.build(
MapperBuilderContext.root(context.mappingLookup().isSourceSynthetic(), false, nonTextFieldsSharedInvertedIndex)
);
return context.mappingLookup().getMapping().mappingUpdate(root);
}

Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -538,7 +538,7 @@ public final MapperBuilderContext createDynamicMapperBuilderContext() {
if (p.endsWith(".")) {
p = p.substring(0, p.length() - 1);
}
return new MapperBuilderContext(p, mappingLookup().isSourceSynthetic(), false);
return new MapperBuilderContext(p, mappingLookup().isSourceSynthetic(), false, false);
}

public abstract XContentParser parser();
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -306,6 +306,7 @@ private KeywordFieldType buildFieldType(MapperBuilderContext context, FieldType
searchAnalyzer,
quoteAnalyzer,
this,
context.isSourceSynthetic(),
context.isSourceSynthetic()
);
}
Expand All @@ -328,7 +329,8 @@ public KeywordFieldMapper build(MapperBuilderContext context) {
multiFieldsBuilder.build(this, context),
copyTo,
context.isSourceSynthetic(),
this
this,
context.isMappingNonTextFieldsSharedInvertedIndex()
);
}
}
Expand All @@ -349,6 +351,7 @@ public static final class KeywordFieldType extends StringFieldType {
private final FieldValues<String> scriptValues;
private final boolean isDimension;
private final boolean isSyntheticSource;
private final boolean mappingNonTextFieldsSharedInvertedIndex;

public KeywordFieldType(
String name,
Expand All @@ -357,7 +360,8 @@ public KeywordFieldType(
NamedAnalyzer searchAnalyzer,
NamedAnalyzer quoteAnalyzer,
Builder builder,
boolean isSyntheticSource
boolean isSyntheticSource,
boolean mappingNonTextFieldsSharedInvertedIndex
) {
super(
name,
Expand All @@ -374,6 +378,7 @@ public KeywordFieldType(
this.scriptValues = builder.scriptValues();
this.isDimension = builder.dimension.getValue();
this.isSyntheticSource = isSyntheticSource;
this.mappingNonTextFieldsSharedInvertedIndex = mappingNonTextFieldsSharedInvertedIndex;
}

public KeywordFieldType(String name, boolean isIndexed, boolean hasDocValues, Map<String, String> meta) {
Expand All @@ -385,6 +390,7 @@ public KeywordFieldType(String name, boolean isIndexed, boolean hasDocValues, Ma
this.scriptValues = null;
this.isDimension = false;
this.isSyntheticSource = false;
this.mappingNonTextFieldsSharedInvertedIndex = false;
}

public KeywordFieldType(String name) {
Expand All @@ -407,6 +413,7 @@ public KeywordFieldType(String name, FieldType fieldType) {
this.scriptValues = null;
this.isDimension = false;
this.isSyntheticSource = false;
this.mappingNonTextFieldsSharedInvertedIndex = false;
}

public KeywordFieldType(String name, NamedAnalyzer analyzer) {
Expand All @@ -418,6 +425,7 @@ public KeywordFieldType(String name, NamedAnalyzer analyzer) {
this.scriptValues = null;
this.isDimension = false;
this.isSyntheticSource = false;
this.mappingNonTextFieldsSharedInvertedIndex = false;
}

@Override
Expand Down Expand Up @@ -851,6 +859,7 @@ public boolean hasNormalizer() {
private final boolean storeIgnored;

private final IndexAnalyzers indexAnalyzers;
private final boolean mappingNonTextFieldsSharedInvertedIndex;

private KeywordFieldMapper(
String simpleName,
Expand All @@ -859,7 +868,8 @@ private KeywordFieldMapper(
MultiFields multiFields,
CopyTo copyTo,
boolean storeIgnored,
Builder builder
Builder builder,
boolean mappingNonTextFieldsSharedInvertedIndex
) {
super(simpleName, mappedFieldType, multiFields, copyTo, builder.script.get() != null, builder.onScriptError.getValue());
assert fieldType.indexOptions().compareTo(IndexOptions.DOCS_AND_FREQS) <= 0;
Expand All @@ -874,6 +884,7 @@ private KeywordFieldMapper(
this.scriptCompiler = builder.scriptCompiler;
this.indexCreatedVersion = builder.indexCreatedVersion;
this.storeIgnored = storeIgnored;
this.mappingNonTextFieldsSharedInvertedIndex = mappingNonTextFieldsSharedInvertedIndex;
}

@Override
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -18,18 +18,20 @@ public class MapperBuilderContext {
/**
* The root context, to be used when building a tree of mappers
*/
public static MapperBuilderContext root(boolean isSourceSynthetic, boolean isDataStream) {
return new MapperBuilderContext(null, isSourceSynthetic, isDataStream);
public static MapperBuilderContext root(boolean isSourceSynthetic, boolean isDataStream, boolean nonTextFieldsSharedInvertedIndex) {
return new MapperBuilderContext(null, isSourceSynthetic, isDataStream, nonTextFieldsSharedInvertedIndex);
}

private final String path;
private final boolean isSourceSynthetic;
private final boolean isDataStream;
private final boolean mappingNonTextFieldsSharedInvertedIndex;

MapperBuilderContext(String path, boolean isSourceSynthetic, boolean isDataStream) {
MapperBuilderContext(String path, boolean isSourceSynthetic, boolean isDataStream, boolean mappingNonTextFieldsSharedInvertedIndex) {
this.path = path;
this.isSourceSynthetic = isSourceSynthetic;
this.isDataStream = isDataStream;
this.mappingNonTextFieldsSharedInvertedIndex = mappingNonTextFieldsSharedInvertedIndex;
}

/**
Expand All @@ -38,7 +40,7 @@ public static MapperBuilderContext root(boolean isSourceSynthetic, boolean isDat
* @return a new MapperBuilderContext with this context as its parent
*/
public MapperBuilderContext createChildContext(String name) {
return new MapperBuilderContext(buildFullName(name), isSourceSynthetic, isDataStream);
return new MapperBuilderContext(buildFullName(name), isSourceSynthetic, isDataStream, mappingNonTextFieldsSharedInvertedIndex);
}

/**
Expand All @@ -64,4 +66,8 @@ public boolean isSourceSynthetic() {
public boolean isDataStream() {
return isDataStream;
}

public boolean isMappingNonTextFieldsSharedInvertedIndex() {
return mappingNonTextFieldsSharedInvertedIndex;
}
}
Original file line number Diff line number Diff line change
Expand Up @@ -21,7 +21,7 @@ public final class MapperMergeContext {
* The root context, to be used when merging a tree of mappers
*/
public static MapperMergeContext root(boolean isSourceSynthetic, boolean isDataStream) {
return new MapperMergeContext(MapperBuilderContext.root(isSourceSynthetic, isDataStream));
return new MapperMergeContext(MapperBuilderContext.root(isSourceSynthetic, isDataStream, false));
}

/**
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -30,7 +30,7 @@ public final class Mapping implements ToXContentFragment {

public static final Mapping EMPTY = new Mapping(
new RootObjectMapper.Builder(MapperService.SINGLE_MAPPING_NAME, ObjectMapper.Defaults.SUBOBJECTS).build(
MapperBuilderContext.root(false, false)
MapperBuilderContext.root(false, false, false)
),
new MetadataFieldMapper[0],
null
Expand Down

0 comments on commit c74d5a2

Please sign in to comment.