Skip to content

Commit

Permalink
Merge remote-tracking branch 'origin/main' into flush-time-excluding-…
Browse files Browse the repository at this point in the history 
…waiting
  • Loading branch information
@arteam
arteam committed Apr 11, 2024
2 parents ea22d6e + c8c9187 commit 4ab7771
Show file tree
Hide file tree
Showing 11 changed files with 821 additions and 62 deletions.
6 changes: 6 additions & 0 deletions docs/changelog/107224.yaml
View file
Original file line number Diff line number Diff line change
@@ -0,0 +1,6 @@
pr: 107224
summary: "Enable 'encoder' and 'tags_schema' highlighting settings at field level"
area: Highlighting
type: enhancement
issues:
- 94028
6 changes: 6 additions & 0 deletions docs/changelog/107291.yaml
View file
Original file line number Diff line number Diff line change
@@ -0,0 +1,6 @@
pr: 107291
summary: Support data streams in enrich policy indices
area: Ingest Node
type: enhancement
issues:
- 98836
502 changes: 502 additions & 0 deletions modules/ingest-geoip/src/test/java/org/elasticsearch/ingest/geoip/MaxMindSupportTests.java
View file

Large diffs are not rendered by default.

3 changes: 2 additions & 1 deletion server/src/main/java/org/elasticsearch/TransportVersions.java
View file
Original file line number Diff line number Diff line change
Expand Up @@ -170,7 +170,8 @@ static TransportVersion def(int id) {
public static final TransportVersion ML_INFERENCE_TIMEOUT_ADDED = def(8_629_00_0);
public static final TransportVersion MODIFY_DATA_STREAM_FAILURE_STORES = def(8_630_00_0);
public static final TransportVersion ML_INFERENCE_RERANK_NEW_RESPONSE_FORMAT = def(8_631_00_0);
public static final TransportVersion ACTUAL_FLUSH_STATS = def(8_632_00_0);
public static final TransportVersion HIGHLIGHTERS_TAGS_ON_FIELD_LEVEL = def(8_632_00_0);
public static final TransportVersion ACTUAL_FLUSH_STATS = def(8_633_00_0);

/*
* STOP! READ THIS FIRST! No, really,
Expand Down
55 changes: 55 additions & 0 deletions ...in/java/org/elasticsearch/search/fetch/subphase/highlight/AbstractHighlighterBuilder.java
View file
Original file line number Diff line number Diff line change
Expand Up @@ -54,6 +54,7 @@ public abstract class AbstractHighlighterBuilder<HB extends AbstractHighlighterB
public static final ParseField FRAGMENT_OFFSET_FIELD = new ParseField("fragment_offset");
public static final ParseField NUMBER_OF_FRAGMENTS_FIELD = new ParseField("number_of_fragments");
public static final ParseField ENCODER_FIELD = new ParseField("encoder");
public static final ParseField TAGS_SCHEMA_FIELD = new ParseField("tags_schema");
public static final ParseField REQUIRE_FIELD_MATCH_FIELD = new ParseField("require_field_match");
public static final ParseField BOUNDARY_SCANNER_FIELD = new ParseField("boundary_scanner");
public static final ParseField BOUNDARY_MAX_SCAN_FIELD = new ParseField("boundary_max_scan");
Expand All @@ -69,6 +70,8 @@ public abstract class AbstractHighlighterBuilder<HB extends AbstractHighlighterB
public static final ParseField MATCHED_FIELDS_FIELD = new ParseField("matched_fields");
public static final ParseField MAX_ANALYZED_OFFSET_FIELD = new ParseField("max_analyzed_offset");

protected String encoder;

protected String[] preTags;

protected String[] postTags;
Expand Down Expand Up @@ -112,6 +115,7 @@ protected AbstractHighlighterBuilder(AbstractHighlighterBuilder<?> template, Que
postTags = template.postTags;
fragmentSize = template.fragmentSize;
numOfFragments = template.numOfFragments;
encoder = template.encoder;
highlighterType = template.highlighterType;
fragmenter = template.fragmenter;
highlightQuery = queryBuilder;
Expand All @@ -137,6 +141,9 @@ protected AbstractHighlighterBuilder(StreamInput in) throws IOException {
postTags(in.readOptionalStringArray());
fragmentSize(in.readOptionalVInt());
numOfFragments(in.readOptionalVInt());
if (in.getTransportVersion().onOrAfter(TransportVersions.HIGHLIGHTERS_TAGS_ON_FIELD_LEVEL)) {
encoder(in.readOptionalString());
}
highlighterType(in.readOptionalString());
fragmenter(in.readOptionalString());
if (in.readBoolean()) {
Expand Down Expand Up @@ -173,6 +180,9 @@ public final void writeTo(StreamOutput out) throws IOException {
out.writeOptionalStringArray(postTags);
out.writeOptionalVInt(fragmentSize);
out.writeOptionalVInt(numOfFragments);
if (out.getTransportVersion().onOrAfter(TransportVersions.HIGHLIGHTERS_TAGS_ON_FIELD_LEVEL)) {
out.writeOptionalString(encoder);
}
out.writeOptionalString(highlighterType);
out.writeOptionalString(fragmenter);
boolean hasQuery = highlightQuery != null;
Expand Down Expand Up @@ -275,6 +285,44 @@ public Integer numOfFragments() {
return this.numOfFragments;
}

/**
* Set the encoder, defaults to {@link HighlightBuilder#DEFAULT_ENCODER}
*/
@SuppressWarnings("unchecked")
public HB encoder(String encoder) {
this.encoder = encoder;
return (HB) this;
}

/**
* @return the value set by {@link #encoder(String)}
*/
public String encoder() {
return this.encoder;
}

/**
* Set a tag scheme that encapsulates a built in pre and post tags. The allowed schemes
* are {@code styled} and {@code default}.
*
* @param schemaName The tag scheme name
*/
@SuppressWarnings("unchecked")
public HB tagsSchema(String schemaName) {
switch (schemaName) {
case "default" -> {
preTags(HighlightBuilder.DEFAULT_PRE_TAGS);
postTags(HighlightBuilder.DEFAULT_POST_TAGS);
}
case "styled" -> {
preTags(HighlightBuilder.DEFAULT_STYLED_PRE_TAG);
postTags(HighlightBuilder.DEFAULT_STYLED_POST_TAGS);
}
default -> throw new IllegalArgumentException("Unknown tag schema [" + schemaName + "]");
}
return (HB) this;
}

/**
* Set type of highlighter to use. Out of the box supported types
* are {@code unified}, {@code plain} and {@code fvh}.
Expand Down Expand Up @@ -561,6 +609,9 @@ void commonOptionsToXContent(XContentBuilder builder) throws IOException {
if (numOfFragments != null) {
builder.field(NUMBER_OF_FRAGMENTS_FIELD.getPreferredName(), numOfFragments);
}
if (encoder != null) {
builder.field(ENCODER_FIELD.getPreferredName(), encoder);
}
if (highlighterType != null) {
builder.field(TYPE_FIELD.getPreferredName(), highlighterType);
}
Expand Down Expand Up @@ -612,6 +663,8 @@ static <HB extends AbstractHighlighterBuilder<HB>> BiFunction<XContentParser, HB
parser.declareBoolean(HB::highlightFilter, HIGHLIGHT_FILTER_FIELD);
parser.declareInt(HB::fragmentSize, FRAGMENT_SIZE_FIELD);
parser.declareInt(HB::numOfFragments, NUMBER_OF_FRAGMENTS_FIELD);
parser.declareString(HB::encoder, ENCODER_FIELD);
parser.declareString(HB::tagsSchema, TAGS_SCHEMA_FIELD);
parser.declareBoolean(HB::requireFieldMatch, REQUIRE_FIELD_MATCH_FIELD);
parser.declareString(HB::boundaryScannerType, BOUNDARY_SCANNER_FIELD);
parser.declareInt(HB::boundaryMaxScan, BOUNDARY_MAX_SCAN_FIELD);
Expand Down Expand Up @@ -661,6 +714,7 @@ public final int hashCode() {
Arrays.hashCode(postTags),
fragmentSize,
numOfFragments,
encoder,
highlighterType,
fragmenter,
highlightQuery,
Expand Down Expand Up @@ -698,6 +752,7 @@ public final boolean equals(Object obj) {
&& Arrays.equals(postTags, other.postTags)
&& Objects.equals(fragmentSize, other.fragmentSize)
&& Objects.equals(numOfFragments, other.numOfFragments)
&& Objects.equals(encoder, other.encoder)
&& Objects.equals(highlighterType, other.highlighterType)
&& Objects.equals(fragmenter, other.fragmenter)
&& Objects.equals(highlightQuery, other.highlightQuery)
Expand Down
68 changes: 13 additions & 55 deletions server/src/main/java/org/elasticsearch/search/fetch/subphase/highlight/HighlightBuilder.java
View file
Original file line number Diff line number Diff line change
Expand Up @@ -10,6 +10,7 @@

import org.apache.lucene.search.Query;
import org.apache.lucene.search.vectorhighlight.SimpleBoundaryScanner;
import org.elasticsearch.TransportVersions;
import org.elasticsearch.common.io.stream.StreamInput;
import org.elasticsearch.common.io.stream.StreamOutput;
import org.elasticsearch.common.io.stream.Writeable;
Expand All @@ -21,7 +22,6 @@
import org.elasticsearch.search.fetch.subphase.highlight.SearchHighlightContext.FieldOptions;
import org.elasticsearch.xcontent.ObjectParser;
import org.elasticsearch.xcontent.ObjectParser.NamedObjectParser;
import org.elasticsearch.xcontent.ParseField;
import org.elasticsearch.xcontent.XContentBuilder;
import org.elasticsearch.xcontent.XContentParser;

Expand Down Expand Up @@ -104,8 +104,6 @@ public final class HighlightBuilder extends AbstractHighlighterBuilder<Highlight

private final List<Field> fields;

private String encoder;

private boolean useExplicitFieldOrder = false;

public HighlightBuilder() {
Expand All @@ -114,7 +112,6 @@ public HighlightBuilder() {

public HighlightBuilder(HighlightBuilder template, QueryBuilder highlightQuery, List<Field> fields) {
super(template, highlightQuery);
this.encoder = template.encoder;
this.useExplicitFieldOrder = template.useExplicitFieldOrder;
this.fields = fields;
}
Expand All @@ -124,15 +121,19 @@ public HighlightBuilder(HighlightBuilder template, QueryBuilder highlightQuery,
*/
public HighlightBuilder(StreamInput in) throws IOException {
super(in);
encoder(in.readOptionalString());
if (in.getTransportVersion().before(TransportVersions.HIGHLIGHTERS_TAGS_ON_FIELD_LEVEL)) {
encoder(in.readOptionalString());
}
useExplicitFieldOrder(in.readBoolean());
this.fields = in.readCollectionAsList(Field::new);
assert this.equals(new HighlightBuilder(this, highlightQuery, fields)) : "copy constructor is broken";
}

@Override
protected void doWriteTo(StreamOutput out) throws IOException {
out.writeOptionalString(encoder);
if (out.getTransportVersion().before(TransportVersions.HIGHLIGHTERS_TAGS_ON_FIELD_LEVEL)) {
out.writeOptionalString(encoder);
}
out.writeBoolean(useExplicitFieldOrder);
out.writeCollection(fields);
}
Expand Down Expand Up @@ -185,45 +186,6 @@ public List<Field> fields() {
return this.fields;
}

/**
* Set a tag scheme that encapsulates a built in pre and post tags. The allowed schemes
* are {@code styled} and {@code default}.
*
* @param schemaName The tag scheme name
*/
public HighlightBuilder tagsSchema(String schemaName) {
switch (schemaName) {
case "default" -> {
preTags(DEFAULT_PRE_TAGS);
postTags(DEFAULT_POST_TAGS);
}
case "styled" -> {
preTags(DEFAULT_STYLED_PRE_TAG);
postTags(DEFAULT_STYLED_POST_TAGS);
}
default -> throw new IllegalArgumentException("Unknown tag schema [" + schemaName + "]");
}
return this;
}

/**
* Set encoder for the highlighting
* are {@code html} and {@code default}.
*
* @param encoder name
*/
public HighlightBuilder encoder(String encoder) {
this.encoder = encoder;
return this;
}

/**
* Getter for {@link #encoder(String)}
*/
public String encoder() {
return this.encoder;
}

/**
* Send the fields to be highlighted using a syntax that is specific about the order in which they should be highlighted.
* @return this for chaining
Expand Down Expand Up @@ -251,8 +213,6 @@ public XContentBuilder toXContent(XContentBuilder builder, Params params) throws
private static final BiFunction<XContentParser, HighlightBuilder, HighlightBuilder> PARSER;
static {
ObjectParser<HighlightBuilder, Void> parser = new ObjectParser<>("highlight");
parser.declareString(HighlightBuilder::tagsSchema, new ParseField("tags_schema"));
parser.declareString(HighlightBuilder::encoder, ENCODER_FIELD);
parser.declareNamedObjects(
HighlightBuilder::fields,
Field.PARSER,
Expand All @@ -269,7 +229,7 @@ public static HighlightBuilder fromXContent(XContentParser p) {
public SearchHighlightContext build(SearchExecutionContext context) throws IOException {
// create template global options that are later merged with any partial field options
final SearchHighlightContext.FieldOptions.Builder globalOptionsBuilder = new SearchHighlightContext.FieldOptions.Builder();
globalOptionsBuilder.encoder(this.encoder);

transferOptions(this, globalOptionsBuilder, context);

// overwrite unset global options by default values
Expand Down Expand Up @@ -325,6 +285,9 @@ private static void transferOptions(
if (highlighterBuilder.numOfFragments != null) {
targetOptionsBuilder.numberOfFragments(highlighterBuilder.numOfFragments);
}
if (highlighterBuilder.encoder != null) {
targetOptionsBuilder.encoder(highlighterBuilder.encoder);
}
if (highlighterBuilder.requireFieldMatch != null) {
targetOptionsBuilder.requireFieldMatch(highlighterBuilder.requireFieldMatch);
}
Expand Down Expand Up @@ -379,9 +342,6 @@ public void innerXContent(XContentBuilder builder) throws IOException {
// first write common options
commonOptionsToXContent(builder);
// special options for top-level highlighter
if (encoder != null) {
builder.field(ENCODER_FIELD.getPreferredName(), encoder);
}
if (fields.size() > 0) {
if (useExplicitFieldOrder) {
builder.startArray(FIELDS_FIELD.getPreferredName());
Expand All @@ -407,14 +367,12 @@ public void innerXContent(XContentBuilder builder) throws IOException {

@Override
protected int doHashCode() {
return Objects.hash(encoder, useExplicitFieldOrder, fields);
return Objects.hash(useExplicitFieldOrder, fields);
}

@Override
protected boolean doEquals(HighlightBuilder other) {
return Objects.equals(encoder, other.encoder)
&& Objects.equals(useExplicitFieldOrder, other.useExplicitFieldOrder)
&& Objects.equals(fields, other.fields);
return Objects.equals(useExplicitFieldOrder, other.useExplicitFieldOrder) && Objects.equals(fields, other.fields);
}

@Override
Expand Down
52 changes: 47 additions & 5 deletions ...rc/test/java/org/elasticsearch/search/fetch/subphase/highlight/HighlightBuilderTests.java
View file
Original file line number Diff line number Diff line change
Expand Up @@ -332,8 +332,6 @@ public MappedFieldType getFieldType(String name) {
highlightBuilder = Rewriteable.rewrite(highlightBuilder, mockContext);
SearchHighlightContext highlight = highlightBuilder.build(mockContext);
for (SearchHighlightContext.Field field : highlight.fields()) {
String encoder = highlightBuilder.encoder() != null ? highlightBuilder.encoder() : HighlightBuilder.DEFAULT_ENCODER;
assertEquals(encoder, field.fieldOptions().encoder());
final Field fieldBuilder = getFieldBuilderByName(highlightBuilder, field.field());
assertNotNull("expected a highlight builder for field " + field.field(), fieldBuilder);
FieldOptions fieldOptions = field.fieldOptions();
Expand All @@ -344,6 +342,7 @@ public MappedFieldType getFieldType(String name) {
fieldOptions
);

checkSame.accept(AbstractHighlighterBuilder::encoder, FieldOptions::encoder);
checkSame.accept(AbstractHighlighterBuilder::boundaryChars, FieldOptions::boundaryChars);
checkSame.accept(AbstractHighlighterBuilder::boundaryScannerType, FieldOptions::boundaryScannerType);
checkSame.accept(AbstractHighlighterBuilder::boundaryMaxScan, FieldOptions::boundaryMaxScan);
Expand Down Expand Up @@ -477,6 +476,49 @@ public void testParsingTagsSchema() throws IOException {
highlightBuilder.postTags()
);

}

highlightElement = """
{
"fields": {
"default_string": {
"tags_schema": "default"
},
"styled_string": {
"tags_schema": "styled"
}
}
}
""";
try (XContentParser parser = createParser(JsonXContent.jsonXContent, highlightElement)) {

HighlightBuilder highlightBuilder = HighlightBuilder.fromXContent(parser);
Field defaultStringField = getFieldBuilderByName(highlightBuilder, "default_string");
assertNotNull(defaultStringField);
assertArrayEquals(
"setting tags_schema 'default' at the field level should alter pre_tags for that field",
HighlightBuilder.DEFAULT_PRE_TAGS,
defaultStringField.preTags()
);
assertArrayEquals(
"setting tags_schema 'default' at the field level should alter post_tags for that that field",
HighlightBuilder.DEFAULT_POST_TAGS,
defaultStringField.postTags()
);

Field styledStringField = getFieldBuilderByName(highlightBuilder, "styled_string");
assertNotNull(styledStringField);
assertArrayEquals(
"setting tags_schema 'styled' at the field level should alter pre_tags for that that field",
HighlightBuilder.DEFAULT_STYLED_PRE_TAG,
styledStringField.preTags()
);
assertArrayEquals(
"setting tags_schema 'styled' at the field level should alter post_tags for that that field",
HighlightBuilder.DEFAULT_STYLED_POST_TAGS,
styledStringField.postTags()
);

XContentParseException e = expectParseThrows(XContentParseException.class, """
{
"tags_schema" : "somthing_else"
Expand Down Expand Up @@ -592,9 +634,6 @@ public static HighlightBuilder randomHighlighterBuilder() {
HighlightBuilder testHighlighter = new HighlightBuilder();
setRandomCommonOptions(testHighlighter);
testHighlighter.useExplicitFieldOrder(randomBoolean());
if (randomBoolean()) {
testHighlighter.encoder(randomFrom(Arrays.asList(new String[] { "default", "html" })));
}
int numberOfFields = randomIntBetween(1, 5);
for (int i = 0; i < numberOfFields; i++) {
Field field = new Field(i + "_" + randomAlphaOfLengthBetween(1, 10));
Expand All @@ -617,6 +656,9 @@ private static void setRandomCommonOptions(AbstractHighlighterBuilder highlightB
highlightBuilder.preTags(randomStringArray(1, 3));
highlightBuilder.postTags(randomStringArray(1, 3));
}
if (randomBoolean()) {
highlightBuilder.encoder(randomFrom(Arrays.asList("default", "html")));
}
if (randomBoolean()) {
highlightBuilder.fragmentSize(randomIntBetween(0, 100));
}
Expand Down
Loading

0 comments on commit 4ab7771

Please sign in to comment.