From 498a43fa78a7d423c72508a44f0690cab695ff82 Mon Sep 17 00:00:00 2001 From: Parker Timmins Date: Wed, 3 Sep 2025 17:26:53 -0500 Subject: [PATCH 1/6] Setup esql blockloader for pattern_text --- .../index/mapper/BlockDocValuesReader.java | 48 ++++++-- .../test/rest/ESRestTestCase.java | 2 +- .../PatternedTextBlockLoader.java | 40 +++++++ .../patternedtext/PatternedTextFieldType.java | 2 +- .../test/patternedtext/40_esql.yml | 108 ++++++++++++++++++ 5 files changed, 189 insertions(+), 11 deletions(-) create mode 100644 x-pack/plugin/logsdb/src/main/java/org/elasticsearch/xpack/logsdb/patternedtext/PatternedTextBlockLoader.java create mode 100644 x-pack/plugin/logsdb/src/yamlRestTest/resources/rest-api-spec/test/patternedtext/40_esql.yml diff --git a/server/src/main/java/org/elasticsearch/index/mapper/BlockDocValuesReader.java b/server/src/main/java/org/elasticsearch/index/mapper/BlockDocValuesReader.java index f2324d70ce750..56402982a15c3 100644 --- a/server/src/main/java/org/elasticsearch/index/mapper/BlockDocValuesReader.java +++ b/server/src/main/java/org/elasticsearch/index/mapper/BlockDocValuesReader.java @@ -886,12 +886,10 @@ public AllReader reader(LeafReaderContext context) throws IOException { } } - private static class BytesRefsFromBinary extends BlockDocValuesReader { - private final BinaryDocValues docValues; - private final ByteArrayStreamInput in = new ByteArrayStreamInput(); - private final BytesRef scratch = new BytesRef(); + abstract static class AbstractBytesRefsFromBinary extends BlockDocValuesReader { + protected final BinaryDocValues docValues; - BytesRefsFromBinary(BinaryDocValues docValues) { + AbstractBytesRefsFromBinary(BinaryDocValues docValues) { this.docValues = docValues; } @@ -911,7 +909,24 @@ public void read(int docId, BlockLoader.StoredFields storedFields, Builder build read(docId, (BytesRefBuilder) builder); } - private void read(int doc, BytesRefBuilder builder) throws IOException { + @Override + public int docId() { + return docValues.docID(); + } + + abstract void read(int docId, BytesRefBuilder builder) throws IOException; + } + + static class BytesRefsFromBinary extends AbstractBytesRefsFromBinary { + private final ByteArrayStreamInput in = new ByteArrayStreamInput(); + private final BytesRef scratch = new BytesRef(); + + BytesRefsFromBinary(BinaryDocValues docValues) { + super(docValues); + } + + @Override + void read(int doc, BytesRefBuilder builder) throws IOException { if (false == docValues.advanceExact(doc)) { builder.appendNull(); return; @@ -937,15 +952,30 @@ private void read(int doc, BytesRefBuilder builder) throws IOException { } builder.endPositionEntry(); } + @Override + public String toString() { + return "BlockDocValuesReader.Bytes"; + } + } + + public static class BytesRefsFromSimpleBinary extends AbstractBytesRefsFromBinary { + public BytesRefsFromSimpleBinary(BinaryDocValues docValues) { + super(docValues); + } @Override - public int docId() { - return docValues.docID(); + void read(int doc, BytesRefBuilder builder) throws IOException { + if (false == docValues.advanceExact(doc)) { + builder.appendNull(); + return; + } + BytesRef bytes = docValues.binaryValue(); + builder.appendBytesRef(bytes); } @Override public String toString() { - return "BlockDocValuesReader.Bytes"; + return "BlockDocValuesReader.BytesSimple"; } } diff --git a/test/framework/src/main/java/org/elasticsearch/test/rest/ESRestTestCase.java b/test/framework/src/main/java/org/elasticsearch/test/rest/ESRestTestCase.java index 3b0470968e2f6..daed1c03cda4c 100644 --- a/test/framework/src/main/java/org/elasticsearch/test/rest/ESRestTestCase.java +++ b/test/framework/src/main/java/org/elasticsearch/test/rest/ESRestTestCase.java @@ -1746,7 +1746,7 @@ protected static void doConfigureClient(RestClientBuilder builder, Settings sett defaultHeaders[i++] = new BasicHeader(entry.getKey(), entry.getValue()); } builder.setDefaultHeaders(defaultHeaders); - final String socketTimeoutString = Objects.requireNonNullElse(settings.get(CLIENT_SOCKET_TIMEOUT), "60s"); + final String socketTimeoutString = Objects.requireNonNullElse(settings.get(CLIENT_SOCKET_TIMEOUT), "6000s"); final TimeValue socketTimeout = TimeValue.parseTimeValue(socketTimeoutString, CLIENT_SOCKET_TIMEOUT); builder.setRequestConfigCallback(conf -> conf.setSocketTimeout(Math.toIntExact(socketTimeout.getMillis()))); if (settings.hasValue(CLIENT_PATH_PREFIX)) { diff --git a/x-pack/plugin/logsdb/src/main/java/org/elasticsearch/xpack/logsdb/patternedtext/PatternedTextBlockLoader.java b/x-pack/plugin/logsdb/src/main/java/org/elasticsearch/xpack/logsdb/patternedtext/PatternedTextBlockLoader.java new file mode 100644 index 0000000000000..a24344cb2cb2a --- /dev/null +++ b/x-pack/plugin/logsdb/src/main/java/org/elasticsearch/xpack/logsdb/patternedtext/PatternedTextBlockLoader.java @@ -0,0 +1,40 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License + * 2.0; you may not use this file except in compliance with the Elastic License + * 2.0. + */ + +package org.elasticsearch.xpack.logsdb.patternedtext; + +import org.apache.lucene.index.LeafReaderContext; +import org.elasticsearch.index.mapper.BlockDocValuesReader; + +import java.io.IOException; + +public class PatternedTextBlockLoader extends BlockDocValuesReader.DocValuesBlockLoader { + + private final String templateFieldName; + private final String argsFieldName; + private final String argsInfoFieldName; + + PatternedTextBlockLoader(String templateFieldName, String argsFieldName, String argsInfoFieldName) { + this.templateFieldName = templateFieldName; + this.argsFieldName = argsFieldName; + this.argsInfoFieldName = argsInfoFieldName; + } + + @Override + public BytesRefBuilder builder(BlockFactory factory, int expectedCount) { + return factory.bytesRefs(expectedCount); + } + + @Override + public AllReader reader(LeafReaderContext context) throws IOException { + var docValues = PatternedTextDocValues.from(context.reader(), templateFieldName, argsFieldName, argsInfoFieldName); + if (docValues == null) { + return new ConstantNullsReader(); + } + return new BlockDocValuesReader.BytesRefsFromSimpleBinary(docValues); + } +} diff --git a/x-pack/plugin/logsdb/src/main/java/org/elasticsearch/xpack/logsdb/patternedtext/PatternedTextFieldType.java b/x-pack/plugin/logsdb/src/main/java/org/elasticsearch/xpack/logsdb/patternedtext/PatternedTextFieldType.java index 2c7f9b59ee4dc..c35b4aa3ba999 100644 --- a/x-pack/plugin/logsdb/src/main/java/org/elasticsearch/xpack/logsdb/patternedtext/PatternedTextFieldType.java +++ b/x-pack/plugin/logsdb/src/main/java/org/elasticsearch/xpack/logsdb/patternedtext/PatternedTextFieldType.java @@ -252,7 +252,7 @@ public Query phrasePrefixQuery(TokenStream stream, int slop, int maxExpansions, @Override public BlockLoader blockLoader(BlockLoaderContext blContext) { - return new BlockDocValuesReader.BytesRefsFromBinaryBlockLoader(name()); + return new PatternedTextBlockLoader(templateFieldName(), argsFieldName(), argsInfoFieldName()); } @Override diff --git a/x-pack/plugin/logsdb/src/yamlRestTest/resources/rest-api-spec/test/patternedtext/40_esql.yml b/x-pack/plugin/logsdb/src/yamlRestTest/resources/rest-api-spec/test/patternedtext/40_esql.yml new file mode 100644 index 0000000000000..b04691290e1a4 --- /dev/null +++ b/x-pack/plugin/logsdb/src/yamlRestTest/resources/rest-api-spec/test/patternedtext/40_esql.yml @@ -0,0 +1,108 @@ +--- +setup: + - requires: + cluster_features: [ "mapper.patterned_text" ] + reason: "patterned_text mappings are used in this test" + + - do: + indices.create: + index: test + body: + settings: + index: + mapping.source.mode: synthetic + mappings: + properties: + "@timestamp": + type: date + message: + type: patterned_text + # defaults to index_options: docs + + - do: + bulk: + index: test + refresh: true + body: + - { "index": { "_id": "1" } } + - { "@timestamp": "2025-07-17T00:00:01Z" } + - { "index": { "_id": "2" } } + - { "message": "Found 5 errors for service [cheddar1]", "@timestamp": "2025-07-17T00:00:02Z" } + # template_id: mOVsnxlxdac + - { "index": { "_id": "3" } } + - { "message": "[2020-08-18T00:58:56] Found 123 errors for service [cheddar1]", "@timestamp": "2025-07-17T00:00:03Z" } + # template_id: 1l_PtCLQ5xY + - { "index": { "_id": "4" } } + - { "message": "Found some errors for cheddar data service", "@timestamp": "2025-07-17T00:00:04Z"} + # template_id: k-2qtjujOCw + - { "index": { "_id": "5" } } + - { "message": "Found 123 errors for service [gorgonzola-24]", "@timestamp": "2025-07-17T00:00:05Z" } + # template_id: mOVsnxlxdac + +--- +teardown: + - do: + indices.delete: + index: test + +--- +"Simple from": + - do: + esql.query: + body: + query: 'FROM test | SORT @timestamp | KEEP message, message.template_id | LIMIT 10' + + - match: {columns.0.name: "message"} + - match: {columns.0.type: "text"} + - match: {columns.1.name: "message.template_id"} + - match: {columns.1.type: "keyword"} + + - length: {values: 5} + - match: {values.0.0: null } + - match: {values.0.1: null } + - match: {values.1.0: "Found 5 errors for service [cheddar1]" } + - match: {values.1.1: "mOVsnxlxdac" } + - match: {values.2.0: "[2020-08-18T00:58:56] Found 123 errors for service [cheddar1]" } + - match: {values.2.1: "1l_PtCLQ5xY" } + - match: {values.3.0: "Found some errors for cheddar data service" } + - match: {values.3.1: "k-2qtjujOCw" } + - match: {values.4.0: "Found 123 errors for service [gorgonzola-24]" } + - match: {values.4.1: "mOVsnxlxdac" } + +--- +"match query": + - do: + esql.query: + body: + query: 'FROM test | WHERE MATCH(message, "gorgonzola-24") | KEEP message | LIMIT 10' + + - length: {values: 1} + - match: {values.0.0: "Found 123 errors for service [gorgonzola-24]" } + +--- +"match phrase query": + - do: + esql.query: + body: + query: 'FROM test | WHERE MATCH_PHRASE(message, "123 errors") | SORT @timestamp | KEEP message | LIMIT 10' + + - length: {values: 2} + - match: {values.0.0: "[2020-08-18T00:58:56] Found 123 errors for service [cheddar1]" } + - match: {values.1.0: "Found 123 errors for service [gorgonzola-24]" } + +--- +"template_id stats": + - do: + esql.query: + body: + query: 'FROM test | STATS count(*) BY message.template_id | SORT message.template_id | LIMIT 10' + + - length: {values: 4} + - match: {values.0.0: 1 } + - match: {values.0.1: "1l_PtCLQ5xY" } + - match: {values.1.0: 1 } + - match: {values.1.1: "k-2qtjujOCw" } + - match: {values.2.0: 2 } + - match: {values.2.1: "mOVsnxlxdac" } + - match: {values.3.0: 1 } + - match: {values.3.1: null } From 3f182d84743ee65780d2d12718b52325af0b8e30 Mon Sep 17 00:00:00 2001 From: Parker Timmins Date: Wed, 3 Sep 2025 17:28:10 -0500 Subject: [PATCH 2/6] remove changes used for debugging --- .../main/java/org/elasticsearch/test/rest/ESRestTestCase.java | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/test/framework/src/main/java/org/elasticsearch/test/rest/ESRestTestCase.java b/test/framework/src/main/java/org/elasticsearch/test/rest/ESRestTestCase.java index daed1c03cda4c..3b0470968e2f6 100644 --- a/test/framework/src/main/java/org/elasticsearch/test/rest/ESRestTestCase.java +++ b/test/framework/src/main/java/org/elasticsearch/test/rest/ESRestTestCase.java @@ -1746,7 +1746,7 @@ protected static void doConfigureClient(RestClientBuilder builder, Settings sett defaultHeaders[i++] = new BasicHeader(entry.getKey(), entry.getValue()); } builder.setDefaultHeaders(defaultHeaders); - final String socketTimeoutString = Objects.requireNonNullElse(settings.get(CLIENT_SOCKET_TIMEOUT), "6000s"); + final String socketTimeoutString = Objects.requireNonNullElse(settings.get(CLIENT_SOCKET_TIMEOUT), "60s"); final TimeValue socketTimeout = TimeValue.parseTimeValue(socketTimeoutString, CLIENT_SOCKET_TIMEOUT); builder.setRequestConfigCallback(conf -> conf.setSocketTimeout(Math.toIntExact(socketTimeout.getMillis()))); if (settings.hasValue(CLIENT_PATH_PREFIX)) { From 3cd7d84c956db0ba280c14a043838844dcd3063d Mon Sep 17 00:00:00 2001 From: elasticsearchmachine Date: Wed, 3 Sep 2025 22:38:00 +0000 Subject: [PATCH 3/6] [CI] Auto commit changes from spotless --- .../org/elasticsearch/index/mapper/BlockDocValuesReader.java | 1 + .../xpack/logsdb/patternedtext/PatternedTextFieldType.java | 1 - 2 files changed, 1 insertion(+), 1 deletion(-) diff --git a/server/src/main/java/org/elasticsearch/index/mapper/BlockDocValuesReader.java b/server/src/main/java/org/elasticsearch/index/mapper/BlockDocValuesReader.java index 56402982a15c3..1b778cca5d529 100644 --- a/server/src/main/java/org/elasticsearch/index/mapper/BlockDocValuesReader.java +++ b/server/src/main/java/org/elasticsearch/index/mapper/BlockDocValuesReader.java @@ -952,6 +952,7 @@ void read(int doc, BytesRefBuilder builder) throws IOException { } builder.endPositionEntry(); } + @Override public String toString() { return "BlockDocValuesReader.Bytes"; diff --git a/x-pack/plugin/logsdb/src/main/java/org/elasticsearch/xpack/logsdb/patternedtext/PatternedTextFieldType.java b/x-pack/plugin/logsdb/src/main/java/org/elasticsearch/xpack/logsdb/patternedtext/PatternedTextFieldType.java index c35b4aa3ba999..e195e4f744ded 100644 --- a/x-pack/plugin/logsdb/src/main/java/org/elasticsearch/xpack/logsdb/patternedtext/PatternedTextFieldType.java +++ b/x-pack/plugin/logsdb/src/main/java/org/elasticsearch/xpack/logsdb/patternedtext/PatternedTextFieldType.java @@ -30,7 +30,6 @@ import org.elasticsearch.index.fielddata.FieldDataContext; import org.elasticsearch.index.fielddata.IndexFieldData; import org.elasticsearch.index.fielddata.SourceValueFetcherSortedBinaryIndexFieldData; -import org.elasticsearch.index.mapper.BlockDocValuesReader; import org.elasticsearch.index.mapper.BlockLoader; import org.elasticsearch.index.mapper.SourceValueFetcher; import org.elasticsearch.index.mapper.StringFieldType; From 3aa135fa5cd530ed5ccbe2d94556e4cf5aac238e Mon Sep 17 00:00:00 2001 From: Parker Timmins Date: Wed, 3 Sep 2025 18:49:56 -0500 Subject: [PATCH 4/6] Add explanation of BytesRef encodings --- .../elasticsearch/index/mapper/BlockDocValuesReader.java | 7 +++++++ 1 file changed, 7 insertions(+) diff --git a/server/src/main/java/org/elasticsearch/index/mapper/BlockDocValuesReader.java b/server/src/main/java/org/elasticsearch/index/mapper/BlockDocValuesReader.java index 1b778cca5d529..c3985881c3e5a 100644 --- a/server/src/main/java/org/elasticsearch/index/mapper/BlockDocValuesReader.java +++ b/server/src/main/java/org/elasticsearch/index/mapper/BlockDocValuesReader.java @@ -917,6 +917,9 @@ public int docId() { abstract void read(int docId, BytesRefBuilder builder) throws IOException; } + /** + * Read BinaryDocValues encoded by {@link BinaryFieldMapper.CustomBinaryDocValuesField} + */ static class BytesRefsFromBinary extends AbstractBytesRefsFromBinary { private final ByteArrayStreamInput in = new ByteArrayStreamInput(); private final BytesRef scratch = new BytesRef(); @@ -959,6 +962,10 @@ public String toString() { } } + /** + * Read BinaryDocValues with no additional structure in the BytesRefs. + * Each BytesRef from the doc values maps directly to a value in the block loader. + */ public static class BytesRefsFromSimpleBinary extends AbstractBytesRefsFromBinary { public BytesRefsFromSimpleBinary(BinaryDocValues docValues) { super(docValues); From 62c0ed9770a9854717e9df3f18bfb55aa1d5d9ee Mon Sep 17 00:00:00 2001 From: Parker Timmins Date: Thu, 4 Sep 2025 09:51:34 -0500 Subject: [PATCH 5/6] Rename BytesRefsFromBinary to BytesRefsFromCustomBinary --- .../index/mapper/BlockDocValuesReader.java | 10 +++++----- .../xpack/wildcard/mapper/WildcardFieldMapper.java | 2 +- 2 files changed, 6 insertions(+), 6 deletions(-) diff --git a/server/src/main/java/org/elasticsearch/index/mapper/BlockDocValuesReader.java b/server/src/main/java/org/elasticsearch/index/mapper/BlockDocValuesReader.java index c3985881c3e5a..2f9571f9cd5e0 100644 --- a/server/src/main/java/org/elasticsearch/index/mapper/BlockDocValuesReader.java +++ b/server/src/main/java/org/elasticsearch/index/mapper/BlockDocValuesReader.java @@ -864,10 +864,10 @@ public String toString() { } } - public static class BytesRefsFromBinaryBlockLoader extends DocValuesBlockLoader { + public static class BytesRefsFromCustomBinaryBlockLoader extends DocValuesBlockLoader { private final String fieldName; - public BytesRefsFromBinaryBlockLoader(String fieldName) { + public BytesRefsFromCustomBinaryBlockLoader(String fieldName) { this.fieldName = fieldName; } @@ -882,7 +882,7 @@ public AllReader reader(LeafReaderContext context) throws IOException { if (docValues == null) { return new ConstantNullsReader(); } - return new BytesRefsFromBinary(docValues); + return new BytesRefsFromCustomBinary(docValues); } } @@ -920,11 +920,11 @@ public int docId() { /** * Read BinaryDocValues encoded by {@link BinaryFieldMapper.CustomBinaryDocValuesField} */ - static class BytesRefsFromBinary extends AbstractBytesRefsFromBinary { + static class BytesRefsFromCustomBinary extends AbstractBytesRefsFromBinary { private final ByteArrayStreamInput in = new ByteArrayStreamInput(); private final BytesRef scratch = new BytesRef(); - BytesRefsFromBinary(BinaryDocValues docValues) { + BytesRefsFromCustomBinary(BinaryDocValues docValues) { super(docValues); } diff --git a/x-pack/plugin/wildcard/src/main/java/org/elasticsearch/xpack/wildcard/mapper/WildcardFieldMapper.java b/x-pack/plugin/wildcard/src/main/java/org/elasticsearch/xpack/wildcard/mapper/WildcardFieldMapper.java index 3aa43fa925747..ff89074c3f063 100644 --- a/x-pack/plugin/wildcard/src/main/java/org/elasticsearch/xpack/wildcard/mapper/WildcardFieldMapper.java +++ b/x-pack/plugin/wildcard/src/main/java/org/elasticsearch/xpack/wildcard/mapper/WildcardFieldMapper.java @@ -956,7 +956,7 @@ public Query prefixQuery( @Override public BlockLoader blockLoader(BlockLoaderContext blContext) { if (hasDocValues()) { - return new BlockDocValuesReader.BytesRefsFromBinaryBlockLoader(name()); + return new BlockDocValuesReader.BytesRefsFromCustomBinaryBlockLoader(name()); } return null; } From 2328dc70952aa8bce9e2eab8bc74d88e36776ac6 Mon Sep 17 00:00:00 2001 From: Parker Timmins Date: Thu, 4 Sep 2025 10:22:23 -0500 Subject: [PATCH 6/6] Rename BytesRefsFromSimpleBinary to BytesRefsFromBinary --- .../elasticsearch/index/mapper/BlockDocValuesReader.java | 8 ++++---- .../logsdb/patternedtext/PatternedTextBlockLoader.java | 2 +- 2 files changed, 5 insertions(+), 5 deletions(-) diff --git a/server/src/main/java/org/elasticsearch/index/mapper/BlockDocValuesReader.java b/server/src/main/java/org/elasticsearch/index/mapper/BlockDocValuesReader.java index 2f9571f9cd5e0..6061682d7146a 100644 --- a/server/src/main/java/org/elasticsearch/index/mapper/BlockDocValuesReader.java +++ b/server/src/main/java/org/elasticsearch/index/mapper/BlockDocValuesReader.java @@ -958,7 +958,7 @@ void read(int doc, BytesRefBuilder builder) throws IOException { @Override public String toString() { - return "BlockDocValuesReader.Bytes"; + return "BlockDocValuesReader.BytesCustom"; } } @@ -966,8 +966,8 @@ public String toString() { * Read BinaryDocValues with no additional structure in the BytesRefs. * Each BytesRef from the doc values maps directly to a value in the block loader. */ - public static class BytesRefsFromSimpleBinary extends AbstractBytesRefsFromBinary { - public BytesRefsFromSimpleBinary(BinaryDocValues docValues) { + public static class BytesRefsFromBinary extends AbstractBytesRefsFromBinary { + public BytesRefsFromBinary(BinaryDocValues docValues) { super(docValues); } @@ -983,7 +983,7 @@ void read(int doc, BytesRefBuilder builder) throws IOException { @Override public String toString() { - return "BlockDocValuesReader.BytesSimple"; + return "BlockDocValuesReader.Bytes"; } } diff --git a/x-pack/plugin/logsdb/src/main/java/org/elasticsearch/xpack/logsdb/patternedtext/PatternedTextBlockLoader.java b/x-pack/plugin/logsdb/src/main/java/org/elasticsearch/xpack/logsdb/patternedtext/PatternedTextBlockLoader.java index a24344cb2cb2a..a4ca13520d7e8 100644 --- a/x-pack/plugin/logsdb/src/main/java/org/elasticsearch/xpack/logsdb/patternedtext/PatternedTextBlockLoader.java +++ b/x-pack/plugin/logsdb/src/main/java/org/elasticsearch/xpack/logsdb/patternedtext/PatternedTextBlockLoader.java @@ -35,6 +35,6 @@ public AllReader reader(LeafReaderContext context) throws IOException { if (docValues == null) { return new ConstantNullsReader(); } - return new BlockDocValuesReader.BytesRefsFromSimpleBinary(docValues); + return new BlockDocValuesReader.BytesRefsFromBinary(docValues); } }