diff --git a/modules/percolator/src/main/java/module-info.java b/modules/percolator/src/main/java/module-info.java index 51848d92a5e6c..392d5d959be4e 100644 --- a/modules/percolator/src/main/java/module-info.java +++ b/modules/percolator/src/main/java/module-info.java @@ -16,4 +16,5 @@ requires org.apache.lucene.memory; requires org.apache.lucene.queries; requires org.apache.lucene.sandbox; + requires org.elasticsearch.logging; } diff --git a/modules/percolator/src/main/java/org/elasticsearch/percolator/PercolateQueryBuilder.java b/modules/percolator/src/main/java/org/elasticsearch/percolator/PercolateQueryBuilder.java index 79f5dd06f8d2d..2ce0f1af5b0c4 100644 --- a/modules/percolator/src/main/java/org/elasticsearch/percolator/PercolateQueryBuilder.java +++ b/modules/percolator/src/main/java/org/elasticsearch/percolator/PercolateQueryBuilder.java @@ -36,6 +36,7 @@ import org.elasticsearch.ResourceNotFoundException; import org.elasticsearch.TransportVersion; import org.elasticsearch.action.get.GetRequest; +import org.elasticsearch.common.CheckedSupplier; import org.elasticsearch.common.bytes.BytesReference; import org.elasticsearch.common.io.stream.InputStreamStreamInput; import org.elasticsearch.common.io.stream.NamedWriteableAwareStreamInput; @@ -62,6 +63,11 @@ import org.elasticsearch.index.query.SearchExecutionContext; import org.elasticsearch.indices.breaker.CircuitBreakerService; import org.elasticsearch.indices.breaker.NoneCircuitBreakerService; +import org.elasticsearch.logging.LogManager; +import org.elasticsearch.logging.Logger; +import org.elasticsearch.search.lookup.Source; +import org.elasticsearch.search.lookup.SourceFilter; +import org.elasticsearch.search.lookup.SourceProvider; import org.elasticsearch.xcontent.ConstructingObjectParser; import org.elasticsearch.xcontent.NamedXContentRegistry; import org.elasticsearch.xcontent.ParseField; @@ -85,6 +91,8 @@ import static org.elasticsearch.xcontent.ConstructingObjectParser.optionalConstructorArg; public class PercolateQueryBuilder extends AbstractQueryBuilder { + private static final Logger LOGGER = LogManager.getLogger(PercolateQueryBuilder.class); + public static final String NAME = "percolate"; static final ParseField DOCUMENT_FIELD = new ParseField("document"); @@ -540,34 +548,32 @@ static PercolateQuery.QueryStore createStore(MappedFieldType queryBuilderFieldTy return docId -> { if (binaryDocValues.advanceExact(docId)) { BytesRef qbSource = binaryDocValues.binaryValue(); - try ( - InputStream in = new ByteArrayInputStream(qbSource.bytes, qbSource.offset, qbSource.length); - StreamInput input = new NamedWriteableAwareStreamInput(new InputStreamStreamInput(in, qbSource.length), registry) - ) { - // Query builder's content is stored via BinaryFieldMapper, which has a custom encoding - // to encode multiple binary values into a single binary doc values field. - // This is the reason we need to first read the number of values and - // then the length of the field value in bytes. - int numValues = input.readVInt(); - assert numValues == 1; - int valueLength = input.readVInt(); - assert valueLength > 0; - - TransportVersion transportVersion; - if (indexVersion.before(IndexVersions.V_8_8_0)) { - transportVersion = TransportVersion.fromId(indexVersion.id()); - } else { - transportVersion = TransportVersion.readVersion(input); + QueryBuilder queryBuilder = readQueryBuilder(qbSource, registry, indexVersion, () -> { + // query builder is written in an incompatible format, fall-back to reading it from source + if (context.isSourceEnabled() == false) { + throw new ElasticsearchException( + "Unable to read percolator query. Original transport version is incompatible and source is " + + "unavailable on index [{}].", + context.index().getName() + ); } - // set the transportversion here - only read vints so far, so can change the version freely at this point - input.setTransportVersion(transportVersion); - - QueryBuilder queryBuilder = input.readNamedWriteable(QueryBuilder.class); - assert in.read() == -1; - queryBuilder = Rewriteable.rewrite(queryBuilder, context); - return queryBuilder.toQuery(context); - } - + LOGGER.warn( + "Reading percolator query from source. For best performance, reindexing of index [{}] is required.", + context.index().getName() + ); + SourceProvider sourceProvider = context.createSourceProvider(new SourceFilter(null, null)); + Source source = sourceProvider.getSource(ctx, docId); + SourceToParse sourceToParse = new SourceToParse( + String.valueOf(docId), + source.internalSourceRef(), + source.sourceContentType() + ); + + return context.parseDocument(sourceToParse).rootDoc().getBinaryValue(queryBuilderFieldType.name()); + }); + + queryBuilder = Rewriteable.rewrite(queryBuilder, context); + return queryBuilder.toQuery(context); } else { return null; } @@ -575,6 +581,48 @@ static PercolateQuery.QueryStore createStore(MappedFieldType queryBuilderFieldTy }; } + private static QueryBuilder readQueryBuilder( + BytesRef bytesRef, + NamedWriteableRegistry registry, + IndexVersion indexVersion, + CheckedSupplier fallbackSource + ) throws IOException { + try ( + InputStream in = new ByteArrayInputStream(bytesRef.bytes, bytesRef.offset, bytesRef.length); + StreamInput input = new NamedWriteableAwareStreamInput(new InputStreamStreamInput(in, bytesRef.length), registry) + ) { + // Query builder's content is stored via BinaryFieldMapper, which has a custom encoding + // to encode multiple binary values into a single binary doc values field. + // This is the reason we need to first read the number of values and + // then the length of the field value in bytes. + int numValues = input.readVInt(); + assert numValues == 1; + int valueLength = input.readVInt(); + assert valueLength > 0; + + TransportVersion transportVersion; + if (indexVersion.before(IndexVersions.V_8_8_0)) { + transportVersion = TransportVersion.fromId(indexVersion.id()); + } else { + transportVersion = TransportVersion.readVersion(input); + } + + QueryBuilder queryBuilder; + + if (TransportVersion.isCompatible(transportVersion) || fallbackSource == null) { + // set the transportversion here - only read vints so far, so can change the version freely at this point + input.setTransportVersion(transportVersion); + queryBuilder = input.readNamedWriteable(QueryBuilder.class); + assert in.read() == -1; + } else { + // incompatible transport version, try the fallback + queryBuilder = readQueryBuilder(fallbackSource.get(), registry, indexVersion, null); + } + + return queryBuilder; + } + } + static SearchExecutionContext wrap(SearchExecutionContext delegate) { return new SearchExecutionContext(delegate) { diff --git a/qa/full-cluster-restart/src/javaRestTest/java/org/elasticsearch/upgrades/QueryBuilderBWCIT.java b/qa/full-cluster-restart/src/javaRestTest/java/org/elasticsearch/upgrades/QueryBuilderBWCIT.java index 701bf83e5e6eb..670b90dec6afb 100644 --- a/qa/full-cluster-restart/src/javaRestTest/java/org/elasticsearch/upgrades/QueryBuilderBWCIT.java +++ b/qa/full-cluster-restart/src/javaRestTest/java/org/elasticsearch/upgrades/QueryBuilderBWCIT.java @@ -11,18 +11,10 @@ import com.carrotsearch.randomizedtesting.annotations.Name; -import org.elasticsearch.TransportVersion; -import org.elasticsearch.Version; import org.elasticsearch.client.Request; import org.elasticsearch.client.Response; import org.elasticsearch.common.Strings; -import org.elasticsearch.common.io.stream.InputStreamStreamInput; -import org.elasticsearch.common.io.stream.NamedWriteableAwareStreamInput; -import org.elasticsearch.common.io.stream.NamedWriteableRegistry; -import org.elasticsearch.common.io.stream.StreamInput; -import org.elasticsearch.common.settings.Settings; import org.elasticsearch.common.unit.Fuzziness; -import org.elasticsearch.core.UpdateForV10; import org.elasticsearch.index.query.BoolQueryBuilder; import org.elasticsearch.index.query.ConstantScoreQueryBuilder; import org.elasticsearch.index.query.DisMaxQueryBuilder; @@ -37,33 +29,23 @@ import org.elasticsearch.index.query.SpanTermQueryBuilder; import org.elasticsearch.index.query.functionscore.FunctionScoreQueryBuilder; import org.elasticsearch.index.query.functionscore.RandomScoreFunctionBuilder; -import org.elasticsearch.search.SearchModule; import org.elasticsearch.test.cluster.ElasticsearchCluster; import org.elasticsearch.test.cluster.local.LocalClusterConfigProvider; import org.elasticsearch.test.cluster.local.distribution.DistributionType; import org.elasticsearch.xcontent.XContentBuilder; import org.junit.ClassRule; -import java.io.ByteArrayInputStream; -import java.io.InputStream; import java.util.ArrayList; -import java.util.Base64; -import java.util.Collections; import java.util.List; -import java.util.Map; -import static org.elasticsearch.cluster.ClusterState.VERSION_INTRODUCING_TRANSPORT_VERSIONS; import static org.elasticsearch.xcontent.XContentFactory.jsonBuilder; /** * An integration test that tests whether percolator queries stored in older supported ES version can still be read by the * current ES version. Percolator queries are stored in the binary format in a dedicated doc values field (see - * PercolatorFieldMapper#createQueryBuilderField(...) method). Using the query builders writable contract. This test - * does best effort verifying that we don't break bwc for query builders between the first previous major version and - * the latest current major release. - * - * The queries to test are specified in json format, which turns out to work because we tend break here rarely. If the - * json format of a query being tested here then feel free to change this. + * PercolatorFieldMapper#createQueryBuilderField(...) method). We don't attempt to assert anything on results here, simply executing + * a percolator query will force deserialization of the old query builder. This also verifies that our fallback compatibility + * functionality is working correctly, otherwise the search request will throw an exception. */ public class QueryBuilderBWCIT extends ParameterizedFullClusterRestartTestCase { private static final List CANDIDATES = new ArrayList<>(); @@ -227,43 +209,20 @@ public void testQueryBuilderBWC() throws Exception { assertEquals(201, rsp.getStatusLine().getStatusCode()); } } else { - NamedWriteableRegistry registry = new NamedWriteableRegistry( - new SearchModule(Settings.EMPTY, Collections.emptyList()).getNamedWriteables() - ); - - for (int i = 0; i < CANDIDATES.size(); i++) { - QueryBuilder expectedQueryBuilder = (QueryBuilder) CANDIDATES.get(i)[1]; - Request request = new Request("GET", "/" + index + "/_search"); - request.setJsonEntity(Strings.format(""" - {"query": {"ids": {"values": ["%s"]}}, "docvalue_fields": [{"field":"query.query_builder_field"}]} - """, i)); - Response rsp = client().performRequest(request); - assertEquals(200, rsp.getStatusLine().getStatusCode()); - var hitRsp = (Map) ((List) ((Map) responseAsMap(rsp).get("hits")).get("hits")).get(0); - String queryBuilderStr = (String) ((List) ((Map) hitRsp.get("fields")).get("query.query_builder_field")).get(0); - byte[] qbSource = Base64.getDecoder().decode(queryBuilderStr); - try ( - InputStream in = new ByteArrayInputStream(qbSource, 0, qbSource.length); - StreamInput input = new NamedWriteableAwareStreamInput(new InputStreamStreamInput(in), registry) - ) { - @UpdateForV10(owner = UpdateForV10.Owner.SEARCH_FOUNDATIONS) // won't need to read <8.8 data anymore - boolean originalClusterHasTransportVersion = parseLegacyVersion(getOldClusterVersion()).map( - v -> v.onOrAfter(VERSION_INTRODUCING_TRANSPORT_VERSIONS) - ).orElse(true); - TransportVersion transportVersion; - if (originalClusterHasTransportVersion == false) { - transportVersion = TransportVersion.fromId( - parseLegacyVersion(getOldClusterVersion()).map(Version::id).orElse(TransportVersion.minimumCompatible().id()) - ); - } else { - transportVersion = TransportVersion.readVersion(input); + Request request = new Request("GET", "/" + index + "/_search"); + request.setJsonEntity(""" + { + "query": { + "percolate": { + "field": "query", + "document": { + "foo": "bar" + } } - input.setTransportVersion(transportVersion); - QueryBuilder queryBuilder = input.readNamedWriteable(QueryBuilder.class); - assert in.read() == -1; - assertEquals(expectedQueryBuilder, queryBuilder); - } - } + } + }"""); + Response rsp = client().performRequest(request); + assertEquals(200, rsp.getStatusLine().getStatusCode()); } } }