HSEARCH-4497 + HSEARCH-4505 Add compatibility with Elasticsearch 8.1.0

CONTRIBUTING.md

@@ -238,7 +238,8 @@ The following profiles are available:
   * `elasticsearch-7.10` for 7.10
   * `elasticsearch-7.11` for 7.11 ([not open-source starting with this version](https://opensource.org/node/1099))
   * `elasticsearch-7.12` for 7.12 to 7.17
-  * `elasticsearch-8.0` for 8.0+ (**the default**)
+  * `elasticsearch-8.0` for 8.0
+  * `elasticsearch-8.1` for 8.1+ (**the default**)
 * [OpenSearch](https://www.opensearch.org/)
   * `opensearch-1.0` for 1.0
   * `opensearch-1.2` for 1.2+

Jenkinsfile

@@ -256,7 +256,10 @@ stage('Configure') {
 							condition: TestCondition.ON_DEMAND),
 					new EsLocalBuildEnvironment(versionRange: '[7.12,8.0)', mavenProfile: 'elasticsearch-7.12',
 							condition: TestCondition.AFTER_MERGE),
-					new EsLocalBuildEnvironment(versionRange: '[8.0,8.x)', mavenProfile: 'elasticsearch-8.0',
+					// Not testing 8.0 because we know there are problems in 8.0.1 (see https://hibernate.atlassian.net/browse/HSEARCH-4497)
+					new EsLocalBuildEnvironment(versionRange: '[8.0,8.1)', mavenProfile: 'elasticsearch-8.0',
+							condition: TestCondition.ON_DEMAND),
+					new EsLocalBuildEnvironment(versionRange: '[8.1,8.x)', mavenProfile: 'elasticsearch-8.1',
 							condition: TestCondition.BEFORE_MERGE,
 							isDefault: true),
 

backend/elasticsearch/src/main/java/org/hibernate/search/backend/elasticsearch/dialect/impl/ElasticsearchDialectFactory.java

@@ -22,6 +22,7 @@
 import org.hibernate.search.backend.elasticsearch.dialect.protocol.impl.Elasticsearch67ProtocolDialect;
 import org.hibernate.search.backend.elasticsearch.dialect.protocol.impl.Elasticsearch70ProtocolDialect;
 import org.hibernate.search.backend.elasticsearch.dialect.protocol.impl.Elasticsearch80ProtocolDialect;
+import org.hibernate.search.backend.elasticsearch.dialect.protocol.impl.Elasticsearch81ProtocolDialect;
 import org.hibernate.search.backend.elasticsearch.dialect.protocol.impl.ElasticsearchProtocolDialect;
 import org.hibernate.search.backend.elasticsearch.logging.impl.Log;
 import org.hibernate.search.util.common.AssertionFailure;
@@ -125,7 +126,7 @@ else if ( major == 8 ) {
 		}
 		else {
 			log.unknownElasticsearchVersion( version );
-			return new Elasticsearch80ProtocolDialect();
+			return new Elasticsearch81ProtocolDialect();
 		}
 	}
 
@@ -165,10 +166,13 @@ private ElasticsearchProtocolDialect createProtocolDialectElasticV7(Elasticsearc
 	}
 
 	private ElasticsearchProtocolDialect createProtocolDialectElasticV8(ElasticsearchVersion version, int minor) {
-		if ( minor > 0 ) {
+		if ( minor > 1 ) {
 			log.unknownElasticsearchVersion( version );
 		}
-		return new Elasticsearch80ProtocolDialect();
+		else if ( minor == 0 ) {
+			return new Elasticsearch80ProtocolDialect();
+		}
+		return new Elasticsearch81ProtocolDialect();
 	}
 
 	private ElasticsearchProtocolDialect createProtocolDialectOpenSearch(ElasticsearchVersion version) {

backend/elasticsearch/src/main/java/org/hibernate/search/backend/elasticsearch/dialect/protocol/impl/Elasticsearch80ProtocolDialect.java

@@ -17,7 +17,7 @@
 import org.hibernate.search.backend.elasticsearch.work.builder.factory.impl.ElasticsearchWorkBuilderFactory;
 
 /**
- * The protocol dialect for Elasticsearch 8.0 and later 8.x.
+ * The protocol dialect for Elasticsearch 8.0.
  */
 public class Elasticsearch80ProtocolDialect implements ElasticsearchProtocolDialect {
 

backend/elasticsearch/src/main/java/org/hibernate/search/backend/elasticsearch/dialect/protocol/impl/Elasticsearch81ProtocolDialect.java

@@ -0,0 +1,43 @@
+/*
+ * Hibernate Search, full-text search for your domain model
+ *
+ * License: GNU Lesser General Public License (LGPL), version 2.1 or later
+ * See the lgpl.txt file in the root directory or <http://www.gnu.org/licenses/lgpl-2.1.html>.
+ */
+package org.hibernate.search.backend.elasticsearch.dialect.protocol.impl;
+
+import org.hibernate.search.backend.elasticsearch.gson.spi.GsonProvider;
+import org.hibernate.search.backend.elasticsearch.lowlevel.syntax.metadata.impl.Elasticsearch64IndexMetadataSyntax;
+import org.hibernate.search.backend.elasticsearch.lowlevel.syntax.metadata.impl.ElasticsearchIndexMetadataSyntax;
+import org.hibernate.search.backend.elasticsearch.lowlevel.syntax.search.impl.Elasticsearch81SearchSyntax;
+import org.hibernate.search.backend.elasticsearch.lowlevel.syntax.search.impl.ElasticsearchSearchSyntax;
+import org.hibernate.search.backend.elasticsearch.search.query.impl.Elasticsearch7SearchResultExtractorFactory;
+import org.hibernate.search.backend.elasticsearch.search.query.impl.ElasticsearchSearchResultExtractorFactory;
+import org.hibernate.search.backend.elasticsearch.work.builder.factory.impl.Elasticsearch7WorkBuilderFactory;
+import org.hibernate.search.backend.elasticsearch.work.builder.factory.impl.ElasticsearchWorkBuilderFactory;
+
+/**
+ * The protocol dialect for Elasticsearch 8.1 and later 8.x.
+ */
+public class Elasticsearch81ProtocolDialect implements ElasticsearchProtocolDialect {
+
+	@Override
+	public ElasticsearchIndexMetadataSyntax createIndexMetadataSyntax() {
+		return new Elasticsearch64IndexMetadataSyntax();
+	}
+
+	@Override
+	public ElasticsearchSearchSyntax createSearchSyntax() {
+		return new Elasticsearch81SearchSyntax();
+	}
+
+	@Override
+	public ElasticsearchWorkBuilderFactory createWorkBuilderFactory(GsonProvider gsonProvider) {
+		return new Elasticsearch7WorkBuilderFactory( gsonProvider );
+	}
+
+	@Override
+	public ElasticsearchSearchResultExtractorFactory createSearchResultExtractorFactory() {
+		return new Elasticsearch7SearchResultExtractorFactory();
+	}
+}

backend/elasticsearch/src/main/java/org/hibernate/search/backend/elasticsearch/lowlevel/syntax/search/impl/Elasticsearch7SearchSyntax.java

@@ -6,60 +6,17 @@
  */
 package org.hibernate.search.backend.elasticsearch.lowlevel.syntax.search.impl;
 
-import java.util.List;
-
-import org.hibernate.search.backend.elasticsearch.gson.impl.JsonAccessor;
-import org.hibernate.search.backend.elasticsearch.gson.impl.JsonArrayAccessor;
-
 import com.google.gson.JsonElement;
-import com.google.gson.JsonObject;
-import com.google.gson.JsonPrimitive;
+import com.google.gson.LongSerializationPolicy;
 
 /**
- * The search syntax for ES7.0 and later.
+ * The search syntax for ES7.0 to ES8.0.
  */
-public class Elasticsearch7SearchSyntax implements ElasticsearchSearchSyntax {
-
-	private static final JsonArrayAccessor DOCVALUE_FIELDS_ACCESSOR =
-			JsonAccessor.root().property( "docvalue_fields" ).asArray();
-
-	private static final JsonAccessor<JsonElement> NESTED_ACCESSOR = JsonAccessor.root().property( "nested" );
-	private static final JsonAccessor<JsonElement> PATH_ACCESSOR = JsonAccessor.root().property( "path" );
-	private static final JsonAccessor<JsonElement> FILTER_ACCESSOR = JsonAccessor.root().property( "filter" );
-	private static final JsonAccessor<Boolean> IGNORE_UNMAPPED_ACCESSOR =
-			JsonAccessor.root().property( "ignore_unmapped" ).asBoolean();
-
-	@Override
-	public String getTermAggregationOrderByTermToken() {
-		return "_key";
-	}
-
-	@Override
-	public void requestDocValues(JsonObject requestBody, JsonPrimitive fieldName) {
-		// The default format is the format defined in the mapping, which is what we want
-		DOCVALUE_FIELDS_ACCESSOR.addElementIfAbsent( requestBody, fieldName );
-	}
-
-	@Override
-	public void requestNestedSort(List<String> nestedPathHierarchy, JsonObject innerObject, JsonObject filterOrNull) {
-		JsonObject nextNestedObjectTarget = innerObject;
-		for ( int i = 0; i < nestedPathHierarchy.size(); i++ ) {
-			String nestedPath = nestedPathHierarchy.get( i );
-
-			JsonObject nestedObject = new JsonObject();
-			PATH_ACCESSOR.set( nestedObject, new JsonPrimitive( nestedPath ) );
-			NESTED_ACCESSOR.set( nextNestedObjectTarget, nestedObject );
-			if ( i == (nestedPathHierarchy.size() - 1) && filterOrNull != null ) {
-				FILTER_ACCESSOR.set( nestedObject, filterOrNull );
-			}
-
-			// the new api requires a recursion on the path hierarchy
-			nextNestedObjectTarget = nestedObject;
-		}
-	}
+public class Elasticsearch7SearchSyntax extends Elasticsearch81SearchSyntax {
 
 	@Override
-	public void requestGeoDistanceSortIgnoreUnmapped(JsonObject innerObject) {
-		IGNORE_UNMAPPED_ACCESSOR.set( innerObject, true );
+	public JsonElement encodeLongForAggregation(Long value) {
+		// Workaround for https://github.com/elastic/elasticsearch/issues/81529
+		return LongSerializationPolicy.STRING.serialize( value );
 	}
 }

backend/elasticsearch/src/main/java/org/hibernate/search/backend/elasticsearch/lowlevel/syntax/search/impl/Elasticsearch81SearchSyntax.java

@@ -0,0 +1,72 @@
+/*
+ * Hibernate Search, full-text search for your domain model
+ *
+ * License: GNU Lesser General Public License (LGPL), version 2.1 or later
+ * See the lgpl.txt file in the root directory or <http://www.gnu.org/licenses/lgpl-2.1.html>.
+ */
+package org.hibernate.search.backend.elasticsearch.lowlevel.syntax.search.impl;
+
+import java.util.List;
+
+import org.hibernate.search.backend.elasticsearch.gson.impl.JsonAccessor;
+import org.hibernate.search.backend.elasticsearch.gson.impl.JsonArrayAccessor;
+
+import com.google.gson.JsonElement;
+import com.google.gson.JsonNull;
+import com.google.gson.JsonObject;
+import com.google.gson.JsonPrimitive;
+
+/**
+ * The search syntax for ES7.0 and later.
+ */
+public class Elasticsearch81SearchSyntax implements ElasticsearchSearchSyntax {
+
+	private static final JsonArrayAccessor DOCVALUE_FIELDS_ACCESSOR =
+			JsonAccessor.root().property( "docvalue_fields" ).asArray();
+
+	private static final JsonAccessor<JsonElement> NESTED_ACCESSOR = JsonAccessor.root().property( "nested" );
+	private static final JsonAccessor<JsonElement> PATH_ACCESSOR = JsonAccessor.root().property( "path" );
+	private static final JsonAccessor<JsonElement> FILTER_ACCESSOR = JsonAccessor.root().property( "filter" );
+	private static final JsonAccessor<Boolean> IGNORE_UNMAPPED_ACCESSOR =
+			JsonAccessor.root().property( "ignore_unmapped" ).asBoolean();
+
+	@Override
+	public String getTermAggregationOrderByTermToken() {
+		return "_key";
+	}
+
+	@Override
+	public void requestDocValues(JsonObject requestBody, JsonPrimitive fieldName) {
+		// The default format is the format defined in the mapping, which is what we want
+		DOCVALUE_FIELDS_ACCESSOR.addElementIfAbsent( requestBody, fieldName );
+	}
+
+	@Override
+	public void requestNestedSort(List<String> nestedPathHierarchy, JsonObject innerObject, JsonObject filterOrNull) {
+		JsonObject nextNestedObjectTarget = innerObject;
+		for ( int i = 0; i < nestedPathHierarchy.size(); i++ ) {
+			String nestedPath = nestedPathHierarchy.get( i );
+
+			JsonObject nestedObject = new JsonObject();
+			PATH_ACCESSOR.set( nestedObject, new JsonPrimitive( nestedPath ) );
+			NESTED_ACCESSOR.set( nextNestedObjectTarget, nestedObject );
+			if ( i == (nestedPathHierarchy.size() - 1) && filterOrNull != null ) {
+				FILTER_ACCESSOR.set( nestedObject, filterOrNull );
+			}
+
+			// the new api requires a recursion on the path hierarchy
+			nextNestedObjectTarget = nestedObject;
+		}
+	}
+
+	@Override
+	public void requestGeoDistanceSortIgnoreUnmapped(JsonObject innerObject) {
+		IGNORE_UNMAPPED_ACCESSOR.set( innerObject, true );
+	}
+
+	@Override
+	public JsonElement encodeLongForAggregation(Long value) {
+		// https://github.com/elastic/elasticsearch/issues/81529 was solved in ES8.1
+		return value == null ? JsonNull.INSTANCE : new JsonPrimitive( value );
+	}
+}

backend/elasticsearch/src/main/java/org/hibernate/search/backend/elasticsearch/lowlevel/syntax/search/impl/ElasticsearchSearchSyntax.java

@@ -8,6 +8,7 @@
 
 import java.util.List;
 
+import com.google.gson.JsonElement;
 import com.google.gson.JsonObject;
 import com.google.gson.JsonPrimitive;
 
@@ -21,4 +22,6 @@ public interface ElasticsearchSearchSyntax {
 
 	void requestGeoDistanceSortIgnoreUnmapped(JsonObject innerObject);
 
+	JsonElement encodeLongForAggregation(Long value);
+
 }

backend/elasticsearch/src/main/java/org/hibernate/search/backend/elasticsearch/search/aggregation/impl/ElasticsearchRangeAggregation.java

@@ -153,7 +153,7 @@ public ElasticsearchRangeAggregation<F, K> build() {
 		private JsonElement convertToFieldValue(K value) {
 			try {
 				F converted = toFieldValueConverter.toDocumentValue( value, scope.toDocumentValueConvertContext() );
-				return codec.encodeForAggregation( converted );
+				return codec.encodeForAggregation( scope.searchSyntax(), converted );
 			}
 			catch (RuntimeException e) {
 				throw log.cannotConvertDslParameter( e.getMessage(), e, field.eventContext() );

backend/elasticsearch/src/main/java/org/hibernate/search/backend/elasticsearch/types/codec/impl/ElasticsearchFieldCodec.java

@@ -6,6 +6,8 @@
  */
 package org.hibernate.search.backend.elasticsearch.types.codec.impl;
 
+import org.hibernate.search.backend.elasticsearch.lowlevel.syntax.search.impl.ElasticsearchSearchSyntax;
+
 import com.google.gson.JsonElement;
 
 /**
@@ -26,10 +28,12 @@ default JsonElement encodeForMissing(F value) {
 	/**
 	 * Encodes a value for inclusion in an aggregation request.
 	 *
+	 *
+	 * @param searchSyntax The search syntax.
 	 * @param value The value to encode.
 	 * @return The encoded value.
 	 */
-	default JsonElement encodeForAggregation(F value) {
+	default JsonElement encodeForAggregation(ElasticsearchSearchSyntax searchSyntax, F value) {
 		return encode( value );
 	}
 

backend/elasticsearch/src/main/java/org/hibernate/search/backend/elasticsearch/types/codec/impl/ElasticsearchLongFieldCodec.java

@@ -7,11 +7,11 @@
 package org.hibernate.search.backend.elasticsearch.types.codec.impl;
 
 import org.hibernate.search.backend.elasticsearch.gson.impl.JsonElementTypes;
+import org.hibernate.search.backend.elasticsearch.lowlevel.syntax.search.impl.ElasticsearchSearchSyntax;
 
 import com.google.gson.JsonElement;
 import com.google.gson.JsonNull;
 import com.google.gson.JsonPrimitive;
-import com.google.gson.LongSerializationPolicy;
 
 public class ElasticsearchLongFieldCodec implements ElasticsearchFieldCodec<Long> {
 	public static final ElasticsearchLongFieldCodec INSTANCE = new ElasticsearchLongFieldCodec();
@@ -28,9 +28,8 @@ public JsonElement encode(Long value) {
 	}
 
 	@Override
-	public JsonElement encodeForAggregation(Long value) {
-		// Workaround for https://github.com/elastic/elasticsearch/issues/81529
-		return LongSerializationPolicy.STRING.serialize( value );
+	public JsonElement encodeForAggregation(ElasticsearchSearchSyntax searchSyntax, Long value) {
+		return searchSyntax.encodeLongForAggregation( value );
 	}
 
 	@Override

backend/elasticsearch/src/test/java/org/hibernate/search/backend/elasticsearch/dialect/impl/ElasticsearchDialectFactoryTest.java

@@ -23,6 +23,7 @@
 import org.hibernate.search.backend.elasticsearch.dialect.protocol.impl.Elasticsearch67ProtocolDialect;
 import org.hibernate.search.backend.elasticsearch.dialect.protocol.impl.Elasticsearch70ProtocolDialect;
 import org.hibernate.search.backend.elasticsearch.dialect.protocol.impl.Elasticsearch80ProtocolDialect;
+import org.hibernate.search.backend.elasticsearch.dialect.protocol.impl.Elasticsearch81ProtocolDialect;
 import org.hibernate.search.backend.elasticsearch.dialect.protocol.impl.ElasticsearchProtocolDialect;
 import org.hibernate.search.util.common.SearchException;
 import org.hibernate.search.util.impl.test.annotation.TestForIssue;
@@ -582,8 +583,8 @@ public void elastic_7_18_0() {
 	@TestForIssue(jiraKey = "HSEARCH-4475")
 	public void elastic_8() {
 		testSuccess(
-				ElasticsearchDistributionName.ELASTIC, "8", "8.0.0",
-				Elasticsearch8ModelDialect.class, Elasticsearch80ProtocolDialect.class
+				ElasticsearchDistributionName.ELASTIC, "8", "8.1.0",
+				Elasticsearch8ModelDialect.class, Elasticsearch81ProtocolDialect.class
 		);
 	}
 
@@ -606,28 +607,46 @@ public void elastic_8_0_0() {
 	}
 
 	@Test
-	@TestForIssue(jiraKey = "HSEARCH-4475")
+	@TestForIssue(jiraKey = "HSEARCH-4505")
 	public void elastic_8_1() {
-		testSuccessWithWarning(
+		testSuccess(
 				ElasticsearchDistributionName.ELASTIC, "8.1", "8.1.0",
-				Elasticsearch8ModelDialect.class, Elasticsearch80ProtocolDialect.class
+				Elasticsearch8ModelDialect.class, Elasticsearch81ProtocolDialect.class
 		);
 	}
 
 	@Test
-	@TestForIssue(jiraKey = "HSEARCH-4475")
+	@TestForIssue(jiraKey = "HSEARCH-4505")
 	public void elastic_8_1_0() {
-		testSuccessWithWarning(
+		testSuccess(
 				ElasticsearchDistributionName.ELASTIC, "8.1.0", "8.1.0",
-				Elasticsearch8ModelDialect.class, Elasticsearch80ProtocolDialect.class
+				Elasticsearch8ModelDialect.class, Elasticsearch81ProtocolDialect.class
+		);
+	}
+
+	@Test
+	@TestForIssue(jiraKey = "HSEARCH-4505")
+	public void elastic_8_2() {
+		testSuccessWithWarning(
+				ElasticsearchDistributionName.ELASTIC, "8.2", "8.2.0",
+				Elasticsearch8ModelDialect.class, Elasticsearch81ProtocolDialect.class
+		);
+	}
+
+	@Test
+	@TestForIssue(jiraKey = "HSEARCH-4505")
+	public void elastic_8_2_0() {
+		testSuccessWithWarning(
+				ElasticsearchDistributionName.ELASTIC, "8.2.0", "8.2.0",
+				Elasticsearch8ModelDialect.class, Elasticsearch81ProtocolDialect.class
 		);
 	}
 
 	@Test
 	public void elastic_9_0_0() {
 		testSuccessWithWarning(
 				ElasticsearchDistributionName.ELASTIC, "9.0.0", "9.0.0",
-				Elasticsearch8ModelDialect.class, Elasticsearch80ProtocolDialect.class
+				Elasticsearch8ModelDialect.class, Elasticsearch81ProtocolDialect.class
 		);
 	}