HSEARCH-3943 Projection on object fields: preserving the document structure in projections

backend/elasticsearch/src/main/java/org/hibernate/search/backend/elasticsearch/index/impl/IndexManagerBackendContext.java

@@ -156,7 +156,7 @@ public <H> ElasticsearchSearchQueryBuilder<H> createSearchQueryBuilder(
 			ElasticsearchSearchIndexScope<?> scope,
 			BackendSessionContext sessionContext,
 			SearchLoadingContextBuilder<?, ?, ?> loadingContextBuilder,
-			ElasticsearchSearchProjection<?, H> rootProjection) {
+			ElasticsearchSearchProjection<H> rootProjection) {
 		multiTenancyStrategy.documentIdHelper().checkTenantId( sessionContext.tenantIdentifier(), eventContext );
 		return new ElasticsearchSearchQueryBuilder<>(
 				link.getWorkBuilderFactory(), link.getSearchResultExtractorFactory(),

backend/elasticsearch/src/main/java/org/hibernate/search/backend/elasticsearch/logging/impl/Log.java

@@ -709,4 +709,21 @@ SearchException customIndexMappingErrorOnLoading(String filePath, String causeMe
 	SearchException customIndexMappingJsonSyntaxErrors(String filePath, String causeMessage, @Cause Exception cause,
 			@Param EventContext context);
 
+	@Message(id = ID_OFFSET + 154,
+			value = "Invalid context for projection on field '%1$s': the surrounding projection"
+					+ " is executed for each object in field '%2$s', which is not a parent of field '%1$s'."
+					+ " Check the structure of your projections.")
+	SearchException invalidContextForProjectionOnField(String absolutePath,
+			String objectFieldAbsolutePath);
+
+	@Message(id = ID_OFFSET + 155,
+			value = "Invalid cardinality for projection on field '%1$s': the projection is single-valued,"
+					+ " but this field is effectively multi-valued in this context,"
+					+ " because parent object field '%2$s' is multi-valued."
+					+ " Either call '.multi()' when you create the projection on field '%1$s',"
+					+ " or wrap that projection in an object projection like this:"
+					+ " 'f.object(\"%2$s\").from(<the projection on field %1$s>).as(...).multi()'.")
+	SearchException invalidSingleValuedProjectionOnValueFieldInMultiValuedObjectField(String absolutePath,
+			String objectFieldAbsolutePath);
+
 }

backend/elasticsearch/src/main/java/org/hibernate/search/backend/elasticsearch/scope/model/impl/ElasticsearchSearchIndexScopeImpl.java

@@ -192,6 +192,11 @@ public ElasticsearchSearchAggregationFactory aggregationFactory() {
 		return new ElasticsearchSearchAggregationFactoryImpl( SearchAggregationDslContext.root( this, predicateFactory() ) );
 	}
 
+	@Override
+	public ElasticsearchSearchIndexNodeContext field(String fieldPath) {
+		return super.field( fieldPath );
+	}
+
 	@Override
 	public Gson userFacingGson() {
 		return userFacingGson;

backend/elasticsearch/src/main/java/org/hibernate/search/backend/elasticsearch/search/common/impl/ElasticsearchMultiIndexSearchIndexCompositeNodeContext.java

@@ -19,7 +19,7 @@ public final class ElasticsearchMultiIndexSearchIndexCompositeNodeContext
 						ElasticsearchSearchIndexNodeContext
 				>
 		implements ElasticsearchSearchIndexCompositeNodeContext,
-		ElasticsearchSearchIndexCompositeNodeTypeContext {
+				ElasticsearchSearchIndexCompositeNodeTypeContext {
 
 	public ElasticsearchMultiIndexSearchIndexCompositeNodeContext(ElasticsearchSearchIndexScope<?> scope,
 			String absolutePath,

backend/elasticsearch/src/main/java/org/hibernate/search/backend/elasticsearch/search/common/impl/ElasticsearchMultiIndexSearchIndexValueFieldContext.java

@@ -49,6 +49,11 @@ public ElasticsearchSearchIndexCompositeNodeContext toComposite() {
 		return SearchIndexSchemaElementContextHelper.throwingToComposite( this );
 	}
 
+	@Override
+	public ElasticsearchSearchIndexCompositeNodeContext toObjectField() {
+		return SearchIndexSchemaElementContextHelper.throwingToObjectField( this );
+	}
+
 	@Override
 	public JsonPrimitive elasticsearchTypeAsJson() {
 		return fromTypeIfCompatible( ElasticsearchSearchIndexValueFieldTypeContext::elasticsearchTypeAsJson, Object::equals,

backend/elasticsearch/src/main/java/org/hibernate/search/backend/elasticsearch/search/common/impl/ElasticsearchSearchIndexNodeContext.java

@@ -14,6 +14,9 @@ public interface ElasticsearchSearchIndexNodeContext
 	@Override
 	ElasticsearchSearchIndexCompositeNodeContext toComposite();
 
+	@Override
+	ElasticsearchSearchIndexCompositeNodeContext toObjectField();
+
 	@Override
 	ElasticsearchSearchIndexValueFieldContext<?> toValueField();
 

backend/elasticsearch/src/main/java/org/hibernate/search/backend/elasticsearch/search/common/impl/ElasticsearchSearchIndexScope.java

@@ -25,6 +25,8 @@ public interface ElasticsearchSearchIndexScope<S extends ElasticsearchSearchInde
 	@Override
 	ElasticsearchSearchIndexNodeContext child(SearchIndexCompositeNodeContext<?> parent, String name);
 
+	ElasticsearchSearchIndexNodeContext field(String fieldPath);
+
 	Gson userFacingGson();
 
 	ElasticsearchSearchSyntax searchSyntax();

backend/elasticsearch/src/main/java/org/hibernate/search/backend/elasticsearch/search/projection/impl/AbstractElasticsearchProjection.java

@@ -11,7 +11,7 @@
 import org.hibernate.search.backend.elasticsearch.search.common.impl.ElasticsearchSearchIndexScope;
 import org.hibernate.search.engine.search.projection.spi.SearchProjectionBuilder;
 
-public abstract class AbstractElasticsearchProjection<E, P> implements ElasticsearchSearchProjection<E, P> {
+public abstract class AbstractElasticsearchProjection<P> implements ElasticsearchSearchProjection<P> {
 
 	protected final Set<String> indexNames;
 

backend/elasticsearch/src/main/java/org/hibernate/search/backend/elasticsearch/search/projection/impl/AccumulatingSourceExtractor.java

@@ -0,0 +1,118 @@
+/*
+ * Hibernate Search, full-text search for your domain model
+ *
+ * License: GNU Lesser General Public License (LGPL), version 2.1 or later
+ * See the lgpl.txt file in the root directory or <http://www.gnu.org/licenses/lgpl-2.1.html>.
+ */
+package org.hibernate.search.backend.elasticsearch.search.projection.impl;
+
+import java.util.Arrays;
+import java.util.stream.Collectors;
+
+import org.hibernate.search.backend.elasticsearch.gson.impl.JsonAccessor;
+import org.hibernate.search.backend.elasticsearch.gson.impl.JsonArrayAccessor;
+import org.hibernate.search.backend.elasticsearch.gson.impl.JsonElementTypes;
+import org.hibernate.search.backend.elasticsearch.gson.impl.UnexpectedJsonElementTypeException;
+import org.hibernate.search.engine.search.loading.spi.ProjectionHitMapper;
+import org.hibernate.search.engine.search.projection.spi.ProjectionAccumulator;
+
+import com.google.gson.JsonElement;
+import com.google.gson.JsonObject;
+
+abstract class AccumulatingSourceExtractor<E, V, A, P>
+		implements ElasticsearchSearchProjection.Extractor<A, P> {
+
+	static final JsonArrayAccessor REQUEST_SOURCE_ACCESSOR =
+			JsonAccessor.root().property( "_source" ).asArray();
+
+	private final String[] fieldPathComponents;
+	final ProjectionAccumulator<E, V, A, P> accumulator;
+
+	public AccumulatingSourceExtractor(String[] fieldPathComponents,
+			ProjectionAccumulator<E, V, A, P> accumulator) {
+		this.fieldPathComponents = fieldPathComponents;
+		this.accumulator = accumulator;
+	}
+
+	@Override
+	public final A extract(ProjectionHitMapper<?, ?> projectionHitMapper, JsonObject hit,
+			JsonObject source, ProjectionExtractContext context) {
+		A accumulated = accumulator.createInitial();
+		accumulated = collect( projectionHitMapper, hit, source, context, accumulated, 0 );
+		return accumulated;
+	}
+
+	private A collect(ProjectionHitMapper<?, ?> projectionHitMapper, JsonObject hit,
+			JsonObject parent, ProjectionExtractContext context,
+			A accumulated, int currentPathComponentIndex) {
+		if ( parent == null ) {
+			return accumulated;
+		}
+
+		JsonElement child = parent.get( fieldPathComponents[currentPathComponentIndex] );
+
+		if ( currentPathComponentIndex == ( fieldPathComponents.length - 1) ) {
+			// We reached the field we want to collect.
+			return collectTargetField( projectionHitMapper, hit, child, context, accumulated );
+		}
+		else {
+			// We just reached an intermediary object field leading to the field we want to collect.
+			if ( child == null || child.isJsonNull() ) {
+				// Not present
+				return accumulated;
+			}
+			else if ( child.isJsonArray() ) {
+				for ( JsonElement childElement : child.getAsJsonArray() ) {
+					JsonObject childElementAsObject = toJsonObject( childElement, currentPathComponentIndex );
+					accumulated = collect( projectionHitMapper, hit, childElementAsObject, context,
+							accumulated, currentPathComponentIndex + 1 );
+				}
+				return accumulated;
+			}
+			else {
+				JsonObject childAsObject = toJsonObject( child, currentPathComponentIndex );
+				return collect( projectionHitMapper, hit, childAsObject, context,
+						accumulated, currentPathComponentIndex + 1 );
+			}
+		}
+	}
+
+	private A collectTargetField(ProjectionHitMapper<?, ?> projectionHitMapper, JsonObject hit, JsonElement fieldValue,
+			ProjectionExtractContext context, A accumulated) {
+		if ( fieldValue == null ) {
+			// Not present
+			return accumulated;
+		}
+		else if ( fieldValue.isJsonNull() ) {
+			// Present, but null
+			return accumulator.accumulate( accumulated, extract( projectionHitMapper, hit, fieldValue, context ) );
+		}
+		else if ( fieldValue.isJsonArray() ) {
+			for ( JsonElement childElement : fieldValue.getAsJsonArray() ) {
+				accumulated = accumulator.accumulate( accumulated,
+						extract( projectionHitMapper, hit, childElement, context ) );
+			}
+			return accumulated;
+		}
+		else {
+			return accumulator.accumulate( accumulated, extract( projectionHitMapper, hit, fieldValue, context ) );
+		}
+	}
+
+	protected abstract E extract(ProjectionHitMapper<?, ?> projectionHitMapper, JsonObject hit,
+			JsonElement sourceElement, ProjectionExtractContext context);
+
+	private JsonObject toJsonObject(JsonElement childElement, int currentPathComponentIndex) {
+		if ( childElement == null || childElement.isJsonNull() ) {
+			return null;
+		}
+		if ( !JsonElementTypes.OBJECT.isInstance( childElement ) ) {
+			throw new UnexpectedJsonElementTypeException(
+					Arrays.stream( fieldPathComponents, 0, currentPathComponentIndex + 1 )
+							.collect( Collectors.joining( "." ) ),
+					JsonElementTypes.OBJECT, childElement
+			);
+		}
+		return childElement.getAsJsonObject();
+	}
+}

backend/elasticsearch/src/main/java/org/hibernate/search/backend/elasticsearch/search/projection/impl/ElasticsearchCompositeProjection.java

@@ -12,82 +12,123 @@
 import org.hibernate.search.engine.search.loading.spi.LoadingResult;
 import org.hibernate.search.engine.search.loading.spi.ProjectionHitMapper;
 import org.hibernate.search.engine.search.projection.SearchProjection;
+import org.hibernate.search.engine.search.projection.spi.ProjectionAccumulator;
 import org.hibernate.search.engine.search.projection.spi.ProjectionCompositor;
 import org.hibernate.search.engine.search.projection.spi.CompositeProjectionBuilder;
 
 import com.google.gson.JsonObject;
 
-class ElasticsearchCompositeProjection<E, V>
-		extends AbstractElasticsearchProjection<E, V> {
+/**
+ * A projection that composes the result of multiple inner projections into a single value.
+ * <p>
+ * Not to be confused with {@link ElasticsearchObjectProjection}.
+ *
+ * @param <E> The type of the temporary storage for component values.
+ * @param <V> The type of a single composed value.
+ * @param <A> The type of the temporary storage for accumulated values, before and after being composed.
+ * @param <P> The type of the final projection result representing an accumulation of composed values of type {@code V}.
+ */
+class ElasticsearchCompositeProjection<E, V, A, P>
+		extends AbstractElasticsearchProjection<P> {
 
+	private final ElasticsearchSearchProjection<?>[] inners;
 	private final ProjectionCompositor<E, V> compositor;
-	private final ElasticsearchSearchProjection<?, ?>[] inners;
+	private final ProjectionAccumulator<E, V, A, P> accumulator;
 
-	private ElasticsearchCompositeProjection(Builder<E, V> builder) {
+	public ElasticsearchCompositeProjection(Builder builder, ElasticsearchSearchProjection<?>[] inners,
+			ProjectionCompositor<E, V> compositor, ProjectionAccumulator<E, V, A, P> accumulator) {
 		super( builder.scope );
-		this.compositor = builder.compositor;
-		this.inners = builder.inners;
+		this.inners = inners;
+		this.compositor = compositor;
+		this.accumulator = accumulator;
 	}
 
 	@Override
 	public String toString() {
 		return getClass().getSimpleName() + "["
 				+ "inners=" + Arrays.toString( inners )
 				+ ", compositor=" + compositor
+				+ ", accumulator=" + accumulator
 				+ "]";
 	}
 
 	@Override
-	public void request(JsonObject requestBody, ProjectionRequestContext context) {
-		for ( ElasticsearchSearchProjection<?, ?> inner : inners ) {
-			inner.request( requestBody, context );
+	public Extractor<A, P> request(JsonObject requestBody, ProjectionRequestContext context) {
+		Extractor<?, ?>[] innerExtractors = new Extractor[inners.length];
+		for ( int i = 0; i < inners.length; i++ ) {
+			innerExtractors[i] = inners[i].request( requestBody, context );
 		}
+		return new CompositeExtractor( innerExtractors );
 	}
 
-	@Override
-	public E extract(ProjectionHitMapper<?, ?> projectionHitMapper, JsonObject hit,
-			ProjectionExtractContext context) {
-		E extractedData = compositor.createInitial();
+	private class CompositeExtractor implements Extractor<A, P> {
+		private final Extractor<?, ?>[] inners;
 
-		for ( int i = 0; i < inners.length; i++ ) {
-			Object extractedDataForInner = inners[i].extract( projectionHitMapper, hit, context );
-			extractedData = compositor.set( extractedData, i, extractedDataForInner );
+		private CompositeExtractor(Extractor<?, ?>[] inners) {
+			this.inners = inners;
 		}
 
-		return extractedData;
-	}
+		@Override
+		public String toString() {
+			return getClass().getSimpleName() + "["
+					+ "inners=" + Arrays.toString( inners )
+					+ ", compositor=" + compositor
+					+ ", accumulator=" + accumulator
+					+ "]";
+		}
 
-	@Override
-	public final V transform(LoadingResult<?, ?> loadingResult, E extractedData,
-			ProjectionTransformContext context) {
-		E transformedData = extractedData;
-		// Transform in-place
-		for ( int i = 0; i < inners.length; i++ ) {
-			Object extractedDataForInner = compositor.get( transformedData, i );
-			Object transformedDataForInner = ElasticsearchSearchProjection.transformUnsafe( inners[i], loadingResult,
-					extractedDataForInner, context );
-			transformedData = compositor.set( transformedData, i, transformedDataForInner );
+		@Override
+		public A extract(ProjectionHitMapper<?, ?> projectionHitMapper, JsonObject hit,
+				JsonObject source, ProjectionExtractContext context) {
+			A accumulated = accumulator.createInitial();
+
+			E components = compositor.createInitial();
+			for ( int i = 0; i < inners.length; i++ ) {
+				Object extractedDataForInner = inners[i].extract( projectionHitMapper, hit, source, context );
+				components = compositor.set( components, i, extractedDataForInner );
+			}
+			accumulated = accumulator.accumulate( accumulated, components );
+
+			return accumulated;
 		}
 
-		return compositor.finish( transformedData );
+		@Override
+		public final P transform(LoadingResult<?, ?> loadingResult, A accumulated,
+				ProjectionTransformContext context) {
+			for ( int i = 0; i < accumulator.size( accumulated ); i++ ) {
+				E transformedData = accumulator.get( accumulated, i );
+				// Transform in-place
+				for ( int j = 0; j < inners.length; j++ ) {
+					Object extractedDataForInner = compositor.get( transformedData, j );
+					Object transformedDataForInner = Extractor.transformUnsafe( inners[j],
+							loadingResult, extractedDataForInner, context );
+					transformedData = compositor.set( transformedData, j, transformedDataForInner );
+				}
+
+				accumulated = accumulator.transform( accumulated, i, compositor.finish( transformedData ) );
+			}
+			return accumulator.finish( accumulated );
+		}
 	}
 
-	static class Builder<E, V> implements CompositeProjectionBuilder<V> {
+	static class Builder implements CompositeProjectionBuilder {
 
 		private final ElasticsearchSearchIndexScope<?> scope;
-		private final ProjectionCompositor<E, V> compositor;
-		private final ElasticsearchSearchProjection<?, ?>[] inners;
 
-		Builder(ElasticsearchSearchIndexScope<?> scope, ProjectionCompositor<E, V> compositor,
-				ElasticsearchSearchProjection<?, ?> ... inners) {
+		Builder(ElasticsearchSearchIndexScope<?> scope) {
 			this.scope = scope;
-			this.compositor = compositor;
-			this.inners = inners;
 		}
 
 		@Override
-		public SearchProjection<V> build() {
-			return new ElasticsearchCompositeProjection<>( this );
+		public <E, V, P> SearchProjection<P> build(SearchProjection<?>[] inners, ProjectionCompositor<E, V> compositor,
+				ProjectionAccumulator.Provider<V, P> accumulatorProvider) {
+			ElasticsearchSearchProjection<?>[] typedInners =
+					new ElasticsearchSearchProjection<?>[ inners.length ];
+			for ( int i = 0; i < inners.length; i++ ) {
+				typedInners[i] = ElasticsearchSearchProjection.from( scope, inners[i] );
+			}
+			return new ElasticsearchCompositeProjection<>( this, typedInners,
+					compositor, accumulatorProvider.get() );
 		}
 	}
 }