HSEARCH-4591 Allow object projections on single-valued, flattened obj…

…ect fields with the Lucene backend

backend/lucene/src/main/java/org/hibernate/search/backend/lucene/logging/impl/Log.java

@@ -614,4 +614,10 @@ SearchException invalidSingleValuedProjectionOnValueFieldInMultiValuedObjectFiel
 					+ " The document was probably indexed with a different configuration: full reindexing is necessary.")
 	SearchException unexpectedMappedTypeNameForByMappedTypeProjection(String typeName, Set<String> expectedTypeNames);
 
+	@Message(value = "This multi-valued field has a 'FLATTENED' structure,"
+			+ " which means the structure of objects is not preserved upon indexing,"
+			+ " making object projections impossible."
+			+ " Try setting the field structure to 'NESTED' and reindexing all your data.")
+	String missingSupportHintForObjectProjectionOnMultiValuedFlattenedObjectNode();
+
 }

backend/lucene/src/main/java/org/hibernate/search/backend/lucene/search/projection/impl/LuceneObjectProjection.java

@@ -14,13 +14,21 @@
 import org.hibernate.search.backend.lucene.search.common.impl.AbstractLuceneCompositeNodeSearchQueryElementFactory;
 import org.hibernate.search.backend.lucene.search.common.impl.LuceneSearchIndexCompositeNodeContext;
 import org.hibernate.search.backend.lucene.search.common.impl.LuceneSearchIndexScope;
+import org.hibernate.search.backend.lucene.search.predicate.impl.LuceneSearchPredicate;
+import org.hibernate.search.backend.lucene.search.predicate.impl.PredicateRequestContext;
 import org.hibernate.search.engine.search.loading.spi.LoadingResult;
+import org.hibernate.search.engine.search.predicate.spi.PredicateTypeKeys;
 import org.hibernate.search.engine.search.projection.SearchProjection;
 import org.hibernate.search.engine.search.projection.spi.CompositeProjectionBuilder;
 import org.hibernate.search.engine.search.projection.spi.ProjectionAccumulator;
 import org.hibernate.search.engine.search.projection.spi.ProjectionCompositor;
+import org.hibernate.search.engine.search.projection.spi.ProjectionTypeKeys;
+import org.hibernate.search.util.common.SearchException;
 
 import org.apache.lucene.index.LeafReaderContext;
+import org.apache.lucene.search.Query;
+import org.apache.lucene.search.join.QueryBitSetProducer;
+import org.apache.lucene.util.BitSet;
 
 /**
  * A projection that yields one composite value per object in a given object field.
@@ -35,6 +43,8 @@ public class LuceneObjectProjection<E, V, P>
 		extends AbstractLuceneProjection<P> {
 
 	private final String absoluteFieldPath;
+	private final boolean nested;
+	private final Query filter;
 	private final String nestedDocumentPath;
 	private final String requiredContextAbsoluteFieldPath;
 	private final LuceneSearchProjection<?>[] inners;
@@ -45,6 +55,8 @@ public LuceneObjectProjection(Builder builder, LuceneSearchProjection<?>[] inner
 			ProjectionCompositor<E, V> compositor, ProjectionAccumulator.Provider<V, P> accumulatorProvider) {
 		super( builder.scope );
 		this.absoluteFieldPath = builder.objectField.absolutePath();
+		this.nested = builder.objectField.type().nested();
+		this.filter = builder.filter;
 		this.nestedDocumentPath = builder.objectField.nestedDocumentPath();
 		this.requiredContextAbsoluteFieldPath = accumulatorProvider.isSingleValued()
 				? builder.objectField.closestMultiValuedParentAbsolutePath() : null;
@@ -64,7 +76,14 @@ public String toString() {
 
 	@Override
 	public Extractor<?, P> request(ProjectionRequestContext context) {
-		ProjectionRequestContext innerContext = context.forField( absoluteFieldPath );
+		ProjectionRequestContext innerContext;
+		if ( nested ) {
+			innerContext = context.forField( absoluteFieldPath );
+		}
+		else {
+			context.checkValidField( absoluteFieldPath );
+			innerContext = context;
+		}
 		if ( requiredContextAbsoluteFieldPath != null
 				&& !requiredContextAbsoluteFieldPath.equals( context.absoluteCurrentFieldPath() ) ) {
 			throw log.invalidSingleValuedProjectionOnValueFieldInMultiValuedObjectField(
@@ -113,22 +132,35 @@ public Values<A> values(ProjectionExtractContext context) {
 
 		private class ObjectFieldValues extends AbstractNestingAwareAccumulatingValues<E, A> {
 			private final Values<?>[] inners;
+			private final QueryBitSetProducer filterBitSetProducer;
+
+			private BitSet filterMatchedBitSet;
 
 			private ObjectFieldValues(TopDocsDataCollectorExecutionContext context, Values<?>[] inners) {
 				super( contextAbsoluteFieldPath, nestedDocumentPath, ObjectFieldExtractor.this.accumulator, context );
 				this.inners = inners;
+				this.filterBitSetProducer = filter == null ? null : new QueryBitSetProducer( filter );
+
 			}
 
 			@Override
 			public void context(LeafReaderContext context) throws IOException {
 				super.context( context );
+				if ( filterBitSetProducer != null ) {
+					filterMatchedBitSet = filterBitSetProducer.getBitSet( context );
+				}
 				for ( Values<?> inner : inners ) {
 					inner.context( context );
 				}
 			}
 
 			@Override
 			protected A accumulate(A accumulated, int docId) throws IOException {
+				if ( filterBitSetProducer != null && ( filterMatchedBitSet == null || !filterMatchedBitSet.get( docId ) ) ) {
+					// The object didn't match the given filter: act as if it didn't exist.
+					// Note that filters are used to detect flattened objects that were null upon indexing.
+					return accumulated;
+				}
 				E components = compositor.createInitial();
 				for ( int i = 0; i < inners.length; i++ ) {
 					Object extractedDataForInner = inners[i].get( docId );
@@ -160,18 +192,38 @@ public static class Factory
 			extends AbstractLuceneCompositeNodeSearchQueryElementFactory<Builder> {
 		@Override
 		public Builder create(LuceneSearchIndexScope<?> scope, LuceneSearchIndexCompositeNodeContext node) {
-			return new Builder( scope, node );
+			Query filter = null;
+			if ( !node.type().nested() ) {
+				if ( node.multiValued() ) {
+					throw node.cannotUseQueryElement( ProjectionTypeKeys.OBJECT,
+							log.missingSupportHintForObjectProjectionOnMultiValuedFlattenedObjectNode(), null );
+				}
+				try {
+					filter = LuceneSearchPredicate.from( scope, node.queryElement( PredicateTypeKeys.EXISTS, scope ).build() )
+							.toQuery( PredicateRequestContext.root() );
+				}
+				catch (SearchException e) {
+					throw node.cannotUseQueryElement( ProjectionTypeKeys.OBJECT, e.getMessage(), e );
+				}
+			}
+			if ( node.multiValued() && !node.type().nested() ) {
+				throw node.cannotUseQueryElement( ProjectionTypeKeys.OBJECT,
+						log.missingSupportHintForObjectProjectionOnMultiValuedFlattenedObjectNode(), null );
+			}
+			return new Builder( scope, node, filter );
 		}
 	}
 
 	static class Builder implements CompositeProjectionBuilder {
 
 		private final LuceneSearchIndexScope<?> scope;
 		private final LuceneSearchIndexCompositeNodeContext objectField;
+		private final Query filter;
 
-		Builder(LuceneSearchIndexScope<?> scope, LuceneSearchIndexCompositeNodeContext objectField) {
+		Builder(LuceneSearchIndexScope<?> scope, LuceneSearchIndexCompositeNodeContext objectField, Query filter) {
 			this.scope = scope;
 			this.objectField = objectField;
+			this.filter = filter;
 		}
 
 		@Override

backend/lucene/src/main/java/org/hibernate/search/backend/lucene/types/impl/LuceneIndexCompositeNodeType.java

@@ -34,9 +34,9 @@ public static class Builder
 		public Builder(ObjectStructure objectStructure) {
 			super( objectStructure );
 			queryElementFactory( PredicateTypeKeys.EXISTS, LuceneObjectExistsPredicate.Factory.INSTANCE );
+			queryElementFactory( ProjectionTypeKeys.OBJECT, new LuceneObjectProjection.Factory() );
 			if ( ObjectStructure.NESTED.equals( objectStructure ) ) {
 				queryElementFactory( PredicateTypeKeys.NESTED, LuceneNestedPredicate.Factory.INSTANCE );
-				queryElementFactory( ProjectionTypeKeys.OBJECT, new LuceneObjectProjection.Factory() );
 			}
 		}
 

engine/src/main/java/org/hibernate/search/engine/backend/document/model/spi/AbstractIndexNode.java

@@ -11,6 +11,7 @@
 import org.hibernate.search.engine.search.common.spi.SearchIndexScope;
 import org.hibernate.search.engine.search.common.spi.SearchQueryElementFactory;
 import org.hibernate.search.engine.search.common.spi.SearchQueryElementTypeKey;
+import org.hibernate.search.util.common.SearchException;
 import org.hibernate.search.util.common.reporting.EventContext;
 
 public abstract class AbstractIndexNode<
@@ -42,6 +43,11 @@ public final <T> T queryElement(SearchQueryElementTypeKey<T> key, SC scope) {
 		return helper().queryElement( key, factory, scope, self() );
 	}
 
+	@Override
+	public SearchException cannotUseQueryElement(SearchQueryElementTypeKey<?> key, String hint, Exception causeOrNull) {
+		return helper().cannotUseQueryElement( key, self(), hint, causeOrNull );
+	}
+
 	abstract SearchIndexSchemaElementContextHelper helper();
 
 }

engine/src/main/java/org/hibernate/search/engine/logging/impl/Log.java

@@ -468,10 +468,10 @@ SearchException inconsistentSupportForQueryElement(SearchQueryElementTypeKey<?>
 			value = "Attribute '%1$s' differs: '%2$s' vs. '%3$s'.")
 	SearchException differentAttribute(String attributeName, Object component1, Object component2);
 
-	@Message(id = ID_OFFSET + 104, value = "Cannot use '%2$s' on %1$s. %3$s")
+	@Message(id = ID_OFFSET + 104, value = "Cannot use '%2$s' on %1$s: %3$s")
 	SearchException cannotUseQueryElementForIndexNode(
 			@FormatWith(EventContextNoPrefixFormatter.class) EventContext elementContext,
-			SearchQueryElementTypeKey<?> key, String hint, @Param EventContext context);
+			SearchQueryElementTypeKey<?> key, String hint, @Param EventContext context, @Cause Exception cause);
 
 	@Message(value = "Make sure the field is marked as searchable/sortable/projectable/aggregable (whichever is relevant)."
 			+ " If it already is, then '%1$s' is not available for fields of this type.")
@@ -482,12 +482,6 @@ SearchException cannotUseQueryElementForIndexNode(
 			+ " If you are trying to use another feature, it probably isn't available for this field.")
 	String missingSupportHintForCompositeNode();
 
-	@Message(id = ID_OFFSET + 105, value = "Cannot use '%2$s' on %1$s: %3$s")
-	SearchException cannotUseQueryElementForIndexElementBecauseCreationException(
-			@FormatWith(EventContextNoPrefixFormatter.class) EventContext elementContext,
-			SearchQueryElementTypeKey<?> key, String causeMessage, @Cause SearchException cause,
-			@Param EventContext elementContextAsParam);
-
 	@Message(id = ID_OFFSET + 106,
 			value = "'%1$s' can be used in some of the targeted indexes, but not all of them. %2$s")
 	SearchException partialSupportForQueryElement(SearchQueryElementTypeKey<?> key, String hint);

engine/src/main/java/org/hibernate/search/engine/search/common/spi/AbstractMultiIndexSearchIndexNodeContext.java

@@ -112,6 +112,11 @@ public final <T> T queryElement(SearchQueryElementTypeKey<T> key, SC scope) {
 		return helper().queryElement( key, factory, scope, self() );
 	}
 
+	@Override
+	public SearchException cannotUseQueryElement(SearchQueryElementTypeKey<?> key, String hint, Exception causeOrNull) {
+		return helper().cannotUseQueryElement( key, self(), hint, causeOrNull );
+	}
+
 	abstract SearchIndexSchemaElementContextHelper helper();
 
 	@Override

engine/src/main/java/org/hibernate/search/engine/search/common/spi/SearchIndexNodeContext.java

@@ -8,6 +8,7 @@
 
 import java.util.List;
 
+import org.hibernate.search.util.common.SearchException;
 import org.hibernate.search.util.common.reporting.EventContext;
 import org.hibernate.search.util.common.reporting.spi.EventContextProvider;
 
@@ -59,4 +60,6 @@ default String nestedDocumentPath() {
 
 	<T> T queryElement(SearchQueryElementTypeKey<T> key, SC searchContext);
 
+	SearchException cannotUseQueryElement(SearchQueryElementTypeKey<?> key, String hint, Exception causeOrNull);
+
 }

engine/src/main/java/org/hibernate/search/engine/search/common/spi/SearchIndexSchemaElementContextHelper.java

@@ -73,21 +73,24 @@ public String partialSupportHint() {
 		}
 	};
 
+	public <T, SC extends SearchIndexScope<?>, N extends SearchIndexNodeContext<SC>>
+			SearchException cannotUseQueryElement(SearchQueryElementTypeKey<T> key, N node, String hint,
+			Exception causeOrNull) {
+		throw log.cannotUseQueryElementForIndexNode( node.relativeEventContext(), key,
+				hint, node.eventContext(), causeOrNull );
+	}
+
 	public <T, SC extends SearchIndexScope<?>, N extends SearchIndexNodeContext<SC>>
 			T queryElement(SearchQueryElementTypeKey<T> key,
 					SearchQueryElementFactory<? extends T, ? super SC, ? super N> factory, SC scope, N node) {
 		if ( factory == null ) {
-			throw log.cannotUseQueryElementForIndexNode( node.relativeEventContext(), key,
-					missingSupportHint( key ), node.eventContext()
-			);
+			throw cannotUseQueryElement( key, node, missingSupportHint( key ), null );
 		}
 		try {
 			return factory.create( scope, node );
 		}
 		catch (SearchException e) {
-			throw log.cannotUseQueryElementForIndexElementBecauseCreationException( node.relativeEventContext(),
-					key, e.getMessage(), e, node.eventContext()
-			);
+			throw cannotUseQueryElement( key, node, e.getMessage(), e );
 		}
 	}
 

integrationtest/backend/elasticsearch/src/test/java/org/hibernate/search/integrationtest/backend/elasticsearch/testsupport/util/ElasticsearchTckBackendFeatures.java

@@ -211,7 +211,7 @@ public boolean supportsFieldSortWhenNestedFieldMissingInSomeTargetIndexes() {
 	}
 
 	@Override
-	public boolean reliesOnNestedDocumentsForObjectProjection() {
+	public boolean reliesOnNestedDocumentsForMultiValuedObjectProjection() {
 		return false;
 	}
 }

integrationtest/backend/lucene/src/test/java/org/hibernate/search/integrationtest/backend/lucene/testsupport/util/LuceneTckBackendFeatures.java

@@ -6,6 +6,7 @@
  */
 package org.hibernate.search.integrationtest.backend.lucene.testsupport.util;
 
+import org.hibernate.search.engine.backend.types.ObjectStructure;
 import org.hibernate.search.integrationtest.backend.tck.testsupport.util.TckBackendFeatures;
 
 class LuceneTckBackendFeatures extends TckBackendFeatures {
@@ -27,7 +28,15 @@ public boolean fieldsProjectableByDefault() {
 	}
 
 	@Override
-	public boolean reliesOnNestedDocumentsForObjectProjection() {
+	public boolean projectionPreservesEmptySingleValuedObject(ObjectStructure structure) {
+		// For single-valued, flattened object fields,
+		// we cannot distinguish between an empty object (non-null object, but no subfield carries a value)
+		// and an empty object.
+		return ObjectStructure.NESTED.equals( structure );
+	}
+
+	@Override
+	public boolean reliesOnNestedDocumentsForMultiValuedObjectProjection() {
 		return true;
 	}
 }

integrationtest/backend/tck/src/main/java/org/hibernate/search/integrationtest/backend/tck/ObjectStructureIT.java

@@ -170,9 +170,9 @@ public void search_error_nonNestedField() {
 			scope.predicate().nested( "flattenedObject" )
 		)
 				.isInstanceOf( SearchException.class )
-				.hasMessageContainingAll( "Cannot use 'predicate:nested' on field 'flattenedObject'.",
-						"Some object field features require a nested structure; "
-								+ "try setting the field structure to 'NESTED' and reindexing all your data" );
+				.hasMessageContainingAll( "Cannot use 'predicate:nested' on field 'flattenedObject'",
+						"Some object field features require a nested structure",
+						"try setting the field structure to 'NESTED' and reindexing all your data" );
 	}
 
 	@Test
@@ -183,7 +183,7 @@ public void search_error_nonObjectField() {
 				scope.predicate().nested( "flattenedObject.string" )
 		)
 				.isInstanceOf( SearchException.class )
-				.hasMessageContainingAll( "Cannot use 'predicate:nested' on field 'flattenedObject.string'.",
+				.hasMessageContainingAll( "Cannot use 'predicate:nested' on field 'flattenedObject.string'",
 						"'predicate:nested' is not available for fields of this type" );
 	}
 

integrationtest/backend/tck/src/main/java/org/hibernate/search/integrationtest/backend/tck/dynamic/ObjectFieldTemplateIT.java

@@ -507,7 +507,7 @@ private void checkFlattened(String objectFieldPath) {
 						.add( f.match().field( objectFieldPath + "." + LASTNAME_FIELD )
 								.matching( LASTNAME_1 ) )
 		) )
-				.hasMessageContainingAll( "Cannot use 'predicate:nested' on field '" + objectFieldPath + "'.",
+				.hasMessageContainingAll( "Cannot use 'predicate:nested' on field '" + objectFieldPath + "'",
 						"Some object field features require a nested structure; "
 								+ "try setting the field structure to 'NESTED' and reindexing all your data" );
 		assertThatQuery( query(