Skip to content

Commit

Permalink
HSEARCH-4203 Replace EntityIdentifierScroll/EntityLoader/LoadingInter…
Browse files Browse the repository at this point in the history
…ceptor with MassIdentifierLoader/MassEntityLoader

1. This is simpler for users to implement, as they can manage the
   context setup/teardown directly in the class where that context
   is used.
2. This is still highly customizable, as BatchEntityLoadingOptions
   offers ways to pass context to the loaders.
  • Loading branch information
yrodiere committed May 7, 2021
1 parent 5abcc1d commit c8da380
Show file tree
Hide file tree
Showing 86 changed files with 1,806 additions and 1,455 deletions.
Expand Up @@ -36,7 +36,7 @@
import org.hibernate.search.integrationtest.mapper.pojo.testsupport.util.rule.JavaBeanMappingSetupHelper;
import org.hibernate.search.mapper.javabean.mapping.SearchMapping;
import org.hibernate.search.mapper.javabean.massindexing.MassIndexer;
import org.hibernate.search.mapper.javabean.massindexing.loader.JavaBeanIndexingStrategies;
import org.hibernate.search.mapper.javabean.loading.MassLoadingStrategies;
import org.hibernate.search.mapper.javabean.session.SearchSession;
import org.hibernate.search.mapper.pojo.mapping.definition.annotation.DocumentId;
import org.hibernate.search.util.impl.integrationtest.common.rule.ThreadSpy;
Expand Down Expand Up @@ -460,7 +460,7 @@ private SearchMapping setup() {

private SearchSession session(SearchMapping mapping) {
return mapping.createSessionWithOptions().loading( (o) -> {
o.massIndexingLoadingStrategy( Book.class, JavaBeanIndexingStrategies.from( booksmap ) );
o.massLoadingStrategy( Book.class, MassLoadingStrategies.from( booksmap ) );
} ).build();
}

Expand Down
Expand Up @@ -21,6 +21,7 @@
import org.hibernate.search.engine.backend.work.execution.DocumentRefreshStrategy;
import org.hibernate.search.engine.cfg.EngineSettings;
import org.hibernate.search.engine.cfg.spi.EngineSpiSettings;
import org.hibernate.search.mapper.javabean.loading.MassLoadingStrategies;
import org.hibernate.search.mapper.pojo.massindexing.MassIndexingFailureHandler;
import org.hibernate.search.mapper.pojo.mapping.definition.annotation.GenericField;
import org.hibernate.search.mapper.pojo.mapping.definition.annotation.Indexed;
Expand All @@ -39,7 +40,6 @@
import org.hibernate.search.integrationtest.mapper.pojo.testsupport.util.rule.JavaBeanMappingSetupHelper;
import org.hibernate.search.mapper.javabean.mapping.SearchMapping;
import org.hibernate.search.mapper.javabean.massindexing.MassIndexer;
import org.hibernate.search.mapper.javabean.massindexing.loader.JavaBeanIndexingStrategies;
import org.hibernate.search.mapper.javabean.session.SearchSession;
import org.hibernate.search.mapper.pojo.mapping.definition.annotation.DocumentId;

Expand Down Expand Up @@ -657,7 +657,7 @@ private SearchMapping setup() {

private SearchSession session(SearchMapping mapping) {
return mapping.createSessionWithOptions().loading( (o) -> {
o.massIndexingLoadingStrategy( Book.class, JavaBeanIndexingStrategies.from( booksmap ) );
o.massLoadingStrategy( Book.class, MassLoadingStrategies.from( booksmap ) );
} ).build();
}

Expand Down
Expand Up @@ -15,9 +15,9 @@
import org.hibernate.search.engine.backend.work.execution.DocumentCommitStrategy;
import org.hibernate.search.engine.backend.work.execution.DocumentRefreshStrategy;
import org.hibernate.search.integrationtest.mapper.pojo.testsupport.util.rule.JavaBeanMappingSetupHelper;
import org.hibernate.search.mapper.javabean.loading.MassLoadingStrategies;
import org.hibernate.search.mapper.javabean.mapping.SearchMapping;
import org.hibernate.search.mapper.javabean.massindexing.MassIndexer;
import org.hibernate.search.mapper.javabean.massindexing.loader.JavaBeanIndexingStrategies;
import org.hibernate.search.mapper.javabean.session.SearchSession;
import org.hibernate.search.mapper.pojo.mapping.definition.annotation.DocumentId;
import org.hibernate.search.mapper.pojo.mapping.definition.annotation.GenericField;
Expand Down Expand Up @@ -232,7 +232,7 @@ public void lazyCreateSearchSessionAfterJavaBeanSessionIsClosed_createMassIndexe

private SearchSession createSessionFromMap() {
return mapping.createSessionWithOptions().loading( (o) -> {
o.massIndexingLoadingStrategy( Book.class, JavaBeanIndexingStrategies.from( booksmap ) );
o.massLoadingStrategy( Book.class, MassLoadingStrategies.from( booksmap ) );
} ).build();
}

Expand Down
Expand Up @@ -14,9 +14,9 @@
import org.hibernate.search.engine.backend.work.execution.DocumentCommitStrategy;
import org.hibernate.search.engine.backend.work.execution.DocumentRefreshStrategy;
import org.hibernate.search.integrationtest.mapper.pojo.testsupport.util.rule.JavaBeanMappingSetupHelper;
import org.hibernate.search.mapper.javabean.loading.MassLoadingStrategies;
import org.hibernate.search.mapper.javabean.mapping.SearchMapping;
import org.hibernate.search.mapper.javabean.massindexing.MassIndexer;
import org.hibernate.search.mapper.javabean.massindexing.loader.JavaBeanIndexingStrategies;
import org.hibernate.search.mapper.javabean.session.SearchSession;
import org.hibernate.search.mapper.pojo.mapping.definition.annotation.DocumentId;
import org.hibernate.search.mapper.pojo.mapping.definition.annotation.GenericField;
Expand Down Expand Up @@ -200,8 +200,8 @@ public void rootIndexed_someSubclassesIndexed_requestMassIndexingOnIndexedSubcla

private SearchSession createSession() {
return mapping.createSessionWithOptions().loading( (o) -> {
o.massIndexingLoadingStrategy( H1_Root_NotIndexed.class, JavaBeanIndexingStrategies.from( h1map ) );
o.massIndexingLoadingStrategy( H2_Root_Indexed.class, JavaBeanIndexingStrategies.from( h2map ) );
o.massLoadingStrategy( H1_Root_NotIndexed.class, MassLoadingStrategies.from( h1map ) );
o.massLoadingStrategy( H2_Root_Indexed.class, MassLoadingStrategies.from( h2map ) );
} ).build();
}

Expand Down
Expand Up @@ -6,33 +6,37 @@
*/
package org.hibernate.search.integrationtest.mapper.pojo.massindexing;

import static org.assertj.core.api.Assertions.assertThat;
import static org.assertj.core.api.Fail.fail;

import java.lang.invoke.MethodHandles;
import java.util.LinkedHashMap;
import java.util.List;
import java.util.Map;
import java.util.stream.Collectors;
import static org.assertj.core.api.Assertions.assertThat;
import static org.assertj.core.api.Fail.fail;

import org.hibernate.search.integrationtest.mapper.pojo.testsupport.util.rule.JavaBeanMappingSetupHelper;
import org.hibernate.search.mapper.javabean.loading.MassIdentifierLoader;
import org.hibernate.search.mapper.javabean.loading.MassIdentifierSink;
import org.hibernate.search.mapper.javabean.loading.MassEntityLoader;
import org.hibernate.search.mapper.javabean.loading.MassLoadingOptions;
import org.hibernate.search.mapper.javabean.loading.LoadingTypeGroup;
import org.hibernate.search.mapper.javabean.loading.MassEntitySink;
import org.hibernate.search.mapper.javabean.loading.MassLoadingStrategy;
import org.hibernate.search.mapper.javabean.mapping.SearchMapping;
import org.hibernate.search.mapper.javabean.massindexing.MassIndexer;
import org.hibernate.search.mapper.javabean.massindexing.loader.JavaBeanIndexingOptions;
import org.hibernate.search.mapper.javabean.session.SearchSession;
import org.hibernate.search.mapper.pojo.loading.EntityIdentifierScroll;
import org.hibernate.search.mapper.pojo.loading.EntityLoader;
import org.hibernate.search.mapper.pojo.mapping.definition.annotation.DocumentId;
import org.hibernate.search.mapper.pojo.mapping.definition.annotation.GenericField;
import org.hibernate.search.mapper.pojo.mapping.definition.annotation.Indexed;
import org.hibernate.search.mapper.pojo.massindexing.MassIndexingFailureContext;
import org.hibernate.search.mapper.pojo.massindexing.MassIndexingFailureHandler;
import org.hibernate.search.mapper.pojo.massindexing.loader.MassIndexingEntityLoadingStrategy;
import org.hibernate.search.mapper.pojo.massindexing.loader.MassIndexingEntityLoadingTypeGroup;
import org.hibernate.search.mapper.pojo.massindexing.loader.MassIndexingThreadContext;
import org.hibernate.search.util.impl.integrationtest.common.rule.BackendMock;

import org.junit.Before;
import org.junit.Rule;
import org.junit.Test;

import org.mockito.ArgumentCaptor;
import org.mockito.Mock;
import org.mockito.Mockito;
Expand All @@ -43,7 +47,7 @@
/**
* Test that the {@link MassIndexer} correctly indexes even complex entity hierarchies
* where superclasses are indexed but not all of their subclasses, and vice-versa.
* It also tests {@link MassIndexingEntityLoadingTypeGroup#includedEntityMap()},
* It also tests {@link LoadingTypeGroup#includedTypesMap()},
* and the position of the first item for the group super type.
*/
public class MassIndexingIncludedEntityMapHierarchyIT {
Expand Down Expand Up @@ -250,27 +254,38 @@ public void rootIndexed_someSubclassesIndexed_requestMassIndexingOnIndexedSubcla

private SearchSession createSession() {
return mapping.createSessionWithOptions().loading( (o) -> {
o.massIndexingLoadingStrategy( H1_Root_NotIndexed.class, exeptingStrategy() );
o.massIndexingLoadingStrategy( H2_Root_Indexed.class, exeptingStrategy() );
o.massLoadingStrategy( H1_Root_NotIndexed.class, failingStrategy() );
o.massLoadingStrategy( H2_Root_Indexed.class, failingStrategy() );
} ).build();
}

public static ExceptingMapIndexingStrategy exeptingStrategy() {
return new ExceptingMapIndexingStrategy<>();
public static FailingMassIndexingStrategy failingStrategy() {
return new FailingMassIndexingStrategy<>();
}

public static class ExceptingMapIndexingStrategy<E> implements MassIndexingEntityLoadingStrategy<E, JavaBeanIndexingOptions> {
public static class FailingMassIndexingStrategy<E> implements MassLoadingStrategy<E, Object> {

@Override
public EntityIdentifierScroll createIdentifierScroll(JavaBeanIndexingOptions options, MassIndexingThreadContext context,
MassIndexingEntityLoadingTypeGroup<? extends E> loadingTypeGroup) {
throw new SimulatedFailure( loadingTypeGroup.includedEntityMap().keySet().stream().collect( Collectors.joining( "," ) ) );
public MassIdentifierLoader createIdentifierLoader(LoadingTypeGroup<E> includedTypes,
MassIdentifierSink<Object> sink, MassLoadingOptions options) {
throw new SimulatedFailure( includedTypes.includedTypesMap().keySet().stream()
.collect( Collectors.joining( "," ) ) );
}

@Override
public EntityLoader<E> createLoader(JavaBeanIndexingOptions options, MassIndexingThreadContext context,
MassIndexingEntityLoadingTypeGroup<? extends E> loadingTypeGroup) {
return identifiers -> null;
public MassEntityLoader<Object> createEntityLoader(LoadingTypeGroup<E> includedTypes, MassEntitySink<E> sink,
MassLoadingOptions options) {
return new MassEntityLoader<Object>() {
@Override
public void close() {
// Nothing to do.
}

@Override
public void load(List<Object> identifiers) {
throw new UnsupportedOperationException( "Didn't expect this to be called" );
}
};
}

}
Expand Down
Expand Up @@ -18,6 +18,7 @@
import org.hibernate.search.engine.backend.work.execution.DocumentCommitStrategy;
import org.hibernate.search.engine.backend.work.execution.DocumentRefreshStrategy;
import org.hibernate.search.engine.cfg.spi.EngineSpiSettings;
import org.hibernate.search.mapper.javabean.loading.MassLoadingStrategies;
import org.hibernate.search.mapper.pojo.mapping.definition.annotation.GenericField;
import org.hibernate.search.mapper.pojo.mapping.definition.annotation.Indexed;
import org.hibernate.search.util.impl.integrationtest.common.rule.BackendMock;
Expand All @@ -31,7 +32,6 @@
import org.hibernate.search.integrationtest.mapper.pojo.testsupport.util.rule.JavaBeanMappingSetupHelper;
import org.hibernate.search.mapper.javabean.mapping.SearchMapping;
import org.hibernate.search.mapper.javabean.massindexing.MassIndexer;
import org.hibernate.search.mapper.javabean.massindexing.loader.JavaBeanIndexingStrategies;
import org.hibernate.search.mapper.javabean.session.SearchSession;
import org.hibernate.search.mapper.pojo.mapping.definition.annotation.DocumentId;

Expand Down Expand Up @@ -189,7 +189,7 @@ private void waitForMassIndexingThreadsToTerminate(int expectedThreadCount) {

private SearchSession createSessionFromMap() {
return mapping.createSessionWithOptions().loading( (o) -> {
o.massIndexingLoadingStrategy( Book.class, JavaBeanIndexingStrategies.from( booksmap ) );
o.massLoadingStrategy( Book.class, MassLoadingStrategies.from( booksmap ) );
} ).build();
}

Expand Down
Expand Up @@ -20,7 +20,7 @@
import org.hibernate.search.integrationtest.mapper.pojo.testsupport.util.rule.JavaBeanMappingSetupHelper;
import org.hibernate.search.mapper.javabean.mapping.SearchMapping;
import org.hibernate.search.mapper.javabean.massindexing.MassIndexer;
import org.hibernate.search.mapper.javabean.massindexing.loader.JavaBeanIndexingStrategies;
import org.hibernate.search.mapper.javabean.loading.MassLoadingStrategies;
import org.hibernate.search.mapper.javabean.session.SearchSession;
import org.hibernate.search.mapper.pojo.mapping.definition.annotation.DocumentId;
import org.hibernate.search.mapper.pojo.massindexing.MassIndexingMonitor;
Expand Down Expand Up @@ -131,7 +131,7 @@ public void simple() {

private SearchSession createSessionFromMap(SearchMapping mapping) {
return mapping.createSessionWithOptions().loading( (o) -> {
o.massIndexingLoadingStrategy( Book.class, JavaBeanIndexingStrategies.from( booksmap ) );
o.massLoadingStrategy( Book.class, MassLoadingStrategies.from( booksmap ) );
} ).build();
}

Expand Down
Expand Up @@ -14,9 +14,9 @@
import org.hibernate.search.engine.backend.work.execution.DocumentCommitStrategy;
import org.hibernate.search.engine.backend.work.execution.DocumentRefreshStrategy;
import org.hibernate.search.integrationtest.mapper.pojo.testsupport.util.rule.JavaBeanMappingSetupHelper;
import org.hibernate.search.mapper.javabean.loading.MassLoadingStrategies;
import org.hibernate.search.mapper.javabean.mapping.SearchMapping;
import org.hibernate.search.mapper.javabean.massindexing.MassIndexer;
import org.hibernate.search.mapper.javabean.massindexing.loader.JavaBeanIndexingStrategies;
import org.hibernate.search.mapper.javabean.session.SearchSession;
import org.hibernate.search.mapper.pojo.mapping.definition.annotation.DocumentId;
import org.hibernate.search.mapper.pojo.mapping.definition.annotation.Indexed;
Expand Down Expand Up @@ -94,7 +94,7 @@ public void entityWithPrimitiveId() {

private SearchSession createSessionFromMap() {
return mapping.createSessionWithOptions().loading( (o) -> {
o.massIndexingLoadingStrategy( EntityWithPrimitiveId.class, JavaBeanIndexingStrategies.from( entitymap ) );
o.massLoadingStrategy( EntityWithPrimitiveId.class, MassLoadingStrategies.from( entitymap ) );
} ).build();
}

Expand Down
Expand Up @@ -8,6 +8,20 @@

import java.util.List;

/**
* A loader for loading a small selection of entities, used in particular during search.
* <p>
* Compared to {@link MassEntityLoader}, this loader:
* <ul>
* <li>Receives batches of identifiers from the caller.</li>
* <li>Is expected to load a small number of entities, potentially in a single batch.</li>
* <li>Returns loaded entities as a list.</li>
* <li>Relies on a pre-existing loading context (a session, a transaction, ...).</li>
* <li>Must ensure entities remain usable (lazy-loading, ...) as long as the supporting context is active.</li>
* </ul>
*
* @param <E> The type of loaded entities.
*/
public interface EntityLoader<E> {

/**
Expand Down
Expand Up @@ -6,17 +6,10 @@
*/
package org.hibernate.search.mapper.javabean.loading;

import org.hibernate.search.mapper.javabean.massindexing.loader.JavaBeanIndexingOptions;
import org.hibernate.search.mapper.pojo.loading.LoadingInterceptor;
import org.hibernate.search.mapper.pojo.massindexing.loader.MassIndexingEntityLoadingStrategy;

public interface LoadingOptions {

<T> LoadingOptions registerLoader(Class<T> type, EntityLoader<T> loader);

<T> void massIndexingLoadingStrategy(Class<T> type, MassIndexingEntityLoadingStrategy<T, JavaBeanIndexingOptions> loadingStrategy);

void identifierInterceptor(LoadingInterceptor interceptor);
<T> void massLoadingStrategy(Class<T> type, MassLoadingStrategy<T, ?> loadingStrategy);

void documentInterceptor(LoadingInterceptor interceptor);
}
Expand Up @@ -4,20 +4,22 @@
* License: GNU Lesser General Public License (LGPL), version 2.1 or later
* See the lgpl.txt file in the root directory or <http://www.gnu.org/licenses/lgpl-2.1.html>.
*/
package org.hibernate.search.mapper.pojo.massindexing.loader;
package org.hibernate.search.mapper.javabean.loading;

import java.util.Map;

/**
* A group entity types for entity loading entities during mass indexing.
* @param <E> The resulting entity type (output)
* A group of entity types for entity loading.
*
* @param <E> The type of loaded entities.
*/
public interface MassIndexingEntityLoadingTypeGroup<E> {
public interface LoadingTypeGroup<E> {

/**
* @return The names of all entity types included in this group.
* @return A map of entity classes by entity name,
* representing all entity types included in this group, and only these types.
*/
Map<String, Class<? extends E>> includedEntityMap();
Map<String, Class<? extends E>> includedTypesMap();

/**
* @param entity An entity of any type.
Expand Down
@@ -0,0 +1,47 @@
/*
* Hibernate Search, full-text search for your domain model
*
* License: GNU Lesser General Public License (LGPL), version 2.1 or later
* See the lgpl.txt file in the root directory or <http://www.gnu.org/licenses/lgpl-2.1.html>.
*/
package org.hibernate.search.mapper.javabean.loading;

import java.util.List;

/**
* A loader for mass loading of entities, used in particular during mass indexing.
* <p>
* Compared to {@link EntityLoader}, this loader:
* <ul>
* <li>Receives batches of identifiers from a {@link MassIdentifierLoader}</li>
* <li>Is expected to load a very large number of entities in multiple small batches.</li>
* <li>Pushes loaded entities to a sink.</li>
* <li>Sets up its own context (session, transactions, ...), instead of potentially relying on a pre-existing context.</li>
* <li>Is free to discard the entities after the sink is done processing them.</li>
* </ul>
*
* @param <I> The type of entity identifiers.
*/
public interface MassEntityLoader<I> extends AutoCloseable {

/**
* Closes this {@link MassEntityLoader}.
*/
@Override
void close();

/**
* Loads the entities corresponding to the given identifiers and adds them to the given sink,
* blocking the current thread while doing so.
* <p>
* Calls to the sink must be performed synchronously (before this method returns).
* <p>
* Entities must be passed to the sink using a single call to {@link MassEntitySink#accept(List)}.
* <p>
* Entities passed to the sink do not need to be the same order as {@code identifiers}.
*
* @param identifiers A list of identifiers of entities to load.
*/
void load(List<I> identifiers);

}
@@ -0,0 +1,29 @@
/*
* Hibernate Search, full-text search for your domain model
*
* License: GNU Lesser General Public License (LGPL), version 2.1 or later
* See the lgpl.txt file in the root directory or <http://www.gnu.org/licenses/lgpl-2.1.html>.
*/
package org.hibernate.search.mapper.javabean.loading;

import java.util.List;

/**
* A sink for use by a {@link MassIdentifierLoader}.
*
* @param <E> The type of loaded entities.
*/
public interface MassEntitySink<E> {

/**
* Adds a batch of entities to the sink.
* <p>
* The list and entities need to stay usable at least until this method returns,
* as they will be consumed synchronously.
* Afterwards, they can be discarded or reused at will.
*
* @param batch The next batch of identifiers. Never {@code null}, never empty.
*/
void accept(List<? extends E> batch);

}

0 comments on commit c8da380

Please sign in to comment.