From 41deddf395aa7ac86dc9f6a31e6df7675a73699b Mon Sep 17 00:00:00 2001 From: Fabio Massimo Ercoli Date: Thu, 5 May 2022 22:49:05 +0200 Subject: [PATCH 01/14] HSEARCH-4566 Introduce shard holder pre-start method --- .../index/impl/LuceneIndexManagerImpl.java | 3 +++ .../lucene/index/impl/ShardHolder.java | 23 ++++++++++++++----- 2 files changed, 20 insertions(+), 6 deletions(-) diff --git a/backend/lucene/src/main/java/org/hibernate/search/backend/lucene/index/impl/LuceneIndexManagerImpl.java b/backend/lucene/src/main/java/org/hibernate/search/backend/lucene/index/impl/LuceneIndexManagerImpl.java index eff7ef1ecff..57aaf32fa1c 100644 --- a/backend/lucene/src/main/java/org/hibernate/search/backend/lucene/index/impl/LuceneIndexManagerImpl.java +++ b/backend/lucene/src/main/java/org/hibernate/search/backend/lucene/index/impl/LuceneIndexManagerImpl.java @@ -82,6 +82,9 @@ public String toString() { @Override public void start(IndexManagerStartContext context) { + // TODO HSEARCH-4545 Move it to preStart: + shardHolder.preStart( context ); + shardHolder.start( context ); } diff --git a/backend/lucene/src/main/java/org/hibernate/search/backend/lucene/index/impl/ShardHolder.java b/backend/lucene/src/main/java/org/hibernate/search/backend/lucene/index/impl/ShardHolder.java index cb7860b018c..993beb6df81 100644 --- a/backend/lucene/src/main/java/org/hibernate/search/backend/lucene/index/impl/ShardHolder.java +++ b/backend/lucene/src/main/java/org/hibernate/search/backend/lucene/index/impl/ShardHolder.java @@ -50,19 +50,31 @@ public String toString() { return getClass().getSimpleName() + "[indexName=" + model.hibernateSearchName() + "]"; } - void start(IndexManagerStartContext startContext) { + void preStart(IndexManagerStartContext startContext) { ConfigurationPropertySource propertySource = startContext.configurationPropertySource(); try { ShardingStrategyInitializationContextImpl initializationContext = new ShardingStrategyInitializationContextImpl( backendContext, model, startContext, propertySource ); this.shardingStrategyHolder = initializationContext.create( shards ); + } + catch (RuntimeException e) { + new SuppressingCloser( e ) + .pushAll( Shard::stop, shards.values() ); + shards.clear(); + throw e; + } + } - if ( startContext.failureCollector().hasFailure() ) { - // At least one shard creation failed; abort and don't even try to start shards. - return; - } + void start(IndexManagerStartContext startContext) { + if ( startContext.failureCollector().hasFailure() ) { + // At least one shard creation failed; abort and don't even try to start shards. + return; + } + ConfigurationPropertySource propertySource = startContext.configurationPropertySource(); + + try { for ( Shard shard : shards.values() ) { shard.start( propertySource ); managementOrchestrators.add( shard.managementOrchestrator() ); @@ -71,7 +83,6 @@ void start(IndexManagerStartContext startContext) { catch (RuntimeException e) { new SuppressingCloser( e ) .pushAll( Shard::stop, shards.values() ); - shards.clear(); managementOrchestrators.clear(); throw e; } From 7d2d89261ef93a3d8747d3e1be0a28c58c435296 Mon Sep 17 00:00:00 2001 From: Fabio Massimo Ercoli Date: Tue, 3 May 2022 16:41:01 +0200 Subject: [PATCH 02/14] HSEARCH-4566 Introduce saved state abstraction --- .../index/spi/IndexManagerImplementor.java | 6 ++ .../search/engine/backend/spi/SavedState.java | 92 +++++++++++++++++++ .../impl/IndexManagerNonStartedState.java | 6 +- .../common/impl/SearchIntegrationImpl.java | 12 +++ ...earchIntegrationPartialBuildStateImpl.java | 29 ++++++ 5 files changed, 144 insertions(+), 1 deletion(-) create mode 100644 engine/src/main/java/org/hibernate/search/engine/backend/spi/SavedState.java diff --git a/engine/src/main/java/org/hibernate/search/engine/backend/index/spi/IndexManagerImplementor.java b/engine/src/main/java/org/hibernate/search/engine/backend/index/spi/IndexManagerImplementor.java index 6a829d95d23..3a088150c1b 100644 --- a/engine/src/main/java/org/hibernate/search/engine/backend/index/spi/IndexManagerImplementor.java +++ b/engine/src/main/java/org/hibernate/search/engine/backend/index/spi/IndexManagerImplementor.java @@ -9,6 +9,7 @@ import java.util.concurrent.CompletableFuture; import org.hibernate.search.engine.backend.schema.management.spi.IndexSchemaManager; +import org.hibernate.search.engine.backend.spi.SavedState; import org.hibernate.search.engine.backend.work.execution.DocumentCommitStrategy; import org.hibernate.search.engine.backend.work.execution.DocumentRefreshStrategy; import org.hibernate.search.engine.backend.index.IndexManager; @@ -28,6 +29,11 @@ */ public interface IndexManagerImplementor { + default SavedState saveForRestart() { + // TODO HSEARCH-4545 Implement this method + return SavedState.empty(); + } + /** * Start any resource necessary to operate the index manager at runtime. *

diff --git a/engine/src/main/java/org/hibernate/search/engine/backend/spi/SavedState.java b/engine/src/main/java/org/hibernate/search/engine/backend/spi/SavedState.java new file mode 100644 index 00000000000..a019df930b1 --- /dev/null +++ b/engine/src/main/java/org/hibernate/search/engine/backend/spi/SavedState.java @@ -0,0 +1,92 @@ +/* + * Hibernate Search, full-text search for your domain model + * + * License: GNU Lesser General Public License (LGPL), version 2.1 or later + * See the lgpl.txt file in the root directory or . + */ +package org.hibernate.search.engine.backend.spi; + +import java.util.LinkedHashMap; +import java.util.Map; +import java.util.Objects; +import java.util.Optional; + +import org.hibernate.search.util.common.impl.Contracts; + +public class SavedState { + + private static final SavedState EMPTY = new SavedState.Builder().build(); + + public static SavedState empty() { + return EMPTY; + } + + private final Map, Object> content; + + private SavedState(Builder builder) { + this.content = builder.content; + } + + @SuppressWarnings("unchecked") // values have always the corresponding key generic type + public Optional get(Key key) { + T value = (T) content.get( key ); + return Optional.ofNullable( value ); + } + + public static Key key(String name) { + Contracts.assertNotNullNorEmpty( name, "name" ); + return new Key<>( name ); + } + + public static final class Key { + + private final String name; + + private Key(String name) { + this.name = name; + } + + @Override + public String toString() { + return getClass().getSimpleName() + "[" + name + "]"; + } + + @Override + public boolean equals(Object o) { + if ( this == o ) { + return true; + } + if ( o == null || getClass() != o.getClass() ) { + return false; + } + Key key = (Key) o; + return Objects.equals( name, key.name ); + } + + @Override + public int hashCode() { + return Objects.hash( name ); + } + } + + public static Builder builder() { + return new Builder(); + } + + public static final class Builder { + private final Map, Object> content = new LinkedHashMap<>(); + + private Builder() { + } + + // values have always the corresponding key generic type + public Builder put(Key key, T value) { + content.put( key, value ); + return this; + } + + public SavedState build() { + return new SavedState( this ); + } + } +} diff --git a/engine/src/main/java/org/hibernate/search/engine/common/impl/IndexManagerNonStartedState.java b/engine/src/main/java/org/hibernate/search/engine/common/impl/IndexManagerNonStartedState.java index 59faa480bb2..bbeb37a1226 100644 --- a/engine/src/main/java/org/hibernate/search/engine/common/impl/IndexManagerNonStartedState.java +++ b/engine/src/main/java/org/hibernate/search/engine/common/impl/IndexManagerNonStartedState.java @@ -7,6 +7,7 @@ package org.hibernate.search.engine.common.impl; import org.hibernate.search.engine.backend.index.spi.IndexManagerImplementor; +import org.hibernate.search.engine.backend.spi.SavedState; import org.hibernate.search.engine.cfg.impl.ConfigurationPropertySourceExtractor; import org.hibernate.search.engine.cfg.ConfigurationPropertySource; import org.hibernate.search.engine.environment.bean.BeanResolver; @@ -32,6 +33,10 @@ void closeOnFailure() { indexManager.stop(); } + void preStart(SavedState savedState) { + // TODO HSEARCH-4545 Implement this method + } + IndexManagerImplementor start(RootFailureCollector rootFailureCollector, BeanResolver beanResolver, ConfigurationPropertySource rootPropertySource) { @@ -48,5 +53,4 @@ IndexManagerImplementor start(RootFailureCollector rootFailureCollector, } return indexManager; // The index is now started } - } diff --git a/engine/src/main/java/org/hibernate/search/engine/common/impl/SearchIntegrationImpl.java b/engine/src/main/java/org/hibernate/search/engine/common/impl/SearchIntegrationImpl.java index 607fef6c91c..067b7f33894 100644 --- a/engine/src/main/java/org/hibernate/search/engine/common/impl/SearchIntegrationImpl.java +++ b/engine/src/main/java/org/hibernate/search/engine/common/impl/SearchIntegrationImpl.java @@ -7,6 +7,7 @@ package org.hibernate.search.engine.common.impl; import java.lang.invoke.MethodHandles; +import java.util.HashMap; import java.util.Map; import java.util.concurrent.CompletableFuture; import java.util.function.BiConsumer; @@ -16,6 +17,7 @@ import org.hibernate.search.engine.backend.index.IndexManager; import org.hibernate.search.engine.backend.index.spi.IndexManagerImplementor; import org.hibernate.search.engine.backend.spi.BackendImplementor; +import org.hibernate.search.engine.backend.spi.SavedState; import org.hibernate.search.engine.common.resources.impl.EngineThreads; import org.hibernate.search.engine.common.spi.SearchIntegration; import org.hibernate.search.engine.common.timing.spi.TimingSource; @@ -38,6 +40,8 @@ public class SearchIntegrationImpl implements SearchIntegration { + static final SavedState.Key> INDEX_MANAGERS_KEY = SavedState.key( "index_managers" ); + private static final Log log = LoggerFactory.make( Log.class, MethodHandles.lookup() ); private final BeanProvider beanProvider; @@ -95,6 +99,14 @@ public IndexManager indexManager(String indexManagerName) { return indexManager.toAPI(); } + public SavedState saveForRestart() { + HashMap states = new HashMap<>(); + for ( Map.Entry indexManager : indexManagers.entrySet() ) { + states.put( indexManager.getKey(), indexManager.getValue().saveForRestart() ); + } + return SavedState.builder().put( INDEX_MANAGERS_KEY, states ).build(); + } + @Override public void close() { RootFailureCollector rootFailureCollector = diff --git a/engine/src/main/java/org/hibernate/search/engine/common/impl/SearchIntegrationPartialBuildStateImpl.java b/engine/src/main/java/org/hibernate/search/engine/common/impl/SearchIntegrationPartialBuildStateImpl.java index d5b9d85a9ae..7f4d283b68f 100644 --- a/engine/src/main/java/org/hibernate/search/engine/common/impl/SearchIntegrationPartialBuildStateImpl.java +++ b/engine/src/main/java/org/hibernate/search/engine/common/impl/SearchIntegrationPartialBuildStateImpl.java @@ -6,12 +6,17 @@ */ package org.hibernate.search.engine.common.impl; +import static org.hibernate.search.engine.common.impl.SearchIntegrationImpl.INDEX_MANAGERS_KEY; + +import java.util.Collections; import java.util.LinkedHashMap; import java.util.Map; +import java.util.Optional; import java.util.concurrent.CompletableFuture; import org.hibernate.search.engine.backend.index.spi.IndexManagerImplementor; import org.hibernate.search.engine.backend.spi.BackendImplementor; +import org.hibernate.search.engine.backend.spi.SavedState; import org.hibernate.search.engine.cfg.spi.ConfigurationPropertyChecker; import org.hibernate.search.engine.cfg.ConfigurationPropertySource; import org.hibernate.search.engine.common.resources.impl.EngineThreads; @@ -58,6 +63,9 @@ class SearchIntegrationPartialBuildStateImpl implements SearchIntegrationPartial private final EngineThreads engineThreads; private final TimingSource timingSource; + // TODO HSEARCH-4545 Load instance here + private final Optional previousIntegration = Optional.empty(); + SearchIntegrationPartialBuildStateImpl( BeanProvider beanProvider, BeanResolver beanResolver, BeanHolder failureHandlerHolder, @@ -164,6 +172,27 @@ public SearchIntegration finalizeIntegration() { } failureCollector.checkNoFailure(); + // Pre-Start indexes + Map indexManagersStates = null; + + for ( Map.Entry entry : nonStartedIndexManagers.entrySet() ) { + SavedState savedState = SavedState.empty(); + if ( previousIntegration.isPresent() ) { + if ( indexManagersStates == null ) { + indexManagersStates = previousIntegration.get().saveForRestart() + .get( INDEX_MANAGERS_KEY ).orElse( Collections.emptyMap() ); + } + + savedState = indexManagersStates.get( entry.getKey() ); + } + + entry.getValue().preStart( savedState ); + } + + if ( previousIntegration.isPresent() ) { + previousIntegration.get().close(); + } + // Start indexes for ( Map.Entry entry : nonStartedIndexManagers.entrySet() ) { startedIndexManagers.put( From c6eb01705802774cff8cdcdb1859d87d618eef91 Mon Sep 17 00:00:00 2001 From: Fabio Massimo Ercoli Date: Wed, 4 May 2022 10:04:57 +0200 Subject: [PATCH 03/14] HSEARCH-4566 Add restart search integration spi --- .../engine/common/impl/SearchIntegrationBuilder.java | 8 ++++++-- .../engine/common/impl/SearchIntegrationImpl.java | 7 +++++++ .../impl/SearchIntegrationPartialBuildStateImpl.java | 12 ++++++++---- .../search/engine/common/spi/SearchIntegration.java | 6 +++++- 4 files changed, 26 insertions(+), 7 deletions(-) diff --git a/engine/src/main/java/org/hibernate/search/engine/common/impl/SearchIntegrationBuilder.java b/engine/src/main/java/org/hibernate/search/engine/common/impl/SearchIntegrationBuilder.java index b8f5e94ac6e..5c46709f906 100644 --- a/engine/src/main/java/org/hibernate/search/engine/common/impl/SearchIntegrationBuilder.java +++ b/engine/src/main/java/org/hibernate/search/engine/common/impl/SearchIntegrationBuilder.java @@ -11,6 +11,7 @@ import java.util.LinkedHashMap; import java.util.List; import java.util.Map; +import java.util.Optional; import org.hibernate.search.engine.cfg.EngineSettings; import org.hibernate.search.engine.cfg.spi.ConfigurationProperty; @@ -63,12 +64,15 @@ public class SearchIntegrationBuilder implements SearchIntegration.Builder { .build(); private final SearchIntegrationEnvironment environment; + private final Optional previousIntegration; private final Map, MappingInitiator> mappingInitiators = new LinkedHashMap<>(); private boolean frozen = false; - public SearchIntegrationBuilder(SearchIntegrationEnvironment environment) { + public SearchIntegrationBuilder(SearchIntegrationEnvironment environment, + Optional previousIntegration) { this.environment = environment; + this.previousIntegration = previousIntegration; environment.propertyChecker().beforeBoot(); } @@ -195,7 +199,7 @@ public SearchIntegrationPartialBuildState prepareBuild() { indexManagerBuildingStateHolder.getBackendNonStartedStates(), indexManagerBuildingStateHolder.getIndexManagersNonStartedStates(), environment.propertyChecker(), - engineThreads, timingSource + engineThreads, timingSource, previousIntegration ); } catch (RuntimeException e) { diff --git a/engine/src/main/java/org/hibernate/search/engine/common/impl/SearchIntegrationImpl.java b/engine/src/main/java/org/hibernate/search/engine/common/impl/SearchIntegrationImpl.java index 067b7f33894..4447469d355 100644 --- a/engine/src/main/java/org/hibernate/search/engine/common/impl/SearchIntegrationImpl.java +++ b/engine/src/main/java/org/hibernate/search/engine/common/impl/SearchIntegrationImpl.java @@ -9,6 +9,7 @@ import java.lang.invoke.MethodHandles; import java.util.HashMap; import java.util.Map; +import java.util.Optional; import java.util.concurrent.CompletableFuture; import java.util.function.BiConsumer; import java.util.function.BiFunction; @@ -20,6 +21,7 @@ import org.hibernate.search.engine.backend.spi.SavedState; import org.hibernate.search.engine.common.resources.impl.EngineThreads; import org.hibernate.search.engine.common.spi.SearchIntegration; +import org.hibernate.search.engine.common.spi.SearchIntegrationEnvironment; import org.hibernate.search.engine.common.timing.spi.TimingSource; import org.hibernate.search.engine.environment.bean.BeanHolder; import org.hibernate.search.engine.environment.bean.spi.BeanProvider; @@ -107,6 +109,11 @@ public SavedState saveForRestart() { return SavedState.builder().put( INDEX_MANAGERS_KEY, states ).build(); } + @Override + public Builder restartBuilder(SearchIntegrationEnvironment environment) { + return new SearchIntegrationBuilder( environment, Optional.of( this ) ); + } + @Override public void close() { RootFailureCollector rootFailureCollector = diff --git a/engine/src/main/java/org/hibernate/search/engine/common/impl/SearchIntegrationPartialBuildStateImpl.java b/engine/src/main/java/org/hibernate/search/engine/common/impl/SearchIntegrationPartialBuildStateImpl.java index 7f4d283b68f..300aacdd586 100644 --- a/engine/src/main/java/org/hibernate/search/engine/common/impl/SearchIntegrationPartialBuildStateImpl.java +++ b/engine/src/main/java/org/hibernate/search/engine/common/impl/SearchIntegrationPartialBuildStateImpl.java @@ -62,9 +62,7 @@ class SearchIntegrationPartialBuildStateImpl implements SearchIntegrationPartial private final EngineThreads engineThreads; private final TimingSource timingSource; - - // TODO HSEARCH-4545 Load instance here - private final Optional previousIntegration = Optional.empty(); + private final Optional previousIntegration; SearchIntegrationPartialBuildStateImpl( BeanProvider beanProvider, BeanResolver beanResolver, @@ -74,7 +72,8 @@ class SearchIntegrationPartialBuildStateImpl implements SearchIntegrationPartial Map nonStartedBackends, Map nonStartedIndexManagers, ConfigurationPropertyChecker partialConfigurationPropertyChecker, - EngineThreads engineThreads, TimingSource timingSource) { + EngineThreads engineThreads, TimingSource timingSource, + Optional previousIntegration) { this.beanProvider = beanProvider; this.beanResolver = beanResolver; this.failureHandlerHolder = failureHandlerHolder; @@ -85,6 +84,7 @@ class SearchIntegrationPartialBuildStateImpl implements SearchIntegrationPartial this.partialConfigurationPropertyChecker = partialConfigurationPropertyChecker; this.engineThreads = engineThreads; this.timingSource = timingSource; + this.previousIntegration = previousIntegration; } @Override @@ -102,6 +102,10 @@ public void closeOnFailure() { closer.pushAll( BeanProvider::close, beanProvider ); closer.pushAll( EngineThreads::onStop, engineThreads ); closer.pushAll( TimingSource::stop, timingSource ); + + if ( previousIntegration.isPresent() ) { + closer.pushAll( SearchIntegration::close, previousIntegration.get() ); + } } } diff --git a/engine/src/main/java/org/hibernate/search/engine/common/spi/SearchIntegration.java b/engine/src/main/java/org/hibernate/search/engine/common/spi/SearchIntegration.java index 2acf89d82cc..ed75b6a7e0f 100644 --- a/engine/src/main/java/org/hibernate/search/engine/common/spi/SearchIntegration.java +++ b/engine/src/main/java/org/hibernate/search/engine/common/spi/SearchIntegration.java @@ -6,6 +6,8 @@ */ package org.hibernate.search.engine.common.spi; +import java.util.Optional; + import org.hibernate.search.engine.backend.Backend; import org.hibernate.search.engine.backend.index.IndexManager; import org.hibernate.search.engine.common.impl.SearchIntegrationBuilder; @@ -21,11 +23,13 @@ public interface SearchIntegration extends AutoCloseable { IndexManager indexManager(String indexManagerName); + Builder restartBuilder(SearchIntegrationEnvironment environment); + @Override void close(); static Builder builder(SearchIntegrationEnvironment environment) { - return new SearchIntegrationBuilder( environment ); + return new SearchIntegrationBuilder( environment, Optional.empty() ); } interface Builder { From 183abfb6d0842a866ebd2d20fa2f2cc7474d0311 Mon Sep 17 00:00:00 2001 From: Fabio Massimo Ercoli Date: Thu, 5 May 2022 23:16:59 +0200 Subject: [PATCH 04/14] HSEARCH-4566 Implement save for restart chain --- .../index/impl/LuceneIndexManagerImpl.java | 12 +++++++++- .../backend/lucene/index/impl/Shard.java | 24 ++++++++++++++++++- .../lucene/index/impl/ShardHolder.java | 12 ++++++++++ .../index/impl/IndexAccessorImpl.java | 14 +++++++++++ .../index/spi/IndexManagerImplementor.java | 1 - 5 files changed, 60 insertions(+), 3 deletions(-) diff --git a/backend/lucene/src/main/java/org/hibernate/search/backend/lucene/index/impl/LuceneIndexManagerImpl.java b/backend/lucene/src/main/java/org/hibernate/search/backend/lucene/index/impl/LuceneIndexManagerImpl.java index 57aaf32fa1c..38b1772d4ac 100644 --- a/backend/lucene/src/main/java/org/hibernate/search/backend/lucene/index/impl/LuceneIndexManagerImpl.java +++ b/backend/lucene/src/main/java/org/hibernate/search/backend/lucene/index/impl/LuceneIndexManagerImpl.java @@ -29,6 +29,7 @@ import org.hibernate.search.engine.backend.scope.spi.IndexScopeBuilder; import org.hibernate.search.engine.backend.session.spi.BackendSessionContext; import org.hibernate.search.engine.backend.session.spi.DetachedBackendSessionContext; +import org.hibernate.search.engine.backend.spi.SavedState; import org.hibernate.search.engine.backend.work.execution.DocumentCommitStrategy; import org.hibernate.search.engine.backend.work.execution.DocumentRefreshStrategy; import org.hibernate.search.engine.backend.work.execution.spi.IndexIndexer; @@ -47,6 +48,8 @@ public class LuceneIndexManagerImpl implements IndexManagerImplementor, LuceneIndexManager, LuceneScopeIndexManagerContext { + private static final SavedState.Key SHARD_HOLDER_KEY = SavedState.key( "shard_holder" ); + private static final Log log = LoggerFactory.make( Log.class, MethodHandles.lookup() ); private final IndexManagerBackendContext backendContext; @@ -80,6 +83,13 @@ public String toString() { .toString(); } + @Override + public SavedState saveForRestart() { + return SavedState.builder() + .put( SHARD_HOLDER_KEY, shardHolder.saveForRestart() ) + .build(); + } + @Override public void start(IndexManagerStartContext context) { // TODO HSEARCH-4545 Move it to preStart: @@ -138,7 +148,7 @@ public IndexScopeBuilder createScopeBuilder(BackendMappingContext mappingContext @Override public void addTo(IndexScopeBuilder builder) { - if ( ! ( builder instanceof LuceneIndexScopeBuilder ) ) { + if ( !( builder instanceof LuceneIndexScopeBuilder ) ) { throw log.cannotMixLuceneScopeWithOtherType( builder, this, backendContext.getEventContext() ); diff --git a/backend/lucene/src/main/java/org/hibernate/search/backend/lucene/index/impl/Shard.java b/backend/lucene/src/main/java/org/hibernate/search/backend/lucene/index/impl/Shard.java index ad146c218d5..41a0bc66771 100644 --- a/backend/lucene/src/main/java/org/hibernate/search/backend/lucene/index/impl/Shard.java +++ b/backend/lucene/src/main/java/org/hibernate/search/backend/lucene/index/impl/Shard.java @@ -11,11 +11,13 @@ import java.util.concurrent.CompletableFuture; import org.hibernate.search.backend.lucene.logging.impl.Log; +import org.hibernate.search.backend.lucene.lowlevel.directory.spi.DirectoryHolder; import org.hibernate.search.backend.lucene.lowlevel.index.impl.IndexAccessorImpl; import org.hibernate.search.backend.lucene.orchestration.impl.LuceneParallelWorkOrchestrator; import org.hibernate.search.backend.lucene.orchestration.impl.LuceneParallelWorkOrchestratorImpl; import org.hibernate.search.backend.lucene.orchestration.impl.LuceneSerialWorkOrchestrator; import org.hibernate.search.backend.lucene.orchestration.impl.LuceneSerialWorkOrchestratorImpl; +import org.hibernate.search.engine.backend.spi.SavedState; import org.hibernate.search.engine.cfg.ConfigurationPropertySource; import org.hibernate.search.util.common.impl.Closer; import org.hibernate.search.util.common.impl.SuppressingCloser; @@ -26,6 +28,8 @@ public final class Shard { + private static final SavedState.Key DIRECTORY_HOLDER_KEY = SavedState.key( "directory_holder" ); + private static final Log log = LoggerFactory.make( Log.class, MethodHandles.lookup() ); private final EventContext eventContext; @@ -33,6 +37,8 @@ public final class Shard { private final LuceneParallelWorkOrchestratorImpl managementOrchestrator; private final LuceneSerialWorkOrchestratorImpl indexingOrchestrator; + private boolean savedForRestart = false; + Shard(EventContext eventContext, IndexAccessorImpl indexAccessor, LuceneParallelWorkOrchestratorImpl managementOrchestrator, LuceneSerialWorkOrchestratorImpl indexingOrchestrator) { @@ -42,6 +48,17 @@ public final class Shard { this.indexingOrchestrator = indexingOrchestrator; } + public SavedState saveForRestart() { + try { + return SavedState.builder() + .put( DIRECTORY_HOLDER_KEY, indexAccessor.directoryHolder() ) + .build(); + } + finally { + savedForRestart = true; + } + } + void start(ConfigurationPropertySource propertySource) { try { indexAccessor.start(); @@ -70,7 +87,12 @@ void stop() { closer.push( LuceneSerialWorkOrchestratorImpl::stop, indexingOrchestrator ); closer.push( LuceneParallelWorkOrchestratorImpl::stop, managementOrchestrator ); // Close the index writer after the orchestrators, when we're sure all works have been performed - closer.push( IndexAccessorImpl::close, indexAccessor ); + if ( savedForRestart ) { + closer.push( IndexAccessorImpl::closeNoDir, indexAccessor ); + } + else { + closer.push( IndexAccessorImpl::close, indexAccessor ); + } } } diff --git a/backend/lucene/src/main/java/org/hibernate/search/backend/lucene/index/impl/ShardHolder.java b/backend/lucene/src/main/java/org/hibernate/search/backend/lucene/index/impl/ShardHolder.java index 993beb6df81..4557f7e2542 100644 --- a/backend/lucene/src/main/java/org/hibernate/search/backend/lucene/index/impl/ShardHolder.java +++ b/backend/lucene/src/main/java/org/hibernate/search/backend/lucene/index/impl/ShardHolder.java @@ -9,6 +9,7 @@ import java.io.IOException; import java.util.ArrayList; import java.util.Collection; +import java.util.HashMap; import java.util.HashSet; import java.util.LinkedHashMap; import java.util.List; @@ -25,6 +26,7 @@ import org.hibernate.search.backend.lucene.schema.management.impl.SchemaManagementIndexManagerContext; import org.hibernate.search.backend.lucene.work.execution.impl.WorkExecutionIndexManagerContext; import org.hibernate.search.engine.backend.index.spi.IndexManagerStartContext; +import org.hibernate.search.engine.backend.spi.SavedState; import org.hibernate.search.engine.cfg.ConfigurationPropertySource; import org.hibernate.search.engine.environment.bean.BeanHolder; import org.hibernate.search.util.common.impl.Closer; @@ -33,6 +35,8 @@ class ShardHolder implements ReadIndexManagerContext, WorkExecutionIndexManagerContext, SchemaManagementIndexManagerContext { + private static final SavedState.Key> SHARDS_KEY = SavedState.key( "shards" ); + private final IndexManagerBackendContext backendContext; private final LuceneIndexModel model; @@ -50,6 +54,14 @@ public String toString() { return getClass().getSimpleName() + "[indexName=" + model.hibernateSearchName() + "]"; } + public SavedState saveForRestart() { + HashMap states = new HashMap<>(); + for ( Map.Entry shard : shards.entrySet() ) { + states.put( shard.getKey(), shard.getValue().saveForRestart() ); + } + return SavedState.builder().put( SHARDS_KEY, states ).build(); + } + void preStart(IndexManagerStartContext startContext) { ConfigurationPropertySource propertySource = startContext.configurationPropertySource(); diff --git a/backend/lucene/src/main/java/org/hibernate/search/backend/lucene/lowlevel/index/impl/IndexAccessorImpl.java b/backend/lucene/src/main/java/org/hibernate/search/backend/lucene/lowlevel/index/impl/IndexAccessorImpl.java index 5cbaa4c34db..4e6c2e4ae8d 100644 --- a/backend/lucene/src/main/java/org/hibernate/search/backend/lucene/lowlevel/index/impl/IndexAccessorImpl.java +++ b/backend/lucene/src/main/java/org/hibernate/search/backend/lucene/lowlevel/index/impl/IndexAccessorImpl.java @@ -65,6 +65,16 @@ public void close() { } } + public void closeNoDir() { + try ( Closer closer = new Closer<>() ) { + closer.push( IndexWriterProvider::clear, indexWriterProvider ); + closer.push( IndexReaderProvider::clear, indexReaderProvider ); + } + catch (RuntimeException | IOException e) { + throw log.unableToShutdownIndexAccessor( e.getMessage(), e ); + } + } + @Override public void createIndexIfMissing() { try { @@ -206,6 +216,10 @@ public long computeSizeInBytes() { return totalSize; } + public DirectoryHolder directoryHolder() { + return directoryHolder; + } + public Directory getDirectoryForTests() { return directoryHolder.get(); } diff --git a/engine/src/main/java/org/hibernate/search/engine/backend/index/spi/IndexManagerImplementor.java b/engine/src/main/java/org/hibernate/search/engine/backend/index/spi/IndexManagerImplementor.java index 3a088150c1b..d9cd0a9963f 100644 --- a/engine/src/main/java/org/hibernate/search/engine/backend/index/spi/IndexManagerImplementor.java +++ b/engine/src/main/java/org/hibernate/search/engine/backend/index/spi/IndexManagerImplementor.java @@ -30,7 +30,6 @@ public interface IndexManagerImplementor { default SavedState saveForRestart() { - // TODO HSEARCH-4545 Implement this method return SavedState.empty(); } From cc97d016082c9755fa64dd9ca9503e96dd1bb1f2 Mon Sep 17 00:00:00 2001 From: Fabio Massimo Ercoli Date: Wed, 4 May 2022 16:17:43 +0200 Subject: [PATCH 05/14] HSEARCH-4566 Implement pre start chain --- .../impl/IndexManagerBackendContext.java | 6 ++- .../index/impl/LuceneIndexManagerImpl.java | 8 ++-- .../backend/lucene/index/impl/Shard.java | 11 ++++-- .../lucene/index/impl/ShardHolder.java | 7 +++- ...dingStrategyInitializationContextImpl.java | 38 +++++++++++++------ .../impl/LocalHeapDirectoryHolder.java | 2 +- .../index/spi/IndexManagerImplementor.java | 19 +++++++++- .../impl/IndexManagerNonStartedState.java | 26 ++++++++----- ...earchIntegrationPartialBuildStateImpl.java | 5 ++- 9 files changed, 88 insertions(+), 34 deletions(-) diff --git a/backend/lucene/src/main/java/org/hibernate/search/backend/lucene/index/impl/IndexManagerBackendContext.java b/backend/lucene/src/main/java/org/hibernate/search/backend/lucene/index/impl/IndexManagerBackendContext.java index 5f56f491759..5fd57cd5177 100644 --- a/backend/lucene/src/main/java/org/hibernate/search/backend/lucene/index/impl/IndexManagerBackendContext.java +++ b/backend/lucene/src/main/java/org/hibernate/search/backend/lucene/index/impl/IndexManagerBackendContext.java @@ -197,7 +197,8 @@ LuceneIndexSchemaManager createSchemaManager(SchemaManagementIndexManagerContext } Shard createShard(LuceneIndexModel model, EventContext shardEventContext, DirectoryHolder directoryHolder, - IOStrategy ioStrategy, ConfigurationPropertySource propertySource) { + IOStrategy ioStrategy, ConfigurationPropertySource propertySource, + boolean reuseAlreadyStaredDirectoryHolder) { LuceneParallelWorkOrchestratorImpl managementOrchestrator; LuceneSerialWorkOrchestratorImpl indexingOrchestrator; IndexAccessorImpl indexAccessor = null; @@ -215,7 +216,8 @@ Shard createShard(LuceneIndexModel model, EventContext shardEventContext, Direct Shard shard = new Shard( shardEventContext, indexAccessor, - managementOrchestrator, indexingOrchestrator + managementOrchestrator, indexingOrchestrator, + reuseAlreadyStaredDirectoryHolder ); return shard; } diff --git a/backend/lucene/src/main/java/org/hibernate/search/backend/lucene/index/impl/LuceneIndexManagerImpl.java b/backend/lucene/src/main/java/org/hibernate/search/backend/lucene/index/impl/LuceneIndexManagerImpl.java index 38b1772d4ac..3b54f5a26a0 100644 --- a/backend/lucene/src/main/java/org/hibernate/search/backend/lucene/index/impl/LuceneIndexManagerImpl.java +++ b/backend/lucene/src/main/java/org/hibernate/search/backend/lucene/index/impl/LuceneIndexManagerImpl.java @@ -91,10 +91,12 @@ public SavedState saveForRestart() { } @Override - public void start(IndexManagerStartContext context) { - // TODO HSEARCH-4545 Move it to preStart: - shardHolder.preStart( context ); + public void preStart(IndexManagerStartContext context, SavedState savedState) { + shardHolder.preStart( context, savedState.get( SHARD_HOLDER_KEY ).orElse( SavedState.empty() ) ); + } + @Override + public void start(IndexManagerStartContext context) { shardHolder.start( context ); } diff --git a/backend/lucene/src/main/java/org/hibernate/search/backend/lucene/index/impl/Shard.java b/backend/lucene/src/main/java/org/hibernate/search/backend/lucene/index/impl/Shard.java index 41a0bc66771..e42e10305fc 100644 --- a/backend/lucene/src/main/java/org/hibernate/search/backend/lucene/index/impl/Shard.java +++ b/backend/lucene/src/main/java/org/hibernate/search/backend/lucene/index/impl/Shard.java @@ -28,7 +28,7 @@ public final class Shard { - private static final SavedState.Key DIRECTORY_HOLDER_KEY = SavedState.key( "directory_holder" ); + static final SavedState.Key DIRECTORY_HOLDER_KEY = SavedState.key( "directory_holder" ); private static final Log log = LoggerFactory.make( Log.class, MethodHandles.lookup() ); @@ -36,16 +36,19 @@ public final class Shard { private final IndexAccessorImpl indexAccessor; private final LuceneParallelWorkOrchestratorImpl managementOrchestrator; private final LuceneSerialWorkOrchestratorImpl indexingOrchestrator; + private final boolean reuseAlreadyStaredDirectoryHolder; private boolean savedForRestart = false; Shard(EventContext eventContext, IndexAccessorImpl indexAccessor, LuceneParallelWorkOrchestratorImpl managementOrchestrator, - LuceneSerialWorkOrchestratorImpl indexingOrchestrator) { + LuceneSerialWorkOrchestratorImpl indexingOrchestrator, + boolean reuseAlreadyStaredDirectoryHolder) { this.eventContext = eventContext; this.indexAccessor = indexAccessor; this.managementOrchestrator = managementOrchestrator; this.indexingOrchestrator = indexingOrchestrator; + this.reuseAlreadyStaredDirectoryHolder = reuseAlreadyStaredDirectoryHolder; } public SavedState saveForRestart() { @@ -61,7 +64,9 @@ public SavedState saveForRestart() { void start(ConfigurationPropertySource propertySource) { try { - indexAccessor.start(); + if ( !reuseAlreadyStaredDirectoryHolder ) { + indexAccessor.start(); + } managementOrchestrator.start( propertySource ); indexingOrchestrator.start( propertySource ); } diff --git a/backend/lucene/src/main/java/org/hibernate/search/backend/lucene/index/impl/ShardHolder.java b/backend/lucene/src/main/java/org/hibernate/search/backend/lucene/index/impl/ShardHolder.java index 4557f7e2542..1534517d06c 100644 --- a/backend/lucene/src/main/java/org/hibernate/search/backend/lucene/index/impl/ShardHolder.java +++ b/backend/lucene/src/main/java/org/hibernate/search/backend/lucene/index/impl/ShardHolder.java @@ -9,6 +9,7 @@ import java.io.IOException; import java.util.ArrayList; import java.util.Collection; +import java.util.Collections; import java.util.HashMap; import java.util.HashSet; import java.util.LinkedHashMap; @@ -62,13 +63,15 @@ public SavedState saveForRestart() { return SavedState.builder().put( SHARDS_KEY, states ).build(); } - void preStart(IndexManagerStartContext startContext) { + void preStart(IndexManagerStartContext startContext, SavedState savedState) { ConfigurationPropertySource propertySource = startContext.configurationPropertySource(); try { ShardingStrategyInitializationContextImpl initializationContext = new ShardingStrategyInitializationContextImpl( backendContext, model, startContext, propertySource ); - this.shardingStrategyHolder = initializationContext.create( shards ); + Map states = savedState.get( SHARDS_KEY ).orElse( Collections.emptyMap() ); + + this.shardingStrategyHolder = initializationContext.create( shards, states ); } catch (RuntimeException e) { new SuppressingCloser( e ) diff --git a/backend/lucene/src/main/java/org/hibernate/search/backend/lucene/index/impl/ShardingStrategyInitializationContextImpl.java b/backend/lucene/src/main/java/org/hibernate/search/backend/lucene/index/impl/ShardingStrategyInitializationContextImpl.java index 4ac2bd0f008..6274b16c57d 100644 --- a/backend/lucene/src/main/java/org/hibernate/search/backend/lucene/index/impl/ShardingStrategyInitializationContextImpl.java +++ b/backend/lucene/src/main/java/org/hibernate/search/backend/lucene/index/impl/ShardingStrategyInitializationContextImpl.java @@ -23,6 +23,7 @@ import org.hibernate.search.backend.lucene.lowlevel.directory.spi.DirectoryProvider; import org.hibernate.search.backend.lucene.lowlevel.index.impl.IOStrategy; import org.hibernate.search.engine.backend.index.spi.IndexManagerStartContext; +import org.hibernate.search.engine.backend.spi.SavedState; import org.hibernate.search.engine.cfg.spi.ConfigurationProperty; import org.hibernate.search.engine.cfg.ConfigurationPropertySource; import org.hibernate.search.engine.environment.bean.BeanHolder; @@ -96,7 +97,8 @@ public ConfigurationPropertySource configurationPropertySource() { return shardingPropertySource; } - public BeanHolder create(Map shardCollector) { + public BeanHolder create(Map shardCollector, + Map states) { BeanHolder shardingStrategyHolder = SHARDING_STRATEGY.getAndTransform( shardingPropertySource, beanResolver()::resolve ); @@ -104,7 +106,8 @@ public BeanHolder create(Map shardCol if ( shardIdentifiers == null ) { // Sharding is disabled => single shard - contributeShardWithSilentFailure( shardCollector, Optional.empty() ); + contributeShardWithSilentFailure( shardCollector, Optional.empty(), + states.getOrDefault( null, SavedState.empty() ) ); return null; } @@ -115,13 +118,15 @@ public BeanHolder create(Map shardCol } for ( String shardIdentifier : shardIdentifiers ) { - contributeShardWithSilentFailure( shardCollector, Optional.of( shardIdentifier ) ); + contributeShardWithSilentFailure( shardCollector, Optional.of( shardIdentifier ), + states.getOrDefault( shardIdentifier, SavedState.empty() ) ); } return shardingStrategyHolder; } - private void contributeShardWithSilentFailure(Map shardCollector, Optional shardId) { + private void contributeShardWithSilentFailure(Map shardCollector, Optional shardId, + SavedState savedState) { EventContext shardEventContext = EventContexts.fromIndexNameAndShardId( indexName(), shardId ); ConfigurationPropertySource shardPropertySource = shardId.isPresent() ? @@ -129,17 +134,28 @@ private void contributeShardWithSilentFailure(Map shardCollector, .withFallback( indexPropertySource ) : indexPropertySource; + + Optional savedDirectoryHolder = savedState.get( Shard.DIRECTORY_HOLDER_KEY ); DirectoryHolder directoryHolder = null; - try ( BeanHolder directoryProviderHolder = - DIRECTORY_TYPE.getAndTransform( shardPropertySource, startContext.beanResolver()::resolve ) ) { - DirectoryCreationContext context = new DirectoryCreationContextImpl( shardEventContext, - indexName(), shardId, beanResolver(), shardPropertySource.withMask( "directory" ) ); - directoryHolder = directoryProviderHolder.get().createDirectoryHolder( context ); + boolean reuseAlreadyStaredDirectoryHolder = false; - IOStrategy ioStrategy = backendContext.createIOStrategy( shardPropertySource ); + try { + if ( savedDirectoryHolder.isPresent() ) { + directoryHolder = savedDirectoryHolder.get(); + reuseAlreadyStaredDirectoryHolder = true; + } + else { + try ( BeanHolder directoryProviderHolder = + DIRECTORY_TYPE.getAndTransform( shardPropertySource, startContext.beanResolver()::resolve ) ) { + DirectoryCreationContext context = new DirectoryCreationContextImpl( shardEventContext, + indexName(), shardId, beanResolver(), shardPropertySource.withMask( "directory" ) ); + directoryHolder = directoryProviderHolder.get().createDirectoryHolder( context ); + } + } + IOStrategy ioStrategy = backendContext.createIOStrategy( shardPropertySource ); Shard shard = backendContext.createShard( model, shardEventContext, directoryHolder, ioStrategy, - shardPropertySource ); + shardPropertySource, reuseAlreadyStaredDirectoryHolder ); shardCollector.put( shardId.orElse( null ), shard ); } catch (RuntimeException e) { diff --git a/backend/lucene/src/main/java/org/hibernate/search/backend/lucene/lowlevel/directory/impl/LocalHeapDirectoryHolder.java b/backend/lucene/src/main/java/org/hibernate/search/backend/lucene/lowlevel/directory/impl/LocalHeapDirectoryHolder.java index 1e5df79766e..fe7c901395d 100644 --- a/backend/lucene/src/main/java/org/hibernate/search/backend/lucene/lowlevel/directory/impl/LocalHeapDirectoryHolder.java +++ b/backend/lucene/src/main/java/org/hibernate/search/backend/lucene/lowlevel/directory/impl/LocalHeapDirectoryHolder.java @@ -26,7 +26,7 @@ final class LocalHeapDirectoryHolder implements DirectoryHolder { @Override public void start() { - this.directory = new ByteBuffersDirectory( lockFactory ); + directory = new ByteBuffersDirectory( lockFactory ); } @Override diff --git a/engine/src/main/java/org/hibernate/search/engine/backend/index/spi/IndexManagerImplementor.java b/engine/src/main/java/org/hibernate/search/engine/backend/index/spi/IndexManagerImplementor.java index d9cd0a9963f..685508bf1ae 100644 --- a/engine/src/main/java/org/hibernate/search/engine/backend/index/spi/IndexManagerImplementor.java +++ b/engine/src/main/java/org/hibernate/search/engine/backend/index/spi/IndexManagerImplementor.java @@ -34,13 +34,30 @@ default SavedState saveForRestart() { } /** - * Start any resource necessary to operate the index manager at runtime. + * Starts a subset of resources that are necessary to operate the index manager at runtime, and are expected to be reused upon restarts. + * The resources may be retrieved them from the saved state, + * or created if they are not present in the saved state. *

* Called by the engine once after bootstrap, after * {@link org.hibernate.search.engine.backend.spi.BackendImplementor#start(BackendStartContext)} * was called on the corresponding backend. * * @param context The start context. + * @param savedState The saved state returned by the corresponding index manager in the Hibernate Search integration + * being restarted, or {@link SavedState#empty()} on the first start. + */ + default void preStart(IndexManagerStartContext context, SavedState savedState) { + // do nothing by default + } + + /** + * Start any resource necessary to operate the index manager at runtime. + *

+ * Called by the engine once just after + * {@link #preStart(IndexManagerStartContext, SavedState)} + * was called on the corresponding backend. + * + * @param context The start context. */ void start(IndexManagerStartContext context); diff --git a/engine/src/main/java/org/hibernate/search/engine/common/impl/IndexManagerNonStartedState.java b/engine/src/main/java/org/hibernate/search/engine/common/impl/IndexManagerNonStartedState.java index bbeb37a1226..ba5ed9aceb7 100644 --- a/engine/src/main/java/org/hibernate/search/engine/common/impl/IndexManagerNonStartedState.java +++ b/engine/src/main/java/org/hibernate/search/engine/common/impl/IndexManagerNonStartedState.java @@ -21,6 +21,10 @@ class IndexManagerNonStartedState { private final ConfigurationPropertySourceExtractor propertySourceExtractor; private final IndexManagerImplementor indexManager; + // created on pre-start + private IndexManagerStartContextImpl startContext; + private ContextualFailureCollector indexFailureCollector; + IndexManagerNonStartedState(EventContext eventContext, ConfigurationPropertySourceExtractor propertySourceExtractor, IndexManagerImplementor indexManager) { @@ -33,18 +37,22 @@ void closeOnFailure() { indexManager.stop(); } - void preStart(SavedState savedState) { - // TODO HSEARCH-4545 Implement this method - } - - IndexManagerImplementor start(RootFailureCollector rootFailureCollector, - BeanResolver beanResolver, - ConfigurationPropertySource rootPropertySource) { - ContextualFailureCollector indexFailureCollector = rootFailureCollector.withContext( eventContext ); + void preStart(RootFailureCollector rootFailureCollector, BeanResolver beanResolver, + ConfigurationPropertySource rootPropertySource, SavedState savedState) { + indexFailureCollector = rootFailureCollector.withContext( eventContext ); ConfigurationPropertySource indexPropertySource = propertySourceExtractor.extract( rootPropertySource ); - IndexManagerStartContextImpl startContext = new IndexManagerStartContextImpl( + startContext = new IndexManagerStartContextImpl( indexFailureCollector, beanResolver, indexPropertySource ); + try { + indexManager.preStart( startContext, savedState ); + } + catch (RuntimeException e) { + indexFailureCollector.add( e ); + } + } + + IndexManagerImplementor start() { try { indexManager.start( startContext ); } diff --git a/engine/src/main/java/org/hibernate/search/engine/common/impl/SearchIntegrationPartialBuildStateImpl.java b/engine/src/main/java/org/hibernate/search/engine/common/impl/SearchIntegrationPartialBuildStateImpl.java index 300aacdd586..28d8814da3f 100644 --- a/engine/src/main/java/org/hibernate/search/engine/common/impl/SearchIntegrationPartialBuildStateImpl.java +++ b/engine/src/main/java/org/hibernate/search/engine/common/impl/SearchIntegrationPartialBuildStateImpl.java @@ -190,8 +190,9 @@ public SearchIntegration finalizeIntegration() { savedState = indexManagersStates.get( entry.getKey() ); } - entry.getValue().preStart( savedState ); + entry.getValue().preStart( failureCollector, beanResolver, propertySource, savedState ); } + failureCollector.checkNoFailure(); if ( previousIntegration.isPresent() ) { previousIntegration.get().close(); @@ -201,7 +202,7 @@ public SearchIntegration finalizeIntegration() { for ( Map.Entry entry : nonStartedIndexManagers.entrySet() ) { startedIndexManagers.put( entry.getKey(), - entry.getValue().start( failureCollector, beanResolver, propertySource ) + entry.getValue().start() ); } failureCollector.checkNoFailure(); From 74988bae5b3a1e1811bac6c4097d9dcf8bbea279 Mon Sep 17 00:00:00 2001 From: Fabio Massimo Ercoli Date: Tue, 3 May 2022 09:43:03 +0200 Subject: [PATCH 06/14] HSEARCH-4566 Test transfer directories state --- ...IndexRestartFromPreviousIntegrationIT.java | 98 +++++++++++++++++++ .../util/rule/SearchSetupHelper.java | 12 ++- 2 files changed, 109 insertions(+), 1 deletion(-) create mode 100644 integrationtest/backend/lucene/src/test/java/org/hibernate/search/integrationtest/backend/lucene/index/LuceneIndexRestartFromPreviousIntegrationIT.java diff --git a/integrationtest/backend/lucene/src/test/java/org/hibernate/search/integrationtest/backend/lucene/index/LuceneIndexRestartFromPreviousIntegrationIT.java b/integrationtest/backend/lucene/src/test/java/org/hibernate/search/integrationtest/backend/lucene/index/LuceneIndexRestartFromPreviousIntegrationIT.java new file mode 100644 index 00000000000..1a287c722e2 --- /dev/null +++ b/integrationtest/backend/lucene/src/test/java/org/hibernate/search/integrationtest/backend/lucene/index/LuceneIndexRestartFromPreviousIntegrationIT.java @@ -0,0 +1,98 @@ +/* + * Hibernate Search, full-text search for your domain model + * + * License: GNU Lesser General Public License (LGPL), version 2.1 or later + * See the lgpl.txt file in the root directory or . + */ +package org.hibernate.search.integrationtest.backend.lucene.index; + +import static org.hibernate.search.util.impl.integrationtest.common.assertion.SearchResultAssert.assertThatQuery; + +import java.util.Arrays; +import java.util.List; + +import org.hibernate.search.backend.lucene.cfg.LuceneIndexSettings; +import org.hibernate.search.engine.backend.document.IndexFieldReference; +import org.hibernate.search.engine.backend.document.model.dsl.IndexSchemaElement; +import org.hibernate.search.engine.common.spi.SearchIntegration; +import org.hibernate.search.integrationtest.backend.tck.testsupport.util.rule.SearchSetupHelper; +import org.hibernate.search.util.impl.integrationtest.mapper.stub.BulkIndexer; +import org.hibernate.search.util.impl.integrationtest.mapper.stub.SimpleMappedIndex; +import org.hibernate.search.util.impl.integrationtest.mapper.stub.StubMappingSchemaManagementStrategy; + +import org.junit.Rule; +import org.junit.Test; +import org.junit.runner.RunWith; +import org.junit.runners.Parameterized; + +@RunWith(Parameterized.class) +public class LuceneIndexRestartFromPreviousIntegrationIT { + + @Parameterized.Parameters(name = "{0}") + public static List params() { + return Arrays.asList( "local-heap", "local-filesystem" ); + } + + @Parameterized.Parameter + public String directoryType; + + @Rule + public final SearchSetupHelper setupHelper = new SearchSetupHelper(); + + private final SimpleMappedIndex indexV1 = SimpleMappedIndex.of( IndexBindingV1::new ); + private final SimpleMappedIndex indexV2 = SimpleMappedIndex.of( IndexBindingV2::new ); + + @Test + public void addNewFieldOnExistingIndex() { + SearchIntegration integrationV1 = setupHelper.start() + .withSchemaManagement( StubMappingSchemaManagementStrategy.DROP_AND_CREATE_ON_STARTUP_ONLY ) + .withIndex( indexV1 ) + .withBackendProperty( LuceneIndexSettings.DIRECTORY_TYPE, directoryType ) + .setup(); + + BulkIndexer indexer1 = indexV1.bulkIndexer(); + indexer1.add( "1", doc -> { + doc.addValue( indexV1.binding().name, "Fabio" ); + } ); + indexer1.join(); + + assertThatQuery( indexV1.query().where( f -> f.match().field( "name" ).matching( "Fabio" ) ) ) + .hasDocRefHitsAnyOrder( indexV1.typeName(), "1" ); + + setupHelper.start() + .withSchemaManagement( StubMappingSchemaManagementStrategy.DROP_ON_SHUTDOWN_ONLY ) + .withIndex( indexV2 ) + .withBackendProperty( LuceneIndexSettings.DIRECTORY_TYPE, directoryType ) + .setup( integrationV1 ); + + BulkIndexer indexer2 = indexV2.bulkIndexer(); + indexer2.add( "2", doc -> { + doc.addValue( indexV2.binding().name, "Fabio" ); + doc.addValue( indexV2.binding().surname, "Ercoli" ); + } ); + indexer2.join(); + + assertThatQuery( indexV2.query().where( f -> f.match().field( "surname" ).matching( "Ercoli" ) ) ) + .hasDocRefHitsAnyOrder( indexV2.typeName(), "2" ); + assertThatQuery( indexV2.query().where( f -> f.match().field( "name" ).matching( "Fabio" ) ) ) + .hasDocRefHitsAnyOrder( indexV2.typeName(), "1", "2" ); + } + + private static class IndexBindingV1 { + final IndexFieldReference name; + + IndexBindingV1(IndexSchemaElement root) { + name = root.field( "name", c -> c.asString() ).toReference(); + } + } + + private static class IndexBindingV2 { + final IndexFieldReference name; + final IndexFieldReference surname; + + IndexBindingV2(IndexSchemaElement root) { + name = root.field( "name", c -> c.asString() ).toReference(); + surname = root.field( "surname", c -> c.asString() ).toReference(); + } + } +} diff --git a/integrationtest/backend/tck/src/main/java/org/hibernate/search/integrationtest/backend/tck/testsupport/util/rule/SearchSetupHelper.java b/integrationtest/backend/tck/src/main/java/org/hibernate/search/integrationtest/backend/tck/testsupport/util/rule/SearchSetupHelper.java index e124981b274..3137904c4ac 100644 --- a/integrationtest/backend/tck/src/main/java/org/hibernate/search/integrationtest/backend/tck/testsupport/util/rule/SearchSetupHelper.java +++ b/integrationtest/backend/tck/src/main/java/org/hibernate/search/integrationtest/backend/tck/testsupport/util/rule/SearchSetupHelper.java @@ -229,13 +229,19 @@ public SetupContext withSchemaManagement(StubMappingSchemaManagementStrategy sch } public PartialSetup setupFirstPhaseOnly() { + return setupFirstPhaseOnly( Optional.empty() ); + } + + public PartialSetup setupFirstPhaseOnly(Optional previousIntegration) { SearchIntegrationEnvironment environment = SearchIntegrationEnvironment.builder( propertySource, unusedPropertyChecker ) .beanProvider( beanProvider ) .build(); environments.add( environment ); - SearchIntegration.Builder integrationBuilder = SearchIntegration.builder( environment ); + SearchIntegration.Builder integrationBuilder = (previousIntegration.isPresent()) ? + previousIntegration.get().restartBuilder( environment ) : + SearchIntegration.builder( environment ); StubMappingInitiator initiator = new StubMappingInitiator( tenancyMode ); mappedIndexes.forEach( initiator::add ); @@ -265,6 +271,10 @@ public SearchIntegration setup() { return setupFirstPhaseOnly().doSecondPhase(); } + public SearchIntegration setup(SearchIntegration previousIntegration) { + return setupFirstPhaseOnly( Optional.of( previousIntegration ) ).doSecondPhase(); + } + } public interface PartialSetup { From 90e763cb940c68fb50d5ce1d3663179cbb3497f1 Mon Sep 17 00:00:00 2001 From: Fabio Massimo Ercoli Date: Fri, 6 May 2022 10:11:24 +0200 Subject: [PATCH 07/14] HSEARCH-4566 Promote closing operator to spi --- .../org/hibernate/search/util/common/impl/AbstractCloser.java | 2 ++ .../hibernate/search/util/common/impl/SuppressingCloser.java | 2 ++ .../search/util/common/{impl => spi}/ClosingOperator.java | 2 +- 3 files changed, 5 insertions(+), 1 deletion(-) rename util/common/src/main/java/org/hibernate/search/util/common/{impl => spi}/ClosingOperator.java (88%) diff --git a/util/common/src/main/java/org/hibernate/search/util/common/impl/AbstractCloser.java b/util/common/src/main/java/org/hibernate/search/util/common/impl/AbstractCloser.java index 44fdf19ae18..cc32bd52e44 100644 --- a/util/common/src/main/java/org/hibernate/search/util/common/impl/AbstractCloser.java +++ b/util/common/src/main/java/org/hibernate/search/util/common/impl/AbstractCloser.java @@ -8,6 +8,8 @@ import java.util.function.Function; +import org.hibernate.search.util.common.spi.ClosingOperator; + /** * A base class implementing the logic behind {@link Closer} and {@link SuppressingCloser}. * diff --git a/util/common/src/main/java/org/hibernate/search/util/common/impl/SuppressingCloser.java b/util/common/src/main/java/org/hibernate/search/util/common/impl/SuppressingCloser.java index b05c495c816..e804fc2efb4 100644 --- a/util/common/src/main/java/org/hibernate/search/util/common/impl/SuppressingCloser.java +++ b/util/common/src/main/java/org/hibernate/search/util/common/impl/SuppressingCloser.java @@ -8,6 +8,8 @@ import java.util.function.Function; +import org.hibernate.search.util.common.spi.ClosingOperator; + /** * A helper for closing multiple resources and re-throwing a provided exception, * {@link Throwable#addSuppressed(Throwable) suppressing} any exceptions caught while closing. diff --git a/util/common/src/main/java/org/hibernate/search/util/common/impl/ClosingOperator.java b/util/common/src/main/java/org/hibernate/search/util/common/spi/ClosingOperator.java similarity index 88% rename from util/common/src/main/java/org/hibernate/search/util/common/impl/ClosingOperator.java rename to util/common/src/main/java/org/hibernate/search/util/common/spi/ClosingOperator.java index 558b617881d..0183cf1296b 100644 --- a/util/common/src/main/java/org/hibernate/search/util/common/impl/ClosingOperator.java +++ b/util/common/src/main/java/org/hibernate/search/util/common/spi/ClosingOperator.java @@ -4,7 +4,7 @@ * License: GNU Lesser General Public License (LGPL), version 2.1 or later * See the lgpl.txt file in the root directory or . */ -package org.hibernate.search.util.common.impl; +package org.hibernate.search.util.common.spi; @FunctionalInterface public interface ClosingOperator { From b56525ebdc28bc2c1b58a408d3b843f17a7a5a2e Mon Sep 17 00:00:00 2001 From: Fabio Massimo Ercoli Date: Fri, 6 May 2022 09:52:36 +0200 Subject: [PATCH 08/14] HSEARCH-4566 Close saved resources not used on restart --- .../backend/lucene/index/impl/Shard.java | 2 +- .../search/engine/backend/spi/SavedState.java | 63 +++++++++++++++++-- ...earchIntegrationPartialBuildStateImpl.java | 7 ++- 3 files changed, 64 insertions(+), 8 deletions(-) diff --git a/backend/lucene/src/main/java/org/hibernate/search/backend/lucene/index/impl/Shard.java b/backend/lucene/src/main/java/org/hibernate/search/backend/lucene/index/impl/Shard.java index e42e10305fc..3b1be36aa0e 100644 --- a/backend/lucene/src/main/java/org/hibernate/search/backend/lucene/index/impl/Shard.java +++ b/backend/lucene/src/main/java/org/hibernate/search/backend/lucene/index/impl/Shard.java @@ -54,7 +54,7 @@ public final class Shard { public SavedState saveForRestart() { try { return SavedState.builder() - .put( DIRECTORY_HOLDER_KEY, indexAccessor.directoryHolder() ) + .put( DIRECTORY_HOLDER_KEY, indexAccessor.directoryHolder(), DirectoryHolder::close ) .build(); } finally { diff --git a/engine/src/main/java/org/hibernate/search/engine/backend/spi/SavedState.java b/engine/src/main/java/org/hibernate/search/engine/backend/spi/SavedState.java index a019df930b1..1a117ff64c5 100644 --- a/engine/src/main/java/org/hibernate/search/engine/backend/spi/SavedState.java +++ b/engine/src/main/java/org/hibernate/search/engine/backend/spi/SavedState.java @@ -12,8 +12,9 @@ import java.util.Optional; import org.hibernate.search.util.common.impl.Contracts; +import org.hibernate.search.util.common.spi.ClosingOperator; -public class SavedState { +public class SavedState implements AutoCloseable { private static final SavedState EMPTY = new SavedState.Builder().build(); @@ -21,7 +22,7 @@ public static SavedState empty() { return EMPTY; } - private final Map, Object> content; + private final Map, SavedValue> content; private SavedState(Builder builder) { this.content = builder.content; @@ -29,7 +30,12 @@ private SavedState(Builder builder) { @SuppressWarnings("unchecked") // values have always the corresponding key generic type public Optional get(Key key) { - T value = (T) content.get( key ); + SavedValue savedValue = content.get( key ); + if ( savedValue == null ) { + return Optional.empty(); + } + + T value = (T) savedValue.value(); return Optional.ofNullable( value ); } @@ -38,6 +44,11 @@ public static Key key(String name) { return new Key<>( name ); } + @Override + public void close() { + content.values().forEach( SavedValue::close ); + } + public static final class Key { private final String name; @@ -74,14 +85,23 @@ public static Builder builder() { } public static final class Builder { - private final Map, Object> content = new LinkedHashMap<>(); + + private final Map, SavedValue> content = new LinkedHashMap<>(); private Builder() { } + public Builder put(Key key, SavedState value) { + return put( key, value, SavedState::close ); + } + + public Builder put(SavedState.Key> key, Map value) { + return put( key, value, (map) -> map.forEach( (k, v) -> v.close() ) ); + } + // values have always the corresponding key generic type - public Builder put(Key key, T value) { - content.put( key, value ); + public Builder put(Key key, T value, ClosingOperator operator) { + content.put( key, new SavedValue<>( value, operator ) ); return this; } @@ -89,4 +109,35 @@ public SavedState build() { return new SavedState( this ); } } + + public static final class SavedValue { + + private final T value; + private final ClosingOperator operator; + + private boolean close = true; + + public SavedValue(T value, ClosingOperator operator) { + this.value = value; + this.operator = operator; + } + + public T value() { + close = false; + return value; + } + + public void close() { + if ( !close ) { + return; + } + + try { + operator.close( value ); + } + catch (Exception e) { + throw new RuntimeException( e ); + } + } + } } diff --git a/engine/src/main/java/org/hibernate/search/engine/common/impl/SearchIntegrationPartialBuildStateImpl.java b/engine/src/main/java/org/hibernate/search/engine/common/impl/SearchIntegrationPartialBuildStateImpl.java index 28d8814da3f..b0b5b3e2beb 100644 --- a/engine/src/main/java/org/hibernate/search/engine/common/impl/SearchIntegrationPartialBuildStateImpl.java +++ b/engine/src/main/java/org/hibernate/search/engine/common/impl/SearchIntegrationPartialBuildStateImpl.java @@ -190,7 +190,12 @@ public SearchIntegration finalizeIntegration() { savedState = indexManagersStates.get( entry.getKey() ); } - entry.getValue().preStart( failureCollector, beanResolver, propertySource, savedState ); + try { + entry.getValue().preStart( failureCollector, beanResolver, propertySource, savedState ); + } + finally { + savedState.close(); + } } failureCollector.checkNoFailure(); From da9a8cc2cba31400adc87c7b2225c3a49cb9a8c2 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Yoann=20Rodi=C3=A8re?= Date: Fri, 6 May 2022 15:49:55 +0200 Subject: [PATCH 09/14] HSEARCH-4566 Close saved index managers not reused on restart --- ...earchIntegrationPartialBuildStateImpl.java | 22 +++++-------------- 1 file changed, 5 insertions(+), 17 deletions(-) diff --git a/engine/src/main/java/org/hibernate/search/engine/common/impl/SearchIntegrationPartialBuildStateImpl.java b/engine/src/main/java/org/hibernate/search/engine/common/impl/SearchIntegrationPartialBuildStateImpl.java index b0b5b3e2beb..d8085dbe15a 100644 --- a/engine/src/main/java/org/hibernate/search/engine/common/impl/SearchIntegrationPartialBuildStateImpl.java +++ b/engine/src/main/java/org/hibernate/search/engine/common/impl/SearchIntegrationPartialBuildStateImpl.java @@ -177,25 +177,13 @@ public SearchIntegration finalizeIntegration() { failureCollector.checkNoFailure(); // Pre-Start indexes - Map indexManagersStates = null; - - for ( Map.Entry entry : nonStartedIndexManagers.entrySet() ) { - SavedState savedState = SavedState.empty(); - if ( previousIntegration.isPresent() ) { - if ( indexManagersStates == null ) { - indexManagersStates = previousIntegration.get().saveForRestart() - .get( INDEX_MANAGERS_KEY ).orElse( Collections.emptyMap() ); - } - - savedState = indexManagersStates.get( entry.getKey() ); - } - - try { + try ( SavedState previousIntegrationSavedState = + previousIntegration.map( SearchIntegrationImpl::saveForRestart ).orElse( SavedState.empty() ) ) { + for ( Map.Entry entry : nonStartedIndexManagers.entrySet() ) { + SavedState savedState = previousIntegrationSavedState.get( INDEX_MANAGERS_KEY ) + .orElse( Collections.emptyMap() ).getOrDefault( entry.getKey(), SavedState.empty() ); entry.getValue().preStart( failureCollector, beanResolver, propertySource, savedState ); } - finally { - savedState.close(); - } } failureCollector.checkNoFailure(); From 42b891e4ed8f5b0b66a38cb713d4a417c120bbef Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Yoann=20Rodi=C3=A8re?= Date: Fri, 6 May 2022 15:57:10 +0200 Subject: [PATCH 10/14] HSEARCH-4566 Safer closing of resources in SavedState --- .../search/engine/backend/spi/SavedState.java | 13 +++++++++++-- 1 file changed, 11 insertions(+), 2 deletions(-) diff --git a/engine/src/main/java/org/hibernate/search/engine/backend/spi/SavedState.java b/engine/src/main/java/org/hibernate/search/engine/backend/spi/SavedState.java index 1a117ff64c5..e4cdf4fa77d 100644 --- a/engine/src/main/java/org/hibernate/search/engine/backend/spi/SavedState.java +++ b/engine/src/main/java/org/hibernate/search/engine/backend/spi/SavedState.java @@ -11,6 +11,7 @@ import java.util.Objects; import java.util.Optional; +import org.hibernate.search.util.common.impl.Closer; import org.hibernate.search.util.common.impl.Contracts; import org.hibernate.search.util.common.spi.ClosingOperator; @@ -46,7 +47,9 @@ public static Key key(String name) { @Override public void close() { - content.values().forEach( SavedValue::close ); + try ( Closer closer = new Closer<>() ) { + closer.pushAll( SavedValue::close, content.values() ); + } } public static final class Key { @@ -84,6 +87,12 @@ public static Builder builder() { return new Builder(); } + private static void closeAll(Map map) { + try ( Closer closer = new Closer<>() ) { + closer.pushAll( SavedState::close, map.values() ); + } + } + public static final class Builder { private final Map, SavedValue> content = new LinkedHashMap<>(); @@ -96,7 +105,7 @@ public Builder put(Key key, SavedState value) { } public Builder put(SavedState.Key> key, Map value) { - return put( key, value, (map) -> map.forEach( (k, v) -> v.close() ) ); + return put( key, value, SavedState::closeAll ); } // values have always the corresponding key generic type From 7730de51f3e5293986719103180e1ec8ff9ddea2 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Yoann=20Rodi=C3=A8re?= Date: Fri, 6 May 2022 15:58:24 +0200 Subject: [PATCH 11/14] HSEARCH-4566 Move SavedState to a more appropriate package --- .../backend/lucene/index/impl/LuceneIndexManagerImpl.java | 2 +- .../org/hibernate/search/backend/lucene/index/impl/Shard.java | 2 +- .../hibernate/search/backend/lucene/index/impl/ShardHolder.java | 2 +- .../index/impl/ShardingStrategyInitializationContextImpl.java | 2 +- .../engine/backend/index/spi/IndexManagerImplementor.java | 2 +- .../search/engine/common/impl/IndexManagerNonStartedState.java | 2 +- .../search/engine/common/impl/SearchIntegrationImpl.java | 2 +- .../common/impl/SearchIntegrationPartialBuildStateImpl.java | 2 +- .../engine/{backend => common/resources}/spi/SavedState.java | 2 +- 9 files changed, 9 insertions(+), 9 deletions(-) rename engine/src/main/java/org/hibernate/search/engine/{backend => common/resources}/spi/SavedState.java (98%) diff --git a/backend/lucene/src/main/java/org/hibernate/search/backend/lucene/index/impl/LuceneIndexManagerImpl.java b/backend/lucene/src/main/java/org/hibernate/search/backend/lucene/index/impl/LuceneIndexManagerImpl.java index 3b54f5a26a0..57583adc525 100644 --- a/backend/lucene/src/main/java/org/hibernate/search/backend/lucene/index/impl/LuceneIndexManagerImpl.java +++ b/backend/lucene/src/main/java/org/hibernate/search/backend/lucene/index/impl/LuceneIndexManagerImpl.java @@ -29,7 +29,7 @@ import org.hibernate.search.engine.backend.scope.spi.IndexScopeBuilder; import org.hibernate.search.engine.backend.session.spi.BackendSessionContext; import org.hibernate.search.engine.backend.session.spi.DetachedBackendSessionContext; -import org.hibernate.search.engine.backend.spi.SavedState; +import org.hibernate.search.engine.common.resources.spi.SavedState; import org.hibernate.search.engine.backend.work.execution.DocumentCommitStrategy; import org.hibernate.search.engine.backend.work.execution.DocumentRefreshStrategy; import org.hibernate.search.engine.backend.work.execution.spi.IndexIndexer; diff --git a/backend/lucene/src/main/java/org/hibernate/search/backend/lucene/index/impl/Shard.java b/backend/lucene/src/main/java/org/hibernate/search/backend/lucene/index/impl/Shard.java index 3b1be36aa0e..ca4b1620bf2 100644 --- a/backend/lucene/src/main/java/org/hibernate/search/backend/lucene/index/impl/Shard.java +++ b/backend/lucene/src/main/java/org/hibernate/search/backend/lucene/index/impl/Shard.java @@ -17,7 +17,7 @@ import org.hibernate.search.backend.lucene.orchestration.impl.LuceneParallelWorkOrchestratorImpl; import org.hibernate.search.backend.lucene.orchestration.impl.LuceneSerialWorkOrchestrator; import org.hibernate.search.backend.lucene.orchestration.impl.LuceneSerialWorkOrchestratorImpl; -import org.hibernate.search.engine.backend.spi.SavedState; +import org.hibernate.search.engine.common.resources.spi.SavedState; import org.hibernate.search.engine.cfg.ConfigurationPropertySource; import org.hibernate.search.util.common.impl.Closer; import org.hibernate.search.util.common.impl.SuppressingCloser; diff --git a/backend/lucene/src/main/java/org/hibernate/search/backend/lucene/index/impl/ShardHolder.java b/backend/lucene/src/main/java/org/hibernate/search/backend/lucene/index/impl/ShardHolder.java index 1534517d06c..0e3a7767881 100644 --- a/backend/lucene/src/main/java/org/hibernate/search/backend/lucene/index/impl/ShardHolder.java +++ b/backend/lucene/src/main/java/org/hibernate/search/backend/lucene/index/impl/ShardHolder.java @@ -27,7 +27,7 @@ import org.hibernate.search.backend.lucene.schema.management.impl.SchemaManagementIndexManagerContext; import org.hibernate.search.backend.lucene.work.execution.impl.WorkExecutionIndexManagerContext; import org.hibernate.search.engine.backend.index.spi.IndexManagerStartContext; -import org.hibernate.search.engine.backend.spi.SavedState; +import org.hibernate.search.engine.common.resources.spi.SavedState; import org.hibernate.search.engine.cfg.ConfigurationPropertySource; import org.hibernate.search.engine.environment.bean.BeanHolder; import org.hibernate.search.util.common.impl.Closer; diff --git a/backend/lucene/src/main/java/org/hibernate/search/backend/lucene/index/impl/ShardingStrategyInitializationContextImpl.java b/backend/lucene/src/main/java/org/hibernate/search/backend/lucene/index/impl/ShardingStrategyInitializationContextImpl.java index 6274b16c57d..ad101113dcc 100644 --- a/backend/lucene/src/main/java/org/hibernate/search/backend/lucene/index/impl/ShardingStrategyInitializationContextImpl.java +++ b/backend/lucene/src/main/java/org/hibernate/search/backend/lucene/index/impl/ShardingStrategyInitializationContextImpl.java @@ -23,7 +23,7 @@ import org.hibernate.search.backend.lucene.lowlevel.directory.spi.DirectoryProvider; import org.hibernate.search.backend.lucene.lowlevel.index.impl.IOStrategy; import org.hibernate.search.engine.backend.index.spi.IndexManagerStartContext; -import org.hibernate.search.engine.backend.spi.SavedState; +import org.hibernate.search.engine.common.resources.spi.SavedState; import org.hibernate.search.engine.cfg.spi.ConfigurationProperty; import org.hibernate.search.engine.cfg.ConfigurationPropertySource; import org.hibernate.search.engine.environment.bean.BeanHolder; diff --git a/engine/src/main/java/org/hibernate/search/engine/backend/index/spi/IndexManagerImplementor.java b/engine/src/main/java/org/hibernate/search/engine/backend/index/spi/IndexManagerImplementor.java index 685508bf1ae..24672213f7c 100644 --- a/engine/src/main/java/org/hibernate/search/engine/backend/index/spi/IndexManagerImplementor.java +++ b/engine/src/main/java/org/hibernate/search/engine/backend/index/spi/IndexManagerImplementor.java @@ -9,7 +9,7 @@ import java.util.concurrent.CompletableFuture; import org.hibernate.search.engine.backend.schema.management.spi.IndexSchemaManager; -import org.hibernate.search.engine.backend.spi.SavedState; +import org.hibernate.search.engine.common.resources.spi.SavedState; import org.hibernate.search.engine.backend.work.execution.DocumentCommitStrategy; import org.hibernate.search.engine.backend.work.execution.DocumentRefreshStrategy; import org.hibernate.search.engine.backend.index.IndexManager; diff --git a/engine/src/main/java/org/hibernate/search/engine/common/impl/IndexManagerNonStartedState.java b/engine/src/main/java/org/hibernate/search/engine/common/impl/IndexManagerNonStartedState.java index ba5ed9aceb7..1f393832ac1 100644 --- a/engine/src/main/java/org/hibernate/search/engine/common/impl/IndexManagerNonStartedState.java +++ b/engine/src/main/java/org/hibernate/search/engine/common/impl/IndexManagerNonStartedState.java @@ -7,7 +7,7 @@ package org.hibernate.search.engine.common.impl; import org.hibernate.search.engine.backend.index.spi.IndexManagerImplementor; -import org.hibernate.search.engine.backend.spi.SavedState; +import org.hibernate.search.engine.common.resources.spi.SavedState; import org.hibernate.search.engine.cfg.impl.ConfigurationPropertySourceExtractor; import org.hibernate.search.engine.cfg.ConfigurationPropertySource; import org.hibernate.search.engine.environment.bean.BeanResolver; diff --git a/engine/src/main/java/org/hibernate/search/engine/common/impl/SearchIntegrationImpl.java b/engine/src/main/java/org/hibernate/search/engine/common/impl/SearchIntegrationImpl.java index 4447469d355..1627b0be1b6 100644 --- a/engine/src/main/java/org/hibernate/search/engine/common/impl/SearchIntegrationImpl.java +++ b/engine/src/main/java/org/hibernate/search/engine/common/impl/SearchIntegrationImpl.java @@ -18,7 +18,7 @@ import org.hibernate.search.engine.backend.index.IndexManager; import org.hibernate.search.engine.backend.index.spi.IndexManagerImplementor; import org.hibernate.search.engine.backend.spi.BackendImplementor; -import org.hibernate.search.engine.backend.spi.SavedState; +import org.hibernate.search.engine.common.resources.spi.SavedState; import org.hibernate.search.engine.common.resources.impl.EngineThreads; import org.hibernate.search.engine.common.spi.SearchIntegration; import org.hibernate.search.engine.common.spi.SearchIntegrationEnvironment; diff --git a/engine/src/main/java/org/hibernate/search/engine/common/impl/SearchIntegrationPartialBuildStateImpl.java b/engine/src/main/java/org/hibernate/search/engine/common/impl/SearchIntegrationPartialBuildStateImpl.java index d8085dbe15a..feb4da904be 100644 --- a/engine/src/main/java/org/hibernate/search/engine/common/impl/SearchIntegrationPartialBuildStateImpl.java +++ b/engine/src/main/java/org/hibernate/search/engine/common/impl/SearchIntegrationPartialBuildStateImpl.java @@ -16,7 +16,7 @@ import org.hibernate.search.engine.backend.index.spi.IndexManagerImplementor; import org.hibernate.search.engine.backend.spi.BackendImplementor; -import org.hibernate.search.engine.backend.spi.SavedState; +import org.hibernate.search.engine.common.resources.spi.SavedState; import org.hibernate.search.engine.cfg.spi.ConfigurationPropertyChecker; import org.hibernate.search.engine.cfg.ConfigurationPropertySource; import org.hibernate.search.engine.common.resources.impl.EngineThreads; diff --git a/engine/src/main/java/org/hibernate/search/engine/backend/spi/SavedState.java b/engine/src/main/java/org/hibernate/search/engine/common/resources/spi/SavedState.java similarity index 98% rename from engine/src/main/java/org/hibernate/search/engine/backend/spi/SavedState.java rename to engine/src/main/java/org/hibernate/search/engine/common/resources/spi/SavedState.java index e4cdf4fa77d..d81e913c579 100644 --- a/engine/src/main/java/org/hibernate/search/engine/backend/spi/SavedState.java +++ b/engine/src/main/java/org/hibernate/search/engine/common/resources/spi/SavedState.java @@ -4,7 +4,7 @@ * License: GNU Lesser General Public License (LGPL), version 2.1 or later * See the lgpl.txt file in the root directory or . */ -package org.hibernate.search.engine.backend.spi; +package org.hibernate.search.engine.common.resources.spi; import java.util.LinkedHashMap; import java.util.Map; From 5bceb66b1d8715e7237c16e98d08236a303133e4 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Yoann=20Rodi=C3=A8re?= Date: Fri, 6 May 2022 15:58:50 +0200 Subject: [PATCH 12/14] HSEARCH-4566 Don't expose SearchIntegrationImpl#saveForRestart more than necessary --- .../search/engine/common/impl/SearchIntegrationImpl.java | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/engine/src/main/java/org/hibernate/search/engine/common/impl/SearchIntegrationImpl.java b/engine/src/main/java/org/hibernate/search/engine/common/impl/SearchIntegrationImpl.java index 1627b0be1b6..fdd427b5119 100644 --- a/engine/src/main/java/org/hibernate/search/engine/common/impl/SearchIntegrationImpl.java +++ b/engine/src/main/java/org/hibernate/search/engine/common/impl/SearchIntegrationImpl.java @@ -101,7 +101,7 @@ public IndexManager indexManager(String indexManagerName) { return indexManager.toAPI(); } - public SavedState saveForRestart() { + SavedState saveForRestart() { HashMap states = new HashMap<>(); for ( Map.Entry indexManager : indexManagers.entrySet() ) { states.put( indexManager.getKey(), indexManager.getValue().saveForRestart() ); From 463ce2d63259eca96c93462efd2903cc80a0e4eb Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Yoann=20Rodi=C3=A8re?= Date: Fri, 6 May 2022 16:00:11 +0200 Subject: [PATCH 13/14] HSEARCH-4566 Fix javadoc --- .../engine/backend/index/spi/IndexManagerImplementor.java | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/engine/src/main/java/org/hibernate/search/engine/backend/index/spi/IndexManagerImplementor.java b/engine/src/main/java/org/hibernate/search/engine/backend/index/spi/IndexManagerImplementor.java index 24672213f7c..c3c75984702 100644 --- a/engine/src/main/java/org/hibernate/search/engine/backend/index/spi/IndexManagerImplementor.java +++ b/engine/src/main/java/org/hibernate/search/engine/backend/index/spi/IndexManagerImplementor.java @@ -54,8 +54,7 @@ default void preStart(IndexManagerStartContext context, SavedState savedState) { * Start any resource necessary to operate the index manager at runtime. *

* Called by the engine once just after - * {@link #preStart(IndexManagerStartContext, SavedState)} - * was called on the corresponding backend. + * {@link #preStart(IndexManagerStartContext, SavedState)}. * * @param context The start context. */ From 80fd40dada1861508134910be3ec65fbdbe1b923 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Yoann=20Rodi=C3=A8re?= Date: Mon, 9 May 2022 12:47:15 +0200 Subject: [PATCH 14/14] HSEARCH-4566 Simplify shard startup in Lucene backend --- .../impl/IndexManagerBackendContext.java | 38 ++---- .../backend/lucene/index/impl/Shard.java | 115 ++++++++++++------ .../lucene/index/impl/ShardHolder.java | 55 ++++++--- ...dingStrategyInitializationContextImpl.java | 72 +---------- .../backend/lucene/logging/impl/Log.java | 8 +- .../index/impl/IndexAccessorImpl.java | 24 +--- .../index/impl/IndexAccessorTest.java | 2 - .../reporting/spi/RootFailureCollector.java | 6 + 8 files changed, 147 insertions(+), 173 deletions(-) diff --git a/backend/lucene/src/main/java/org/hibernate/search/backend/lucene/index/impl/IndexManagerBackendContext.java b/backend/lucene/src/main/java/org/hibernate/search/backend/lucene/index/impl/IndexManagerBackendContext.java index 5fd57cd5177..5cceaa1d99e 100644 --- a/backend/lucene/src/main/java/org/hibernate/search/backend/lucene/index/impl/IndexManagerBackendContext.java +++ b/backend/lucene/src/main/java/org/hibernate/search/backend/lucene/index/impl/IndexManagerBackendContext.java @@ -53,7 +53,6 @@ import org.hibernate.search.engine.common.timing.spi.TimingSource; import org.hibernate.search.engine.reporting.FailureHandler; import org.hibernate.search.engine.search.loading.spi.SearchLoadingContextBuilder; -import org.hibernate.search.util.common.impl.SuppressingCloser; import org.hibernate.search.util.common.reporting.EventContext; import org.apache.lucene.search.similarities.Similarity; @@ -196,40 +195,19 @@ LuceneIndexSchemaManager createSchemaManager(SchemaManagementIndexManagerContext return new LuceneIndexSchemaManager( workFactory, context ); } - Shard createShard(LuceneIndexModel model, EventContext shardEventContext, DirectoryHolder directoryHolder, - IOStrategy ioStrategy, ConfigurationPropertySource propertySource, - boolean reuseAlreadyStaredDirectoryHolder) { - LuceneParallelWorkOrchestratorImpl managementOrchestrator; - LuceneSerialWorkOrchestratorImpl indexingOrchestrator; - IndexAccessorImpl indexAccessor = null; + IndexAccessorImpl createIndexAccessor(LuceneIndexModel model, EventContext shardEventContext, + DirectoryHolder directoryHolder, IOStrategy ioStrategy, + ConfigurationPropertySource propertySource) { String indexName = model.hibernateSearchName(); IndexWriterConfigSource writerConfigSource = IndexWriterConfigSource.create( similarity, model.getIndexingAnalyzer(), propertySource, shardEventContext ); - - try { - indexAccessor = ioStrategy.createIndexAccessor( - indexName, shardEventContext, directoryHolder, writerConfigSource - ); - managementOrchestrator = createIndexManagementOrchestrator( shardEventContext, indexAccessor ); - indexingOrchestrator = createIndexingOrchestrator( shardEventContext, indexAccessor ); - - Shard shard = new Shard( - shardEventContext, indexAccessor, - managementOrchestrator, indexingOrchestrator, - reuseAlreadyStaredDirectoryHolder - ); - return shard; - } - catch (RuntimeException e) { - new SuppressingCloser( e ) - // No need to stop the orchestrators, we didn't start them - .push( indexAccessor ); - throw e; - } + return ioStrategy.createIndexAccessor( + indexName, shardEventContext, directoryHolder, writerConfigSource + ); } - private LuceneParallelWorkOrchestratorImpl createIndexManagementOrchestrator(EventContext eventContext, + LuceneParallelWorkOrchestratorImpl createIndexManagementOrchestrator(EventContext eventContext, IndexAccessorImpl indexAccessor) { return new LuceneParallelWorkOrchestratorImpl( "Lucene index management orchestrator for " + eventContext.render(), @@ -239,7 +217,7 @@ private LuceneParallelWorkOrchestratorImpl createIndexManagementOrchestrator(Eve ); } - private LuceneSerialWorkOrchestratorImpl createIndexingOrchestrator(EventContext eventContext, + LuceneSerialWorkOrchestratorImpl createIndexingOrchestrator(EventContext eventContext, IndexAccessorImpl indexAccessor) { return new LuceneSerialWorkOrchestratorImpl( "Lucene indexing orchestrator for " + eventContext.render(), diff --git a/backend/lucene/src/main/java/org/hibernate/search/backend/lucene/index/impl/Shard.java b/backend/lucene/src/main/java/org/hibernate/search/backend/lucene/index/impl/Shard.java index ca4b1620bf2..53fb384e78b 100644 --- a/backend/lucene/src/main/java/org/hibernate/search/backend/lucene/index/impl/Shard.java +++ b/backend/lucene/src/main/java/org/hibernate/search/backend/lucene/index/impl/Shard.java @@ -8,19 +8,30 @@ import java.io.IOException; import java.lang.invoke.MethodHandles; +import java.util.Optional; import java.util.concurrent.CompletableFuture; +import org.hibernate.search.backend.lucene.cfg.LuceneIndexSettings; +import org.hibernate.search.backend.lucene.document.model.impl.LuceneIndexModel; import org.hibernate.search.backend.lucene.logging.impl.Log; +import org.hibernate.search.backend.lucene.lowlevel.directory.impl.DirectoryCreationContextImpl; +import org.hibernate.search.backend.lucene.lowlevel.directory.spi.DirectoryCreationContext; import org.hibernate.search.backend.lucene.lowlevel.directory.spi.DirectoryHolder; +import org.hibernate.search.backend.lucene.lowlevel.directory.spi.DirectoryProvider; +import org.hibernate.search.backend.lucene.lowlevel.index.impl.IOStrategy; import org.hibernate.search.backend.lucene.lowlevel.index.impl.IndexAccessorImpl; import org.hibernate.search.backend.lucene.orchestration.impl.LuceneParallelWorkOrchestrator; import org.hibernate.search.backend.lucene.orchestration.impl.LuceneParallelWorkOrchestratorImpl; import org.hibernate.search.backend.lucene.orchestration.impl.LuceneSerialWorkOrchestrator; import org.hibernate.search.backend.lucene.orchestration.impl.LuceneSerialWorkOrchestratorImpl; +import org.hibernate.search.engine.cfg.spi.ConfigurationProperty; import org.hibernate.search.engine.common.resources.spi.SavedState; import org.hibernate.search.engine.cfg.ConfigurationPropertySource; +import org.hibernate.search.engine.environment.bean.BeanHolder; +import org.hibernate.search.engine.environment.bean.BeanReference; +import org.hibernate.search.engine.environment.bean.BeanResolver; +import org.hibernate.search.engine.reporting.spi.EventContexts; import org.hibernate.search.util.common.impl.Closer; -import org.hibernate.search.util.common.impl.SuppressingCloser; import org.hibernate.search.util.common.logging.impl.LoggerFactory; import org.hibernate.search.util.common.reporting.EventContext; @@ -28,33 +39,37 @@ public final class Shard { - static final SavedState.Key DIRECTORY_HOLDER_KEY = SavedState.key( "directory_holder" ); + private static final ConfigurationProperty> DIRECTORY_TYPE = + ConfigurationProperty.forKey( LuceneIndexSettings.DIRECTORY_TYPE ) + .asBeanReference( DirectoryProvider.class ) + .withDefault( BeanReference.of( DirectoryProvider.class, LuceneIndexSettings.Defaults.DIRECTORY_TYPE ) ) + .build(); + + private static final SavedState.Key DIRECTORY_HOLDER_KEY = SavedState.key( "directory_holder" ); private static final Log log = LoggerFactory.make( Log.class, MethodHandles.lookup() ); - private final EventContext eventContext; - private final IndexAccessorImpl indexAccessor; - private final LuceneParallelWorkOrchestratorImpl managementOrchestrator; - private final LuceneSerialWorkOrchestratorImpl indexingOrchestrator; - private final boolean reuseAlreadyStaredDirectoryHolder; + private final Optional shardId; + private final IndexManagerBackendContext backendContext; + private final LuceneIndexModel model; + + private DirectoryHolder directoryHolder; + private IndexAccessorImpl indexAccessor; + private LuceneParallelWorkOrchestratorImpl managementOrchestrator; + private LuceneSerialWorkOrchestratorImpl indexingOrchestrator; private boolean savedForRestart = false; - Shard(EventContext eventContext, IndexAccessorImpl indexAccessor, - LuceneParallelWorkOrchestratorImpl managementOrchestrator, - LuceneSerialWorkOrchestratorImpl indexingOrchestrator, - boolean reuseAlreadyStaredDirectoryHolder) { - this.eventContext = eventContext; - this.indexAccessor = indexAccessor; - this.managementOrchestrator = managementOrchestrator; - this.indexingOrchestrator = indexingOrchestrator; - this.reuseAlreadyStaredDirectoryHolder = reuseAlreadyStaredDirectoryHolder; + Shard(Optional shardId, IndexManagerBackendContext backendContext, LuceneIndexModel model) { + this.shardId = shardId; + this.backendContext = backendContext; + this.model = model; } public SavedState saveForRestart() { try { return SavedState.builder() - .put( DIRECTORY_HOLDER_KEY, indexAccessor.directoryHolder(), DirectoryHolder::close ) + .put( DIRECTORY_HOLDER_KEY, directoryHolder, DirectoryHolder::close ) .build(); } finally { @@ -62,24 +77,47 @@ public SavedState saveForRestart() { } } - void start(ConfigurationPropertySource propertySource) { + void preStart(ConfigurationPropertySource propertySource, BeanResolver beanResolver, SavedState savedState) { + Optional savedDirectoryHolder = savedState.get( Shard.DIRECTORY_HOLDER_KEY ); try { - if ( !reuseAlreadyStaredDirectoryHolder ) { - indexAccessor.start(); + if ( savedDirectoryHolder.isPresent() ) { + directoryHolder = savedDirectoryHolder.get(); + } + else { + try ( BeanHolder directoryProviderHolder = + DIRECTORY_TYPE.getAndTransform( propertySource, beanResolver::resolve ) ) { + String indexName = model.hibernateSearchName(); + EventContext indexAndShardEventContext = EventContexts.fromIndexNameAndShardId( indexName, shardId ); + DirectoryCreationContext context = new DirectoryCreationContextImpl( indexAndShardEventContext, + indexName, shardId, beanResolver, + propertySource.withMask( "directory" ) ); + directoryHolder = directoryProviderHolder.get().createDirectoryHolder( context ); + } + directoryHolder.start(); } + } + catch (IOException | RuntimeException e) { + throw log.unableToStartShard( e.getMessage(), e ); + } + } + + void start(ConfigurationPropertySource propertySource) { + String indexName = model.hibernateSearchName(); + EventContext indexAndShardEventContext = EventContexts.fromIndexNameAndShardId( indexName, shardId ); + try { + IOStrategy ioStrategy = backendContext.createIOStrategy( propertySource ); + indexAccessor = backendContext.createIndexAccessor( model, indexAndShardEventContext, directoryHolder, + ioStrategy, propertySource ); + managementOrchestrator = + backendContext.createIndexManagementOrchestrator( indexAndShardEventContext, indexAccessor ); + indexingOrchestrator = + backendContext.createIndexingOrchestrator( indexAndShardEventContext, indexAccessor ); + managementOrchestrator.start( propertySource ); indexingOrchestrator.start( propertySource ); } - catch (IOException | RuntimeException e) { - new SuppressingCloser( e ) - .push( indexAccessor ) - .push( LuceneSerialWorkOrchestratorImpl::stop, indexingOrchestrator ) - .push( LuceneParallelWorkOrchestratorImpl::stop, managementOrchestrator ); - throw log.unableToInitializeIndexDirectory( - e.getMessage(), - eventContext, - e - ); + catch (RuntimeException e) { + throw log.unableToStartShard( e.getMessage(), e ); } } @@ -88,17 +126,22 @@ CompletableFuture preStop() { } void stop() { - try ( Closer closer = new Closer<>() ) { + try ( Closer closer = new Closer<>() ) { closer.push( LuceneSerialWorkOrchestratorImpl::stop, indexingOrchestrator ); closer.push( LuceneParallelWorkOrchestratorImpl::stop, managementOrchestrator ); // Close the index writer after the orchestrators, when we're sure all works have been performed - if ( savedForRestart ) { - closer.push( IndexAccessorImpl::closeNoDir, indexAccessor ); - } - else { - closer.push( IndexAccessorImpl::close, indexAccessor ); + closer.push( IndexAccessorImpl::close, indexAccessor ); + if ( !savedForRestart ) { + closer.push( DirectoryHolder::close, directoryHolder ); } } + catch (RuntimeException | IOException e) { + throw log.unableToShutdownShard( + e.getMessage(), + shardId.map( EventContexts::fromShardId ).orElse( null ), + e + ); + } } DirectoryReader openReader() throws IOException { diff --git a/backend/lucene/src/main/java/org/hibernate/search/backend/lucene/index/impl/ShardHolder.java b/backend/lucene/src/main/java/org/hibernate/search/backend/lucene/index/impl/ShardHolder.java index 0e3a7767881..7238cd9ac82 100644 --- a/backend/lucene/src/main/java/org/hibernate/search/backend/lucene/index/impl/ShardHolder.java +++ b/backend/lucene/src/main/java/org/hibernate/search/backend/lucene/index/impl/ShardHolder.java @@ -18,6 +18,7 @@ import java.util.Set; import java.util.concurrent.CompletableFuture; +import org.hibernate.search.backend.lucene.cfg.LuceneIndexSettings; import org.hibernate.search.backend.lucene.document.model.impl.LuceneIndexModel; import org.hibernate.search.backend.lucene.index.spi.ShardingStrategy; import org.hibernate.search.backend.lucene.lowlevel.reader.impl.DirectoryReaderCollector; @@ -30,6 +31,7 @@ import org.hibernate.search.engine.common.resources.spi.SavedState; import org.hibernate.search.engine.cfg.ConfigurationPropertySource; import org.hibernate.search.engine.environment.bean.BeanHolder; +import org.hibernate.search.engine.reporting.spi.EventContexts; import org.hibernate.search.util.common.impl.Closer; import org.hibernate.search.util.common.impl.SuppressingCloser; @@ -63,15 +65,36 @@ public SavedState saveForRestart() { return SavedState.builder().put( SHARDS_KEY, states ).build(); } - void preStart(IndexManagerStartContext startContext, SavedState savedState) { - ConfigurationPropertySource propertySource = startContext.configurationPropertySource(); + private ConfigurationPropertySource toShardPropertySource(ConfigurationPropertySource indexPropertySource, String shardIdOrNull) { + return shardIdOrNull != null + ? indexPropertySource.withMask( LuceneIndexSettings.SHARDS ).withMask( shardIdOrNull ) + .withFallback( indexPropertySource ) + : indexPropertySource; + } + void preStart(IndexManagerStartContext startContext, SavedState savedState) { + ConfigurationPropertySource indexPropertySource = startContext.configurationPropertySource(); try { ShardingStrategyInitializationContextImpl initializationContext = - new ShardingStrategyInitializationContextImpl( backendContext, model, startContext, propertySource ); + new ShardingStrategyInitializationContextImpl( backendContext, model, startContext, indexPropertySource ); Map states = savedState.get( SHARDS_KEY ).orElse( Collections.emptyMap() ); - this.shardingStrategyHolder = initializationContext.create( shards, states ); + this.shardingStrategyHolder = initializationContext.create( shards ); + + for ( Map.Entry entry : shards.entrySet() ) { + String shardId = entry.getKey(); + Shard shard = entry.getValue(); + ConfigurationPropertySource shardPropertySource = toShardPropertySource( indexPropertySource, shardId ); + try { + shard.preStart( shardPropertySource, startContext.beanResolver(), + states.getOrDefault( entry.getKey(), SavedState.empty() ) ); + } + catch (RuntimeException e) { + startContext.failureCollector() + .withContext( shardId == null ? null : EventContexts.fromShardId( shardId ) ) + .add( e ); + } + } } catch (RuntimeException e) { new SuppressingCloser( e ) @@ -82,17 +105,21 @@ void preStart(IndexManagerStartContext startContext, SavedState savedState) { } void start(IndexManagerStartContext startContext) { - if ( startContext.failureCollector().hasFailure() ) { - // At least one shard creation failed; abort and don't even try to start shards. - return; - } - - ConfigurationPropertySource propertySource = startContext.configurationPropertySource(); - + ConfigurationPropertySource indexPropertySource = startContext.configurationPropertySource(); try { - for ( Shard shard : shards.values() ) { - shard.start( propertySource ); - managementOrchestrators.add( shard.managementOrchestrator() ); + for ( Map.Entry entry : shards.entrySet() ) { + String shardId = entry.getKey(); + Shard shard = entry.getValue(); + ConfigurationPropertySource shardPropertySource = toShardPropertySource( indexPropertySource, shardId ); + try { + shard.start( shardPropertySource ); + managementOrchestrators.add( shard.managementOrchestrator() ); + } + catch (RuntimeException e) { + startContext.failureCollector() + .withContext( shardId == null ? null : EventContexts.fromShardId( shardId ) ) + .add( e ); + } } } catch (RuntimeException e) { diff --git a/backend/lucene/src/main/java/org/hibernate/search/backend/lucene/index/impl/ShardingStrategyInitializationContextImpl.java b/backend/lucene/src/main/java/org/hibernate/search/backend/lucene/index/impl/ShardingStrategyInitializationContextImpl.java index ad101113dcc..477325c3f9e 100644 --- a/backend/lucene/src/main/java/org/hibernate/search/backend/lucene/index/impl/ShardingStrategyInitializationContextImpl.java +++ b/backend/lucene/src/main/java/org/hibernate/search/backend/lucene/index/impl/ShardingStrategyInitializationContextImpl.java @@ -17,23 +17,13 @@ import org.hibernate.search.backend.lucene.index.spi.ShardingStrategy; import org.hibernate.search.backend.lucene.index.spi.ShardingStrategyInitializationContext; import org.hibernate.search.backend.lucene.logging.impl.Log; -import org.hibernate.search.backend.lucene.lowlevel.directory.impl.DirectoryCreationContextImpl; -import org.hibernate.search.backend.lucene.lowlevel.directory.spi.DirectoryCreationContext; -import org.hibernate.search.backend.lucene.lowlevel.directory.spi.DirectoryHolder; -import org.hibernate.search.backend.lucene.lowlevel.directory.spi.DirectoryProvider; -import org.hibernate.search.backend.lucene.lowlevel.index.impl.IOStrategy; import org.hibernate.search.engine.backend.index.spi.IndexManagerStartContext; -import org.hibernate.search.engine.common.resources.spi.SavedState; import org.hibernate.search.engine.cfg.spi.ConfigurationProperty; import org.hibernate.search.engine.cfg.ConfigurationPropertySource; import org.hibernate.search.engine.environment.bean.BeanHolder; import org.hibernate.search.engine.environment.bean.BeanReference; import org.hibernate.search.engine.environment.bean.BeanResolver; -import org.hibernate.search.engine.reporting.spi.ContextualFailureCollector; -import org.hibernate.search.engine.reporting.spi.EventContexts; -import org.hibernate.search.util.common.impl.SuppressingCloser; import org.hibernate.search.util.common.logging.impl.LoggerFactory; -import org.hibernate.search.util.common.reporting.EventContext; class ShardingStrategyInitializationContextImpl implements ShardingStrategyInitializationContext { @@ -47,16 +37,9 @@ class ShardingStrategyInitializationContextImpl implements ShardingStrategyIniti ) ) .build(); - private static final ConfigurationProperty> DIRECTORY_TYPE = - ConfigurationProperty.forKey( LuceneIndexSettings.DIRECTORY_TYPE ) - .asBeanReference( DirectoryProvider.class ) - .withDefault( BeanReference.of( DirectoryProvider.class, LuceneIndexSettings.Defaults.DIRECTORY_TYPE ) ) - .build(); - private final IndexManagerBackendContext backendContext; private final LuceneIndexModel model; private final IndexManagerStartContext startContext; - private final ConfigurationPropertySource indexPropertySource; private final ConfigurationPropertySource shardingPropertySource; private Set shardIdentifiers = new LinkedHashSet<>(); @@ -67,7 +50,6 @@ class ShardingStrategyInitializationContextImpl implements ShardingStrategyIniti this.backendContext = backendContext; this.model = model; this.startContext = startContext; - this.indexPropertySource = indexPropertySource; this.shardingPropertySource = indexPropertySource.withMask( "sharding" ); } @@ -97,8 +79,7 @@ public ConfigurationPropertySource configurationPropertySource() { return shardingPropertySource; } - public BeanHolder create(Map shardCollector, - Map states) { + public BeanHolder create(Map shardCollector) { BeanHolder shardingStrategyHolder = SHARDING_STRATEGY.getAndTransform( shardingPropertySource, beanResolver()::resolve ); @@ -106,8 +87,7 @@ public BeanHolder create(Map shardCol if ( shardIdentifiers == null ) { // Sharding is disabled => single shard - contributeShardWithSilentFailure( shardCollector, Optional.empty(), - states.getOrDefault( null, SavedState.empty() ) ); + contributeShard( shardCollector, Optional.empty() ); return null; } @@ -118,54 +98,14 @@ public BeanHolder create(Map shardCol } for ( String shardIdentifier : shardIdentifiers ) { - contributeShardWithSilentFailure( shardCollector, Optional.of( shardIdentifier ), - states.getOrDefault( shardIdentifier, SavedState.empty() ) ); + contributeShard( shardCollector, Optional.of( shardIdentifier ) ); } return shardingStrategyHolder; } - private void contributeShardWithSilentFailure(Map shardCollector, Optional shardId, - SavedState savedState) { - EventContext shardEventContext = EventContexts.fromIndexNameAndShardId( indexName(), shardId ); - ConfigurationPropertySource shardPropertySource = - shardId.isPresent() ? - indexPropertySource.withMask( LuceneIndexSettings.SHARDS ).withMask( shardId.get() ) - .withFallback( indexPropertySource ) - : indexPropertySource; - - - Optional savedDirectoryHolder = savedState.get( Shard.DIRECTORY_HOLDER_KEY ); - DirectoryHolder directoryHolder = null; - boolean reuseAlreadyStaredDirectoryHolder = false; - - try { - if ( savedDirectoryHolder.isPresent() ) { - directoryHolder = savedDirectoryHolder.get(); - reuseAlreadyStaredDirectoryHolder = true; - } - else { - try ( BeanHolder directoryProviderHolder = - DIRECTORY_TYPE.getAndTransform( shardPropertySource, startContext.beanResolver()::resolve ) ) { - DirectoryCreationContext context = new DirectoryCreationContextImpl( shardEventContext, - indexName(), shardId, beanResolver(), shardPropertySource.withMask( "directory" ) ); - directoryHolder = directoryProviderHolder.get().createDirectoryHolder( context ); - } - } - - IOStrategy ioStrategy = backendContext.createIOStrategy( shardPropertySource ); - Shard shard = backendContext.createShard( model, shardEventContext, directoryHolder, ioStrategy, - shardPropertySource, reuseAlreadyStaredDirectoryHolder ); - shardCollector.put( shardId.orElse( null ), shard ); - } - catch (RuntimeException e) { - new SuppressingCloser( e ).push( directoryHolder ); - - ContextualFailureCollector failureCollector = startContext.failureCollector(); - if ( shardId.isPresent() ) { - failureCollector = failureCollector.withContext( EventContexts.fromShardId( shardId.get() ) ); - } - failureCollector.add( e ); - } + private void contributeShard(Map shardCollector, Optional shardId) { + Shard shard = new Shard( shardId, backendContext, model ); + shardCollector.put( shardId.orElse( null ), shard ); } } diff --git a/backend/lucene/src/main/java/org/hibernate/search/backend/lucene/logging/impl/Log.java b/backend/lucene/src/main/java/org/hibernate/search/backend/lucene/logging/impl/Log.java index 1d99b1e9c67..b293db8056d 100644 --- a/backend/lucene/src/main/java/org/hibernate/search/backend/lucene/logging/impl/Log.java +++ b/backend/lucene/src/main/java/org/hibernate/search/backend/lucene/logging/impl/Log.java @@ -299,8 +299,8 @@ SearchException indexManagerUnwrappingWithUnknownType(@FormatWith(ClassFormatter value = "Invalid search projection: '%1$s'. You must build the projection from a Lucene search scope.") SearchException cannotMixLuceneSearchQueryWithOtherProjections(SearchProjection projection); - @Message(id = ID_OFFSET + 61, value = "Unable to shut down index accessor: %1$s") - SearchException unableToShutdownIndexAccessor(String causeMessage, @Cause Exception cause); + @Message(id = ID_OFFSET + 61, value = "Unable to shut down index: %1$s") + SearchException unableToShutdownShard(String causeMessage, @Param EventContext context, @Cause Exception cause); @Message(id = ID_OFFSET + 62, value = "No built-in index field type for class: '%1$s'.") SearchException cannotGuessFieldType(@FormatWith(ClassFormatter.class) Class inputType, @Param EventContext context); @@ -592,4 +592,8 @@ SearchException indexSchemaNamedPredicateNameConflict(String relativeFilterName, value = "Offset + limit should be lower than Integer.MAX_VALUE, offset: '%1$s', limit: '%2$s'.") IOException offsetLimitExceedsMaxValue(int offset, Integer limit); + @Message(id = ID_OFFSET + 154, + value = "Unable to start index: %1$s") + SearchException unableToStartShard(String causeMessage, @Cause Exception cause); + } diff --git a/backend/lucene/src/main/java/org/hibernate/search/backend/lucene/lowlevel/index/impl/IndexAccessorImpl.java b/backend/lucene/src/main/java/org/hibernate/search/backend/lucene/lowlevel/index/impl/IndexAccessorImpl.java index 4e6c2e4ae8d..34797fbb053 100644 --- a/backend/lucene/src/main/java/org/hibernate/search/backend/lucene/lowlevel/index/impl/IndexAccessorImpl.java +++ b/backend/lucene/src/main/java/org/hibernate/search/backend/lucene/lowlevel/index/impl/IndexAccessorImpl.java @@ -49,30 +49,12 @@ public IndexAccessorImpl(EventContext eventContext, this.indexReaderProvider = indexReaderProvider; } - public void start() throws IOException { - directoryHolder.start(); - } - @Override - public void close() { - try ( Closer closer = new Closer<>() ) { - closer.push( IndexWriterProvider::clear, indexWriterProvider ); - closer.push( IndexReaderProvider::clear, indexReaderProvider ); - closer.push( DirectoryHolder::close, directoryHolder ); - } - catch (RuntimeException | IOException e) { - throw log.unableToShutdownIndexAccessor( e.getMessage(), e ); - } - } - - public void closeNoDir() { + public void close() throws IOException { try ( Closer closer = new Closer<>() ) { closer.push( IndexWriterProvider::clear, indexWriterProvider ); closer.push( IndexReaderProvider::clear, indexReaderProvider ); } - catch (RuntimeException | IOException e) { - throw log.unableToShutdownIndexAccessor( e.getMessage(), e ); - } } @Override @@ -216,10 +198,6 @@ public long computeSizeInBytes() { return totalSize; } - public DirectoryHolder directoryHolder() { - return directoryHolder; - } - public Directory getDirectoryForTests() { return directoryHolder.get(); } diff --git a/backend/lucene/src/test/java/org/hibernate/search/backend/lucene/lowlevel/index/impl/IndexAccessorTest.java b/backend/lucene/src/test/java/org/hibernate/search/backend/lucene/lowlevel/index/impl/IndexAccessorTest.java index 9522acbb08e..7fa2faf1f2f 100644 --- a/backend/lucene/src/test/java/org/hibernate/search/backend/lucene/lowlevel/index/impl/IndexAccessorTest.java +++ b/backend/lucene/src/test/java/org/hibernate/search/backend/lucene/lowlevel/index/impl/IndexAccessorTest.java @@ -64,8 +64,6 @@ public class IndexAccessorTest { public void start() throws IOException { accessor = new IndexAccessorImpl( indexEventContext, directoryHolderMock, indexWriterProviderMock, indexReaderProviderMock ); - accessor.start(); - verify( directoryHolderMock ).start(); } @After diff --git a/engine/src/main/java/org/hibernate/search/engine/reporting/spi/RootFailureCollector.java b/engine/src/main/java/org/hibernate/search/engine/reporting/spi/RootFailureCollector.java index 973fdf610b2..4937ae57e81 100644 --- a/engine/src/main/java/org/hibernate/search/engine/reporting/spi/RootFailureCollector.java +++ b/engine/src/main/java/org/hibernate/search/engine/reporting/spi/RootFailureCollector.java @@ -98,6 +98,9 @@ protected NonRootFailureCollector(NonRootFailureCollector parent) { @Override public synchronized ContextualFailureCollectorImpl withContext(EventContext context) { + if ( context == null ) { + return withDefaultContext(); + } List elements = context.elements(); try { NonRootFailureCollector failureCollector = this; @@ -116,6 +119,9 @@ public synchronized ContextualFailureCollectorImpl withContext(EventContext cont @Override public synchronized ContextualFailureCollectorImpl withContext(EventContextElement contextElement) { + if ( contextElement == null ) { + return withDefaultContext(); + } return children.computeIfAbsent( contextElement, element -> new ContextualFailureCollectorImpl( this, element )