HSEARCH-2675 In the JGroups backend, create the delegate backend on d…

…emand

So that we support dynamic master node re-election. Previously the
delegate backend was only created if the current node was initially a
master, preventing a former slave to take on the role of master.

backends/jgroups/src/main/java/org/hibernate/search/backend/jgroups/impl/JGroupsBackend.java

@@ -131,17 +131,16 @@ public JGroupsBackendQueueProcessor createQueueProcessor(IndexManager indexManag
 				messageSender, indexManager, masterNodeSelector, luceneWorkSerializer,
 				block, messageTimeout );
 
-		BackendQueueProcessor delegateQueueProcessor = null;
-		if ( selectionStrategy.isIndexOwnerLocal() ) {
-			delegateQueueProcessor = BackendFactory.createBackend(
-					delegateBackendName, indexManager, context, properties );
-		}
-
 		JGroupsBackendQueueProcessor queueProcessor = new JGroupsBackendQueueProcessor(
-				selectionStrategy, jgroupsProcessor, delegateQueueProcessor );
+				selectionStrategy, jgroupsProcessor,
+				() -> createDelegateQueueProcessor( indexManager, context ) );
 		return queueProcessor;
 	}
 
+	private BackendQueueProcessor createDelegateQueueProcessor(IndexManager indexManager, WorkerBuildContext context) {
+		return BackendFactory.createBackend(delegateBackendName, indexManager, context, properties );
+	}
+
 	protected NodeSelectorStrategy createNodeSelectorStrategy(IndexManager indexManager) {
 		return new AutoNodeSelector( indexManager.getIndexName() );
 	}

backends/jgroups/src/main/java/org/hibernate/search/backend/jgroups/impl/JGroupsBackendQueueProcessor.java

@@ -8,6 +8,7 @@
 
 import java.util.Collections;
 import java.util.List;
+import java.util.function.Supplier;
 
 import org.hibernate.search.backend.IndexingMonitor;
 import org.hibernate.search.backend.LuceneWork;
@@ -26,28 +27,36 @@ public class JGroupsBackendQueueProcessor implements BackendQueueProcessor {
 
 	private final NodeSelectorStrategy selectionStrategy;
 	private final JGroupsBackendQueueTask jgroupsProcessor;
-	private final BackendQueueProcessor delegatedBackend;
+	private final Supplier<BackendQueueProcessor> delegatedBackendFactory;
+	private volatile BackendQueueProcessor delegate;
 
 	public JGroupsBackendQueueProcessor(NodeSelectorStrategy selectionStrategy,
 			JGroupsBackendQueueTask jgroupsProcessor,
-			BackendQueueProcessor delegatedBackend) {
+			Supplier<BackendQueueProcessor> delegatedBackendFactory) {
 		this.selectionStrategy = selectionStrategy;
 		this.jgroupsProcessor = jgroupsProcessor;
-		this.delegatedBackend = delegatedBackend;
+		this.delegatedBackendFactory = delegatedBackendFactory;
+		if ( selectionStrategy.isIndexOwnerLocal() ) {
+			/*
+			 * Eager initialization if we know from the start we are the master.
+			 * This allows in particular the delegate backend to fail fast
+			 * if there is a configuration issue.
+			 */
+			getOrCreateDelegate();
+		}
 	}
 
 	@Override
-	public void close() {
-		if ( selectionStrategy.isIndexOwnerLocal() ) {
-			//TODO verify all delegates have been closed when ownership was lost before [HSEARCH-2060]
-			delegatedBackend.close();
+	public synchronized void close() {
+		if ( delegate != null ) {
+			delegate.close();
 		}
 	}
 
 	@Override
 	public void applyWork(List<LuceneWork> workList, IndexingMonitor monitor) {
 		if ( selectionStrategy.isIndexOwnerLocal() ) {
-			delegatedBackend.applyWork( workList, monitor );
+			getOrCreateDelegate().applyWork( workList, monitor );
 		}
 		else {
 			if ( workList == null ) {
@@ -60,20 +69,33 @@ public void applyWork(List<LuceneWork> workList, IndexingMonitor monitor) {
 	@Override
 	public void applyStreamWork(LuceneWork singleOperation, IndexingMonitor monitor) {
 		if ( selectionStrategy.isIndexOwnerLocal() ) {
-			delegatedBackend.applyStreamWork( singleOperation, monitor );
+			getOrCreateDelegate().applyStreamWork( singleOperation, monitor );
 		}
 		else {
 			//TODO optimize for single operation?
 			jgroupsProcessor.sendLuceneWorkList( Collections.singletonList( singleOperation ) );
 		}
 	}
 
+	private BackendQueueProcessor getOrCreateDelegate() {
+		if ( delegate != null ) {
+			return delegate;
+		}
+		synchronized ( this ) {
+			if ( delegate != null ) {
+				return delegate;
+			}
+			delegate = delegatedBackendFactory.get();
+			return delegate;
+		}
+	}
+
 	public boolean blocksForACK() {
 		return jgroupsProcessor.blocksForACK();
 	}
 
-	public BackendQueueProcessor getDelegatedBackend() {
-		return delegatedBackend;
+	public BackendQueueProcessor getExistingDelegate() {
+		return delegate;
 	}
 
 	public long getMessageTimeout() {

backends/jgroups/src/test/java/org/hibernate/search/backend/jgroups/impl/SyncJGroupsBackendTest.java

@@ -125,7 +125,7 @@ public void testSynchAsConfigured() {
 	public void alternativeBackendConfiguration() {
 		BackendQueueProcessor backendQueueProcessor = extractBackendQueue( masterNode, "dvds" );
 		JGroupsReceivingMockBackendQueueProcessor jgroupsProcessor = (JGroupsReceivingMockBackendQueueProcessor) backendQueueProcessor;
-		BackendQueueProcessor delegatedBackend = jgroupsProcessor.getDelegate().getDelegatedBackend();
+		BackendQueueProcessor delegatedBackend = jgroupsProcessor.getDelegate().getExistingDelegate();
 		Assert.assertTrue( "dvds backend was configured with a delegate to blackhole but it's not using it", delegatedBackend instanceof BlackHoleBackendQueueProcessor );
 	}