Rebuild the index if the last committed transaction id has changed si…

…nce the index was last shut down.

This can happen if the index has been disabled for a while and new data has been committed during this time.

enterprise/fulltext-addon/src/main/java/org/neo4j/kernel/api/impl/fulltext/FulltextIndexConfiguration.java

@@ -19,8 +19,8 @@
  */
 package org.neo4j.kernel.api.impl.fulltext;
 
-import org.apache.lucene.analysis.Analyzer;
 import org.apache.lucene.document.Document;
+import org.apache.lucene.document.LongField;
 import org.apache.lucene.document.StoredField;
 import org.apache.lucene.document.StringField;
 import org.apache.lucene.index.Term;
@@ -30,8 +30,10 @@
 import java.util.Set;
 
 import static org.apache.lucene.document.Field.Store.NO;
+import static org.apache.lucene.document.Field.Store.YES;
 import static org.neo4j.kernel.api.impl.fulltext.FulltextProvider.FIELD_CONFIG_ANALYZER;
 import static org.neo4j.kernel.api.impl.fulltext.FulltextProvider.FIELD_CONFIG_PROPERTIES;
+import static org.neo4j.kernel.api.impl.fulltext.FulltextProvider.FIELD_LAST_COMMITTED_TX_ID;
 import static org.neo4j.kernel.api.impl.fulltext.FulltextProvider.FIELD_METADATA_DOC;
 
 class FulltextIndexConfiguration
@@ -40,29 +42,51 @@ class FulltextIndexConfiguration
 
     private final Set<String> properties;
     private final String analyzerClassName;
+    private final long txId;
 
     FulltextIndexConfiguration( Document doc )
     {
         properties = new HashSet<>( Arrays.asList( doc.getValues( FIELD_CONFIG_PROPERTIES ) ) );
         analyzerClassName = doc.get( FIELD_CONFIG_ANALYZER );
+        txId = Long.parseLong( doc.get( FIELD_LAST_COMMITTED_TX_ID ) );
     }
 
-    FulltextIndexConfiguration( Analyzer analyzer, Set<String> properties )
+    FulltextIndexConfiguration( String analyzerClassName, Set<String> properties, long txId )
     {
         this.properties = properties;
-        this.analyzerClassName = analyzer.getClass().getCanonicalName();
+        this.analyzerClassName = analyzerClassName;
+        this.txId = txId;
     }
 
-    boolean matches( String analyzerClassName, Set<String> properties )
+    @Override
+    public boolean equals( Object o )
     {
-        return this.analyzerClassName.equals( analyzerClassName ) && this.properties.equals( properties );
+        if ( this == o )
+        { return true; }
+        if ( o == null || getClass() != o.getClass() )
+        { return false; }
+
+        FulltextIndexConfiguration that = (FulltextIndexConfiguration) o;
+
+        return txId == that.txId && properties.equals( that.properties ) &&
+               analyzerClassName.equals( that.analyzerClassName );
+    }
+
+    @Override
+    public int hashCode()
+    {
+        int result = properties.hashCode();
+        result = 31 * result + analyzerClassName.hashCode();
+        result = 31 * result + (int) (txId ^ (txId >>> 32));
+        return result;
     }
 
     Document asDocument()
     {
         Document doc = new Document();
         doc.add( new StringField( FIELD_METADATA_DOC, "", NO ) );
         doc.add( new StoredField( FIELD_CONFIG_ANALYZER, analyzerClassName ) );
+        doc.add( new LongField( FIELD_LAST_COMMITTED_TX_ID, txId, YES ) );
         for ( String property : properties )
         {
             doc.add( new StoredField( FIELD_CONFIG_PROPERTIES, property ) );

enterprise/fulltext-addon/src/main/java/org/neo4j/kernel/api/impl/fulltext/FulltextProvider.java

@@ -28,6 +28,7 @@
 
 import org.neo4j.graphdb.GraphDatabaseService;
 import org.neo4j.kernel.AvailabilityGuard;
+import org.neo4j.kernel.impl.transaction.log.TransactionIdStore;
 import org.neo4j.logging.Log;
 import org.neo4j.scheduler.JobScheduler;
 
@@ -41,9 +42,11 @@ public class FulltextProvider implements AutoCloseable
     public static final String FIELD_METADATA_DOC = LUCENE_FULLTEXT_ADDON_PREFIX + "metadata__doc__field__";
     public static final String FIELD_CONFIG_ANALYZER = LUCENE_FULLTEXT_ADDON_PREFIX + "analyzer";
     public static final String FIELD_CONFIG_PROPERTIES = LUCENE_FULLTEXT_ADDON_PREFIX + "properties";
+    public static final String FIELD_LAST_COMMITTED_TX_ID = LUCENE_FULLTEXT_ADDON_PREFIX + "tx__id";
 
     private final GraphDatabaseService db;
     private final Log log;
+    private final TransactionIdStore transactionIdStore;
     private final FulltextTransactionEventUpdater fulltextTransactionEventUpdater;
     private final Set<String> nodeProperties;
     private final Set<String> relationshipProperties;
@@ -59,12 +62,14 @@ public class FulltextProvider implements AutoCloseable
      * @param log For logging errors.
      * @param availabilityGuard Used for waiting with populating the index until the database is available.
      * @param scheduler For background work.
+     * @param transactionIdStore
      */
     public FulltextProvider( GraphDatabaseService db, Log log, AvailabilityGuard availabilityGuard,
-                             JobScheduler scheduler )
+                             JobScheduler scheduler, TransactionIdStore transactionIdStore )
     {
         this.db = db;
         this.log = log;
+        this.transactionIdStore = transactionIdStore;
         applier = new FulltextUpdateApplier( log, availabilityGuard, scheduler );
         applier.start();
         fulltextTransactionEventUpdater = new FulltextTransactionEventUpdater( this, log, applier );
@@ -103,10 +108,14 @@ public void init() throws IOException
 
     private boolean matchesConfiguration( WritableFulltext index ) throws IOException
     {
+        long txId = transactionIdStore.getLastCommittedTransactionId();
+        FulltextIndexConfiguration currentConfig =
+                new FulltextIndexConfiguration( index.getAnalyzerName(), index.getProperties(), txId );
+
         try ( ReadOnlyFulltext indexReader = index.getIndexReader() )
         {
-            FulltextIndexConfiguration config = indexReader.getConfigurationDocument();
-            return config != null && config.matches( index.getAnalyzerName(), index.getProperties() );
+            FulltextIndexConfiguration storedConfig = indexReader.getConfigurationDocument();
+            return storedConfig != null && storedConfig.equals( currentConfig );
         }
     }
 
@@ -134,6 +143,7 @@ public void close()
         {
             try
             {
+                luceneFulltextIndex.saveConfiguration( transactionIdStore.getLastCommittedTransactionId() );
                 luceneFulltextIndex.close();
             }
             catch ( IOException e )

enterprise/fulltext-addon/src/main/java/org/neo4j/kernel/api/impl/fulltext/LuceneFulltext.java

@@ -42,8 +42,9 @@ class LuceneFulltext extends AbstractLuceneIndex
     private final FulltextProvider.FulltextIndexType type;
     private final Set<String> properties;
 
-    LuceneFulltext( PartitionedIndexStorage indexStorage, IndexPartitionFactory partitionFactory, List<String> properties, Analyzer analyzer,
-            String identifier, FulltextProvider.FulltextIndexType type )
+    LuceneFulltext( PartitionedIndexStorage indexStorage, IndexPartitionFactory partitionFactory,
+                    List<String> properties, Analyzer analyzer, String identifier,
+                    FulltextProvider.FulltextIndexType type )
     {
         super( indexStorage, partitionFactory );
         this.properties = Collections.unmodifiableSet( new HashSet<>( properties ) );
@@ -119,13 +120,12 @@ private PartitionedFulltextReader createPartitionedReader( List<AbstractIndexPar
         return new PartitionedFulltextReader( searchers, properties.toArray( new String[0] ), analyzer );
     }
 
-    @Override
-    public void close() throws IOException
+    public void saveConfiguration( long txId ) throws IOException
     {
         PartitionedIndexWriter writer = getIndexWriter( new WritableFulltext( this ) );
-        FulltextIndexConfiguration config = new FulltextIndexConfiguration( analyzer, properties );
+        String analyzerName = analyzer.getClass().getCanonicalName();
+        FulltextIndexConfiguration config = new FulltextIndexConfiguration( analyzerName, properties, txId );
         writer.updateDocument( FulltextIndexConfiguration.TERM, config.asDocument() );
-        super.close();
     }
 
     public String getAnalyzerName()

enterprise/fulltext-addon/src/main/java/org/neo4j/kernel/api/impl/fulltext/integrations/bloom/LoadableBloomFulltextConfig.java → enterprise/fulltext-addon/src/main/java/org/neo4j/kernel/api/impl/fulltext/integrations/bloom/BloomFulltextConfig.java

@@ -27,7 +27,6 @@
 import org.neo4j.configuration.Internal;
 import org.neo4j.configuration.LoadableConfig;
 import org.neo4j.graphdb.config.Setting;
-import org.neo4j.kernel.configuration.Settings;
 
 import static org.neo4j.kernel.api.impl.fulltext.FulltextProvider.LUCENE_FULLTEXT_ADDON_PREFIX;
 import static org.neo4j.kernel.configuration.Settings.BOOLEAN;
@@ -42,9 +41,8 @@
 /**
  * Configuration parameters for the bloom fulltext addon.
  */
-public class LoadableBloomFulltextConfig implements LoadableConfig
+public class BloomFulltextConfig implements LoadableConfig
 {
-
     public static final String UNSUPPORTED_PROPERTY_KEY_REGEX = "^(?!" + LUCENE_FULLTEXT_ADDON_PREFIX + ").+$";
     public static final BiFunction<List<String>,Function<String,String>,List<String>> ILLEGAL_VALUE_CONSTRAINT =
             illegalValueMessage( "Must not contain '" + LUCENE_FULLTEXT_ADDON_PREFIX + "'", matchesAny(
@@ -57,9 +55,12 @@ public class LoadableBloomFulltextConfig implements LoadableConfig
     @Description( "Property keys to index" )
     @Internal
     static final Setting<List<String>> bloom_indexed_properties =
-            buildSetting( "unsupported.dbms.bloom_indexed_properties", STRING_LIST, "" ).constraint( ILLEGAL_VALUE_CONSTRAINT ).build();
+            buildSetting( "unsupported.dbms.bloom_indexed_properties", STRING_LIST, "" )
+                    .constraint( ILLEGAL_VALUE_CONSTRAINT ).build();
 
-    @Description( "Define the analyzer to use for the bloom index. Expects the fully qualified classname of the analyzer to use" )
+    @Description( "Define the analyzer to use for the bloom index. Expects the fully qualified classname of the " +
+                  "analyzer to use" )
     @Internal
-    static final Setting<String> bloom_analyzer = setting( "unsupported.dbms.bloom_analyzer", STRING, "org.apache.lucene.analysis.standard.StandardAnalyzer" );
+    static final Setting<String> bloom_analyzer = setting( "unsupported.dbms.bloom_analyzer", STRING,
+            "org.apache.lucene.analysis.standard.StandardAnalyzer" );
 }

enterprise/fulltext-addon/src/main/java/org/neo4j/kernel/api/impl/fulltext/integrations/bloom/BloomKernelExtension.java

@@ -22,6 +22,7 @@
 import java.io.File;
 import java.io.IOException;
 import java.util.List;
+import java.util.function.Supplier;
 
 import org.neo4j.graphdb.GraphDatabaseService;
 import org.neo4j.io.fs.FileSystemAbstraction;
@@ -32,6 +33,7 @@
 import org.neo4j.kernel.configuration.Config;
 import org.neo4j.kernel.impl.logging.LogService;
 import org.neo4j.kernel.impl.proc.Procedures;
+import org.neo4j.kernel.impl.transaction.log.TransactionIdStore;
 import org.neo4j.kernel.lifecycle.LifecycleAdapter;
 import org.neo4j.logging.Log;
 import org.neo4j.scheduler.JobScheduler;
@@ -51,12 +53,14 @@ class BloomKernelExtension extends LifecycleAdapter
     private LogService logService;
     private final AvailabilityGuard availabilityGuard;
     private final JobScheduler scheduler;
+    private final Supplier<TransactionIdStore> transactionIdStore;
     private FulltextProvider provider;
 
     BloomKernelExtension( FileSystemAbstraction fileSystem, File storeDir, Config config,
                           GraphDatabaseService db, Procedures procedures,
                           LogService logService, AvailabilityGuard availabilityGuard,
-                          JobScheduler scheduler )
+                          JobScheduler scheduler,
+                          Supplier<TransactionIdStore> transactionIdStore )
     {
         this.storeDir = storeDir;
         this.config = config;
@@ -66,18 +70,19 @@ class BloomKernelExtension extends LifecycleAdapter
         this.logService = logService;
         this.availabilityGuard = availabilityGuard;
         this.scheduler = scheduler;
+        this.transactionIdStore = transactionIdStore;
     }
 
     @Override
-    public void init() throws IOException, KernelException
+    public void start() throws IOException, KernelException
     {
-        if ( config.get( LoadableBloomFulltextConfig.bloom_enabled ) )
+        if ( config.get( BloomFulltextConfig.bloom_enabled ) )
         {
             List<String> properties = getProperties();
-            String analyzer = config.get( LoadableBloomFulltextConfig.bloom_analyzer );
+            String analyzer = config.get( BloomFulltextConfig.bloom_analyzer );
 
             Log log = logService.getInternalLog( FulltextProvider.class );
-            provider = new FulltextProvider( db, log, availabilityGuard, scheduler );
+            provider = new FulltextProvider( db, log, availabilityGuard, scheduler, transactionIdStore.get() );
             FulltextFactory fulltextFactory = new FulltextFactory( fileSystem, storeDir, analyzer, provider );
             fulltextFactory.createFulltextIndex( BLOOM_NODES, NODES, properties );
             fulltextFactory.createFulltextIndex( BLOOM_RELATIONSHIPS, RELATIONSHIPS, properties );
@@ -90,7 +95,7 @@ public void init() throws IOException, KernelException
 
     private List<String> getProperties()
     {
-        List<String> properties = config.get( LoadableBloomFulltextConfig.bloom_indexed_properties );
+        List<String> properties = config.get( BloomFulltextConfig.bloom_indexed_properties );
         if ( properties.isEmpty() )
         {
             throw new RuntimeException( "Properties to index must be configured for bloom fulltext" );
@@ -99,7 +104,7 @@ private List<String> getProperties()
     }
 
     @Override
-    public void shutdown() throws Exception
+    public void stop() throws Exception
     {
         if ( provider != null )
         {

enterprise/fulltext-addon/src/main/java/org/neo4j/kernel/api/impl/fulltext/integrations/bloom/BloomKernelExtensionFactory.java

@@ -20,6 +20,7 @@
 package org.neo4j.kernel.api.impl.fulltext.integrations.bloom;
 
 import java.io.File;
+import java.util.function.Supplier;
 
 import org.neo4j.graphdb.GraphDatabaseService;
 import org.neo4j.helpers.Service;
@@ -30,14 +31,15 @@
 import org.neo4j.kernel.impl.logging.LogService;
 import org.neo4j.kernel.impl.proc.Procedures;
 import org.neo4j.kernel.impl.spi.KernelContext;
+import org.neo4j.kernel.impl.transaction.log.TransactionIdStore;
 import org.neo4j.kernel.lifecycle.Lifecycle;
 import org.neo4j.scheduler.JobScheduler;
 
 /**
  * A {@link KernelExtensionFactory} for the bloom fulltext addon.
  *
  * @see BloomProcedures
- * @see LoadableBloomFulltextConfig
+ * @see BloomFulltextConfig
  */
 @Service.Implementation( KernelExtensionFactory.class )
 public class BloomKernelExtensionFactory extends KernelExtensionFactory<BloomKernelExtensionFactory.Dependencies>
@@ -62,6 +64,8 @@ public interface Dependencies
         AvailabilityGuard availabilityGuard();
 
         JobScheduler scheduler();
+
+        TransactionIdStore transactionIdStore();
     }
 
     public BloomKernelExtensionFactory()
@@ -80,7 +84,8 @@ public Lifecycle newInstance( KernelContext context, Dependencies dependencies )
         LogService logService = dependencies.logService();
         AvailabilityGuard availabilityGuard = dependencies.availabilityGuard();
         JobScheduler scheduler = dependencies.scheduler();
+        Supplier<TransactionIdStore> transactionIdStore = dependencies::transactionIdStore;
         return new BloomKernelExtension(
-                fs, storeDir, config, db, procedures, logService, availabilityGuard, scheduler );
+                fs, storeDir, config, db, procedures, logService, availabilityGuard, scheduler, transactionIdStore );
     }
 }

enterprise/fulltext-addon/src/main/resources/META-INF/services/org.neo4j.configuration.LoadableConfig

@@ -1 +1 @@
-org.neo4j.kernel.api.impl.fulltext.integrations.bloom.LoadableBloomFulltextConfig
+org.neo4j.kernel.api.impl.fulltext.integrations.bloom.BloomFulltextConfig

enterprise/fulltext-addon/src/test/java/org/neo4j/kernel/api/impl/fulltext/LuceneFulltextTestSupport.java

@@ -36,6 +36,7 @@
 import org.neo4j.graphdb.Transaction;
 import org.neo4j.io.fs.FileSystemAbstraction;
 import org.neo4j.kernel.AvailabilityGuard;
+import org.neo4j.kernel.impl.transaction.log.TransactionIdStore;
 import org.neo4j.kernel.internal.GraphDatabaseAPI;
 import org.neo4j.logging.Log;
 import org.neo4j.logging.NullLog;
@@ -61,6 +62,7 @@ public class LuceneFulltextTestSupport
     protected JobScheduler scheduler;
     protected FileSystemAbstraction fs;
     protected File storeDir;
+    private TransactionIdStore transactionIdStore;
 
     @Before
     public void setUp() throws Exception
@@ -69,11 +71,12 @@ public void setUp() throws Exception
         scheduler = dbRule.resolveDependency( JobScheduler.class );
         fs = dbRule.resolveDependency( FileSystemAbstraction.class );
         storeDir = dbRule.getStoreDir();
+        transactionIdStore = dbRule.resolveDependency( TransactionIdStore.class );
     }
 
     protected FulltextProvider createProvider() throws IOException
     {
-        FulltextProvider provider = new FulltextProvider( db, LOG, availabilityGuard, scheduler );
+        FulltextProvider provider = new FulltextProvider( db, LOG, availabilityGuard, scheduler, transactionIdStore );
         fulltextFactory = new FulltextFactory( fs, storeDir, ANALYZER, provider );
         return provider;
     }

enterprise/fulltext-addon/src/test/java/org/neo4j/kernel/api/impl/fulltext/integrations/bloom/BloomFulltextIndexBenchmarks.java

@@ -81,7 +81,7 @@ public void fiveHundredThousandOnlineUpdates() throws Exception
         createTestGraphDatabaseFactory();
         configureBloomExtension();
         GraphDatabaseBuilder builder = factory.newEmbeddedDatabaseBuilder( testDirectory.graphDbDir() );
-        builder.setConfig( LoadableBloomFulltextConfig.bloom_indexed_properties, "prop" );
+        builder.setConfig( BloomFulltextConfig.bloom_indexed_properties, "prop" );
         db = builder.newGraphDatabase();
 
         int trials = 50;
@@ -152,7 +152,7 @@ public void fiveHundredThousandNodesForPopulation() throws Exception
         // Then measure startup performance
         configureBloomExtension();
         builder = factory.newEmbeddedDatabaseBuilder( testDirectory.graphDbDir() );
-        builder.setConfig( LoadableBloomFulltextConfig.bloom_indexed_properties, "prop" );
+        builder.setConfig( BloomFulltextConfig.bloom_indexed_properties, "prop" );
 
         for ( int i = 0; i < 50; i++ )
         {