Only drop and rebuild fulltext indices if needed.

enterprise/fulltext-addon/src/main/java/org/neo4j/kernel/api/impl/fulltext/FulltextFactory.java

@@ -31,6 +31,7 @@
 import org.neo4j.kernel.api.impl.index.IndexWriterConfigs;
 import org.neo4j.kernel.api.impl.index.builder.LuceneIndexStorageBuilder;
 import org.neo4j.kernel.api.impl.index.partition.WritableIndexPartitionFactory;
+import org.neo4j.kernel.api.impl.index.storage.PartitionedIndexStorage;
 
 /**
  * Used for creating {@link LuceneFulltext} and registering those to a {@link FulltextProvider}.
@@ -39,44 +40,64 @@ public class FulltextFactory
 {
     public static final String INDEX_DIR = "fulltext";
     private final FileSystemAbstraction fileSystem;
+    private final FulltextProvider provider;
     private final WritableIndexPartitionFactory partitionFactory;
     private final File indexDir;
     private final Analyzer analyzer;
 
     /**
      * Creates a factory for the specified location and analyzer.
+     *
      * @param fileSystem The filesystem to use.
      * @param storeDir Store directory of the database.
-     * @param analyzer The Lucene analyzer to use for the {@link LuceneFulltext} created by this factory.
+     * @param analyzerClassName The Lucene analyzer to use for the {@link LuceneFulltext} created by this factory.
+     * @param provider The {@link FulltextProvider} to register the indexes with.
      * @throws IOException
      */
-    public FulltextFactory( FileSystemAbstraction fileSystem, File storeDir, Analyzer analyzer ) throws IOException
+    public FulltextFactory( FileSystemAbstraction fileSystem, File storeDir, String analyzerClassName,
+                            FulltextProvider provider ) throws IOException
     {
-        this.analyzer = analyzer;
+        this.analyzer = getAnalyzer( analyzerClassName );
         this.fileSystem = fileSystem;
+        this.provider = provider;
         Factory<IndexWriterConfig> indexWriterConfigFactory = () -> IndexWriterConfigs.standard( analyzer );
         partitionFactory = new WritableIndexPartitionFactory( indexWriterConfigFactory );
         indexDir = new File( storeDir, INDEX_DIR );
     }
 
+    private Analyzer getAnalyzer( String analyzerClassName )
+    {
+        Analyzer analyzer;
+        try
+        {
+            Class configuredAnalyzer = Class.forName( analyzerClassName );
+            analyzer = (Analyzer) configuredAnalyzer.newInstance();
+        }
+        catch ( Exception e )
+        {
+            throw new RuntimeException( "Could not create the configured analyzer", e );
+        }
+        return analyzer;
+    }
+
     /**
      * Creates an instance of {@link LuceneFulltext} and registers it with the supplied {@link FulltextProvider}.
+     *
      * @param identifier The identifier of the new fulltext index
      * @param type The type of the new fulltext index
      * @param properties The properties to index
-     * @param provider The provider to register with
      * @throws IOException
      */
-    public void createFulltextIndex( String identifier, FulltextProvider.FulltextIndexType type, List<String> properties, FulltextProvider provider )
+    public void createFulltextIndex( String identifier, FulltextProvider.FulltextIndexType type,
+                                     List<String> properties )
             throws IOException
     {
-        // First delete any existing index, since we don't know if whatever it might contain actually matches our
-        // current configuration:
         File indexRootFolder = new File( indexDir, identifier );
-        fileSystem.deleteRecursively( indexRootFolder );
-
         LuceneIndexStorageBuilder storageBuilder = LuceneIndexStorageBuilder.create();
         storageBuilder.withFileSystem( fileSystem ).withIndexFolder( indexRootFolder );
-        provider.register( new LuceneFulltext( storageBuilder.build(), partitionFactory, properties, analyzer, identifier, type ) );
+        PartitionedIndexStorage storage = storageBuilder.build();
+        LuceneFulltext index = new LuceneFulltext( storage, partitionFactory, properties, analyzer, identifier, type );
+
+        provider.register( index );
     }
 }

enterprise/fulltext-addon/src/main/java/org/neo4j/kernel/api/impl/fulltext/FulltextIndexConfiguration.java

@@ -0,0 +1,73 @@
+/*
+ * Copyright (c) 2002-2017 "Neo Technology,"
+ * Network Engine for Objects in Lund AB [http://neotechnology.com]
+ *
+ * This file is part of Neo4j.
+ *
+ * Neo4j is free software: you can redistribute it and/or modify
+ * it under the terms of the GNU General Public License as published by
+ * the Free Software Foundation, either version 3 of the License, or
+ * (at your option) any later version.
+ *
+ * This program is distributed in the hope that it will be useful,
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+ * GNU General Public License for more details.
+ *
+ * You should have received a copy of the GNU General Public License
+ * along with this program.  If not, see <http://www.gnu.org/licenses/>.
+ */
+package org.neo4j.kernel.api.impl.fulltext;
+
+import org.apache.lucene.analysis.Analyzer;
+import org.apache.lucene.document.Document;
+import org.apache.lucene.document.StoredField;
+import org.apache.lucene.document.StringField;
+import org.apache.lucene.index.Term;
+
+import java.util.Arrays;
+import java.util.HashSet;
+import java.util.Set;
+
+import static org.apache.lucene.document.Field.Store.NO;
+import static org.apache.lucene.document.Field.Store.YES;
+import static org.neo4j.kernel.api.impl.fulltext.FulltextProvider.FIELD_CONFIG_ANALYZER;
+import static org.neo4j.kernel.api.impl.fulltext.FulltextProvider.FIELD_CONFIG_PROPERTIES;
+import static org.neo4j.kernel.api.impl.fulltext.FulltextProvider.FIELD_METADATA_DOC;
+
+public class FulltextIndexConfiguration
+{
+    public static Term TERM = new Term( FIELD_METADATA_DOC );
+
+    private final Set<String> properties;
+    private final String analyzerClassName;
+
+    public FulltextIndexConfiguration( Document doc )
+    {
+        properties = new HashSet<String>( Arrays.asList( doc.getValues( FIELD_CONFIG_PROPERTIES ) ) );
+        analyzerClassName = doc.get( FIELD_CONFIG_ANALYZER );
+    }
+
+    public FulltextIndexConfiguration( Analyzer analyzer, Set<String> properties )
+    {
+        this.properties = properties;
+        this.analyzerClassName = analyzer.getClass().getCanonicalName();
+    }
+
+    public boolean matches( String analyzerClassName, Set<String> properties )
+    {
+        return this.analyzerClassName.equals( analyzerClassName ) && this.properties.equals( properties );
+    }
+
+    public Document asDocument()
+    {
+        Document doc = new Document();
+        doc.add( new StringField( FIELD_METADATA_DOC, "", NO ) );
+        doc.add( new StoredField( FIELD_CONFIG_ANALYZER, analyzerClassName ) );
+        for ( String property : properties )
+        {
+            doc.add( new StoredField( FIELD_CONFIG_PROPERTIES, property ) );
+        }
+        return doc;
+    }
+}

enterprise/fulltext-addon/src/main/java/org/neo4j/kernel/api/impl/fulltext/FulltextProvider.java

@@ -36,7 +36,12 @@
  */
 public class FulltextProvider implements AutoCloseable
 {
-    public static final String LUCENE_FULLTEXT_ADDON_INTERNAL_ID = "__lucene__fulltext__addon__internal__id__";
+    public static final String LUCENE_FULLTEXT_ADDON_PREFIX = "__lucene__fulltext__addon__";
+    public static final String FIELD_ENTITY_ID = LUCENE_FULLTEXT_ADDON_PREFIX + "internal__id__";
+    public static final String FIELD_METADATA_DOC = LUCENE_FULLTEXT_ADDON_PREFIX + "metadata__doc__field__";
+    public static final String FIELD_CONFIG_ANALYZER = LUCENE_FULLTEXT_ADDON_PREFIX + "analyzer";
+    public static final String FIELD_CONFIG_PROPERTIES = LUCENE_FULLTEXT_ADDON_PREFIX + "properties";
+
     private final GraphDatabaseService db;
     private final Log log;
     private final FulltextTransactionEventUpdater fulltextTransactionEventUpdater;
@@ -75,15 +80,36 @@ public void init() throws IOException
     {
         for ( WritableFulltext index : writableNodeIndices )
         {
-            applier.populateNodes( index, db );
+            index.open();
+            if ( !matchesConfiguration( index ) )
+            {
+                index.drop();
+                index.open();
+                applier.populateNodes( index, db );
+            }
         }
         for ( WritableFulltext index : writableRelationshipIndices )
         {
-            applier.populateRelationships( index, db );
+            index.open();
+            if ( !matchesConfiguration( index ) )
+            {
+                index.drop();
+                index.open();
+                applier.populateRelationships( index, db );
+            }
         }
         db.registerTransactionEventHandler( fulltextTransactionEventUpdater );
     }
 
+    private boolean matchesConfiguration( WritableFulltext index ) throws IOException
+    {
+        try ( ReadOnlyFulltext indexReader = index.getIndexReader() )
+        {
+            FulltextIndexConfiguration config = indexReader.getConfigurationDocument();
+            return config != null && config.matches( index.getAnalyzerName(), index.getProperties() );
+        }
+    }
+
     /**
      * Wait for the asynchronous background population, if one is on-going, to complete.
      *
@@ -130,7 +156,6 @@ public void close()
 
     void register( LuceneFulltext fulltextIndex ) throws IOException
     {
-        fulltextIndex.open();
         if ( fulltextIndex.getType() == FulltextIndexType.NODES )
         {
             nodeIndices.put( fulltextIndex.getIdentifier(), fulltextIndex );

enterprise/fulltext-addon/src/main/java/org/neo4j/kernel/api/impl/fulltext/FulltextUpdateApplier.java

@@ -81,7 +81,7 @@ <E extends Entity> AsyncFulltextIndexOperation updatePropertyData(
             PartitionedIndexWriter indexWriter = index.getIndexWriter();
             for ( Map.Entry<Long,Map<String,Object>> stateEntry : state.entrySet() )
             {
-                Set<String> indexedProperties = index.properties();
+                Set<String> indexedProperties = index.getProperties();
                 if ( !Collections.disjoint( indexedProperties, stateEntry.getValue().keySet() ) )
                 {
                     long entityId = stateEntry.getKey();
@@ -120,7 +120,7 @@ <E extends Entity> AsyncFulltextIndexOperation removePropertyData(
         {
             for ( PropertyEntry<E> propertyEntry : propertyEntries )
             {
-                if ( index.properties().contains( propertyEntry.key() ) )
+                if ( index.getProperties().contains( propertyEntry.key() ) )
                 {
                     long entityId = propertyEntry.entity().getId();
                     Map<String,Object> allProperties = state.get( entityId );
@@ -163,7 +163,7 @@ private AsyncFulltextIndexOperation enqueuePopulateIndex(
         FulltextIndexUpdate population = () ->
         {
             PartitionedIndexWriter indexWriter = index.getIndexWriter();
-            String[] indexedPropertyKeys = index.properties().toArray( new String[0] );
+            String[] indexedPropertyKeys = index.getProperties().toArray( new String[0] );
             ArrayList<Supplier<Document>> documents = new ArrayList<>();
             try ( Transaction ignore = db.beginTx( 1, TimeUnit.DAYS ) )
             {

enterprise/fulltext-addon/src/main/java/org/neo4j/kernel/api/impl/fulltext/LuceneFulltext.java

@@ -118,4 +118,18 @@ private PartitionedFulltextReader createPartitionedReader( List<AbstractIndexPar
         List<PartitionSearcher> searchers = acquireSearchers( partitions );
         return new PartitionedFulltextReader( searchers, properties.toArray( new String[0] ), analyzer );
     }
+
+    @Override
+    public void close() throws IOException
+    {
+        PartitionedIndexWriter writer = getIndexWriter( new WritableFulltext( this ) );
+        FulltextIndexConfiguration config = new FulltextIndexConfiguration( analyzer, properties );
+        writer.updateDocument( FulltextIndexConfiguration.TERM, config.asDocument() );
+        super.close();
+    }
+
+    public String getAnalyzerName()
+    {
+        return analyzer.getClass().getCanonicalName();
+    }
 }

enterprise/fulltext-addon/src/main/java/org/neo4j/kernel/api/impl/fulltext/LuceneFulltextDocumentStructure.java

@@ -70,8 +70,8 @@ private static class DocWithId
 
         private DocWithId()
         {
-            idField = new StringField( FulltextProvider.LUCENE_FULLTEXT_ADDON_INTERNAL_ID, "", NO );
-            idValueField = new NumericDocValuesField( FulltextProvider.LUCENE_FULLTEXT_ADDON_INTERNAL_ID, 0L );
+            idField = new StringField( FulltextProvider.FIELD_ENTITY_ID, "", NO );
+            idValueField = new NumericDocValuesField( FulltextProvider.FIELD_ENTITY_ID, 0L );
             document = new Document();
             document.add( idField );
             document.add( idValueField );
@@ -100,7 +100,7 @@ private void removeAllValueFields()
             {
                 IndexableField field = it.next();
                 String fieldName = field.name();
-                if ( !fieldName.equals( FulltextProvider.LUCENE_FULLTEXT_ADDON_INTERNAL_ID ) )
+                if ( !fieldName.equals( FulltextProvider.FIELD_ENTITY_ID ) )
                 {
                     it.remove();
                 }
@@ -111,7 +111,6 @@ private void removeAllValueFields()
 
     static Term newTermForChangeOrRemove( long id )
     {
-        return new Term( FulltextProvider.LUCENE_FULLTEXT_ADDON_INTERNAL_ID, "" + id );
+        return new Term( FulltextProvider.FIELD_ENTITY_ID, "" + id );
     }
-
 }

enterprise/fulltext-addon/src/main/java/org/neo4j/kernel/api/impl/fulltext/PartitionedFulltextReader.java

@@ -90,6 +90,20 @@ public void close()
         }
     }
 
+    @Override
+    public FulltextIndexConfiguration getConfigurationDocument() throws IOException
+    {
+        for ( ReadOnlyFulltext indexReader : indexReaders )
+        {
+            FulltextIndexConfiguration config = indexReader.getConfigurationDocument();
+            if ( config != null )
+            {
+                return config;
+            }
+        }
+        return null;
+    }
+
     private PrimitiveLongIterator partitionedOperation( Function<ReadOnlyFulltext,PrimitiveLongIterator> readerFunction )
     {
         return PrimitiveLongCollections.concat( indexReaders.parallelStream().map( readerFunction ).collect( Collectors.toList() ) );

enterprise/fulltext-addon/src/main/java/org/neo4j/kernel/api/impl/fulltext/ReadOnlyFulltext.java

@@ -19,6 +19,8 @@
  */
 package org.neo4j.kernel.api.impl.fulltext;
 
+import java.io.IOException;
+
 import org.neo4j.collection.primitive.PrimitiveLongIterator;
 
 public interface ReadOnlyFulltext extends AutoCloseable
@@ -38,4 +40,9 @@ public interface ReadOnlyFulltext extends AutoCloseable
      * @return An iterator over the matching entityIDs, ordered by lucene scoring of the match.
      */
     PrimitiveLongIterator fuzzyQuery( String... terms );
+
+    @Override
+    void close();
+
+    FulltextIndexConfiguration getConfigurationDocument() throws IOException;
 }

enterprise/fulltext-addon/src/main/java/org/neo4j/kernel/api/impl/fulltext/SimpleFulltextReader.java

@@ -26,6 +26,8 @@
 import org.apache.lucene.search.IndexSearcher;
 import org.apache.lucene.search.Query;
 import org.apache.lucene.search.Sort;
+import org.apache.lucene.search.TermQuery;
+import org.apache.lucene.search.TopDocs;
 
 import java.io.IOException;
 
@@ -37,7 +39,7 @@
 
 import static java.util.Arrays.stream;
 import static java.util.stream.Collectors.joining;
-import static org.neo4j.kernel.api.impl.fulltext.FulltextProvider.LUCENE_FULLTEXT_ADDON_INTERNAL_ID;
+import static org.neo4j.kernel.api.impl.fulltext.FulltextProvider.FIELD_ENTITY_ID;
 
 /**
  * Lucene index reader that is able to read/sample a single partition of a partitioned Lucene index.
@@ -84,6 +86,18 @@ public void close()
         }
     }
 
+    @Override
+    public FulltextIndexConfiguration getConfigurationDocument() throws IOException
+    {
+        IndexSearcher indexSearcher = getIndexSearcher();
+        TopDocs docs = indexSearcher.search( new TermQuery( FulltextIndexConfiguration.TERM ), 1 );
+        if ( docs.scoreDocs.length < 1 )
+        {
+            return null;
+        }
+        return new FulltextIndexConfiguration( indexSearcher.doc( docs.scoreDocs[0].doc ) );
+    }
+
     private PrimitiveLongIterator innerQuery( String queryString )
     {
         MultiFieldQueryParser multiFieldQueryParser = new MultiFieldQueryParser( properties, analyzer );
@@ -107,7 +121,7 @@ private PrimitiveLongIterator indexQuery( Query query )
         {
             DocValuesCollector docValuesCollector = new DocValuesCollector( true );
             getIndexSearcher().search( query, docValuesCollector );
-            return docValuesCollector.getSortedValuesIterator( LUCENE_FULLTEXT_ADDON_INTERNAL_ID, Sort.RELEVANCE );
+            return docValuesCollector.getSortedValuesIterator( FIELD_ENTITY_ID, Sort.RELEVANCE );
         }
         catch ( IOException e )
         {

enterprise/fulltext-addon/src/main/java/org/neo4j/kernel/api/impl/fulltext/WritableFulltext.java

@@ -41,8 +41,18 @@ PartitionedIndexWriter getIndexWriter()
         return indexWriter;
     }
 
-    Set<String> properties()
+    Set<String> getProperties()
     {
         return luceneIndex.getProperties();
     }
+
+    public ReadOnlyFulltext getIndexReader() throws IOException
+    {
+        return luceneIndex.getIndexReader();
+    }
+
+    public String getAnalyzerName()
+    {
+        return luceneIndex.getAnalyzerName();
+    }
 }