Insight index support multiple properties per node

community/lucene-index/src/main/java/org/neo4j/kernel/api/impl/index/IndexWriterConfigs.java

@@ -19,6 +19,7 @@
  */
 package org.neo4j.kernel.api.impl.index;
 
+import org.apache.lucene.analysis.Analyzer;
 import org.apache.lucene.codecs.PostingsFormat;
 import org.apache.lucene.codecs.blocktreeords.BlockTreeOrdsPostingsFormat;
 import org.apache.lucene.codecs.lucene54.Lucene54Codec;
@@ -71,7 +72,13 @@ private IndexWriterConfigs()
 
     public static IndexWriterConfig standard()
     {
-        IndexWriterConfig writerConfig = new IndexWriterConfig( LuceneDataSource.KEYWORD_ANALYZER );
+        Analyzer analyzer = LuceneDataSource.KEYWORD_ANALYZER;
+        return getIndexWriterConfigWithAnalyzer( analyzer );
+    }
+
+    private static IndexWriterConfig getIndexWriterConfigWithAnalyzer( Analyzer analyzer )
+    {
+        IndexWriterConfig writerConfig = new IndexWriterConfig( analyzer );
 
         writerConfig.setMaxBufferedDocs( MAX_BUFFERED_DOCS );
         writerConfig.setMaxBufferedDeleteTerms( MAX_BUFFERED_DELETE_TERMS );
@@ -108,4 +115,12 @@ public static IndexWriterConfig population()
         writerConfig.setRAMBufferSizeMB( POPULATION_RAM_BUFFER_SIZE_MB );
         return writerConfig;
     }
+
+    public static IndexWriterConfig population( Analyzer analyzer )
+    {
+        IndexWriterConfig writerConfig = getIndexWriterConfigWithAnalyzer( analyzer );
+        writerConfig.setMaxBufferedDocs( POPULATION_MAX_BUFFERED_DOCS );
+        writerConfig.setRAMBufferSizeMB( POPULATION_RAM_BUFFER_SIZE_MB );
+        return writerConfig;
+    }
 }

community/lucene-index/src/main/java/org/neo4j/kernel/api/impl/insight/InsightFieldEncoding.java

@@ -19,18 +19,13 @@
  */
 package org.neo4j.kernel.api.impl.insight;
 
-import org.apache.lucene.analysis.TokenStream;
-import org.apache.lucene.analysis.en.EnglishAnalyzer;
 import org.apache.lucene.document.Field;
-import org.apache.lucene.document.StringField;
 import org.apache.lucene.document.TextField;
 
 import org.neo4j.values.storable.TextValue;
 import org.neo4j.values.storable.Value;
 import org.neo4j.values.storable.Values;
 
-import static org.apache.lucene.document.Field.Store.NO;
-
 /**
  * Enumeration representing all possible property types with corresponding encodings and query structures for Lucene
  * schema indexes.
@@ -156,8 +151,7 @@ Field encodeField( String name, Value value )
                 {
                     java.lang.String stringValue = ((TextValue) value).stringValue();
 
-                    TokenStream tokenStream = englishAnalyzer.tokenStream( name, stringValue );
-                    TextField field = new TextField( name, tokenStream );
+                    TextField field = new TextField( name, stringValue, Field.Store.NO );
                     return field;
                 }
 
@@ -175,7 +169,6 @@ Field encodeField( String name, Value value )
 //                }
             };
 
-    private static EnglishAnalyzer englishAnalyzer = new EnglishAnalyzer();
     private static final InsightFieldEncoding[] AllEncodings = values();
 
     abstract String key();

community/lucene-index/src/main/java/org/neo4j/kernel/api/impl/insight/InsightIndex.java

@@ -19,11 +19,14 @@
  */
 package org.neo4j.kernel.api.impl.insight;
 
+import org.apache.lucene.analysis.en.EnglishAnalyzer;
+import org.apache.lucene.index.IndexWriterConfig;
+
 import java.io.File;
 import java.io.IOException;
 import java.nio.file.Paths;
-import java.util.Arrays;
 
+import org.neo4j.function.Factory;
 import org.neo4j.io.fs.FileSystemAbstraction;
 import org.neo4j.kernel.api.impl.index.IndexWriterConfigs;
 import org.neo4j.kernel.api.impl.index.builder.LuceneIndexStorageBuilder;
@@ -35,18 +38,16 @@ public class InsightIndex implements AutoCloseable
 {
     private final InsightLuceneIndex nodeIndex;
 
-    public InsightIndex( FileSystemAbstraction fileSystem, File file, int[] properties ) throws IOException
+    public InsightIndex( FileSystemAbstraction fileSystem, File file, String... properties ) throws IOException
     {
         LuceneIndexStorageBuilder storageBuilder = LuceneIndexStorageBuilder.create();
         storageBuilder.withFileSystem( fileSystem ).withIndexIdentifier( "insightNodes" )
                 .withDirectoryFactory( directoryFactory( false, fileSystem ) )
                 .withIndexRootFolder( Paths.get( file.getAbsolutePath(),"insightindex" ).toFile() );
 
-        WritableIndexPartitionFactory partitionFactory = new WritableIndexPartitionFactory(
-                IndexWriterConfigs::population );
-        String[] propertyIdsAsStrings = Arrays.stream( properties ).sorted().mapToObj( String::valueOf )
-                .toArray( String[]::new );
-        nodeIndex = new InsightLuceneIndex( storageBuilder.build(), partitionFactory, propertyIdsAsStrings );
+        Factory<IndexWriterConfig> population = () -> IndexWriterConfigs.population( new EnglishAnalyzer() );
+        WritableIndexPartitionFactory partitionFactory = new WritableIndexPartitionFactory( population );
+        nodeIndex = new InsightLuceneIndex( storageBuilder.build(), partitionFactory, properties );
         nodeIndex.open();
     }
 

community/lucene-index/src/main/java/org/neo4j/kernel/api/impl/insight/InsightIndexTransactionEventUpdater.java

@@ -22,13 +22,12 @@
 import org.apache.lucene.document.Document;
 
 import java.io.IOException;
+import java.util.HashMap;
+import java.util.Map;
 
 import org.neo4j.graphdb.Node;
-import org.neo4j.graphdb.event.PropertyEntry;
 import org.neo4j.graphdb.event.TransactionData;
 import org.neo4j.graphdb.event.TransactionEventHandler;
-import org.neo4j.kernel.api.properties.PropertyKeyValue;
-import org.neo4j.values.storable.Values;
 
 public class InsightIndexTransactionEventUpdater implements TransactionEventHandler<Object>
 {
@@ -42,24 +41,26 @@ public class InsightIndexTransactionEventUpdater implements TransactionEventHand
     @Override
     public Object beforeCommit( TransactionData data ) throws Exception
     {
-        return null;
+        Map<Long,Map<String,Object>> map = new HashMap<Long,Map<String,Object>>();
+        data.createdNodes().forEach( node -> map.put( node.getId(), node.getAllProperties() ) );
+        return map;
     }
 
     @Override
     public void afterCommit( TransactionData data, Object state )
     {
-        writeNodeData( data.assignedNodeProperties() );
+        writeNodeData( data.createdNodes(), (Map<Long,Map<String,Object>>) state );
         deleteNodeData( data.deletedNodes() );
     }
 
-    private void writeNodeData( Iterable<PropertyEntry<Node>> propertyEntries )
+    private void writeNodeData( Iterable<Node> propertyEntries, Map<Long,Map<String,Object>> state )
     {
-        for ( PropertyEntry<Node> propertyEntry : propertyEntries )
+        for ( Node node : propertyEntries )
         {
-//            propertyEntry.previouslyCommitedValue()
-            Document document = LuceneInsightDocumentStructure.documentRepresentingProperties( propertyEntry.entity().getId(),
-                    //TODO THIS IS BAD
-                    new PropertyKeyValue( Integer.parseInt( "1" ), Values.of( propertyEntry.value() ) ) );
+
+            Map<String,Object> allProperties = state.get( node.getId() );
+
+            Document document = LuceneInsightDocumentStructure.documentRepresentingProperties( node.getId(), allProperties );
             try
             {
                 writableDatabaseInsightIndex.getIndexWriter().addDocument( document );

community/lucene-index/src/main/java/org/neo4j/kernel/api/impl/insight/LuceneInsightDocumentStructure.java

@@ -27,9 +27,10 @@
 import org.apache.lucene.index.Term;
 
 import java.util.Iterator;
+import java.util.Map;
 
-import org.neo4j.kernel.api.properties.PropertyKeyValue;
 import org.neo4j.values.storable.Value;
+import org.neo4j.values.storable.Values;
 
 import static org.apache.lucene.document.Field.Store.YES;
 
@@ -50,7 +51,7 @@ private static DocWithId reuseDocument( long nodeId )
         return doc;
     }
 
-    public static Document documentRepresentingProperties( long nodeId, PropertyKeyValue... values )
+    public static Document documentRepresentingProperties( long nodeId, Map<String,Object> values )
     {
         DocWithId document = reuseDocument( nodeId );
         document.setValues( values );
@@ -62,10 +63,10 @@ public static long getNodeId( Document from )
         return Long.parseLong( from.get( NODE_ID_KEY ) );
     }
 
-    static Field encodeValueField( int propertyNumber, Value value )
+    static Field encodeValueField( String propertyKey, Value value )
     {
         InsightFieldEncoding encoding = InsightFieldEncoding.forValue( value );
-        return encoding.encodeField( Integer.toString( propertyNumber ), value );
+        return encoding.encodeField( propertyKey, value );
     }
 
     private static class DocWithId
@@ -91,11 +92,11 @@ private void setId( long id )
             idValueField.setLongValue( id );
         }
 
-        private void setValues( PropertyKeyValue... values )
+        private void setValues( Map<String,Object> values )
         {
-            for ( int i = 0; i < values.length; i++ )
+            for ( Map.Entry<String,Object> entry : values.entrySet() )
             {
-                Field field = encodeValueField( values[i].propertyKeyId(), values[i].value() );
+                Field field = encodeValueField( entry.getKey(), Values.of( entry.getValue() ) );
                 document.add( field );
             }
         }

community/lucene-index/src/main/java/org/neo4j/kernel/api/impl/insight/SimpleInsightIndexReader.java

@@ -61,7 +61,11 @@ public PrimitiveLongIterator query( String... query )
         BooleanQuery.Builder builder = new BooleanQuery.Builder();
         for ( String property : properties )
         {
-            builder.add( new PhraseQuery( property, query ), BooleanClause.Occur.SHOULD );
+            builder.add( new PhraseQuery( 2, property, query ), BooleanClause.Occur.SHOULD );
+            for ( String s : query )
+            {
+                builder.add( new PhraseQuery( 2, property, s ), BooleanClause.Occur.SHOULD );
+            }
         }
         return query( builder.build() );
     }

community/lucene-index/src/test/java/org/neo4j/kernel/api/impl/insight/InsightLuceneIndexUpdaterTest.java

@@ -23,8 +23,7 @@
 import org.junit.Rule;
 import org.junit.Test;
 
-import java.io.IOException;
-
+import org.neo4j.collection.primitive.PrimitiveLongIterator;
 import org.neo4j.graphdb.Label;
 import org.neo4j.graphdb.Node;
 import org.neo4j.graphdb.Transaction;
@@ -33,7 +32,6 @@
 import org.neo4j.test.rule.EmbeddedDatabaseRule;
 import org.neo4j.test.rule.TestDirectory;
 import org.neo4j.test.rule.fs.DefaultFileSystemRule;
-import org.neo4j.test.rule.fs.EphemeralFileSystemRule;
 import org.neo4j.test.rule.fs.FileSystemRule;
 
 import static org.junit.Assert.assertEquals;
@@ -54,7 +52,8 @@ public class InsightLuceneIndexUpdaterTest
     public void shouldFindNodeWithString() throws Exception
     {
         GraphDatabaseAPI db = dbRule.getGraphDatabaseAPI();
-        try (InsightIndex insightIndex = new InsightIndex(fileSystemRule, testDirectory.graphDbDir(), new int[]{1})) {
+        try ( InsightIndex insightIndex = new InsightIndex( fileSystemRule, testDirectory.graphDbDir(), "prop" ) )
+        {
             db.registerTransactionEventHandler(insightIndex.getUpdater());
 
             long firstID;
@@ -86,7 +85,8 @@ public void shouldFindNodeWithString() throws Exception
     public void shouldNotFindRemovedNodes() throws Exception
     {
         GraphDatabaseAPI db = dbRule.getGraphDatabaseAPI();
-        try (InsightIndex insightIndex = new InsightIndex(fileSystemRule, testDirectory.graphDbDir(), new int[]{1})) {
+        try ( InsightIndex insightIndex = new InsightIndex( fileSystemRule, testDirectory.graphDbDir(), "prop" ) )
+        {
             db.registerTransactionEventHandler(insightIndex.getUpdater());
 
             long firstID;
@@ -120,4 +120,42 @@ public void shouldNotFindRemovedNodes() throws Exception
             }
         }
     }
+
+    @Test
+    public void shouldOrderResults() throws Exception
+    {
+        GraphDatabaseAPI db = dbRule.getGraphDatabaseAPI();
+        try ( InsightIndex insightIndex = new InsightIndex( fileSystemRule, testDirectory.graphDbDir(), "prop", "prop2" ) )
+        {
+            db.registerTransactionEventHandler( insightIndex.getUpdater() );
+
+            long firstID;
+            long secondID;
+            long thirdID;
+            try ( Transaction tx = db.beginTx() )
+            {
+                Node node = db.createNode( LABEL );
+                firstID = node.getId();
+                node.setProperty( "prop", "Tomtar tomtar oftsat i tomteutstyrsel." );
+                Node node2 = db.createNode( LABEL );
+                secondID = node2.getId();
+                node2.setProperty( "prop", "tomtar tomtar tomtar tomtar tomtar." );
+                node2.setProperty( "prop2", "tomtar tomtar tomtar tomtar tomtar tomtar karl" );
+                Node node3 = db.createNode( LABEL );
+                thirdID = node3.getId();
+                node3.setProperty( "prop", "Tomtar som tomtar ser upp till tomtar som inte tomtar." );
+
+                tx.success();
+            }
+
+            try ( InsightIndexReader reader = insightIndex.getReader() )
+            {
+
+                PrimitiveLongIterator iterator = reader.query( "tomtar", "karl" );
+                assertEquals( secondID, iterator.next() );
+                assertEquals( firstID, iterator.next() );
+                assertEquals( thirdID, iterator.next() );
+            }
+        }
+    }
 }