Skip to content

Commit

Permalink
Insight index support multiple properties per node
Browse files Browse the repository at this point in the history
  • Loading branch information
@ragadeeshu
ragadeeshu committed Sep 12, 2017
1 parent 0aca7b3 commit 1fc8258
Show file tree
Hide file tree
Showing 7 changed files with 93 additions and 40 deletions.
17 changes: 16 additions & 1 deletion community/lucene-index/src/main/java/org/neo4j/kernel/api/impl/index/IndexWriterConfigs.java
View file
Expand Up @@ -19,6 +19,7 @@
*/
package org.neo4j.kernel.api.impl.index;

import org.apache.lucene.analysis.Analyzer;
import org.apache.lucene.codecs.PostingsFormat;
import org.apache.lucene.codecs.blocktreeords.BlockTreeOrdsPostingsFormat;
import org.apache.lucene.codecs.lucene54.Lucene54Codec;
Expand Down Expand Up @@ -71,7 +72,13 @@ private IndexWriterConfigs()

public static IndexWriterConfig standard()
{
IndexWriterConfig writerConfig = new IndexWriterConfig( LuceneDataSource.KEYWORD_ANALYZER );
Analyzer analyzer = LuceneDataSource.KEYWORD_ANALYZER;
return getIndexWriterConfigWithAnalyzer( analyzer );
}

private static IndexWriterConfig getIndexWriterConfigWithAnalyzer( Analyzer analyzer )
{
IndexWriterConfig writerConfig = new IndexWriterConfig( analyzer );

writerConfig.setMaxBufferedDocs( MAX_BUFFERED_DOCS );
writerConfig.setMaxBufferedDeleteTerms( MAX_BUFFERED_DELETE_TERMS );
Expand Down Expand Up @@ -108,4 +115,12 @@ public static IndexWriterConfig population()
writerConfig.setRAMBufferSizeMB( POPULATION_RAM_BUFFER_SIZE_MB );
return writerConfig;
}

public static IndexWriterConfig population( Analyzer analyzer )
{
IndexWriterConfig writerConfig = getIndexWriterConfigWithAnalyzer( analyzer );
writerConfig.setMaxBufferedDocs( POPULATION_MAX_BUFFERED_DOCS );
writerConfig.setRAMBufferSizeMB( POPULATION_RAM_BUFFER_SIZE_MB );
return writerConfig;
}
}
9 changes: 1 addition & 8 deletions ...ty/lucene-index/src/main/java/org/neo4j/kernel/api/impl/insight/InsightFieldEncoding.java
View file
Expand Up @@ -19,18 +19,13 @@
*/
package org.neo4j.kernel.api.impl.insight;

import org.apache.lucene.analysis.TokenStream;
import org.apache.lucene.analysis.en.EnglishAnalyzer;
import org.apache.lucene.document.Field;
import org.apache.lucene.document.StringField;
import org.apache.lucene.document.TextField;

import org.neo4j.values.storable.TextValue;
import org.neo4j.values.storable.Value;
import org.neo4j.values.storable.Values;

import static org.apache.lucene.document.Field.Store.NO;

/**
* Enumeration representing all possible property types with corresponding encodings and query structures for Lucene
* schema indexes.
Expand Down Expand Up @@ -156,8 +151,7 @@ Field encodeField( String name, Value value )
{
java.lang.String stringValue = ((TextValue) value).stringValue();

TokenStream tokenStream = englishAnalyzer.tokenStream( name, stringValue );
TextField field = new TextField( name, tokenStream );
TextField field = new TextField( name, stringValue, Field.Store.NO );
return field;
}

Expand All @@ -175,7 +169,6 @@ Field encodeField( String name, Value value )
// }
};

private static EnglishAnalyzer englishAnalyzer = new EnglishAnalyzer();
private static final InsightFieldEncoding[] AllEncodings = values();

abstract String key();
Expand Down
15 changes: 8 additions & 7 deletions community/lucene-index/src/main/java/org/neo4j/kernel/api/impl/insight/InsightIndex.java
View file
Expand Up @@ -19,11 +19,14 @@
*/
package org.neo4j.kernel.api.impl.insight;

import org.apache.lucene.analysis.en.EnglishAnalyzer;
import org.apache.lucene.index.IndexWriterConfig;

import java.io.File;
import java.io.IOException;
import java.nio.file.Paths;
import java.util.Arrays;

import org.neo4j.function.Factory;
import org.neo4j.io.fs.FileSystemAbstraction;
import org.neo4j.kernel.api.impl.index.IndexWriterConfigs;
import org.neo4j.kernel.api.impl.index.builder.LuceneIndexStorageBuilder;
Expand All @@ -35,18 +38,16 @@ public class InsightIndex implements AutoCloseable
{
private final InsightLuceneIndex nodeIndex;

public InsightIndex( FileSystemAbstraction fileSystem, File file, int[] properties ) throws IOException
public InsightIndex( FileSystemAbstraction fileSystem, File file, String... properties ) throws IOException
{
LuceneIndexStorageBuilder storageBuilder = LuceneIndexStorageBuilder.create();
storageBuilder.withFileSystem( fileSystem ).withIndexIdentifier( "insightNodes" )
.withDirectoryFactory( directoryFactory( false, fileSystem ) )
.withIndexRootFolder( Paths.get( file.getAbsolutePath(),"insightindex" ).toFile() );

WritableIndexPartitionFactory partitionFactory = new WritableIndexPartitionFactory(
IndexWriterConfigs::population );
String[] propertyIdsAsStrings = Arrays.stream( properties ).sorted().mapToObj( String::valueOf )
.toArray( String[]::new );
nodeIndex = new InsightLuceneIndex( storageBuilder.build(), partitionFactory, propertyIdsAsStrings );
Factory<IndexWriterConfig> population = () -> IndexWriterConfigs.population( new EnglishAnalyzer() );
WritableIndexPartitionFactory partitionFactory = new WritableIndexPartitionFactory( population );
nodeIndex = new InsightLuceneIndex( storageBuilder.build(), partitionFactory, properties );
nodeIndex.open();
}

Expand Down
23 changes: 12 additions & 11 deletions .../src/main/java/org/neo4j/kernel/api/impl/insight/InsightIndexTransactionEventUpdater.java
View file
Expand Up @@ -22,13 +22,12 @@
import org.apache.lucene.document.Document;

import java.io.IOException;
import java.util.HashMap;
import java.util.Map;

import org.neo4j.graphdb.Node;
import org.neo4j.graphdb.event.PropertyEntry;
import org.neo4j.graphdb.event.TransactionData;
import org.neo4j.graphdb.event.TransactionEventHandler;
import org.neo4j.kernel.api.properties.PropertyKeyValue;
import org.neo4j.values.storable.Values;

public class InsightIndexTransactionEventUpdater implements TransactionEventHandler<Object>
{
Expand All @@ -42,24 +41,26 @@ public class InsightIndexTransactionEventUpdater implements TransactionEventHand
@Override
public Object beforeCommit( TransactionData data ) throws Exception
{
return null;
Map<Long,Map<String,Object>> map = new HashMap<Long,Map<String,Object>>();
data.createdNodes().forEach( node -> map.put( node.getId(), node.getAllProperties() ) );
return map;
}

@Override
public void afterCommit( TransactionData data, Object state )
{
writeNodeData( data.assignedNodeProperties() );
writeNodeData( data.createdNodes(), (Map<Long,Map<String,Object>>) state );
deleteNodeData( data.deletedNodes() );
}

private void writeNodeData( Iterable<PropertyEntry<Node>> propertyEntries )
private void writeNodeData( Iterable<Node> propertyEntries, Map<Long,Map<String,Object>> state )
{
for ( PropertyEntry<Node> propertyEntry : propertyEntries )
for ( Node node : propertyEntries )
{
// propertyEntry.previouslyCommitedValue()
Document document = LuceneInsightDocumentStructure.documentRepresentingProperties( propertyEntry.entity().getId(),
//TODO THIS IS BAD
new PropertyKeyValue( Integer.parseInt( "1" ), Values.of( propertyEntry.value() ) ) );

Map<String,Object> allProperties = state.get( node.getId() );

Document document = LuceneInsightDocumentStructure.documentRepresentingProperties( node.getId(), allProperties );
try
{
writableDatabaseInsightIndex.getIndexWriter().addDocument( document );
Expand Down
15 changes: 8 additions & 7 deletions ...index/src/main/java/org/neo4j/kernel/api/impl/insight/LuceneInsightDocumentStructure.java
View file
Expand Up @@ -27,9 +27,10 @@
import org.apache.lucene.index.Term;

import java.util.Iterator;
import java.util.Map;

import org.neo4j.kernel.api.properties.PropertyKeyValue;
import org.neo4j.values.storable.Value;
import org.neo4j.values.storable.Values;

import static org.apache.lucene.document.Field.Store.YES;

Expand All @@ -50,7 +51,7 @@ private static DocWithId reuseDocument( long nodeId )
return doc;
}

public static Document documentRepresentingProperties( long nodeId, PropertyKeyValue... values )
public static Document documentRepresentingProperties( long nodeId, Map<String,Object> values )
{
DocWithId document = reuseDocument( nodeId );
document.setValues( values );
Expand All @@ -62,10 +63,10 @@ public static long getNodeId( Document from )
return Long.parseLong( from.get( NODE_ID_KEY ) );
}

static Field encodeValueField( int propertyNumber, Value value )
static Field encodeValueField( String propertyKey, Value value )
{
InsightFieldEncoding encoding = InsightFieldEncoding.forValue( value );
return encoding.encodeField( Integer.toString( propertyNumber ), value );
return encoding.encodeField( propertyKey, value );
}

private static class DocWithId
Expand All @@ -91,11 +92,11 @@ private void setId( long id )
idValueField.setLongValue( id );
}

private void setValues( PropertyKeyValue... values )
private void setValues( Map<String,Object> values )
{
for ( int i = 0; i < values.length; i++ )
for ( Map.Entry<String,Object> entry : values.entrySet() )
{
Field field = encodeValueField( values[i].propertyKeyId(), values[i].value() );
Field field = encodeValueField( entry.getKey(), Values.of( entry.getValue() ) );
document.add( field );
}
}
Expand Down
6 changes: 5 additions & 1 deletion ...ucene-index/src/main/java/org/neo4j/kernel/api/impl/insight/SimpleInsightIndexReader.java
View file
Expand Up @@ -61,7 +61,11 @@ public PrimitiveLongIterator query( String... query )
BooleanQuery.Builder builder = new BooleanQuery.Builder();
for ( String property : properties )
{
builder.add( new PhraseQuery( property, query ), BooleanClause.Occur.SHOULD );
builder.add( new PhraseQuery( 2, property, query ), BooleanClause.Occur.SHOULD );
for ( String s : query )
{
builder.add( new PhraseQuery( 2, property, s ), BooleanClause.Occur.SHOULD );
}
}
return query( builder.build() );
}
Expand Down
48 changes: 43 additions & 5 deletions ...-index/src/test/java/org/neo4j/kernel/api/impl/insight/InsightLuceneIndexUpdaterTest.java
View file
Expand Up @@ -23,8 +23,7 @@
import org.junit.Rule;
import org.junit.Test;

import java.io.IOException;

import org.neo4j.collection.primitive.PrimitiveLongIterator;
import org.neo4j.graphdb.Label;
import org.neo4j.graphdb.Node;
import org.neo4j.graphdb.Transaction;
Expand All @@ -33,7 +32,6 @@
import org.neo4j.test.rule.EmbeddedDatabaseRule;
import org.neo4j.test.rule.TestDirectory;
import org.neo4j.test.rule.fs.DefaultFileSystemRule;
import org.neo4j.test.rule.fs.EphemeralFileSystemRule;
import org.neo4j.test.rule.fs.FileSystemRule;

import static org.junit.Assert.assertEquals;
Expand All @@ -54,7 +52,8 @@ public class InsightLuceneIndexUpdaterTest
public void shouldFindNodeWithString() throws Exception
{
GraphDatabaseAPI db = dbRule.getGraphDatabaseAPI();
try (InsightIndex insightIndex = new InsightIndex(fileSystemRule, testDirectory.graphDbDir(), new int[]{1})) {
try ( InsightIndex insightIndex = new InsightIndex( fileSystemRule, testDirectory.graphDbDir(), "prop" ) )
{
db.registerTransactionEventHandler(insightIndex.getUpdater());

long firstID;
Expand Down Expand Up @@ -86,7 +85,8 @@ public void shouldFindNodeWithString() throws Exception
public void shouldNotFindRemovedNodes() throws Exception
{
GraphDatabaseAPI db = dbRule.getGraphDatabaseAPI();
try (InsightIndex insightIndex = new InsightIndex(fileSystemRule, testDirectory.graphDbDir(), new int[]{1})) {
try ( InsightIndex insightIndex = new InsightIndex( fileSystemRule, testDirectory.graphDbDir(), "prop" ) )
{
db.registerTransactionEventHandler(insightIndex.getUpdater());

long firstID;
Expand Down Expand Up @@ -120,4 +120,42 @@ public void shouldNotFindRemovedNodes() throws Exception
}
}
}

@Test
public void shouldOrderResults() throws Exception
{
GraphDatabaseAPI db = dbRule.getGraphDatabaseAPI();
try ( InsightIndex insightIndex = new InsightIndex( fileSystemRule, testDirectory.graphDbDir(), "prop", "prop2" ) )
{
db.registerTransactionEventHandler( insightIndex.getUpdater() );

long firstID;
long secondID;
long thirdID;
try ( Transaction tx = db.beginTx() )
{
Node node = db.createNode( LABEL );
firstID = node.getId();
node.setProperty( "prop", "Tomtar tomtar oftsat i tomteutstyrsel." );
Node node2 = db.createNode( LABEL );
secondID = node2.getId();
node2.setProperty( "prop", "tomtar tomtar tomtar tomtar tomtar." );
node2.setProperty( "prop2", "tomtar tomtar tomtar tomtar tomtar tomtar karl" );
Node node3 = db.createNode( LABEL );
thirdID = node3.getId();
node3.setProperty( "prop", "Tomtar som tomtar ser upp till tomtar som inte tomtar." );

tx.success();
}

try ( InsightIndexReader reader = insightIndex.getReader() )
{

PrimitiveLongIterator iterator = reader.query( "tomtar", "karl" );
assertEquals( secondID, iterator.next() );
assertEquals( firstID, iterator.next() );
assertEquals( thirdID, iterator.next() );
}
}
}
}

0 comments on commit 1fc8258

Please sign in to comment.