Skip to content

Commit

Permalink
Only index indexed properties in bloom index
Browse files Browse the repository at this point in the history
  • Loading branch information
@ragadeeshu
ragadeeshu committed Sep 12, 2017
1 parent 6af6b14 commit 4e8ba7f
Show file tree
Hide file tree
Showing 3 changed files with 122 additions and 36 deletions.
6 changes: 4 additions & 2 deletions community/lucene-index/src/main/java/org/neo4j/kernel/api/impl/insight/InsightIndex.java
View file
Original file line number Diff line number Diff line change
Expand Up @@ -38,17 +38,19 @@ public class InsightIndex implements AutoCloseable
{
private final InsightLuceneIndex nodeIndex;
private final InsightLuceneIndex relationshipIndex;
private final String[] properties;

public InsightIndex( FileSystemAbstraction fileSystem, File file, String... properties ) throws IOException
{
this.properties = properties;
Factory<IndexWriterConfig> population = () -> IndexWriterConfigs.population( new EnglishAnalyzer() );
WritableIndexPartitionFactory partitionFactory = new WritableIndexPartitionFactory( population );

LuceneIndexStorageBuilder storageBuilder = LuceneIndexStorageBuilder.create();
storageBuilder.withFileSystem( fileSystem ).withIndexIdentifier( "insightNodes" )
.withDirectoryFactory( directoryFactory( false, fileSystem ) )
.withIndexRootFolder( Paths.get( file.getAbsolutePath(),"insightindex" ).toFile() );
nodeIndex = new InsightLuceneIndex( storageBuilder.build(), partitionFactory, properties );
nodeIndex = new InsightLuceneIndex( storageBuilder.build(), partitionFactory, this.properties );
nodeIndex.open();

storageBuilder = LuceneIndexStorageBuilder.create();
Expand All @@ -63,7 +65,7 @@ public InsightIndexTransactionEventUpdater getUpdater() throws IOException
{
WritableDatabaseInsightIndex writableNodeIndex = new WritableDatabaseInsightIndex( nodeIndex );
WritableDatabaseInsightIndex writableRelationshipIndex = new WritableDatabaseInsightIndex( relationshipIndex );
return new InsightIndexTransactionEventUpdater( writableNodeIndex, writableRelationshipIndex );
return new InsightIndexTransactionEventUpdater( writableNodeIndex, writableRelationshipIndex, properties );
}

public InsightIndexReader getNodeReader() throws IOException
Expand Down
10 changes: 6 additions & 4 deletions .../src/main/java/org/neo4j/kernel/api/impl/insight/InsightIndexTransactionEventUpdater.java
View file
Original file line number Diff line number Diff line change
Expand Up @@ -35,11 +35,13 @@ public class InsightIndexTransactionEventUpdater implements TransactionEventHand
{
private final WritableDatabaseInsightIndex nodeIndex;
private final WritableDatabaseInsightIndex relationshipIndex;
private final String[] properties;

InsightIndexTransactionEventUpdater( WritableDatabaseInsightIndex nodeIndex, WritableDatabaseInsightIndex relationshipIndex )
InsightIndexTransactionEventUpdater( WritableDatabaseInsightIndex nodeIndex, WritableDatabaseInsightIndex relationshipIndex, String[] properties )
{
this.nodeIndex = nodeIndex;
this.relationshipIndex = relationshipIndex;
this.properties = properties;
}

@Override
Expand All @@ -49,7 +51,7 @@ public Object beforeCommit( TransactionData data ) throws Exception
data.removedNodeProperties().forEach( propertyEntry -> {
try
{
nodeMap.put( propertyEntry.entity().getId(), propertyEntry.entity().getAllProperties() );
nodeMap.put( propertyEntry.entity().getId(), propertyEntry.entity().getProperties( properties ) );
}
catch ( NotFoundException e )
{
Expand All @@ -63,15 +65,15 @@ public Object beforeCommit( TransactionData data ) throws Exception
data.removedRelationshipProperties().forEach( propertyEntry -> {
try
{
relationshipMap.put( propertyEntry.entity().getId(), propertyEntry.entity().getAllProperties() );
relationshipMap.put( propertyEntry.entity().getId(), propertyEntry.entity().getProperties( properties ) );
}
catch ( NotFoundException e )
{
//This means that the relationship was deleted.
}
} );
data.assignedRelationshipProperties().forEach(
propertyEntry -> relationshipMap.put( propertyEntry.entity().getId(), propertyEntry.entity().getAllProperties() ) );
propertyEntry -> relationshipMap.put( propertyEntry.entity().getId(), propertyEntry.entity().getProperties( properties ) ) );
return new Map[]{nodeMap, relationshipMap};
}

Expand Down
142 changes: 112 additions & 30 deletions ...-index/src/test/java/org/neo4j/kernel/api/impl/insight/InsightLuceneIndexUpdaterTest.java
View file
Original file line number Diff line number Diff line change
Expand Up @@ -57,29 +57,30 @@ public void shouldFindNodeWithString() throws Exception
GraphDatabaseAPI db = dbRule.getGraphDatabaseAPI();
try ( InsightIndex insightIndex = new InsightIndex( fileSystemRule, testDirectory.graphDbDir(), "prop" ) )
{
db.registerTransactionEventHandler(insightIndex.getUpdater());
db.registerTransactionEventHandler( insightIndex.getUpdater() );

long firstID;
long secondID;
try (Transaction tx = db.beginTx()) {
Node node = db.createNode(LABEL);
try ( Transaction tx = db.beginTx() )
{
Node node = db.createNode( LABEL );
firstID = node.getId();
node.setProperty("prop", "Hello. Hello again.");
Node node2 = db.createNode(LABEL);
node.setProperty( "prop", "Hello. Hello again." );
Node node2 = db.createNode( LABEL );
secondID = node2.getId();
node2.setProperty("prop",
"A zebroid (also zedonk, zorse, zebra mule, zonkey, and zebmule) is the offspring of any cross " +
"between a zebra and any other equine: essentially, a zebra hybrid.");
node2.setProperty( "prop", "A zebroid (also zedonk, zorse, zebra mule, zonkey, and zebmule) is the offspring of any cross " +
"between a zebra and any other equine: essentially, a zebra hybrid." );

tx.success();
}

try (InsightIndexReader reader = insightIndex.getNodeReader()) {
try ( InsightIndexReader reader = insightIndex.getNodeReader() )
{

assertEquals(firstID, reader.query("hello").next());
assertEquals(secondID, reader.query("zebra").next());
assertEquals(secondID, reader.query("zedonk").next());
assertEquals(secondID, reader.query("cross").next());
assertEquals( firstID, reader.query( "hello" ).next() );
assertEquals( secondID, reader.query( "zebra" ).next() );
assertEquals( secondID, reader.query( "zedonk" ).next() );
assertEquals( secondID, reader.query( "cross" ).next() );
}
}
}
Expand Down Expand Up @@ -138,36 +139,72 @@ public void shouldNotFindRemovedNodes() throws Exception
GraphDatabaseAPI db = dbRule.getGraphDatabaseAPI();
try ( InsightIndex insightIndex = new InsightIndex( fileSystemRule, testDirectory.graphDbDir(), "prop" ) )
{
db.registerTransactionEventHandler(insightIndex.getUpdater());
db.registerTransactionEventHandler( insightIndex.getUpdater() );

long firstID;
long secondID;
try (Transaction tx = db.beginTx()) {
Node node = db.createNode(LABEL);
try ( Transaction tx = db.beginTx() )
{
Node node = db.createNode( LABEL );
firstID = node.getId();
node.setProperty("prop", "Hello. Hello again.");
Node node2 = db.createNode(LABEL);
node.setProperty( "prop", "Hello. Hello again." );
Node node2 = db.createNode( LABEL );
secondID = node2.getId();
node2.setProperty("prop",
"A zebroid (also zedonk, zorse, zebra mule, zonkey, and zebmule) is the offspring of any cross " +
"between a zebra and any other equine: essentially, a zebra hybrid.");
node2.setProperty( "prop", "A zebroid (also zedonk, zorse, zebra mule, zonkey, and zebmule) is the offspring of any cross " +
"between a zebra and any other equine: essentially, a zebra hybrid." );

tx.success();
}

try (Transaction tx = db.beginTx()) {
db.getNodeById(firstID).delete();
db.getNodeById(secondID).delete();
try ( Transaction tx = db.beginTx() )
{
db.getNodeById( firstID ).delete();
db.getNodeById( secondID ).delete();

tx.success();
}

try (InsightIndexReader reader = insightIndex.getNodeReader()) {
try ( InsightIndexReader reader = insightIndex.getNodeReader() )
{

assertFalse(reader.query("hello").hasNext());
assertFalse(reader.query("zebra").hasNext());
assertFalse(reader.query("zedonk").hasNext());
assertFalse(reader.query("cross").hasNext());
assertFalse( reader.query( "hello" ).hasNext() );
assertFalse( reader.query( "zebra" ).hasNext() );
assertFalse( reader.query( "zedonk" ).hasNext() );
assertFalse( reader.query( "cross" ).hasNext() );
}
}
}

@Test
public void shouldOnlyIndexIndexedProperties() throws Exception
{
GraphDatabaseAPI db = dbRule.getGraphDatabaseAPI();
try ( InsightIndex insightIndex = new InsightIndex( fileSystemRule, testDirectory.graphDbDir(), "prop" ) )
{
db.registerTransactionEventHandler( insightIndex.getUpdater() );

long firstID;
long secondID;
try ( Transaction tx = db.beginTx() )
{
Node node = db.createNode( LABEL );
firstID = node.getId();
node.setProperty( "prop", "Hello. Hello again." );
node.setProperty( "prop2", "zebra" );
Node node2 = db.createNode( LABEL );
node2.setProperty( "prop2", "zebra" );
node2.setProperty( "prop3", "hello" );

tx.success();
}

try ( InsightIndexReader reader = insightIndex.getNodeReader() )
{

PrimitiveLongIterator hello = reader.query( "hello" );
assertEquals( firstID, hello.next() );
assertFalse( hello.hasNext() );
assertFalse( reader.query( "zebra" ).hasNext() );
}
}
}
Expand Down Expand Up @@ -210,6 +247,52 @@ public void shouldSearchAcrossMultipleProperties() throws Exception
}
}

@Test
public void shouldOrderResultsBasedOnRelevance() throws Exception
{
GraphDatabaseAPI db = dbRule.getGraphDatabaseAPI();
try ( InsightIndex insightIndex = new InsightIndex( fileSystemRule, testDirectory.graphDbDir(), "first", "last" ) )
{
db.registerTransactionEventHandler( insightIndex.getUpdater() );

long firstID;
long secondID;
long thirdID;
long fourthID;
try ( Transaction tx = db.beginTx() )
{
Node node = db.createNode( LABEL );
firstID = node.getId();
node.setProperty( "first", "Full" );
node.setProperty( "last", "Hanks" );
Node node2 = db.createNode( LABEL );
secondID = node2.getId();
node2.setProperty( "first", "Tom" );
node2.setProperty( "last", "Hunk" );
Node node3 = db.createNode( LABEL );
thirdID = node3.getId();
node3.setProperty( "first", "Tom" );
node3.setProperty( "last", "Hanks" );
Node node4 = db.createNode( LABEL );
fourthID = node4.getId();
node4.setProperty( "first", "Tom Hanks" );
node4.setProperty( "last", "Tom Hanks" );

tx.success();
}

try ( InsightIndexReader reader = insightIndex.getNodeReader() )
{

PrimitiveLongIterator iterator = reader.query( "Tom", "Hanks" );
assertEquals( fourthID, iterator.next() );
assertEquals( thirdID, iterator.next() );
assertEquals( firstID, iterator.next() );
assertEquals( secondID, iterator.next() );
}
}
}

@Test
public void shouldDifferentiateNodesAndRelationships() throws Exception
{
Expand Down Expand Up @@ -267,5 +350,4 @@ public void shouldDifferentiateNodesAndRelationships() throws Exception
}
}
}

}

0 comments on commit 4e8ba7f

Please sign in to comment.