Skip to content

Commit

Permalink
Simplify the updater for the fulltext addon
Browse files Browse the repository at this point in the history
  • Loading branch information
@ragadeeshu
ragadeeshu committed Sep 12, 2017
1 parent 18a0d6c commit 4c208da
Show file tree
Hide file tree
Showing 3 changed files with 117 additions and 59 deletions.
8 changes: 4 additions & 4 deletions ...lltext-addon/src/main/java/org/neo4j/kernel/api/impl/fulltext/FulltextHelperProvider.java
View file
Expand Up @@ -32,7 +32,7 @@ public class FulltextHelperProvider implements AutoCloseable
{ {
private static FulltextHelperProvider instance; private static FulltextHelperProvider instance;
private final GraphDatabaseService db; private final GraphDatabaseService db;
private final FulltextHelplerTransactionEventUpdater fulltextHelplerTransactionEventUpdater; private final FulltextHelperTransactionEventUpdater fulltextHelperTransactionEventUpdater;
private boolean closed; private boolean closed;
private Set<String> nodeProperties; private Set<String> nodeProperties;
private Set<String> relationshipProperties; private Set<String> relationshipProperties;
Expand All @@ -43,8 +43,8 @@ private FulltextHelperProvider( GraphDatabaseService db )
{ {
this.db = db; this.db = db;
closed = false; closed = false;
fulltextHelplerTransactionEventUpdater = new FulltextHelplerTransactionEventUpdater( this ); fulltextHelperTransactionEventUpdater = new FulltextHelperTransactionEventUpdater( this );
db.registerTransactionEventHandler( fulltextHelplerTransactionEventUpdater ); db.registerTransactionEventHandler( fulltextHelperTransactionEventUpdater );
nodeProperties = new HashSet<>(); nodeProperties = new HashSet<>();
relationshipProperties = new HashSet<>(); relationshipProperties = new HashSet<>();
nodeIndices = new HashMap<>(); nodeIndices = new HashMap<>();
Expand All @@ -66,7 +66,7 @@ public synchronized void close()
if ( !closed ) if ( !closed )
{ {
closed = true; closed = true;
db.unregisterTransactionEventHandler( fulltextHelplerTransactionEventUpdater ); db.unregisterTransactionEventHandler( fulltextHelperTransactionEventUpdater );
nodeIndices.values().forEach( luceneFulltextHelper -> nodeIndices.values().forEach( luceneFulltextHelper ->
{ {
try try
Expand Down
103 changes: 48 additions & 55 deletions ...lltextHelplerTransactionEventUpdater.java → ...ulltextHelperTransactionEventUpdater.java
View file
Expand Up @@ -22,21 +22,24 @@
import org.apache.lucene.document.Document; import org.apache.lucene.document.Document;


import java.io.IOException; import java.io.IOException;
import java.util.Collections;
import java.util.HashMap; import java.util.HashMap;
import java.util.Map; import java.util.Map;
import java.util.Set;
import java.util.stream.Collectors;


import org.neo4j.graphdb.Entity; import org.neo4j.graphdb.Entity;
import org.neo4j.graphdb.NotFoundException; import org.neo4j.graphdb.NotFoundException;
import org.neo4j.graphdb.event.PropertyEntry; import org.neo4j.graphdb.event.PropertyEntry;
import org.neo4j.graphdb.event.TransactionData; import org.neo4j.graphdb.event.TransactionData;
import org.neo4j.graphdb.event.TransactionEventHandler; import org.neo4j.graphdb.event.TransactionEventHandler;


public class FulltextHelplerTransactionEventUpdater implements TransactionEventHandler<Object> public class FulltextHelperTransactionEventUpdater implements TransactionEventHandler<Object>
{ {


private FulltextHelperProvider fulltextHelperProvider; private FulltextHelperProvider fulltextHelperProvider;


public FulltextHelplerTransactionEventUpdater( FulltextHelperProvider fulltextHelperProvider ) public FulltextHelperTransactionEventUpdater( FulltextHelperProvider fulltextHelperProvider )
{ {
this.fulltextHelperProvider = fulltextHelperProvider; this.fulltextHelperProvider = fulltextHelperProvider;
} }
Expand Down Expand Up @@ -82,79 +85,69 @@ public Object beforeCommit( TransactionData data ) throws Exception
public void afterCommit( TransactionData data, Object state ) public void afterCommit( TransactionData data, Object state )
{ {
//update node indices //update node indices
Map<Long,Map<String,Object>> nodeMap = ((Map<Long,Map<String,Object>>[]) state)[0]; try
for ( WritableDatabaseBloomIndex nodeIndex : fulltextHelperProvider.nodeIndices() )
{ {
updatePropertyData( data.removedNodeProperties(), nodeMap, nodeIndex ); for ( WritableDatabaseBloomIndex nodeIndex : fulltextHelperProvider.nodeIndices() )
updatePropertyData( data.assignedNodeProperties(), nodeMap, nodeIndex ); {
deleteIndexData( data.deletedNodes(), nodeIndex ); Map<Long,Map<String,Object>> nodeMap = ((Map<Long,Map<String,Object>>[]) state)[0];
removePropertyData( data.removedNodeProperties(), nodeMap, nodeIndex );
updatePropertyData( nodeMap, nodeIndex );
refreshIndex( nodeIndex );
}
//update relationship indices
for ( WritableDatabaseBloomIndex relationshipIndex : fulltextHelperProvider.relationshipIndices() )
{
Map<Long,Map<String,Object>> relationshipMap = ((Map<Long,Map<String,Object>>[]) state)[1];
removePropertyData( data.removedRelationshipProperties(), relationshipMap, relationshipIndex );
updatePropertyData( relationshipMap, relationshipIndex );
refreshIndex( relationshipIndex );
}
} }
//update relationship index catch ( IOException e )
for ( WritableDatabaseBloomIndex relationshipIndex : fulltextHelperProvider.relationshipIndices() )
{ {
Map<Long,Map<String,Object>> relationshipMap = ((Map<Long,Map<String,Object>>[]) state)[1]; throw new RuntimeException( "Unable to update fulltext helper index", e );
updatePropertyData( data.removedRelationshipProperties(), relationshipMap, relationshipIndex );
updatePropertyData( data.assignedRelationshipProperties(), relationshipMap, relationshipIndex );
deleteIndexData( data.deletedRelationships(), relationshipIndex );
} }
} }


private <E extends Entity> void updatePropertyData( Iterable<PropertyEntry<E>> propertyEntries, Map<Long,Map<String,Object>> state, private <E extends Entity> void updatePropertyData( Map<Long,Map<String,Object>> state, WritableDatabaseBloomIndex index ) throws IOException
WritableDatabaseBloomIndex index )
{ {
for ( PropertyEntry<E> propertyEntry : propertyEntries ) for ( Map.Entry<Long,Map<String,Object>> stateEntry : state.entrySet() )
{ {
if ( index.properites().contains( propertyEntry.key() ) ) Set<String> indexedProperties = index.properites();
if ( !Collections.disjoint( indexedProperties, stateEntry.getValue().keySet() ) )
{ {
long entityId = propertyEntry.entity().getId(); long entityId = stateEntry.getKey();
Map<String,Object> allProperties = state.get( entityId ); Map<String,Object> allProperties =
if ( allProperties == null ) stateEntry.getValue().entrySet().stream().filter( entry -> indexedProperties.contains( entry.getKey() ) ).collect(
Collectors.toMap( entry -> entry.getKey(), entry -> entry.getValue() ) );
if ( !allProperties.isEmpty() )
{ {
try Document document = FulltextHelperDocumentStructure.documentRepresentingProperties( entityId, allProperties );
{ index.getIndexWriter().updateDocument( FulltextHelperDocumentStructure.newTermForChangeOrRemove( entityId ), document );
index.getIndexWriter().deleteDocuments( BloomDocumentStructure.newTermForChangeOrRemove( entityId ) );
}
catch ( IOException e )
{
e.printStackTrace();
}
continue;
}

Document document = BloomDocumentStructure.documentRepresentingProperties( entityId, allProperties );
try
{
index.getIndexWriter().updateDocument( BloomDocumentStructure.newTermForChangeOrRemove( entityId ), document );
}
catch ( IOException e )
{
e.printStackTrace();
} }
} }
} }
try
{
index.maybeRefreshBlocking();
}
catch ( IOException e )
{
e.printStackTrace();
}
} }


private <E extends Entity> void deleteIndexData( Iterable<E> entities, WritableDatabaseBloomIndex index ) private <E extends Entity> void removePropertyData( Iterable<PropertyEntry<E>> propertyEntries, Map<Long,Map<String,Object>> state,
WritableDatabaseBloomIndex index ) throws IOException
{ {
for ( E entity : entities ) for ( PropertyEntry<E> propertyEntry : propertyEntries )
{ {
try if ( index.properites().contains( propertyEntry.key() ) )
{
index.getIndexWriter().deleteDocuments( BloomDocumentStructure.newTermForChangeOrRemove( entity.getId() ) );
}
catch ( IOException e )
{ {
e.printStackTrace(); long entityId = propertyEntry.entity().getId();
Map<String,Object> allProperties = state.get( entityId );
if ( allProperties == null || allProperties.isEmpty() )
{
index.getIndexWriter().deleteDocuments( FulltextHelperDocumentStructure.newTermForChangeOrRemove( entityId ) );
}
} }
} }
}

private void refreshIndex( WritableDatabaseBloomIndex index )
{
try try
{ {
index.maybeRefreshBlocking(); index.maybeRefreshBlocking();
Expand Down
65 changes: 65 additions & 0 deletions ...don/src/test/java/org/neo4j/kernel/api/impl/fulltext/LuceneFulltextHelperUpdaterTest.java
View file
Expand Up @@ -299,6 +299,71 @@ public void shouldNotFindRemovedNodes() throws Exception
} }
} }


@Test
public void shouldNotFindRemovedProperties() throws Exception
{
GraphDatabaseAPI db = dbRule.withSetting( GraphDatabaseSettings.bloom_indexed_properties, "prop, prop2" ).getGraphDatabaseAPI();
Config config = db.getDependencyResolver().resolveDependency( Config.class );
config.augment( GraphDatabaseSettings.bloom_indexed_properties, "prop, prop2" );
FulltextHelperFactory fulltextHelperFactory = new FulltextHelperFactory( fileSystemRule, testDirectory.graphDbDir(), config );
try ( FulltextHelperProvider provider = FulltextHelperProvider.instance( db ) )
{
provider.register( fulltextHelperFactory.createFulltextHelper( "bloomNodes", FULLTEXT_HELPER_TYPE.NODES ) );

long firstID;
long secondID;
long thirdID;
try ( Transaction tx = db.beginTx() )
{
Node node = db.createNode( LABEL );
firstID = node.getId();
Node node2 = db.createNode( LABEL );
secondID = node2.getId();
Node node3 = db.createNode( LABEL );
thirdID = node3.getId();

node.setProperty( "prop", "Hello. Hello again." );
node.setProperty( "prop", "zebra" );

node2.setProperty( "prop", "A zebroid (also zedonk, zorse, zebra mule, zonkey, and zebmule) is the offspring of any cross " +
"between a zebra and any other equine: essentially, a zebra hybrid." );
node2.setProperty( "prop", "Hello. Hello again." );

node3.setProperty( "prop", "Hello. Hello again." );

tx.success();
}

try ( Transaction tx = db.beginTx() )
{
Node node = db.getNodeById( firstID );
Node node2 = db.getNodeById( secondID );
Node node3 = db.getNodeById( thirdID );

node.setProperty( "prop", "tomtar" );
node.setProperty( "prop2", "tomtar" );

node2.setProperty( "prop", "tomtar" );
node2.setProperty( "prop2", "Hello" );

node3.removeProperty( "prop" );

tx.success();
}

try ( BloomIndexReader reader = provider.getReader( "bloomNodes", FULLTEXT_HELPER_TYPE.NODES ) )
{

PrimitiveLongIterator hello = reader.query( "hello" );
assertEquals( secondID, hello.next() );
assertFalse( hello.hasNext() );
assertFalse( reader.query( "zebra" ).hasNext() );
assertFalse( reader.query( "zedonk" ).hasNext() );
assertFalse( reader.query( "cross" ).hasNext() );
}
}
}

@Test @Test
public void shouldOnlyIndexIndexedProperties() throws Exception public void shouldOnlyIndexIndexedProperties() throws Exception
{ {
Expand Down

0 comments on commit 4c208da

Please sign in to comment.