From 4c208da3065e3c5fcc0c82048bc07d0f2d163e89 Mon Sep 17 00:00:00 2001 From: Ragnar Mellbin Date: Thu, 31 Aug 2017 14:02:09 +0200 Subject: [PATCH] Simplify the updater for the fulltext addon --- .../impl/fulltext/FulltextHelperProvider.java | 8 +- ...ulltextHelperTransactionEventUpdater.java} | 103 ++++++++---------- .../LuceneFulltextHelperUpdaterTest.java | 65 +++++++++++ 3 files changed, 117 insertions(+), 59 deletions(-) rename enterprise/fulltext-addon/src/main/java/org/neo4j/kernel/api/impl/fulltext/{FulltextHelplerTransactionEventUpdater.java => FulltextHelperTransactionEventUpdater.java} (59%) diff --git a/enterprise/fulltext-addon/src/main/java/org/neo4j/kernel/api/impl/fulltext/FulltextHelperProvider.java b/enterprise/fulltext-addon/src/main/java/org/neo4j/kernel/api/impl/fulltext/FulltextHelperProvider.java index 063497224b303..846e0418e607c 100644 --- a/enterprise/fulltext-addon/src/main/java/org/neo4j/kernel/api/impl/fulltext/FulltextHelperProvider.java +++ b/enterprise/fulltext-addon/src/main/java/org/neo4j/kernel/api/impl/fulltext/FulltextHelperProvider.java @@ -32,7 +32,7 @@ public class FulltextHelperProvider implements AutoCloseable { private static FulltextHelperProvider instance; private final GraphDatabaseService db; - private final FulltextHelplerTransactionEventUpdater fulltextHelplerTransactionEventUpdater; + private final FulltextHelperTransactionEventUpdater fulltextHelperTransactionEventUpdater; private boolean closed; private Set nodeProperties; private Set relationshipProperties; @@ -43,8 +43,8 @@ private FulltextHelperProvider( GraphDatabaseService db ) { this.db = db; closed = false; - fulltextHelplerTransactionEventUpdater = new FulltextHelplerTransactionEventUpdater( this ); - db.registerTransactionEventHandler( fulltextHelplerTransactionEventUpdater ); + fulltextHelperTransactionEventUpdater = new FulltextHelperTransactionEventUpdater( this ); + db.registerTransactionEventHandler( fulltextHelperTransactionEventUpdater ); nodeProperties = new HashSet<>(); relationshipProperties = new HashSet<>(); nodeIndices = new HashMap<>(); @@ -66,7 +66,7 @@ public synchronized void close() if ( !closed ) { closed = true; - db.unregisterTransactionEventHandler( fulltextHelplerTransactionEventUpdater ); + db.unregisterTransactionEventHandler( fulltextHelperTransactionEventUpdater ); nodeIndices.values().forEach( luceneFulltextHelper -> { try diff --git a/enterprise/fulltext-addon/src/main/java/org/neo4j/kernel/api/impl/fulltext/FulltextHelplerTransactionEventUpdater.java b/enterprise/fulltext-addon/src/main/java/org/neo4j/kernel/api/impl/fulltext/FulltextHelperTransactionEventUpdater.java similarity index 59% rename from enterprise/fulltext-addon/src/main/java/org/neo4j/kernel/api/impl/fulltext/FulltextHelplerTransactionEventUpdater.java rename to enterprise/fulltext-addon/src/main/java/org/neo4j/kernel/api/impl/fulltext/FulltextHelperTransactionEventUpdater.java index 9772cd365cc94..11ba41981b055 100644 --- a/enterprise/fulltext-addon/src/main/java/org/neo4j/kernel/api/impl/fulltext/FulltextHelplerTransactionEventUpdater.java +++ b/enterprise/fulltext-addon/src/main/java/org/neo4j/kernel/api/impl/fulltext/FulltextHelperTransactionEventUpdater.java @@ -22,8 +22,11 @@ import org.apache.lucene.document.Document; import java.io.IOException; +import java.util.Collections; import java.util.HashMap; import java.util.Map; +import java.util.Set; +import java.util.stream.Collectors; import org.neo4j.graphdb.Entity; import org.neo4j.graphdb.NotFoundException; @@ -31,12 +34,12 @@ import org.neo4j.graphdb.event.TransactionData; import org.neo4j.graphdb.event.TransactionEventHandler; -public class FulltextHelplerTransactionEventUpdater implements TransactionEventHandler +public class FulltextHelperTransactionEventUpdater implements TransactionEventHandler { private FulltextHelperProvider fulltextHelperProvider; - public FulltextHelplerTransactionEventUpdater( FulltextHelperProvider fulltextHelperProvider ) + public FulltextHelperTransactionEventUpdater( FulltextHelperProvider fulltextHelperProvider ) { this.fulltextHelperProvider = fulltextHelperProvider; } @@ -82,79 +85,69 @@ public Object beforeCommit( TransactionData data ) throws Exception public void afterCommit( TransactionData data, Object state ) { //update node indices - Map> nodeMap = ((Map>[]) state)[0]; - for ( WritableDatabaseBloomIndex nodeIndex : fulltextHelperProvider.nodeIndices() ) + try { - updatePropertyData( data.removedNodeProperties(), nodeMap, nodeIndex ); - updatePropertyData( data.assignedNodeProperties(), nodeMap, nodeIndex ); - deleteIndexData( data.deletedNodes(), nodeIndex ); + for ( WritableDatabaseBloomIndex nodeIndex : fulltextHelperProvider.nodeIndices() ) + { + Map> nodeMap = ((Map>[]) state)[0]; + removePropertyData( data.removedNodeProperties(), nodeMap, nodeIndex ); + updatePropertyData( nodeMap, nodeIndex ); + refreshIndex( nodeIndex ); + } + //update relationship indices + for ( WritableDatabaseBloomIndex relationshipIndex : fulltextHelperProvider.relationshipIndices() ) + { + Map> relationshipMap = ((Map>[]) state)[1]; + removePropertyData( data.removedRelationshipProperties(), relationshipMap, relationshipIndex ); + updatePropertyData( relationshipMap, relationshipIndex ); + refreshIndex( relationshipIndex ); + } } - //update relationship index - for ( WritableDatabaseBloomIndex relationshipIndex : fulltextHelperProvider.relationshipIndices() ) + catch ( IOException e ) { - Map> relationshipMap = ((Map>[]) state)[1]; - updatePropertyData( data.removedRelationshipProperties(), relationshipMap, relationshipIndex ); - updatePropertyData( data.assignedRelationshipProperties(), relationshipMap, relationshipIndex ); - deleteIndexData( data.deletedRelationships(), relationshipIndex ); + throw new RuntimeException( "Unable to update fulltext helper index", e ); } } - private void updatePropertyData( Iterable> propertyEntries, Map> state, - WritableDatabaseBloomIndex index ) + private void updatePropertyData( Map> state, WritableDatabaseBloomIndex index ) throws IOException { - for ( PropertyEntry propertyEntry : propertyEntries ) + for ( Map.Entry> stateEntry : state.entrySet() ) { - if ( index.properites().contains( propertyEntry.key() ) ) + Set indexedProperties = index.properites(); + if ( !Collections.disjoint( indexedProperties, stateEntry.getValue().keySet() ) ) { - long entityId = propertyEntry.entity().getId(); - Map allProperties = state.get( entityId ); - if ( allProperties == null ) + long entityId = stateEntry.getKey(); + Map allProperties = + stateEntry.getValue().entrySet().stream().filter( entry -> indexedProperties.contains( entry.getKey() ) ).collect( + Collectors.toMap( entry -> entry.getKey(), entry -> entry.getValue() ) ); + if ( !allProperties.isEmpty() ) { - try - { - index.getIndexWriter().deleteDocuments( BloomDocumentStructure.newTermForChangeOrRemove( entityId ) ); - } - catch ( IOException e ) - { - e.printStackTrace(); - } - continue; - } - - Document document = BloomDocumentStructure.documentRepresentingProperties( entityId, allProperties ); - try - { - index.getIndexWriter().updateDocument( BloomDocumentStructure.newTermForChangeOrRemove( entityId ), document ); - } - catch ( IOException e ) - { - e.printStackTrace(); + Document document = FulltextHelperDocumentStructure.documentRepresentingProperties( entityId, allProperties ); + index.getIndexWriter().updateDocument( FulltextHelperDocumentStructure.newTermForChangeOrRemove( entityId ), document ); } } } - try - { - index.maybeRefreshBlocking(); - } - catch ( IOException e ) - { - e.printStackTrace(); - } } - private void deleteIndexData( Iterable entities, WritableDatabaseBloomIndex index ) + private void removePropertyData( Iterable> propertyEntries, Map> state, + WritableDatabaseBloomIndex index ) throws IOException { - for ( E entity : entities ) + for ( PropertyEntry propertyEntry : propertyEntries ) { - try - { - index.getIndexWriter().deleteDocuments( BloomDocumentStructure.newTermForChangeOrRemove( entity.getId() ) ); - } - catch ( IOException e ) + if ( index.properites().contains( propertyEntry.key() ) ) { - e.printStackTrace(); + long entityId = propertyEntry.entity().getId(); + Map allProperties = state.get( entityId ); + if ( allProperties == null || allProperties.isEmpty() ) + { + index.getIndexWriter().deleteDocuments( FulltextHelperDocumentStructure.newTermForChangeOrRemove( entityId ) ); + } } } + } + + private void refreshIndex( WritableDatabaseBloomIndex index ) + { try { index.maybeRefreshBlocking(); diff --git a/enterprise/fulltext-addon/src/test/java/org/neo4j/kernel/api/impl/fulltext/LuceneFulltextHelperUpdaterTest.java b/enterprise/fulltext-addon/src/test/java/org/neo4j/kernel/api/impl/fulltext/LuceneFulltextHelperUpdaterTest.java index db8c58496a2f4..ce848b3ee2d15 100644 --- a/enterprise/fulltext-addon/src/test/java/org/neo4j/kernel/api/impl/fulltext/LuceneFulltextHelperUpdaterTest.java +++ b/enterprise/fulltext-addon/src/test/java/org/neo4j/kernel/api/impl/fulltext/LuceneFulltextHelperUpdaterTest.java @@ -299,6 +299,71 @@ public void shouldNotFindRemovedNodes() throws Exception } } + @Test + public void shouldNotFindRemovedProperties() throws Exception + { + GraphDatabaseAPI db = dbRule.withSetting( GraphDatabaseSettings.bloom_indexed_properties, "prop, prop2" ).getGraphDatabaseAPI(); + Config config = db.getDependencyResolver().resolveDependency( Config.class ); + config.augment( GraphDatabaseSettings.bloom_indexed_properties, "prop, prop2" ); + FulltextHelperFactory fulltextHelperFactory = new FulltextHelperFactory( fileSystemRule, testDirectory.graphDbDir(), config ); + try ( FulltextHelperProvider provider = FulltextHelperProvider.instance( db ) ) + { + provider.register( fulltextHelperFactory.createFulltextHelper( "bloomNodes", FULLTEXT_HELPER_TYPE.NODES ) ); + + long firstID; + long secondID; + long thirdID; + try ( Transaction tx = db.beginTx() ) + { + Node node = db.createNode( LABEL ); + firstID = node.getId(); + Node node2 = db.createNode( LABEL ); + secondID = node2.getId(); + Node node3 = db.createNode( LABEL ); + thirdID = node3.getId(); + + node.setProperty( "prop", "Hello. Hello again." ); + node.setProperty( "prop", "zebra" ); + + node2.setProperty( "prop", "A zebroid (also zedonk, zorse, zebra mule, zonkey, and zebmule) is the offspring of any cross " + + "between a zebra and any other equine: essentially, a zebra hybrid." ); + node2.setProperty( "prop", "Hello. Hello again." ); + + node3.setProperty( "prop", "Hello. Hello again." ); + + tx.success(); + } + + try ( Transaction tx = db.beginTx() ) + { + Node node = db.getNodeById( firstID ); + Node node2 = db.getNodeById( secondID ); + Node node3 = db.getNodeById( thirdID ); + + node.setProperty( "prop", "tomtar" ); + node.setProperty( "prop2", "tomtar" ); + + node2.setProperty( "prop", "tomtar" ); + node2.setProperty( "prop2", "Hello" ); + + node3.removeProperty( "prop" ); + + tx.success(); + } + + try ( BloomIndexReader reader = provider.getReader( "bloomNodes", FULLTEXT_HELPER_TYPE.NODES ) ) + { + + PrimitiveLongIterator hello = reader.query( "hello" ); + assertEquals( secondID, hello.next() ); + assertFalse( hello.hasNext() ); + assertFalse( reader.query( "zebra" ).hasNext() ); + assertFalse( reader.query( "zedonk" ).hasNext() ); + assertFalse( reader.query( "cross" ).hasNext() ); + } + } + } + @Test public void shouldOnlyIndexIndexedProperties() throws Exception {