Skip to content

Commit

Permalink
Clean up the procedures surface for the Bloom fulltext index add-on
Browse files Browse the repository at this point in the history 
Collect them under the `bloom` namespace, give them more descriptive names, and make them symmetrical around nodes and relationships.
  • Loading branch information
@chrisvest
chrisvest committed Oct 5, 2017
1 parent a903fd0 commit 397e673
Show file tree
Hide file tree
Showing 2 changed files with 86 additions and 60 deletions.
55 changes: 36 additions & 19 deletions .../src/main/java/org/neo4j/kernel/api/impl/fulltext/integrations/bloom/BloomProcedures.java
View file
Expand Up @@ -19,7 +19,6 @@
*/
package org.neo4j.kernel.api.impl.fulltext.integrations.bloom;

import java.util.Arrays;
import java.util.Iterator;
import java.util.List;
import java.util.Spliterator;
Expand Down Expand Up @@ -53,38 +52,54 @@ public class BloomProcedures
public FulltextProvider provider;

@Description( "Await the completion of any background index population or updates" )
@Procedure( name = "db.fulltext.bloomAwaitPopulation", mode = READ )
@Procedure( name = "bloom.awaitPopulation", mode = READ )
public void awaitPopulation() throws Exception
{
provider.awaitPopulation();
}

@Description( "Returns the property keys indexed by the bloom addon" )
@Procedure( name = "db.fulltext.bloomFulltextGetPropertyKeys", mode = READ )
public Stream<PropertyOutput> bloomFulltextGetPropertyKeys() throws Exception
@Description( "Returns the node property keys indexed by the Bloom fulltext index add-on" )
@Procedure( name = "bloom.getIndexedNodePropertyKeys", mode = READ )
public Stream<PropertyOutput> getIndexedNodePropertyKeys() throws Exception
{
return provider.getProperties( BLOOM_NODES, NODES ).stream().map( PropertyOutput::new );
}

@Description( "Check the status of the bloom addon" )
@Procedure( name = "db.fulltext.bloomFulltextStatus", mode = READ )
public Stream<StatusOutput> bloomFulltextStatus() throws Exception
@Description( "Returns the relationship property keys indexed by the Bloom fulltext index add-on" )
@Procedure( name = "bloom.getIndexedRelationshipPropertyKeys", mode = READ )
public Stream<PropertyOutput> getIndexedRelationshipPropertyKeys() throws Exception
{
List<InternalIndexState> states = Arrays.asList( provider.getState( BLOOM_NODES, NODES ), provider.getState( BLOOM_RELATIONSHIPS, RELATIONSHIPS ) );
return states.stream().map( StatusOutput::new );
return provider.getProperties( BLOOM_NODES, NODES ).stream().map( PropertyOutput::new );
}

@Description( "Set the property keys to index" )
@Procedure( name = "db.fulltext.bloomFulltextSetPropertyKeys", mode = SCHEMA )
public void bloomFulltextSetPropertyKeys( @Name( "propertyKeys" ) List<String> propertyKeys ) throws Exception
@Description( "Set the node property keys to index" )
@Procedure( name = "bloom.setIndexedNodePropertyKeys", mode = SCHEMA )
public void setIndexedNodePropertyKeys( @Name( "propertyKeys" ) List<String> propertyKeys ) throws Exception
{
provider.changeIndexedProperties( BLOOM_NODES, NODES, propertyKeys );
}

@Description( "Set the relationship property keys to index" )
@Procedure( name = "bloom.setIndexedRelationshipPropertyKeys", mode = SCHEMA )
public void setIndexedRelationshipPropertyKeys( @Name( "propertyKeys" ) List<String> propertyKeys ) throws Exception
{
provider.changeIndexedProperties( BLOOM_RELATIONSHIPS, RELATIONSHIPS, propertyKeys );
}

@Description( "Queries the bloom index for nodes" )
@Procedure( name = "db.fulltext.bloomFulltextNodes", mode = READ )
public Stream<EntityOutput> bloomFulltextNodes( @Name( "terms" ) List<String> terms, @Name( value = "fuzzy", defaultValue = "true" ) boolean fuzzy,
@Description( "Check the status of the Bloom fulltext index add-on" )
@Procedure( name = "bloom.indexStatus", mode = READ )
public Stream<StatusOutput> indexStatus() throws Exception
{
InternalIndexState nodeIndexState = provider.getState( BLOOM_NODES, NODES );
InternalIndexState relationshipIndexState = provider.getState( BLOOM_RELATIONSHIPS, RELATIONSHIPS );
return Stream.of( nodeIndexState, relationshipIndexState ).map( StatusOutput::new );
}

@Description( "Query the Bloom fulltext index for nodes" )
@Procedure( name = "bloom.searchNodes", mode = READ )
public Stream<EntityOutput> bloomFulltextNodes(
@Name( "terms" ) List<String> terms,
@Name( value = "fuzzy", defaultValue = "true" ) boolean fuzzy,
@Name( value = "matchAll", defaultValue = "false" ) boolean matchAll ) throws Exception
{
try ( ReadOnlyFulltext indexReader = provider.getReader( BLOOM_NODES, NODES ) )
Expand All @@ -93,9 +108,11 @@ public Stream<EntityOutput> bloomFulltextNodes( @Name( "terms" ) List<String> te
}
}

@Description( "Queries the bloom index for relationships" )
@Procedure( name = "db.fulltext.bloomFulltextRelationships", mode = READ )
public Stream<EntityOutput> bloomFulltextRelationships( @Name( "terms" ) List<String> terms, @Name( value = "fuzzy", defaultValue = "true" ) boolean fuzzy,
@Description( "Query the Bloom fulltext index for relationships" )
@Procedure( name = "bloom.searchRelationships", mode = READ )
public Stream<EntityOutput> bloomFulltextRelationships(
@Name( "terms" ) List<String> terms,
@Name( value = "fuzzy", defaultValue = "true" ) boolean fuzzy,
@Name( value = "matchAll", defaultValue = "false" ) boolean matchAll ) throws Exception
{
try ( ReadOnlyFulltext indexReader = provider.getReader( BLOOM_RELATIONSHIPS, RELATIONSHIPS ) )
Expand Down
91 changes: 50 additions & 41 deletions ...xt-addon/src/test/java/org/neo4j/kernel/api/impl/fulltext/integrations/bloom/BloomIT.java
View file
Expand Up @@ -63,12 +63,17 @@

public class BloomIT
{
private static final String NODES = "CALL db.fulltext.bloomFulltextNodes([%s])";
private static final String NODES_ADVANCED = "CALL db.fulltext.bloomFulltextNodes([%s], %b, %b)";
private static final String RELS = "CALL db.fulltext.bloomFulltextRelationships([%s])";
private static final String RELS_ADVANCED = "CALL db.fulltext.bloomFulltextRelationships([%s], %b, %b)";
private static final String NODES = "CALL bloom.searchNodes([%s])";
private static final String NODES_ADVANCED = "CALL bloom.searchNodes([%s], %b, %b)";
private static final String RELS = "CALL bloom.searchRelationships([%s])";
private static final String RELS_ADVANCED = "CALL bloom.searchRelationships([%s], %b, %b)";
private static final String ENTITYID = "entityid";
private static final String KEYS = "CALL db.fulltext.bloomFulltextSetPropertyKeys([%s])";
private static final String SET_NODE_KEYS = "CALL bloom.setIndexedNodePropertyKeys([%s])";
private static final String SET_REL_KEYS = "CALL bloom.setIndexedRelationshipPropertyKeys([%s])";
private static final String GET_NODE_KEYS = "CALL bloom.getIndexedNodePropertyKeys";
private static final String GET_REL_KEYS = "CALL bloom.getIndexedNodePropertyKeys";
private static final String AWAIT_POPULATION = "CALL bloom.awaitPopulation";
private static final String STATUS = "CALL bloom.indexStatus";

@Rule
public final DefaultFileSystemRule fs = new DefaultFileSystemRule();
Expand Down Expand Up @@ -108,7 +113,8 @@ private GraphDatabaseService getDb() throws KernelException
public void shouldPopulateAndQueryIndexes() throws Exception
{
db = getDb();
db.execute( String.format( KEYS, "\"prop\", \"relprop\"" ) );
db.execute( String.format( SET_NODE_KEYS, "\"prop\", \"relprop\"" ) );
db.execute( String.format( SET_REL_KEYS, "\"prop\", \"relprop\"" ) );
try ( Transaction transaction = db.beginTx() )
{
Node node1 = db.createNode();
Expand Down Expand Up @@ -136,7 +142,8 @@ public void shouldPopulateAndQueryIndexes() throws Exception
public void exactQueryShouldBeExact() throws Exception
{
db = getDb();
db.execute( String.format( KEYS, "\"prop\"" ) );
db.execute( String.format( SET_NODE_KEYS, "\"prop\"" ) );
db.execute( String.format( SET_REL_KEYS, "\"prop\"" ) );
try ( Transaction transaction = db.beginTx() )
{
Node node1 = db.createNode();
Expand Down Expand Up @@ -166,7 +173,8 @@ public void exactQueryShouldBeExact() throws Exception
public void matchAllQueryShouldMatchAll() throws Exception
{
db = getDb();
db.execute( String.format( KEYS, "\"prop\"" ) );
db.execute( String.format( SET_NODE_KEYS, "\"prop\"" ) );
db.execute( String.format( SET_REL_KEYS, "\"prop\"" ) );
try ( Transaction transaction = db.beginTx() )
{
Node node1 = db.createNode();
Expand Down Expand Up @@ -194,7 +202,7 @@ public void shouldBeAbleToConfigureAnalyzer() throws Exception
{
builder.setConfig( BloomFulltextConfig.bloom_analyzer, "org.apache.lucene.analysis.sv.SwedishAnalyzer" );
db = getDb();
db.execute( String.format( KEYS, "\"prop\"" ) );
db.execute( String.format( SET_NODE_KEYS, "\"prop\"" ) );
try ( Transaction transaction = db.beginTx() )
{
Node node1 = db.createNode();
Expand All @@ -220,7 +228,7 @@ public void shouldBeAbleToConfigureAnalyzer() throws Exception
public void shouldPopulateIndexWithExistingDataOnIndexCreate() throws Exception
{
db = getDb();
db.execute( String.format( KEYS, "\"something\"" ) );
db.execute( String.format( SET_NODE_KEYS, "\"something\"" ) );

long nodeId;
try ( Transaction tx = db.beginTx() )
Expand All @@ -236,9 +244,9 @@ public void shouldPopulateIndexWithExistingDataOnIndexCreate() throws Exception

builder.setConfig( BloomFulltextConfig.bloom_analyzer, "org.apache.lucene.analysis.da.DanishAnalyzer" );
db = getDb();
db.execute( String.format( KEYS, "\"prop\"" ) );
db.execute( String.format( SET_NODE_KEYS, "\"prop\"" ) );

db.execute( "CALL db.fulltext.bloomAwaitPopulation" ).close();
db.execute( AWAIT_POPULATION ).close();

result = db.execute( String.format( NODES, "\"Roskildevej\"") );
assertTrue( result.hasNext() );
Expand All @@ -251,7 +259,7 @@ public void startupPopulationShouldNotCauseDuplicates() throws Exception
{

db = getDb();
db.execute( String.format( KEYS, "\"prop\"" ) );
db.execute( String.format( SET_NODE_KEYS, "\"prop\"" ) );
long nodeId;
try ( Transaction tx = db.beginTx() )
{
Expand All @@ -262,7 +270,7 @@ public void startupPopulationShouldNotCauseDuplicates() throws Exception
}

// Verify it's indexed exactly once
db.execute( "CALL db.fulltext.bloomAwaitPopulation" ).close();
db.execute( AWAIT_POPULATION ).close();
Result result = db.execute( String.format( NODES, "\"Jyllingevej\"") );
assertTrue( result.hasNext() );
assertEquals( nodeId, result.next().get( ENTITYID ) );
Expand All @@ -271,7 +279,7 @@ public void startupPopulationShouldNotCauseDuplicates() throws Exception
db.shutdown();
db = getDb();

db.execute( "CALL db.fulltext.bloomAwaitPopulation" ).close();
db.execute( AWAIT_POPULATION ).close();
// Verify it's STILL indexed exactly once
result = db.execute( String.format( NODES, "\"Jyllingevej\"") );
assertTrue( result.hasNext() );
Expand All @@ -284,7 +292,7 @@ public void shouldNotBeAbleToFindNodesAfterRemovingIndex() throws Exception
{

db = getDb();
db.execute( String.format( KEYS, "\"prop\"" ) );
db.execute( String.format( SET_NODE_KEYS, "\"prop\"" ) );
long nodeId;
try ( Transaction tx = db.beginTx() )
{
Expand All @@ -295,14 +303,14 @@ public void shouldNotBeAbleToFindNodesAfterRemovingIndex() throws Exception
}

// Verify it's indexed exactly once
db.execute( "CALL db.fulltext.bloomAwaitPopulation" ).close();
db.execute( AWAIT_POPULATION ).close();
Result result = db.execute( String.format( NODES, "\"Jyllingevej\"" ) );
assertTrue( result.hasNext() );
assertEquals( nodeId, result.next().get( ENTITYID ) );
assertFalse( result.hasNext() );
db.execute( String.format( KEYS, "" ) );
db.execute( String.format( SET_NODE_KEYS, "" ) );

db.execute( "CALL db.fulltext.bloomAwaitPopulation" ).close();
db.execute( AWAIT_POPULATION ).close();
// Verify it's nowhere to be found now
result = db.execute( String.format( NODES, "\"Jyllingevej\"" ) );
assertFalse( result.hasNext() );
Expand All @@ -319,7 +327,7 @@ public void staleDataFromEntityDeleteShouldNotBeAccessibleAfterConfigurationChan
{

db = getDb();
db.execute( String.format( KEYS, "\"prop\"" ) );
db.execute( String.format( SET_NODE_KEYS, "\"prop\"" ) );
long nodeId;
try ( Transaction tx = db.beginTx() )
{
Expand All @@ -331,7 +339,7 @@ public void staleDataFromEntityDeleteShouldNotBeAccessibleAfterConfigurationChan

db.shutdown();
db = getDb();
db.execute( String.format( KEYS, "\"not-prop\"" ) );
db.execute( String.format( SET_NODE_KEYS, "\"not-prop\"" ) );

try ( Transaction tx = db.beginTx() )
{
Expand All @@ -342,10 +350,10 @@ public void staleDataFromEntityDeleteShouldNotBeAccessibleAfterConfigurationChan

db.shutdown();
db = getDb();
db.execute( String.format( KEYS, "\"prop\"" ) );
db.execute( String.format( SET_NODE_KEYS, "\"prop\"" ) );

// Verify that the node is no longer indexed
db.execute( "CALL db.fulltext.bloomAwaitPopulation" ).close();
db.execute( AWAIT_POPULATION ).close();
Result result = db.execute( String.format( NODES, "\"Esplanaden\"") );
assertFalse( result.hasNext() );
result.close();
Expand All @@ -356,7 +364,7 @@ public void staleDataFromPropertyRemovalShouldNotBeAccessibleAfterConfigurationC
{

db = getDb();
db.execute( String.format( KEYS, "\"prop\"" ) );
db.execute( String.format( SET_NODE_KEYS, "\"prop\"" ) );
long nodeId;
try ( Transaction tx = db.beginTx() )
{
Expand All @@ -368,7 +376,7 @@ public void staleDataFromPropertyRemovalShouldNotBeAccessibleAfterConfigurationC

db.shutdown();
db = getDb();
db.execute( String.format( KEYS, "\"not-prop\"" ) );
db.execute( String.format( SET_NODE_KEYS, "\"not-prop\"" ) );

try ( Transaction tx = db.beginTx() )
{
Expand All @@ -379,10 +387,10 @@ public void staleDataFromPropertyRemovalShouldNotBeAccessibleAfterConfigurationC

db.shutdown();
db = getDb();
db.execute( String.format( KEYS, "\"prop\"" ) );
db.execute( String.format( SET_NODE_KEYS, "\"prop\"" ) );

// Verify that the node is no longer indexed
db.execute( "CALL db.fulltext.bloomAwaitPopulation" ).close();
db.execute( AWAIT_POPULATION ).close();
Result result = db.execute( String.format( NODES, "\"Esplanaden\"") );
assertFalse( result.hasNext() );
result.close();
Expand All @@ -392,7 +400,7 @@ public void staleDataFromPropertyRemovalShouldNotBeAccessibleAfterConfigurationC
public void updatesAreAvailableToConcurrentReadTransactions() throws Exception
{
db = getDb();
db.execute( String.format( KEYS, "\"prop\"" ) );
db.execute( String.format( SET_NODE_KEYS, "\"prop\"" ) );

try ( Transaction tx = db.beginTx() )
{
Expand Down Expand Up @@ -430,14 +438,14 @@ public void shouldNotBeAbleToStartWithIllegalPropertyKey() throws Exception
{
expectedException.expectMessage( "It is not possible to index property keys starting with __lucene__fulltext__addon__" );
db = getDb();
db.execute( String.format( KEYS, "\"prop\", \"" + FulltextProvider.FIELD_ENTITY_ID + "\", \"hello\"" ) );
db.execute( String.format( SET_NODE_KEYS, "\"prop\", \"" + FulltextProvider.FIELD_ENTITY_ID + "\", \"hello\"" ) );
}

@Test
public void shouldBeAbleToRunConsistencyCheck() throws Exception
{
db = getDb();
db.execute( String.format( KEYS, "\"prop\"" ) );
db.execute( String.format( SET_NODE_KEYS, "\"prop\"" ) );

try ( Transaction tx = db.beginTx() )
{
Expand All @@ -464,7 +472,7 @@ public void shouldReindexNodesWhenAnalyzerIsChanged() throws Exception
builder.setConfig( BloomFulltextConfig.bloom_analyzer, ENGLISH );

db = getDb();
db.execute( String.format( KEYS, "\"prop\"" ) );
db.execute( String.format( SET_NODE_KEYS, "\"prop\"" ) );
try ( Transaction tx = db.beginTx() )
{
db.createNode().setProperty( "prop", "Hello and hello again." );
Expand All @@ -487,7 +495,7 @@ public void shouldReindexNodesWhenAnalyzerIsChanged() throws Exception
db.shutdown();
builder.setConfig( BloomFulltextConfig.bloom_analyzer, SWEDISH );
db = getDb();
db.execute( "CALL db.fulltext.bloomAwaitPopulation" ).close();
db.execute( AWAIT_POPULATION ).close();

try ( Transaction ignore = db.beginTx() )
{
Expand All @@ -508,7 +516,7 @@ public void shouldReindexAfterBeingTemporarilyDisabled() throws Exception

// Create a node while the index is enabled.
db = getDb();
db.execute( String.format( KEYS, "\"prop\"" ) );
db.execute( String.format( SET_NODE_KEYS, "\"prop\"" ) );
try ( Transaction tx = db.beginTx() )
{
db.createNode().setProperty( "prop", "Hello and hello again." );
Expand All @@ -531,7 +539,7 @@ public void shouldReindexAfterBeingTemporarilyDisabled() throws Exception
db.shutdown();
builder.setConfig( bloom_enabled, "true" );
db = getDb();
db.execute( "CALL db.fulltext.bloomAwaitPopulation" ).close();
db.execute( AWAIT_POPULATION ).close();

// Now we should be able to find the node that was added while the index was disabled.
try ( Transaction ignore = db.beginTx() )
Expand All @@ -552,7 +560,7 @@ public void indexedPropertiesShouldBeSetByProcedure() throws Exception
{
db = getDb();

db.execute( String.format( KEYS, "\"prop\"" ) );
db.execute( String.format( SET_NODE_KEYS, "\"prop\"" ) );
// Create a node while the index is enabled.
try ( Transaction tx = db.beginTx() )
{
Expand All @@ -574,9 +582,9 @@ public void shouldBeAbleToQueryForIndexedProperties() throws Exception
{
db = getDb();

db.execute( String.format( KEYS, "\"prop\", \"otherprop\", \"proppmatt\"" ) );
db.execute( String.format( SET_NODE_KEYS, "\"prop\", \"otherprop\", \"proppmatt\"" ) );

Result result = db.execute( "CALL db.fulltext.bloomFulltextGetPropertyKeys" );
Result result = db.execute( GET_NODE_KEYS );
assertEquals( "otherprop", result.next().get( "propertyKey" ) );
assertEquals( "prop", result.next().get( "propertyKey" ) );
assertEquals( "proppmatt", result.next().get( "propertyKey" ) );
Expand All @@ -587,10 +595,11 @@ public void shouldBeAbleToQueryForIndexedProperties() throws Exception
public void onlineIndexShouldBeReportedAsOnline() throws Exception
{
db = getDb();
db.execute( String.format( KEYS, "\"prop, otherprop, proppmatt\"" ) );
db.execute( String.format( SET_NODE_KEYS, "\"prop, otherprop, proppmatt\"" ) );
db.execute( String.format( SET_REL_KEYS, "\"prop, otherprop, proppmatt\"" ) );

db.execute( "CALL db.fulltext.bloomAwaitPopulation" );
Result result = db.execute( "CALL db.fulltext.bloomFulltextStatus" );
db.execute( AWAIT_POPULATION );
Result result = db.execute( STATUS );
assertEquals( "ONLINE", result.next().get( "state" ) );
assertEquals( "ONLINE", result.next().get( "state" ) );
assertFalse( result.hasNext() );
Expand All @@ -605,7 +614,7 @@ public void failureToStartUpMustNotPreventShutDown() throws Exception

// Create the store directory and all its files, and add a bit of data to it
GraphDatabaseService db = getDb();
db.execute( String.format( KEYS, "\"prop\"" ) );
db.execute( String.format( SET_NODE_KEYS, "\"prop\"" ) );

try ( Transaction tx = db.beginTx() )
{
Expand Down

0 comments on commit 397e673

Please sign in to comment.