Skip to content

Commit

Permalink
Make bloom extension loadable, disabled by default
Browse files Browse the repository at this point in the history
  • Loading branch information
@ragadeeshu
ragadeeshu committed Sep 20, 2017
1 parent 64367fd commit 8466351
Show file tree
Hide file tree
Showing 5 changed files with 85 additions and 50 deletions.
5 changes: 5 additions & 0 deletions enterprise/fulltext-addon/pom.xml
View file
Expand Up @@ -62,6 +62,11 @@
<version>${project.version}</version>
<scope>test</scope>
</dependency>
<dependency>
<groupId>org.neo4j</groupId>
<artifactId>neo4j-consistency-check</artifactId>
<version>${project.version}</version>
</dependency>
<dependency>
<groupId>org.neo4j</groupId>
<artifactId>neo4j-kernel</artifactId>
Expand Down
28 changes: 22 additions & 6 deletions ...main/java/org/neo4j/kernel/api/impl/fulltext/integrations/bloom/BloomKernelExtension.java
View file
Expand Up @@ -70,18 +70,31 @@ class BloomKernelExtension extends LifecycleAdapter
@Override
public void init() throws IOException, KernelException
{
List<String> properties = config.get( LoadableBloomFulltextConfig.bloom_indexed_properties );
Analyzer analyzer = getAnalyzer();
if ( config.get( LoadableBloomFulltextConfig.bloom_enabled ) )
{
List<String> properties = getProperties();
Analyzer analyzer = getAnalyzer();

Log log = logService.getInternalLog( FulltextProvider.class );
provider = new FulltextProvider( db, log, availabilityGuard, scheduler );
FulltextFactory fulltextFactory = new FulltextFactory( fileSystemAbstraction, storeDir, analyzer );
fulltextFactory.createFulltextIndex( BLOOM_NODES, FulltextProvider.FulltextIndexType.NODES, properties, provider );
fulltextFactory.createFulltextIndex( BLOOM_RELATIONSHIPS, FulltextProvider.FulltextIndexType.RELATIONSHIPS, properties, provider );

provider.init();
procedures.registerComponent( FulltextProvider.class, context -> provider, true );
procedures.registerProcedure( BloomProcedures.class );
provider.init();
procedures.registerComponent( FulltextProvider.class, context -> provider, true );
procedures.registerProcedure( BloomProcedures.class );
}
}

private List<String> getProperties()
{
List<String> properties = config.get( LoadableBloomFulltextConfig.bloom_indexed_properties );
if ( properties.isEmpty() )
{
throw new RuntimeException( "Properties to index must be configured for bloom fulltext" );
}
return properties;
}

private Analyzer getAnalyzer()
Expand All @@ -102,6 +115,9 @@ private Analyzer getAnalyzer()
@Override
public void shutdown() throws Exception
{
provider.close();
if ( provider != null )
{
provider.close();
}
}
}
4 changes: 2 additions & 2 deletions ...va/org/neo4j/kernel/api/impl/fulltext/integrations/bloom/BloomKernelExtensionFactory.java
View file
Expand Up @@ -41,7 +41,7 @@
public class BloomKernelExtensionFactory extends KernelExtensionFactory<BloomKernelExtensionFactory.Dependencies>
{

private static final String SERVICE_NAME = "bloom";
static final String SERVICE_NAME = "bloom";

public interface Dependencies
{
Expand All @@ -60,7 +60,7 @@ public interface Dependencies
JobScheduler scheduler();
}

BloomKernelExtensionFactory()
public BloomKernelExtensionFactory()
{
super( SERVICE_NAME );
}
Expand Down
12 changes: 8 additions & 4 deletions ...va/org/neo4j/kernel/api/impl/fulltext/integrations/bloom/LoadableBloomFulltextConfig.java
View file
Expand Up @@ -30,6 +30,8 @@
import org.neo4j.kernel.configuration.Settings;

import static org.neo4j.kernel.api.impl.fulltext.FulltextProvider.LUCENE_FULLTEXT_ADDON_INTERNAL_ID;
import static org.neo4j.kernel.configuration.Settings.BOOLEAN;
import static org.neo4j.kernel.configuration.Settings.FALSE;
import static org.neo4j.kernel.configuration.Settings.STRING;
import static org.neo4j.kernel.configuration.Settings.STRING_LIST;
import static org.neo4j.kernel.configuration.Settings.buildSetting;
Expand All @@ -46,13 +48,15 @@ public class LoadableBloomFulltextConfig implements LoadableConfig
public static final String UNSUPPORTED_PROPERY_KEY_REGEX = "^(?!" + LUCENE_FULLTEXT_ADDON_INTERNAL_ID + ").+$";
public static final BiFunction<List<String>,Function<String,String>,List<String>> ILLEGAL_VALUE_CONSTRAINT =
illegalValueMessage( "Must not contain '" + LUCENE_FULLTEXT_ADDON_INTERNAL_ID + "'", matchesAny( UNSUPPORTED_PROPERY_KEY_REGEX ) );
public static final BiFunction<List<String>,Function<String,String>,List<String>> MUST_NOT_BE_EMPTY_CONSTRAINT =
illegalValueMessage( "Must not be empty", Settings.nonEmptyList );

@Description( "Enable the fulltext addon for bloom." )
@Internal
static final Setting<Boolean> bloom_enabled = setting( "unsupported.dbms.bloom_enabled", BOOLEAN, FALSE );

@Description( "Property keys to index" )
@Internal
static final Setting<List<String>> bloom_indexed_properties =
buildSetting( "unsupported.dbms.bloom_indexed_properties", STRING_LIST, "" ).constraint( ILLEGAL_VALUE_CONSTRAINT ).constraint(
MUST_NOT_BE_EMPTY_CONSTRAINT ).build();
buildSetting( "unsupported.dbms.bloom_indexed_properties", STRING_LIST, "" ).constraint( ILLEGAL_VALUE_CONSTRAINT ).build();

@Description( "Define the analyzer to use for the bloom index. Expects the fully qualified classname of the analyzer to use" )
@Internal
Expand Down
86 changes: 48 additions & 38 deletions ...xt-addon/src/test/java/org/neo4j/kernel/api/impl/fulltext/integrations/bloom/BloomIT.java
View file
Expand Up @@ -25,21 +25,23 @@
import org.junit.Test;
import org.junit.rules.ExpectedException;

import java.util.Collections;
import java.util.HashMap;
import java.util.Map;
import java.util.Date;

import org.neo4j.consistency.ConsistencyCheckService;
import org.neo4j.consistency.checking.full.CheckConsistencyConfig;
import org.neo4j.graphdb.GraphDatabaseService;
import org.neo4j.graphdb.Node;
import org.neo4j.graphdb.Relationship;
import org.neo4j.graphdb.RelationshipType;
import org.neo4j.graphdb.Result;
import org.neo4j.graphdb.Transaction;
import org.neo4j.graphdb.config.InvalidSettingException;
import org.neo4j.graphdb.config.Setting;
import org.neo4j.graphdb.factory.GraphDatabaseBuilder;
import org.neo4j.helpers.collection.Iterators;
import org.neo4j.helpers.progress.ProgressMonitorFactory;
import org.neo4j.kernel.api.impl.fulltext.FulltextProvider;
import org.neo4j.kernel.configuration.Config;
import org.neo4j.logging.NullLogProvider;
import org.neo4j.test.TestGraphDatabaseFactory;
import org.neo4j.test.mockito.matcher.RootCauseMatcher;
import org.neo4j.test.rule.TestDirectory;
Expand All @@ -50,6 +52,7 @@
import static org.junit.Assert.assertFalse;
import static org.junit.Assert.assertThat;
import static org.junit.Assert.assertTrue;
import static org.neo4j.kernel.api.impl.fulltext.integrations.bloom.LoadableBloomFulltextConfig.bloom_enabled;

public class BloomIT
{
Expand All @@ -63,32 +66,23 @@ public class BloomIT
@Rule
public final ExpectedException expectedException = ExpectedException.none();

private TestGraphDatabaseFactory factory;
private GraphDatabaseService db;
private GraphDatabaseBuilder builder;

@Before
public void before() throws Exception
{
createTestGraphDatabaseFactory();
configureBloomExtension();
}

private void createTestGraphDatabaseFactory()
{
factory = new TestGraphDatabaseFactory();
TestGraphDatabaseFactory factory = new TestGraphDatabaseFactory();
factory.setFileSystem( fs.get() );
}

private void configureBloomExtension()
{
factory.addKernelExtensions( Collections.singletonList( new BloomKernelExtensionFactory() ) );
builder = factory.newEmbeddedDatabaseBuilder( testDirectory.graphDbDir() );
builder.setConfig( bloom_enabled, "true" );
}

@Test
public void shouldPopulateAndQueryIndexes() throws Exception
{
db = factory.newImpermanentDatabase( testDirectory.graphDbDir(),
Collections.singletonMap( LoadableBloomFulltextConfig.bloom_indexed_properties, "prop, relprop" ) );
builder.setConfig( LoadableBloomFulltextConfig.bloom_indexed_properties, "prop, relprop" );
db = builder.newGraphDatabase();
try ( Transaction transaction = db.beginTx() )
{
Node node1 = db.createNode();
Expand All @@ -112,10 +106,9 @@ public void shouldPopulateAndQueryIndexes() throws Exception
@Test
public void shouldBeAbleToConfigureAnalyzer() throws Exception
{
Map<Setting<?>,String> config = new HashMap<>();
config.put( LoadableBloomFulltextConfig.bloom_indexed_properties, "prop" );
config.put( LoadableBloomFulltextConfig.bloom_analyzer, "org.apache.lucene.analysis.sv.SwedishAnalyzer" );
db = factory.newImpermanentDatabase( testDirectory.graphDbDir(), config );
builder.setConfig( LoadableBloomFulltextConfig.bloom_indexed_properties, "prop" );
builder.setConfig( LoadableBloomFulltextConfig.bloom_analyzer, "org.apache.lucene.analysis.sv.SwedishAnalyzer" );
db = builder.newGraphDatabase();
try ( Transaction transaction = db.beginTx() )
{
Node node1 = db.createNode();
Expand All @@ -140,8 +133,9 @@ public void shouldBeAbleToConfigureAnalyzer() throws Exception
@Test
public void shouldPopulateIndexWithExistingDataOnIndexCreate() throws Exception
{
createTestGraphDatabaseFactory();
db = factory.newEmbeddedDatabase( testDirectory.graphDbDir() );
builder.setConfig( LoadableBloomFulltextConfig.bloom_indexed_properties, "something" );
db = builder.newGraphDatabase();

long nodeId;
try ( Transaction tx = db.beginTx() )
{
Expand All @@ -150,17 +144,17 @@ public void shouldPopulateIndexWithExistingDataOnIndexCreate() throws Exception
nodeId = node.getId();
tx.success();
}
Result result = db.execute( String.format( NODES, "Roskildevej" ) );
assertFalse( result.hasNext() );
db.shutdown();

configureBloomExtension();
GraphDatabaseBuilder builder = factory.newEmbeddedDatabaseBuilder( testDirectory.graphDbDir() );
builder.setConfig( LoadableBloomFulltextConfig.bloom_indexed_properties, "prop" );
builder.setConfig( LoadableBloomFulltextConfig.bloom_analyzer, "org.apache.lucene.analysis.da.DanishAnalyzer" );
db = builder.newGraphDatabase();

db.execute( "CALL db.fulltext.bloomAwaitPopulation" ).close();

Result result = db.execute( String.format( NODES, "Roskildevej" ) );
result = db.execute( String.format( NODES, "Roskildevej" ) );
assertTrue( result.hasNext() );
assertEquals( nodeId, result.next().get( ENTITYID ) );
assertFalse( result.hasNext() );
Expand All @@ -169,7 +163,6 @@ public void shouldPopulateIndexWithExistingDataOnIndexCreate() throws Exception
@Test
public void startupPopulationShouldNotCauseDuplicates() throws Exception
{
GraphDatabaseBuilder builder = factory.newEmbeddedDatabaseBuilder( testDirectory.graphDbDir() );
builder.setConfig( LoadableBloomFulltextConfig.bloom_indexed_properties, "prop" );

db = builder.newGraphDatabase();
Expand Down Expand Up @@ -203,7 +196,6 @@ public void startupPopulationShouldNotCauseDuplicates() throws Exception
@Test
public void staleDataFromEntityDeleteShouldNotBeAccessibleAfterConfigurationChange() throws Exception
{
GraphDatabaseBuilder builder = factory.newEmbeddedDatabaseBuilder( testDirectory.graphDbDir() );
builder.setConfig( LoadableBloomFulltextConfig.bloom_indexed_properties, "prop" );

db = builder.newGraphDatabase();
Expand Down Expand Up @@ -241,7 +233,6 @@ public void staleDataFromEntityDeleteShouldNotBeAccessibleAfterConfigurationChan
@Test
public void staleDataFromPropertyRemovalShouldNotBeAccessibleAfterConfigurationChange() throws Exception
{
GraphDatabaseBuilder builder = factory.newEmbeddedDatabaseBuilder( testDirectory.graphDbDir() );
builder.setConfig( LoadableBloomFulltextConfig.bloom_indexed_properties, "prop" );

db = builder.newGraphDatabase();
Expand Down Expand Up @@ -279,7 +270,6 @@ public void staleDataFromPropertyRemovalShouldNotBeAccessibleAfterConfigurationC
@Test
public void updatesAreAvailableToConcurrentReadTransactions() throws Exception
{
GraphDatabaseBuilder builder = factory.newEmbeddedDatabaseBuilder( testDirectory.graphDbDir() );
builder.setConfig( LoadableBloomFulltextConfig.bloom_indexed_properties, "prop" );
db = builder.newGraphDatabase();

Expand Down Expand Up @@ -317,18 +307,38 @@ public void updatesAreAvailableToConcurrentReadTransactions() throws Exception
@Test
public void shouldNotBeAbleToStartWithoutConfiguringProperties() throws Exception
{
Map<Setting<?>,String> config = new HashMap<>();
expectedException.expect( new RootCauseMatcher<>( InvalidSettingException.class, "Bad value" ) );
db = factory.newImpermanentDatabase( testDirectory.graphDbDir(), config );
expectedException.expect( new RootCauseMatcher<>( RuntimeException.class, "Properties to index must be configured for bloom fulltext" ) );
db = builder.newGraphDatabase();
}

@Test
public void shouldNotBeAbleToStartWithIllegalPropertyKey() throws Exception
{
Map<Setting<?>,String> config = new HashMap<>();
config.put( LoadableBloomFulltextConfig.bloom_indexed_properties, "prop, " + FulltextProvider.LUCENE_FULLTEXT_ADDON_INTERNAL_ID + ", hello" );
expectedException.expect( InvalidSettingException.class );
db = factory.newImpermanentDatabase( testDirectory.graphDbDir(), config );
builder.setConfig( LoadableBloomFulltextConfig.bloom_indexed_properties, "prop, " + FulltextProvider.LUCENE_FULLTEXT_ADDON_INTERNAL_ID + ", hello" );
db = builder.newGraphDatabase();
}

@Test
public void shouldBeAbleToRunConsistencyCheck() throws Exception
{
builder.setConfig( LoadableBloomFulltextConfig.bloom_indexed_properties, "prop" );
db = builder.newGraphDatabase();

try ( Transaction tx = db.beginTx() )
{
db.createNode().setProperty( "prop", "Langelinie Pavillinen" );
tx.success();
}
db.shutdown();

Config config = Config.defaults( LoadableBloomFulltextConfig.bloom_indexed_properties, "prop" );
ConsistencyCheckService consistencyCheckService = new ConsistencyCheckService( new Date() );
CheckConsistencyConfig checkConsistencyConfig = new CheckConsistencyConfig( true, true, true, true );
ConsistencyCheckService.Result result =
consistencyCheckService.runFullConsistencyCheck( testDirectory.graphDbDir(), config, ProgressMonitorFactory.NONE, NullLogProvider.getInstance(),
true, checkConsistencyConfig );
assertTrue( result.isSuccessful() );
}

@After
Expand Down

0 comments on commit 8466351

Please sign in to comment.