Skip to content

Commit

Permalink
Introduce builders for all lucene indexes
Browse files Browse the repository at this point in the history
Add additional builder for label scan store index,
separate builder for index storeage creation,
update code and tests to use new builder
  • Loading branch information
MishaDemianenko committed Jan 21, 2016
1 parent 1ad1c64 commit ca4b790
Show file tree
Hide file tree
Showing 22 changed files with 367 additions and 99 deletions.
19 changes: 19 additions & 0 deletions community/io/src/test/java/org/neo4j/io/IOUtilsTest.java
@@ -1,3 +1,22 @@
/*
* Copyright (c) 2002-2016 "Neo Technology,"
* Network Engine for Objects in Lund AB [http://neotechnology.com]
*
* This file is part of Neo4j.
*
* Neo4j is free software: you can redistribute it and/or modify
* it under the terms of the GNU General Public License as published by
* the Free Software Foundation, either version 3 of the License, or
* (at your option) any later version.
*
* This program is distributed in the hope that it will be useful,
* but WITHOUT ANY WARRANTY; without even the implied warranty of
* MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
* GNU General Public License for more details.
*
* You should have received a copy of the GNU General Public License
* along with this program. If not, see <http://www.gnu.org/licenses/>.
*/
package org.neo4j.io;

import org.junit.Rule;
Expand Down
@@ -1,3 +1,22 @@
/*
* Copyright (c) 2002-2016 "Neo Technology,"
* Network Engine for Objects in Lund AB [http://neotechnology.com]
*
* This file is part of Neo4j.
*
* Neo4j is free software: you can redistribute it and/or modify
* it under the terms of the GNU General Public License as published by
* the Free Software Foundation, either version 3 of the License, or
* (at your option) any later version.
*
* This program is distributed in the hope that it will be useful,
* but WITHOUT ANY WARRANTY; without even the implied warranty of
* MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
* GNU General Public License for more details.
*
* You should have received a copy of the GNU General Public License
* along with this program. If not, see <http://www.gnu.org/licenses/>.
*/
package org.neo4j.test;

import org.hamcrest.Description;
Expand Down
Expand Up @@ -25,13 +25,16 @@
import org.neo4j.io.fs.FileSystemAbstraction;
import org.neo4j.kernel.api.impl.index.LuceneLabelScanIndex;
import org.neo4j.kernel.api.impl.index.LuceneLabelScanStore;
import org.neo4j.kernel.api.impl.index.builder.LuceneLabelScanIndexBuilder;
import org.neo4j.kernel.api.impl.index.storage.DirectoryFactory;
import org.neo4j.kernel.api.impl.index.storage.PartitionedIndexStorage;
import org.neo4j.kernel.api.labelscan.LabelScanStore;
import org.neo4j.kernel.impl.api.scan.LabelScanStoreProvider;
import org.neo4j.kernel.impl.api.scan.LabelScanStoreProvider.FullStoreChangeStream;
import org.neo4j.logging.LogProvider;

import static org.neo4j.kernel.impl.api.scan.LabelScanStoreProvider.fullStoreLabelUpdateStream;

/**
* Means of obtaining a {@link LabelScanStore}, independent of the {@link org.neo4j.kernel.extension.KernelExtensions}
* mechanism, when you need to access the store without running a full database. This is used during consistency
Expand Down Expand Up @@ -62,12 +65,12 @@ public LabelScanStore build()
if ( null == labelScanStore )
{
// TODO: Replace with kernel extension based lookup
PartitionedIndexStorage indexStorage = new PartitionedIndexStorage( DirectoryFactory.PERSISTENT, fileSystem,
LabelScanStoreProvider.getStoreDirectory( storeDir ),
LuceneLabelScanStore.INDEX_IDENTIFIER );
LuceneLabelScanIndex index = new LuceneLabelScanIndex( indexStorage );
labelScanStore = new LuceneLabelScanStore( index, fullStoreStream, logProvider,
LuceneLabelScanStore.Monitor.EMPTY );
LuceneLabelScanIndex index = LuceneLabelScanIndexBuilder.create()
.withFileSystem( fileSystem )
.withIndexRootFolder( LabelScanStoreProvider.getStoreDirectory( storeDir ) )
.build();
labelScanStore = new LuceneLabelScanStore( index, fullStoreStream,
logProvider, LuceneLabelScanStore.Monitor.EMPTY );

try
{
Expand Down
Expand Up @@ -37,11 +37,6 @@ public class LuceneLabelScanIndex extends AbstractLuceneIndex
private final BitmapDocumentFormat format;
private final LabelScanStorageStrategy storageStrategy;

public LuceneLabelScanIndex( PartitionedIndexStorage indexStorage )
{
this( BitmapDocumentFormat._32, indexStorage );
}

public LuceneLabelScanIndex( BitmapDocumentFormat format, PartitionedIndexStorage indexStorage )
{
super( indexStorage );
Expand Down
Expand Up @@ -38,8 +38,6 @@

public class LuceneLabelScanStore implements LabelScanStore
{
public static final String INDEX_IDENTIFIER = "labelStore";

private final LuceneLabelScanIndex luceneIndex;
// We get in a full store stream here in case we need to fully rebuild the store if it's missing or corrupted.
private final FullStoreChangeStream fullStoreStream;
Expand Down
Expand Up @@ -24,6 +24,7 @@
import org.neo4j.helpers.Service;
import org.neo4j.kernel.NeoStoreDataSource;
import org.neo4j.kernel.api.impl.index.LuceneLabelScanStore.Monitor;
import org.neo4j.kernel.api.impl.index.builder.LuceneLabelScanIndexBuilder;
import org.neo4j.kernel.api.impl.index.storage.DirectoryFactory;
import org.neo4j.kernel.api.impl.index.storage.PartitionedIndexStorage;
import org.neo4j.kernel.configuration.Config;
Expand Down Expand Up @@ -88,10 +89,11 @@ public LabelScanStoreProvider newInstance( KernelContext context, Dependencies d

private LuceneLabelScanIndex getLuceneIndex( KernelContext context, DirectoryFactory directoryFactory )
{
PartitionedIndexStorage indexStorage = new PartitionedIndexStorage( directoryFactory, context.fileSystem(),
LabelScanStoreProvider.getStoreDirectory( context.storeDir() ),
LuceneLabelScanStore.INDEX_IDENTIFIER );
return new LuceneLabelScanIndex( indexStorage );
return LuceneLabelScanIndexBuilder.create()
.withDirectoryFactory( directoryFactory )
.withFileSystem( context.fileSystem() )
.withIndexRootFolder( LabelScanStoreProvider.getStoreDirectory( context.storeDir() ) )
.build();
}

}
Expand Up @@ -30,6 +30,7 @@

import org.neo4j.io.fs.FileSystemAbstraction;
import org.neo4j.io.pagecache.PageCache;
import org.neo4j.kernel.api.impl.index.builder.LuceneSchemaIndexBuilder;
import org.neo4j.kernel.api.impl.index.populator.DeferredConstraintVerificationUniqueLuceneIndexPopulator;
import org.neo4j.kernel.api.impl.index.populator.NonUniqueLuceneIndexPopulator;
import org.neo4j.kernel.api.impl.index.storage.DirectoryFactory;
Expand All @@ -41,7 +42,6 @@
import org.neo4j.kernel.api.index.IndexPopulator;
import org.neo4j.kernel.api.index.InternalIndexState;
import org.neo4j.kernel.api.index.SchemaIndexProvider;
import org.neo4j.kernel.configuration.Config;
import org.neo4j.kernel.impl.api.index.sampling.IndexSamplingConfig;
import org.neo4j.kernel.impl.api.scan.LabelScanStoreProvider;
import org.neo4j.kernel.impl.storemigration.StoreMigrationParticipant;
Expand Down Expand Up @@ -73,8 +73,11 @@ public LuceneSchemaIndexProvider( FileSystemAbstraction fileSystem, DirectoryFac
public IndexPopulator getPopulator( long indexId, IndexDescriptor descriptor,
IndexConfiguration config, IndexSamplingConfig samplingConfig )
{
PartitionedIndexStorage indexStorage = getIndexStorage( indexId );
LuceneSchemaIndex luceneIndex = new LuceneSchemaIndex( indexStorage, config, samplingConfig );
LuceneSchemaIndex luceneIndex = LuceneSchemaIndexBuilder.create()
.withIndexConfig( config )
.withSamplingConfig( samplingConfig )
.withIndexStorage( getIndexStorage( indexId ) )
.build();
if ( config.isUnique() )
{
return new DeferredConstraintVerificationUniqueLuceneIndexPopulator( luceneIndex, descriptor );
Expand All @@ -89,8 +92,11 @@ public IndexPopulator getPopulator( long indexId, IndexDescriptor descriptor,
public IndexAccessor getOnlineAccessor( long indexId, IndexConfiguration config,
IndexSamplingConfig samplingConfig ) throws IOException
{
PartitionedIndexStorage indexStorage = getIndexStorage( indexId );
LuceneSchemaIndex luceneIndex = new LuceneSchemaIndex( indexStorage, config, samplingConfig );
LuceneSchemaIndex luceneIndex = LuceneSchemaIndexBuilder.create()
.withIndexConfig( config )
.withSamplingConfig( samplingConfig )
.withIndexStorage( getIndexStorage( indexId ) )
.build();
luceneIndex.open();
return new LuceneIndexAccessor( luceneIndex );
}
Expand Down Expand Up @@ -164,8 +170,7 @@ private PartitionedIndexStorage getIndexStorage( long indexId )

private boolean indexIsOnline( PartitionedIndexStorage indexStorage ) throws IOException
{
IndexSamplingConfig samplingConfig = new IndexSamplingConfig( new Config() );
try ( LuceneSchemaIndex index = new LuceneSchemaIndex( indexStorage, IndexConfiguration.NON_UNIQUE, samplingConfig ) )
try ( LuceneSchemaIndex index = LuceneSchemaIndexBuilder.create().withIndexStorage( indexStorage ).build() )
{
if ( index.exists() )
{
Expand Down
@@ -0,0 +1,62 @@
/*
* Copyright (c) 2002-2016 "Neo Technology,"
* Network Engine for Objects in Lund AB [http://neotechnology.com]
*
* This file is part of Neo4j.
*
* Neo4j is free software: you can redistribute it and/or modify
* it under the terms of the GNU General Public License as published by
* the Free Software Foundation, either version 3 of the License, or
* (at your option) any later version.
*
* This program is distributed in the hope that it will be useful,
* but WITHOUT ANY WARRANTY; without even the implied warranty of
* MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
* GNU General Public License for more details.
*
* You should have received a copy of the GNU General Public License
* along with this program. If not, see <http://www.gnu.org/licenses/>.
*/
package org.neo4j.kernel.api.impl.index.builder;

import java.io.File;

import org.neo4j.io.fs.FileSystemAbstraction;
import org.neo4j.kernel.api.impl.index.storage.DirectoryFactory;
import org.neo4j.kernel.api.impl.index.storage.PartitionedIndexStorage;

abstract class AbstractLuceneIndexBuilder<T extends AbstractLuceneIndexBuilder>
{
protected LuceneIndexStorageBuilder storageBuilder = LuceneIndexStorageBuilder.create();

public T withIndexStorage( PartitionedIndexStorage indexStorage )
{
storageBuilder.withIndexStorage( indexStorage );
return (T) this;
}

public T withDirectoryFactory( DirectoryFactory directoryFactory )
{
storageBuilder.withDirectoryFactory( directoryFactory );
return (T) this;
}

public T withFileSystem( FileSystemAbstraction fileSystem )
{
storageBuilder.withFileSystem( fileSystem );
return (T) this;
}

public T withIndexRootFolder( File indexRootFolder )
{
storageBuilder.withIndexRootFolder( indexRootFolder );
return (T) this;
}

public T withIndexIdentifier( String indexIdentifier )
{
storageBuilder.withIndexIdentifier( indexIdentifier );
return (T) this;
}

}
@@ -0,0 +1,90 @@
/*
* Copyright (c) 2002-2016 "Neo Technology,"
* Network Engine for Objects in Lund AB [http://neotechnology.com]
*
* This file is part of Neo4j.
*
* Neo4j is free software: you can redistribute it and/or modify
* it under the terms of the GNU General Public License as published by
* the Free Software Foundation, either version 3 of the License, or
* (at your option) any later version.
*
* This program is distributed in the hope that it will be useful,
* but WITHOUT ANY WARRANTY; without even the implied warranty of
* MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
* GNU General Public License for more details.
*
* You should have received a copy of the GNU General Public License
* along with this program. If not, see <http://www.gnu.org/licenses/>.
*/
package org.neo4j.kernel.api.impl.index.builder;

import java.io.File;
import java.util.Objects;

import org.neo4j.io.fs.DefaultFileSystemAbstraction;
import org.neo4j.io.fs.FileSystemAbstraction;
import org.neo4j.kernel.api.impl.index.storage.DirectoryFactory;
import org.neo4j.kernel.api.impl.index.storage.PartitionedIndexStorage;

class LuceneIndexStorageBuilder
{
private DirectoryFactory directoryFactory = DirectoryFactory.PERSISTENT;
private FileSystemAbstraction fileSystem = new DefaultFileSystemAbstraction();
private File indexRootFolder;
private String indexIdentifier;
private PartitionedIndexStorage indexStorage;

private LuceneIndexStorageBuilder()
{
}

static LuceneIndexStorageBuilder create()
{
return new LuceneIndexStorageBuilder();
}

PartitionedIndexStorage buildIndexStorage()
{
if ( indexStorage == null )
{
Objects.requireNonNull( directoryFactory );
Objects.requireNonNull( fileSystem );
Objects.requireNonNull( indexRootFolder );
Objects.requireNonNull( indexIdentifier );
indexStorage =
new PartitionedIndexStorage( directoryFactory, fileSystem, indexRootFolder, indexIdentifier );
}
return indexStorage;
}

LuceneIndexStorageBuilder withIndexIdentifier( String indexIdentifier )
{
this.indexIdentifier = indexIdentifier;
return this;
}

LuceneIndexStorageBuilder withDirectoryFactory( DirectoryFactory directoryFactory )
{
this.directoryFactory = directoryFactory;
return this;
}

LuceneIndexStorageBuilder withFileSystem( FileSystemAbstraction fileSystem )
{
this.fileSystem = fileSystem;
return this;
}

LuceneIndexStorageBuilder withIndexRootFolder( File indexRootFolder )
{
this.indexRootFolder = indexRootFolder;
return this;
}

LuceneIndexStorageBuilder withIndexStorage( PartitionedIndexStorage indexStorage )
{
this.indexStorage = indexStorage;
return this;
}
}
@@ -0,0 +1,52 @@
/*
* Copyright (c) 2002-2016 "Neo Technology,"
* Network Engine for Objects in Lund AB [http://neotechnology.com]
*
* This file is part of Neo4j.
*
* Neo4j is free software: you can redistribute it and/or modify
* it under the terms of the GNU General Public License as published by
* the Free Software Foundation, either version 3 of the License, or
* (at your option) any later version.
*
* This program is distributed in the hope that it will be useful,
* but WITHOUT ANY WARRANTY; without even the implied warranty of
* MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
* GNU General Public License for more details.
*
* You should have received a copy of the GNU General Public License
* along with this program. If not, see <http://www.gnu.org/licenses/>.
*/
package org.neo4j.kernel.api.impl.index.builder;

import org.neo4j.kernel.api.impl.index.BitmapDocumentFormat;
import org.neo4j.kernel.api.impl.index.LuceneLabelScanIndex;

public class LuceneLabelScanIndexBuilder extends AbstractLuceneIndexBuilder<LuceneLabelScanIndexBuilder>
{
public static final String DEFAULT_INDEX_IDENTIFIER = "labelStore";

private BitmapDocumentFormat format = BitmapDocumentFormat._32;

private LuceneLabelScanIndexBuilder()
{
super();
storageBuilder.withIndexIdentifier( DEFAULT_INDEX_IDENTIFIER );
}

public static LuceneLabelScanIndexBuilder create()
{
return new LuceneLabelScanIndexBuilder();
}

public LuceneLabelScanIndexBuilder withDocumentFormat( BitmapDocumentFormat format )
{
this.format = format;
return this;
}

public LuceneLabelScanIndex build()
{
return new LuceneLabelScanIndex( format, storageBuilder.buildIndexStorage() );
}
}

0 comments on commit ca4b790

Please sign in to comment.