Skip to content

Commit

Permalink
Update LuceneBatchInserterIndexProviderNewImpl expectation of specifi…
Browse files Browse the repository at this point in the history
…c batch inserter

Introduce new interface IndexConfigStoreProvider that responsible for providing config store
and use it as an expected type in LuceneBatchInserterIndexProviderNewImpl instead of BatchInserterImpl.
Modifications here is quite limited since most of the involved classes are part of public API
  • Loading branch information
MishaDemianenko committed Nov 28, 2016
1 parent b1cf408 commit 803d31d
Show file tree
Hide file tree
Showing 7 changed files with 149 additions and 28 deletions.
Expand Up @@ -29,6 +29,7 @@
import org.neo4j.kernel.extension.KernelExtensionFactory; import org.neo4j.kernel.extension.KernelExtensionFactory;
import org.neo4j.unsafe.batchinsert.internal.BatchInserterImpl; import org.neo4j.unsafe.batchinsert.internal.BatchInserterImpl;
import org.neo4j.unsafe.batchinsert.internal.FileSystemClosingBatchInserter; import org.neo4j.unsafe.batchinsert.internal.FileSystemClosingBatchInserter;
import org.neo4j.unsafe.batchinsert.internal.IndexConfigStoreProvider;


import static org.neo4j.helpers.collection.MapUtil.stringMap; import static org.neo4j.helpers.collection.MapUtil.stringMap;


Expand All @@ -47,7 +48,8 @@ public final class BatchInserters
public static BatchInserter inserter( File storeDir ) throws IOException public static BatchInserter inserter( File storeDir ) throws IOException
{ {
DefaultFileSystemAbstraction fileSystem = createFileSystem(); DefaultFileSystemAbstraction fileSystem = createFileSystem();
return new FileSystemClosingBatchInserter( inserter( storeDir, fileSystem, stringMap() ), fileSystem ); BatchInserter batchInserter = inserter( storeDir, fileSystem, stringMap() );
return new FileSystemClosingBatchInserter( batchInserter, (IndexConfigStoreProvider) batchInserter, fileSystem );
} }


public static BatchInserter inserter( File storeDir, FileSystemAbstraction fs ) throws IOException public static BatchInserter inserter( File storeDir, FileSystemAbstraction fs ) throws IOException
Expand All @@ -58,8 +60,8 @@ public static BatchInserter inserter( File storeDir, FileSystemAbstraction fs )
public static BatchInserter inserter( File storeDir, Map<String,String> config ) throws IOException public static BatchInserter inserter( File storeDir, Map<String,String> config ) throws IOException
{ {
DefaultFileSystemAbstraction fileSystem = createFileSystem(); DefaultFileSystemAbstraction fileSystem = createFileSystem();
return new FileSystemClosingBatchInserter( inserter( storeDir, fileSystem, config, loadKernelExtension() ), BatchInserter inserter = inserter( storeDir, fileSystem, config, loadKernelExtension() );
fileSystem ); return new FileSystemClosingBatchInserter( inserter, (IndexConfigStoreProvider) inserter, fileSystem );
} }


public static BatchInserter inserter( File storeDir, FileSystemAbstraction fs, Map<String,String> config ) throws IOException public static BatchInserter inserter( File storeDir, FileSystemAbstraction fs, Map<String,String> config ) throws IOException
Expand All @@ -71,7 +73,8 @@ public static BatchInserter inserter( File storeDir,
Map<String, String> config, Iterable<KernelExtensionFactory<?>> kernelExtensions ) throws IOException Map<String, String> config, Iterable<KernelExtensionFactory<?>> kernelExtensions ) throws IOException
{ {
DefaultFileSystemAbstraction fileSystem = createFileSystem(); DefaultFileSystemAbstraction fileSystem = createFileSystem();
return new FileSystemClosingBatchInserter( new BatchInserterImpl( storeDir, fileSystem, config, kernelExtensions ), fileSystem ); BatchInserterImpl inserter = new BatchInserterImpl( storeDir, fileSystem, config, kernelExtensions );
return new FileSystemClosingBatchInserter( inserter, inserter, fileSystem );
} }


public static BatchInserter inserter( File storeDir, FileSystemAbstraction fileSystem, public static BatchInserter inserter( File storeDir, FileSystemAbstraction fileSystem,
Expand Down
Expand Up @@ -162,7 +162,7 @@
import static org.neo4j.kernel.impl.store.PropertyStore.encodeString; import static org.neo4j.kernel.impl.store.PropertyStore.encodeString;
import static org.neo4j.kernel.impl.util.IoPrimitiveUtils.safeCastLongToInt; import static org.neo4j.kernel.impl.util.IoPrimitiveUtils.safeCastLongToInt;


public class BatchInserterImpl implements BatchInserter public class BatchInserterImpl implements BatchInserter, IndexConfigStoreProvider
{ {
private final LifeSupport life; private final LifeSupport life;
private final NeoStores neoStores; private final NeoStores neoStores;
Expand Down Expand Up @@ -1069,8 +1069,8 @@ public String getStoreDir()
return storeDir.getPath(); return storeDir.getPath();
} }


// needed by lucene-index @Override
public IndexConfigStore getIndexStore() public IndexConfigStore getIndexConfigStore()
{ {
return this.indexStore; return this.indexStore;
} }
Expand Down
Expand Up @@ -28,17 +28,21 @@
import org.neo4j.graphdb.schema.ConstraintCreator; import org.neo4j.graphdb.schema.ConstraintCreator;
import org.neo4j.graphdb.schema.IndexCreator; import org.neo4j.graphdb.schema.IndexCreator;
import org.neo4j.io.fs.FileSystemAbstraction; import org.neo4j.io.fs.FileSystemAbstraction;
import org.neo4j.kernel.impl.index.IndexConfigStore;
import org.neo4j.unsafe.batchinsert.BatchInserter; import org.neo4j.unsafe.batchinsert.BatchInserter;
import org.neo4j.unsafe.batchinsert.BatchRelationship; import org.neo4j.unsafe.batchinsert.BatchRelationship;


public class FileSystemClosingBatchInserter implements BatchInserter public class FileSystemClosingBatchInserter implements BatchInserter, IndexConfigStoreProvider
{ {
private final BatchInserter delegate; private final BatchInserter delegate;
private final IndexConfigStoreProvider configStoreProvider;
private final FileSystemAbstraction fileSystem; private final FileSystemAbstraction fileSystem;


public FileSystemClosingBatchInserter( BatchInserter delegate, FileSystemAbstraction fileSystem ) public FileSystemClosingBatchInserter( BatchInserter delegate, IndexConfigStoreProvider configStoreProvider,
FileSystemAbstraction fileSystem )
{ {
this.delegate = delegate; this.delegate = delegate;
this.configStoreProvider = configStoreProvider;
this.fileSystem = fileSystem; this.fileSystem = fileSystem;
} }


Expand Down Expand Up @@ -203,4 +207,10 @@ private void closeFileSystem()
throw new UncheckedIOException( e ); throw new UncheckedIOException( e );
} }
} }

@Override
public IndexConfigStore getIndexConfigStore()
{
return configStoreProvider.getIndexConfigStore();
}
} }
@@ -0,0 +1,33 @@
/*
* Copyright (c) 2002-2016 "Neo Technology,"
* Network Engine for Objects in Lund AB [http://neotechnology.com]
*
* This file is part of Neo4j.
*
* Neo4j is free software: you can redistribute it and/or modify
* it under the terms of the GNU General Public License as published by
* the Free Software Foundation, either version 3 of the License, or
* (at your option) any later version.
*
* This program is distributed in the hope that it will be useful,
* but WITHOUT ANY WARRANTY; without even the implied warranty of
* MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
* GNU General Public License for more details.
*
* You should have received a copy of the GNU General Public License
* along with this program. If not, see <http://www.gnu.org/licenses/>.
*/
package org.neo4j.unsafe.batchinsert.internal;


import org.neo4j.kernel.impl.index.IndexConfigStore;
import org.neo4j.unsafe.batchinsert.BatchInserterIndexProvider;

/**
* Provider of index store for batch inserter index providers
* @see BatchInserterIndexProvider
*/
public interface IndexConfigStoreProvider
{
IndexConfigStore getIndexConfigStore();
}
Expand Up @@ -35,9 +35,10 @@ public class FileSystemClosingBatchInserterTest
public void closeFileSystemOnShutdown() throws Exception public void closeFileSystemOnShutdown() throws Exception
{ {
BatchInserter batchInserter = mock( BatchInserter.class ); BatchInserter batchInserter = mock( BatchInserter.class );
IndexConfigStoreProvider configStoreProvider = mock( IndexConfigStoreProvider.class );
FileSystemAbstraction fileSystem = mock( FileSystemAbstraction.class ); FileSystemAbstraction fileSystem = mock( FileSystemAbstraction.class );
FileSystemClosingBatchInserter inserter = FileSystemClosingBatchInserter inserter =
new FileSystemClosingBatchInserter( batchInserter, fileSystem ); new FileSystemClosingBatchInserter( batchInserter, configStoreProvider, fileSystem );


inserter.shutdown(); inserter.shutdown();


Expand Down
Expand Up @@ -32,10 +32,10 @@
import org.neo4j.kernel.impl.index.IndexConfigStore; import org.neo4j.kernel.impl.index.IndexConfigStore;
import org.neo4j.kernel.impl.index.IndexEntityType; import org.neo4j.kernel.impl.index.IndexEntityType;
import org.neo4j.unsafe.batchinsert.BatchInserter; import org.neo4j.unsafe.batchinsert.BatchInserter;
import org.neo4j.unsafe.batchinsert.internal.BatchInserterImpl;
import org.neo4j.unsafe.batchinsert.BatchInserterIndex; import org.neo4j.unsafe.batchinsert.BatchInserterIndex;
import org.neo4j.unsafe.batchinsert.BatchInserterIndexProvider; import org.neo4j.unsafe.batchinsert.BatchInserterIndexProvider;
import org.neo4j.unsafe.batchinsert.BatchRelationship; import org.neo4j.unsafe.batchinsert.BatchRelationship;
import org.neo4j.unsafe.batchinsert.internal.IndexConfigStoreProvider;


/** /**
* The {@link BatchInserter} version of {@link LuceneIndexImplementation}. Indexes * The {@link BatchInserter} version of {@link LuceneIndexImplementation}. Indexes
Expand All @@ -52,16 +52,12 @@ public class LuceneBatchInserterIndexProviderNewImpl implements BatchInserterInd
public LuceneBatchInserterIndexProviderNewImpl( final BatchInserter inserter ) public LuceneBatchInserterIndexProviderNewImpl( final BatchInserter inserter )
{ {
this.inserter = inserter; this.inserter = inserter;
this.indexStore = ((BatchInserterImpl) inserter).getIndexStore(); this.indexStore = ((IndexConfigStoreProvider) inserter).getIndexConfigStore();
this.relationshipLookup = new LuceneBatchInserterIndex.RelationshipLookup() this.relationshipLookup = id ->
{ {
@Override // TODO too may objects allocated here
public EntityId lookup( long id ) BatchRelationship rel = inserter.getRelationshipById( id );
{ return new EntityId.RelationshipData( id, rel.getStartNode(), rel.getEndNode() );
// TODO too may objects allocated here
BatchRelationship rel = inserter.getRelationshipById( id );
return new EntityId.RelationshipData( id, rel.getStartNode(), rel.getEndNode() );
}
}; };
} }


Expand Down Expand Up @@ -100,14 +96,14 @@ private BatchInserterIndex index( IndexIdentifier identifier, Map<String, String
{ {
// We don't care about threads here... c'mon... it's a // We don't care about threads here... c'mon... it's a
// single-threaded batch inserter // single-threaded batch inserter
LuceneBatchInserterIndex index = indexes.get( identifier ); return indexes.computeIfAbsent( identifier,
if ( index == null ) k -> getLuceneBatchInserterIndex( identifier, config ) );
{ }
index = new LuceneBatchInserterIndex( new File(inserter.getStoreDir()),
identifier, config, relationshipLookup ); private LuceneBatchInserterIndex getLuceneBatchInserterIndex( IndexIdentifier identifier,
indexes.put( identifier, index ); Map<String,String> config )
} {
return index; return new LuceneBatchInserterIndex( new File( inserter.getStoreDir() ), identifier, config, relationshipLookup );
} }


@Override @Override
Expand Down
@@ -0,0 +1,78 @@
/*
* Copyright (c) 2002-2016 "Neo Technology,"
* Network Engine for Objects in Lund AB [http://neotechnology.com]
*
* This file is part of Neo4j.
*
* Neo4j is free software: you can redistribute it and/or modify
* it under the terms of the GNU General Public License as published by
* the Free Software Foundation, either version 3 of the License, or
* (at your option) any later version.
*
* This program is distributed in the hope that it will be useful,
* but WITHOUT ANY WARRANTY; without even the implied warranty of
* MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
* GNU General Public License for more details.
*
* You should have received a copy of the GNU General Public License
* along with this program. If not, see <http://www.gnu.org/licenses/>.
*/
package org.neo4j.index.impl.lucene.legacy;

import org.junit.Rule;
import org.junit.Test;

import java.io.File;
import java.util.Map;

import org.neo4j.helpers.collection.Iterables;
import org.neo4j.helpers.collection.MapUtil;
import org.neo4j.kernel.extension.KernelExtensionFactory;
import org.neo4j.kernel.impl.api.index.inmemory.InMemoryIndexProviderFactory;
import org.neo4j.kernel.impl.api.scan.InMemoryLabelScanStoreExtension;
import org.neo4j.test.rule.TestDirectory;
import org.neo4j.test.rule.fs.DefaultFileSystemRule;
import org.neo4j.unsafe.batchinsert.BatchInserter;
import org.neo4j.unsafe.batchinsert.BatchInserters;

public class LuceneBatchInserterIndexProviderNewImplTest
{
@Rule
public TestDirectory testDirectory = TestDirectory.testDirectory();
@Rule
public DefaultFileSystemRule fileSystemRule = new DefaultFileSystemRule();

@Test
public void createBatchIndexFromAnyIndexStoreProvider() throws Exception
{
createEndCloseIndexProvider( BatchInserters.inserter( getStoreDir() ) );
createEndCloseIndexProvider( BatchInserters.inserter( getStoreDir(), fileSystemRule.get() ) );
createEndCloseIndexProvider( BatchInserters.inserter( getStoreDir(), getConfig() ) );
createEndCloseIndexProvider( BatchInserters.inserter( getStoreDir(), getConfig(), getExtensions() ) );
createEndCloseIndexProvider( BatchInserters.inserter( getStoreDir(), fileSystemRule.get(), getConfig() ) );
createEndCloseIndexProvider( BatchInserters.inserter( getStoreDir(), fileSystemRule.get(), getConfig(),
getExtensions() ) );
}

private void createEndCloseIndexProvider( BatchInserter inserter )
{
LuceneBatchInserterIndexProviderNewImpl provider = new LuceneBatchInserterIndexProviderNewImpl( inserter );
provider.shutdown();
inserter.shutdown();
}

private Iterable<KernelExtensionFactory<?>> getExtensions()
{
return Iterables.asIterable( new InMemoryIndexProviderFactory(), new InMemoryLabelScanStoreExtension() );
}

private Map<String,String> getConfig()
{
return MapUtil.stringMap();
}

private File getStoreDir()
{
return testDirectory.graphDbDir();
}
}

0 comments on commit 803d31d

Please sign in to comment.