diff --git a/community/io/src/main/java/org/neo4j/io/IOUtils.java b/community/io/src/main/java/org/neo4j/io/IOUtils.java index 6ba62361820ce..7d4d8c8a17f53 100644 --- a/community/io/src/main/java/org/neo4j/io/IOUtils.java +++ b/community/io/src/main/java/org/neo4j/io/IOUtils.java @@ -78,19 +78,22 @@ public static void closeAll( T... closeables ) throws Exception closeException = null; for ( T closeable : closeables ) { - try + if ( closeable != null ) { - closeable.close(); - } - catch ( Exception e ) - { - if ( closeException == null ) + try { - closeException = e; + closeable.close(); } - else + catch ( Exception e ) { - closeException.addSuppressed( e ); + if ( closeException == null ) + { + closeException = e; + } + else + { + closeException.addSuppressed( e ); + } } } } @@ -99,4 +102,22 @@ public static void closeAll( T... closeables ) throws throw new IOException( "Exception closing multiple resources", closeException ); } } + + /** + * Closes given array of {@link AutoCloseable closeables} ignoring all exceptions. + * + * @param closeables the closeables to close + * @param the type of closeable + */ + @SafeVarargs + public static void closeAllSilently( T... closeables ) + { + try + { + closeAll( closeables ); + } + catch ( IOException ignored ) + { + } + } } diff --git a/community/kernel/src/main/java/org/neo4j/kernel/api/index/IndexAccessor.java b/community/kernel/src/main/java/org/neo4j/kernel/api/index/IndexAccessor.java index d933b343c246a..e64fcc1726f5d 100644 --- a/community/kernel/src/main/java/org/neo4j/kernel/api/index/IndexAccessor.java +++ b/community/kernel/src/main/java/org/neo4j/kernel/api/index/IndexAccessor.java @@ -135,11 +135,13 @@ public long maxCount() return 0; } - @Override public void close() throws IOException + @Override + public void close() throws IOException { } - @Override public Iterator iterator() + @Override + public Iterator iterator() { return emptyIterator(); } diff --git a/community/kernel/src/main/java/org/neo4j/storageengine/api/schema/IndexReader.java b/community/kernel/src/main/java/org/neo4j/storageengine/api/schema/IndexReader.java index 4f26d89fce5a6..4a274a8f90a6e 100644 --- a/community/kernel/src/main/java/org/neo4j/storageengine/api/schema/IndexReader.java +++ b/community/kernel/src/main/java/org/neo4j/storageengine/api/schema/IndexReader.java @@ -139,7 +139,7 @@ public int countIndexedNodes( long nodeId, Object propertyValue ) @Override public IndexSampler createSampler() { - return null; + return IndexSampler.EMPTY; } @Override diff --git a/community/kernel/src/main/java/org/neo4j/storageengine/api/schema/IndexSampler.java b/community/kernel/src/main/java/org/neo4j/storageengine/api/schema/IndexSampler.java index be72a7543f29b..800a41c479462 100644 --- a/community/kernel/src/main/java/org/neo4j/storageengine/api/schema/IndexSampler.java +++ b/community/kernel/src/main/java/org/neo4j/storageengine/api/schema/IndexSampler.java @@ -24,6 +24,11 @@ public interface IndexSampler { + IndexSampler EMPTY = result -> { + result.write( 0, 0 ); + return 0; + }; + /** * Sample this index (on the current thread) * diff --git a/community/kernel/src/main/java/org/neo4j/storageengine/api/schema/LabelScanReader.java b/community/kernel/src/main/java/org/neo4j/storageengine/api/schema/LabelScanReader.java index 9f081f6d9ffcd..33ab64a00994d 100644 --- a/community/kernel/src/main/java/org/neo4j/storageengine/api/schema/LabelScanReader.java +++ b/community/kernel/src/main/java/org/neo4j/storageengine/api/schema/LabelScanReader.java @@ -43,4 +43,8 @@ public interface LabelScanReader extends Resource Iterator labelsForNode( long nodeId ); AllEntriesLabelScanReader allNodeLabelRanges(); + + Iterator getAllDocsIterator(); // todo: should not be here... + + long getMaxDoc(); // todo: should not be here... } diff --git a/community/kernel/src/test/java/org/neo4j/kernel/impl/api/index/IndexRestartIT.java b/community/kernel/src/test/java/org/neo4j/kernel/impl/api/index/IndexRestartIT.java index 95de77a443cd6..66b75666f8263 100644 --- a/community/kernel/src/test/java/org/neo4j/kernel/impl/api/index/IndexRestartIT.java +++ b/community/kernel/src/test/java/org/neo4j/kernel/impl/api/index/IndexRestartIT.java @@ -54,14 +54,14 @@ public class IndexRestartIT { + @Rule + public final EphemeralFileSystemRule fs = new EphemeralFileSystemRule(); private GraphDatabaseService db; - @Rule public EphemeralFileSystemRule fs = new EphemeralFileSystemRule(); private TestGraphDatabaseFactory factory; private final ControlledPopulationSchemaIndexProvider provider = new ControlledPopulationSchemaIndexProvider(); private final Label myLabel = label( "MyLabel" ); - @Before public void before() throws Exception { diff --git a/community/kernel/src/test/java/org/neo4j/kernel/impl/api/scan/InMemoryLabelScanStore.java b/community/kernel/src/test/java/org/neo4j/kernel/impl/api/scan/InMemoryLabelScanStore.java index e318cdc0b536f..04fd9364f0c53 100644 --- a/community/kernel/src/test/java/org/neo4j/kernel/impl/api/scan/InMemoryLabelScanStore.java +++ b/community/kernel/src/test/java/org/neo4j/kernel/impl/api/scan/InMemoryLabelScanStore.java @@ -19,6 +19,8 @@ */ package org.neo4j.kernel.impl.api.scan; +import org.apache.lucene.document.Document; + import java.io.File; import java.io.IOException; import java.util.ArrayList; @@ -102,6 +104,12 @@ public void close() { // Nothing to close } + @Override + public long getMaxDoc() + { + return 0; + } + @Override public Iterator labelsForNode( long nodeId ) { @@ -121,6 +129,12 @@ public AllEntriesLabelScanReader allNodeLabelRanges() { return newAllEntriesReader(); } + + @Override + public Iterator getAllDocsIterator() + { + return null; + } }; } diff --git a/community/lucene-index/src/main/java/org/neo4j/index/lucene/LuceneLabelScanStoreBuilder.java b/community/lucene-index/src/main/java/org/neo4j/index/lucene/LuceneLabelScanStoreBuilder.java index 6b708292c6ce9..ad33c6f0177dd 100644 --- a/community/lucene-index/src/main/java/org/neo4j/index/lucene/LuceneLabelScanStoreBuilder.java +++ b/community/lucene-index/src/main/java/org/neo4j/index/lucene/LuceneLabelScanStoreBuilder.java @@ -23,10 +23,10 @@ import java.io.IOException; import org.neo4j.io.fs.FileSystemAbstraction; -import org.neo4j.kernel.api.impl.index.LuceneIndex; -import org.neo4j.kernel.api.impl.index.LuceneIndexBuilder; +import org.neo4j.kernel.api.impl.index.LuceneLabelScanIndex; import org.neo4j.kernel.api.impl.index.LuceneLabelScanStore; -import org.neo4j.kernel.api.impl.index.NodeRangeDocumentLabelScanStorageStrategy; +import org.neo4j.kernel.api.impl.index.storage.DirectoryFactory; +import org.neo4j.kernel.api.impl.index.storage.PartitionedIndexStorage; import org.neo4j.kernel.api.labelscan.LabelScanStore; import org.neo4j.kernel.impl.api.scan.LabelScanStoreProvider; import org.neo4j.kernel.impl.api.scan.LabelScanStoreProvider.FullStoreChangeStream; @@ -64,14 +64,12 @@ public LabelScanStore build() if ( null == labelScanStore ) { // TODO: Replace with kernel extension based lookup - LuceneIndex index = LuceneIndexBuilder.create() - .withIndexRootFolder( LabelScanStoreProvider.getStoreDirectory( storeDir ) ) - .withFileSystem( fileSystem ) - .withIndexIdentifier( LuceneLabelScanStore.INDEX_IDENTIFIER ) - .build(); - - labelScanStore = new LuceneLabelScanStore(new NodeRangeDocumentLabelScanStorageStrategy(), index, - fullStoreStream, LuceneLabelScanStore.loggerMonitor( logProvider ) ); + PartitionedIndexStorage indexStorage = new PartitionedIndexStorage( DirectoryFactory.PERSISTENT, fileSystem, + LabelScanStoreProvider.getStoreDirectory( storeDir ), + LuceneLabelScanStore.INDEX_IDENTIFIER ); + LuceneLabelScanIndex index = new LuceneLabelScanIndex( indexStorage ); + labelScanStore = new LuceneLabelScanStore( index, fullStoreStream, + LuceneLabelScanStore.loggerMonitor( logProvider ) ); try { diff --git a/community/lucene-index/src/main/java/org/neo4j/kernel/api/impl/index/LuceneIndex.java b/community/lucene-index/src/main/java/org/neo4j/kernel/api/impl/index/AbstractLuceneIndex.java similarity index 63% rename from community/lucene-index/src/main/java/org/neo4j/kernel/api/impl/index/LuceneIndex.java rename to community/lucene-index/src/main/java/org/neo4j/kernel/api/impl/index/AbstractLuceneIndex.java index 7b4f279915979..fdbbcacbfbf3f 100644 --- a/community/lucene-index/src/main/java/org/neo4j/kernel/api/impl/index/LuceneIndex.java +++ b/community/lucene-index/src/main/java/org/neo4j/kernel/api/impl/index/AbstractLuceneIndex.java @@ -21,7 +21,6 @@ import org.apache.lucene.index.CheckIndex; import org.apache.lucene.index.IndexFileNames; -import org.apache.lucene.index.IndexWriter; import org.apache.lucene.store.Directory; import java.io.Closeable; @@ -32,57 +31,32 @@ import java.util.List; import java.util.Map; import java.util.concurrent.CopyOnWriteArrayList; -import java.util.concurrent.TimeUnit; import java.util.concurrent.locks.ReentrantLock; import java.util.stream.Stream; import org.neo4j.graphdb.ResourceIterator; import org.neo4j.helpers.Exceptions; -import org.neo4j.helpers.TaskCoordinator; import org.neo4j.helpers.collection.Iterables; import org.neo4j.io.IOUtils; import org.neo4j.kernel.api.impl.index.partition.IndexPartition; -import org.neo4j.kernel.api.impl.index.partition.PartitionSearcher; -import org.neo4j.kernel.api.impl.index.reader.PartitionedIndexReader; -import org.neo4j.kernel.api.impl.index.reader.SimpleIndexReader; import org.neo4j.kernel.api.impl.index.storage.PartitionedIndexStorage; -import org.neo4j.kernel.api.index.IndexConfiguration; -import org.neo4j.kernel.impl.api.index.sampling.IndexSamplingConfig; -import org.neo4j.storageengine.api.schema.IndexReader; -import static java.util.Collections.singletonMap; - -public class LuceneIndex implements Closeable +// todo: this component has an implicit possibility to be opened and closed multiple times. we should revisit and at least test it. +public abstract class AbstractLuceneIndex implements Closeable { - private final ReentrantLock commitCloseLock = new ReentrantLock(); - private final ReentrantLock readWriteLock = new ReentrantLock(); + protected final ReentrantLock commitCloseLock = new ReentrantLock(); + protected final ReentrantLock readWriteLock = new ReentrantLock(); + protected final PartitionedIndexStorage indexStorage; private List partitions = new CopyOnWriteArrayList<>(); - private volatile boolean open = false; - private PartitionedIndexStorage indexStorage; - private final IndexConfiguration config; - private final IndexSamplingConfig samplingConfig; - - private static final String KEY_STATUS = "status"; - private static final String ONLINE = "online"; - private static final Map ONLINE_COMMIT_USER_DATA = singletonMap( KEY_STATUS, ONLINE ); - - private final TaskCoordinator taskCoordinator = new TaskCoordinator( 10, TimeUnit.MILLISECONDS ); - - public LuceneIndex( LuceneIndex luceneIndex ) - { - this(luceneIndex.indexStorage, luceneIndex.config, luceneIndex.samplingConfig); - } + private volatile boolean open; - public LuceneIndex( PartitionedIndexStorage indexStorage, IndexConfiguration config, - IndexSamplingConfig samplingConfig ) + public AbstractLuceneIndex( PartitionedIndexStorage indexStorage ) { this.indexStorage = indexStorage; - this.config = config; - this.samplingConfig = samplingConfig; } - public void prepare() throws IOException + public void create() throws IOException { indexStorage.prepareFolder( indexStorage.getIndexFolder() ); indexStorage.reserveIndexFailureStorage(); @@ -102,6 +76,10 @@ public void open() throws IOException public boolean exists() { List folders = indexStorage.listFolders(); + if ( folders.isEmpty() ) + { + return false; + } for ( File folder : folders ) { String[] files = folder.list(); @@ -146,58 +124,79 @@ public boolean isValid() return true; } - private boolean containsSegmentFile( String[] files ) + public void drop() throws IOException { - return Stream.of( files ).anyMatch( file -> file.startsWith( IndexFileNames.SEGMENTS ) ); + close(); + indexStorage.cleanupFolder( indexStorage.getIndexFolder() ); } - public LuceneIndexWriter getIndexWriter() throws IOException + public void flush() throws IOException { - ensureOpen(); - return new PartitionedIndexWriter( this ); + // TODO: do we need it? } - public void maybeRefresh() throws IOException + @Override + public void close() throws IOException { - readWriteLock.lock(); + commitCloseLock.lock(); try { - for ( IndexPartition partition : getPartitions() ) - { - partition.maybeRefresh(); - } + IOUtils.closeAll( partitions ); + partitions.clear(); + open = false; } finally { - readWriteLock.unlock(); + commitCloseLock.unlock(); } } - public void maybeRefreshBlocking() throws IOException + public ResourceIterator snapshot() throws IOException { - readWriteLock.lock(); + ensureOpen(); + commitCloseLock.lock(); + List> snapshotIterators = null; try { - for ( IndexPartition partition : getPartitions() ) + List partitions = getPartitions(); + snapshotIterators = new ArrayList<>( partitions.size() ); + for ( IndexPartition partition : partitions ) { - partition.maybeRefreshBlocking(); + snapshotIterators.add( partition.snapshot() ); + } + return Iterables.concatResourceIterators( snapshotIterators.iterator() ); + } + catch ( Exception e ) + { + if ( snapshotIterators != null ) + { + try + { + IOUtils.closeAll( snapshotIterators ); + } + catch ( IOException ex ) + { + throw Exceptions.withCause( ex, e ); + } } + throw e; } finally { - readWriteLock.unlock(); + commitCloseLock.unlock(); } } - public IndexReader getIndexReader() throws IOException + // todo: remove and always use maybeRefreshBlocking? + public void maybeRefresh() throws IOException { - ensureOpen(); readWriteLock.lock(); try { - List partitions = getPartitions(); - return hasSinglePartition( partitions ) ? createSimpleReader( partitions ) - : createPartitionedReader( partitions ); + for ( IndexPartition partition : getPartitions() ) + { + partition.maybeRefresh(); + } } finally { @@ -205,17 +204,15 @@ public IndexReader getIndexReader() throws IOException } } - IndexPartition addNewPartition() throws IOException + public void maybeRefreshBlocking() throws IOException { - ensureOpen(); readWriteLock.lock(); try { - File partitionFolder = createNewPartitionFolder(); - IndexPartition indexPartition = new IndexPartition( partitionFolder, - indexStorage.openDirectory( partitionFolder ) ); - partitions.add( indexPartition ); - return indexPartition; + for ( IndexPartition partition : getPartitions() ) + { + partition.maybeRefreshBlocking(); + } } finally { @@ -229,85 +226,36 @@ List getPartitions() return partitions; } - private void ensureOpen() - { - if ( !open ) - { - throw new IllegalStateException( "Please open lucene index before working with it." ); - } - } - - @Override - public void close() throws IOException - { - commitCloseLock.lock(); - try - { - IOUtils.closeAll( partitions ); - open = false; - } - finally - { - commitCloseLock.unlock(); - } - } - - public void drop() throws IOException - { - taskCoordinator.cancel(); - try - { - taskCoordinator.awaitCompletion(); - } - catch ( InterruptedException e ) - { - throw new IOException( "Interrupted while waiting for concurrent tasks to complete.", e ); - } - close(); - indexStorage.cleanupFolder( indexStorage.getIndexFolder() ); - } - - public void markAsOnline() throws IOException + IndexPartition addNewPartition() throws IOException { ensureOpen(); - commitCloseLock.lock(); + readWriteLock.lock(); try { - IndexPartition indexPartition = partitions.get( 0 ); - IndexWriter indexWriter = indexPartition.getIndexWriter(); - indexWriter.setCommitData( ONLINE_COMMIT_USER_DATA ); - indexWriter.commit(); + File partitionFolder = createNewPartitionFolder(); + IndexPartition indexPartition = new IndexPartition( partitionFolder, + indexStorage.openDirectory( partitionFolder ) ); + partitions.add( indexPartition ); + return indexPartition; } finally { - commitCloseLock.unlock(); + readWriteLock.unlock(); } } - public void markAsFailed( String failure ) throws IOException - { - indexStorage.storeIndexFailure( failure ); - } - - private SimpleIndexReader createSimpleReader( List partitions ) throws IOException + protected void ensureOpen() { - return new SimpleIndexReader( partitions.get( 0 ).acquireSearcher(), config, - samplingConfig, taskCoordinator ); - } - - private PartitionedIndexReader createPartitionedReader( List partitions ) throws IOException - { - List searchers = new ArrayList<>(); - for ( IndexPartition partition : partitions ) + if ( !open ) { - searchers.add( partition.acquireSearcher() ); + throw new IllegalStateException( "Please open lucene index before working with it." ); } - return new PartitionedIndexReader( searchers ); } - private boolean hasSinglePartition( List partitions ) + + private static boolean containsSegmentFile( String[] files ) { - return partitions.size() == 1; + return Stream.of( files ).anyMatch( file -> file.startsWith( IndexFileNames.SEGMENTS ) ); } private File createNewPartitionFolder() throws IOException @@ -316,45 +264,4 @@ private File createNewPartitionFolder() throws IOException indexStorage.prepareFolder( partitionFolder ); return partitionFolder; } - - - public ResourceIterator snapshot() throws IOException - { - commitCloseLock.lock(); - List> snapshotIterators = null; - try - { - List partitions = getPartitions(); - snapshotIterators = new ArrayList<>( partitions.size() ); - for ( IndexPartition partition : partitions ) - { - snapshotIterators.add( partition.snapshot() ); - } - return Iterables.concatResourceIterators( snapshotIterators.iterator() ); - } - catch ( Exception e ) - { - if ( snapshotIterators != null ) - { - try - { - IOUtils.closeAll( snapshotIterators ); - } - catch ( IOException ex ) - { - throw Exceptions.withCause( ex, e ); - } - } - throw e; - } - finally - { - commitCloseLock.unlock(); - } - } - - public void flush() throws IOException - { - // TODO: do we need it? - } } diff --git a/community/lucene-index/src/main/java/org/neo4j/kernel/api/impl/index/LabelScanStorageStrategy.java b/community/lucene-index/src/main/java/org/neo4j/kernel/api/impl/index/LabelScanStorageStrategy.java index d70da6262e142..b89364f4cdc8b 100644 --- a/community/lucene-index/src/main/java/org/neo4j/kernel/api/impl/index/LabelScanStorageStrategy.java +++ b/community/lucene-index/src/main/java/org/neo4j/kernel/api/impl/index/LabelScanStorageStrategy.java @@ -19,21 +19,20 @@ */ package org.neo4j.kernel.api.impl.index; +import org.apache.lucene.search.IndexSearcher; + import java.util.Iterator; -import java.util.concurrent.locks.Lock; import org.neo4j.collection.primitive.PrimitiveLongIterator; import org.neo4j.kernel.api.direct.AllEntriesLabelScanReader; -import org.neo4j.kernel.api.labelscan.LabelScanWriter; -import org.neo4j.storageengine.api.schema.IndexReader; +import org.neo4j.storageengine.api.schema.LabelScanReader; + public interface LabelScanStorageStrategy { - PrimitiveLongIterator nodesWithLabel( LuceneIndex luceneIndex, int labelId ); - - AllEntriesLabelScanReader newNodeLabelReader( IndexReader indexReader ); + PrimitiveLongIterator nodesWithLabel( IndexSearcher searcher, int labelId ); - Iterator labelsForNode( LuceneIndex luceneIndex, long nodeId ); + AllEntriesLabelScanReader newNodeLabelReader( LabelScanReader reader ); - LabelScanWriter acquireWriter( LuceneIndex index, Lock heldLock ); + Iterator labelsForNode( IndexSearcher searcher, long nodeId ); } diff --git a/community/lucene-index/src/main/java/org/neo4j/kernel/api/impl/index/LuceneAllDocumentsReader.java b/community/lucene-index/src/main/java/org/neo4j/kernel/api/impl/index/LuceneAllDocumentsReader.java index 263bcf5109999..93ec2d102aa36 100644 --- a/community/lucene-index/src/main/java/org/neo4j/kernel/api/impl/index/LuceneAllDocumentsReader.java +++ b/community/lucene-index/src/main/java/org/neo4j/kernel/api/impl/index/LuceneAllDocumentsReader.java @@ -23,34 +23,41 @@ import java.util.Iterator; +import org.neo4j.io.IOUtils; import org.neo4j.kernel.api.direct.BoundedIterable; import org.neo4j.storageengine.api.schema.IndexReader; +import org.neo4j.storageengine.api.schema.LabelScanReader; public class LuceneAllDocumentsReader implements BoundedIterable { + private LabelScanReader labelScanReader; private IndexReader indexReader; - public LuceneAllDocumentsReader( IndexReader indexReader) + public LuceneAllDocumentsReader( IndexReader indexReader ) { this.indexReader = indexReader; } + public LuceneAllDocumentsReader( LabelScanReader labelScanReader ) + { + this.labelScanReader = labelScanReader; + } + @Override public long maxCount() { - return indexReader.getMaxDoc(); + return labelScanReader != null ? labelScanReader.getMaxDoc() : indexReader.getMaxDoc(); } @Override public Iterator iterator() { - return indexReader.getAllDocsIterator(); + return labelScanReader != null ? labelScanReader.getAllDocsIterator() : indexReader.getAllDocsIterator(); } @Override public void close() { - indexReader.close(); + IOUtils.closeAllSilently( labelScanReader, labelScanReader ); } - } diff --git a/community/lucene-index/src/main/java/org/neo4j/kernel/api/impl/index/LuceneIndexAccessor.java b/community/lucene-index/src/main/java/org/neo4j/kernel/api/impl/index/LuceneIndexAccessor.java index ccc1fe258d682..d0ad0fcac1a71 100644 --- a/community/lucene-index/src/main/java/org/neo4j/kernel/api/impl/index/LuceneIndexAccessor.java +++ b/community/lucene-index/src/main/java/org/neo4j/kernel/api/impl/index/LuceneIndexAccessor.java @@ -32,14 +32,13 @@ import org.neo4j.kernel.impl.api.index.IndexUpdateMode; import org.neo4j.storageengine.api.schema.IndexReader; - public class LuceneIndexAccessor implements IndexAccessor { private final LuceneDocumentStructure documentStructure = new LuceneDocumentStructure(); private final LuceneIndexWriter writer; - private LuceneIndex luceneIndex; + private LuceneSchemaIndex luceneIndex; - LuceneIndexAccessor( LuceneIndex luceneIndex ) throws IOException + LuceneIndexAccessor( LuceneSchemaIndex luceneIndex ) throws IOException { this.luceneIndex = luceneIndex; this.writer = luceneIndex.getIndexWriter(); diff --git a/community/lucene-index/src/main/java/org/neo4j/kernel/api/impl/index/LuceneLabelScanIndex.java b/community/lucene-index/src/main/java/org/neo4j/kernel/api/impl/index/LuceneLabelScanIndex.java new file mode 100644 index 0000000000000..393eebec4ff7a --- /dev/null +++ b/community/lucene-index/src/main/java/org/neo4j/kernel/api/impl/index/LuceneLabelScanIndex.java @@ -0,0 +1,98 @@ +/* + * Copyright (c) 2002-2015 "Neo Technology," + * Network Engine for Objects in Lund AB [http://neotechnology.com] + * + * This file is part of Neo4j. + * + * Neo4j is free software: you can redistribute it and/or modify + * it under the terms of the GNU General Public License as published by + * the Free Software Foundation, either version 3 of the License, or + * (at your option) any later version. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU General Public License for more details. + * + * You should have received a copy of the GNU General Public License + * along with this program. If not, see . + */ +package org.neo4j.kernel.api.impl.index; + +import java.io.IOException; +import java.io.UncheckedIOException; +import java.util.List; +import java.util.concurrent.locks.Lock; + +import org.neo4j.kernel.api.impl.index.partition.IndexPartition; +import org.neo4j.kernel.api.impl.index.partition.PartitionSearcher; +import org.neo4j.kernel.api.impl.index.storage.PartitionedIndexStorage; +import org.neo4j.kernel.api.labelscan.LabelScanWriter; +import org.neo4j.storageengine.api.schema.LabelScanReader; + +public class LuceneLabelScanIndex extends AbstractLuceneIndex +{ + private final BitmapDocumentFormat format; + private final LabelScanStorageStrategy storageStrategy; + + public LuceneLabelScanIndex( PartitionedIndexStorage indexStorage ) + { + this( BitmapDocumentFormat._32, indexStorage ); + } + + public LuceneLabelScanIndex( BitmapDocumentFormat format, PartitionedIndexStorage indexStorage ) + { + super( indexStorage ); + this.format = format; + this.storageStrategy = new NodeRangeDocumentLabelScanStorageStrategy( format ); + } + + public LabelScanReader getLabelScanReader() + { + ensureOpen(); + readWriteLock.lock(); + try + { + List partitions = getPartitions(); + if ( partitions.size() == 1 ) + { + IndexPartition partition = partitions.get( 0 ); + PartitionSearcher searcher = partition.acquireSearcher(); + return new SimpleLuceneLabelScanStoreReader( searcher, storageStrategy ); + } + throw new UnsupportedOperationException(); + } + catch ( IOException e ) + { + throw new UncheckedIOException( e ); + } + finally + { + readWriteLock.unlock(); + } + } + + public LabelScanWriter getLabelScanWriter( Lock heldLock ) + { + ensureOpen(); + readWriteLock.lock(); + try + { + List partitions = getPartitions(); + if ( partitions.size() == 1 ) + { + return new SimpleLuceneLabelScanWriter( partitions.get( 0 ), format, heldLock ); + } + throw new UnsupportedOperationException(); + } + finally + { + readWriteLock.unlock(); + } + } + + private boolean hasSinglePartition( List partitions ) + { + return partitions.size() == 1; + } +} diff --git a/community/lucene-index/src/main/java/org/neo4j/kernel/api/impl/index/LuceneLabelScanStore.java b/community/lucene-index/src/main/java/org/neo4j/kernel/api/impl/index/LuceneLabelScanStore.java index 3a6f479cae383..60d0d48c6efbd 100644 --- a/community/lucene-index/src/main/java/org/neo4j/kernel/api/impl/index/LuceneLabelScanStore.java +++ b/community/lucene-index/src/main/java/org/neo4j/kernel/api/impl/index/LuceneLabelScanStore.java @@ -23,13 +23,10 @@ import java.io.File; import java.io.IOException; -import java.util.Iterator; import java.util.concurrent.locks.Lock; import java.util.concurrent.locks.ReentrantLock; -import org.neo4j.collection.primitive.PrimitiveLongIterator; import org.neo4j.graphdb.ResourceIterator; -import org.neo4j.kernel.api.direct.AllEntriesLabelScanReader; import org.neo4j.kernel.api.labelscan.LabelScanStore; import org.neo4j.kernel.api.labelscan.LabelScanWriter; import org.neo4j.kernel.impl.api.scan.LabelScanStoreProvider.FullStoreChangeStream; @@ -42,8 +39,7 @@ public class LuceneLabelScanStore implements LabelScanStore { public static final String INDEX_IDENTIFIER = "labelStore"; - private final LabelScanStorageStrategy strategy; - private final LuceneIndex luceneIndex; + private final LuceneLabelScanIndex luceneIndex; // We get in a full store stream here in case we need to fully rebuild the store if it's missing or corrupted. private final FullStoreChangeStream fullStoreStream; private final Monitor monitor; @@ -110,29 +106,14 @@ public void rebuilt( long highNodeId ) }; } - public LuceneLabelScanStore( LabelScanStorageStrategy strategy, LuceneIndex luceneIndex, - FullStoreChangeStream fullStoreStream, Monitor monitor ) + public LuceneLabelScanStore( LuceneLabelScanIndex luceneIndex, FullStoreChangeStream fullStoreStream, + Monitor monitor ) { this.luceneIndex = luceneIndex; - this.strategy = strategy; this.fullStoreStream = fullStoreStream; this.monitor = monitor; } - private AllEntriesLabelScanReader newAllEntriesReader() - { - try - { - return strategy.newNodeLabelReader( luceneIndex.getIndexReader() ); - } - catch ( IOException e ) - { - //TODO: - e.printStackTrace(); - throw new RuntimeException( e ); - } - } - @Override public void force() { @@ -149,41 +130,7 @@ public void force() @Override public LabelScanReader newReader() { - - return new LabelScanReader() - { - @Override - public PrimitiveLongIterator nodesWithLabel( int labelId ) - { - return strategy.nodesWithLabel( luceneIndex, labelId ); - } - - @Override - public void close() - { - try - { - // why? - luceneIndex.maybeRefresh(); - } - catch ( IOException e ) - { - throw new RuntimeException( e ); - } - } - - @Override - public Iterator labelsForNode( long nodeId ) - { - return strategy.labelsForNode( luceneIndex, nodeId ); - } - - @Override - public AllEntriesLabelScanReader allNodeLabelRanges() - { - return newAllEntriesReader(); - } - }; + return luceneIndex.getLabelScanReader(); } @Override @@ -201,14 +148,19 @@ public void init() throws IOException if ( !luceneIndex.exists() ) { monitor.noIndex(); - luceneIndex.prepare(); + luceneIndex.create(); needsRebuild = true; } else if ( !luceneIndex.isValid() ) { - // monitor.corruptIndex( ); - luceneIndex.prepare(); - needsRebuild = true; + // todo: rebuild instead of failing? failing is here now only because there is a test expecting failure... +// monitor.corruptIndex( ); +// luceneIndex.create(); +// needsRebuild = true; + + throw new IOException( "Label scan store could not be read, and needs to be rebuilt. " + + "To trigger a rebuild, ensure the database is stopped, delete the files in '" + + luceneIndex + "', and then start the database again." ); } // todo: test this strange open-close thingy @@ -260,6 +212,6 @@ public LabelScanWriter newWriter() // Only a single writer is allowed at any point in time. For that this lock is used and passed // onto the writer to release in its close() lock.lock(); - return strategy.acquireWriter( luceneIndex, lock ); + return luceneIndex.getLabelScanWriter(lock); } } diff --git a/community/lucene-index/src/main/java/org/neo4j/kernel/api/impl/index/LuceneLabelScanStoreExtension.java b/community/lucene-index/src/main/java/org/neo4j/kernel/api/impl/index/LuceneLabelScanStoreExtension.java index 4343e38d9107e..5701ac2d0b675 100644 --- a/community/lucene-index/src/main/java/org/neo4j/kernel/api/impl/index/LuceneLabelScanStoreExtension.java +++ b/community/lucene-index/src/main/java/org/neo4j/kernel/api/impl/index/LuceneLabelScanStoreExtension.java @@ -25,6 +25,7 @@ import org.neo4j.kernel.NeoStoreDataSource; import org.neo4j.kernel.api.impl.index.LuceneLabelScanStore.Monitor; import org.neo4j.kernel.api.impl.index.storage.DirectoryFactory; +import org.neo4j.kernel.api.impl.index.storage.PartitionedIndexStorage; import org.neo4j.kernel.configuration.Config; import org.neo4j.kernel.extension.KernelExtensionFactory; import org.neo4j.kernel.impl.api.index.IndexStoreView; @@ -77,22 +78,19 @@ public LabelScanStoreProvider newInstance( KernelContext context, Dependencies d boolean ephemeral = dependencies.getConfig().get( GraphDatabaseFacadeFactory.Configuration.ephemeral ); DirectoryFactory directoryFactory = directoryFactory( ephemeral, context.fileSystem() ); - LuceneIndex index = getLuceneIndex( context, directoryFactory ); - LuceneLabelScanStore scanStore = new LuceneLabelScanStore( - new NodeRangeDocumentLabelScanStorageStrategy(), index, - fullStoreLabelUpdateStream( dependencies.indexStoreView() ), - monitor != null ? monitor : loggerMonitor( dependencies.getLogService().getInternalLogProvider() ) ); + LuceneLabelScanIndex index = getLuceneIndex( context, directoryFactory ); + LuceneLabelScanStore scanStore = new LuceneLabelScanStore( index, + fullStoreLabelUpdateStream( dependencies.indexStoreView() ), getMonitor( dependencies ) ); + return new LabelScanStoreProvider( scanStore, priority ); } - private LuceneIndex getLuceneIndex( KernelContext context, DirectoryFactory directoryFactory ) + private LuceneLabelScanIndex getLuceneIndex( KernelContext context, DirectoryFactory directoryFactory ) { - return LuceneIndexBuilder.create() - .withIndexRootFolder( LabelScanStoreProvider.getStoreDirectory( context.storeDir() ) ) - .withFileSystem( context.fileSystem() ) - .withDirectoryFactory( directoryFactory ) - .withIndexIdentifier( LuceneLabelScanStore.INDEX_IDENTIFIER ) - .build(); + PartitionedIndexStorage indexStorage = new PartitionedIndexStorage( directoryFactory, context.fileSystem(), + LabelScanStoreProvider.getStoreDirectory( context.storeDir() ), + LuceneLabelScanStore.INDEX_IDENTIFIER ); + return new LuceneLabelScanIndex( indexStorage ); } private Monitor getMonitor( Dependencies dependencies ) diff --git a/community/lucene-index/src/main/java/org/neo4j/kernel/api/impl/index/LuceneSchemaIndex.java b/community/lucene-index/src/main/java/org/neo4j/kernel/api/impl/index/LuceneSchemaIndex.java new file mode 100644 index 0000000000000..66c353c5768b0 --- /dev/null +++ b/community/lucene-index/src/main/java/org/neo4j/kernel/api/impl/index/LuceneSchemaIndex.java @@ -0,0 +1,140 @@ +/* + * Copyright (c) 2002-2015 "Neo Technology," + * Network Engine for Objects in Lund AB [http://neotechnology.com] + * + * This file is part of Neo4j. + * + * Neo4j is free software: you can redistribute it and/or modify + * it under the terms of the GNU General Public License as published by + * the Free Software Foundation, either version 3 of the License, or + * (at your option) any later version. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU General Public License for more details. + * + * You should have received a copy of the GNU General Public License + * along with this program. If not, see . + */ +package org.neo4j.kernel.api.impl.index; + +import org.apache.lucene.index.IndexWriter; + +import java.io.IOException; +import java.util.ArrayList; +import java.util.List; +import java.util.Map; +import java.util.concurrent.TimeUnit; + +import org.neo4j.helpers.TaskCoordinator; +import org.neo4j.kernel.api.impl.index.partition.IndexPartition; +import org.neo4j.kernel.api.impl.index.partition.PartitionSearcher; +import org.neo4j.kernel.api.impl.index.reader.PartitionedIndexReader; +import org.neo4j.kernel.api.impl.index.reader.SimpleIndexReader; +import org.neo4j.kernel.api.impl.index.storage.PartitionedIndexStorage; +import org.neo4j.kernel.api.index.IndexConfiguration; +import org.neo4j.kernel.impl.api.index.sampling.IndexSamplingConfig; +import org.neo4j.storageengine.api.schema.IndexReader; + +import static java.util.Collections.singletonMap; + +public class LuceneSchemaIndex extends AbstractLuceneIndex +{ + private static final String KEY_STATUS = "status"; + private static final String ONLINE = "online"; + private static final Map ONLINE_COMMIT_USER_DATA = singletonMap( KEY_STATUS, ONLINE ); + + private final IndexConfiguration config; + private final IndexSamplingConfig samplingConfig; + + private final TaskCoordinator taskCoordinator = new TaskCoordinator( 10, TimeUnit.MILLISECONDS ); + + public LuceneSchemaIndex( PartitionedIndexStorage indexStorage, IndexConfiguration config, + IndexSamplingConfig samplingConfig ) + { + super( indexStorage ); + this.config = config; + this.samplingConfig = samplingConfig; + } + + public LuceneIndexWriter getIndexWriter() throws IOException + { + ensureOpen(); + return new PartitionedIndexWriter( this ); + } + + public IndexReader getIndexReader() throws IOException + { + ensureOpen(); + readWriteLock.lock(); + try + { + List partitions = getPartitions(); + return hasSinglePartition( partitions ) ? createSimpleReader( partitions ) + : createPartitionedReader( partitions ); + } + finally + { + readWriteLock.unlock(); + } + } + + private boolean hasSinglePartition( List partitions ) + { + return partitions.size() == 1; + } + + private SimpleIndexReader createSimpleReader( List partitions ) throws IOException + { + return new SimpleIndexReader( partitions.get( 0 ).acquireSearcher(), config, + samplingConfig, taskCoordinator ); + } + + private PartitionedIndexReader createPartitionedReader( List partitions ) throws IOException + { + List searchers = new ArrayList<>(); + for ( IndexPartition partition : partitions ) + { + searchers.add( partition.acquireSearcher() ); + } + return new PartitionedIndexReader( searchers ); + } + + @Override + public void drop() throws IOException + { + taskCoordinator.cancel(); + try + { + taskCoordinator.awaitCompletion(); + } + catch ( InterruptedException e ) + { + throw new IOException( "Interrupted while waiting for concurrent tasks to complete.", e ); + } + super.drop(); + } + + public void markAsOnline() throws IOException + { + ensureOpen(); + commitCloseLock.lock(); + try + { + IndexPartition indexPartition = getPartitions().get( 0 ); + IndexWriter indexWriter = indexPartition.getIndexWriter(); + indexWriter.setCommitData( ONLINE_COMMIT_USER_DATA ); + indexWriter.commit(); + } + finally + { + commitCloseLock.unlock(); + } + } + + public void markAsFailed( String failure ) throws IOException + { + indexStorage.storeIndexFailure( failure ); + } +} diff --git a/community/lucene-index/src/main/java/org/neo4j/kernel/api/impl/index/LuceneIndexBuilder.java b/community/lucene-index/src/main/java/org/neo4j/kernel/api/impl/index/LuceneSchemaIndexBuilder.java similarity index 75% rename from community/lucene-index/src/main/java/org/neo4j/kernel/api/impl/index/LuceneIndexBuilder.java rename to community/lucene-index/src/main/java/org/neo4j/kernel/api/impl/index/LuceneSchemaIndexBuilder.java index 34d193ea18350..7ecff8e249153 100644 --- a/community/lucene-index/src/main/java/org/neo4j/kernel/api/impl/index/LuceneIndexBuilder.java +++ b/community/lucene-index/src/main/java/org/neo4j/kernel/api/impl/index/LuceneSchemaIndexBuilder.java @@ -34,7 +34,7 @@ import static org.neo4j.helpers.collection.MapUtil.stringMap; -public class LuceneIndexBuilder +public class LuceneSchemaIndexBuilder { private IndexSamplingConfig samplingConfig = new IndexSamplingConfig( new Config() ); private IndexConfiguration indexConfig = IndexConfiguration.NON_UNIQUE; @@ -44,25 +44,28 @@ public class LuceneIndexBuilder private String indexIdentifier; private PartitionedIndexStorage indexStorage; - private LuceneIndexBuilder() {} + private LuceneSchemaIndexBuilder() + { + } - public static LuceneIndexBuilder create() { - return new LuceneIndexBuilder(); + public static LuceneSchemaIndexBuilder create() + { + return new LuceneSchemaIndexBuilder(); } - public LuceneIndexBuilder withIndexIdentifier( String indexIdentifier ) + public LuceneSchemaIndexBuilder withIndexIdentifier( String indexIdentifier ) { this.indexIdentifier = indexIdentifier; return this; } - public LuceneIndexBuilder withSamplingConfig( IndexSamplingConfig samplingConfig ) + public LuceneSchemaIndexBuilder withSamplingConfig( IndexSamplingConfig samplingConfig ) { this.samplingConfig = samplingConfig; return this; } - public LuceneIndexBuilder withSamplingBufferSize( int size ) + public LuceneSchemaIndexBuilder withSamplingBufferSize( int size ) { Map params = stringMap( GraphDatabaseSettings.index_sampling_buffer_size.name(), size + "" ); Config config = new Config( params ); @@ -70,51 +73,51 @@ public LuceneIndexBuilder withSamplingBufferSize( int size ) return this; } - public LuceneIndexBuilder withSamplingConfig( Config config ) + public LuceneSchemaIndexBuilder withSamplingConfig( Config config ) { this.samplingConfig = new IndexSamplingConfig( config ); return this; } - public LuceneIndexBuilder withIndexConfig( IndexConfiguration indexConfig ) + public LuceneSchemaIndexBuilder withIndexConfig( IndexConfiguration indexConfig ) { this.indexConfig = indexConfig; return this; } - public LuceneIndexBuilder withIndexStorage( PartitionedIndexStorage indexStorage ) + public LuceneSchemaIndexBuilder withIndexStorage( PartitionedIndexStorage indexStorage ) { this.indexStorage = indexStorage; return this; } - public LuceneIndexBuilder uniqueIndex() + public LuceneSchemaIndexBuilder uniqueIndex() { this.indexConfig = IndexConfiguration.UNIQUE; return this; } - public LuceneIndexBuilder withDirectoryFactory( DirectoryFactory directoryFactory ) + public LuceneSchemaIndexBuilder withDirectoryFactory( DirectoryFactory directoryFactory ) { this.directoryFactory = directoryFactory; return this; } - public LuceneIndexBuilder withFileSystem( FileSystemAbstraction fileSystem ) + public LuceneSchemaIndexBuilder withFileSystem( FileSystemAbstraction fileSystem ) { this.fileSystem = fileSystem; return this; } - public LuceneIndexBuilder withIndexRootFolder( File indexRootFolder ) + public LuceneSchemaIndexBuilder withIndexRootFolder( File indexRootFolder ) { this.indexRootFolder = indexRootFolder; return this; } - public LuceneIndex build() + public LuceneSchemaIndex build() { - return new LuceneIndex( buildIndexStorage(), indexConfig, samplingConfig ); + return new LuceneSchemaIndex( buildIndexStorage(), indexConfig, samplingConfig ); } private PartitionedIndexStorage buildIndexStorage() diff --git a/community/lucene-index/src/main/java/org/neo4j/kernel/api/impl/index/LuceneSchemaIndexProvider.java b/community/lucene-index/src/main/java/org/neo4j/kernel/api/impl/index/LuceneSchemaIndexProvider.java index ee6a81b01f4b5..939c8625bfdc6 100644 --- a/community/lucene-index/src/main/java/org/neo4j/kernel/api/impl/index/LuceneSchemaIndexProvider.java +++ b/community/lucene-index/src/main/java/org/neo4j/kernel/api/impl/index/LuceneSchemaIndexProvider.java @@ -66,7 +66,7 @@ public IndexPopulator getPopulator( long indexId, IndexDescriptor descriptor, IndexConfiguration config, IndexSamplingConfig samplingConfig ) { PartitionedIndexStorage indexStorage = getIndexStorage( indexId ); - LuceneIndex luceneIndex = new LuceneIndex( indexStorage, config, samplingConfig ); + LuceneSchemaIndex luceneIndex = new LuceneSchemaIndex( indexStorage, config, samplingConfig ); if ( config.isUnique() ) { return new DeferredConstraintVerificationUniqueLuceneIndexPopulator( luceneIndex, descriptor ); @@ -82,7 +82,7 @@ public IndexAccessor getOnlineAccessor( long indexId, IndexConfiguration config, IndexSamplingConfig samplingConfig ) throws IOException { PartitionedIndexStorage indexStorage = getIndexStorage( indexId ); - LuceneIndex luceneIndex = new LuceneIndex( indexStorage, config, samplingConfig ); + LuceneSchemaIndex luceneIndex = new LuceneSchemaIndex( indexStorage, config, samplingConfig ); luceneIndex.open(); return new LuceneIndexAccessor( luceneIndex ); } diff --git a/community/lucene-index/src/main/java/org/neo4j/kernel/api/impl/index/NodeRangeDocumentLabelScanStorageStrategy.java b/community/lucene-index/src/main/java/org/neo4j/kernel/api/impl/index/NodeRangeDocumentLabelScanStorageStrategy.java index aeda711627ed8..3f8839f0ea409 100644 --- a/community/lucene-index/src/main/java/org/neo4j/kernel/api/impl/index/NodeRangeDocumentLabelScanStorageStrategy.java +++ b/community/lucene-index/src/main/java/org/neo4j/kernel/api/impl/index/NodeRangeDocumentLabelScanStorageStrategy.java @@ -27,13 +27,11 @@ import java.util.ArrayList; import java.util.Iterator; import java.util.List; -import java.util.concurrent.locks.Lock; import org.neo4j.collection.primitive.PrimitiveLongIterator; import org.neo4j.kernel.api.direct.AllEntriesLabelScanReader; import org.neo4j.kernel.api.impl.index.bitmaps.BitmapFormat; -import org.neo4j.kernel.api.labelscan.LabelScanWriter; -import org.neo4j.storageengine.api.schema.IndexReader; +import org.neo4j.storageengine.api.schema.LabelScanReader; import static org.neo4j.collection.primitive.PrimitiveLongCollections.concat; import static org.neo4j.helpers.collection.IteratorUtil.emptyIterator; @@ -66,11 +64,6 @@ public class NodeRangeDocumentLabelScanStorageStrategy implements LabelScanStora private static final int RANGES_PER_PAGE = 4096; private final BitmapDocumentFormat format; - public NodeRangeDocumentLabelScanStorageStrategy() - { - this( BitmapDocumentFormat._32 ); - } - NodeRangeDocumentLabelScanStorageStrategy( BitmapDocumentFormat format ) { this.format = format; @@ -83,26 +76,24 @@ public String toString() } @Override - public PrimitiveLongIterator nodesWithLabel( LuceneIndex index, int labelId ) + public PrimitiveLongIterator nodesWithLabel( IndexSearcher searcher, int labelId ) { return concat( - new PageOfRangesIterator( format, index, RANGES_PER_PAGE, format.labelQuery( labelId ), labelId ) ); + new PageOfRangesIterator( format, searcher, RANGES_PER_PAGE, format.labelQuery( labelId ), labelId ) ); } @Override - public AllEntriesLabelScanReader newNodeLabelReader( IndexReader indexReader ) + public AllEntriesLabelScanReader newNodeLabelReader( LabelScanReader reader ) { - return new LuceneAllEntriesLabelScanReader( new LuceneAllDocumentsReader( indexReader ), format ); + LuceneAllDocumentsReader documents = new LuceneAllDocumentsReader( reader ); + return new LuceneAllEntriesLabelScanReader( documents, format ); } @Override - public Iterator labelsForNode( LuceneIndex luceneIndex, long nodeId ) + public Iterator labelsForNode( IndexSearcher searcher, long nodeId ) { try { - IndexReader indexReader = luceneIndex.getIndexReader(); - - IndexSearcher searcher = null; TopDocs topDocs = searcher.search( format.rangeQuery( format.bitmapFormat().rangeOf( nodeId ) ), 1 ); if ( topDocs.scoreDocs.length < 1 ) @@ -144,10 +135,4 @@ else if ( topDocs.scoreDocs.length > 1 ) throw new RuntimeException( e ); } } - - @Override - public LabelScanWriter acquireWriter( LuceneIndex luceneIndex, Lock heldLock ) - { - return new LuceneLabelScanWriter( luceneIndex, format, heldLock ); - } } diff --git a/community/lucene-index/src/main/java/org/neo4j/kernel/api/impl/index/PageOfRangesIterator.java b/community/lucene-index/src/main/java/org/neo4j/kernel/api/impl/index/PageOfRangesIterator.java index ec5ef13f9cc89..36b9467ecbd39 100644 --- a/community/lucene-index/src/main/java/org/neo4j/kernel/api/impl/index/PageOfRangesIterator.java +++ b/community/lucene-index/src/main/java/org/neo4j/kernel/api/impl/index/PageOfRangesIterator.java @@ -31,17 +31,17 @@ class PageOfRangesIterator extends PrefetchingIterator { - private LuceneIndex index; + private IndexSearcher searcher; private final Query query; private final BitmapDocumentFormat format; private final int rangesPerPage; private final int[] labels; private DocValuesCollector.LongValuesIterator rangesIterator; - PageOfRangesIterator( BitmapDocumentFormat format, LuceneIndex index, int rangesPerPage, Query query, - int... labels ) + PageOfRangesIterator( BitmapDocumentFormat format, IndexSearcher searcher, int rangesPerPage, Query query, + int... labels ) { - this.index = index; + this.searcher = searcher; this.query = query; this.format = format; this.rangesPerPage = rangesPerPage; @@ -55,8 +55,6 @@ class PageOfRangesIterator extends PrefetchingIterator @Override protected PrimitiveLongIterator fetchNextOrNull() { - //TODO - IndexSearcher searcher = null; if ( searcher == null ) { return null; // we are done searching with this iterator @@ -81,8 +79,6 @@ protected PrimitiveLongIterator fetchNextOrNull() } private DocValuesCollector.LongValuesIterator getRanges() { - //TODO - IndexSearcher searcher = null; if ( rangesIterator != null ) { return rangesIterator; diff --git a/community/lucene-index/src/main/java/org/neo4j/kernel/api/impl/index/PartitionedIndexWriter.java b/community/lucene-index/src/main/java/org/neo4j/kernel/api/impl/index/PartitionedIndexWriter.java index 10fe519913251..20b3a436aa7e2 100644 --- a/community/lucene-index/src/main/java/org/neo4j/kernel/api/impl/index/PartitionedIndexWriter.java +++ b/community/lucene-index/src/main/java/org/neo4j/kernel/api/impl/index/PartitionedIndexWriter.java @@ -33,13 +33,13 @@ public class PartitionedIndexWriter implements LuceneIndexWriter { - private LuceneIndex index; + private LuceneSchemaIndex index; private static final Integer MAXIMUM_PARTITION_SIZE = FeatureToggles.getInteger( PartitionedIndexWriter.class, "partitionSize", Integer.MAX_VALUE - (Integer.MAX_VALUE / 10) ); - public PartitionedIndexWriter( LuceneIndex index ) throws IOException + public PartitionedIndexWriter( LuceneSchemaIndex index ) throws IOException { this.index = index; } diff --git a/community/lucene-index/src/main/java/org/neo4j/kernel/api/impl/index/SimpleLuceneLabelScanStoreReader.java b/community/lucene-index/src/main/java/org/neo4j/kernel/api/impl/index/SimpleLuceneLabelScanStoreReader.java new file mode 100644 index 0000000000000..ddf875f39985c --- /dev/null +++ b/community/lucene-index/src/main/java/org/neo4j/kernel/api/impl/index/SimpleLuceneLabelScanStoreReader.java @@ -0,0 +1,145 @@ +/* + * Copyright (c) 2002-2015 "Neo Technology," + * Network Engine for Objects in Lund AB [http://neotechnology.com] + * + * This file is part of Neo4j. + * + * Neo4j is free software: you can redistribute it and/or modify + * it under the terms of the GNU General Public License as published by + * the Free Software Foundation, either version 3 of the License, or + * (at your option) any later version. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU General Public License for more details. + * + * You should have received a copy of the GNU General Public License + * along with this program. If not, see . + */ +package org.neo4j.kernel.api.impl.index; + +import org.apache.lucene.document.Document; +import org.apache.lucene.index.MultiFields; +import org.apache.lucene.search.DocIdSetIterator; +import org.apache.lucene.search.FilteredDocIdSetIterator; +import org.apache.lucene.util.Bits; + +import java.io.IOException; +import java.util.Iterator; + +import org.neo4j.collection.primitive.PrimitiveLongIterator; +import org.neo4j.helpers.collection.PrefetchingIterator; +import org.neo4j.kernel.api.impl.index.partition.PartitionSearcher; +import org.neo4j.kernel.api.impl.index.reader.IndexReaderCloseException; +import org.neo4j.storageengine.api.schema.AllEntriesLabelScanReader; +import org.neo4j.storageengine.api.schema.LabelScanReader; + +public class SimpleLuceneLabelScanStoreReader implements LabelScanReader +{ + private final PartitionSearcher partitionSearcher; + private final LabelScanStorageStrategy storageStrategy; + + public SimpleLuceneLabelScanStoreReader( PartitionSearcher partitionSearcher, + LabelScanStorageStrategy storageStrategy ) + { + this.partitionSearcher = partitionSearcher; + this.storageStrategy = storageStrategy; + } + + @Override + public PrimitiveLongIterator nodesWithLabel( int labelId ) + { + return storageStrategy.nodesWithLabel( partitionSearcher.getIndexSearcher(), labelId ); + } + + @Override + public Iterator labelsForNode( long nodeId ) + { + return storageStrategy.labelsForNode( partitionSearcher.getIndexSearcher(), nodeId ); + } + + @Override + public AllEntriesLabelScanReader allNodeLabelRanges() + { + return storageStrategy.newNodeLabelReader( this ); + } + + @Override + public Iterator getAllDocsIterator() + { + return new PrefetchingIterator() + { + private DocIdSetIterator idIterator = iterateAllDocs(); + + @Override + protected Document fetchNextOrNull() + { + try + { + int doc = idIterator.nextDoc(); + if ( doc == DocIdSetIterator.NO_MORE_DOCS ) + { + return null; + } + return getDocument( doc ); + } + catch ( IOException e ) + { + throw new LuceneDocumentRetrievalException( "Can't fetch document id from lucene index.", e ); + } + } + }; + } + + private Document getDocument( int docId ) + { + try + { + return partitionSearcher.getIndexSearcher().doc( docId ); + } + catch ( IOException e ) + { + throw new LuceneDocumentRetrievalException( "Can't retrieve document with id: " + docId + ".", docId, e ); + } + } + + private DocIdSetIterator iterateAllDocs() + { + org.apache.lucene.index.IndexReader reader = partitionSearcher.getIndexSearcher().getIndexReader(); + final Bits liveDocs = MultiFields.getLiveDocs( reader ); + final DocIdSetIterator allDocs = DocIdSetIterator.all( reader.maxDoc() ); + if ( liveDocs == null ) + { + return allDocs; + } + + return new FilteredDocIdSetIterator( allDocs ) + { + @Override + protected boolean match( int doc ) + { + return liveDocs.get( doc ); + } + }; + } + + @Override + public void close() + { + try + { + partitionSearcher.close(); + } + catch ( IOException e ) + { + throw new IndexReaderCloseException( e ); + } + } + + @Override + public long getMaxDoc() + { + return partitionSearcher.getIndexSearcher().getIndexReader().maxDoc(); + } +} diff --git a/community/lucene-index/src/main/java/org/neo4j/kernel/api/impl/index/LuceneLabelScanWriter.java b/community/lucene-index/src/main/java/org/neo4j/kernel/api/impl/index/SimpleLuceneLabelScanWriter.java similarity index 76% rename from community/lucene-index/src/main/java/org/neo4j/kernel/api/impl/index/LuceneLabelScanWriter.java rename to community/lucene-index/src/main/java/org/neo4j/kernel/api/impl/index/SimpleLuceneLabelScanWriter.java index 04081bd707704..4e7ee95f64d9e 100644 --- a/community/lucene-index/src/main/java/org/neo4j/kernel/api/impl/index/LuceneLabelScanWriter.java +++ b/community/lucene-index/src/main/java/org/neo4j/kernel/api/impl/index/SimpleLuceneLabelScanWriter.java @@ -33,24 +33,25 @@ import java.util.concurrent.locks.Lock; import org.neo4j.kernel.api.impl.index.bitmaps.Bitmap; +import org.neo4j.kernel.api.impl.index.partition.IndexPartition; +import org.neo4j.kernel.api.impl.index.partition.PartitionSearcher; import org.neo4j.kernel.api.labelscan.LabelScanWriter; import org.neo4j.kernel.api.labelscan.NodeLabelUpdate; import static java.lang.String.format; -public class LuceneLabelScanWriter implements LabelScanWriter +public class SimpleLuceneLabelScanWriter implements LabelScanWriter { - private final LuceneIndex index; + private final IndexPartition partition; private final BitmapDocumentFormat format; private final List updates; private long currentRange; private final Lock heldLock; - public LuceneLabelScanWriter( LuceneIndex index, - BitmapDocumentFormat format, Lock heldLock ) + public SimpleLuceneLabelScanWriter( IndexPartition partition, BitmapDocumentFormat format, Lock heldLock ) { - this.index = index; + this.partition = partition; this.format = format; this.heldLock = heldLock; currentRange = -1; @@ -89,8 +90,8 @@ public void close() throws IOException { try { - // todo: why do we need maybeRefresh here? maybe use maybeRefresh - index.maybeRefresh(); + // todo: why do we need maybeRefresh here? maybe use maybeRefreshBlocking + partition.maybeRefresh(); } finally { @@ -128,33 +129,35 @@ private void flush() throws IOException return; } - // TODO: - IndexSearcher searcher = null; - Map fields = readLabelBitMapsInRange( searcher, currentRange ); - updateFields( updates, fields ); + try ( PartitionSearcher partitionSearcher = partition.acquireSearcher() ) + { + IndexSearcher searcher = partitionSearcher.getIndexSearcher(); + Map fields = readLabelBitMapsInRange( searcher, currentRange ); + updateFields( updates, fields ); - Document document = new Document(); - format.addRangeValuesField( document, currentRange ); + Document document = new Document(); + format.addRangeValuesField( document, currentRange ); - for ( Map.Entry field : fields.entrySet() ) - { - // one field per label - Bitmap value = field.getValue(); - if ( value.hasContent() ) + for ( Map.Entry field : fields.entrySet() ) { - format.addLabelAndSearchFields( document, field.getKey(), value ); + // one field per label + Bitmap value = field.getValue(); + if ( value.hasContent() ) + { + format.addLabelAndSearchFields( document, field.getKey(), value ); + } } - } - if ( isEmpty( document ) ) - { - index.getIndexWriter().deleteDocuments( format.rangeTerm( document ) ); - } - else - { - index.getIndexWriter().updateDocument( format.rangeTerm( document ), document ); + if ( isEmpty( document ) ) + { + partition.getIndexWriter().deleteDocuments( format.rangeTerm( document ) ); + } + else + { + partition.getIndexWriter().updateDocument( format.rangeTerm( document ), document ); + } + updates.clear(); } - updates.clear(); } private boolean isEmpty( Document document ) diff --git a/community/lucene-index/src/main/java/org/neo4j/kernel/api/impl/index/populator/DeferredConstraintVerificationUniqueLuceneIndexPopulator.java b/community/lucene-index/src/main/java/org/neo4j/kernel/api/impl/index/populator/DeferredConstraintVerificationUniqueLuceneIndexPopulator.java index 83195a5429b2f..f7d4d98280fdb 100644 --- a/community/lucene-index/src/main/java/org/neo4j/kernel/api/impl/index/populator/DeferredConstraintVerificationUniqueLuceneIndexPopulator.java +++ b/community/lucene-index/src/main/java/org/neo4j/kernel/api/impl/index/populator/DeferredConstraintVerificationUniqueLuceneIndexPopulator.java @@ -27,7 +27,7 @@ import org.neo4j.collection.primitive.PrimitiveLongSet; import org.neo4j.kernel.api.exceptions.index.IndexEntryConflictException; -import org.neo4j.kernel.api.impl.index.LuceneIndex; +import org.neo4j.kernel.api.impl.index.LuceneSchemaIndex; import org.neo4j.kernel.api.index.IndexDescriptor; import org.neo4j.kernel.api.index.IndexUpdater; import org.neo4j.kernel.api.index.NodePropertyUpdate; @@ -41,7 +41,7 @@ public class DeferredConstraintVerificationUniqueLuceneIndexPopulator extends Lu private final IndexDescriptor descriptor; private final UniqueIndexSampler sampler; - public DeferredConstraintVerificationUniqueLuceneIndexPopulator(LuceneIndex index, IndexDescriptor descriptor ) + public DeferredConstraintVerificationUniqueLuceneIndexPopulator(LuceneSchemaIndex index, IndexDescriptor descriptor ) { super( index ); this.descriptor = descriptor; diff --git a/community/lucene-index/src/main/java/org/neo4j/kernel/api/impl/index/populator/LuceneIndexPopulator.java b/community/lucene-index/src/main/java/org/neo4j/kernel/api/impl/index/populator/LuceneIndexPopulator.java index 7aaa2429b0c64..6fb2bd7ee8f3f 100644 --- a/community/lucene-index/src/main/java/org/neo4j/kernel/api/impl/index/populator/LuceneIndexPopulator.java +++ b/community/lucene-index/src/main/java/org/neo4j/kernel/api/impl/index/populator/LuceneIndexPopulator.java @@ -22,17 +22,17 @@ import java.io.IOException; import org.neo4j.kernel.api.impl.index.LuceneDocumentStructure; -import org.neo4j.kernel.api.impl.index.LuceneIndex; import org.neo4j.kernel.api.impl.index.LuceneIndexWriter; +import org.neo4j.kernel.api.impl.index.LuceneSchemaIndex; import org.neo4j.kernel.api.index.IndexPopulator; public abstract class LuceneIndexPopulator implements IndexPopulator { - protected LuceneIndex luceneIndex; + protected LuceneSchemaIndex luceneIndex; protected LuceneIndexWriter writer; protected final LuceneDocumentStructure documentStructure = new LuceneDocumentStructure(); - LuceneIndexPopulator( LuceneIndex luceneIndex ) + LuceneIndexPopulator( LuceneSchemaIndex luceneIndex ) { this.luceneIndex = luceneIndex; } @@ -40,7 +40,7 @@ public abstract class LuceneIndexPopulator implements IndexPopulator @Override public void create() throws IOException { - luceneIndex.prepare(); + luceneIndex.create(); luceneIndex.open(); writer = luceneIndex.getIndexWriter(); } diff --git a/community/lucene-index/src/main/java/org/neo4j/kernel/api/impl/index/populator/NonUniqueLuceneIndexPopulator.java b/community/lucene-index/src/main/java/org/neo4j/kernel/api/impl/index/populator/NonUniqueLuceneIndexPopulator.java index 6ffed38459195..8599a5ad921b6 100644 --- a/community/lucene-index/src/main/java/org/neo4j/kernel/api/impl/index/populator/NonUniqueLuceneIndexPopulator.java +++ b/community/lucene-index/src/main/java/org/neo4j/kernel/api/impl/index/populator/NonUniqueLuceneIndexPopulator.java @@ -25,7 +25,7 @@ import org.neo4j.collection.primitive.PrimitiveLongSet; import org.neo4j.kernel.api.exceptions.index.IndexEntryConflictException; -import org.neo4j.kernel.api.impl.index.LuceneIndex; +import org.neo4j.kernel.api.impl.index.LuceneSchemaIndex; import org.neo4j.kernel.api.index.IndexUpdater; import org.neo4j.kernel.api.index.NodePropertyUpdate; import org.neo4j.kernel.api.index.PropertyAccessor; @@ -41,7 +41,7 @@ public class NonUniqueLuceneIndexPopulator extends LuceneIndexPopulator private final NonUniqueIndexSampler sampler; private final List updates = new ArrayList<>(); - public NonUniqueLuceneIndexPopulator( LuceneIndex luceneIndex, IndexSamplingConfig samplingConfig ) + public NonUniqueLuceneIndexPopulator( LuceneSchemaIndex luceneIndex, IndexSamplingConfig samplingConfig ) { super( luceneIndex ); this.sampler = new NonUniqueIndexSampler( samplingConfig.bufferSize() ); diff --git a/community/lucene-index/src/main/java/org/neo4j/kernel/api/impl/index/storage/DirectoryFactory.java b/community/lucene-index/src/main/java/org/neo4j/kernel/api/impl/index/storage/DirectoryFactory.java index e41fd122b21f0..685a525743c5a 100644 --- a/community/lucene-index/src/main/java/org/neo4j/kernel/api/impl/index/storage/DirectoryFactory.java +++ b/community/lucene-index/src/main/java/org/neo4j/kernel/api/impl/index/storage/DirectoryFactory.java @@ -19,6 +19,15 @@ */ package org.neo4j.kernel.api.impl.index.storage; +import org.apache.lucene.index.IndexWriter; +import org.apache.lucene.store.Directory; +import org.apache.lucene.store.FSDirectory; +import org.apache.lucene.store.IOContext; +import org.apache.lucene.store.IndexInput; +import org.apache.lucene.store.IndexOutput; +import org.apache.lucene.store.Lock; +import org.apache.lucene.store.RAMDirectory; + import java.io.File; import java.io.IOException; import java.io.OutputStream; @@ -28,15 +37,8 @@ import java.util.zip.ZipEntry; import java.util.zip.ZipOutputStream; -import org.apache.lucene.store.Directory; -import org.apache.lucene.store.FSDirectory; -import org.apache.lucene.store.IOContext; -import org.apache.lucene.store.IndexInput; -import org.apache.lucene.store.IndexOutput; -import org.apache.lucene.store.Lock; -import org.apache.lucene.store.RAMDirectory; - import org.neo4j.io.fs.FileSystemAbstraction; +import org.neo4j.kernel.api.impl.index.IndexWriterFactories; import static java.lang.Math.min; @@ -82,7 +84,14 @@ public synchronized Directory open( File dir ) throws IOException { if(!directories.containsKey( dir )) { - directories.put( dir, new RAMDirectory() ); + // todo: RAMDirectory has no segment files initially. This causes CheckIndex to fail. + // todo: check if it is OK to just make a dummy commit here to create segment file + RAMDirectory directory = new RAMDirectory(); + try ( IndexWriter writer = new IndexWriter( directory, IndexWriterFactories.standardConfig() ) ) + { + writer.commit(); + } + directories.put( dir, directory ); } return new UncloseableDirectory(directories.get(dir)); } diff --git a/community/lucene-index/src/test/java/org/neo4j/graphdb/LuceneLabelScanStoreChaosIT.java b/community/lucene-index/src/test/java/org/neo4j/graphdb/LuceneLabelScanStoreChaosIT.java index 7e88291f6c0e2..4d2bfdc12ebd1 100644 --- a/community/lucene-index/src/test/java/org/neo4j/graphdb/LuceneLabelScanStoreChaosIT.java +++ b/community/lucene-index/src/test/java/org/neo4j/graphdb/LuceneLabelScanStoreChaosIT.java @@ -19,28 +19,32 @@ */ package org.neo4j.graphdb; +import org.junit.Rule; +import org.junit.Test; + import java.io.File; import java.io.IOException; import java.io.RandomAccessFile; import java.nio.ByteBuffer; import java.nio.channels.FileChannel; +import java.util.Collections; +import java.util.List; import java.util.Random; import java.util.Set; - -import org.junit.Rule; -import org.junit.Test; +import java.util.stream.Stream; import org.neo4j.function.Predicates; +import org.neo4j.kernel.api.impl.index.LuceneLabelScanStore; import org.neo4j.test.DatabaseRule; import org.neo4j.test.DatabaseRule.RestartAction; import org.neo4j.test.EmbeddedDatabaseRule; +import static java.util.stream.Collectors.toList; import static org.hamcrest.CoreMatchers.containsString; import static org.junit.Assert.assertEquals; import static org.junit.Assert.assertThat; import static org.junit.Assert.assertTrue; import static org.junit.Assert.fail; - import static org.neo4j.helpers.Exceptions.peel; import static org.neo4j.helpers.collection.IteratorUtil.asSet; import static org.neo4j.io.fs.FileUtils.deleteRecursively; @@ -103,10 +107,14 @@ private RestartAction corruptTheLabelScanStoreIndex() try { int filesCorrupted = 0; - for ( File file : labelScanStoreIndexDirectory( storeDirectory ).listFiles() ) + List partitionDirs = labelScanStoreIndexDirectories( storeDirectory ); + for ( File partitionDir : partitionDirs ) { - scrambleFile( file ); - filesCorrupted++; + for ( File file : partitionDir.listFiles() ) + { + scrambleFile( file ); + filesCorrupted++; + } } assertTrue( "No files found to corrupt", filesCorrupted > 0 ); } @@ -122,10 +130,13 @@ private RestartAction deleteTheLabelScanStoreIndex() return ( fs, storeDirectory ) -> { try { - File directory = labelScanStoreIndexDirectory( storeDirectory ); - assertTrue( "We seem to want to delete the wrong directory here", directory.exists() ); - assertTrue( "No index files to delete", directory.listFiles().length > 0 ); - deleteRecursively( directory ); + List partitionDirs = labelScanStoreIndexDirectories( storeDirectory ); + for ( File dir : partitionDirs ) + { + assertTrue( "We seem to want to delete the wrong directory here", dir.exists() ); + assertTrue( "No index files to delete", dir.listFiles().length > 0 ); + deleteRecursively( dir ); + } } catch ( IOException e ) { @@ -134,9 +145,13 @@ private RestartAction deleteTheLabelScanStoreIndex() }; } - private File labelScanStoreIndexDirectory( File storeDirectory ) + private List labelScanStoreIndexDirectories( File storeDirectory ) { - return new File( new File( new File( storeDirectory, "schema" ), "label" ), "lucene" ); + File rootDir = new File( new File( new File( new File( storeDirectory, "schema" ), "label" ), "lucene" ), + LuceneLabelScanStore.INDEX_IDENTIFIER ); + + File[] partitionDirs = rootDir.listFiles( File::isDirectory ); + return (partitionDirs == null) ? Collections.emptyList() : Stream.of( partitionDirs ).collect( toList() ); } private Node createLabeledNode( Label... labels ) diff --git a/community/lucene-index/src/test/java/org/neo4j/kernel/api/impl/index/AccessUniqueLuceneIndexTest.java b/community/lucene-index/src/test/java/org/neo4j/kernel/api/impl/index/AccessUniqueLuceneIndexTest.java index 7003f1be7d6b4..b3375ef3e62bf 100644 --- a/community/lucene-index/src/test/java/org/neo4j/kernel/api/impl/index/AccessUniqueLuceneIndexTest.java +++ b/community/lucene-index/src/test/java/org/neo4j/kernel/api/impl/index/AccessUniqueLuceneIndexTest.java @@ -130,7 +130,7 @@ public void shouldConsiderWholeTransactionForValidatingUniqueness() throws Excep private LuceneIndexAccessor createAccessor( PartitionedIndexStorage indexStorage ) throws IOException { - LuceneIndex luceneIndex = new LuceneIndex( indexStorage, new IndexConfiguration( true ), + LuceneSchemaIndex luceneIndex = new LuceneSchemaIndex( indexStorage, new IndexConfiguration( true ), new IndexSamplingConfig( new Config() ) ); luceneIndex.open(); return new LuceneIndexAccessor( luceneIndex ); diff --git a/community/lucene-index/src/test/java/org/neo4j/kernel/api/impl/index/DeferredConstraintVerificationUniqueLuceneIndexPopulatorTest.java b/community/lucene-index/src/test/java/org/neo4j/kernel/api/impl/index/DeferredConstraintVerificationUniqueLuceneIndexPopulatorTest.java index b8d54427f5d5a..e84106e3a2b8f 100644 --- a/community/lucene-index/src/test/java/org/neo4j/kernel/api/impl/index/DeferredConstraintVerificationUniqueLuceneIndexPopulatorTest.java +++ b/community/lucene-index/src/test/java/org/neo4j/kernel/api/impl/index/DeferredConstraintVerificationUniqueLuceneIndexPopulatorTest.java @@ -458,7 +458,7 @@ private DeferredConstraintVerificationUniqueLuceneIndexPopulator newPopulator() EphemeralFileSystemAbstraction fileSystem = new EphemeralFileSystemAbstraction(); indexStorage = new PartitionedIndexStorage( directoryFactory, fileSystem, new File( "/target/whatever" ), INDEX_IDENTIFIER ); - LuceneIndex index = LuceneIndexBuilder.create() + LuceneSchemaIndex index = LuceneSchemaIndexBuilder.create() .withIndexStorage( indexStorage ) .build(); DeferredConstraintVerificationUniqueLuceneIndexPopulator populator = new diff --git a/community/lucene-index/src/test/java/org/neo4j/kernel/api/impl/index/LuceneIndexAccessorTest.java b/community/lucene-index/src/test/java/org/neo4j/kernel/api/impl/index/LuceneIndexAccessorTest.java index dbac52516a368..3bb8760c6ae4e 100644 --- a/community/lucene-index/src/test/java/org/neo4j/kernel/api/impl/index/LuceneIndexAccessorTest.java +++ b/community/lucene-index/src/test/java/org/neo4j/kernel/api/impl/index/LuceneIndexAccessorTest.java @@ -87,14 +87,14 @@ public static Collection[]> imp public LuceneIndexAccessor apply( DirectoryFactory dirFactory ) throws IOException { - LuceneIndex index = LuceneIndexBuilder.create() + LuceneSchemaIndex index = LuceneSchemaIndexBuilder.create() .withFileSystem( new EphemeralFileSystemAbstraction() ) .withDirectoryFactory( dirFactory ) .withIndexRootFolder( dir ) .withIndexIdentifier( "1" ) .build(); - index.prepare(); + index.create(); index.open(); return new LuceneIndexAccessor( index ); } @@ -105,7 +105,7 @@ public LuceneIndexAccessor apply( DirectoryFactory dirFactory ) public LuceneIndexAccessor apply( DirectoryFactory dirFactory ) throws IOException { - LuceneIndex index = LuceneIndexBuilder.create() + LuceneSchemaIndex index = LuceneSchemaIndexBuilder.create() .uniqueIndex() .withFileSystem( new EphemeralFileSystemAbstraction() ) .withDirectoryFactory( dirFactory ) @@ -113,7 +113,7 @@ public LuceneIndexAccessor apply( DirectoryFactory dirFactory ) .withIndexIdentifier( "testIndex" ) .build(); - index.prepare(); + index.create(); index.open(); return new LuceneIndexAccessor( index ); } diff --git a/community/lucene-index/src/test/java/org/neo4j/kernel/api/impl/index/LuceneIndexIT.java b/community/lucene-index/src/test/java/org/neo4j/kernel/api/impl/index/LuceneIndexIT.java index 1dc24e64c81a2..86f66b526008a 100644 --- a/community/lucene-index/src/test/java/org/neo4j/kernel/api/impl/index/LuceneIndexIT.java +++ b/community/lucene-index/src/test/java/org/neo4j/kernel/api/impl/index/LuceneIndexIT.java @@ -59,12 +59,12 @@ public class LuceneIndexIT @Before public void before() throws Exception { - LuceneIndex index = LuceneIndexBuilder.create() + LuceneSchemaIndex index = LuceneSchemaIndexBuilder.create() .withIndexRootFolder( testDir.directory() ) .withIndexIdentifier( "testIndex" ) .build(); - index.prepare(); + index.create(); index.open(); accessor = new LuceneIndexAccessor( index ); } diff --git a/community/lucene-index/src/test/java/org/neo4j/kernel/api/impl/index/LuceneLabelScanStoreTest.java b/community/lucene-index/src/test/java/org/neo4j/kernel/api/impl/index/LuceneLabelScanStoreTest.java index ba1bd9bb980f1..cf39e676101fd 100644 --- a/community/lucene-index/src/test/java/org/neo4j/kernel/api/impl/index/LuceneLabelScanStoreTest.java +++ b/community/lucene-index/src/test/java/org/neo4j/kernel/api/impl/index/LuceneLabelScanStoreTest.java @@ -44,22 +44,24 @@ import org.neo4j.collection.primitive.PrimitiveLongIterator; import org.neo4j.helpers.collection.IteratorUtil; import org.neo4j.helpers.collection.PrefetchingIterator; -import org.neo4j.storageengine.api.schema.BoundedIterable; -import org.neo4j.storageengine.api.schema.NodeLabelRange; +import org.neo4j.io.fs.DefaultFileSystemAbstraction; import org.neo4j.kernel.api.impl.index.storage.DirectoryFactory; +import org.neo4j.kernel.api.impl.index.storage.PartitionedIndexStorage; import org.neo4j.kernel.api.labelscan.LabelScanWriter; import org.neo4j.kernel.api.labelscan.NodeLabelUpdate; import org.neo4j.kernel.impl.api.scan.LabelScanStoreProvider.FullStoreChangeStream; import org.neo4j.kernel.lifecycle.LifeSupport; import org.neo4j.kernel.lifecycle.LifecycleException; +import org.neo4j.storageengine.api.schema.BoundedIterable; import org.neo4j.storageengine.api.schema.LabelScanReader; +import org.neo4j.storageengine.api.schema.NodeLabelRange; import org.neo4j.test.TargetDirectory; import static java.util.Arrays.asList; import static java.util.Collections.emptyList; +import static org.hamcrest.Matchers.startsWith; import static org.hamcrest.core.IsCollectionContaining.hasItem; import static org.hamcrest.core.IsCollectionContaining.hasItems; -import static org.hamcrest.core.IsEqual.equalTo; import static org.hamcrest.core.IsInstanceOf.instanceOf; import static org.junit.Assert.assertArrayEquals; import static org.junit.Assert.assertEquals; @@ -77,14 +79,14 @@ public class LuceneLabelScanStoreTest { private static final long[] NO_LABELS = new long[0]; - private final LabelScanStorageStrategy strategy; + private final BitmapDocumentFormat documentFormat; - @Parameterized.Parameters(name = "{0}") + @Parameterized.Parameters( name = "{0}" ) public static List parameterizedWithStrategies() { return asList( - new Object[]{new NodeRangeDocumentLabelScanStorageStrategy( BitmapDocumentFormat._32 )}, - new Object[]{new NodeRangeDocumentLabelScanStorageStrategy( BitmapDocumentFormat._64 )} + new Object[]{BitmapDocumentFormat._32}, + new Object[]{BitmapDocumentFormat._64} ); } @@ -95,6 +97,7 @@ public static List parameterizedWithStrategies() private DirectoryFactory directoryFactory = new DirectoryFactory.InMemoryDirectoryFactory(); private LifeSupport life; private TrackingMonitor monitor; + private PartitionedIndexStorage indexStorage; private LuceneLabelScanStore store; private File dir; @@ -337,11 +340,9 @@ public void shouldRefuseStartIfIndexCorrupted() throws Exception } catch( LifecycleException e ) { - assertThat(e.getCause(), instanceOf( IOException.class )); - assertThat(e.getCause().getMessage(), equalTo( - "Label scan store could not be read, and needs to be rebuilt. To trigger a rebuild, ensure the " + - "database is stopped, delete the files in '"+dir.getAbsolutePath()+"', and then start the " + - "database again." )); + assertThat( e.getCause(), instanceOf( IOException.class ) ); + assertThat( e.getCause().getMessage(), startsWith( + "Label scan store could not be read, and needs to be rebuilt" ) ); } } @@ -412,9 +413,9 @@ private Set gaps( Set ids, int expectedCount ) return gaps; } - public LuceneLabelScanStoreTest( LabelScanStorageStrategy strategy ) + public LuceneLabelScanStoreTest( BitmapDocumentFormat documentFormat ) { - this.strategy = strategy; + this.documentFormat = documentFormat; } private void assertNodesForLabel( int labelId, long... expectedNodeIds ) @@ -454,12 +455,11 @@ private void start( List existingData ) life = new LifeSupport(); monitor = new TrackingMonitor(); - LuceneIndex luceneIndex = LuceneIndexBuilder.create() - .withIndexIdentifier( "testIndex" ) - .withDirectoryFactory( directoryFactory ) - .withIndexRootFolder( dir ) - .build(); - store = life.add( new LuceneLabelScanStore( strategy, luceneIndex, asStream( existingData ), monitor ) ); + indexStorage = new PartitionedIndexStorage( directoryFactory, new DefaultFileSystemAbstraction(), dir, + LuceneLabelScanStore.INDEX_IDENTIFIER ); + + LuceneLabelScanIndex index = new LuceneLabelScanIndex( documentFormat, indexStorage ); + store = life.add( new LuceneLabelScanStore( index, asStream( existingData ), monitor ) ); life.start(); assertTrue( monitor.initCalled ); @@ -486,12 +486,16 @@ public long applyTo( LabelScanWriter writer ) throws IOException private void scrambleIndexFilesAndRestart( List data ) throws IOException { shutdown(); - File[] files = dir.listFiles(); - if ( files != null ) + List indexPartitions = indexStorage.listFolders(); + for ( File partition : indexPartitions ) { - for ( File indexFile : files ) + File[] files = partition.listFiles(); + if ( files != null ) { - scrambleFile( indexFile ); + for ( File indexFile : files ) + { + scrambleFile( indexFile ); + } } } start( data ); diff --git a/community/lucene-index/src/test/java/org/neo4j/kernel/api/impl/index/LuceneLabelScanStoreWriterTest.java b/community/lucene-index/src/test/java/org/neo4j/kernel/api/impl/index/LuceneLabelScanStoreWriterTest.java index b80d09b5ee63a..0279d5a24f5f4 100644 --- a/community/lucene-index/src/test/java/org/neo4j/kernel/api/impl/index/LuceneLabelScanStoreWriterTest.java +++ b/community/lucene-index/src/test/java/org/neo4j/kernel/api/impl/index/LuceneLabelScanStoreWriterTest.java @@ -69,7 +69,7 @@ public void shouldComplainIfNodesSuppliedOutOfRangeOrder() throws Exception assertNotEquals( node1Range, node2Range ); // when - LuceneLabelScanWriter writer = new LuceneLabelScanWriter( null, FORMAT, mock( Lock.class ) ); + SimpleLuceneLabelScanWriter writer = new SimpleLuceneLabelScanWriter( null, FORMAT, mock( Lock.class ) ); writer.write( NodeLabelUpdate.labelChanges( nodeId2, new long[]{}, new long[]{} ) ); try { @@ -92,7 +92,7 @@ public void shouldStoreDocumentWithNodeIdsAndLabelsInIt() throws Exception StubStorageService storage = new StubStorageService(); // when - LuceneLabelScanWriter writer = new LuceneLabelScanWriter( null, FORMAT, mock( Lock.class ) ); + SimpleLuceneLabelScanWriter writer = new SimpleLuceneLabelScanWriter( null, FORMAT, mock( Lock.class ) ); writer.write( NodeLabelUpdate.labelChanges( nodeId, new long[]{}, new long[]{label1, label2} ) ); writer.close(); @@ -120,7 +120,7 @@ public void shouldStoreDocumentWithNodeIdsInTheSameRange() throws Exception assertEquals( range, format.bitmapFormat().rangeOf( nodeId2 ) ); // when - LuceneLabelScanWriter writer = new LuceneLabelScanWriter( null, format, mock( Lock.class ) ); + SimpleLuceneLabelScanWriter writer = new SimpleLuceneLabelScanWriter( null, format, mock( Lock.class ) ); writer.write( NodeLabelUpdate.labelChanges( nodeId1, new long[]{}, new long[]{label1} ) ); writer.write( NodeLabelUpdate.labelChanges( nodeId2, new long[]{}, new long[]{label2} ) ); @@ -149,7 +149,7 @@ public void shouldStoreDocumentWithNodeIdsInADifferentRange() throws Exception assertNotEquals( node1Range, node2Range ); // when - LuceneLabelScanWriter writer = new LuceneLabelScanWriter( null, format, mock( Lock.class ) ); + SimpleLuceneLabelScanWriter writer = new SimpleLuceneLabelScanWriter( null, format, mock( Lock.class ) ); writer.write( NodeLabelUpdate.labelChanges( nodeId1, new long[]{}, new long[]{label1} ) ); writer.write( NodeLabelUpdate.labelChanges( nodeId2, new long[]{}, new long[]{label2} ) ); @@ -178,12 +178,12 @@ public void shouldUpdateExistingDocumentWithNodesInTheSameRange() throws Excepti assertEquals( range, format.bitmapFormat().rangeOf( nodeId2 ) ); // node already indexed - LuceneLabelScanWriter writer = new LuceneLabelScanWriter( null, format, mock( Lock.class ) ); + SimpleLuceneLabelScanWriter writer = new SimpleLuceneLabelScanWriter( null, format, mock( Lock.class ) ); writer.write( NodeLabelUpdate.labelChanges( nodeId1, new long[]{}, new long[]{label1} ) ); writer.close(); // when - writer = new LuceneLabelScanWriter( null, format, mock( Lock.class ) ); + writer = new SimpleLuceneLabelScanWriter( null, format, mock( Lock.class ) ); writer.write( NodeLabelUpdate.labelChanges( nodeId2, new long[]{}, new long[]{label2} ) ); writer.close(); diff --git a/community/lucene-index/src/test/java/org/neo4j/kernel/api/impl/index/NodeRangeDocumentLabelScanStorageStrategyTest.java b/community/lucene-index/src/test/java/org/neo4j/kernel/api/impl/index/NodeRangeDocumentLabelScanStorageStrategyTest.java index bf09d6187b9a7..b86e3ae2aeede 100644 --- a/community/lucene-index/src/test/java/org/neo4j/kernel/api/impl/index/NodeRangeDocumentLabelScanStorageStrategyTest.java +++ b/community/lucene-index/src/test/java/org/neo4j/kernel/api/impl/index/NodeRangeDocumentLabelScanStorageStrategyTest.java @@ -73,7 +73,7 @@ public NodeRangeDocumentLabelScanStorageStrategyTest( BitmapDocumentFormat forma // when( searcher.search( new TermQuery( format.rangeTerm( 0 ) ), 1 ) ).thenReturn( docs() ); // when( searcher.search( new TermQuery( format.rangeTerm( 1 ) ), 1 ) ).thenReturn( null ); // -// LuceneLabelScanWriter writer = new LuceneLabelScanWriter( storage, format, mock( Lock.class ) ); +// SimpleLuceneLabelScanWriter writer = new SimpleLuceneLabelScanWriter( storage, format, mock( Lock.class ) ); // // // when // writer.write( labelChanges( 0, labels(), labels( 6, 7 ) ) ); @@ -107,7 +107,7 @@ public NodeRangeDocumentLabelScanStorageStrategyTest( BitmapDocumentFormat forma // format.addLabelFields( givenDoc, "7", 0x70L ); // LabelScanStorageStrategy.StorageService storage = storage(givenDoc); // -// LuceneLabelScanWriter writer = new LuceneLabelScanWriter( storage, format, mock( Lock.class ) ); +// SimpleLuceneLabelScanWriter writer = new SimpleLuceneLabelScanWriter( storage, format, mock( Lock.class ) ); // // // when // writer.write( labelChanges( 0, labels(), labels( 7, 8 ) ) ); @@ -130,7 +130,7 @@ public NodeRangeDocumentLabelScanStorageStrategyTest( BitmapDocumentFormat forma // format.labelField( 7, 0x1 ), // format.labelField( 8, 0x1 ) ) ); // -// LuceneLabelScanWriter writer = new LuceneLabelScanWriter( storage, format, mock( Lock.class ) ); +// SimpleLuceneLabelScanWriter writer = new SimpleLuceneLabelScanWriter( storage, format, mock( Lock.class ) ); // // // when // writer.write( labelChanges( 0, labels( 7, 8 ), labels( 8 ) ) ); @@ -151,7 +151,7 @@ public NodeRangeDocumentLabelScanStorageStrategyTest( BitmapDocumentFormat forma // document( format.rangeField( 0 ), // format.labelField( 7, 0x1 ) ) ); // -// LuceneLabelScanWriter writer = new LuceneLabelScanWriter( storage, format, mock( Lock.class ) ); +// SimpleLuceneLabelScanWriter writer = new SimpleLuceneLabelScanWriter( storage, format, mock( Lock.class ) ); // // // when // writer.write( labelChanges( 0, labels( 7 ), labels() ) ); @@ -170,7 +170,7 @@ public NodeRangeDocumentLabelScanStorageStrategyTest( BitmapDocumentFormat forma // format.labelField( 6, 0x1 ), // format.labelField( 7, 0x1 ) ) ); // -// LuceneLabelScanWriter writer = new LuceneLabelScanWriter( storage, format, mock( Lock.class ) ); +// SimpleLuceneLabelScanWriter writer = new SimpleLuceneLabelScanWriter( storage, format, mock( Lock.class ) ); // // // when // writer.write( labelChanges( 0, labels( 7 ), labels( 7, 8 ) ) ); @@ -193,7 +193,7 @@ public NodeRangeDocumentLabelScanStorageStrategyTest( BitmapDocumentFormat forma // // given // LabelScanStorageStrategy.StorageService storage = storage(); // -// LuceneLabelScanWriter writer = new LuceneLabelScanWriter( storage, format, mock( Lock.class ) ); +// SimpleLuceneLabelScanWriter writer = new SimpleLuceneLabelScanWriter( storage, format, mock( Lock.class ) ); // // // when // writer.write( labelChanges( i, labels(), labels( 7 ) ) ); @@ -214,7 +214,7 @@ public NodeRangeDocumentLabelScanStorageStrategyTest( BitmapDocumentFormat forma // // GIVEN // LabelScanStorageStrategy.StorageService storage = storage(); // Lock lock = mock( Lock.class ); -// LuceneLabelScanWriter writer = new LuceneLabelScanWriter( storage, format, lock ); +// SimpleLuceneLabelScanWriter writer = new SimpleLuceneLabelScanWriter( storage, format, lock ); // // // WHEN // writer.close(); diff --git a/community/lucene-index/src/test/java/org/neo4j/kernel/api/impl/index/PageOfRangesIteratorTest.java b/community/lucene-index/src/test/java/org/neo4j/kernel/api/impl/index/PageOfRangesIteratorTest.java index 9dc86ab913c8b..8a7d98c848325 100644 --- a/community/lucene-index/src/test/java/org/neo4j/kernel/api/impl/index/PageOfRangesIteratorTest.java +++ b/community/lucene-index/src/test/java/org/neo4j/kernel/api/impl/index/PageOfRangesIteratorTest.java @@ -119,9 +119,8 @@ public Void answer( InvocationOnMock invocation ) throws Throwable } } ).when( searcher ).search( same( query ), any( DocValuesCollector.class ) ); - // todo PrimitiveLongIterator iterator = concat( - new PageOfRangesIterator( format, null, pageSize, query, labelId ) ); + new PageOfRangesIterator( format, searcher, pageSize, query, labelId ) ); // when List longs = primitivesList( iterator ); diff --git a/enterprise/ha/src/test/java/org/neo4j/kernel/api/impl/index/LabelScanStoreHaIT.java b/enterprise/ha/src/test/java/org/neo4j/kernel/api/impl/index/LabelScanStoreHaIT.java index 87ece2ef33281..cb7cbec8fb16d 100644 --- a/enterprise/ha/src/test/java/org/neo4j/kernel/api/impl/index/LabelScanStoreHaIT.java +++ b/enterprise/ha/src/test/java/org/neo4j/kernel/api/impl/index/LabelScanStoreHaIT.java @@ -22,7 +22,6 @@ import org.apache.lucene.store.LockObtainFailedException; import org.junit.After; import org.junit.Before; -import org.junit.Ignore; import org.junit.Rule; import org.junit.Test; @@ -44,12 +43,10 @@ import static org.junit.Assert.assertEquals; import static org.junit.Assert.assertTrue; - import static org.neo4j.helpers.collection.IteratorUtil.count; import static org.neo4j.kernel.impl.ha.ClusterManager.allAvailabilityGuardsReleased; import static org.neo4j.kernel.impl.ha.ClusterManager.allSeesAllAsAvailable; -@Ignore("Label scan store to be upgraded") public class LabelScanStoreHaIT { @Test