From b94a609f54d6b5f74b6b265c9b3b2b75da4c243f Mon Sep 17 00:00:00 2001 From: lutovich Date: Wed, 30 Dec 2015 17:04:04 +0100 Subject: [PATCH] Reading all documents moved inside lucene index Consistency checked needs to read all lucene documents from schema indexes and label scan store to verify their correctness. Temporarily logic of reading lucene documents and presenting them as node ids or node label ranges lived in IndexReader and LabelScanReader. This commit moves it to the AbstractLuceneIndex. Index is now capable of giving away an iterator over lucene documents. Callers of this method then interpret documents as node ids or node label ranges. --- .../full/NodeCorrectlyIndexedCheckTest.java | 14 --- .../full/NodeCorrectlyIndexedCheckTest.java | 14 --- .../api/index/DelegatingIndexReader.java | 15 +-- .../storageengine/api/schema/IndexReader.java | 19 +-- .../api/schema/LabelScanReader.java | 4 +- .../api/index/inmemory/HashBasedIndex.java | 14 --- .../impl/api/scan/InMemoryLabelScanStore.java | 14 --- .../api/impl/index/AbstractLuceneIndex.java | 36 ++++++ .../impl/index/LabelScanStorageStrategy.java | 4 +- .../impl/index/LuceneAllDocumentsReader.java | 32 ++--- .../api/impl/index/LuceneIndexAccessor.java | 4 +- .../api/impl/index/LuceneLabelScanIndex.java | 2 +- .../LucenePartitionAllDocumentsReader.java | 119 ++++++++++++++++++ ...RangeDocumentLabelScanStorageStrategy.java | 9 +- .../SimpleLuceneLabelScanStoreReader.java | 78 +----------- .../index/reader/PartitionedIndexReader.java | 14 --- .../impl/index/reader/SimpleIndexReader.java | 73 ----------- 17 files changed, 188 insertions(+), 277 deletions(-) create mode 100644 community/lucene-index/src/main/java/org/neo4j/kernel/api/impl/index/LucenePartitionAllDocumentsReader.java diff --git a/community/consistency-check-legacy/src/test/java/org/neo4j/legacy/consistency/checking/full/NodeCorrectlyIndexedCheckTest.java b/community/consistency-check-legacy/src/test/java/org/neo4j/legacy/consistency/checking/full/NodeCorrectlyIndexedCheckTest.java index f03cbe6e64a2..46ed4ca17ac0 100644 --- a/community/consistency-check-legacy/src/test/java/org/neo4j/legacy/consistency/checking/full/NodeCorrectlyIndexedCheckTest.java +++ b/community/consistency-check-legacy/src/test/java/org/neo4j/legacy/consistency/checking/full/NodeCorrectlyIndexedCheckTest.java @@ -19,7 +19,6 @@ */ package org.neo4j.legacy.consistency.checking.full; -import org.apache.lucene.document.Document; import org.junit.Test; import java.io.File; @@ -27,7 +26,6 @@ import java.util.ArrayList; import java.util.Collections; import java.util.HashMap; -import java.util.Iterator; import java.util.List; import java.util.Map; @@ -291,18 +289,6 @@ public void verifyDeferredConstraints( Object accessor, int propertyKeyId, { } - @Override - public long getMaxDoc() - { - return 0; - } - - @Override - public Iterator getAllDocsIterator() - { - return null; - } - @Override public void close() { diff --git a/community/consistency-check/src/test/java/org/neo4j/consistency/checking/full/NodeCorrectlyIndexedCheckTest.java b/community/consistency-check/src/test/java/org/neo4j/consistency/checking/full/NodeCorrectlyIndexedCheckTest.java index b48089286233..383819496af1 100644 --- a/community/consistency-check/src/test/java/org/neo4j/consistency/checking/full/NodeCorrectlyIndexedCheckTest.java +++ b/community/consistency-check/src/test/java/org/neo4j/consistency/checking/full/NodeCorrectlyIndexedCheckTest.java @@ -19,7 +19,6 @@ */ package org.neo4j.consistency.checking.full; -import org.apache.lucene.document.Document; import org.junit.Test; import java.io.File; @@ -27,7 +26,6 @@ import java.util.ArrayList; import java.util.Collections; import java.util.HashMap; -import java.util.Iterator; import java.util.List; import java.util.Map; @@ -291,18 +289,6 @@ public void verifyDeferredConstraints( Object accessor, int propertyKeyId, { } - @Override - public long getMaxDoc() - { - return 0; - } - - @Override - public Iterator getAllDocsIterator() - { - return null; - } - @Override public void close() { diff --git a/community/kernel/src/main/java/org/neo4j/kernel/api/index/DelegatingIndexReader.java b/community/kernel/src/main/java/org/neo4j/kernel/api/index/DelegatingIndexReader.java index 84f4eb87bbe7..7c621fff4ec4 100644 --- a/community/kernel/src/main/java/org/neo4j/kernel/api/index/DelegatingIndexReader.java +++ b/community/kernel/src/main/java/org/neo4j/kernel/api/index/DelegatingIndexReader.java @@ -19,7 +19,6 @@ */ package org.neo4j.kernel.api.index; -import java.util.Iterator; import java.util.List; import org.neo4j.collection.primitive.PrimitiveLongIterator; @@ -77,19 +76,7 @@ public IndexSampler createSampler() { return delegate.createSampler(); } - - @Override - public long getMaxDoc() - { - return delegate.getMaxDoc(); - } - - @Override - public Iterator getAllDocsIterator() - { - return delegate.getAllDocsIterator(); - } - + @Override public void verifyDeferredConstraints( Object accessor, int propertyKeyId ) throws Exception diff --git a/community/kernel/src/main/java/org/neo4j/storageengine/api/schema/IndexReader.java b/community/kernel/src/main/java/org/neo4j/storageengine/api/schema/IndexReader.java index 4a274a8f90a6..675575409c70 100644 --- a/community/kernel/src/main/java/org/neo4j/storageengine/api/schema/IndexReader.java +++ b/community/kernel/src/main/java/org/neo4j/storageengine/api/schema/IndexReader.java @@ -19,7 +19,6 @@ */ package org.neo4j.storageengine.api.schema; -import java.util.Iterator; import java.util.List; import org.neo4j.collection.primitive.PrimitiveLongCollections; @@ -87,10 +86,6 @@ public interface IndexReader extends Resource //TODO: - long getMaxDoc(); - - Iterator getAllDocsIterator(); - void verifyDeferredConstraints( Object accessor, int propertyKeyId ) throws Exception; void verifyDeferredConstraints( Object accessor, int propertyKeyId, List updatedPropertyValues ) @@ -141,19 +136,7 @@ public IndexSampler createSampler() { return IndexSampler.EMPTY; } - - @Override - public long getMaxDoc() - { - return 0; - } - - @Override - public Iterator getAllDocsIterator() - { - return null; - } - + @Override public void verifyDeferredConstraints( Object accessor, int propertyKeyId ) throws Exception diff --git a/community/kernel/src/main/java/org/neo4j/storageengine/api/schema/LabelScanReader.java b/community/kernel/src/main/java/org/neo4j/storageengine/api/schema/LabelScanReader.java index 33ab64a00994..14b48c9647b9 100644 --- a/community/kernel/src/main/java/org/neo4j/storageengine/api/schema/LabelScanReader.java +++ b/community/kernel/src/main/java/org/neo4j/storageengine/api/schema/LabelScanReader.java @@ -44,7 +44,5 @@ public interface LabelScanReader extends Resource AllEntriesLabelScanReader allNodeLabelRanges(); - Iterator getAllDocsIterator(); // todo: should not be here... - - long getMaxDoc(); // todo: should not be here... + void close(); } diff --git a/community/kernel/src/test/java/org/neo4j/kernel/impl/api/index/inmemory/HashBasedIndex.java b/community/kernel/src/test/java/org/neo4j/kernel/impl/api/index/inmemory/HashBasedIndex.java index 597c3dd15ff0..fc1bdd0963dd 100644 --- a/community/kernel/src/test/java/org/neo4j/kernel/impl/api/index/inmemory/HashBasedIndex.java +++ b/community/kernel/src/test/java/org/neo4j/kernel/impl/api/index/inmemory/HashBasedIndex.java @@ -19,8 +19,6 @@ */ package org.neo4j.kernel.impl.api.index.inmemory; -import org.apache.lucene.document.Document; - import java.util.Collection; import java.util.HashSet; import java.util.Iterator; @@ -264,16 +262,4 @@ public void verifyDeferredConstraints( Object accessor, int propertyKeyId, List< { // TODO: } - - @Override - public long getMaxDoc() - { - return 0; - } - - @Override - public Iterator getAllDocsIterator() - { - return null; - } } diff --git a/community/kernel/src/test/java/org/neo4j/kernel/impl/api/scan/InMemoryLabelScanStore.java b/community/kernel/src/test/java/org/neo4j/kernel/impl/api/scan/InMemoryLabelScanStore.java index 04fd9364f0c5..e318cdc0b536 100644 --- a/community/kernel/src/test/java/org/neo4j/kernel/impl/api/scan/InMemoryLabelScanStore.java +++ b/community/kernel/src/test/java/org/neo4j/kernel/impl/api/scan/InMemoryLabelScanStore.java @@ -19,8 +19,6 @@ */ package org.neo4j.kernel.impl.api.scan; -import org.apache.lucene.document.Document; - import java.io.File; import java.io.IOException; import java.util.ArrayList; @@ -104,12 +102,6 @@ public void close() { // Nothing to close } - @Override - public long getMaxDoc() - { - return 0; - } - @Override public Iterator labelsForNode( long nodeId ) { @@ -129,12 +121,6 @@ public AllEntriesLabelScanReader allNodeLabelRanges() { return newAllEntriesReader(); } - - @Override - public Iterator getAllDocsIterator() - { - return null; - } }; } diff --git a/community/lucene-index/src/main/java/org/neo4j/kernel/api/impl/index/AbstractLuceneIndex.java b/community/lucene-index/src/main/java/org/neo4j/kernel/api/impl/index/AbstractLuceneIndex.java index 75263f664bf4..33558170f6ac 100644 --- a/community/lucene-index/src/main/java/org/neo4j/kernel/api/impl/index/AbstractLuceneIndex.java +++ b/community/lucene-index/src/main/java/org/neo4j/kernel/api/impl/index/AbstractLuceneIndex.java @@ -26,6 +26,7 @@ import java.io.Closeable; import java.io.File; import java.io.IOException; +import java.io.UncheckedIOException; import java.util.ArrayList; import java.util.Collection; import java.util.List; @@ -40,8 +41,11 @@ import org.neo4j.helpers.collection.Iterables; import org.neo4j.io.IOUtils; import org.neo4j.kernel.api.impl.index.partition.IndexPartition; +import org.neo4j.kernel.api.impl.index.partition.PartitionSearcher; import org.neo4j.kernel.api.impl.index.storage.PartitionedIndexStorage; +import static java.util.stream.Collectors.toList; + // todo: this component has an implicit possibility to be opened and closed multiple times. we should revisit and at least test it. public abstract class AbstractLuceneIndex implements Closeable { @@ -157,6 +161,38 @@ public void close() throws IOException } } + public LuceneAllDocumentsReader allDocumentsReader() + { + ensureOpen(); + readWriteLock.lock(); + try + { + List searchers = new ArrayList<>( partitions.size() ); + try + { + for ( IndexPartition partition : partitions ) + { + searchers.add( partition.acquireSearcher() ); + } + + List partitionReaders = searchers.stream() + .map( LucenePartitionAllDocumentsReader::new ) + .collect( toList() ); + + return new LuceneAllDocumentsReader( partitionReaders ); + } + catch ( IOException e ) + { + IOUtils.closeAllSilently( searchers ); + throw new UncheckedIOException( e ); + } + } + finally + { + readWriteLock.unlock(); + } + } + public ResourceIterator snapshot() throws IOException { ensureOpen(); diff --git a/community/lucene-index/src/main/java/org/neo4j/kernel/api/impl/index/LabelScanStorageStrategy.java b/community/lucene-index/src/main/java/org/neo4j/kernel/api/impl/index/LabelScanStorageStrategy.java index b89364f4cdc8..b73d03415c42 100644 --- a/community/lucene-index/src/main/java/org/neo4j/kernel/api/impl/index/LabelScanStorageStrategy.java +++ b/community/lucene-index/src/main/java/org/neo4j/kernel/api/impl/index/LabelScanStorageStrategy.java @@ -25,14 +25,12 @@ import org.neo4j.collection.primitive.PrimitiveLongIterator; import org.neo4j.kernel.api.direct.AllEntriesLabelScanReader; -import org.neo4j.storageengine.api.schema.LabelScanReader; - public interface LabelScanStorageStrategy { PrimitiveLongIterator nodesWithLabel( IndexSearcher searcher, int labelId ); - AllEntriesLabelScanReader newNodeLabelReader( LabelScanReader reader ); + AllEntriesLabelScanReader newNodeLabelReader( LuceneAllDocumentsReader allDocumentsReader ); Iterator labelsForNode( IndexSearcher searcher, long nodeId ); } diff --git a/community/lucene-index/src/main/java/org/neo4j/kernel/api/impl/index/LuceneAllDocumentsReader.java b/community/lucene-index/src/main/java/org/neo4j/kernel/api/impl/index/LuceneAllDocumentsReader.java index 93ec2d102aa3..31248a9f4a53 100644 --- a/community/lucene-index/src/main/java/org/neo4j/kernel/api/impl/index/LuceneAllDocumentsReader.java +++ b/community/lucene-index/src/main/java/org/neo4j/kernel/api/impl/index/LuceneAllDocumentsReader.java @@ -21,43 +21,45 @@ import org.apache.lucene.document.Document; +import java.io.IOException; import java.util.Iterator; +import java.util.List; +import org.neo4j.helpers.collection.Iterables; import org.neo4j.io.IOUtils; import org.neo4j.kernel.api.direct.BoundedIterable; -import org.neo4j.storageengine.api.schema.IndexReader; -import org.neo4j.storageengine.api.schema.LabelScanReader; + +import static java.util.stream.Collectors.toList; public class LuceneAllDocumentsReader implements BoundedIterable { - private LabelScanReader labelScanReader; - private IndexReader indexReader; + private final List partitionReaders; - public LuceneAllDocumentsReader( IndexReader indexReader ) + public LuceneAllDocumentsReader( List partitionReaders ) { - this.indexReader = indexReader; - } - - public LuceneAllDocumentsReader( LabelScanReader labelScanReader ) - { - this.labelScanReader = labelScanReader; + this.partitionReaders = partitionReaders; } @Override public long maxCount() { - return labelScanReader != null ? labelScanReader.getMaxDoc() : indexReader.getMaxDoc(); + return partitionReaders.stream().mapToLong( LucenePartitionAllDocumentsReader::maxCount ).sum(); } @Override public Iterator iterator() { - return labelScanReader != null ? labelScanReader.getAllDocsIterator() : indexReader.getAllDocsIterator(); + Iterator> iterators = partitionReaders.stream() + .map( LucenePartitionAllDocumentsReader::iterator ) + .collect( toList() ) + .iterator(); + + return Iterables.concat( iterators ); } @Override - public void close() + public void close() throws IOException { - IOUtils.closeAllSilently( labelScanReader, labelScanReader ); + IOUtils.closeAll( partitionReaders ); } } diff --git a/community/lucene-index/src/main/java/org/neo4j/kernel/api/impl/index/LuceneIndexAccessor.java b/community/lucene-index/src/main/java/org/neo4j/kernel/api/impl/index/LuceneIndexAccessor.java index d0ad0fcac1a7..a99e97e4076f 100644 --- a/community/lucene-index/src/main/java/org/neo4j/kernel/api/impl/index/LuceneIndexAccessor.java +++ b/community/lucene-index/src/main/java/org/neo4j/kernel/api/impl/index/LuceneIndexAccessor.java @@ -105,9 +105,7 @@ public BoundedIterable newAllEntriesReader() { try { - // TODO: - LuceneAllDocumentsReader allDocumentsReader = new LuceneAllDocumentsReader( luceneIndex.getIndexReader() ); - return new LuceneAllEntriesIndexAccessorReader( allDocumentsReader ); + return new LuceneAllEntriesIndexAccessorReader( luceneIndex.allDocumentsReader() ); } catch ( Exception e ) { diff --git a/community/lucene-index/src/main/java/org/neo4j/kernel/api/impl/index/LuceneLabelScanIndex.java b/community/lucene-index/src/main/java/org/neo4j/kernel/api/impl/index/LuceneLabelScanIndex.java index 393eebec4ff7..c8fbc78f7655 100644 --- a/community/lucene-index/src/main/java/org/neo4j/kernel/api/impl/index/LuceneLabelScanIndex.java +++ b/community/lucene-index/src/main/java/org/neo4j/kernel/api/impl/index/LuceneLabelScanIndex.java @@ -58,7 +58,7 @@ public LabelScanReader getLabelScanReader() { IndexPartition partition = partitions.get( 0 ); PartitionSearcher searcher = partition.acquireSearcher(); - return new SimpleLuceneLabelScanStoreReader( searcher, storageStrategy ); + return new SimpleLuceneLabelScanStoreReader( this, searcher, storageStrategy ); } throw new UnsupportedOperationException(); } diff --git a/community/lucene-index/src/main/java/org/neo4j/kernel/api/impl/index/LucenePartitionAllDocumentsReader.java b/community/lucene-index/src/main/java/org/neo4j/kernel/api/impl/index/LucenePartitionAllDocumentsReader.java new file mode 100644 index 000000000000..d789ff4fdb98 --- /dev/null +++ b/community/lucene-index/src/main/java/org/neo4j/kernel/api/impl/index/LucenePartitionAllDocumentsReader.java @@ -0,0 +1,119 @@ +/* + * Copyright (c) 2002-2015 "Neo Technology," + * Network Engine for Objects in Lund AB [http://neotechnology.com] + * + * This file is part of Neo4j. + * + * Neo4j is free software: you can redistribute it and/or modify + * it under the terms of the GNU General Public License as published by + * the Free Software Foundation, either version 3 of the License, or + * (at your option) any later version. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU General Public License for more details. + * + * You should have received a copy of the GNU General Public License + * along with this program. If not, see . + */ +package org.neo4j.kernel.api.impl.index; + +import org.apache.lucene.document.Document; +import org.apache.lucene.index.IndexReader; +import org.apache.lucene.index.MultiFields; +import org.apache.lucene.search.DocIdSetIterator; +import org.apache.lucene.search.FilteredDocIdSetIterator; +import org.apache.lucene.search.IndexSearcher; +import org.apache.lucene.util.Bits; + +import java.io.IOException; +import java.util.Iterator; + +import org.neo4j.helpers.collection.PrefetchingIterator; +import org.neo4j.kernel.api.direct.BoundedIterable; +import org.neo4j.kernel.api.impl.index.partition.PartitionSearcher; + +public class LucenePartitionAllDocumentsReader implements BoundedIterable +{ + private final PartitionSearcher partitionSearcher; + private final IndexSearcher searcher; + private final IndexReader reader; + + public LucenePartitionAllDocumentsReader( PartitionSearcher partitionSearcher ) + { + this.partitionSearcher = partitionSearcher; + this.searcher = partitionSearcher.getIndexSearcher(); + this.reader = searcher.getIndexReader(); + } + + @Override + public long maxCount() + { + return reader.maxDoc(); + } + + @Override + public Iterator iterator() + { + return new PrefetchingIterator() + { + final DocIdSetIterator idIterator = iterateAllDocs(); + + @Override + protected Document fetchNextOrNull() + { + try + { + int doc = idIterator.nextDoc(); + if ( doc == DocIdSetIterator.NO_MORE_DOCS ) + { + return null; + } + return getDocument( doc ); + } + catch ( IOException e ) + { + throw new LuceneDocumentRetrievalException( "Can't fetch document id from lucene index.", e ); + } + } + }; + } + + @Override + public void close() throws IOException + { + partitionSearcher.close(); + } + + private Document getDocument( int docId ) + { + try + { + return searcher.doc( docId ); + } + catch ( IOException e ) + { + throw new LuceneDocumentRetrievalException( "Can't retrieve document with id: " + docId + ".", docId, e ); + } + } + + private DocIdSetIterator iterateAllDocs() + { + Bits liveDocs = MultiFields.getLiveDocs( reader ); + DocIdSetIterator allDocs = DocIdSetIterator.all( reader.maxDoc() ); + if ( liveDocs == null ) + { + return allDocs; + } + + return new FilteredDocIdSetIterator( allDocs ) + { + @Override + protected boolean match( int doc ) + { + return liveDocs.get( doc ); + } + }; + } +} diff --git a/community/lucene-index/src/main/java/org/neo4j/kernel/api/impl/index/NodeRangeDocumentLabelScanStorageStrategy.java b/community/lucene-index/src/main/java/org/neo4j/kernel/api/impl/index/NodeRangeDocumentLabelScanStorageStrategy.java index 3f8839f0ea40..ddac0d544086 100644 --- a/community/lucene-index/src/main/java/org/neo4j/kernel/api/impl/index/NodeRangeDocumentLabelScanStorageStrategy.java +++ b/community/lucene-index/src/main/java/org/neo4j/kernel/api/impl/index/NodeRangeDocumentLabelScanStorageStrategy.java @@ -31,7 +31,11 @@ import org.neo4j.collection.primitive.PrimitiveLongIterator; import org.neo4j.kernel.api.direct.AllEntriesLabelScanReader; import org.neo4j.kernel.api.impl.index.bitmaps.BitmapFormat; +<<<<<<< b169a7e93e2e6863cdd95e5dd138f05f30892e02 import org.neo4j.storageengine.api.schema.LabelScanReader; +======= +import org.neo4j.storageengine.api.schema.AllEntriesLabelScanReader; +>>>>>>> Reading all documents moved inside lucene index import static org.neo4j.collection.primitive.PrimitiveLongCollections.concat; import static org.neo4j.helpers.collection.IteratorUtil.emptyIterator; @@ -83,10 +87,9 @@ public PrimitiveLongIterator nodesWithLabel( IndexSearcher searcher, int labelId } @Override - public AllEntriesLabelScanReader newNodeLabelReader( LabelScanReader reader ) + public AllEntriesLabelScanReader newNodeLabelReader( LuceneAllDocumentsReader allDocumentsReader ) { - LuceneAllDocumentsReader documents = new LuceneAllDocumentsReader( reader ); - return new LuceneAllEntriesLabelScanReader( documents, format ); + return new LuceneAllEntriesLabelScanReader( allDocumentsReader, format ); } @Override diff --git a/community/lucene-index/src/main/java/org/neo4j/kernel/api/impl/index/SimpleLuceneLabelScanStoreReader.java b/community/lucene-index/src/main/java/org/neo4j/kernel/api/impl/index/SimpleLuceneLabelScanStoreReader.java index ddf875f39985..1b7ebeb3fdbb 100644 --- a/community/lucene-index/src/main/java/org/neo4j/kernel/api/impl/index/SimpleLuceneLabelScanStoreReader.java +++ b/community/lucene-index/src/main/java/org/neo4j/kernel/api/impl/index/SimpleLuceneLabelScanStoreReader.java @@ -19,17 +19,10 @@ */ package org.neo4j.kernel.api.impl.index; -import org.apache.lucene.document.Document; -import org.apache.lucene.index.MultiFields; -import org.apache.lucene.search.DocIdSetIterator; -import org.apache.lucene.search.FilteredDocIdSetIterator; -import org.apache.lucene.util.Bits; - import java.io.IOException; import java.util.Iterator; import org.neo4j.collection.primitive.PrimitiveLongIterator; -import org.neo4j.helpers.collection.PrefetchingIterator; import org.neo4j.kernel.api.impl.index.partition.PartitionSearcher; import org.neo4j.kernel.api.impl.index.reader.IndexReaderCloseException; import org.neo4j.storageengine.api.schema.AllEntriesLabelScanReader; @@ -37,12 +30,14 @@ public class SimpleLuceneLabelScanStoreReader implements LabelScanReader { + private final LuceneLabelScanIndex index; private final PartitionSearcher partitionSearcher; private final LabelScanStorageStrategy storageStrategy; - public SimpleLuceneLabelScanStoreReader( PartitionSearcher partitionSearcher, + public SimpleLuceneLabelScanStoreReader( LuceneLabelScanIndex index, PartitionSearcher partitionSearcher, LabelScanStorageStrategy storageStrategy ) { + this.index = index; this.partitionSearcher = partitionSearcher; this.storageStrategy = storageStrategy; } @@ -62,66 +57,7 @@ public Iterator labelsForNode( long nodeId ) @Override public AllEntriesLabelScanReader allNodeLabelRanges() { - return storageStrategy.newNodeLabelReader( this ); - } - - @Override - public Iterator getAllDocsIterator() - { - return new PrefetchingIterator() - { - private DocIdSetIterator idIterator = iterateAllDocs(); - - @Override - protected Document fetchNextOrNull() - { - try - { - int doc = idIterator.nextDoc(); - if ( doc == DocIdSetIterator.NO_MORE_DOCS ) - { - return null; - } - return getDocument( doc ); - } - catch ( IOException e ) - { - throw new LuceneDocumentRetrievalException( "Can't fetch document id from lucene index.", e ); - } - } - }; - } - - private Document getDocument( int docId ) - { - try - { - return partitionSearcher.getIndexSearcher().doc( docId ); - } - catch ( IOException e ) - { - throw new LuceneDocumentRetrievalException( "Can't retrieve document with id: " + docId + ".", docId, e ); - } - } - - private DocIdSetIterator iterateAllDocs() - { - org.apache.lucene.index.IndexReader reader = partitionSearcher.getIndexSearcher().getIndexReader(); - final Bits liveDocs = MultiFields.getLiveDocs( reader ); - final DocIdSetIterator allDocs = DocIdSetIterator.all( reader.maxDoc() ); - if ( liveDocs == null ) - { - return allDocs; - } - - return new FilteredDocIdSetIterator( allDocs ) - { - @Override - protected boolean match( int doc ) - { - return liveDocs.get( doc ); - } - }; + return storageStrategy.newNodeLabelReader( index.allDocumentsReader() ); } @Override @@ -136,10 +72,4 @@ public void close() throw new IndexReaderCloseException( e ); } } - - @Override - public long getMaxDoc() - { - return partitionSearcher.getIndexSearcher().getIndexReader().maxDoc(); - } } diff --git a/community/lucene-index/src/main/java/org/neo4j/kernel/api/impl/index/reader/PartitionedIndexReader.java b/community/lucene-index/src/main/java/org/neo4j/kernel/api/impl/index/reader/PartitionedIndexReader.java index 9aa5d1307efe..c20fad86ecde 100644 --- a/community/lucene-index/src/main/java/org/neo4j/kernel/api/impl/index/reader/PartitionedIndexReader.java +++ b/community/lucene-index/src/main/java/org/neo4j/kernel/api/impl/index/reader/PartitionedIndexReader.java @@ -19,11 +19,9 @@ */ package org.neo4j.kernel.api.impl.index.reader; -import org.apache.lucene.document.Document; import org.apache.lucene.search.IndexSearcher; import java.io.IOException; -import java.util.Iterator; import java.util.List; import org.neo4j.collection.primitive.PrimitiveLongIterator; @@ -101,18 +99,6 @@ public void verifyDeferredConstraints( Object accessor, int propertyKeyId, } - @Override - public long getMaxDoc() - { - return 0; - } - - @Override - public Iterator getAllDocsIterator() - { - return null; - } - @Override public void close() { diff --git a/community/lucene-index/src/main/java/org/neo4j/kernel/api/impl/index/reader/SimpleIndexReader.java b/community/lucene-index/src/main/java/org/neo4j/kernel/api/impl/index/reader/SimpleIndexReader.java index 6e387df3f2fd..a10f228e066d 100644 --- a/community/lucene-index/src/main/java/org/neo4j/kernel/api/impl/index/reader/SimpleIndexReader.java +++ b/community/lucene-index/src/main/java/org/neo4j/kernel/api/impl/index/reader/SimpleIndexReader.java @@ -19,34 +19,26 @@ */ package org.neo4j.kernel.api.impl.index.reader; -import org.apache.lucene.document.Document; import org.apache.lucene.index.Fields; import org.apache.lucene.index.LeafReaderContext; -import org.apache.lucene.index.MultiFields; import org.apache.lucene.index.Term; import org.apache.lucene.index.TermsEnum; import org.apache.lucene.search.BooleanClause; import org.apache.lucene.search.BooleanQuery; -import org.apache.lucene.search.DocIdSetIterator; -import org.apache.lucene.search.FilteredDocIdSetIterator; import org.apache.lucene.search.IndexSearcher; import org.apache.lucene.search.Query; import org.apache.lucene.search.TermQuery; import org.apache.lucene.search.TotalHitCountCollector; -import org.apache.lucene.util.Bits; import org.apache.lucene.util.BytesRef; import java.io.IOException; -import java.util.Iterator; import java.util.List; import org.neo4j.collection.primitive.PrimitiveLongIterator; import org.neo4j.helpers.TaskControl; import org.neo4j.helpers.TaskCoordinator; -import org.neo4j.helpers.collection.PrefetchingIterator; import org.neo4j.kernel.api.exceptions.index.IndexEntryConflictException; import org.neo4j.kernel.api.impl.index.DocValuesCollector; -import org.neo4j.kernel.api.impl.index.LuceneDocumentRetrievalException; import org.neo4j.kernel.api.impl.index.LuceneDocumentStructure; import org.neo4j.kernel.api.impl.index.partition.PartitionSearcher; import org.neo4j.kernel.api.impl.index.sampler.NonUniqueLuceneIndexSampler; @@ -163,71 +155,6 @@ public void verifyDeferredConstraints( Object accessor, int propertyKeyId, } } - @Override - public long getMaxDoc() - { - return getIndexSearcher().getIndexReader().maxDoc(); - } - - @Override - public Iterator getAllDocsIterator() - { - return new PrefetchingIterator() - { - private DocIdSetIterator idIterator = iterateAllDocs(); - - @Override - protected Document fetchNextOrNull() - { - try - { - int doc = idIterator.nextDoc(); - if ( doc == DocIdSetIterator.NO_MORE_DOCS ) - { - return null; - } - return getDocument( doc ); - } - catch ( IOException e ) - { - throw new LuceneDocumentRetrievalException( "Can't fetch document id from lucene index.", e ); - } - } - }; - } - - private Document getDocument( int docId ) - { - try - { - return getIndexSearcher().doc( docId ); - } - catch ( IOException e ) - { - throw new LuceneDocumentRetrievalException("Can't retrieve document with id: " + docId + ".", docId, e ); - } - } - - private DocIdSetIterator iterateAllDocs() - { - org.apache.lucene.index.IndexReader reader = getIndexSearcher().getIndexReader(); - final Bits liveDocs = MultiFields.getLiveDocs( reader ); - final DocIdSetIterator allDocs = DocIdSetIterator.all( reader.maxDoc() ); - if ( liveDocs == null ) - { - return allDocs; - } - - return new FilteredDocIdSetIterator( allDocs ) - { - @Override - protected boolean match( int doc ) - { - return liveDocs.get( doc ); - } - }; - } - @Override public PrimitiveLongIterator seek( Object value ) {