Reading all documents moved inside lucene index

Consistency checked needs to read all lucene documents from schema indexes and label scan store to verify their correctness. Temporarily logic of reading lucene documents and presenting them as node ids or node label ranges lived in IndexReader and LabelScanReader. This commit moves it to the AbstractLuceneIndex. Index is now capable of giving away an iterator over lucene documents. Callers of this method then interpret documents as node ids or node label ranges.
neo4j · Jan 21, 2016 · b94a609 · b94a609
1 parent b1c25dc
commit b94a609
Show file tree

Hide file tree

Showing 17 changed files with 188 additions and 277 deletions.
diff --git a/...c/test/java/org/neo4j/legacy/consistency/checking/full/NodeCorrectlyIndexedCheckTest.java b/...c/test/java/org/neo4j/legacy/consistency/checking/full/NodeCorrectlyIndexedCheckTest.java
@@ -19,15 +19,13 @@
  */
 package org.neo4j.legacy.consistency.checking.full;
 
-import org.apache.lucene.document.Document;
 import org.junit.Test;
 
 import java.io.File;
 import java.io.IOException;
 import java.util.ArrayList;
 import java.util.Collections;
 import java.util.HashMap;
-import java.util.Iterator;
 import java.util.List;
 import java.util.Map;
 
@@ -291,18 +289,6 @@ public void verifyDeferredConstraints( Object accessor, int propertyKeyId,
                 {
                 }
 
-                @Override
-                public long getMaxDoc()
-                {
-                    return 0;
-                }
-
-                @Override
-                public Iterator<Document> getAllDocsIterator()
-                {
-                    return null;
-                }
-
                 @Override
                 public void close()
                 {

diff --git a/...heck/src/test/java/org/neo4j/consistency/checking/full/NodeCorrectlyIndexedCheckTest.java b/...heck/src/test/java/org/neo4j/consistency/checking/full/NodeCorrectlyIndexedCheckTest.java
@@ -19,15 +19,13 @@
  */
 package org.neo4j.consistency.checking.full;
 
-import org.apache.lucene.document.Document;
 import org.junit.Test;
 
 import java.io.File;
 import java.io.IOException;
 import java.util.ArrayList;
 import java.util.Collections;
 import java.util.HashMap;
-import java.util.Iterator;
 import java.util.List;
 import java.util.Map;
 
@@ -291,18 +289,6 @@ public void verifyDeferredConstraints( Object accessor, int propertyKeyId,
                 {
                 }
 
-                @Override
-                public long getMaxDoc()
-                {
-                    return 0;
-                }
-
-                @Override
-                public Iterator<Document> getAllDocsIterator()
-                {
-                    return null;
-                }
-
                 @Override
                 public void close()
                 {

diff --git a/community/kernel/src/main/java/org/neo4j/kernel/api/index/DelegatingIndexReader.java b/community/kernel/src/main/java/org/neo4j/kernel/api/index/DelegatingIndexReader.java
@@ -19,7 +19,6 @@
  */
 package org.neo4j.kernel.api.index;
 
-import java.util.Iterator;
 import java.util.List;
 
 import org.neo4j.collection.primitive.PrimitiveLongIterator;
@@ -77,19 +76,7 @@ public IndexSampler createSampler()
     {
         return delegate.createSampler();
     }
-
-    @Override
-    public long getMaxDoc()
-    {
-        return delegate.getMaxDoc();
-    }
-
-    @Override
-    public Iterator getAllDocsIterator()
-    {
-        return delegate.getAllDocsIterator();
-    }
-
+
     @Override
     public void verifyDeferredConstraints( Object accessor, int propertyKeyId )
             throws Exception

diff --git a/community/kernel/src/main/java/org/neo4j/storageengine/api/schema/IndexReader.java b/community/kernel/src/main/java/org/neo4j/storageengine/api/schema/IndexReader.java
@@ -19,7 +19,6 @@
  */
 package org.neo4j.storageengine.api.schema;
 
-import java.util.Iterator;
 import java.util.List;
 
 import org.neo4j.collection.primitive.PrimitiveLongCollections;
@@ -87,10 +86,6 @@ public interface IndexReader extends Resource
 
     //TODO:
 
-    long getMaxDoc();
-
-    Iterator getAllDocsIterator();
-
     void verifyDeferredConstraints( Object accessor, int propertyKeyId ) throws Exception;
 
     void verifyDeferredConstraints( Object accessor, int propertyKeyId, List<Object> updatedPropertyValues )
@@ -141,19 +136,7 @@ public IndexSampler createSampler()
         {
             return IndexSampler.EMPTY;
         }
-
-        @Override
-        public long getMaxDoc()
-        {
-            return 0;
-        }
-
-        @Override
-        public Iterator getAllDocsIterator()
-        {
-            return null;
-        }
-
+
         @Override
         public void verifyDeferredConstraints( Object accessor, int propertyKeyId )
                 throws Exception

diff --git a/community/kernel/src/main/java/org/neo4j/storageengine/api/schema/LabelScanReader.java b/community/kernel/src/main/java/org/neo4j/storageengine/api/schema/LabelScanReader.java
@@ -44,7 +44,5 @@ public interface LabelScanReader extends Resource
 
     AllEntriesLabelScanReader allNodeLabelRanges();
 
-    Iterator getAllDocsIterator(); // todo: should not be here...
-
-    long getMaxDoc(); // todo: should not be here...
+    void close();
 }
diff --git a/community/kernel/src/test/java/org/neo4j/kernel/impl/api/index/inmemory/HashBasedIndex.java b/community/kernel/src/test/java/org/neo4j/kernel/impl/api/index/inmemory/HashBasedIndex.java
@@ -19,8 +19,6 @@
  */
 package org.neo4j.kernel.impl.api.index.inmemory;
 
-import org.apache.lucene.document.Document;
-
 import java.util.Collection;
 import java.util.HashSet;
 import java.util.Iterator;
@@ -264,16 +262,4 @@ public void verifyDeferredConstraints( Object accessor, int propertyKeyId, List<
     {
         // TODO:
     }
-
-    @Override
-    public long getMaxDoc()
-    {
-        return 0;
-    }
-
-    @Override
-    public Iterator<Document> getAllDocsIterator()
-    {
-        return null;
-    }
 }
diff --git a/community/kernel/src/test/java/org/neo4j/kernel/impl/api/scan/InMemoryLabelScanStore.java b/community/kernel/src/test/java/org/neo4j/kernel/impl/api/scan/InMemoryLabelScanStore.java
@@ -19,8 +19,6 @@
  */
 package org.neo4j.kernel.impl.api.scan;
 
-import org.apache.lucene.document.Document;
-
 import java.io.File;
 import java.io.IOException;
 import java.util.ArrayList;
@@ -104,12 +102,6 @@ public void close()
             {   // Nothing to close
             }
 
-            @Override
-            public long getMaxDoc()
-            {
-                return 0;
-            }
-
             @Override
             public Iterator<Long> labelsForNode( long nodeId )
             {
@@ -129,12 +121,6 @@ public AllEntriesLabelScanReader allNodeLabelRanges()
             {
                 return newAllEntriesReader();
             }
-
-            @Override
-            public Iterator<Document> getAllDocsIterator()
-            {
-                return null;
-            }
         };
     }
 

diff --git a/...unity/lucene-index/src/main/java/org/neo4j/kernel/api/impl/index/AbstractLuceneIndex.java b/...unity/lucene-index/src/main/java/org/neo4j/kernel/api/impl/index/AbstractLuceneIndex.java
@@ -26,6 +26,7 @@
 import java.io.Closeable;
 import java.io.File;
 import java.io.IOException;
+import java.io.UncheckedIOException;
 import java.util.ArrayList;
 import java.util.Collection;
 import java.util.List;
@@ -40,8 +41,11 @@
 import org.neo4j.helpers.collection.Iterables;
 import org.neo4j.io.IOUtils;
 import org.neo4j.kernel.api.impl.index.partition.IndexPartition;
+import org.neo4j.kernel.api.impl.index.partition.PartitionSearcher;
 import org.neo4j.kernel.api.impl.index.storage.PartitionedIndexStorage;
 
+import static java.util.stream.Collectors.toList;
+
 // todo: this component has an implicit possibility to be opened and closed multiple times. we should revisit and at least test it.
 public abstract class AbstractLuceneIndex implements Closeable
 {
@@ -157,6 +161,38 @@ public void close() throws IOException
         }
     }
 
+    public LuceneAllDocumentsReader allDocumentsReader()
+    {
+        ensureOpen();
+        readWriteLock.lock();
+        try
+        {
+            List<PartitionSearcher> searchers = new ArrayList<>( partitions.size() );
+            try
+            {
+                for ( IndexPartition partition : partitions )
+                {
+                    searchers.add( partition.acquireSearcher() );
+                }
+
+                List<LucenePartitionAllDocumentsReader> partitionReaders = searchers.stream()
+                        .map( LucenePartitionAllDocumentsReader::new )
+                        .collect( toList() );
+
+                return new LuceneAllDocumentsReader( partitionReaders );
+            }
+            catch ( IOException e )
+            {
+                IOUtils.closeAllSilently( searchers );
+                throw new UncheckedIOException( e );
+            }
+        }
+        finally
+        {
+            readWriteLock.unlock();
+        }
+    }
+
     public ResourceIterator<File> snapshot() throws IOException
     {
         ensureOpen();

diff --git a/.../lucene-index/src/main/java/org/neo4j/kernel/api/impl/index/LabelScanStorageStrategy.java b/.../lucene-index/src/main/java/org/neo4j/kernel/api/impl/index/LabelScanStorageStrategy.java
@@ -25,14 +25,12 @@
 
 import org.neo4j.collection.primitive.PrimitiveLongIterator;
 import org.neo4j.kernel.api.direct.AllEntriesLabelScanReader;
-import org.neo4j.storageengine.api.schema.LabelScanReader;
-
 
 public interface LabelScanStorageStrategy
 {
     PrimitiveLongIterator nodesWithLabel( IndexSearcher searcher, int labelId );
 
-    AllEntriesLabelScanReader newNodeLabelReader( LabelScanReader reader );
+    AllEntriesLabelScanReader newNodeLabelReader( LuceneAllDocumentsReader allDocumentsReader );
 
     Iterator<Long> labelsForNode( IndexSearcher searcher, long nodeId );
 }
diff --git a/.../lucene-index/src/main/java/org/neo4j/kernel/api/impl/index/LuceneAllDocumentsReader.java b/.../lucene-index/src/main/java/org/neo4j/kernel/api/impl/index/LuceneAllDocumentsReader.java
@@ -21,43 +21,45 @@
 
 import org.apache.lucene.document.Document;
 
+import java.io.IOException;
 import java.util.Iterator;
+import java.util.List;
 
+import org.neo4j.helpers.collection.Iterables;
 import org.neo4j.io.IOUtils;
 import org.neo4j.kernel.api.direct.BoundedIterable;
-import org.neo4j.storageengine.api.schema.IndexReader;
-import org.neo4j.storageengine.api.schema.LabelScanReader;
+
+import static java.util.stream.Collectors.toList;
 
 public class LuceneAllDocumentsReader implements BoundedIterable<Document>
 {
-    private LabelScanReader labelScanReader;
-    private IndexReader indexReader;
+    private final List<LucenePartitionAllDocumentsReader> partitionReaders;
 
-    public LuceneAllDocumentsReader( IndexReader indexReader )
+    public LuceneAllDocumentsReader( List<LucenePartitionAllDocumentsReader> partitionReaders )
     {
-        this.indexReader = indexReader;
-    }
-
-    public LuceneAllDocumentsReader( LabelScanReader labelScanReader )
-    {
-        this.labelScanReader = labelScanReader;
+        this.partitionReaders = partitionReaders;
     }
 
     @Override
     public long maxCount()
     {
-        return labelScanReader != null ? labelScanReader.getMaxDoc() : indexReader.getMaxDoc();
+        return partitionReaders.stream().mapToLong( LucenePartitionAllDocumentsReader::maxCount ).sum();
     }
 
     @Override
     public Iterator<Document> iterator()
     {
-        return labelScanReader != null ? labelScanReader.getAllDocsIterator() : indexReader.getAllDocsIterator();
+        Iterator<Iterator<Document>> iterators = partitionReaders.stream()
+                .map( LucenePartitionAllDocumentsReader::iterator )
+                .collect( toList() )
+                .iterator();
+
+        return Iterables.concat( iterators );
     }
 
     @Override
-    public void close()
+    public void close() throws IOException
     {
-        IOUtils.closeAllSilently( labelScanReader, labelScanReader );
+        IOUtils.closeAll( partitionReaders );
     }
 }
diff --git a/...unity/lucene-index/src/main/java/org/neo4j/kernel/api/impl/index/LuceneIndexAccessor.java b/...unity/lucene-index/src/main/java/org/neo4j/kernel/api/impl/index/LuceneIndexAccessor.java
@@ -105,9 +105,7 @@ public BoundedIterable<Long> newAllEntriesReader()
     {
         try
         {
-            // TODO:
-            LuceneAllDocumentsReader allDocumentsReader = new LuceneAllDocumentsReader( luceneIndex.getIndexReader() );
-            return new LuceneAllEntriesIndexAccessorReader( allDocumentsReader );
+            return new LuceneAllEntriesIndexAccessorReader( luceneIndex.allDocumentsReader() );
         }
         catch ( Exception e )
         {

diff --git a/...nity/lucene-index/src/main/java/org/neo4j/kernel/api/impl/index/LuceneLabelScanIndex.java b/...nity/lucene-index/src/main/java/org/neo4j/kernel/api/impl/index/LuceneLabelScanIndex.java
@@ -58,7 +58,7 @@ public LabelScanReader getLabelScanReader()
             {
                 IndexPartition partition = partitions.get( 0 );
                 PartitionSearcher searcher = partition.acquireSearcher();
-                return new SimpleLuceneLabelScanStoreReader( searcher, storageStrategy );
+                return new SimpleLuceneLabelScanStoreReader( this, searcher, storageStrategy );
             }
             throw new UnsupportedOperationException();
         }