Finish old tests cleanup and restoration. All old tests restored.

Rewrite and cleanup couple of old tests that where still commented out, minor code cleanups.

community/lucene-index/src/main/java/org/neo4j/kernel/api/impl/index/PartitionedLuceneLabelScanWriter.java

@@ -45,6 +45,7 @@
 public class PartitionedLuceneLabelScanWriter implements LabelScanWriter
 {
 
+    // TODO: Integer.MAX_VALUE usually used as sentinel marker in lucene, test can we use it as max partition size ?
     private final Integer MAXIMUM_PARTITION_SIZE =
             Integer.getInteger( "labelScanStore.maxPartitionSize", Integer.MAX_VALUE );
 

community/lucene-index/src/main/java/org/neo4j/kernel/api/impl/index/builder/LuceneIndexStorageBuilder.java

@@ -27,7 +27,7 @@
 import org.neo4j.kernel.api.impl.index.storage.DirectoryFactory;
 import org.neo4j.kernel.api.impl.index.storage.PartitionedIndexStorage;
 
-class LuceneIndexStorageBuilder
+public class LuceneIndexStorageBuilder
 {
     private DirectoryFactory directoryFactory = DirectoryFactory.PERSISTENT;
     private FileSystemAbstraction fileSystem = new DefaultFileSystemAbstraction();
@@ -39,12 +39,12 @@ private LuceneIndexStorageBuilder()
     {
     }
 
-    static LuceneIndexStorageBuilder create()
+    public static LuceneIndexStorageBuilder create()
     {
         return new LuceneIndexStorageBuilder();
     }
 
-    PartitionedIndexStorage buildIndexStorage()
+    public PartitionedIndexStorage build()
     {
         if ( indexStorage == null )
         {
@@ -58,31 +58,31 @@ PartitionedIndexStorage buildIndexStorage()
         return indexStorage;
     }
 
-    LuceneIndexStorageBuilder withIndexIdentifier( String indexIdentifier )
+    public LuceneIndexStorageBuilder withIndexIdentifier( String indexIdentifier )
     {
         this.indexIdentifier = indexIdentifier;
         return this;
     }
 
-    LuceneIndexStorageBuilder withDirectoryFactory( DirectoryFactory directoryFactory )
+    public LuceneIndexStorageBuilder withDirectoryFactory( DirectoryFactory directoryFactory )
     {
         this.directoryFactory = directoryFactory;
         return this;
     }
 
-    LuceneIndexStorageBuilder withFileSystem( FileSystemAbstraction fileSystem )
+    public LuceneIndexStorageBuilder withFileSystem( FileSystemAbstraction fileSystem )
     {
         this.fileSystem = fileSystem;
         return this;
     }
 
-    LuceneIndexStorageBuilder withIndexRootFolder( File indexRootFolder )
+    public LuceneIndexStorageBuilder withIndexRootFolder( File indexRootFolder )
     {
         this.indexRootFolder = indexRootFolder;
         return this;
     }
 
-    LuceneIndexStorageBuilder withIndexStorage( PartitionedIndexStorage indexStorage )
+    public LuceneIndexStorageBuilder withIndexStorage( PartitionedIndexStorage indexStorage )
     {
         this.indexStorage = indexStorage;
         return this;

community/lucene-index/src/main/java/org/neo4j/kernel/api/impl/index/builder/LuceneLabelScanIndexBuilder.java

@@ -47,6 +47,6 @@ public LuceneLabelScanIndexBuilder withDocumentFormat( BitmapDocumentFormat form
 
     public LuceneLabelScanIndex build()
     {
-        return new LuceneLabelScanIndex( format, storageBuilder.buildIndexStorage() );
+        return new LuceneLabelScanIndex( format, storageBuilder.build() );
     }
 }

community/lucene-index/src/main/java/org/neo4j/kernel/api/impl/index/builder/LuceneSchemaIndexBuilder.java

@@ -77,7 +77,7 @@ public LuceneSchemaIndexBuilder uniqueIndex()
 
     public LuceneSchemaIndex build()
     {
-        return new LuceneSchemaIndex( storageBuilder.buildIndexStorage(), indexConfig, samplingConfig );
+        return new LuceneSchemaIndex( storageBuilder.build(), indexConfig, samplingConfig );
     }
 
 }

community/lucene-index/src/test/java/org/neo4j/graphdb/LuceneLabelScanStoreChaosIT.java

@@ -64,8 +64,6 @@ public void shouldRebuildDeletedLabelScanStoreOnStartup() throws Exception
         deleteNode( node2 ); // just to create a hole in the store
 
         // WHEN
-        // TODO how do we make sure it was deleted and then fully rebuilt? I mean if we somehow deleted
-        // the wrong directory here then it would also work, right?
         dbRule.restartDatabase( deleteTheLabelScanStoreIndex() );
 
         // THEN

community/lucene-index/src/test/java/org/neo4j/index/IndexConstraintsTest.java

@@ -55,13 +55,13 @@ public class IndexConstraintsTest
     @Before
     public void setup() throws IOException
     {
-        this.graphDb = new TestGraphDatabaseFactory().newImpermanentDatabase();
+        graphDb = new TestGraphDatabaseFactory().newImpermanentDatabase();
     }
 
     @After
     public void shutdown() throws IOException
     {
-        this.graphDb.shutdown();
+        graphDb.shutdown();
     }
 
     @Test
@@ -81,26 +81,22 @@ public void testMultipleCreate() throws InterruptedException
                 Executors.newFixedThreadPool( numThreads ) );
         for ( int i = 0; i < numThreads; i++ )
         {
-            ecs.submit( new Callable<Node>()
-            {
-                public Node call() throws Exception
+            ecs.submit( () -> {
+                try ( Transaction tx = graphDb.beginTx() )
                 {
-                    try ( Transaction tx = graphDb.beginTx() )
+                    final Node node = graphDb.createNode();
+                    // Acquire lock
+                    tx.acquireWriteLock( commonNode );
+                    Index<Node> index = graphDb.index().forNodes( "uuids" );
+                    final Node existing = index.get( "uuid", uuid ).getSingle();
+                    if ( existing != null )
                     {
-                        final Node node = graphDb.createNode();
-                        // Acquire lock
-                        tx.acquireWriteLock( commonNode );
-                        Index<Node> index = graphDb.index().forNodes( "uuids" );
-                        final Node existing = index.get( "uuid", uuid ).getSingle();
-                        if ( existing != null )
-                        {
-                            throw new RuntimeException( "Node already exists" );
-                        }
-                        node.setProperty( "uuid", uuid );
-                        index.add( node, "uuid", uuid );
-                        tx.success();
-                        return node;
+                        throw new RuntimeException( "Node already exists" );
                     }
+                    node.setProperty( "uuid", uuid );
+                    index.add( node, "uuid", uuid );
+                    tx.success();
+                    return node;
                 }
             } );
         }

community/lucene-index/src/test/java/org/neo4j/index/impl/lucene/legacy/AbstractLuceneIndexTest.java

@@ -97,15 +97,15 @@ void restartTx()
         beginTx();
     }
 
-    protected static abstract interface EntityCreator<T extends PropertyContainer>
+    protected interface EntityCreator<T extends PropertyContainer>
     {
         T create( Object... properties );
 
         void delete( T entity );
     }
 
-    private static final RelationshipType TEST_TYPE =
-            RelationshipType.withName( "TEST_TYPE" );
+    private static final RelationshipType TEST_TYPE = RelationshipType.withName( "TEST_TYPE" );
+
     protected static final EntityCreator<Node> NODE_CREATOR = new EntityCreator<Node>()
     {
         public Node create( Object... properties )

community/lucene-index/src/test/java/org/neo4j/index/lucene/ConstraintIndexFailureIT.java

@@ -28,9 +28,8 @@
 import org.neo4j.graphdb.ConstraintViolationException;
 import org.neo4j.graphdb.GraphDatabaseService;
 import org.neo4j.graphdb.Transaction;
-import org.neo4j.io.fs.DefaultFileSystemAbstraction;
 import org.neo4j.kernel.api.exceptions.schema.UnableToValidateConstraintKernelException;
-import org.neo4j.kernel.api.impl.index.storage.DirectoryFactory;
+import org.neo4j.kernel.api.impl.index.builder.LuceneIndexStorageBuilder;
 import org.neo4j.kernel.api.impl.index.storage.PartitionedIndexStorage;
 import org.neo4j.test.TargetDirectory;
 import org.neo4j.test.TestGraphDatabaseFactory;
@@ -103,8 +102,9 @@ private void dbWithConstraint()
     private void storeIndexFailure( String failure ) throws IOException
     {
         File luceneRootDirectory = new File( storeDir.directory(), "schema/index/lucene" );
-        PartitionedIndexStorage indexStorage = new PartitionedIndexStorage( DirectoryFactory.PERSISTENT, new
-                DefaultFileSystemAbstraction(), luceneRootDirectory, "1" );
+        PartitionedIndexStorage indexStorage = LuceneIndexStorageBuilder.create()
+                .withIndexRootFolder( luceneRootDirectory )
+                .withIndexIdentifier( "1" ).build();
         indexStorage.storeIndexFailure( failure );
     }
 }

community/lucene-index/src/test/java/org/neo4j/kernel/api/impl/index/IndexReaderStub.java

@@ -81,11 +81,6 @@ public IndexReaderStub( final Map<String,NumericDocValues> ndvs )
         };
     }
 
-    public void throwOnNextFieldsAccess( IOException throwOnFields )
-    {
-        this.throwOnFields = throwOnFields;
-    }
-
     public void setElements( String[] elements )
     {
         this.elements = elements;
@@ -148,9 +143,7 @@ public SortedSetDocValues getSortedSetDocValues( String field ) throws IOExcepti
     @Override
     public Bits getDocsWithField( String field ) throws IOException
     {
-        throw new RuntimeException(
-                "org.neo4j.kernel.api.impl.index.LuceneAllDocumentsReaderTest.ReaderStub.getDocsWithField: Not " +
-                "yet implemented." );
+        throw new RuntimeException( "Not yet implemented." );
     }
 
     @Override
@@ -162,9 +155,7 @@ public NumericDocValues getNormValues( String field ) throws IOException
     @Override
     public FieldInfos getFieldInfos()
     {
-        throw new RuntimeException(
-                "org.neo4j.kernel.api.impl.index.LuceneAllDocumentsReaderTest.ReaderStub.getFieldInfos: Not yet " +
-                "implemented." );
+        throw new RuntimeException( "Not yet implemented." );
     }
 
     @Override
@@ -193,15 +184,12 @@ public int length()
     @Override
     public void checkIntegrity() throws IOException
     {
-
     }
 
     @Override
     public Fields getTermVectors( int docID ) throws IOException
     {
-        throw new RuntimeException(
-                "org.neo4j.kernel.api.impl.index.LuceneAllDocumentsReaderTest.ReaderStub.getTermVectors: Not yet " +
-                "implemented." );
+        throw new RuntimeException( "Not yet implemented." );
     }
 
     @Override
@@ -225,6 +213,5 @@ public void document( int docID, StoredFieldVisitor visitor ) throws IOException
     @Override
     protected void doClose() throws IOException
     {
-
     }
 }

community/lucene-index/src/test/java/org/neo4j/kernel/api/impl/index/LuceneAllDocumentsReaderTest.java

@@ -26,73 +26,94 @@
 import org.junit.Test;
 
 import java.io.IOException;
-import java.util.ArrayList;
-import java.util.Iterator;
+import java.util.Arrays;
 import java.util.List;
 
-import org.neo4j.helpers.collection.IteratorUtil;
+import org.neo4j.helpers.collection.Iterables;
+import org.neo4j.kernel.api.impl.index.partition.PartitionSearcher;
 
 import static org.junit.Assert.assertEquals;
-import static org.junit.Assert.assertFalse;
+import static org.mockito.Mockito.mock;
+import static org.mockito.Mockito.verify;
+import static org.mockito.Mockito.when;
 
 public class LuceneAllDocumentsReaderTest
 {
+
+    private final PartitionSearcher partitionSearcher1 = createPartitionSearcher( 1, 0, 2 );
+    private final PartitionSearcher partitionSearcher2 = createPartitionSearcher( 2, 1, 2 );
+
+    public LuceneAllDocumentsReaderTest() throws IOException
+    {
+    }
+
     @Test
-    public void shouldIterateOverAllLuceneDocumentsWhereNoDocumentsHaveBeenDeleted() throws Exception
+    public void allDocumentsMaxCount()
     {
-        // given
-        String[] elements = {"A", "B", "C"};
-
-//        LuceneAllDocumentsReader reader =
-//                new LuceneAllDocumentsReader(
-//                        new SearcherManagerStub( new SearcherStub( new IndexReaderStub( false, elements ), elements )
-//                        ) );
-//
-//        // when
-//        Iterator<Document> iterator = reader.iterator();
-//        List<Document> actualDocuments = new ArrayList<>( IteratorUtil.asCollection( iterator ) );
-//
-//        // then
-//        for ( int i = 0; i < elements.length; i++ )
-//        {
-//            assertEquals( elements[i], actualDocuments.get( i ).get( "element" ) );
-//        }
+        LuceneAllDocumentsReader allDocumentsReader = createAllDocumentsReader();
+        assertEquals( 3, allDocumentsReader.maxCount());
     }
 
     @Test
-    public void shouldFindNoDocumentsIfTheyHaveAllBeenDeleted() throws Exception
+    public void closeCorrespondingSearcherOnClose() throws IOException
     {
-        // given
-//        final String[] elements = {"A", "B", "C"};
-//
-//        LuceneAllDocumentsReader reader =
-//                new LuceneAllDocumentsReader(
-//                        new SearcherManagerStub( new SearcherStub( new IndexReaderStub( true, elements ), elements )
-//                        ) );
-//
-//        // when
-//        Iterator<Document> iterator = reader.iterator();
-//
-//        // then
-//        assertFalse( iterator.hasNext() );
+        LuceneAllDocumentsReader allDocumentsReader = createAllDocumentsReader();
+        allDocumentsReader.close();
+
+        verify( partitionSearcher1 ).close();
+        verify( partitionSearcher2 ).close();
     }
 
-    private static class SearcherStub extends IndexSearcher
+    @Test
+    public void readAllDocuments()
     {
-        private final String[] elements;
-
-        public SearcherStub( IndexReader r, String[] elements )
-        {
-            super( r );
-            this.elements = elements;
-        }
-
-        @Override
-        public Document doc( int docID ) throws IOException
-        {
-            Document document = new Document();
-            document.add( new StoredField( "element", elements[docID] ) );
-            return document;
-        }
+        LuceneAllDocumentsReader allDocumentsReader = createAllDocumentsReader();
+        List<Document> documents = Iterables.toList( allDocumentsReader.iterator() );
+
+        assertEquals( "Should have 1 document from first partition and 2 from second one.", 3, documents.size() );
+        assertEquals( "1", documents.get( 0 ).getField( "value" ).stringValue() );
+        assertEquals( "3", documents.get( 1 ).getField( "value" ).stringValue() );
+        assertEquals( "4", documents.get( 2 ).getField( "value" ).stringValue() );
+    }
+
+    private LuceneAllDocumentsReader createAllDocumentsReader()
+    {
+        return new LuceneAllDocumentsReader(createPartitionReaders());
+    }
+
+    private List<LucenePartitionAllDocumentsReader> createPartitionReaders()
+    {
+        LucenePartitionAllDocumentsReader reader1 = new LucenePartitionAllDocumentsReader( partitionSearcher1 );
+        LucenePartitionAllDocumentsReader reader2 = new LucenePartitionAllDocumentsReader( partitionSearcher2 );
+        return Arrays.asList(reader1, reader2);
+    }
+
+    private static PartitionSearcher createPartitionSearcher( int maxDoc, int partition, int maxSize )
+            throws IOException
+    {
+        PartitionSearcher partitionSearcher = mock( PartitionSearcher.class );
+        IndexSearcher indexSearcher = mock( IndexSearcher.class );
+        IndexReader indexReader = mock( IndexReader.class );
+
+        when(partitionSearcher.getIndexSearcher()).thenReturn( indexSearcher );
+        when( indexSearcher.getIndexReader() ).thenReturn( indexReader );
+        when( indexReader.maxDoc() ).thenReturn( maxDoc );
+
+        when( indexSearcher.doc( 0 ) ).thenReturn( createDocument( uniqueDocValue( 1, partition, maxSize ) ) );
+        when( indexSearcher.doc( 1 ) ).thenReturn( createDocument( uniqueDocValue( 2, partition, maxSize ) ) );
+        when( indexSearcher.doc( 2 ) ).thenReturn( createDocument( uniqueDocValue( 3, partition, maxSize ) ) );
+
+        return partitionSearcher;
+    }
+
+    private static String uniqueDocValue(int value, int partition, int maxSize )
+    {
+        return String.valueOf( value + (partition * maxSize) );
+    }
+
+    private static Document createDocument(String value) {
+        Document document = new Document();
+        document.add( new StoredField( "value",  value) );
+        return document;
     }
 }