Merge 3.3 into 3.4

community/kernel/src/main/java/org/neo4j/kernel/impl/index/IndexConfigStore.java

@@ -295,8 +295,9 @@ private void write( File file )
         StoreChannel channel = null;
         try
         {
+
             channel = fileSystem.open( file, OpenMode.READ_WRITE );
-            channel.write( ByteBuffer.wrap( MAGICK ) );
+            channel.writeAll( ByteBuffer.wrap( MAGICK ) );
             IoPrimitiveUtils.writeInt( channel, buffer( 4 ), VERSION );
             writeMap( channel, nodeConfig );
             writeMap( channel, relConfig );

community/kernel/src/main/java/org/neo4j/kernel/impl/store/id/IdContainer.java

@@ -138,11 +138,11 @@ private long readAndValidateHeader() throws IOException
     private static long readAndValidate( StoreChannel channel, File fileName ) throws IOException
     {
         ByteBuffer buffer = ByteBuffer.allocate( HEADER_SIZE );
-        int read = channel.read( buffer );
-        if ( read != HEADER_SIZE )
+        int totalBytesRead = readBuffer( channel, buffer );
+        if ( totalBytesRead != HEADER_SIZE )
         {
             throw new InvalidIdGeneratorException(
-                    "Unable to read header, bytes read: " + read );
+                    "Unable to read header, bytes read: " + totalBytesRead );
         }
         buffer.flip();
         byte storageStatus = buffer.get();
@@ -175,7 +175,7 @@ private void markAsSticky() throws IOException
         ByteBuffer buffer = ByteBuffer.allocate( Byte.BYTES );
         buffer.put( STICKY_GENERATOR ).flip();
         fileChannel.position( 0 );
-        fileChannel.write( buffer );
+        fileChannel.writeAll( buffer );
         fileChannel.force( false );
     }
 
@@ -185,7 +185,7 @@ private void markAsCleanlyClosed(  ) throws IOException
         ByteBuffer buffer = ByteBuffer.allocate( Byte.BYTES );
         buffer.put( CLEAN_GENERATOR ).flip();
         fileChannel.position( 0 );
-        fileChannel.write( buffer );
+        fileChannel.writeAll( buffer );
     }
 
     public void close( long highId )
@@ -219,7 +219,7 @@ private void writeHeader( long highId ) throws IOException
         ByteBuffer buffer = ByteBuffer.allocate( HEADER_SIZE );
         buffer.put( STICKY_GENERATOR ).putLong( highId ).flip();
         fileChannel.position( 0 );
-        fileChannel.write( buffer );
+        fileChannel.writeAll( buffer );
     }
 
     public void delete()
@@ -306,7 +306,7 @@ public static void createEmptyIdFile( FileSystemAbstraction fs, File file, long
             channel.truncate( 0 );
             ByteBuffer buffer = ByteBuffer.allocate( HEADER_SIZE );
             buffer.put( CLEAN_GENERATOR ).putLong( highId ).flip();
-            channel.write( buffer );
+            channel.writeAll( buffer );
             channel.force( false );
         }
         catch ( IOException e )
@@ -315,6 +315,22 @@ public static void createEmptyIdFile( FileSystemAbstraction fs, File file, long
         }
     }
 
+    private static int readBuffer( StoreChannel channel, ByteBuffer buffer ) throws IOException
+    {
+        int totalBytesRead = 0;
+        int currentBytesRead;
+        do
+        {
+            currentBytesRead = channel.read( buffer );
+            if ( currentBytesRead > 0 )
+            {
+                totalBytesRead += currentBytesRead;
+            }
+        }
+        while ( buffer.hasRemaining() && currentBytesRead >= 0 );
+        return totalBytesRead;
+    }
+
     @Override
     public String toString()
     {

community/kernel/src/main/java/org/neo4j/kernel/impl/storemigration/participant/CountsMigrator.java

@@ -163,6 +163,7 @@ private void rebuildCountsFromScratch( File storeDirToReadFrom, File migrationDi
                 .openNeoStores( StoreType.NODE, StoreType.RELATIONSHIP, StoreType.LABEL_TOKEN,
                         StoreType.RELATIONSHIP_TYPE_TOKEN ) )
         {
+            neoStores.verifyStoreOk();
             NodeStore nodeStore = neoStores.getNodeStore();
             RelationshipStore relationshipStore = neoStores.getRelationshipStore();
             try ( Lifespan life = new Lifespan() )

community/kernel/src/main/java/org/neo4j/kernel/impl/storemigration/participant/NativeLabelScanStoreMigrator.java

@@ -73,6 +73,7 @@ public void migrate( File storeDir, File migrationDir, ProgressReporter progress
             try ( NeoStores neoStores = storeFactory.openAllNeoStores();
                     Lifespan lifespan = new Lifespan() )
             {
+                neoStores.verifyStoreOk();
                 // Remove any existing file to ensure we always do migration
                 deleteNativeIndexFile( migrationDir );
 

community/kernel/src/main/java/org/neo4j/kernel/impl/transaction/log/entry/LogHeaderWriter.java

@@ -67,7 +67,7 @@ public static void writeLogHeader( StoreChannel channel, long logVersion, long p
     {
         ByteBuffer buffer = ByteBuffer.allocate( LOG_HEADER_SIZE );
         writeLogHeader( buffer, logVersion, previousLastCommittedTxId );
-        channel.write( buffer );
+        channel.writeAll( buffer );
     }
 
     public static long encodeLogVersion( long logVersion )

community/kernel/src/test/java/org/neo4j/kernel/impl/store/id/IdContainerTest.java

@@ -19,13 +19,17 @@
  */
 package org.neo4j.kernel.impl.store.id;
 
-import java.io.File;
-
 import org.junit.Before;
 import org.junit.Rule;
 import org.junit.Test;
 
+import java.io.File;
+import java.io.IOException;
+import java.nio.ByteBuffer;
+
+import org.neo4j.io.fs.DefaultFileSystemAbstraction;
 import org.neo4j.io.fs.FileSystemAbstraction;
+import org.neo4j.io.fs.StoreFileChannel;
 import org.neo4j.kernel.impl.store.InvalidIdGeneratorException;
 import org.neo4j.test.rule.TestDirectory;
 import org.neo4j.test.rule.fs.DefaultFileSystemRule;
@@ -171,8 +175,61 @@ public void shouldReturnTrueOnInitIfAProperFileWasThere() throws Exception
         assertTrue( idContainer.init() );
     }
 
+    @Test
+    public void idContainerReadWriteBySingleByte() throws IOException
+    {
+        SingleByteFileSystemAbstraction fileSystem = new SingleByteFileSystemAbstraction();
+        IdContainer idContainer = new IdContainer( fileSystem, file, 100, false );
+        idContainer.init();
+        idContainer.close( 100 );
+
+        idContainer = new IdContainer( fileSystem, file, 100, false );
+        idContainer.init();
+        assertEquals( 100, idContainer.getInitialHighId() );
+        fileSystem.close();
+    }
+
     private void createEmptyFile()
     {
         IdContainer.createEmptyIdFile( fs, file, 42, false );
     }
+
+    private static class SingleByteFileSystemAbstraction extends DefaultFileSystemAbstraction
+    {
+        @Override
+        public StoreFileChannel open( File fileName, String mode ) throws IOException
+        {
+            return new SingleByteBufferChannel( super.open( fileName, mode ) );
+        }
+    }
+
+    private static class SingleByteBufferChannel extends StoreFileChannel
+    {
+
+        SingleByteBufferChannel( StoreFileChannel channel )
+        {
+            super( channel );
+        }
+
+        @Override
+        public int write( ByteBuffer src ) throws IOException
+        {
+            byte b = src.get();
+            ByteBuffer byteBuffer = ByteBuffer.wrap( new byte[]{b} );
+            return super.write( byteBuffer );
+        }
+
+        @Override
+        public int read( ByteBuffer dst ) throws IOException
+        {
+            ByteBuffer byteBuffer = ByteBuffer.allocate( 1 );
+            int read = super.read( byteBuffer );
+            if ( read > 0 )
+            {
+                byteBuffer.flip();
+                dst.put( byteBuffer.get() );
+            }
+            return read;
+        }
+    }
 }

community/kernel/src/test/java/org/neo4j/kernel/impl/storemigration/participant/NativeLabelScanStoreMigratorTest.java

@@ -44,7 +44,9 @@
 import org.neo4j.kernel.configuration.Config;
 import org.neo4j.kernel.impl.api.scan.FullStoreChangeStream;
 import org.neo4j.kernel.impl.index.labelscan.NativeLabelScanStore;
+import org.neo4j.kernel.impl.store.InvalidIdGeneratorException;
 import org.neo4j.kernel.impl.store.MetaDataStore;
+import org.neo4j.kernel.impl.store.StoreFile;
 import org.neo4j.kernel.impl.store.format.standard.StandardV2_3;
 import org.neo4j.kernel.impl.store.format.standard.StandardV3_2;
 import org.neo4j.kernel.impl.util.monitoring.ProgressReporter;
@@ -101,7 +103,7 @@ public void setUp() throws Exception
     @Test
     public void skipMigrationIfNativeIndexExist() throws Exception
     {
-        ByteBuffer sourceBuffer = writeNativeIndexFile( nativeLabelIndex, new byte[]{1, 2, 3} );
+        ByteBuffer sourceBuffer = writeFile( nativeLabelIndex, new byte[]{1, 2, 3} );
 
         indexMigrator.migrate( storeDir, migrationDir, progressReporter, StandardV3_2.STORE_VERSION, StandardV3_2.STORE_VERSION );
         indexMigrator.moveMigratedFiles( migrationDir, storeDir, StandardV3_2.STORE_VERSION, StandardV3_2.STORE_VERSION );
@@ -111,6 +113,16 @@ public void skipMigrationIfNativeIndexExist() throws Exception
         assertTrue( fileSystem.fileExists( luceneLabelScanStore ) );
     }
 
+    @Test( expected = InvalidIdGeneratorException.class )
+    public void failMigrationWhenNodeIdFileIsBroken() throws Exception
+    {
+        prepareEmpty23Database();
+        File nodeIdFile = new File( storeDir, StoreFile.NODE_STORE.storeFileName() + ".id" );
+        writeFile( nodeIdFile, new byte[]{1, 2, 3} );
+
+        indexMigrator.migrate( storeDir, migrationDir, progressMonitor, StandardV3_2.STORE_VERSION, StandardV3_2.STORE_VERSION );
+    }
+
     @Test
     public void clearMigrationDirFromAnyLabelScanStoreBeforeMigrating() throws Exception
     {
@@ -144,7 +156,7 @@ public void moveCreatedNativeLabelIndexBackToStoreDirectory() throws IOException
         prepareEmpty23Database();
         indexMigrator.migrate( storeDir, migrationDir, progressReporter, StandardV2_3.STORE_VERSION, StandardV3_2.STORE_VERSION );
         File migrationNativeIndex = new File( migrationDir, NativeLabelScanStore.FILE_NAME );
-        ByteBuffer migratedFileContent = writeNativeIndexFile( migrationNativeIndex, new byte[]{5, 4, 3, 2, 1} );
+        ByteBuffer migratedFileContent = writeFile( migrationNativeIndex, new byte[]{5, 4, 3, 2, 1} );
 
         indexMigrator.moveMigratedFiles( migrationDir, storeDir, StandardV2_3.STORE_VERSION, StandardV3_2.STORE_VERSION );
 
@@ -220,7 +232,7 @@ private void assertNoContentInNativeLabelScanStore( File dir )
         }
     }
 
-    private ByteBuffer writeNativeIndexFile( File file, byte[] content ) throws IOException
+    private ByteBuffer writeFile( File file, byte[] content ) throws IOException
     {
         ByteBuffer sourceBuffer = ByteBuffer.wrap( content );
         storeFileContent( file, sourceBuffer );
@@ -272,11 +284,11 @@ private ByteBuffer readFileContent( File nativeLabelIndex, int length ) throws I
         }
     }
 
-    private void storeFileContent( File nativeLabelIndex, ByteBuffer sourceBuffer ) throws IOException
+    private void storeFileContent( File file, ByteBuffer sourceBuffer ) throws IOException
     {
-        try ( StoreChannel storeChannel = fileSystem.create( nativeLabelIndex ) )
+        try ( StoreChannel storeChannel = fileSystem.create( file ) )
         {
-            storeChannel.write( sourceBuffer );
+            storeChannel.writeAll( sourceBuffer );
         }
     }
 }

community/lucene-index/src/main/java/org/neo4j/kernel/api/impl/index/storage/FailureStorage.java

@@ -71,7 +71,7 @@ public synchronized void reserveForIndex() throws IOException
         File failureFile = failureFile();
         try ( StoreChannel channel = fs.create( failureFile ) )
         {
-            channel.write( ByteBuffer.wrap( new byte[MAX_FAILURE_SIZE] ) );
+            channel.writeAll( ByteBuffer.wrap( new byte[MAX_FAILURE_SIZE] ) );
             channel.force( true );
         }
     }
@@ -121,7 +121,7 @@ public synchronized void storeIndexFailure( String failure ) throws IOException
             channel.position( lengthOf( existingData ) );
 
             byte[] data = UTF8.encode( failure );
-            channel.write( ByteBuffer.wrap( data, 0, Math.min( data.length, MAX_FAILURE_SIZE ) ) );
+            channel.writeAll( ByteBuffer.wrap( data, 0, Math.min( data.length, MAX_FAILURE_SIZE ) ) );
 
             channel.force( true );
             channel.close();