Minimum logical log threshold is now 1 Mb

it doesn't really make sense to have threshold smaller than that.

community/kernel/src/main/java/org/neo4j/graphdb/factory/GraphDatabaseSettings.java

@@ -191,7 +191,7 @@ public abstract class GraphDatabaseSettings
 
     @Description( "Specifies at which file size the logical log will auto-rotate. " +
                   "`0` means that no rotation will automatically occur based on file size. " )
-    public static final Setting<Long> logical_log_rotation_threshold = setting( "logical_log_rotation_threshold", BYTES, "250M" );
+    public static final Setting<Long> logical_log_rotation_threshold = setting( "logical_log_rotation_threshold", BYTES, "250M", min( 1024*1024L /*1Mb*/ ) );
 
     @Description("Use a quick approach for rebuilding the ID generators. This give quicker recovery time, " +
             "but will limit the ability to reuse the space of deleted entities.")

community/kernel/src/test/java/org/neo4j/helpers/SettingsTest.java

@@ -19,12 +19,12 @@
  */
 package org.neo4j.helpers;
 
+import org.junit.Test;
+
 import java.io.File;
 import java.net.URI;
 import java.util.List;
 
-import org.junit.Test;
-
 import org.neo4j.graphdb.config.InvalidSettingException;
 import org.neo4j.graphdb.config.Setting;
 import org.neo4j.graphdb.factory.GraphDatabaseSettings;
@@ -39,8 +39,8 @@
 import static org.neo4j.helpers.Settings.DURATION;
 import static org.neo4j.helpers.Settings.INTEGER;
 import static org.neo4j.helpers.Settings.MANDATORY;
-import static org.neo4j.helpers.Settings.NO_DEFAULT;
 import static org.neo4j.helpers.Settings.NORMALIZED_RELATIVE_URI;
+import static org.neo4j.helpers.Settings.NO_DEFAULT;
 import static org.neo4j.helpers.Settings.PATH;
 import static org.neo4j.helpers.Settings.STRING;
 import static org.neo4j.helpers.Settings.basePath;
@@ -283,13 +283,11 @@ public void testLogicalLogRotationThreshold() throws Exception
         // WHEN
         Setting<Long> setting = GraphDatabaseSettings.logical_log_rotation_threshold;
         long defaultValue = setting.apply( map( stringMap() ) );
-        long kiloValue = setting.apply( map( stringMap( setting.name(), "10k" ) ) );
         long megaValue = setting.apply( map( stringMap( setting.name(), "10M" ) ) );
         long gigaValue = setting.apply( map( stringMap( setting.name(), "10g" ) ) );
 
         // THEN
         assertThat( defaultValue, greaterThan( 0L ) );
-        assertEquals( 10 * 1024, kiloValue );
         assertEquals( 10 * 1024 * 1024, megaValue );
         assertEquals( 10L * 1024 * 1024 * 1024, gigaValue );
     }
@@ -335,13 +333,13 @@ public void testMemoryUsageConfigFromSystemMemory() throws Exception
         // When && then
         assertThat(memUse.apply( Functions.<String,String>constant( null ) ), greaterThan( 0l ));
     }
-    
+
     @Test
     public void testNormalizedRelativeURI() throws Exception
     {
         // Given
         Setting<URI> uri = setting( "mySetting", NORMALIZED_RELATIVE_URI, "http://localhost:7474///db///data///" );
-        
+
         // When && then
         assertThat( uri.apply( Functions.<String,String>constant( null ) ).toString(), equalTo( "/db/data" ) );
     }

community/kernel/src/test/java/org/neo4j/kernel/impl/store/counts/CountsRotationTest.java

@@ -32,7 +32,6 @@
 import org.neo4j.graphdb.Label;
 import org.neo4j.graphdb.Transaction;
 import org.neo4j.graphdb.factory.GraphDatabaseBuilder;
-import org.neo4j.graphdb.factory.GraphDatabaseSettings;
 import org.neo4j.helpers.Pair;
 import org.neo4j.io.fs.FileSystemAbstraction;
 import org.neo4j.io.pagecache.PageCache;
@@ -43,17 +42,19 @@
 import org.neo4j.kernel.impl.store.counts.keys.CountsKey;
 import org.neo4j.kernel.impl.store.kvstore.KeyValueRecordVisitor;
 import org.neo4j.kernel.impl.store.kvstore.SortedKeyValueStore;
+import org.neo4j.kernel.impl.transaction.log.LogRotation;
 import org.neo4j.register.Register.CopyableDoubleLongRegister;
+import org.neo4j.register.Register.DoubleLongRegister;
 import org.neo4j.test.EphemeralFileSystemRule;
 import org.neo4j.test.PageCacheRule;
 import org.neo4j.test.TargetDirectory;
 import org.neo4j.test.TestGraphDatabaseFactory;
 
 import static org.junit.Assert.assertEquals;
 import static org.junit.Assert.assertTrue;
+
 import static org.neo4j.kernel.impl.store.kvstore.SortedKeyValueStoreHeader.BASE_MINOR_VERSION;
 import static org.neo4j.kernel.impl.transaction.log.TransactionIdStore.BASE_TX_ID;
-import static org.neo4j.register.Register.DoubleLongRegister;
 import static org.neo4j.register.Registers.newDoubleLongRegister;
 
 public class CountsRotationTest
@@ -121,16 +122,17 @@ public void shouldRotateCountsStoreWhenClosingTheDatabase() throws IOException
     public void shouldRotateCountsStoreWhenRotatingLog() throws IOException
     {
         // GIVEN
-        GraphDatabaseAPI db = (GraphDatabaseAPI) dbBuilder.
-                setConfig( GraphDatabaseSettings.logical_log_rotation_threshold, "1" ).newGraphDatabase();
+        GraphDatabaseAPI db = (GraphDatabaseAPI) dbBuilder.newGraphDatabase();
+
+        // WHEN doing a transaction (actually two, the label-mini-tx also counts)
         try ( Transaction tx = db.beginTx() )
         {
             db.createNode( B );
             tx.success();
         }
-
-        // WHEN
-        // we do a log rotation _BEFORE_ this transaction
+        // and rotating the log (which implies flushing)
+        rotateLog( db );
+        // and creating another node after it
         try ( Transaction tx = db.beginTx() )
         {
             db.createNode( C );
@@ -142,7 +144,7 @@ public void shouldRotateCountsStoreWhenRotatingLog() throws IOException
         assertTrue( fs.fileExists( betaStoreFile() ) );
 
         final PageCache pageCache = db.getDependencyResolver().resolveDependency( PageCache.class );
-        try ( CountsStore store = CountsStore.open( fs, pageCache, alphaStoreFile() ) )
+        try ( CountsStore store = CountsStore.open( fs, pageCache, betaStoreFile() ) )
         {
             // NOTE since the rotation happens before the second transaction is committed we do not see those changes
             // in the stats
@@ -165,6 +167,11 @@ public void shouldRotateCountsStoreWhenRotatingLog() throws IOException
         db.shutdown();
     }
 
+    private void rotateLog( GraphDatabaseAPI db ) throws IOException
+    {
+        db.getDependencyResolver().resolveDependency( LogRotation.class ).rotateLogFile();
+    }
+
     private final Label A = DynamicLabel.label( "A" );
     private final Label B = DynamicLabel.label( "B" );
     private final Label C = DynamicLabel.label( "C" );

community/kernel/src/test/java/org/neo4j/kernel/impl/transaction/PartialTransactionFailureIT.java

@@ -19,14 +19,14 @@
  */
 package org.neo4j.kernel.impl.transaction;
 
+import org.junit.Rule;
+import org.junit.Test;
+
 import java.util.Arrays;
 import java.util.Iterator;
 import java.util.Map;
 import java.util.concurrent.CountDownLatch;
 
-import org.junit.Rule;
-import org.junit.Test;
-
 import org.neo4j.adversaries.ClassGuardedAdversary;
 import org.neo4j.adversaries.CountingAdversary;
 import org.neo4j.adversaries.fs.AdversarialFileSystemAbstraction;
@@ -41,6 +41,7 @@
 import org.neo4j.kernel.impl.api.scan.InMemoryLabelScanStoreExtension;
 import org.neo4j.kernel.impl.cache.CacheProvider;
 import org.neo4j.kernel.impl.cache.SoftCacheProvider;
+import org.neo4j.kernel.impl.transaction.log.LogRotation;
 import org.neo4j.test.TargetDirectory;
 
 import static org.junit.Assert.assertEquals;
@@ -68,10 +69,8 @@ public void concurrentlyCommittingTransactionsMustNotRotateOutLoggedCommandsOfFa
                 "org.neo4j.kernel.impl.nioneo.xa.Command$RelationshipCommand" );
         adversary.disable();
 
-        Map<String, String> params = stringMap(
-                "logical_log_rotation_threshold", "1");
         String storeDir = dir.directory().getAbsolutePath();
-        final EmbeddedGraphDatabase db = new TestEmbeddedGraphDatabase( storeDir, params ) {
+        final EmbeddedGraphDatabase db = new TestEmbeddedGraphDatabase( storeDir, stringMap() ) {
             @Override
             protected FileSystemAbstraction createFileSystemAbstraction()
             {
@@ -179,6 +178,7 @@ public void run()
                     x.createRelationshipTo( y, DynamicRelationshipType.withName( "r" ) );
                     tx.success();
                     latch.await();
+                    db.getDependencyResolver().resolveDependency( LogRotation.class ).rotateLogFile();
                     tx.finish();
                 }
                 catch ( Exception ignore )

community/kernel/src/test/java/org/neo4j/kernel/impl/transaction/ReadTransactionLogWritingTest.java

@@ -19,13 +19,13 @@
  */
 package org.neo4j.kernel.impl.transaction;
 
-import java.io.File;
-import java.io.IOException;
-
 import org.junit.Before;
 import org.junit.Rule;
 import org.junit.Test;
 
+import java.io.File;
+import java.io.IOException;
+
 import org.neo4j.graphdb.Label;
 import org.neo4j.graphdb.Node;
 import org.neo4j.graphdb.PropertyContainer;
@@ -37,7 +37,6 @@
 import org.neo4j.kernel.GraphDatabaseAPI;
 import org.neo4j.kernel.impl.MyRelTypes;
 import org.neo4j.kernel.impl.transaction.log.LogFileInformation;
-import org.neo4j.kernel.impl.transaction.log.PhysicalLogFile;
 import org.neo4j.kernel.impl.transaction.log.entry.LogEntry;
 import org.neo4j.test.DatabaseRule;
 import org.neo4j.test.ImpermanentDatabaseRule;
@@ -76,7 +75,6 @@ public void shouldNotWriteAnyLogCommandInPureReadTransaction() throws Exception
         protected void configure( GraphDatabaseBuilder builder )
         {
             builder.setConfig( GraphDatabaseSettings.cache_type, "none" );
-            builder.setConfig( GraphDatabaseSettings.logical_log_rotation_threshold, "5");
         };
     };
 
@@ -99,7 +97,6 @@ public void createDataset() throws IOException
             relationship.setProperty( "long", longString( 300 ) );
             tx.success();
         }
-        db.getDependencyResolver().resolveDependency( PhysicalLogFile.class ).rotationNeeded();
         logEntriesWrittenBeforeReadOperations = countLogEntries();
     }
 

community/kernel/src/test/java/org/neo4j/kernel/impl/transaction/log/pruning/TestLogPruning.java

@@ -19,19 +19,19 @@
  */
 package org.neo4j.kernel.impl.transaction.log.pruning;
 
-import java.io.File;
-import java.io.IOException;
-
 import org.junit.After;
-import org.junit.Ignore;
 import org.junit.Test;
 
+import java.io.File;
+import java.io.IOException;
+
 import org.neo4j.graphdb.Node;
 import org.neo4j.graphdb.Transaction;
 import org.neo4j.graphdb.mockfs.EphemeralFileSystemAbstraction;
 import org.neo4j.io.fs.FileSystemAbstraction;
 import org.neo4j.io.fs.StoreChannel;
 import org.neo4j.kernel.GraphDatabaseAPI;
+import org.neo4j.kernel.impl.transaction.log.LogRotation;
 import org.neo4j.kernel.impl.transaction.log.LogVersionBridge;
 import org.neo4j.kernel.impl.transaction.log.LogVersionedStoreChannel;
 import org.neo4j.kernel.impl.transaction.log.PhysicalLogFile;
@@ -47,7 +47,6 @@
 import static org.junit.Assert.assertTrue;
 
 import static org.neo4j.graphdb.factory.GraphDatabaseSettings.keep_logical_logs;
-import static org.neo4j.graphdb.factory.GraphDatabaseSettings.logical_log_rotation_threshold;
 import static org.neo4j.helpers.collection.MapUtil.stringMap;
 import static org.neo4j.kernel.impl.transaction.log.entry.LogVersions.CURRENT_LOG_VERSION;
 
@@ -58,11 +57,10 @@ private interface Extractor
         int extract( File from ) throws IOException;
     }
 
-    // From the last measurement by figureOutSampleTransactionSizeBytes
-    private final int transactionLogSize = 358;
     private GraphDatabaseAPI db;
     private FileSystemAbstraction fs;
     private PhysicalLogFiles files;
+    private int rotateEveryNTransactions, performedTransactions;
 
     @After
     public void after() throws Exception
@@ -76,7 +74,7 @@ public void after() throws Exception
     @Test
     public void noPruning() throws Exception
     {
-        newDb( "true", transactionLogSize * 2 );
+        newDb( "true", 2 );
 
         for ( int i = 0; i < 100; i++ )
         {
@@ -95,25 +93,27 @@ public void noPruning() throws Exception
     public void pruneByFileSize() throws Exception
     {
         // Given
-        int size = 1050;
-        int logThreshold = transactionLogSize * 3;
-        newDb( size + " size", logThreshold );
+        int transactionByteSize = figureOutSampleTransactionSizeBytes();
+        int transactionsPerFile = 3;
+        int logThreshold = transactionByteSize * transactionsPerFile;
+        newDb( logThreshold + " size", 1 );
 
         // When
         for ( int i = 0; i < 100; i++ )
         {
             doTransaction();
         }
 
-        int logFileSize = logFileSize();
-        assertTrue( logFileSize >= size - logThreshold && logFileSize <= size + logThreshold );
+        int totalLogFileSize = logFileSize();
+        double totalTransactions = (double) totalLogFileSize / transactionByteSize;
+        assertTrue( totalTransactions >= 3 && totalTransactions < 4 );
     }
 
     @Test
     public void pruneByFileCount() throws Exception
     {
         int logsToKeep = 5;
-        newDb( logsToKeep + " files", transactionLogSize * 3 );
+        newDb( logsToKeep + " files", 3 );
 
         for ( int i = 0; i < 100; i++ )
         {
@@ -128,9 +128,8 @@ public void pruneByFileCount() throws Exception
     public void pruneByTransactionCount() throws Exception
     {
         int transactionsToKeep = 100;
-
         int transactionsPerLog = 3;
-        newDb( transactionsToKeep + " txs", transactionsToKeep * transactionsPerLog );
+        newDb( transactionsToKeep + " txs", 3 );
 
         for ( int i = 0; i < 100; i++ )
         {
@@ -145,12 +144,12 @@ public void pruneByTransactionCount() throws Exception
                 transactionCount <= (transactionsToKeep + transactionsPerLog) );
     }
 
-    private GraphDatabaseAPI newDb( String logPruning, int rotateThreshold )
+    private GraphDatabaseAPI newDb( String logPruning, int rotateEveryNTransactions )
     {
+        this.rotateEveryNTransactions = rotateEveryNTransactions;
         fs = new EphemeralFileSystemAbstraction();
         GraphDatabaseAPI db = new ImpermanentGraphDatabase( stringMap(
-                keep_logical_logs.name(), logPruning,
-                logical_log_rotation_threshold.name(), "" + rotateThreshold
+                keep_logical_logs.name(), logPruning
         ) )
         {
             @Override
@@ -164,8 +163,14 @@ protected FileSystemAbstraction createFileSystemAbstraction()
         return db;
     }
 
-    private void doTransaction()
+    private void doTransaction() throws IOException
     {
+        if ( ++performedTransactions >= rotateEveryNTransactions )
+        {
+            db.getDependencyResolver().resolveDependency( LogRotation.class ).rotateLogFile();
+            performedTransactions = 0;
+        }
+
         try ( Transaction tx = db.beginTx() )
         {
             Node node = db.createNode();
@@ -174,14 +179,12 @@ private void doTransaction()
         }
     }
 
-    @Test
-    @Ignore( "Here as a helper to figure out the transaction size of the sample transaction on disk" )
-    public void figureOutSampleTransactionSizeBytes()
+    private int figureOutSampleTransactionSizeBytes() throws IOException
     {
-        db = newDb( "true", transactionLogSize * 2 );
+        db = newDb( "true", 5 );
         doTransaction();
         db.shutdown();
-        System.out.println( fs.getFileSize( files.getLogFileForVersion( 0 ) ) );
+        return (int) fs.getFileSize( files.getLogFileForVersion( 0 ) );
     }
 
     private int aggregateLogData( Extractor extractor ) throws IOException