Merge pull request #7960 from tinwelint/3.1-reduce-build-time

Reduce build time by making some tests faster

community/io/src/main/java/org/neo4j/io/fs/DefaultFileSystemAbstraction.java

@@ -46,6 +46,9 @@
  */
 public class DefaultFileSystemAbstraction implements FileSystemAbstraction
 {
+    // Named this way for better readability when statically importing
+    public static final FileSystemAbstraction REAL_FS = new DefaultFileSystemAbstraction();
+
     static final String UNABLE_TO_CREATE_DIRECTORY_FORMAT = "Unable to create directory path [%s] for Neo4j store.";
 
     @Override

community/io/src/test/java/org/neo4j/test/rule/TestDirectory.java

@@ -161,15 +161,14 @@ public File graphDbDir()
         return directory( "graph-db" );
     }
 
-    public File makeGraphDbDir()
+    public File makeGraphDbDir() throws IOException
     {
         return cleanDirectory( "graph-db" );
     }
 
     public void cleanup() throws IOException
     {
-        fileSystem.deleteRecursively( testClassBaseFolder );
-        fileSystem.mkdirs( testClassBaseFolder );
+        clean( fileSystem, testClassBaseFolder );
     }
 
     @Override
@@ -179,14 +178,18 @@ public String toString()
         return format( "%s[%s]", getClass().getSimpleName(), testDirectoryName );
     }
 
-    public File cleanDirectory( String name )
+    public File cleanDirectory( String name ) throws IOException
     {
-        File dir = new File( ensureBase(), name );
-        if ( fileSystem.fileExists( dir ) )
+        return clean( fileSystem, new File( ensureBase(), name ) );
+    }
+
+    private static File clean( FileSystemAbstraction fs, File dir ) throws IOException
+    {
+        if ( fs.fileExists( dir ) )
         {
-            deleteSilently( dir );
+            fs.deleteRecursively( dir );
         }
-        fileSystem.mkdir( dir );
+        fs.mkdirs( dir );
         return dir;
     }
 
@@ -211,19 +214,7 @@ private void complete( boolean success )
         testDirectory = null;
     }
 
-    private void deleteSilently( File dir )
-    {
-        try
-        {
-            fileSystem.deleteRecursively( dir );
-        }
-        catch ( IOException e )
-        {
-            throw new RuntimeException( e );
-        }
-    }
-
-    private File directoryForDescription( Description description )
+    private File directoryForDescription( Description description ) throws IOException
     {
         if ( owningTest == null )
         {
@@ -246,8 +237,26 @@ private void evaluateClassBaseTestFolder( )
         {
             throw new IllegalStateException(" Test owning class is not defined" );
         }
+        try
+        {
+            testClassBaseFolder = testDataDirectoryOf( fileSystem, owningTest, false );
+        }
+        catch ( IOException e )
+        {
+            throw new RuntimeException( e );
+        }
+    }
+
+    private static File testDataDirectoryOf( FileSystemAbstraction fs, Class<?> owningTest, boolean clean )
+            throws IOException
+    {
         File testData = new File( locateTarget( owningTest ), "test-data" );
-        testClassBaseFolder = new File( testData, owningTest.getName() ).getAbsoluteFile();
+        File result = new File( testData, owningTest.getName() ).getAbsoluteFile();
+        if ( clean )
+        {
+            clean( fs, result );
+        }
+        return result;
     }
 
     private void register( String test, String dir )

community/jmx/src/main/java/org/neo4j/jmx/impl/ManagementData.java

@@ -30,7 +30,7 @@ public final class ManagementData extends DependencyResolver.Adapter
     private final ManagementSupport support;
     final ManagementBeanProvider provider;
 
-    ManagementData( ManagementBeanProvider provider, KernelData kernel, ManagementSupport support )
+    public ManagementData( ManagementBeanProvider provider, KernelData kernel, ManagementSupport support )
     {
         this.provider = provider;
         this.kernel = kernel;

community/jmx/src/main/java/org/neo4j/jmx/impl/ManagementSupport.java

@@ -35,7 +35,7 @@
 
 public class ManagementSupport
 {
-    static final ManagementSupport load()
+    public static final ManagementSupport load()
     {
         ManagementSupport support = new ManagementSupport();
         for ( ManagementSupport candidate : Service.load( ManagementSupport.class ) )
@@ -66,7 +66,7 @@ protected <T> T makeProxy( KernelBean kernel, ObjectName name, Class<T> beanInte
 
     final <T> Collection<T> getProxiesFor( Class<T> beanInterface, KernelBean kernel )
     {
-        Collection<T> result = new ArrayList<T>();
+        Collection<T> result = new ArrayList<>();
         ObjectName query = createObjectNameQuery( kernel.getInstanceId(), beanInterface );
         for ( ObjectName name : getMBeanServer().queryNames( query, null ) )
         {
@@ -115,7 +115,7 @@ protected String getBeanName( Class<?> beanInterface )
 
     protected ObjectName createObjectName( String instanceId, String beanName, boolean query, String... extraNaming )
     {
-        Hashtable<String, String> properties = new Hashtable<String, String>();
+        Hashtable<String, String> properties = new Hashtable<>();
         properties.put( "instance", "kernel#" + instanceId );
         properties.put( "name", beanName );
         for ( int i = 0; i < extraNaming.length; i++ )

community/kernel/src/main/java/org/neo4j/kernel/impl/util/DebugUtil.java

@@ -29,13 +29,17 @@
 import java.lang.reflect.Method;
 import java.util.HashMap;
 import java.util.Map;
+import java.util.concurrent.TimeUnit;
 import java.util.concurrent.atomic.AtomicInteger;
 import java.util.function.Predicate;
 
 import static java.lang.String.format;
+import static java.lang.System.currentTimeMillis;
+import static java.lang.System.nanoTime;
 import static java.lang.reflect.Modifier.isPublic;
 import static java.lang.reflect.Modifier.isStatic;
 import static org.neo4j.helpers.Exceptions.stringify;
+import static org.neo4j.helpers.Format.duration;
 
 public class DebugUtil
 {
@@ -364,4 +368,78 @@ private static boolean hasTestAnnotation( Method method )
         }
         return false;
     }
+
+    /**
+     * Super simple utility for determining where most time is spent when you don't know where to even start.
+     * It could be used to home in on right place in a test or in a sequence of operations or similar.
+     */
+    public abstract static class Timer
+    {
+        private final TimeUnit unit;
+        private long startTime;
+
+        protected Timer( TimeUnit unit )
+        {
+            this.unit = unit;
+            this.startTime = currentTime();
+        }
+
+        protected abstract long currentTime();
+
+        public void reset()
+        {
+            startTime = currentTime();
+        }
+
+        public void at( String point )
+        {
+            long duration = currentTime() - startTime;
+            System.out.println( duration( unit.toMillis( duration ) ) + " @ " + point );
+            startTime = currentTime();
+        }
+
+        public static Timer millis()
+        {
+            return new Millis();
+        }
+
+        private static class Millis extends Timer
+        {
+            Millis()
+            {
+                super( TimeUnit.MILLISECONDS );
+            }
+
+            @Override
+            protected long currentTime()
+            {
+                return currentTimeMillis();
+            }
+        }
+
+        public static Timer nanos()
+        {
+            return new Nanos();
+        }
+
+        private static class Nanos extends Timer
+        {
+            Nanos()
+            {
+                super( TimeUnit.NANOSECONDS );
+            }
+
+            @Override
+            protected long currentTime()
+            {
+                return nanoTime();
+            }
+        }
+    }
+
+    public static long time( long startTime, String message )
+    {
+        System.out.println( duration( (currentTimeMillis() - startTime) ) + ": " + message );
+        return currentTimeMillis();
+    }
 }

community/kernel/src/main/java/org/neo4j/unsafe/batchinsert/internal/BatchInserterImpl.java

@@ -750,8 +750,7 @@ protected PropertyBlock underlyingObjectToObject( Entry<String, Object> property
     private void setNodeLabels( NodeRecord nodeRecord, Label... labels )
     {
         NodeLabels nodeLabels = parseLabelsField( nodeRecord );
-        nodeStore.updateDynamicLabelRecords( nodeLabels.put( getOrCreateLabelIds( labels ), nodeStore,
-                nodeStore.getDynamicLabelStore() ) );
+        nodeLabels.put( getOrCreateLabelIds( labels ), nodeStore, nodeStore.getDynamicLabelStore() );
         labelsTouched = true;
     }
 
@@ -970,21 +969,23 @@ public void shutdown()
         {
             throw new RuntimeException( e );
         }
+        finally
+        {
+            cursors.close();
+            neoStores.close();
 
-        cursors.close();
-        neoStores.close();
+            try
+            {
+                storeLocker.release();
+            }
+            catch ( IOException e )
+            {
+                throw new UnderlyingStorageException( "Could not release store lock", e );
+            }
 
-        try
-        {
-            storeLocker.release();
-        }
-        catch ( IOException e )
-        {
-            throw new UnderlyingStorageException( "Could not release store lock", e );
+            msgLog.info( Thread.currentThread() + " Clean shutdown on BatchInserter(" + this + ")", true );
+            life.shutdown();
         }
-
-        msgLog.info( Thread.currentThread() + " Clean shutdown on BatchInserter(" + this + ")", true );
-        life.shutdown();
     }
 
     @Override
@@ -1102,6 +1103,17 @@ private void dumpConfiguration( Map<String,String> config, PrintStream out )
         }
     }
 
+    // test-access
+    NeoStores getNeoStores()
+    {
+        return neoStores;
+    }
+
+    void forceFlushChanges()
+    {
+        flushStrategy.forceFlush();
+    }
+
     private class BatchSchemaActions implements InternalSchemaActions
     {
         @Override

community/kernel/src/main/java/org/neo4j/unsafe/impl/batchimport/RelationshipGroupDefragmenter.java

@@ -43,13 +43,38 @@
  */
 public class RelationshipGroupDefragmenter
 {
+    public interface Monitor
+    {
+        /**
+         * When defragmenting the relationship group store it may happen in chunks, selected by node range.
+         * Every time a chunk is selected this method is called.
+         *
+         * @param fromNodeId low node id in the range to process (inclusive).
+         * @param toNodeId high node id in the range to process (exclusive).
+         */
+        default void defragmentingNodeRange( long fromNodeId, long toNodeId )
+        {   // empty
+        }
+
+        Monitor EMPTY = new Monitor()
+        {   // empty
+        };
+    }
+
     private final Configuration config;
     private final ExecutionMonitor executionMonitor;
+    private final Monitor monitor;
 
     public RelationshipGroupDefragmenter( Configuration config, ExecutionMonitor executionMonitor )
+    {
+        this( config, executionMonitor, Monitor.EMPTY );
+    }
+
+    public RelationshipGroupDefragmenter( Configuration config, ExecutionMonitor executionMonitor, Monitor monitor )
     {
         this.config = config;
         this.executionMonitor = executionMonitor;
+        this.monitor = monitor;
     }
 
     public void run( long memoryWeCanHoldForCertain, BatchingNeoStores neoStore, long highNodeId )
@@ -75,6 +100,7 @@ public void run( long memoryWeCanHoldForCertain, BatchingNeoStores neoStore, lon
                     // See how many nodes' groups we can fit into the cache this iteration of the loop.
                     // Groups that doesn't fit in this round will be included in consecutive rounds.
                     toNodeId = groupCache.prepare( fromNodeId );
+                    monitor.defragmentingNodeRange( fromNodeId, toNodeId );
                     // Cache those groups
                     executeStage( new ScanAndCacheGroupsStage( groupConfig, fromStore, groupCache ) );
                     // And write them in sequential order in the store

community/kernel/src/test/java/org/neo4j/graphdb/ConsistentPropertyReadsTest.java → community/kernel/src/test/java/org/neo4j/graphdb/ConsistentPropertyReadsIT.java

@@ -37,7 +37,7 @@
  * Test for how properties are read and that they should be read consistently, i.e. adhere to neo4j's
  * interpretation of the ACID guarantees.
  */
-public class ConsistentPropertyReadsTest
+public class ConsistentPropertyReadsIT
 {
     @Rule
     public DatabaseRule db = new EmbeddedDatabaseRule( getClass() );