Considers double record units in disk estimation

neo4j · Dec 12, 2017 · 8345060 · 8345060
1 parent bcc3b31
commit 8345060
Show file tree

Hide file tree

Showing 3 changed files with 23 additions and 12 deletions.
diff --git a/community/kernel/src/main/java/org/neo4j/unsafe/impl/batchimport/ImportLogic.java b/community/kernel/src/main/java/org/neo4j/unsafe/impl/batchimport/ImportLogic.java
@@ -72,10 +72,8 @@
 
 import static org.neo4j.helpers.Format.bytes;
 import static org.neo4j.helpers.Format.duration;
-import static org.neo4j.unsafe.impl.batchimport.SourceOrCachedInputIterable.cachedForSure;
 import static org.neo4j.unsafe.impl.batchimport.cache.NodeRelationshipCache.calculateMaxMemoryUsage;
 import static org.neo4j.unsafe.impl.batchimport.cache.NumberArrayFactory.auto;
-import static org.neo4j.unsafe.impl.batchimport.input.InputCache.MAIN;
 import static org.neo4j.unsafe.impl.batchimport.staging.ExecutionSupervisors.superviseExecution;
 
 /**
@@ -88,7 +86,6 @@
  */
 public class ImportLogic implements Closeable
 {
-    private static final long DOUBLE_RELATIONSHIP_RECORD_UNIT_THRESHOLD = 2L << 34;
     private final File storeDir;
     private final FileSystemAbstraction fileSystem;
     private final BatchingNeoStores neoStore;
@@ -119,7 +116,6 @@ public class ImportLogic implements Closeable
     private MemoryUsageStatsProvider memoryUsageStats;
     private InputIterable<InputNode> nodes;
     private InputIterable<InputRelationship> relationships;
-    private InputIterable<InputNode> cachedNodes;
     private long peakMemoryUsage;
     private long availableMemoryForLinking;
 
@@ -162,16 +158,11 @@ public void initialize( Input input ) throws IOException
         memoryUsageStats = new MemoryUsageStatsProvider( nodeRelationshipCache, idMapper, neoStore );
         nodes = input.nodes();
         relationships = input.relationships();
-        cachedNodes = cachedForSure( nodes, inputCache.nodes( MAIN, true ) );
         Estimates inputEstimates = input.calculateEstimates( neoStore.getPropertyStore().newValueEncodedSizeCalculator() );
         sanityCheckEstimatesWithRecordFormat( inputEstimates );
         dependencies.satisfyDependencies( inputEstimates, idMapper, neoStore, nodeRelationshipCache );
 
-        doubleRelationshipRecordUnits =
+        if ( neoStore.determineDoubleRelationshipRecordUnits( inputEstimates ) )
-                // TODO figure out in a good way if this is the high limit format
-                recordFormats.relationship().getMaxId() > 2L << 36 &&
-                inputEstimates.numberOfRelationships() > DOUBLE_RELATIONSHIP_RECORD_UNIT_THRESHOLD;
-        if ( doubleRelationshipRecordUnits )
         {
             System.out.println( "Will use double record units for all relationships" );
         }

diff --git a/...n/java/org/neo4j/unsafe/impl/batchimport/staging/HumanUnderstandableExecutionMonitor.java b/...n/java/org/neo4j/unsafe/impl/batchimport/staging/HumanUnderstandableExecutionMonitor.java
@@ -24,12 +24,12 @@
 
 import org.neo4j.graphdb.DependencyResolver;
 import org.neo4j.unsafe.impl.batchimport.CountGroupsStage;
+import org.neo4j.unsafe.impl.batchimport.DataStatistics;
 import org.neo4j.unsafe.impl.batchimport.IdMapperPreparationStage;
 import org.neo4j.unsafe.impl.batchimport.NodeDegreeCountStage;
 import org.neo4j.unsafe.impl.batchimport.NodeStage;
 import org.neo4j.unsafe.impl.batchimport.RelationshipGroupStage;
 import org.neo4j.unsafe.impl.batchimport.RelationshipStage;
-import org.neo4j.unsafe.impl.batchimport.DataStatistics;
 import org.neo4j.unsafe.impl.batchimport.ScanAndCacheGroupsStage;
 import org.neo4j.unsafe.impl.batchimport.SparseNodeFirstRelationshipStage;
 import org.neo4j.unsafe.impl.batchimport.cache.NodeRelationshipCache;
@@ -163,7 +163,8 @@ private static long nodesDiskUsage( Estimates estimates, BatchingNeoStores neoSt
 
     private static long relationshipsDiskUsage( Estimates estimates, BatchingNeoStores neoStores )
     {
-        return estimates.numberOfRelationships() * neoStores.getRelationshipStore().getRecordSize();
+        return estimates.numberOfRelationships() * neoStores.getRelationshipStore().getRecordSize() *
+                (neoStores.usesDoubleRelationshipRecordUnits() ? 2 : 1);
     }
 
     @Override

diff --git a/...unity/kernel/src/main/java/org/neo4j/unsafe/impl/batchimport/store/BatchingNeoStores.java b/...unity/kernel/src/main/java/org/neo4j/unsafe/impl/batchimport/store/BatchingNeoStores.java
@@ -58,12 +58,14 @@
 import org.neo4j.unsafe.impl.batchimport.AdditionalInitialIds;
 import org.neo4j.unsafe.impl.batchimport.Configuration;
 import org.neo4j.unsafe.impl.batchimport.cache.MemoryStatsVisitor;
+import org.neo4j.unsafe.impl.batchimport.input.Input.Estimates;
 import org.neo4j.unsafe.impl.batchimport.store.BatchingTokenRepository.BatchingLabelTokenRepository;
 import org.neo4j.unsafe.impl.batchimport.store.BatchingTokenRepository.BatchingPropertyKeyTokenRepository;
 import org.neo4j.unsafe.impl.batchimport.store.BatchingTokenRepository.BatchingRelationshipTypeTokenRepository;
 import org.neo4j.unsafe.impl.batchimport.store.io.IoTracer;
 
 import static java.lang.String.valueOf;
+
 import static org.neo4j.graphdb.factory.GraphDatabaseSettings.dense_node_threshold;
 import static org.neo4j.graphdb.factory.GraphDatabaseSettings.pagecache_memory;
 import static org.neo4j.helpers.collection.MapUtil.stringMap;
@@ -78,6 +80,8 @@
  */
 public class BatchingNeoStores implements AutoCloseable, MemoryStatsVisitor.Visitable
 {
+    // Empirical and slightly defensive threshold where relationship records seem to start requiring double record units.
+    private static final long DOUBLE_RELATIONSHIP_RECORD_UNIT_THRESHOLD = 1L << 33;
     private static final String TEMP_NEOSTORE_NAME = "temp." + DEFAULT_NAME;
 
     private final FileSystemAbstraction fileSystem;
@@ -101,6 +105,7 @@ public class BatchingNeoStores implements AutoCloseable, MemoryStatsVisitor.Visi
     private LifeSupport life = new LifeSupport();
     private LabelScanStore labelScanStore;
     private PageCacheFlusher flusher;
+    private boolean doubleRelationshipRecordUnits;
 
     private boolean successful;
 
@@ -420,4 +425,18 @@ public void success()
     {
         successful = true;
     }
+
+    public boolean determineDoubleRelationshipRecordUnits( Estimates inputEstimates )
+    {
+        doubleRelationshipRecordUnits =
+                // TODO figure out in a good way if this is the high limit format
+                recordFormats.relationship().getMaxId() > 2L << 36 &&
+                inputEstimates.numberOfRelationships() > DOUBLE_RELATIONSHIP_RECORD_UNIT_THRESHOLD;
+        return doubleRelationshipRecordUnits;
+    }
+
+    public boolean usesDoubleRelationshipRecordUnits()
+    {
+        return doubleRelationshipRecordUnits;
+    }
 }