Skip to content

Commit

Permalink
Includes more information in overview print
Browse files Browse the repository at this point in the history
of the import tool:
- number of processors in overview
- peak memory usage (from caching) during the import
  • Loading branch information
tinwelint committed Aug 29, 2016
1 parent 8c8bab8 commit 1762664
Show file tree
Hide file tree
Showing 2 changed files with 15 additions and 6 deletions.
Expand Up @@ -384,7 +384,7 @@ idType, csvConfiguration( args, defaultSettingsSuitableForTests ), badCollector,
logService, logService,
ExecutionMonitors.defaultVisible(), ExecutionMonitors.defaultVisible(),
dbConfig ); dbConfig );
printOverview( storeDir, nodesFiles, relationshipsFiles ); printOverview( storeDir, nodesFiles, relationshipsFiles, configuration );
success = false; success = false;
try try
{ {
Expand Down Expand Up @@ -437,16 +437,18 @@ private static Config loadDbConfig( File file ) throws IOException
} }


private static void printOverview( File storeDir, Collection<Option<File[]>> nodesFiles, private static void printOverview( File storeDir, Collection<Option<File[]>> nodesFiles,
Collection<Option<File[]>> relationshipsFiles ) Collection<Option<File[]>> relationshipsFiles,
org.neo4j.unsafe.impl.batchimport.Configuration configuration )
{ {
System.out.println( "Neo4j version: " + Version.getKernel().getReleaseVersion() ); System.out.println( "Neo4j version: " + Version.getKernel().getReleaseVersion() );
System.out.println( "Importing the contents of these files into " + storeDir + ":" ); System.out.println( "Importing the contents of these files into " + storeDir + ":" );
printInputFiles( "Nodes", nodesFiles ); printInputFiles( "Nodes", nodesFiles );
printInputFiles( "Relationships", relationshipsFiles ); printInputFiles( "Relationships", relationshipsFiles );
System.out.println(); System.out.println();
System.out.println( "Available memory:" ); System.out.println( "Available resources:" );
printIndented( "Free machine memory: " + bytes( OsBeanUtil.getFreePhysicalMemory() ) ); printIndented( "Free machine memory: " + bytes( OsBeanUtil.getFreePhysicalMemory() ) );
printIndented( "Max heap memory : " + bytes( Runtime.getRuntime().maxMemory() ) ); printIndented( "Max heap memory : " + bytes( Runtime.getRuntime().maxMemory() ) );
printIndented( "Processors: " + configuration.maxNumberOfProcessors() );
System.out.println(); System.out.println();
} }


Expand Down
Expand Up @@ -56,7 +56,10 @@
import org.neo4j.unsafe.impl.batchimport.store.io.IoMonitor; import org.neo4j.unsafe.impl.batchimport.store.io.IoMonitor;


import static java.lang.Math.max; import static java.lang.Math.max;
import static java.lang.String.format;
import static java.lang.System.currentTimeMillis; import static java.lang.System.currentTimeMillis;

import static org.neo4j.helpers.Format.bytes;
import static org.neo4j.helpers.collection.Iterators.asSet; import static org.neo4j.helpers.collection.Iterators.asSet;
import static org.neo4j.io.ByteUnit.mebiBytes; import static org.neo4j.io.ByteUnit.mebiBytes;
import static org.neo4j.unsafe.impl.batchimport.AdditionalInitialIds.EMPTY; import static org.neo4j.unsafe.impl.batchimport.AdditionalInitialIds.EMPTY;
Expand Down Expand Up @@ -182,15 +185,15 @@ public void doImport( Input input ) throws IOException
calculateDenseNodesStage.getRelationshipTypes( 100 ) ); calculateDenseNodesStage.getRelationshipTypes( 100 ) );


// Release this potentially really big piece of cached data // Release this potentially really big piece of cached data
long memoryWeCanHoldForCertain = totalMemoryUsageOf( idMapper, nodeRelationshipCache ); long peakMemoryUsage = totalMemoryUsageOf( idMapper, nodeRelationshipCache );
long highNodeId = nodeRelationshipCache.getHighNodeId(); long highNodeId = nodeRelationshipCache.getHighNodeId();
idMapper.close(); idMapper.close();
idMapper = null; idMapper = null;
nodeRelationshipCache.close(); nodeRelationshipCache.close();
nodeRelationshipCache = null; nodeRelationshipCache = null;


new RelationshipGroupDefragmenter( config, executionMonitor ).run( new RelationshipGroupDefragmenter( config, executionMonitor ).run(
max( max( memoryWeCanHoldForCertain, highNodeId * 4 ), mebiBytes( 1 ) ), neoStore, highNodeId ); max( max( peakMemoryUsage, highNodeId * 4 ), mebiBytes( 1 ) ), neoStore, highNodeId );


// Stage 6 -- count nodes per label and labels per node // Stage 6 -- count nodes per label and labels per node
nodeLabelsCache = new NodeLabelsCache( AUTO, neoStore.getLabelRepository().getHighId() ); nodeLabelsCache = new NodeLabelsCache( AUTO, neoStore.getLabelRepository().getHighId() );
Expand All @@ -204,7 +207,11 @@ public void doImport( Input input ) throws IOException


// We're done, do some final logging about it // We're done, do some final logging about it
long totalTimeMillis = currentTimeMillis() - startTime; long totalTimeMillis = currentTimeMillis() - startTime;
executionMonitor.done( totalTimeMillis, storeUpdateMonitor.toString() ); executionMonitor.done( totalTimeMillis,
format( "%n" ) +
storeUpdateMonitor.toString() +
format( "%n" ) +
"Peak memory usage: " + bytes( peakMemoryUsage ) );
log.info( "Import completed, took " + Format.duration( totalTimeMillis ) + ". " + storeUpdateMonitor ); log.info( "Import completed, took " + Format.duration( totalTimeMillis ) + ". " + storeUpdateMonitor );
hasBadEntries = badCollector.badEntries() > 0; hasBadEntries = badCollector.badEntries() > 0;
if ( hasBadEntries ) if ( hasBadEntries )
Expand Down

0 comments on commit 1762664

Please sign in to comment.