Skip to content

Commit

Permalink
Includes more information in overview print
Browse files Browse the repository at this point in the history
of the import tool:
- number of processors in overview
- peak memory usage (from caching) during the import
  • Loading branch information
tinwelint committed Aug 29, 2016
1 parent 8c8bab8 commit 1762664
Show file tree
Hide file tree
Showing 2 changed files with 15 additions and 6 deletions.
Expand Up @@ -384,7 +384,7 @@ idType, csvConfiguration( args, defaultSettingsSuitableForTests ), badCollector,
logService,
ExecutionMonitors.defaultVisible(),
dbConfig );
printOverview( storeDir, nodesFiles, relationshipsFiles );
printOverview( storeDir, nodesFiles, relationshipsFiles, configuration );
success = false;
try
{
Expand Down Expand Up @@ -437,16 +437,18 @@ private static Config loadDbConfig( File file ) throws IOException
}

private static void printOverview( File storeDir, Collection<Option<File[]>> nodesFiles,
Collection<Option<File[]>> relationshipsFiles )
Collection<Option<File[]>> relationshipsFiles,
org.neo4j.unsafe.impl.batchimport.Configuration configuration )
{
System.out.println( "Neo4j version: " + Version.getKernel().getReleaseVersion() );
System.out.println( "Importing the contents of these files into " + storeDir + ":" );
printInputFiles( "Nodes", nodesFiles );
printInputFiles( "Relationships", relationshipsFiles );
System.out.println();
System.out.println( "Available memory:" );
System.out.println( "Available resources:" );
printIndented( "Free machine memory: " + bytes( OsBeanUtil.getFreePhysicalMemory() ) );
printIndented( "Max heap memory : " + bytes( Runtime.getRuntime().maxMemory() ) );
printIndented( "Processors: " + configuration.maxNumberOfProcessors() );
System.out.println();
}

Expand Down
Expand Up @@ -56,7 +56,10 @@
import org.neo4j.unsafe.impl.batchimport.store.io.IoMonitor;

import static java.lang.Math.max;
import static java.lang.String.format;
import static java.lang.System.currentTimeMillis;

import static org.neo4j.helpers.Format.bytes;
import static org.neo4j.helpers.collection.Iterators.asSet;
import static org.neo4j.io.ByteUnit.mebiBytes;
import static org.neo4j.unsafe.impl.batchimport.AdditionalInitialIds.EMPTY;
Expand Down Expand Up @@ -182,15 +185,15 @@ public void doImport( Input input ) throws IOException
calculateDenseNodesStage.getRelationshipTypes( 100 ) );

// Release this potentially really big piece of cached data
long memoryWeCanHoldForCertain = totalMemoryUsageOf( idMapper, nodeRelationshipCache );
long peakMemoryUsage = totalMemoryUsageOf( idMapper, nodeRelationshipCache );
long highNodeId = nodeRelationshipCache.getHighNodeId();
idMapper.close();
idMapper = null;
nodeRelationshipCache.close();
nodeRelationshipCache = null;

new RelationshipGroupDefragmenter( config, executionMonitor ).run(
max( max( memoryWeCanHoldForCertain, highNodeId * 4 ), mebiBytes( 1 ) ), neoStore, highNodeId );
max( max( peakMemoryUsage, highNodeId * 4 ), mebiBytes( 1 ) ), neoStore, highNodeId );

// Stage 6 -- count nodes per label and labels per node
nodeLabelsCache = new NodeLabelsCache( AUTO, neoStore.getLabelRepository().getHighId() );
Expand All @@ -204,7 +207,11 @@ public void doImport( Input input ) throws IOException

// We're done, do some final logging about it
long totalTimeMillis = currentTimeMillis() - startTime;
executionMonitor.done( totalTimeMillis, storeUpdateMonitor.toString() );
executionMonitor.done( totalTimeMillis,
format( "%n" ) +
storeUpdateMonitor.toString() +
format( "%n" ) +
"Peak memory usage: " + bytes( peakMemoryUsage ) );
log.info( "Import completed, took " + Format.duration( totalTimeMillis ) + ". " + storeUpdateMonitor );
hasBadEntries = badCollector.badEntries() > 0;
if ( hasBadEntries )
Expand Down

0 comments on commit 1762664

Please sign in to comment.