Skip to content

Commit

Permalink
neo4j-admin memrec calculates lucene/native sizes for db
Browse files Browse the repository at this point in the history 
And prints it as a note, rather than as a page cacge setting.
The idea is that those calculations are more suited for
extrapolation and understanding, rather than definitive page cache
setting in the config file.
  • Loading branch information
@tinwelint
tinwelint committed Apr 23, 2018
1 parent b5cbc62 commit 173972f
Show file tree
Hide file tree
Showing 2 changed files with 63 additions and 41 deletions.
68 changes: 38 additions & 30 deletions community/dbms/src/main/java/org/neo4j/commandline/dbms/MemoryRecommendationsCommand.java
View file
Expand Up @@ -30,10 +30,7 @@
import org.neo4j.commandline.admin.IncorrectUsage;
import org.neo4j.commandline.admin.OutsideWorld;
import org.neo4j.commandline.arguments.Arguments;
import org.neo4j.commandline.arguments.OptionalBooleanArg;
import org.neo4j.commandline.arguments.OptionalNamedArg;
import org.neo4j.commandline.arguments.common.Database;
import org.neo4j.graphdb.factory.GraphDatabaseSettings;
import org.neo4j.io.os.OsBeanUtil;
import org.neo4j.kernel.api.impl.index.storage.FailureStorage;
import org.neo4j.kernel.configuration.Config;
Expand Down Expand Up @@ -222,8 +219,7 @@ public void execute( String[] args ) throws IncorrectUsage, CommandFailed
print( "# Based on the above, the following memory settings are recommended:" );
print( initialHeapSize.name() + "=" + heap );
print( maxHeapSize.name() + "=" + heap );
// print this generic memory calculation as commented out, for reference
print( (specificDb ? "#" : "") + pagecache_memory.name() + "=" + pagecache );
print( pagecache_memory.name() + "=" + pagecache );

if ( !specificDb )
{
Expand All @@ -232,42 +228,54 @@ public void execute( String[] args ) throws IncorrectUsage, CommandFailed
String databaseName = arguments.get( ARG_DATABASE );
File configFile = configDir.resolve( Config.DEFAULT_CONFIG_FILE_NAME ).toFile();
File storeDir = getConfig( configFile, databaseName ).get( database_path );
long pageCacheSize = dbSpecificPageCacheSize( storeDir );
long luceneSize = dbSpecificLuceneSize( storeDir );

print( "#" );
print( "# A minimal page cache memory setting to fully map the database contents of '" + storeDir.getAbsolutePath() + "':" );
print( pagecache_memory.name() + "=" + dbSpecificPageCacheMemory( storeDir ) );
print( "# The numbers below have been derived based on your current data volume in database and index configuration of database '" + databaseName +
"'." );
print( "# They can be used as an input into more detailed memory analysis." );
print( "# Lucene indexes: " + bytesToString( luceneSize ) );
print( "# Data volume and native indexes: " + bytesToString( pageCacheSize ) );
}

private FilenameFilter getNativeIndexFileFilter( Path baseSchemaIndexPath )
private long dbSpecificPageCacheSize( File storeDir )
{
return sumStoreFiles( storeDir ) + sumIndexFiles( baseSchemaIndexFolder( storeDir ), getNativeIndexFileFilter( false ) );
}

private long dbSpecificLuceneSize( File storeDir )
{
return sumIndexFiles( baseSchemaIndexFolder( storeDir ), getNativeIndexFileFilter( true ) );
}

private FilenameFilter getNativeIndexFileFilter( boolean inverse )
{
return ( dir, name ) ->
{
File file = new File( dir, name );
// Lucene index files lives in:
// - schema/index/lucene/<indexId>/<partition>/.....
// - schema/index/lucene_native-x.y/<indexId>/lucene-x.y/x/.....
if ( outsideWorld.fileSystem().isDirectory( file ) && (file.toPath().getParent().equals( baseSchemaIndexPath ) && name.equals( "lucene" ) ||
file.toPath().getNameCount() - baseSchemaIndexPath.getNameCount() == 3 && name.startsWith( "lucene-" )) )
if ( outsideWorld.fileSystem().isDirectory( file ) )
{
// Always go down directories
return true;
}
if ( name.equals( FailureStorage.DEFAULT_FAILURE_FILE_NAME ) )
{
// Do not go down lucene directory
// Never include failure-storage files
return false;
}
return true;
};
}

private String dbSpecificPageCacheMemory( File storeDir )
{
long total = 0;

// All neostore files
total += sumStoreFiles( storeDir );

// All native index files
File baseSchemaIndexDir = baseSchemaIndexFolder( storeDir );
total += sumNativeIndexFiles( baseSchemaIndexDir, getNativeIndexFileFilter( baseSchemaIndexDir.toPath() ) );
Path path = file.toPath();
int nameCount = path.getNameCount();
// Lucene index files lives in:
// - schema/index/lucene_native-x.y/<indexId>/lucene-x.y/x/.....
boolean isLuceneFilePart1 = nameCount >= 3 && path.getName( nameCount - 3 ).toString().startsWith( "lucene-" );
// - schema/index/lucene/<indexId>/<partition>/.....
boolean isLuceneFilePart2 = nameCount >= 4 && path.getName( nameCount - 4 ).toString().equals( "lucene" );

return bytesToString( total );
boolean isLuceneFile = isLuceneFilePart1 || isLuceneFilePart2;
return inverse == isLuceneFile;
};
}

private long sumStoreFiles( File storeDir )
Expand All @@ -287,7 +295,7 @@ private long sumStoreFiles( File storeDir )
return total;
}

private long sumNativeIndexFiles( File file, FilenameFilter filter )
private long sumIndexFiles( File file, FilenameFilter filter )
{
long total = 0;
if ( outsideWorld.fileSystem().isDirectory( file ) )
Expand All @@ -297,7 +305,7 @@ private long sumNativeIndexFiles( File file, FilenameFilter filter )
{
for ( File child : children )
{
total += sumNativeIndexFiles( child, filter );
total += sumIndexFiles( child, filter );
}
}
}
Expand Down
36 changes: 25 additions & 11 deletions ...unity/dbms/src/test/java/org/neo4j/commandline/dbms/MemoryRecommendationsCommandTest.java
View file
Expand Up @@ -60,6 +60,7 @@
import org.neo4j.values.storable.Values;

import static org.hamcrest.Matchers.both;
import static org.hamcrest.Matchers.containsString;
import static org.hamcrest.Matchers.greaterThan;
import static org.hamcrest.Matchers.greaterThanOrEqualTo;
import static org.hamcrest.Matchers.is;
Expand Down Expand Up @@ -199,26 +200,37 @@ public void stdOutLine( String text )
}
};
MemoryRecommendationsCommand command = new MemoryRecommendationsCommand( homeDir, configDir, outsideWorld );
String heap = bytesToString( recommendHeapMemory( gibiBytes( 8 ) ) );
String pagecache = bytesToString( recommendPageCacheMemory( gibiBytes( 8 ) ) );

// when
command.execute( array( "--database", databaseName, "--memory", "8g" ) );

// then
Map<String,String> stringMap = MapUtil.load( new StringReader( output.toString() ) );
long expectedSize = calculatePageCacheFileSize( storeDir );
assertThat( stringMap.get( pagecache_memory.name() ), is( bytesToString( expectedSize ) ) );
String memrecString = output.toString();
Map<String,String> stringMap = MapUtil.load( new StringReader( memrecString ) );
assertThat( stringMap.get( initialHeapSize.name() ), is( heap ) );
assertThat( stringMap.get( maxHeapSize.name() ), is( heap ) );
assertThat( stringMap.get( pagecache_memory.name() ), is( pagecache ) );

long[] expectedSizes = calculatePageCacheFileSize( storeDir );
long expectedPageCacheSize = expectedSizes[0];
long expectedLuceneSize = expectedSizes[1];
assertThat( memrecString, containsString( "Lucene indexes: " + bytesToString( expectedLuceneSize ) ) );
assertThat( memrecString, containsString( "Data volume and native indexes: " + bytesToString( expectedPageCacheSize ) ) );
}

private long calculatePageCacheFileSize( File storeDir ) throws IOException
private long[] calculatePageCacheFileSize( File storeDir ) throws IOException
{
MutableLong total = new MutableLong();
MutableLong pageCacheTotal = new MutableLong();
MutableLong luceneTotal = new MutableLong();
for ( StoreType storeType : StoreType.values() )
{
if ( storeType.isRecordStore() )
{
File file = new File( storeDir, storeType.getStoreFile().storeFileName() );
long length = file.length();
total.add( length );
pageCacheTotal.add( length );
}
}

Expand All @@ -231,14 +243,14 @@ public FileVisitResult visitFile( Path path, BasicFileAttributes attrs ) throws
Path name = path.getName( path.getNameCount() - 3 );
boolean isLuceneFile = (path.getNameCount() >= 3 && name.toString().startsWith( "lucene-" )) ||
(path.getNameCount() >= 4 && path.getName( path.getNameCount() - 4 ).toString().equals( "lucene" ));
if ( !FailureStorage.DEFAULT_FAILURE_FILE_NAME.equals( file.getName() ) && !isLuceneFile )
if ( !FailureStorage.DEFAULT_FAILURE_FILE_NAME.equals( file.getName() ) )
{
total.add( file.length() );
(isLuceneFile ? luceneTotal : pageCacheTotal).add( file.length() );
}
return FileVisitResult.CONTINUE;
}
} );
return total.longValue();
return new long[]{pageCacheTotal.longValue(), luceneTotal.longValue()};
}

private void createDatabaseWithNativeIndexes( File storeDir )
Expand All @@ -260,7 +272,7 @@ private void createDatabaseWithNativeIndexes( File storeDir )

try ( Transaction tx = db.beginTx() )
{
for ( int i = 0; i < 1_000; i++ )
for ( int i = 0; i < 10_000; i++ )
{
db.createNode( LABEL_ONE ).setProperty( key, randomIndexValue( i ) );
}
Expand All @@ -276,7 +288,7 @@ private void createDatabaseWithNativeIndexes( File storeDir )

private Object randomIndexValue( int i )
{
switch ( i % 10 )
switch ( i % 11 )
{
case 0:
return i;
Expand All @@ -298,6 +310,8 @@ private Object randomIndexValue( int i )
return Values.pointValue( CoordinateReferenceSystem.Cartesian, 1, 2 );
case 9:
return Values.pointValue( CoordinateReferenceSystem.Cartesian_3D, 1, 2, 3 );
case 10:
return new long[]{i, i + 1, i + 2, i + 3, i + 4, i + 5};
default:
throw new UnsupportedOperationException( "Unexpected" );
}
Expand Down

0 comments on commit 173972f

Please sign in to comment.