Skip to content

Commit

Permalink
Extracted common size calculations to FileUtils.
Browse files Browse the repository at this point in the history
Added jmx bean to causal cluster and added basic information:
* Role in core cluster
* Size of raft log
* Size of replicated states
  • Loading branch information
klaren committed Jul 3, 2017
1 parent 70c1919 commit c6be09e
Show file tree
Hide file tree
Showing 10 changed files with 374 additions and 63 deletions.
33 changes: 32 additions & 1 deletion community/io/src/main/java/org/neo4j/io/fs/FileUtils.java
Expand Up @@ -68,6 +68,7 @@ public class FileUtils


private FileUtils() private FileUtils()
{ {
throw new AssertionError();
} }


public static void deleteRecursively( File directory ) throws IOException public static void deleteRecursively( File directory ) throws IOException
Expand Down Expand Up @@ -515,7 +516,7 @@ public interface LineListener


public static LineListener echo( final PrintStream target ) public static LineListener echo( final PrintStream target )
{ {
return line -> target.println( line ); return target::println;
} }


public static void readTextFile( File file, LineListener listener ) throws IOException public static void readTextFile( File file, LineListener listener ) throws IOException
Expand Down Expand Up @@ -762,4 +763,34 @@ public FileVisitResult postVisitDirectory( Path dir, IOException exc )


return size.getValue(); return size.getValue();
} }

/**
* Calculates the size of a given directory or file given the provided abstract filesystem.
*
* @param fs the filesystem abstraction to use
* @param path to the file or directory.
* @return the size, in bytes, of the file or the total size of the content in the directory, including
* subdirectories.
*/
public static long size( FileSystemAbstraction fs, File path )
{
if ( fs.isDirectory( path ) )
{
long size = 0L;
File[] files = fs.listFiles( path );
if ( files == null )
{
return 0L;
}
for ( File child : files )
{
size += size( fs, child );
}
return size;
}
else
{
return fs.getFileSize( path );
}
}
} }
39 changes: 9 additions & 30 deletions community/jmx/src/main/java/org/neo4j/jmx/impl/StoreFileBean.java
Expand Up @@ -25,6 +25,7 @@


import org.neo4j.helpers.Service; import org.neo4j.helpers.Service;
import org.neo4j.io.fs.FileSystemAbstraction; import org.neo4j.io.fs.FileSystemAbstraction;
import org.neo4j.io.fs.FileUtils;
import org.neo4j.jmx.StoreFile; import org.neo4j.jmx.StoreFile;
import org.neo4j.kernel.NeoStoreDataSource; import org.neo4j.kernel.NeoStoreDataSource;
import org.neo4j.kernel.api.index.SchemaIndexProvider; import org.neo4j.kernel.api.index.SchemaIndexProvider;
Expand Down Expand Up @@ -123,40 +124,18 @@ private File resolvePath( NeoStoreDataSource ds )
@Override @Override
public long getTotalStoreSize() public long getTotalStoreSize()
{ {
return storePath == null ? 0 : sizeOf( fs, storePath ); return storePath == null ? 0 : FileUtils.size( fs, storePath );
} }


@Override @Override
public long getLogicalLogSize() public long getLogicalLogSize()
{ {
return logFile == null ? 0 : sizeOf( fs, logFile.currentLogFile() ); return logFile == null ? 0 : FileUtils.size( fs, logFile.currentLogFile() );
}

private static long sizeOf( FileSystemAbstraction fs, File file )
{
if ( fs.isDirectory( file ) )
{
long size = 0;
File[] files = fs.listFiles( file );
if ( files == null )
{
return 0;
}
for ( File child : files )
{
size += sizeOf( fs, child );
}
return size;
}
else
{
return fs.getFileSize( file );
}
} }


private long sizeOf( String name ) private long sizeOf( String name )
{ {
return storePath == null ? 0 : sizeOf( fs, new File( storePath, name ) ); return storePath == null ? 0 : FileUtils.size( fs, new File( storePath, name ) );
} }


@Override @Override
Expand All @@ -178,19 +157,19 @@ public long getAllLogicalLogsSize()
@Override @Override
public long getIndexStoreSize() public long getIndexStoreSize()
{ {
long size = 0; long size = 0L;


// Add legacy indices // Add legacy indices
for ( IndexImplementation index : legacyIndexProviderLookup.all() ) for ( IndexImplementation index : legacyIndexProviderLookup.all() )
{ {
size += sizeOf( fs, index.getIndexImplementationDirectory( storePath ) ); size += FileUtils.size( fs, index.getIndexImplementationDirectory( storePath ) );
} }


// Add schema index // Add schema index
size += sizeOf( fs, schemaIndexProvider.getSchemaIndexStoreDirectory( storePath ) ); size += FileUtils.size( fs, schemaIndexProvider.getSchemaIndexStoreDirectory( storePath ) );


// Add label index // Add label index
size += sizeOf( fs, labelScanStore.getLabelScanStoreFile() ); size += FileUtils.size( fs, labelScanStore.getLabelScanStoreFile() );


return size; return size;
} }
Expand Down Expand Up @@ -237,7 +216,7 @@ long getTotalSize()
@Override @Override
public void visit( File file, long logVersion ) public void visit( File file, long logVersion )
{ {
totalSize += sizeOf( fs, file ); totalSize += FileUtils.size( fs, file );
} }
} }
} }
Expand Down
Expand Up @@ -24,6 +24,7 @@
import org.mockito.Mockito; import org.mockito.Mockito;


import java.io.File; import java.io.File;
import java.io.IOException;
import java.nio.ByteBuffer; import java.nio.ByteBuffer;


import org.neo4j.graphdb.mockfs.EphemeralFileSystemAbstraction; import org.neo4j.graphdb.mockfs.EphemeralFileSystemAbstraction;
Expand Down Expand Up @@ -94,19 +95,8 @@ public void setUp() throws Throwable
@Test @Test
public void shouldCountAllLogFiles() throws Throwable public void shouldCountAllLogFiles() throws Throwable
{ {
try ( StoreChannel storeChannel = fs.create( physicalLogFiles.getLogFileForVersion( 0 ) ) ) createFileOfSize( physicalLogFiles.getLogFileForVersion( 0 ), 10 );
{ createFileOfSize( physicalLogFiles.getLogFileForVersion( 1 ), 20 );
byte[] bytes = new byte[10];
ByteBuffer buffer = ByteBuffer.wrap( bytes );
storeChannel.writeAll( buffer );
}

try ( StoreChannel storeChannel = fs.create( physicalLogFiles.getLogFileForVersion( 1 ) ) )
{
byte[] bytes = new byte[20];
ByteBuffer buffer = ByteBuffer.wrap( bytes );
storeChannel.writeAll( buffer );
}


assertEquals( 30, storeFileBean.getAllLogicalLogsSize() ); assertEquals( 30, storeFileBean.getAllLogicalLogsSize() );
} }
Expand All @@ -116,38 +106,33 @@ public void shouldCountAllIndexFiles() throws Exception
{ {
// Legacy index file // Legacy index file
File legacyIndex = new File( storeDir, "legacyIndex" ); File legacyIndex = new File( storeDir, "legacyIndex" );
try ( StoreChannel storeChannel = fs.create( legacyIndex ) ) createFileOfSize( legacyIndex, 10 );
{
byte[] bytes = new byte[10];
ByteBuffer buffer = ByteBuffer.wrap( bytes );
storeChannel.writeAll( buffer );
}


IndexImplementation indexImplementation = mock( IndexImplementation.class ); IndexImplementation indexImplementation = mock( IndexImplementation.class );
when( indexImplementation.getIndexImplementationDirectory( Mockito.any() ) ).thenReturn( legacyIndex ); when( indexImplementation.getIndexImplementationDirectory( Mockito.any() ) ).thenReturn( legacyIndex );
when( legacyIndexProviderLookup.all() ).thenReturn( iterable( indexImplementation ) ); when( legacyIndexProviderLookup.all() ).thenReturn( iterable( indexImplementation ) );


// Legacy index file // Legacy index file
File schemaIndex = new File( storeDir, "schemaIndex" ); File schemaIndex = new File( storeDir, "schemaIndex" );
try ( StoreChannel storeChannel = fs.create( schemaIndex ) ) createFileOfSize( schemaIndex, 5 );
{
byte[] bytes = new byte[5];
ByteBuffer buffer = ByteBuffer.wrap( bytes );
storeChannel.writeAll( buffer );
}
when( schemaIndexProvider.getSchemaIndexStoreDirectory( Mockito.any() ) ).thenReturn( schemaIndex ); when( schemaIndexProvider.getSchemaIndexStoreDirectory( Mockito.any() ) ).thenReturn( schemaIndex );


// Label scan store // Label scan store
File labelScan = new File( storeDir, "labelScanStore" ); File labelScan = new File( storeDir, "labelScanStore" );
try ( StoreChannel storeChannel = fs.create( labelScan ) ) createFileOfSize( labelScan, 20 );
{
byte[] bytes = new byte[20];
ByteBuffer buffer = ByteBuffer.wrap( bytes );
storeChannel.writeAll( buffer );
}
when( labelScanStore.getLabelScanStoreFile() ).thenReturn( labelScan ); when( labelScanStore.getLabelScanStoreFile() ).thenReturn( labelScan );


// Count all files // Count all files
assertEquals( 35, storeFileBean.getIndexStoreSize() ); assertEquals( 35, storeFileBean.getIndexStoreSize() );
} }

private void createFileOfSize( File file, int size ) throws IOException
{
try ( StoreChannel storeChannel = fs.create( file ) )
{
byte[] bytes = new byte[size];
ByteBuffer buffer = ByteBuffer.wrap( bytes );
storeChannel.writeAll( buffer );
}
}
} }
11 changes: 11 additions & 0 deletions enterprise/causal-clustering/pom.xml
Expand Up @@ -73,6 +73,17 @@
<version>${project.version}</version> <version>${project.version}</version>
</dependency> </dependency>


<dependency>
<groupId>org.neo4j</groupId>
<artifactId>neo4j-jmx</artifactId>
<version>${project.version}</version>
</dependency>
<dependency>
<groupId>org.neo4j</groupId>
<artifactId>neo4j-management</artifactId>
<version>${project.version}</version>
</dependency>

<dependency> <dependency>
<groupId>org.neo4j</groupId> <groupId>org.neo4j</groupId>
<artifactId>neo4j-consistency-check</artifactId> <artifactId>neo4j-consistency-check</artifactId>
Expand Down
Expand Up @@ -171,6 +171,7 @@ public void registerEditionSpecificProcedures( Procedures procedures ) throws Ke
{ {
throw new RuntimeException( e ); throw new RuntimeException( e );
} }
dependencies.satisfyDependency( clusterStateDirectory );


eligibleForIdReuse = IdReuseEligibility.ALWAYS; eligibleForIdReuse = IdReuseEligibility.ALWAYS;


Expand Down
@@ -0,0 +1,135 @@
/*
* Copyright (c) 2002-2017 "Neo Technology,"
* Network Engine for Objects in Lund AB [http://neotechnology.com]
*
* This file is part of Neo4j.
*
* Neo4j is free software: you can redistribute it and/or modify
* it under the terms of the GNU Affero General Public License as
* published by the Free Software Foundation, either version 3 of the
* License, or (at your option) any later version.
*
* This program is distributed in the hope that it will be useful,
* but WITHOUT ANY WARRANTY; without even the implied warranty of
* MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
* GNU Affero General Public License for more details.
*
* You should have received a copy of the GNU Affero General Public License
* along with this program. If not, see <http://www.gnu.org/licenses/>.
*/
package org.neo4j.causalclustering.management;


import java.io.File;
import javax.management.NotCompliantMBeanException;

import org.neo4j.causalclustering.core.CoreGraphDatabase;
import org.neo4j.causalclustering.core.consensus.RaftMachine;
import org.neo4j.causalclustering.core.state.ClusterStateDirectory;
import org.neo4j.helpers.Service;
import org.neo4j.io.fs.FileSystemAbstraction;
import org.neo4j.io.fs.FileUtils;
import org.neo4j.jmx.impl.ManagementBeanProvider;
import org.neo4j.jmx.impl.ManagementData;
import org.neo4j.jmx.impl.Neo4jMBean;
import org.neo4j.management.CausalClustering;

import static org.neo4j.causalclustering.core.consensus.log.RaftLog.RAFT_LOG_DIRECTORY_NAME;

@Service.Implementation( ManagementBeanProvider.class )
public class CausalClusteringBean extends ManagementBeanProvider
{
CausalClusteringBean()
{
super( CausalClustering.class );
}

@Override
protected Neo4jMBean createMBean( ManagementData management ) throws NotCompliantMBeanException
{
if ( isCausalClustering( management ) )
{
return new CausalClusteringBeanImpl( management );
}
return null;
}

@Override
protected Neo4jMBean createMXBean( ManagementData management ) throws NotCompliantMBeanException
{
if ( isCausalClustering( management ) )
{
return new CausalClusteringBeanImpl( management, true );
}
return null;
}

private static boolean isCausalClustering( ManagementData management )
{
return management.getKernelData().graphDatabase() instanceof CoreGraphDatabase;
}

private static class CausalClusteringBeanImpl extends Neo4jMBean implements CausalClustering
{
private final ClusterStateDirectory clusterStateDirectory;
private final RaftMachine raftMachine;
private final FileSystemAbstraction fs;

CausalClusteringBeanImpl( ManagementData management ) throws NotCompliantMBeanException
{
super( management );
clusterStateDirectory = management.resolveDependency( ClusterStateDirectory.class );
raftMachine = management.resolveDependency( RaftMachine.class );

fs = management.getKernelData().getFilesystemAbstraction();
}

CausalClusteringBeanImpl( ManagementData management, boolean isMXBean )
{
super( management, isMXBean );
clusterStateDirectory = management.resolveDependency( ClusterStateDirectory.class );
raftMachine = management.resolveDependency( RaftMachine.class );

fs = management.getKernelData().getFilesystemAbstraction();
}

@Override
public String getRole()
{
return raftMachine.currentRole().toString();
}

@Override
public long getRaftLogSize()
{
File raftLogDirectory = new File( clusterStateDirectory.get(), RAFT_LOG_DIRECTORY_NAME );
return FileUtils.size( fs, raftLogDirectory );
}

@Override
public long getReplicatedStateSize()
{
File replicatedStateDirectory = clusterStateDirectory.get();

File[] files = fs.listFiles( replicatedStateDirectory );
if ( files == null )
{
return 0L;
}

long size = 0L;
for ( File file : files )
{
// Exclude raft log that resides in same directory
if ( fs.isDirectory( file ) && file.getName().equals( RAFT_LOG_DIRECTORY_NAME ) )
{
continue;
}

size += FileUtils.size( fs, file );
}

return size;
}
}
}
@@ -0,0 +1 @@
org.neo4j.causalclustering.management.CausalClusteringBean

0 comments on commit c6be09e

Please sign in to comment.