Skip to content

Commit

Permalink
Report command handles several dbs and allows for specifying which
Browse files Browse the repository at this point in the history 
By default report will collect for all databases
  • Loading branch information
@emmaholmbergohlsson
emmaholmbergohlsson committed Feb 14, 2024
1 parent a0f1bdc commit 07826ae
Show file tree
Hide file tree
Showing 9 changed files with 135 additions and 65 deletions.
49 changes: 25 additions & 24 deletions ...unity-it/dbms-it/src/test/java/org/neo4j/commandline/dbms/DiagnosticsReportCommandIT.java
View file
Expand Up @@ -19,6 +19,7 @@
*/
package org.neo4j.commandline.dbms;

import java.nio.file.StandardOpenOption;
import org.junit.jupiter.api.AfterEach;
import org.junit.jupiter.api.BeforeEach;
import org.junit.jupiter.api.Test;
Expand Down Expand Up @@ -75,6 +76,7 @@ class DiagnosticsReportCommandIT
void setUp() throws Exception
{
homeDir = testDirectory.directory( "home-dir" );
createDatabaseDir(homeDir);
configDir = testDirectory.directory( "config-dir" );

// Touch config
Expand All @@ -87,6 +89,12 @@ void setUp() throws Exception
ctx = new ExecutionContext( homeDir, configDir, System.out, System.err, fs );
}

private void createDatabaseDir( Path homeDir ) throws IOException
{
// Database directory needed for command to be able to collect anything
Files.createDirectories( homeDir.resolve( "data" ).resolve( "databases" ).resolve( "neo4j" ));
}

@AfterEach
void tearDown()
{
Expand All @@ -100,19 +108,15 @@ void shouldBeAbleToAttachToPidAndRunThreadDump() throws IOException
long pid = getPID();
assertThat( pid ).isNotEqualTo( 0 );

// Write config file
Files.createFile( testDirectory.file( "neo4j.conf" ) );

// write neo4j.pid file
Path run = testDirectory.directory( "run" );
Path run = testDirectory.directory( "run", homeDir.getFileName().toString());
Files.write( run.resolve( "neo4j.pid" ), String.valueOf( pid ).getBytes() );

// Run command, should detect running instance
try
{
String[] args = {"threads", "--to=" + testDirectory.absolutePath() + "/reports"};
Path homeDir = testDirectory.homePath();
var ctx = new ExecutionContext( homeDir, homeDir, System.out, System.err, testDirectory.getFileSystem() );
var ctx = new ExecutionContext( homeDir, configDir, System.out, System.err, testDirectory.getFileSystem() );
DiagnosticsReportCommand diagnosticsReportCommand = new DiagnosticsReportCommand( ctx );
CommandLine.populateCommand( diagnosticsReportCommand, args );
diagnosticsReportCommand.execute();
Expand Down Expand Up @@ -148,19 +152,15 @@ void shouldBeAbleToAttachToPidAndRunHeapDump() throws IOException
long pid = getPID();
assertThat( pid ).isNotEqualTo( 0 );

// Write config file
Files.createFile( testDirectory.file( "neo4j.conf" ) );

// write neo4j.pid file
Path run = testDirectory.directory( "run" );
Files.write( run.resolve( "neo4j.pid" ), String.valueOf( pid ).getBytes() );
Path run = testDirectory.directory( "run", homeDir.getFileName().toString() );
Files.write( run.resolve( "neo4j.pid" ), String.valueOf( pid ).getBytes(), StandardOpenOption.CREATE );

// Run command, should detect running instance
try
{
String[] args = {"heap", "--to=" + testDirectory.absolutePath() + "/reports"};
Path homeDir = testDirectory.homePath();
var ctx = new ExecutionContext( homeDir, homeDir, System.out, System.err, testDirectory.getFileSystem() );
var ctx = new ExecutionContext( homeDir, configDir, System.out, System.err, testDirectory.getFileSystem() );
DiagnosticsReportCommand diagnosticsReportCommand = new DiagnosticsReportCommand( ctx );
CommandLine.populateCommand( diagnosticsReportCommand, args );
diagnosticsReportCommand.execute();
Expand Down Expand Up @@ -190,19 +190,20 @@ void shouldBeAbleToAttachToPidAndRunHeapDump() throws IOException
void shouldHandleRotatedLogFiles() throws IOException
{
// Write config file and specify a custom name for the neo4j.log file.
Path confFile = testDirectory.createFile( "neo4j.conf" );
Files.write( confFile, singletonList( GraphDatabaseSettings.store_user_log_path.name() + "=custom.neo4j.log.name" ) );

// Create some log files that should be found. debug.log has already been created during setup.
testDirectory.directory( "logs" );
testDirectory.createFile( "logs/debug.log" );
testDirectory.createFile( "logs/debug.log.1.zip" );
testDirectory.createFile( "logs/custom.neo4j.log.name" );
testDirectory.createFile( "logs/custom.neo4j.log.name.1" );
Files.write(
configDir.resolve("neo4j.conf"), singletonList(GraphDatabaseSettings.store_user_log_path.name() + "=custom.neo4j.log.name"));

// Create some log files that should be found.
Path logDir =
testDirectory.directory( "logs", homeDir.getFileName().toString() );
FileSystemAbstraction fs = testDirectory.getFileSystem();
fs.write( logDir.resolve( "debug.log" ) );
fs.write( logDir.resolve( "debug.log.1.zip" ) );
fs.write( logDir.resolve( "custom.neo4j.log.name" ) );
fs.write( logDir.resolve( "custom.neo4j.log.name.1" ) );

String[] args = {"logs", "--to=" + testDirectory.absolutePath() + "/reports"};
Path homeDir = testDirectory.homePath();
var ctx = new ExecutionContext( homeDir, homeDir, System.out, System.err, testDirectory.getFileSystem() );
var ctx = new ExecutionContext( homeDir, configDir, System.out, System.err, testDirectory.getFileSystem() );
DiagnosticsReportCommand diagnosticsReportCommand = new DiagnosticsReportCommand( ctx );
CommandLine.populateCommand( diagnosticsReportCommand, args );
diagnosticsReportCommand.execute();
Expand Down
80 changes: 64 additions & 16 deletions community/dbms/src/main/java/org/neo4j/commandline/dbms/DiagnosticsReportCommand.java
View file
Expand Up @@ -19,8 +19,12 @@
*/
package org.neo4j.commandline.dbms;

import org.jutils.jprocesses.JProcesses;
import org.jutils.jprocesses.model.ProcessInfo;
import static java.lang.String.join;
import static org.apache.commons.text.StringEscapeUtils.escapeCsv;
import static picocli.CommandLine.Command;
import static picocli.CommandLine.Help.Visibility.NEVER;
import static picocli.CommandLine.Option;
import static picocli.CommandLine.Parameters;

import java.io.IOException;
import java.net.InetAddress;
Expand All @@ -33,30 +37,28 @@
import java.util.Optional;
import java.util.Set;
import java.util.TreeSet;

import org.eclipse.collections.impl.factory.Sets;
import org.jutils.jprocesses.JProcesses;
import org.jutils.jprocesses.model.ProcessInfo;
import org.neo4j.cli.AbstractCommand;
import org.neo4j.cli.CommandFailedException;
import org.neo4j.cli.Converters;
import org.neo4j.cli.ExecutionContext;
import org.neo4j.configuration.Config;
import org.neo4j.configuration.ConfigUtils;
import org.neo4j.configuration.GraphDatabaseSettings;
import org.neo4j.configuration.helpers.DatabaseNamePattern;
import org.neo4j.dbms.diagnostics.jmx.JMXDumper;
import org.neo4j.dbms.diagnostics.jmx.JmxDump;
import org.neo4j.io.fs.FileSystemAbstraction;
import org.neo4j.io.layout.Neo4jLayout;
import org.neo4j.kernel.diagnostics.DiagnosticsReportSource;
import org.neo4j.kernel.diagnostics.DiagnosticsReportSources;
import org.neo4j.kernel.diagnostics.DiagnosticsReporter;
import org.neo4j.kernel.diagnostics.DiagnosticsReporterProgress;
import org.neo4j.kernel.diagnostics.InteractiveProgress;
import org.neo4j.kernel.diagnostics.NonInteractiveProgress;

import static java.lang.String.join;
import static org.apache.commons.text.StringEscapeUtils.escapeCsv;
import static org.neo4j.configuration.GraphDatabaseInternalSettings.databases_root_path;
import static picocli.CommandLine.Command;
import static picocli.CommandLine.Help.Visibility.NEVER;
import static picocli.CommandLine.Option;
import static picocli.CommandLine.Parameters;

@Command(
name = "report",
header = "Produces a zip/tar of the most common information needed for remote assessments.",
Expand All @@ -70,6 +72,16 @@ public class DiagnosticsReportCommand extends AbstractCommand
private static final DateTimeFormatter filenameDateTimeFormatter = DateTimeFormatter.ofPattern("yyyy-MM-dd_HHmmss" );
private static final long NO_PID = 0;

@Option(
names = "--database",
paramLabel = "<database>",
defaultValue = "*",
description = "Name of the database to report for. Can contain * and ? for globbing. "
+ "Note that * and ? have special meaning in some shells "
+ "and might need to be escaped or used with quotes.",
converter = Converters.DatabaseNamePatternConverter.class )
private DatabaseNamePattern database;

@Option( names = "--list", arity = "0", description = "List all available classifiers" )
private boolean list;

Expand Down Expand Up @@ -98,7 +110,8 @@ public void execute()
Config config = getConfig( configFile() );

jmxDumper = new JMXDumper( config, ctx.fs(), ctx.out(), ctx.err(), verbose );
DiagnosticsReporter reporter = createAndRegisterSources( config );
Set<String> dbNames = getDbNames(config, ctx.fs(), database);
DiagnosticsReporter reporter = createAndRegisterSources(config, dbNames);

if ( list )
{
Expand Down Expand Up @@ -182,13 +195,11 @@ private void listClassifiers( Set<String> availableClassifiers )
}
}

private DiagnosticsReporter createAndRegisterSources( Config config )
private DiagnosticsReporter createAndRegisterSources( Config config, Set<String> databaseNames )
{
DiagnosticsReporter reporter = new DiagnosticsReporter();

Path storeDirectory = config.get( databases_root_path );

reporter.registerAllOfflineProviders( config, storeDirectory, ctx.fs(), config.get( GraphDatabaseSettings.default_database ) );
reporter.registerAllOfflineProviders( config, ctx.fs(), databaseNames );

// Register sources provided by this tool
reporter.registerSource( "config",
Expand Down Expand Up @@ -309,4 +320,41 @@ private static DiagnosticsReportSource runningProcesses()
return sb.toString();
} );
}

private Set<String> getDbNames( Config config, FileSystemAbstraction fs, DatabaseNamePattern database )
{
if ( !database.containsPattern() )
{
return Set.of( database.getDatabaseName() );
}
else
{
Set<String> dbNames = Sets.mutable.empty();
Path databasesDir = Neo4jLayout.of( config ).databasesDirectory();
try
{
for ( Path path : fs.listFiles( databasesDir ) )
{
if ( fs.isDirectory( path ) )
{
String name = path.getFileName().toString();
if ( database.matches( name ) )
{
dbNames.add( name );
}
}
}
}
catch ( IOException e )
{
throw new CommandFailedException( "Failed to list databases", e );
}
if ( dbNames.isEmpty() )
{
throw new CommandFailedException(
"Pattern '" + database.getDatabaseName() + "' did not match any database" );
}
return dbNames;
}
}
}
10 changes: 8 additions & 2 deletions community/dbms/src/test/java/org/neo4j/commandline/dbms/DiagnosticsReportCommandTest.java
View file
Expand Up @@ -46,8 +46,8 @@ void printUsageHelp()
"%n" +
"USAGE%n" +
"%n" +
"report [--expand-commands] [--force] [--list] [--verbose] [--pid=<pid>]%n" +
" [--to=<path>] [<classifier>...]%n" +
"report [--expand-commands] [--force] [--list] [--verbose]%n" +
" [--database=<database>] [--pid=<pid>] [--to=<path>] [<classifier>...]%n" +
"%n" +
"DESCRIPTION%n" +
"%n" +
Expand All @@ -64,6 +64,12 @@ void printUsageHelp()
"%n" +
" --verbose Enable verbose output.%n" +
" --expand-commands Allow command expansion in config value evaluation.%n" +
" --database=<database>%n" +
" Name of the database to report for. Can contain * and%n" +
" ? for globbing. Note that * and ? have special%n" +
" meaning in some shells and might need to be escaped%n" +
" or used with quotes.%n" +
" Default: *%n" +
" --list List all available classifiers%n" +
" --force Ignore disk full warning%n" +
" --to=<path> Destination directory for reports. Defaults to a%n" +
Expand Down
13 changes: 6 additions & 7 deletions ...y/kernel/src/main/java/org/neo4j/kernel/diagnostics/DiagnosticsOfflineReportProvider.java
View file
Expand Up @@ -19,7 +19,6 @@
*/
package org.neo4j.kernel.diagnostics;

import java.nio.file.Path;
import java.util.Arrays;
import java.util.HashSet;
import java.util.List;
Expand All @@ -33,7 +32,7 @@
* Base class for a provider of offline reports. Offline reports does not require a running instance of the database
* and is intended to be use as a way to gather information even if the database cannot be started. All implementing
* classes is service loaded and initialized through the
* {@link DiagnosticsOfflineReportProvider#init(FileSystemAbstraction, String, Config, Path)} method.
* {@link DiagnosticsOfflineReportProvider#init(FileSystemAbstraction, Config, Set)} method.
*/
@Service
public abstract class DiagnosticsOfflineReportProvider
Expand All @@ -55,12 +54,12 @@ protected DiagnosticsOfflineReportProvider( String classifier, String... classif

/**
* Called after service loading to initialize the class.
* @param fs filesystem to use for file access.
* @param defaultDatabaseName identifier for default database
* @param config configuration file in use.
* @param storeDirectory directory of the database files.
*
* @param fs filesystem to use for file access.
* @param config configuration file in use.
* @param databaseNames the databases to report for.
*/
public abstract void init( FileSystemAbstraction fs, String defaultDatabaseName, Config config, Path storeDirectory );
public abstract void init( FileSystemAbstraction fs, Config config, Set<String> databaseNames );

/**
* Returns a list of source that matches the given classifiers.
Expand Down
4 changes: 2 additions & 2 deletions community/kernel/src/main/java/org/neo4j/kernel/diagnostics/DiagnosticsReporter.java
View file
Expand Up @@ -143,11 +143,11 @@ public Set<String> getAvailableClassifiers()
return availableClassifiers;
}

public void registerAllOfflineProviders( Config config, Path storeDirectory, FileSystemAbstraction fs, String defaultDatabaseName )
public void registerAllOfflineProviders( Config config, FileSystemAbstraction fs, Set<String> databaseNames )
{
for ( DiagnosticsOfflineReportProvider provider : Services.loadAll( DiagnosticsOfflineReportProvider.class ) )
{
provider.init( fs, defaultDatabaseName, config, storeDirectory );
provider.init( fs, config, databaseNames );
registerOfflineProvider( provider );
}
}
Expand Down

0 comments on commit 07826ae

Please sign in to comment.