Skip to content

Commit

Permalink
Refactor ImportTool to allow output to be handled in a better way.
Browse files Browse the repository at this point in the history
  • Loading branch information
@srbaker
srbaker committed Sep 23, 2016
1 parent e0267f4 commit 77c25dc
Showing 1 changed file with 70 additions and 62 deletions.
132 changes: 70 additions & 62 deletions community/import-tool/src/main/java/org/neo4j/tooling/ImportTool.java
View file
Expand Up @@ -28,7 +28,6 @@
import java.util.Arrays;
import java.util.Collection;
import java.util.Map.Entry;
import java.util.function.BiFunction;
import java.util.function.Function;

import org.neo4j.csv.reader.IllegalMultilineFieldException;
Expand Down Expand Up @@ -315,10 +314,13 @@ public static void main( String[] incomingArguments ) throws IOException
*/
public static void main( String[] incomingArguments, boolean defaultSettingsSuitableForTests ) throws IOException
{
PrintStream out = System.out;
PrintStream err = System.err;
Args args = Args.parse( incomingArguments );

if ( ArrayUtil.isEmpty( incomingArguments ) || asksForUsage( args ) )
{
printUsage( System.out );
printUsage( out );
return;
}

Expand All @@ -344,8 +346,8 @@ public static void main( String[] incomingArguments, boolean defaultSettingsSuit

File badFile = new File( storeDir, BAD_FILE_NAME );
badOutput = new BufferedOutputStream( fs.openAsOutputStream( badFile, false ) );
nodesFiles = INPUT_FILES_EXTRACTOR.apply( args, Options.NODE_DATA.key() );
relationshipsFiles = INPUT_FILES_EXTRACTOR.apply( args, Options.RELATIONSHIP_DATA.key() );
nodesFiles = extractInputFiles( args, Options.NODE_DATA.key(), err );
relationshipsFiles = extractInputFiles( args, Options.RELATIONSHIP_DATA.key(), err );
validateInputFiles( nodesFiles, relationshipsFiles );
enableStacktrace = args.getBoolean( Options.STACKTRACE.key(), Boolean.FALSE, Boolean.TRUE );
processors = args.getNumber( Options.PROCESSORS.key(), null );
Expand Down Expand Up @@ -380,11 +382,11 @@ idType, csvConfiguration( args, defaultSettingsSuitableForTests ), badCollector,
}
catch ( IllegalArgumentException e )
{
throw andPrintError( "Input error", e, false );
throw andPrintError( "Input error", e, false, err );
}
catch ( IOException e )
{
throw andPrintError( "File error", e, false );
throw andPrintError( "File error", e, false, err );
}
finally
{
Expand All @@ -404,7 +406,7 @@ idType, csvConfiguration( args, defaultSettingsSuitableForTests ), badCollector,
logService,
ExecutionMonitors.defaultVisible(),
dbConfig );
printOverview( storeDir, nodesFiles, relationshipsFiles, configuration );
printOverview( storeDir, nodesFiles, relationshipsFiles, configuration, out );
success = false;
try
{
Expand All @@ -413,7 +415,7 @@ idType, csvConfiguration( args, defaultSettingsSuitableForTests ), badCollector,
}
catch ( Exception e )
{
throw andPrintError( "Import error", e, enableStacktrace );
throw andPrintError( "Import error", e, enableStacktrace, err );
}
finally
{
Expand All @@ -425,7 +427,7 @@ idType, csvConfiguration( args, defaultSettingsSuitableForTests ), badCollector,
File badFile = new File( storeDir, BAD_FILE_NAME );
if ( badFile.exists() )
{
System.out.println(
out.println(
"There were bad entries which were skipped and logged into " + badFile.getAbsolutePath() );
}
}
Expand All @@ -441,7 +443,7 @@ idType, csvConfiguration( args, defaultSettingsSuitableForTests ), badCollector,
}
catch ( IOException e )
{
System.err.println( "Unable to delete store files after an aborted import " + e );
err.println( "Unable to delete store files after an aborted import " + e );
if ( enableStacktrace )
{
e.printStackTrace();
Expand All @@ -451,56 +453,81 @@ idType, csvConfiguration( args, defaultSettingsSuitableForTests ), badCollector,
}
}

private static Collection<Option<File[]>> extractInputFiles( Args args, String key, PrintStream err )
{
return args
.interpretOptionsWithMetadata( key, Converters.<File[]>optional(),
Converters.toFiles( MULTI_FILE_DELIMITER, Converters.regexFiles( true ) ), filesExist(
err ),
Validators.<File>atLeast( "--" + key, 1 ) );
}

private static Validator<File[]> filesExist( PrintStream err )
{
return files -> {
for ( File file : files )
{
if ( file.getName().startsWith( ":" ) )
{
err.println( "It looks like you're trying to specify default label or relationship type (" +
file.getName() + "). Please put such directly on the key, f.ex. " +
Options.NODE_DATA.argument() + ":MyLabel" );
}
Validators.REGEX_FILE_EXISTS.validate( file );
}
};
}

private static Config loadDbConfig( File file ) throws IOException
{
return file != null && file.exists() ? new Config( MapUtil.load( file ) ) : Config.defaults();
}

private static void printOverview( File storeDir, Collection<Option<File[]>> nodesFiles,
Collection<Option<File[]>> relationshipsFiles,
org.neo4j.unsafe.impl.batchimport.Configuration configuration )
org.neo4j.unsafe.impl.batchimport.Configuration configuration, PrintStream out )
{
System.out.println( "Neo4j version: " + Version.getKernel().getReleaseVersion() );
System.out.println( "Importing the contents of these files into " + storeDir + ":" );
printInputFiles( "Nodes", nodesFiles );
printInputFiles( "Relationships", relationshipsFiles );
System.out.println();
System.out.println( "Available resources:" );
printIndented( "Free machine memory: " + bytes( OsBeanUtil.getFreePhysicalMemory() ) );
printIndented( "Max heap memory : " + bytes( Runtime.getRuntime().maxMemory() ) );
printIndented( "Processors: " + configuration.maxNumberOfProcessors() );
System.out.println();
out.println( "Neo4j version: " + Version.getKernel().getReleaseVersion() );
out.println( "Importing the contents of these files into " + storeDir + ":" );
printInputFiles( "Nodes", nodesFiles, out );
printInputFiles( "Relationships", relationshipsFiles, out );
out.println();
out.println( "Available resources:" );
printIndented( "Free machine memory: " + bytes( OsBeanUtil.getFreePhysicalMemory() ), out );
printIndented( "Max heap memory : " + bytes( Runtime.getRuntime().maxMemory() ), out );
printIndented( "Processors: " + configuration.maxNumberOfProcessors(), out );
out.println();
}

private static void printInputFiles( String name, Collection<Option<File[]>> files )
private static void printInputFiles( String name, Collection<Option<File[]>> files, PrintStream out )
{
if ( files.isEmpty() )
{
return;
}

System.out.println( name + ":" );
out.println( name + ":" );
int i = 0;
for ( Option<File[]> group : files )
{
if ( i++ > 0 )
{
System.out.println();
out.println();
}
if ( group.metadata() != null )
{
printIndented( ":" + group.metadata() );
printIndented( ":" + group.metadata(), out );
}
for ( File file : group.value() )
{
printIndented( file );
printIndented( file, out );
}
}
}

private static void printIndented( Object value )
private static void printIndented( Object value, PrintStream out )
{
System.out.println( " " + value );
out.println( " " + value );
}

private static void validateInputFiles( Collection<Option<File[]>> nodesFiles,
Expand Down Expand Up @@ -562,22 +589,26 @@ private static String manualReference( ManualPage page, Anchor anchor )
/**
* Method name looks strange, but look at how it's used and you'll see why it's named like that.
* @param stackTrace whether or not to also print the stack trace of the error.
* @param err
*/
private static RuntimeException andPrintError( String typeOfError, Exception e, boolean stackTrace )
private static RuntimeException andPrintError( String typeOfError, Exception e, boolean stackTrace,
PrintStream err )
{
// List of common errors that can be explained to the user
if ( DuplicateInputIdException.class.equals( e.getClass() ) )
{
printErrorMessage( "Duplicate input ids that would otherwise clash can be put into separate id space, " +
"read more about how to use id spaces in the manual:" +
manualReference( ManualPage.IMPORT_TOOL_FORMAT, Anchor.ID_SPACES ), e, stackTrace );
manualReference( ManualPage.IMPORT_TOOL_FORMAT, Anchor.ID_SPACES ), e, stackTrace,
err );
}
else if ( MissingRelationshipDataException.class.equals( e.getClass() ) )
{
printErrorMessage( "Relationship missing mandatory field '" +
((MissingRelationshipDataException) e).getFieldType() + "', read more about " +
"relationship format in the manual: " +
manualReference( ManualPage.IMPORT_TOOL_FORMAT, Anchor.RELATIONSHIP ), e, stackTrace );
manualReference( ManualPage.IMPORT_TOOL_FORMAT, Anchor.RELATIONSHIP ), e, stackTrace,
err );
}
// This type of exception is wrapped since our input code throws InputException consistently,
// and so IllegalMultilineFieldException comes from the csv component, which has no access to InputException
Expand All @@ -587,18 +618,18 @@ else if ( Exceptions.contains( e, IllegalMultilineFieldException.class ) )
printErrorMessage( "Detected field which spanned multiple lines for an import where " +
Options.MULTILINE_FIELDS.argument() + "=false. If you know that your input data " +
"include fields containing new-line characters then import with this option set to " +
"true.", e, stackTrace );
"true.", e, stackTrace, err );
}
else if ( Exceptions.contains( e, InputException.class ) )
{
printErrorMessage( "Error in input data", e, stackTrace );
printErrorMessage( "Error in input data", e, stackTrace, err );
}
// Fallback to printing generic error and stack trace
else
{
printErrorMessage( typeOfError + ": " + e.getMessage(), e, true );
printErrorMessage( typeOfError + ": " + e.getMessage(), e, true, err );
}
System.err.println();
err.println();

// Mute the stack trace that the default exception handler would have liked to print.
// Calling System.exit( 1 ) or similar would be convenient on one hand since we can set
Expand All @@ -608,13 +639,13 @@ else if ( Exceptions.contains( e, InputException.class ) )
return launderedException( e ); // throw in order to have process exit with !0
}

private static void printErrorMessage( String string, Exception e, boolean stackTrace )
private static void printErrorMessage( String string, Exception e, boolean stackTrace, PrintStream err )
{
System.err.println( string );
System.err.println( "Caused by:" + e.getMessage() );
err.println( string );
err.println( "Caused by:" + e.getMessage() );
if ( stackTrace )
{
e.printStackTrace( System.err );
e.printStackTrace( err );
}
}

Expand Down Expand Up @@ -771,29 +802,6 @@ public boolean trimStrings()

private static final Function<String,Character> CHARACTER_CONVERTER = new CharacterConverter();

static final Validator<File[]> FILES_EXISTS = files -> {
for ( File file : files )
{
if ( file.getName().startsWith( ":" ) )
{
warn( "It looks like you're trying to specify default label or relationship type (" +
file.getName() + "). Please put such directly on the key, f.ex. " +
Options.NODE_DATA.argument() + ":MyLabel" );
}
Validators.REGEX_FILE_EXISTS.validate( file );
}
};

private static final BiFunction<Args,String,Collection<Option<File[]>>> INPUT_FILES_EXTRACTOR =
( args, key ) -> args.interpretOptionsWithMetadata( key, Converters.<File[]>optional(),
Converters.toFiles( MULTI_FILE_DELIMITER, Converters.regexFiles( true ) ), FILES_EXISTS,
Validators.<File>atLeast( "--" + key, 1 ) );

static void warn( String warning )
{
System.err.println( warning );
}

private enum ManualPage
{
IMPORT_TOOL_FORMAT( "import-tool-header-format.html" );
Expand Down

0 comments on commit 77c25dc

Please sign in to comment.