Skip to content

Commit

Permalink
Removes --bad import tool option
Browse files Browse the repository at this point in the history
so that it cannot be misused or misplaced. It will always end up in
<into>/bad.log
  • Loading branch information
tinwelint committed Apr 13, 2015
1 parent 5c18872 commit d6578ac
Show file tree
Hide file tree
Showing 4 changed files with 19 additions and 50 deletions.
Expand Up @@ -65,6 +65,7 @@
import static org.neo4j.helpers.Exceptions.launderedException;
import static org.neo4j.helpers.collection.MapUtil.stringMap;
import static org.neo4j.kernel.impl.util.Converters.withDefault;
import static org.neo4j.unsafe.impl.batchimport.Configuration.BAD_FILE_NAME;
import static org.neo4j.unsafe.impl.batchimport.input.InputEntityDecorators.NO_NODE_DECORATOR;
import static org.neo4j.unsafe.impl.batchimport.input.InputEntityDecorators.additiveLabels;
import static org.neo4j.unsafe.impl.batchimport.input.InputEntityDecorators.defaultRelationshipType;
Expand Down Expand Up @@ -133,11 +134,6 @@ enum Options
STACKTRACE( "stacktrace", null,
"",
"Enable printing of error stack traces." ),
BAD( "bad", null,
"<file name>",
"Relationships that refer to nodes that cannot be found can, instead of making the import fail,"
+ " be logged to a file specified by this option. Can be relative (to store directory)"
+ " or absolute" ),
BAD_TOLERANCE( "bad-tolerance", 1000,
"<max number of bad entries>",
"Number of bad entries before the import is considered failed. This tolerance threshold is "
Expand Down Expand Up @@ -250,7 +246,6 @@ public static void main( String[] incomingArguments, boolean defaultSettingsSuit
boolean enableStacktrace;
Number processors = null;
Input input = null;
String badFileName;
int badTolerance;
Charset inputEncoding;
try
Expand All @@ -266,7 +261,6 @@ public static void main( String[] incomingArguments, boolean defaultSettingsSuit
withDefault( (IdType)Options.ID_TYPE.defaultValue() ), TO_ID_TYPE );
badTolerance = args.getNumber( Options.BAD_TOLERANCE.key(),
(Number) Options.BAD_TOLERANCE.defaultValue() ).intValue();
badFileName = args.get( Options.BAD.key() );
inputEncoding = Charset.forName( args.get( Options.INPUT_ENCODING.key(), defaultCharset().name() ) );
input = new CsvInput(
nodeData( inputEncoding, nodesFiles ), defaultFormatNodeFileHeader(),
Expand All @@ -284,7 +278,7 @@ idType, csvConfiguration( args, defaultSettingsSuitableForTests ),
new Config( stringMap( store_dir.name(), storeDir.getAbsolutePath() ) ) ) );
life.start();
org.neo4j.unsafe.impl.batchimport.Configuration config =
importConfiguration( processors, badFileName, defaultSettingsSuitableForTests );
importConfiguration( processors, defaultSettingsSuitableForTests );
BatchImporter importer = new ParallelBatchImporter( storeDir.getPath(),
config,
logging,
Expand All @@ -301,11 +295,11 @@ idType, csvConfiguration( args, defaultSettingsSuitableForTests ),
}
finally
{
File badRelationships = config.badFile( storeDir );
File badRelationships = new File( storeDir, BAD_FILE_NAME );
if ( badRelationships.exists() )
{
out.println("There were bad relationships which were skipped " +
"and logged into " + badRelationships.getAbsolutePath());
out.println( "There were bad relationships which were skipped " +
"and logged into " + badRelationships.getAbsolutePath() );
}

life.shutdown();
Expand Down Expand Up @@ -343,7 +337,7 @@ private static void validateInputFiles( Collection<Option<File[]>> nodesFiles,
}

private static org.neo4j.unsafe.impl.batchimport.Configuration importConfiguration( final Number processors,
final String badFileName, final boolean defaultSettingsSuitableForTests )
final boolean defaultSettingsSuitableForTests )
{
return new org.neo4j.unsafe.impl.batchimport.Configuration.Default()
{
Expand All @@ -353,18 +347,6 @@ public int maxNumberOfProcessors()
return processors != null ? processors.intValue() : super.maxNumberOfProcessors();
}

@Override
public File badFile( File storeDirectory )
{
if ( badFileName == null )
{
return super.badFile( storeDirectory );
}

File part = new File( badFileName );
return part.isAbsolute() ? part : new File( storeDirectory, badFileName );
}

@Override
public int bigFileChannelBufferSizeMultiplier()
{
Expand Down
Expand Up @@ -42,11 +42,13 @@
import org.neo4j.test.TargetDirectory.TestDirectory;
import org.neo4j.test.TestGraphDatabaseFactory;
import org.neo4j.tooling.ImportTool.Options;
import org.neo4j.unsafe.impl.batchimport.Configuration;

import static org.junit.Assert.assertEquals;
import static org.junit.Assert.assertTrue;

import static org.neo4j.helpers.ArrayUtil.join;
import static org.neo4j.io.fs.FileUtils.copyFile;
import static org.neo4j.tooling.ImportTool.MULTI_FILE_DELIMITER;

public class ImportToolDocIT
Expand Down Expand Up @@ -513,15 +515,16 @@ public void badRelationshipsDefault() throws IOException
}

// WHEN
File badFile = file( "ops", "bad-relationships-default-not-imported.bad.adoc" ).getAbsoluteFile();
File badFile = new File( directory.directory(), Configuration.BAD_FILE_NAME );
String[] arguments = arguments(
"--into", directory.absolutePath(),
"--nodes", movies.getAbsolutePath(),
"--nodes", actors.getAbsolutePath(),
"--relationships", roles.getAbsolutePath(),
"--bad", badFile.getAbsolutePath() );
"--relationships", roles.getAbsolutePath() );
importTool( arguments );
assertTrue( badFile.exists() );
// There's a reference in the manual to this file
copyFile( badFile, file( "ops", "bad-relationships-default-not-imported.bad.adoc" ) );

// DOCS
String realDir = movies.getParentFile().getAbsolutePath();
Expand Down
Expand Up @@ -19,8 +19,6 @@
*/
package org.neo4j.unsafe.impl.batchimport;

import java.io.File;

import org.neo4j.graphdb.factory.GraphDatabaseSettings;
import org.neo4j.kernel.configuration.Config;

Expand All @@ -32,6 +30,12 @@
*/
public interface Configuration extends org.neo4j.unsafe.impl.batchimport.staging.Configuration
{
/**
* File name in which bad entries from the import will end up. This file will be created in the
* database directory of the imported database, i.e. <into>/bad.log.
*/
String BAD_FILE_NAME = "bad.log";

/**
* Memory dedicated to buffering data to be written to each store file.
*/
Expand Down Expand Up @@ -73,14 +77,6 @@ public interface Configuration extends org.neo4j.unsafe.impl.batchimport.staging
*/
int maxNumberOfProcessors();

/**
* File name of log accepting bad entries encountered during import. Can be relative (to where the
* store directory of the database that gets created) or absolute.
*
* @param the directory of the target database.
*/
File badFile( File storeDirectory );

class Default
extends org.neo4j.unsafe.impl.batchimport.staging.Configuration.Default
implements Configuration
Expand Down Expand Up @@ -145,12 +141,6 @@ public int movingAverageSize()
{
return 100;
}

@Override
public File badFile( File storeDirectory )
{
return new File( storeDirectory, "not-imported.bad" );
}
}

Configuration DEFAULT = new Default();
Expand Down Expand Up @@ -209,11 +199,5 @@ public int movingAverageSize()
{
return defaults.movingAverageSize();
}

@Override
public File badFile( File storeDirectory )
{
return defaults.badFile( storeDirectory );
}
}
}
Expand Up @@ -124,7 +124,7 @@ public void doImport( Input input ) throws IOException
NodeLabelsCache nodeLabelsCache = null;
long startTime = currentTimeMillis();
boolean hasBadRelationships = false;
File badFile = config.badFile( storeDir );
File badFile = new File( storeDir, Configuration.BAD_FILE_NAME );
try ( BatchingNeoStore neoStore = new BatchingNeoStore( fileSystem, storeDir, config,
writeMonitor, logging, monitors, writerFactory, additionalInitialIds );
OutputStream badRelationshipsOutput = new BufferedOutputStream(
Expand Down

0 comments on commit d6578ac

Please sign in to comment.