From d820493f464ec611ca99924cec8949b7fbba6be1 Mon Sep 17 00:00:00 2001 From: Jonas Kalderstam Date: Wed, 28 Oct 2015 14:05:19 +0100 Subject: [PATCH] Handle extra columns in CSV files using bad file Now either fails with an exception or logs to bad file. --- .../ParallelBatchImporterTest.java | 4 +- .../java/org/neo4j/tooling/ImportTool.java | 44 ++++++++++--- .../neo4j/tooling/CsvDataGeneratorInput.java | 8 ++- .../org/neo4j/tooling/ImportToolTest.java | 41 +++++++++++- .../java/org/neo4j/tooling/QuickImport.java | 4 +- .../impl/storemigration/StoreMigrator.java | 9 ++- .../batchimport/ParallelBatchImporter.java | 3 +- .../impl/batchimport/input/BadCollector.java | 25 +++++++- .../impl/batchimport/input/Collector.java | 2 + .../impl/batchimport/input/Collectors.java | 21 ++++++- .../unsafe/impl/batchimport/input/Input.java | 2 +- .../unsafe/impl/batchimport/input/Inputs.java | 10 +-- .../impl/batchimport/input/csv/CsvInput.java | 14 ++--- .../csv/ExternalPropertiesDecorator.java | 5 +- .../input/csv/InputEntityDeserializer.java | 28 ++------- .../input/csv/CsvInputBatchImportIT.java | 4 +- .../batchimport/input/csv/CsvInputTest.java | 62 +++++++++---------- .../csv/ExternalPropertiesDecoratorTest.java | 7 ++- 18 files changed, 197 insertions(+), 96 deletions(-) diff --git a/community/consistency-check/src/test/java/org/neo4j/unsafe/impl/batchimport/ParallelBatchImporterTest.java b/community/consistency-check/src/test/java/org/neo4j/unsafe/impl/batchimport/ParallelBatchImporterTest.java index 8e438b29b6b2a..016d24f142bf3 100644 --- a/community/consistency-check/src/test/java/org/neo4j/unsafe/impl/batchimport/ParallelBatchImporterTest.java +++ b/community/consistency-check/src/test/java/org/neo4j/unsafe/impl/batchimport/ParallelBatchImporterTest.java @@ -93,6 +93,7 @@ import static org.neo4j.unsafe.impl.batchimport.cache.idmapping.IdGenerators.startingFromTheBeginning; import static org.neo4j.unsafe.impl.batchimport.cache.idmapping.IdMappers.longs; import static org.neo4j.unsafe.impl.batchimport.cache.idmapping.IdMappers.strings; +import static org.neo4j.unsafe.impl.batchimport.input.Collectors.silentBadCollector; import static org.neo4j.unsafe.impl.batchimport.staging.ProcessorAssignmentStrategies.eagerRandomSaturation; import static org.neo4j.unsafe.impl.batchimport.store.BatchingPageCache.SYNCHRONOUS; @@ -178,7 +179,8 @@ public void shouldImportCsvData() throws Exception nodes( nodeRandomSeed, NODE_COUNT, inputIdGenerator, groups ), relationships( relationshipRandomSeed, RELATIONSHIP_COUNT, inputIdGenerator, groups ), idMapper, idGenerator, false, - RELATIONSHIP_COUNT/*insanely high bad tolerance, but it will actually never be that many*/ ) ); + /*insanely high bad tolerance, but it will actually never be that many*/ + silentBadCollector( RELATIONSHIP_COUNT ) ) ); // THEN GraphDatabaseService db = new TestGraphDatabaseFactory().newEmbeddedDatabase( directory.absolutePath() ); diff --git a/community/import-tool/src/main/java/org/neo4j/tooling/ImportTool.java b/community/import-tool/src/main/java/org/neo4j/tooling/ImportTool.java index 67fbc63b10d90..e005eb444d82d 100644 --- a/community/import-tool/src/main/java/org/neo4j/tooling/ImportTool.java +++ b/community/import-tool/src/main/java/org/neo4j/tooling/ImportTool.java @@ -19,8 +19,10 @@ */ package org.neo4j.tooling; +import java.io.BufferedOutputStream; import java.io.File; import java.io.IOException; +import java.io.OutputStream; import java.io.PrintStream; import java.lang.Thread.UncaughtExceptionHandler; import java.nio.charset.Charset; @@ -53,6 +55,7 @@ import org.neo4j.unsafe.impl.batchimport.BatchImporter; import org.neo4j.unsafe.impl.batchimport.ParallelBatchImporter; import org.neo4j.unsafe.impl.batchimport.cache.idmapping.string.DuplicateInputIdException; +import org.neo4j.unsafe.impl.batchimport.input.Collector; import org.neo4j.unsafe.impl.batchimport.input.Input; import org.neo4j.unsafe.impl.batchimport.input.InputNode; import org.neo4j.unsafe.impl.batchimport.input.InputRelationship; @@ -176,7 +179,12 @@ enum Options + "nodes within the same group having the same id, the first encountered will be imported " + "whereas consecutive such nodes will be skipped. " + "Skipped nodes will be logged" - + ", containing at most number of entities specified by " + BAD_TOLERANCE.key() + "." ); + + ", containing at most number of entities specified by " + BAD_TOLERANCE.key() + "." ), + IGNORE_EXTRA_COLUMNS( "ignore-extra-columns", Boolean.FALSE, + "", + "Whether or not to ignore extra columns in the data not specified by the header. " + + "Skipped columns will be logged, containing at most number of entities specified by " + + BAD_TOLERANCE.key() + "." ); private final String key; private final Object defaultValue; @@ -293,12 +301,15 @@ public static void main( String[] incomingArguments, boolean defaultSettingsSuit Input input = null; int badTolerance; Charset inputEncoding; - boolean skipBadRelationships, skipDuplicateNodes; + boolean skipBadRelationships, skipDuplicateNodes, ignoreExtraColumns; try { storeDir = args.interpretOption( Options.STORE_DIR.key(), Converters.mandatory(), Converters.toFile(), Validators.DIRECTORY_IS_WRITABLE, Validators.CONTAINS_NO_EXISTING_DATABASE ); + + File badFile = new File( storeDir, BAD_FILE_NAME ); + OutputStream badOutput = new BufferedOutputStream( fs.openAsOutputStream( badFile, false ) ); nodesFiles = INPUT_FILES_EXTRACTOR.apply( args, Options.NODE_DATA.key() ); relationshipsFiles = INPUT_FILES_EXTRACTOR.apply( args, Options.RELATIONSHIP_DATA.key() ); validateInputFiles( nodesFiles, relationshipsFiles ); @@ -313,16 +324,24 @@ public static void main( String[] incomingArguments, boolean defaultSettingsSuit (Boolean)Options.SKIP_BAD_RELATIONSHIPS.defaultValue(), true ); skipDuplicateNodes = args.getBoolean( Options.SKIP_DUPLICATE_NODES.key(), (Boolean)Options.SKIP_DUPLICATE_NODES.defaultValue(), true ); - input = new CsvInput( - nodeData( inputEncoding, nodesFiles ), defaultFormatNodeFileHeader(), + ignoreExtraColumns = args.getBoolean( Options.IGNORE_EXTRA_COLUMNS.key(), + (Boolean)Options.IGNORE_EXTRA_COLUMNS.defaultValue(), true ); + + Collector badCollector = badCollector( badOutput, badTolerance, collect( skipBadRelationships, + skipDuplicateNodes, ignoreExtraColumns ) ); + + input = new CsvInput( nodeData( inputEncoding, nodesFiles ), defaultFormatNodeFileHeader(), relationshipData( inputEncoding, relationshipsFiles ), defaultFormatRelationshipFileHeader(), - idType, csvConfiguration( args, defaultSettingsSuitableForTests ), - badCollector( badTolerance, collect( skipBadRelationships, skipDuplicateNodes ) ) ); + idType, csvConfiguration( args, defaultSettingsSuitableForTests ), badCollector ); } catch ( IllegalArgumentException e ) { throw andPrintError( "Input error", e, false ); } + catch ( IOException e ) + { + throw andPrintError( "File error", e, false ); + } LifeSupport life = new LifeSupport(); Logging logging = life.add( new ClassicLoggingService( @@ -347,11 +366,16 @@ idType, csvConfiguration( args, defaultSettingsSuitableForTests ), } finally { - File badFile = new File( storeDir, BAD_FILE_NAME ); - if ( badFile.exists() ) + input.badCollector().close(); + + if ( input.badCollector().badEntries() > 0 ) { - out.println( "There were bad entries which were skipped and logged into " + - badFile.getAbsolutePath() ); + File badFile = new File( storeDir, BAD_FILE_NAME ); + if ( badFile.exists() ) + { + out.println( + "There were bad entries which were skipped and logged into " + badFile.getAbsolutePath() ); + } } life.shutdown(); diff --git a/community/import-tool/src/test/java/org/neo4j/tooling/CsvDataGeneratorInput.java b/community/import-tool/src/test/java/org/neo4j/tooling/CsvDataGeneratorInput.java index e48d97e2cb91a..cfc034f12aaf7 100644 --- a/community/import-tool/src/test/java/org/neo4j/tooling/CsvDataGeneratorInput.java +++ b/community/import-tool/src/test/java/org/neo4j/tooling/CsvDataGeneratorInput.java @@ -47,10 +47,11 @@ public class CsvDataGeneratorInput extends CsvDataGenerator implements Input { private final IdType idType; + private final Collector badCollector; public CsvDataGeneratorInput( final Header nodeHeader, final Header relationshipHeader, Configuration config, long nodes, long relationships, final Groups groups, final IdType idType, - int numberOfLabels, int numberOfRelationshipTypes ) + int numberOfLabels, int numberOfRelationshipTypes, Collector badCollector ) { super( nodeHeader, relationshipHeader, config, nodes, relationships, new Function>() @@ -71,6 +72,7 @@ public Deserialization apply( SourceTraceability from ) throw }, numberOfLabels, numberOfRelationshipTypes ); this.idType = idType; + this.badCollector = badCollector; } @Override @@ -130,8 +132,8 @@ public boolean specificRelationshipIds() } @Override - public Collector badCollector( OutputStream out ) + public Collector badCollector() { - return Collectors.badCollector( out, 0 ); + return badCollector; } } diff --git a/community/import-tool/src/test/java/org/neo4j/tooling/ImportToolTest.java b/community/import-tool/src/test/java/org/neo4j/tooling/ImportToolTest.java index 16ac6c8d16ff8..02893d97d0e25 100644 --- a/community/import-tool/src/test/java/org/neo4j/tooling/ImportToolTest.java +++ b/community/import-tool/src/test/java/org/neo4j/tooling/ImportToolTest.java @@ -193,7 +193,7 @@ public void import4097Labels() throws Exception } @Test - public void shouldPrintWarningsIfHeaderHasLessColumnsThanData() throws Exception + public void shouldFailIfHeaderHasLessColumnsThanData() throws Exception { // GIVEN List nodeIds = nodeIds(); @@ -201,8 +201,42 @@ public void shouldPrintWarningsIfHeaderHasLessColumnsThanData() throws Exception // WHEN data file contains more columns than header file int extraColumns = 3; - String output = executeImportAndCatchOutput( + try + { + executeImportAndCatchOutput( + "--into", dbRule.getStoreDirAbsolutePath(), + "--delimiter", "TAB", + "--array-delimiter", String.valueOf( config.arrayDelimiter() ), + "--nodes", nodeHeader( config ).getAbsolutePath() + MULTI_FILE_DELIMITER + + nodeData( false, config, nodeIds, alwaysTrue(), Charset.defaultCharset(), extraColumns ) + .getAbsolutePath(), + "--relationships", relationshipHeader( config ).getAbsolutePath() + MULTI_FILE_DELIMITER + + relationshipData( false, config, nodeIds, alwaysTrue(), true ).getAbsolutePath() ); + + fail( "Should have thrown exception" ); + } + catch ( InputException e ) + { + // THEN + assertTrue( e.getMessage().contains( "Extra column not present in header on line" ) ); + } + } + + @Test + public void shouldWarnIfHeaderHasLessColumnsThanDataWhenToldTo() throws Exception + { + // GIVEN + List nodeIds = nodeIds(); + Configuration config = Configuration.TABS; + File bad = file( "bad.log" ); + + // WHEN data file contains more columns than header file + int extraColumns = 3; + executeImportAndCatchOutput( "--into", dbRule.getStoreDirAbsolutePath(), + "--bad", bad.getAbsolutePath(), + "--bad-tolerance", Integer.toString( nodeIds.size() * extraColumns ), + "--ignore-extra-columns", "--delimiter", "TAB", "--array-delimiter", String.valueOf( config.arrayDelimiter() ), "--nodes", nodeHeader( config ).getAbsolutePath() + MULTI_FILE_DELIMITER + @@ -212,7 +246,8 @@ public void shouldPrintWarningsIfHeaderHasLessColumnsThanData() throws Exception relationshipData( false, config, nodeIds, alwaysTrue(), true ).getAbsolutePath() ); // THEN - assertTrue( "Should warn when columns are ignored:\n" + output, output.contains( "Warning: ignored columns" ) ); + String badContents = FileUtils.readTextFile( bad, Charset.defaultCharset() ); + assertTrue( badContents.contains( "Extra column not present in header on line" ) ); } @Test diff --git a/community/import-tool/src/test/java/org/neo4j/tooling/QuickImport.java b/community/import-tool/src/test/java/org/neo4j/tooling/QuickImport.java index ac7023c9a6d32..8a58c080c96ee 100644 --- a/community/import-tool/src/test/java/org/neo4j/tooling/QuickImport.java +++ b/community/import-tool/src/test/java/org/neo4j/tooling/QuickImport.java @@ -40,6 +40,7 @@ import static org.neo4j.tooling.CsvDataGenerator.bareboneNodeHeader; import static org.neo4j.tooling.CsvDataGenerator.bareboneRelationshipHeader; import static org.neo4j.unsafe.impl.batchimport.Configuration.DEFAULT; +import static org.neo4j.unsafe.impl.batchimport.input.Collectors.silentBadCollector; import static org.neo4j.unsafe.impl.batchimport.input.csv.Configuration.COMMAS; import static org.neo4j.unsafe.impl.batchimport.input.csv.DataFactories.defaultFormatNodeFileHeader; import static org.neo4j.unsafe.impl.batchimport.input.csv.DataFactories.defaultFormatRelationshipFileHeader; @@ -78,7 +79,8 @@ public static void main( String[] arguments ) throws IOException Input input = new CsvDataGeneratorInput( nodeHeader, relationshipHeader, - COMMAS, nodeCount, relationshipCount, new Groups(), idType, labelCount, relationshipTypeCount ); + COMMAS, nodeCount, relationshipCount, new Groups(), idType, labelCount, relationshipTypeCount, + silentBadCollector( 0 )); BatchImporter importer = new ParallelBatchImporter( dir, DEFAULT, new SystemOutLogging(), defaultVisible() ); importer.doImport( input ); } diff --git a/community/kernel/src/main/java/org/neo4j/kernel/impl/storemigration/StoreMigrator.java b/community/kernel/src/main/java/org/neo4j/kernel/impl/storemigration/StoreMigrator.java index bf474793e30ad..2277b99d2dddb 100644 --- a/community/kernel/src/main/java/org/neo4j/kernel/impl/storemigration/StoreMigrator.java +++ b/community/kernel/src/main/java/org/neo4j/kernel/impl/storemigration/StoreMigrator.java @@ -19,9 +19,12 @@ */ package org.neo4j.kernel.impl.storemigration; +import java.io.BufferedOutputStream; import java.io.File; import java.io.FileNotFoundException; +import java.io.FileOutputStream; import java.io.IOException; +import java.io.OutputStream; import java.io.Reader; import java.io.Writer; import java.util.ArrayList; @@ -85,6 +88,7 @@ import org.neo4j.unsafe.impl.batchimport.ParallelBatchImporter; import org.neo4j.unsafe.impl.batchimport.cache.idmapping.IdGenerators; import org.neo4j.unsafe.impl.batchimport.cache.idmapping.IdMappers; +import org.neo4j.unsafe.impl.batchimport.input.Collectors; import org.neo4j.unsafe.impl.batchimport.input.InputEntity; import org.neo4j.unsafe.impl.batchimport.input.InputNode; import org.neo4j.unsafe.impl.batchimport.input.InputRelationship; @@ -339,13 +343,16 @@ private void migrateWithBatchImporter( File storeDir, File migrationDir, long la } Configuration importConfig = new Configuration.Overridden( config ); + File badFile = new File( storeDir, Configuration.BAD_FILE_NAME ); + OutputStream badOutput = new BufferedOutputStream( new FileOutputStream( badFile, false ) ); BatchImporter importer = new ParallelBatchImporter( migrationDir.getAbsolutePath(), fileSystem, importConfig, logging, withDynamicProcessorAssignment( migrationBatchImporterMonitor( legacyStore, progressMonitor ), importConfig ), parallel(), readAdditionalIds( storeDir, lastTxId, lastTxChecksum ) ); InputIterable nodes = legacyNodesAsInput( legacyStore ); InputIterable relationships = legacyRelationshipsAsInput( legacyStore ); - importer.doImport( Inputs.input( nodes, relationships, IdMappers.actual(), IdGenerators.fromInput(), true, 0 ) ); + importer.doImport( Inputs.input( nodes, relationships, IdMappers.actual(), IdGenerators.fromInput(), true, + Collectors.badCollector( badOutput, 0 )) ); // During migration the batch importer only writes node, relationship, relationship group and counts stores. // Delete the property store files from the batch import migration so that even if we won't diff --git a/community/kernel/src/main/java/org/neo4j/unsafe/impl/batchimport/ParallelBatchImporter.java b/community/kernel/src/main/java/org/neo4j/unsafe/impl/batchimport/ParallelBatchImporter.java index be9911bbc2fa8..62fd909da0418 100644 --- a/community/kernel/src/main/java/org/neo4j/unsafe/impl/batchimport/ParallelBatchImporter.java +++ b/community/kernel/src/main/java/org/neo4j/unsafe/impl/batchimport/ParallelBatchImporter.java @@ -128,12 +128,11 @@ public void doImport( Input input ) throws IOException CountingStoreUpdateMonitor storeUpdateMonitor = new CountingStoreUpdateMonitor(); try ( BatchingNeoStore neoStore = new BatchingNeoStore( fileSystem, storeDir, config, writeMonitor, logging, monitors, writerFactory, additionalInitialIds ); - OutputStream badOutput = new BufferedOutputStream( fileSystem.openAsOutputStream( badFile, false ) ); - Collector badCollector = input.badCollector( badOutput ); CountsAccessor.Updater countsUpdater = neoStore.getCountsStore().reset( neoStore.getLastCommittedTransactionId() ); InputCache inputCache = new InputCache( fileSystem, storeDir ) ) { + Collector badCollector = input.badCollector(); // Some temporary caches and indexes in the import IdMapper idMapper = input.idMapper(); IdGenerator idGenerator = input.idGenerator(); diff --git a/community/kernel/src/main/java/org/neo4j/unsafe/impl/batchimport/input/BadCollector.java b/community/kernel/src/main/java/org/neo4j/unsafe/impl/batchimport/input/BadCollector.java index 4bbdfb7e12dee..7c70641246306 100644 --- a/community/kernel/src/main/java/org/neo4j/unsafe/impl/batchimport/input/BadCollector.java +++ b/community/kernel/src/main/java/org/neo4j/unsafe/impl/batchimport/input/BadCollector.java @@ -48,7 +48,8 @@ private interface ProblemReporter public static final int BAD_RELATIONSHIPS = 0x1; public static final int DUPLICATE_NODES = 0x2; - public static final int COLLECT_ALL = BAD_RELATIONSHIPS | DUPLICATE_NODES; + public static final int EXTRA_COLUMNS = 0x4; + public static final int COLLECT_ALL = BAD_RELATIONSHIPS | DUPLICATE_NODES | EXTRA_COLUMNS; private final PrintStream out; private final int tolerance; @@ -114,6 +115,28 @@ public InputException exception() leftOverDuplicateNodeIds[leftOverDuplicateNodeIdsCursor++] = actualId; } + @Override + public void collectExtraColumns( final String source, final long row, final String value ) + { + checkTolerance( EXTRA_COLUMNS, new ProblemReporter() + { + private final String message = format( "Extra column not present in header on line %d in %s with value %s", + row, source, value ); + + @Override + public String message() + { + return message; + } + + @Override + public InputException exception() + { + return new InputException( message ); + } + } ); + } + @Override public PrimitiveLongIterator leftOverDuplicateNodesIds() { diff --git a/community/kernel/src/main/java/org/neo4j/unsafe/impl/batchimport/input/Collector.java b/community/kernel/src/main/java/org/neo4j/unsafe/impl/batchimport/input/Collector.java index b2c51a98da2c4..de7d795655238 100644 --- a/community/kernel/src/main/java/org/neo4j/unsafe/impl/batchimport/input/Collector.java +++ b/community/kernel/src/main/java/org/neo4j/unsafe/impl/batchimport/input/Collector.java @@ -31,6 +31,8 @@ public interface Collector extends AutoCloseable void collectDuplicateNode( Object id, long actualId, String group, String firstSource, String otherSource ); + void collectExtraColumns( final String source, final long row, final String value ); + int badEntries(); /** diff --git a/community/kernel/src/main/java/org/neo4j/unsafe/impl/batchimport/input/Collectors.java b/community/kernel/src/main/java/org/neo4j/unsafe/impl/batchimport/input/Collectors.java index 4f67831260103..ae3d622546313 100644 --- a/community/kernel/src/main/java/org/neo4j/unsafe/impl/batchimport/input/Collectors.java +++ b/community/kernel/src/main/java/org/neo4j/unsafe/impl/batchimport/input/Collectors.java @@ -19,6 +19,7 @@ */ package org.neo4j.unsafe.impl.batchimport.input; +import java.io.IOException; import java.io.OutputStream; import org.neo4j.function.Function; @@ -28,6 +29,21 @@ */ public class Collectors { + public static Collector silentBadCollector( int tolerance ) { + return silentBadCollector( tolerance, BadCollector.COLLECT_ALL ); + } + + public static Collector silentBadCollector( int tolerance, int collect ) { + return badCollector( new OutputStream() + { + @Override + public void write( int i ) throws IOException + { + // ignored + } + }, tolerance, collect ); + } + public static Collector badCollector( OutputStream out, int tolerance ) { return badCollector( out, tolerance, BadCollector.COLLECT_ALL ); @@ -55,9 +71,10 @@ public Collector apply( OutputStream out ) throws RuntimeException }; } - public static int collect( boolean skipBadRelationships, boolean skipDuplicateNodes ) + public static int collect( boolean skipBadRelationships, boolean skipDuplicateNodes, boolean ignoreExtraColumns ) { return (skipBadRelationships ? BadCollector.BAD_RELATIONSHIPS : 0 ) | - (skipDuplicateNodes ? BadCollector.DUPLICATE_NODES : 0 ); + (skipDuplicateNodes ? BadCollector.DUPLICATE_NODES : 0 ) | + (ignoreExtraColumns ? BadCollector.EXTRA_COLUMNS : 0 ); } } diff --git a/community/kernel/src/main/java/org/neo4j/unsafe/impl/batchimport/input/Input.java b/community/kernel/src/main/java/org/neo4j/unsafe/impl/batchimport/input/Input.java index 2f748833e2bce..68b0891fc605a 100644 --- a/community/kernel/src/main/java/org/neo4j/unsafe/impl/batchimport/input/Input.java +++ b/community/kernel/src/main/java/org/neo4j/unsafe/impl/batchimport/input/Input.java @@ -71,5 +71,5 @@ public interface Input * @return a {@link Collector} capable of writing {@link InputRelationship bad relationships} * and {@link InputNode duplicate nodes} to an output stream for later handling. */ - Collector badCollector( OutputStream out ); + Collector badCollector(); } diff --git a/community/kernel/src/main/java/org/neo4j/unsafe/impl/batchimport/input/Inputs.java b/community/kernel/src/main/java/org/neo4j/unsafe/impl/batchimport/input/Inputs.java index 1ad372b55adf3..067206f46242c 100644 --- a/community/kernel/src/main/java/org/neo4j/unsafe/impl/batchimport/input/Inputs.java +++ b/community/kernel/src/main/java/org/neo4j/unsafe/impl/batchimport/input/Inputs.java @@ -44,7 +44,7 @@ public class Inputs public static Input input( final InputIterable nodes, final InputIterable relationships, final IdMapper idMapper, final IdGenerator idGenerator, final boolean specificRelationshipIds, - final int badTolerance ) + final Collector badCollector ) { return new Input() { @@ -79,20 +79,20 @@ public boolean specificRelationshipIds() } @Override - public Collector badCollector( OutputStream out ) + public Collector badCollector() { - return Collectors.badCollector( out, badTolerance ); + return badCollector; } }; } public static Input csv( File nodes, File relationships, IdType idType, - Configuration configuration ) + Configuration configuration, Collector badCollector ) { return new CsvInput( nodeData( data( NO_NODE_DECORATOR, defaultCharset(), nodes ) ), defaultFormatNodeFileHeader(), relationshipData( data( NO_RELATIONSHIP_DECORATOR, defaultCharset(), relationships ) ), defaultFormatRelationshipFileHeader(), idType, configuration, - Collectors.badCollector( 0 ) ); + badCollector ); } } diff --git a/community/kernel/src/main/java/org/neo4j/unsafe/impl/batchimport/input/csv/CsvInput.java b/community/kernel/src/main/java/org/neo4j/unsafe/impl/batchimport/input/csv/CsvInput.java index cc4bb7dcc259a..66cd7288a4534 100644 --- a/community/kernel/src/main/java/org/neo4j/unsafe/impl/batchimport/input/csv/CsvInput.java +++ b/community/kernel/src/main/java/org/neo4j/unsafe/impl/batchimport/input/csv/CsvInput.java @@ -52,7 +52,7 @@ public class CsvInput implements Input private final IdType idType; private final Configuration config; private final Groups groups = new Groups(); - private final Function collectorFactory; + private final Collector badCollector; /** * @param nodeDataFactory multiple {@link DataFactory} instances providing data, each {@link DataFactory} @@ -69,7 +69,7 @@ public class CsvInput implements Input public CsvInput( Iterable> nodeDataFactory, Header.Factory nodeHeaderFactory, Iterable> relationshipDataFactory, Header.Factory relationshipHeaderFactory, - IdType idType, Configuration config, Function collectorFactory ) + IdType idType, Configuration config, Collector badCollector ) { assertSaneConfiguration( config ); @@ -79,7 +79,7 @@ public CsvInput( this.relationshipHeaderFactory = relationshipHeaderFactory; this.idType = idType; this.config = config; - this.collectorFactory = collectorFactory; + this.badCollector = badCollector; } private void assertSaneConfiguration( Configuration config ) @@ -117,7 +117,7 @@ protected InputEntityDeserializer entityDeserializer( CharSeeker data { return new InputEntityDeserializer<>( dataHeader, dataStream, config.delimiter(), new InputNodeDeserialization( dataStream, dataHeader, groups, idType.idsAreExternal() ), - decorator, Validators.emptyValidator() ); + decorator, Validators.emptyValidator(), badCollector ); } }; } @@ -165,7 +165,7 @@ public void validate( InputRelationship entity ) throw new DataException( entity + " is missing " + Type.TYPE + " field" ); } } - } ); + }, badCollector ); } }; } @@ -197,8 +197,8 @@ public boolean specificRelationshipIds() } @Override - public Collector badCollector( OutputStream out ) + public Collector badCollector() { - return collectorFactory.apply( out ); + return badCollector; } } diff --git a/community/kernel/src/main/java/org/neo4j/unsafe/impl/batchimport/input/csv/ExternalPropertiesDecorator.java b/community/kernel/src/main/java/org/neo4j/unsafe/impl/batchimport/input/csv/ExternalPropertiesDecorator.java index ae88d27b46e3d..84960b20d98d6 100644 --- a/community/kernel/src/main/java/org/neo4j/unsafe/impl/batchimport/input/csv/ExternalPropertiesDecorator.java +++ b/community/kernel/src/main/java/org/neo4j/unsafe/impl/batchimport/input/csv/ExternalPropertiesDecorator.java @@ -23,6 +23,7 @@ import org.neo4j.function.Function; import org.neo4j.function.Functions; import org.neo4j.kernel.impl.util.Validators; +import org.neo4j.unsafe.impl.batchimport.input.Collector; import org.neo4j.unsafe.impl.batchimport.input.Groups; import org.neo4j.unsafe.impl.batchimport.input.InputNode; import org.neo4j.unsafe.impl.batchimport.input.UpdateBehaviour; @@ -58,14 +59,14 @@ public class ExternalPropertiesDecorator implements Function data, Header.Factory headerFactory, - Configuration config, IdType idType, UpdateBehaviour updateBehaviour ) + Configuration config, IdType idType, UpdateBehaviour updateBehaviour, Collector badCollector ) { this.updateBehaviour = updateBehaviour; CharSeeker dataStream = data.create( config ).stream(); Header header = headerFactory.create( dataStream, config, idType ); this.deserializer = new InputEntityDeserializer<>( header, dataStream, config.delimiter(), new InputNodeDeserialization( dataStream, header, new Groups(), idType.idsAreExternal() ), - Functions.identity(), Validators.emptyValidator() ); + Functions.identity(), Validators.emptyValidator(), badCollector ); } @Override diff --git a/community/kernel/src/main/java/org/neo4j/unsafe/impl/batchimport/input/csv/InputEntityDeserializer.java b/community/kernel/src/main/java/org/neo4j/unsafe/impl/batchimport/input/csv/InputEntityDeserializer.java index 7a4bcb392a805..535e19aa1a0b1 100644 --- a/community/kernel/src/main/java/org/neo4j/unsafe/impl/batchimport/input/csv/InputEntityDeserializer.java +++ b/community/kernel/src/main/java/org/neo4j/unsafe/impl/batchimport/input/csv/InputEntityDeserializer.java @@ -31,6 +31,7 @@ import org.neo4j.helpers.collection.PrefetchingIterator; import org.neo4j.kernel.impl.util.Validator; import org.neo4j.unsafe.impl.batchimport.InputIterator; +import org.neo4j.unsafe.impl.batchimport.input.Collector; import org.neo4j.unsafe.impl.batchimport.input.InputEntity; import org.neo4j.unsafe.impl.batchimport.input.InputException; import org.neo4j.unsafe.impl.batchimport.input.UnexpectedEndOfInputException; @@ -51,12 +52,12 @@ public class InputEntityDeserializer private final Function decorator; private final Deserialization deserialization; private final Validator validator; - private final ArrayList> ignoredRowsSample = new ArrayList<>(); private final Extractors.StringExtractor stringExtractor = new Extractors.StringExtractor( false ); + private final Collector badCollector; InputEntityDeserializer( Header header, CharSeeker data, int delimiter, Deserialization deserialization, Function decorator, - Validator validator ) + Validator validator, Collector badCollector ) { this.header = header; this.data = data; @@ -64,6 +65,7 @@ public class InputEntityDeserializer this.deserialization = deserialization; this.decorator = decorator; this.validator = validator; + this.badCollector = badCollector; } public void initialize() @@ -87,24 +89,16 @@ protected ENTITY fetchNextOrNull() // Ignore additional values on this, but log it in case user doesn't realise that the header specifies // less columns than the data. Prints in close() so it only happens once per file. - int ignoredValues = 0; - StringBuilder restOfLine = new StringBuilder(); while ( !mark.isEndOfLine() ) { data.seek( mark, delimiter ); data.extract( mark, stringExtractor ); - restOfLine.append( String.valueOf( Character.toChars( delimiter ) ) ).append( stringExtractor.value() ); - ignoredValues++; + badCollector.collectExtraColumns( data.sourceDescription(), data.lineNumber(), stringExtractor.value() ); } entity = decorator.apply( entity ); validator.validate( entity ); - if ( ignoredValues > 0 && ignoredRowsSample.size() < 9 ) - { - ignoredRowsSample.add( Pair.of( data.lineNumber(), restOfLine.toString() ) ); - } - return entity; } catch ( IOException e ) @@ -191,18 +185,6 @@ private boolean deserializeNextFromSource() throws IOException @Override public void close() { - if ( ignoredRowsSample.size() > 0 ) - { - System.out.println( "\nWarning: ignored columns which were not present in header for" ); - System.out.println( String.format( " %s", data.sourceDescription() ) ); - - for ( Pair x : ignoredRowsSample ) - { - System.out.println( String.format( "Row %d: %s", x.first(), x.other() ) ); - } - System.out.println( "...\n" ); - } - try { data.close(); diff --git a/community/kernel/src/test/java/org/neo4j/unsafe/impl/batchimport/input/csv/CsvInputBatchImportIT.java b/community/kernel/src/test/java/org/neo4j/unsafe/impl/batchimport/input/csv/CsvInputBatchImportIT.java index 7b2f2b74f5fd1..e1b0f132f42e0 100644 --- a/community/kernel/src/test/java/org/neo4j/unsafe/impl/batchimport/input/csv/CsvInputBatchImportIT.java +++ b/community/kernel/src/test/java/org/neo4j/unsafe/impl/batchimport/input/csv/CsvInputBatchImportIT.java @@ -62,6 +62,7 @@ import org.neo4j.unsafe.impl.batchimport.Configuration; import org.neo4j.unsafe.impl.batchimport.Configuration.Default; import org.neo4j.unsafe.impl.batchimport.ParallelBatchImporter; +import org.neo4j.unsafe.impl.batchimport.input.Collectors; import org.neo4j.unsafe.impl.batchimport.input.InputNode; import org.neo4j.unsafe.impl.batchimport.input.InputRelationship; @@ -74,6 +75,7 @@ import static org.neo4j.kernel.impl.util.AutoCreatingHashMap.nested; import static org.neo4j.kernel.impl.util.AutoCreatingHashMap.values; import static org.neo4j.register.Registers.newDoubleLongRegister; +import static org.neo4j.unsafe.impl.batchimport.input.Collectors.silentBadCollector; import static org.neo4j.unsafe.impl.batchimport.input.InputEntity.NO_PROPERTIES; import static org.neo4j.unsafe.impl.batchimport.input.Inputs.csv; import static org.neo4j.unsafe.impl.batchimport.input.csv.Configuration.COMMAS; @@ -95,7 +97,7 @@ public void shouldImportDataComingFromCsvFiles() throws Exception try { importer.doImport( csv( nodeDataAsFile( nodeData ), relationshipDataAsFile( relationshipData ), - IdType.STRING, lowBufferSize( COMMAS ) ) ); + IdType.STRING, lowBufferSize( COMMAS ), silentBadCollector( 0 ) ) ); // THEN verifyImportedData( nodeData, relationshipData ); success = true; diff --git a/community/kernel/src/test/java/org/neo4j/unsafe/impl/batchimport/input/csv/CsvInputTest.java b/community/kernel/src/test/java/org/neo4j/unsafe/impl/batchimport/input/csv/CsvInputTest.java index 6f068018dc1fd..03f8def9bd1f2 100644 --- a/community/kernel/src/test/java/org/neo4j/unsafe/impl/batchimport/input/csv/CsvInputTest.java +++ b/community/kernel/src/test/java/org/neo4j/unsafe/impl/batchimport/input/csv/CsvInputTest.java @@ -60,7 +60,7 @@ import static org.neo4j.csv.reader.Readables.wrap; import static org.neo4j.helpers.ArrayUtil.union; import static org.neo4j.helpers.collection.IteratorUtil.asSet; -import static org.neo4j.unsafe.impl.batchimport.input.Collectors.badCollector; +import static org.neo4j.unsafe.impl.batchimport.input.Collectors.silentBadCollector; import static org.neo4j.unsafe.impl.batchimport.input.Group.GLOBAL; import static org.neo4j.unsafe.impl.batchimport.input.InputEntity.NO_PROPERTIES; import static org.neo4j.unsafe.impl.batchimport.input.InputEntityDecorators.additiveLabels; @@ -83,7 +83,7 @@ public void shouldProvideNodesFromCsvInput() throws Exception header( entry( null, Type.ID, idType.extractor( extractors ) ), entry( "name", Type.PROPERTY, extractors.string() ), entry( "labels", Type.LABEL, extractors.string() ) ), - null, null, idType, COMMAS, badCollector( 0 ) ); + null, null, idType, COMMAS, silentBadCollector( 0 ) ); // WHEN/THEN Iterator nodes = input.nodes().iterator(); @@ -105,7 +105,7 @@ public void shouldProvideRelationshipsFromCsvInput() throws Exception entry( "to", Type.END_ID, idType.extractor( extractors ) ), entry( "type", Type.TYPE, extractors.string() ), entry( "since", Type.PROPERTY, extractors.long_() ) ), idType, COMMAS, - badCollector( 0 ) ); + silentBadCollector( 0 ) ); // WHEN/THEN Iterator relationships = input.relationships().iterator(); @@ -129,7 +129,7 @@ nodeDataIterable, header( relationshipDataIterable, header( entry( null, Type.START_ID, idType.extractor( extractors ) ), entry( null, Type.END_ID, idType.extractor( extractors ) ) ), - idType, COMMAS, badCollector( 0 ) ); + idType, COMMAS, silentBadCollector( 0 ) ); // WHEN try ( ResourceIterator iterator = input.nodes().iterator() ) @@ -160,7 +160,7 @@ public void shouldCopeWithLinesThatHasTooFewValuesButStillValidates() throws Exc entry( "unit", Type.PROPERTY, extractors.string() ), entry( "type", Type.LABEL, extractors.string() ), entry( "kills", Type.PROPERTY, extractors.int_() ) ), - null, null, IdType.ACTUAL, Configuration.COMMAS, badCollector( 0 ) ); + null, null, IdType.ACTUAL, Configuration.COMMAS, silentBadCollector( 0 ) ); // WHEN try ( ResourceIterator nodes = input.nodes().iterator() ) @@ -184,7 +184,7 @@ public void shouldIgnoreValuesAfterHeaderEntries() throws Exception header( entry( null, Type.ID, extractors.long_() ), entry( "name", Type.PROPERTY, extractors.string() ) ), - null, null, IdType.ACTUAL, Configuration.COMMAS, badCollector( 0 ) ); + null, null, IdType.ACTUAL, Configuration.COMMAS, silentBadCollector( 4 ) ); // WHEN try ( ResourceIterator nodes = input.nodes().iterator() ) @@ -209,7 +209,7 @@ public void shouldHandleMultipleInputGroups() throws Exception Iterable> data = dataIterable( group1, group2 ); Input input = new CsvInput( data, defaultFormatNodeFileHeader(), null, null, - IdType.STRING, Configuration.COMMAS, badCollector( 0 ) ); + IdType.STRING, Configuration.COMMAS, silentBadCollector( 0 ) ); // WHEN iterating over them, THEN the expected data should come out ResourceIterator nodes = input.nodes().iterator(); @@ -232,7 +232,7 @@ public void shouldProvideAdditiveLabels() throws Exception additiveLabels( addedLabels ) ); Iterable> dataIterable = dataIterable( data ); Input input = new CsvInput( dataIterable, defaultFormatNodeFileHeader(), - null, null, IdType.ACTUAL, Configuration.COMMAS, badCollector( 0 ) ); + null, null, IdType.ACTUAL, Configuration.COMMAS, silentBadCollector( 0 ) ); // WHEN/THEN try ( ResourceIterator nodes = input.nodes().iterator() ) @@ -261,7 +261,7 @@ public void shouldProvideDefaultRelationshipType() throws Exception Iterable> dataIterable = dataIterable( data ); Input input = new CsvInput( null, null, dataIterable, defaultFormatRelationshipFileHeader(), IdType.ACTUAL, Configuration.COMMAS, - badCollector( 0 ) ); + silentBadCollector( 0 ) ); // WHEN/THEN try ( ResourceIterator relationships = input.relationships().iterator() ) @@ -284,7 +284,7 @@ public void shouldFailOnMissingRelationshipType() throws Exception Iterable> dataIterable = dataIterable( data ); Input input = new CsvInput( null, null, dataIterable, defaultFormatRelationshipFileHeader(), IdType.ACTUAL, Configuration.COMMAS, - badCollector( 0 ) ); + silentBadCollector( 0 ) ); // WHEN/THEN try ( ResourceIterator relationships = input.relationships().iterator() ) @@ -312,7 +312,7 @@ public void shouldAllowNodesWithoutIdHeader() throws Exception "Johan,2\n" ); Iterable> dataIterable = dataIterable( data ); Input input = new CsvInput( dataIterable, defaultFormatNodeFileHeader(), null, null, IdType.STRING, COMMAS, - badCollector( 0 ) ); + silentBadCollector( 0 ) ); // WHEN try ( ResourceIterator nodes = input.nodes().iterator() ) @@ -334,7 +334,7 @@ public void shouldAllowSomeNodesToBeAnonymous() throws Exception ",Johan,2\n" ); // this node is anonymous Iterable> dataIterable = dataIterable( data ); Input input = new CsvInput( dataIterable, defaultFormatNodeFileHeader(), null, null, IdType.STRING, COMMAS, - badCollector( 0 ) ); + silentBadCollector( 0 ) ); // WHEN try ( ResourceIterator nodes = input.nodes().iterator() ) @@ -356,7 +356,7 @@ public void shouldAllowNodesToBeAnonymousEvenIfIdHeaderIsNamed() throws Exceptio ",Johan,2\n" ); // this node is anonymous Iterable> dataIterable = dataIterable( data ); Input input = new CsvInput( dataIterable, defaultFormatNodeFileHeader(), null, null, IdType.STRING, COMMAS, - badCollector( 0 ) ); + silentBadCollector( 0 ) ); // WHEN try ( ResourceIterator nodes = input.nodes().iterator() ) @@ -378,7 +378,7 @@ public void shouldHaveIdSetAsPropertyIfIdHeaderEntryIsNamed() throws Exception "def,Johan,2\n" ); // this node is anonymous Iterable> dataIterable = dataIterable( data ); Input input = new CsvInput( dataIterable, defaultFormatNodeFileHeader(), null, null, IdType.STRING, COMMAS, - badCollector( 0 ) ); + silentBadCollector( 0 ) ); // WHEN try ( ResourceIterator nodes = input.nodes().iterator() ) @@ -400,7 +400,7 @@ public void shouldNotHaveIdSetAsPropertyIfIdHeaderEntryIsNamedForActualIds() thr "1,Johan,2\n" ); // this node is anonymous Iterable> dataIterable = dataIterable( data ); Input input = new CsvInput( dataIterable, defaultFormatNodeFileHeader(), null, null, IdType.ACTUAL, COMMAS, - badCollector( 0 ) ); + silentBadCollector( 0 ) ); // WHEN try ( ResourceIterator nodes = input.nodes().iterator() ) @@ -422,7 +422,7 @@ public void shouldIgnoreEmptyPropertyValues() throws Exception "1,Johan,Additional\n" ); Iterable> dataIterable = dataIterable( data ); Input input = new CsvInput( dataIterable, defaultFormatNodeFileHeader(), null, null, IdType.ACTUAL, COMMAS, - badCollector( 0 ) ); + silentBadCollector( 0 ) ); // WHEN try ( ResourceIterator nodes = input.nodes().iterator() ) @@ -444,7 +444,7 @@ public void shouldIgnoreEmptyIntPropertyValues() throws Exception "1,Johan,10\n" ); Iterable> dataIterable = dataIterable( data ); Input input = new CsvInput( dataIterable, defaultFormatNodeFileHeader(), null, null, IdType.ACTUAL, COMMAS, - badCollector( 0 ) ); + silentBadCollector( 0 ) ); // WHEN try ( ResourceIterator nodes = input.nodes().iterator() ) @@ -463,7 +463,7 @@ public void shouldFailOnArrayDelimiterBeingSameAsDelimiter() throws Exception try { new CsvInput( null, null, null, null, IdType.ACTUAL, customConfig( ',', ',', '"' ), - badCollector( 0 ) ); + silentBadCollector( 0 ) ); fail( "Should not be possible" ); } catch ( IllegalArgumentException e ) @@ -480,7 +480,7 @@ public void shouldFailOnQuotationCharacterBeingSameAsDelimiter() throws Exceptio try { new CsvInput( null, null, null, null, IdType.ACTUAL, customConfig( ',', ';', ',' ), - badCollector( 0 ) ); + silentBadCollector( 0 ) ); fail( "Should not be possible" ); } catch ( IllegalArgumentException e ) @@ -498,7 +498,7 @@ public void shouldFailOnQuotationCharacterBeingSameAsArrayDelimiter() throws Exc try { new CsvInput( null, null, null, null, IdType.ACTUAL, customConfig( ',', ';', ';' ), - badCollector( 0 ) ); + silentBadCollector( 0 ) ); fail( "Should not be possible" ); } catch ( IllegalArgumentException e ) @@ -524,7 +524,7 @@ public void shouldHaveNodesBelongToGroupSpecifiedInHeader() throws Exception header( entry( null, Type.ID, group.name(), idType.extractor( extractors ) ), entry( "name", Type.PROPERTY, extractors.string() ) ), null, null, idType, COMMAS, - badCollector( 0 ) ); + silentBadCollector( 0 ) ); // WHEN/THEN Iterator nodes = input.nodes().iterator(); @@ -550,7 +550,7 @@ public void shouldHaveRelationshipsSpecifyStartEndNodeIdGroupsInHeader() throws entry( null, Type.TYPE, extractors.string() ), entry( null, Type.END_ID, endNodeGroup.name(), idType.extractor( extractors ) ) ), idType, COMMAS, - badCollector( 0 ) ); + silentBadCollector( 0 ) ); // WHEN/THEN Iterator relationships = input.relationships().iterator(); @@ -571,7 +571,7 @@ public void shouldDoWithoutRelationshipTypeHeaderIfDefaultSupplied() throws Exce Iterable> dataIterable = dataIterable( data ); Input input = new CsvInput( null, null, dataIterable, defaultFormatRelationshipFileHeader(), IdType.ACTUAL, COMMAS, - badCollector( 0 ) ); + silentBadCollector( 0 ) ); // WHEN try ( ResourceIterator relationships = input.relationships().iterator() ) @@ -594,7 +594,7 @@ public void shouldIncludeDataSourceInformationOnBadFieldValueOrLine() throws Exc "3,Emil,12" ) ); Input input = new CsvInput( data, DataFactories.defaultFormatNodeFileHeader(), null, null, IdType.INTEGER, Configuration.COMMAS, - badCollector( 0 ) ); + silentBadCollector( 0 ) ); // WHEN try ( InputIterator nodes = input.nodes().iterator() ) @@ -624,7 +624,7 @@ public void shouldIgnoreNodeEntriesMarkedIgnoreUsingHeader() throws Exception "2,Johan,111,Person\n" + "3,Emil,12,Person" ) ); Input input = new CsvInput( data, defaultFormatNodeFileHeader(), null, null, IdType.INTEGER, COMMAS, - badCollector( 0 ) ); + silentBadCollector( 0 ) ); // WHEN try ( InputIterator nodes = input.nodes().iterator() ) @@ -646,7 +646,7 @@ public void shouldIgnoreRelationshipEntriesMarkedIgnoreUsingHeader() throws Exce "2,KNOWS,3,Johan,111\n" + "3,KNOWS,4,Emil,12" ) ); Input input = new CsvInput( null, null, data, defaultFormatRelationshipFileHeader(), IdType.INTEGER, COMMAS, - badCollector( 0 ) ); + silentBadCollector( 0 ) ); // WHEN try ( InputIterator relationships = input.relationships().iterator() ) @@ -667,7 +667,7 @@ public void shouldPropagateExceptionFromFailingDecorator() throws Exception DataFactories.nodeData( CsvInputTest.data( ":ID,name\n1,Mattias", new FailingNodeDecorator( failure ) ) ); Input input = new CsvInput( data, defaultFormatNodeFileHeader(), null, null, IdType.INTEGER, COMMAS, - badCollector( 0 ) ); + silentBadCollector( 0 ) ); // WHEN try ( InputIterator nodes = input.nodes().iterator() ) @@ -691,7 +691,7 @@ public void shouldNotIncludeEmptyArraysInEntities() throws Exception "2,a;b,10;20" ) ); Input input = new CsvInput( data, defaultFormatNodeFileHeader(), null, null, IdType.INTEGER, COMMAS, - badCollector( 0 ) ); + silentBadCollector( 0 ) ); // WHEN/THEN try ( InputIterator nodes = input.nodes().iterator() ) @@ -711,7 +711,7 @@ public void shouldFailOnRelationshipWithMissingStartIdField() throws Exception ":START_ID,:END_ID,:TYPE\n" + ",1," ) ); Input input = new CsvInput( null, null, data, defaultFormatRelationshipFileHeader(), IdType.INTEGER, COMMAS, - badCollector( 0 ) ); + silentBadCollector( 0 ) ); // WHEN try ( InputIterator relationships = input.relationships().iterator() ) @@ -734,7 +734,7 @@ public void shouldFailOnRelationshipWithMissingEndIdField() throws Exception ":START_ID,:END_ID,:TYPE\n" + "1,," ) ); Input input = new CsvInput( null, null, data, defaultFormatRelationshipFileHeader(), IdType.INTEGER, COMMAS, - badCollector( 0 ) ); + silentBadCollector( 0 ) ); // WHEN try ( InputIterator relationships = input.relationships().iterator() ) @@ -765,7 +765,7 @@ public boolean emptyQuotedStringsAsNull() } }; Input input = new CsvInput( data, defaultFormatNodeFileHeader(), - null, null, IdType.INTEGER, config, badCollector( 0 ) ); + null, null, IdType.INTEGER, config, silentBadCollector( 0 ) ); // WHEN try ( InputIterator nodes = input.nodes().iterator() ) diff --git a/community/kernel/src/test/java/org/neo4j/unsafe/impl/batchimport/input/csv/ExternalPropertiesDecoratorTest.java b/community/kernel/src/test/java/org/neo4j/unsafe/impl/batchimport/input/csv/ExternalPropertiesDecoratorTest.java index aa64a9887137b..2c22a77df988e 100644 --- a/community/kernel/src/test/java/org/neo4j/unsafe/impl/batchimport/input/csv/ExternalPropertiesDecoratorTest.java +++ b/community/kernel/src/test/java/org/neo4j/unsafe/impl/batchimport/input/csv/ExternalPropertiesDecoratorTest.java @@ -38,6 +38,7 @@ import static org.junit.Assert.assertNotNull; import static org.neo4j.helpers.collection.MapUtil.map; +import static org.neo4j.unsafe.impl.batchimport.input.Collectors.silentBadCollector; import static org.neo4j.unsafe.impl.batchimport.input.InputEntityDecorators.NO_NODE_DECORATOR; import static org.neo4j.unsafe.impl.batchimport.input.csv.DataFactories.defaultFormatNodeFileHeader; @@ -57,7 +58,8 @@ public void shouldCombineNodesWithExternalPropertiesFile() throws Exception IdType idType = IdType.STRING; Function externalPropertiesDecorator = new ExternalPropertiesDecorator( DataFactories.data( NO_NODE_DECORATOR, readable( propertyData ) ), - defaultFormatNodeFileHeader(), config, idType, UpdateBehaviour.ADD ); + defaultFormatNodeFileHeader(), config, idType, UpdateBehaviour.ADD, + silentBadCollector( 0 )); // WHEN assertProperties( externalPropertiesDecorator.apply( node( "1", "key", "value1" ) ), @@ -84,7 +86,8 @@ public void shouldCombineNodesWithExternalPropertyArraysFile() throws Exception IdType idType = IdType.STRING; Function externalPropertiesDecorator = new ExternalPropertiesDecorator( DataFactories.data( NO_NODE_DECORATOR, readable( propertyData ) ), - defaultFormatNodeFileHeader(), config, idType, UpdateBehaviour.ADD ); + defaultFormatNodeFileHeader(), config, idType, UpdateBehaviour.ADD, + silentBadCollector( 0 )); // WHEN assertProperties( externalPropertiesDecorator.apply( node( "1", "key", "value1", "email", "existing" ) ),