Skip to content

Commit

Permalink
Some class extractions and removal of unused code
Browse files Browse the repository at this point in the history
  • Loading branch information
@tinwelint
tinwelint committed Aug 13, 2016
1 parent 8ff9689 commit fd3ca7b
Show file tree
Hide file tree
Showing 5 changed files with 60 additions and 158 deletions.
19 changes: 1 addition & 18 deletions community/kernel/src/main/java/org/neo4j/unsafe/impl/batchimport/input/csv/CsvInput.java
View file
Expand Up @@ -34,7 +34,6 @@
import org.neo4j.unsafe.impl.batchimport.input.Input;
import org.neo4j.unsafe.impl.batchimport.input.InputNode;
import org.neo4j.unsafe.impl.batchimport.input.InputRelationship;
import org.neo4j.unsafe.impl.batchimport.input.MissingRelationshipDataException;

/**
* Provides {@link Input} from data contained in tabular/csv form. Expects factories for instantiating
Expand Down Expand Up @@ -145,23 +144,7 @@ protected InputEntityDeserializer<InputRelationship> entityDeserializer( CharSee
{
return new InputEntityDeserializer<>( dataHeader, dataStream, config.delimiter(),
new InputRelationshipDeserialization( dataStream, dataHeader, groups ),
decorator, entity -> {
if ( entity.startNode() == null )
{
throw new MissingRelationshipDataException(Type.START_ID,
entity + " is missing " + Type.START_ID + " field" );
}
if ( entity.endNode() == null )
{
throw new MissingRelationshipDataException(Type.END_ID,
entity + " is missing " + Type.END_ID + " field" );
}
if ( !entity.hasTypeId() && entity.type() == null )
{
throw new MissingRelationshipDataException(Type.TYPE,
entity + " is missing " + Type.TYPE + " field" );
}
}, badCollector );
decorator, new InputRelationshipValidator(), badCollector );
}
};
}
Expand Down
120 changes: 7 additions & 113 deletions ...unity/kernel/src/main/java/org/neo4j/unsafe/impl/batchimport/input/csv/DataFactories.java
View file
Expand Up @@ -123,17 +123,7 @@ public Function<ENTITY,ENTITY> decorator()
*/
public static Header.Factory defaultFormatNodeFileHeader()
{
return new DefaultNodeFileHeaderParser( READ_FROM_DATA_SEEKER );
}

/**
* Header parser that will read header information, using the default node header format,
* from a {@link Readable} containing that data.
* @param reader {@link Readable} containing header data.
*/
public static Header.Factory defaultFormatNodeFileHeader( CharReadable reader )
{
return new DefaultNodeFileHeaderParser( new HeaderFromSeparateReaderFactory( reader ) );
return new DefaultNodeFileHeaderParser();
}

/**
Expand All @@ -145,115 +135,31 @@ public static Header.Factory defaultFormatNodeFileHeader( CharReadable reader )
*/
public static Header.Factory defaultFormatRelationshipFileHeader()
{
return new DefaultRelationshipFileHeaderParser( READ_FROM_DATA_SEEKER );
}

/**
* Header parser that will read header information, using the default relationship header format,
* from a {@link Readable} containing that data.
* @param reader {@link Readable} containing header data.
*/
public static Header.Factory defaultFormatRelationshipFileHeader( CharReadable reader )
{
return new DefaultRelationshipFileHeaderParser( new HeaderFromSeparateReaderFactory( reader ) );
}

/**
* Provides {@link CharSeeker} to read and parse header information from.
*/
private interface HeaderCharSeekerFactory
{
/**
* @param seeker the {@link CharSeeker} for the data file, if that's what we want.
* @param config
* @return the {@link CharSeeker} to extract header information from.
* @throws IOException if {@link CharSeeker} couldn't be provided.
*/
CharSeeker open( CharSeeker seeker, Configuration config ) throws IOException;

/**
* Closes the header {@link CharSeeker}. Only close if {@link #open(CharSeeker, Configuration)} opens its own.
* @param seeker {@link CharSeeker} returned from {@link #open(CharSeeker, Configuration)}.
*/
void close( CharSeeker seeker );
}

/**
* Just uses the provided {@link CharSeeker} containing the data itself.
*/
private static final HeaderCharSeekerFactory READ_FROM_DATA_SEEKER = new HeaderCharSeekerFactory()
{
@Override
public CharSeeker open( CharSeeker seeker, Configuration config )
{
return seeker;
}

@Override
public void close( CharSeeker seeker )
{ // Leave it open for data reading later
}
};

private static abstract class SeparateHeaderReaderFactory implements HeaderCharSeekerFactory
{
@Override
public void close( CharSeeker seeker )
{
try
{
seeker.close();
}
catch ( IOException e )
{
throw new RuntimeException( "Unable to close header reader", e );
}
}
}

private static class HeaderFromSeparateReaderFactory extends SeparateHeaderReaderFactory
{
private final CharReadable readable;

HeaderFromSeparateReaderFactory( CharReadable readable )
{
this.readable = readable;
}

@Override
public CharSeeker open( CharSeeker seeker, Configuration config ) throws IOException
{
return charSeeker( readable, config, true );
}
return new DefaultRelationshipFileHeaderParser();
}

private static abstract class AbstractDefaultFileHeaderParser implements Header.Factory
{
private final Type[] mandatoryTypes;
private final HeaderCharSeekerFactory headerCharSeekerFactory;

protected AbstractDefaultFileHeaderParser( HeaderCharSeekerFactory headerCharSeekerFactory,
Type... mandatoryTypes )
protected AbstractDefaultFileHeaderParser( Type... mandatoryTypes )
{
this.headerCharSeekerFactory = headerCharSeekerFactory;
this.mandatoryTypes = mandatoryTypes;
}

@Override
public Header create( CharSeeker dataSeeker, Configuration config, IdType idType )
{
CharSeeker headerSeeker = null;
try
{
headerSeeker = headerCharSeekerFactory.open( dataSeeker, config );
Mark mark = new Mark();
Extractors extractors = new Extractors( config.arrayDelimiter(), config.emptyQuotedStringsAsNull() );
Extractor<?> idExtractor = idType.extractor( extractors );
int delimiter = config.delimiter();
List<Header.Entry> columns = new ArrayList<>();
for ( int i = 0; !mark.isEndOfLine() && headerSeeker.seek( mark, delimiter ); i++ )
for ( int i = 0; !mark.isEndOfLine() && dataSeeker.seek( mark, delimiter ); i++ )
{
String entryString = headerSeeker.tryExtract( mark, extractors.string() )
String entryString = dataSeeker.tryExtract( mark, extractors.string() )
? extractors.string().value() : null;
HeaderEntrySpec spec = new HeaderEntrySpec( entryString );

Expand All @@ -275,13 +181,6 @@ public Header create( CharSeeker dataSeeker, Configuration config, IdType idType
{
throw new RuntimeException( e );
}
finally
{
if ( headerSeeker != null )
{
headerCharSeekerFactory.close( headerSeeker );
}
}
}

private void validateHeader( Entry[] entries )
Expand Down Expand Up @@ -383,11 +282,6 @@ private static class HeaderEntrySpec

private static class DefaultNodeFileHeaderParser extends AbstractDefaultFileHeaderParser
{
public DefaultNodeFileHeaderParser( HeaderCharSeekerFactory headerCharSeekerFactory )
{
super( headerCharSeekerFactory );
}

@Override
protected Header.Entry entry( int index, String name, String typeSpec, String groupName, Extractors extractors,
Extractor<?> idExtractor )
Expand Down Expand Up @@ -427,10 +321,10 @@ else if ( isRecognizedType( typeSpec ) )

private static class DefaultRelationshipFileHeaderParser extends AbstractDefaultFileHeaderParser
{
protected DefaultRelationshipFileHeaderParser( HeaderCharSeekerFactory headerCharSeekerFactory )
protected DefaultRelationshipFileHeaderParser()
{
// Don't have TYPE as mandatory since a decorator could provide that
super( headerCharSeekerFactory, Type.START_ID, Type.END_ID );
super( Type.START_ID, Type.END_ID );
}

@Override
Expand Down
47 changes: 47 additions & 0 deletions ...src/main/java/org/neo4j/unsafe/impl/batchimport/input/csv/InputRelationshipValidator.java
View file
@@ -0,0 +1,47 @@
/*
* Copyright (c) 2002-2016 "Neo Technology,"
* Network Engine for Objects in Lund AB [http://neotechnology.com]
*
* This file is part of Neo4j.
*
* Neo4j is free software: you can redistribute it and/or modify
* it under the terms of the GNU General Public License as published by
* the Free Software Foundation, either version 3 of the License, or
* (at your option) any later version.
*
* This program is distributed in the hope that it will be useful,
* but WITHOUT ANY WARRANTY; without even the implied warranty of
* MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
* GNU General Public License for more details.
*
* You should have received a copy of the GNU General Public License
* along with this program. If not, see <http://www.gnu.org/licenses/>.
*/
package org.neo4j.unsafe.impl.batchimport.input.csv;

import org.neo4j.kernel.impl.util.Validator;
import org.neo4j.unsafe.impl.batchimport.input.InputRelationship;
import org.neo4j.unsafe.impl.batchimport.input.MissingRelationshipDataException;

class InputRelationshipValidator implements Validator<InputRelationship>
{
@Override
public void validate( InputRelationship entity )
{
if ( entity.startNode() == null )
{
throw new MissingRelationshipDataException(Type.START_ID,
entity + " is missing " + Type.START_ID + " field" );
}
if ( entity.endNode() == null )
{
throw new MissingRelationshipDataException(Type.END_ID,
entity + " is missing " + Type.END_ID + " field" );
}
if ( !entity.hasTypeId() && entity.type() == null )
{
throw new MissingRelationshipDataException(Type.TYPE,
entity + " is missing " + Type.TYPE + " field" );
}
}
}
4 changes: 2 additions & 2 deletions community/kernel/src/test/java/org/neo4j/unsafe/impl/batchimport/input/csv/CsvInputTest.java
View file
Expand Up @@ -27,8 +27,8 @@
import org.junit.Rule;
import org.junit.Test;

import org.neo4j.csv.reader.BufferedCharSeeker;
import org.neo4j.csv.reader.CharSeeker;
import org.neo4j.csv.reader.CharSeekers;
import org.neo4j.csv.reader.Extractor;
import org.neo4j.csv.reader.Extractors;
import org.neo4j.graphdb.ResourceIterator;
Expand Down Expand Up @@ -940,7 +940,7 @@ public int bufferSize()

private static CharSeeker charSeeker( String data )
{
return new BufferedCharSeeker( wrap( new StringReader( data ) ), SEEKER_CONFIG );
return CharSeekers.charSeeker( wrap( new StringReader( data ) ), SEEKER_CONFIG, false );
}

@SuppressWarnings( { "rawtypes", "unchecked" } )
Expand Down
28 changes: 3 additions & 25 deletions ...y/kernel/src/test/java/org/neo4j/unsafe/impl/batchimport/input/csv/DataFactoriesTest.java
View file
Expand Up @@ -25,8 +25,8 @@

import org.junit.Test;

import org.neo4j.csv.reader.BufferedCharSeeker;
import org.neo4j.csv.reader.CharSeeker;
import org.neo4j.csv.reader.CharSeekers;
import org.neo4j.csv.reader.Extractor;
import org.neo4j.csv.reader.Extractors;
import org.neo4j.unsafe.impl.batchimport.input.DuplicateHeaderException;
Expand All @@ -38,8 +38,7 @@
import static org.junit.Assert.assertEquals;
import static org.junit.Assert.assertThat;
import static org.junit.Assert.fail;
import static org.mockito.Mockito.mock;
import static org.mockito.Mockito.verifyZeroInteractions;

import static org.neo4j.csv.reader.Readables.sources;
import static org.neo4j.csv.reader.Readables.wrap;
import static org.neo4j.helpers.ArrayUtil.array;
Expand Down Expand Up @@ -175,27 +174,6 @@ public void shouldAllowMissingIdHeaderEntry() throws Exception
seeker.close();
}

@Test
public void shouldParseHeaderFromSeparateReader() throws Exception
{
// GIVEN
CharSeeker dataSeeker = mock( CharSeeker.class );
Header.Factory headerFactory =
defaultFormatNodeFileHeader( wrap( new StringReader( "id:ID\tname:String\tbirth_date:long" ) ) );
Extractors extractors = new Extractors( ';' );

// WHEN
Header header = headerFactory.create( dataSeeker, TABS, IdType.ACTUAL );

// THEN
assertArrayEquals( array(
entry( "id", Type.ID, extractors.long_() ),
entry( "name", Type.PROPERTY, extractors.string() ),
entry( "birth_date", Type.PROPERTY, extractors.long_() ) ), header.entries() );
verifyZeroInteractions( dataSeeker );
dataSeeker.close();
}

@Test
public void shouldParseHeaderFromFirstLineOfFirstInputFile() throws Exception
{
Expand Down Expand Up @@ -300,7 +278,7 @@ public int bufferSize()

private CharSeeker seeker( String data )
{
return new BufferedCharSeeker( wrap( new StringReader( data ) ), SEEKER_CONFIG );
return CharSeekers.charSeeker( wrap( new StringReader( data ) ), SEEKER_CONFIG, false );
}

private static Configuration withBufferSize( Configuration config, final int bufferSize )
Expand Down

0 comments on commit fd3ca7b

Please sign in to comment.