Makes :TYPE header in import input optional

which is OK if a relationship decorator that provides the relationship
type instead is supplied. For the import tool it may look something like:

  --nodes mynodes.csv --relationships:MY_TYPE myrels.csv

where the header of myrels.csv may look something like:

  :START_ID,:END_ID

i.e. no :TYPE in there.

community/import-tool/src/test/java/org/neo4j/tooling/ImportToolTest.java

@@ -239,7 +239,7 @@ public void shouldImportGroupsOfOverlappingIds() throws Exception
                                    nodeData( false, config, groupOneNodeIds, alwaysTrue() ),
                 "--nodes",         nodeHeader( config, groupTwo ) + MULTI_FILE_DELIMITER +
                                    nodeData( false, config, groupTwoNodeIds, alwaysTrue() ),
-                "--relationships", relationshipHeader( config, groupOne, groupTwo ) + MULTI_FILE_DELIMITER +
+                "--relationships", relationshipHeader( config, groupOne, groupTwo, true ) + MULTI_FILE_DELIMITER +
                                    relationshipData( false, config, rels.iterator(), alwaysTrue(), true )
 
                 ) );
@@ -290,6 +290,26 @@ public void shouldNotBeAbleToMixSpecifiedAndUnspecifiedGroups() throws Exception
         }
     }
 
+    @Test
+    public void shouldImportWithoutTypeSpecifiedInRelationshipHeaderbutWithDefaultTypeInArgument() throws Exception
+    {
+        // GIVEN
+        List<String> nodeIds = nodeIds();
+        Configuration config = Configuration.COMMAS;
+        String type = randomType();
+
+        // WHEN
+        ImportTool.main( arguments(
+                "--into",          directory.absolutePath(),
+                "--nodes",         nodeData( true, config, nodeIds, alwaysTrue() ).getAbsolutePath(),
+                                   // there will be no :TYPE specified in the header of the relationships below
+                "--relationships:" + type,
+                                   relationshipData( true, config, nodeIds, alwaysTrue(), false ).getAbsolutePath() ) );
+
+        // THEN
+        verifyData();
+    }
+
     protected void assertNodeHasLabels( Node node, String[] names )
     {
         for ( String name : names )
@@ -457,7 +477,7 @@ private File relationshipData( boolean includeHeader, Configuration config,
         {
             if ( includeHeader )
             {
-                writeRelationshipHeader( writer, config, null, null );
+                writeRelationshipHeader( writer, config, null, null, specifyType );
             }
             writeRelationshipData( writer, config, data, linePredicate, specifyType );
         }
@@ -466,16 +486,16 @@ private File relationshipData( boolean includeHeader, Configuration config,
 
     private File relationshipHeader( Configuration config ) throws FileNotFoundException
     {
-        return relationshipHeader( config, null, null );
+        return relationshipHeader( config, null, null, true );
     }
 
-    private File relationshipHeader( Configuration config, String startIdGroup, String endIdGroup )
+    private File relationshipHeader( Configuration config, String startIdGroup, String endIdGroup, boolean specifyType )
             throws FileNotFoundException
     {
         File file = directory.file( fileName( "relationships-header.csv" ) );
         try ( PrintStream writer = new PrintStream( file ) )
         {
-            writeRelationshipHeader( writer, config, startIdGroup, endIdGroup );
+            writeRelationshipHeader( writer, config, startIdGroup, endIdGroup, specifyType );
         }
         return file;
     }
@@ -486,13 +506,14 @@ private String fileName( String name )
     }
 
     private void writeRelationshipHeader( PrintStream writer, Configuration config,
-            String startIdGroup, String endIdGroup )
+            String startIdGroup, String endIdGroup, boolean specifyType )
     {
         char delimiter = config.delimiter();
         writer.println(
                 idEntry( null, Type.START_ID, startIdGroup ) + delimiter +
-                idEntry( null, Type.END_ID, endIdGroup ) + delimiter +
-                ":" + Type.TYPE + delimiter + "created:long" );
+                idEntry( null, Type.END_ID, endIdGroup ) +
+                (specifyType ? (delimiter + ":" + Type.TYPE) : "") +
+                delimiter + "created:long" );
     }
 
     private void writeRelationshipData( PrintStream writer, Configuration config,
@@ -508,11 +529,10 @@ private void writeRelationshipData( PrintStream writer, Configuration config,
                     break;
                 }
                 Triplet<String,String,String> entry = data.next();
-                writer.println(
-                        entry.first() + delimiter +
-                        entry.second() + delimiter +
-                        (specifyType ? entry.third() : "") + delimiter +
-                        currentTimeMillis() );
+                writer.println( entry.first() +
+                        delimiter + entry.second() +
+                        (specifyType ? (delimiter + entry.third()) : "") +
+                        delimiter + currentTimeMillis() );
             }
         }
     }

community/kernel/src/main/java/org/neo4j/unsafe/impl/batchimport/input/DataException.java

@@ -0,0 +1,28 @@
+/**
+ * Copyright (c) 2002-2015 "Neo Technology,"
+ * Network Engine for Objects in Lund AB [http://neotechnology.com]
+ *
+ * This file is part of Neo4j.
+ *
+ * Neo4j is free software: you can redistribute it and/or modify
+ * it under the terms of the GNU General Public License as published by
+ * the Free Software Foundation, either version 3 of the License, or
+ * (at your option) any later version.
+ *
+ * This program is distributed in the hope that it will be useful,
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+ * GNU General Public License for more details.
+ *
+ * You should have received a copy of the GNU General Public License
+ * along with this program.  If not, see <http://www.gnu.org/licenses/>.
+ */
+package org.neo4j.unsafe.impl.batchimport.input;
+
+public class DataException extends InputException
+{
+    public DataException( String message )
+    {
+        super( message );
+    }
+}

community/kernel/src/main/java/org/neo4j/unsafe/impl/batchimport/input/HeaderException.java

@@ -25,9 +25,4 @@ public HeaderException( String message )
     {
         super( message );
     }
-
-    public HeaderException( String message, Throwable cause )
-    {
-        super( message, cause );
-    }
 }

community/kernel/src/main/java/org/neo4j/unsafe/impl/batchimport/input/csv/DataFactories.java

@@ -457,7 +457,8 @@ private static class DefaultRelationshipFileHeaderParser extends AbstractDefault
     {
         protected DefaultRelationshipFileHeaderParser( HeaderCharSeekerFactory headerCharSeekerFactory )
         {
-            super( headerCharSeekerFactory, Type.START_ID, Type.END_ID, Type.TYPE );
+            // Don't have TYPE as mandatory since a decorator could provide that
+            super( headerCharSeekerFactory, Type.START_ID, Type.END_ID );
         }
 
         @Override

community/kernel/src/main/java/org/neo4j/unsafe/impl/batchimport/input/csv/InputEntityDeserializer.java

@@ -26,6 +26,7 @@
 import org.neo4j.csv.reader.Mark;
 import org.neo4j.function.Function;
 import org.neo4j.helpers.collection.PrefetchingResourceIterator;
+import org.neo4j.unsafe.impl.batchimport.input.DataException;
 import org.neo4j.unsafe.impl.batchimport.input.InputEntity;
 import org.neo4j.unsafe.impl.batchimport.input.InputException;
 import org.neo4j.unsafe.impl.batchimport.input.UnexpectedEndOfInputException;
@@ -117,7 +118,9 @@ protected ENTITY fetchNextOrNull()
                 data.seek( mark, delimiter );
             }
 
-            return decorator.apply( entity );
+            entity = decorator.apply( entity );
+            validate( entity );
+            return entity;
         }
         catch ( IOException e )
         {
@@ -129,6 +132,14 @@ protected ENTITY fetchNextOrNull()
         }
     }
 
+    /**
+     * Called after the entity has been fully populated.
+     * @throws DataException on validation error.
+     */
+    protected void validate( ENTITY entity )
+    {   // No default validation
+    }
+
     protected void addProperty( Header.Entry entry, Object value )
     {
         if ( value != null )

community/kernel/src/main/java/org/neo4j/unsafe/impl/batchimport/input/csv/InputRelationshipDeserializer.java

@@ -22,6 +22,7 @@
 import org.neo4j.csv.reader.CharSeeker;
 import org.neo4j.function.Function;
 import org.neo4j.kernel.impl.store.id.IdSequence;
+import org.neo4j.unsafe.impl.batchimport.input.DataException;
 import org.neo4j.unsafe.impl.batchimport.input.Group;
 import org.neo4j.unsafe.impl.batchimport.input.Groups;
 import org.neo4j.unsafe.impl.batchimport.input.InputRelationship;
@@ -73,4 +74,13 @@ protected InputRelationship convertToInputEntity( Object[] properties )
         return new InputRelationship( idSequence.nextId(), properties, null,
                 startNodeGroup, startNode, endNodeGroup, endNode, type, null );
     }
+
+    @Override
+    protected void validate( InputRelationship entity )
+    {
+        if ( !entity.hasTypeId() && entity.type() == null )
+        {
+            throw new DataException( entity + " is missing " + Type.TYPE + " field" );
+        }
+    }
 }

community/kernel/src/test/java/org/neo4j/unsafe/impl/batchimport/input/csv/CsvInputTest.java

@@ -36,6 +36,7 @@
 import org.neo4j.helpers.collection.Iterables;
 import org.neo4j.test.TargetDirectory;
 import org.neo4j.test.TargetDirectory.TestDirectory;
+import org.neo4j.unsafe.impl.batchimport.input.DataException;
 import org.neo4j.unsafe.impl.batchimport.input.Group;
 import org.neo4j.unsafe.impl.batchimport.input.Groups;
 import org.neo4j.unsafe.impl.batchimport.input.Input;
@@ -113,7 +114,8 @@ public void shouldCloseDataIteratorsInTheEnd() throws Exception
         CharSeeker relationshipData = spy( charSeeker( "test" ) );
         IdType idType = IdType.STRING;
         Iterable<DataFactory<InputNode>> nodeDataIterable = dataIterable( given( nodeData ) );
-        Iterable<DataFactory<InputRelationship>> relationshipDataIterable = dataIterable( given( relationshipData ) );
+        Iterable<DataFactory<InputRelationship>> relationshipDataIterable =
+                dataIterable( data( relationshipData, defaultRelationshipType( "TYPE" ) ) );
         Input input = new CsvInput(
                 nodeDataIterable, header(
                         entry( null, Type.ID, idType.extractor( extractors ) ) ),
@@ -263,6 +265,34 @@ public void shouldProvideDefaultRelationshipType() throws Exception
         }
     }
 
+    @Test
+    public void shouldFailOnMissingRelationshipType() throws Exception
+    {
+        // GIVEN
+        String type = "CUSTOM";
+        DataFactory<InputRelationship> data = data( ":START_ID,:END_ID,:TYPE\n" +
+                "0,1," + type + "\n" +
+                "1,2," );
+        Iterable<DataFactory<InputRelationship>> dataIterable = dataIterable( data );
+        Input input = new CsvInput( null, null,
+                dataIterable, defaultFormatRelationshipFileHeader(), IdType.ACTUAL, Configuration.COMMAS );
+
+        // WHEN/THEN
+        try ( ResourceIterator<InputRelationship> relationships = input.relationships().iterator() )
+        {
+            assertRelationship( relationships.next(), 0L, 0L, 1L, type, NO_PROPERTIES );
+            try
+            {
+                relationships.next();
+                fail( "Should have failed" );
+            }
+            catch ( DataException e )
+            {
+                assertTrue( e.getMessage().contains( Type.TYPE.name() ) );
+            }
+        }
+    }
+
     @Test
     public void shouldAllowNodesWithoutIdHeader() throws Exception
     {
@@ -532,19 +562,55 @@ public char arrayDelimiter()
         };
     }
 
+    @Test
+    public void shouldDoWithoutRelationshipTypeHeaderIfDefaultSupplied() throws Exception
+    {
+        // GIVEN relationship data w/o :TYPE header
+        String defaultType = "HERE";
+        DataFactory<InputRelationship> data = data(
+                ":START_ID,:END_ID,name\n" +
+                "0,1,First\n" +
+                "2,3,Second\n", defaultRelationshipType( defaultType ) );
+        Iterable<DataFactory<InputRelationship>> dataIterable = dataIterable( data );
+        Input input = new CsvInput( null, null, dataIterable, defaultFormatRelationshipFileHeader(),
+                IdType.ACTUAL, COMMAS );
+
+        // WHEN
+        try ( ResourceIterator<InputRelationship> relationships = input.relationships().iterator() )
+        {
+            // THEN
+            assertRelationship( relationships.next(), 0, 0L, 1L, defaultType, properties( "name", "First" ) );
+            assertRelationship( relationships.next(), 1, 2L, 3L, defaultType, properties( "name", "Second" ) );
+            assertFalse( relationships.hasNext() );
+        }
+    }
+
     private <ENTITY extends InputEntity> DataFactory<ENTITY> given( final CharSeeker data )
     {
         return new DataFactory<ENTITY>()
         {
             @Override
             public Data<ENTITY> create( Configuration config )
             {
-                return noDecoratorData( data, Functions.<ENTITY>identity() );
+                return dataItem( data, Functions.<ENTITY>identity() );
+            }
+        };
+    }
+
+    private <ENTITY extends InputEntity> DataFactory<ENTITY> data( final CharSeeker data,
+            final Function<ENTITY,ENTITY> decorator )
+    {
+        return new DataFactory<ENTITY>()
+        {
+            @Override
+            public Data<ENTITY> create( Configuration config )
+            {
+                return dataItem( data, decorator );
             }
         };
     }
 
-    private <ENTITY extends InputEntity> Data<ENTITY> noDecoratorData( final CharSeeker data,
+    private <ENTITY extends InputEntity> Data<ENTITY> dataItem( final CharSeeker data,
             final Function<ENTITY,ENTITY> decorator )
     {
         return new Data<ENTITY>()
@@ -641,7 +707,7 @@ private <ENTITY extends InputEntity> DataFactory<ENTITY> data( final String data
             @Override
             public Data<ENTITY> create( Configuration config )
             {
-                return noDecoratorData( charSeeker( data ), decorator );
+                return dataItem( charSeeker( data ), decorator );
             }
         };
     }