Fixes neo4j-contrib#2932: The apoc.import.csv skipLines config doesn'…

…t work correctly (neo4j-contrib#2984) (neo4j-contrib#3012) Co-authored-by: Giuseppe Villani <giuseppe.villani@larus-ba.it>
gem-neo4j · Jul 12, 2022 · f0fffbb · f0fffbb
1 parent d9f5b70
commit f0fffbb
Show file tree

Hide file tree

Showing 3 changed files with 36 additions and 3 deletions.
diff --git a/core/src/main/java/apoc/export/csv/CsvEntityLoader.java b/core/src/main/java/apoc/export/csv/CsvEntityLoader.java
@@ -7,7 +7,9 @@
 import apoc.load.Mapping;
 import apoc.load.util.Results;
 import apoc.util.FileUtils;
+import com.opencsv.CSVParserBuilder;
 import com.opencsv.CSVReader;
+import com.opencsv.CSVReaderBuilder;
 import org.neo4j.graphdb.*;
 import org.neo4j.logging.Log;
 
@@ -48,7 +50,6 @@ public void loadNodes(final Object fileName, final List<String> labels, final Gr
 
         try (final CountingReader reader = FileUtils.readerFor(fileName, clc.getCompressionAlgo())) {
             final String header = readFirstLine(reader);
-            reader.skip(clc.getSkipLines() - 1);
             final List<CsvHeaderField> fields = CsvHeaderFields.processHeader(header, clc.getDelimiter(), clc.getQuotationCharacter());
 
             final Optional<CsvHeaderField> idField = fields.stream()
@@ -67,7 +68,13 @@ public void loadNodes(final Object fileName, final List<String> labels, final Gr
 
             final Map<String, Mapping> mapping = getMapping(fields);
 
-            final CSVReader csv = new CSVReader(reader, clc.getDelimiter(), clc.getQuotationCharacter());
+            final CSVReader csv = new CSVReaderBuilder(reader)
+                    .withCSVParser(new CSVParserBuilder()
+                            .withSeparator(clc.getDelimiter())
+                            .withQuoteChar(clc.getQuotationCharacter())
+                            .build())
+                    .withSkipLines(clc.getSkipLines() - 1)
+                    .build();
 
             final String[] loadCsvCompatibleHeader = fields.stream().map(f -> f.getName()).toArray(String[]::new);
             int lineNo = 0;

diff --git a/core/src/main/java/apoc/export/csv/CsvLoaderConfig.java b/core/src/main/java/apoc/export/csv/CsvLoaderConfig.java
@@ -2,6 +2,7 @@
 
 import apoc.util.CompressionAlgo;
 import apoc.util.CompressionConfig;
+import apoc.util.Util;
 
 import java.util.Map;
 
@@ -121,7 +122,7 @@ public static CsvLoaderConfig from(Map<String, Object> config) {
         if (config.get(ARRAY_DELIMITER) != null) builder.arrayDelimiter(getCharacterOrString(config, ARRAY_DELIMITER));
         if (config.get(QUOTATION_CHARACTER) != null) builder.quotationCharacter(getCharacterOrString(config, QUOTATION_CHARACTER));
         if (config.get(STRING_IDS) != null) builder.stringIds((boolean) config.get(STRING_IDS));
-        if (config.get(SKIP_LINES) != null) builder.skipLines((int) config.get(SKIP_LINES));
+        if (config.get(SKIP_LINES) != null) builder.skipLines(Util.toInteger(config.get(SKIP_LINES)));
         if (config.get(BATCH_SIZE) != null) builder.batchSize((int) config.get(BATCH_SIZE));
         if (config.get(IGNORE_DUPLICATE_NODES) != null) builder.ignoreDuplicateNodes((boolean) config.get(IGNORE_DUPLICATE_NODES));
         if (config.get(IGNORE_BLANK_STRING) != null) builder.ignoreBlankString((boolean) config.get(IGNORE_BLANK_STRING));

diff --git a/core/src/test/java/apoc/export/csv/ImportCsvTest.java b/core/src/test/java/apoc/export/csv/ImportCsvTest.java
@@ -189,6 +189,31 @@ public void testNodesWithIds() {
         List<Long> ids = TestUtil.firstColumn(db, "MATCH (n:Person) RETURN n.id AS id ORDER BY id");
         assertThat(ids, Matchers.contains(1L, 2L));
     }
+
+    @Test
+    public void testImportCsvWithSkipLines() {
+        // skip only-header (default config)
+        testSkipLine(1L, 2);
+
+        // skip header and another one
+        testSkipLine(2L, 1);
+
+        // skip header and another two (no result because the file has 3 lines)
+        testSkipLine(3L, 0);
+    }
+
+    private void testSkipLine(long skipLine, int nodes) {
+        TestUtil.testCall(db,
+                "call apoc.import.csv([{fileName: 'id-idspaces.csv', labels: ['SkipLine']}], [], $config)",
+                map("config", 
+                        map("delimiter", '|', "skipLines", skipLine)),
+                (r) -> assertEquals((long) nodes, r.get("nodes"))
+        );
+
+        TestUtil.testCallCount(db, "MATCH (n:SkipLine) RETURN n", nodes);
+
+        db.executeTransactionally("MATCH (n:SkipLine) DETACH DELETE n");
+    }
 
     @Test
     public void testNodesAndRelsWithMultiTypes() {