Skip to content

HTTPS clone URL

Subversion checkout URL

You can clone with
or
.
Download ZIP
Browse files

fixed typed properties for relationships, added tests

  • Loading branch information...
commit 06b9a53d6ad7a7640a024067a884fed9d7b15893 1 parent 81ff8e4
@jexp authored
View
6 pom.xml
@@ -20,6 +20,12 @@
<scope>test</scope>
</dependency>
<dependency>
+ <groupId>org.mockito</groupId>
+ <artifactId>mockito-core</artifactId>
+ <version>1.8.5</version>
+ <scope>test</scope>
+ </dependency>
+ <dependency>
<groupId>org.neo4j</groupId>
<artifactId>neo4j-kernel</artifactId>
<version>1.8-SNAPSHOT</version>
View
54 src/main/java/org/neo4j/batchimport/Importer.java
@@ -8,11 +8,7 @@
import org.neo4j.unsafe.batchinsert.BatchInserterIndex;
import org.neo4j.unsafe.batchinsert.LuceneBatchInserterIndexProvider;
-import java.io.BufferedReader;
-import java.io.File;
-import java.io.FileReader;
-import java.io.FileWriter;
-import java.io.IOException;
+import java.io.*;
import java.util.*;
import org.neo4j.helpers.collection.MapUtil;
@@ -53,9 +49,21 @@ public Importer(File graphDb) {
System.out.println(e.getMessage());
}
- db = BatchInserters.inserter(graphDb.getAbsolutePath(), config);
- lucene = new LuceneBatchInserterIndexProvider(db);
- report = new Report(10 * 1000 * 1000, 100);
+ db = createBatchInserter(graphDb, config);
+ lucene = createIndexProvider();
+ report = createReport();
+ }
+
+ protected StdOutReport createReport() {
+ return new StdOutReport(10 * 1000 * 1000, 100);
+ }
+
+ protected LuceneBatchInserterIndexProvider createIndexProvider() {
+ return new LuceneBatchInserterIndexProvider(db);
+ }
+
+ protected BatchInserter createBatchInserter(File graphDb, Map<String, String> config) {
+ return BatchInserters.inserter(graphDb.getAbsolutePath(), config);
}
public static void main(String[] args) throws IOException {
@@ -74,22 +82,22 @@ public static void main(String[] args) throws IOException {
}
Importer importBatch = new Importer(graphDb);
try {
- if (nodesFile.exists()) importBatch.importNodes(nodesFile);
- if (relationshipsFile.exists()) importBatch.importRelationships(relationshipsFile);
+ if (nodesFile.exists()) importBatch.importNodes(new FileReader(nodesFile));
+ if (relationshipsFile.exists()) importBatch.importRelationships(new FileReader(relationshipsFile));
for (int i = 3; i < args.length; i = i + 4) {
indexFile = new File(args[i + 3]);
if (!indexFile.exists()) continue;
indexName = args[i+1];
indexType = args[i+2];
BatchInserterIndex index = args[i].equals("node_index") ? importBatch.nodeIndexFor(indexName, indexType) : importBatch.relationshipIndexFor(indexName, indexType);
- importBatch.importIndex(indexFile, indexName, index);
+ importBatch.importIndex(indexName, index, new FileReader(indexFile));
}
} finally {
importBatch.finish();
}
}
- private void finish() {
+ void finish() {
lucene.shutdown();
db.shutdown();
report.finish();
@@ -151,7 +159,7 @@ private int split(String line) {
}
if (i >= offset && lineData[i]!=null) {
data[count++]=fields[i];
- data[count++]=types[i-offset].convert(lineData[i]);
+ data[count++]=types[i].convert(lineData[i]);
}
}
return count;
@@ -173,26 +181,29 @@ private int split(String line) {
}
- static class Report {
+ static class StdOutReport implements Report {
private final long batch;
private final long dots;
private long count;
private long total = System.currentTimeMillis(), time, batchTime;
- public Report(long batch, int dots) {
+ public StdOutReport(long batch, int dots) {
this.batch = batch;
this.dots = batch / dots;
}
+ @Override
public void reset() {
count = 0;
batchTime = time = System.currentTimeMillis();
}
+ @Override
public void finish() {
System.out.println("\nTotal import time: "+ (System.currentTimeMillis() - total) / 1000 + " seconds ");
}
+ @Override
public void dots() {
if ((++count % dots) != 0) return;
System.out.print(".");
@@ -202,13 +213,14 @@ public void dots() {
batchTime = now;
}
+ @Override
public void finishImport(String type) {
System.out.println("\nImporting " + count + " " + type + " took " + (System.currentTimeMillis() - time) / 1000 + " seconds ");
}
}
- private void importNodes(File file) throws IOException {
- BufferedReader bf = new BufferedReader(new FileReader(file));
+ void importNodes(Reader reader) throws IOException {
+ BufferedReader bf = new BufferedReader(reader);
final Data data = new Data(bf.readLine(), "\t", 0);
String line;
report.reset();
@@ -219,8 +231,8 @@ private void importNodes(File file) throws IOException {
report.finishImport("Nodes");
}
- private void importRelationships(File file) throws IOException {
- BufferedReader bf = new BufferedReader(new FileReader(file));
+ void importRelationships(Reader reader) throws IOException {
+ BufferedReader bf = new BufferedReader(reader);
final Data data = new Data(bf.readLine(), "\t", 3);
Object[] rel = new Object[3];
final RelType relType = new RelType();
@@ -234,9 +246,9 @@ private void importRelationships(File file) throws IOException {
report.finishImport("Relationships");
}
- private void importIndex(File file, String indexName, BatchInserterIndex index) throws IOException {
+ void importIndex(String indexName, BatchInserterIndex index, Reader reader) throws IOException {
- BufferedReader bf = new BufferedReader(new FileReader(file));
+ BufferedReader bf = new BufferedReader(reader);
final Data data = new Data(bf.readLine(), "\t", 1);
Object[] node = new Object[1];
View
15 src/main/java/org/neo4j/batchimport/Report.java
@@ -0,0 +1,15 @@
+package org.neo4j.batchimport;
+
+/**
+ * @author mh
+ * @since 21.08.12
+ */
+public interface Report {
+ void reset();
+
+ void finish();
+
+ void dots();
+
+ void finishImport(String type);
+}
View
76 src/test/java/org/neo4j/batchimport/ImporterTest.java
@@ -0,0 +1,76 @@
+package org.neo4j.batchimport;
+
+import org.junit.Before;
+import org.junit.Test;
+import org.mockito.Mockito;
+import org.neo4j.unsafe.batchinsert.BatchInserter;
+import org.neo4j.unsafe.batchinsert.LuceneBatchInserterIndexProvider;
+
+import java.io.File;
+import java.io.StringReader;
+import java.util.Map;
+
+import static org.mockito.Matchers.any;
+import static org.mockito.Matchers.argThat;
+import static org.mockito.Matchers.eq;
+import static org.mockito.Mockito.atLeastOnce;
+import static org.mockito.Mockito.verify;
+import static org.neo4j.helpers.collection.MapUtil.map;
+
+public class ImporterTest {
+
+ private BatchInserter inserter;
+ private LuceneBatchInserterIndexProvider index;
+ private Importer importer;
+
+ @Before
+ public void setUp() throws Exception {
+ inserter = Mockito.mock(BatchInserter.class);
+ index = Mockito.mock(LuceneBatchInserterIndexProvider.class);
+
+ importer = new Importer(File.createTempFile("test", "db")) {
+ @Override
+ protected BatchInserter createBatchInserter(File graphDb, Map<String, String> config) {
+ return inserter;
+ }
+
+ @Override
+ protected LuceneBatchInserterIndexProvider createIndexProvider() {
+ return index;
+ }
+ };
+ }
+
+ @Test
+ public void testImportSimpleNode() throws Exception {
+ importer.importNodes(new StringReader("a\nfoo"));
+ importer.finish();
+ verify(inserter, atLeastOnce()).createNode(eq(map("a", "foo")));
+ }
+ @Test
+ public void testImportNodeWithMultipleProps() throws Exception {
+ importer.importNodes(new StringReader("a\tb\nfoo\tbar"));
+ importer.finish();
+ verify(inserter, atLeastOnce()).createNode(eq(map("a", "foo","b","bar")));
+ }
+
+ @Test
+ public void testImportNodeWithIndividualTypes() throws Exception {
+ importer.importNodes(new StringReader("a:int\tb:float\tc:float\n10\t10.0\t1E+10"));
+ importer.finish();
+ verify(inserter, atLeastOnce()).createNode(eq(map("a", 10,"b",10.0F,"c",1E+10F)));
+ }
+
+ @Test
+ public void testImportSimpleRelationship() throws Exception {
+ importer.importRelationships(new StringReader("star\tend\ttype\ta\n1\t2\tTYPE\tfoo"));
+ importer.finish();
+ verify(inserter, atLeastOnce()).createRelationship(eq(1L), eq(2L), argThat(new RelationshipMatcher("TYPE")), eq(map("a", "foo")));
+ }
+ @Test
+ public void testImportRelationshipWithIndividualTypes() throws Exception {
+ importer.importRelationships(new StringReader("star\tend\ttype\ta:int\tb:float\tc:float\n1\t2\tTYPE\t10\t10.0\t1E+10"));
+ importer.finish();
+ verify(inserter, atLeastOnce()).createRelationship(eq(1L), eq(2L), argThat(new RelationshipMatcher("TYPE")), eq(map("a", 10,"b",10.0F,"c",1E+10F)));
+ }
+}
View
28 src/test/java/org/neo4j/batchimport/RelationshipMatcher.java
@@ -0,0 +1,28 @@
+package org.neo4j.batchimport;
+
+import org.hamcrest.BaseMatcher;
+import org.hamcrest.Description;
+import org.neo4j.graphdb.DynamicRelationshipType;
+import org.neo4j.graphdb.RelationshipType;
+
+class RelationshipMatcher extends BaseMatcher<RelationshipType> {
+ public RelationshipType type;
+ private Object other;
+
+ RelationshipMatcher(RelationshipType type) {
+ this.type = type;
+ }
+ RelationshipMatcher(String name) {
+ this(DynamicRelationshipType.withName(name));
+ }
+
+ public boolean matches(Object other) {
+ this.other = other;
+ return ((RelationshipType)other).name().equals(type.name());
+ }
+
+ @Override
+ public void describeTo(Description description) {
+ description.appendText("Expected "+ type +" but got "+other);
+ }
+}
Please sign in to comment.
Something went wrong with that request. Please try again.