Skip to content

Commit

Permalink
fixes #1257: apoc.load.json file roundtrip from apoc.export.json.all (#…
Browse files Browse the repository at this point in the history
  • Loading branch information
conker84 authored and mneedham committed Apr 8, 2020
1 parent 9729fc6 commit 5d99359
Show file tree
Hide file tree
Showing 5 changed files with 590 additions and 0 deletions.
25 changes: 25 additions & 0 deletions docs/asciidoc/import/loadjson.adoc
Original file line number Diff line number Diff line change
Expand Up @@ -195,3 +195,28 @@ You will receive the following response:
"weight": 0.02548018842935562
}
----

== Import JSON

If you used the `apoc.export.json.all` to export the graph, we provide the `apoc.import.json` to reimport that data.

[source,cypher]
----
CALL apoc.import.json($file, $config)
----

The `$config` parameter is a map

[opts=header,cols="m,m"]
|===
| name | default | description
| unwindBatchSize | 5000 | the batch size of the unwind
| txBatchSize | 5000 | the batch size of the transacttion
| importIdName | neo4jImportId | the name of the "id" field into the used for the import it refers to the "id" field into the root object of the json.
| nodePropertyMappings | {} | The mapping label/property name/property type for Custom Neo4j types (point date). I.e. { User: { born: 'Point', dateOfBirth: 'Datetime' } }
| relPropertyMappings | {} | The mapping rel type/property name/property type for Custom Neo4j types (point date). I.e. { KNOWS: { since: 'Datetime' } }
|===

N.b. Supported Neo4j types via config mappings are:

Point, Localdate, Localtime, Localdatetime, Duration, offsettime, Zoneddatetime
45 changes: 45 additions & 0 deletions src/main/java/apoc/export/json/ImportJson.java
Original file line number Diff line number Diff line change
@@ -0,0 +1,45 @@
package apoc.export.json;

import apoc.export.util.CountingReader;
import apoc.export.util.ProgressReporter;
import apoc.result.ProgressInfo;
import apoc.util.FileUtils;
import apoc.util.JsonUtil;
import apoc.util.Util;
import org.neo4j.graphdb.GraphDatabaseService;
import org.neo4j.procedure.Context;
import org.neo4j.procedure.Description;
import org.neo4j.procedure.Mode;
import org.neo4j.procedure.Name;
import org.neo4j.procedure.Procedure;

import java.util.Map;
import java.util.Scanner;
import java.util.stream.Stream;

public class ImportJson {
@Context
public GraphDatabaseService db;

@Procedure(value = "apoc.import.json", mode = Mode.WRITE)
@Description("apoc.import.json(file,config) - imports the json list to the provided file")
public Stream<ProgressInfo> all(@Name("file") String fileName, @Name(value = "config", defaultValue = "{}") Map<String, Object> config) {
ProgressInfo result =
Util.inThread(() -> {
ImportJsonConfig importJsonConfig = new ImportJsonConfig(config);
ProgressReporter reporter = new ProgressReporter(null, null, new ProgressInfo(fileName, "file", "json"));

try (final CountingReader reader = FileUtils.readerFor(fileName);
final Scanner scanner = new Scanner(reader).useDelimiter("\n|\r");
JsonImporter jsonImporter = new JsonImporter(importJsonConfig, db, reporter)) {
while (scanner.hasNext()) {
Map<String, Object> row = JsonUtil.OBJECT_MAPPER.readValue(scanner.nextLine(), Map.class);
jsonImporter.importRow(row);
}
}

return reporter.getTotal();
});
return Stream.of(result);
}
}
52 changes: 52 additions & 0 deletions src/main/java/apoc/export/json/ImportJsonConfig.java
Original file line number Diff line number Diff line change
@@ -0,0 +1,52 @@
package apoc.export.json;

import apoc.util.Util;
import org.apache.commons.lang3.StringUtils;

import java.util.Collection;
import java.util.Collections;
import java.util.Map;

public class ImportJsonConfig {

private final Map<String, Map<String, String>> nodePropertyMappings;
private final Map<String, Map<String, String>> relPropertyMappings;

private final int unwindBatchSize;
private final int txBatchSize;

private final String importIdName;

public ImportJsonConfig(Map<String, Object> config) {
config = config == null ? Collections.emptyMap() : config;
this.nodePropertyMappings = (Map<String, Map<String, String>>) config.getOrDefault("nodePropertyMappings", Collections.emptyMap());
this.relPropertyMappings = (Map<String, Map<String, String>>) config.getOrDefault("relPropertyMappings", Collections.emptyMap());
this.unwindBatchSize = Util.toInteger(config.getOrDefault("unwindBatchSize", 5000));
this.txBatchSize = Util.toInteger(config.getOrDefault("txBatchSize", 5000));
this.importIdName = (String) config.getOrDefault("importIdName", "neo4jImportId");
}

public String typeForNode(Collection<String> labels, String property) {
return labels.stream()
.map(label -> nodePropertyMappings.getOrDefault(label, Collections.emptyMap()).get(property))
.filter(StringUtils::isNotBlank)
.findFirst()
.orElse(null);
}

public String typeForRel(String type, String property) {
return relPropertyMappings.getOrDefault(type, Collections.emptyMap()).get(property);
}

public int getUnwindBatchSize() {
return unwindBatchSize;
}

public int getTxBatchSize() {
return txBatchSize;
}

public String getImportIdName() {
return importIdName;
}
}

0 comments on commit 5d99359

Please sign in to comment.