Skip to content

Commit

Permalink
fix: json importer did not use merge strategy correctly
Browse files Browse the repository at this point in the history 
Fixed issue #1492
  • Loading branch information
@lvca
lvca committed Mar 1, 2024
1 parent 111f9ad commit 5ecb28e
Show file tree
Hide file tree
Showing 3 changed files with 73 additions and 42 deletions.
61 changes: 39 additions & 22 deletions integration/src/main/java/com/arcadedb/integration/importer/format/JSONImporterFormat.java
View file
Original file line number Diff line number Diff line change
Expand Up @@ -56,17 +56,18 @@

public class JSONImporterFormat implements FormatImporter {
static class CascadingProperties {
CascadingProperties parent;
Map<String, Object> map;
final CascadingProperties parent;
final Map<String, Object> map;

public CascadingProperties(CascadingProperties parent, Map<String, Object> map) {
public CascadingProperties(final CascadingProperties parent, final Map<String, Object> map) {
this.parent = parent;
this.map = map;
}
}

@Override
public void load(final SourceSchema sourceSchema, final AnalyzedEntity.ENTITY_TYPE entityType, final Parser parser, final DatabaseInternal database,
public void load(final SourceSchema sourceSchema, final AnalyzedEntity.ENTITY_TYPE entityType, final Parser parser,
final DatabaseInternal database,
final ImporterContext context, final ImporterSettings settings) throws IOException {

final JSONObject mapping = settings.mapping != null ? new JSONObject(settings.mapping) : null;
Expand Down Expand Up @@ -119,7 +120,8 @@ public String getFormat() {
return "JSON";
}

private void parseRecords(final JsonReader reader, final Database database, final ImporterSettings settings, final ImporterContext context,
private void parseRecords(final JsonReader reader, final Database database, final ImporterSettings settings,
final ImporterContext context,
final JSONArray mapping, boolean ignore) throws IOException {
reader.beginArray();

Expand Down Expand Up @@ -148,13 +150,15 @@ private void parseRecords(final JsonReader reader, final Database database, fina
reader.endArray();
}

private static MutableDocument saveAnonymousRecord(final Database database, final ImporterSettings settings, final Map<String, Object> map) {
private static MutableDocument saveAnonymousRecord(final Database database, final ImporterSettings settings,
final Map<String, Object> map) {
// NO MAPPING, SAVE THE RECORD AS A DOCUMENT
database.getSchema().getOrCreateDocumentType(settings.documentTypeName);
return database.newDocument(settings.documentTypeName).set(map).save();
}

private Object parseRecord(final JsonReader reader, final ImporterSettings settings, final ImporterContext context, final Database database,
private Object parseRecord(final JsonReader reader, final ImporterSettings settings, final ImporterContext context,
final Database database,
final JSONObject mapping, final boolean ignore) throws IOException {
final CascadingProperties attributes = ignore ? null : new CascadingProperties(null, new LinkedHashMap<>());

Expand Down Expand Up @@ -250,20 +254,23 @@ private Object getAttribute(final CascadingProperties properties, final String n
return properties.map.get(name);
}

private Document createRecord(final Database database, final ImporterContext context, final CascadingProperties attributes, final JSONObject mapping,
private Document createRecord(final Database database, final ImporterContext context, final CascadingProperties attributes,
final JSONObject mapping,
final ImporterSettings settings) {
if (mapping == null)
return null;
//return saveAnonymousRecord(database, settings, (Map<String, Object>) attributes.map);

if (!mapping.has("@cat")) {
LogManager.instance().log(this, Level.WARNING, "No @cat tag defined in mapping object. The following object will be skipped %s", attributes);
LogManager.instance()
.log(this, Level.WARNING, "No @cat tag defined in mapping object. The following object will be skipped %s", attributes);
context.errors.incrementAndGet();
return null;
}

if (!mapping.has("@type")) {
LogManager.instance().log(this, Level.WARNING, "No @type tag defined in mapping object. The following object will be skipped %s", attributes);
LogManager.instance()
.log(this, Level.WARNING, "No @type tag defined in mapping object. The following object will be skipped %s", attributes);
context.errors.incrementAndGet();
return null;
}
Expand Down Expand Up @@ -314,7 +321,8 @@ private Document createRecord(final Database database, final ImporterContext con
if (prop == null) {
if (idValue == null) {
// NO ID FOUND, SKIP THE RECORD
LogManager.instance().log(this, Level.WARNING, "@id property not found on current record, skipping record: %s", attributes);
LogManager.instance()
.log(this, Level.WARNING, "@id property not found on current record, skipping record: %s", attributes);
context.errors.incrementAndGet();
return null;
}
Expand Down Expand Up @@ -362,7 +370,8 @@ record = database.newDocument(typeName);
return record;
}

private void applyMappingRules(final Database database, final ImporterContext context, final MutableDocument record, final CascadingProperties attributes,
private void applyMappingRules(final Database database, final ImporterContext context, final MutableDocument record,
final CascadingProperties attributes,
final JSONObject mapping, final ImporterSettings settings) {
resolveProperties(mapping, attributes);

Expand Down Expand Up @@ -390,7 +399,8 @@ private void applyMappingRules(final Database database, final ImporterContext co
} else if (mappingValue instanceof JSONArray) {
if (!(attributeValue instanceof Collection)) {
LogManager.instance()
.log(this, Level.WARNING, "Defined an array on mapping for property '%s' but found the object of class %s as attribute", mappingName,
.log(this, Level.WARNING,
"Defined an array on mapping for property '%s' but found the object of class %s as attribute", mappingName,
attributeValue.getClass());
context.errors.incrementAndGet();
continue;
Expand All @@ -411,7 +421,8 @@ private void applyMappingRules(final Database database, final ImporterContext co
}
}

private List<Object> parseArray(final JsonReader reader, final ImporterSettings settings, final ImporterContext context, final Database database,
private List<Object> parseArray(final JsonReader reader, final ImporterSettings settings, final ImporterContext context,
final Database database,
final JSONArray mapping, boolean ignore) throws IOException {
final List<Object> list = ignore ? null : new ArrayList<>();
reader.beginArray();
Expand Down Expand Up @@ -455,7 +466,8 @@ private List<Object> parseArray(final JsonReader reader, final ImporterSettings
return list;
}

private Object convertMap(final Database database, final ImporterContext context, final MutableDocument record, final Object value, final Object mapping,
private Object convertMap(final Database database, final ImporterContext context, final MutableDocument record,
final Object value, final Object mapping,
final CascadingProperties attributes, final ImporterSettings settings) {
if (mapping instanceof JSONObject) {
final JSONObject mappingObject = (JSONObject) mapping;
Expand All @@ -464,24 +476,25 @@ private Object convertMap(final Database database, final ImporterContext context
if (value instanceof Map)
// CONVERT EMBEDDED MAP INTO A RECORD
attributeMap = new LinkedHashMap<>((Map<String, Object>) value);
else {
else
// TREAT THE VALUE AS ID
attributeMap = new LinkedHashMap<>();
}

final String subCategory = mappingObject.has("@cat") ? mappingObject.getString("@cat") : null;
final String subTypeName = mappingObject.has("@type") ? mappingObject.getString("@type") : null;

if ("e".equals(subCategory)) {
// TRANSFORM INTO AN EDGE
if (subTypeName == null) {
LogManager.instance().log(this, Level.WARNING, "Cannot convert object into an edge because the edge @type is not defined");
LogManager.instance()
.log(this, Level.WARNING, "Cannot convert object into an edge because the edge @type is not defined");
context.errors.incrementAndGet();
return null;
}

if (!(record instanceof Vertex)) {
LogManager.instance().log(this, Level.WARNING, "Cannot convert object into an edge because the root record is not a vertex");
LogManager.instance()
.log(this, Level.WARNING, "Cannot convert object into an edge because the root record is not a vertex");
context.errors.incrementAndGet();
return null;
}
Expand All @@ -497,15 +510,18 @@ private Object convertMap(final Database database, final ImporterContext context
destVertexItem = attributeMap.get(inVertex);
} else if (inValue instanceof JSONObject) {
destVertexMappingObject = (JSONObject) inValue;
attributeMap.put((String) destVertexMappingObject.get("@id"), value);
destVertexItem = attributeMap;
} else {
LogManager.instance()
.log(this, Level.WARNING, "Cannot convert object into an edge because the destination vertx @in type is not supported: " + inValue);
.log(this, Level.WARNING,
"Cannot convert object into an edge because the destination vertx @in type is not supported: " + inValue);
context.errors.incrementAndGet();
return null;
}
} else {
LogManager.instance().log(this, Level.WARNING, "Cannot convert object into an edge because the destination vertx @in is not defined");
LogManager.instance()
.log(this, Level.WARNING, "Cannot convert object into an edge because the destination vertx @in is not defined");
context.errors.incrementAndGet();
return null;
}
Expand All @@ -514,7 +530,8 @@ private Object convertMap(final Database database, final ImporterContext context
if (destVertexItem instanceof Document)
destVertex = (MutableVertex) destVertexItem;
else if (destVertexItem instanceof Map) {
destVertex = (MutableVertex) createRecord(record.getDatabase(), context, new CascadingProperties(attributes, (Map<String, Object>) destVertexItem),
destVertex = (MutableVertex) createRecord(record.getDatabase(), context,
new CascadingProperties(attributes, (Map<String, Object>) destVertexItem),
destVertexMappingObject, settings);
if (destVertex == null) {
LogManager.instance().log(this, Level.WARNING, "Cannot convert inner map into destination vertex: %s", destVertexItem);
Expand Down
37 changes: 19 additions & 18 deletions integration/src/test/java/com/arcadedb/integration/importer/JSONImporterIT.java
View file
Original file line number Diff line number Diff line change
Expand Up @@ -38,7 +38,8 @@ public void importSingleObject() throws IOException {
final String databasePath = "target/databases/test-import-graph";

Importer importer = new Importer(
("-url file://src/test/resources/importer-one-object.json -database " + databasePath + " -documentType Food -forceDatabaseCreate true").split(" "));
("-url file://src/test/resources/importer-one-object.json -database " + databasePath
+ " -documentType Food -forceDatabaseCreate true").split(" "));
importer.load();

try (final Database db = new DatabaseFactory(databasePath).open()) {
Expand Down Expand Up @@ -102,7 +103,8 @@ public void importEmployees() throws IOException {
"}";

Importer importer = new Importer(
new String[] { "-url", "file://src/test/resources/importer-employees.json", "-database", databasePath, "-forceDatabaseCreate", "true", "-mapping",
new String[] { "-url", "file://src/test/resources/importer-employees.json", "-database", databasePath,
"-forceDatabaseCreate", "true", "-mapping",
mapping });
importer.load();

Expand All @@ -114,26 +116,25 @@ public void importEmployees() throws IOException {

if ("Marcus".equalsIgnoreCase(name)) {
Assertions.assertEquals("1234", vertex.getString("id"));
for (Vertex v : vertex.getVertices(Vertex.DIRECTION.OUT))
v.getString("1230");
for (Vertex v : vertex.getVertices(Vertex.DIRECTION.IN))
Assertions.fail();
Assertions.assertEquals(0, vertex.countEdges(Vertex.DIRECTION.OUT, "HAS_MANAGER"));
Assertions.assertEquals(2, vertex.countEdges(Vertex.DIRECTION.IN, "HAS_MANAGER"));
} else if ("Win".equals(name)) {
Assertions.assertEquals("1230", vertex.getString("id"));
for (Vertex v : vertex.getVertices(Vertex.DIRECTION.IN))
v.getString("1234");
for (Vertex v : vertex.getVertices(Vertex.DIRECTION.OUT))
v.getString("1231");
} else {
Assertions.assertEquals("1231", vertex.getString("id"));
for (Vertex v : vertex.getVertices(Vertex.DIRECTION.IN))
v.getString("1230");
for (Vertex v : vertex.getVertices(Vertex.DIRECTION.OUT))
Assertions.fail();
}
Assertions.assertEquals(1, vertex.countEdges(Vertex.DIRECTION.OUT, "HAS_MANAGER"));
Assertions.assertEquals(0, vertex.countEdges(Vertex.DIRECTION.IN, "HAS_MANAGER"));
} else if ("Dave".equals(name)) {
Assertions.assertEquals("1232", vertex.getString("id"));
Assertions.assertEquals(1, vertex.countEdges(Vertex.DIRECTION.OUT, "HAS_MANAGER"));
Assertions.assertEquals(1, vertex.countEdges(Vertex.DIRECTION.IN, "HAS_MANAGER"));
} else if ("Albert".equals(name)) {
Assertions.assertEquals("1239", vertex.getString("id"));
Assertions.assertEquals(1, vertex.countEdges(Vertex.DIRECTION.OUT, "HAS_MANAGER"));
Assertions.assertEquals(0, vertex.countEdges(Vertex.DIRECTION.IN, "HAS_MANAGER"));
} else
Assertions.fail();
}

Assertions.assertEquals(3, db.countType("User", true));
Assertions.assertEquals(4, db.countType("User", true));
}

TestHelper.checkActiveDatabases();
Expand Down
17 changes: 15 additions & 2 deletions integration/src/test/resources/importer-employees.json
View file
Original file line number Diff line number Diff line change
@@ -1,14 +1,27 @@
{
"Users": [
{
"id": "1",
"EmployeeID": "1234",
"ManagerID": "1230",
"Name": "Marcus"
},
{
"id": "2",
"EmployeeID": "1230",
"ManagerID": "1231",
"ManagerID": "1234",
"Name": "Win"
},
{
"id": "3",
"EmployeeID": "1232",
"ManagerID": "1234",
"Name": "Dave"
},
{
"id": "4",
"EmployeeID": "1239",
"ManagerID": "1232",
"Name": "Albert"
}
]
}

0 comments on commit 5ecb28e

Please sign in to comment.