diff --git a/README.md b/README.md
index c05fe4e..639a7fc 100644
--- a/README.md
+++ b/README.md
@@ -43,6 +43,12 @@ pass `XML_VALIDATION_ENABLED` property as `true`.
newName
+
+ dtype
+
+
+ dtype
+
yyyy-MM-dd'T'HH:mm:ss.SSS'Z'
@@ -66,6 +72,12 @@ pass `XML_VALIDATION_ENABLED` property as `true`.
User
task_id
Task
+
+ dtype
+
+
+ dtype
+
HAS_TASK
@@ -102,32 +114,44 @@ pass `XML_VALIDATION_ENABLED` property as `true`.
8) `` - column tag, contains table name.
9) `` - (optional for `node` migration) columns to be renamed
during migration. It means data from column with ``
- will be stored as `` property;
-10) `` - (optional for `node` migration) format of timestamp to
+ will be stored as `` property.
+10) `` - (optional for `node`, `relationship` migration) follows only
+ that rows which cells in these columns are equal to `value` attribute
+ of `` tag. If multiple columns are provided, all columns match is
+ required to migrate it.
+11) `` - (optional for `node`, `relationship` migration) skips only
+ that rows which cells in these columns are equal to `value` attribute
+ of `` tag. If multiple columns are provided, at least one match is
+ required to skip it.
+12) `` - (optional for `node` migration) format of timestamp to
store in Neo4j. It is needed to store LocalDateTime and access it without
converters in code.
-11) `` - (optional for `node` migration) collection of labels to be
+13) `` - (optional for `node` migration) collection of labels to be
added
to Nodes.
-12) `` - label tag, defines its name.
-13) `` - column with foreign key to entity table. Relationship will
+14) `` - label tag, defines its name.
+15) `` - column with foreign key to entity table. Relationship
+ will
be started from Node from that table by this foreign key. Inner field will
be added to node with this primary key.
-14) `` - column with foreign key to entity table. Relationship will
+16) `` - column with foreign key to entity table. Relationship
+ will
be ended with Node from that table by this foreign key.
-15) `` - (optional for `migration`, `innerField` migration) specifies
+17) `` - (optional for `relationship`, `innerField` migration)
+ specifies
label of
start
node to find it by
foreign key.
-16) `` - (optional for `migration` migration) specifies label of end
+18) `` - (optional for `relationship` migration) specifies label of
+ end
node to
find it by foreign
key.
-17) `` - type of the relationship.
-18) `` - name of column with value for inner field migration.
-19) `` - name of inner field of node to set value to.
-20) `` - (optional for `innerField` migration) specify whether values in
+19) `` - type of the relationship.
+20) `` - name of column with value for inner field migration.
+21) `` - name of inner field of node to set value to.
+22) `` - (optional for `innerField` migration) specify whether values in
inner field must be unique. False if not present.
### NOTE
@@ -144,6 +168,13 @@ Note that at first we exclude columns and only after rename them. So if you will
rename excluded columns, it was excluded and no columns with this name will be
renamed.
+We first handle `` rows, it means if row matches `` rule and it
+matches `` rule, it won`t be migrated.
+
+By providing several values for one column, they are considered as array of
+available values. If this array contains cell value, `` rule will skip
+this row, `` rule will follow this row.
+
We handle Postgres types in generated JSON the following way:
- `integer`, `bigserial`, `biginteger` are considered numeric values
@@ -151,6 +182,10 @@ We handle Postgres types in generated JSON the following way:
- `timestamp`, `timestamp without time zone` as timestamp
- other types - strings.
+If there are `null` in cell, we store this as `null` too, so it won`t be saved
+to
+node.
+
We recommend to fill up all tags to be sure that correct data will
be saved to Neo4j.
diff --git a/src/main/java/com/example/postgresneo4jmigrationtool/generator/dumper/CSVPostgresDumper.java b/src/main/java/com/example/postgresneo4jmigrationtool/generator/dumper/CSVPostgresDumper.java
index 7b5d4cc..2a56925 100644
--- a/src/main/java/com/example/postgresneo4jmigrationtool/generator/dumper/CSVPostgresDumper.java
+++ b/src/main/java/com/example/postgresneo4jmigrationtool/generator/dumper/CSVPostgresDumper.java
@@ -1,6 +1,6 @@
package com.example.postgresneo4jmigrationtool.generator.dumper;
-import com.example.postgresneo4jmigrationtool.model.DumpResult;
+import com.example.postgresneo4jmigrationtool.model.MigrationData;
import com.example.postgresneo4jmigrationtool.model.exception.MigrationException;
import com.example.postgresneo4jmigrationtool.repository.postgres.PostgresRepository;
import lombok.RequiredArgsConstructor;
@@ -14,6 +14,7 @@
import java.io.InputStream;
import java.io.PrintWriter;
import java.util.Collection;
+import java.util.List;
import java.util.Map;
@Service
@@ -28,40 +29,78 @@ public class CSVPostgresDumper implements PostgresDumper {
private String delimiter;
@Override
- public DumpResult dump(String tableName, Collection columnsToDump) {
- DumpResult dumpResult = new DumpResult();
- dumpResult.add("dumpDirectory", dumpDirectory);
+ public MigrationData dump(String tableName, Collection columnsToDump, MigrationData params) {
+ MigrationData migrationData = new MigrationData();
+ migrationData.add("dumpDirectory", dumpDirectory);
File dumpScript = new File(dumpDirectory + "/" + dumpScriptFileName);
createFile(dumpScript);
String columns = String.join(",", columnsToDump);
+ Map> followRows = (Map>) params.get("followRows");
+ Map> skipRows = (Map>) params.get("skipRows");
+ String follow = "true = true";
+ if (!followRows.isEmpty()) {
+ for (String key : followRows.keySet()) {
+ List values = followRows.get(key);
+ String valuesString = "'" + String.join("', '", values) + "'";
+ follow += String.format(" AND %s IN (%s)", key, valuesString);
+ }
+ }
+ String skip = "true = true";
+ if (!followRows.isEmpty()) {
+ for (String key : skipRows.keySet()) {
+ List values = skipRows.get(key);
+ String valuesString = "'" + String.join("', '", values) + "'";
+ skip += String.format(" AND %s NOT IN (%s)", key, valuesString);
+ }
+ }
try (PrintWriter writer = new PrintWriter(dumpScript)) {
- writer.printf("psql -U %s -c \"COPY (SELECT %s FROM %s) TO STDOUT WITH CSV DELIMITER '%s' HEADER\" %s > %s.csv",
- postgresRepository.getUsername(), columns, tableName, delimiter,
+ writer.printf("psql -U %s -c \"COPY (SELECT %s FROM %s WHERE %s AND %s) TO STDOUT WITH CSV DELIMITER '%s' HEADER\" %s > %s.csv",
+ postgresRepository.getUsername(), columns, tableName, follow, skip, delimiter,
postgresRepository.getDatabaseName(), tableName);
} catch (IOException e) {
throw new MigrationException("Exception during dumping: " + e.getMessage());
}
runScript(dumpScript);
- addInputStream(dumpResult, tableName);
- return dumpResult;
+ addInputStream(migrationData, tableName);
+ return migrationData;
}
@Override
- public DumpResult dumpWithForeignKeys(String tableName, String columnFrom, String columnTo) {
- DumpResult dumpResult = new DumpResult();
- dumpResult.add("dumpDirectory", dumpDirectory);
+ public MigrationData dumpWithForeignKeys(String tableName, String columnFrom, String columnTo, MigrationData params) {
+ MigrationData migrationData = new MigrationData();
+ migrationData.add("dumpDirectory", dumpDirectory);
File dumpScript = new File(dumpDirectory + "/" + dumpScriptFileName);
createFile(dumpScript);
String foreignColumnFrom = postgresRepository.getForeignColumnName(tableName, columnFrom);
String foreignColumnTo = postgresRepository.getForeignColumnName(tableName, columnTo);
+ Map> followRows = (Map>) params.get("followRows");
+ Map> skipRows = (Map>) params.get("skipRows");
+ String follow = "true = true";
+ if (!followRows.isEmpty()) {
+ for (String key : followRows.keySet()) {
+ List values = followRows.get(key);
+ String valuesString = "'" + String.join("', '", values) + "'";
+ follow += String.format(" AND %s IN (%s)", key, valuesString);
+ }
+ }
+ String skip = "true = true";
+ if (!followRows.isEmpty()) {
+ for (String key : skipRows.keySet()) {
+ List values = skipRows.get(key);
+ String valuesString = "'" + String.join("', '", values) + "'";
+ skip += String.format(" AND %s NOT IN (%s)", key, valuesString);
+ }
+ }
try (PrintWriter writer = new PrintWriter(dumpScript)) {
- writer.printf("psql -U %s -c \"COPY (SELECT %s as %s, %s as %s FROM %s) TO STDOUT WITH CSV DELIMITER '%s' HEADER\" %s > %s.csv",
+ writer.printf("psql -U %s -c \"COPY (SELECT %s as %s, %s as %s FROM %s WHERE %s AND %s) TO STDOUT WITH CSV DELIMITER '%s' HEADER\" %s > %s.csv",
postgresRepository.getUsername(),
columnFrom,
foreignColumnFrom,
columnTo,
foreignColumnTo,
tableName,
+ follow,
+ skip,
delimiter,
postgresRepository.getDatabaseName(),
tableName);
@@ -69,14 +108,14 @@ public DumpResult dumpWithForeignKeys(String tableName, String columnFrom, Strin
throw new MigrationException("Exception during dumping: " + e.getMessage());
}
runScript(dumpScript);
- addInputStream(dumpResult, tableName);
- return dumpResult;
+ addInputStream(migrationData, tableName);
+ return migrationData;
}
@Override
- public DumpResult dumpInnerFields(String tableName, String columnFrom, String valueColumn) {
- DumpResult dumpResult = new DumpResult();
- dumpResult.add("dumpDirectory", dumpDirectory);
+ public MigrationData dumpInnerFields(String tableName, String columnFrom, String valueColumn) {
+ MigrationData migrationData = new MigrationData();
+ migrationData.add("dumpDirectory", dumpDirectory);
File dumpScript = new File(dumpDirectory + "/" + dumpScriptFileName);
createFile(dumpScript);
String foreignColumnFrom = postgresRepository.getForeignColumnName(tableName, columnFrom);
@@ -94,8 +133,8 @@ public DumpResult dumpInnerFields(String tableName, String columnFrom, String va
throw new MigrationException("Exception during dumping: " + e.getMessage());
}
runScript(dumpScript);
- addInputStream(dumpResult, tableName);
- return dumpResult;
+ addInputStream(migrationData, tableName);
+ return migrationData;
}
private void createFile(File file) {
@@ -121,10 +160,10 @@ private void runScript(File dumpScript) {
}
}
- private void addInputStream(DumpResult dumpResult, String tableName) {
+ private void addInputStream(MigrationData migrationData, String tableName) {
try {
InputStream inputStream = new FileInputStream(dumpDirectory + "/" + tableName + ".csv");
- dumpResult.add("inputStream", inputStream);
+ migrationData.add("inputStream", inputStream);
} catch (FileNotFoundException e) {
throw new MigrationException("Migration script file was not found.");
}
diff --git a/src/main/java/com/example/postgresneo4jmigrationtool/generator/dumper/PostgresDumper.java b/src/main/java/com/example/postgresneo4jmigrationtool/generator/dumper/PostgresDumper.java
index f308ee9..d5ce6bd 100644
--- a/src/main/java/com/example/postgresneo4jmigrationtool/generator/dumper/PostgresDumper.java
+++ b/src/main/java/com/example/postgresneo4jmigrationtool/generator/dumper/PostgresDumper.java
@@ -1,15 +1,15 @@
package com.example.postgresneo4jmigrationtool.generator.dumper;
-import com.example.postgresneo4jmigrationtool.model.DumpResult;
+import com.example.postgresneo4jmigrationtool.model.MigrationData;
import java.util.Collection;
public interface PostgresDumper {
- DumpResult dump(String tableName, Collection columnsToDump);
+ MigrationData dump(String tableName, Collection columnsToDump, MigrationData params);
- DumpResult dumpWithForeignKeys(String tableName, String columnFrom, String columnTo);
+ MigrationData dumpWithForeignKeys(String tableName, String columnFrom, String columnTo, MigrationData params);
- DumpResult dumpInnerFields(String tableName, String columnFrom, String valueColumn);
+ MigrationData dumpInnerFields(String tableName, String columnFrom, String valueColumn);
}
diff --git a/src/main/java/com/example/postgresneo4jmigrationtool/generator/uploader/CSVNeo4jUploader.java b/src/main/java/com/example/postgresneo4jmigrationtool/generator/uploader/CSVNeo4jUploader.java
index b5b626f..db5ddf5 100644
--- a/src/main/java/com/example/postgresneo4jmigrationtool/generator/uploader/CSVNeo4jUploader.java
+++ b/src/main/java/com/example/postgresneo4jmigrationtool/generator/uploader/CSVNeo4jUploader.java
@@ -1,20 +1,21 @@
package com.example.postgresneo4jmigrationtool.generator.uploader;
import com.example.postgresneo4jmigrationtool.model.InnerField;
+import com.example.postgresneo4jmigrationtool.model.MigrationData;
import com.example.postgresneo4jmigrationtool.model.Node;
import com.example.postgresneo4jmigrationtool.model.Relationship;
-import com.example.postgresneo4jmigrationtool.model.UploadParams;
-import com.example.postgresneo4jmigrationtool.model.UploadResult;
import com.example.postgresneo4jmigrationtool.model.exception.InvalidConfigurationException;
import com.example.postgresneo4jmigrationtool.model.exception.InvalidFieldException;
import com.example.postgresneo4jmigrationtool.model.exception.MigrationException;
import com.example.postgresneo4jmigrationtool.repository.neo4j.Neo4jRepository;
import lombok.RequiredArgsConstructor;
+import lombok.SneakyThrows;
import org.springframework.beans.factory.annotation.Value;
import org.springframework.stereotype.Service;
import java.io.InputStream;
import java.util.ArrayList;
+import java.util.Arrays;
import java.util.Collection;
import java.util.HashMap;
import java.util.HashSet;
@@ -31,9 +32,10 @@ public class CSVNeo4jUploader implements Neo4jUploader {
@Value("${xml.delimiter}")
private String delimiter;
+ @SneakyThrows
@Override
- public UploadResult createNode(InputStream inputStream, UploadParams params) {
- UploadResult uploadResult = new UploadResult();
+ public MigrationData createNode(InputStream inputStream, MigrationData params) {
+ MigrationData result = new MigrationData();
try (Scanner scanner = new Scanner(inputStream)) {
String headers = scanner.nextLine();
String[] columnNames = headers.split(delimiter);
@@ -61,19 +63,32 @@ public UploadResult createNode(InputStream inputStream, UploadParams params) {
while (scanner.hasNextLine()) {
String data = scanner.nextLine();
String[] values = data.split(delimiter);
+ while (values[values.length - 1].startsWith("\"") && !values[values.length - 1].endsWith("\"")) {
+ String line = scanner.nextLine();
+ String[] additionalData = line.split(delimiter);
+ if (additionalData.length == 1) {
+ values[values.length - 1] += "\n" + line + " ";
+ } else {
+ values[values.length - 1] += additionalData[0];
+ int previousLength = values.length;
+ values = Arrays.copyOf(values, values.length + additionalData.length - 1);
+ System.arraycopy(additionalData, 1, values, previousLength - 1 + 1, additionalData.length - 1);
+ }
+ }
Node node = new Node(columnNames, values, types.toArray(new String[0]), labels.toArray(new String[0]));
node.setTimeFormat(timeFormat);
neo4jRepository.addNode(node);
+ System.out.println("Node " + nodeCounter + " added.");
nodeCounter++;
}
- uploadResult.add("nodeCounter", nodeCounter);
+ result.add("nodeCounter", nodeCounter);
}
- return uploadResult;
+ return result;
}
@Override
- public UploadResult createRelationship(InputStream inputStream, UploadParams params) {
- UploadResult uploadResult = new UploadResult();
+ public MigrationData createRelationship(InputStream inputStream, MigrationData params) {
+ MigrationData result = new MigrationData();
try (Scanner scanner = new Scanner(inputStream)) {
String headers = scanner.nextLine();
String[] columnNames = headers.split(delimiter);
@@ -97,20 +112,33 @@ public UploadResult createRelationship(InputStream inputStream, UploadParams par
while (scanner.hasNextLine()) {
String data = scanner.nextLine();
String[] values = data.split(delimiter);
+ while (values[values.length - 1].startsWith("\"") && !values[values.length - 1].endsWith("\"")) {
+ String line = scanner.nextLine();
+ String[] additionalData = line.split(delimiter);
+ if (additionalData.length == 1) {
+ values[values.length - 1] += "\n" + line;
+ } else {
+ values[values.length - 1] += additionalData[0];
+ int previousLength = values.length;
+ values = Arrays.copyOf(values, values.length + additionalData.length - 1);
+ System.arraycopy(additionalData, 1, values, previousLength - 1 + 1, additionalData.length - 1);
+ }
+ }
Node source = new Node(columnNames[0], values[0], sourceColumnType, sourceLabel);
Node target = new Node(columnNames[1], values[1], targetColumnType, targetLabel);
Relationship relationship = new Relationship(source, target);
neo4jRepository.addRelationship(relationship, type);
+ System.out.println("Relationship " + relationshipCounter + " added.");
relationshipCounter++;
}
- uploadResult.add("relationshipCounter", relationshipCounter);
+ result.add("relationshipCounter", relationshipCounter);
}
- return uploadResult;
+ return result;
}
@Override
- public UploadResult createInnerField(InputStream inputStream, UploadParams params) {
- UploadResult uploadResult = new UploadResult();
+ public MigrationData createInnerField(InputStream inputStream, MigrationData params) {
+ MigrationData result = new MigrationData();
try (Scanner scanner = new Scanner(inputStream)) {
String headers = scanner.nextLine();
String[] columnNames = headers.split(delimiter);
@@ -156,9 +184,9 @@ public UploadResult createInnerField(InputStream inputStream, UploadParams param
InnerField innerField = new InnerField(node, fieldName, valueColumnType, fields.get(key));
neo4jRepository.addInnerField(innerField);
}
- uploadResult.add("objectCounter", objectCounter);
+ result.add("objectCounter", objectCounter);
}
- return uploadResult;
+ return result;
}
}
diff --git a/src/main/java/com/example/postgresneo4jmigrationtool/generator/uploader/Neo4jUploader.java b/src/main/java/com/example/postgresneo4jmigrationtool/generator/uploader/Neo4jUploader.java
index bcf19f8..54cea67 100644
--- a/src/main/java/com/example/postgresneo4jmigrationtool/generator/uploader/Neo4jUploader.java
+++ b/src/main/java/com/example/postgresneo4jmigrationtool/generator/uploader/Neo4jUploader.java
@@ -1,16 +1,15 @@
package com.example.postgresneo4jmigrationtool.generator.uploader;
-import com.example.postgresneo4jmigrationtool.model.UploadParams;
-import com.example.postgresneo4jmigrationtool.model.UploadResult;
+import com.example.postgresneo4jmigrationtool.model.MigrationData;
import java.io.InputStream;
public interface Neo4jUploader {
- UploadResult createNode(InputStream inputStream, UploadParams params);
+ MigrationData createNode(InputStream inputStream, MigrationData params);
- UploadResult createRelationship(InputStream inputStream, UploadParams params);
+ MigrationData createRelationship(InputStream inputStream, MigrationData params);
- UploadResult createInnerField(InputStream inputStream, UploadParams params);
+ MigrationData createInnerField(InputStream inputStream, MigrationData params);
}
diff --git a/src/main/java/com/example/postgresneo4jmigrationtool/model/DumpResult.java b/src/main/java/com/example/postgresneo4jmigrationtool/model/MigrationData.java
similarity index 88%
rename from src/main/java/com/example/postgresneo4jmigrationtool/model/DumpResult.java
rename to src/main/java/com/example/postgresneo4jmigrationtool/model/MigrationData.java
index 39eea78..0ad6dfe 100644
--- a/src/main/java/com/example/postgresneo4jmigrationtool/model/DumpResult.java
+++ b/src/main/java/com/example/postgresneo4jmigrationtool/model/MigrationData.java
@@ -8,11 +8,11 @@
@Data
@AllArgsConstructor
-public class DumpResult {
+public class MigrationData {
private Map params;
- public DumpResult() {
+ public MigrationData() {
this.params = new HashMap<>();
}
diff --git a/src/main/java/com/example/postgresneo4jmigrationtool/model/Node.java b/src/main/java/com/example/postgresneo4jmigrationtool/model/Node.java
index 75dc1d7..df94980 100644
--- a/src/main/java/com/example/postgresneo4jmigrationtool/model/Node.java
+++ b/src/main/java/com/example/postgresneo4jmigrationtool/model/Node.java
@@ -58,33 +58,49 @@ public String toString() {
break;
}
switch (types[i]) {
- case "integer", "bigint", "bigserial" ->
+ case "integer", "bigint", "bigserial" -> {
+ if (((String) values[i]).isEmpty()) {
+ result.append("null");
+ } else {
result.append(values[i]);
+ }
+ }
case "boolean", "bool" -> result.append(values[i].equals("t"));
case "timestamp", "timestamp without time zone" -> {
- result.append("\"");
- if (timeFormat == null || timeFormat.isEmpty()) {
- result.append(Timestamp.valueOf((String) values[i]));
- } else {
- try {
+ try {
+ if (timeFormat == null || timeFormat.isEmpty()) {
+ String resultTime = Timestamp.valueOf((String) values[i]).toString();
+ result.append("\"");
+ result.append(resultTime);
+ result.append("\"");
+ } else {
Timestamp time = Timestamp.valueOf(values[i].toString());
LocalDateTime localDateTime = time.toLocalDateTime();
OffsetDateTime offsetDateTime = localDateTime.atOffset(ZoneOffset.UTC);
DateTimeFormatter formatter = DateTimeFormatter.ofPattern(timeFormat);
String resultTime = offsetDateTime.format(formatter);
+ result.append("\"");
result.append(resultTime);
- } catch (IllegalArgumentException |
- DateTimeException e) {
- throw new InvalidConfigurationException("Invalid time format configuration: " + e.getMessage());
+ result.append("\"");
}
+ } catch (DateTimeException e) {
+ throw new InvalidConfigurationException("Invalid time format configuration: " + e.getMessage());
+ } catch (IllegalArgumentException e) {
+ result.append("null");
}
- result.append("\"");
}
default -> {
if (values[i].equals("\"\"")) {
result.append("\"\"");
} else if (((String) values[i]).isEmpty()) {
result.append("null");
+ } else if (((String) values[i]).startsWith("\"") && ((String) values[i]).endsWith("\"")) {
+ if (((String) values[i]).endsWith("\"\"\"")) {
+ values[i] = ((String) values[i]).substring(((String) values[i]).length() - 3) + "\"";
+ }
+ String s = ((String) values[i]).replaceAll("\"\"\"", "\"\\\\\"")
+ .replaceAll("\"\"", "\\\\\"");
+ result.append(s);
} else {
result.append("\"");
result.append(values[i]);
diff --git a/src/main/java/com/example/postgresneo4jmigrationtool/model/UploadParams.java b/src/main/java/com/example/postgresneo4jmigrationtool/model/UploadParams.java
deleted file mode 100644
index 1db39ad..0000000
--- a/src/main/java/com/example/postgresneo4jmigrationtool/model/UploadParams.java
+++ /dev/null
@@ -1,27 +0,0 @@
-package com.example.postgresneo4jmigrationtool.model;
-
-import lombok.AllArgsConstructor;
-import lombok.Data;
-
-import java.util.HashMap;
-import java.util.Map;
-
-@Data
-@AllArgsConstructor
-public class UploadParams {
-
- private Map params;
-
- public UploadParams() {
- this.params = new HashMap<>();
- }
-
- public void add(String key, Object object) {
- params.put(key, object);
- }
-
- public Object get(String key) {
- return params.get(key);
- }
-
-}
diff --git a/src/main/java/com/example/postgresneo4jmigrationtool/model/UploadResult.java b/src/main/java/com/example/postgresneo4jmigrationtool/model/UploadResult.java
deleted file mode 100644
index ceebc9b..0000000
--- a/src/main/java/com/example/postgresneo4jmigrationtool/model/UploadResult.java
+++ /dev/null
@@ -1,27 +0,0 @@
-package com.example.postgresneo4jmigrationtool.model;
-
-import lombok.AllArgsConstructor;
-import lombok.Data;
-
-import java.util.HashMap;
-import java.util.Map;
-
-@Data
-@AllArgsConstructor
-public class UploadResult {
-
- private Map params;
-
- public UploadResult() {
- this.params = new HashMap<>();
- }
-
- public void add(String key, Object object) {
- params.put(key, object);
- }
-
- public Object get(String key) {
- return params.get(key);
- }
-
-}
diff --git a/src/main/java/com/example/postgresneo4jmigrationtool/parser/InnerFieldXmlParser.java b/src/main/java/com/example/postgresneo4jmigrationtool/parser/InnerFieldXmlParser.java
index d670ceb..84945aa 100644
--- a/src/main/java/com/example/postgresneo4jmigrationtool/parser/InnerFieldXmlParser.java
+++ b/src/main/java/com/example/postgresneo4jmigrationtool/parser/InnerFieldXmlParser.java
@@ -2,9 +2,7 @@
import com.example.postgresneo4jmigrationtool.generator.dumper.PostgresDumper;
import com.example.postgresneo4jmigrationtool.generator.uploader.Neo4jUploader;
-import com.example.postgresneo4jmigrationtool.model.DumpResult;
-import com.example.postgresneo4jmigrationtool.model.UploadParams;
-import com.example.postgresneo4jmigrationtool.model.UploadResult;
+import com.example.postgresneo4jmigrationtool.model.MigrationData;
import com.example.postgresneo4jmigrationtool.repository.postgres.PostgresRepository;
import com.jcabi.xml.XML;
import lombok.Data;
@@ -34,14 +32,14 @@ public void parse() {
String fieldName = new TextXpath(table).getInnerValue("configuration", "fieldName");
String sourceColumnType = postgresRepository.getColumnType(tableName, sourceColumn);
String valueColumnType = postgresRepository.getColumnType(tableName, valueColumn);
- DumpResult dumpResult = dumper.dumpInnerFields(tableName, sourceColumn, valueColumn);
- UploadParams uploadParams = new UploadParams();
+ MigrationData migrationData = dumper.dumpInnerFields(tableName, sourceColumn, valueColumn);
+ MigrationData uploadParams = new MigrationData();
uploadParams.add("sourceLabel", sourceLabel);
uploadParams.add("sourceColumnType", sourceColumnType);
uploadParams.add("valueColumnType", valueColumnType);
uploadParams.add("unique", unique.equals("true"));
uploadParams.add("fieldName", fieldName);
- UploadResult uploadResult = uploader.createInnerField((InputStream) dumpResult.get("inputStream"), uploadParams);
+ MigrationData uploadResult = uploader.createInnerField((InputStream) migrationData.get("inputStream"), uploadParams);
System.out.println("Table " + tableName + " successfully uploaded to Neo4j.");
System.out.println("Created " + uploadResult.get("objectCounter") + " objects.\n");
}
diff --git a/src/main/java/com/example/postgresneo4jmigrationtool/parser/NodeXmlParser.java b/src/main/java/com/example/postgresneo4jmigrationtool/parser/NodeXmlParser.java
index 5240e43..2148d69 100644
--- a/src/main/java/com/example/postgresneo4jmigrationtool/parser/NodeXmlParser.java
+++ b/src/main/java/com/example/postgresneo4jmigrationtool/parser/NodeXmlParser.java
@@ -2,9 +2,7 @@
import com.example.postgresneo4jmigrationtool.generator.dumper.PostgresDumper;
import com.example.postgresneo4jmigrationtool.generator.uploader.Neo4jUploader;
-import com.example.postgresneo4jmigrationtool.model.DumpResult;
-import com.example.postgresneo4jmigrationtool.model.UploadParams;
-import com.example.postgresneo4jmigrationtool.model.UploadResult;
+import com.example.postgresneo4jmigrationtool.model.MigrationData;
import com.example.postgresneo4jmigrationtool.model.exception.InvalidConfigurationException;
import com.example.postgresneo4jmigrationtool.repository.postgres.PostgresRepository;
import com.jcabi.xml.XML;
@@ -34,22 +32,29 @@ public void parse() {
List configurationTag = table.nodes("configuration");
List excludedColumns = new ArrayList<>();
Map renamedColumns = new LinkedHashMap<>();
+ Map> followRows = new LinkedHashMap<>();
+ Map> skipRows = new LinkedHashMap<>();
String timeFormat = "";
if (!configurationTag.isEmpty()) {
XML configuration = configurationTag.get(0);
excludedColumns = getExcludedColumns(configuration);
renamedColumns = getRenamedColumns(configuration);
+ followRows = new TextXpath(configuration).getPreferredColumns("follow");
+ skipRows = new TextXpath(configuration).getPreferredColumns("skip");
timeFormat = new TextXpath(table).getInnerValue("configuration", "timeFormat");
}
List labels = getLabels(table);
Map tablesToDump = getColumns(tableName, excludedColumns);
- DumpResult dumpResult = dumper.dump(tableName, tablesToDump.keySet());
- UploadParams uploadParams = new UploadParams();
+ MigrationData dumpParams = new MigrationData();
+ dumpParams.add("followRows", followRows);
+ dumpParams.add("skipRows", skipRows);
+ MigrationData migrationData = dumper.dump(tableName, tablesToDump.keySet(), dumpParams);
+ MigrationData uploadParams = new MigrationData();
uploadParams.add("newNames", renamedColumns);
uploadParams.add("labels", labels);
uploadParams.add("types", tablesToDump.values());
uploadParams.add("timeFormat", timeFormat);
- UploadResult uploadResult = uploader.createNode((InputStream) dumpResult.get("inputStream"), uploadParams);
+ MigrationData uploadResult = uploader.createNode((InputStream) migrationData.get("inputStream"), uploadParams);
System.out.println("Table " + tableName + " successfully uploaded to Neo4j.");
System.out.println("Created " + uploadResult.get("nodeCounter") + " nodes.\n");
}
diff --git a/src/main/java/com/example/postgresneo4jmigrationtool/parser/RelationshipXmlParser.java b/src/main/java/com/example/postgresneo4jmigrationtool/parser/RelationshipXmlParser.java
index 1383f72..052c1e1 100644
--- a/src/main/java/com/example/postgresneo4jmigrationtool/parser/RelationshipXmlParser.java
+++ b/src/main/java/com/example/postgresneo4jmigrationtool/parser/RelationshipXmlParser.java
@@ -2,9 +2,7 @@
import com.example.postgresneo4jmigrationtool.generator.dumper.PostgresDumper;
import com.example.postgresneo4jmigrationtool.generator.uploader.Neo4jUploader;
-import com.example.postgresneo4jmigrationtool.model.DumpResult;
-import com.example.postgresneo4jmigrationtool.model.UploadParams;
-import com.example.postgresneo4jmigrationtool.model.UploadResult;
+import com.example.postgresneo4jmigrationtool.model.MigrationData;
import com.example.postgresneo4jmigrationtool.model.exception.InvalidConfigurationException;
import com.example.postgresneo4jmigrationtool.repository.postgres.PostgresRepository;
import com.jcabi.xml.XML;
@@ -12,7 +10,9 @@
import org.springframework.stereotype.Service;
import java.io.InputStream;
+import java.util.LinkedHashMap;
import java.util.List;
+import java.util.Map;
@Service(value = "relationshipXmlParser")
@Data
@@ -28,24 +28,35 @@ public class RelationshipXmlParser implements Parser {
public void parse() {
for (XML table : tables) {
String tableName = new TextXpath(table).getAttribute("name");
+ List configurationTag = table.nodes("configuration");
String sourceColumn = new TextXpath(table).getInnerValue("configuration", "sourceColumn");
String targetColumn = new TextXpath(table).getInnerValue("configuration", "targetColumn");
String sourceLabel = new TextXpath(table).getInnerValue("configuration", "sourceLabel");
String targetLabel = new TextXpath(table).getInnerValue("configuration", "targetLabel");
- String type = new TextXpath(table).getValue("type");
+ Map> followRows = new LinkedHashMap<>();
+ Map> skipRows = new LinkedHashMap<>();
+ String type = new TextXpath(table).getInnerValue("type");
if (type.isEmpty()) {
throw new InvalidConfigurationException("Relationship must have type.");
}
+ if (!configurationTag.isEmpty()) {
+ XML configuration = configurationTag.get(0);
+ followRows = new TextXpath(configuration).getPreferredColumns("follow");
+ skipRows = new TextXpath(configuration).getPreferredColumns("skip");
+ }
String sourceColumnType = postgresRepository.getColumnType(tableName, sourceColumn);
String targetColumnType = postgresRepository.getColumnType(tableName, sourceColumn);
- DumpResult dumpResult = dumper.dumpWithForeignKeys(tableName, sourceColumn, targetColumn);
- UploadParams uploadParams = new UploadParams();
+ MigrationData dumpParams = new MigrationData();
+ dumpParams.add("followRows", followRows);
+ dumpParams.add("skipRows", skipRows);
+ MigrationData migrationData = dumper.dumpWithForeignKeys(tableName, sourceColumn, targetColumn, dumpParams);
+ MigrationData uploadParams = new MigrationData();
uploadParams.add("type", type);
uploadParams.add("sourceLabel", sourceLabel);
uploadParams.add("targetLabel", targetLabel);
uploadParams.add("sourceColumnType", sourceColumnType);
uploadParams.add("targetColumnType", targetColumnType);
- UploadResult uploadResult = uploader.createRelationship((InputStream) dumpResult.get("inputStream"), uploadParams);
+ MigrationData uploadResult = uploader.createRelationship((InputStream) migrationData.get("inputStream"), uploadParams);
System.out.println("Table " + tableName + " successfully uploaded to Neo4j.");
System.out.println("Created " + uploadResult.get("relationshipCounter") + " relationships.\n");
}
diff --git a/src/main/java/com/example/postgresneo4jmigrationtool/parser/TextXpath.java b/src/main/java/com/example/postgresneo4jmigrationtool/parser/TextXpath.java
index 2f3013f..e71da98 100644
--- a/src/main/java/com/example/postgresneo4jmigrationtool/parser/TextXpath.java
+++ b/src/main/java/com/example/postgresneo4jmigrationtool/parser/TextXpath.java
@@ -5,7 +5,10 @@
import lombok.RequiredArgsConstructor;
import org.w3c.dom.Node;
+import java.util.ArrayList;
+import java.util.LinkedHashMap;
import java.util.List;
+import java.util.Map;
@RequiredArgsConstructor
public final class TextXpath {
@@ -22,7 +25,12 @@ public String getAttribute(String attribute) {
return value.getNodeValue();
}
- public String getValue(String node) {
+ public String getValue() {
+ return xml.xpath("text()")
+ .get(0);
+ }
+
+ public String getInnerValue(String node) {
List type = xml.nodes(node);
return type.get(0)
.xpath("text()")
@@ -43,4 +51,34 @@ public String getInnerValue(String node, String tag) {
}
}
+ public Map> getPreferredColumns(String tag) {
+ List followColumns = xml.nodes(tag);
+ if (!followColumns.isEmpty()) {
+ List columns = followColumns.get(0)
+ .nodes("column");
+ List columnNames = columns.stream()
+ .map(c -> new TextXpath(c).getValue())
+ .toList();
+ List values = columns.stream()
+ .map(c -> new TextXpath(c).getAttribute("value"))
+ .toList();
+ if (columnNames.size() != values.size()) {
+ throw new InvalidConfigurationException("Amount of column names and values must be the same.");
+ }
+ Map> followMap = new LinkedHashMap<>();
+ for (int i = 0; i < columnNames.size(); i++) {
+ List list;
+ if (followMap.containsKey(columnNames.get(i))) {
+ list = followMap.get(columnNames.get(i));
+ } else {
+ list = new ArrayList<>();
+ }
+ list.add(values.get(i));
+ followMap.put(columnNames.get(i), list);
+ }
+ return followMap;
+ }
+ return new LinkedHashMap<>();
+ }
+
}
diff --git a/src/main/java/com/example/postgresneo4jmigrationtool/repository/neo4j/Neo4jRepositoryImpl.java b/src/main/java/com/example/postgresneo4jmigrationtool/repository/neo4j/Neo4jRepositoryImpl.java
index fe664cd..6ac0d8a 100644
--- a/src/main/java/com/example/postgresneo4jmigrationtool/repository/neo4j/Neo4jRepositoryImpl.java
+++ b/src/main/java/com/example/postgresneo4jmigrationtool/repository/neo4j/Neo4jRepositoryImpl.java
@@ -21,8 +21,9 @@ public void addNode(Node node) {
String data = node.toString();
String preparedQuery = String.format(query, data);
if (node.getLabels().length > 0) {
- preparedQuery += " SET n :%s";
- preparedQuery = String.format(preparedQuery, String.join(" :", node.getLabels()));
+ String labels = " SET n: %s";
+ labels = String.format(labels, String.join(" :", node.getLabels()));
+ preparedQuery += labels;
}
neo4jClient.query(preparedQuery).fetch().all();
}
diff --git a/src/main/java/com/example/postgresneo4jmigrationtool/repository/postgres/PostgresRepositoryImpl.java b/src/main/java/com/example/postgresneo4jmigrationtool/repository/postgres/PostgresRepositoryImpl.java
index ba7206f..0703eee 100644
--- a/src/main/java/com/example/postgresneo4jmigrationtool/repository/postgres/PostgresRepositoryImpl.java
+++ b/src/main/java/com/example/postgresneo4jmigrationtool/repository/postgres/PostgresRepositoryImpl.java
@@ -97,9 +97,13 @@ public String getForeignColumnName(String tableName, String columnName) {
AND kcu.column_name = '%s';
""";
String formattedQuery = String.format(query, tableName, columnName);
- return jdbcTemplate.query(formattedQuery, (rs, rowNum) ->
- rs.getString("column_name"))
- .get(0);
+ List rows = jdbcTemplate.query(formattedQuery, (rs, rowNum) ->
+ rs.getString("column_name"));
+ if (rows.isEmpty()) {
+ return columnName;
+ } else {
+ return rows.get(0);
+ }
}
}
diff --git a/src/main/resources/application.yaml b/src/main/resources/application.yaml
index 2fe8691..60ddf77 100644
--- a/src/main/resources/application.yaml
+++ b/src/main/resources/application.yaml
@@ -31,6 +31,6 @@ xml:
enabled: ${XML_VALIDATION_ENABLED}
schema:
path: ${XSD_SCHEMA_LOCATION}
- delimiter: ;
+ delimiter: '~'
script:
path: ${XML_CONFIG_LOCATION}
\ No newline at end of file
diff --git a/src/main/resources/xml/schema.xsd b/src/main/resources/xml/schema.xsd
index cf08732..5fb4161 100644
--- a/src/main/resources/xml/schema.xsd
+++ b/src/main/resources/xml/schema.xsd
@@ -40,6 +40,8 @@
minOccurs="0"/>
+
+
@@ -53,7 +55,8 @@
-
+
@@ -64,6 +67,28 @@
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
diff --git a/src/main/resources/xml/script.xml b/src/main/resources/xml/script.xml
index f9b5cab..2ff820d 100644
--- a/src/main/resources/xml/script.xml
+++ b/src/main/resources/xml/script.xml
@@ -12,6 +12,12 @@
newName
+
+ dtype
+
+
+ dtype
+
yyyy-MM-dd'T'HH:mm:ss.SSS'Z'
@@ -35,6 +41,12 @@
User
task_id
Task
+
+ dtype
+
+
+ dtype
+
HAS_TASK