Merge pull request #27 from IlyaLisov/#24

#24 Implement inner field migration, refactor classes

README.md

@@ -11,7 +11,8 @@ User can:
 - rename columns
 - add labels to generated nodes
 - migrate relationships by migrating foreign keys
-- reformat timestamp to time format
+- reformat timestamp to custom time format
+- migrate tables as inner fields
 
 ### How to use?
 
@@ -60,15 +61,28 @@ You can validate your schema with `schema.xsd`schema.
         <tables>
             <table name="users_tasks">
                 <configuration>
-                    <columnFrom>user_id</columnFrom>
-                    <labelFrom>User</labelFrom>
-                    <columnTo>task_id</columnTo>
-                    <labelTo>Task</labelTo>
+                    <sourceColumn>user_id</sourceColumn>
+                    <sourceLabel>User</sourceLabel>
+                    <targetColumn>task_id</targetColumn>
+                    <targetLabel>Task</targetLabel>
                 </configuration>
                 <type>HAS_TASK</type>
             </table>
         </tables>
     </relationship>
+    <innerField>
+        <tables>
+            <table name="users_roles">
+                <configuration>
+                    <sourceColumn>user_id</sourceColumn>
+                    <sourceLabel>User</sourceLabel>
+                    <valueColumn>role</valueColumn>
+                    <fieldName>userRole</fieldName>
+                    <unique>true</unique>
+                </configuration>
+            </table>
+        </tables>
+    </innerField>
 </migration>
 ```
 
@@ -95,19 +109,25 @@ You can validate your schema with `schema.xsd`schema.
     added
     to Nodes.
 12) `<label>` - label tag, defines its name.
-13) `<columnFrom>` - column with foreign key to entity table. Relationship will
-    be started from Node from that table by this foreign key.
-14) `<columnTo>` - column with foreign key to entity table. Relationship will
+13) `<sourceColumn>` - column with foreign key to entity table. Relationship will
+    be started from Node from that table by this foreign key. Inner field will
+    be added to node with this primary key.
+14) `<targetColumn>` - column with foreign key to entity table. Relationship will
     be ended with Node from that table by this foreign key.
-15) `<labelFrom>` - (optional for `migration` migration) specifies label of
+15) `<sourceLabel>` - (optional for `migration`, `innerField` migration) specifies
+    label of
     start
     node to find it by
     foreign key.
-16) `<labelTo>` - (optional for `migration` migration) specifies label of end
+16) `<targetLabel>` - (optional for `migration` migration) specifies label of end
     node to
     find it by foreign
     key.
 17) `<type>` - type of the relationship.
+18) `<valueColumn>` - name of column with value for inner field migration.
+19) `<fieldName>` - name of inner field of node to set value to.
+20) `<unique>` - (optional for `innerField` migration) specify whether values in
+    inner field must be unique. False if not present.
 
 ### NOTE
 
@@ -145,7 +165,7 @@ what data was dumped and uploaded to Neo4j.
 
 Relationship migration is provided by matching nodes with provided primary key.
 So if some of your nodes have similar id, relationship will be added to each of
-them. It can be avoided but providing `<labelFrom>` and `<labelTo>` tags.
+them. It can be avoided but providing `<sourceLabel>` and `<targetLabel>` tags.
 
 We parse timestamp from database then format it to provided time format (
 from optional `<timeFormat>` tag).

src/main/java/com/example/postgresneo4jmigrationtool/PostgresNeo4jMigrationToolApplication.java

@@ -1,6 +1,7 @@
 package com.example.postgresneo4jmigrationtool;
 
 import com.example.postgresneo4jmigrationtool.parser.Parser;
+import com.example.postgresneo4jmigrationtool.parser.XmlParser;
 import lombok.RequiredArgsConstructor;
 import org.springframework.boot.SpringApplication;
 import org.springframework.boot.autoconfigure.SpringBootApplication;
@@ -13,7 +14,7 @@ public class PostgresNeo4jMigrationToolApplication {
     public static void main(String[] args) {
         ConfigurableApplicationContext appContext = SpringApplication.run(PostgresNeo4jMigrationToolApplication.class, args);
         try {
-            Parser parser = appContext.getBean(Parser.class);
+            Parser parser = appContext.getBean(XmlParser.class);
             parser.parse();
         } catch (Exception e) {
             System.out.println(e.getMessage());

src/main/java/com/example/postgresneo4jmigrationtool/generator/dumper/CSVPostgresDumper.java

@@ -4,6 +4,7 @@
 import com.example.postgresneo4jmigrationtool.model.exception.MigrationException;
 import com.example.postgresneo4jmigrationtool.repository.postgres.PostgresRepository;
 import lombok.RequiredArgsConstructor;
+import org.springframework.beans.factory.annotation.Value;
 import org.springframework.stereotype.Service;
 
 import java.io.File;
@@ -22,13 +23,14 @@ public class CSVPostgresDumper implements PostgresDumper {
     private final PostgresRepository postgresRepository;
     private final String dumpDirectory = "dump";
     private final String dumpScriptFileName = "dump_script.sh";
-    private final String delimiter = ";";
+
+    @Value("${xml.delimiter}")
+    private String delimiter;
 
     @Override
     public DumpResult dump(String tableName, Collection<String> columnsToDump) {
         DumpResult dumpResult = new DumpResult();
         dumpResult.add("dumpDirectory", dumpDirectory);
-        dumpResult.add("delimiter", delimiter);
         File dumpScript = new File(dumpDirectory + "/" + dumpScriptFileName);
         createFile(dumpScript);
         String columns = String.join(",", columnsToDump);
@@ -48,7 +50,6 @@ public DumpResult dump(String tableName, Collection<String> columnsToDump) {
     public DumpResult dumpWithForeignKeys(String tableName, String columnFrom, String columnTo) {
         DumpResult dumpResult = new DumpResult();
         dumpResult.add("dumpDirectory", dumpDirectory);
-        dumpResult.add("delimiter", delimiter);
         File dumpScript = new File(dumpDirectory + "/" + dumpScriptFileName);
         createFile(dumpScript);
         String foreignColumnFrom = postgresRepository.getForeignColumnName(tableName, columnFrom);
@@ -72,6 +73,31 @@ public DumpResult dumpWithForeignKeys(String tableName, String columnFrom, Strin
         return dumpResult;
     }
 
+    @Override
+    public DumpResult dumpInnerFields(String tableName, String columnFrom, String valueColumn) {
+        DumpResult dumpResult = new DumpResult();
+        dumpResult.add("dumpDirectory", dumpDirectory);
+        File dumpScript = new File(dumpDirectory + "/" + dumpScriptFileName);
+        createFile(dumpScript);
+        String foreignColumnFrom = postgresRepository.getForeignColumnName(tableName, columnFrom);
+        try (PrintWriter writer = new PrintWriter(dumpScript)) {
+            writer.printf("psql -U %s -c \"COPY (SELECT %s as %s, %s FROM %s) TO STDOUT WITH CSV DELIMITER '%s' HEADER\" %s > %s.csv",
+                    postgresRepository.getUsername(),
+                    columnFrom,
+                    foreignColumnFrom,
+                    valueColumn,
+                    tableName,
+                    delimiter,
+                    postgresRepository.getDatabaseName(),
+                    tableName);
+        } catch (IOException e) {
+            throw new MigrationException("Exception during dumping: " + e.getMessage());
+        }
+        runScript(dumpScript);
+        addInputStream(dumpResult, tableName);
+        return dumpResult;
+    }
+
     private void createFile(File file) {
         try {
             if (!file.exists()) {

src/main/java/com/example/postgresneo4jmigrationtool/generator/dumper/PostgresDumper.java

@@ -10,4 +10,6 @@ public interface PostgresDumper {
 
     DumpResult dumpWithForeignKeys(String tableName, String columnFrom, String columnTo);
 
+    DumpResult dumpInnerFields(String tableName, String columnFrom, String valueColumn);
+
 }

src/main/java/com/example/postgresneo4jmigrationtool/generator/uploader/CSVNeo4jUploader.java

@@ -1,5 +1,6 @@
 package com.example.postgresneo4jmigrationtool.generator.uploader;
 
+import com.example.postgresneo4jmigrationtool.model.InnerField;
 import com.example.postgresneo4jmigrationtool.model.Node;
 import com.example.postgresneo4jmigrationtool.model.Relationship;
 import com.example.postgresneo4jmigrationtool.model.UploadParams;
@@ -9,10 +10,14 @@
 import com.example.postgresneo4jmigrationtool.model.exception.MigrationException;
 import com.example.postgresneo4jmigrationtool.repository.neo4j.Neo4jRepository;
 import lombok.RequiredArgsConstructor;
+import org.springframework.beans.factory.annotation.Value;
 import org.springframework.stereotype.Service;
 
 import java.io.InputStream;
+import java.util.ArrayList;
 import java.util.Collection;
+import java.util.HashMap;
+import java.util.HashSet;
 import java.util.List;
 import java.util.Map;
 import java.util.Scanner;
@@ -23,12 +28,15 @@ public class CSVNeo4jUploader implements Neo4jUploader {
 
     private final Neo4jRepository neo4jRepository;
 
+    @Value("${xml.delimiter}")
+    private String delimiter;
+
     @Override
     public UploadResult createNode(InputStream inputStream, UploadParams params) {
         UploadResult uploadResult = new UploadResult();
         try (Scanner scanner = new Scanner(inputStream)) {
             String headers = scanner.nextLine();
-            String[] columnNames = headers.split(String.valueOf(params.get("delimiter")));
+            String[] columnNames = headers.split(delimiter);
             if (columnNames.length == 0) {
                 throw new MigrationException("First row of dumped file must contain column names.");
             }
@@ -52,10 +60,10 @@ public UploadResult createNode(InputStream inputStream, UploadParams params) {
             int nodeCounter = 0;
             while (scanner.hasNextLine()) {
                 String data = scanner.nextLine();
-                String[] values = data.split(String.valueOf(params.get("delimiter")));
-                Node node = new Node(columnNames, values, types.toArray(new String[0]));
+                String[] values = data.split(delimiter);
+                Node node = new Node(columnNames, values, types.toArray(new String[0]), labels.toArray(new String[0]));
                 node.setTimeFormat(timeFormat);
-                neo4jRepository.addNode(node, labels.toArray(new String[0]));
+                neo4jRepository.addNode(node);
                 nodeCounter++;
             }
             uploadResult.add("nodeCounter", nodeCounter);
@@ -68,9 +76,9 @@ public UploadResult createRelationship(InputStream inputStream, UploadParams par
         UploadResult uploadResult = new UploadResult();
         try (Scanner scanner = new Scanner(inputStream)) {
             String headers = scanner.nextLine();
-            String[] columnNames = headers.split(String.valueOf(params.get("delimiter")));
-            String columnFromType = (String) params.get("columnFromType");
-            String columnToType = (String) params.get("columnToType");
+            String[] columnNames = headers.split(delimiter);
+            String sourceColumnType = (String) params.get("sourceColumnType");
+            String targetColumnType = (String) params.get("targetColumnType");
             if (columnNames.length == 0) {
                 throw new MigrationException("First row of dumped file must contain column names.");
             }
@@ -83,15 +91,15 @@ public UploadResult createRelationship(InputStream inputStream, UploadParams par
             if (type == null || type.isEmpty()) {
                 throw new InvalidConfigurationException("Relationship must have type.");
             }
-            String labelFrom = (String) params.get("labelFrom");
-            String labelTo = (String) params.get("labelTo");
+            String sourceLabel = (String) params.get("sourceLabel");
+            String targetLabel = (String) params.get("targetLabel");
             int relationshipCounter = 0;
             while (scanner.hasNextLine()) {
                 String data = scanner.nextLine();
-                String[] values = data.split(String.valueOf(params.get("delimiter")));
-                Node nodeFrom = new Node(new String[]{columnNames[0]}, new String[]{values[0]}, new String[]{columnFromType});
-                Node nodeTo = new Node(new String[]{columnNames[1]}, new String[]{values[1]}, new String[]{columnToType});
-                Relationship relationship = new Relationship(nodeFrom, nodeTo, labelFrom, labelTo);
+                String[] values = data.split(delimiter);
+                Node source = new Node(columnNames[0], values[0], sourceColumnType, sourceLabel);
+                Node target = new Node(columnNames[1], values[1], targetColumnType, targetLabel);
+                Relationship relationship = new Relationship(source, target);
                 neo4jRepository.addRelationship(relationship, type);
                 relationshipCounter++;
             }
@@ -100,4 +108,57 @@ public UploadResult createRelationship(InputStream inputStream, UploadParams par
         return uploadResult;
     }
 
+    @Override
+    public UploadResult createInnerField(InputStream inputStream, UploadParams params) {
+        UploadResult uploadResult = new UploadResult();
+        try (Scanner scanner = new Scanner(inputStream)) {
+            String headers = scanner.nextLine();
+            String[] columnNames = headers.split(delimiter);
+            String sourceColumnType = (String) params.get("sourceColumnType");
+            String valueColumnType = (String) params.get("valueColumnType");
+            boolean unique = (boolean) params.get("unique");
+            String fieldName = (String) params.get("fieldName");
+            String sourceLabel = (String) params.get("sourceLabel");
+            if (columnNames.length == 0) {
+                throw new MigrationException("First row of dumped file must contain column names.");
+            }
+            for (String name : columnNames) {
+                if (name.contains(" ")) {
+                    throw new InvalidFieldException("Field name can not include spaces: " + name);
+                }
+            }
+            int objectCounter = 0;
+            Map<Object, List<Object>> fields = new HashMap<>();
+            while (scanner.hasNextLine()) {
+                String data = scanner.nextLine();
+                String[] values = data.split(delimiter);
+                String id = values[0];
+                String value = values[1];
+                if (fields.containsKey(id)) {
+                    List<Object> objects = fields.get(id);
+                    objects.add(value);
+                    fields.put(id, objects);
+                } else {
+                    List<Object> objects = new ArrayList<>();
+                    objects.add(value);
+                    fields.put(id, objects);
+                }
+                objectCounter++;
+            }
+            if (unique) {
+                for (Object key : fields.keySet()) {
+                    List<Object> values = fields.get(key);
+                    fields.put(key, new ArrayList<>(new HashSet<>(values)));
+                }
+            }
+            for (Object key : fields.keySet()) {
+                Node node = new Node(columnNames[0], (String) key, sourceColumnType, sourceLabel);
+                InnerField innerField = new InnerField(node, fieldName, valueColumnType, fields.get(key));
+                neo4jRepository.addInnerField(innerField);
+            }
+            uploadResult.add("objectCounter", objectCounter);
+        }
+        return uploadResult;
+    }
+
 }

src/main/java/com/example/postgresneo4jmigrationtool/generator/uploader/Neo4jUploader.java

@@ -11,4 +11,6 @@ public interface Neo4jUploader {
 
     UploadResult createRelationship(InputStream inputStream, UploadParams params);
 
+    UploadResult createInnerField(InputStream inputStream, UploadParams params);
+
 }

src/main/java/com/example/postgresneo4jmigrationtool/model/InnerField.java

@@ -0,0 +1,37 @@
+package com.example.postgresneo4jmigrationtool.model;
+
+import lombok.AllArgsConstructor;
+import lombok.Data;
+
+import java.util.List;
+
+@Data
+@AllArgsConstructor
+public class InnerField {
+
+    private Node source;
+    private String fieldName;
+    private String valueType;
+    private List<Object> values;
+
+    @Override
+    public String toString() {
+        StringBuilder result = new StringBuilder();
+        result.append("[");
+        for (Object value : values) {
+            switch (valueType) {
+                case "integer", "bigint", "bigserial" -> result.append(value);
+                default -> {
+                    result.append("\"");
+                    result.append(value);
+                    result.append("\"");
+                }
+            }
+            result.append(", ");
+        }
+        result.delete(result.lastIndexOf(", "), result.length() - 1);
+        result.append("]");
+        return result.toString();
+    }
+
+}