Merge pull request #22 from IlyaLisov/#12

#12 Implement timestamp and enum types handling

README.md

@@ -11,10 +11,7 @@ User can:
 - rename columns
 - add labels to generated nodes
 - migrate relationships by migrating foreign keys
-
-Future features:
-
-- time and boolean formatting
+- reformat timestamp to time format
 
 ### How to use?
 
@@ -44,6 +41,7 @@ You can validate your schema with `schema.xsd`schema.
                             <newName>newName</newName>
                         </columns>
                     </renamedColumns>
+                    <timeFormat>yyyy-MM-dd'T'HH:mm:ss.SSS'Z'</timeFormat>
                 </configuration>
                 <labels>
                     <label>User</label>
@@ -90,21 +88,26 @@ You can validate your schema with `schema.xsd`schema.
 9) `<renamedColumns>` - (optional for `node` migration) columns to be renamed
    during migration. It means data from column with `<previousName>`
    will be stored as `<newName>` property;
-10) `<labels>` - (optional for `node` migration) collection of labels to be
+10) `<timeFormat>` - (optional for `node` migration) format of timestamp to
+    store in Neo4j. It is needed to store LocalDateTime and access it without
+    converters in code.
+11) `<labels>` - (optional for `node` migration) collection of labels to be
     added
     to Nodes.
-11) `<label>` - label tag, defines its name.
-12) `<columnFrom>` - column with foreign key to entity table. Relationship will
+12) `<label>` - label tag, defines its name.
+13) `<columnFrom>` - column with foreign key to entity table. Relationship will
     be started from Node from that table by this foreign key.
-13) `<columnTo>` - column with foreign key to entity table. Relationship will
+14) `<columnTo>` - column with foreign key to entity table. Relationship will
     be ended with Node from that table by this foreign key.
-14) `<labelFrom>` - (optional for `migration` mode) specifies label of start
+15) `<labelFrom>` - (optional for `migration` migration) specifies label of
+    start
     node to find it by
     foreign key.
-15) `<labelTo>` - (optional for `migration` mode) specifies label of end node to
+16) `<labelTo>` - (optional for `migration` migration) specifies label of end
+    node to
     find it by foreign
     key.
-16) `<type>` - type of the relationship.
+17) `<type>` - type of the relationship.
 
 ### NOTE
 
@@ -120,6 +123,13 @@ Note that at first we exclude columns and only after rename them. So if you will
 rename excluded columns, it was excluded and no columns with this name will be
 renamed.
 
+We handle Postgres types in generated JSON the following way:
+
+- `integer`, `bigserial`, `biginteger` are considered numeric values
+- `bool`, `boolean` are considered boolean values
+- `timestamp`, `timestamp without time zone` as timestamp
+- other types - strings.
+
 We recommend to fill up all tags to be sure that correct data will
 be saved to Neo4j.
 
@@ -137,5 +147,8 @@ Relationship migration is provided by matching nodes with provided primary key.
 So if some of your nodes have similar id, relationship will be added to each of
 them. It can be avoided but providing `<labelFrom>` and `<labelTo>` tags.
 
+We parse timestamp from database then format it to provided time format (
+from optional `<timeFormat>` tag).
+
 If no exceptions were thrown, you will see messages in logs with amount of
 created nodes and relationships.

src/main/java/com/example/postgresneo4jmigrationtool/generator/uploader/CSVNeo4jUploader.java

@@ -12,6 +12,7 @@
 import org.springframework.stereotype.Service;
 
 import java.io.InputStream;
+import java.util.Collection;
 import java.util.List;
 import java.util.Map;
 import java.util.Scanner;
@@ -33,6 +34,8 @@ public UploadResult createNode(InputStream inputStream, UploadParams params) {
             }
             List<String> labels = (List<String>) params.get("labels");
             Map<String, String> newNames = (Map<String, String>) params.get("newNames");
+            Collection<String> types = (Collection<String>) params.get("types");
+            String timeFormat = (String) params.get("timeFormat");
             for (String name : newNames.values()) {
                 if (name.contains(" ")) {
                     throw new InvalidFieldException("Field name can not include spaces: " + name);
@@ -50,7 +53,8 @@ public UploadResult createNode(InputStream inputStream, UploadParams params) {
             while (scanner.hasNextLine()) {
                 String data = scanner.nextLine();
                 String[] values = data.split(String.valueOf(params.get("delimiter")));
-                Node node = new Node(columnNames, values);
+                Node node = new Node(columnNames, values, types.toArray(new String[0]));
+                node.setTimeFormat(timeFormat);
                 neo4jRepository.addNode(node, labels.toArray(new String[0]));
                 nodeCounter++;
             }
@@ -65,6 +69,8 @@ public UploadResult createRelationship(InputStream inputStream, UploadParams par
         try (Scanner scanner = new Scanner(inputStream)) {
             String headers = scanner.nextLine();
             String[] columnNames = headers.split(String.valueOf(params.get("delimiter")));
+            String columnFromType = (String) params.get("columnFromType");
+            String columnToType = (String) params.get("columnToType");
             if (columnNames.length == 0) {
                 throw new MigrationException("First row of dumped file must contain column names.");
             }
@@ -83,8 +89,8 @@ public UploadResult createRelationship(InputStream inputStream, UploadParams par
             while (scanner.hasNextLine()) {
                 String data = scanner.nextLine();
                 String[] values = data.split(String.valueOf(params.get("delimiter")));
-                Node nodeFrom = new Node(new String[]{columnNames[0]}, new String[]{values[0]});
-                Node nodeTo = new Node(new String[]{columnNames[1]}, new String[]{values[1]});
+                Node nodeFrom = new Node(new String[]{columnNames[0]}, new String[]{values[0]}, new String[]{columnFromType});
+                Node nodeTo = new Node(new String[]{columnNames[1]}, new String[]{values[1]}, new String[]{columnToType});
                 Relationship relationship = new Relationship(nodeFrom, nodeTo, labelFrom, labelTo);
                 neo4jRepository.addRelationship(relationship, type);
                 relationshipCounter++;

src/main/java/com/example/postgresneo4jmigrationtool/model/Node.java

@@ -1,26 +1,69 @@
 package com.example.postgresneo4jmigrationtool.model;
 
-import lombok.AllArgsConstructor;
+import com.example.postgresneo4jmigrationtool.model.exception.InvalidConfigurationException;
 import lombok.Data;
 
+import java.sql.Timestamp;
+import java.time.DateTimeException;
+import java.time.LocalDateTime;
+import java.time.OffsetDateTime;
+import java.time.ZoneOffset;
+import java.time.format.DateTimeFormatter;
+import java.util.Arrays;
+
 @Data
-@AllArgsConstructor
 public class Node {
 
     private String[] names;
     private Object[] values;
+    private String[] types;
+    private String timeFormat;
+
+    public Node(String[] names, Object[] values, String[] types) {
+        this.names = names;
+        this.types = types;
+        this.values = values;
+        if (values.length < types.length) {
+            this.values = Arrays.copyOf(values, values.length + 1);
+        }
+    }
 
     @Override
     public String toString() {
         StringBuilder result = new StringBuilder("{");
         for (int i = 0; i < names.length; i++) {
             result.append(names[i]).append(": ");
-            if (values[i] instanceof String) {
-                result.append("\"");
-                result.append(values[i]);
-                result.append("\"");
-            } else {
-                result.append(values[i]);
+            if (values[i] == null) {
+                break;
+            }
+            switch (types[i]) {
+                case "integer", "bigint", "bigserial" ->
+                        result.append(values[i]);
+                case "boolean", "bool" -> result.append(values[i].equals("t"));
+                case "timestamp", "timestamp without time zone" -> {
+                    result.append("\"");
+                    if (timeFormat == null || timeFormat.isEmpty()) {
+                        result.append(Timestamp.valueOf((String) values[i]));
+                    } else {
+                        try {
+                            Timestamp time = Timestamp.valueOf(values[i].toString());
+                            LocalDateTime localDateTime = time.toLocalDateTime();
+                            OffsetDateTime offsetDateTime = localDateTime.atOffset(ZoneOffset.UTC);
+                            DateTimeFormatter formatter = DateTimeFormatter.ofPattern(timeFormat);
+                            String resultTime = offsetDateTime.format(formatter);
+                            result.append(resultTime);
+                        } catch (IllegalArgumentException |
+                                 DateTimeException e) {
+                            throw new InvalidConfigurationException("Invalid time format configuration: " + e.getMessage());
+                        }
+                    }
+                    result.append("\"");
+                }
+                default -> {
+                    result.append("\"");
+                    result.append(values[i]);
+                    result.append("\"");
+                }
             }
             result.append(", ");
         }

src/main/java/com/example/postgresneo4jmigrationtool/parser/XmlParser.java

@@ -15,7 +15,7 @@
 import java.io.InputStream;
 import java.util.ArrayList;
 import java.util.Arrays;
-import java.util.HashMap;
+import java.util.LinkedHashMap;
 import java.util.List;
 import java.util.Map;
 
@@ -66,19 +66,23 @@ private void parseNodeMigration(List<XML> tables) {
             String tableName = getTableName(table);
             List<XML> configurationTag = table.nodes("configuration");
             List<String> excludedColumns = new ArrayList<>();
-            Map<String, String> renamedColumns = new HashMap<>();
+            Map<String, String> renamedColumns = new LinkedHashMap<>();
+            String timeFormat = "";
             if (!configurationTag.isEmpty()) {
                 XML configuration = configurationTag.get(0);
                 excludedColumns = getExcludedColumns(configuration);
                 renamedColumns = getRenamedColumns(configuration);
+                timeFormat = getConfigurationTagValue(table, "timeFormat");
             }
             List<String> labels = getLabels(table);
-            Map<String, String> tablesToDump = getTables(tableName, excludedColumns);
+            Map<String, String> tablesToDump = getColumns(tableName, excludedColumns);
             DumpResult dumpResult = dumper.dump(tableName, tablesToDump.keySet());
             UploadParams uploadParams = new UploadParams();
             uploadParams.add("newNames", renamedColumns);
             uploadParams.add("delimiter", dumpResult.get("delimiter"));
             uploadParams.add("labels", labels);
+            uploadParams.add("types", tablesToDump.values());
+            uploadParams.add("timeFormat", timeFormat);
             UploadResult uploadResult = neo4jUploader.createNode((InputStream) dumpResult.get("inputStream"), uploadParams);
             System.out.println("Table " + tableName + " successfully uploaded to Neo4j.");
             System.out.println("Created " + uploadResult.get("nodeCounter") + " nodes.\n");
@@ -93,12 +97,16 @@ private void parseRelationshipMigration(List<XML> tables) {
             String labelFrom = getConfigurationTagValue(table, "labelFrom");
             String labelTo = getConfigurationTagValue(table, "labelTo");
             String type = getType(table);
+            String columnFromType = postgresRepository.getColumnType(tableName, columnFrom);
+            String columnToType = postgresRepository.getColumnType(tableName, columnFrom);
             DumpResult dumpResult = dumper.dumpWithForeignKeys(tableName, columnFrom, columnTo);
             UploadParams uploadParams = new UploadParams();
             uploadParams.add("delimiter", dumpResult.get("delimiter"));
             uploadParams.add("type", type);
             uploadParams.add("labelFrom", labelFrom);
             uploadParams.add("labelTo", labelTo);
+            uploadParams.add("columnFromType", columnFromType);
+            uploadParams.add("columnToType", columnToType);
             UploadResult uploadResult = neo4jUploader.createRelationship((InputStream) dumpResult.get("inputStream"), uploadParams);
             System.out.println("Table " + tableName + " successfully uploaded to Neo4j.");
             System.out.println("Created " + uploadResult.get("relationshipCounter") + " relationships.\n");
@@ -144,16 +152,16 @@ private Map<String, String> getRenamedColumns(XML configuration) {
             if (previousNames.size() != newNames.size()) {
                 throw new InvalidConfigurationException("Amount of <previousName> and <newName> tags must be the same.");
             }
-            Map<String, String> renamedColumnsMap = new HashMap<>();
+            Map<String, String> renamedColumnsMap = new LinkedHashMap<>();
             for (int i = 0; i < previousNames.size(); i++) {
                 renamedColumnsMap.put(previousNames.get(i), newNames.get(i));
             }
             return renamedColumnsMap;
         }
-        return new HashMap<>();
+        return new LinkedHashMap<>();
     }
 
-    private Map<String, String> getTables(String tableName, List<String> excludedColumns) {
+    private Map<String, String> getColumns(String tableName, List<String> excludedColumns) {
         Map<String, String> tables = postgresRepository.getColumnsInfo(tableName);
         for (String name : excludedColumns) {
             tables.remove(name);

src/main/java/com/example/postgresneo4jmigrationtool/repository/neo4j/Neo4jRepositoryImpl.java

@@ -29,7 +29,7 @@ public void addNode(Node node, String... labels) {
     @Override
     @Transactional
     public void addRelationship(Relationship relationship, String type) {
-        String query = "MATCH(nodeFrom %s {%s: '%s'}) MATCH(nodeTo %s {%s: '%s'}) CREATE (nodeFrom)-[:%s]->(nodeTo)";
+        String query = "MATCH(nodeFrom %s %s) MATCH(nodeTo %s %s) CREATE (nodeFrom)-[:%s]->(nodeTo)";
         if (!relationship.getLabelFrom().isEmpty()) {
             relationship.setLabelFrom(": " + relationship.getLabelFrom());
         }
@@ -38,11 +38,9 @@ public void addRelationship(Relationship relationship, String type) {
         }
         String preparedQuery = String.format(query,
                 relationship.getLabelFrom(),
-                relationship.getNodeFrom().getNames()[0],
-                relationship.getNodeFrom().getValues()[0],
+                relationship.getNodeFrom().toString(),
                 relationship.getLabelTo(),
-                relationship.getNodeTo().getNames()[0],
-                relationship.getNodeTo().getValues()[0],
+                relationship.getNodeTo().toString(),
                 type);
         neo4jClient.query(preparedQuery).fetch().all();
     }

src/main/java/com/example/postgresneo4jmigrationtool/repository/postgres/PostgresRepository.java

@@ -14,6 +14,8 @@ public interface PostgresRepository {
 
     Map<String, String> getColumnsInfo(String tableName);
 
+    String getColumnType(String tableName, String column);
+
     String getForeignColumnName(String tableName, String columnName);
 
 }

src/main/java/com/example/postgresneo4jmigrationtool/repository/postgres/PostgresRepositoryImpl.java

@@ -5,7 +5,8 @@
 import org.springframework.jdbc.core.JdbcTemplate;
 import org.springframework.stereotype.Repository;
 
-import java.util.HashMap;
+import java.util.LinkedHashMap;
+import java.util.List;
 import java.util.Map;
 
 @Repository
@@ -55,7 +56,7 @@ public Map<String, String> getColumnsInfo(String tableName) {
                 AND table_name   = '%s';
                 """;
         String formattedQuery = String.format(query, getSchemaName(), tableName);
-        Map<String, String> columnsInfo = new HashMap<>();
+        Map<String, String> columnsInfo = new LinkedHashMap<>();
         jdbcTemplate.query(formattedQuery, (rs, rowNum) -> {
             String columnName = rs.getString("column_name");
             String columnType = rs.getString("data_type");
@@ -65,6 +66,21 @@ public Map<String, String> getColumnsInfo(String tableName) {
         return columnsInfo;
     }
 
+    @Override
+    public String getColumnType(String tableName, String column) {
+        String query = """
+                SELECT data_type
+                FROM information_schema.columns
+                WHERE table_schema = '%s'
+                AND table_name   = '%s'
+                AND column_name = '%s';
+                """;
+        String formattedQuery = String.format(query, getSchemaName(), tableName, column);
+        List<String> type = jdbcTemplate.query(formattedQuery, (rs, rowNum) ->
+                rs.getString("data_type"));
+        return type.get(0);
+    }
+
     @Override
     public String getForeignColumnName(String tableName, String columnName) {
         String query = """

src/main/resources/xml/schema.xsd

@@ -39,6 +39,7 @@
                         minOccurs="0"/>
             <xs:element name="renamedColumns" type="renamedColumnsType"
                         minOccurs="0"/>
+            <xs:element name="timeFormat" type="xs:string" minOccurs="0"/>
         </xs:sequence>
     </xs:complexType>
 

src/main/resources/xml/script.xml

@@ -12,6 +12,7 @@
                             <newName>newName</newName>
                         </columns>
                     </renamedColumns>
+                    <timeFormat>yyyy-MM-dd'T'HH:mm:ss.SSS'Z'</timeFormat>
                 </configuration>
                 <labels>
                     <label>User</label>