[#877] added support for property value null (#908)

fixes #877
dbs-leipzig · Jul 31, 2018 · 1ede21b · 1ede21b
1 parent 68efc3d
commit 1ede21b
Show file tree

Hide file tree

Showing 13 changed files with 41 additions and 14 deletions.
diff --git a/gradoop-common/src/test/java/org/gradoop/common/GradoopTestUtils.java b/gradoop-common/src/test/java/org/gradoop/common/GradoopTestUtils.java
@@ -78,6 +78,7 @@ public class GradoopTestUtils {
   public static final String KEY_c = "keyc";
   public static final String KEY_d = "keyd";
   public static final String KEY_e = "keye";
+  public static final String KEY_f = "keyf";
 
   public static final Object      NULL_VAL_0                        = null;
   public static final boolean     BOOL_VAL_1                        = true;

diff --git a/gradoop-flink/src/main/java/org/gradoop/flink/io/impl/csv/metadata/MetaDataParser.java b/gradoop-flink/src/main/java/org/gradoop/flink/io/impl/csv/metadata/MetaDataParser.java
@@ -77,6 +77,7 @@ private static Map<String, Function<String, Object>> getTypeParserMap() {
     map.put(TypeString.LOCALDATE.getTypeString(), LocalDate::parse);
     map.put(TypeString.LOCALTIME.getTypeString(), LocalTime::parse);
     map.put(TypeString.LOCALDATETIME.getTypeString(), LocalDateTime::parse);
+    map.put(TypeString.NULL.getTypeString(), MetaDataParser::parseNullProperty);
     return Collections.unmodifiableMap(map);
   }
 
@@ -284,14 +285,31 @@ private static Object parseMapProperty(
       .collect(Collectors.toMap(e -> PropertyValue.create(e[0]), e -> PropertyValue.create(e[1])));
   }
 
+  /**
+   * Parse function to create null from the null string representation.
+   *
+   * @param nullString The string representing null.
+   * @throws IllegalArgumentException The string that is passed has to represent null.
+   * @return Returns null
+   */
+  private static Object parseNullProperty(String nullString) throws IllegalArgumentException {
+    if (nullString != null && nullString.equalsIgnoreCase(TypeString.NULL.getTypeString())) {
+      return null;
+    } else {
+      throw new IllegalArgumentException("Only null represents a null string.");
+    }
+  }
+
   /**
    * Returns the type string for the specified property value.
    *
    * @param propertyValue property value
    * @return property type string
    */
   public static String getTypeString(PropertyValue propertyValue) {
-    if (propertyValue.isShort()) {
+    if (propertyValue.isNull()) {
+      return TypeString.NULL.getTypeString();
+    } else if (propertyValue.isShort()) {
       return TypeString.SHORT.getTypeString();
     } else if (propertyValue.isInt()) {
       return TypeString.INTEGER.getTypeString();
@@ -336,6 +354,10 @@ public static String getTypeString(PropertyValue propertyValue) {
    * Supported type strings for the CSV format.
    */
   private enum TypeString {
+    /**
+     * Null type
+     */
+    NULL("null"),
     /**
      * Boolean type
      */

diff --git a/gradoop-flink/src/test/java/org/gradoop/flink/io/impl/csv/CSVDataSinkTest.java b/gradoop-flink/src/test/java/org/gradoop/flink/io/impl/csv/CSVDataSinkTest.java
@@ -143,7 +143,6 @@ public void testWriteExtendedProperties() throws Exception {
     DataSource csvDataSource = new CSVDataSource(tmpPath, getConfig());
     LogicalGraph sourceLogicalGraph = csvDataSource.getLogicalGraph();
 
-    collectAndAssertTrue(logicalGraph.equalsByElementData(sourceLogicalGraph));
     collectAndAssertTrue(logicalGraph.equalsByData(sourceLogicalGraph));
 
     sourceLogicalGraph.getEdges().collect().forEach(this::checkProperties);

diff --git a/gradoop-flink/src/test/java/org/gradoop/flink/io/impl/csv/CSVDataSourceTest.java b/gradoop-flink/src/test/java/org/gradoop/flink/io/impl/csv/CSVDataSourceTest.java
@@ -57,7 +57,6 @@ public void testReadExtendedProperties() throws Exception {
     DataSource dataSource = new CSVDataSource(csvPath, getConfig());
     LogicalGraph sourceLogicalGraph = dataSource.getLogicalGraph();
 
-    collectAndAssertTrue(sourceLogicalGraph.equalsByElementData(expected));
     collectAndAssertTrue(sourceLogicalGraph.equalsByData(expected));
 
     dataSource.getLogicalGraph().getEdges().collect()

diff --git a/gradoop-flink/src/test/java/org/gradoop/flink/io/impl/csv/CSVTestBase.java b/gradoop-flink/src/test/java/org/gradoop/flink/io/impl/csv/CSVTestBase.java
@@ -88,6 +88,7 @@ private static Map<String, Object> getPropertyMap() {
     propertyMap.put(GradoopTestUtils.KEY_c, stringList);
     propertyMap.put(GradoopTestUtils.KEY_d, intList);
     propertyMap.put(GradoopTestUtils.KEY_e, GradoopTestUtils.SHORT_VAL_e);
+    propertyMap.put(GradoopTestUtils.KEY_f, GradoopTestUtils.NULL_VAL_0);
     return Collections.unmodifiableMap(propertyMap);
   }
 
@@ -142,6 +143,7 @@ protected void checkProperties(EPGMElement epgmElement) {
     assertTrue(epgmElement.hasProperty(GradoopTestUtils.KEY_c));
     assertTrue(epgmElement.hasProperty(GradoopTestUtils.KEY_d));
     assertTrue(epgmElement.hasProperty(GradoopTestUtils.KEY_e));
+    assertTrue(epgmElement.hasProperty(GradoopTestUtils.KEY_f));
 
     // assert that the properties have valid data types
     assertTrue(epgmElement.getPropertyValue(GradoopTestUtils.KEY_0).isBoolean());
@@ -159,6 +161,7 @@ protected void checkProperties(EPGMElement epgmElement) {
     assertTrue(epgmElement.getPropertyValue(GradoopTestUtils.KEY_c).isList());
     assertTrue(epgmElement.getPropertyValue(GradoopTestUtils.KEY_d).isList());
     assertTrue(epgmElement.getPropertyValue(GradoopTestUtils.KEY_e).isShort());
+    assertTrue(epgmElement.getPropertyValue(GradoopTestUtils.KEY_f).isNull());
 
     // assert that the properties have valid values
     assertEquals(epgmElement.getPropertyValue(GradoopTestUtils.KEY_0).getBoolean(),
@@ -191,6 +194,8 @@ protected void checkProperties(EPGMElement epgmElement) {
       PROPERTY_MAP.get(GradoopTestUtils.KEY_d));
     assertEquals(epgmElement.getPropertyValue(GradoopTestUtils.KEY_e).getShort(),
       PROPERTY_MAP.get(GradoopTestUtils.KEY_e));
+    assertEquals(epgmElement.getPropertyValue(GradoopTestUtils.KEY_f).getObject(),
+      PROPERTY_MAP.get(GradoopTestUtils.KEY_f));
   }
 
   /**
@@ -214,5 +219,6 @@ protected void checkMetadataCsvLine(String line) {
     assertTrue(line.contains(GradoopTestUtils.KEY_c + ":list:string"));
     assertTrue(line.contains(GradoopTestUtils.KEY_d + ":list:int"));
     assertTrue(line.contains(GradoopTestUtils.KEY_e + ":short"));
+    assertTrue(line.contains(GradoopTestUtils.KEY_f + ":null"));
   }
 }
diff --git a/gradoop-flink/src/test/resources/data/csv/expected/expected.gdl b/gradoop-flink/src/test/resources/data/csv/expected/expected.gdl
@@ -1,5 +1,5 @@
 expected [
- (v0:A {a:"foo",b:42,c:13.37f})-[e0:a {a:1234,b:13.37f}]->(v1:A {a:"bar",b:23,c:19.84f}),
+ (v0:A {a:"foo",b:42,c:13.37f,d: NULL})-[e0:a {a:1234,b:13.37f}]->(v1:A {a:"bar",b:23,c:19.84f}),
  (v1)-[e1:a {a:5678,b:23.42f}]->(v0),
  (v1)-[e2:b {a:3141L}]->(v2:B {a:1234L,b:true,c:0.123d}),
  (v2)-[e3:b {a:2718L}]->(v3:B {a:5678L,b:false,c:4.123d}),

diff --git a/gradoop-flink/src/test/resources/data/csv/input/metadata.csv b/gradoop-flink/src/test/resources/data/csv/input/metadata.csv
@@ -1,4 +1,4 @@
-v;A;a:string,b:int,c:float
+v;A;a:string,b:int,c:float,d:null
 v;B;a:long,b:boolean,c:double
 e;a;a:int,b:float
 e;b;a:long
diff --git a/gradoop-flink/src/test/resources/data/csv/input/vertices.csv b/gradoop-flink/src/test/resources/data/csv/input/vertices.csv
@@ -1,4 +1,4 @@
-000000000000000000000000;A;foo|42|13.37
+000000000000000000000000;A;foo|42|13.37|null
 000000000000000000000001;A;bar|23|19.84
 000000000000000000000002;B;1234|true|0.123
 000000000000000000000003;B;5678|false|4.123

diff --git a/gradoop-flink/src/test/resources/data/csv/input_extended_properties/edges.csv b/gradoop-flink/src/test/resources/data/csv/input_extended_properties/edges.csv
@@ -1 +1 @@
-000000000000000000000002;000000000000000000000000;000000000000000000000001;creatorOf;true|23|23|2.3|2.3|23|000000000000000000000001|2018-06-01|18:06:01|2018-06-01T18:06:01|23|{myString1=12.345, myString2=67.89}|[myString1, myString2]|[1234, 5678]|23
+000000000000000000000002;000000000000000000000000;000000000000000000000001;creatorOf;true|23|23|2.3|2.3|23|000000000000000000000001|2018-06-01|18:06:01|2018-06-01T18:06:01|23|{myString1=12.345, myString2=67.89}|[myString1, myString2]|[1234, 5678]|23|null
diff --git a/gradoop-flink/src/test/resources/data/csv/input_extended_properties/metadata.csv b/gradoop-flink/src/test/resources/data/csv/input_extended_properties/metadata.csv
@@ -1,3 +1,3 @@
-v;User;key0:boolean,key1:int,key2:long,key3:float,key4:double,key5:string,key6:gradoopid,key7:localdate,key8:localtime,key9:localdatetime,keya:bigdecimal,keyb:map:string:double,keyc:list:string,keyd:list:int,keye:short
-v;Post;key0:boolean,key1:int,key2:long,key3:float,key4:double,key5:string,key6:gradoopid,key7:localdate,key8:localtime,key9:localdatetime,keya:bigdecimal,keyb:map:string:double,keyc:list:string,keyd:list:int,keye:short
-e;creatorOf;key0:boolean,key1:int,key2:long,key3:float,key4:double,key5:string,key6:gradoopid,key7:localdate,key8:localtime,key9:localdatetime,keya:bigdecimal,keyb:map:string:double,keyc:list:string,keyd:list:int,keye:short
+v;User;key0:boolean,key1:int,key2:long,key3:float,key4:double,key5:string,key6:gradoopid,key7:localdate,key8:localtime,key9:localdatetime,keya:bigdecimal,keyb:map:string:double,keyc:list:string,keyd:list:int,keye:short,keyf:null
+v;Post;key0:boolean,key1:int,key2:long,key3:float,key4:double,key5:string,key6:gradoopid,key7:localdate,key8:localtime,key9:localdatetime,keya:bigdecimal,keyb:map:string:double,keyc:list:string,keyd:list:int,keye:short,keyf:null
+e;creatorOf;key0:boolean,key1:int,key2:long,key3:float,key4:double,key5:string,key6:gradoopid,key7:localdate,key8:localtime,key9:localdatetime,keya:bigdecimal,keyb:map:string:double,keyc:list:string,keyd:list:int,keye:short,keyf:null
diff --git a/gradoop-flink/src/test/resources/data/csv/input_extended_properties/vertices.csv b/gradoop-flink/src/test/resources/data/csv/input_extended_properties/vertices.csv
@@ -1,2 +1,2 @@
-000000000000000000000000;User;true|23|23|2.3|2.3|23|000000000000000000000001|2018-06-01|18:06:01|2018-06-01T18:06:01|23|{myString1=12.345, myString2=67.89}|[myString1, myString2]|[1234, 5678]|23
-000000000000000000000001;Post;true|23|23|2.3|2.3|23|000000000000000000000001|2018-06-01|18:06:01|2018-06-01T18:06:01|23|{myString1=12.345, myString2=67.89}|[myString1, myString2]|[1234, 5678]|23
+000000000000000000000000;User;true|23|23|2.3|2.3|23|000000000000000000000001|2018-06-01|18:06:01|2018-06-01T18:06:01|23|{myString1=12.345, myString2=67.89}|[myString1, myString2]|[1234, 5678]|23|null
+000000000000000000000001;Post;true|23|23|2.3|2.3|23|000000000000000000000001|2018-06-01|18:06:01|2018-06-01T18:06:01|23|{myString1=12.345, myString2=67.89}|[myString1, myString2]|[1234, 5678]|23|null
diff --git a/gradoop-flink/src/test/resources/data/csv/input_indexed/metadata.csv b/gradoop-flink/src/test/resources/data/csv/input_indexed/metadata.csv
@@ -1,4 +1,4 @@
-v;A;a:string,b:int,c:float
+v;A;a:string,b:int,c:float,d:null
 v;B;a:long,b:boolean,c:double
 e;a;a:int,b:float
 e;b;a:long
diff --git a/gradoop-flink/src/test/resources/data/csv/input_indexed/vertices/A/data.csv b/gradoop-flink/src/test/resources/data/csv/input_indexed/vertices/A/data.csv
@@ -1,2 +1,2 @@
-000000000000000000000000;A;foo|42|13.37
+000000000000000000000000;A;foo|42|13.37|null
 000000000000000000000001;A;bar|23|19.84