Add some refactoring, fix issue #53

orientechnologies · Apr 3, 2015 · 225b7db · 225b7db
1 parent bc0525e
commit 225b7db
Show file tree

Hide file tree

Showing 2 changed files with 25 additions and 22 deletions.
diff --git a/src/main/java/com/orientechnologies/orient/etl/transformer/OCSVTransformer.java b/src/main/java/com/orientechnologies/orient/etl/transformer/OCSVTransformer.java
@@ -112,26 +112,15 @@ public Object executeTransform(final Object input) {
 
         final OType fieldType = columnTypes != null ? columnTypes.get(i) : null;
 
-        if (fieldType != null && fieldType != OType.ANY) {
-          // DEFINED TYPE
-          fieldValue = getCellContent(fieldStringValue);
-          try {
-            fieldValue = OType.convert(fieldValue, fieldType.getDefaultJavaType());
-            doc.field(fieldName, fieldValue);
-          } catch (Exception e) {
-            processor.getStats().incrementErrors();
-            log(OETLProcessor.LOG_LEVELS.ERROR, "Error on converting row %d field '%s' (%d), value '%s' (class:%s) to type: %s",
-                processor.getExtractor().getProgress(), fieldName, i, fieldValue, fieldValue.getClass().getName(), fieldType);
+          if (fieldType != null && fieldType != OType.ANY) {
+              // DEFINED TYPE
+              fieldValue = processKnownType(doc, i, fieldName, fieldStringValue, fieldType);
+          } else {
+              // DETERMINE THE TYPE
+              if (fieldStringValue == null) fieldValue = null;
+              else fieldValue = determineTheType(fieldStringValue);
           }
-        } else if (fieldStringValue != null && !fieldStringValue.isEmpty()) {
-          // DETERMINE THE TYPE
-            fieldValue = determineTheType(fieldStringValue);
-            if (nullValue != null && nullValue.equals(fieldValue))
-            // NULL VALUE, SKIP
-            continue;
-
           doc.field(fieldName, fieldValue);
-        }
 
       } catch (Exception e) {
         processor.getStats().incrementErrors();
@@ -144,6 +133,20 @@ public Object executeTransform(final Object input) {
     return doc;
   }
 
+    private Object processKnownType(ODocument doc, int i, String fieldName, String fieldStringValue, OType fieldType) {
+        Object fieldValue;
+        fieldValue = getCellContent(fieldStringValue);
+        try {
+            fieldValue = OType.convert(fieldValue, fieldType.getDefaultJavaType());
+            doc.field(fieldName, fieldValue);
+        } catch (Exception e) {
+            processor.getStats().incrementErrors();
+            log(OETLProcessor.LOG_LEVELS.ERROR, "Error on converting row %d field '%s' (%d), value '%s' (class:%s) to type: %s",
+                    processor.getExtractor().getProgress(), fieldName, i, fieldValue, fieldValue.getClass().getName(), fieldType);
+        }
+        return fieldValue;
+    }
+
     private Object determineTheType(String fieldStringValue) {
         Object fieldValue;
         if ((fieldValue = transformToDate(fieldStringValue)) == null)// try maybe Date type
@@ -225,7 +228,7 @@ protected boolean isFinite(Float f) {
 
   // TODO Test, and double doubleqoutes case
   public String getCellContent(String iValue) {
-    if (iValue == null)
+      if (iValue == null || iValue.length() == 0 || "NULL".equals(iValue))
       return null;
 
     if (iValue.length() > 1 && (iValue.charAt(0) == stringCharacter && iValue.charAt(iValue.length() - 1) == stringCharacter))

diff --git a/src/test/java/com/orientechnologies/orient/etl/transformer/OCSVTransformerTest.java b/src/test/java/com/orientechnologies/orient/etl/transformer/OCSVTransformerTest.java
@@ -234,7 +234,7 @@ public void testNullCell() {
         List<ODocument> res = getResult();
         ODocument doc = res.get(0);
         assertEquals(new Integer(1), (Integer) doc.field("id"));
-        assertNull((Integer) doc.field("postId"));
+        assertNull(doc.field("postId"));
         assertEquals("Hello", (String) doc.field("text"));
     }
 
@@ -245,13 +245,13 @@ public void testNullValueInCell() {
         List<ODocument> res = getResult();
         ODocument doc = res.get(0);
         assertEquals(new Integer(1), (Integer) doc.field("id"));
-        assertNull((Integer) doc.field("postId"));
+        assertNull(doc.field("postId"));
         assertEquals("Hello", (String) doc.field("text"));
     }
 
     @Test
     public void testNullValueInCellEmptyString() {
-        String cfgJson = "{source: { content: { value: 'id,title,text\n1,,Hello'} }, extractor : { row : {} }, transformers : [{ csv : {nullValue: 'NULL'} }], loader : { test: {} } }";
+        String cfgJson = "{source: { content: { value: 'id,title,text\n1,\"\",Hello'} }, extractor : { row : {} }, transformers : [{ csv : {nullValue: 'NULL'} }], loader : { test: {} } }";
         process(cfgJson);
         List<ODocument> res = getResult();
         ODocument doc = res.get(0);