Fix json field extraction with mix of nested objects.

elastic · May 19, 2015 · 60bd896 · 60bd896
1 parent e3bb04e
commit 60bd896
Show file tree

Hide file tree

Showing 3 changed files with 41 additions and 37 deletions.
diff --git a/mr/src/main/java/org/elasticsearch/hadoop/serialization/ParsingUtils.java b/mr/src/main/java/org/elasticsearch/hadoop/serialization/ParsingUtils.java
@@ -158,38 +158,18 @@ public static List<Object> values(Parser parser, String... paths) {
     }
 
     private static void doFind(Parser parser, List<Matcher> currentMatchers, int level, int maxNesting) {
-        String currentName;
         Token token = parser.currentToken();
-
         if (token == null) {
-            token = parser.nextToken();
+            // advance to the initial START_OBJECT token
+            parser.nextToken();
         }
-        List<Matcher> nextLevel = null;
-
-        while ((token = parser.nextToken()) != null) {
-            if (token == Token.START_OBJECT) {
-                if (level < maxNesting) {
-                    if (nextLevel != null) {
-                        doFind(parser, nextLevel, level + 1, maxNesting);
-                    }
-                    // first round - a bit exceptional
-                    else if (level == -1) {
-                        doFind(parser, currentMatchers, level + 1, maxNesting);
-                    }
-                    // no need to go deeper, there are no matchers
-                    else {
-                        parser.skipChildren();
-                    }
-                }
-                else {
-                    parser.skipChildren();
-                }
-            }
-            else if (token == Token.FIELD_NAME) {
-                currentName = parser.currentName();
 
+        while ((token = parser.nextToken()) != null && token != Token.END_OBJECT) {
+            if (token == Token.FIELD_NAME) {
+                String currentName = parser.currentName();
                 Object value = null;
                 boolean valueRead = false;
+                List<Matcher> nextLevel = null;
 
                 for (Matcher matcher : currentMatchers) {
                     if (matcher.matches(currentName, level)) {
@@ -225,15 +205,24 @@ else if (token == Token.FIELD_NAME) {
                         }
                     }
                 }
+
+                if (!valueRead) {
+                    // must parse or skip the value
+                    switch (parser.nextToken()) {
+                        case START_OBJECT:
+                            if (level < maxNesting && nextLevel != null) {
+                                doFind(parser, nextLevel, level + 1, maxNesting);
+                            } else {
+                                parser.skipChildren();
+                            }
+                            break;
+                        case START_ARRAY:
+                            // arrays are not handled; simply ignore
+                            parser.skipChildren();
+                            break;
+                    }
+                }
             }
-            else if (token == Token.END_OBJECT) {
-                // end current block
-            }
-            // arrays are not handled; simply ignore
-            else if (token == Token.START_ARRAY) {
-                parser.skipChildren();
-            }
-            // ignore other tokens
         }
     }
 

diff --git a/mr/src/test/java/org/elasticsearch/hadoop/serialization/JsonValuePathTest.java b/mr/src/test/java/org/elasticsearch/hadoop/serialization/JsonValuePathTest.java
@@ -104,4 +104,14 @@ public void testCorrectLevelMatched() throws Exception {
         assertEquals(1, vals.size());
         assertThat(vals.get(0).toString(), containsString("CA"));
     }
+
+    @Test
+    public void testMixedLevels() throws Exception {
+        List<Object> vals = ParsingUtils.values(parser, "firstName", "address.building.floors", "address.decor.walls", "zzz");
+        assertEquals(4, vals.size());
+        assertEquals("John", vals.get(0));
+        assertEquals(10, vals.get(1));
+        assertEquals("white", vals.get(2));
+        assertEquals("end", vals.get(3));
+    }
 }
diff --git a/mr/src/test/resources/org/elasticsearch/hadoop/serialization/parser-test-nested.json b/mr/src/test/resources/org/elasticsearch/hadoop/serialization/parser-test-nested.json
@@ -6,7 +6,7 @@
         "bogus":true,
         "state":"bogus",
         "building": {
-            "hight":"bogus",
+            "height":"bogus",
             "floors": 0,
             "flats": 0
         }
@@ -17,10 +17,14 @@
         "state": "NY",
         "postalCode": 10021,
         "building": {
-            "hight":"tall",
+            "height":"tall",
             "floors": 10,
             "flats": 40
         },
+        "decor": {
+            "walls": "white",
+            "floors": "parquet"
+        },
         "firstName":"should-not-be-picked"
     },
     "phoneNumbers": [
@@ -44,5 +48,6 @@
     },
     "small-array": [
         "foo", "bar"
-    ]
+    ],
+    "zzz": "end"
 }