Permalink
Browse files

Made JsonMap look at nested map values to make everything type safe f…

…or Pig
  • Loading branch information...
1 parent 02a331a commit 03740c8cb17cc3d9589007f8aa7cf9a381da4119 @xstevens xstevens committed Aug 12, 2011
@@ -33,13 +33,13 @@
private static BagFactory bagFactory = BagFactory.getInstance();
private static TupleFactory tupleFactory = TupleFactory.getInstance();
+ @SuppressWarnings("unchecked")
@Override
public DataBag exec(Tuple input) throws IOException {
if (input == null || input.size() == 0) {
return null;
}
- @SuppressWarnings("unchecked")
Map<Object,Object> m = (Map<Object,Object>)input.get(0);
DataBag output = bagFactory.newDefaultBag();
@@ -66,28 +66,38 @@ private DataBag convertListToBag(List<Object> l) {
return dbag;
}
-
+
+ /**
+ * Convert map and its values to types that Pig can handle
+ * @param m
+ * @return
+ */
@SuppressWarnings("unchecked")
+ private Map<String,Object> makeSafe(Map<String,Object> m) {
+ Map<String,Object> safeValues = new HashMap<String,Object>();
+ for (Map.Entry<String, Object> entry : m.entrySet()) {
+ Object v = entry.getValue();
+ if (v != null && v instanceof List) {
+ DataBag db = convertListToBag((List<Object>)v);
+ safeValues.put(entry.getKey(), db);
+ } else if (v != null && v instanceof Map) {
+ safeValues.put(entry.getKey(), makeSafe((Map<String,Object>)v));
+ } else {
+ safeValues.put(entry.getKey(), entry.getValue());
+ }
+ }
+
+ return safeValues;
+ }
+
public Map<String, Object> exec(Tuple input) throws IOException {
if (input == null || input.size() == 0) {
return null;
}
try {
- reporter.progress();
Map<String,Object> values = jsonMapper.readValue((String)input.get(0), new TypeReference<Map<String,Object>>() { });
- Map<String,Object> safeValues = new HashMap<String,Object>();
- for (Map.Entry<String, Object> entry : values.entrySet()) {
- Object v = entry.getValue();
- if (v != null && v instanceof List) {
- DataBag db = convertListToBag((List<Object>)v);
- safeValues.put(entry.getKey(), db);
- } else {
- safeValues.put(entry.getKey(), entry.getValue());
- }
- }
-
- return safeValues;
+ return makeSafe(values);
} catch(JsonParseException e) {
pigLogger.warn(this, "JSON Parse Error", ERRORS.JSONParseError);
} catch(JsonMappingException e) {
@@ -0,0 +1,27 @@
+package com.mozilla.pig.filter.map;
+
+import java.io.IOException;
+import java.util.Map;
+
+import org.apache.pig.FilterFunc;
+import org.apache.pig.data.Tuple;
+
+/**
+ * Determines if the given map has contains the given key
+ */
+public class ContainsKey extends FilterFunc {
+
+ @SuppressWarnings("rawtypes")
+ @Override
+ public Boolean exec(Tuple input) throws IOException {
+ if (input == null || input.size() < 2) {
+ return false;
+ }
+
+ Map m = (Map)input.get(0);
+ Object k = input.get(1);
+
+ return m.containsKey(k);
+ }
+
+}
@@ -0,0 +1,47 @@
+package com.mozilla.telemetry.pig.eval;
+
+import java.io.IOException;
+import java.util.Map;
+
+import org.apache.pig.EvalFunc;
+import org.apache.pig.data.BagFactory;
+import org.apache.pig.data.DataBag;
+import org.apache.pig.data.Tuple;
+import org.apache.pig.data.TupleFactory;
+
+public class HistogramValueTuples extends EvalFunc<DataBag> {
+
+ private static BagFactory bagFactory = BagFactory.getInstance();
+ private static TupleFactory tupleFactory = TupleFactory.getInstance();
+
+ @SuppressWarnings("unchecked")
+ @Override
+ public DataBag exec(Tuple input) throws IOException {
+ if (input == null || input.size() == 0) {
+ return null;
+ }
+
+ DataBag output = bagFactory.newDefaultBag();
+ Map<String,Map<String,Map<String,Object>>> m = (Map<String,Map<String,Map<String,Object>>>)input.get(0);
+ if (m != null) {
+ for (Map.Entry<String, Map<String,Map<String,Object>>> hist : m.entrySet()) {
+ Map<String,Map<String,Object>> hv = hist.getValue();
+ if (hv != null) {
+ Map<String,Object> values = hv.get("values");
+ if (values != null) {
+ for (Map.Entry<String, Object> v : values.entrySet()) {
+ Tuple t = tupleFactory.newTuple(3);
+ t.set(0, hist.getKey());
+ t.set(1, v.getKey());
+ t.set(2, v.getValue());
+ output.add(t);
+ }
+ }
+ }
+ }
+ }
+
+ return output;
+ }
+
+}
@@ -0,0 +1,63 @@
+package com.mozilla.pig.eval.json;
+
+import static org.junit.Assert.assertEquals;
+import static org.junit.Assert.assertTrue;
+
+import java.io.BufferedReader;
+import java.io.FileInputStream;
+import java.io.IOException;
+import java.io.InputStreamReader;
+import java.util.Iterator;
+import java.util.Map;
+
+import org.apache.pig.data.DataBag;
+import org.apache.pig.data.Tuple;
+import org.apache.pig.data.TupleFactory;
+import org.junit.Test;
+
+import com.mozilla.telemetry.pig.eval.HistogramValueTuples;
+
+public class JsonMapTest {
+
+ private JsonMap jsonMap = new JsonMap();
+ private TupleFactory tupleFactory = TupleFactory.getInstance();
+
+ private String readFile(String file) throws IOException {
+ StringBuilder sb = new StringBuilder();
+ BufferedReader reader = null;
+ try {
+ reader = new BufferedReader(new InputStreamReader(new FileInputStream(file), "UTF-8"));
+ String line = null;
+ while ((line = reader.readLine()) != null) {
+ sb.append(line);
+ }
+ } finally {
+ if (reader != null) {
+ reader.close();
+ }
+ }
+
+ return sb.toString();
+ }
+
+ @Test
+ public void testExec1() throws IOException {
+ Tuple input = tupleFactory.newTuple();
+ input.append("{ \"foo\": \"bar\" }");
+ Map<String,Object> myMap = jsonMap.exec(input);
+ assertTrue(myMap.containsKey("foo"));
+ assertEquals(myMap.get("foo"), "bar");
+ }
+
+ @SuppressWarnings("unchecked")
+ @Test
+ public void testExec2() throws IOException {
+ Tuple input = tupleFactory.newTuple();
+ input.append(readFile(System.getProperty("basedir") + "/src/test/resources/telemetry.js"));
+ Map<String,Object> myMap = jsonMap.exec(input);
+ Map<String,Object> histograms = (Map<String,Object>)myMap.get("histograms");
+
+ assertTrue(histograms.containsKey("MOZ_SQLITE_OTHER_SYNC_MAIN_THREAD_MS"));
+ }
+
+}
Oops, something went wrong.

0 comments on commit 03740c8

Please sign in to comment.