Permalink
Browse files

Checking in latest changes for pig scripts and UDF changes

  • Loading branch information...
1 parent a3c9de5 commit e2eeafd4818a8a2376c27bd23546164270af1e3f @xstevens xstevens committed Feb 1, 2013
@@ -74,7 +74,7 @@
private static final Logger LOG = Logger.getLogger(FHRConsumer.class);
private static final String GEO_COUNTRY_FIELD = "geoCountry";
- private static final String UNKNOWN = "Unknown";
+ private static final String UNKNOWN_COUNTRY_CODE = "--";
private ObjectMapper jsonMapper;
private LookupService geoIpLookupService;
@@ -224,15 +224,16 @@ public Void call() throws Exception {
// do a geoip lookup on the IP if we have one
if (bmsg.hasIpAddr()) {
Location location = geoIpLookupService.getLocation(InetAddress.getByAddress(bmsg.getIpAddr().toByteArray()));
- if (location != null && !"--".equals(location.countryCode)) {
+ if (location != null && !UNKNOWN_COUNTRY_CODE.equals(location.countryCode) &&
+ location.countryCode.trim().length() > 0) {
document.put(GEO_COUNTRY_FIELD, location.countryCode);
} else {
unknownGeoIpMeter.mark();
- document.put(GEO_COUNTRY_FIELD, UNKNOWN);
+ document.put(GEO_COUNTRY_FIELD, UNKNOWN_COUNTRY_CODE);
}
} else {
unknownGeoIpMeter.mark();
- document.put(GEO_COUNTRY_FIELD, UNKNOWN);
+ document.put(GEO_COUNTRY_FIELD, UNKNOWN_COUNTRY_CODE);
}
// store the document
@@ -20,42 +20,34 @@
package com.mozilla.fhr.pig.eval;
import java.io.IOException;
-import java.text.ParseException;
-import java.text.SimpleDateFormat;
import java.util.Calendar;
-import java.util.Date;
import org.apache.pig.EvalFunc;
import org.apache.pig.data.Tuple;
-public class ProfileAgeTime extends EvalFunc<Long> {
+public class ProfileCreationTime extends EvalFunc<Long> {
public static enum ERRORS { DateParseError };
+
+ private Calendar cal;
- private SimpleDateFormat sdf;
-
- public ProfileAgeTime(String pingTimeFormat) {
- sdf = new SimpleDateFormat(pingTimeFormat);
+ public ProfileCreationTime() {
+ cal = Calendar.getInstance();
}
@Override
public Long exec(Tuple input) throws IOException {
- if (input == null || input.size() != 2) {
+ if (input == null || input.size() == 0 ||
+ input.get(0) == null) {
return null;
}
-
- Long t = null;
- try {
- Date d = sdf.parse((String)input.get(0));
- Calendar cal = Calendar.getInstance();
- cal.setTime(d);
- cal.add(Calendar.DATE, -((Number)input.get(1)).intValue());
- t = cal.getTimeInMillis();
- } catch (ParseException e) {
- pigLogger.warn(this, "Date parse error", ERRORS.DateParseError);
- }
+
+ // reset to epoch
+ cal.setTimeInMillis(0);
+ // profileCreation is in days since epoch
+ cal.add(Calendar.DATE, ((Number)input.get(0)).intValue());
- return t;
+ return cal.getTimeInMillis();
}
}
@@ -24,6 +24,7 @@
import java.text.SimpleDateFormat;
import java.util.Date;
import java.util.Iterator;
+import java.util.Map;
import org.apache.pig.EvalFunc;
import org.apache.pig.data.DataBag;
@@ -33,6 +34,10 @@
public static enum ERRORS { ParseError };
+ private static final String APPINFO_VERSIONS_FIELD = "org.mozilla.appInfo.versions";
+ private static final String VERSION = "version";
+ private static final String MULTI_VERSION_DELIMITER = "|";
+
private final SimpleDateFormat sdf;
private long perspectiveTime;
@@ -46,6 +51,7 @@ public VersionOnDate(String dateFormat, String perspectiveDate) {
}
}
+ @SuppressWarnings("unchecked")
@Override
public String exec(Tuple input) throws IOException {
if (input == null || input.size() == 0) {
@@ -55,13 +61,33 @@ public String exec(Tuple input) throws IOException {
String latestVersion = null;
long latestTime = 0;
try {
- DataBag versions = (DataBag)input.get(0);
- Iterator<Tuple> iter = versions.iterator();
- while (iter.hasNext()) {
- Tuple t = iter.next();
- Date d = sdf.parse((String)t.get(0));
- if (d.getTime() <= perspectiveTime && d.getTime() > latestTime) {
- latestVersion = (String)t.get(1);
+ Map<String,Object> dataPoints = (Map<String,Object>)input.get(0);
+ for (Map.Entry<String,Object> dayEntry : dataPoints.entrySet()) {
+ Map<String,Object> dayMap = (Map<String,Object>)dayEntry.getValue();
+ if (dayMap.containsKey(APPINFO_VERSIONS_FIELD)) {
+ Date d = sdf.parse(dayEntry.getKey());
+ if (d.getTime() <= perspectiveTime && d.getTime() > latestTime) {
+ Map<String,Object> appInfoVersionMap = (Map<String,Object>)dayMap.get(APPINFO_VERSIONS_FIELD);
+ if (appInfoVersionMap.containsKey(VERSION)) {
+ DataBag versionBag = (DataBag)appInfoVersionMap.get(VERSION);
+ StringBuilder sb = new StringBuilder();
+ Iterator<Tuple> vbIter = versionBag.iterator();
+ for (int i=0; i < versionBag.size() && vbIter.hasNext(); i++) {
+ Tuple versionTuple = vbIter.next();
+ if (versionTuple.size() > 0) {
+ sb.append(versionTuple.get(0));
+ if (vbIter.hasNext()) {
+ sb.append(MULTI_VERSION_DELIMITER);
+ }
+ }
+ }
+
+ if (sb.length() > 0) {
+ latestVersion = sb.toString();
+ latestTime = d.getTime();
+ }
+ }
+ }
}
}
} catch (ParseException e) {
Oops, something went wrong.

0 comments on commit e2eeafd

Please sign in to comment.