Merge pull request #104 from nlevitt/aitfive-39

change field names to match new druid config
internetarchive · Oct 27, 2014 · f25c171 · f25c171
2 parents 0712cb3 + 13a9d3a
commit f25c171
Showing 1 changed file with 12 additions and 6 deletions.
diff --git a/contrib/src/main/java/org/archive/modules/postprocessor/AMQPCrawlLogFeed.java b/contrib/src/main/java/org/archive/modules/postprocessor/AMQPCrawlLogFeed.java
@@ -110,11 +110,11 @@ protected byte[] buildMessage(CrawlURI curi) {
             jo.put(entry.getKey(), entry.getValue());
         }
 
-        jo.put("bytes_received", curi.isHttpTransaction() && curi.getContentLength() >= 0 ? curi.getContentLength() : JSONObject.NULL);
-        jo.put("stored_size", curi.getContentSize() > 0 ? curi.getContentSize() : JSONObject.NULL);
+        jo.put("content_length", curi.isHttpTransaction() && curi.getContentLength() >= 0 ? curi.getContentLength() : JSONObject.NULL);
+        jo.put("size", curi.getContentSize() > 0 ? curi.getContentSize() : JSONObject.NULL);
 
-        jo.put("response_code", checkForNull(curi.getFetchStatus()));
-        jo.put("document_url", checkForNull(curi.getUURI().toString()));
+        jo.put("status_code", checkForNull(curi.getFetchStatus()));
+        jo.put("url", checkForNull(curi.getUURI().toString()));
         jo.put("hop_path", checkForNull(curi.getPathFromSeed()));
         jo.put("via", checkForNull(curi.flattenVia()));
         jo.put("mimetype", checkForNull(MimetypeUtils.truncate(curi.getContentType())));
@@ -129,7 +129,7 @@ protected byte[] buildMessage(CrawlURI curi) {
             jo.put("start_time_plus_duration", JSONObject.NULL);
         }
 
-        jo.put("payload_hash", checkForNull(curi.getContentDigestSchemeString()));
+        jo.put("content_digest", checkForNull(curi.getContentDigestSchemeString()));
         jo.put("seed", checkForNull(curi.getSourceTag()));
 
         CrawlHost host = getServerCache().getHostFor(curi.getUURI());
@@ -141,7 +141,13 @@ protected byte[] buildMessage(CrawlURI curi) {
 
         jo.put("annotations", checkForNull(StringUtils.join(curi.getAnnotations(), ",")));
 
-        jo.put("extra_info", checkForNull(curi.getExtraInfo()));
+        JSONObject ei = curi.getExtraInfo();
+        if (ei == null) {
+            ei = new JSONObject();
+        }
+        jo.put("warc_filename", checkForNull(ei.get("warcFilename")));
+        jo.put("warc_offset", checkForNull(ei.get("warcOffset")));
+        jo.put("extra_info", ei);
 
         String str = jo.toString();
         try {