Fix for #116 - Store segment path of url content in SOLR

Add segment field to the managed schema Add segment field to the managed schema
USCDataScience · Apr 9, 2018 · 0de316c · 0de316c
1 parent 946f20e
commit 0de316c
Show file tree

Hide file tree

Showing 5 changed files with 14 additions and 0 deletions.
diff --git a/conf/solr/crawldb/conf/managed-schema b/conf/solr/crawldb/conf/managed-schema
@@ -146,6 +146,7 @@
    <field name="fetch_depth" type="int" indexed="true" stored="true" default="0" multiValued="false" docValues="true" />
    <field name="relative_path" type="string" indexed="true" stored="true" multiValued="false" docValues="true" />
    <field name="parent" type="string" indexed="true" stored="true" multiValued="false" docValues="true" />
+   <field name="segment" type="string" indexed="true" stored="true" multiValued="false" docValues="true" />
 
    <field name="modified_time" type="date" indexed="true" stored="true" multiValued="false" default="NOW" />
    <field name="retries_since_fetch" type="int" indexed="true" stored="true" default="-1" multiValued="false" docValues="true" />

diff --git a/sparkler-api/src/main/java/edu/usc/irds/sparkler/Constants.java b/sparkler-api/src/main/java/edu/usc/irds/sparkler/Constants.java
@@ -141,6 +141,7 @@ interface solr { // Solr Fields
         String MD_SUFFIX = "_md";
         String HDR_SUFFIX = "_hd";
         String RESPONSE_TIME = "response_time";
+        String SEGMENT = "segment";
     }
 
 }
diff --git a/sparkler-api/src/main/java/edu/usc/irds/sparkler/model/FetchedData.java b/sparkler-api/src/main/java/edu/usc/irds/sparkler/model/FetchedData.java
@@ -37,6 +37,7 @@ public class FetchedData implements Serializable {
     private int responseCode;
     private long responseTime = -1;
 
+    private String segment;
 
     public FetchedData() {
     }
@@ -92,6 +93,15 @@ public void setContentLength(Integer contentLength) {
         this.contentLength = contentLength;
     }
 
+    public String getSegment() {
+        return segment;
+    }
+
+    public void setSegment(String segment) {
+        this.segment = segment;
+    }
+
+
     public void setContent(byte[] content) {
         this.content = content;
     }

diff --git a/sparkler-app/src/main/scala/edu/usc/irds/sparkler/pipeline/FairFetcher.scala b/sparkler-app/src/main/scala/edu/usc/irds/sparkler/pipeline/FairFetcher.scala
@@ -53,6 +53,7 @@ class FairFetcher(val job: SparklerJob, val resources: Iterator[Resource], val d
     data.fetchedData = fetchedData.next
     val endTime = System.currentTimeMillis()
     data.fetchedData.getResource.setFetchTimestamp(data.fetchedData.getFetchedAt)
+    data.fetchedData.setSegment(job.currentTask)
     lastHit = data.fetchedData.getResource.getUrl
     if (data.fetchedData.getResponseTime < 0) {
       data.fetchedData.setResponseTime(endTime - startTime)

diff --git a/sparkler-app/src/main/scala/edu/usc/irds/sparkler/pipeline/StatusUpdateSolrTransformer.scala b/sparkler-app/src/main/scala/edu/usc/irds/sparkler/pipeline/StatusUpdateSolrTransformer.scala
@@ -54,6 +54,7 @@ object StatusUpdateSolrTransformer extends (CrawlData => SolrInputDocument ) wit
     sUpdate.setField(Constants.solr.SIGNATURE, hashFunction.hashBytes(data.fetchedData.getContent).toString)
     sUpdate.setField(Constants.solr.RELATIVE_PATH, URLUtil.reverseUrl(data.fetchedData.getResource.getUrl))
     sUpdate.setField(Constants.solr.OUTLINKS, data.parsedData.outlinks.toArray)
+    sUpdate.setField(Constants.solr.SEGMENT, data.fetchedData.getSegment)
     sUpdate.setField(Constants.solr.RESPONSE_TIME, data.fetchedData.getResponseTime)
     for ((scoreKey, score) <- data.fetchedData.getResource.getScore) {
       sUpdate.setField(scoreKey, Map("set" -> score).asJava)