Adding getCrawlYear in ArchiveRecords, #104 (#105)

Will test locally to see if this fits use case.
archivesunleashed · Oct 26, 2017 · 010fe24 · 010fe24
1 parent a539a1c
commit 010fe24
Show file tree

Hide file tree

Showing 4 changed files with 14 additions and 0 deletions.
diff --git a/src/main/scala/io/archivesunleashed/spark/archive/io/ArcRecord.scala b/src/main/scala/io/archivesunleashed/spark/archive/io/ArcRecord.scala
@@ -27,6 +27,8 @@ class ArcRecord(r: SerializableWritable[ArcRecordWritable]) extends ArchiveRecor
 
   val getCrawlMonth: String = ExtractDate(r.t.getRecord.getMetaData.getDate, DateComponent.YYYYMM)
 
+  val getCrawlYear: String = ExtractDate(r.t.getRecord.getMetaData.getDate, DateComponent.YYYY)
+
   val getMimeType: String = r.t.getRecord.getMetaData.getMimetype
 
   val getUrl: String = r.t.getRecord.getMetaData.getUrl

diff --git a/src/main/scala/io/archivesunleashed/spark/archive/io/ArchiveRecord.scala b/src/main/scala/io/archivesunleashed/spark/archive/io/ArchiveRecord.scala
@@ -21,6 +21,8 @@ trait ArchiveRecord extends Serializable {
 
   val getCrawlMonth: String
 
+  val getCrawlYear: String
+
   val getUrl: String
 
   val getDomain: String

diff --git a/src/main/scala/io/archivesunleashed/spark/archive/io/GenericArchiveRecord.scala b/src/main/scala/io/archivesunleashed/spark/archive/io/GenericArchiveRecord.scala
@@ -57,6 +57,14 @@ class GenericArchiveRecord(r: SerializableWritable[GenericArchiveRecordWritable]
     }
   }
 
+  val getCrawlYear: String = {
+    if (r.t.getFormat == ArchiveFormat.ARC) {
+      ExtractDate(arcRecord.getMetaData.getDate, DateComponent.YYYY)
+    } else {
+      ExtractDate(ArchiveUtils.get14DigitDate(ISO8601.parse(warcRecord.getHeader.getDate)), DateComponent.YYYY)
+    }
+  }
+
   val getContentBytes: Array[Byte] = {
     if (r.t.getFormat == ArchiveFormat.ARC) {
       ArcRecordUtils.getBodyContent(arcRecord)

diff --git a/src/main/scala/io/archivesunleashed/spark/archive/io/WarcRecord.scala b/src/main/scala/io/archivesunleashed/spark/archive/io/WarcRecord.scala
@@ -32,6 +32,8 @@ class WarcRecord(r: SerializableWritable[WarcRecordWritable]) extends ArchiveRec
 
   val getCrawlMonth: String = ExtractDate(ArchiveUtils.get14DigitDate(ISO8601.parse(r.t.getRecord.getHeader.getDate)), DateComponent.YYYYMM)
 
+  val getCrawlYear: String = ExtractDate(ArchiveUtils.get14DigitDate(ISO8601.parse(r.t.getRecord.getHeader.getDate)), DateComponent.YYYY)
+
   val getContentBytes: Array[Byte] = WarcRecordUtils.getContent(r.t.getRecord)
 
   val getContentString: String = new String(getContentBytes)