Also allow XHTML through, as per #252.

lintool · Sep 29, 2016 · a190cea · a190cea
1 parent 5938479
commit a190cea
Showing 1 changed file with 1 addition and 0 deletions.
diff --git a/warcbase-core/src/main/scala/org/warcbase/spark/rdd/RecordRDD.scala b/warcbase-core/src/main/scala/org/warcbase/spark/rdd/RecordRDD.scala
@@ -52,6 +52,7 @@ object RecordRDD extends java.io.Serializable {
       rdd.filter(r =>
         r.getCrawlDate != null
           && (r.getMimeType == "text/html"
+          || r.getMimeType == "application/xhtml+xml"
           || r.getUrl.endsWith("htm")
           || r.getUrl.endsWith("html"))
           && !r.getUrl.endsWith("robots.txt"))