Add html sanitization

Remove unnecessary rules

Author: Kikobeats

index.js

@@ -1,8 +1,9 @@
 'use strict'
 
-const rules = require('req-all')('./src/rules')
 const reduce = require('lodash.reduce')
-const cheerio = require('cheerio')
+
+const rules = require('req-all')('./src/rules')
+const loadHtml = require('./src/html')
 
 const isValid = result => result !== null && result !== undefined && result !== ''
 
@@ -18,9 +19,7 @@ const getValue = ($, conditions) => {
 }
 
 module.exports = rawHtml => {
-  const html = cheerio.load(rawHtml, {
-    lowerCaseAttributeNames: true
-  })
+  const html = loadHtml(rawHtml)
 
   return reduce(rules, (acc, conditions, ruleName) => {
     const value = getValue(html, conditions)

package.json

@@ -27,6 +27,7 @@
     "lodash.reduce": "~4.6.0",
     "normalize-url": "~1.9.1",
     "req-all": "~1.0.0",
+    "sanitize-html": "~1.14.1",
     "to-title-case": "~1.0.0",
     "url-regex": "~4.1.1"
   },

src/html/index.js

@@ -0,0 +1,23 @@
+'use strict'
+
+const sanitizeHtml = require('sanitize-html')
+const flow = require('lodash.flow')
+const cheerio = require('cheerio')
+
+const sanitize = html => sanitizeHtml(html, {
+  allowedTags: false,
+  allowedAttributes: false,
+  transformTags: {
+    meta: (tagName, attribs) => {
+      if (attribs.name) attribs.name = attribs.name.toLowerCase()
+      return {tagName, attribs}
+    }
+  }
+})
+
+const load = cheerio.load.bind(cheerio)
+
+module.exports = flow([
+  sanitize,
+  load
+])

src/rules/date.js

@@ -36,16 +36,11 @@ const wrap = rule => $ => {
 module.exports = [
   wrap($ => $('meta[property="article:published_time"]').attr('content')),
   wrap($ => $('meta[name="dc.date"]').attr('content')),
-  wrap($ => $('meta[name="DC.date"]').attr('content')),
   wrap($ => $('meta[name="dc.date.issued"]').attr('content')),
-  wrap($ => $('meta[name="DC.date.issued"]').attr('content')),
   wrap($ => $('meta[name="dc.date.created"]').attr('content')),
-  wrap($ => $('meta[name="DC.date.created"]').attr('content')),
-  wrap($ => $('meta[name="DC.Date"]').attr('content')),
   wrap($ => $('meta[name="date"]').attr('content')),
   wrap($ => $('meta[name="dcterms.date"]').attr('content')),
   wrap($ => $('[itemprop="datePublished"]').attr('content')),
-  wrap($ => $('time[itemprop*="pubDate"]').attr('datetime')),
   wrap($ => $('time[itemprop*="pubdate"]').attr('datetime')),
   wrap($ => $('[property*="dc:date"]').attr('content')),
   wrap($ => $('[property*="dc:created"]').attr('content')),