Add logo support

Author: Kikobeats

index.js

@@ -1,6 +1,6 @@
 'use strict'
 
-const reduce = require('lodash.reduce')
+const {reduce} = require('lodash')
 const {ensureAsync} = require('async')
 const {promisify} = require('util')
 

package.json

@@ -22,10 +22,9 @@
     "cheerio": "~1.0.0-rc.2",
     "chrono-node": "~1.3.4",
     "condense-whitespace": "~1.0.0",
+    "is-relative-url": "~2.0.0",
     "isostring": "0.0.1",
-    "lodash.flow": "~3.5.0",
-    "lodash.isstring": "~4.0.1",
-    "lodash.reduce": "~4.6.0",
+    "lodash": "~4.17.4",
     "normalize-url": "~1.9.1",
     "req-all": "~1.0.0",
     "sanitize-html": "~1.14.1",

src/get-data/rules/author.js

@@ -1,10 +1,9 @@
 'use strict'
 
 const condenseWhitespace = require('condense-whitespace')
-const isString = require('lodash.isstring')
+const {isString, flow} = require('lodash')
 const toTitle = require('to-title-case')
 const urlRegex = require('url-regex')
-const flow = require('lodash.flow')
 
 const REGEX_BY = /^[\s\n]*by[\s\n]*/im
 const REGEX_STRICT = /^\S+\s+\S+/

src/get-data/rules/description.js

@@ -1,9 +1,8 @@
 'use strict'
 
 const condenseWhitespace = require('condense-whitespace')
-const isString = require('lodash.isstring')
 const smartquotes = require('smartquotes')
-const flow = require('lodash.flow')
+const {isString, flow} = require('lodash')
 
 const REGEX_LOCATION = /^[A-Z\s]+\s+[-—–]\s+/
 

src/get-data/rules/logo.js

@@ -0,0 +1,63 @@
+'use strict'
+
+const isRelativeUrl = require('is-relative-url')
+const sanetizeUrl = require('normalize-url')
+const {flow, chain, first, isString, concat, toNumber, split} = require('lodash')
+const {resolve: resolveUrl} = require('url')
+
+const normalizeUrl = url => sanetizeUrl(url, {stripWWW: false})
+
+/**
+ * Wrap a rule with validation and formatting logic.
+ *
+ * @param {Function} rule
+ * @return {Function} wrapped
+ */
+
+const wrap = rule => (htmlDom, baseUrl) => {
+  const value = rule(htmlDom)
+  if (!isString(value)) return
+  const url = isRelativeUrl(value) ? resolveUrl(baseUrl, value) : value
+  return normalizeUrl(url)
+}
+
+const getSize = flow([str => split(str, 'x'), first, toNumber])
+
+const getDomNodeSizes = (domNodes, attr) =>
+  chain(domNodes)
+  .map(({attribs}) => ({
+    size: getSize(attribs.sizes),
+    link: attribs[attr]
+  }))
+  .value()
+
+const getSizes = ($, collection) => chain(collection)
+  .reduce((acc, {tag, attr}) => {
+    const domNode = $(tag).get()
+    const selectors = getDomNodeSizes(domNode, attr)
+    return concat(acc, selectors)
+  }, [])
+  .sortBy(({size}) => -size)
+  .value()
+
+const sizeSelectors = [
+  {tag: 'meta[property="og:logo"]', attr: 'content'},
+  {tag: 'link[rel="apple-touch-icon"]', attr: 'href'},
+  {tag: 'link[rel="apple-touch-icon-precomposed"]', attr: 'href'},
+  {tag: 'meta[name="msapplication-TileImage"]', attr: 'content'},
+  {tag: 'link[rel="icon"]', attr: 'href'},
+  {tag: 'link[rel="shortcut icon"]', attr: 'href'}
+]
+
+/**
+ * Rules.
+ */
+
+module.exports = [
+  wrap($ => {
+    const sizes = getSizes($, sizeSelectors)
+    const size = chain(sizes).first().get('link').value()
+    return size
+  }),
+  wrap($ => '/favicon.ico')
+]

src/get-data/rules/publisher.js

@@ -1,6 +1,6 @@
 'use strict'
 
-const isString = require('lodash.isstring')
+const {isString} = require('lodash')
 const condenseWhitespace = require('condense-whitespace')
 
 const REGEX_RSS = /^(.*?)\s[-|]\satom$/i

src/get-data/rules/title.js

@@ -1,9 +1,8 @@
 'use strict'
 
 const condenseWhitespace = require('condense-whitespace')
-const isString = require('lodash.isstring')
+const {isString, flow} = require('lodash')
 const smartquotes = require('smartquotes')
-const flow = require('lodash.flow')
 
 const sanetize = flow([
   condenseWhitespace,

src/get-data/rules/url.js

@@ -1,7 +1,7 @@
 'use strict'
 
 const sanetizeUrl = require('normalize-url')
-const isString = require('lodash.isstring')
+const {isString} = require('lodash')
 
 const normalizeUrl = url => sanetizeUrl(url, {stripWWW: false})
 

src/html/index.js

@@ -1,7 +1,7 @@
 'use strict'
 
 const sanitizeHtml = require('sanitize-html')
-const flow = require('lodash.flow')
+const {flow} = require('lodash')
 const cheerio = require('cheerio')
 
 const sanitize = html => sanitizeHtml(html, {
@@ -15,6 +15,10 @@ const sanitize = html => sanitizeHtml(html, {
     a: (tagName, attribs) => {
       if (attribs.href) attribs.href = attribs.href.toLowerCase()
       return {tagName, attribs}
+    },
+    link: (tagName, attribs) => {
+      if (attribs.rel) attribs.rel = attribs.rel.toLowerCase()
+      return {tagName, attribs}
     }
   }
 })

test/web/anandtech/index.js

@@ -9,9 +9,10 @@ const fs = require('fs')
 const getMetaData = require('../../..')
 const readFile = promisify(fs.readFile)
 
-it('arstechnica', async () => {
+it('anandtech', async () => {
   const html = await readFile(resolve(__dirname, 'input.html'))
   const json = await loadJSON(resolve(__dirname, 'output.json'))
-  const metadata = await getMetaData({html})
+  const {url} = json
+  const metadata = await getMetaData({html, url})
   should(metadata).be.eql(json)
 })