Support URL

Author: Kikobeats

index.js

@@ -1,18 +1,18 @@
 'use strict'
 
-const cb2promise = require('cb2promise')
 const reduce = require('lodash.reduce')
 const {ensureAsync} = require('async')
+const {promisify} = require('util')
 
 const getData = require('./src/get-data')
 const loadHtml = require('./src/html')
 const {props} = getData
 
-const getMetaData = ensureAsync((rawHtml, cb) => {
-  const html = loadHtml(rawHtml)
+const getMetaData = ensureAsync(({url, html}, cb) => {
+  const htmlDom = loadHtml(html)
 
   const output = reduce(props, (acc, conditions, propName) => {
-    const value = getData(html, conditions)
+    const value = getData({htmlDom, url, conditions})
     // TODO: Avoid response nil values
     acc[propName] = value
     return acc
@@ -21,8 +21,8 @@ const getMetaData = ensureAsync((rawHtml, cb) => {
   return cb(null, output)
 })
 
-module.exports = (html, cb) => {
-  return cb
-    ? getMetaData(html, cb)
-    : cb2promise(getMetaData, html)
-}
+const getMetaDataPromise = promisify(getMetaData)
+
+module.exports = ({url, html}, cb) => (
+  cb ? getMetaData({url, html}, cb) : getMetaDataPromise({url, html})
+)

package.json

@@ -19,7 +19,6 @@
   ],
   "dependencies": {
     "async": "~2.5.0",
-    "cb2promise": "~1.1.0",
     "cheerio": "~1.0.0-rc.2",
     "chrono-node": "~1.3.4",
     "condense-whitespace": "~1.0.0",

src/get-data/index.js

@@ -4,19 +4,20 @@ const rules = require('req-all')('./rules')
 
 const isValid = result => result !== null && result !== undefined && result !== ''
 
-const getValue = ($, conditions) => {
+const getValue = ({htmlDom, url, conditions}) => {
   const size = conditions.length
   let index = -1
   let value = null
 
   while (!isValid(value) && index++ < size - 1) {
-    value = conditions[index]($)
+    value = conditions[index](htmlDom, url)
   }
+
   return value
 }
 
-const getData = ($, conditions) => {
-  const data = getValue($, conditions)
+const getData = ({htmlDom, url, conditions}) => {
+  const data = getValue({htmlDom, url, conditions})
   return isValid(data) ? data : null
 }
 

src/get-data/rules/date.js

@@ -50,9 +50,13 @@ module.exports = [
   wrap($ => $('time[datetime]').attr('datetime')),
   wrap($ => $('[class*="byline"]').text()),
   wrap($ => $('[class*="dateline"]').text()),
-  wrap($ => $('[class*="date"]').text()),
   wrap($ => $('[id*="date"]').text()),
+  wrap($ => $('[class*="date"]').text()),
+  wrap($ => $('[id*="publish"]').text()),
+  wrap($ => $('[class*="publish"]').text()),
   wrap($ => $('[class*="post-timestamp"]').text()),
   wrap($ => $('[class*="post-meta"]').text()),
-  wrap($ => $('[class*="metadata"]').text())
+  wrap($ => $('[class*="metadata"]').text()),
+  wrap($ => $('[id*="time"]').text()),
+  wrap($ => $('[class*="time"]').text())
 ]

src/get-data/rules/image.js

@@ -1,9 +1,11 @@
 'use strict'
 
-const normalizeUrl = require('normalize-url')
+const sanetizeUrl = require('normalize-url')
+const {resolve: resolveUrl} = require('url')
 const urlRegex = require('url-regex')
 
 const isUrl = value => urlRegex().test(value)
+const normalizeUrl = url => sanetizeUrl(url, {stripWWW: false})
 
 /**
  * Wrap a rule with validation and formatting logic.
@@ -12,19 +14,19 @@ const isUrl = value => urlRegex().test(value)
  * @return {Function} wrapped
  */
 
-const wrap = rule => $ => {
-  const value = rule($)
+const wrap = rule => (htmlDom, url) => {
+  const imageUrl = rule(htmlDom)
+  if (!imageUrl) return
 
-  if (!isUrl(value)) return
-  return normalizeUrl(value, {
-    stripWWW: false
-  })
+  if (isUrl(imageUrl)) return normalizeUrl(imageUrl)
+
+  const absoluteImageUrl = resolveUrl(url, imageUrl)
+  if (isUrl(absoluteImageUrl)) return normalizeUrl(absoluteImageUrl)
 }
 
 /**
  * Rules.
  */
-
 module.exports = [
   wrap($ => $('meta[property="og:image:secure_url"]').attr('content')),
   wrap($ => $('meta[property="og:image:url"]').attr('content')),
@@ -37,5 +39,7 @@ module.exports = [
   wrap($ => $('meta[name="sailthru.image.full"]').attr('content')),
   wrap($ => $('meta[name="sailthru.image.thumb"]').attr('content')),
   wrap($ => $('article img[src]').first().attr('src')),
-  wrap($ => $('#content img[src]').first().attr('src'))
+  wrap($ => $('#content img[src]').first().attr('src')),
+  wrap($ => $('img[alt*="author"]').attr('src')),
+  wrap($ => $('img[src]').attr('src'))
 ]

test/web/bloomberg/index.js

@@ -12,6 +12,6 @@ const readFile = promisify(fs.readFile)
 it(basename(__dirname), async () => {
   const html = await readFile(resolve(__dirname, 'input.html'))
   const json = await loadJSON(resolve(__dirname, 'output.json'))
-  const metadata = await getMetaData(html)
+  const metadata = await getMetaData({html})
   should(metadata).be.eql(json)
 })

test/web/business-today/index.js

@@ -12,6 +12,6 @@ const readFile = promisify(fs.readFile)
 it(basename(__dirname), async () => {
   const html = await readFile(resolve(__dirname, 'input.html'))
   const json = await loadJSON(resolve(__dirname, 'output.json'))
-  const metadata = await getMetaData(html)
+  const metadata = await getMetaData({html})
   should(metadata).be.eql(json)
 })

test/web/cbr/index.js

@@ -12,6 +12,6 @@ const readFile = promisify(fs.readFile)
 it(basename(__dirname), async () => {
   const html = await readFile(resolve(__dirname, 'input.html'))
   const json = await loadJSON(resolve(__dirname, 'output.json'))
-  const metadata = await getMetaData(html)
+  const metadata = await getMetaData({html})
   should(metadata).be.eql(json)
 })

test/web/cio/index.js

@@ -12,6 +12,6 @@ const readFile = promisify(fs.readFile)
 it(basename(__dirname), async () => {
   const html = await readFile(resolve(__dirname, 'input.html'))
   const json = await loadJSON(resolve(__dirname, 'output.json'))
-  const metadata = await getMetaData(html)
+  const metadata = await getMetaData({html})
   should(metadata).be.eql(json)
 })

test/web/cloud-pro/index.js

@@ -12,6 +12,6 @@ const readFile = promisify(fs.readFile)
 it(basename(__dirname), async () => {
   const html = await readFile(resolve(__dirname, 'input.html'))
   const json = await loadJSON(resolve(__dirname, 'output.json'))
-  const metadata = await getMetaData(html)
+  const metadata = await getMetaData({html})
   should(metadata).be.eql(json)
 })