Add metascraper-audio (#113)

Author: Kikobeats

README.md

@@ -103,6 +103,9 @@ Here is an example of the metadata that **metascraper** can collect:
 
 - `description` — eg. *Venture capitalists are raising money at the fastest rate...*<br/>
   The publisher's chosen description of the article.
+  
+- `audio` — eg. *https://cf-media.sndcdn.com/U78RIfDPV6ok.128.mp3*<br/>
+A audio URL that best represents the article.
 
 - `video` — eg. *https://assets.entrepreneur.com/content/preview.mp4*<br/>
   A video URL that best represents the article.
@@ -185,6 +188,7 @@ const metascraper = require('metascraper')([
 | [`metascraper-author`](https://github.com/microlinkhq/metascraper/tree/master/packages/metascraper-author) | [![npm](https://img.shields.io/npm/v/metascraper-author.svg?style=flat-square)](https://www.npmjs.com/package/metascraper-author) | [![Dependency Status](https://david-dm.org/microlinkhq/metascraper.svg?path=packages/metascraper-author&style=flat-square)](https://david-dm.org/microlinkhq/metascraper?path=packages/metascraper-author) |
 | [`metascraper-date`](https://github.com/microlinkhq/metascraper/tree/master/packages/metascraper-date) | [![npm](https://img.shields.io/npm/v/metascraper-date.svg?style=flat-square)](https://www.npmjs.com/package/metascraper-date) | [![Dependency Status](https://david-dm.org/microlinkhq/metascraper.svg?path=packages/metascraper-date&style=flat-square)](https://david-dm.org/microlinkhq/metascraper?path=packages/metascraper-date) |
 | [`metascraper-description`](https://github.com/microlinkhq/metascraper/tree/master/packages/metascraper-description) | [![npm](https://img.shields.io/npm/v/metascraper-description.svg?style=flat-square)](https://www.npmjs.com/package/metascraper-description) | [![Dependency Status](https://david-dm.org/microlinkhq/metascraper.svg?path=packages/metascraper-description&style=flat-square)](https://david-dm.org/microlinkhq/metascraper?path=packages/metascraper-description) |
+| [`metascraper-audio`](https://github.com/microlinkhq/metascraper/tree/master/packages/metascraper-audio) | [![npm](https://img.shields.io/npm/v/metascraper-audio.svg?style=flat-square)](https://www.npmjs.com/package/metascraper-audio) | [![Dependency Status](https://david-dm.org/microlinkhq/metascraper.svg?path=packages/metascraper-audio&style=flat-square)](https://david-dm.org/microlinkhq/metascraper?path=packages/metascraper-audio) |
 | [`metascraper-video`](https://github.com/microlinkhq/metascraper/tree/master/packages/metascraper-video) | [![npm](https://img.shields.io/npm/v/metascraper-video.svg?style=flat-square)](https://www.npmjs.com/package/metascraper-video) | [![Dependency Status](https://david-dm.org/microlinkhq/metascraper.svg?path=packages/metascraper-video&style=flat-square)](https://david-dm.org/microlinkhq/metascraper?path=packages/metascraper-video) |
 | [`metascraper-image`](https://github.com/microlinkhq/metascraper/tree/master/packages/metascraper-image) | [![npm](https://img.shields.io/npm/v/metascraper-image.svg?style=flat-square)](https://www.npmjs.com/package/metascraper-image) | [![Dependency Status](https://david-dm.org/microlinkhq/metascraper.svg?path=packages/metascraper-image&style=flat-square)](https://david-dm.org/microlinkhq/metascraper?path=packages/metascraper-image) |
 | [`metascraper-logo`](https://github.com/microlinkhq/metascraper/tree/master/packages/metascraper-logo) | [![npm](https://img.shields.io/npm/v/metascraper-logo.svg?style=flat-square)](https://www.npmjs.com/package/metascraper-logo) | [![Dependency Status](https://david-dm.org/microlinkhq/metascraper.svg?path=packages/metascraper-logo&style=flat-square)](https://david-dm.org/microlinkhq/metascraper?path=packages/metascraper-logo) |

packages/metascraper-audio/.npmrc

@@ -0,0 +1,4 @@
+unsafe-perm=true
+save-prefix=~
+shrinkwrap=false
+save=false

packages/metascraper-audio/README.md

@@ -0,0 +1,19 @@
+# metascraper-audio
+
+[![npm](https://img.shields.io/npm/v/metascraper-audio.svg?style=flat-square)](https://www.npmjs.com/package/metascraper-audio)
+[![Dependency Status](https://david-dm.org/microlinkhq/metascraper.svg?path=packages/metascraper-audio&style=flat-square)](https://david-dm.org/microlinkhq/metascraper?path=packages/metascraper-audio)
+
+> Get audio property from HTML markup.
+
+## Install
+
+```bash
+$ npm install metascraper-audio --save
+```
+
+## License
+
+**metascraper-audio** © [microlink.io](https://microlink.io), Released under the [MIT](https://github.com/microlinkhq/metascraper-audio/blob/master/LICENSE.md) License.<br>
+Authored and maintained by microlink.io with help from [contributors](https://github.com/microlinkhq/metascraper-audio/contributors).
+
+> [microlink.io](https://microlink.io) · GitHub [@microlink.io](https://github.com/microlinkhq) · Twitter [@microlinkhq](https://twitter.com/microlinkhq)

packages/metascraper-audio/__snapshots__/index.js.snap-shot

@@ -0,0 +1,20 @@
+exports['og:audio 1'] = {
+  "audio": "https://browserless.js.org/static/demo.mp3"
+}
+
+exports['og:audio:secure_url 1'] = {
+  "audio": "https://browserless.js.org/static/demo.mp3"
+}
+
+exports['twitter:player:stream 1'] = {
+  "audio": "https://browserless.js.org/static/demo.mp3"
+}
+
+exports['audio:src 1'] = {
+  "audio": "https://browserless.js.org/static/demo.mp3"
+}
+
+exports['audio:source:src 1'] = {
+  "audio": "https://browserless.js.org/static/demo.mp3"
+}
+

packages/metascraper-audio/index.js

@@ -0,0 +1,44 @@
+'use strict'
+
+const { isMime, url: urlFn, isAudioUrl } = require('@metascraper/helpers')
+
+/**
+ * Wrap a rule with validation and formatting logic.
+ *
+ * @param {Function} rule
+ * @return {Function} wrapped
+ */
+
+const createWrapper = fn => rule => ({ htmlDom, url }) => {
+  const value = rule(htmlDom)
+  return fn(value, url)
+}
+
+const wrapAudio = createWrapper((value, url) => {
+  const urlValue = urlFn(value, { url })
+  return isAudioUrl(urlValue) && urlValue
+})
+
+const withContentType = (url, contentType) =>
+  isMime(contentType, 'audio') ? url : false
+
+/**
+ * Rules.
+ */
+module.exports = () => ({
+  audio: [
+    wrapAudio($ => $('meta[property="og:audio:secure_url"]').attr('content')),
+    wrapAudio($ => $('meta[property="og:audio"]').attr('content')),
+    wrapAudio($ => {
+      const contentType = $(
+        'meta[property="twitter:player:stream:content_type"]'
+      ).attr('content')
+      const streamUrl = $('meta[property="twitter:player:stream"]').attr(
+        'content'
+      )
+      return contentType ? withContentType(streamUrl, contentType) : streamUrl
+    }),
+    wrapAudio($ => $('audio').attr('src')),
+    wrapAudio($ => $('audio > source').attr('src'))
+  ]
+})

packages/metascraper-audio/package.json

@@ -0,0 +1,47 @@
+{
+  "name": "metascraper-audio",
+  "description": "Get audio property from HTML markup",
+  "homepage": "https://metascraper.js.org",
+  "version": "0.0.0",
+  "main": "index.js",
+  "author": {
+    "email": "ian@ianstormtaylor.com",
+    "name": "Ian Storm Taylor"
+  },
+  "repository": {
+    "type": "git",
+    "url": "https://github.com/microlinkhq/metascraper/tree/master/packages/metascraper-audio"
+  },
+  "bugs": {
+    "url": "https://github.com/microlinkhq/metascraper/issues"
+  },
+  "dependencies": {
+    "@metascraper/helpers": "^4.0.1"
+  },
+  "devDependencies": {
+    "lodash": "latest",
+    "mocha": "latest",
+    "nyc": "latest",
+    "should": "latest",
+    "snap-shot": "latest",
+    "standard": "11"
+  },
+  "engines": {
+    "node": ">= 8"
+  },
+  "files": [
+    "index.js"
+  ],
+  "scripts": {
+    "test": "NODE_PATH=.. TZ=UTC NODE_ENV=test nyc mocha test"
+  },
+  "license": "MIT",
+  "peerDependencies": {
+    "metascraper": "^4"
+  },
+  "standard": {
+    "env": [
+      "mocha"
+    ]
+  }
+}

packages/metascraper-audio/test/index.js

@@ -0,0 +1,42 @@
+'use strict'
+
+const snapshot = require('snap-shot')
+
+const metascraper = require('metascraper')([require('metascraper-audio')()])
+
+describe('metascraper-audio', () => {
+  it('og:audio', async () => {
+    const html = `<meta property="og:audio" content="https://browserless.js.org/static/demo.mp3">`
+    const url = 'https://browserless.js.org'
+    const metadata = await metascraper({ html, url })
+    snapshot(metadata)
+  })
+
+  it('og:audio:secure_url', async () => {
+    const html = `<meta property="og:audio:secure_url" content="https://browserless.js.org/static/demo.mp3">`
+    const url = 'https://browserless.js.org'
+    const metadata = await metascraper({ html, url })
+    snapshot(metadata)
+  })
+
+  it('twitter:player:stream', async () => {
+    const html = `<meta property="twitter:player:stream" content="https://browserless.js.org/static/demo.mp3">`
+    const url = 'https://browserless.js.org'
+    const metadata = await metascraper({ html, url })
+    snapshot(metadata)
+  })
+
+  it('audio:src', async () => {
+    const html = `<audio src="https://browserless.js.org/static/demo.mp3">`
+    const url = 'https://browserless.js.org'
+    const metadata = await metascraper({ html, url })
+    snapshot(metadata)
+  })
+
+  it('audio:source:src', async () => {
+    const html = `<audio><source src="https://browserless.js.org/static/demo.mp3"></source></audio>`
+    const url = 'https://browserless.js.org'
+    const metadata = await metascraper({ html, url })
+    snapshot(metadata)
+  })
+})

packages/metascraper-audio/test/mocha.opts

@@ -0,0 +1,6 @@
+--require should
+--reporter spec
+--timeout 120000
+--slow 300
+--bail
+--recursive

packages/metascraper-helpers/index.js

@@ -13,20 +13,29 @@ const {
 } = require('lodash')
 
 const imageExtensions = difference(require('image-extensions'), ['gif'])
+const audioExtensions = difference(require('audio-extensions'), ['mp4'])
 const videoExtensions = union(require('video-extensions'), ['gif'])
+const langs = require('iso-639-3').map(({ iso6391 }) => iso6391)
 const condenseWhitespace = require('condense-whitespace')
-const audioExtensions = require('audio-extensions')
+const urlRegex = require('url-regex')({ exact: true })
 const isRelativeUrl = require('is-relative-url')
 const fileExtension = require('file-extension')
 const { resolve: resolveUrl } = require('url')
 const _normalizeUrl = require('normalize-url')
 const smartquotes = require('smartquotes')
+const mimeTypes = require('mime-types')
 const chrono = require('chrono-node')
-const urlRegex = require('url-regex')({ exact: true })
 const isIso = require('isostring')
 const toTitle = require('title')
+
 const { URL } = require('url')
 
+const MIMES_EXTENSIONS = {
+  audio: audioExtensions,
+  video: videoExtensions,
+  image: imageExtensions
+}
+
 const REGEX_BY = /^[\s\n]*by|@[\s\n]*/i
 
 const REGEX_LOCATION = /^[A-Z\s]+\s+[-—–]\s+/
@@ -83,12 +92,21 @@ const protocol = url => {
 const createUrlExtensionValidator = collection => url =>
   isUrl(url) && includes(collection, extension(url))
 
+const createExtensionValidator = collection => url =>
+  includes(collection, extension(url))
+
 const isVideoUrl = createUrlExtensionValidator(videoExtensions)
 
 const isAudioUrl = createUrlExtensionValidator(audioExtensions)
 
 const isImageUrl = createUrlExtensionValidator(imageExtensions)
 
+const isVideoExtension = createExtensionValidator(videoExtensions)
+
+const isAudioExtension = createExtensionValidator(audioExtensions)
+
+const isImageExtension = createExtensionValidator(imageExtensions)
+
 const extension = url => fileExtension(url).split('?')[0]
 
 const description = value => isString(value) && getDescription(value)
@@ -120,10 +138,21 @@ const date = value => {
   if (parsed) return parsed.toISOString()
 }
 
-const lang = value => isString(value) && toLower(value.substring(0, 2))
+const lang = value => {
+  if (isEmpty(value)) return false
+  const lang = toLower(value.trim().substring(0, 2))
+  const isLang = includes(langs, lang)
+  return isLang ? lang : false
+}
 
 const title = value => isString(value) && titleize(value)
 
+const isMime = (type, mime) => {
+  const extension = mimeTypes.extension(type)
+  const collection = MIMES_EXTENSIONS[extension]
+  return includes(collection, mime)
+}
+
 module.exports = {
   author,
   title,
@@ -139,8 +168,12 @@ module.exports = {
   protocol,
   publisher,
   normalizeUrl,
+  isMime,
   isUrl,
   isVideoUrl,
   isAudioUrl,
-  isImageUrl
+  isImageUrl,
+  isVideoExtension,
+  isAudioExtension,
+  isImageExtension
 }

packages/metascraper-helpers/package.json

@@ -22,8 +22,10 @@
     "file-extension": "~4.0.5",
     "image-extensions": "~1.1.0",
     "is-relative-url": "~2.0.0",
+    "iso-639-3": "~1.1.0",
     "isostring": "0.0.1",
     "lodash": "~4.17.10",
+    "mime-types": "~2.1.20",
     "normalize-url": "~3.3.0",
     "smartquotes": "~2.3.1",
     "title": "~3.3.2",