Extract more video fields

Author: Kikobeats

packages/metascraper-author/index.js

@@ -16,8 +16,8 @@ const wrap = rule => ({ htmlDom }) => {
   const value = rule(htmlDom)
 
   return isString(value) &&
-  !isUrl(value, {relative: false}) &&
-  titleize(value, {removeBy: true})
+    !isUrl(value, {relative: false}) &&
+    titleize(value, {removeBy: true})
 }
 
 /**

packages/metascraper-video-provider/__snapshots__/index.js.snap-shot

@@ -0,0 +1,44 @@
+exports['twitter 1'] = {
+  "author": "The Verge",
+  "publisher": "TwitterCard",
+  "date": "2018-02-11T12:00:00.000Z",
+  "description": "“Is it bad to blow into game cartridges? https://t.co/Y3yAimrUnP”",
+  "image": "https://pbs.twimg.com/media/DRg1OMRVwAEuwTK.jpg",
+  "lang": "es",
+  "logo": "https://abs.twimg.com/icons/apple-touch-icon-192x192.png",
+  "url": "https://twitter.com/verge/status/957383241714970624"
+}
+
+exports['facebook 1'] = {
+  "author": "AFC Ajax",
+  "publisher": "Facebook",
+  "date": "2018-04-02T18:57:00.000Z",
+  "description": null,
+  "image": null,
+  "lang": "es",
+  "logo": "https://static.xx.fbcdn.net/rsrc.php/yp/r/1Dxu6XIjaTc.ico",
+  "url": "https://www.facebook.com/afcajax/videos/1686831701364171"
+}
+
+exports['youtube 1'] = {
+  "author": "ONE Media",
+  "publisher": "Youtube",
+  "date": "2017-04-14T12:00:00.000Z",
+  "description": "Star Wars 8 El Ultimo JEDI Trailer Espanol (Subtitulado) - 2017\n© 2017 - Disney",
+  "image": "https://i.ytimg.com/vi/hwMkbaS_M_c/mqdefault.jpg",
+  "lang": null,
+  "logo": "https://www.youtube.com/yts/img/favicon_144-vfliLAfaB.png",
+  "url": "https://www.youtube.com/watch?v=hwMkbaS_M_c"
+}
+
+exports['vimeo 1'] = {
+  "author": "pleid",
+  "publisher": "Vimeo",
+  "date": "2016-10-20T13:06:52.000Z",
+  "description": "Converse has spent a good part of this year updating some of their classics. Our past is constantly catching up to us, but we rarely get to see the relationship…",
+  "image": "https://i.vimeocdn.com/filter/overlay?src0=https://i.vimeocdn.com/video/598160082_1280x720.jpg&src1=https://f.vimeocdn.com/images_v6/share/play_icon_overlay.png",
+  "lang": "en",
+  "logo": "https://i.vimeocdn.com/favicon/main-touch_180",
+  "url": "https://vimeo.com/188175573"
+}
+

packages/metascraper-video-provider/index.js

@@ -1,41 +1,92 @@
 'use strict'
 
-const { isUrl } = require('@metascraper/helpers')
+const { round, size, get, chain, find, isString } = require('lodash')
+const { isUrl, titleize } = require('@metascraper/helpers')
 const youtubedl = require('youtube-dl')
 const { promisify } = require('util')
 const path = require('path')
 
-const getVideoInfo = promisify(youtubedl.getInfo)
+const getInfo = promisify(youtubedl.getInfo)
+
+let cachedVideoInfoUrl
+let cachedVideoInfo
 
 /**
- * Get a Video source quality not too high
+ * Get the video info.
+ * Avoid do more one request for the same URL.
  */
-const getVideoUrl = ({formats}) => {
-  const urls = formats
-    .filter(item =>
-      item.protocol === 'https' &&
-      (item.ext === 'mp4' ||
-      path.extname(item.url).startsWith('.mp4'))
-    )
-    .map(item => item.url)
-
-  const index = Math.round(urls.length / 2) - 1
-  return urls[index]
-}
-
-const getVideoProvider = async url => {
+const getVideoInfo = async url => {
+  if (url === cachedVideoInfoUrl) return cachedVideoInfo
   try {
-    const info = await getVideoInfo(url)
-    const videoUrl = getVideoUrl(info)
-    return isUrl(videoUrl) && videoUrl
+    const info = await getInfo(url)
+    cachedVideoInfoUrl = url
+    cachedVideoInfo = info
+    return info
   } catch (err) {
-    return false
+    return {}
   }
 }
 
+const isMp4 = format => format.ext === 'mp4' || path.extname(format.url).startsWith('.mp4')
+const isHttp = format => format.protocol === 'https' || format.protocol === 'http'
+
+/**
+ * Get a Video source quality enough good
+ * compatible to be consumed for the browser.
+ */
+const getVideoUrl = formats => {
+  const urls = chain(formats)
+    .filter(format => isHttp(format) && isMp4(format))
+    .map('url')
+    .value()
+
+  const index = round(size(urls) / 2) - 1
+  return get(urls, index)
+}
+
+/**
+ * Get a URL-like video source.
+ */
+const getVideoProvider = async ({url}) => {
+  const { formats } = await getVideoInfo(url)
+  const videoUrl = getVideoUrl(formats)
+  return isUrl(videoUrl) && videoUrl
+}
+
+/**
+ * Get the Author of the video.
+ */
+const getVideoAuthor = async ({url}) => {
+  const { uploader, creator, uploader_id: uploaderId } = await getVideoInfo(url)
+  const author = find([creator, uploader, uploaderId], str => (
+    isString(str) && !isUrl(str, {relative: false})
+  ))
+  return author && titleize(author, {removeBy: true})
+}
+
+const getVideoPublisher = async ({url}) => {
+  const { extractor_key: extractorKey } = await getVideoInfo(url)
+  return isString(extractorKey) && extractorKey
+}
+
+const getVideoTitle = async ({url}) => {
+  const { title: mainTitle, alt_title: secondaryTitle } = await getVideoInfo(url)
+  const title = find([mainTitle, secondaryTitle], isString)
+  return title && titleize(title)
+}
+
+const getVideoDate = async ({url}) => {
+  const { timestamp } = await getVideoInfo(url)
+  return timestamp && new Date(timestamp * 1000).toISOString()
+}
+
 module.exports = () => {
   return {
-    video: ({url}) => getVideoProvider(url)
+    video: getVideoProvider,
+    author: getVideoAuthor,
+    publisher: getVideoPublisher,
+    title: getVideoTitle,
+    date: getVideoDate
   }
 }
 

packages/metascraper-video-provider/package.json

@@ -16,15 +16,16 @@
     "url": "https://github.com/microlinkhq/metascraper/issues"
   },
   "dependencies": {
+    "lodash": "~4.17.5",
     "@metascraper/helpers": "^3.9.3",
     "youtube-dl": "~1.12.2"
   },
   "devDependencies": {
-    "lodash": "latest",
     "metascraper": "latest",
     "mocha": "latest",
     "nyc": "latest",
     "should": "latest",
+    "snap-shot": "latest",
     "standard": "latest"
   },
   "engines": {