Better twitter video extraction (#104)

Author: Kikobeats

packages/metascraper-video-provider/package.json

@@ -3,7 +3,7 @@
   "description": "Get video property from HTML markup",
   "homepage": "https://metascraper.js.org",
   "version": "3.12.1",
-  "main": "index.js",
+  "main": "src/index",
   "author": {
     "email": "ian@ianstormtaylor.com",
     "name": "Ian Storm Taylor"
@@ -17,13 +17,12 @@
   },
   "dependencies": {
     "@metascraper/helpers": "^3.12.1",
+    "got": "~9.0.0",
     "json-future": "~2.1.2",
     "lodash": "~4.17.10",
-    "twdown": "~1.0.3",
     "youtube-dl": "~1.12.2"
   },
   "devDependencies": {
-    "browserless": "latest",
     "mocha": "latest",
     "nyc": "latest",
     "puppeteer": "latest",
@@ -35,8 +34,7 @@
     "node": ">= 8"
   },
   "files": [
-    "index.js",
-    "scripts"
+    "src"
   ],
   "scripts": {
     "test": "NODE_PATH=.. TZ=UTC NODE_ENV=test nyc mocha test"

packages/metascraper-video-provider/src/get-video-info/index.js

@@ -0,0 +1,40 @@
+'use strict'
+
+const {isTwitterUrl, getTwitterVideoInfo} = require('./twitter-video-info')
+const getVideoInfo = require('./video-info')
+const { chain } = require('lodash')
+
+const getInfo = async url => {
+  if (!isTwitterUrl(url)) return getVideoInfo(url)
+
+  const [videoInfo, twitterVideos] = await Promise.all([
+    getVideoInfo(url),
+    getTwitterVideoInfo(url)
+  ])
+
+  const formats = chain(videoInfo.formats)
+    .reduce((acc, format, index) => {
+      const { url } = twitterVideos[index]
+      const item = {...format, url}
+      return [...acc, item]
+    }, [])
+    .value()
+
+  return {...videoInfo, formats}
+}
+
+// Local cache for successive calls
+let cachedVideoInfoUrl
+let cachedVideoInfo
+
+module.exports = async url => {
+  if (url === cachedVideoInfoUrl) return cachedVideoInfo
+  cachedVideoInfoUrl = url
+
+  try {
+    cachedVideoInfo = await getInfo(url)
+  } catch (err) {
+    cachedVideoInfo = {}
+  }
+  return cachedVideoInfo
+}

packages/metascraper-video-provider/src/get-video-info/twitter-video-info.js

@@ -0,0 +1,42 @@
+'use strict'
+
+const { chain } = require('lodash')
+const { URL } = require('url')
+const got = require('got')
+
+// twitter guest web token
+// https://github.com/soimort/you-get/blob/da8c982608c9308765e0960e08fc28cccb74b215/src/you_get/extractors/twitter.py#L72
+const TWITTER_BEARER_TOKEN = 'Bearer AAAAAAAAAAAAAAAAAAAAANRILgAAAAAAnNwIzUejRCOuH5E6I8xnZz4puTs%3D1Zv7ttfk8LF81IUq16cHjhLTvJu4FA33AGWWjCpTnA'
+
+const TWITTER_HOSTNAMES = ['twitter.com', 'mobile.twitter.com']
+
+const isTwitterUrl = url => TWITTER_HOSTNAMES.includes(new URL(url).hostname)
+
+const getTweetId = url => url.split('/').reverse()[0]
+
+const getGuestToken = async () => {
+  const { body } = await got.post('https://api.twitter.com/1.1/guest/activate.json', {
+    headers: { authorization: TWITTER_BEARER_TOKEN },
+    json: true
+  })
+  return body.guest_token
+}
+
+const getTwitterVideoInfo = async url => {
+  const tweetId = getTweetId(url)
+  const apiUrl = `https://api.twitter.com/2/timeline/conversation/${tweetId}.json?tweet_mode=extended`
+  const { body } = await got(apiUrl, {
+    json: true,
+    headers: {
+      authorization: TWITTER_BEARER_TOKEN,
+      'x-guest-token': await getGuestToken()
+    }
+  })
+
+  return chain(body)
+    .get(`globalObjects.tweets.${tweetId}.extended_entities.media.0.video_info.variants`)
+    .orderBy('bitrate', 'asc')
+    .value()
+}
+
+module.exports = { getTwitterVideoInfo, isTwitterUrl }

packages/metascraper-video-provider/src/get-video-info/video-info.js

@@ -0,0 +1,6 @@
+'use strict'
+
+const youtubedl = require('youtube-dl')
+const { promisify } = require('util')
+
+module.exports = promisify(youtubedl.getInfo)

packages/metascraper-video-provider/index.js renamed to packages/metascraper-video-provider/src/index.js

@@ -2,33 +2,9 @@
 
 const { overEvery, isEmpty, eq, has, round, size, get, chain, find, isString } = require('lodash')
 const { isUrl, titleize } = require('@metascraper/helpers')
-const youtubedl = require('youtube-dl')
-const { promisify } = require('util')
-const twdown = require('twdown')
-const { URL } = require('url')
 const path = require('path')
 
-const getInfo = promisify(youtubedl.getInfo)
-
-const TWITTER_HOSTNAMES = ['twitter.com', 'mobile.twitter.com']
-
-const isTwitterUrl = url => TWITTER_HOSTNAMES.includes(new URL(url).hostname)
-
-// Local cache for successive calls
-let cachedVideoInfoUrl
-let cachedVideoInfo
-
-const getVideoInfo = async url => {
-  if (url === cachedVideoInfoUrl) return cachedVideoInfo
-  cachedVideoInfoUrl = url
-
-  try {
-    cachedVideoInfo = await getInfo(url)
-  } catch (err) {
-    cachedVideoInfo = {}
-  }
-  return cachedVideoInfo
-}
+const getVideoInfo = require('./get-video-info')
 
 const isMp4 = video =>
   eq(get(video, 'ext'), 'mp4') || path.extname(get(video, 'url')).startsWith('.mp4')
@@ -52,12 +28,10 @@ const getVideoUrl = (videos, filters = []) => {
 }
 
 /**
- * Get a URL-like video source.
+ * Get a URL-like video source
  */
-const getVideoProvider = getBrowserless => async ({ url }) => {
-  const formats = !isTwitterUrl(url)
-    ? (await getVideoInfo(url)).formats
-    : await twdown({ url, browserless: await getBrowserless() })
+const getVideoProvider = async ({ url }) => {
+  const { formats } = await getVideoInfo(url)
 
   const videoUrl =
     getVideoUrl(formats, [isMp4, isHttps, hasAudio]) ||
@@ -97,9 +71,9 @@ const getVideoDate = async ({ url }) => {
   return timestamp && new Date(timestamp * 1000).toISOString()
 }
 
-module.exports = ({ getBrowserless }) => {
+module.exports = () => {
   return {
-    video: getVideoProvider(getBrowserless),
+    video: getVideoProvider,
     author: getVideoAuthor,
     publisher: getVideoPublisher,
     title: getVideoTitle,

packages/metascraper-video-provider/test/index.js

@@ -1,7 +1,6 @@
 'use strict'
 
 const { isUrl } = require('@metascraper/helpers')
-const browserless = require('browserless')()
 const { isString } = require('lodash')
 const snapshot = require('snap-shot')
 const { promisify } = require('util')
@@ -11,7 +10,7 @@ const should = require('should')
 const fs = require('fs')
 
 const metascraper = require('metascraper').load([
-  require('metascraper-video-provider')({ getBrowserless: () => browserless }),
+  require('metascraper-video-provider')(),
   require('metascraper-author')(),
   require('metascraper-date')(),
   require('metascraper-description')(),