Detect extractors without HTTP request

Author: Kikobeats

packages/metascraper-video-provider/package.json

@@ -17,7 +17,6 @@
   },
   "dependencies": {
     "@metascraper/helpers": "^3.11.8",
-    "cheerio": "~1.0.0-rc.2",
     "json-future": "~2.1.2",
     "lodash": "~4.17.10",
     "parse-domain": "~2.1.2",

packages/metascraper-video-provider/scripts/postinstall.js

@@ -1,18 +1,21 @@
 'use strict'
 
+const parseDomain = require('parse-domain')
 const jsonFuture = require('json-future')
-const cheerio = require('cheerio')
-const got = require('got')
+const youtubedl = require('youtube-dl')
+const { promisify } = require('util')
+
+const getExtractors = promisify(youtubedl.getExtractors)
 
 ;(async () => {
-  const { body } = await got('https://rg3.github.io/youtube-dl/supportedsites.html')
-  const $ = cheerio.load(body)
-  const set = new Set()
+  const extractors = await getExtractors()
 
-  $('li').each(function (i, el) {
-    const domain = $(el).text().split(':')[0].toLowerCase()
-    set.add(domain)
-  })
+  const providers = extractors.reduce((set, extractor) => {
+    const provider = extractor.split(':')[0].toLowerCase()
+    const { domain = '' } = parseDomain(provider) || {}
+    set.add(domain || provider)
+    return set
+  }, new Set())
 
-  await jsonFuture.saveAsync('providers.json', Array.from(set))
+  await jsonFuture.saveAsync('providers.json', Array.from(providers))
 })()