perf: remove reachable-url when is not necessary (#516)

* test: rename variable meta → metadata * perf: use a memoizer * test: use same style wip * perf: remove reachable-url Since we want toe player HTML, we don't need to check reachability
microlinkhq · Mar 27, 2022 · 3e9f58d · 3e9f58d
1 parent 8b3ec1e
commit 3e9f58d
Show file tree

Hide file tree

Showing 7 changed files with 26 additions and 32 deletions.
diff --git a/packages/metascraper-audio/index.js b/packages/metascraper-audio/index.js
@@ -11,7 +11,6 @@ const {
   toRule
 } = require('@metascraper/helpers')
 
-const reachableUrl = require('reachable-url')
 const memoize = require('@keyvhq/memoize')
 const pReflect = require('p-reflect')
 const cheerio = require('cheerio')
@@ -49,16 +48,11 @@ const _getIframe = async (url, { src }) => {
 
 const createGetPlayer = ({ gotOpts, keyvOpts }) => {
   const getPlayer = async playerUrl => {
-    const response = await reachableUrl(playerUrl, gotOpts)
-    if (!reachableUrl.isReachable(response)) return
+    const { value: response } = await pReflect(got(playerUrl, gotOpts))
     const contentType = response.headers['content-type']
     if (!contentType || !contentType.startsWith('text')) return
-    const { value: html } = await pReflect(
-      got(playerUrl, { resolveBodyOnly: true, ...gotOpts })
-    )
-    return html
+    return response.body
   }
-
   return memoize(getPlayer, keyvOpts)
 }
 

diff --git a/packages/metascraper-audio/package.json b/packages/metascraper-audio/package.json
@@ -26,8 +26,7 @@
     "@metascraper/helpers": "^5.26.0",
     "cheerio": "~1.0.0-rc.10",
     "got": "~11.8.3",
-    "p-reflect": "~2.1.0",
-    "reachable-url": "~1.6.7"
+    "p-reflect": "~2.1.0"
   },
   "devDependencies": {
     "mocha": "latest",

diff --git a/packages/metascraper-audio/test/index.js b/packages/metascraper-audio/test/index.js
@@ -7,7 +7,7 @@ const createMetascraper = (...args) =>
   require('metascraper')([require('..')(...args)])
 
 describe('metascraper-audio', () => {
-  it('allow to customize keyOpts', async () => {
+  it('allow to customize keyv options', async () => {
     const cache = new Map()
     const html =
       '<meta property="twitter:player" content="https://twitter-card-player.vercel.app/audio.html">'

diff --git a/packages/metascraper-spotify/test/index.js b/packages/metascraper-spotify/test/index.js
@@ -45,6 +45,16 @@ const spotifyUrls = [
 ]
 
 describe('metascraper-spotify', () => {
+  it('allow to customize keyv options', async () => {
+    const url = 'https://open.spotify.com/playlist/0Lt5S4hGarhtZmtz7BNTeX'
+    const cache = new Map()
+    const metascraper = createMetascraper({ keyvOpts: { store: cache } })
+    await metascraper({ url })
+    await metascraper({ url })
+    await metascraper({ url })
+    should(cache.size > 0).be.true()
+  })
+
   describe('.isvalidUrl', () => {
     describe('true', () => {
       spotifyUrls.forEach(url => {
@@ -53,6 +63,7 @@ describe('metascraper-spotify', () => {
         })
       })
     })
+
     describe('false', () => {
       ;[
         'https://soundcloud.com/beautybrainsp/beauty-brain-swag-bandicoot'
@@ -73,14 +84,4 @@ describe('metascraper-spotify', () => {
       })
     })
   })
-
-  it('allow to customise keyv options', async () => {
-    const url = 'https://open.spotify.com/playlist/0Lt5S4hGarhtZmtz7BNTeX'
-    const cache = new Map()
-    const metascraper = createMetascraper({ keyvOpts: { store: cache } })
-    await metascraper({ url })
-    await metascraper({ url })
-    await metascraper({ url })
-    should(cache.size > 0).be.true()
-  })
 })
diff --git a/packages/metascraper-video/index.js b/packages/metascraper-video/index.js
@@ -9,7 +9,6 @@ const {
   video
 } = require('@metascraper/helpers')
 
-const reachableUrl = require('reachable-url')
 const memoize = require('@keyvhq/memoize')
 const pReflect = require('p-reflect')
 const { chain } = require('lodash')
@@ -43,16 +42,11 @@ const videoRules = [
 
 const createGetPlayer = ({ gotOpts, keyvOpts }) => {
   const getPlayer = async playerUrl => {
-    const response = await reachableUrl(playerUrl, gotOpts)
-    if (!reachableUrl.isReachable(response)) return
+    const { value: response } = await pReflect(got(playerUrl, gotOpts))
     const contentType = response.headers['content-type']
     if (!contentType || !contentType.startsWith('text')) return
-    const { value: html } = await pReflect(
-      got(playerUrl, { resolveBodyOnly: true, ...gotOpts })
-    )
-    return html
+    return response.body
   }
-
   return memoize(getPlayer, keyvOpts)
 }
 

diff --git a/packages/metascraper-video/package.json b/packages/metascraper-video/package.json
@@ -27,8 +27,7 @@
     "cheerio": "~1.0.0-rc.10",
     "got": "~11.8.3",
     "lodash": "~4.17.21",
-    "p-reflect": "~2.1.0",
-    "reachable-url": "~1.6.7"
+    "p-reflect": "~2.1.0"
   },
   "devDependencies": {
     "mocha": "latest",

diff --git a/packages/metascraper-video/test/index.js b/packages/metascraper-video/test/index.js
@@ -10,7 +10,7 @@ const createMetascraper = (...args) =>
   require('metascraper')([require('..')(...args)])
 
 describe('metascraper-video', () => {
-  it('allow to customize keyOpts', async () => {
+  it('allow to customize keyv options', async () => {
     const cache = new Map()
     const html =
       '<meta name="twitter:player" content="https://twitter-card-player.vercel.app/container.html">'
@@ -32,6 +32,7 @@ describe('metascraper-video', () => {
       snapshot(metadata)
     })
   })
+
   describe('video', () => {
     describe('<video />', () => {
       it('source:src', async () => {
@@ -44,6 +45,7 @@ describe('metascraper-video', () => {
         const metadata = await metascraper({ html, url })
         snapshot(metadata)
       })
+
       it('multiple source:src', async () => {
         const html = `
         <video controls>
@@ -59,6 +61,7 @@ describe('metascraper-video', () => {
         const metadata = await metascraper({ html, url })
         snapshot(metadata)
       })
+
       it('multiple source:src with no valid video values', async () => {
         const html = await readFile(
           resolve(__dirname, 'fixtures/providers/bluecadet.com.html')
@@ -69,6 +72,7 @@ describe('metascraper-video', () => {
         snapshot(metadata)
       })
     })
+
     it('<source src />', async () => {
       const html = await readFile(
         resolve(__dirname, 'fixtures/source-src.html')
@@ -78,13 +82,15 @@ describe('metascraper-video', () => {
       const metadata = await metascraper({ html, url })
       snapshot(metadata)
     })
+
     it('og:video', async () => {
       const html = await readFile(resolve(__dirname, 'fixtures/tweet.html'))
       const url = 'https://twitter.com/_developit/status/955905369242513414'
       const metascraper = createMetascraper()
       const metadata = await metascraper({ html, url })
       snapshot(metadata)
     })
+
     it('jsonld:contentUrl', async () => {
       const html = `<script type="application/ld+json">
         {"@context":"http://schema.org","@type":"VideoObject","@id":"https://example.com/video.mp4","contentUrl":"https://example.com/video.mp4"}
@@ -94,6 +100,7 @@ describe('metascraper-video', () => {
       const metadata = await metascraper({ html, url })
       snapshot(metadata)
     })
+
     it('twitter:player', async () => {
       const html =
         '<meta name="twitter:player" content="https://twitter-card-player.vercel.app/container.html">'