refactor(readability): avoid worker_threads

packages/metascraper-readability/src/index.js

@@ -1,22 +1,44 @@
 'use strict'
 
 const { memoizeOne, composeRule } = require('@metascraper/helpers')
+const { Readability } = require('@mozilla/readability')
 const asyncMemoizeOne = require('async-memoize-one')
-const { Worker } = require('worker_threads')
-const path = require('path')
+const { Browser } = require('happy-dom')
 
-const SCRIPT_PATH = path.resolve(__dirname, 'worker.js')
+const parseReader = reader => {
+  const parsed = reader.parse()
+  return parsed || {}
+}
 
-const readability = asyncMemoizeOne((url, html, readabilityOpts) => {
-  const worker = new Worker(SCRIPT_PATH, {
-    workerData: { url, html, readabilityOpts },
-    stdout: true,
-    stderr: true
+const getDocument = ({ url, html }) => {
+  const browser = new Browser({
+    settings: {
+      disableComputedStyleRendering: true,
+      disableCSSFileLoading: true,
+      disableIframePageLoading: true,
+      disableJavaScriptEvaluation: true,
+      disableJavaScriptFileLoading: true
+    }
   })
-  const { promise, resolve, reject } = Promise.withResolvers()
-  worker.on('message', message => resolve(JSON.parse(message)))
-  worker.on('error', reject)
-  return promise
+
+  const page = browser.newPage()
+  page.url = url
+  page.content = html
+  return {
+    document: page.mainFrame.document,
+    teardown: () => browser.close()
+  }
+}
+
+const readability = asyncMemoizeOne(async (url, html, readabilityOpts) => {
+  const { document, teardown } = getDocument({ url, html })
+  try {
+    const reader = new Readability(document, readabilityOpts)
+    const result = parseReader(reader)
+    return result
+  } finally {
+    await teardown()
+  }
 }, memoizeOne.EqualityFirstArgument)
 
 module.exports = ({ readabilityOpts } = {}) => {

packages/metascraper/test/integration/atlasobscura/index.js

@@ -21,7 +21,7 @@ const metascraper = require('../../..')([
   require('metascraper-readability')()
 ])
 
-const url = 'http://www.atlasobscura.com/articles/ikea-bowl-blanda-blank-fire'
+const url = 'https://www.atlasobscura.com/articles/ikea-bowl-blanda-blank-fire'
 
 test('atlasobscura', async t => {
   const html = await readFile(resolve(__dirname, 'input.html'))

packages/metascraper/test/integration/atlasobscura/snapshots/index.js.md

@@ -11,11 +11,11 @@ Generated by [AVA](https://avajs.dev).
     {
       audio: null,
       author: 'Cara Giaimo',
-      date: '2017-06-22T20:16:06.000Z',
+      date: '2025-08-07T13:37:22.000Z',
       description: 'The innocent-looking “Blanda blank” is a pyromaniac.',
       image: 'https://img.atlasobscura.com/F7MQkybuKDEHefjywNqMPp7XRlY_rmrLGo4zMtlPo5E/rt:fit/w:600/c:1148:765:nowe:17:199/q:81/sm:1/scp:1/ar:1/aHR0cHM6Ly9hdGxh/cy1kZXYuczMuYW1h/em9uYXdzLmNvbS91/cGxvYWRzL2Fzc2V0/cy9kN2YwODk2ZGI4/MDYyODhmZjdfU2Ny/ZWVuIFNob3QgMjAx/Ny0wNi0yMSBhdCA5/LjM2LjQ0IEFNLnBu/Zw.png',
       lang: 'en',
-      logo: 'https://img.atlasobscura.com/jthYPtckohvAoDRucK2XgRa0uaiuPPDLSMiSzYOdNWU/q:81/sm:1/scp:1/aHR0cHM6Ly9zMy5h/bWF6b25hd3MuY29t/L2F0bGFzLWRldi9t/aXNjL2ljb25zL2Fw/cGxlLXRvdWNoLWlj/b24tMTQ0eDE0NC1w/cmVjb21wb3NlZC5w/bmc',
+      logo: 'https://s3.amazonaws.com/atlas-dev/misc/icons/favicon.ico',
       publisher: 'Atlas Obscura',
       title: 'This Ikea Bowl Has Been Setting Things on Fire',
       url: 'http://www.atlasobscura.com/articles/ikea-bowl-blanda-blank-fire',