Merge bde8630 into a79aa7f

extractus · Mar 28, 2023 · b69462f · b69462f
2 parents a79aa7f + bde8630
commit b69462f
Show file tree

Hide file tree

Showing 7 changed files with 39 additions and 37 deletions.
diff --git a/README.md b/README.md
@@ -160,7 +160,7 @@ For example:
 import { extract } from '@extractus/article-extractor'
 
 const url = 'https://www.cnbc.com/2022/09/21/what-another-major-rate-hike-by-the-federal-reserve-means-to-you.html'
-const article = await extract(url, null, {
+const article = await extract(url, {}, {
   headers: {
     'user-agent': 'Opera/9.60 (Windows NT 6.0; U; en) Presto/2.1.1'
   }
@@ -178,7 +178,7 @@ import { extract } from '@extractus/article-extractor'
 
 const url = 'https://www.cnbc.com/2022/09/21/what-another-major-rate-hike-by-the-federal-reserve-means-to-you.html'
 
-await extract(url, null, {
+await extract(url, {}, {
   headers: {
     'user-agent': 'Opera/9.60 (Windows NT 6.0; U; en) Presto/2.1.1'
   },

diff --git a/dist/article-extractor.esm.js b/dist/article-extractor.esm.js
diff --git a/dist/cjs/article-extractor.js b/dist/cjs/article-extractor.js
diff --git a/dist/cjs/package.json b/dist/cjs/package.json
@@ -1,5 +1,5 @@
 {
   "name": "@extractus/article-extractor",
-  "version": "7.2.11",
+  "version": "7.2.12",
   "main": "./article-extractor.js"
 }
diff --git a/package.json b/package.json
@@ -1,5 +1,5 @@
 {
-  "version": "7.2.11",
+  "version": "7.2.12",
   "name": "@extractus/article-extractor",
   "description": "To extract main article from given URL",
   "homepage": "https://github.com/extractus/article-extractor",
@@ -34,16 +34,16 @@
     "reset": "node reset"
   },
   "dependencies": {
-    "@mozilla/readability": "^0.4.2",
+    "@mozilla/readability": "^0.4.3",
     "bellajs": "^11.1.2",
     "cross-fetch": "^3.1.5",
-    "linkedom": "^0.14.24",
+    "linkedom": "^0.14.25",
     "sanitize-html": "2.10.0",
     "string-similarity": "^4.0.4"
   },
   "devDependencies": {
-    "@types/sanitize-html": "^2.8.1",
-    "esbuild": "^0.17.11",
+    "@types/sanitize-html": "^2.9.0",
+    "esbuild": "^0.17.14",
     "eslint": "^8.36.0",
     "jest": "^29.5.0",
     "nock": "^13.3.0"

diff --git a/src/main.js b/src/main.js
@@ -14,14 +14,14 @@ export const extract = async (input, parserOptions = {}, fetchOptions = {}) => {
   }
 
   if (!isValidUrl(input)) {
-    return parseFromHtml(input, null, parserOptions)
+    return parseFromHtml(input, null, parserOptions || {})
   }
   const html = await retrieve(input, fetchOptions)
   if (!html) {
     return null
   }
 
-  return parseFromHtml(html, input, parserOptions)
+  return parseFromHtml(html, input, parserOptions || {})
 }
 
 export const extractFromHtml = async (html, url, parserOptions = {}) => {

diff --git a/src/utils/retrieve.js b/src/utils/retrieve.js
@@ -15,7 +15,9 @@ const profetch = async (url, proxy = {}) => {
 
 export default async (url, options = {}) => {
   const {
-    headers = {},
+    headers = {
+      'user-agent': 'Mozilla/5.0 (X11; Linux x86_64; rv:109.0) Gecko/20100101 Firefox/111.0',
+    },
     proxy = null,
   } = options