Merge 0e39547 into e4065e0

extractus · Jul 1, 2023 · b279218 · b279218
2 parents e4065e0 + 0e39547
commit b279218
Show file tree

Hide file tree

Showing 8 changed files with 19 additions and 6 deletions.
diff --git a/.github/workflows/ci-test.yml b/.github/workflows/ci-test.yml
@@ -8,7 +8,7 @@ on: [push, pull_request]
 jobs:
   test:
 
-    runs-on: ubuntu-22.04
+    runs-on: ubuntu-latest
 
     strategy:
       matrix:

diff --git a/README.md b/README.md
@@ -208,6 +208,8 @@ console.log('Run article-extractor with proxy:', proxy)
 console.log(art)
 ```
 
+For more info about [https-proxy-agent](https://www.npmjs.com/package/https-proxy-agent), check [its repo](https://github.com/TooTallNate/proxy-agents).
+
 For more info about proxy authentication, please refer [HTTP authentication](https://developer.mozilla.org/en-US/docs/Web/HTTP/Authentication)
 
 For a deeper customization, you can consider using [Proxy](https://developer.mozilla.org/en-US/docs/Web/JavaScript/Reference/Global_Objects/Proxy) to replace `fetch` behaviors with your own handlers.

diff --git a/dist/article-extractor.esm.js b/dist/article-extractor.esm.js
diff --git a/dist/cjs/article-extractor.js b/dist/cjs/article-extractor.js
diff --git a/dist/cjs/index.d.ts b/dist/cjs/index.d.ts
@@ -57,6 +57,12 @@ export interface FetchOptions {
    * default: null
    */
   proxy?: ProxyConfig;
+
+  /**
+   * http proxy agent
+   * default: null
+   */
+  agent?: object;
 }
 
 export interface ArticleData {

diff --git a/index.d.ts b/index.d.ts
@@ -57,6 +57,12 @@ export interface FetchOptions {
    * default: null
    */
   proxy?: ProxyConfig;
+
+  /**
+   * http proxy agent
+   * default: null
+   */
+  agent?: object;
 }
 
 export interface ArticleData {

diff --git a/package.json b/package.json
@@ -41,7 +41,6 @@
     "@types/sanitize-html": "^2.9.0",
     "esbuild": "^0.18.10",
     "eslint": "^8.44.0",
-    "https-proxy-agent": "^7.0.0",
     "jest": "^29.5.0",
     "nock": "^13.3.1"
   },

diff --git a/src/utils/extractWithReadability.js b/src/utils/extractWithReadability.js
@@ -4,13 +4,13 @@ import { Readability } from '@mozilla/readability'
 import { DOMParser } from 'linkedom'
 import { isString } from 'bellajs'
 
-export default (html, inputUrl = '') => {
+export default (html, url = '') => {
   if (!isString(html)) {
     return null
   }
   const doc = new DOMParser().parseFromString(html, 'text/html')
   const base = doc.createElement('base')
-  base.setAttribute('href', inputUrl)
+  base.setAttribute('href', url)
   doc.head.appendChild(base)
   const reader = new Readability(doc, {
     keepClasses: true,