Skip to content

Commit

Permalink
fix: Ne pas télécharger les fichiers audio / vidéos.
Browse files Browse the repository at this point in the history
  • Loading branch information
regseb committed Mar 30, 2020
1 parent 8a82ee6 commit 449cc0e
Show file tree
Hide file tree
Showing 4 changed files with 32 additions and 9 deletions.
15 changes: 15 additions & 0 deletions package-lock.json

Some generated files are not rendered by default. Learn more about how customized files appear on GitHub.

1 change: 1 addition & 0 deletions package.json
Original file line number Diff line number Diff line change
Expand Up @@ -37,6 +37,7 @@
"clean": "rm -rf .nyc_output/ build/ coverage/ jsdocs/ node_modules/"
},
"devDependencies": {
"abort-controller": "3.0.0",
"addons-linter": "^1.22.0",
"david": "^12.0.0",
"eslint": "^6.8.0",
Expand Down
7 changes: 6 additions & 1 deletion src/core/scrapers.js
Original file line number Diff line number Diff line change
Expand Up @@ -116,14 +116,19 @@ export const extract = async function (url, options) {
const content = {
html: cacheable(async () => {
try {
const response = await fetch(url.href);
const controller = new AbortController();
const response = await fetch(url.href, {
signal: controller.signal,
});
const contentType = response.headers.get("Content-Type");
if (null !== contentType &&
(contentType.startsWith("text/html") ||
contentType.startsWith("application/xhtml+xml"))) {
const text = await response.text();
return new DOMParser().parseFromString(text, "text/html");
}
// Si ce n'est pas du HTML : annuler la requête.
controller.abort();
} catch {
// Ignorer le cas où l'URL n'est pas accessible.
}
Expand Down
18 changes: 10 additions & 8 deletions test/polyfill.js
Original file line number Diff line number Diff line change
@@ -1,9 +1,11 @@
import { URL } from "url";
import { JSDOM } from "jsdom";
import { browser } from "./polyfill/browser.js";
import { fetch } from "./polyfill/fetch.js";
import { URL } from "url";
import AbortController from "abort-controller";
import { JSDOM } from "jsdom";
import { browser } from "./polyfill/browser.js";
import { fetch } from "./polyfill/fetch.js";

globalThis.URL = URL;
globalThis.DOMParser = new JSDOM().window.DOMParser;
globalThis.browser = browser;
globalThis.fetch = fetch;
globalThis.URL = URL;
globalThis.AbortController = AbortController;
globalThis.DOMParser = new JSDOM().window.DOMParser;
globalThis.browser = browser;
globalThis.fetch = fetch;

0 comments on commit 449cc0e

Please sign in to comment.