-
Notifications
You must be signed in to change notification settings - Fork 44
Commit
This commit does not belong to any branch on this repository, and may belong to a fork outside of the repository.
Add throttling logic per origin, skip SVG resources (#1356)
* Add throttling logic per origin This adds a generic throttled queue per origin class that replaces the previous `throttle` function to add serialization (and sleep interval) per origin. The code is inspired by: https://github.com/w3c/cg-monitor/blob/5ceea24a1eaa7c1736d38909bcbb47df4ae297a3/lib/caching-queued-fetch.js#L68 (Note plan is to reuse the same class in Reffy) * Intercept and skip network requests on SVG files SVG files were not intercepted in Reffy because a couple of specs were using a clunky dynamic PNG fallback mechanism that created an infinite loop. These specs were fixed some time ago, so I don't think that's still an issue and, for some reason, the drafts CSS server takes a lot of time to serve SVG files linked from specs. --------- Co-authored-by: Dominique Hazael-Massieux <dom@w3.org>
- Loading branch information
1 parent
1c6f61e
commit 7e3dc35
Showing
5 changed files
with
137 additions
and
48 deletions.
There are no files selected for viewing
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
This file was deleted.
Oops, something went wrong.
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
| Original file line number | Diff line number | Diff line change |
|---|---|---|
| @@ -0,0 +1,120 @@ | ||
| /** | ||
| * Helper function to sleep for a specified number of milliseconds | ||
| */ | ||
| function sleep(ms) { | ||
| return new Promise(resolve => setTimeout(resolve, ms, 'slept')); | ||
| } | ||
|
|
||
|
|
||
| /** | ||
| * Helper function that returns the "origin" of a URL, defined in a loose way | ||
| * as the part of the true origin that identifies the server that's going to | ||
| * serve the resource. | ||
| * | ||
| * For example "github.io" for all specs under github.io, "whatwg.org" for | ||
| * all WHATWG specs, "csswg.org" for CSS specs at large (including Houdini | ||
| * and FXTF specs since they are served by the same server). | ||
| */ | ||
| function getOrigin(url) { | ||
| if (!url) { | ||
| return ''; | ||
| } | ||
| const origin = (new URL(url)).origin; | ||
| if (origin.endsWith('.whatwg.org')) { | ||
| return 'whatwg.org'; | ||
| } | ||
| else if (origin.endsWith('.github.io')) { | ||
| return 'github.io'; | ||
| } | ||
| else if (origin.endsWith('.csswg.org') || | ||
| origin.endsWith('.css-houdini.org') || | ||
| origin.endsWith('.fxtf.org')) { | ||
| return 'csswg.org'; | ||
| } | ||
| else { | ||
| return origin; | ||
| } | ||
| } | ||
|
|
||
|
|
||
| /** | ||
| * The ThrottledQueue class can be used to run a series of tasks that send | ||
| * network requests to an origin server in parallel, up to a certain limit, | ||
| * while guaranteeing that only one request will be sent to a given origin | ||
| * server at a time. | ||
| */ | ||
| module.exports = class ThrottledQueue { | ||
| originQueue = {}; | ||
| maxParallel = 4; | ||
| ongoing = 0; | ||
| pending = []; | ||
|
|
||
| constructor(maxParallel) { | ||
| if (maxParallel >= 0) { | ||
| this.maxParallel = maxParallel; | ||
| } | ||
| } | ||
|
|
||
| /** | ||
| * Run the given processing function with the given parameters, immediately | ||
| * if possible or as soon as possible when too many tasks are already running | ||
| * in parallel. | ||
| * | ||
| * Note this function has no notion of origin. Users may call the function | ||
| * directly if they don't need any throttling per origin. | ||
| */ | ||
| async runThrottled(processFunction, ...params) { | ||
| if (this.ongoing >= this.maxParallel) { | ||
| return new Promise((resolve, reject) => { | ||
| this.pending.push({ params, resolve, reject }); | ||
| }); | ||
| } | ||
| else { | ||
| this.ongoing += 1; | ||
| const result = await processFunction.call(null, ...params); | ||
| this.ongoing -= 1; | ||
|
|
||
| // Done with current task, trigger next pending task in the background | ||
| setTimeout(_ => { | ||
| if (this.pending.length && this.ongoing < this.maxParallel) { | ||
| const next = this.pending.shift(); | ||
| this.runThrottled(processFunction, ...next.params) | ||
| .then(result => next.resolve(result)) | ||
| .catch(err => next.reject(err)); | ||
| } | ||
| }, 0); | ||
|
|
||
| return result; | ||
| } | ||
| } | ||
|
|
||
| /** | ||
| * Run the given processing function with the given parameters, immediately | ||
| * if possible or as soon as possible when too many tasks are already running | ||
| * in parallel, or when there's already a task being run against the same | ||
| * origin as that of the provided URL. | ||
| * | ||
| * Said differently, the function serializes tasks per origin, and calls | ||
| * "runThrottled" to restrict the number of tasks that run in parallel to the | ||
| * requested maximum. | ||
| * | ||
| * Additionally, the function forces a 2 second sleep after processing to | ||
| * keep a low network profile. | ||
| */ | ||
| async runThrottledPerOrigin(url, processFunction, ...params) { | ||
| const origin = getOrigin(url); | ||
| if (!this.originQueue[origin]) { | ||
| this.originQueue[origin] = Promise.resolve(true); | ||
| } | ||
| return new Promise((resolve, reject) => { | ||
| this.originQueue[origin] = this.originQueue[origin] | ||
| .then(async _ => this.runThrottled(processFunction, ...params)) | ||
| .then(async result => { | ||
| await sleep(2000); | ||
| return result; | ||
| }) | ||
| .then(resolve) | ||
| .catch(reject); | ||
| }); | ||
| } | ||
| } |