-
Notifications
You must be signed in to change notification settings - Fork 667
Commit
This commit does not belong to any branch on this repository, and may belong to a fork outside of the repository.
Refactor how the interaction with the debugging protocol is handled while fixing some of the most recent issues found. - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - Fix #1398 Fix #1395 Fix #1403 Fix webhintio/rfcs#44
- Loading branch information
Showing
5 changed files
with
406 additions
and
347 deletions.
There are no files selected for viewing
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
271 changes: 271 additions & 0 deletions
271
packages/utils-debugging-protocol-common/src/RequestResponse.ts
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
| Original file line number | Diff line number | Diff line change |
|---|---|---|
| @@ -0,0 +1,271 @@ | ||
| import { atob } from 'abab'; | ||
| import { Crdp } from 'chrome-remote-debug-protocol'; | ||
|
|
||
| import { getContentTypeData} from 'hint/dist/src/lib/utils/content-type'; | ||
| import { HttpHeaders, Response } from 'hint/dist/src/lib/types'; | ||
| import { debug as d } from 'hint/dist/src/lib/utils/debug'; | ||
| import { Requester } from '@hint/utils-connector-tools/dist/src/requester'; | ||
| import { normalizeHeaders } from '@hint/utils-connector-tools/dist/src/normalize-headers'; | ||
|
|
||
| import { AsyncHTMLElement } from './cdp-async-html'; | ||
|
|
||
| const debug: debug.IDebugger = d(__filename); | ||
|
|
||
| export enum RequestStatus { | ||
| willBeSent = 'willBeSent', | ||
| responseReceived = 'responseReceived', | ||
| loadingFinished = 'loadingFinished', | ||
| loadingFailed = 'loadingFailed' | ||
| } | ||
|
|
||
| export class RequestResponse { | ||
| private _overrideInvalidCert: boolean; | ||
|
|
||
| /** The debugging protocol Network Client used to download the response body */ | ||
| private _network: Crdp.NetworkClient; | ||
|
|
||
| private _requestId: string; | ||
| /** The associated `requestId` for this network request. */ | ||
| public get requestId() { | ||
| return this._requestId; | ||
| } | ||
|
|
||
| private _status: RequestStatus; | ||
| /** The current status of the request. */ | ||
| public get status() { | ||
| return this._status; | ||
| } | ||
|
|
||
| private _hops: string[] = []; | ||
| /** The redirects done to get to the final resource. */ | ||
| public get hops() { | ||
| return this._hops; | ||
| } | ||
|
|
||
| private _originalUrl: string; | ||
| /** The initial URL that started the request process before any redirects. */ | ||
| public get originalUrl() { | ||
| return this._originalUrl; | ||
| } | ||
|
|
||
| /** The final URL after redirects. Could be the same as `originalUrl`. */ | ||
| public get finalUrl(): string { | ||
| // `willBeSent` gets updated with each redirect so this is the final requested url | ||
| return this.willBeSent.request.url; | ||
| } | ||
|
|
||
| private _willBeSent: Crdp.Network.RequestWillBeSentEvent | undefined; | ||
| /** The payload returned by the debugging protocol in the `RequestWillBeSent` event. */ | ||
| public get willBeSent(): Crdp.Network.RequestWillBeSentEvent { | ||
| return this._willBeSent!; | ||
| } | ||
|
|
||
| private _responseReceived: Crdp.Network.ResponseReceivedEvent | undefined; | ||
| /** The payload returned by the debugging protocol in the `ResponseReceived` event. */ | ||
| public get responseReceived() { | ||
| return this._responseReceived; | ||
| } | ||
|
|
||
| private _loadingFinished: Crdp.Network.LoadingFinishedEvent | undefined; | ||
| /** The payload returned by the debugging protocol in the `LoadingFinished` event. */ | ||
| public get loadingFinished() { | ||
| return this._loadingFinished; | ||
| } | ||
|
|
||
| private _loadingFailed: Crdp.Network.LoadingFailedEvent | undefined; | ||
| /** The payload returned by the debugging protocol in the `LoadingFailed` event. */ | ||
| public get loadingFailed() { | ||
| return this._loadingFailed; | ||
| } | ||
|
|
||
| private _responseBody: Crdp.Network.GetResponseBodyResponse | undefined; | ||
| /** The payload returned by calling `getResponseBody` for the request. */ | ||
| public get responseBody() { | ||
| return this._responseBody; | ||
| } | ||
|
|
||
| private fetchContent(href: string, headers: HttpHeaders) { | ||
| const options = { | ||
| headers, | ||
| // we sync the ignore SSL error options with `request`. This is neeeded for local https tests | ||
| rejectUnauthorized: !this._overrideInvalidCert, | ||
| strictSSL: !this._overrideInvalidCert | ||
| }; | ||
|
|
||
| const request: Requester = new Requester(options); | ||
|
|
||
| return request.get(href); | ||
| } | ||
|
|
||
| private _rawResponse: Buffer | undefined; | ||
| private getRawResponse(): Promise<Buffer> { | ||
| const that = this; | ||
|
|
||
| if (this._rawResponse) { | ||
| return Promise.resolve(this._rawResponse); | ||
| } | ||
|
|
||
| const { rawContent } = this._response!.body; | ||
|
|
||
| if (rawContent && rawContent.length.toString() === this._responseReceived!.response.headers['Content-Length']) { | ||
| // Response wasn't compressed so both buffers are the same | ||
| return Promise.resolve(rawContent); | ||
| } | ||
|
|
||
| const { url: responseUrl, requestHeaders: headers } = this._responseReceived!.response; | ||
|
|
||
| /* | ||
| * Real browser connectors automatically request using HTTP2. This spec has | ||
| * [`Pseudo-Header Fields`](https://tools.ietf.org/html/rfc7540#section-8.1.2.3): | ||
| * `:authority`, `:method`, `:path` and `:scheme`. | ||
| * | ||
| * An example of request with those `Pseudo-Header Fields` to google.com: | ||
| * | ||
| * ``` | ||
| * :authority:www.google.com | ||
| * :method:GET | ||
| * :path:/images/branding/googlelogo/2x/googlelogo_color_120x44dp.png | ||
| * :scheme:https | ||
| * accept:text/html,application/xhtml+xml,application/xml;q=0.9,image/webp,image/apng,*\/*;q=0.8 | ||
| * accept-encoding:gzip, deflate, br | ||
| * ... | ||
| * ``` | ||
| * | ||
| * The `request` module doesn't support HTTP2 yet: https://github.com/request/request/issues/2033 | ||
| * so the request need to be transformed to valid HTTP 1.1 ones, basically removing those headers. | ||
| * | ||
| */ | ||
|
|
||
| const validHeaders = Object.entries(headers || {}).reduce((final, [key, value]) => { | ||
| if (key.startsWith(':')) { | ||
| return final; | ||
| } | ||
|
|
||
| final[key] = value; | ||
|
|
||
| return final; | ||
| }, {} as Crdp.Network.Headers); | ||
|
|
||
| return this.fetchContent(responseUrl, validHeaders) | ||
| .then((result) => { | ||
| const { response: { body: { rawResponse: rr } } } = result; | ||
|
|
||
| return rr(); | ||
| }) | ||
| .then((value) => { | ||
| that._rawResponse = value; | ||
|
|
||
| return value; | ||
| }); | ||
| } | ||
|
|
||
| private _response: Response | undefined; | ||
| /** The `Response` associated to this request to be sent on the `fetch::end` event. */ | ||
| public getResponse(element: AsyncHTMLElement | null): Response { | ||
| if (!this._response) { | ||
|
|
||
| const { headers, status } = this.responseReceived!.response; | ||
| const normalizedHeaders = normalizeHeaders(headers); | ||
| const that = this; | ||
|
|
||
| let rawContent = Buffer.alloc(0); | ||
| let rBody = { | ||
| content: '', | ||
| rawContent, | ||
| rawResponse: () => { | ||
| return Promise.resolve(Buffer.alloc(0)); | ||
| } | ||
| }; | ||
|
|
||
|
|
||
| if (this._responseBody) { | ||
| const { body, base64Encoded } = this._responseBody; | ||
| const encoding = base64Encoded ? 'base64' : 'utf-8'; | ||
| const content = base64Encoded ? atob(body) : body; // There are some JS responses that are base64 encoded for some reason | ||
|
|
||
| rawContent = Buffer.from(body, encoding); | ||
|
|
||
| rBody = { | ||
| content, | ||
| rawContent, | ||
| rawResponse: () => { | ||
| return that.getRawResponse(); | ||
| } | ||
| }; | ||
| } | ||
|
|
||
| this._response = { | ||
| body: rBody, | ||
| charset: null!, | ||
| headers: normalizedHeaders!, | ||
| hops: this.hops, | ||
| mediaType: null!, | ||
| statusCode: status, | ||
| url: this.finalUrl | ||
| }; | ||
|
|
||
| const { charset, mediaType } = getContentTypeData(element, this.originalUrl, normalizedHeaders, rawContent); | ||
|
|
||
| this._response.mediaType = mediaType!; | ||
| this._response.charset = charset!; | ||
| } | ||
|
|
||
| return this._response; | ||
| } | ||
|
|
||
| private logInfo(message: string) { | ||
| debug(`(${this._requestId}) ${message}`); | ||
| } | ||
|
|
||
| /** | ||
| * Update the payload received on the `RequestWillBeSent` event for this `requestId`, | ||
| * taking care of updating other related data if there are any redirects. | ||
| */ | ||
| public updateRequestWillBeSent(event: Crdp.Network.RequestWillBeSentEvent) { | ||
| if (event.redirectResponse) { | ||
| this._hops.push(event.redirectResponse.url); | ||
| } | ||
|
|
||
| this._willBeSent = event; | ||
| this.logInfo(RequestStatus.willBeSent); | ||
| } | ||
|
|
||
| /** Update the payload received on the `ResponseReceived` event for this `requestId`. */ | ||
| public updateResponseReceived(event: Crdp.Network.ResponseReceivedEvent) { | ||
| this._responseReceived = event; | ||
| this._status = RequestStatus.responseReceived; | ||
|
|
||
| this.logInfo(RequestStatus.responseReceived); | ||
| } | ||
|
|
||
| /** Update the payload received on the `LoadingFinished` event for this `requestId`. */ | ||
| public async updateLoadingFinished(event: Crdp.Network.LoadingFinishedEvent) { | ||
| this._loadingFinished = event; | ||
| try { | ||
| this._responseBody = await this._network.getResponseBody!({ requestId: event.requestId }); | ||
| } catch (e) { | ||
| this.logInfo(`Error getting body`); | ||
| } | ||
|
|
||
| this._status = RequestStatus.loadingFinished; | ||
|
|
||
| this.logInfo(RequestStatus.loadingFinished); | ||
| } | ||
|
|
||
| /** Update the payload received on the `LoadingFailed` event for this `requestId`. */ | ||
| public updateLoadingFailed(event: Crdp.Network.LoadingFailedEvent) { | ||
| this._loadingFailed = event; | ||
|
|
||
| this._status = RequestStatus.loadingFailed; | ||
| this.logInfo(RequestStatus.loadingFailed); | ||
| } | ||
|
|
||
| public constructor(network: Crdp.NetworkClient, event: Crdp.Network.RequestWillBeSentEvent, overrideInvalidCert: boolean) { | ||
| this._network = network; | ||
| this._status = RequestStatus.willBeSent; | ||
| this._willBeSent = event; | ||
| this._requestId = event.requestId; | ||
| this._originalUrl = event.request.url; | ||
| this._overrideInvalidCert = overrideInvalidCert; | ||
| } | ||
| } |
Oops, something went wrong.