From bad343923bcf3a051fc3a85dd61eb5116b13307a Mon Sep 17 00:00:00 2001 From: Max Schmitt Date: Wed, 22 Feb 2023 16:17:46 +0100 Subject: [PATCH] chore: consolidate http/https fetching --- .../src/server/chromium/chromium.ts | 27 +++++-------------- packages/playwright-core/src/server/fetch.ts | 4 +-- .../playwright-core/src/server/transport.ts | 2 +- .../src/{server => utils}/happy-eyeballs.ts | 5 ++-- packages/playwright-core/src/utils/network.ts | 14 +++++++--- .../src/plugins/webServerPlugin.ts | 18 +++++-------- 6 files changed, 30 insertions(+), 40 deletions(-) rename packages/playwright-core/src/{server => utils}/happy-eyeballs.ts (96%) diff --git a/packages/playwright-core/src/server/chromium/chromium.ts b/packages/playwright-core/src/server/chromium/chromium.ts index f8c5b8675e326..af72f730feeaf 100644 --- a/packages/playwright-core/src/server/chromium/chromium.ts +++ b/packages/playwright-core/src/server/chromium/chromium.ts @@ -33,11 +33,10 @@ import { Browser } from '../browser'; import type * as types from '../types'; import type * as channels from '@protocol/channels'; import type { HTTPRequestParams } from '../../utils/network'; -import { NET_DEFAULT_TIMEOUT } from '../../utils/network'; import { fetchData } from '../../utils/network'; import { getUserAgent } from '../../utils/userAgent'; import { wrapInASCIIBox } from '../../utils/ascii'; -import { debugMode, headersArrayToObject, } from '../../utils'; +import { debugMode, headersArrayToObject, } from '../../utils'; import { removeFolders } from '../../utils/fileUtils'; import { RecentLogsCollector } from '../../common/debugLogger'; import type { Progress } from '../progress'; @@ -45,13 +44,11 @@ import { ProgressController } from '../progress'; import { TimeoutSettings } from '../../common/timeoutSettings'; import { helper } from '../helper'; import type { CallMetadata } from '../instrumentation'; -import http from 'http'; -import https from 'https'; +import type http from 'http'; import { registry } from '../registry'; import { ManualPromise } from '../../utils/manualPromise'; import { validateBrowserContextOptions } from '../browserContext'; import { chromiumSwitches } from './chromiumSwitches'; -import { httpHappyEyeballsAgent, httpsHappyEyeballsAgent } from '../happy-eyeballs'; const ARTIFACTS_FOLDER = path.join(os.tmpdir(), 'playwright-artifacts-'); @@ -338,21 +335,11 @@ async function urlToWSEndpoint(progress: Progress, endpointURL: string) { return endpointURL; progress.log(` retrieving websocket url from ${endpointURL}`); const httpURL = endpointURL.endsWith('/') ? `${endpointURL}json/version/` : `${endpointURL}/json/version/`; - const isHTTPS = endpointURL.startsWith('https://'); - const json = await new Promise((resolve, reject) => { - (isHTTPS ? https : http).get(httpURL, { - timeout: NET_DEFAULT_TIMEOUT, - agent: isHTTPS ? httpsHappyEyeballsAgent : httpHappyEyeballsAgent, - }, resp => { - if (resp.statusCode! < 200 || resp.statusCode! >= 400) { - reject(new Error(`Unexpected status ${resp.statusCode} when connecting to ${httpURL}.\n` + - `This does not look like a DevTools server, try connecting via ws://.`)); - } - let data = ''; - resp.on('data', chunk => data += chunk); - resp.on('end', () => resolve(data)); - }).on('error', reject); - }); + const json = await fetchData({ + url: httpURL, + }, async (_, resp) => new Error(`Unexpected status ${resp.statusCode} when connecting to ${httpURL}.\n` + + `This does not look like a DevTools server, try connecting via ws://.`) + ); return JSON.parse(json).webSocketDebuggerUrl; } diff --git a/packages/playwright-core/src/server/fetch.ts b/packages/playwright-core/src/server/fetch.ts index ae04287919ec1..29f4893f7a80d 100644 --- a/packages/playwright-core/src/server/fetch.ts +++ b/packages/playwright-core/src/server/fetch.ts @@ -30,7 +30,7 @@ import { HttpsProxyAgent, SocksProxyAgent } from '../utilsBundle'; import { BrowserContext } from './browserContext'; import { CookieStore, domainMatches } from './cookieStore'; import { MultipartFormData } from './formData'; -import { httpHappyEyeballsAgent, httpsHappyEyeballsAgent } from './happy-eyeballs'; +import { httpHappyEyeballsAgent, httpsHappyEyeballsAgent } from '../utils/happy-eyeballs'; import type { CallMetadata } from './instrumentation'; import { SdkObject } from './instrumentation'; import type { Playwright } from './playwright'; @@ -69,7 +69,7 @@ export type APIRequestFinishedEvent = { body?: Buffer; }; -export type SendRequestOptions = https.RequestOptions & { +type SendRequestOptions = https.RequestOptions & { maxRedirects: number, deadline: number, __testHookLookup?: (hostname: string) => LookupAddress[] diff --git a/packages/playwright-core/src/server/transport.ts b/packages/playwright-core/src/server/transport.ts index e20b1ec085d78..b6222386e64a8 100644 --- a/packages/playwright-core/src/server/transport.ts +++ b/packages/playwright-core/src/server/transport.ts @@ -20,7 +20,7 @@ import type { WebSocket } from '../utilsBundle'; import type { ClientRequest, IncomingMessage } from 'http'; import type { Progress } from './progress'; import { makeWaitForNextTask } from '../utils'; -import { httpHappyEyeballsAgent, httpsHappyEyeballsAgent } from './happy-eyeballs'; +import { httpHappyEyeballsAgent, httpsHappyEyeballsAgent } from '../utils/happy-eyeballs'; export type ProtocolRequest = { id: number; diff --git a/packages/playwright-core/src/server/happy-eyeballs.ts b/packages/playwright-core/src/utils/happy-eyeballs.ts similarity index 96% rename from packages/playwright-core/src/server/happy-eyeballs.ts rename to packages/playwright-core/src/utils/happy-eyeballs.ts index 4b80c010a4123..e17d6a70262c0 100644 --- a/packages/playwright-core/src/server/happy-eyeballs.ts +++ b/packages/playwright-core/src/utils/happy-eyeballs.ts @@ -19,8 +19,7 @@ import * as http from 'http'; import * as https from 'https'; import * as net from 'net'; import * as tls from 'tls'; -import { ManualPromise } from '../utils/manualPromise'; -import type { SendRequestOptions } from './fetch'; +import { ManualPromise } from './manualPromise'; // Implementation(partial) of Happy Eyeballs 2 algorithm described in // https://www.rfc-editor.org/rfc/rfc8305 @@ -50,7 +49,7 @@ export const httpsHappyEyeballsAgent = new HttpsHappyEyeballsAgent(); export const httpHappyEyeballsAgent = new HttpHappyEyeballsAgent(); async function createConnectionAsync(options: http.ClientRequestArgs, oncreate: ((err: Error | null, socket?: net.Socket) => void) | undefined, useTLS: boolean) { - const lookup = (options as SendRequestOptions).__testHookLookup || lookupAddresses; + const lookup = (options as any).__testHookLookup || lookupAddresses; const hostname = clientRequestArgsToHostName(options); const addresses = await lookup(hostname); const sockets = new Set(); diff --git a/packages/playwright-core/src/utils/network.ts b/packages/playwright-core/src/utils/network.ts index 1bb8ede2978a7..1a85f88a75367 100644 --- a/packages/playwright-core/src/utils/network.ts +++ b/packages/playwright-core/src/utils/network.ts @@ -24,6 +24,7 @@ import * as URL from 'url'; import type { URLMatch } from '../common/types'; import { isString, isRegExp } from './rtti'; import { globToRegex } from './glob'; +import { httpHappyEyeballsAgent, httpsHappyEyeballsAgent } from './happy-eyeballs'; export async function createSocket(host: string, port: number): Promise { return new Promise((resolve, reject) => { @@ -39,15 +40,22 @@ export type HTTPRequestParams = { headers?: http.OutgoingHttpHeaders, data?: string | Buffer, timeout?: number, + rejectUnauthorized?: boolean, }; export const NET_DEFAULT_TIMEOUT = 30_000; export function httpRequest(params: HTTPRequestParams, onResponse: (r: http.IncomingMessage) => void, onError: (error: Error) => void) { const parsedUrl = URL.parse(params.url); - let options: https.RequestOptions = { ...parsedUrl }; - options.method = params.method || 'GET'; - options.headers = params.headers; + let options: https.RequestOptions = { + ...parsedUrl, + agent: parsedUrl.protocol === 'https:' ? httpsHappyEyeballsAgent : httpHappyEyeballsAgent, + method: params.method || 'GET', + headers: params.headers, + }; + if (params.rejectUnauthorized !== undefined) + options.rejectUnauthorized = params.rejectUnauthorized; + const timeout = params.timeout ?? NET_DEFAULT_TIMEOUT; const proxyURL = getProxyForUrl(params.url); diff --git a/packages/playwright-test/src/plugins/webServerPlugin.ts b/packages/playwright-test/src/plugins/webServerPlugin.ts index 60bee7e54e1ff..8ca2dccc13b1c 100644 --- a/packages/playwright-test/src/plugins/webServerPlugin.ts +++ b/packages/playwright-test/src/plugins/webServerPlugin.ts @@ -13,13 +13,11 @@ * See the License for the specific language governing permissions and * limitations under the License. */ -import http from 'http'; -import https from 'https'; import path from 'path'; import net from 'net'; import { debug } from 'playwright-core/lib/utilsBundle'; -import { raceAgainstTimeout, launchProcess } from 'playwright-core/lib/utils'; +import { raceAgainstTimeout, launchProcess, httpRequest } from 'playwright-core/lib/utils'; import type { FullConfig, Reporter } from '../../types/testReporter'; import type { TestRunnerPlugin } from '.'; @@ -159,20 +157,18 @@ async function isURLAvailable(url: URL, ignoreHTTPSErrors: boolean, onStdErr: Re } async function httpStatusCode(url: URL, ignoreHTTPSErrors: boolean, onStdErr: Reporter['onStdErr']): Promise { - const commonRequestOptions = { headers: { Accept: '*/*' } }; - const isHttps = url.protocol === 'https:'; - const requestOptions = isHttps ? { - ...commonRequestOptions, - rejectUnauthorized: !ignoreHTTPSErrors, - } : commonRequestOptions; return new Promise(resolve => { debugWebServer(`HTTP GET: ${url}`); - (isHttps ? https : http).get(url, requestOptions, res => { + httpRequest({ + url: url.toString(), + headers: { Accept: '*/*' }, + rejectUnauthorized: !ignoreHTTPSErrors + }, res => { res.resume(); const statusCode = res.statusCode ?? 0; debugWebServer(`HTTP Status: ${statusCode}`); resolve(statusCode); - }).on('error', error => { + }, error => { if ((error as NodeJS.ErrnoException).code === 'DEPTH_ZERO_SELF_SIGNED_CERT') onStdErr?.(`[WebServer] Self-signed certificate detected. Try adding ignoreHTTPSErrors: true to config.webServer.`); debugWebServer(`Error while checking if ${url} is available: ${error.message}`);