feat(rest): createAxios function instantiates axios with HTTP proxy s…

…upport

Making Axios support enterprise proxies is rather onerous so now Serenity/JS can do this for you

packages/rest/spec/io/axiosProxyOverridesFor.spec.ts

@@ -0,0 +1,39 @@
+import { expect } from '@integration/testing-tools';
+import axios from 'axios';
+import { describe, it } from 'mocha';
+
+import { axiosProxyOverridesFor } from '../../src/io/axiosProxyOverridesFor';
+import { ProxyAgent } from '../../src/io/ProxyAgent';
+
+describe('axiosProxyOverridesFor', () => {
+
+    describe('when no proxy overrides are defined', () => {
+
+        it('overrides Axios built-in proxy mechanism to use env variables', () => {
+
+            const overrides = axiosProxyOverridesFor(axios.create({}).defaults);
+
+            expect(overrides.proxy).to.equal(false);
+            expect(overrides.httpAgent).to.be.instanceof(ProxyAgent);
+            expect(overrides.httpsAgent).to.be.instanceof(ProxyAgent);
+        });
+    });
+
+    describe('when proxy overrides are defined', () => {
+
+        it('creates an Axios instance that overrides Axios built-in proxy mechanism', () => {
+
+            const overrides = axiosProxyOverridesFor(axios.create({
+                proxy: {
+                    protocol: 'http',
+                    host: 'proxy.mycompany.com',
+                    port: 9000,
+                }
+            }).defaults);
+
+            expect(overrides.proxy).to.equal(false);
+            expect(overrides.httpAgent).to.be.instanceof(ProxyAgent);
+            expect(overrides.httpsAgent).to.be.instanceof(ProxyAgent);
+        });
+    });
+});

packages/rest/spec/io/createUrl.spec.ts

@@ -0,0 +1,128 @@
+import { describe, it } from 'mocha';
+import { given } from 'mocha-testdata';
+
+import { createUrl } from '../../src/io/createUrl';
+import { expect } from '../expect';
+
+describe('createUrl', () => {
+
+    given([
+        {
+            description: 'default port for HTTP gets ignored',
+            options: {
+                protocol: 'http',
+                hostname: 'example.org',
+                port: '80',
+            },
+            expected: 'http://example.org/'
+        },
+        {
+            description: 'default port for HTTPs gets ignored',
+            options: {
+                protocol: 'https',
+                hostname: 'example.org',
+                port: '443',
+            },
+            expected: 'https://example.org/'
+        },
+        {
+            description: 'non-default HTTP port gets added',
+            options: {
+                protocol: 'http',
+                hostname: 'example.org',
+                port: '8080',
+            },
+            expected: 'http://example.org:8080/'
+        },
+        {
+            description: 'non-default HTTPs port gets added',
+            options: {
+                protocol: 'https',
+                hostname: 'example.org',
+                port: '8443',
+            },
+            expected: 'https://example.org:8443/'
+        },
+    ]).
+    it('should create a URL when given valid configuration', ({ options, expected }) => {
+
+        const url = createUrl(options);
+
+        expect(url.toString()).equals(expected);
+    });
+
+    describe('when the protocol is undefined', () => {
+
+        it('adds the port number', () => {
+
+            const url = createUrl({ protocol: undefined, hostname: 'example.org', port: 80 });
+
+            expect(url.toString()).equals('example.org:80');
+        });
+
+        it(`defaults to port 80 when port is not specified`, () => {
+
+            const url = createUrl({ protocol: undefined, hostname: 'example.org', port: undefined });
+
+            expect(url.toString()).equals('example.org:80');
+        });
+    });
+
+    given([
+        'https://',
+        'https:',
+        'https',
+        'HTTPS://',
+        'HTTPS:',
+        'HTTPS',
+    ]).
+    it('cleans up the protocol name', (protocol) => {
+
+        const url = createUrl({ protocol, hostname: `example.org`, port: undefined });
+
+        expect(url.protocol).to.equal('https:');
+        expect(url.toString()).to.equal('https://example.org/');
+    });
+
+    describe('when dealing with credentials', () => {
+
+        it('adds the url-encoded username to the resulting url', () => {
+            const url = createUrl({
+                protocol: 'https',
+                hostname: `example.org`,
+                username: 'alice.jones@example.org'
+            });
+
+            expect(url.username).to.equal('alice.jones%40example.org');
+            expect(url.toString()).to.equal('https://alice.jones%40example.org@example.org/');
+        });
+
+        it('adds the url-encoded username and password to the resulting url', () => {
+            const url = createUrl({
+                protocol: 'https',
+                hostname: `example.org`,
+                username: 'alice.jones@example.org',
+                password: '//P@55w0rd!'
+            });
+
+            expect(url.username).to.equal('alice.jones%40example.org');
+            expect(url.password).to.equal('%2F%2FP%4055w0rd!');
+            expect(url.toString()).to.equal('https://alice.jones%40example.org:%2F%2FP%4055w0rd!@example.org/');
+        });
+    });
+
+    describe('when handling errors', () => {
+
+        it('complains when the hostname is not specified', () => {
+            expect(() => {
+                createUrl({ protocol: 'http', hostname: undefined, port: undefined });
+            }).to.throw(Error, 'hostname should be a string');
+        });
+
+        it('complains when the hostname is blank', () => {
+            expect(() => {
+                createUrl({ protocol: 'http', hostname: '', port: undefined });
+            }).to.throw(Error, 'hostname should not be blank');
+        });
+    });
+});

packages/rest/src/index.ts

@@ -1 +1,2 @@
+export * from './io';
 export * from './screenplay';

packages/rest/src/io/AxiosRequestConfigDefaults.ts

@@ -0,0 +1,15 @@
+import { type CreateAxiosDefaults } from 'axios';
+
+export type AxiosRequestConfigProxyDefaults = {
+    host: string;
+    port?: number;          // SOCKS proxies don't require port number
+    auth?: {
+        username: string;
+        password: string;
+    };
+    protocol?: string;
+}
+
+export type AxiosRequestConfigDefaults<Data = any> = Omit<CreateAxiosDefaults<Data>, 'proxy'> & {
+    proxy?: AxiosRequestConfigProxyDefaults | false;
+}

packages/rest/src/io/ProxyAgent.ts

@@ -0,0 +1,129 @@
+import { ConfigurationError } from '@serenity-js/core';
+import type { AgentConnectOpts } from 'agent-base';
+import { Agent } from 'agent-base';
+import * as http from 'http';
+import { HttpProxyAgent, type HttpProxyAgentOptions } from 'http-proxy-agent';
+import * as https from 'https';
+import { HttpsProxyAgent, type HttpsProxyAgentOptions } from 'https-proxy-agent';
+import { LRUCache } from 'lru-cache';
+import { getProxyForUrl as envGetProxyForUrl } from 'proxy-from-env';
+
+const protocols = [
+    ...HttpProxyAgent.protocols,
+] as const;
+
+type AgentConstructor = new (proxy: URL | string, options?: ProxyAgentOptions) => Agent;
+
+type ValidProtocol = (typeof protocols)[number];
+
+type GetProxyForUrlCallback = (url: string) => string;
+
+export type ProxyAgentOptions =
+    HttpProxyAgentOptions<''> &
+    HttpsProxyAgentOptions<''> & {
+        /**
+         * Default `http.Agent` instance to use when no proxy is
+         * configured for a request. Defaults to a new `http.Agent()`
+         * instance with the proxy agent options passed in.
+         */
+        httpAgent?: http.Agent;
+        /**
+         * Default `http.Agent` instance to use when no proxy is
+         * configured for a request. Defaults to a new `https.Agent()`
+         * instance with the proxy agent options passed in.
+         */
+        httpsAgent?: http.Agent;
+        /**
+         * A callback for dynamic provision of proxy for url.
+         * Defaults to standard proxy environment variables,
+         * see https://www.npmjs.com/package/proxy-from-env for details
+         */
+        getProxyForUrl?: GetProxyForUrlCallback;
+    };
+
+/**
+ * A simplified version of the original
+ * [`ProxyAgent`](https://github.com/TooTallNate/proxy-agents/blob/5923589c2e5206504772c250ac4f20fc31122d3b/packages/proxy-agent/src/index.ts)
+ * with fewer dependencies.
+ *
+ * Delegates requests to the appropriate `Agent` subclass based on the "proxy"
+ * environment variables, or the provided `agentOptions.getProxyForUrl` callback.
+ *
+ * Uses an LRU cache to prevent unnecessary creation of proxy `http.Agent` instances.
+ */
+export class ProxyAgent extends Agent {
+
+    private static proxyAgents: { [P in ValidProtocol]: [ AgentConstructor, AgentConstructor ] } = {
+        http:   [ HttpProxyAgent, HttpsProxyAgent ],
+        https:  [ HttpProxyAgent, HttpsProxyAgent ],
+    };
+
+    /**
+     * Cache for `Agent` instances.
+     */
+    private readonly cache = new LRUCache<string, Agent>({
+        max: 20,
+        dispose: (value: Agent, key: string) => value.destroy(),
+    });
+
+    private readonly httpAgent: http.Agent;
+    private readonly httpsAgent: http.Agent;
+    private readonly getProxyForUrl: GetProxyForUrlCallback;
+
+    constructor(private readonly agentOptions: ProxyAgentOptions) {
+        super(agentOptions);
+        this.httpAgent      = agentOptions?.httpAgent       || new http.Agent(agentOptions);
+        this.httpsAgent     = agentOptions?.httpsAgent      || new https.Agent(agentOptions as https.AgentOptions);
+        this.getProxyForUrl = agentOptions?.getProxyForUrl  || envGetProxyForUrl;
+    }
+
+    override async connect(request: http.ClientRequest, options: AgentConnectOpts): Promise<http.Agent> {
+        const { secureEndpoint } = options;
+        const isWebSocket = request.getHeader('upgrade') === 'websocket';
+        const protocol = secureEndpoint
+            ? (isWebSocket ? 'wss:' : 'https:')
+            : (isWebSocket ? 'ws:' : 'http:');
+        const host = request.getHeader('host');
+        const url = new URL(request.path, `${protocol}//${host}`).href;
+        const proxy = this.getProxyForUrl(url);
+
+        if (! proxy) {
+            return secureEndpoint
+                ? this.httpsAgent
+                : this.httpAgent;
+        }
+
+        // attempt to get a cached `http.Agent` instance first
+        const cacheKey = `${ protocol }+${ proxy }`;
+        let agent = this.cache.get(cacheKey);
+        if (! agent) {
+            agent = this.createAgent(new URL(proxy), secureEndpoint || isWebSocket);
+
+            this.cache.set(cacheKey, agent);
+        }
+
+        return agent;
+    }
+
+    private createAgent(proxyUrl: URL, requiresTls: boolean): Agent {
+
+        const protocol = proxyUrl.protocol.replace(':', '');
+
+        if (! this.isValidProtocol(protocol)) {
+            throw new ConfigurationError(`Unsupported protocol for proxy URL: ${ proxyUrl }`);
+        }
+
+        const ctor = ProxyAgent.proxyAgents[protocol][requiresTls ? 1 : 0];
+
+        return new ctor(proxyUrl, this.agentOptions);
+    }
+
+    private isValidProtocol(v: string): v is ValidProtocol {
+        return (protocols as readonly string[]).includes(v);
+    }
+
+    override destroy(): void {
+        this.cache.clear();
+        super.destroy();
+    }
+}

packages/rest/src/io/axiosProxyOverridesFor.ts

@@ -0,0 +1,39 @@
+import type * as http from 'http';
+import { ensure, isDefined } from 'tiny-types';
+
+import type { AxiosRequestConfigDefaults } from './AxiosRequestConfigDefaults';
+import { createUrl } from './createUrl';
+import { ProxyAgent } from './ProxyAgent';
+
+/**
+ * @param options
+ */
+export function axiosProxyOverridesFor<Data = any>(options: AxiosRequestConfigDefaults<Data>): {
+    proxy: false, httpAgent: http.Agent, httpsAgent: http.Agent
+} {
+    const envProxyOverride: string | false = options.proxy
+        && createUrl({
+            username: options.proxy?.auth?.username,
+            password: options.proxy?.auth?.password,
+            protocol: options.proxy?.protocol,
+            hostname: ensure('proxy.host', options.proxy?.host, isDefined()),
+            port: options.proxy?.port
+        }).toString();
+
+    const agent = new ProxyAgent({
+        httpAgent: options.httpAgent,
+        httpsAgent: options.httpsAgent,
+
+        // if there's a specific proxy override configured, use it
+        // if not - detect proxy automatically based on env variables
+        getProxyForUrl: envProxyOverride
+            ? (url_: string) => envProxyOverride
+            : undefined,
+    });
+
+    return {
+        proxy: false,
+        httpAgent: agent,
+        httpsAgent: agent,
+    };
+}