From 1202a41fdccf7ccea67c557715eef66ea19a1c6f Mon Sep 17 00:00:00 2001 From: Tim Date: Tue, 14 May 2024 08:26:28 +0200 Subject: [PATCH] Ignore Headers / Send Proxy Port / Dump Matcher Fails (#604) I added three flags that are needed in my situation and might be helpful for others: `--ignore-headers`: Allows users to specify a list of headers that should be ignored by Proxay. This is especially useful together with the `--exact-request-matching` flag. `--send-proxy-port`: Sends the proxays port to the proxied host, similar to nginx' `proxy_set_header Host $host:$server_port;`. This helps against redirect issues. `--dump-matcher-fails`: Dumps the headers from the current and recorded request, if they are not matching. This helps to find headers that should be ignored in the specific situation. For all three parts there are simple tests. I also added a paragraph in the Readme that documents those flags. --------- Co-authored-by: Tim Dawborn --- README.md | 17 +++ src/cli.ts | 29 +++++- src/matcher.ts | 4 + src/sender.ts | 5 +- src/server.ts | 19 ++++ src/similarity.spec.ts | 168 ++++++++++++++++++++++++++++++ src/similarity.ts | 28 ++++- src/tests/send-proxy-port.spec.ts | 22 ++++ src/tests/setup.ts | 3 + src/tests/testserver.ts | 6 +- 10 files changed, 294 insertions(+), 7 deletions(-) create mode 100644 src/tests/send-proxy-port.spec.ts diff --git a/README.md b/README.md index e5fdad53..487201c6 100644 --- a/README.md +++ b/README.md @@ -65,6 +65,23 @@ In replay mode, this same file will be read from your tapes directory. You can leverage this by picking a tape based on the current test's name in the `beforeEach` block of your test suite. +## (Some) Options + +`--send-proxy-port`: This flag enables the forwarding of the Proxay's local port number to the proxied host in the host header. It is particularly useful for handling redirect issues where the proxied host needs to be aware of the port on which Proxay is running to construct accurate redirect URLs. This is similar to nginx' `proxy_set_header Host $host:$server_port;`. + + +`--ignore-headers `: Allows users to specify a list of headers that should be ignored by Proxay's matching algorithm during request comparison. This is useful for bypassing headers that do not influence the behavior of the request but may cause mismatches, such as `x-forwarded-for` or `x-real-ip`. The headers should be provided as a comma-separated list. + +Note: there is already a hardcoded list of headers that get ignored: +`accept`, `accept-encoding`, `age`, `cache-control`, `clear-site-data`, `connection`, `expires`, `from`, `host`, `postman-token`, `pragma`, `referer`, `referer-policy`, `te`, `trailer`, `transfer-encoding`, `user-agent`, `warning`, `x-datadog-trace-id`, `x-datadog-parent-id`, `traceparent` + + +`-r, --redact-headers `: This option enables the redaction of specific HTTP header values, which are replaced by `XXXX` to maintain privacy or confidentiality during the recording of network interactions. The headers should be provided as a comma-separated list. + + +`--debug-matcher-fails`: When exact request matching is enabled, this flag provides some debug information on why a request did not match any recorded tape. It is useful for troubleshooting and refining the conditions under which requests are considered equivalent, focusing on differences in headers and query parameters. + + ## Typical use case Let's say you're writing tests for your client. You want your tests to run as diff --git a/src/cli.ts b/src/cli.ts index aed9140e..c07d5750 100644 --- a/src/cli.ts +++ b/src/cli.ts @@ -57,13 +57,21 @@ async function main(argv: string[]) { .option("--default-tape ", "Name of the default tape", "default") .option("-h, --host ", "Host to proxy (not required in replay mode)") .option("-p, --port ", "Local port to serve on", "3000") + .option( + "--send-proxy-port", + "Sends the proxays port to the proxied host (helps for redirect issues)", + ) .option( "--exact-request-matching", "Perform exact request matching instead of best-effort request matching during replay.", ) + .option( + "--debug-matcher-fails", + "In exact request matching mode, shows debug information about failed matches for headers and queries.", + ) .option( "-r, --redact-headers ", - "Request headers to redact", + "Request headers to redact (values will be replaced by XXXX)", commaSeparatedList, ) .option( @@ -88,6 +96,11 @@ async function main(argv: string[]) { rewriteRule, new RewriteRules(), ) + .option( + "--ignore-headers ", + "Save headers to be ignored for the matching algorithm", + commaSeparatedList, + ) .parse(argv); const options = program.opts(); @@ -96,6 +109,8 @@ async function main(argv: string[]) { const defaultTapeName: string = options.defaultTape; const host: string = options.host; const port = parseInt(options.port, 10); + const sendProxyPort: boolean = + options.sendProxyPort === undefined ? false : options.sendProxyPort; const redactHeaders: string[] = options.redactHeaders; const preventConditionalRequests: boolean = !!options.dropConditionalRequestHeaders; @@ -103,10 +118,13 @@ async function main(argv: string[]) { const httpsKey: string = options.httpsKey; const httpsCert: string = options.httpsCert; const rewriteBeforeDiffRules: RewriteRules = options.rewriteBeforeDiff; + const ignoreHeaders: string[] = options.ignoreHeaders; const exactRequestMatching: boolean = options.exactRequestMatching === undefined ? false : options.exactRequestMatching; + const debugMatcherFails: boolean = + options.debugMatcherFails === undefined ? false : options.debugMatcherFails; switch (initialMode) { case "record": @@ -146,10 +164,17 @@ async function main(argv: string[]) { } } + if (debugMatcherFails && !exactRequestMatching) { + panic( + "The --debug-matcher-fails flag can only be used with the --exact-request-matching flag.", + ); + } + const server = new RecordReplayServer({ initialMode, tapeDir, host, + proxyPortToSend: sendProxyPort ? port : undefined, defaultTapeName, enableLogging: true, redactHeaders, @@ -158,7 +183,9 @@ async function main(argv: string[]) { httpsKey, httpsCert, rewriteBeforeDiffRules, + ignoreHeaders, exactRequestMatching, + debugMatcherFails, }); await server.start(port); console.log(chalk.green(`Proxying in ${initialMode} mode on port ${port}.`)); diff --git a/src/matcher.ts b/src/matcher.ts index 78e27a0d..2449c3db 100644 --- a/src/matcher.ts +++ b/src/matcher.ts @@ -48,6 +48,8 @@ export function findRecordMatches( tapeRecords: TapeRecord[], rewriteBeforeDiffRules: RewriteRules, exactRequestMatching: boolean, + debugMatcherFails: boolean, + ignoreHeaders: string[], ): TapeRecord[] { let bestSimilarityScore = +Infinity; if (exactRequestMatching) { @@ -59,6 +61,8 @@ export function findRecordMatches( request, potentialMatch, rewriteBeforeDiffRules, + ignoreHeaders, + debugMatcherFails, ); if (similarityScore < bestSimilarityScore) { diff --git a/src/sender.ts b/src/sender.ts index ac856955..5be509b8 100644 --- a/src/sender.ts +++ b/src/sender.ts @@ -13,6 +13,7 @@ export async function send( options: { loggingEnabled?: boolean; timeout?: number; + proxyPortToSend?: number; }, ): Promise { try { @@ -27,7 +28,9 @@ export async function send( port, headers: { ...request.headers, - host: hostname, + host: + hostname + + (options.proxyPortToSend ? `:${options.proxyPortToSend}` : ""), }, timeout: options.timeout, }; diff --git a/src/server.ts b/src/server.ts index 9919edfd..be62d91c 100644 --- a/src/server.ts +++ b/src/server.ts @@ -22,6 +22,7 @@ export class RecordReplayServer { private mode: Mode; private proxiedHost?: string; + private proxyPortToSend?: number; private timeout: number; private currentTapeRecords: TapeRecord[] = []; private currentTape!: string; @@ -30,13 +31,16 @@ export class RecordReplayServer { private replayedTapes: Set = new Set(); private preventConditionalRequests?: boolean; private rewriteBeforeDiffRules: RewriteRules; + private ignoreHeaders: string[]; private exactRequestMatching: boolean; + private debugMatcherFails: boolean; constructor(options: { initialMode: Mode; tapeDir: string; defaultTapeName: string; host?: string; + proxyPortToSend?: number; timeout?: number; enableLogging?: boolean; redactHeaders?: string[]; @@ -45,11 +49,14 @@ export class RecordReplayServer { httpsKey?: string; httpsCert?: string; rewriteBeforeDiffRules?: RewriteRules; + ignoreHeaders?: string[]; exactRequestMatching?: boolean; + debugMatcherFails?: boolean; }) { this.currentTapeRecords = []; this.mode = options.initialMode; this.proxiedHost = options.host; + this.proxyPortToSend = options.proxyPortToSend; this.timeout = options.timeout || 5000; this.loggingEnabled = options.enableLogging || false; const redactHeaders = options.redactHeaders || []; @@ -58,10 +65,15 @@ export class RecordReplayServer { this.preventConditionalRequests = options.preventConditionalRequests; this.rewriteBeforeDiffRules = options.rewriteBeforeDiffRules || new RewriteRules(); + this.ignoreHeaders = options.ignoreHeaders || []; this.exactRequestMatching = options.exactRequestMatching === undefined ? false : options.exactRequestMatching; + this.debugMatcherFails = + options.debugMatcherFails === undefined + ? false + : options.debugMatcherFails; this.loadTape(this.defaultTape); const handler = async ( @@ -291,6 +303,8 @@ export class RecordReplayServer { this.currentTapeRecords, this.rewriteBeforeDiffRules, this.exactRequestMatching, + this.debugMatcherFails, + this.ignoreHeaders, ), this.replayedTapes, ); @@ -331,6 +345,7 @@ export class RecordReplayServer { { loggingEnabled: this.loggingEnabled, timeout: this.timeout, + proxyPortToSend: this.proxyPortToSend, }, ); this.addRecordToTape(record); @@ -352,6 +367,8 @@ export class RecordReplayServer { this.currentTapeRecords, this.rewriteBeforeDiffRules, this.exactRequestMatching, + this.debugMatcherFails, + this.ignoreHeaders, ), this.replayedTapes, ); @@ -375,6 +392,7 @@ export class RecordReplayServer { { loggingEnabled: this.loggingEnabled, timeout: this.timeout, + proxyPortToSend: this.proxyPortToSend, }, ); this.addRecordToTape(record); @@ -405,6 +423,7 @@ export class RecordReplayServer { { loggingEnabled: this.loggingEnabled, timeout: this.timeout, + proxyPortToSend: this.proxyPortToSend, }, ); if (this.loggingEnabled) { diff --git a/src/similarity.spec.ts b/src/similarity.spec.ts index d775e651..18d9bd84 100644 --- a/src/similarity.spec.ts +++ b/src/similarity.spec.ts @@ -31,6 +31,7 @@ describe("similarity", () => { response: DUMMY_RESPONSE, }, new RewriteRules(), + [], ), ).toBe(0); }); @@ -54,6 +55,7 @@ describe("similarity", () => { response: DUMMY_RESPONSE, }, new RewriteRules(), + [], ), ).toBe(Infinity); }); @@ -77,6 +79,7 @@ describe("similarity", () => { response: DUMMY_RESPONSE, }, new RewriteRules(), + [], ), ).toBe(Infinity); }); @@ -100,6 +103,7 @@ describe("similarity", () => { response: DUMMY_RESPONSE, }, new RewriteRules(), + [], ), ).toBe(1); expect( @@ -120,6 +124,7 @@ describe("similarity", () => { response: DUMMY_RESPONSE, }, new RewriteRules(), + [], ), ).toBe(1); expect( @@ -140,6 +145,7 @@ describe("similarity", () => { response: DUMMY_RESPONSE, }, new RewriteRules(), + [], ), ).toBe(2); expect( @@ -160,6 +166,7 @@ describe("similarity", () => { response: DUMMY_RESPONSE, }, new RewriteRules(), + [], ), ).toBe(0); expect( @@ -180,6 +187,7 @@ describe("similarity", () => { response: DUMMY_RESPONSE, }, new RewriteRules(), + [], ), ).toBe(1); }); @@ -214,6 +222,7 @@ describe("similarity", () => { response: DUMMY_RESPONSE, }, new RewriteRules(), + [], ), ).toBe(0); expect( @@ -244,6 +253,7 @@ describe("similarity", () => { response: DUMMY_RESPONSE, }, new RewriteRules(), + [], ), ).toBe(1); expect( @@ -275,10 +285,154 @@ describe("similarity", () => { response: DUMMY_RESPONSE, }, new RewriteRules(), + [], ), ).toBe(1); }); + it("counts headers differences (ignoring specified ones)", () => { + expect( + computeSimilarity( + { + method: "POST", + path: "/test", + headers: { + one: "one", + two: "two", + three: "three", + four: "four", + }, + body: Buffer.from([]), + }, + { + request: { + method: "POST", + path: "/test", + headers: { + // one is missing + two: "different two", + three: "different three", + four: "four", // four is the same + }, + body: Buffer.from([]), + }, + response: DUMMY_RESPONSE, + }, + new RewriteRules(), + ["one", "two", "three"], + ), + ).toBe(0); + expect( + computeSimilarity( + { + method: "POST", + path: "/test", + headers: { + one: "one", + two: "two", + three: "three", + four: "four", + }, + body: Buffer.from([]), + }, + { + request: { + method: "POST", + path: "/test", + headers: { + // one is missing + two: "different two", + three: "different three", + four: "different four", + }, + body: Buffer.from([]), + }, + response: DUMMY_RESPONSE, + }, + new RewriteRules(), + ["one", "two", "three"], + ), + ).toBe(1); + }); + + it("dumps headers if non equal", () => { + const logSpy = jest.spyOn(console, "log"); + + computeSimilarity( + { + method: "POST", + path: "/test", + headers: { + ignore: "ignore", + one: "one", + two: "two", + three: "three", + four: "four", + }, + body: Buffer.from([]), + }, + { + request: { + method: "POST", + path: "/test", + headers: { + ignore: "different ignore", + one: "one", + two: "two", + three: "three", + four: "four", + }, + body: Buffer.from([]), + }, + response: DUMMY_RESPONSE, + }, + new RewriteRules(), + ["ignore"], + true, + ); + + expect(logSpy).not.toHaveBeenCalled(); + + computeSimilarity( + { + method: "POST", + path: "/test", + headers: { + ignore: "ignore", + one: "one", + two: "two", + three: "three", + four: "four", + }, + body: Buffer.from([]), + }, + { + request: { + method: "POST", + path: "/test", + headers: { + ignore: "different ignore", + // one is missing + two: "different two", + three: "different three", + four: "four", // four is the same + }, + body: Buffer.from([]), + }, + response: DUMMY_RESPONSE, + }, + new RewriteRules(), + ["ignore"], + true, + ); + + expect(logSpy).toHaveBeenCalledWith( + 'debug: a: {"one":"one","two":"two","three":"three","four":"four"} / b: {"two":"different two","three":"different three","four":"four"}', + ); + + logSpy.mockRestore(); + }); + describe("JSON payload types", () => { it("reports no differences when the paylods are the same", () => { // The following payloads are identical, but formatted differently. @@ -312,6 +466,7 @@ describe("similarity", () => { response: DUMMY_RESPONSE, }, new RewriteRules(), + [], ), ).toBe(0); }); @@ -351,6 +506,7 @@ describe("similarity", () => { response: DUMMY_RESPONSE, }, new RewriteRules(), + [], ), ).toBe(1); @@ -385,6 +541,7 @@ describe("similarity", () => { response: DUMMY_RESPONSE, }, new RewriteRules(), + [], ), ).toBe(6); @@ -425,6 +582,7 @@ describe("similarity", () => { "$1", ), ), + [], ), ).toBe(0); }); @@ -454,6 +612,7 @@ describe("similarity", () => { response: DUMMY_RESPONSE, }, new RewriteRules(), + [], ), ).toBe(0); }); @@ -481,6 +640,7 @@ describe("similarity", () => { response: DUMMY_RESPONSE, }, new RewriteRules(), + [], ), ).toBe(6); }); @@ -517,6 +677,7 @@ describe("similarity", () => { response: DUMMY_RESPONSE, }, new RewriteRules(), + [], ), ).toBe(0); }); @@ -544,6 +705,7 @@ describe("similarity", () => { response: DUMMY_RESPONSE, }, new RewriteRules(), + [], ), ).toBe(5149); }); @@ -591,6 +753,7 @@ describe("similarity", () => { response: DUMMY_RESPONSE, }, new RewriteRules(), + [], ), ).toBe(0); }); @@ -655,6 +818,7 @@ describe("similarity", () => { response: DUMMY_RESPONSE, }, new RewriteRules(), + [], ), ).toBe(1); }); @@ -724,6 +888,7 @@ describe("similarity", () => { "$1", ), ), + [], ), ).toBe(0); }); @@ -756,6 +921,7 @@ describe("similarity", () => { response: DUMMY_RESPONSE, }, new RewriteRules(), + [], ), ).toBe(0); }); @@ -788,6 +954,7 @@ describe("similarity", () => { response: DUMMY_RESPONSE, }, new RewriteRules(), + [], ), ).toBe(1); }); @@ -825,6 +992,7 @@ describe("similarity", () => { "$1", ), ), + [], ), ).toBe(0); }); diff --git a/src/similarity.ts b/src/similarity.ts index 75f05f12..36f1699c 100644 --- a/src/similarity.ts +++ b/src/similarity.ts @@ -27,6 +27,8 @@ export function computeSimilarity( request: HttpRequest, compareTo: TapeRecord, rewriteBeforeDiffRules: RewriteRules, + ignoreHeaders: string[], + debugMatcherFails: boolean = false, ): number { // If the HTTP method is different, no match. if (request.method !== compareTo.request.method) { @@ -50,17 +52,20 @@ export function computeSimilarity( parsedQueryParameters, parsedCompareToQueryParameters, rewriteBeforeDiffRules, + debugMatcherFails, ); // Compare the cleaned headers. - const cleanedHeaders = stripExtraneousHeaders(request.headers); + const cleanedHeaders = stripExtraneousHeaders(request.headers, ignoreHeaders); const cleanedCompareToHeaders = stripExtraneousHeaders( compareTo.request.headers, + ignoreHeaders, ); const differencesHeaders = countObjectDifferences( cleanedHeaders, cleanedCompareToHeaders, rewriteBeforeDiffRules, + debugMatcherFails, ); // Compare the bodies. @@ -69,6 +74,9 @@ export function computeSimilarity( compareTo.request, rewriteBeforeDiffRules, ); + if (debugMatcherFails && differencesBody > 0) { + console.log(`debug: body is different`); + } return differencesQueryParameters + differencesHeaders + differencesBody; } @@ -267,11 +275,18 @@ function countObjectDifferences( a: object, b: object, rewriteRules: RewriteRules, + debugMatcherFails: boolean = false, ): number { a = rewriteRules.apply(a); b = rewriteRules.apply(b); - return (diff(a, b) || []).length; + const result = (diff(a, b) || []).length; + + if (debugMatcherFails && result > 0) { + console.log(`debug: a: ${JSON.stringify(a)} / b: ${JSON.stringify(b)}`); + } + + return result; } /** @@ -317,7 +332,10 @@ function parseQueryParameters(path: string): ParsedUrlQuery { /** * Strips out headers that are likely to result in false negatives. */ -function stripExtraneousHeaders(headers: HttpHeaders): HttpHeaders { +function stripExtraneousHeaders( + headers: HttpHeaders, + ignoreHeaders: string[], +): HttpHeaders { const safeHeaders: HttpHeaders = {}; for (const key of Object.keys(headers)) { switch (key) { @@ -345,7 +363,9 @@ function stripExtraneousHeaders(headers: HttpHeaders): HttpHeaders { // Ignore. continue; default: - safeHeaders[key] = headers[key]; + if (!ignoreHeaders.find((header) => header === key)) { + safeHeaders[key] = headers[key]; + } } } return safeHeaders; diff --git a/src/tests/send-proxy-port.spec.ts b/src/tests/send-proxy-port.spec.ts new file mode 100644 index 00000000..c5da57c8 --- /dev/null +++ b/src/tests/send-proxy-port.spec.ts @@ -0,0 +1,22 @@ +import { setupServers } from "./setup"; +import axios from "axios"; +import { PROXAY_HOST, PROXAY_PORT } from "./config"; +import { JSON_IDENTITY_PATH } from "./testserver"; + +describe("SendProxyPort", () => { + describe("regular sendProxyPort mode", () => { + setupServers({ mode: "passthrough" }); + test("response: hostname without port", async () => { + const response = await axios.get(`${PROXAY_HOST}${JSON_IDENTITY_PATH}`); + expect(response.data.hostname).toBe("localhost"); + }); + }); + + describe("regular sendProxyPort mode", () => { + setupServers({ mode: "passthrough", sendProxyPort: true }); + test("response: hostname with port", async () => { + const response = await axios.get(`${PROXAY_HOST}${JSON_IDENTITY_PATH}`); + expect(response.data.hostname).toBe("localhost:" + PROXAY_PORT); + }); + }); +}); diff --git a/src/tests/setup.ts b/src/tests/setup.ts index 65781248..7d8c9b85 100644 --- a/src/tests/setup.ts +++ b/src/tests/setup.ts @@ -10,11 +10,13 @@ export function setupServers({ tapeDirName = mode, defaultTapeName = "default", exactRequestMatching, + sendProxyPort = false, }: { mode: Mode; tapeDirName?: string; defaultTapeName?: string; exactRequestMatching?: boolean; + sendProxyPort?: boolean; }) { const tapeDir = path.join(__dirname, "tapes", tapeDirName); const servers = { tapeDir } as { @@ -39,6 +41,7 @@ export function setupServers({ timeout: 100, enableLogging: true, exactRequestMatching, + proxyPortToSend: sendProxyPort ? PROXAY_PORT : undefined, }); await Promise.all([ servers.proxy.start(PROXAY_PORT), diff --git a/src/tests/testserver.ts b/src/tests/testserver.ts index 081c7818..971236c3 100644 --- a/src/tests/testserver.ts +++ b/src/tests/testserver.ts @@ -41,7 +41,11 @@ export class TestServer { res.send(BINARY_RESPONSE); }); this.app.get(JSON_IDENTITY_PATH, (req, res) => { - res.json({ data: req.path, requestCount: this.requestCount }); + res.json({ + data: req.path, + requestCount: this.requestCount, + hostname: req.header("host"), + }); }); this.app.post(JSON_IDENTITY_PATH, (req, res) => { res.json({ ...req.body, requestCount: this.requestCount });