Ignore Headers / Send Proxy Port / Dump Matcher Fails (#604)

I added three flags that are needed in my situation and might be helpful
for others:

`--ignore-headers`: Allows users to specify a list of headers that
should be ignored by Proxay. This is especially useful together with the
`--exact-request-matching` flag.

`--send-proxy-port`: Sends the proxays port to the proxied host, similar
to nginx' `proxy_set_header Host $host:$server_port;`. This helps
against redirect issues.

`--dump-matcher-fails`: Dumps the headers from the current and recorded
request, if they are not matching. This helps to find headers that
should be ignored in the specific situation.

For all three parts there are simple tests. I also added a paragraph in
the Readme that documents those flags.

---------

Co-authored-by: Tim Dawborn <tim.dawborn@gmail.com>

README.md

@@ -65,6 +65,23 @@ In replay mode, this same file will be read from your tapes directory.
 You can leverage this by picking a tape based on the current test's name in the `beforeEach`
 block of your test suite.
 
+## (Some) Options
+
+`--send-proxy-port`: This flag enables the forwarding of the Proxay's local port number to the proxied host in the host header. It is particularly useful for handling redirect issues where the proxied host needs to be aware of the port on which Proxay is running to construct accurate redirect URLs. This is similar to nginx' `proxy_set_header Host $host:$server_port;`.
+
+
+`--ignore-headers <headers>`: Allows users to specify a list of headers that should be ignored by Proxay's matching algorithm during request comparison. This is useful for bypassing headers that do not influence the behavior of the request but may cause mismatches, such as `x-forwarded-for` or `x-real-ip`. The headers should be provided as a comma-separated list.
+
+Note: there is already a hardcoded list of headers that get ignored:
+`accept`, `accept-encoding`, `age`, `cache-control`, `clear-site-data`, `connection`, `expires`, `from`, `host`, `postman-token`, `pragma`, `referer`, `referer-policy`, `te`, `trailer`, `transfer-encoding`, `user-agent`, `warning`, `x-datadog-trace-id`, `x-datadog-parent-id`, `traceparent`
+
+
+`-r, --redact-headers <headers>`: This option enables the redaction of specific HTTP header values, which are replaced by `XXXX` to maintain privacy or confidentiality during the recording of network interactions. The headers should be provided as a comma-separated list.
+
+
+`--debug-matcher-fails`: When exact request matching is enabled, this flag provides some debug information on why a request did not match any recorded tape. It is useful for troubleshooting and refining the conditions under which requests are considered equivalent, focusing on differences in headers and query parameters.
+
+
 ## Typical use case
 
 Let's say you're writing tests for your client. You want your tests to run as

src/cli.ts

@@ -57,13 +57,21 @@ async function main(argv: string[]) {
     .option("--default-tape <tape-name>", "Name of the default tape", "default")
     .option("-h, --host <host>", "Host to proxy (not required in replay mode)")
     .option("-p, --port <port>", "Local port to serve on", "3000")
+    .option(
+      "--send-proxy-port",
+      "Sends the proxays port to the proxied host (helps for redirect issues)",
+    )
     .option(
       "--exact-request-matching",
       "Perform exact request matching instead of best-effort request matching during replay.",
     )
+    .option(
+      "--debug-matcher-fails",
+      "In exact request matching mode, shows debug information about failed matches for headers and queries.",
+    )
     .option(
       "-r, --redact-headers <headers>",
-      "Request headers to redact",
+      "Request headers to redact (values will be replaced by XXXX)",
       commaSeparatedList,
     )
     .option(
@@ -88,6 +96,11 @@ async function main(argv: string[]) {
       rewriteRule,
       new RewriteRules(),
     )
+    .option(
+      "--ignore-headers <headers>",
+      "Save headers to be ignored for the matching algorithm",
+      commaSeparatedList,
+    )
     .parse(argv);
 
   const options = program.opts();
@@ -96,17 +109,22 @@ async function main(argv: string[]) {
   const defaultTapeName: string = options.defaultTape;
   const host: string = options.host;
   const port = parseInt(options.port, 10);
+  const sendProxyPort: boolean =
+    options.sendProxyPort === undefined ? false : options.sendProxyPort;
   const redactHeaders: string[] = options.redactHeaders;
   const preventConditionalRequests: boolean =
     !!options.dropConditionalRequestHeaders;
   const httpsCA: string = options.httpsCa || "";
   const httpsKey: string = options.httpsKey;
   const httpsCert: string = options.httpsCert;
   const rewriteBeforeDiffRules: RewriteRules = options.rewriteBeforeDiff;
+  const ignoreHeaders: string[] = options.ignoreHeaders;
   const exactRequestMatching: boolean =
     options.exactRequestMatching === undefined
       ? false
       : options.exactRequestMatching;
+  const debugMatcherFails: boolean =
+    options.debugMatcherFails === undefined ? false : options.debugMatcherFails;
 
   switch (initialMode) {
     case "record":
@@ -146,10 +164,17 @@ async function main(argv: string[]) {
     }
   }
 
+  if (debugMatcherFails && !exactRequestMatching) {
+    panic(
+      "The --debug-matcher-fails flag can only be used with the --exact-request-matching flag.",
+    );
+  }
+
   const server = new RecordReplayServer({
     initialMode,
     tapeDir,
     host,
+    proxyPortToSend: sendProxyPort ? port : undefined,
     defaultTapeName,
     enableLogging: true,
     redactHeaders,
@@ -158,7 +183,9 @@ async function main(argv: string[]) {
     httpsKey,
     httpsCert,
     rewriteBeforeDiffRules,
+    ignoreHeaders,
     exactRequestMatching,
+    debugMatcherFails,
   });
   await server.start(port);
   console.log(chalk.green(`Proxying in ${initialMode} mode on port ${port}.`));

src/matcher.ts

@@ -48,6 +48,8 @@ export function findRecordMatches(
   tapeRecords: TapeRecord[],
   rewriteBeforeDiffRules: RewriteRules,
   exactRequestMatching: boolean,
+  debugMatcherFails: boolean,
+  ignoreHeaders: string[],
 ): TapeRecord[] {
   let bestSimilarityScore = +Infinity;
   if (exactRequestMatching) {
@@ -59,6 +61,8 @@ export function findRecordMatches(
       request,
       potentialMatch,
       rewriteBeforeDiffRules,
+      ignoreHeaders,
+      debugMatcherFails,
     );
 
     if (similarityScore < bestSimilarityScore) {

src/sender.ts

@@ -13,6 +13,7 @@ export async function send(
   options: {
     loggingEnabled?: boolean;
     timeout?: number;
+    proxyPortToSend?: number;
   },
 ): Promise<TapeRecord> {
   try {
@@ -27,7 +28,9 @@ export async function send(
           port,
           headers: {
             ...request.headers,
-            host: hostname,
+            host:
+              hostname +
+              (options.proxyPortToSend ? `:${options.proxyPortToSend}` : ""),
           },
           timeout: options.timeout,
         };

src/server.ts

@@ -22,6 +22,7 @@ export class RecordReplayServer {
 
   private mode: Mode;
   private proxiedHost?: string;
+  private proxyPortToSend?: number;
   private timeout: number;
   private currentTapeRecords: TapeRecord[] = [];
   private currentTape!: string;
@@ -30,13 +31,16 @@ export class RecordReplayServer {
   private replayedTapes: Set<TapeRecord> = new Set();
   private preventConditionalRequests?: boolean;
   private rewriteBeforeDiffRules: RewriteRules;
+  private ignoreHeaders: string[];
   private exactRequestMatching: boolean;
+  private debugMatcherFails: boolean;
 
   constructor(options: {
     initialMode: Mode;
     tapeDir: string;
     defaultTapeName: string;
     host?: string;
+    proxyPortToSend?: number;
     timeout?: number;
     enableLogging?: boolean;
     redactHeaders?: string[];
@@ -45,11 +49,14 @@ export class RecordReplayServer {
     httpsKey?: string;
     httpsCert?: string;
     rewriteBeforeDiffRules?: RewriteRules;
+    ignoreHeaders?: string[];
     exactRequestMatching?: boolean;
+    debugMatcherFails?: boolean;
   }) {
     this.currentTapeRecords = [];
     this.mode = options.initialMode;
     this.proxiedHost = options.host;
+    this.proxyPortToSend = options.proxyPortToSend;
     this.timeout = options.timeout || 5000;
     this.loggingEnabled = options.enableLogging || false;
     const redactHeaders = options.redactHeaders || [];
@@ -58,10 +65,15 @@ export class RecordReplayServer {
     this.preventConditionalRequests = options.preventConditionalRequests;
     this.rewriteBeforeDiffRules =
       options.rewriteBeforeDiffRules || new RewriteRules();
+    this.ignoreHeaders = options.ignoreHeaders || [];
     this.exactRequestMatching =
       options.exactRequestMatching === undefined
         ? false
         : options.exactRequestMatching;
+    this.debugMatcherFails =
+      options.debugMatcherFails === undefined
+        ? false
+        : options.debugMatcherFails;
     this.loadTape(this.defaultTape);
 
     const handler = async (
@@ -291,6 +303,8 @@ export class RecordReplayServer {
         this.currentTapeRecords,
         this.rewriteBeforeDiffRules,
         this.exactRequestMatching,
+        this.debugMatcherFails,
+        this.ignoreHeaders,
       ),
       this.replayedTapes,
     );
@@ -331,6 +345,7 @@ export class RecordReplayServer {
       {
         loggingEnabled: this.loggingEnabled,
         timeout: this.timeout,
+        proxyPortToSend: this.proxyPortToSend,
       },
     );
     this.addRecordToTape(record);
@@ -352,6 +367,8 @@ export class RecordReplayServer {
         this.currentTapeRecords,
         this.rewriteBeforeDiffRules,
         this.exactRequestMatching,
+        this.debugMatcherFails,
+        this.ignoreHeaders,
       ),
       this.replayedTapes,
     );
@@ -375,6 +392,7 @@ export class RecordReplayServer {
         {
           loggingEnabled: this.loggingEnabled,
           timeout: this.timeout,
+          proxyPortToSend: this.proxyPortToSend,
         },
       );
       this.addRecordToTape(record);
@@ -405,6 +423,7 @@ export class RecordReplayServer {
       {
         loggingEnabled: this.loggingEnabled,
         timeout: this.timeout,
+        proxyPortToSend: this.proxyPortToSend,
       },
     );
     if (this.loggingEnabled) {