[TEX-537] Refactor HTTP request and response representations (#554)

This PR does a bunch of initial refactoring and cleaning up of the
representations of HTTP request and response objects throughout the
codebase, ahead of making some larger changes to support gRPC-web
request comparisons.

The changes in this PR are mostly code structure in nature rather than
functional. There is one functionality change in the decision on how to
compare two HTTP request bodies, which I've called out in comments.

package.json

@@ -54,13 +54,14 @@
     "brotli": "^1.3.3",
     "chalk": "^4.1.0",
     "commander": "^10.0.1",
+    "content-type": "^1.0.5",
     "deep-diff": "^1.0.2",
     "fs-extra": "^11.1.1",
     "js-yaml": "^4.1.0",
-    "query-string": "^6.13.6",
     "string-similarity": "^4.0.4"
   },
   "devDependencies": {
+    "@types/content-type": "^1.1.8",
     "@types/deep-diff": "^1.0.5",
     "@types/express": "^4.17.21",
     "@types/fs-extra": "^11.0.4",

src/http.ts

@@ -0,0 +1,83 @@
+import brotli from "brotli";
+import zlib from "zlib";
+import { ParsedMediaType as ParsedContentType } from "content-type";
+
+/**
+ * Headers of a request or response.
+ */
+export interface HttpHeaders {
+  [headerName: string]: string | string[] | undefined;
+}
+
+/**
+ * The common fields of a HTTP request.
+ */
+export interface HttpRequest {
+  host?: string;
+  method: string;
+  path: string;
+  headers: HttpHeaders;
+  body: Buffer;
+}
+
+export interface HttpRequestWithHost extends HttpRequest {
+  host: string;
+}
+
+/**
+ * The common fields of a HTTP response.
+ */
+export interface HttpResponse {
+  status: {
+    code: number;
+  };
+  headers: HttpHeaders;
+  body: Buffer;
+}
+
+export function getHeaderAsString(
+  headers: HttpHeaders,
+  headerName: string,
+): string {
+  const rawValue = headers[headerName];
+  if (rawValue === undefined) {
+    return "";
+  } else if (typeof rawValue === "string") {
+    return rawValue;
+  } else {
+    return rawValue[0];
+  }
+}
+
+export function getHttpRequestContentType(request: HttpRequest): string {
+  return (
+    getHeaderAsString(request.headers, "content-type") ||
+    "application/octet-stream"
+  );
+}
+
+export function getHttpRequestBodyDecoded(request: HttpRequest): Buffer {
+  // Process the content-encoding before looking at the content-type.
+  const contentEncoding = getHeaderAsString(
+    request.headers,
+    "content-encoding",
+  );
+  switch (contentEncoding) {
+    case "":
+      return request.body;
+    case "br":
+      return Buffer.from(brotli.decompress(request.body));
+    case "gzip":
+      return zlib.gunzipSync(request.body);
+    default:
+      throw Error(`Unhandled content-encoding value "${contentEncoding}"`);
+  }
+}
+
+export function decodeHttpRequestBodyToString(
+  request: HttpRequest,
+  contentType: ParsedContentType,
+): string {
+  const encoding = contentType.parameters.charset as BufferEncoding | undefined;
+  return getHttpRequestBodyDecoded(request).toString(encoding || "utf-8");
+}

src/matcher.ts

@@ -1,6 +1,7 @@
+import { HttpRequest } from "./http";
 import { RewriteRules } from "./rewrite";
 import { computeSimilarity } from "./similarity";
-import { Headers, TapeRecord } from "./tape";
+import { TapeRecord } from "./tape";
 
 /**
  * Returns the first of a list of records that hasn't been replayed before.
@@ -42,21 +43,15 @@ export function findNextRecordToReplay(
  * against different paths).
  */
 export function findRecordMatches(
+  request: HttpRequest,
   tapeRecords: TapeRecord[],
-  requestMethod: string,
-  requestPath: string,
-  requestHeaders: Headers,
-  requestBody: Buffer,
   rewriteBeforeDiffRules: RewriteRules,
 ): TapeRecord[] {
   let bestSimilarityScore = +Infinity;
   let bestMatches: TapeRecord[] = [];
   for (const potentialMatch of tapeRecords) {
     const similarityScore = computeSimilarity(
-      requestMethod,
-      requestPath,
-      requestHeaders,
-      requestBody,
+      request,
       potentialMatch,
       rewriteBeforeDiffRules,
     );

src/persistence.ts

@@ -3,9 +3,9 @@ import fs from "fs-extra";
 import yaml from "js-yaml";
 import path from "path";
 import { gunzipSync, gzipSync } from "zlib";
+import { HttpHeaders } from "./http";
 import {
   CompressionAlgorithm,
-  Headers,
   PersistedBuffer,
   PersistedTapeRecord,
   TapeRecord,
@@ -122,7 +122,7 @@ export function reviveTape(persistedRecord: PersistedTapeRecord): TapeRecord {
 
 export function serialiseBuffer(
   buffer: Buffer,
-  headers: Headers,
+  headers: HttpHeaders,
 ): PersistedBuffer {
   const header = headers["content-encoding"];
   const contentEncoding = typeof header === "string" ? header : undefined;

src/sender.ts

@@ -2,13 +2,14 @@ import chalk from "chalk";
 import http from "http";
 import https from "https";
 import { ensureBuffer } from "./buffer";
-import { Headers, TapeRecord } from "./tape";
+import { HttpRequestWithHost } from "./http";
+import { TapeRecord } from "./tape";
 
 /**
  * Sends a network request and returns the recorded tape.
  */
 export async function send(
-  request: RequestWithHost,
+  request: HttpRequestWithHost,
   options: {
     loggingEnabled?: boolean;
     timeout?: number;
@@ -76,15 +77,3 @@ export async function send(
     throw e;
   }
 }
-
-export interface Request {
-  host?: string;
-  method: string;
-  path: string;
-  headers: Headers;
-  body: Buffer;
-}
-
-export interface RequestWithHost extends Request {
-  host: string;
-}

src/server.ts

@@ -5,11 +5,12 @@ import http from "http";
 import https from "https";
 import net from "net";
 import { ensureBuffer } from "./buffer";
+import { HttpRequest } from "./http";
 import { findNextRecordToReplay, findRecordMatches } from "./matcher";
 import { Mode } from "./modes";
 import { Persistence } from "./persistence";
 import { RewriteRules } from "./rewrite";
-import { Request, send } from "./sender";
+import { send } from "./sender";
 import { TapeRecord } from "./tape";
 
 /**
@@ -79,7 +80,7 @@ export class RecordReplayServer {
       }
 
       try {
-        const request: Request = {
+        const request: HttpRequest = {
           method: req.method,
           path: extractPath(req.url),
           headers: req.headers,
@@ -186,7 +187,7 @@ export class RecordReplayServer {
   /**
    * Handles requests that are intended for Proxay itself.
    */
-  private handleProxayApi(request: Request, res: http.ServerResponse) {
+  private handleProxayApi(request: HttpRequest, res: http.ServerResponse) {
     // Sending a request to /__proxay will return a 200 (so tests can identify whether
     // their backend is Proxay or not).
     if (
@@ -247,7 +248,7 @@ export class RecordReplayServer {
   /**
    * Potentially rewrite the request before processing it.
    */
-  private rewriteRequest(request: Request) {
+  private rewriteRequest(request: HttpRequest) {
     // Grab the `host` header of the request.
     const hostname = (request.headers.host || null) as string | null;
 
@@ -276,7 +277,7 @@ export class RecordReplayServer {
   /**
    * Rewrite a gRPC-web+json request to be unframed.
    */
-  private rewriteGrpcWebJsonRequest(request: Request) {
+  private rewriteGrpcWebJsonRequest(request: HttpRequest) {
     /**
      * From the gRPC specification (https://github.com/grpc/grpc/blob/master/doc/PROTOCOL-HTTP2.md)
      *
@@ -314,7 +315,9 @@ export class RecordReplayServer {
     request.body = request.body.subarray(5);
   }
 
-  private async fetchResponse(request: Request): Promise<TapeRecord | null> {
+  private async fetchResponse(
+    request: HttpRequest,
+  ): Promise<TapeRecord | null> {
     switch (this.mode) {
       case "replay":
         return this.fetchReplayResponse(request);
@@ -333,15 +336,12 @@ export class RecordReplayServer {
    * Fetches the response from the tape, returning null otherwise.
    */
   private async fetchReplayResponse(
-    request: Request,
+    request: HttpRequest,
   ): Promise<TapeRecord | null> {
     const record = findNextRecordToReplay(
       findRecordMatches(
+        request,
         this.currentTapeRecords,
-        request.method,
-        request.path,
-        request.headers,
-        request.body,
         this.rewriteBeforeDiffRules,
       ),
       this.replayedTapes,
@@ -367,7 +367,7 @@ export class RecordReplayServer {
    * Fetches the response directly from the proxied host and records it.
    */
   private async fetchRecordResponse(
-    request: Request,
+    request: HttpRequest,
   ): Promise<TapeRecord | null> {
     if (!this.proxiedHost) {
       throw new Error("Missing proxied host");
@@ -396,15 +396,12 @@ export class RecordReplayServer {
    * Fetches the response from the tape if present, otherwise from the proxied host.
    */
   private async fetchMimicResponse(
-    request: Request,
+    request: HttpRequest,
   ): Promise<TapeRecord | null> {
     let record = findNextRecordToReplay(
       findRecordMatches(
+        request,
         this.currentTapeRecords,
-        request.method,
-        request.path,
-        request.headers,
-        request.body,
         this.rewriteBeforeDiffRules,
       ),
       this.replayedTapes,
@@ -443,7 +440,7 @@ export class RecordReplayServer {
    * Fetches the response directly from the proxied host without recording it.
    */
   private async fetchPassthroughResponse(
-    request: Request,
+    request: HttpRequest,
   ): Promise<TapeRecord | null> {
     if (!this.proxiedHost) {
       throw new Error("Missing proxied host");