diff --git a/package.json b/package.json index 14a24fde..be87313e 100644 --- a/package.json +++ b/package.json @@ -51,7 +51,6 @@ }, "dependencies": { "assert-never": "^1.2.1", - "brotli": "^1.3.3", "chalk": "^4.1.0", "commander": "^10.0.1", "content-type": "^1.0.5", diff --git a/src/brotli.d.ts b/src/brotli.d.ts deleted file mode 100644 index d1788237..00000000 --- a/src/brotli.d.ts +++ /dev/null @@ -1,16 +0,0 @@ -declare module "brotli" { - /** - * Compresses the data. - * - * If the compressed size isn't better, returns null. - */ - function compress(data: Uint8Array): Uint8Array | null; - - /** - * Uncompresses the data. - */ - function decompress( - compressedData: Uint8Array, - decompressedLength?: number, - ): Uint8Array; -} diff --git a/src/compression.spec.ts b/src/compression.spec.ts new file mode 100644 index 00000000..ec72b91d --- /dev/null +++ b/src/compression.spec.ts @@ -0,0 +1,362 @@ +import { + compressBuffer, + CompressionAlgorithm, + convertHttpContentEncodingToCompressionAlgorithm, + decompressBuffer, +} from "./compression"; + +const empty = Buffer.from([]); +const small = Buffer.from([1, 2, 3, 4, 5]); +const favicon = Buffer.from([ + 137, 80, 78, 71, 13, 10, 26, 10, 0, 0, 0, 13, 73, 72, 68, 82, 0, 0, 0, 57, 0, + 0, 0, 57, 8, 6, 0, 0, 0, 140, 24, 131, 133, 0, 0, 0, 9, 112, 72, 89, 115, 0, + 0, 11, 19, 0, 0, 11, 19, 1, 0, 154, 156, 24, 0, 0, 0, 1, 115, 82, 71, 66, 0, + 174, 206, 28, 233, 0, 0, 0, 4, 103, 65, 77, 65, 0, 0, 177, 143, 11, 252, 97, + 5, 0, 0, 3, 132, 73, 68, 65, 84, 120, 1, 237, 154, 221, 81, 219, 64, 16, 199, + 119, 79, 146, 163, 129, 153, 140, 58, 136, 233, 0, 42, 8, 84, 16, 83, 65, 148, + 4, 50, 121, 195, 169, 32, 80, 1, 230, 45, 19, 200, 96, 58, 128, 14, 220, 65, + 220, 65, 92, 130, 94, 32, 138, 101, 105, 115, 39, 227, 144, 113, 140, 181, 39, + 233, 78, 246, 192, 239, 193, 2, 188, 30, 230, 175, 93, 239, 215, 9, 224, 9, + 128, 160, 73, 16, 82, 16, 251, 16, 64, 67, 248, 49, 68, 81, 31, 35, 157, 207, + 184, 160, 201, 111, 55, 9, 69, 42, 78, 161, 33, 198, 46, 157, 200, 203, 177, + 206, 103, 4, 104, 242, 98, 226, 245, 229, 69, 235, 78, 54, 141, 182, 72, 21, + 42, 68, 217, 9, 172, 17, 218, 34, 21, 191, 46, 90, 61, 249, 101, 30, 192, 154, + 80, 74, 164, 34, 117, 156, 119, 128, 56, 130, 53, 160, 180, 200, 248, 43, 142, + 18, 74, 247, 215, 65, 104, 105, 145, 138, 228, 188, 53, 204, 132, 216, 91, + 117, 161, 149, 68, 42, 148, 71, 239, 190, 57, 91, 25, 192, 202, 134, 111, 101, + 145, 51, 226, 115, 183, 159, 139, 37, 218, 147, 73, 233, 12, 41, 79, 76, 139, + 74, 77, 4, 132, 67, 155, 55, 68, 187, 25, 40, 34, 190, 240, 6, 48, 151, 121, + 253, 79, 212, 206, 223, 147, 94, 247, 194, 241, 118, 203, 19, 167, 68, 176, + 13, 150, 168, 205, 147, 203, 80, 226, 84, 59, 182, 249, 49, 61, 245, 60, 241, + 131, 0, 118, 193, 34, 86, 68, 110, 30, 78, 194, 177, 155, 254, 36, 162, 238, + 227, 86, 50, 132, 13, 81, 123, 184, 254, 139, 31, 82, 219, 241, 210, 203, 220, + 115, 5, 163, 0, 2, 157, 73, 187, 75, 48, 128, 81, 79, 10, 55, 61, 230, 133, + 38, 14, 83, 162, 17, 24, 194, 152, 72, 53, 146, 73, 247, 188, 229, 216, 42, + 47, 130, 65, 140, 137, 76, 188, 180, 195, 181, 149, 45, 226, 0, 12, 98, 50, + 92, 153, 94, 132, 129, 202, 190, 96, 16, 35, 34, 85, 194, 209, 40, 19, 87, + 249, 107, 134, 109, 48, 132, 17, 145, 194, 157, 132, 92, 91, 47, 113, 174, + 193, 48, 102, 194, 21, 5, 43, 84, 165, 97, 95, 119, 95, 83, 134, 218, 69, 250, + 7, 201, 46, 192, 180, 141, 43, 4, 233, 6, 44, 80, 123, 51, 32, 0, 67, 150, 33, + 230, 211, 139, 241, 80, 85, 212, 31, 174, 8, 111, 88, 118, 100, 111, 125, 82, + 171, 72, 213, 163, 202, 11, 107, 39, 155, 81, 118, 5, 150, 168, 219, 147, 188, + 132, 35, 67, 245, 126, 36, 179, 66, 109, 34, 245, 106, 163, 61, 47, 42, 106, + 19, 137, 110, 194, 110, 227, 50, 225, 246, 193, 34, 245, 137, 68, 231, 136, + 101, 103, 161, 141, 155, 167, 22, 145, 90, 181, 113, 214, 198, 89, 164, 22, + 145, 236, 218, 8, 230, 39, 142, 69, 212, 19, 174, 136, 175, 57, 102, 4, 116, + 99, 59, 84, 21, 149, 69, 78, 107, 35, 47, 84, 165, 199, 173, 116, 56, 255, + 255, 223, 234, 48, 155, 113, 121, 43, 16, 95, 65, 3, 84, 18, 169, 57, 55, 74, + 149, 116, 188, 113, 152, 94, 78, 19, 149, 61, 42, 53, 232, 114, 19, 183, 75, + 160, 11, 133, 2, 49, 220, 56, 152, 68, 136, 112, 191, 134, 196, 64, 174, 43, + 219, 96, 136, 74, 34, 9, 240, 139, 122, 45, 5, 66, 240, 16, 5, 84, 226, 233, + 5, 62, 165, 195, 117, 243, 112, 188, 173, 81, 27, 27, 165, 180, 72, 34, 209, + 133, 53, 161, 124, 226, 121, 172, 54, 170, 211, 170, 21, 59, 194, 43, 37, 114, + 105, 109, 148, 9, 100, 213, 78, 160, 75, 137, 148, 153, 112, 233, 196, 225, + 33, 118, 90, 99, 177, 131, 25, 24, 221, 140, 115, 209, 22, 169, 106, 163, 244, + 210, 242, 21, 7, 97, 62, 145, 220, 126, 119, 187, 8, 217, 14, 90, 92, 117, 44, + 66, 91, 164, 170, 141, 12, 179, 64, 61, 185, 165, 126, 184, 61, 111, 13, 111, + 47, 220, 189, 204, 113, 182, 136, 178, 207, 75, 78, 160, 141, 161, 93, 39, + 101, 109, 60, 226, 212, 70, 129, 66, 121, 187, 55, 251, 253, 190, 49, 239, + 205, 254, 22, 116, 41, 136, 227, 135, 125, 16, 78, 146, 14, 162, 153, 199, + 217, 180, 68, 170, 218, 40, 39, 9, 214, 49, 184, 188, 25, 163, 101, 239, 71, + 189, 124, 169, 252, 215, 163, 254, 123, 217, 1, 57, 96, 4, 173, 112, 213, 169, + 141, 54, 183, 113, 69, 232, 125, 39, 153, 115, 163, 237, 109, 92, 17, 108, + 145, 27, 31, 38, 29, 118, 27, 215, 112, 54, 157, 135, 239, 73, 4, 246, 54, 46, + 129, 116, 37, 234, 227, 12, 150, 200, 105, 109, 228, 14, 199, 56, 84, 143, + 163, 193, 10, 193, 18, 201, 172, 141, 57, 166, 207, 255, 203, 192, 18, 57, + 173, 141, 60, 154, 216, 198, 21, 193, 170, 147, 153, 35, 246, 129, 73, 217, + 109, 156, 255, 210, 185, 150, 205, 193, 160, 208, 206, 135, 232, 14, 158, 121, + 154, 252, 1, 139, 232, 48, 101, 122, 172, 204, 168, 0, 0, 0, 0, 73, 69, 78, + 68, 174, 66, 96, 130, +]); + +// RFC 1952. https://www.ietf.org/rfc/rfc1952.txt +// The 10th byte in a gzip encoding specifies an identifier for the operating system that the +// compression happened on. +// To make our tests pass on different OS's, we dynamically insert that byte into the hardcoded +// gzip compression payloads. +function makeGzipEncodingPlatformAgnostic(buffer: Buffer): Buffer { + const newBuffer = Buffer.from(buffer); + newBuffer[9] = 0; // Always make it a fixed value for cross-platform consistency. + return newBuffer; +} + +const emptyGzip = makeGzipEncodingPlatformAgnostic( + Buffer.from([31, 139, 8, 0, 0, 0, 0, 0, 0, 19, 3, 0, 0, 0, 0, 0, 0, 0, 0, 0]), +); +const smallGzip = makeGzipEncodingPlatformAgnostic( + Buffer.from([ + 31, 139, 8, 0, 0, 0, 0, 0, 0, 19, 99, 100, 98, 102, 97, 5, 0, 244, 153, 11, + 71, 5, 0, 0, 0, + ]), +); +const faviconGzip = makeGzipEncodingPlatformAgnostic( + Buffer.from([ + 31, 139, 8, 0, 0, 0, 0, 0, 0, 19, 1, 239, 3, 16, 252, 137, 80, 78, 71, 13, + 10, 26, 10, 0, 0, 0, 13, 73, 72, 68, 82, 0, 0, 0, 57, 0, 0, 0, 57, 8, 6, 0, + 0, 0, 140, 24, 131, 133, 0, 0, 0, 9, 112, 72, 89, 115, 0, 0, 11, 19, 0, 0, + 11, 19, 1, 0, 154, 156, 24, 0, 0, 0, 1, 115, 82, 71, 66, 0, 174, 206, 28, + 233, 0, 0, 0, 4, 103, 65, 77, 65, 0, 0, 177, 143, 11, 252, 97, 5, 0, 0, 3, + 132, 73, 68, 65, 84, 120, 1, 237, 154, 221, 81, 219, 64, 16, 199, 119, 79, + 146, 163, 129, 153, 140, 58, 136, 233, 0, 42, 8, 84, 16, 83, 65, 148, 4, 50, + 121, 195, 169, 32, 80, 1, 230, 45, 19, 200, 96, 58, 128, 14, 220, 65, 220, + 65, 92, 130, 94, 32, 138, 101, 105, 115, 39, 227, 144, 113, 140, 181, 39, + 233, 78, 246, 192, 239, 193, 2, 188, 30, 230, 175, 93, 239, 215, 9, 224, 9, + 128, 160, 73, 16, 82, 16, 251, 16, 64, 67, 248, 49, 68, 81, 31, 35, 157, + 207, 184, 160, 201, 111, 55, 9, 69, 42, 78, 161, 33, 198, 46, 157, 200, 203, + 177, 206, 103, 4, 104, 242, 98, 226, 245, 229, 69, 235, 78, 54, 141, 182, + 72, 21, 42, 68, 217, 9, 172, 17, 218, 34, 21, 191, 46, 90, 61, 249, 101, 30, + 192, 154, 80, 74, 164, 34, 117, 156, 119, 128, 56, 130, 53, 160, 180, 200, + 248, 43, 142, 18, 74, 247, 215, 65, 104, 105, 145, 138, 228, 188, 53, 204, + 132, 216, 91, 117, 161, 149, 68, 42, 148, 71, 239, 190, 57, 91, 25, 192, + 202, 134, 111, 101, 145, 51, 226, 115, 183, 159, 139, 37, 218, 147, 73, 233, + 12, 41, 79, 76, 139, 74, 77, 4, 132, 67, 155, 55, 68, 187, 25, 40, 34, 190, + 240, 6, 48, 151, 121, 253, 79, 212, 206, 223, 147, 94, 247, 194, 241, 118, + 203, 19, 167, 68, 176, 13, 150, 168, 205, 147, 203, 80, 226, 84, 59, 182, + 249, 49, 61, 245, 60, 241, 131, 0, 118, 193, 34, 86, 68, 110, 30, 78, 194, + 177, 155, 254, 36, 162, 238, 227, 86, 50, 132, 13, 81, 123, 184, 254, 139, + 31, 82, 219, 241, 210, 203, 220, 115, 5, 163, 0, 2, 157, 73, 187, 75, 48, + 128, 81, 79, 10, 55, 61, 230, 133, 38, 14, 83, 162, 17, 24, 194, 152, 72, + 53, 146, 73, 247, 188, 229, 216, 42, 47, 130, 65, 140, 137, 76, 188, 180, + 195, 181, 149, 45, 226, 0, 12, 98, 50, 92, 153, 94, 132, 129, 202, 190, 96, + 16, 35, 34, 85, 194, 209, 40, 19, 87, 249, 107, 134, 109, 48, 132, 17, 145, + 194, 157, 132, 92, 91, 47, 113, 174, 193, 48, 102, 194, 21, 5, 43, 84, 165, + 97, 95, 119, 95, 83, 134, 218, 69, 250, 7, 201, 46, 192, 180, 141, 43, 4, + 233, 6, 44, 80, 123, 51, 32, 0, 67, 150, 33, 230, 211, 139, 241, 80, 85, + 212, 31, 174, 8, 111, 88, 118, 100, 111, 125, 82, 171, 72, 213, 163, 202, + 11, 107, 39, 155, 81, 118, 5, 150, 168, 219, 147, 188, 132, 35, 67, 245, + 126, 36, 179, 66, 109, 34, 245, 106, 163, 61, 47, 42, 106, 19, 137, 110, + 194, 110, 227, 50, 225, 246, 193, 34, 245, 137, 68, 231, 136, 101, 103, 161, + 141, 155, 167, 22, 145, 90, 181, 113, 214, 198, 89, 164, 22, 145, 236, 218, + 8, 230, 39, 142, 69, 212, 19, 174, 136, 175, 57, 102, 4, 116, 99, 59, 84, + 21, 149, 69, 78, 107, 35, 47, 84, 165, 199, 173, 116, 56, 255, 255, 223, + 234, 48, 155, 113, 121, 43, 16, 95, 65, 3, 84, 18, 169, 57, 55, 74, 149, + 116, 188, 113, 152, 94, 78, 19, 149, 61, 42, 53, 232, 114, 19, 183, 75, 160, + 11, 133, 2, 49, 220, 56, 152, 68, 136, 112, 191, 134, 196, 64, 174, 43, 219, + 96, 136, 74, 34, 9, 240, 139, 122, 45, 5, 66, 240, 16, 5, 84, 226, 233, 5, + 62, 165, 195, 117, 243, 112, 188, 173, 81, 27, 27, 165, 180, 72, 34, 209, + 133, 53, 161, 124, 226, 121, 172, 54, 170, 211, 170, 21, 59, 194, 43, 37, + 114, 105, 109, 148, 9, 100, 213, 78, 160, 75, 137, 148, 153, 112, 233, 196, + 225, 33, 118, 90, 99, 177, 131, 25, 24, 221, 140, 115, 209, 22, 169, 106, + 163, 244, 210, 242, 21, 7, 97, 62, 145, 220, 126, 119, 187, 8, 217, 14, 90, + 92, 117, 44, 66, 91, 164, 170, 141, 12, 179, 64, 61, 185, 165, 126, 184, 61, + 111, 13, 111, 47, 220, 189, 204, 113, 182, 136, 178, 207, 75, 78, 160, 141, + 161, 93, 39, 101, 109, 60, 226, 212, 70, 129, 66, 121, 187, 55, 251, 253, + 190, 49, 239, 205, 254, 22, 116, 41, 136, 227, 135, 125, 16, 78, 146, 14, + 162, 153, 199, 217, 180, 68, 170, 218, 40, 39, 9, 214, 49, 184, 188, 25, + 163, 101, 239, 71, 189, 124, 169, 252, 215, 163, 254, 123, 217, 1, 57, 96, + 4, 173, 112, 213, 169, 141, 54, 183, 113, 69, 232, 125, 39, 153, 115, 163, + 237, 109, 92, 17, 108, 145, 27, 31, 38, 29, 118, 27, 215, 112, 54, 157, 135, + 239, 73, 4, 246, 54, 46, 129, 116, 37, 234, 227, 12, 150, 200, 105, 109, + 228, 14, 199, 56, 84, 143, 163, 193, 10, 193, 18, 201, 172, 141, 57, 166, + 207, 255, 203, 192, 18, 57, 173, 141, 60, 154, 216, 198, 21, 193, 170, 147, + 153, 35, 246, 129, 73, 217, 109, 156, 255, 210, 185, 150, 205, 193, 160, + 208, 206, 135, 232, 14, 158, 121, 154, 252, 1, 139, 232, 48, 101, 122, 172, + 204, 168, 0, 0, 0, 0, 73, 69, 78, 68, 174, 66, 96, 130, 145, 24, 164, 164, + 239, 3, 0, 0, + ]), +); + +const emptyBr = Buffer.from([59]); +const smallBr = Buffer.from([11, 2, 128, 1, 2, 3, 4, 5, 3]); +const faviconBr = Buffer.from([ + 11, 247, 129, 137, 80, 78, 71, 13, 10, 26, 10, 0, 0, 0, 13, 73, 72, 68, 82, 0, + 0, 0, 57, 0, 0, 0, 57, 8, 6, 0, 0, 0, 140, 24, 131, 133, 0, 0, 0, 9, 112, 72, + 89, 115, 0, 0, 11, 19, 0, 0, 11, 19, 1, 0, 154, 156, 24, 0, 0, 0, 1, 115, 82, + 71, 66, 0, 174, 206, 28, 233, 0, 0, 0, 4, 103, 65, 77, 65, 0, 0, 177, 143, 11, + 252, 97, 5, 0, 0, 3, 132, 73, 68, 65, 84, 120, 1, 237, 154, 221, 81, 219, 64, + 16, 199, 119, 79, 146, 163, 129, 153, 140, 58, 136, 233, 0, 42, 8, 84, 16, 83, + 65, 148, 4, 50, 121, 195, 169, 32, 80, 1, 230, 45, 19, 200, 96, 58, 128, 14, + 220, 65, 220, 65, 92, 130, 94, 32, 138, 101, 105, 115, 39, 227, 144, 113, 140, + 181, 39, 233, 78, 246, 192, 239, 193, 2, 188, 30, 230, 175, 93, 239, 215, 9, + 224, 9, 128, 160, 73, 16, 82, 16, 251, 16, 64, 67, 248, 49, 68, 81, 31, 35, + 157, 207, 184, 160, 201, 111, 55, 9, 69, 42, 78, 161, 33, 198, 46, 157, 200, + 203, 177, 206, 103, 4, 104, 242, 98, 226, 245, 229, 69, 235, 78, 54, 141, 182, + 72, 21, 42, 68, 217, 9, 172, 17, 218, 34, 21, 191, 46, 90, 61, 249, 101, 30, + 192, 154, 80, 74, 164, 34, 117, 156, 119, 128, 56, 130, 53, 160, 180, 200, + 248, 43, 142, 18, 74, 247, 215, 65, 104, 105, 145, 138, 228, 188, 53, 204, + 132, 216, 91, 117, 161, 149, 68, 42, 148, 71, 239, 190, 57, 91, 25, 192, 202, + 134, 111, 101, 145, 51, 226, 115, 183, 159, 139, 37, 218, 147, 73, 233, 12, + 41, 79, 76, 139, 74, 77, 4, 132, 67, 155, 55, 68, 187, 25, 40, 34, 190, 240, + 6, 48, 151, 121, 253, 79, 212, 206, 223, 147, 94, 247, 194, 241, 118, 203, 19, + 167, 68, 176, 13, 150, 168, 205, 147, 203, 80, 226, 84, 59, 182, 249, 49, 61, + 245, 60, 241, 131, 0, 118, 193, 34, 86, 68, 110, 30, 78, 194, 177, 155, 254, + 36, 162, 238, 227, 86, 50, 132, 13, 81, 123, 184, 254, 139, 31, 82, 219, 241, + 210, 203, 220, 115, 5, 163, 0, 2, 157, 73, 187, 75, 48, 128, 81, 79, 10, 55, + 61, 230, 133, 38, 14, 83, 162, 17, 24, 194, 152, 72, 53, 146, 73, 247, 188, + 229, 216, 42, 47, 130, 65, 140, 137, 76, 188, 180, 195, 181, 149, 45, 226, 0, + 12, 98, 50, 92, 153, 94, 132, 129, 202, 190, 96, 16, 35, 34, 85, 194, 209, 40, + 19, 87, 249, 107, 134, 109, 48, 132, 17, 145, 194, 157, 132, 92, 91, 47, 113, + 174, 193, 48, 102, 194, 21, 5, 43, 84, 165, 97, 95, 119, 95, 83, 134, 218, 69, + 250, 7, 201, 46, 192, 180, 141, 43, 4, 233, 6, 44, 80, 123, 51, 32, 0, 67, + 150, 33, 230, 211, 139, 241, 80, 85, 212, 31, 174, 8, 111, 88, 118, 100, 111, + 125, 82, 171, 72, 213, 163, 202, 11, 107, 39, 155, 81, 118, 5, 150, 168, 219, + 147, 188, 132, 35, 67, 245, 126, 36, 179, 66, 109, 34, 245, 106, 163, 61, 47, + 42, 106, 19, 137, 110, 194, 110, 227, 50, 225, 246, 193, 34, 245, 137, 68, + 231, 136, 101, 103, 161, 141, 155, 167, 22, 145, 90, 181, 113, 214, 198, 89, + 164, 22, 145, 236, 218, 8, 230, 39, 142, 69, 212, 19, 174, 136, 175, 57, 102, + 4, 116, 99, 59, 84, 21, 149, 69, 78, 107, 35, 47, 84, 165, 199, 173, 116, 56, + 255, 255, 223, 234, 48, 155, 113, 121, 43, 16, 95, 65, 3, 84, 18, 169, 57, 55, + 74, 149, 116, 188, 113, 152, 94, 78, 19, 149, 61, 42, 53, 232, 114, 19, 183, + 75, 160, 11, 133, 2, 49, 220, 56, 152, 68, 136, 112, 191, 134, 196, 64, 174, + 43, 219, 96, 136, 74, 34, 9, 240, 139, 122, 45, 5, 66, 240, 16, 5, 84, 226, + 233, 5, 62, 165, 195, 117, 243, 112, 188, 173, 81, 27, 27, 165, 180, 72, 34, + 209, 133, 53, 161, 124, 226, 121, 172, 54, 170, 211, 170, 21, 59, 194, 43, 37, + 114, 105, 109, 148, 9, 100, 213, 78, 160, 75, 137, 148, 153, 112, 233, 196, + 225, 33, 118, 90, 99, 177, 131, 25, 24, 221, 140, 115, 209, 22, 169, 106, 163, + 244, 210, 242, 21, 7, 97, 62, 145, 220, 126, 119, 187, 8, 217, 14, 90, 92, + 117, 44, 66, 91, 164, 170, 141, 12, 179, 64, 61, 185, 165, 126, 184, 61, 111, + 13, 111, 47, 220, 189, 204, 113, 182, 136, 178, 207, 75, 78, 160, 141, 161, + 93, 39, 101, 109, 60, 226, 212, 70, 129, 66, 121, 187, 55, 251, 253, 190, 49, + 239, 205, 254, 22, 116, 41, 136, 227, 135, 125, 16, 78, 146, 14, 162, 153, + 199, 217, 180, 68, 170, 218, 40, 39, 9, 214, 49, 184, 188, 25, 163, 101, 239, + 71, 189, 124, 169, 252, 215, 163, 254, 123, 217, 1, 57, 96, 4, 173, 112, 213, + 169, 141, 54, 183, 113, 69, 232, 125, 39, 153, 115, 163, 237, 109, 92, 17, + 108, 145, 27, 31, 38, 29, 118, 27, 215, 112, 54, 157, 135, 239, 73, 4, 246, + 54, 46, 129, 116, 37, 234, 227, 12, 150, 200, 105, 109, 228, 14, 199, 56, 84, + 143, 163, 193, 10, 193, 18, 201, 172, 141, 57, 166, 207, 255, 203, 192, 18, + 57, 173, 141, 60, 154, 216, 198, 21, 193, 170, 147, 153, 35, 246, 129, 73, + 217, 109, 156, 255, 210, 185, 150, 205, 193, 160, 208, 206, 135, 232, 14, 158, + 121, 154, 252, 1, 139, 232, 48, 101, 122, 172, 204, 168, 0, 0, 0, 0, 73, 69, + 78, 68, 174, 66, 96, 130, 3, +]); + +describe("convertHttpContentEncodingToCompressionAlgorithm", () => { + it("maps the expected values correctly", () => { + expect(convertHttpContentEncodingToCompressionAlgorithm("")).toEqual( + "none", + ); + expect(convertHttpContentEncodingToCompressionAlgorithm("br")).toEqual( + "br", + ); + expect(convertHttpContentEncodingToCompressionAlgorithm("gzip")).toEqual( + "gzip", + ); + }); + + it("throws an exception on unexpected values", () => { + expect(() => { + convertHttpContentEncodingToCompressionAlgorithm("chicken"); + }).toThrow(); + expect(() => { + convertHttpContentEncodingToCompressionAlgorithm("thrift"); + }).toThrow(); + }); +}); + +describe("compressBuffer", () => { + describe("none", () => { + const algorithm: CompressionAlgorithm = "none"; + + it("handles an empty buffer correctly", () => { + expect(compressBuffer(algorithm, empty)).toEqual(empty); + }); + + it("handles a small buffer correctly", () => { + expect(compressBuffer(algorithm, small)).toEqual(small); + }); + + it("handles a large buffer correctly", () => { + expect(compressBuffer(algorithm, favicon)).toEqual(favicon); + }); + }); + + describe("gzip", () => { + const algorithm: CompressionAlgorithm = "gzip"; + + it("handles an empty buffer correctly", () => { + expect( + makeGzipEncodingPlatformAgnostic(compressBuffer(algorithm, empty)), + ).toEqual(emptyGzip); + }); + + it("handles a small buffer correctly", () => { + expect( + makeGzipEncodingPlatformAgnostic(compressBuffer(algorithm, small)), + ).toEqual(smallGzip); + }); + + it("handles a large buffer correctly", () => { + expect( + makeGzipEncodingPlatformAgnostic(compressBuffer(algorithm, favicon)), + ).toEqual(faviconGzip); + }); + }); + + describe("br", () => { + const algorithm: CompressionAlgorithm = "br"; + + it("handles an empty buffer correctly", () => { + expect(compressBuffer(algorithm, empty)).toEqual(emptyBr); + }); + + it("handles a small buffer correctly", () => { + expect(compressBuffer(algorithm, small)).toEqual(smallBr); + }); + + it("handles a large buffer correctly", () => { + expect(compressBuffer(algorithm, favicon)).toEqual(faviconBr); + }); + }); + + it("throws an excpetion on unexpected values ", () => { + expect(() => { + compressBuffer("chicken" as CompressionAlgorithm, small); + }).toThrow(); + }); +}); + +describe("decompressBuffer", () => { + describe("none", () => { + const algorithm: CompressionAlgorithm = "none"; + + it("handles an empty buffer correctly", () => { + expect(decompressBuffer(algorithm, empty)).toEqual(empty); + }); + + it("handles a small buffer correctly", () => { + expect(decompressBuffer(algorithm, small)).toEqual(small); + }); + + it("handles a large buffer correctly", () => { + expect(decompressBuffer(algorithm, favicon)).toEqual(favicon); + }); + }); + + describe("gzip", () => { + const algorithm: CompressionAlgorithm = "gzip"; + + it("handles an empty buffer correctly", () => { + expect(decompressBuffer(algorithm, emptyGzip)).toEqual(empty); + }); + + it("handles a small buffer correctly", () => { + expect(decompressBuffer(algorithm, smallGzip)).toEqual(small); + }); + + it("handles a large buffer correctly", () => { + expect(decompressBuffer(algorithm, faviconGzip)).toEqual(favicon); + }); + }); + + describe("br", () => { + const algorithm: CompressionAlgorithm = "br"; + + it("handles an empty buffer correctly", () => { + expect(decompressBuffer(algorithm, emptyBr)).toEqual(empty); + }); + + it("handles a small buffer correctly", () => { + expect(decompressBuffer(algorithm, smallBr)).toEqual(small); + }); + + it("handles a large buffer correctly", () => { + expect(decompressBuffer(algorithm, faviconBr)).toEqual(favicon); + }); + }); + + it("throws an excpetion on unexpected values ", () => { + expect(() => { + decompressBuffer("chicken" as CompressionAlgorithm, small); + }).toThrow(); + }); +}); diff --git a/src/compression.ts b/src/compression.ts new file mode 100644 index 00000000..345c6a46 --- /dev/null +++ b/src/compression.ts @@ -0,0 +1,50 @@ +import zlib from "zlib"; + +export type CompressionAlgorithm = "br" | "gzip" | "none"; + +export function compressBuffer( + algorithm: CompressionAlgorithm, + buffer: Buffer, +): Buffer { + switch (algorithm) { + case "none": + return buffer; + case "br": + return zlib.brotliCompressSync(buffer); + case "gzip": + return zlib.gzipSync(buffer); + default: + throw new Error(`Unhandled compression algorithm value "${algorithm}"`); + } +} + +export function decompressBuffer( + algorithm: CompressionAlgorithm, + buffer: Buffer, +): Buffer { + switch (algorithm) { + case "none": + return buffer; + case "br": + return zlib.brotliDecompressSync(buffer); + case "gzip": + return zlib.gunzipSync(buffer); + default: + throw new Error(`Unhandled compression algorithm value "${algorithm}"`); + } +} + +export function convertHttpContentEncodingToCompressionAlgorithm( + contentEncoding: string, +): CompressionAlgorithm { + switch (contentEncoding) { + case "": + return "none"; + case "br": + return "br"; + case "gzip": + return "gzip"; + default: + throw new Error(`Unhandled content-encoding value "${contentEncoding}"`); + } +} diff --git a/src/http.ts b/src/http.ts index a703b2cd..d300217a 100644 --- a/src/http.ts +++ b/src/http.ts @@ -1,6 +1,8 @@ -import brotli from "brotli"; -import zlib from "zlib"; import { ParsedMediaType as ParsedContentType } from "content-type"; +import { + convertHttpContentEncodingToCompressionAlgorithm, + decompressBuffer, +} from "./compression"; /** * Headers of a request or response. @@ -35,7 +37,7 @@ export interface HttpResponse { body: Buffer; } -export function getHeaderAsString( +export function getHttpHeaderAsString( headers: HttpHeaders, headerName: string, ): string { @@ -49,35 +51,28 @@ export function getHeaderAsString( } } -export function getHttpRequestContentType(request: HttpRequest): string { +export function getHttpContentEncoding(r: HttpRequest | HttpResponse): string { + return getHttpHeaderAsString(r.headers, "content-encoding"); +} + +export function getHttpContentType(r: HttpRequest | HttpResponse): string { return ( - getHeaderAsString(request.headers, "content-type") || + getHttpHeaderAsString(r.headers, "content-type") || "application/octet-stream" ); } -export function getHttpRequestBodyDecoded(request: HttpRequest): Buffer { - // Process the content-encoding before looking at the content-type. - const contentEncoding = getHeaderAsString( - request.headers, - "content-encoding", - ); - switch (contentEncoding) { - case "": - return request.body; - case "br": - return Buffer.from(brotli.decompress(request.body)); - case "gzip": - return zlib.gunzipSync(request.body); - default: - throw Error(`Unhandled content-encoding value "${contentEncoding}"`); - } +export function getHttpBodyDecoded(r: HttpRequest | HttpResponse): Buffer { + const contentEncoding = getHttpHeaderAsString(r.headers, "content-encoding"); + const compressionAlgorithm = + convertHttpContentEncodingToCompressionAlgorithm(contentEncoding); + return decompressBuffer(compressionAlgorithm, r.body); } -export function decodeHttpRequestBodyToString( - request: HttpRequest, +export function decodeHttpBodyToString( + r: HttpRequest | HttpResponse, contentType: ParsedContentType, ): string { const encoding = contentType.parameters.charset as BufferEncoding | undefined; - return getHttpRequestBodyDecoded(request).toString(encoding || "utf-8"); + return getHttpBodyDecoded(r).toString(encoding || "utf-8"); } diff --git a/src/persistence.spec.ts b/src/persistence.spec.ts index f5d3be2a..7aa35131 100644 --- a/src/persistence.spec.ts +++ b/src/persistence.spec.ts @@ -1,5 +1,4 @@ -import brotli from "brotli"; -import { gzipSync } from "zlib"; +import { brotliCompressSync, gzipSync } from "zlib"; import { persistTape, reviveTape, redactRequestHeaders } from "./persistence"; // Note the repetition. This is necessary otherwise Brotli compression @@ -20,13 +19,15 @@ const BINARY_RESPONSE = Buffer.from([ ]); const UTF8_REQUEST_BROTLI = Buffer.from( - brotli.compress(Buffer.from(UTF8_REQUEST, "utf8"))!, + brotliCompressSync(Buffer.from(UTF8_REQUEST, "utf8"))!, ); const UTF8_RESPONSE_BROTLI = Buffer.from( - brotli.compress(Buffer.from(UTF8_RESPONSE, "utf8"))!, + brotliCompressSync(Buffer.from(UTF8_RESPONSE, "utf8"))!, +); +const BINARY_REQUEST_BROTLI = Buffer.from(brotliCompressSync(BINARY_REQUEST)!); +const BINARY_RESPONSE_BROTLI = Buffer.from( + brotliCompressSync(BINARY_RESPONSE)!, ); -const BINARY_REQUEST_BROTLI = Buffer.from(brotli.compress(BINARY_REQUEST)!); -const BINARY_RESPONSE_BROTLI = Buffer.from(brotli.compress(BINARY_RESPONSE)!); const UTF8_REQUEST_GZIP = gzipSync(Buffer.from(UTF8_REQUEST, "utf8")); const UTF8_RESPONSE_GZIP = gzipSync(Buffer.from(UTF8_RESPONSE, "utf8")); @@ -221,7 +222,7 @@ describe("Persistence", () => { }, body: { encoding: "base64", - data: "GxcAAI6UrMm1WkAERl0HoDFuCn3CIekc", + data: "GxcA+I+UrMm1WkAERl0HoDFuCn3CAZLOAQ==", }, }, response: { @@ -233,7 +234,7 @@ describe("Persistence", () => { }, body: { encoding: "base64", - data: "GxcAAI6UrPmFmgFmOV+HoM3+C33CIe4U", + data: "GxcA+I+UrPmFmgFmOV+HoM3+C33CAeJOAQ==", }, }, }); diff --git a/src/persistence.ts b/src/persistence.ts index c43cc035..455cb169 100644 --- a/src/persistence.ts +++ b/src/persistence.ts @@ -1,15 +1,17 @@ -import brotli from "brotli"; import fs from "fs-extra"; import yaml from "js-yaml"; import path from "path"; -import { gunzipSync, gzipSync } from "zlib"; -import { HttpHeaders } from "./http"; import { - CompressionAlgorithm, - PersistedBuffer, - PersistedTapeRecord, - TapeRecord, -} from "./tape"; + compressBuffer, + convertHttpContentEncodingToCompressionAlgorithm, +} from "./compression"; +import { + getHttpBodyDecoded, + getHttpContentEncoding, + HttpRequest, + HttpResponse, +} from "./http"; +import { PersistedBuffer, PersistedTapeRecord, TapeRecord } from "./tape"; /** * Persistence layer to save tapes to disk and read them from disk. @@ -94,12 +96,12 @@ export function persistTape(record: TapeRecord): PersistedTapeRecord { method: record.request.method, path: record.request.path, headers: record.request.headers, - body: serialiseBuffer(record.request.body, record.request.headers), + body: serialiseForTape(record.request), }, response: { status: record.response.status, headers: record.response.headers, - body: serialiseBuffer(record.response.body, record.response.headers), + body: serialiseForTape(record.response), }, }; } @@ -120,22 +122,12 @@ export function reviveTape(persistedRecord: PersistedTapeRecord): TapeRecord { }; } -export function serialiseBuffer( - buffer: Buffer, - headers: HttpHeaders, -): PersistedBuffer { - const header = headers["content-encoding"]; - const contentEncoding = typeof header === "string" ? header : undefined; - const originalBuffer = buffer; - let compression: CompressionAlgorithm = "none"; - if (contentEncoding === "br") { - buffer = Buffer.from(brotli.decompress(buffer)); - compression = "br"; - } - if (contentEncoding === "gzip") { - buffer = gunzipSync(buffer); - compression = "gzip"; - } +function serialiseForTape(r: HttpRequest | HttpResponse): PersistedBuffer { + const buffer = getHttpBodyDecoded(r); + const contentEncoding = getHttpContentEncoding(r); + const compressionAlgorithm = + convertHttpContentEncodingToCompressionAlgorithm(contentEncoding); + const utf8Representation = buffer.toString("utf8"); try { // Can it be safely stored and recreated in YAML? @@ -148,17 +140,18 @@ export function serialiseBuffer( return { encoding: "utf8", data: utf8Representation, - compression, + compression: compressionAlgorithm, }; } } catch { // Fall through. } + // No luck. Fall back to Base64, persisting the original buffer // since we might as well store it in its compressed state. return { encoding: "base64", - data: originalBuffer.toString("base64"), + data: r.body.toString("base64"), }; } @@ -170,21 +163,7 @@ function unserialiseBuffer(persisted: PersistedBuffer): Buffer { break; case "utf8": buffer = Buffer.from(persisted.data, "utf8"); - if (persisted.compression === "br") { - // TODO: Find a workaround for the new compressed message not necessarily - // being identical to what was originally sent (update Content-Length?). - const compressed = brotli.compress(buffer); - if (compressed) { - buffer = Buffer.from(compressed); - } else { - throw new Error(`Brotli compression failed!`); - } - } - if (persisted.compression === "gzip") { - // TODO: Find a workaround for the new compressed message not necessarily - // being identical to what was originally sent (update Content-Length?). - buffer = gzipSync(buffer); - } + buffer = compressBuffer(persisted.compression || "none", buffer); break; default: throw new Error(`Unsupported encoding!`); diff --git a/src/similarity.ts b/src/similarity.ts index 2000e04d..e70276d8 100644 --- a/src/similarity.ts +++ b/src/similarity.ts @@ -8,8 +8,8 @@ import { compareTwoStrings } from "string-similarity"; import { RewriteRules } from "./rewrite"; import { TapeRecord } from "./tape"; import { - decodeHttpRequestBodyToString, - getHttpRequestContentType, + decodeHttpBodyToString, + getHttpContentType, HttpHeaders, HttpRequest, } from "./http"; @@ -79,8 +79,8 @@ function countBodyDifferences( request2: HttpRequest, rewriteBeforeDiffRules: RewriteRules, ): number { - const contentType1 = parseContentType(getHttpRequestContentType(request1)); - const contentType2 = parseContentType(getHttpRequestContentType(request1)); + const contentType1 = parseContentType(getHttpContentType(request1)); + const contentType2 = parseContentType(getHttpContentType(request1)); // If the content types are not the same, we cannot compare. if (contentType1.type !== contentType2.type) { @@ -118,8 +118,8 @@ function countBodyDifferencesApplicationJson( rewriteBeforeDiffRules: RewriteRules, ): number { // Decode the bodies to strings. - const body1 = decodeHttpRequestBodyToString(request1, contentType1); - const body2 = decodeHttpRequestBodyToString(request2, contentType2); + const body1 = decodeHttpBodyToString(request1, contentType1); + const body2 = decodeHttpBodyToString(request2, contentType2); // Early bail if bodies are empty. if (body1.length === 0 && body1.length === body2.length) { @@ -149,8 +149,8 @@ function countBodyDifferencesText( rewriteBeforeDiffRules: RewriteRules, ): number { // Decode the bodies to strings. - const body1 = decodeHttpRequestBodyToString(request1, contentType1); - const body2 = decodeHttpRequestBodyToString(request2, contentType2); + const body1 = decodeHttpBodyToString(request1, contentType1); + const body2 = decodeHttpBodyToString(request2, contentType2); // Early bail if bodies are empty. if (body1.length === 0 && body1.length === body2.length) { diff --git a/src/tape.ts b/src/tape.ts index e9836f6d..917c2f69 100644 --- a/src/tape.ts +++ b/src/tape.ts @@ -1,3 +1,4 @@ +import { CompressionAlgorithm } from "./compression"; import { HttpHeaders, HttpRequest, HttpResponse } from "./http"; /** @@ -37,8 +38,6 @@ export type PersistedBuffer = } | { encoding: "utf8"; - compression: CompressionAlgorithm; + compression: CompressionAlgorithm | undefined; data: string; }; - -export type CompressionAlgorithm = "br" | "gzip" | "none"; diff --git a/src/tests/setup.ts b/src/tests/setup.ts index c883c346..82224fc2 100644 --- a/src/tests/setup.ts +++ b/src/tests/setup.ts @@ -37,6 +37,7 @@ export function setupServers({ defaultTapeName, host: TEST_SERVER_HOST, timeout: 100, + enableLogging: true, unframeGrpcWebJsonRequestsHostnames, }); await Promise.all([ diff --git a/tsconfig.json b/tsconfig.json index 9dd5e53f..e17fc4ca 100644 --- a/tsconfig.json +++ b/tsconfig.json @@ -38,9 +38,7 @@ /* Module Resolution Options */ // "moduleResolution": "node", /* Specify module resolution strategy: 'node' (Node.js) or 'classic' (TypeScript pre-1.6). */ "baseUrl": ".", /* Base directory to resolve non-absolute module names. */ - "paths": { - "brotli": ["src/brotli.d.ts"] - }, /* A series of entries which re-map imports to lookup locations relative to the 'baseUrl'. */ + "paths": {}, /* A series of entries which re-map imports to lookup locations relative to the 'baseUrl'. */ "rootDirs": ["src"], /* List of root folders whose combined content represents the structure of the project at runtime. */ // "typeRoots": ["], /* List of folders to include type definitions from. */ // "types": [], /* Type declaration files to be included in compilation. */ @@ -58,4 +56,4 @@ // "experimentalDecorators": true, /* Enables experimental support for ES7 decorators. */ // "emitDecoratorMetadata": true, /* Enables experimental support for emitting type metadata for decorators. */ } -} \ No newline at end of file +} diff --git a/yarn.lock b/yarn.lock index abe90f82..c4222d8c 100644 --- a/yarn.lock +++ b/yarn.lock @@ -1015,11 +1015,6 @@ balanced-match@^1.0.0: resolved "https://registry.npmjs.org/balanced-match/-/balanced-match-1.0.2.tgz" integrity sha512-3oSeUO0TMV67hN1AmbXsK4yaqU7tjiHlbxRDZOpH0KW9+CeX4bRAaX0Anxt0tx2MrpRpWwQaPwIlISEJhYU5Pw== -base64-js@^1.1.2: - version "1.5.1" - resolved "https://registry.npmjs.org/base64-js/-/base64-js-1.5.1.tgz" - integrity sha512-AKpaYlHn8t4SVbOHCy+b5+KKgvR4vrsD8vbvrbiQJps7fKDTkjkDry6ji0rUJjC0kzbNePLwzxq8iypo41qeWA== - binary-extensions@^2.0.0: version "2.2.0" resolved "https://registry.npmjs.org/binary-extensions/-/binary-extensions-2.2.0.tgz" @@ -1065,13 +1060,6 @@ braces@^3.0.2, braces@~3.0.2: dependencies: fill-range "^7.0.1" -brotli@^1.3.3: - version "1.3.3" - resolved "https://registry.npmjs.org/brotli/-/brotli-1.3.3.tgz" - integrity sha512-oTKjJdShmDuGW94SyyaoQvAjf30dZaHnjJ8uAF+u2/vGJkJbJPJAT1gDiOJP5v1Zb6f9KEyW/1HpuaWIXtGHPg== - dependencies: - base64-js "^1.1.2" - browserslist@^4.21.9: version "4.22.1" resolved "https://registry.npmjs.org/browserslist/-/browserslist-4.22.1.tgz"