Skip to content
New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Jump to bottom

[TEX-537] Refactor compression #555

Merged
merged 12 commits into from
Dec 10, 2023
Merged

[TEX-537] Refactor compression #555

Show file tree
Hide file tree
Changes from 6 commits
Commits
Show all changes
12 commits
Select commit Hold shift + click to select a range
c2e7f52
[TEX-537] Refactor HTTP request and response logic.
timdawborn Dec 8, 2023
6474958
Add forgotten dependency updates.
timdawborn Dec 8, 2023
2bcf63d
Update binary comparison logic to be the same as before.
timdawborn Dec 8, 2023
eb337d7
[TEX-537] Refactor compression.
timdawborn Dec 8, 2023
8d91ff3
Fix old tapes issue.
timdawborn Dec 8, 2023
a9efac5
Add unit test.
timdawborn Dec 8, 2023
1ca870c
Use builtin brotli functionality.
timdawborn Dec 8, 2023
4ed1aa3
Add brotli tests.
timdawborn Dec 8, 2023
26e0391
Cleanup gzip mess.
timdawborn Dec 8, 2023
fa3152b
Try enabling NodeJS 20 again.
timdawborn Dec 8, 2023
c1a3a68
Revert "Try enabling NodeJS 20 again."
timdawborn Dec 8, 2023
17fc13c
Merge branch 'master' into TEX-537-refactor-compression
timdawborn Dec 10, 2023
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
3 changes: 2 additions & 1 deletion package.json
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -54,13 +54,14 @@
"brotli": "^1.3.3",
Copy link
Contributor Author

@timdawborn timdawborn Dec 8, 2023 •
edited
Loading

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

Replacing this third-party library with the brotli implementation provided by the NodeJS builtin zlib module.

"chalk": "^4.1.0",
"commander": "^10.0.1",
"content-type": "^1.0.5",
"deep-diff": "^1.0.2",
"fs-extra": "^11.1.1",
"js-yaml": "^4.1.0",
"query-string": "^6.13.6",
"string-similarity": "^4.0.4"
},
"devDependencies": {
"@types/content-type": "^1.1.8",
"@types/deep-diff": "^1.0.5",
"@types/express": "^4.17.21",
"@types/fs-extra": "^11.0.4",
Expand Down
235 changes: 235 additions & 0 deletions src/compression.spec.ts
View file
Open in desktop
Original file line number Diff line number Diff line change
@@ -0,0 +1,235 @@
import {
compressBuffer,
CompressionAlgorithm,
convertHttpContentEncodingToCompressionAlgorithm,
decompressBuffer,
} from "./compression";

const empty = Buffer.from([]);
const small = Buffer.from([1, 2, 3, 4, 5]);
const favicon = Buffer.from([
137, 80, 78, 71, 13, 10, 26, 10, 0, 0, 0, 13, 73, 72, 68, 82, 0, 0, 0, 57, 0,
0, 0, 57, 8, 6, 0, 0, 0, 140, 24, 131, 133, 0, 0, 0, 9, 112, 72, 89, 115, 0,
0, 11, 19, 0, 0, 11, 19, 1, 0, 154, 156, 24, 0, 0, 0, 1, 115, 82, 71, 66, 0,
174, 206, 28, 233, 0, 0, 0, 4, 103, 65, 77, 65, 0, 0, 177, 143, 11, 252, 97,
5, 0, 0, 3, 132, 73, 68, 65, 84, 120, 1, 237, 154, 221, 81, 219, 64, 16, 199,
119, 79, 146, 163, 129, 153, 140, 58, 136, 233, 0, 42, 8, 84, 16, 83, 65, 148,
4, 50, 121, 195, 169, 32, 80, 1, 230, 45, 19, 200, 96, 58, 128, 14, 220, 65,
220, 65, 92, 130, 94, 32, 138, 101, 105, 115, 39, 227, 144, 113, 140, 181, 39,
233, 78, 246, 192, 239, 193, 2, 188, 30, 230, 175, 93, 239, 215, 9, 224, 9,
128, 160, 73, 16, 82, 16, 251, 16, 64, 67, 248, 49, 68, 81, 31, 35, 157, 207,
184, 160, 201, 111, 55, 9, 69, 42, 78, 161, 33, 198, 46, 157, 200, 203, 177,
206, 103, 4, 104, 242, 98, 226, 245, 229, 69, 235, 78, 54, 141, 182, 72, 21,
42, 68, 217, 9, 172, 17, 218, 34, 21, 191, 46, 90, 61, 249, 101, 30, 192, 154,
80, 74, 164, 34, 117, 156, 119, 128, 56, 130, 53, 160, 180, 200, 248, 43, 142,
18, 74, 247, 215, 65, 104, 105, 145, 138, 228, 188, 53, 204, 132, 216, 91,
117, 161, 149, 68, 42, 148, 71, 239, 190, 57, 91, 25, 192, 202, 134, 111, 101,
145, 51, 226, 115, 183, 159, 139, 37, 218, 147, 73, 233, 12, 41, 79, 76, 139,
74, 77, 4, 132, 67, 155, 55, 68, 187, 25, 40, 34, 190, 240, 6, 48, 151, 121,
253, 79, 212, 206, 223, 147, 94, 247, 194, 241, 118, 203, 19, 167, 68, 176,
13, 150, 168, 205, 147, 203, 80, 226, 84, 59, 182, 249, 49, 61, 245, 60, 241,
131, 0, 118, 193, 34, 86, 68, 110, 30, 78, 194, 177, 155, 254, 36, 162, 238,
227, 86, 50, 132, 13, 81, 123, 184, 254, 139, 31, 82, 219, 241, 210, 203, 220,
115, 5, 163, 0, 2, 157, 73, 187, 75, 48, 128, 81, 79, 10, 55, 61, 230, 133,
38, 14, 83, 162, 17, 24, 194, 152, 72, 53, 146, 73, 247, 188, 229, 216, 42,
47, 130, 65, 140, 137, 76, 188, 180, 195, 181, 149, 45, 226, 0, 12, 98, 50,
92, 153, 94, 132, 129, 202, 190, 96, 16, 35, 34, 85, 194, 209, 40, 19, 87,
249, 107, 134, 109, 48, 132, 17, 145, 194, 157, 132, 92, 91, 47, 113, 174,
193, 48, 102, 194, 21, 5, 43, 84, 165, 97, 95, 119, 95, 83, 134, 218, 69, 250,
7, 201, 46, 192, 180, 141, 43, 4, 233, 6, 44, 80, 123, 51, 32, 0, 67, 150, 33,
230, 211, 139, 241, 80, 85, 212, 31, 174, 8, 111, 88, 118, 100, 111, 125, 82,
171, 72, 213, 163, 202, 11, 107, 39, 155, 81, 118, 5, 150, 168, 219, 147, 188,
132, 35, 67, 245, 126, 36, 179, 66, 109, 34, 245, 106, 163, 61, 47, 42, 106,
19, 137, 110, 194, 110, 227, 50, 225, 246, 193, 34, 245, 137, 68, 231, 136,
101, 103, 161, 141, 155, 167, 22, 145, 90, 181, 113, 214, 198, 89, 164, 22,
145, 236, 218, 8, 230, 39, 142, 69, 212, 19, 174, 136, 175, 57, 102, 4, 116,
99, 59, 84, 21, 149, 69, 78, 107, 35, 47, 84, 165, 199, 173, 116, 56, 255,
255, 223, 234, 48, 155, 113, 121, 43, 16, 95, 65, 3, 84, 18, 169, 57, 55, 74,
149, 116, 188, 113, 152, 94, 78, 19, 149, 61, 42, 53, 232, 114, 19, 183, 75,
160, 11, 133, 2, 49, 220, 56, 152, 68, 136, 112, 191, 134, 196, 64, 174, 43,
219, 96, 136, 74, 34, 9, 240, 139, 122, 45, 5, 66, 240, 16, 5, 84, 226, 233,
5, 62, 165, 195, 117, 243, 112, 188, 173, 81, 27, 27, 165, 180, 72, 34, 209,
133, 53, 161, 124, 226, 121, 172, 54, 170, 211, 170, 21, 59, 194, 43, 37, 114,
105, 109, 148, 9, 100, 213, 78, 160, 75, 137, 148, 153, 112, 233, 196, 225,
33, 118, 90, 99, 177, 131, 25, 24, 221, 140, 115, 209, 22, 169, 106, 163, 244,
210, 242, 21, 7, 97, 62, 145, 220, 126, 119, 187, 8, 217, 14, 90, 92, 117, 44,
66, 91, 164, 170, 141, 12, 179, 64, 61, 185, 165, 126, 184, 61, 111, 13, 111,
47, 220, 189, 204, 113, 182, 136, 178, 207, 75, 78, 160, 141, 161, 93, 39,
101, 109, 60, 226, 212, 70, 129, 66, 121, 187, 55, 251, 253, 190, 49, 239,
205, 254, 22, 116, 41, 136, 227, 135, 125, 16, 78, 146, 14, 162, 153, 199,
217, 180, 68, 170, 218, 40, 39, 9, 214, 49, 184, 188, 25, 163, 101, 239, 71,
189, 124, 169, 252, 215, 163, 254, 123, 217, 1, 57, 96, 4, 173, 112, 213, 169,
141, 54, 183, 113, 69, 232, 125, 39, 153, 115, 163, 237, 109, 92, 17, 108,
145, 27, 31, 38, 29, 118, 27, 215, 112, 54, 157, 135, 239, 73, 4, 246, 54, 46,
129, 116, 37, 234, 227, 12, 150, 200, 105, 109, 228, 14, 199, 56, 84, 143,
163, 193, 10, 193, 18, 201, 172, 141, 57, 166, 207, 255, 203, 192, 18, 57,
173, 141, 60, 154, 216, 198, 21, 193, 170, 147, 153, 35, 246, 129, 73, 217,
109, 156, 255, 210, 185, 150, 205, 193, 160, 208, 206, 135, 232, 14, 158, 121,
154, 252, 1, 139, 232, 48, 101, 122, 172, 204, 168, 0, 0, 0, 0, 73, 69, 78,
68, 174, 66, 96, 130,
]);

const emptyGzip = Buffer.from([
31, 139, 8, 0, 0, 0, 0, 0, 0, 19, 3, 0, 0, 0, 0, 0, 0, 0, 0, 0,
]);
const smallGzip = Buffer.from([
31, 139, 8, 0, 0, 0, 0, 0, 0, 19, 99, 100, 98, 102, 97, 5, 0, 244, 153, 11,
71, 5, 0, 0, 0,
]);
const faviconGzip = Buffer.from([
31, 139, 8, 0, 0, 0, 0, 0, 0, 19, 1, 239, 3, 16, 252, 137, 80, 78, 71, 13, 10,
26, 10, 0, 0, 0, 13, 73, 72, 68, 82, 0, 0, 0, 57, 0, 0, 0, 57, 8, 6, 0, 0, 0,
140, 24, 131, 133, 0, 0, 0, 9, 112, 72, 89, 115, 0, 0, 11, 19, 0, 0, 11, 19,
1, 0, 154, 156, 24, 0, 0, 0, 1, 115, 82, 71, 66, 0, 174, 206, 28, 233, 0, 0,
0, 4, 103, 65, 77, 65, 0, 0, 177, 143, 11, 252, 97, 5, 0, 0, 3, 132, 73, 68,
65, 84, 120, 1, 237, 154, 221, 81, 219, 64, 16, 199, 119, 79, 146, 163, 129,
153, 140, 58, 136, 233, 0, 42, 8, 84, 16, 83, 65, 148, 4, 50, 121, 195, 169,
32, 80, 1, 230, 45, 19, 200, 96, 58, 128, 14, 220, 65, 220, 65, 92, 130, 94,
32, 138, 101, 105, 115, 39, 227, 144, 113, 140, 181, 39, 233, 78, 246, 192,
239, 193, 2, 188, 30, 230, 175, 93, 239, 215, 9, 224, 9, 128, 160, 73, 16, 82,
16, 251, 16, 64, 67, 248, 49, 68, 81, 31, 35, 157, 207, 184, 160, 201, 111,
55, 9, 69, 42, 78, 161, 33, 198, 46, 157, 200, 203, 177, 206, 103, 4, 104,
242, 98, 226, 245, 229, 69, 235, 78, 54, 141, 182, 72, 21, 42, 68, 217, 9,
172, 17, 218, 34, 21, 191, 46, 90, 61, 249, 101, 30, 192, 154, 80, 74, 164,
34, 117, 156, 119, 128, 56, 130, 53, 160, 180, 200, 248, 43, 142, 18, 74, 247,
215, 65, 104, 105, 145, 138, 228, 188, 53, 204, 132, 216, 91, 117, 161, 149,
68, 42, 148, 71, 239, 190, 57, 91, 25, 192, 202, 134, 111, 101, 145, 51, 226,
115, 183, 159, 139, 37, 218, 147, 73, 233, 12, 41, 79, 76, 139, 74, 77, 4,
132, 67, 155, 55, 68, 187, 25, 40, 34, 190, 240, 6, 48, 151, 121, 253, 79,
212, 206, 223, 147, 94, 247, 194, 241, 118, 203, 19, 167, 68, 176, 13, 150,
168, 205, 147, 203, 80, 226, 84, 59, 182, 249, 49, 61, 245, 60, 241, 131, 0,
118, 193, 34, 86, 68, 110, 30, 78, 194, 177, 155, 254, 36, 162, 238, 227, 86,
50, 132, 13, 81, 123, 184, 254, 139, 31, 82, 219, 241, 210, 203, 220, 115, 5,
163, 0, 2, 157, 73, 187, 75, 48, 128, 81, 79, 10, 55, 61, 230, 133, 38, 14,
83, 162, 17, 24, 194, 152, 72, 53, 146, 73, 247, 188, 229, 216, 42, 47, 130,
65, 140, 137, 76, 188, 180, 195, 181, 149, 45, 226, 0, 12, 98, 50, 92, 153,
94, 132, 129, 202, 190, 96, 16, 35, 34, 85, 194, 209, 40, 19, 87, 249, 107,
134, 109, 48, 132, 17, 145, 194, 157, 132, 92, 91, 47, 113, 174, 193, 48, 102,
194, 21, 5, 43, 84, 165, 97, 95, 119, 95, 83, 134, 218, 69, 250, 7, 201, 46,
192, 180, 141, 43, 4, 233, 6, 44, 80, 123, 51, 32, 0, 67, 150, 33, 230, 211,
139, 241, 80, 85, 212, 31, 174, 8, 111, 88, 118, 100, 111, 125, 82, 171, 72,
213, 163, 202, 11, 107, 39, 155, 81, 118, 5, 150, 168, 219, 147, 188, 132, 35,
67, 245, 126, 36, 179, 66, 109, 34, 245, 106, 163, 61, 47, 42, 106, 19, 137,
110, 194, 110, 227, 50, 225, 246, 193, 34, 245, 137, 68, 231, 136, 101, 103,
161, 141, 155, 167, 22, 145, 90, 181, 113, 214, 198, 89, 164, 22, 145, 236,
218, 8, 230, 39, 142, 69, 212, 19, 174, 136, 175, 57, 102, 4, 116, 99, 59, 84,
21, 149, 69, 78, 107, 35, 47, 84, 165, 199, 173, 116, 56, 255, 255, 223, 234,
48, 155, 113, 121, 43, 16, 95, 65, 3, 84, 18, 169, 57, 55, 74, 149, 116, 188,
113, 152, 94, 78, 19, 149, 61, 42, 53, 232, 114, 19, 183, 75, 160, 11, 133, 2,
49, 220, 56, 152, 68, 136, 112, 191, 134, 196, 64, 174, 43, 219, 96, 136, 74,
34, 9, 240, 139, 122, 45, 5, 66, 240, 16, 5, 84, 226, 233, 5, 62, 165, 195,
117, 243, 112, 188, 173, 81, 27, 27, 165, 180, 72, 34, 209, 133, 53, 161, 124,
226, 121, 172, 54, 170, 211, 170, 21, 59, 194, 43, 37, 114, 105, 109, 148, 9,
100, 213, 78, 160, 75, 137, 148, 153, 112, 233, 196, 225, 33, 118, 90, 99,
177, 131, 25, 24, 221, 140, 115, 209, 22, 169, 106, 163, 244, 210, 242, 21, 7,
97, 62, 145, 220, 126, 119, 187, 8, 217, 14, 90, 92, 117, 44, 66, 91, 164,
170, 141, 12, 179, 64, 61, 185, 165, 126, 184, 61, 111, 13, 111, 47, 220, 189,
204, 113, 182, 136, 178, 207, 75, 78, 160, 141, 161, 93, 39, 101, 109, 60,
226, 212, 70, 129, 66, 121, 187, 55, 251, 253, 190, 49, 239, 205, 254, 22,
116, 41, 136, 227, 135, 125, 16, 78, 146, 14, 162, 153, 199, 217, 180, 68,
170, 218, 40, 39, 9, 214, 49, 184, 188, 25, 163, 101, 239, 71, 189, 124, 169,
252, 215, 163, 254, 123, 217, 1, 57, 96, 4, 173, 112, 213, 169, 141, 54, 183,
113, 69, 232, 125, 39, 153, 115, 163, 237, 109, 92, 17, 108, 145, 27, 31, 38,
29, 118, 27, 215, 112, 54, 157, 135, 239, 73, 4, 246, 54, 46, 129, 116, 37,
234, 227, 12, 150, 200, 105, 109, 228, 14, 199, 56, 84, 143, 163, 193, 10,
193, 18, 201, 172, 141, 57, 166, 207, 255, 203, 192, 18, 57, 173, 141, 60,
154, 216, 198, 21, 193, 170, 147, 153, 35, 246, 129, 73, 217, 109, 156, 255,
210, 185, 150, 205, 193, 160, 208, 206, 135, 232, 14, 158, 121, 154, 252, 1,
139, 232, 48, 101, 122, 172, 204, 168, 0, 0, 0, 0, 73, 69, 78, 68, 174, 66,
96, 130, 145, 24, 164, 164, 239, 3, 0, 0,
]);

describe("convertHttpContentEncodingToCompressionAlgorithm", () => {
it("maps the expected values correctly", () => {
expect(convertHttpContentEncodingToCompressionAlgorithm("")).toEqual(
"none",
);
expect(convertHttpContentEncodingToCompressionAlgorithm("br")).toEqual(
"br",
);
expect(convertHttpContentEncodingToCompressionAlgorithm("gzip")).toEqual(
"gzip",
);
});

it("throws an exception on unexpected values", () => {
expect(() => {
convertHttpContentEncodingToCompressionAlgorithm("chicken");
}).toThrow();
expect(() => {
convertHttpContentEncodingToCompressionAlgorithm("thrift");
}).toThrow();
});
});

describe("compressBuffer", () => {
describe("none", () => {
it("handles an empty buffer correctly", () => {
expect(compressBuffer("none", empty)).toEqual(empty);
});

it("handles a small buffer correctly", () => {
expect(compressBuffer("none", small)).toEqual(small);
});

it("handles a large buffer correctly", () => {
expect(compressBuffer("none", favicon)).toEqual(favicon);
});
});

describe("gzip", () => {
it("handles an empty buffer correctly", () => {
expect(compressBuffer("gzip", empty)).toEqual(emptyGzip);
});

it("handles a small buffer correctly", () => {
expect(compressBuffer("gzip", small)).toEqual(smallGzip);
});

it("handles a large buffer correctly", () => {
expect(compressBuffer("gzip", favicon)).toEqual(faviconGzip);
});
});

it("throws an excpetion on unexpected values ", () => {
expect(() => {
compressBuffer("chicken" as CompressionAlgorithm, small);
}).toThrow();
});
});

describe("decompressBuffer", () => {
describe("none", () => {
it("handles an empty buffer correctly", () => {
expect(decompressBuffer("none", empty)).toEqual(empty);
});

it("handles a small buffer correctly", () => {
expect(decompressBuffer("none", small)).toEqual(small);
});

it("handles a large buffer correctly", () => {
expect(decompressBuffer("none", favicon)).toEqual(favicon);
});
});

describe("gzip", () => {
it("handles an empty buffer correctly", () => {
expect(decompressBuffer("gzip", emptyGzip)).toEqual(empty);
});

it("handles a small buffer correctly", () => {
expect(decompressBuffer("gzip", smallGzip)).toEqual(small);
});

it("handles a large buffer correctly", () => {
expect(decompressBuffer("gzip", faviconGzip)).toEqual(favicon);
});
});

it("throws an excpetion on unexpected values ", () => {
expect(() => {
decompressBuffer("chicken" as CompressionAlgorithm, small);
}).toThrow();
});
});
56 changes: 56 additions & 0 deletions src/compression.ts
View file
Open in desktop
Original file line number Diff line number Diff line change
@@ -0,0 +1,56 @@
import brotli from "brotli";
import zlib from "zlib";

export type CompressionAlgorithm = "br" | "gzip" | "none";

export function compressBuffer(
algorithm: CompressionAlgorithm,
buffer: Buffer,
): Buffer {
switch (algorithm) {
case "none":
return buffer;
case "br":
const compressed = brotli.compress(buffer);
if (compressed !== null) {
return Buffer.from(compressed);
} else {
throw new Error(`Brotli compression failed!`);
}
case "gzip":
return zlib.gzipSync(buffer);
default:
throw new Error(`Unhandled compression algorithm value "${algorithm}"`);
}
}

export function decompressBuffer(
algorithm: CompressionAlgorithm,
buffer: Buffer,
): Buffer {
switch (algorithm) {
case "none":
return buffer;
case "br":
return Buffer.from(brotli.decompress(buffer));
case "gzip":
return zlib.gunzipSync(buffer);
default:
throw new Error(`Unhandled compression algorithm value "${algorithm}"`);
}
}

export function convertHttpContentEncodingToCompressionAlgorithm(
contentEncoding: string,
): CompressionAlgorithm {
switch (contentEncoding) {
case "":
return "none";
case "br":
return "br";
case "gzip":
return "gzip";
default:
throw new Error(`Unhandled content-encoding value "${contentEncoding}"`);
}
}
78 changes: 78 additions & 0 deletions src/http.ts
View file
Open in desktop
Original file line number Diff line number Diff line change
@@ -0,0 +1,78 @@
import { ParsedMediaType as ParsedContentType } from "content-type";
import {
convertHttpContentEncodingToCompressionAlgorithm,
decompressBuffer,
} from "./compression";

/**
* Headers of a request or response.
*/
export interface HttpHeaders {
[headerName: string]: string | string[] | undefined;
}

/**
* The common fields of a HTTP request.
*/
export interface HttpRequest {
host?: string;
method: string;
path: string;
headers: HttpHeaders;
body: Buffer;
}

export interface HttpRequestWithHost extends HttpRequest {
host: string;
}

/**
* The common fields of a HTTP response.
*/
export interface HttpResponse {
status: {
code: number;
};
headers: HttpHeaders;
body: Buffer;
}

export function getHttpHeaderAsString(
headers: HttpHeaders,
headerName: string,
): string {
const rawValue = headers[headerName];
if (rawValue === undefined) {
return "";
} else if (typeof rawValue === "string") {
return rawValue;
} else {
return rawValue[0];
}
}

export function getHttpContentEncoding(r: HttpRequest | HttpResponse): string {
return getHttpHeaderAsString(r.headers, "content-encoding");
}

export function getHttpContentType(r: HttpRequest | HttpResponse): string {
return (
getHttpHeaderAsString(r.headers, "content-type") ||
"application/octet-stream"
);
}

export function getHttpBodyDecoded(r: HttpRequest | HttpResponse): Buffer {
const contentEncoding = getHttpHeaderAsString(r.headers, "content-encoding");
const compressionAlgorithm =
convertHttpContentEncodingToCompressionAlgorithm(contentEncoding);
return decompressBuffer(compressionAlgorithm, r.body);
}

export function decodeHttpBodyToString(
r: HttpRequest | HttpResponse,
contentType: ParsedContentType,
): string {
const encoding = contentType.parameters.charset as BufferEncoding | undefined;
return getHttpBodyDecoded(r).toString(encoding || "utf-8");
}
Loading