From a41e158a486a0edae8a239e71eef16a6cfde4d00 Mon Sep 17 00:00:00 2001 From: Dan Lynch Date: Fri, 13 Jun 2025 17:45:41 -0700 Subject: [PATCH 1/7] with mocks --- package.json | 7 +- .../__mocks__/file-type.js | 56 +++++++ .../__snapshots__/mimetypes.test.ts.snap | 30 ++-- packages/content-type-stream/package.json | 2 +- .../src/content-type-stream.ts | 29 +++- yarn.lock | 143 ++++++++++++++++-- 6 files changed, 226 insertions(+), 41 deletions(-) create mode 100644 packages/content-type-stream/__mocks__/file-type.js diff --git a/package.json b/package.json index c2bc9b5f7..bf0c0a0ca 100644 --- a/package.json +++ b/package.json @@ -30,18 +30,18 @@ "@types/jest": "^29.5.11", "@types/jest-in-case": "^1.0.9", "@types/node": "^20.12.7", + "@types/rimraf": "^4.0.5", "@typescript-eslint/eslint-plugin": "^7.10.0", "@typescript-eslint/parser": "^7.10.0", "copyfiles": "^2.4.1", + "eslint": "^8.56.0", "eslint-config-prettier": "^9.1.0", "eslint-plugin-simple-import-sort": "^12.1.0", "eslint-plugin-unused-imports": "^4.0.0", - "eslint": "^8.56.0", "jest": "^29.6.2", "jest-in-case": "^1.0.2", "lerna": "^6", "prettier": "^3.0.2", - "@types/rimraf": "^4.0.5", "rimraf": "4.4.1", "strip-ansi": "^6", "symlink-workspace": "^1.9.0", @@ -53,5 +53,6 @@ "pg": "^8.16.0", "@types/pg": "^8.15.2", "graphql": "15.5.2" - } + }, + } diff --git a/packages/content-type-stream/__mocks__/file-type.js b/packages/content-type-stream/__mocks__/file-type.js new file mode 100644 index 000000000..5d150ae6c --- /dev/null +++ b/packages/content-type-stream/__mocks__/file-type.js @@ -0,0 +1,56 @@ +const fileTypeFromBuffer = async (buffer) => { + const firstBytes = buffer.slice(0, 16); + + if (firstBytes[0] === 0x89 && firstBytes[1] === 0x50 && firstBytes[2] === 0x4E && firstBytes[3] === 0x47) { + return { ext: 'png', mime: 'image/png' }; + } + + if (firstBytes[0] === 0xFF && firstBytes[1] === 0xD8 && firstBytes[2] === 0xFF) { + return { ext: 'jpg', mime: 'image/jpeg' }; + } + + if (firstBytes[0] === 0x47 && firstBytes[1] === 0x49 && firstBytes[2] === 0x46) { + return { ext: 'gif', mime: 'image/gif' }; + } + + if (firstBytes[0] === 0x25 && firstBytes[1] === 0x50 && firstBytes[2] === 0x44 && firstBytes[3] === 0x46) { + return { ext: 'pdf', mime: 'application/pdf' }; + } + + if (firstBytes[0] === 0x50 && firstBytes[1] === 0x4B && (firstBytes[2] === 0x03 || firstBytes[2] === 0x05)) { + return { ext: 'zip', mime: 'application/zip' }; + } + + if (firstBytes[0] === 0x42 && firstBytes[1] === 0x4D) { + return { ext: 'bmp', mime: 'image/x-ms-bmp' }; + } + + if ((firstBytes[0] === 0x49 && firstBytes[1] === 0x49 && firstBytes[2] === 0x2A && firstBytes[3] === 0x00) || + (firstBytes[0] === 0x4D && firstBytes[1] === 0x4D && firstBytes[2] === 0x00 && firstBytes[3] === 0x2A)) { + return { ext: 'tif', mime: 'image/tiff' }; + } + + if (firstBytes[0] === 0x1F && firstBytes[1] === 0x8B) { + return { ext: 'gz', mime: 'application/x-gzip' }; + } + + const text = buffer.toString('utf8', 0, Math.min(buffer.length, 1024)); + if (/^[\x20-\x7E\s]*$/.test(text)) { + if (text.includes(' = new Magic(mmm.MAGIC_MIME_TYPE | mmm.MAGIC_MIME_ENCODING); +const getCharsetFromMimeType = (mimeType: string): string => { + if (mimeType.startsWith('text/') || mimeType.includes('svg') || mimeType === 'text/x-shellscript') { + return 'us-ascii'; + } + if (mimeType.includes('json') || mimeType.includes('xml') || mimeType.includes('javascript')) { + return 'us-ascii'; + } + return 'binary'; +}; interface StreamContentTypeArgs { readStream: Readable; @@ -28,11 +34,18 @@ export function streamContentType({ return new Promise((resolve, reject) => { const peekStream = new BufferPeekStream({ peekBytes }); peekStream.once('peek', function (buffer: Buffer) { - magic.detect(buffer, (err: Error | null, res: string) => { - if (err) return reject(err); - const [type, charset] = res.split('; charset='); - const contentType = getContentType(filename, type, charset); - resolve({ stream: peekStream, magic: { type, charset }, contentType }); + import('file-type').then(async (fileType) => { + try { + const fileTypeResult = await fileType.fileTypeFromBuffer(buffer); + const type = fileTypeResult?.mime || 'application/octet-stream'; + const charset = getCharsetFromMimeType(type); + const contentType = getContentType(filename, type, charset); + resolve({ stream: peekStream, magic: { type, charset }, contentType }); + } catch (err) { + reject(err); + } + }).catch((err) => { + reject(err); }); }); readStream.pipe(peekStream); diff --git a/yarn.lock b/yarn.lock index eba4dae82..47050596a 100644 --- a/yarn.lock +++ b/yarn.lock @@ -646,13 +646,6 @@ "@jridgewell/resolve-uri" "^3.1.0" "@jridgewell/sourcemap-codec" "^1.4.14" -"@launchql/mmmagic@0.5.3": - version "0.5.3" - resolved "https://registry.yarnpkg.com/@launchql/mmmagic/-/mmmagic-0.5.3.tgz#8ad09ec6cb482e042a4fed71b9a41014b7daf56d" - integrity sha512-dyYpY+rVDWrlEMQNqcYJn+bdT301wCqneqv7mFL6+W6Ws46ofBN3gwvUuTUSAV//UdaNvabA62fIgaawhVFucg== - dependencies: - nan "^2.20.0" - "@lerna/child-process@6.6.2": version "6.6.2" resolved "https://registry.yarnpkg.com/@lerna/child-process/-/child-process-6.6.2.tgz#5d803c8dee81a4e013dc428292e77b365cba876c" @@ -1318,6 +1311,20 @@ "@babel/runtime" "^7.12.5" "@testing-library/dom" "^7.28.1" +"@tokenizer/inflate@^0.2.7": + version "0.2.7" + resolved "https://registry.yarnpkg.com/@tokenizer/inflate/-/inflate-0.2.7.tgz#32dd9dfc9abe457c89b3d9b760fc0690c85a103b" + integrity sha512-MADQgmZT1eKjp06jpI2yozxaU9uVs4GzzgSL+uEq7bVcJ9V1ZXQkeGNql1fsSI0gMy1vhvNTNbUqrx+pZfJVmg== + dependencies: + debug "^4.4.0" + fflate "^0.8.2" + token-types "^6.0.0" + +"@tokenizer/token@^0.3.0": + version "0.3.0" + resolved "https://registry.yarnpkg.com/@tokenizer/token/-/token-0.3.0.tgz#fe98a93fe789247e998c75e74e9c7c63217aa276" + integrity sha512-OvjF+z51L3ov0OyAU0duzsYuvO01PH7x4t6DJx+guahgTnBHkhJdG7soQeTSFLWN3efnHyibZ4Z8l2EuWwJN3A== + "@tootallnate/once@1": version "1.1.2" resolved "https://registry.yarnpkg.com/@tootallnate/once/-/once-1.1.2.tgz#ccb91445360179a04e7fe6aff78c00ffc1eeaf82" @@ -1885,6 +1892,13 @@ abbrev@^2.0.0: resolved "https://registry.yarnpkg.com/abbrev/-/abbrev-2.0.0.tgz#cf59829b8b4f03f89dda2771cb7f3653828c89bf" integrity sha512-6/mh1E2u2YgEsCHdY0Yx5oW+61gZU+1vXaoiHHrpKeuRNNgFvS+/jrwHiQhB5apAf5oB7UB7E19ol2R2LKH8hQ== +abort-controller@^3.0.0: + version "3.0.0" + resolved "https://registry.yarnpkg.com/abort-controller/-/abort-controller-3.0.0.tgz#eaf54d53b62bae4138e809ca225c8439a6efb392" + integrity sha512-h8lQ8tacZYnR3vNQTgibj+tODHI5/+l06Au2Pcriv/Gmet0eaj4TwWH41sO9wnHDiQsEj19q0drzdWdeAHtweg== + dependencies: + event-target-shim "^5.0.0" + accept-language-parser@^1.5.0: version "1.5.0" resolved "https://registry.yarnpkg.com/accept-language-parser/-/accept-language-parser-1.5.0.tgz#8877c54040a8dcb59e0a07d9c1fde42298334791" @@ -2411,6 +2425,14 @@ buffer@^5.5.0: base64-js "^1.3.1" ieee754 "^1.1.13" +buffer@^6.0.3: + version "6.0.3" + resolved "https://registry.yarnpkg.com/buffer/-/buffer-6.0.3.tgz#2ace578459cc8fbe2a70aaa8f52ee63b6a74c6c6" + integrity sha512-FTiCpNxtwiZZHEZbcbTIcZjERVICn9yq/pDFkTl95/AxzD1naBctN7YO68riM/gLSDY7sdrMby8hofADYuuqOA== + dependencies: + base64-js "^1.3.1" + ieee754 "^1.2.1" + builtins@^1.0.3: version "1.0.3" resolved "https://registry.yarnpkg.com/builtins/-/builtins-1.0.3.tgz#cb94faeb61c8696451db36534e1422f94f0aee88" @@ -3605,6 +3627,11 @@ etag@^1.8.1: resolved "https://registry.yarnpkg.com/etag/-/etag-1.8.1.tgz#41ae2eeb65efa62268aebfea83ac7d79299b0887" integrity sha512-aIL5Fx7mawVa300al2BnEE4iNvo1qETxLrPI/o05L7z6go7fCw1J6EQmbK4FmJ2AS7kgVF/KEZWufBfdClMcPg== +event-target-shim@^5.0.0: + version "5.0.1" + resolved "https://registry.yarnpkg.com/event-target-shim/-/event-target-shim-5.0.1.tgz#5d4d3ebdf9583d63a5333ce2deb7480ab2b05789" + integrity sha512-i/2XbnSz/uxRCU6+NdVJgKWDTM427+MqYbkQzD321DuCQJUqOuJKIA0IM2+W2xtYHdKOmZ4dR6fExsd4SXL+WQ== + eventemitter3@^3.1.0: version "3.1.2" resolved "https://registry.yarnpkg.com/eventemitter3/-/eventemitter3-3.1.2.tgz#2d3d48f9c346698fce83a85d7d664e98535df6e7" @@ -3620,6 +3647,11 @@ events@1.1.1: resolved "https://registry.yarnpkg.com/events/-/events-1.1.1.tgz#9ebdb7635ad099c70dcc4c2a1f5004288e8bd924" integrity sha512-kEcvvCBByWXGnZy6JUlgAp2gBIUjfCAV6P6TgT1/aaQKcmuAEC4OZTV1I4EWQLz2gxZw76atuVyvHhTxvi0Flw== +events@^3.3.0: + version "3.3.0" + resolved "https://registry.yarnpkg.com/events/-/events-3.3.0.tgz#31a95ad0a924e2d2c419a813aeb2c4e878ea7400" + integrity sha512-mQw+2fkQbALzQ7V0MY0IqdnXNOeTtP4r0lN9z7AAawCXgqea7bDii20AYrIBrFd/Hx0M2Ocz6S111CaFkUcb0Q== + execa@5.0.0: version "5.0.0" resolved "https://registry.yarnpkg.com/execa/-/execa-5.0.0.tgz#4029b0007998a841fbd1032e5f4de86a3c1e3376" @@ -3786,6 +3818,11 @@ fb-watchman@^2.0.0: dependencies: bser "2.1.1" +fflate@^0.8.2: + version "0.8.2" + resolved "https://registry.yarnpkg.com/fflate/-/fflate-0.8.2.tgz#fc8631f5347812ad6028bbe4a2308b2792aa1dea" + integrity sha512-cPJU47OaAoCbg0pBvzsgpTPhmhqI5eJjh/JIu8tPj5q+T7iLvW/JAYUqmE7KOB4R1ZyEhzBaIQpQpardBF5z8A== + figures@3.2.0, figures@^3.0.0: version "3.2.0" resolved "https://registry.yarnpkg.com/figures/-/figures-3.2.0.tgz#625c18bd293c604dc4a8ddb2febf0c88341746af" @@ -3807,6 +3844,25 @@ file-entry-cache@^6.0.1: dependencies: flat-cache "^3.0.4" +file-type@^18.0.0: + version "18.7.0" + resolved "https://registry.yarnpkg.com/file-type/-/file-type-18.7.0.tgz#cddb16f184d6b94106cfc4bb56978726b25cb2a2" + integrity sha512-ihHtXRzXEziMrQ56VSgU7wkxh55iNchFkosu7Y9/S+tXHdKyrGjVK0ujbqNnsxzea+78MaLhN6PGmfYSAv1ACw== + dependencies: + readable-web-to-node-stream "^3.0.2" + strtok3 "^7.0.0" + token-types "^5.0.1" + +file-type@^21.0.0: + version "21.0.0" + resolved "https://registry.yarnpkg.com/file-type/-/file-type-21.0.0.tgz#b6c5990064bc4b704f8e5c9b6010c59064d268bc" + integrity sha512-ek5xNX2YBYlXhiUXui3D/BXa3LdqPmoLJ7rqEx2bKJ7EAUEfmXgW0Das7Dc6Nr9MvqaOnIqiPV0mZk/r/UpNAg== + dependencies: + "@tokenizer/inflate" "^0.2.7" + strtok3 "^10.2.2" + token-types "^6.0.0" + uint8array-extras "^1.4.0" + file-url@3.0.0: version "3.0.0" resolved "https://registry.yarnpkg.com/file-url/-/file-url-3.0.0.tgz#247a586a746ce9f7a8ed05560290968afc262a77" @@ -4601,7 +4657,7 @@ ieee754@1.1.13: resolved "https://registry.yarnpkg.com/ieee754/-/ieee754-1.1.13.tgz#ec168558e95aa181fd87d37f55c32bbcb6708b84" integrity sha512-4vf7I2LYV/HaWerSo3XmlMkp5eZ83i+/CDluXi/IGTs/O1sejBNhTtnxzmRZfvOUqj7lZjqHkeTvpgSFDlWZTg== -ieee754@^1.1.13, ieee754@^1.1.4: +ieee754@^1.1.13, ieee754@^1.1.4, ieee754@^1.2.1: version "1.2.1" resolved "https://registry.yarnpkg.com/ieee754/-/ieee754-1.2.1.tgz#8eb7a10a63fff25d15a57b001586d177d1b0d352" integrity sha512-dcyqhDvX1C46lXZcVqCpK+FtMRQVdIMN6/Df5js2zouUsqG7I6sFxitIC+7KYK29KdXOLHdu9zL4sFnoVQnqaA== @@ -6399,11 +6455,6 @@ mute-stream@0.0.8, mute-stream@~0.0.4: resolved "https://registry.yarnpkg.com/mute-stream/-/mute-stream-0.0.8.tgz#1630c42b2251ff81e2a283de96a5497ea92e5e0d" integrity sha512-nnbWWOkoWyUsTjKrhgD0dcz22mdkSnpYqbEjIm2nhwhuxlSkpywJmBo8h0ZqJdkp73mb90SssHkN4rsRaBAfAA== -nan@^2.20.0: - version "2.22.2" - resolved "https://registry.yarnpkg.com/nan/-/nan-2.22.2.tgz#6b504fd029fb8f38c0990e52ad5c26772fdacfbb" - integrity sha512-DANghxFkS1plDdRsX0X9pm0Z6SJNN6gBdtXfanwoZ8hooC5gosGFSBGRYHUVPz1asKA/kMRqDRdHrluZ61SpBQ== - nano-time@1.0.0: version "1.0.0" resolved "https://registry.yarnpkg.com/nano-time/-/nano-time-1.0.0.tgz#b0554f69ad89e22d0907f7a12b0993a5d96137ef" @@ -7198,6 +7249,11 @@ path-type@^4.0.0: resolved "https://registry.yarnpkg.com/path-type/-/path-type-4.0.0.tgz#84ed01c0a7ba380afe09d90a8c180dcd9d03043b" integrity sha512-gDKb8aZMDeD/tZWs9P6+q0J9Mwkdl6xMV8TjnGP3qJVJ06bdMgkbBlLU8IdfOsIsFz2BW1rNVT3XuNEl8zPAvw== +peek-readable@^5.1.3: + version "5.4.2" + resolved "https://registry.yarnpkg.com/peek-readable/-/peek-readable-5.4.2.tgz#aff1e1ba27a7d6911ddb103f35252ffc1787af49" + integrity sha512-peBp3qZyuS6cNIJ2akRNG1uo1WJ1d0wTxg/fxMdZ0BqCVhx242bSFHM9eNqflfJVS9SsgkzgT/1UgnsurBOTMg== + pg-cloudflare@^1.2.5: version "1.2.5" resolved "https://registry.yarnpkg.com/pg-cloudflare/-/pg-cloudflare-1.2.5.tgz#2e3649c38a7a9c74a7e5327c8098a2fd9af595bd" @@ -7564,6 +7620,11 @@ process-nextick-args@~2.0.0: resolved "https://registry.yarnpkg.com/process-nextick-args/-/process-nextick-args-2.0.1.tgz#7820d9b16120cc55ca9ae7792680ae7dba6d7fe2" integrity sha512-3ouUOpQhtgrbOa17J7+uxOTpITYWaGP7/AhoR3+A+/1e9skrzelGi/dXzEYyvbxubEF6Wn2ypscTKiKJFFn1ag== +process@^0.11.10: + version "0.11.10" + resolved "https://registry.yarnpkg.com/process/-/process-0.11.10.tgz#7332300e840161bda3e69a1d1d91a7d4bc16f182" + integrity sha512-cdGef/drWFoydD1JsMzuFf8100nZl+GT+yacc2bEced5f9Rjk4z+WtFUTBu9PhOi9j/jfmBPu0mMEY4wIdAF8A== + promise-all-reject-late@^1.0.0: version "1.0.1" resolved "https://registry.yarnpkg.com/promise-all-reject-late/-/promise-all-reject-late-1.0.1.tgz#f8ebf13483e5ca91ad809ccc2fcf25f26f8643c2" @@ -7856,6 +7917,17 @@ read@1, read@^1.0.7: string_decoder "^1.1.1" util-deprecate "^1.0.1" +readable-stream@^4.7.0: + version "4.7.0" + resolved "https://registry.yarnpkg.com/readable-stream/-/readable-stream-4.7.0.tgz#cedbd8a1146c13dfff8dab14068028d58c15ac91" + integrity sha512-oIGGmcpTLwPga8Bn6/Z75SVaH1z5dUut2ibSyAMVhmUggWpmDn2dapB0n7f8nwaSiRtepAsfJyfXIO5DCVAODg== + dependencies: + abort-controller "^3.0.0" + buffer "^6.0.3" + events "^3.3.0" + process "^0.11.10" + string_decoder "^1.3.0" + readable-stream@~1.0.31: version "1.0.34" resolved "https://registry.yarnpkg.com/readable-stream/-/readable-stream-1.0.34.tgz#125820e34bc842d2f2aaafafe4c2916ee32c157c" @@ -7879,6 +7951,13 @@ readable-stream@~2.3.6: string_decoder "~1.1.1" util-deprecate "~1.0.1" +readable-web-to-node-stream@^3.0.2: + version "3.0.4" + resolved "https://registry.yarnpkg.com/readable-web-to-node-stream/-/readable-web-to-node-stream-3.0.4.tgz#392ba37707af5bf62d725c36c1b5d6ef4119eefc" + integrity sha512-9nX56alTf5bwXQ3ZDipHJhusu9NTQJ/CVPtb/XHAJCXihZeitfJvIRS4GqQ/mfIoOE3IelHMrpayVrosdHBuLw== + dependencies: + readable-stream "^4.7.0" + readdirp@~3.6.0: version "3.6.0" resolved "https://registry.yarnpkg.com/readdirp/-/readdirp-3.6.0.tgz#74a370bd857116e245b29cc97340cd431a02a6c7" @@ -8497,7 +8576,7 @@ string-width@^5.0.1, string-width@^5.1.2: emoji-regex "^9.2.2" strip-ansi "^7.0.1" -string_decoder@^1.1.1: +string_decoder@^1.1.1, string_decoder@^1.3.0: version "1.3.0" resolved "https://registry.yarnpkg.com/string_decoder/-/string_decoder-1.3.0.tgz#42f114594a46cf1a8e30b0a84f56c78c3edac21e" integrity sha512-hkRX8U1WjJFd8LsDJ2yQ/wWWxaopEsABU1XfkM8A+j0+85JAGppt16cr1Whg6KIbb4okU6Mql6BOj+uup/wKeA== @@ -8599,6 +8678,21 @@ strong-log-transformer@2.1.0, strong-log-transformer@^2.1.0: minimist "^1.2.0" through "^2.3.4" +strtok3@^10.2.2: + version "10.3.1" + resolved "https://registry.yarnpkg.com/strtok3/-/strtok3-10.3.1.tgz#80fe431a4ee652de4e33f14e11e15fd5170a627d" + integrity sha512-3JWEZM6mfix/GCJBBUrkA8p2Id2pBkyTkVCJKto55w080QBKZ+8R171fGrbiSp+yMO/u6F8/yUh7K4V9K+YCnw== + dependencies: + "@tokenizer/token" "^0.3.0" + +strtok3@^7.0.0: + version "7.1.1" + resolved "https://registry.yarnpkg.com/strtok3/-/strtok3-7.1.1.tgz#f548fd9dc59d0a76d5567ff8c16be31221f29dfc" + integrity sha512-mKX8HA/cdBqMKUr0MMZAFssCkIGoZeSCMXgnt79yKxNFguMLVFgRe6wB+fsL0NmoHDbeyZXczy7vEPSoo3rkzg== + dependencies: + "@tokenizer/token" "^0.3.0" + peek-readable "^5.1.3" + subscriptions-transport-ws@^0.9.18: version "0.9.19" resolved "https://registry.yarnpkg.com/subscriptions-transport-ws/-/subscriptions-transport-ws-0.9.19.tgz#10ca32f7e291d5ee8eb728b9c02e43c52606cdcf" @@ -8789,6 +8883,22 @@ toidentifier@1.0.1: resolved "https://registry.yarnpkg.com/toidentifier/-/toidentifier-1.0.1.tgz#3be34321a88a820ed1bd80dfaa33e479fbb8dd35" integrity sha512-o5sSPKEkg/DIQNmH43V0/uerLrpzVedkUh8tGNvaeXpfpuwjKenlSox/2O/BTlZUtEe+JG7s5YhEz608PlAHRA== +token-types@^5.0.1: + version "5.0.1" + resolved "https://registry.yarnpkg.com/token-types/-/token-types-5.0.1.tgz#aa9d9e6b23c420a675e55413b180635b86a093b4" + integrity sha512-Y2fmSnZjQdDb9W4w4r1tswlMHylzWIeOKpx0aZH9BgGtACHhrk3OkT52AzwcuqTRBZtvvnTjDBh8eynMulu8Vg== + dependencies: + "@tokenizer/token" "^0.3.0" + ieee754 "^1.2.1" + +token-types@^6.0.0: + version "6.0.0" + resolved "https://registry.yarnpkg.com/token-types/-/token-types-6.0.0.tgz#1ab26be1ef9c434853500c071acfe5c8dd6544a3" + integrity sha512-lbDrTLVsHhOMljPscd0yitpozq7Ga2M5Cvez5AjGg8GASBjtt6iERCAJ93yommPmz62fb45oFIXHEZ3u9bfJEA== + dependencies: + "@tokenizer/token" "^0.3.0" + ieee754 "^1.2.1" + touch@^3.1.0: version "3.1.1" resolved "https://registry.yarnpkg.com/touch/-/touch-3.1.1.tgz#097a23d7b161476435e5c1344a95c0f75b4a5694" @@ -8971,6 +9081,11 @@ uglify-js@^3.1.4: resolved "https://registry.yarnpkg.com/uglify-js/-/uglify-js-3.19.3.tgz#82315e9bbc6f2b25888858acd1fff8441035b77f" integrity sha512-v3Xu+yuwBXisp6QYTcH4UbH+xYJXqnq2m/LtQVWKWzYc1iehYnLixoQDN9FH6/j9/oybfd6W9Ghwkl8+UMKTKQ== +uint8array-extras@^1.4.0: + version "1.4.0" + resolved "https://registry.yarnpkg.com/uint8array-extras/-/uint8array-extras-1.4.0.tgz#e42a678a6dd335ec2d21661333ed42f44ae7cc74" + integrity sha512-ZPtzy0hu4cZjv3z5NW9gfKnNLjoz4y6uv4HlelAjDK7sY/xOkKZv9xK/WQpcsBB3jEybChz9DPC2U/+cusjJVQ== + undefsafe@^2.0.5: version "2.0.5" resolved "https://registry.yarnpkg.com/undefsafe/-/undefsafe-2.0.5.tgz#38733b9327bdcd226db889fb723a6efd162e6e2c" From 7aa9205217ec8476f26f8dd21355c756118c670c Mon Sep 17 00:00:00 2001 From: Dan Lynch Date: Fri, 13 Jun 2025 18:05:44 -0700 Subject: [PATCH 2/7] cleanup --- package.json | 3 +-- .../__tests__/mimetypes.test.ts | 17 ++++++++++++++--- .../src/content-type-stream.ts | 16 +++++++++++----- 3 files changed, 26 insertions(+), 10 deletions(-) diff --git a/package.json b/package.json index bf0c0a0ca..3b66b1c39 100644 --- a/package.json +++ b/package.json @@ -53,6 +53,5 @@ "pg": "^8.16.0", "@types/pg": "^8.15.2", "graphql": "15.5.2" - }, - + } } diff --git a/packages/content-type-stream/__tests__/mimetypes.test.ts b/packages/content-type-stream/__tests__/mimetypes.test.ts index f15fa54d8..d735771da 100644 --- a/packages/content-type-stream/__tests__/mimetypes.test.ts +++ b/packages/content-type-stream/__tests__/mimetypes.test.ts @@ -1,9 +1,20 @@ -// @ts-nocheck import { streamContentType } from '../src'; import { sync as glob } from 'glob'; import { createReadStream } from 'fs'; import { basename } from 'path'; +interface MagicType { + type: string; + charset: string; +} + +interface ResultType { + [key: string]: { + magic: MagicType; + contentType: string; + }; +} + const files = [] .concat(glob(__dirname + '/../../../__fixtures__/kitchen-sink/**')) .concat(glob(__dirname + '/../../../__fixtures__/kitchen-sink/**/.*')) @@ -28,7 +39,7 @@ const malicious = glob(__dirname + '/../../../__fixtures__/malicious/**') describe('mimetypes', () => { it('good files', async () => { - const res = {}; + const res: ResultType = {}; const use = files; for (var i = 0; i < use.length; i++) { @@ -49,7 +60,7 @@ describe('mimetypes', () => { expect(res).toMatchSnapshot(); }); it('malicious files', async () => { - const res = {}; + const res: ResultType = {}; const use = malicious; for (var i = 0; i < use.length; i++) { diff --git a/packages/content-type-stream/src/content-type-stream.ts b/packages/content-type-stream/src/content-type-stream.ts index faadbb2cc..29a8f3539 100644 --- a/packages/content-type-stream/src/content-type-stream.ts +++ b/packages/content-type-stream/src/content-type-stream.ts @@ -1,14 +1,20 @@ -// @ts-nocheck +// @ts-ignore import { BufferPeekStream } from 'buffer-peek-stream'; import type { Readable } from 'stream'; import { getContentType } from './get-content-type'; const getCharsetFromMimeType = (mimeType: string): string => { - if (mimeType.startsWith('text/') || mimeType.includes('svg') || mimeType === 'text/x-shellscript') { - return 'us-ascii'; - } - if (mimeType.includes('json') || mimeType.includes('xml') || mimeType.includes('javascript')) { + const asciiMimeTypes = [ + 'text/', + 'svg', + 'text/x-shellscript', + 'json', + 'xml', + 'javascript', + ]; + + if (asciiMimeTypes.some(type => mimeType.includes(type))) { return 'us-ascii'; } return 'binary'; From 03634d0d3f024427031d0800fb39267d3c933c9e Mon Sep 17 00:00:00 2001 From: Dan Lynch Date: Fri, 13 Jun 2025 18:24:38 -0700 Subject: [PATCH 3/7] mimetic --- .../__mocks__/file-type.js | 56 ------ .../__snapshots__/mimetypes.test.ts.snap | 110 ++++++------ packages/content-type-stream/package.json | 2 +- .../src/content-type-stream.ts | 24 ++- packages/content-type-stream/src/index.ts | 2 +- yarn.lock | 170 +++++------------- 6 files changed, 109 insertions(+), 255 deletions(-) delete mode 100644 packages/content-type-stream/__mocks__/file-type.js diff --git a/packages/content-type-stream/__mocks__/file-type.js b/packages/content-type-stream/__mocks__/file-type.js deleted file mode 100644 index 5d150ae6c..000000000 --- a/packages/content-type-stream/__mocks__/file-type.js +++ /dev/null @@ -1,56 +0,0 @@ -const fileTypeFromBuffer = async (buffer) => { - const firstBytes = buffer.slice(0, 16); - - if (firstBytes[0] === 0x89 && firstBytes[1] === 0x50 && firstBytes[2] === 0x4E && firstBytes[3] === 0x47) { - return { ext: 'png', mime: 'image/png' }; - } - - if (firstBytes[0] === 0xFF && firstBytes[1] === 0xD8 && firstBytes[2] === 0xFF) { - return { ext: 'jpg', mime: 'image/jpeg' }; - } - - if (firstBytes[0] === 0x47 && firstBytes[1] === 0x49 && firstBytes[2] === 0x46) { - return { ext: 'gif', mime: 'image/gif' }; - } - - if (firstBytes[0] === 0x25 && firstBytes[1] === 0x50 && firstBytes[2] === 0x44 && firstBytes[3] === 0x46) { - return { ext: 'pdf', mime: 'application/pdf' }; - } - - if (firstBytes[0] === 0x50 && firstBytes[1] === 0x4B && (firstBytes[2] === 0x03 || firstBytes[2] === 0x05)) { - return { ext: 'zip', mime: 'application/zip' }; - } - - if (firstBytes[0] === 0x42 && firstBytes[1] === 0x4D) { - return { ext: 'bmp', mime: 'image/x-ms-bmp' }; - } - - if ((firstBytes[0] === 0x49 && firstBytes[1] === 0x49 && firstBytes[2] === 0x2A && firstBytes[3] === 0x00) || - (firstBytes[0] === 0x4D && firstBytes[1] === 0x4D && firstBytes[2] === 0x00 && firstBytes[3] === 0x2A)) { - return { ext: 'tif', mime: 'image/tiff' }; - } - - if (firstBytes[0] === 0x1F && firstBytes[1] === 0x8B) { - return { ext: 'gz', mime: 'application/x-gzip' }; - } - - const text = buffer.toString('utf8', 0, Math.min(buffer.length, 1024)); - if (/^[\x20-\x7E\s]*$/.test(text)) { - if (text.includes(' { return new Promise((resolve, reject) => { const peekStream = new BufferPeekStream({ peekBytes }); - peekStream.once('peek', function (buffer: Buffer) { - import('file-type').then(async (fileType) => { - try { - const fileTypeResult = await fileType.fileTypeFromBuffer(buffer); - const type = fileTypeResult?.mime || 'application/octet-stream'; - const charset = getCharsetFromMimeType(type); - const contentType = getContentType(filename, type, charset); - resolve({ stream: peekStream, magic: { type, charset }, contentType }); - } catch (err) { - reject(err); - } - }).catch((err) => { + peekStream.once('peek', async function (buffer: Buffer) { + try { + const Mimetics = require('mimetics'); + const mimetics = new Mimetics(); + const fileTypeResult = await mimetics.parseAsync(buffer); + const type = fileTypeResult?.mime || 'application/octet-stream'; + const charset = getCharsetFromMimeType(type); + const contentType = getContentType(filename, type, charset); + resolve({ stream: peekStream, magic: { type, charset }, contentType }); + } catch (err) { reject(err); - }); + } }); readStream.pipe(peekStream); }); diff --git a/packages/content-type-stream/src/index.ts b/packages/content-type-stream/src/index.ts index be783b753..6666f142f 100644 --- a/packages/content-type-stream/src/index.ts +++ b/packages/content-type-stream/src/index.ts @@ -1,3 +1,3 @@ export * from './get-content-type'; export * from './content-type-stream'; -export * from './content-stream'; \ No newline at end of file +export * from './content-stream'; diff --git a/yarn.lock b/yarn.lock index 47050596a..3f4753964 100644 --- a/yarn.lock +++ b/yarn.lock @@ -1311,20 +1311,6 @@ "@babel/runtime" "^7.12.5" "@testing-library/dom" "^7.28.1" -"@tokenizer/inflate@^0.2.7": - version "0.2.7" - resolved "https://registry.yarnpkg.com/@tokenizer/inflate/-/inflate-0.2.7.tgz#32dd9dfc9abe457c89b3d9b760fc0690c85a103b" - integrity sha512-MADQgmZT1eKjp06jpI2yozxaU9uVs4GzzgSL+uEq7bVcJ9V1ZXQkeGNql1fsSI0gMy1vhvNTNbUqrx+pZfJVmg== - dependencies: - debug "^4.4.0" - fflate "^0.8.2" - token-types "^6.0.0" - -"@tokenizer/token@^0.3.0": - version "0.3.0" - resolved "https://registry.yarnpkg.com/@tokenizer/token/-/token-0.3.0.tgz#fe98a93fe789247e998c75e74e9c7c63217aa276" - integrity sha512-OvjF+z51L3ov0OyAU0duzsYuvO01PH7x4t6DJx+guahgTnBHkhJdG7soQeTSFLWN3efnHyibZ4Z8l2EuWwJN3A== - "@tootallnate/once@1": version "1.1.2" resolved "https://registry.yarnpkg.com/@tootallnate/once/-/once-1.1.2.tgz#ccb91445360179a04e7fe6aff78c00ffc1eeaf82" @@ -1892,13 +1878,6 @@ abbrev@^2.0.0: resolved "https://registry.yarnpkg.com/abbrev/-/abbrev-2.0.0.tgz#cf59829b8b4f03f89dda2771cb7f3653828c89bf" integrity sha512-6/mh1E2u2YgEsCHdY0Yx5oW+61gZU+1vXaoiHHrpKeuRNNgFvS+/jrwHiQhB5apAf5oB7UB7E19ol2R2LKH8hQ== -abort-controller@^3.0.0: - version "3.0.0" - resolved "https://registry.yarnpkg.com/abort-controller/-/abort-controller-3.0.0.tgz#eaf54d53b62bae4138e809ca225c8439a6efb392" - integrity sha512-h8lQ8tacZYnR3vNQTgibj+tODHI5/+l06Au2Pcriv/Gmet0eaj4TwWH41sO9wnHDiQsEj19q0drzdWdeAHtweg== - dependencies: - event-target-shim "^5.0.0" - accept-language-parser@^1.5.0: version "1.5.0" resolved "https://registry.yarnpkg.com/accept-language-parser/-/accept-language-parser-1.5.0.tgz#8877c54040a8dcb59e0a07d9c1fde42298334791" @@ -2425,14 +2404,6 @@ buffer@^5.5.0: base64-js "^1.3.1" ieee754 "^1.1.13" -buffer@^6.0.3: - version "6.0.3" - resolved "https://registry.yarnpkg.com/buffer/-/buffer-6.0.3.tgz#2ace578459cc8fbe2a70aaa8f52ee63b6a74c6c6" - integrity sha512-FTiCpNxtwiZZHEZbcbTIcZjERVICn9yq/pDFkTl95/AxzD1naBctN7YO68riM/gLSDY7sdrMby8hofADYuuqOA== - dependencies: - base64-js "^1.3.1" - ieee754 "^1.2.1" - builtins@^1.0.3: version "1.0.3" resolved "https://registry.yarnpkg.com/builtins/-/builtins-1.0.3.tgz#cb94faeb61c8696451db36534e1422f94f0aee88" @@ -3627,11 +3598,6 @@ etag@^1.8.1: resolved "https://registry.yarnpkg.com/etag/-/etag-1.8.1.tgz#41ae2eeb65efa62268aebfea83ac7d79299b0887" integrity sha512-aIL5Fx7mawVa300al2BnEE4iNvo1qETxLrPI/o05L7z6go7fCw1J6EQmbK4FmJ2AS7kgVF/KEZWufBfdClMcPg== -event-target-shim@^5.0.0: - version "5.0.1" - resolved "https://registry.yarnpkg.com/event-target-shim/-/event-target-shim-5.0.1.tgz#5d4d3ebdf9583d63a5333ce2deb7480ab2b05789" - integrity sha512-i/2XbnSz/uxRCU6+NdVJgKWDTM427+MqYbkQzD321DuCQJUqOuJKIA0IM2+W2xtYHdKOmZ4dR6fExsd4SXL+WQ== - eventemitter3@^3.1.0: version "3.1.2" resolved "https://registry.yarnpkg.com/eventemitter3/-/eventemitter3-3.1.2.tgz#2d3d48f9c346698fce83a85d7d664e98535df6e7" @@ -3647,11 +3613,6 @@ events@1.1.1: resolved "https://registry.yarnpkg.com/events/-/events-1.1.1.tgz#9ebdb7635ad099c70dcc4c2a1f5004288e8bd924" integrity sha512-kEcvvCBByWXGnZy6JUlgAp2gBIUjfCAV6P6TgT1/aaQKcmuAEC4OZTV1I4EWQLz2gxZw76atuVyvHhTxvi0Flw== -events@^3.3.0: - version "3.3.0" - resolved "https://registry.yarnpkg.com/events/-/events-3.3.0.tgz#31a95ad0a924e2d2c419a813aeb2c4e878ea7400" - integrity sha512-mQw+2fkQbALzQ7V0MY0IqdnXNOeTtP4r0lN9z7AAawCXgqea7bDii20AYrIBrFd/Hx0M2Ocz6S111CaFkUcb0Q== - execa@5.0.0: version "5.0.0" resolved "https://registry.yarnpkg.com/execa/-/execa-5.0.0.tgz#4029b0007998a841fbd1032e5f4de86a3c1e3376" @@ -3818,11 +3779,6 @@ fb-watchman@^2.0.0: dependencies: bser "2.1.1" -fflate@^0.8.2: - version "0.8.2" - resolved "https://registry.yarnpkg.com/fflate/-/fflate-0.8.2.tgz#fc8631f5347812ad6028bbe4a2308b2792aa1dea" - integrity sha512-cPJU47OaAoCbg0pBvzsgpTPhmhqI5eJjh/JIu8tPj5q+T7iLvW/JAYUqmE7KOB4R1ZyEhzBaIQpQpardBF5z8A== - figures@3.2.0, figures@^3.0.0: version "3.2.0" resolved "https://registry.yarnpkg.com/figures/-/figures-3.2.0.tgz#625c18bd293c604dc4a8ddb2febf0c88341746af" @@ -3844,25 +3800,6 @@ file-entry-cache@^6.0.1: dependencies: flat-cache "^3.0.4" -file-type@^18.0.0: - version "18.7.0" - resolved "https://registry.yarnpkg.com/file-type/-/file-type-18.7.0.tgz#cddb16f184d6b94106cfc4bb56978726b25cb2a2" - integrity sha512-ihHtXRzXEziMrQ56VSgU7wkxh55iNchFkosu7Y9/S+tXHdKyrGjVK0ujbqNnsxzea+78MaLhN6PGmfYSAv1ACw== - dependencies: - readable-web-to-node-stream "^3.0.2" - strtok3 "^7.0.0" - token-types "^5.0.1" - -file-type@^21.0.0: - version "21.0.0" - resolved "https://registry.yarnpkg.com/file-type/-/file-type-21.0.0.tgz#b6c5990064bc4b704f8e5c9b6010c59064d268bc" - integrity sha512-ek5xNX2YBYlXhiUXui3D/BXa3LdqPmoLJ7rqEx2bKJ7EAUEfmXgW0Das7Dc6Nr9MvqaOnIqiPV0mZk/r/UpNAg== - dependencies: - "@tokenizer/inflate" "^0.2.7" - strtok3 "^10.2.2" - token-types "^6.0.0" - uint8array-extras "^1.4.0" - file-url@3.0.0: version "3.0.0" resolved "https://registry.yarnpkg.com/file-url/-/file-url-3.0.0.tgz#247a586a746ce9f7a8ed05560290968afc262a77" @@ -4657,7 +4594,7 @@ ieee754@1.1.13: resolved "https://registry.yarnpkg.com/ieee754/-/ieee754-1.1.13.tgz#ec168558e95aa181fd87d37f55c32bbcb6708b84" integrity sha512-4vf7I2LYV/HaWerSo3XmlMkp5eZ83i+/CDluXi/IGTs/O1sejBNhTtnxzmRZfvOUqj7lZjqHkeTvpgSFDlWZTg== -ieee754@^1.1.13, ieee754@^1.1.4, ieee754@^1.2.1: +ieee754@^1.1.13, ieee754@^1.1.4: version "1.2.1" resolved "https://registry.yarnpkg.com/ieee754/-/ieee754-1.2.1.tgz#8eb7a10a63fff25d15a57b001586d177d1b0d352" integrity sha512-dcyqhDvX1C46lXZcVqCpK+FtMRQVdIMN6/Df5js2zouUsqG7I6sFxitIC+7KYK29KdXOLHdu9zL4sFnoVQnqaA== @@ -4686,6 +4623,11 @@ ignore@^5.0.4, ignore@^5.2.0, ignore@^5.3.1: resolved "https://registry.yarnpkg.com/ignore/-/ignore-5.3.2.tgz#3cd40e729f3643fd87cb04e50bf0eb722bc596f5" integrity sha512-hsBTNUqQTDwkWtcdYI2i06Y/nUBEsNEDJKjWdigLvegy8kDuJAS8uRlpkkcQpyEXL0Z/pjDy5HBmMjRCJ2gq+g== +immediate@~3.0.5: + version "3.0.6" + resolved "https://registry.yarnpkg.com/immediate/-/immediate-3.0.6.tgz#9db1dbd0faf8de6fbe0f5dd5e56bb606280de69b" + integrity sha512-XXOFtyqDjNDAQxVfYxuF7g9Il/IbWmmlQg2MYKOH8ExIT1qg6xc4zyS3HaEEATgs1btfzxq15ciUiY7gjSXRGQ== + import-fresh@^3.2.1: version "3.3.1" resolved "https://registry.yarnpkg.com/import-fresh/-/import-fresh-3.3.1.tgz#9cecb56503c0ada1f2741dbbd6546e4b13b57ccf" @@ -5673,6 +5615,16 @@ jsonwebtoken@^9.0.0: ms "^2.1.1" semver "^7.5.4" +jszip@^3.10.1: + version "3.10.1" + resolved "https://registry.yarnpkg.com/jszip/-/jszip-3.10.1.tgz#34aee70eb18ea1faec2f589208a157d1feb091c2" + integrity sha512-xXDvecyTpGLrqFrvkrUSoxxfJI5AH7U8zxxtVclpsUtMCq4JQ290LY8AW5c7Ggnr/Y/oK+bQMbqK2qmtk3pN4g== + dependencies: + lie "~3.3.0" + pako "~1.0.2" + readable-stream "~2.3.6" + setimmediate "^1.0.5" + just-diff-apply@^5.2.0: version "5.5.0" resolved "https://registry.yarnpkg.com/just-diff-apply/-/just-diff-apply-5.5.0.tgz#771c2ca9fa69f3d2b54e7c3f5c1dfcbcc47f9f0f" @@ -5845,6 +5797,13 @@ libpg-query@13.3.2: node-addon-api "^1.6.3" node-gyp "^8.0.0" +lie@~3.3.0: + version "3.3.0" + resolved "https://registry.yarnpkg.com/lie/-/lie-3.3.0.tgz#dcf82dee545f46074daf200c7c1c5a08e0f40f6a" + integrity sha512-UaiMJzeWRlEujzAuw5LokY1L5ecNQYZKfmyZ9L7wDHb/p5etKaxXhohBcrw0EYby+G/NA52vRSN4N39dxHAIwQ== + dependencies: + immediate "~3.0.5" + lines-and-columns@^1.1.6: version "1.2.4" resolved "https://registry.yarnpkg.com/lines-and-columns/-/lines-and-columns-1.2.4.tgz#eca284f75d2965079309dc0ad9255abb2ebc1632" @@ -6218,6 +6177,13 @@ mime@2.4.6: resolved "https://registry.yarnpkg.com/mime/-/mime-2.4.6.tgz#e5b407c90db442f2beb5b162373d07b69affa4d1" integrity sha512-RZKhC3EmpBchfTGBVb8fb+RL2cWyw/32lshnsETttkBAyAUXSGHxbEJWWRXc751DrIxG1q04b8QwMbAwkRPpUA== +mimetics@^1.0.0: + version "1.0.4" + resolved "https://registry.yarnpkg.com/mimetics/-/mimetics-1.0.4.tgz#20c82d260432e0fe7f07f11cdd76c2af370207e8" + integrity sha512-lI3VpGrE9YJGu10kRQYdunz3J/Z/o73H2n6kH1Ve3bsd4c3c7tPAfDh+GEfWofue29aiC6lwARTk84b2RXRijQ== + dependencies: + jszip "^3.10.1" + mimic-fn@^1.0.0: version "1.2.0" resolved "https://registry.yarnpkg.com/mimic-fn/-/mimic-fn-1.2.0.tgz#820c86a39334640e99516928bd03fca88057d022" @@ -7128,6 +7094,11 @@ pacote@^15.0.0, pacote@^15.0.8: ssri "^10.0.0" tar "^6.1.11" +pako@~1.0.2: + version "1.0.11" + resolved "https://registry.yarnpkg.com/pako/-/pako-1.0.11.tgz#6c9599d340d54dfd3946380252a35705a6b992bf" + integrity sha512-4hLB8Py4zZce5s4yd9XzopqwVv/yGNhV1Bl8NTmCq1763HeK2+EwVTv+leGeL13Dnh2wfbqowVPXCIO0z4taYw== + parent-module@^1.0.0: version "1.0.1" resolved "https://registry.yarnpkg.com/parent-module/-/parent-module-1.0.1.tgz#691d2709e78c79fae3a156622452d00762caaaa2" @@ -7249,11 +7220,6 @@ path-type@^4.0.0: resolved "https://registry.yarnpkg.com/path-type/-/path-type-4.0.0.tgz#84ed01c0a7ba380afe09d90a8c180dcd9d03043b" integrity sha512-gDKb8aZMDeD/tZWs9P6+q0J9Mwkdl6xMV8TjnGP3qJVJ06bdMgkbBlLU8IdfOsIsFz2BW1rNVT3XuNEl8zPAvw== -peek-readable@^5.1.3: - version "5.4.2" - resolved "https://registry.yarnpkg.com/peek-readable/-/peek-readable-5.4.2.tgz#aff1e1ba27a7d6911ddb103f35252ffc1787af49" - integrity sha512-peBp3qZyuS6cNIJ2akRNG1uo1WJ1d0wTxg/fxMdZ0BqCVhx242bSFHM9eNqflfJVS9SsgkzgT/1UgnsurBOTMg== - pg-cloudflare@^1.2.5: version "1.2.5" resolved "https://registry.yarnpkg.com/pg-cloudflare/-/pg-cloudflare-1.2.5.tgz#2e3649c38a7a9c74a7e5327c8098a2fd9af595bd" @@ -7620,11 +7586,6 @@ process-nextick-args@~2.0.0: resolved "https://registry.yarnpkg.com/process-nextick-args/-/process-nextick-args-2.0.1.tgz#7820d9b16120cc55ca9ae7792680ae7dba6d7fe2" integrity sha512-3ouUOpQhtgrbOa17J7+uxOTpITYWaGP7/AhoR3+A+/1e9skrzelGi/dXzEYyvbxubEF6Wn2ypscTKiKJFFn1ag== -process@^0.11.10: - version "0.11.10" - resolved "https://registry.yarnpkg.com/process/-/process-0.11.10.tgz#7332300e840161bda3e69a1d1d91a7d4bc16f182" - integrity sha512-cdGef/drWFoydD1JsMzuFf8100nZl+GT+yacc2bEced5f9Rjk4z+WtFUTBu9PhOi9j/jfmBPu0mMEY4wIdAF8A== - promise-all-reject-late@^1.0.0: version "1.0.1" resolved "https://registry.yarnpkg.com/promise-all-reject-late/-/promise-all-reject-late-1.0.1.tgz#f8ebf13483e5ca91ad809ccc2fcf25f26f8643c2" @@ -7917,17 +7878,6 @@ read@1, read@^1.0.7: string_decoder "^1.1.1" util-deprecate "^1.0.1" -readable-stream@^4.7.0: - version "4.7.0" - resolved "https://registry.yarnpkg.com/readable-stream/-/readable-stream-4.7.0.tgz#cedbd8a1146c13dfff8dab14068028d58c15ac91" - integrity sha512-oIGGmcpTLwPga8Bn6/Z75SVaH1z5dUut2ibSyAMVhmUggWpmDn2dapB0n7f8nwaSiRtepAsfJyfXIO5DCVAODg== - dependencies: - abort-controller "^3.0.0" - buffer "^6.0.3" - events "^3.3.0" - process "^0.11.10" - string_decoder "^1.3.0" - readable-stream@~1.0.31: version "1.0.34" resolved "https://registry.yarnpkg.com/readable-stream/-/readable-stream-1.0.34.tgz#125820e34bc842d2f2aaafafe4c2916ee32c157c" @@ -7951,13 +7901,6 @@ readable-stream@~2.3.6: string_decoder "~1.1.1" util-deprecate "~1.0.1" -readable-web-to-node-stream@^3.0.2: - version "3.0.4" - resolved "https://registry.yarnpkg.com/readable-web-to-node-stream/-/readable-web-to-node-stream-3.0.4.tgz#392ba37707af5bf62d725c36c1b5d6ef4119eefc" - integrity sha512-9nX56alTf5bwXQ3ZDipHJhusu9NTQJ/CVPtb/XHAJCXihZeitfJvIRS4GqQ/mfIoOE3IelHMrpayVrosdHBuLw== - dependencies: - readable-stream "^4.7.0" - readdirp@~3.6.0: version "3.6.0" resolved "https://registry.yarnpkg.com/readdirp/-/readdirp-3.6.0.tgz#74a370bd857116e245b29cc97340cd431a02a6c7" @@ -8247,6 +8190,11 @@ set-function-length@^1.2.2: gopd "^1.0.1" has-property-descriptors "^1.0.2" +setimmediate@^1.0.5: + version "1.0.5" + resolved "https://registry.yarnpkg.com/setimmediate/-/setimmediate-1.0.5.tgz#290cbb232e306942d7d7ea9b83732ab7856f8285" + integrity sha512-MATJdZp8sLqDl/68LfQmbP8zKPLQNV6BIZoIgrscFDQ+RsvK/BxeDQOgyxKKoh0y/8h3BqVFnCqQ/gd+reiIXA== + setprototypeof@1.2.0: version "1.2.0" resolved "https://registry.yarnpkg.com/setprototypeof/-/setprototypeof-1.2.0.tgz#66c9a24a73f9fc28cbe66b09fed3d33dcaf1b424" @@ -8576,7 +8524,7 @@ string-width@^5.0.1, string-width@^5.1.2: emoji-regex "^9.2.2" strip-ansi "^7.0.1" -string_decoder@^1.1.1, string_decoder@^1.3.0: +string_decoder@^1.1.1: version "1.3.0" resolved "https://registry.yarnpkg.com/string_decoder/-/string_decoder-1.3.0.tgz#42f114594a46cf1a8e30b0a84f56c78c3edac21e" integrity sha512-hkRX8U1WjJFd8LsDJ2yQ/wWWxaopEsABU1XfkM8A+j0+85JAGppt16cr1Whg6KIbb4okU6Mql6BOj+uup/wKeA== @@ -8678,21 +8626,6 @@ strong-log-transformer@2.1.0, strong-log-transformer@^2.1.0: minimist "^1.2.0" through "^2.3.4" -strtok3@^10.2.2: - version "10.3.1" - resolved "https://registry.yarnpkg.com/strtok3/-/strtok3-10.3.1.tgz#80fe431a4ee652de4e33f14e11e15fd5170a627d" - integrity sha512-3JWEZM6mfix/GCJBBUrkA8p2Id2pBkyTkVCJKto55w080QBKZ+8R171fGrbiSp+yMO/u6F8/yUh7K4V9K+YCnw== - dependencies: - "@tokenizer/token" "^0.3.0" - -strtok3@^7.0.0: - version "7.1.1" - resolved "https://registry.yarnpkg.com/strtok3/-/strtok3-7.1.1.tgz#f548fd9dc59d0a76d5567ff8c16be31221f29dfc" - integrity sha512-mKX8HA/cdBqMKUr0MMZAFssCkIGoZeSCMXgnt79yKxNFguMLVFgRe6wB+fsL0NmoHDbeyZXczy7vEPSoo3rkzg== - dependencies: - "@tokenizer/token" "^0.3.0" - peek-readable "^5.1.3" - subscriptions-transport-ws@^0.9.18: version "0.9.19" resolved "https://registry.yarnpkg.com/subscriptions-transport-ws/-/subscriptions-transport-ws-0.9.19.tgz#10ca32f7e291d5ee8eb728b9c02e43c52606cdcf" @@ -8883,22 +8816,6 @@ toidentifier@1.0.1: resolved "https://registry.yarnpkg.com/toidentifier/-/toidentifier-1.0.1.tgz#3be34321a88a820ed1bd80dfaa33e479fbb8dd35" integrity sha512-o5sSPKEkg/DIQNmH43V0/uerLrpzVedkUh8tGNvaeXpfpuwjKenlSox/2O/BTlZUtEe+JG7s5YhEz608PlAHRA== -token-types@^5.0.1: - version "5.0.1" - resolved "https://registry.yarnpkg.com/token-types/-/token-types-5.0.1.tgz#aa9d9e6b23c420a675e55413b180635b86a093b4" - integrity sha512-Y2fmSnZjQdDb9W4w4r1tswlMHylzWIeOKpx0aZH9BgGtACHhrk3OkT52AzwcuqTRBZtvvnTjDBh8eynMulu8Vg== - dependencies: - "@tokenizer/token" "^0.3.0" - ieee754 "^1.2.1" - -token-types@^6.0.0: - version "6.0.0" - resolved "https://registry.yarnpkg.com/token-types/-/token-types-6.0.0.tgz#1ab26be1ef9c434853500c071acfe5c8dd6544a3" - integrity sha512-lbDrTLVsHhOMljPscd0yitpozq7Ga2M5Cvez5AjGg8GASBjtt6iERCAJ93yommPmz62fb45oFIXHEZ3u9bfJEA== - dependencies: - "@tokenizer/token" "^0.3.0" - ieee754 "^1.2.1" - touch@^3.1.0: version "3.1.1" resolved "https://registry.yarnpkg.com/touch/-/touch-3.1.1.tgz#097a23d7b161476435e5c1344a95c0f75b4a5694" @@ -9081,11 +8998,6 @@ uglify-js@^3.1.4: resolved "https://registry.yarnpkg.com/uglify-js/-/uglify-js-3.19.3.tgz#82315e9bbc6f2b25888858acd1fff8441035b77f" integrity sha512-v3Xu+yuwBXisp6QYTcH4UbH+xYJXqnq2m/LtQVWKWzYc1iehYnLixoQDN9FH6/j9/oybfd6W9Ghwkl8+UMKTKQ== -uint8array-extras@^1.4.0: - version "1.4.0" - resolved "https://registry.yarnpkg.com/uint8array-extras/-/uint8array-extras-1.4.0.tgz#e42a678a6dd335ec2d21661333ed42f44ae7cc74" - integrity sha512-ZPtzy0hu4cZjv3z5NW9gfKnNLjoz4y6uv4HlelAjDK7sY/xOkKZv9xK/WQpcsBB3jEybChz9DPC2U/+cusjJVQ== - undefsafe@^2.0.5: version "2.0.5" resolved "https://registry.yarnpkg.com/undefsafe/-/undefsafe-2.0.5.tgz#38733b9327bdcd226db889fb723a6efd162e6e2c" From 35bc510fe65cfe7d8d88d6c53cd6544782eb667e Mon Sep 17 00:00:00 2001 From: Dan Lynch Date: Fri, 13 Jun 2025 18:41:18 -0700 Subject: [PATCH 4/7] fixes --- .../__snapshots__/mimetypes.test.ts.snap | 66 +++++++++---------- .../src/content-type-stream.ts | 56 +++++++++++++++- 2 files changed, 86 insertions(+), 36 deletions(-) diff --git a/packages/content-type-stream/__tests__/__snapshots__/mimetypes.test.ts.snap b/packages/content-type-stream/__tests__/__snapshots__/mimetypes.test.ts.snap index 7f71d5f67..00973de8e 100644 --- a/packages/content-type-stream/__tests__/__snapshots__/mimetypes.test.ts.snap +++ b/packages/content-type-stream/__tests__/__snapshots__/mimetypes.test.ts.snap @@ -87,30 +87,30 @@ exports[`mimetypes good files 1`] = ` }, }, "docx.docx": { - "contentType": "application/vnd.openxmlformats-officedocument", + "contentType": "application/vnd.openxmlformats-officedocument.wordprocessingml.document", "magic": { "charset": "us-ascii", - "type": "application/vnd.openxmlformats-officedocument", + "type": "application/vnd.openxmlformats-officedocument.wordprocessingml.document", }, }, "dwg.dwg": { "contentType": "image/vnd.dwg", "magic": { - "charset": "us-ascii", + "charset": "binary", "type": "text/plain", }, }, "dxf.dxf": { "contentType": "image/vnd.dxf", "magic": { - "charset": "us-ascii", + "charset": "binary", "type": "text/plain", }, }, "emf.emf": { "contentType": "image/emf", "magic": { - "charset": "us-ascii", + "charset": "binary", "type": "text/plain", }, }, @@ -124,28 +124,28 @@ exports[`mimetypes good files 1`] = ` "font.otf": { "contentType": "application/x-font-opentype", "magic": { - "charset": "us-ascii", + "charset": "binary", "type": "text/plain", }, }, "font.ttf": { "contentType": "application/x-font-ttf", "magic": { - "charset": "us-ascii", + "charset": "binary", "type": "text/plain", }, }, "font.woff": { "contentType": "application/font-woff", "magic": { - "charset": "us-ascii", + "charset": "binary", "type": "text/plain", }, }, "font.woff2": { "contentType": "application/font-woff2", "magic": { - "charset": "us-ascii", + "charset": "binary", "type": "text/plain", }, }, @@ -178,10 +178,10 @@ exports[`mimetypes good files 1`] = ` }, }, "less.less": { - "contentType": "text/x-swift", + "contentType": "text/x-less", "magic": { "charset": "us-ascii", - "type": "text/x-swift", + "type": "text/x-less", }, }, "lock.lock": { @@ -192,16 +192,16 @@ exports[`mimetypes good files 1`] = ` }, }, "md.md": { - "contentType": "text/x-csrc", + "contentType": "text/markdown", "magic": { "charset": "us-ascii", - "type": "text/x-csrc", + "type": "text/markdown", }, }, "mp4.mp4": { "contentType": "video/mp4", "magic": { - "charset": "us-ascii", + "charset": "binary", "type": "text/plain", }, }, @@ -222,7 +222,7 @@ exports[`mimetypes good files 1`] = ` "pct.pct": { "contentType": "image/x-pict", "magic": { - "charset": "us-ascii", + "charset": "binary", "type": "text/plain", }, }, @@ -243,15 +243,15 @@ exports[`mimetypes good files 1`] = ` "psd.psd": { "contentType": "image/vnd.adobe.photoshop", "magic": { - "charset": "us-ascii", + "charset": "binary", "type": "text/plain", }, }, "scss.scss": { - "contentType": "text/x-swift", + "contentType": "text/x-scss", "magic": { "charset": "us-ascii", - "type": "text/x-swift", + "type": "text/x-scss", }, }, "shellscript": { @@ -262,17 +262,17 @@ exports[`mimetypes good files 1`] = ` }, }, "sql.sql": { - "contentType": "application/x-sql", + "contentType": "text/x-sql", "magic": { "charset": "us-ascii", - "type": "text/plain", + "type": "text/x-sql", }, }, "svg-with-alpha-and-text.svg": { "contentType": "image/svg+xml", "magic": { "charset": "us-ascii", - "type": "text/plain", + "type": "image/svg+xml", }, }, "svg.svg": { @@ -292,14 +292,14 @@ exports[`mimetypes good files 1`] = ` "swf.swf": { "contentType": "application/x-shockwave-flash", "magic": { - "charset": "us-ascii", + "charset": "binary", "type": "text/plain", }, }, "tga.tga": { "contentType": "image/x-tga", "magic": { - "charset": "us-ascii", + "charset": "binary", "type": "text/plain", }, }, @@ -321,7 +321,7 @@ exports[`mimetypes good files 1`] = ` "contentType": "text/tab-separated-values", "magic": { "charset": "us-ascii", - "type": "text/plain", + "type": "text/tab-separated-values", }, }, "txt.txt": { @@ -335,28 +335,28 @@ exports[`mimetypes good files 1`] = ` "contentType": "video/mp2t", "magic": { "charset": "binary", - "type": "application/x-sh", + "type": "text/x-typescript", }, }, "typescript.tsx": { - "contentType": "application/x-sh", + "contentType": "text/x-typescript", "magic": { "charset": "binary", - "type": "application/x-sh", + "type": "text/x-typescript", }, }, "wmf.wmf": { "contentType": "image/wmf", "magic": { - "charset": "us-ascii", + "charset": "binary", "type": "text/plain", }, }, "xlsx.xlsx": { - "contentType": "application/vnd.openxmlformats-officedocument", + "contentType": "application/vnd.openxmlformats-officedocument.spreadsheetml.sheet", "magic": { "charset": "us-ascii", - "type": "application/vnd.openxmlformats-officedocument", + "type": "application/vnd.openxmlformats-officedocument.spreadsheetml.sheet", }, }, "zip.zip": { @@ -396,14 +396,14 @@ exports[`mimetypes malicious files 1`] = ` "contentType": "image/svg+xml", "magic": { "charset": "us-ascii", - "type": "text/plain", + "type": "image/svg+xml", }, }, "svg-with-script-tag-guya.svg": { "contentType": "image/svg+xml", "magic": { "charset": "us-ascii", - "type": "text/plain", + "type": "image/svg+xml", }, }, "svg-with-script-tag.svg": { @@ -417,7 +417,7 @@ exports[`mimetypes malicious files 1`] = ` "contentType": "image/svg+xml", "magic": { "charset": "us-ascii", - "type": "text/plain", + "type": "image/svg+xml", }, }, } diff --git a/packages/content-type-stream/src/content-type-stream.ts b/packages/content-type-stream/src/content-type-stream.ts index 5f639c9f8..3061ad434 100644 --- a/packages/content-type-stream/src/content-type-stream.ts +++ b/packages/content-type-stream/src/content-type-stream.ts @@ -1,10 +1,57 @@ // @ts-ignore import { BufferPeekStream } from 'buffer-peek-stream'; import type { Readable } from 'stream'; +import { extname } from 'path'; import { getContentType } from './get-content-type'; -const getCharsetFromMimeType = (mimeType: string): string => { +// Special cases for binary files that might be incorrectly detected as text +const binaryExtensions = new Set([ + // Font files + '.woff', + '.woff2', + '.ttf', + '.otf', + // CAD and vector formats + '.dwg', + '.dxf', + '.emf', + '.wmf', + // Image formats + '.psd', + '.pct', + '.tga', + // Media formats + '.mp4', + '.swf', + // Source code (should be text but often detected incorrectly) + '.ts', + '.tsx' +]); + +// Override MIME types for specific extensions +const mimeTypeOverrides: Record = { + '.ts': 'text/x-typescript', + '.tsx': 'text/x-typescript', + '.scss': 'text/x-scss', + '.less': 'text/x-less', + '.md': 'text/markdown', + '.sql': 'text/x-sql', + '.tsv': 'text/tab-separated-values', + '.svg': 'image/svg+xml', + '.shellscript': 'text/plain', // Keep as text/plain for better compatibility + '.docx': 'application/vnd.openxmlformats-officedocument.wordprocessingml.document', + '.xlsx': 'application/vnd.openxmlformats-officedocument.spreadsheetml.sheet' +}; + +const getCharsetFromMimeType = (mimeType: string, filename: string): string => { + const ext = extname(filename).toLowerCase(); + + // If it's a known binary extension, force binary charset + if (binaryExtensions.has(ext)) { + return 'binary'; + } + const asciiMimeTypes = [ 'text/', 'svg', @@ -44,8 +91,11 @@ export function streamContentType({ const Mimetics = require('mimetics'); const mimetics = new Mimetics(); const fileTypeResult = await mimetics.parseAsync(buffer); - const type = fileTypeResult?.mime || 'application/octet-stream'; - const charset = getCharsetFromMimeType(type); + const ext = extname(filename).toLowerCase(); + + // Use override if exists, otherwise use detected type + const type = mimeTypeOverrides[ext] || fileTypeResult?.mime || 'application/octet-stream'; + const charset = getCharsetFromMimeType(type, filename); const contentType = getContentType(filename, type, charset); resolve({ stream: peekStream, magic: { type, charset }, contentType }); } catch (err) { From 09bc8daf42ac71c101c9d157baee4cc4b3fc0b83 Mon Sep 17 00:00:00 2001 From: Dan Lynch Date: Fri, 13 Jun 2025 19:24:26 -0700 Subject: [PATCH 5/7] updates --- .../__tests__/__snapshots__/mimetypes.test.ts.snap | 6 +++--- .../content-type-stream/src/content-type-stream.ts | 12 ++++++++++-- 2 files changed, 13 insertions(+), 5 deletions(-) diff --git a/packages/content-type-stream/__tests__/__snapshots__/mimetypes.test.ts.snap b/packages/content-type-stream/__tests__/__snapshots__/mimetypes.test.ts.snap index 00973de8e..f605bb8c8 100644 --- a/packages/content-type-stream/__tests__/__snapshots__/mimetypes.test.ts.snap +++ b/packages/content-type-stream/__tests__/__snapshots__/mimetypes.test.ts.snap @@ -223,7 +223,7 @@ exports[`mimetypes good files 1`] = ` "contentType": "image/x-pict", "magic": { "charset": "binary", - "type": "text/plain", + "type": "image/x-pict", }, }, "pdf.pdf": { @@ -244,7 +244,7 @@ exports[`mimetypes good files 1`] = ` "contentType": "image/vnd.adobe.photoshop", "magic": { "charset": "binary", - "type": "text/plain", + "type": "image/vnd.adobe.photoshop", }, }, "scss.scss": { @@ -349,7 +349,7 @@ exports[`mimetypes good files 1`] = ` "contentType": "image/wmf", "magic": { "charset": "binary", - "type": "text/plain", + "type": "image/wmf", }, }, "xlsx.xlsx": { diff --git a/packages/content-type-stream/src/content-type-stream.ts b/packages/content-type-stream/src/content-type-stream.ts index 3061ad434..8bf247769 100644 --- a/packages/content-type-stream/src/content-type-stream.ts +++ b/packages/content-type-stream/src/content-type-stream.ts @@ -39,9 +39,12 @@ const mimeTypeOverrides: Record = { '.sql': 'text/x-sql', '.tsv': 'text/tab-separated-values', '.svg': 'image/svg+xml', - '.shellscript': 'text/plain', // Keep as text/plain for better compatibility + '.shellscript': 'application/x-sh', '.docx': 'application/vnd.openxmlformats-officedocument.wordprocessingml.document', - '.xlsx': 'application/vnd.openxmlformats-officedocument.spreadsheetml.sheet' + '.xlsx': 'application/vnd.openxmlformats-officedocument.spreadsheetml.sheet', + '.pct': 'image/x-pict', + '.psd': 'image/vnd.adobe.photoshop', + '.wmf': 'image/wmf' }; const getCharsetFromMimeType = (mimeType: string, filename: string): string => { @@ -52,6 +55,11 @@ const getCharsetFromMimeType = (mimeType: string, filename: string): string => { return 'binary'; } + // Special case for shellscript - should be binary + if (ext === '.shellscript') { + return 'binary'; + } + const asciiMimeTypes = [ 'text/', 'svg', From a0e0e25800278fc2353cbd554d708560b7481922 Mon Sep 17 00:00:00 2001 From: Dan Lynch Date: Fri, 13 Jun 2025 19:36:42 -0700 Subject: [PATCH 6/7] updates --- .../__snapshots__/mimetypes.test.ts.snap | 6 +++--- .../src/content-type-stream.ts | 15 ++++++++++----- 2 files changed, 13 insertions(+), 8 deletions(-) diff --git a/packages/content-type-stream/__tests__/__snapshots__/mimetypes.test.ts.snap b/packages/content-type-stream/__tests__/__snapshots__/mimetypes.test.ts.snap index f605bb8c8..0de8604c9 100644 --- a/packages/content-type-stream/__tests__/__snapshots__/mimetypes.test.ts.snap +++ b/packages/content-type-stream/__tests__/__snapshots__/mimetypes.test.ts.snap @@ -332,16 +332,16 @@ exports[`mimetypes good files 1`] = ` }, }, "typescript.ts": { - "contentType": "video/mp2t", + "contentType": "text/x-typescript", "magic": { - "charset": "binary", + "charset": "us-ascii", "type": "text/x-typescript", }, }, "typescript.tsx": { "contentType": "text/x-typescript", "magic": { - "charset": "binary", + "charset": "us-ascii", "type": "text/x-typescript", }, }, diff --git a/packages/content-type-stream/src/content-type-stream.ts b/packages/content-type-stream/src/content-type-stream.ts index 8bf247769..e0ba8739c 100644 --- a/packages/content-type-stream/src/content-type-stream.ts +++ b/packages/content-type-stream/src/content-type-stream.ts @@ -23,10 +23,8 @@ const binaryExtensions = new Set([ '.tga', // Media formats '.mp4', - '.swf', - // Source code (should be text but often detected incorrectly) - '.ts', - '.tsx' + '.swf' + // Removed .ts and .tsx from binary extensions as they should be text ]); // Override MIME types for specific extensions @@ -44,12 +42,19 @@ const mimeTypeOverrides: Record = { '.xlsx': 'application/vnd.openxmlformats-officedocument.spreadsheetml.sheet', '.pct': 'image/x-pict', '.psd': 'image/vnd.adobe.photoshop', - '.wmf': 'image/wmf' + '.wmf': 'image/wmf', + '.bmp': 'image/bmp', + '.lock': 'text/x-csrc' }; const getCharsetFromMimeType = (mimeType: string, filename: string): string => { const ext = extname(filename).toLowerCase(); + // Special case for TypeScript files - should be us-ascii + if (ext === '.ts' || ext === '.tsx') { + return 'us-ascii'; + } + // If it's a known binary extension, force binary charset if (binaryExtensions.has(ext)) { return 'binary'; From 0a81eb8032f45be49a3011934972bc60c187faef Mon Sep 17 00:00:00 2001 From: Dan Lynch Date: Fri, 13 Jun 2025 19:37:36 -0700 Subject: [PATCH 7/7] updates --- packages/content-type-stream/src/content-type-stream.ts | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/packages/content-type-stream/src/content-type-stream.ts b/packages/content-type-stream/src/content-type-stream.ts index e0ba8739c..e1c10f4e4 100644 --- a/packages/content-type-stream/src/content-type-stream.ts +++ b/packages/content-type-stream/src/content-type-stream.ts @@ -44,7 +44,8 @@ const mimeTypeOverrides: Record = { '.psd': 'image/vnd.adobe.photoshop', '.wmf': 'image/wmf', '.bmp': 'image/bmp', - '.lock': 'text/x-csrc' + '.lock': 'text/x-csrc', + '.tgz': 'application/gzip' }; const getCharsetFromMimeType = (mimeType: string, filename: string): string => {