From 260b05be1987c4e63c11eb368b8ea14677d41235 Mon Sep 17 00:00:00 2001 From: Samuel Attard Date: Mon, 30 Aug 2021 13:05:58 -0700 Subject: [PATCH 01/14] feat: add file hashes to asar header --- lib/filesystem.js | 3 ++ lib/hash.js | 28 ++++++++++++++++++ test/expected/packthis-all-unpacked.asar | Bin 348 -> 748 bytes test/expected/packthis-transformed.asar | Bin 530 -> 934 bytes test/expected/packthis-unicode-path.asar | Bin 124 -> 256 bytes test/expected/packthis-unpack-dir-glob.asar | Bin 460 -> 860 bytes .../packthis-unpack-dir-globstar.asar | Bin 478 -> 878 bytes test/expected/packthis-unpack-dir.asar | Bin 298 -> 634 bytes test/expected/packthis-unpack.asar | Bin 285 -> 617 bytes test/expected/packthis-without-hidden.asar | Bin 467 -> 803 bytes test/expected/packthis.asar | Bin 530 -> 934 bytes 11 files changed, 31 insertions(+) create mode 100644 lib/hash.js diff --git a/lib/filesystem.js b/lib/filesystem.js index 2a6579d..e40c0af 100644 --- a/lib/filesystem.js +++ b/lib/filesystem.js @@ -5,6 +5,7 @@ const os = require('os') const path = require('path') const { promisify } = require('util') const stream = require('stream') +const hashFile = require('./hash') const UINT32_MAX = 2 ** 32 - 1 @@ -57,6 +58,7 @@ class Filesystem { if (shouldUnpack || dirNode.unpacked) { node.size = file.stat.size node.unpacked = true + node.hashes = await hashFile(p) return Promise.resolve() } @@ -86,6 +88,7 @@ class Filesystem { node.size = size node.offset = this.offset.toString() + node.hashes = await hashFile(p) if (process.platform !== 'win32' && (file.stat.mode & 0o100)) { node.executable = true } diff --git a/lib/hash.js b/lib/hash.js new file mode 100644 index 0000000..c92a96c --- /dev/null +++ b/lib/hash.js @@ -0,0 +1,28 @@ +const crypto = require('crypto') +const fs = require('fs') +const stream = require('stream') +const { promisify } = require('util') + +const ALGORITHMS = ['SHA256'] + +const pipeline = promisify(stream.pipeline) + +async function hashFile (path) { + const hashes = {} + + await Promise.all(ALGORITHMS.map(async (algo) => { + const read = fs.createReadStream(path) + const hash = crypto.createHash(algo) + hash.setEncoding('base64') + await pipeline( + read, + hash + ) + + hashes[algo] = hash.read() + })) + + return hashes +} + +module.exports = hashFile diff --git a/test/expected/packthis-all-unpacked.asar b/test/expected/packthis-all-unpacked.asar index f699be095056799e4297c16dbfb5e73035ea2dbd..cb6376bd8871bf6b5c27f8bd2a4d3525f7e36c68 100644 GIT binary patch literal 748 zcma)(%Wi@|6oyIDCz-8AZUSP`1*jD9ibx6Ws0@N5IAK7Arcc^U7k!XQ?9Fyj7AKiG z$(i$g|J+g(^>#y1ujhI>*Aob$9VDRp1X^L_`!K2Qt?ZOlEfSRwbXmCx@&tQp1bxuW z(i}EH=#lU;X6Xtih@cBtrN6?AoCt-5;8H8uvTZ98o30sjbPIVV?Rs16={u}%?!fbN z6ZlceR$G3*{Y6wA*lU)0DCR~p7&^2fJ5M@==+8;uqLmaa7Hf1U9EGSMVfdN+=ngBC z4;b!W7_Nnr&5vL@T&-lvfs#UpH%0{sW-(*f4@#dKC^J`%NJT!bFl!^L(U^^pd7^QEkMocrVj1g8AfA9L6{+{(d@BYhPRH z4@|u6=&I#4=;bhW8iLV-xd|627N!$yIIDX;v~tbONYKl z=e|gB=Okx2!#AAw*2jIjg<;rF6T`l%el{;{M(W_ci+qUi!7FYmLYHCj0Zs|SO-k|! zQ3R71(q{{vYf4fL*J)`esl?1*SSb5Sf*yknkoM^)Z zhOvuIg)Jb_M`Taqx6b6o6oY!E90?h;d+M}6o)(j;wHAimk)SzzfHF|nUX%#sS@udm z-`8ou}v_l1alo4QOqpnscLV=J>Se_t75KqjYp;SW8FQ_r(+0wc3n!deyCT2ml5t! Z-NA{jB$lzT;8ET(1*)7X78G>lI0El6zS5i>|5-rZGN>#El&{4`y zODj$V3Mm;V=_q9+7H2?o2YWagnVJEGP0U?f0}HY&Je-0Pwe>?hbHlYwy_{1^15<;t pb4~qx3%tUk+>JAWOWi_km1=8sl%6i#{J3%5)3p<5ZB;E00008XP<;RZ delta 31 hcmZo*s^Mi}U|=W#;sPMfn#k)gae@YWEeJ3$003S12K@j4 diff --git a/test/expected/packthis-unpack-dir-glob.asar b/test/expected/packthis-unpack-dir-glob.asar index c0b55c9abd175d14d71ab3c066b40a40e02bd8ac..3317af022cf9a94c32e646a485c8a297abf3e45b 100644 GIT binary patch literal 860 zcmbV~TaTJR6vs*0r~Mf7(g#T8qFhY+uvKsYK@>ss4PlU}y4)|zX1{x(q;5%Vn{<*h zIhi?`|8M^1e8ez}{D5IZ8=u;E1p&Ii3`}1^O?7+m9J}5b`e2%;1ti|DyE^J~Ugup^ zh8hT_Jn>t3=w@O6ocZ)fPgpPo<(bz5(k2@hp_=-Gg@&$fbX;{z-L^s}6aCVe>7(QRC&~-c zq1;UB5@sk(kpzf>AcL(lfCL1%7Vp<9Jez^fpKUUZbZp2I`zKy zW6+ET-d97k4vHPJv;%qT1kK-P5mjm{neI`xcBku_OEpQDEG4;g_M8x9yVc?mA8~ex cG>?2|!8)-#Od)*evLGtbK0#%FlA-$HJER8?egFUf delta 55 zcmcb^c7~acg@J)#0V4y$93Y;`IFT=4;zEzfI*ewMmol16mSJ+8yoAYu!lBTO@A-7m7MS7 zpWsisC{VK>qOe~<+01w>lg(xu8@W47az}_PQ(u{+?%Ag6L|!Hbr8hAL`|TUb6W5{K zLh24&e>6i=AP&O}waEY)P~aaK802pIv|hvvUp@Z1cKk(8x)g9SV zTyEG6ZWH)e#;}heeM$yP@&YOC3T?`{RWX-N`JtIjppSzyDylR};6tTV$2>xrkKB(f|RGb)M?S2x$M(hPfksy=K- z&G%{APkXb){709pGi$|$d)3(-&P(B9DAH)IsHL|Rq&V9wX7}Vka7%3X^g9dD4a=ho LLU*rW;ri|cQa)DUInH5|BMfMkm diff --git a/test/expected/packthis-unpack-dir.asar b/test/expected/packthis-unpack-dir.asar index 02cbd14331e1cc27eed62604267ba5a4d87b84ba..443d761681bcb1263cd34a8f2912dcb9ac462d6a 100644 GIT binary patch literal 634 zcma))U2lRg6o$#NA7XkdkuO0p*#$Tiaeg2}rr8Z@I}j;^!X}vg_tBW?%v`c2Cp~$S zljnU;d(SY;>m9?qRAp6l1Sr@;0y;;q4N`&LXMr|CBRgbNB|!;6M`b~b5P_8hc)$YR zBmQaHHZ)OID%pDLn>^>+fs^BQp=@UaEyGnSn9n!id*NzC_X#ge)%9}t06we3QnE-H zIZnJy2)CeqE}HeIR#Nx8yB zA=b)|J}S8;wR3_>*Qup{k&?&_U0d%y$%2kTAGg%QEFbA3qvSY4(d*Gt8K@i#U_Sm1 z4x=P1=$fy2{EscGx3}DEJ@%|**EV$9t8>r8$gRs}7v>h6hmkxPC#Jg^NWPu-ol7kK z8Y5J>&ae3g$MW{g_jt$o%A)K(wRct}ptSDA<^~Qe13UAH1LOW$F32fV^}lGKT8v?X MHhgUp;|${L3$dKWQ~&?~ delta 65 zcmeyxvWkh1g@J*AhmnDS1BjUzC-T`(oZvHAo>6^rC!^NnZ;YzawY9Yj3~8A;sS5dd Rsd^xWfkJYAUP)?R2>{=05uN}5 diff --git a/test/expected/packthis-unpack.asar b/test/expected/packthis-unpack.asar index 82ffbfaa20af6d327b5fa4ba7391f85a155d30d9..6cfb4e5182521383b8de7d3c4b0a40cb76311912 100644 GIT binary patch literal 617 zcmZ{gTW^9e7(mIgA7c6{k((fxEE#wi;uRSZeM4;pkwPdq!R)^eW;Qi*O}_NYNt%<> zXNscqCyMHp^IFasApZafXrIB>Pq^y-1=Yee(oXk6KFdQ_X8wWE>mL>P_&htZ!eV*n1CmTlJsF zbPvsAv}ZhNQFKP~-Yfx1>TYCg;K0aWF|T(YX3T7+SPJMS9;Hyy3so?#?v0P&>dP1x zAIGxuOTO!8`2AjfUAM0ohOt`=W1NGnQ~N>fuq31eQ+D{qdIszXB0FTAXWi-o;>CWE z4NCE8L8Bm9NwMp0l4FS=RS50!b9o(@_w4&=DmOEg1`inV)}*A-JndTk5{`NroCk7D z51>R*lELalk1EiwnU9-hCQng&}z&Ruh9LP`MVuAe-rNd$Nd%%aapPM iVCy*HE;?}G-r8W(F*4ts`FO9dg_53`^O(=p#m_%G19bcV delta 45 ycmZ3?cA1%vg@J+L4-o$V;!g|{`Rpf7u$;`ps5ZHjQD^cKMs?=e+SONYKl z=e|gB=Okx2!#AAw*2jIjg<;rF6T`l%el{;{M(W_ci+qUi!7FYmLYHCj0Zs|SO-k|! zQ3R71(q{{vYf4fL*J)`esl?1*SSb5Sf*yknkoM^)Z zhOvuIg)Jb_M`Taqx6b6o6oY!E90?h;d+M}6o)(j;wHAimk)SzzfHF|nUX%#sS@udm z-`8ou}v_l1alo4QOqpnscLV=J>Se_t75KqjYp;SW8FQ_r(+0wc3n!deyCT2ml5t! Z-NA{jB$lzT;8ET(1*)7X7 Date: Mon, 30 Aug 2021 13:10:16 -0700 Subject: [PATCH 02/14] feat: add getRawHeader method to public API --- lib/asar.js | 4 ++++ lib/disk.js | 2 +- lib/index.d.ts | 8 ++++++++ 3 files changed, 13 insertions(+), 1 deletion(-) diff --git a/lib/asar.js b/lib/asar.js index c902e6a..a5e577c 100644 --- a/lib/asar.js +++ b/lib/asar.js @@ -157,6 +157,10 @@ module.exports.statFile = function (archive, filename, followLinks) { return filesystem.getFile(filename, followLinks) } +module.exports.getRawHeader = function (archive) { + return disk.readArchiveHeaderSync(archive); +} + module.exports.listPackage = function (archive, options) { return disk.readFilesystemSync(archive).listFiles(options) } diff --git a/lib/disk.js b/lib/disk.js index 72a3949..34569a4 100644 --- a/lib/disk.js +++ b/lib/disk.js @@ -76,7 +76,7 @@ module.exports.readArchiveHeaderSync = function (archive) { const headerPickle = pickle.createFromBuffer(headerBuf) const header = headerPickle.createIterator().readString() - return { header: JSON.parse(header), headerSize: size } + return { headerString: header, header: JSON.parse(header), headerSize: size } } module.exports.readFilesystemSync = function (archive) { diff --git a/lib/index.d.ts b/lib/index.d.ts index a95ba9a..d73bef5 100644 --- a/lib/index.d.ts +++ b/lib/index.d.ts @@ -44,6 +44,13 @@ export type InputMetadata = { } }; +export type ArchiveHeader = { + // The JSON parsed header string + header: any; + headerString: string; + headerSize: number; +} + export function createPackage(src: string, dest: string): Promise; export function createPackageWithOptions( src: string, @@ -59,6 +66,7 @@ export function createPackageFromFiles( ): Promise; export function statFile(archive: string, filename: string, followLinks?: boolean): Metadata; +export function getRawHeader(archive: string): ArchiveHeader; export function listPackage(archive: string, options?: ListOptions): string[]; export function extractFile(archive: string, filename: string): Buffer; export function extractAll(archive: string, dest: string): void; From e5c3b611422a92e3dad6695fa412d7da83eaf8db Mon Sep 17 00:00:00 2001 From: Samuel Attard Date: Mon, 30 Aug 2021 13:12:55 -0700 Subject: [PATCH 03/14] chore: fix lint --- lib/asar.js | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/lib/asar.js b/lib/asar.js index a5e577c..db869e9 100644 --- a/lib/asar.js +++ b/lib/asar.js @@ -158,7 +158,7 @@ module.exports.statFile = function (archive, filename, followLinks) { } module.exports.getRawHeader = function (archive) { - return disk.readArchiveHeaderSync(archive); + return disk.readArchiveHeaderSync(archive) } module.exports.listPackage = function (archive, options) { From 79f9712d02550ca8e4d05635a6852e2d80f54cd0 Mon Sep 17 00:00:00 2001 From: Samuel Attard Date: Mon, 30 Aug 2021 13:30:59 -0700 Subject: [PATCH 04/14] chore: update docs --- README.md | 19 ++++++++++++++++--- lib/index.d.ts | 13 ++++++++++++- 2 files changed, 28 insertions(+), 4 deletions(-) diff --git a/README.md b/README.md index 6dbd11c..97465b2 100644 --- a/README.md +++ b/README.md @@ -153,12 +153,18 @@ Structure of `header` is something like this: "ls": { "offset": "0", "size": 100, - "executable": true + "executable": true, + "hashes": { + "SHA256": "..." + } }, "cd": { "offset": "100", "size": 100, - "executable": true + "executable": true, + "hashes": { + "SHA256": "..." + } } } } @@ -168,7 +174,10 @@ Structure of `header` is something like this: "files": { "hosts": { "offset": "200", - "size": 32 + "size": 32, + "hashes": { + "SHA256": "..." + } } } } @@ -187,6 +196,10 @@ precisely represent UINT64 in JavaScript `Number`. `size` is a JavaScript because file size in Node.js is represented as `Number` and it is not safe to convert `Number` to UINT64. +`hashes` is an object consisting of key-value pairs where the key is the +identifier of a hashing algorithm and the value is the base64 encoded hash +of the file. + [pickle]: https://chromium.googlesource.com/chromium/src/+/master/base/pickle.h [node-pickle]: https://www.npmjs.org/package/chromium-pickle [grunt-asar]: https://github.com/bwin/grunt-asar diff --git a/lib/index.d.ts b/lib/index.d.ts index d73bef5..33dc677 100644 --- a/lib/index.d.ts +++ b/lib/index.d.ts @@ -44,9 +44,20 @@ export type InputMetadata = { } }; +export type DirectoryRecord = { + files: Record; +}; + +export type FileRecord = { + offset: string; + size: number; + executable?: boolean; + hashes: Record; +} + export type ArchiveHeader = { // The JSON parsed header string - header: any; + header: DirectoryRecord; headerString: string; headerSize: number; } From 949be1a807dae061ea4157d0fc7465ba9d04eee6 Mon Sep 17 00:00:00 2001 From: Samuel Attard Date: Tue, 31 Aug 2021 13:12:15 -0700 Subject: [PATCH 05/14] refactor: use integrity instead of hash pairs --- README.md | 20 +++++++------ lib/filesystem.js | 6 ++-- lib/hash.js | 28 ------------------ lib/index.d.ts | 5 +++- lib/integrity.js | 25 ++++++++++++++++ test/expected/packthis-all-unpacked.asar | Bin 748 -> 1000 bytes test/expected/packthis-transformed.asar | Bin 934 -> 1186 bytes test/expected/packthis-unicode-path.asar | Bin 256 -> 340 bytes test/expected/packthis-unpack-dir-glob.asar | Bin 860 -> 1112 bytes .../packthis-unpack-dir-globstar.asar | Bin 878 -> 1130 bytes test/expected/packthis-unpack-dir.asar | Bin 634 -> 842 bytes test/expected/packthis-unpack.asar | Bin 617 -> 829 bytes test/expected/packthis-without-hidden.asar | Bin 803 -> 1011 bytes test/expected/packthis.asar | Bin 934 -> 1186 bytes 14 files changed, 43 insertions(+), 41 deletions(-) delete mode 100644 lib/hash.js create mode 100644 lib/integrity.js diff --git a/README.md b/README.md index 97465b2..2b3fef5 100644 --- a/README.md +++ b/README.md @@ -154,16 +154,18 @@ Structure of `header` is something like this: "offset": "0", "size": 100, "executable": true, - "hashes": { - "SHA256": "..." + "integrity": { + "algorithm": "SHA256", + "hash": "..." } }, "cd": { "offset": "100", "size": 100, "executable": true, - "hashes": { - "SHA256": "..." + "integrity": { + "algorithm": "SHA256", + "hash": "..." } } } @@ -175,8 +177,9 @@ Structure of `header` is something like this: "hosts": { "offset": "200", "size": 32, - "hashes": { - "SHA256": "..." + "integrity": { + "algorithm": "SHA256", + "hash": "..." } } } @@ -196,9 +199,8 @@ precisely represent UINT64 in JavaScript `Number`. `size` is a JavaScript because file size in Node.js is represented as `Number` and it is not safe to convert `Number` to UINT64. -`hashes` is an object consisting of key-value pairs where the key is the -identifier of a hashing algorithm and the value is the base64 encoded hash -of the file. +`integrity` is an object consisting of a hashing `algorithm` and a hex encoded +`hash` value. [pickle]: https://chromium.googlesource.com/chromium/src/+/master/base/pickle.h [node-pickle]: https://www.npmjs.org/package/chromium-pickle diff --git a/lib/filesystem.js b/lib/filesystem.js index e40c0af..85e1eb0 100644 --- a/lib/filesystem.js +++ b/lib/filesystem.js @@ -5,7 +5,7 @@ const os = require('os') const path = require('path') const { promisify } = require('util') const stream = require('stream') -const hashFile = require('./hash') +const getFileIntegrity = require('./integrity') const UINT32_MAX = 2 ** 32 - 1 @@ -58,7 +58,7 @@ class Filesystem { if (shouldUnpack || dirNode.unpacked) { node.size = file.stat.size node.unpacked = true - node.hashes = await hashFile(p) + node.integrity = await getFileIntegrity(p) return Promise.resolve() } @@ -88,7 +88,7 @@ class Filesystem { node.size = size node.offset = this.offset.toString() - node.hashes = await hashFile(p) + node.integrity = await getFileIntegrity(p) if (process.platform !== 'win32' && (file.stat.mode & 0o100)) { node.executable = true } diff --git a/lib/hash.js b/lib/hash.js deleted file mode 100644 index c92a96c..0000000 --- a/lib/hash.js +++ /dev/null @@ -1,28 +0,0 @@ -const crypto = require('crypto') -const fs = require('fs') -const stream = require('stream') -const { promisify } = require('util') - -const ALGORITHMS = ['SHA256'] - -const pipeline = promisify(stream.pipeline) - -async function hashFile (path) { - const hashes = {} - - await Promise.all(ALGORITHMS.map(async (algo) => { - const read = fs.createReadStream(path) - const hash = crypto.createHash(algo) - hash.setEncoding('base64') - await pipeline( - read, - hash - ) - - hashes[algo] = hash.read() - })) - - return hashes -} - -module.exports = hashFile diff --git a/lib/index.d.ts b/lib/index.d.ts index 33dc677..efa63d6 100644 --- a/lib/index.d.ts +++ b/lib/index.d.ts @@ -52,7 +52,10 @@ export type FileRecord = { offset: string; size: number; executable?: boolean; - hashes: Record; + integrity: { + hash: string; + algorithm: 'SHA256'; + }; } export type ArchiveHeader = { diff --git a/lib/integrity.js b/lib/integrity.js new file mode 100644 index 0000000..e8ef9ed --- /dev/null +++ b/lib/integrity.js @@ -0,0 +1,25 @@ +const crypto = require('crypto') +const fs = require('fs') +const stream = require('stream') +const { promisify } = require('util') + +const ALGORITHM = 'SHA256' + +const pipeline = promisify(stream.pipeline) + +async function getFileIntegrity (path) { + const read = fs.createReadStream(path) + const hash = crypto.createHash(ALGORITHM) + hash.setEncoding('hex') + await pipeline( + read, + hash + ) + + return { + algorithm: ALGORITHM, + hash: hash.read() + } +} + +module.exports = getFileIntegrity diff --git a/test/expected/packthis-all-unpacked.asar b/test/expected/packthis-all-unpacked.asar index cb6376bd8871bf6b5c27f8bd2a4d3525f7e36c68..446b0550921b64eda011a9d2fb61692ac480a166 100644 GIT binary patch literal 1000 zcmbu8y>1&Z41f=4pG31^kP`JnZ{5202`GuQ4^14yz5q>wJZF~HQonGAT*=-MV?|NM0?ec9|uP*aV4kPXa&{;BB%9=e1We%7_&q0wO z)ShY%!~s(9MyeV?fpLrg|q*|3p9B^;vTF-Uzn<73tq>C zw%iLNt9pt!%BkjKfeKhs>mje@^x_Bf8@B&1w%?wP`#)qCe}nDT2zlcY2{8QBAQZ`=t-)-qCCZ)CLy zkPJ>ULlel#T(l(1mc?h$@Fr95Ps`K4!wEN#X{gYMJQr)&M#sgfQ7g>JDtlVf6DlR* z%7K~?M`o;fj72RmzT1LpTX>5uLg8GhmSR@2#2lcEDGP*MP MV;vu?zUrEO17`g%X8-^I literal 748 zcma)(%Wi@|6oyIDCz-8AZUSP`1*jD9ibx6Ws0@N5IAK7Arcc^U7k!XQ?9Fyj7AKiG z$(i$g|J+g(^>#y1ujhI>*Aob$9VDRp1X^L_`!K2Qt?ZOlEfSRwbXmCx@&tQp1bxuW z(i}EH=#lU;X6Xtih@cBtrN6?AoCt-5;8H8uvTZ98o30sjbPIVV?Rs16={u}%?!fbN z6ZlceR$G3*{Y6wA*lU)0DCR~p7&^2fJ5M@==+8;uqLmaa7Hf1U9EGSMVfdN+=ngBC z4;b!W7_Nnr&5vL@T&-lvfs#UpH%0{sW-(*f4@#dKC^J`%NJT!bFl!^L(U^^pd7^QEkMocrVj1g8AfA9L6{+{(d@BYhPRH z4@|u6=&I#4=;bhW8iLV-xd|627N!$yIIDX;v~tbT5IJY#C93iil;@H>gCU3I!DcH+5zZA%%voqP4jjUn>%^2Th`dEW(Uu$ZG5@j zs56;iMa%YD7zaT}Fjfi3R8&k@p$w;r2gM^5gu9SA#f9M{;|xTMQ4-6ov@}Yr)#?8J zY#Q<%S~j{6S{~@2#D0a;3?or6LWKz&v}D={XBBW3F%i8A+E5cC6Oy8M2TW?>2-TSn zT$SR3W+?GoB)cS%)w=CwA4kplKr%gyOz#C@PMIut!yI!IxXl7Fl1f5&&6;NS5IEAUNKyhz>Pen`Jm8X&4cVfeC_QQ;nS=cmYmZA}q=T01B3< zXpD|J0|m-5Wg|Z)7q`2`?&V8I288ipSViuL7lIiLo_d_F1qZ@wV={1tiL$$(QDnk6 z!-coPNMoo|cmJv zl2R^(vYJYVo8!)4;qGp<@?N#1RYm`QVXK{Xti4=aS(==A(zUq!wRSA3qDE@`)z5dUSaWB2p}sNw(4SoU>ysCiH=|>}ZdKosvCa1fm6?Oa r!N_4{>f|Iz#_lhDIsJ5cjGrXSiz^F9^N*iX#tZykyW^`QZsqU~0R&n& delta 429 zcmZw9J#K7g?o9-wdog?icLyGV>g?;R3_h zp<`i#O!PCVr}0~7B2;?gvCb$*Acb~Mofes=MU-l-!LU05n!^WKN*1<B`JU?1skvgB{YnLvIkDX44m|J z|K9!LFBxOo1!HB-h>g}QR z{bi1*`p^Vl9m8{1u7ylPqEcNIZky=XAPUNz;f5nAryZJ{D0P|yicDGL7E>fm2^xhY biMY;kDReF+jAI1T8G>lI0El6zS5i>|5-rZGN>#El&{4`y zODj$V3Mm;V=_q9+7H2?o2YWagnVJEGP0U?f0}HY&Je-0Pwe>?hbHlYwy_{1^15<;t pb4~qx3%tUk+>JAWOWi_km1=8sl%6i#{J3%5)3p<5ZB;E00008XP<;RZ diff --git a/test/expected/packthis-unpack-dir-glob.asar b/test/expected/packthis-unpack-dir-glob.asar index 3317af022cf9a94c32e646a485c8a297abf3e45b..ac3cc76d6f8345db8a261f86c544290ae313c792 100644 GIT binary patch literal 1112 zcmb`G!H&}~5QcNu(>{i|aEOpS_SoazxN+wRY|nTbv20PBLbog4oyiso5fZQxksM8& z|M1P{@y##{M)(f zY-s!~K0VBGx1Z+6B~|Y4c3N*gnvHq??k!+6quG|jR==x+ZaMXUDKsurprslxWGsNd zJqX(-rIQHPn4wo|#Rr>12oss4PlU}y4)|zX1{x(q;5%Vn{<*h zIhi?`|8M^1e8ez}{D5IZ8=u;E1p&Ii3`}1^O?7+m9J}5b`e2%;1ti|DyE^J~Ugup^ zh8hT_Jn>t3=w@O6ocZ)fPgpPo<(bz5(k2@hp_=-Gg@&$fbX;{z-L^s}6aCVe>7(QRC&~-c zq1;UB5@sk(kpzf>AcL(lfCL1%7Vp<9Jez^fpKUUZbZp2I`zKy zW6+ET-d97k4vHPJv;%qT1kK-P5mjm{neI`xcBku_OEpQDEG4;g_M8x9yVc?mA8~ex cG>?2|!8)-#Od)*evLGtbK0#%FlA-$HJER8?egFUf diff --git a/test/expected/packthis-unpack-dir-globstar.asar b/test/expected/packthis-unpack-dir-globstar.asar index 13c522873bb4b6ea51656a537189313cb266ff17..785ea2fcbdc56f66aa35d7f0b6fdc597ee648a52 100644 GIT binary patch literal 1130 zcmb`GL2nZ=5QX>9Q-6lf5wd4I_Bc0AaNvN%FR(r1CZe<`*@`q3|DD+YK@o=*(UrWi zci-WAZ=SEFY5I6IO&@H%v-RZ0_Ez_9{p1c|c|Y8Jt^Ko#ThF@(UH$pJ=)#-%FfXe8 z_NZ>ntH0#iyIF5`+xc-()!UnU+i$+Owfpq`El_eRx2gM$otwttC67oscBx#U=N2gz zsz4wks`##Tu&T6FU^MR)qc1VWMkToK_N?3io`r_jgCvS#s@= zeQKl&nP<+u)RaON#LlG;Q&{vy>?3xfoYfF5)#s}aFM-Q{R6&|*Yol4F3K2!9gTj!K zgb@QnDh4a1n9A20D%CK&87m-W?ROQ9C+KskFK7n6o`)Q3>oFT8^~hzQ_am_9)Ou|R zZO_dH8wPqBxj-j?LBcx3t|zgYqtS*6H3bY^&eYR_HE(vR*QWt&57$2cf!bQ# literal 878 zcmbV~U2md56oyH5ulh6Q7P9i8e3u!lBTO@A-7m7MS7 zpWsisC{VK>qOe~<+01w>lg(xu8@W47az}_PQ(u{+?%Ag6L|!Hbr8hAL`|TUb6W5{K zLh24&e>6i=AP&O}waEY)P~aaK802pIv|hvvUp@Z1cKk(8x)g9SV zTyEG6ZWH)e#;}heeM$yP@&YOC3T?`{RWX-N`JtIjppSzyDylR};6tTV$2>xrkKB(f|RGb)M?S2x$M(hPfksy=K- z&G%{APkXb){709pGi$|$d)3(-&P(B9DAH)IsHL|Rq&V9wX7}Vka7%3X^g9dD4a=ho LLU*rW;ri|cIJRi=jH|{*aeAvl~Hi?q0L|gIh%mNBTT&Nhy8r$QM zfBM_+hhdoB4a3K7AG*DcHb3ZP+`f)&KI3{{6RU%p+RnQ3!6?0z$2qUo;>{|sI$1C=PD zRiQ!!&E?QQJbD1v5(8-Hcdps2N{Gr*p&FpKYNj3hC!V`i()|Zh4#8*kLcl~Ojj?x4 zy|uNgL=MI6`6;zh8;z7r*Rf0YVufv{_J-MTjsF=mOWXi8=&W*;4-zF&?J rLXWhV~^FMCUo?guZB`@m9?qRAp6l1Sr@;0y;;q4N`&LXMr|CBRgbNB|!;6M`b~b5P_8hc)$YR zBmQaHHZ)OID%pDLn>^>+fs^BQp=@UaEyGnSn9n!id*NzC_X#ge)%9}t06we3QnE-H zIZnJy2)CeqE}HeIR#Nx8yB zA=b)|J}S8;wR3_>*Qup{k&?&_U0d%y$%2kTAGg%QEFbA3qvSY4(d*Gt8K@i#U_Sm1 z4x=P1=$fy2{EscGx3}DEJ@%|**EV$9t8>r8$gRs}7v>h6hmkxPC#Jg^NWPu-ol7kK z8Y5J>&ae3g$MW{g_jt$o%A)K(wRct}ptSDA<^~Qe13UAH1LOW$F32fV^}lGKT8v?X MHhgUp;|${L3$dKWQ~&?~ diff --git a/test/expected/packthis-unpack.asar b/test/expected/packthis-unpack.asar index 6cfb4e5182521383b8de7d3c4b0a40cb76311912..a5e6c4e18da3693a4aa8801a69e98c8b47d16ca2 100644 GIT binary patch literal 829 zcmb7?%Zghu3_vfmU(i(-T?}4_Aj$GW*>%@_zrYVE37ttoatAsc`tS7}$~2*?7>s3E zpu@2~PSf<|!!&&!`_tI3&i0Rbahq4y_h($r_rwz5dj2^t*~|Wyx{a*d(d?ol<-o1n z{xIwIyq{l|lJdAcj(GPp9>0J4N+FJlUAgSWoe7Yod&_~ca4pF^=}2fxJqwB|alk}e zjY5VBm}Bcc`PyU8`kar zMe`+gkbnWe_sjqq$5~o2YZ8;PG-w7GteNQmUA-yvx90qnIl0GNi7^xoLcl~OopH3x zqp$VGR&xO^1JcaU3M3WDnxiF4g;G)f*PK3`=9lG1Z?WJC3+Wy%^%WX3Nw$V zXNscqCyMHp^IFasApZafXrIB>Pq^y-1=Yee(oXk6KFdQ_X8wWE>mL>P_&htZ!eV*n1CmTlJsF zbPvsAv}ZhNQFKP~-Yfx1>TYCg;K0aWF|T(YX3T7+SPx8x04(scSr7l#$Y2h zf?%U)nKD&C5X3HEVP})pTJ(PqabA$?+xQhMny+ z$maI5qO&V1f0n2gqFFvFtL&ijIAwFt$a;u_M5&BtjjYox)8=lc+^b6b_GXXPTRXJ5 z-fl^gQ^l4aY*DNw7a9tM&PWG3c#BFVN~CBD2Le)(+6XIkKxrHgT(~?I%BoQ0g_GG6 zdKwj@{EtyK``yjIjKax(C}9-J7&Vu{LS96WK?Z@!=LH8N4CktFn)8&C_t7Ip$ZgUP zc|@SsHvmQSVWD%d9FQSBBDUgZ31w~ zBa#i!~CH79Ok%$vH%4y|Icrx=%_Ao>P8a~eG&9r7?S(b+la)=LqxY?yn_tO|4c3Tlf0EXewHto(s&|O&!RX{Nz&7dOU4+6yrE}EbnI*6cgf)<)ypgY46dWjyR zS7~~IdH}xU-9C%&@Am7aaSZ^N)B!Lqq58Rj9%;ZRVO;l}a}Cs#WUkBEuevEf>PB9c zc5?Y;&O$$4wj;+`$HyEarcoxeEM?`7)bqDmMHB;cAh>0Ya~4{OV0sJGZw`>&3JbW4@n3v2R$!y}pi8Pz;O}reosGI4YT7K4fz3Pr&D;{?nNaaP* ln{+m|9c;q`AMDIE8QDhaIWv#!^p%t|mHW61fb#C@=O1V{afJW? diff --git a/test/expected/packthis.asar b/test/expected/packthis.asar index ba0f5b63f41a7746fe22caed070e7168e0c1afb5..38383fbc073b9db9324cd6cb9430498827301cae 100644 GIT binary patch literal 1186 zcmb7^ziU)M5XU!qi7^rj3kj0qc#R0**q?8A(_Ki&1vP{uVC>vqlRHT6j@$!{A&uAw zf{mhO$}|RqfZ7Et>}=9ni~bKH&I=*lVRgm4*_k(w`MmG$Y=sc==`tb5Xz$VXGH7k3 zUN*az)tapsQ@f07#gn2;^;%C;Hp}cx)&b}x%w;s2$y)6qZSJ&+-Ll4QZFX?o+{Tw{ zjXGC3Ry19&iE)&aM(ebMTqn(?liCWVMbsiONrjKO*FsuBbHPETloWNsDaVq+T9fVX z&t$Q+!^%b%pyh%MBK8w9I}D=alt~)}7{!g1-f0j#VIl<`jb%0^E)_#~4_q1QDKmM9 zLYMH-a6~+ZNN2Nnd0AS## zPS%=aa?qe1*Di^3<dbKwtrP*MSEajI}v<%VpVJvLth9 zy%jPzX_d9iYdiwpD0QI~9n^{qiZv9IWX@{ilQ95BJEa9Rc%X!2G{-1;u+CAdS>lYc zq!nXAN$m_%9w*12iG{Pf-fZ=%rCL?={|2_&X{XwYmF30BiCYs`Ho4T8UqwG&+V~}W z-rc_Q9onzq$k)zc#N<@5uu-ga zpm^+d66rb|{#rW~Rav7o@$$#p6|A|vG+$qzdJs;o{0ZcF<@M;;&l}Y@bZq0@L1p@& raWHaNxp{Iz$k@Hb&!-7g?o9-wdog?icLyGV>g?;R3_h zp<`i#O!PCVr}0~7B2;?gvCb$*Acb~Mofes=MU-l-!LU05n!^WKN*1< Date: Tue, 31 Aug 2021 17:19:02 -0700 Subject: [PATCH 06/14] feat: add block hashes --- lib/integrity.js | 39 +++++++++++++++++- test/expected/packthis-all-unpacked.asar | Bin 1000 -> 1696 bytes test/expected/packthis-transformed.asar | Bin 1186 -> 8038 bytes test/expected/packthis-unicode-path.asar | Bin 340 -> 496 bytes test/expected/packthis-unpack-dir-glob.asar | Bin 1112 -> 1580 bytes .../packthis-unpack-dir-globstar.asar | Bin 1130 -> 1598 bytes test/expected/packthis-unpack-dir.asar | Bin 842 -> 1234 bytes test/expected/packthis-unpack.asar | Bin 829 -> 1217 bytes test/expected/packthis-without-hidden.asar | Bin 1011 -> 1403 bytes test/expected/packthis.asar | Bin 1186 -> 8038 bytes test/util/compareFiles.js | 3 ++ 11 files changed, 41 insertions(+), 1 deletion(-) diff --git a/lib/integrity.js b/lib/integrity.js index e8ef9ed..6060dfb 100644 --- a/lib/integrity.js +++ b/lib/integrity.js @@ -4,6 +4,8 @@ const stream = require('stream') const { promisify } = require('util') const ALGORITHM = 'SHA256' +// 4MB default block size +const BLOCK_SIZE = 4 * 1024 * 1024 const pipeline = promisify(stream.pipeline) @@ -16,9 +18,44 @@ async function getFileIntegrity (path) { hash ) + const blocks = [] + let currentBlockSize = 0 + let currentBlock = [] + + await pipeline( + fs.createReadStream(path), + new stream.PassThrough({ + decodeStrings: false, + transform (_chunk, encoding, callback) { + function handleChunk (chunk) { + const diffToSlice = Math.min(BLOCK_SIZE - currentBlockSize, chunk.byteLength) + currentBlockSize += diffToSlice + currentBlock.push(chunk.slice(0, diffToSlice)) + if (currentBlockSize === BLOCK_SIZE) { + blocks.push(Buffer.concat(currentBlock)) + currentBlock = [] + currentBlockSize = 0 + } + if (diffToSlice < chunk.byteLength) { + handleChunk(chunk.slice(diffToSlice)) + } + } + handleChunk(_chunk) + callback() + }, + flush (callback) { + blocks.push(Buffer.concat(currentBlock)) + currentBlock = [] + callback() + } + }) + ) + return { algorithm: ALGORITHM, - hash: hash.read() + hash: hash.read(), + blockSize: BLOCK_SIZE, + blocks: blocks.map(block => crypto.createHash(ALGORITHM).update(block).digest('hex')) } } diff --git a/test/expected/packthis-all-unpacked.asar b/test/expected/packthis-all-unpacked.asar index 446b0550921b64eda011a9d2fb61692ac480a166..7f24f480060423f1d10b6f6b1b48b543209aac93 100644 GIT binary patch delta 351 zcmaFCzJS+=g@J)#1{(vz6d;}e#MMe^nK`M&N>W7 z_}aQY#mv;u%+$!-($K&(Inm7A+}O~4piTnHF1Ht20q*1~-za6APR;LBdKY znMH<^3mFwB-tIw`pRCI0#27m{opC3M=smEg0@D-}(IO_0+AU18P(*o{L85-lQ&B`0 MGCQ#XEv~Hv03<7BHvj+t delta 60 zcmZ3$`+{AHg@J+L0W$-`9U#6A#MMe^nK`M&N>&H(%tL`>V6NC)|BGLt~epRolUeE9Ms=KD>UY2F| z7qYAb+Xs8tkE6XX?O%V`9}KtO-!3PIA$6vsj{pT`YPmm&aT?10^*-t^_ebM0>>Q5D z2Wgbq+nGRncONcpZ>|z9V8pJO?n2)Sjs+)DW5k^iQc|oCM@T5;BeUKKZHSXrN-Q~v z$S@HiCdkU@Rd5s{*}aLo2fyvz`GdXp=+Sb2usiZTj1%KQc^?@4#S`lpmWlMah%$t{ zHj+glgCxu_sVSD!OBygk)OukhWtt*ENn{}ejtn*2;J{&Qy4MWRd4p9%{pb)UcBi>b z4czA3rYz)nC^LdE)1F4fiPFp(87T?i2u^yXu?~SSjzJ6?30KG>tc3GaCNaDsEy{BT z1Ij78K$@H(O$G$FQ!cSCG3P0F5&- zIt|N#X@^`ym`4|5XwjZKSe5CNSlUi3on=V*xz@4~0@ftg1TjLQ_6!3D1w^8EELvp{ zGr&5=h@(L=Xsv=0QATNqG@<>_&K;~tY}h|2ADn$?QH!U*=gDNC7CFXIeYX19w%RU7aNWi{t=YY4E5<@IcAysetv=yg6dz^>pwT&A62Y zk7}%^XL)n@$eTOo%?Ty1?hf&bYIv5f)Y#RX0-XYF1)i_FSJM8!4)zb{|MsM=*D25` z@V`lThEY7{=1DgwU596tKm?!{scfLZN*DqRe|e8Y5=Y@cvK^Jpmcu( zDjB+}Q{Z1!;P&olTBPqe1>oPGWQ)l-3_iViW8?D0S1!U2y1cbHyaiZ&1XxT@`WySN z{|t~%Z*8n)Pky@g8+3X*t8cAl**BkG`rTZB{)-2j?@Y7or5}>zKTaNda4wbWcgpQa zEI%}dAv-Ye#cwbUWx)n$KtKECtD7+9wXNam?WMQf$qRqD?33OX7Z!efx%d(--1+*r wckOudIDg!`dUBFw3vX_G|Ma`3-@=q^YyHOX$=cnIF`*^I8&6z;!rmzU2{d>m`v3p{ delta 127 zcmaE6w}?}Ug@J)#2Qvf179d^=#MMe^nK`M&N>OcrF;nHV!Z diff --git a/test/expected/packthis-unicode-path.asar b/test/expected/packthis-unicode-path.asar index 43bbd56f335bda047aa9f4e4fd63047b09495ce9..dee750de69dd2f89fe5a645a389371ce48373eca 100644 GIT binary patch delta 66 zcmcb@^nsa|g@J+L1tSB)6Ci#tk#`NZj#5%iesXrPl2!C%9!4j|*oix*q6@Oe)`9^5 Ddi@ka delta 32 icmeyse1(aZg@J*=hmnE71BhKF@~)Z8&N!8=77PGz5eOpy diff --git a/test/expected/packthis-unpack-dir-glob.asar b/test/expected/packthis-unpack-dir-glob.asar index ac3cc76d6f8345db8a261f86c544290ae313c792..59e056ac061a4d49cbe4e71cc7594a2c6c8aa280 100644 GIT binary patch delta 152 zcmcb?v4)3_g#ifI7#RMrGBA8+oyfO`TSqA=CqFs6Sjj4SVuurB?8H-Bk;Nw4G6DrB q*E23g5q$s_RbpC*BHG9VQhS(b6N;!YGf1?Qc^iu8OR%Umi!K0>@Gh|c delta 46 zcmZ3(bAyAAg@J)V2S{iDu`P(bUne55bgCaMX38dpH(-9QWaAuI`Oy+GUqP#31QGXU`0RB`j A0ssI2 delta 80 zcmdnT^NNFyg@J(~07&=%u{+B|zCEn9K%g^Okws;)BV*5GX{N=Kd6+j(=4ClDS)O&9 ZG)NW%7}7FxQcE(5Qd7A=tm4cHE&ys67`p%f diff --git a/test/expected/packthis-unpack-dir.asar b/test/expected/packthis-unpack-dir.asar index 6630436c189b2e0e2fd96ad89d51194c452ab9e5..391c33858b1231bba58fe29b6114c26e0b980cf2 100644 GIT binary patch delta 155 zcmX@bc8QaZg@J)#3y|0V#4A}Q@~!69QA*0mPtGn@vWlKq?!*{7apz8CvB?IEK*7mH zj8jlVkAOwln5LtMMl*rb&Sjc}BKnreNiMb)2!M8{W#*(RGjJ0NLOwL;wH) delta 47 zcmey(^_iWIg@J)VhM9pu0*Hl~C-SYH%+9!M@^8k;lkYIioP3yh3QH{zOkT}$0RS9p B4-fzV diff --git a/test/expected/packthis.asar b/test/expected/packthis.asar index 38383fbc073b9db9324cd6cb9430498827301cae..2a0df909bb2e1c99bbb1fb6dc06f9e4853aa8932 100644 GIT binary patch literal 8038 zcmeHM&ubk;9N*Q~7+WgELkZHu@<2ogkNN$hrzE_P1htf=U`52v&*ahMy^yzJYbgZr zA_!g-J?7X`K@h}Sz=Jn0y?Pe?A4L4l?i$|`goXzqvI8@pneWbgcR!!+?9L9;%d+hL zLY9?~_aQ&*$I)Jx_OCze4~Ekpq6{Ig zjbu^CAPF-}YKkTGk_OBWwO&|BnWjik5?Kg=BSQ^0IB@8j?lnVnUS}0iKRU#T-Dzx7 z1-Ci3DGPZX%8Ve)w5L&VqBOHcMoI!0f|FiptV1A-V-Uke!WFUzE8#qqNer(@i}Kvg zfO3j1kS1qHlfl7wXC}~!v?49jgni~I?HohIN-ez8#1e}mHNqp!G0bb~6~rzGK;uk| zPQ!Fy+94Ma=F!C%TD0eORz*4mmbL>+XBkp{uDNW4fH}!EL5z^7J;T630g>n(i&h!L z46u$d;%JZzTC1Q$lu;TYO=!Qgb31DS8}<*%2WMYe)Z!`dc_JC8MUF9|Rdm?FTsF~0 z=9JbVT1{9Y!*F1NX{#x7hH0)fwh|`1Rh+jXIk&UZNTPNmaXXafqsXw)QrVzXj7l0V zDN^t@QihR$R)~K8ntm${ z9@SV6&+=yTkvDhFn;lAC-5vZFRrf4ksj(|N1v&-V3OrvoucY;V9r8cy|J#GQR;NIx z!2e2tyf@q)t^uWHZ9PLS+3z;iR6l|YSapY@^%DS%v=lEvt`b}?QU!3O!J`_%0;&5E zsATAhPJw?_f!n*IX_3xz3c$aQWQ)l-3_iViW8?D0S1!T>UEbOp-U6%+0T$Dn{>J|6 zKLg~`TN`WHlb^2r294g%>RYQ>_RZ&)em57O{o=vqJJT$C>4%i_A19AJ7?;ZRJLUEy zmLHnKkR2E}@f(aoS+D^b(9eGP>L&DgZELuCd+BX=^1>f3`=s~9g@s>VF1|zycfLOE vT|3@9&L8)#o}6Ua!kZi4KmG3Mw=g8zTE8)TvUc}lOlabR3O9k{@p1VlG3F)v delta 127 zcmaE6w}?}Ug@J)#2Qvf179d^=#MMe^nK`M&N>OcrF;nHV!Z diff --git a/test/util/compareFiles.js b/test/util/compareFiles.js index 3862079..485ba2b 100644 --- a/test/util/compareFiles.js +++ b/test/util/compareFiles.js @@ -4,6 +4,9 @@ const assert = require('assert') const fs = require('../../lib/wrapped-fs') module.exports = async function (actualFilePath, expectedFilePath) { + if (process.env.ELECTRON_ASAR_SPEC_UPDATE) { + await fs.writeFile(expectedFilePath, await fs.readFile(actualFilePath)) + } const [actual, expected] = await Promise.all([fs.readFile(actualFilePath, 'utf8'), fs.readFile(expectedFilePath, 'utf8')]) assert.strictEqual(actual, expected) } From 6b936a54627adbc3da71cc11e60fe1d04eb59560 Mon Sep 17 00:00:00 2001 From: Samuel Attard Date: Fri, 3 Sep 2021 11:42:24 -0700 Subject: [PATCH 07/14] fix: ensure executables are extracted with executable permission --- lib/asar.js | 3 +++ 1 file changed, 3 insertions(+) diff --git a/lib/asar.js b/lib/asar.js index db869e9..050e1a7 100644 --- a/lib/asar.js +++ b/lib/asar.js @@ -203,6 +203,9 @@ module.exports.extractAll = function (archive, dest) { // it's a file, extract it const content = disk.readFileSync(filesystem, filename, file) fs.writeFileSync(destFilename, content) + if (file.executable) { + fs.chmodSync(destFilename, '755') + } } } } From 94606dd4d00d800eaeb3f08e0aa0df4cf91b08fc Mon Sep 17 00:00:00 2001 From: Samuel Attard Date: Fri, 3 Sep 2021 11:42:41 -0700 Subject: [PATCH 08/14] fix: ensure symlinks are not deeply resolved when packaging --- lib/crawlfs.js | 14 ++++++++++++-- 1 file changed, 12 insertions(+), 2 deletions(-) diff --git a/lib/crawlfs.js b/lib/crawlfs.js index cc4a796..468272c 100644 --- a/lib/crawlfs.js +++ b/lib/crawlfs.js @@ -9,10 +9,10 @@ async function determineFileType (filename) { const stat = await fs.lstat(filename) if (stat.isFile()) { return { type: 'file', stat } - } else if (stat.isDirectory()) { - return { type: 'directory', stat } } else if (stat.isSymbolicLink()) { return { type: 'link', stat } + } else if (stat.isDirectory()) { + return { type: 'directory', stat } } } @@ -20,11 +20,21 @@ module.exports = async function (dir, options) { const metadata = {} const crawled = await glob(dir, options) const results = await Promise.all(crawled.map(async filename => [filename, await determineFileType(filename)])) + const links = [] const filenames = results.map(([filename, type]) => { if (type) { metadata[filename] = type + if (type.type === 'link') links.push(filename) } return filename + }).filter((filename) => { + // Newer glob can return files inside symlinked directories, to avoid + // those appearing in archives we need to manually exclude theme here + const exactLinkIndex = links.findIndex(link => filename === link) + return links.every((link, index) => { + if (index === exactLinkIndex) return true + return !filename.startsWith(link) + }) }) return [filenames, metadata] } From 425a4552d93fd830443376e5cebedf6fb101852d Mon Sep 17 00:00:00 2001 From: Samuel Attard Date: Fri, 3 Sep 2021 12:09:58 -0700 Subject: [PATCH 09/14] chore: update test files --- test/expected/packthis-all-unpacked.asar | Bin 1696 -> 2088 bytes test/expected/packthis-transformed.asar | Bin 8038 -> 14578 bytes test/expected/packthis-unicode-path.asar | Bin 496 -> 536 bytes test/expected/packthis-unpack-dir-glob.asar | Bin 1580 -> 1700 bytes .../packthis-unpack-dir-globstar.asar | Bin 1598 -> 1718 bytes test/expected/packthis-unpack-dir.asar | Bin 1234 -> 1334 bytes test/expected/packthis-unpack.asar | Bin 1217 -> 1317 bytes test/expected/packthis-without-hidden.asar | Bin 1403 -> 1503 bytes test/expected/packthis.asar | Bin 8038 -> 14578 bytes 9 files changed, 0 insertions(+), 0 deletions(-) diff --git a/test/expected/packthis-all-unpacked.asar b/test/expected/packthis-all-unpacked.asar index 7f24f480060423f1d10b6f6b1b48b543209aac93..5db9eb76b168d83c7f57f50323047fce0f427431 100644 GIT binary patch delta 191 zcmZ3$yF!4Eg@J)VfrEiT28cyDCh{%k4bH4eRkAWMv@|g`FqydS1FFE}>5Rn_rDvfE zPFxo>nTu&Us^I2XOni)}LX#IVzmz~RUq>k^CqFs6Sjj4SvLlPrr8fILQ^uiov95ZZ)5_LFG1BlxsJJz#n8yq zV6p(S?qqoubgc<2*;&!$Jy?s7UPYqN!IZe|ma*xbOzA~AWo+<5>? Ckwp;z delta 121 zcmexV_{@%vg@J*gf{lTp1c-ClCh{$rcz~EhgKumP|g*S^(sj83H*vlQY$$<17r(5CDV< B4l)1$ delta 33 mcmbQi@`0I;g@J+L1tSB)6Ci%TIFWDD#4C>{Uh!n91p)w}>w5Ch{HT4bH4eRkAWMv@|g`FxhyDixE|1@>#|oXd?TV YzM+b2Udzmmrfu_D7H%}5$s1X#0p)Wepa1{> delta 47 zcmZ3&yM~94g#ifI7#RMrGBA8+oyd21<5OnF$$uC>PG)3&znF!EakB&~3*%%1wh{m< C<_$mq diff --git a/test/expected/packthis-unpack-dir-globstar.asar b/test/expected/packthis-unpack-dir-globstar.asar index 27dbff47cdfd0147b3c4fe3c8d4ca25e08a40e33..3ae71943bee3fdb14f27068f53e9d9b924273527 100644 GIT binary patch delta 132 zcmdnTvyGRRg@J)#4jTi*3?QC7k@pC1aAsAil9h>}rHQeD$;MM$jHn`$&ocf%71_Lx Yi33g1=Kai)=t37+xY2|rUuDe%029R{wg3PC delta 48 zcmdnSyN`#Lg@J)Vg^hth0f?n1@*dgvl$mkzAI6WH8JQUwH;b_dF>bbDWnr8g!j=R8 D2yP2# diff --git a/test/expected/packthis-unpack-dir.asar b/test/expected/packthis-unpack-dir.asar index 391c33858b1231bba58fe29b6114c26e0b980cf2..51d6439b42570c8e7ed90064122b3e2b43e46aef 100644 GIT binary patch delta 127 zcmcb_xs8jDg@J)Vf|Y?m1c-TAC-QCL4bH4eRkAWMv@|g`Fxj}1j}cX5@&U$|5-4hP hl#+7tle3GJtfD6;FgZr8fILQ^uiov95ZZ)5_LFG1BlxsJJz#n8yq zV6p(S?qqoubgc<2*;&!$Jy?s7UPYqN!IZe|ma*xbOzA~AWo+<5>? Ckwp;z delta 121 zcmexV_{@%vg@J*gf{lTp1c-ClCh{$rcz~EhgKumP|g*S^(sj83H*vlQY Date: Fri, 3 Sep 2021 12:14:05 -0700 Subject: [PATCH 10/14] chore: remove DS_Store --- test/expected/packthis-all-unpacked.asar | Bin 2088 -> 1588 bytes test/expected/packthis-transformed.asar | Bin 14578 -> 1774 bytes test/expected/packthis.asar | Bin 14578 -> 1774 bytes 3 files changed, 0 insertions(+), 0 deletions(-) diff --git a/test/expected/packthis-all-unpacked.asar b/test/expected/packthis-all-unpacked.asar index 5db9eb76b168d83c7f57f50323047fce0f427431..f809530da995810dd178bebc63f9f7fd24153fcb 100644 GIT binary patch delta 41 tcmZ1>u!Tp8g@J)VhmC3of(+rIalFZXmEK-w=CvH@7q0k7W*xFj1jZ*TA;zPlqn=KgsurM(&001tEVwV5_ diff --git a/test/expected/packthis-transformed.asar b/test/expected/packthis-transformed.asar index c36db8e1d2c5f1feae336915d29e2681062dba87..9960c6ea823dbc4fb004fc694f8925cb8a36f95b 100644 GIT binary patch delta 126 zcmexV_>NbKg@J*Ag$+mm@lRF;hH9m>%$(F>C9CSm0*va629xbsbtb1X?wBmWtiWgq z=4CM7W-_t>GfP;GF&i0z6#W3w#t>3&9Dh%z+qI=qJjfzF3b{DmjPQi+ZP4bf6!QGiv*=&orHJ3W_bfXtx()-&C(3E2qQmkU2UP(YO6mO z`<=u7_)e^4_I8GFy}J)Lds}OS3#hScM!S$N1jm9CsWIZt2q`I6h$AGFDgv{G6WS0b zt&~`D5|ClS2TYKap-|pYc#_>4y7zkJn=q^yMXWA^_kDYQ1Ml3sx7up&_KU&~q6+Qt zhrseL?pU8-9)*~RJVU6^MzSEJmxLK6HN}z^l6uS#wS};fGEI@7BrxwiM}``1u;)-W z+B${&yar10MgI^-#xwYUz(UTX#DBvLq5^o@0Z#TmCKU5oXfuK^(*+HR6Q!9oGEm~7 zz&lwejkWiLaSX0u1K|o;gq3gwmC-c{MNV~qxeb5=ME?gDAX6?7l_2ed!On@HpvZ}_ z09p_>ix$o?M6A@JaGF?RaiB&NNOKI+0WB1`vKN5HnGl?Y5yP}YE+EW<3&EeN#oPwU z!4QelX&0bVv^k2?+rORwjSw)Fxh9Aa5_G{Z5L3V|6pjU}3}ObP2qEBLkn~zBuSAeR z8X}E9eFB`@KG;RE3r_aQjVWXw8Ua1h2G+a`oV1!eKk${}H zK^ud##GYV;z2q2E&ZJS8Tli4eQ+dyApzQRx3dq;rFL0J!t`-C(W-rGZPX|~q#6`o? z;h7UfdG)LlCFwFLkQ7J?Bn6TJNr6RF0KVBVEMKfJrAdLLKvLkb0&#w5LX6m{aZ+J9 zbwE;X0RS6NTn4y~dy3emS!1WhNri9+LS8imteWQa79+2kLwU2V*Qs$*Vbwg#>&-`A zTh8kpN?uxr=QqphS>CBIrAdLLz_bF-R@uw(`@aG4Z}$JwjZ!WtkQDe|DUdh1z3w_t zs=uvIk&ETG>(Y2Vfe2Vuhh%CI0GaqHUV&H!xZjKyfSnpA6+#Jwv=As|NJ&!QOcl7f z+aERKxla?YItB;(=~&=QJ0TfM3M2)R0!e|Sz``g{t;^G~z`}G$dXc2S0x1yZhbF{` zof;<B?`AY3!`Mv6f}u ze17qfIS2XY4z}JJW!dvT#F+m$yk9`!%6Q|}xHk;r56q#@4h;OCrakcECTpWM{p^>o zu0x$yw!3RLSKf5T=l^ioCyg)8E&cjJ^GmdJ>+7S&m7}er{C?xbNbKg@J*Ag$+mm@lRF;hH9m>%$(F>C9CSm0*va629xbsbtb1X?wBmWtiWgq z=4CM7W-_t>GfP;GF&i0z6#W3w#t>3&9Dh%z+qI=qJjfzF3b{DmjPQi+ZP4bf6!QGiv*=&orHJ3W_bfXtx()-&C(3E2qQmkU2UP(YO6mO z`<=u7_)e^4_I8GFy}J)Lds}OS3#hScM!S$N1jm9CsWIZt2q`I6h$AGFDgv{G6WS0b zt&~`D5|ClS2TYKap-|pY_$9kHbno@bCt+AKidbC+&-?cL2Hv@MZ?)Cl?H7d~L>1cO z4}s-h+_65vJPI)rd4^D-jbuScF9|bDYKkQ-B=wjfY71c{Wtt*ENnqZ4jtn*2V9%j$ zv~>#kc@320i~b>wjA!rxfrXq&iT{QhL4(h*+sb;WV+t;y{fkkmeYs16n9>WiJ4YGa)z)BZg^*TtJuy7lJ=ki@6Py zgCP>9(=I@#Xmb>&w|_kW8X;gTb4?H#q`8N>`o5kkPhAnCPMUWp)s zG(;MI`UE()f!YQ9{=xW8WIs(FJwaa2Lf%u09AiYQ;IMM1& z!!sv}^6Ixvl%&g~KvEznkQ7J?Bn1{x0eEM}uzaz?lqLm|0!e|#3dH%L2{B@)#z}?c z)B#Dk1psV7aT(w`?kQrMW{sU1Cl$gS2zk{MuxgsuTa3JF4&}|dUZ=)Mg;nz`uQwlg zZ8@)ZD0yiee!p2(&+<-%DNPC_1*R2vw#r_P@Ba;mf3yFeZj^FKfuz9yN`bu5?RD3I zQvGgyid-zeU6;n|2}HoMIwVtz0La8w@e0H;!2M>t0PNH_sSrvaq=i5!LrRhYXR5%> z-TtT<&wZMJ)iF5OPsajh+6l>6QXnai6i5mr1r|nuYF(a=1s0}5(t{)g7D$0OKQtjm z?9@1^u$(#|nfgZnZpFIkSRe|KLQ-JC6-dVd3*JHL@sa{(pg=knI0KQBiKM_HDNvpW z3Gf-ymoB_?0Uqws_Ez@>VEG}ya$MWLw*TtS0Qu;~=6d$vrz^ifrm?g3##)wr z^ZCU`<{adoJJ@<_lx5HV5M%!1@O}Y>E8~q@ Date: Tue, 7 Sep 2021 16:17:50 -0700 Subject: [PATCH 11/14] perf: generate block hashes as we parse the stream --- lib/integrity.js | 10 +++++++--- 1 file changed, 7 insertions(+), 3 deletions(-) diff --git a/lib/integrity.js b/lib/integrity.js index 6060dfb..daf0c0e 100644 --- a/lib/integrity.js +++ b/lib/integrity.js @@ -9,6 +9,10 @@ const BLOCK_SIZE = 4 * 1024 * 1024 const pipeline = promisify(stream.pipeline) +function hashBlock (block) { + return crypto.createHash(ALGORITHM).update(block).digest('hex') +} + async function getFileIntegrity (path) { const read = fs.createReadStream(path) const hash = crypto.createHash(ALGORITHM) @@ -32,7 +36,7 @@ async function getFileIntegrity (path) { currentBlockSize += diffToSlice currentBlock.push(chunk.slice(0, diffToSlice)) if (currentBlockSize === BLOCK_SIZE) { - blocks.push(Buffer.concat(currentBlock)) + blocks.push(hashBlock(Buffer.concat(currentBlock))) currentBlock = [] currentBlockSize = 0 } @@ -44,7 +48,7 @@ async function getFileIntegrity (path) { callback() }, flush (callback) { - blocks.push(Buffer.concat(currentBlock)) + blocks.push(hashBlock(Buffer.concat(currentBlock))) currentBlock = [] callback() } @@ -55,7 +59,7 @@ async function getFileIntegrity (path) { algorithm: ALGORITHM, hash: hash.read(), blockSize: BLOCK_SIZE, - blocks: blocks.map(block => crypto.createHash(ALGORITHM).update(block).digest('hex')) + blocks: blocks } } From 927723a6d65b2057b5294a8e789fb9ae2f4e3ab6 Mon Sep 17 00:00:00 2001 From: Samuel Attard Date: Tue, 7 Sep 2021 16:21:22 -0700 Subject: [PATCH 12/14] docs: update README with new options --- README.md | 7 +++++-- 1 file changed, 5 insertions(+), 2 deletions(-) diff --git a/README.md b/README.md index 2b3fef5..06015bb 100644 --- a/README.md +++ b/README.md @@ -199,8 +199,11 @@ precisely represent UINT64 in JavaScript `Number`. `size` is a JavaScript because file size in Node.js is represented as `Number` and it is not safe to convert `Number` to UINT64. -`integrity` is an object consisting of a hashing `algorithm` and a hex encoded -`hash` value. +`integrity` is an object consisting of a few keys: +* A hashing `algorithm`, currently only `SHA256` is supported. +* A hex encoded `hash` value representing the hash of the entire file. +* An array of hex encoded hashes for the `blocks` of the file. i.e. for a blockSize of 4KB this array contains the hash of every block if you split the file into N 4KB blocks. +* A integer value `blockSize` representing the size in bytes of each block in the `blocks` hashes above [pickle]: https://chromium.googlesource.com/chromium/src/+/master/base/pickle.h [node-pickle]: https://www.npmjs.org/package/chromium-pickle From 420a3a44d5ab7b72df1ee62091a3b969ecd2f8fa Mon Sep 17 00:00:00 2001 From: Samuel Attard Date: Tue, 7 Sep 2021 16:34:34 -0700 Subject: [PATCH 13/14] revert --- lib/crawlfs.js | 4 ++-- lib/index.d.ts | 2 ++ 2 files changed, 4 insertions(+), 2 deletions(-) diff --git a/lib/crawlfs.js b/lib/crawlfs.js index 468272c..a26c3eb 100644 --- a/lib/crawlfs.js +++ b/lib/crawlfs.js @@ -9,10 +9,10 @@ async function determineFileType (filename) { const stat = await fs.lstat(filename) if (stat.isFile()) { return { type: 'file', stat } - } else if (stat.isSymbolicLink()) { - return { type: 'link', stat } } else if (stat.isDirectory()) { return { type: 'directory', stat } + } else if (stat.isSymbolicLink()) { + return { type: 'link', stat } } } diff --git a/lib/index.d.ts b/lib/index.d.ts index efa63d6..b3790ec 100644 --- a/lib/index.d.ts +++ b/lib/index.d.ts @@ -55,6 +55,8 @@ export type FileRecord = { integrity: { hash: string; algorithm: 'SHA256'; + blocks: string[]; + blockSize: number; }; } From 83a62654de51e34dbc18ea886cbf7fb5f0931990 Mon Sep 17 00:00:00 2001 From: Samuel Attard Date: Thu, 9 Sep 2021 10:41:01 -0700 Subject: [PATCH 14/14] chore: update per feedback --- README.md | 12 +++++++++--- lib/integrity.js | 12 ++++-------- 2 files changed, 13 insertions(+), 11 deletions(-) diff --git a/README.md b/README.md index 06015bb..e05497f 100644 --- a/README.md +++ b/README.md @@ -156,7 +156,9 @@ Structure of `header` is something like this: "executable": true, "integrity": { "algorithm": "SHA256", - "hash": "..." + "hash": "...", + "blockSize": 1024, + "blocks": ["...", "..."] } }, "cd": { @@ -165,7 +167,9 @@ Structure of `header` is something like this: "executable": true, "integrity": { "algorithm": "SHA256", - "hash": "..." + "hash": "...", + "blockSize": 1024, + "blocks": ["...", "..."] } } } @@ -179,7 +183,9 @@ Structure of `header` is something like this: "size": 32, "integrity": { "algorithm": "SHA256", - "hash": "..." + "hash": "...", + "blockSize": 1024, + "blocks": ["...", "..."] } } } diff --git a/lib/integrity.js b/lib/integrity.js index daf0c0e..6fabee4 100644 --- a/lib/integrity.js +++ b/lib/integrity.js @@ -14,13 +14,7 @@ function hashBlock (block) { } async function getFileIntegrity (path) { - const read = fs.createReadStream(path) - const hash = crypto.createHash(ALGORITHM) - hash.setEncoding('hex') - await pipeline( - read, - hash - ) + const fileHash = crypto.createHash(ALGORITHM) const blocks = [] let currentBlockSize = 0 @@ -31,6 +25,8 @@ async function getFileIntegrity (path) { new stream.PassThrough({ decodeStrings: false, transform (_chunk, encoding, callback) { + fileHash.update(_chunk) + function handleChunk (chunk) { const diffToSlice = Math.min(BLOCK_SIZE - currentBlockSize, chunk.byteLength) currentBlockSize += diffToSlice @@ -57,7 +53,7 @@ async function getFileIntegrity (path) { return { algorithm: ALGORITHM, - hash: hash.read(), + hash: fileHash.digest('hex'), blockSize: BLOCK_SIZE, blocks: blocks }