From 2dc6dd4ea0b97d3f5e0364a2ca05b31352575125 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Ji=C5=99=C3=AD=20Morav=C4=8D=C3=ADk?= Date: Wed, 21 Jul 2021 10:35:34 +0200 Subject: [PATCH] fix: flushing rows in the end; correctly wait for drain event when writing to transform stream. --- src/XLSXRowTransform.js | 10 ++++++++-- src/XLSXTransformStream.js | 21 ++++++++++----------- test/XLSXRowTransform.spec.js | 7 ++++--- test/XLSXTransformStream.spec.js | 8 ++++++-- test/test.xlsx | Bin 2959 -> 8659 bytes 5 files changed, 28 insertions(+), 18 deletions(-) diff --git a/src/XLSXRowTransform.js b/src/XLSXRowTransform.js index f1c050a..7adcf3d 100644 --- a/src/XLSXRowTransform.js +++ b/src/XLSXRowTransform.js @@ -1,12 +1,13 @@ import { Transform } from 'stream'; -import { Row } from './templates'; +import { Row, SheetHeader, SheetFooter } from './templates'; -/** Class representing a XLSX Row transformation from array to Row. */ +/** Class representing a XLSX Row transformation from array to Row. Also adds the necessary XLSX header and footer. */ export default class XLSXRowTransform extends Transform { constructor(shouldFormat) { super({ objectMode: true }); this.rowCount = 0; this.shouldFormat = shouldFormat; + this.push(SheetHeader); } /** * Transform array to row string @@ -18,4 +19,9 @@ export default class XLSXRowTransform extends Transform { this.rowCount++; callback(null, xlsxRow); } + + _flush(callback) { + this.push(SheetFooter); + callback(); + } } diff --git a/src/XLSXTransformStream.js b/src/XLSXTransformStream.js index 89ec211..01ed4a2 100644 --- a/src/XLSXTransformStream.js +++ b/src/XLSXTransformStream.js @@ -1,5 +1,5 @@ import Archiver from 'archiver'; -import { PassThrough, Transform } from 'stream'; +import { Transform } from 'stream'; import XLSXRowTransform from './XLSXRowTransform'; import * as templates from './templates'; @@ -16,10 +16,7 @@ export default class XLSXTransformStream extends Transform { this.initializeArchiver(); this.rowTransform = new XLSXRowTransform(this.options.shouldFormat); - this.sheetStream = new PassThrough(); - this.sheetStream.write(templates.SheetHeader); - this.rowTransform.pipe(this.sheetStream); - this.zip.append(this.sheetStream, { + this.zip.append(this.rowTransform, { name: 'xl/worksheets/sheet1.xml', }); } @@ -65,13 +62,15 @@ export default class XLSXTransformStream extends Transform { } _transform(row, encoding, callback) { - this.rowTransform.write(row); - callback(); + if (this.rowTransform.write(row)) { + process.nextTick(callback); + } else { + this.rowTransform.once('drain', callback); + } } - async _flush(callback) { - this.sheetStream.end(templates.SheetFooter); - await this.zip.finalize(); - callback(); + _flush(callback) { + this.rowTransform.end(); + this.zip.finalize().then(callback); } } diff --git a/test/XLSXRowTransform.spec.js b/test/XLSXRowTransform.spec.js index 02b4654..1c3a97e 100644 --- a/test/XLSXRowTransform.spec.js +++ b/test/XLSXRowTransform.spec.js @@ -2,14 +2,15 @@ import { expect } from 'chai'; import { Readable, PassThrough } from 'stream'; import XLSXRowTransform from '../src/XLSXRowTransform'; +import { SheetHeader, SheetFooter } from '../src/templates'; describe('The XLSXRowTransform', () => { - it('Correctly transforms an array of data into a XLSX row format', async () => { - const expectedResult = ` + it('Correctly transforms an array of data into a XLSX row format with header and footer', async () => { + const expectedResult = `${SheetHeader} test 123 - `; + ${SheetFooter}`; const inputStream = Readable.from([['test', 123]]); const transform = new XLSXRowTransform(); const outputStream = new PassThrough(); diff --git a/test/XLSXTransformStream.spec.js b/test/XLSXTransformStream.spec.js index bc288c2..09e7233 100644 --- a/test/XLSXTransformStream.spec.js +++ b/test/XLSXTransformStream.spec.js @@ -5,8 +5,10 @@ import path from 'path'; import unzipper from 'unzipper'; import XLSXTransformStream from '../src/XLSXTransformStream'; +const TEST_ROWS_COUNT = 500; + describe('The XLSXTransformStream', () => { - it('The transformed xlsx file corresponds to the snapshot xlsx file', async () => { + it(`The transformed xlsx file corresponds to the snapshot xlsx file (${TEST_ROWS_COUNT} rows)`, async () => { const snapshotFiles = {}; await fs.createReadStream(path.resolve(__dirname, 'test.xlsx')) .pipe(unzipper.Parse()) @@ -18,7 +20,9 @@ describe('The XLSXTransformStream', () => { const inputStream = new Readable({ objectMode: true }); const testFiles = {}; - inputStream.push(['Testing', 1]); + for (let i = 0; i < TEST_ROWS_COUNT; i++) { + inputStream.push(['Testing', i + 1]); + } inputStream.push(null); await inputStream diff --git a/test/test.xlsx b/test/test.xlsx index 7c4e1520976150798f9b0c8f30041d070a177ea2..5c762bbc66e6a72e7cbe54be4153dfbb90332d68 100644 GIT binary patch literal 8659 zcmcgy2{@Gd+m>W0B4x>zE&G;b)WG*IC+CgHzWF)` zJX@)|{N{oB&BNN%*A?Y{Ucv{9$%`-4-+hcWWKl&Ukx9gFhx2b+@Z@tH6mL6LJlBsu?~JIm9p>ff zSRs3OBl6S}6?>b`?67l_`uu}un=67xdJHM2OfWh8f|&alj90)U!Y1GGG5aLV9>XkY z`;WCq;mimA=zfU^zmI!9qfNMr%yWn5$R0jvR-l~WXXsN#R2mWtqJ$ah4}VhokgKAB zHLwd=aB5l{9aBzZWHXd})R3TK=uFgF)km#r;^oE?Ha=$-({ZtVEu#<$ao^SLO2sXF zv+(8!`9{f5)XS18FR|HuL|Uzws?9iC1?12^b`JH*&qQGT%dre4bk^dxm%&U>fNO}t3kMb;Dk z4k4`jbUaK-g>O@kdlo}yUHeu!^s83r0!8(GLbZmi=iGJtl}HSBe!mmiuMfMw5=>X^ zSd>aJkcwG@NK;^kWO!clz+YC~FMjohLf z3qy;tFIO#=4)kWx1X>?pyL;Xfwd{BK(T9=dF7y)bk;KOw1Riy6tK&ISLBsV6^G*>A z=>^7U$ME44atW5;@=F^_<8LpKk+rd5ch`&5`Q7GEyWbEdx3T7Eb0@1mqFQhTZ%tC5vI^ru6H z818MIWQ145P1p5+I8PJr#(epKzWKq;(8wgu!y~Uz4(&iilr(f6?d)af;nH2Qv zF_&{sZ+MK}`cVH~f{Z9^5iZ0~YJWv;XzaAxoO8-RTx?i5Lurps2}l2pcAt39$G&f@ z&R*UPcEU7ED_IZh!ru-a9oz|b4_^$*9ol2_K^L49L%^W5?eZ;F^czIOJ>z)I2OW9| zeW$K?zM#n62*P9s>#Yt7?%oq`_@uv@u>5JMqHOjS_Oi*|{TJ>dbQ@VI4;~>GIhZ8< zEb?7!eY7`SP$4qws^}>@3!~jG%+>{kOxg4bnG3+oC;p*xu;y(auUZ4Y0@F)f(~~G+ zpDa9gmu}(!j|uv;@o>N)j%yWx;S{!v3qo<)D&+iO0Rt=-(vc^{h|;exX2WuZyy2g` zjGc-`WkXO>M;otan5VW;C!QjIBjW`BzW~k7WZpD9$q`{6XKMb;wzKCn7C>0nb1vl z%n4|Umg!vMd`@Ja3h<@t623`~vmUzknU!)ElJ5iZQf5$V&fWd)EgFm`*`t`=VRC!# zpV6ckHXCM%@#|QB!%HKOq_R-`Q7Y5{yK?_h-SHVO?yJvhk6KyLn>{Jz?lKmak8*Eo ziZwfFCzN!L@0ippdLtFOa>Bj6)xG{btewkq_CC2Dr#)}jx~AIN%><(558LR=C>DF) zoc?MQY*zc{ruo_2mwz5b*=O^=(?UeW*!gwRE{dn6s+K$+5%yPnCe7Yrpsf6+)X0r@ zxSWZ-)Ztjzo(vu>Yo4i&-CeoE@d3A)5_@T`^rNW^gV;A08JAd1eN^s8&YskJO~sU2 zXfa*2Dw*Ws_feRA`t8%zVrQix{U+7`cNU(|{((QHZ(nL2IYprL^U_X9UcbInBST^3 z{kqe=Lt|L|`kSb!XIG}8j!li`%cA@bcwHFmjVzKw()uqCa2~80-E~-e`nf`>k(9W2 zk27=4;M3;Z}tH&q3X*V2uZQu886pvU>oFV{!W|N z;EjOQ_*;d7lsWHuE2zv@XM7`2Glf$eYQo9x!q?74u9B%;Ao9kWqm$|aR@Kx3$usBL zhFe~=)Z;^s`uJps#bnfFJvbm_$#pJOwqoHmk%~fCV1J!Ef3zqKSE=5r+Q6ZDBZnIl z$wL|FRGm|?r^9avv*sK(I`_vB1uBz!@0D`wry6zdd3ioRv+Pgh#ZLBn*~R1g*=8@H z8CK-lttovtZeNV?*Ub~}FC=Yi@=&xg-U}WC1ie*3Jr*S9G`sC*; z9ZD<%S3hy_AEQhOC9ATR_RX?PM3$5ZU&o_1SBEZ}DXIv{P;{tlx+r7ta+^aDGdyWGe_0}KJvRmN4tsB#L%*`NkaE<^ShOewdtjm$>!Oy+0jwg zvI29PRwoebMdsbBVyfWO|k{-&c23@7#`2y?rMjK%rtFOR$vV z4*gTY^8CvD+;pGnaC5?JqGV{ZfokAt4Pkkd_|9v^$4{j{AX4!5?TOLxnmd){wK)vlnDIQYcTKPtoZh}M`(_VD85 z+zG`Om8vCZt&QR3HIr4Xitg{WC=F>M9 zSxTsIsvZdls8}CeTeptQmb7-EZ^mDcc>ntAJA#WUSH3}gH?hWTUL}8X%AN*IPM@K`dk^GyvImL zbZUXqNFC?1Ouna>k*j)qPd_zeiOwbv1#ooT!NznE{`Y-bnRcI!&86h+V)i|;&h1$> z=4)@efNVqHR8Ag(qMBZI;kTnpE(1#$fk%&FPX33GlITLlANzA&6Q@Tj;zIWUQWE@X>>sg~N zJ5iUNWQ{0JL6EGm7#S?7l6^f6pf%V-d|@k|R7xj1oKrBIG@%Ybpd`_V#%9rxXngOu z7;wz5?)aNDp$kD^CRyiUv&2c(RzzJZ5{=^G6wn%+BrH}2OR7d+pFW%%1m#OCOY(OweV>xzMe$G1?L1>gDo6D5V%M*o@29= zNi_V4y8a{@3&kl5TQt6Nd|LEj)`>>bpoG*nO~odbw#s{&>K6VuElM!A^JsU*x?Qf zz=T?B$n7QVuMb@cyqzbCuc0-J{HV2$v#DQfaYA>pC_Z|=@vq+Q%Ia@ zxm&kF>94L!-p%h|Xm=&}Ige0KBqQY3$CS7I!POVO4+oR9mBbDRt@PXI8>1h(NO9fJ z#!K*k)?Z#A)34D40gj`z1Q_#lsX-1(vTj>*K-O zK5_K~z@Vy9>Fg~yd^a#4_x-jE`K7LhF~3&3Ix`7P66SQu|R<249o zcPP9|2!`F$#%se&gf!C4_CicTg&D<7U?%!;=vy$8pa|qXCWuL_DZgb7!|~X8MW*mj z5poSm+(8yFfqD_fJTn+(5r-~>VeJvfTo^`whTrlL%%q&R-4AB6n>n0W3}#ZOjlT#p zxvr6ZbRWc|UW_sCEOl!AigM)3VCEp+;8tN;Fz*cJaC31OR;-Q3z_1dHbWa#YCBZ0u z4rbyOhaQEQq(vfiSl|TC^IMj|Og{6rzlWJ*u!NgSQoOQT|D^H$UF3E9ct>O zDC8t7`B-7V5xy@cH19UpxLc}&v!Jb+o&>`}r5VLhFf2I^Ee^w~qL7h?;RGD`En8tG z$NAdbU?#(d!@Fc)CT-gIyD*b5&2%QZpjnH`j5r@{K12Hgg{m}ggmn4ZZ^E!(ws2-S z7}lwce*nV@G}BRZa0lcV^PFKO1#xIwm`PkTQj{H@N-V!+AIt>9*WMQb4m(j>spNo= zbWiF;8#v6RacEB%_AVOP48vkw_$>!v*zbJp;V68L5Xw1B-I9BKHMp3zfnP^_mal&iRe-6%V%;G?T- zVi1ifn*w|5VAQ|}bz(FpZSs_JK?{xpP@>?f_$zeDz6pGW2rns%qWmgp7dMKgy8V-KbOZtJVaxK=bxE&YK(x ztCYifF}gSoa!-3;cm%Yv^x#6l)}WA-H^w<7z_o`hK_PK*v+o;$YoEYJ5P)ls<48d$ zK(!wjX}~vvLISt}SOP*KtKjy9t`4@CXaH8?zw`y^+|TI^DRlvqfSMgTj6u7NOt6#E z?T7;w{YPH^!56e%Kc@#y=STxsWa%EDi_Y5>iUAZ9PW=BC0GeZ%GoU1Jv*M~k1hfX0 zw#xSZ-mI@XJ>~;c`Jb4EZ;Ap)Nn;#1$@T?ljd9+9l)yv8d3L*)plbfS2j7$kd~f|t z)dAVpivbsiH<(T@rhIX08veln7DQCB` z*mj#<=-y&i@fSY;HL7S`_~KT;-i`}-nisOe;NnpQqc%;D10zD9pdhyB{!*l#kg;8&;Hn642tX6V?4a#O@O3DlmA0*D zdjb7}70n2}t!Qe1_Q8sFXj{=T+5|$OHHUf@U)?q$v{SqSNL(OaPD9{o{8w&ClY4Dj zD51PSp#;Yu;+~YOLc~r>Fw@%s{{LeQ|BJC8P5GB?f!WJ|E>mR>(Hi*zO zA+I0;i^O8E8L+$1A)r{^Vjjnl1S`yH)q}y2`y_>-HwWGtqp-pR6#O-~^2kF#7{L%l zh3o4!oi1*sB!!{M1!ny#AY$Ml47)a1=<3 zU{;V8fg!$eE9-@JGu_G!(xrbq1aNNJV$P;muk^v-z;>PfiV=9sDy*c+B}vu8p21)< zl2n;b%~WmWD!Q;}vz08Dtao?KGfkz^4Pal^?JCw7RgwKrR%Zt^BDsSS3w& zRPxRCxew$DomIeu+gE>@Sx+5!bc9^ELnn-%dixLQLa%jjuF@BK(;AKbvX2FM;F|!d z*cfz3`0t9hlP>Tn6<#!rna99uXM1nHn`lMw=wbxc{}ptKfga&g4A_btZ_uajgpBWt zx05b(Xod!4z!7+>Z?E`wo3sEPnIIRp9&bN_pJ@S}+i#Y?NtbBj3{G;nCrxVIqs;LXe;!oUH92Wp>hC&4EnqV1doInfaN)B5&CQAxaoI zH0ygO`{&;_;5mB!>~evM^{LBGZ(`fjo$I(!j*TaJ`&`kP+gd#qDH{IY_cvj8?xxcA zUv6ISzn@@_yeqp`xVq8mgUhrkz!WX8?d+L+4gXIISFTP-HdH(bFOWD#7 zd?Jr-FIbVe=Ah}!xyKI(Yb?5)qq*eVx0jP%oQ<9kcl_p01V2xH_Xr>hjOjj}w+16Zo)G zv+aqw%k789oMWB%KGl0_{15e8_1o}&eqc_?q4Q7bZqM2+lvgBRYF739hqdg!1-aIG z3nLmN$|J8XH&ttH^gglq=cA*kCXw0K#4o#AbwoHZDt)c0;5?f3ubyvC=;OGw3$;29 z1)OFU(jVB}S4{Dp`skIQ@a+p)C#Ozw_MVkhR$98$?dEb%u?0cDR|OO$+Rx>`^Dx-x z(!=!`=Ym(qZT}H^{92oejM?K!-U5YQisz>(%=ne~^(oVlwizew!)0GdJr20rkkRpD z=Y*?1C#Tyl{>2>N&Ca3F!T(`DBLhP$GXq0_HzSiSGdw{|_EE3{Cx~7JGcbKj!3s<> cD_VeQJ4IK1U_uE(GGp>2MQgU7oD4t!02&G%VE_OC