From a39d43e6b698bdaee33e9f0c4ab7266b53685ef7 Mon Sep 17 00:00:00 2001 From: Brian Becker Date: Tue, 29 Aug 2023 21:08:04 -0400 Subject: [PATCH 1/2] v5.3.4 - OutFile improvements --- source/HttpCommand.dyalog | 156 ++++++++++++++++++++++++-------------- tests/teardown.dyalog | 4 - tests/test_outfile.aplf | 33 ++++++++ 3 files changed, 132 insertions(+), 61 deletions(-) delete mode 100644 tests/teardown.dyalog create mode 100644 tests/test_outfile.aplf diff --git a/source/HttpCommand.dyalog b/source/HttpCommand.dyalog index c2835a2..ad92522 100644 --- a/source/HttpCommand.dyalog +++ b/source/HttpCommand.dyalog @@ -7,7 +7,7 @@ ∇ r←Version ⍝ Return the current version :Access public shared - r←'HttpCommand' '5.3.3' '2023-08-14' + r←'HttpCommand' '5.3.4' '2023-08-28' ∇ ⍝ Request-related fields @@ -399,7 +399,7 @@ ∆FAIL:(rc secureParams)←¯1 msg ⍝ failure ∇ - ∇ {r}←certs HttpCmd args;url;parms;hdrs;urlparms;p;b;secure;port;host;path;auth;req;err;done;data;datalen;rc;donetime;ind;len;obj;evt;dat;z;msg;timedOut;certfile;keyfile;simpleChar;defaultPort;cookies;domain;t;replace;outFile;toFile;startSize;options;congaPath;progress;starttime;outTn;secureParams;ct;forceClose;headers;cmd;file;protocol;conx;proxied;proxy;cert;noCT;simpleParms;noContentLength;connectionClose + ∇ {r}←certs HttpCmd args;url;parms;hdrs;urlparms;p;b;secure;port;host;path;auth;req;err;done;data;datalen;rc;donetime;ind;len;obj;evt;dat;z;msg;timedOut;certfile;keyfile;simpleChar;defaultPort;cookies;domain;t;replace;outFile;toFile;startSize;options;congaPath;progress;starttime;outTn;secureParams;ct;forceClose;headers;cmd;file;protocol;conx;proxied;proxy;cert;noCT;simpleParms;noContentLength;connectionClose;tmpFile;tmpTn;redirected;encoding;compType;isutf8 ⍝ issue an HTTP command ⍝ certs - X509Cert|(PublicCertFile PrivateKeyFile) SSLValidation Priority PublicCertFile PrivateKeyFile ⍝ args - [1] HTTP method @@ -430,7 +430,8 @@ url←,url url←BaseURL makeURL url cmd←uc,cmd - + toFile←redirected←outTn←tmpTn←0 ⍝ initial settings + tmpFile←'' ∆GET: ⍝ do header initialization here because we may return here on a redirect @@ -568,36 +569,40 @@ →∆EXIT :EndIf - outTn←0 (outFile replace)←2↑{⍵,(≢⍵)↓'' 0}eis OutFile - :If toFile←~0∊⍴outFile - :Trap Debug↓0 - outFile←1 ⎕NPARTS outFile - :If ~⎕NEXISTS⊃outFile - →∆END⊣r.msg←'Output file folder "',(⊃outFile),'" does not exist' - :EndIf - :If 0∊⍴∊1↓outFile ⍝ no file name specified, try to use the name from the URL - :If ~0∊⍴file←∊1↓1 ⎕NPARTS path - outFile←(⊃outFile),file - :Else ⍝ no file name specified and none in the URL - →∆END⊣r.msg←'No file name specified in OutFile or URL' + :If 0=outTn ⍝ if we don't already have an output file tied + :If toFile←~0∊⍴outFile ⍝ if we're directing the response payload to file + :Trap Debug↓0 + outFile←1 ⎕NPARTS outFile + :If ~⎕NEXISTS⊃outFile + →∆END⊣r.msg←'Output file folder "',(⊃outFile),'" does not exist' :EndIf - :EndIf - :If ⎕NEXISTS outFile←∊outFile - :If (0=replace)∧0≠2 ⎕NINFO outFile - →∆END⊣r.msg←'Output file "',outFile,'" is not empty' + :If 0∊⍴∊1↓outFile ⍝ no file name specified, try to use the name from the URL + :If ~0∊⍴file←∊1↓1 ⎕NPARTS path + outFile←(⊃outFile),file + :Else ⍝ no file name specified and none in the URL + →∆END⊣r.msg←'No file name specified in OutFile or URL' + :EndIf + :EndIf + :If ⎕NEXISTS outFile←∊outFile + :If (0=replace)∧0≠2 ⎕NINFO outFile + →∆END⊣r.msg←'Output file "',outFile,'" is not empty' + :Else + outTn←outFile ⎕NTIE 0 + {}0 ⎕NRESIZE⍣(1=replace)⊢outTn + :EndIf :Else - outTn←outFile ⎕NTIE 0 - {}0 ⎕NRESIZE⍣(1=replace)⊢outTn + outTn←outFile ⎕NCREATE 0 :EndIf + startSize←⎕NSIZE outTn + r.OutFile←outFile + tmpFile←tempFolder,'/',(∊1↓1 ⎕NPARTS outFile) ⍝ create temporary file to work with + tmpTn←tmpFile(⎕NCREATE⍠'Unique' 1)0 ⍝ create with a unique name + tmpFile←∊1 ⎕NPARTS ⎕NNAMES[⎕NNUMS⍳tmpTn;] ⍝ save the name for ⎕NDELETE later :Else - outTn←outFile ⎕NCREATE 0 - :EndIf - startSize←⎕NSIZE outTn - r.OutFile←outFile - :Else - →∆END⊣r.msg←({⍺,(~0∊⍴⍵)/' (',⍵,')'}/⎕DMX.(EM Message)),' occurred while trying to initialize output file "',(⍕outFile),'"' - :EndTrap + →∆END⊣r.msg←({⍺,(~0∊⍴⍵)/' (',⍵,')'}/⎕DMX.(EM Message)),' occurred while trying to initialize output file "',(⍕outFile),'"' + :EndTrap + :EndIf :EndIf secureParams←'' @@ -704,6 +709,7 @@ :Else r.(HttpVersion HttpStatus HttpMessage Headers)←4↑dat r.HttpStatus←toInt r.HttpStatus + redirected←3=⌊0.01×r.HttpStatus datalen←⊃toInt{0∊⍴⍵:'¯1' ⋄ ⍵}r.GetHeader'Content-Length' ⍝ ¯1 if no content length not specified connectionClose←'close'≡lc r.GetHeader'Connection' noContentLength←datalen=¯1 @@ -712,9 +718,10 @@ :EndIf :Case 'HTTPBody' →∆END⍴⍨forceClose←r CheckPayloadSize(≢data)+≢dat - :If toFile - →∆END⍴⍨forceClose←r CheckPayloadSize(⎕NSIZE outTn)+≢dat - dat ⎕NAPPEND outTn + :If toFile>redirected ⍝ don't write redirect response payload to file + →∆END⍴⍨forceClose←r CheckPayloadSize(⎕NSIZE tmpTn)+≢dat + dat ⎕NAPPEND tmpTn + ⎕NUNTIE ⍬ :Else data,←dat :EndIf @@ -722,9 +729,10 @@ :Case 'HTTPChunk' :If 1=≡dat →∆END⊣r.(Data msg)←dat'Conga failed to parse the response HTTP chunk' ⍝ HTTP chunk parsing failed? - :ElseIf toFile - →∆END⍴⍨forceClose←r CheckPayloadSize(⎕NSIZE outTn)+≢1⊃dat - (1⊃dat)⎕NAPPEND outTn + :ElseIf toFile>redirected ⍝ don't write redirect response payload to file + →∆END⍴⍨forceClose←r CheckPayloadSize(⎕NSIZE tmpTn)+≢1⊃dat + (1⊃dat)⎕NAPPEND tmpTn + ⎕NUNTIE ⍬ :Else →∆END⍴⍨forceClose←r CheckPayloadSize(≢data)+≢1⊃dat data,←1⊃dat @@ -752,9 +760,10 @@ :EndIf :Case 'BlockLast' ⍝ BlockLast included for pre-Conga v3.4 compatibility for RFC7230 (Sec 3.3.3 item 7) →∆END⍴⍨forceClose←r CheckPayloadSize(≢data)+≢dat - :If toFile - →∆END⍴⍨forceClose←r CheckPayloadSize(⎕NSIZE outTn)+≢dat - dat ⎕NAPPEND outTn + :If toFileforceClose:⍵ ⋄ ''⊣LDRC.Close ⍵}Client ∆EXIT: ∇ @@ -898,6 +921,24 @@ ∆EXIT: ∇ + ∇ {r}←type UnzipFile tn;data + :Access public shared + ⍝ Unzip an output file + ⍝ type is compression type: ¯2 for gzip, ¯3 for deflate + ⍝ tn is the tie number of the file to unzip + ⍝ r is 0 for success or ⎕EN + :Trap 0 + data←⎕NREAD tn 83,(⎕NSIZE tn),0 + data←⎕UCS 256|type Zipper data + 0 ⎕NRESIZE tn + data ⎕NAPPEND tn + ⎕NUNTIE ⍬ + r←0 + :Else + r←⎕EN + :EndTrap + ∇ + NL←⎕UCS 13 10 toChar←{(⎕DR'')⎕DR ⍵} fromutf8←{0::(⎕AV,'?')[⎕AVU⍳⍵] ⋄ 'UTF-8'⎕UCS ⍵} ⍝ Turn raw UTF-8 input into text @@ -934,6 +975,7 @@ seconds←{⍵÷86400} ⍝ convert seconds to fractional day (for cookie max-age) atLeast←{a←(≢⍵)↑⍺ ⋄ ⊃((~∧\⍵=a)/a>⍵),1} ⍝ checks if ⍺ is at least version ⍵ Zipper←219⌶ + tempFolder←739⌶0 makeURL←{ ⍝ build URL from BaseURL (⍺) and URL (⍵) ~0∊⍴'^https?\:\/\/'⎕S 3⍠('IC' 1)⊢⍵:⍵ ⍝ URL begins with http:// or https:// diff --git a/tests/teardown.dyalog b/tests/teardown.dyalog deleted file mode 100644 index 6cf370a..0000000 --- a/tests/teardown.dyalog +++ /dev/null @@ -1,4 +0,0 @@ - r←teardown dummy -⍝ teardown DRC and Conga - r←'' - {}#.⎕EX'DRC' 'Conga' 'HttpCommand' 'httpcommand_test' diff --git a/tests/test_outfile.aplf b/tests/test_outfile.aplf new file mode 100644 index 0000000..8315a21 --- /dev/null +++ b/tests/test_outfile.aplf @@ -0,0 +1,33 @@ + {r}←test_outfile dummy;tmp;tn;fname;h;resp;fileSize;if;notEmpty;contains;nnums;data;size;readFile + nnums←⎕NNUMS + if←⍴⍨ + notEmpty←~∘(0∘∊)∘≢ + contains←(∨/)⍷⍨ + fileSize←{⊃2 ⎕NINFO ⍵} + readFile←{tn←⍵ ⎕NTIE 0 ⋄ (⎕NUNTIE tn)⊢'UTF-8'⎕UCS ⎕UCS ⎕NREAD tn,(⎕DR''),2↑⎕NSIZE tn} + r←'' + tmp←739⌶0 ⍝ temporary folder + tn←(tmp,'/outfile-test')(⎕NCREATE ⎕OPT'Unique' 1)0 ⍝ unique file + fname←(⎕NNUMS⍳tn)⊃↓⎕NNAMES ⍝ get its name + ⎕NUNTIE tn + h←#.HttpCommand.New'get' 'dyalog.com' + resp←h.Run + data←resp.Data ⍝ note size of data + h.OutFile←fname + resp←h.Run + →Cleanup if notEmpty r←(data≢readFile fname)/'OutFile data does not match reference data' + →Cleanup if notEmpty r←(resp.BytesWritten≠fileSize fname)/'File size does not match BytesWritten' + resp←h.Run + →Cleanup if notEmpty r←((resp.rc=¯1)⍲resp.msg contains'not empty')/'Error in overwrite protection' + h.OutFile←fname 1 ⍝ overwrite file + resp←h.Run + →Cleanup if notEmpty r←(data≢readFile fname)/'Overwrite OutFile data does not match reference data' + →Cleanup if notEmpty r←(resp.BytesWritten≠fileSize fname)/'Overwrite file size does not match BytesWritten' + h.OutFile←fname 2 ⍝ append to file + size←fileSize fname + resp←h.Run + →Cleanup if notEmpty r←((data⍴⍨2×⍴data)≢readFile fname)/'Append OutFile data does not match reference data' + →Cleanup if notEmpty r←((size+resp.BytesWritten)≠fileSize fname)/'Append file size does not match previous size + BytesWritten' +Cleanup: + ⎕NUNTIE ⎕NNUMS~nnums + ⎕NDELETE fname From 221b73a3e2d5168c47d6b58362967f3c1bf13b06 Mon Sep 17 00:00:00 2001 From: Brian Becker Date: Wed, 30 Aug 2023 09:34:11 -0400 Subject: [PATCH 2/2] test_outfile improvements per Michael --- tests/test_outfile.aplf | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/tests/test_outfile.aplf b/tests/test_outfile.aplf index 8315a21..d3b101d 100644 --- a/tests/test_outfile.aplf +++ b/tests/test_outfile.aplf @@ -12,20 +12,24 @@ ⎕NUNTIE tn h←#.HttpCommand.New'get' 'dyalog.com' resp←h.Run + →Cleanup if notEmpty r←(0 200≢resp.(rc HttpStatus))/⍕resp data←resp.Data ⍝ note size of data h.OutFile←fname resp←h.Run + →Cleanup if notEmpty r←(0 200≢resp.(rc HttpStatus))/⍕resp →Cleanup if notEmpty r←(data≢readFile fname)/'OutFile data does not match reference data' →Cleanup if notEmpty r←(resp.BytesWritten≠fileSize fname)/'File size does not match BytesWritten' resp←h.Run →Cleanup if notEmpty r←((resp.rc=¯1)⍲resp.msg contains'not empty')/'Error in overwrite protection' h.OutFile←fname 1 ⍝ overwrite file resp←h.Run + →Cleanup if notEmpty r←(0 200≢resp.(rc HttpStatus))/⍕resp →Cleanup if notEmpty r←(data≢readFile fname)/'Overwrite OutFile data does not match reference data' →Cleanup if notEmpty r←(resp.BytesWritten≠fileSize fname)/'Overwrite file size does not match BytesWritten' h.OutFile←fname 2 ⍝ append to file size←fileSize fname resp←h.Run + →Cleanup if notEmpty r←(0 200≢resp.(rc HttpStatus))/⍕resp →Cleanup if notEmpty r←((data⍴⍨2×⍴data)≢readFile fname)/'Append OutFile data does not match reference data' →Cleanup if notEmpty r←((size+resp.BytesWritten)≠fileSize fname)/'Append file size does not match previous size + BytesWritten' Cleanup: