From 50c3f8ccbf4d19d7110e0b3ae75fe9c6da39ff64 Mon Sep 17 00:00:00 2001 From: Emanuele Torre Date: Sat, 21 Oct 2023 19:54:29 +0200 Subject: [PATCH] tool/-T: perform non-chunked transfer for stdin if it is a regular file curl will now also compute the content-length of the transfer if stdin is the file to upload and stdin is a regular file, using its file size. Since, while being a regular file, stdin could not have its offset at the start of the file, curl will now also get the current offset into the upload file's file descriptor and use (filesize - offset) as content-length for transfer instead of just using the full filesize. This also fixes a bug on BSDs where open("/dev/fd/N") behaves like dup(N), so, if N is a file descriptor to a regular file, the file offset of the file descriptor returned by open() may not have been at the start of the file despite curl's previous assumption. Since I don't know anything about VMS systems, I left the behaviour for VMS unchanged; on VMS, curl will still perform a chunked transfer if the upload file is stdin. I updated tests that were testing chunked transfers for stdin to use since now curl only performs a chunked transfer by default if the upload file (stdin or not) is not a regular file. And I added a new test (258) that verifies that stdin is uploaded with content-length (not transfer-encoding:chunked) if it is a regular file. Fixes #12171 Fixes #12177 --- src/tool_operate.c | 87 ++++++++++++++++++++++++++--------------- tests/data/Makefile.inc | 2 +- tests/data/test1068 | 4 +- tests/data/test1069 | 4 +- tests/data/test1072 | 12 +++--- tests/data/test1073 | 12 +++--- tests/data/test268 | 49 +++++++++++++++++++++++ tests/data/test60 | 4 +- tests/data/test98 | 4 +- 9 files changed, 125 insertions(+), 53 deletions(-) create mode 100644 tests/data/test268 diff --git a/src/tool_operate.c b/src/tool_operate.c index 697b64e38a220d..5728736c8a3536 100644 --- a/src/tool_operate.c +++ b/src/tool_operate.c @@ -267,55 +267,78 @@ static CURLcode pre_transfer(struct GlobalConfig *global, struct_stat fileinfo; CURLcode result = CURLE_OK; - if(per->uploadfile && !stdin_upload(per->uploadfile)) { - /* VMS Note: - * - * Reading binary from files can be a problem... Only FIXED, VAR - * etc WITHOUT implied CC will work. Others need a \n appended to - * a line - * - * - Stat gives a size but this is UNRELIABLE in VMS. E.g. - * a fixed file with implied CC needs to have a byte added for every - * record processed, this can be derived from Filesize & recordsize - * for VARiable record files the records need to be counted! for - * every record add 1 for linefeed and subtract 2 for the record - * header for VARIABLE header files only the bare record data needs - * to be considered with one appended if implied CC - */ + if(per->uploadfile) { + bool isstdin = stdin_upload(per->uploadfile); + if(!isstdin) { #ifdef __VMS - /* Calculate the real upload size for VMS */ - per->infd = -1; - if(stat(per->uploadfile, &fileinfo) == 0) { - fileinfo.st_size = VmsSpecialSize(uploadfile, &fileinfo); - switch(fileinfo.st_fab_rfm) { - case FAB$C_VAR: - case FAB$C_VFC: - case FAB$C_STMCR: - per->infd = open(per->uploadfile, O_RDONLY | O_BINARY); - break; - default: - per->infd = open(per->uploadfile, O_RDONLY | O_BINARY, - "rfm=stmlf", "ctx=stm"); + /* VMS Note: + * + * Reading binary from files can be a problem... Only FIXED, VAR + * etc WITHOUT implied CC will work. Others need a \n appended to + * a line + * + * - Stat gives a size but this is UNRELIABLE in VMS. E.g. a + * fixed file with implied CC needs to have a byte added for every + * record processed, this can be derived from Filesize & + * recordsize for VARiable record files the records need to be + * counted! for every record add 1 for linefeed and subtract 2 + * for the record header for VARIABLE header files only the bare + * record data needs to be considered with one appended if implied + * CC + */ + /* Calculate the real upload size for VMS */ + per->infd = -1; + if(stat(per->uploadfile, &fileinfo) == 0) { + fileinfo.st_size = VmsSpecialSize(uploadfile, &fileinfo); + switch(fileinfo.st_fab_rfm) { + case FAB$C_VAR: + case FAB$C_VFC: + case FAB$C_STMCR: + per->infd = open(per->uploadfile, O_RDONLY | O_BINARY); + break; + default: + per->infd = open(per->uploadfile, O_RDONLY | O_BINARY, + "rfm=stmlf", "ctx=stm"); + } } +#else + per->infd = open(per->uploadfile, O_RDONLY | O_BINARY); +#endif } + +#ifdef __VMS if(per->infd == -1) #else - per->infd = open(per->uploadfile, O_RDONLY | O_BINARY); - if((per->infd == -1) || fstat(per->infd, &fileinfo)) + if(per->infd == -1 || fstat(per->infd, &fileinfo)) #endif { helpf(tool_stderr, "Can't open '%s'", per->uploadfile); - if(per->infd != -1) { + if(!isstdin && per->infd != -1) { close(per->infd); per->infd = STDIN_FILENO; } return CURLE_READ_ERROR; } - per->infdopen = TRUE; + per->infdopen = !isstdin; /* we ignore file size for char/block devices, sockets, etc. */ if(S_ISREG(fileinfo.st_mode)) +#ifdef __VMS uploadfilesize = fileinfo.st_size; +#else + { + /* When the upload file is stdin, or when the upload file is + /dev/std{in,out,err} or /dev/fd/N on BSDs, the offset may not + be 0 */ + off_t offset = lseek(per->infd, 0, SEEK_CUR); + if(offset >= 0) + uploadfilesize = fileinfo.st_size - offset; + else { + warnf(global, "Can't get file position of file descriptor %d ('%s')", + per->infd, per->uploadfile); + } + } +#endif #ifdef DEBUGBUILD /* allow dedicated test cases to override */ diff --git a/tests/data/Makefile.inc b/tests/data/Makefile.inc index a8be7f5ec7c0fc..c49028ba8538d2 100644 --- a/tests/data/Makefile.inc +++ b/tests/data/Makefile.inc @@ -51,7 +51,7 @@ test226 test227 test228 test229 test230 test231 test232 test233 test234 \ test235 test236 test237 test238 test239 test240 test241 test242 test243 \ test244 test245 test246 test247 test248 test249 test250 test251 test252 \ test253 test254 test255 test256 test257 test258 test259 test260 test261 \ -test262 test263 test264 test265 test266 test267 test269 test270 \ +test262 test263 test264 test265 test266 test267 test269 test268 test270 \ test271 test272 test273 test274 test275 test276 test277 test278 test279 \ test280 test281 test282 test283 test284 test285 test286 test287 test288 \ test289 test290 test291 test292 test293 test294 test295 test296 test297 \ diff --git a/tests/data/test1068 b/tests/data/test1068 index 982385400ed624..2e7416b9954d88 100644 --- a/tests/data/test1068 +++ b/tests/data/test1068 @@ -26,12 +26,12 @@ blablabla http -HTTP PUT from stdin +HTTP PUT from stdin pipe http://%HOSTIP:%HTTPPORT/bzz/%TESTNUMBER -T - - + more than one byte diff --git a/tests/data/test1069 b/tests/data/test1069 index 51569418336afa..53c966a11ae5af 100644 --- a/tests/data/test1069 +++ b/tests/data/test1069 @@ -17,12 +17,12 @@ HTTP/1.0 http -HTTP 1.0 PUT from stdin with no content length +HTTP 1.0 PUT from stdin pipe with no content length http://%HOSTIP:%HTTPPORT/bzz/%TESTNUMBER -T - -0 - + this data can't be sent diff --git a/tests/data/test1072 b/tests/data/test1072 index f0390fb7f3cb5b..e41e65cc2aba2c 100644 --- a/tests/data/test1072 +++ b/tests/data/test1072 @@ -42,9 +42,9 @@ HTTP chunked PUT to HTTP 1.0 server with authorization http://%HOSTIP:%HTTPPORT/%TESTNUMBER -T - -u testuser:testpass --anyauth - + This is data we upload with PUT -it comes from stdin so MUST be sent +it comes from a pipe so MUST be sent with chunked encoding which is impossible in HTTP/1.0 @@ -53,7 +53,7 @@ which is impossible in HTTP/1.0 # Verify data after the test has been "shot" -25 +65 PUT /%TESTNUMBER HTTP/1.1 @@ -64,12 +64,12 @@ Transfer-Encoding: chunked Expect: 100-continue %if hyper -7A +7B %else -7a +7b %endif This is data we upload with PUT -it comes from stdin so MUST be sent +it comes from a pipe so MUST be sent with chunked encoding which is impossible in HTTP/1.0 diff --git a/tests/data/test1073 b/tests/data/test1073 index 2bc40f057778be..f8ba0db3667409 100644 --- a/tests/data/test1073 +++ b/tests/data/test1073 @@ -36,9 +36,9 @@ HTTP chunked PUT to HTTP 1.0 server with redirect http://%HOSTIP:%HTTPPORT/%TESTNUMBER -T - -L - + This is data we upload with PUT -it comes from stdin so MUST be sent +it comes from a pipe so MUST be sent with chunked encoding which is impossible in HTTP/1.0 @@ -47,7 +47,7 @@ which is impossible in HTTP/1.0 # Verify data after the test has been "shot" -25 +65 PUT /%TESTNUMBER HTTP/1.1 @@ -58,12 +58,12 @@ Transfer-Encoding: chunked Expect: 100-continue %if hyper -7A +7B %else -7a +7b %endif This is data we upload with PUT -it comes from stdin so MUST be sent +it comes from a pipe so MUST be sent with chunked encoding which is impossible in HTTP/1.0 diff --git a/tests/data/test268 b/tests/data/test268 new file mode 100644 index 00000000000000..303b66b5be5508 --- /dev/null +++ b/tests/data/test268 @@ -0,0 +1,49 @@ + + + +HTTP +HTTP PUT + + + +# Server-side + + +HTTP/1.0 200 OK swsclose +Date: Tue, 09 Nov 2010 14:49:00 GMT +Server: test-server/fake + +blablabla + + + + +# Client-side + + +http + + +HTTP PUT of regular file stdin + + +http://%HOSTIP:%HTTPPORT/bzz/%TESTNUMBER -T - + + +more than one byte + + + +# Verify data after the test has been "shot" + + +PUT /bzz/%TESTNUMBER HTTP/1.1 +Host: %HOSTIP:%HTTPPORT +User-Agent: curl/%VERSION +Accept: */* +Content-Length: 19 + +more than one byte + + + diff --git a/tests/data/test60 b/tests/data/test60 index 14338b44010ddd..78c9da8bbbda3d 100644 --- a/tests/data/test60 +++ b/tests/data/test60 @@ -25,12 +25,12 @@ blablabla http -HTTP PUT from stdin with wrong content-length +HTTP PUT from stdin pipe with wrong content-length http://%HOSTIP:%HTTPPORT/bzz/%TESTNUMBER -T - -H "Content-Length: 1" - + more than one byte diff --git a/tests/data/test98 b/tests/data/test98 index ced184b3906008..5f60c1a23a2bec 100644 --- a/tests/data/test98 +++ b/tests/data/test98 @@ -26,12 +26,12 @@ blaha http -HTTP PUT from stdin with set size, disabling chunked transfer-encoding +HTTP PUT from a pipe with set size, disabling chunked transfer-encoding -T - -H "Transfer-Encoding:" -H "Content-Length: 14" http://%HOSTIP:%HTTPPORT/%TESTNUMBER - + data on stdin