tool/-T: perform non-chunked transfer for stdin if it is a regular file

curl will now also compute the content-length of the transfer if stdin
is the file to upload and stdin is a regular file, using its file size.

Since, while being a regular file, stdin could not have its offset at
the start of the file, curl will now also get the current offset into
the upload file's file descriptor and use (filesize - offset) as
content-length for transfer instead of just using the full filesize.
This also fixes a bug on BSDs where open("/dev/fd/N") behaves like
dup(N), so, if N is a file descriptor to a regular file, the file offset
of the file descriptor returned by open() may not have been at the start
of the file despite curl's previous assumption.

Since I don't know anything about VMS systems, I left the behaviour for
VMS unchanged; on VMS, curl will still perform a chunked transfer if the
upload file is stdin.

I updated tests that were testing chunked transfers for stdin to use
<stdin pipe="yes"> since now curl only performs a chunked transfer by
default if the upload file (stdin or not) is not a regular file.
And I added a new test (268) that verifies that stdin is uploaded with
content-length (not transfer-encoding:chunked) if it is a regular file.

Fixes #12171
Fixes #12177

src/tool_operate.c

@@ -267,55 +267,78 @@ static CURLcode pre_transfer(struct GlobalConfig *global,
   struct_stat fileinfo;
   CURLcode result = CURLE_OK;
 
-  if(per->uploadfile && !stdin_upload(per->uploadfile)) {
-    /* VMS Note:
-     *
-     * Reading binary from files can be a problem...  Only FIXED, VAR
-     * etc WITHOUT implied CC will work. Others need a \n appended to
-     * a line
-     *
-     * - Stat gives a size but this is UNRELIABLE in VMS. E.g.
-     * a fixed file with implied CC needs to have a byte added for every
-     * record processed, this can be derived from Filesize & recordsize
-     * for VARiable record files the records need to be counted!  for
-     * every record add 1 for linefeed and subtract 2 for the record
-     * header for VARIABLE header files only the bare record data needs
-     * to be considered with one appended if implied CC
-     */
+  if(per->uploadfile) {
+    bool isstdin = stdin_upload(per->uploadfile);
+    if(!isstdin) {
 #ifdef __VMS
-    /* Calculate the real upload size for VMS */
-    per->infd = -1;
-    if(stat(per->uploadfile, &fileinfo) == 0) {
-      fileinfo.st_size = VmsSpecialSize(uploadfile, &fileinfo);
-      switch(fileinfo.st_fab_rfm) {
-      case FAB$C_VAR:
-      case FAB$C_VFC:
-      case FAB$C_STMCR:
-        per->infd = open(per->uploadfile, O_RDONLY | O_BINARY);
-        break;
-      default:
-        per->infd = open(per->uploadfile, O_RDONLY | O_BINARY,
-                        "rfm=stmlf", "ctx=stm");
+      /* VMS Note:
+       *
+       * Reading binary from files can be a problem...  Only FIXED, VAR
+       * etc WITHOUT implied CC will work. Others need a \n appended to
+       * a line
+       *
+       * - Stat gives a size but this is UNRELIABLE in VMS. E.g.  a
+       * fixed file with implied CC needs to have a byte added for every
+       * record processed, this can be derived from Filesize &
+       * recordsize for VARiable record files the records need to be
+       * counted!  for every record add 1 for linefeed and subtract 2
+       * for the record header for VARIABLE header files only the bare
+       * record data needs to be considered with one appended if implied
+       * CC
+       */
+      /* Calculate the real upload size for VMS */
+      per->infd = -1;
+      if(stat(per->uploadfile, &fileinfo) == 0) {
+        fileinfo.st_size = VmsSpecialSize(uploadfile, &fileinfo);
+        switch(fileinfo.st_fab_rfm) {
+        case FAB$C_VAR:
+        case FAB$C_VFC:
+        case FAB$C_STMCR:
+          per->infd = open(per->uploadfile, O_RDONLY | O_BINARY);
+          break;
+        default:
+          per->infd = open(per->uploadfile, O_RDONLY | O_BINARY,
+                           "rfm=stmlf", "ctx=stm");
+        }
       }
+#else
+      per->infd = open(per->uploadfile, O_RDONLY | O_BINARY);
+#endif
     }
+
+#ifdef __VMS
     if(per->infd == -1)
 #else
-      per->infd = open(per->uploadfile, O_RDONLY | O_BINARY);
-    if((per->infd == -1) || fstat(per->infd, &fileinfo))
+    if(per->infd == -1 || fstat(per->infd, &fileinfo))
 #endif
     {
       helpf(tool_stderr, "Can't open '%s'", per->uploadfile);
-      if(per->infd != -1) {
+      if(!isstdin && per->infd != -1) {
         close(per->infd);
         per->infd = STDIN_FILENO;
       }
       return CURLE_READ_ERROR;
     }
-    per->infdopen = TRUE;
+    per->infdopen = !isstdin;
 
     /* we ignore file size for char/block devices, sockets, etc. */
     if(S_ISREG(fileinfo.st_mode))
+#ifdef __VMS
       uploadfilesize = fileinfo.st_size;
+#else
+    {
+      /* When the upload file is stdin, or when the upload file is
+         /dev/std{in,out,err} or /dev/fd/N on BSDs, the offset may not
+         be 0 */
+      off_t offset = lseek(per->infd, 0, SEEK_CUR);
+      if(offset >= 0)
+        uploadfilesize = fileinfo.st_size - offset;
+      else {
+        warnf(global, "Can't get file position of file descriptor %d ('%s')",
+              per->infd, per->uploadfile);
+      }
+    }
+#endif
 
 #ifdef DEBUGBUILD
     /* allow dedicated test cases to override */

tests/data/Makefile.inc

@@ -51,7 +51,7 @@ test226 test227 test228 test229 test230 test231 test232 test233 test234 \
 test235 test236 test237 test238 test239 test240 test241 test242 test243 \
 test244 test245 test246 test247 test248 test249 test250 test251 test252 \
 test253 test254 test255 test256 test257 test258 test259 test260 test261 \
-test262 test263 test264 test265 test266 test267 test269 test270 \
+test262 test263 test264 test265 test266 test267 test268 test269 test270 \
 test271 test272 test273 test274 test275 test276 test277 test278 test279 \
 test280 test281 test282 test283 test284 test285 test286 test287 test288 \
 test289 test290 test291 test292 test293 test294 test295 test296 test297 \

tests/data/test1068

@@ -26,12 +26,12 @@ blablabla
 http
 </server>
 <name>
-HTTP PUT from stdin
+HTTP PUT from stdin pipe
 </name>
 <command>
 http://%HOSTIP:%HTTPPORT/bzz/%TESTNUMBER -T -
 </command>
-<stdin>
+<stdin pipe="yes">
 more than one byte
 </stdin>
 </client>

tests/data/test1069

@@ -17,12 +17,12 @@ HTTP/1.0
 http
 </server>
 <name>
-HTTP 1.0 PUT from stdin with no content length
+HTTP 1.0 PUT from stdin pipe with no content length
 </name>
 <command>
 http://%HOSTIP:%HTTPPORT/bzz/%TESTNUMBER -T - -0
 </command>
-<stdin>
+<stdin pipe="yes">
 this data can't be sent
 </stdin>
 </client>

tests/data/test1072

@@ -42,9 +42,9 @@ HTTP chunked PUT to HTTP 1.0 server with authorization
 <command>
 http://%HOSTIP:%HTTPPORT/%TESTNUMBER -T - -u testuser:testpass --anyauth
 </command>
-<stdin>
+<stdin pipe="yes">
 This is data we upload with PUT
-it comes from stdin so MUST be sent
+it comes from a pipe so MUST be sent
 with chunked encoding
 which is impossible in HTTP/1.0
 </stdin>
@@ -53,7 +53,7 @@ which is impossible in HTTP/1.0
 # Verify data after the test has been "shot"
 <verify>
 <errorcode>
-25
+65
 </errorcode>
 <protocol>
 PUT /%TESTNUMBER HTTP/1.1
@@ -64,12 +64,12 @@ Transfer-Encoding: chunked
 Expect: 100-continue
 
 %if hyper
-7A
+7B
 %else
-7a
+7b
 %endif
 This is data we upload with PUT
-it comes from stdin so MUST be sent
+it comes from a pipe so MUST be sent
 with chunked encoding
 which is impossible in HTTP/1.0
 

tests/data/test1073

@@ -36,9 +36,9 @@ HTTP chunked PUT to HTTP 1.0 server with redirect
 <command>
 http://%HOSTIP:%HTTPPORT/%TESTNUMBER -T - -L
 </command>
-<stdin>
+<stdin pipe="yes">
 This is data we upload with PUT
-it comes from stdin so MUST be sent
+it comes from a pipe so MUST be sent
 with chunked encoding
 which is impossible in HTTP/1.0
 </stdin>
@@ -47,7 +47,7 @@ which is impossible in HTTP/1.0
 # Verify data after the test has been "shot"
 <verify>
 <errorcode>
-25
+65
 </errorcode>
 <protocol>
 PUT /%TESTNUMBER HTTP/1.1
@@ -58,12 +58,12 @@ Transfer-Encoding: chunked
 Expect: 100-continue
 
 %if hyper
-7A
+7B
 %else
-7a
+7b
 %endif
 This is data we upload with PUT
-it comes from stdin so MUST be sent
+it comes from a pipe so MUST be sent
 with chunked encoding
 which is impossible in HTTP/1.0
 

tests/data/test268

@@ -0,0 +1,49 @@
+<testcase>
+<info>
+<keywords>
+HTTP
+HTTP PUT
+</keywords>
+</info>
+
+# Server-side
+<reply>
+<data>
+HTTP/1.0 200 OK swsclose
+Date: Tue, 09 Nov 2010 14:49:00 GMT
+Server: test-server/fake
+
+blablabla
+
+</data>
+</reply>
+
+# Client-side
+<client>
+<server>
+http
+</server>
+<name>
+HTTP PUT of regular file stdin
+</name>
+<command>
+http://%HOSTIP:%HTTPPORT/bzz/%TESTNUMBER -T -
+</command>
+<stdin>
+more than one byte
+</stdin>
+</client>
+
+# Verify data after the test has been "shot"
+<verify>
+<protocol>
+PUT /bzz/%TESTNUMBER HTTP/1.1
+Host: %HOSTIP:%HTTPPORT
+User-Agent: curl/%VERSION
+Accept: */*
+Content-Length: 19
+
+more than one byte
+</protocol>
+</verify>
+</testcase>

tests/data/test60

@@ -25,12 +25,12 @@ blablabla
 http
 </server>
 <name>
-HTTP PUT from stdin with wrong content-length
+HTTP PUT from stdin pipe with wrong content-length
 </name>
 <command>
 http://%HOSTIP:%HTTPPORT/bzz/%TESTNUMBER -T - -H "Content-Length: 1"
 </command>
-<stdin>
+<stdin pipe="yes">
 more than one byte
 </stdin>
 </client>

tests/data/test98

@@ -26,12 +26,12 @@ blaha
 http
 </server>
 <name>
-HTTP PUT from stdin with set size, disabling chunked transfer-encoding
+HTTP PUT from a pipe with set size, disabling chunked transfer-encoding
 </name>
 <command>
  -T - -H "Transfer-Encoding:" -H "Content-Length: 14" http://%HOSTIP:%HTTPPORT/%TESTNUMBER
 </command>
-<stdin>
+<stdin pipe="yes">
 data on stdin
 </stdin>
 </client>