Skip to content
New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

cli tool: reimplement stdin buffering in -F. #1985

Closed
wants to merge 2 commits into from
Closed
Show file tree
Hide file tree
Changes from 1 commit
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
9 changes: 6 additions & 3 deletions docs/cmdline-opts/form.d
Original file line number Diff line number Diff line change
Expand Up @@ -25,9 +25,12 @@ the form-field to which portrait.jpg will be the input:
curl -F profile=@portrait.jpg https://example.com/upload.cgi

To read content from stdin instead of a file, use - as the filename. This goes
for both @ and < constructs. For this case, as well as for others in which the
full data size cannot be determined before the transfer starts (as named pipes
or similar), data is transferred as chunks by HTTP and rejected by IMAP.
for both @ and < constructs. If stdin is not attached to a regular file, it is
buffered first to determine its size and allow a possible resend. Defining a
part's data from a named non-regular file (such as a named pipe or similar) is
unfortunately not subject to buffering and will be effectively read at
transmission time; since the full size is unknown before the transfer starts,
data is sent as chunks by HTTP and rejected by IMAP.

You can also tell curl what Content-Type to use by using 'type=', in a manner
similar to:
Expand Down
151 changes: 146 additions & 5 deletions src/tool_formparse.c
Original file line number Diff line number Diff line change
Expand Up @@ -31,10 +31,21 @@
#include "tool_cfgable.h"
#include "tool_convert.h"
#include "tool_msgs.h"
#include "tool_binmode.h"
#include "tool_formparse.h"

#include "memdebug.h" /* keep this as LAST include */

#define STDIN_BUFSIZE 0x4000 /* Always have 16K to read more stdin data. */

/* Stdin parameters. */
typedef struct {
char *data; /* Memory data. */
curl_off_t origin; /* File read origin offset. */
curl_off_t size; /* Data size. */
curl_off_t curpos; /* Current read position. */
} standard_input;


/*
* helper function to get a word from form param
Expand Down Expand Up @@ -372,16 +383,146 @@ static int get_param_part(struct OperationConfig *config, char **str,
return sep & 0xFF;
}

/* Check if file is "-". If so, use a callback to read OUR stdin (to

/* Mime part callbacks for stdin. */
static size_t stdin_read(char *buffer, size_t size, size_t nitems, void *arg)
Copy link
Member

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

There's already a file2memory function you should use instead!

{
standard_input *sip = (standard_input *) arg;
curl_off_t bytesleft;
(void) size; /* Always 1: ignored. */

if(sip->curpos >= sip->size)
return 0; /* At eof. */
bytesleft = sip->size - sip->curpos;
if((curl_off_t) nitems > bytesleft)
nitems = (size_t) bytesleft;
if(sip->data) {
/* Return data from memory. */
memcpy(buffer, sip->data + (size_t) sip->curpos, nitems);
}
else {
/* Read from stdin. */
nitems = fread(buffer, 1, nitems, stdin);
}
sip->curpos += nitems;
return nitems;
}

static int stdin_seek(void *instream, curl_off_t offset, int whence)
{
standard_input *sip = (standard_input *) instream;

switch(whence) {
case SEEK_CUR:
offset += sip->curpos;
break;
case SEEK_END:
offset += sip->size;
break;
}
if(offset < 0)
return CURL_SEEKFUNC_CANTSEEK;
if(!sip->data) {
if(fseek(stdin, offset + sip->origin, SEEK_SET))
return CURL_SEEKFUNC_CANTSEEK;
}
sip->curpos = offset;
return CURL_SEEKFUNC_OK;
}

static void stdin_free(void *ptr)
{
standard_input *sip = (standard_input *) ptr;

Curl_safefree(sip->data);
free(sip);
}

/* Set a part's data from a file, taking care about the pseudo filename "-" as
* a shortcut to read stdin: if so, use a callback to read OUR stdin (to
* workaround Windows DLL file handle caveat).
* Else use curl_mime_filedata(). */
* If stdin is a regular file opened in binary mode, save current offset as
* origin for rewind and do not buffer data. Else read to EOF and keep in
* memory. In all cases, compute the stdin data size.
*/
static CURLcode file_or_stdin(curl_mimepart *part, const char *file)
{
standard_input *sip = NULL;
int fd = -1;
CURLcode result = CURLE_OK;
struct stat sbuf;

if(strcmp(file, "-"))
return curl_mime_filedata(part, file);

return curl_mime_data_cb(part, -1, (curl_read_callback) fread,
(curl_seek_callback) fseek, NULL, stdin);
sip = (standard_input *) malloc(sizeof *sip);
if(!sip)
return CURLE_OUT_OF_MEMORY;

memset((char *) sip, 0, sizeof *sip);
set_binmode(stdin);

/* If stdin is a regular file, do not buffer data but read it when needed. */
fd = fileno(stdin);
sip->origin = ftell(stdin);
if(fd >= 0 && sip->origin >= 0 && !fstat(fd, &sbuf) &&
#ifdef __VMS
sbuf.st_fab_rfm != FAB$C_VAR && sbuf.st_fab_rfm != FAB$C_VFC &&
#endif
S_ISREG(sbuf.st_mode)) {
sip->size = sbuf.st_size - sip->origin;
if(sip->size < 0)
sip->size = 0;
}
else {
/* Not suitable for direct use, buffer stdin data. */
sip->origin = 0;
sip->size = 0;
sip->curpos = STDIN_BUFSIZE;
sip->data = malloc(STDIN_BUFSIZE);
if(!sip->data) {
stdin_free(sip);
return CURLE_OUT_OF_MEMORY;
}
for(;;) {
size_t wantbytes = (size_t) (sip->curpos - sip->size);
size_t havebytes = fread(sip->data + (size_t) sip->size, 1, wantbytes,
stdin);
char *p;

sip->size += havebytes;
if(ferror(stdin)) {
stdin_free(sip);
return CURLE_READ_ERROR;
}
if(!havebytes || feof(stdin))
break;

/* Enlarge data buffer. */
p = realloc(sip->data, (size_t) sip->size + STDIN_BUFSIZE);
if(!p) {
stdin_free(sip);
return CURLE_OUT_OF_MEMORY;
}
sip->data = p;
sip->curpos = sip->size + STDIN_BUFSIZE;
}
/* Shrink buffer to spare memory resources. */
if(sip->size < sip->curpos)
sip->data = realloc(sip->data, (size_t) sip->size);
Copy link
Member

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

Even shrinking can fail so this return code must be checked and bail out if you fail.

}
sip->curpos = 0; /* Rewind. */

/* Set remote file name. */
result = curl_mime_filename(part, file);

/* Set part's data from callback. */
if(!result)
result = curl_mime_data_cb(part, sip->size,
stdin_read, stdin_seek, stdin_free, sip);
if(result)
stdin_free(sip);
return result;
}


Expand Down Expand Up @@ -650,7 +791,7 @@ int formparse(struct OperationConfig *config,
if(*contp == '<' && !literal_value) {
++contp;
sep = get_param_part(config, &contp,
&data, &type, &filename, &encoder, &headers);
&data, &type, NULL, &encoder, &headers);
if(sep < 0) {
Curl_safefree(contents);
return 21;
Expand Down
7 changes: 1 addition & 6 deletions tests/data/test173
Original file line number Diff line number Diff line change
Expand Up @@ -53,11 +53,9 @@ POST /we/want/173 HTTP/1.1
User-Agent: curl/7.12.1-CVS (i686-pc-linux-gnu) libcurl/7.12.1-CVS OpenSSL/0.9.6b ipv6 zlib/1.1.4 GSS libidn/0.4.6
Host: %HOSTIP:%HTTPPORT
Accept: */*
Transfer-Encoding: chunked
Expect: 100-continue
Content-Length: 360
Content-Type: multipart/form-data; boundary=----------------------------5dbea401cd8c

168
------------------------------5dbea401cd8c
Content-Disposition: form-data; name="field1"

Expand All @@ -76,9 +74,6 @@ line7
line8

------------------------------5dbea401cd8c--

0

</protocol>
</verify>
</testcase>