Skip to content
Permalink
Browse files

curl: support parallel transfers

This is done by making sure each individual transfer is first added to a
linked list as then they can be performed serially, or at will, in
parallel.

Closes #3804
  • Loading branch information...
bagder committed Jul 20, 2019
1 parent 14a385b commit b88940850002a3f1c25bc6488b95ad30eb80d696
@@ -860,6 +860,7 @@ check_symbol_exists(strlcat "${CURL_INCLUDES}" HAVE_STRLCAT)
check_symbol_exists(getpwuid "${CURL_INCLUDES}" HAVE_GETPWUID)
check_symbol_exists(getpwuid_r "${CURL_INCLUDES}" HAVE_GETPWUID_R)
check_symbol_exists(geteuid "${CURL_INCLUDES}" HAVE_GETEUID)
check_symbol_exists(usleep "${CURL_INCLUDES}" HAVE_USLEEP)
check_symbol_exists(utime "${CURL_INCLUDES}" HAVE_UTIME)
check_symbol_exists(gmtime_r "${CURL_INCLUDES}" HAVE_GMTIME_R)
check_symbol_exists(localtime_r "${CURL_INCLUDES}" HAVE_LOCALTIME_R)
@@ -3671,6 +3671,7 @@ AC_CHECK_FUNCS([fnmatch \
setlocale \
setmode \
setrlimit \
usleep \
utime \
utimes
],[
@@ -100,7 +100,10 @@ DPAGES = \
noproxy.d \
ntlm.d ntlm-wb.d \
oauth2-bearer.d \
output.d pass.d \
output.d \
pass.d \
parallel.d \
parallel-max.d \
path-as-is.d \
pinnedpubkey.d \
post301.d \
@@ -0,0 +1,9 @@
Long: parallel-max
Help: Maximum concurrency for parallel transfers
Added: 7.66.0
See-also: parallel
---
When asked to do parallel transfers, using --parallel, this option controls
the maximum amount of transfers to do simultaneously.

The default is 50.
@@ -0,0 +1,7 @@
Short: Z
Long: parallel
Help: Perform transfers in parallel
Added: 7.66.0
---
Makes curl perform its transfers in parallel as compared to the regular serial
manner.
@@ -54,6 +54,7 @@ CURL_CFILES = \
tool_panykey.c \
tool_paramhlp.c \
tool_parsecfg.c \
tool_progress.c \
tool_strdup.c \
tool_setopt.c \
tool_sleep.c \
@@ -95,6 +96,7 @@ CURL_HFILES = \
tool_panykey.h \
tool_paramhlp.h \
tool_parsecfg.h \
tool_progress.h \
tool_sdecls.h \
tool_setopt.h \
tool_setup.h \
@@ -5,7 +5,7 @@
* | (__| |_| | _ <| |___
* \___|\___/|_| \_\_____|
*
* Copyright (C) 1998 - 2018, Daniel Stenberg, <daniel@haxx.se>, et al.
* Copyright (C) 1998 - 2019, Daniel Stenberg, <daniel@haxx.se>, et al.
*
* This software is licensed as described in the file COPYING, which
* you should have received as part of this distribution. The terms
@@ -32,6 +32,7 @@
#include "tool_msgs.h"
#include "tool_cb_hdr.h"
#include "tool_cb_wrt.h"
#include "tool_operate.h"

#include "memdebug.h" /* keep this as LAST include */

@@ -54,9 +55,10 @@ static char *parse_filename(const char *ptr, size_t len);

size_t tool_header_cb(char *ptr, size_t size, size_t nmemb, void *userdata)
{
struct HdrCbData *hdrcbdata = userdata;
struct OutStruct *outs = hdrcbdata->outs;
struct OutStruct *heads = hdrcbdata->heads;
struct per_transfer *per = userdata;
struct HdrCbData *hdrcbdata = &per->hdrcbdata;
struct OutStruct *outs = &per->outs;
struct OutStruct *heads = &per->heads;
const char *str = ptr;
const size_t cb = size * nmemb;
const char *end = (char *)ptr + cb;
@@ -100,7 +102,7 @@ size_t tool_header_cb(char *ptr, size_t size, size_t nmemb, void *userdata)
* Content-Disposition header specifying a filename property.
*/

curl_easy_getinfo(outs->config->easy, CURLINFO_PROTOCOL, &protocol);
curl_easy_getinfo(per->curl, CURLINFO_PROTOCOL, &protocol);
if(hdrcbdata->honor_cd_filename &&
(cb > 20) && checkprefix("Content-disposition:", str) &&
(protocol & (CURLPROTO_HTTPS|CURLPROTO_HTTP))) {
@@ -5,7 +5,7 @@
* | (__| |_| | _ <| |___
* \___|\___/|_| \_\_____|
*
* Copyright (C) 1998 - 2018, Daniel Stenberg, <daniel@haxx.se>, et al.
* Copyright (C) 1998 - 2019, Daniel Stenberg, <daniel@haxx.se>, et al.
*
* This software is licensed as described in the file COPYING, which
* you should have received as part of this distribution. The terms
@@ -28,6 +28,7 @@
#include "tool_cfgable.h"
#include "tool_msgs.h"
#include "tool_cb_wrt.h"
#include "tool_operate.h"

#include "memdebug.h" /* keep this as LAST include */

@@ -75,7 +76,8 @@ bool tool_create_output_file(struct OutStruct *outs)
size_t tool_write_cb(char *buffer, size_t sz, size_t nmemb, void *userdata)
{
size_t rc;
struct OutStruct *outs = userdata;
struct per_transfer *per = userdata;
struct OutStruct *outs = &per->outs;
struct OperationConfig *config = outs->config;
size_t bytes = sz * nmemb;
bool is_tty = config->global->isatty;
@@ -202,7 +204,7 @@ size_t tool_write_cb(char *buffer, size_t sz, size_t nmemb, void *userdata)

if(config->readbusy) {
config->readbusy = FALSE;
curl_easy_pause(config->easy, CURLPAUSE_CONT);
curl_easy_pause(per->curl, CURLPAUSE_CONT);
}

if(config->nobuffer) {
@@ -38,7 +38,6 @@ typedef enum {
struct GlobalConfig;

struct OperationConfig {
CURL *easy; /* A copy of the handle from GlobalConfig */
bool remote_time;
char *random_file;
char *egd_file;
@@ -242,9 +241,6 @@ struct OperationConfig {
bool use_metalink; /* process given URLs as metalink XML file */
metalinkfile *metalinkfile_list; /* point to the first node */
metalinkfile *metalinkfile_last; /* point to the last/current node */
#ifdef CURLDEBUG
bool test_event_based;
#endif
char *oauth_bearer; /* OAuth 2.0 bearer token */
bool nonpn; /* enable/disable TLS NPN extension */
bool noalpn; /* enable/disable TLS ALPN extension */
@@ -268,7 +264,6 @@ struct OperationConfig {
};

struct GlobalConfig {
CURL *easy; /* Once we have one, we keep it here */
int showerror; /* -1 == unset, default => show errors
0 => -s is used to NOT show errors
1 => -S has been used to show errors */
@@ -286,6 +281,11 @@ struct GlobalConfig {
char *libcurl; /* Output libcurl code to this file name */
bool fail_early; /* exit on first transfer error */
bool styled_output; /* enable fancy output style detection */
#ifdef CURLDEBUG
bool test_event_based;
#endif
bool parallel;
long parallel_max;
struct OperationConfig *first;
struct OperationConfig *current;
struct OperationConfig *last; /* Always last in the struct */
@@ -40,6 +40,7 @@
#include "tool_msgs.h"
#include "tool_paramhlp.h"
#include "tool_parsecfg.h"
#include "tool_main.h"

#include "memdebug.h" /* keep this as LAST include */

@@ -316,6 +317,8 @@ static const struct LongShort aliases[]= {
{"Y", "speed-limit", ARG_STRING},
{"y", "speed-time", ARG_STRING},
{"z", "time-cond", ARG_STRING},
{"Z", "parallel", ARG_BOOL},
{"Zb", "parallel-max", ARG_STRING},
{"#", "progress-bar", ARG_BOOL},
{":", "next", ARG_NONE},
};
@@ -1104,7 +1107,7 @@ ParameterError getparameter(const char *flag, /* f or -long-flag */
break;
case 'L': /* --test-event */
#ifdef CURLDEBUG
config->test_event_based = toggle;
global->test_event_based = toggle;
#else
warnf(global, "--test-event is ignored unless a debug build!\n");
#endif
@@ -1356,7 +1359,7 @@ ParameterError getparameter(const char *flag, /* f or -long-flag */
size = 0;
}
else {
char *enc = curl_easy_escape(config->easy, postdata, (int)size);
char *enc = curl_easy_escape(NULL, postdata, (int)size);
Curl_safefree(postdata); /* no matter if it worked or not */
if(enc) {
/* now make a string with the name from above and append the
@@ -2127,6 +2130,21 @@ ParameterError getparameter(const char *flag, /* f or -long-flag */
if(!config->low_speed_time)
config->low_speed_time = 30;
break;
case 'Z':
switch(subletter) {
case '\0': /* --parallel */
global->parallel = toggle;
break;
case 'b': /* --parallel-max */
err = str2unum(&global->parallel_max, nextarg);
if(err)
return err;
if((global->parallel_max > MAX_PARALLEL) ||
(global->parallel_max < 1))
global->parallel_max = PARALLEL_DEFAULT;
break;
}
break;
case 'z': /* time condition coming up */
switch(*nextarg) {
case '+':
@@ -2176,14 +2194,14 @@ ParameterError getparameter(const char *flag, /* f or -long-flag */
return PARAM_OK;
}

ParameterError parse_args(struct GlobalConfig *config, int argc,
ParameterError parse_args(struct GlobalConfig *global, int argc,
argv_item_t argv[])
{
int i;
bool stillflags;
char *orig_opt = NULL;
ParameterError result = PARAM_OK;
struct OperationConfig *operation = config->first;
struct OperationConfig *config = global->first;

for(i = 1, stillflags = TRUE; i < argc && !result; i++) {
orig_opt = argv[i];
@@ -2199,31 +2217,28 @@ ParameterError parse_args(struct GlobalConfig *config, int argc,
else {
char *nextarg = (i < (argc - 1)) ? argv[i + 1] : NULL;

result = getparameter(flag, nextarg, &passarg, config, operation);
result = getparameter(flag, nextarg, &passarg, global, config);
if(result == PARAM_NEXT_OPERATION) {
/* Reset result as PARAM_NEXT_OPERATION is only used here and not
returned from this function */
result = PARAM_OK;

if(operation->url_list && operation->url_list->url) {
if(config->url_list && config->url_list->url) {
/* Allocate the next config */
operation->next = malloc(sizeof(struct OperationConfig));
if(operation->next) {
config->next = malloc(sizeof(struct OperationConfig));
if(config->next) {
/* Initialise the newly created config */
config_init(operation->next);

/* Copy the easy handle */
operation->next->easy = config->easy;
config_init(config->next);

/* Set the global config pointer */
operation->next->global = config;
config->next->global = global;

/* Update the last operation pointer */
config->last = operation->next;
/* Update the last config pointer */
global->last = config->next;

/* Move onto the new config */
operation->next->prev = operation;
operation = operation->next;
config->next->prev = config;
config = config->next;
}
else
result = PARAM_NO_MEM;
@@ -2237,8 +2252,8 @@ ParameterError parse_args(struct GlobalConfig *config, int argc,
bool used;

/* Just add the URL please */
result = getparameter((char *)"--url", argv[i], &used, config,
operation);
result = getparameter((char *)"--url", argv[i], &used, global,
config);
}
}

@@ -2249,9 +2264,9 @@ ParameterError parse_args(struct GlobalConfig *config, int argc,
const char *reason = param2text(result);

if(orig_opt && strcmp(":", orig_opt))
helpf(config->errors, "option %s: %s\n", orig_opt, reason);
helpf(global->errors, "option %s: %s\n", orig_opt, reason);
else
helpf(config->errors, "%s\n", reason);
helpf(global->errors, "%s\n", reason);
}

return result;
@@ -273,6 +273,10 @@ static const struct helptxt helptext[] = {
"OAuth 2 Bearer Token"},
{"-o, --output <file>",
"Write to file instead of stdout"},
{"-Z, --parallel",
"Perform transfers in parallel"},
{" --parallel-max",
"Maximum concurrency for parallel transfers"},
{" --pass <phrase>",
"Pass phrase for the private key"},
{" --path-as-is",
@@ -602,8 +606,9 @@ void tool_version_info(void)
}
}

void tool_list_engines(CURL *curl)
void tool_list_engines(void)
{
CURL *curl = curl_easy_init();
struct curl_slist *engines = NULL;

/* Get the list of engines */
@@ -620,4 +625,5 @@ void tool_list_engines(CURL *curl)

/* Cleanup the list of engines */
curl_slist_free_all(engines);
curl_easy_cleanup(curl);
}
@@ -7,7 +7,7 @@
* | (__| |_| | _ <| |___
* \___|\___/|_| \_\_____|
*
* Copyright (C) 1998 - 2014, Daniel Stenberg, <daniel@haxx.se>, et al.
* Copyright (C) 1998 - 2019, Daniel Stenberg, <daniel@haxx.se>, et al.
*
* This software is licensed as described in the file COPYING, which
* you should have received as part of this distribution. The terms
@@ -24,7 +24,7 @@
#include "tool_setup.h"

void tool_help(void);
void tool_list_engines(CURL *curl);
void tool_list_engines(void);
void tool_version_info(void);

#endif /* HEADER_CURL_TOOL_HELP_H */

0 comments on commit b889408

Please sign in to comment.
You can’t perform that action at this time.