curl: support parallel transfers

This is done by making sure each individual transfer is first added to a linked list as then they can be performed serially, or at will, in parallel.
curl · Apr 28, 2019 · 79de0ba · 79de0ba
1 parent 2fe2da9
commit 79de0ba
Show file tree

Hide file tree

Showing 35 changed files with 1,465 additions and 771 deletions.
diff --git a/CMakeLists.txt b/CMakeLists.txt
@@ -860,6 +860,7 @@ check_symbol_exists(strlcat       "${CURL_INCLUDES}" HAVE_STRLCAT)
 check_symbol_exists(getpwuid      "${CURL_INCLUDES}" HAVE_GETPWUID)
 check_symbol_exists(getpwuid_r    "${CURL_INCLUDES}" HAVE_GETPWUID_R)
 check_symbol_exists(geteuid       "${CURL_INCLUDES}" HAVE_GETEUID)
+check_symbol_exists(usleep        "${CURL_INCLUDES}" HAVE_USLEEP)
 check_symbol_exists(utime         "${CURL_INCLUDES}" HAVE_UTIME)
 check_symbol_exists(gmtime_r      "${CURL_INCLUDES}" HAVE_GMTIME_R)
 check_symbol_exists(localtime_r   "${CURL_INCLUDES}" HAVE_LOCALTIME_R)

diff --git a/configure.ac b/configure.ac
@@ -3770,6 +3770,7 @@ AC_CHECK_FUNCS([fnmatch \
   setlocale \
   setmode \
   setrlimit \
+  usleep \
   utime \
   utimes
 ],[

diff --git a/docs/cmdline-opts/Makefile.inc b/docs/cmdline-opts/Makefile.inc
@@ -100,7 +100,10 @@ DPAGES =					\
   noproxy.d					\
   ntlm.d ntlm-wb.d				\
   oauth2-bearer.d				\
-  output.d pass.d				\
+  output.d                                      \
+  pass.d                			\
+  parallel.d                                    \
+  parallel-max.d                                \
   path-as-is.d					\
   pinnedpubkey.d				\
   post301.d					\

diff --git a/docs/cmdline-opts/parallel-max.d b/docs/cmdline-opts/parallel-max.d
@@ -0,0 +1,9 @@
+Long: parallel-max
+Help: Maximum concurrency for parallel transfers
+Added: 7.66.0
+See-also: parallel
+---
+When asked to do parallel transfers, using --parallel, this option controls
+the maximum amount of transfers to do simultaneously.
+
+The default is 50.
diff --git a/docs/cmdline-opts/parallel.d b/docs/cmdline-opts/parallel.d
@@ -0,0 +1,7 @@
+Short: Z
+Long: parallel
+Help: Perform transfers in parallel
+Added: 7.66.0
+---
+Makes curl perform its transfers in parallel as compared to the regular serial
+manner.
diff --git a/src/Makefile.inc b/src/Makefile.inc
@@ -54,6 +54,7 @@ CURL_CFILES = \
   tool_panykey.c \
   tool_paramhlp.c \
   tool_parsecfg.c \
+  tool_progress.c \
   tool_strdup.c \
   tool_setopt.c \
   tool_sleep.c \
@@ -95,6 +96,7 @@ CURL_HFILES = \
   tool_panykey.h \
   tool_paramhlp.h \
   tool_parsecfg.h \
+  tool_progress.h \
   tool_sdecls.h \
   tool_setopt.h \
   tool_setup.h \

diff --git a/src/tool_cb_hdr.c b/src/tool_cb_hdr.c
@@ -5,7 +5,7 @@
  *                            | (__| |_| |  _ <| |___
  *                             \___|\___/|_| \_\_____|
  *
- * Copyright (C) 1998 - 2018, Daniel Stenberg, <daniel@haxx.se>, et al.
+ * Copyright (C) 1998 - 2019, Daniel Stenberg, <daniel@haxx.se>, et al.
  *
  * This software is licensed as described in the file COPYING, which
  * you should have received as part of this distribution. The terms
@@ -32,6 +32,7 @@
 #include "tool_msgs.h"
 #include "tool_cb_hdr.h"
 #include "tool_cb_wrt.h"
+#include "tool_operate.h"
 
 #include "memdebug.h" /* keep this as LAST include */
 
@@ -54,9 +55,10 @@ static char *parse_filename(const char *ptr, size_t len);
 
 size_t tool_header_cb(char *ptr, size_t size, size_t nmemb, void *userdata)
 {
-  struct HdrCbData *hdrcbdata = userdata;
-  struct OutStruct *outs = hdrcbdata->outs;
-  struct OutStruct *heads = hdrcbdata->heads;
+  struct per_transfer *per = userdata;
+  struct HdrCbData *hdrcbdata = &per->hdrcbdata;
+  struct OutStruct *outs = &per->outs;
+  struct OutStruct *heads = &per->heads;
   const char *str = ptr;
   const size_t cb = size * nmemb;
   const char *end = (char *)ptr + cb;
@@ -100,7 +102,7 @@ size_t tool_header_cb(char *ptr, size_t size, size_t nmemb, void *userdata)
    * Content-Disposition header specifying a filename property.
    */
 
-  curl_easy_getinfo(outs->config->easy, CURLINFO_PROTOCOL, &protocol);
+  curl_easy_getinfo(per->curl, CURLINFO_PROTOCOL, &protocol);
   if(hdrcbdata->honor_cd_filename &&
      (cb > 20) && checkprefix("Content-disposition:", str) &&
      (protocol & (CURLPROTO_HTTPS|CURLPROTO_HTTP))) {

diff --git a/src/tool_cb_wrt.c b/src/tool_cb_wrt.c
@@ -5,7 +5,7 @@
  *                            | (__| |_| |  _ <| |___
  *                             \___|\___/|_| \_\_____|
  *
- * Copyright (C) 1998 - 2018, Daniel Stenberg, <daniel@haxx.se>, et al.
+ * Copyright (C) 1998 - 2019, Daniel Stenberg, <daniel@haxx.se>, et al.
  *
  * This software is licensed as described in the file COPYING, which
  * you should have received as part of this distribution. The terms
@@ -28,6 +28,7 @@
 #include "tool_cfgable.h"
 #include "tool_msgs.h"
 #include "tool_cb_wrt.h"
+#include "tool_operate.h"
 
 #include "memdebug.h" /* keep this as LAST include */
 
@@ -75,7 +76,8 @@ bool tool_create_output_file(struct OutStruct *outs)
 size_t tool_write_cb(char *buffer, size_t sz, size_t nmemb, void *userdata)
 {
   size_t rc;
-  struct OutStruct *outs = userdata;
+  struct per_transfer *per = userdata;
+  struct OutStruct *outs = &per->outs;
   struct OperationConfig *config = outs->config;
   size_t bytes = sz * nmemb;
   bool is_tty = config->global->isatty;
@@ -202,7 +204,7 @@ size_t tool_write_cb(char *buffer, size_t sz, size_t nmemb, void *userdata)
 
   if(config->readbusy) {
     config->readbusy = FALSE;
-    curl_easy_pause(config->easy, CURLPAUSE_CONT);
+    curl_easy_pause(per->curl, CURLPAUSE_CONT);
   }
 
   if(config->nobuffer) {

diff --git a/src/tool_cfgable.c b/src/tool_cfgable.c
@@ -44,6 +44,7 @@ void config_init(struct OperationConfig* config)
   config->tcp_nodelay = TRUE; /* enabled by default */
   config->happy_eyeballs_timeout_ms = CURL_HET_DEFAULT;
   config->http09_allowed = TRUE;
+  config->parallel_max = PARALLEL_DEFAULT;
 }
 
 static void free_config_fields(struct OperationConfig *config)

diff --git a/src/tool_cfgable.h b/src/tool_cfgable.h
@@ -38,7 +38,6 @@ typedef enum {
 struct GlobalConfig;
 
 struct OperationConfig {
-  CURL *easy;               /* A copy of the handle from GlobalConfig */
   bool remote_time;
   char *random_file;
   char *egd_file;
@@ -262,13 +261,14 @@ struct OperationConfig {
                                      0 is valid. default: CURL_HET_DEFAULT. */
   bool haproxy_protocol;          /* whether to send HAProxy protocol v1 */
   bool disallow_username_in_url;  /* disallow usernames in URLs */
+  bool parallel;
+  long parallel_max;
   struct GlobalConfig *global;
   struct OperationConfig *prev;
   struct OperationConfig *next;   /* Always last in the struct */
 };
 
 struct GlobalConfig {
-  CURL *easy;                     /* Once we have one, we keep it here */
   int showerror;                  /* -1 == unset, default => show errors
                                       0 => -s is used to NOT show errors
                                       1 => -S has been used to show errors */

diff --git a/src/tool_getparam.c b/src/tool_getparam.c
@@ -40,6 +40,7 @@
 #include "tool_msgs.h"
 #include "tool_paramhlp.h"
 #include "tool_parsecfg.h"
+#include "tool_main.h"
 
 #include "memdebug.h" /* keep this as LAST include */
 
@@ -316,6 +317,8 @@ static const struct LongShort aliases[]= {
   {"Y",  "speed-limit",              ARG_STRING},
   {"y",  "speed-time",               ARG_STRING},
   {"z",  "time-cond",                ARG_STRING},
+  {"Z",  "parallel",                 ARG_BOOL},
+  {"Zb", "parallel-max",             ARG_STRING},
   {"#",  "progress-bar",             ARG_BOOL},
   {":",  "next",                     ARG_NONE},
 };
@@ -1351,7 +1354,7 @@ ParameterError getparameter(const char *flag, /* f or -long-flag */
           size = 0;
         }
         else {
-          char *enc = curl_easy_escape(config->easy, postdata, (int)size);
+          char *enc = curl_easy_escape(NULL, postdata, (int)size);
           Curl_safefree(postdata); /* no matter if it worked or not */
           if(enc) {
             /* now make a string with the name from above and append the
@@ -2122,6 +2125,21 @@ ParameterError getparameter(const char *flag, /* f or -long-flag */
       if(!config->low_speed_time)
         config->low_speed_time = 30;
       break;
+    case 'Z':
+      switch(subletter) {
+      case '\0':  /* --parallel */
+        config->parallel = toggle;
+        break;
+      case 'b':   /* --parallel-max */
+        err = str2unum(&config->parallel_max, nextarg);
+        if(err)
+          return err;
+        if((config->parallel_max > MAX_PARALLEL) ||
+           (config->parallel_max < 1))
+          config->parallel_max = PARALLEL_DEFAULT;
+        break;
+      }
+      break;
     case 'z': /* time condition coming up */
       switch(*nextarg) {
       case '+':
@@ -2207,9 +2225,6 @@ ParameterError parse_args(struct GlobalConfig *config, int argc,
               /* Initialise the newly created config */
               config_init(operation->next);
 
-              /* Copy the easy handle */
-              operation->next->easy = config->easy;
-
               /* Set the global config pointer */
               operation->next->global = config;
 

diff --git a/src/tool_help.c b/src/tool_help.c
@@ -273,6 +273,10 @@ static const struct helptxt helptext[] = {
    "OAuth 2 Bearer Token"},
   {"-o, --output <file>",
    "Write to file instead of stdout"},
+  {"-Z, --parallel",
+   "Perform transfers in parallel"},
+  {"    --parallel-max",
+   "Maximum concurrency for parallel transfers"},
   {"    --pass <phrase>",
    "Pass phrase for the private key"},
   {"    --path-as-is",
@@ -602,8 +606,9 @@ void tool_version_info(void)
   }
 }
 
-void tool_list_engines(CURL *curl)
+void tool_list_engines(void)
 {
+  CURL *curl = curl_easy_init();
   struct curl_slist *engines = NULL;
 
   /* Get the list of engines */
@@ -620,4 +625,5 @@ void tool_list_engines(CURL *curl)
 
   /* Cleanup the list of engines */
   curl_slist_free_all(engines);
+  curl_easy_cleanup(curl);
 }
diff --git a/src/tool_help.h b/src/tool_help.h
@@ -7,7 +7,7 @@
  *                            | (__| |_| |  _ <| |___
  *                             \___|\___/|_| \_\_____|
  *
- * Copyright (C) 1998 - 2014, Daniel Stenberg, <daniel@haxx.se>, et al.
+ * Copyright (C) 1998 - 2019, Daniel Stenberg, <daniel@haxx.se>, et al.
  *
  * This software is licensed as described in the file COPYING, which
  * you should have received as part of this distribution. The terms
@@ -24,7 +24,7 @@
 #include "tool_setup.h"
 
 void tool_help(void);
-void tool_list_engines(CURL *curl);
+void tool_list_engines(void);
 void tool_version_info(void);
 
 #endif /* HEADER_CURL_TOOL_HELP_H */
diff --git a/src/tool_main.c b/src/tool_main.c
@@ -160,19 +160,9 @@ static CURLcode main_init(struct GlobalConfig *config)
       result = get_libcurl_info();
 
       if(!result) {
-        /* Get a curl handle to use for all forthcoming curl transfers */
-        config->easy = curl_easy_init();
-        if(config->easy) {
-          /* Initialise the config */
-          config_init(config->first);
-          config->first->easy = config->easy;
-          config->first->global = config;
-        }
-        else {
-          helpf(stderr, "error initializing curl easy handle\n");
-          result = CURLE_FAILED_INIT;
-          free(config->first);
-        }
+        /* Initialise the config */
+        config_init(config->first);
+        config->first->global = config;
       }
       else {
         helpf(stderr, "error retrieving curl library information\n");
@@ -214,9 +204,6 @@ static void free_globalconfig(struct GlobalConfig *config)
 static void main_free(struct GlobalConfig *config)
 {
   /* Cleanup the easy handle */
-  curl_easy_cleanup(config->easy);
-  config->easy = NULL;
-
   /* Main cleanup */
   curl_global_cleanup();
   convert_cleanup();

diff --git a/src/tool_main.h b/src/tool_main.h
@@ -7,7 +7,7 @@
  *                            | (__| |_| |  _ <| |___
  *                             \___|\___/|_| \_\_____|
  *
- * Copyright (C) 1998 - 2012, Daniel Stenberg, <daniel@haxx.se>, et al.
+ * Copyright (C) 1998 - 2019, Daniel Stenberg, <daniel@haxx.se>, et al.
  *
  * This software is licensed as described in the file COPYING, which
  * you should have received as part of this distribution. The terms
@@ -28,6 +28,9 @@
 #define RETRY_SLEEP_DEFAULT 1000L   /* ms */
 #define RETRY_SLEEP_MAX     600000L /* ms == 10 minutes */
 
+#define MAX_PARALLEL 300 /* conservative */
+#define PARALLEL_DEFAULT 50
+
 #ifndef STDIN_FILENO
 #  define STDIN_FILENO  fileno(stdin)
 #endif

diff --git a/src/tool_metalink.c b/src/tool_metalink.c
@@ -5,7 +5,7 @@
  *                            | (__| |_| |  _ <| |___
  *                             \___|\___/|_| \_\_____|
  *
- * Copyright (C) 1998 - 2018, Daniel Stenberg, <daniel@haxx.se>, et al.
+ * Copyright (C) 1998 - 2019, Daniel Stenberg, <daniel@haxx.se>, et al.
  *
  * This software is licensed as described in the file COPYING, which
  * you should have received as part of this distribution. The terms
@@ -104,6 +104,7 @@ struct win32_crypto_hash {
 #include "tool_paramhlp.h"
 #include "tool_cfgable.h"
 #include "tool_metalink.h"
+#include "tool_operate.h"
 #include "tool_msgs.h"
 
 #include "memdebug.h" /* keep this as LAST include */
@@ -674,8 +675,9 @@ int metalink_check_hash(struct GlobalConfig *config,
   return rv;
 }
 
-static metalink_checksum *new_metalink_checksum_from_hex_digest
-(const metalink_digest_def *digest_def, const char *hex_digest)
+static metalink_checksum *
+checksum_from_hex_digest(const metalink_digest_def *digest_def,
+                         const char *hex_digest)
 {
   metalink_checksum *chksum;
   unsigned char *digest;
@@ -754,8 +756,8 @@ static metalinkfile *new_metalinkfile(metalink_file_t *fileinfo)
         if(curl_strequal(digest_alias->alias_name, (*p)->type) &&
            check_hex_digest((*p)->hash, digest_alias->digest_def)) {
           f->checksum =
-            new_metalink_checksum_from_hex_digest(digest_alias->digest_def,
-                                                  (*p)->hash);
+            checksum_from_hex_digest(digest_alias->digest_def,
+                                     (*p)->hash);
           break;
         }
       }
@@ -891,7 +893,8 @@ int parse_metalink(struct OperationConfig *config, struct OutStruct *outs,
 size_t metalink_write_cb(void *buffer, size_t sz, size_t nmemb,
                          void *userdata)
 {
-  struct OutStruct *outs = userdata;
+  struct per_transfer *per = userdata;
+  struct OutStruct *outs = &per->outs;
   struct OperationConfig *config = outs->config;
   int rv;
 

diff --git a/src/tool_metalink.h b/src/tool_metalink.h
@@ -7,7 +7,7 @@
  *                            | (__| |_| |  _ <| |___
  *                             \___|\___/|_| \_\_____|
  *
- * Copyright (C) 1998 - 2014, Daniel Stenberg, <daniel@haxx.se>, et al.
+ * Copyright (C) 1998 - 2014, 2019, Daniel Stenberg, <daniel@haxx.se>, et al.
  *
  * This software is licensed as described in the file COPYING, which
  * you should have received as part of this distribution. The terms
@@ -22,6 +22,7 @@
  *
  ***************************************************************************/
 #include "tool_setup.h"
+#include "tool_sdecls.h"
 
 struct GlobalConfig;
 struct OperationConfig;