CURLOPT_DEFAULT_PROTOCOL: added

- Add new option CURLOPT_DEFAULT_PROTOCOL to allow specifying a default
protocol for schemeless URLs.

- Add new tool option --proto-default to expose
CURLOPT_DEFAULT_PROTOCOL.

In the case of schemeless URLs libcurl will behave in this way:

When the option is used libcurl will use the supplied default.

When the option is not used, libcurl will follow its usual plan of
guessing from the hostname and falling back to 'http'.

docs/curl.1

@@ -1312,6 +1312,25 @@ This option can be used multiple times, in which case the effect is the same
 as concatenating the protocols into one instance of the option.
 
 (Added in 7.20.2)
+.IP "--proto-default <protocol>"
+Tells curl to use \fIprotocol\fP for any URL missing a scheme name.
+
+Example:
+
+.RS
+.IP "--proto-default https ftp.mozilla.org"
+https://ftp.mozilla.org
+.RE
+
+An unknown or unsupported protocol causes error
+\fICURLE_UNSUPPORTED_PROTOCOL\fP.
+
+This option does not change the default proxy protocol (http).
+
+Without this option curl would make a guess based on the host, see \fI--url\fP
+for details.
+
+(Added in 7.45.0)
 .IP "--proto-redir <protocols>"
 Tells curl to use the listed protocols on redirect. See --proto for how
 protocols are represented.
@@ -1773,6 +1792,12 @@ If this option is used several times, the last one will be used.
 Specify a URL to fetch. This option is mostly handy when you want to specify
 URL(s) in a config file.
 
+If the given URL is missing a scheme name (such as "http://" or "ftp://" etc)
+then curl will make a guess based on the host. If the outermost sub-domain name
+matches DICT, FTP, IMAP, LDAP, POP3 or SMTP then that protocol will be used,
+otherwise HTTP will be used. Since 7.45.0 guessing can be disabled by setting a
+default protocol, see \fI--proto-default\fP for details.
+
 This option may be used any number of times. To control where this URL is
 written, use the \fI-o, --output\fP or the \fI-O, --remote-name\fP options.
 .IP "-v, --verbose"

docs/libcurl/curl_easy_setopt.3

@@ -153,6 +153,8 @@ Disable squashing /../ and /./ sequences in the path. See \fICURLOPT_PATH_AS_IS(
 Allowed protocols. See \fICURLOPT_PROTOCOLS(3)\fP
 .IP CURLOPT_REDIR_PROTOCOLS
 Protocols to allow redirects to. See \fICURLOPT_REDIR_PROTOCOLS(3)\fP
+.IP CURLOPT_DEFAULT_PROTOCOL
+Default protocol. See \fICURLOPT_DEFAULT_PROTOCOL(3)\fP
 .IP CURLOPT_PROXY
 Proxy to use. See \fICURLOPT_PROXY(3)\fP
 .IP CURLOPT_PROXYPORT

docs/libcurl/opts/CURLOPT_DEFAULT_PROTOCOL.3

@@ -0,0 +1,79 @@
+.\" **************************************************************************
+.\" *                                  _   _ ____  _
+.\" *  Project                     ___| | | |  _ \| |
+.\" *                             / __| | | | |_) | |
+.\" *                            | (__| |_| |  _ <| |___
+.\" *                             \___|\___/|_| \_\_____|
+.\" *
+.\" * Copyright (C) 1998 - 2015, Daniel Stenberg, <daniel@haxx.se>, et al.
+.\" *
+.\" * This software is licensed as described in the file COPYING, which
+.\" * you should have received as part of this distribution. The terms
+.\" * are also available at http://curl.haxx.se/docs/copyright.html.
+.\" *
+.\" * You may opt to use, copy, modify, merge, publish, distribute and/or sell
+.\" * copies of the Software, and permit persons to whom the Software is
+.\" * furnished to do so, under the terms of the COPYING file.
+.\" *
+.\" * This software is distributed on an "AS IS" basis, WITHOUT WARRANTY OF ANY
+.\" * KIND, either express or implied.
+.\" *
+.\" **************************************************************************
+.\"
+.TH CURLOPT_DEFAULT_PROTOCOL 3 "18 Aug 2015" "libcurl 7.45.0" "curl_easy_setopt options"
+.SH NAME
+CURLOPT_DEFAULT_PROTOCOL \- default protocol to use if the URL is missing a
+scheme name
+.SH SYNOPSIS
+#include <curl/curl.h>
+
+CURLcode curl_easy_setopt(CURL *handle, CURLOPT_DEFAULT_PROTOCOL, char
+*protocol);
+.SH DESCRIPTION
+This option tells libcurl to use \fIprotocol\fP if the URL is missing a scheme
+name.
+
+Use one of these protocol (scheme) names:
+
+dict, file, ftp, ftps, gopher, http, https, imap, imaps, ldap, ldaps, pop3,
+pop3s, rtsp, scp, sftp, smb, smbs, smtp, smtps, telnet, tftp
+
+An unknown or unsupported protocol causes error
+\fICURLE_UNSUPPORTED_PROTOCOL\fP when libcurl parses a schemeless URL. Parsing
+happens when \fIcurl_easy_perform(3)\fP or \fIcurl_multi_perform(3)\fP is
+called. The protocols supported by libcurl will vary depending on how it was
+built. Use \fIcurl_version_info(3)\fP if you need a list of protocol names
+supported by the build of libcurl that you are using.
+
+This option does not change the default proxy protocol (http).
+
+Without this option libcurl would make a guess based on the host, see
+\fICURLOPT_URL(3)\fP for details.
+.SH DEFAULT
+NULL (make a guess based on the host)
+.SH PROTOCOLS
+All
+.SH EXAMPLE
+.nf
+curl = curl_easy_init();
+if(curl) {
+  /* set a URL without a scheme */
+  curl_easy_setopt(curl, CURLOPT_URL, "example.com");
+
+  /* set the default protocol (scheme) for schemeless URLs */
+  curl_easy_setopt(curl, CURLOPT_DEFAULT_PROTOCOL, "https");
+
+  /* Perform the request */
+  curl_easy_perform(curl);
+}
+.fi
+.SH AVAILABILITY
+Added in 7.45.0
+.SH RETURN VALUE
+CURLE_OK if the option is supported.
+
+CURLE_OUT_OF_MEMORY if there was insufficient heap space.
+
+CURLE_UNKNOWN_OPTION if the option is not supported.
+.SH "SEE ALSO"
+.BR CURLOPT_URL "(3), "

docs/libcurl/opts/CURLOPT_URL.3

@@ -40,9 +40,11 @@ libcurl doesn't validate the syntax or use this variable until the transfer is
 issued. Even if you set a crazy value here, \fIcurl_easy_setopt(3)\fP will
 still return \fICURLE_OK\fP.
 
-If the given URL lacks the scheme (such as "http://" or "ftp://" etc) then
-libcurl will attempt to resolve the protocol based on one of the following
-given host names: HTTP, FTP, DICT, LDAP, IMAP, POP3 or SMTP
+If the given URL is missing a scheme name (such as "http://" or "ftp://" etc)
+then libcurl will make a guess based on the host. If the outermost sub-domain
+name matches DICT, FTP, IMAP, LDAP, POP3 or SMTP then that protocol will be
+used, otherwise HTTP will be used. Since 7.45.0 guessing can be disabled by
+setting a default protocol, see \fICURLOPT_DEFAULT_PROTOCOL(3)\fP for details.
 
 Should the protocol, either that specified by the scheme or deduced by libcurl
 from the host name, not be supported by libcurl then

docs/libcurl/opts/Makefile.am

@@ -115,7 +115,7 @@ man_MANS = CURLOPT_ACCEPT_ENCODING.3 CURLOPT_ACCEPTTIMEOUT_MS.3		\
  CURLMOPT_TIMERFUNCTION.3 CURLOPT_UNIX_SOCKET_PATH.3			\
  CURLOPT_PATH_AS_IS.3 CURLOPT_PROXY_SERVICE_NAME.3			\
  CURLOPT_SERVICE_NAME.3 CURLOPT_PIPEWAIT.3 CURLMOPT_PUSHDATA.3		\
- CURLMOPT_PUSHFUNCTION.3
+ CURLMOPT_PUSHFUNCTION.3 CURLOPT_DEFAULT_PROTOCOL.3
 
 HTMLPAGES = CURLOPT_ACCEPT_ENCODING.html CURLOPT_ACCEPTTIMEOUT_MS.html	\
  CURLOPT_ADDRESS_SCOPE.html CURLOPT_APPEND.html				\
@@ -224,7 +224,7 @@ HTMLPAGES = CURLOPT_ACCEPT_ENCODING.html CURLOPT_ACCEPTTIMEOUT_MS.html	\
  CURLOPT_UNIX_SOCKET_PATH.html CURLOPT_PATH_AS_IS.html			\
  CURLOPT_PROXY_SERVICE_NAME.html CURLOPT_SERVICE_NAME.html		\
  CURLOPT_PIPEWAIT.html CURLMOPT_PUSHDATA.html				\
- CURLMOPT_PUSHFUNCTION.html
+ CURLMOPT_PUSHFUNCTION.html CURLOPT_DEFAULT_PROTOCOL.html
 
 PDFPAGES = CURLOPT_ACCEPT_ENCODING.pdf CURLOPT_ACCEPTTIMEOUT_MS.pdf	\
  CURLOPT_ADDRESS_SCOPE.pdf CURLOPT_APPEND.pdf CURLOPT_AUTOREFERER.pdf	\
@@ -330,7 +330,8 @@ PDFPAGES = CURLOPT_ACCEPT_ENCODING.pdf CURLOPT_ACCEPTTIMEOUT_MS.pdf	\
  CURLMOPT_TIMERDATA.pdf CURLMOPT_TIMERFUNCTION.pdf			\
  CURLOPT_UNIX_SOCKET_PATH.pdf CURLOPT_PATH_AS_IS.pdf			\
  CURLOPT_PROXY_SERVICE_NAME.pdf CURLOPT_SERVICE_NAME.pdf		\
- CURLOPT_PIPEWAIT.pdf CURLMOPT_PUSHDATA.pdf CURLMOPT_PUSHFUNCTION.pdf
+ CURLOPT_PIPEWAIT.pdf CURLMOPT_PUSHDATA.pdf CURLMOPT_PUSHFUNCTION.pdf	\
+ CURLOPT_DEFAULT_PROTOCOL.pdf
 
 CLEANFILES = $(HTMLPAGES) $(PDFPAGES)
 

docs/libcurl/symbols-in-versions

@@ -339,6 +339,7 @@ CURLOPT_CRLFILE                 7.19.0
 CURLOPT_CUSTOMREQUEST           7.1
 CURLOPT_DEBUGDATA               7.9.6
 CURLOPT_DEBUGFUNCTION           7.9.6
+CURLOPT_DEFAULT_PROTOCOL        7.45.0
 CURLOPT_DIRLISTONLY             7.17.0
 CURLOPT_DNS_CACHE_TIMEOUT       7.9.3
 CURLOPT_DNS_INTERFACE           7.33.0

include/curl/curl.h

@@ -1645,6 +1645,9 @@ typedef enum {
   /* Wait/don't wait for pipe/mutex to clarify */
   CINIT(PIPEWAIT, LONG, 237),
 
+  /* Set the protocol used when curl is given a URL without a protocol */
+  CINIT(DEFAULT_PROTOCOL, OBJECTPOINT, 238),
+
   CURLOPT_LASTENTRY /* the last unused */
 } CURLoption;
 

lib/url.c

@@ -2441,6 +2441,12 @@ CURLcode Curl_setopt(struct SessionHandle *data, CURLoption option,
     data->set.redir_protocols = va_arg(param, long);
     break;
 
+  case CURLOPT_DEFAULT_PROTOCOL:
+    /* Set the protocol to use when the URL doesn't include any protocol */
+    result = setstropt(&data->set.str[STRING_DEFAULT_PROTOCOL],
+                       va_arg(param, char *));
+    break;
+
   case CURLOPT_MAIL_FROM:
     /* Set the SMTP mail originator */
     result = setstropt(&data->set.str[STRING_MAIL_FROM],
@@ -4028,27 +4034,30 @@ static CURLcode parseurlandfillconn(struct SessionHandle *data,
       }
 
       /*
-       * Since there was no protocol part specified, we guess what protocol it
-       * is based on the first letters of the server name.
+       * Since there was no protocol part specified in the URL use the
+       * user-specified default protocol. If we weren't given a default make a
+       * guess by matching some protocols against the host's outermost
+       * sub-domain name. Finally if there was no match use HTTP.
        */
 
-      /* Note: if you add a new protocol, please update the list in
-       * lib/version.c too! */
-
-      if(checkprefix("FTP.", conn->host.name))
-        protop = "ftp";
-      else if(checkprefix("DICT.", conn->host.name))
-        protop = "DICT";
-      else if(checkprefix("LDAP.", conn->host.name))
-        protop = "LDAP";
-      else if(checkprefix("IMAP.", conn->host.name))
-        protop = "IMAP";
-      else if(checkprefix("SMTP.", conn->host.name))
-        protop = "smtp";
-      else if(checkprefix("POP3.", conn->host.name))
-        protop = "pop3";
-      else {
-        protop = "http";
+      protop = data->set.str[STRING_DEFAULT_PROTOCOL];
+      if(!protop) {
+        /* Note: if you add a new protocol, please update the list in
+         * lib/version.c too! */
+        if(checkprefix("FTP.", conn->host.name))
+          protop = "ftp";
+        else if(checkprefix("DICT.", conn->host.name))
+          protop = "DICT";
+        else if(checkprefix("LDAP.", conn->host.name))
+          protop = "LDAP";
+        else if(checkprefix("IMAP.", conn->host.name))
+          protop = "IMAP";
+        else if(checkprefix("SMTP.", conn->host.name))
+          protop = "smtp";
+        else if(checkprefix("POP3.", conn->host.name))
+          protop = "pop3";
+        else
+          protop = "http";
       }
 
       *prot_missing = TRUE; /* not given in URL */

lib/urldata.h

@@ -1350,6 +1350,7 @@ enum dupstring {
   STRING_COOKIE,          /* HTTP cookie string to send */
   STRING_COOKIEJAR,       /* dump all cookies to this file */
   STRING_CUSTOMREQUEST,   /* HTTP/FTP/RTSP request/method to use */
+  STRING_DEFAULT_PROTOCOL, /* Protocol to use when the URL doesn't specify */
   STRING_DEVICE,          /* local network interface/address to use */
   STRING_ENCODING,        /* Accept-Encoding string */
   STRING_FTP_ACCOUNT,     /* ftp account data */

src/tool_cfgable.c

@@ -40,6 +40,7 @@ void config_init(struct OperationConfig* config)
                         ~(CURLPROTO_FILE | CURLPROTO_SCP | CURLPROTO_SMB |
                           CURLPROTO_SMBS);
   config->proto_redir_present = FALSE;
+  config->proto_default = NULL;
 }
 
 static void free_config_fields(struct OperationConfig *config)

src/tool_cfgable.h

@@ -51,6 +51,7 @@ struct OperationConfig {
   bool proto_present;
   long proto_redir;
   bool proto_redir_present;
+  char *proto_default;
   curl_off_t resume_from;
   char *postfields;
   curl_off_t postfieldsize;

src/tool_getparam.c

@@ -160,8 +160,6 @@ static const struct LongShort aliases[]= {
   {"$5", "noproxy",                  TRUE},
   {"$6", "socks5-gssapi-service",    TRUE},
   {"$7", "socks5-gssapi-nec",        FALSE},
-  {"$O", "proxy-service-name",       TRUE},
-  {"$P", "service-name",             TRUE},
   {"$8", "proxy1.0",                 TRUE},
   {"$9", "tftp-blksize",             TRUE},
   {"$A", "mail-from",                TRUE},
@@ -178,6 +176,9 @@ static const struct LongShort aliases[]= {
   {"$L", "test-event",               FALSE},
   {"$M", "unix-socket",              TRUE},
   {"$N", "path-as-is",               FALSE},
+  {"$O", "proxy-service-name",       TRUE},
+  {"$P", "service-name",             TRUE},
+  {"$Q", "proto-default",            TRUE},
   {"0",   "http1.0",                 FALSE},
   {"01",  "http1.1",                 FALSE},
   {"02",  "http2",                   FALSE},
@@ -903,12 +904,6 @@ ParameterError getparameter(char *flag,    /* f or -long-flag */
       case '7': /* --socks5-gssapi-nec*/
         config->socks5_gssapi_nec = toggle;
         break;
-      case 'O': /* --proxy-service-name */
-        GetStr(&config->proxy_service_name, nextarg);
-        break;
-      case 'P': /* --service-name */
-        GetStr(&config->service_name, nextarg);
-        break;
       case '8': /* --proxy1.0 */
         /* http 1.0 proxy */
         GetStr(&config->proxy, nextarg);
@@ -992,6 +987,18 @@ ParameterError getparameter(char *flag,    /* f or -long-flag */
       case 'N': /* --path-as-is */
         config->path_as_is = toggle;
         break;
+      case 'O': /* --proxy-service-name */
+        GetStr(&config->proxy_service_name, nextarg);
+        break;
+      case 'P': /* --service-name */
+        GetStr(&config->service_name, nextarg);
+        break;
+      case 'Q': /* --proto-default */
+        GetStr(&config->proto_default, nextarg);
+        err = check_protocol(config->proto_default);
+        if(err)
+          return err;
+        break;
       }
       break;
     case '#': /* --progress-bar */

src/tool_getparam.h

@@ -37,6 +37,7 @@ typedef enum {
   PARAM_BAD_NUMERIC,
   PARAM_NEGATIVE_NUMERIC,
   PARAM_LIBCURL_DOESNT_SUPPORT,
+  PARAM_LIBCURL_UNSUPPORTED_PROTOCOL,
   PARAM_NO_MEM,
   PARAM_NEXT_OPERATION,
   PARAM_LAST

src/tool_help.c

@@ -165,7 +165,8 @@ static const char *const helptext[] = {
   "Do not switch to GET after following a 303 redirect (H)",
   " -#, --progress-bar  Display transfer progress as a progress bar",
   "     --proto PROTOCOLS  Enable/disable PROTOCOLS",
-  "     --proto-redir PROTOCOLS  Enable/disable PROTOCOLS on redirect",
+  "     --proto-default PROTOCOL  Use PROTOCOL for any URL missing a scheme",
+  "     --proto-redir PROTOCOLS   Enable/disable PROTOCOLS on redirect",
   " -x, --proxy [PROTOCOL://]HOST[:PORT]  Use proxy on given port",
   "     --proxy-anyauth  Pick \"any\" proxy authentication method (H)",
   "     --proxy-basic   Use Basic authentication on the proxy (H)",

src/tool_helpers.c

@@ -58,6 +58,8 @@ const char *param2text(int res)
     return "expected a positive numerical parameter";
   case PARAM_LIBCURL_DOESNT_SUPPORT:
     return "the installed libcurl version doesn't support this";
+  case PARAM_LIBCURL_UNSUPPORTED_PROTOCOL:
+    return "a specified protocol is unsupported by libcurl";
   case PARAM_NO_MEM:
     return "out of memory";
   default:

src/tool_operate.c

@@ -1355,6 +1355,10 @@ static CURLcode operate_do(struct GlobalConfig *global,
           my_setopt_str(curl, CURLOPT_UNIX_SOCKET_PATH,
                         config->unix_socket_path);
 
+        /* new in 7.45.0 */
+        if(config->proto_default)
+          my_setopt_str(curl, CURLOPT_DEFAULT_PROTOCOL, config->proto_default);
+
         /* initialize retry vars for loop below */
         retry_sleep_default = (config->retry_delay) ?
           config->retry_delay*1000L : RETRY_SLEEP_DEFAULT; /* ms */
@@ -1850,6 +1854,8 @@ CURLcode operate(struct GlobalConfig *config, int argc, argv_item_t argv[])
       /* Check if we were asked to list the SSL engines */
       else if(res == PARAM_ENGINES_REQUESTED)
         tool_list_engines(config->easy);
+      else if(res == PARAM_LIBCURL_UNSUPPORTED_PROTOCOL)
+        result = CURLE_UNSUPPORTED_PROTOCOL;
       else
         result = CURLE_FAILED_INIT;
     }