mime: use percent-escaping for multipart form field and file names

Until now, form field and file names where escaped using the backslash-escaping algorithm defined for multipart mails. This commit replaces this with the percent-escaping method for URLs. As this may introduce incompatibilities with server-side applications, a libcurl option CURLOPT_FORM_ESCAPE_AS_MIME is introduced to revert to legacy use of backslash-escaping. This is controlled by new cli tool option --mime-escape. New tests and documentation are provided for this feature. Reported by: Ryan Sleevi Fixes curl#7789
monnerat · Oct 1, 2021 · d94d8ee · d94d8ee
1 parent 85f9124
commit d94d8ee
Show file tree

Hide file tree

Showing 23 changed files with 452 additions and 30 deletions.
diff --git a/docs/cmdline-opts/Makefile.inc b/docs/cmdline-opts/Makefile.inc
@@ -131,6 +131,7 @@ DPAGES = \
   max-redirs.d \
   max-time.d \
   metalink.d \
+  mime-escape.d \
   negotiate.d \
   netrc-file.d \
   netrc-optional.d \

diff --git a/docs/cmdline-opts/mime-escape.d b/docs/cmdline-opts/mime-escape.d
@@ -0,0 +1,11 @@
+Long: mime-escape
+Help: Escape multipart form field/file names with the MIME algorithm
+Protocols: HTTP
+See-also: --form
+Added: 7.80.0
+Category: http post
+Example: --mime-escape --form 'field\\name=curl' 'file=@load"this' $URL
+---
+Tells curl to escape multipart form field and file names using the
+backslash-escaping algorithm defined for mime mail rather than
+the percent-escaping used in URI.
diff --git a/docs/libcurl/curl_easy_setopt.3 b/docs/libcurl/curl_easy_setopt.3
@@ -367,6 +367,9 @@ This HTTP/2 stream depends on another exclusively. See
 \fICURLOPT_STREAM_DEPENDS_E(3)\fP
 .IP CURLOPT_STREAM_WEIGHT
 Set this HTTP/2 stream's weight. See \fICURLOPT_STREAM_WEIGHT(3)\fP
+.IP CURLOPT_FORM_ESCAPE_AS_MIME
+Use backslash-escaping in form field and file names rather than
+percent-escaping. See \fICURLOPT_FORM_ESCAPE_AS_MIME(3)\fP
 .SH SMTP OPTIONS
 .IP CURLOPT_MAIL_FROM
 Address of the sender. See \fICURLOPT_MAIL_FROM(3)\fP

diff --git a/docs/libcurl/opts/CURLOPT_FORM_ESCAPE_AS_MIME.3 b/docs/libcurl/opts/CURLOPT_FORM_ESCAPE_AS_MIME.3
@@ -0,0 +1,75 @@
+.\" **************************************************************************
+.\" *                                  _   _ ____  _
+.\" *  Project                     ___| | | |  _ \| |
+.\" *                             / __| | | | |_) | |
+.\" *                            | (__| |_| |  _ <| |___
+.\" *                             \___|\___/|_| \_\_____|
+.\" *
+.\" * Copyright (C) 1998 - 2021, Daniel Stenberg, <daniel@haxx.se>, et al.
+.\" *
+.\" * This software is licensed as described in the file COPYING, which
+.\" * you should have received as part of this distribution. The terms
+.\" * are also available at https://curl.se/docs/copyright.html.
+.\" *
+.\" * You may opt to use, copy, modify, merge, publish, distribute and/or sell
+.\" * copies of the Software, and permit persons to whom the Software is
+.\" * furnished to do so, under the terms of the COPYING file.
+.\" *
+.\" * This software is distributed on an "AS IS" basis, WITHOUT WARRANTY OF ANY
+.\" * KIND, either express or implied.
+.\" *
+.\" **************************************************************************
+.\"
+.TH CURLOPT_FORM_ESCAPE_AS_MIME 3 "14 Oct 2021" "libcurl 7.80.0" "curl_easy_setopt options"
+.SH NAME
+CURLOPT_FORM_ESCAPE_AS_MIME \- change form field/file names escaping
+.SH SYNOPSIS
+#include <curl/curl.h>
+
+CURLcode curl_easy_setopt(CURL *handle, CURLOPT_FORM_ESCAPE_AS_MIME, long onoff);
+.SH DESCRIPTION
+Set the \fIonoff\fP parameter to 1 to escape multipart form field and file
+names using the backslash-escaping algorithm defined for mime mail rather than
+the percent-escaping used in URI.
+
+As an example, consider field or file name \fIstrange\\name"kind\fP.
+When the containing multipart form is sent, this is normally transmitted as
+\fIstrange\\name%22kind\fP. When this option is set, it is sent as
+\fIstrange\\\\name\\"kind\fP.
+.SH DEFAULT
+0, meaning disabled.
+.SH PROTOCOLS
+HTTP
+.SH EXAMPLE
+.nf
+CURL *curl = curl_easy_init();
+curl_mime *form = NULL;
+
+if(curl) {
+  curl_easy_setopt(curl, CURLOPT_URL, "https://example.com");
+  curl_easy_setopt(curl, CURLOPT_FORM_ESCAPE_AS_MIME, 1L);
+
+  form = curl_mime_init(curl);
+  if(form) {
+    curl_mimepart *part = curl_mime_addpart(form);
+
+    if(part) {
+      curl_mime_filedata(part, "strange\\\\file\\\\name");
+      curl_mime_name(part, "strange\\"field\\"name");
+      curl_easy_setopt(curl, CURLOPT_MIMEPOST, form);
+
+      /* Perform the request */
+      curl_easy_perform(curl);
+    }
+  }
+
+  curl_easy_cleanup(curl);
+  curl_mime_free(mime);
+}
+.fi
+.SH AVAILABILITY
+Option added in 7.80.0. Before this version, percent-escaping was never applied.
+.SH RETURN VALUE
+Returns CURLE_OK
+.SH "SEE ALSO"
+.BR CURLOPT_MIMEPOST "(3), " CURLOPT_HTTPPOST "(3)"
diff --git a/docs/libcurl/opts/Makefile.inc b/docs/libcurl/opts/Makefile.inc
@@ -165,6 +165,7 @@ man_MANS =                                      \
   CURLOPT_FNMATCH_FUNCTION.3                    \
   CURLOPT_FOLLOWLOCATION.3                      \
   CURLOPT_FORBID_REUSE.3                        \
+  CURLOPT_FORM_ESCAPE_AS_MIME.3                 \
   CURLOPT_FRESH_CONNECT.3                       \
   CURLOPT_FTPPORT.3                             \
   CURLOPT_FTPSSLAUTH.3                          \

diff --git a/docs/libcurl/symbols-in-versions b/docs/libcurl/symbols-in-versions
@@ -429,6 +429,7 @@ CURLOPT_FNMATCH_DATA            7.21.0
 CURLOPT_FNMATCH_FUNCTION        7.21.0
 CURLOPT_FOLLOWLOCATION          7.1
 CURLOPT_FORBID_REUSE            7.7
+CURLOPT_FORM_ESCAPE_AS_MIME     7.80.0
 CURLOPT_FRESH_CONNECT           7.7
 CURLOPT_FTPAPPEND               7.1           7.16.4
 CURLOPT_FTPASCII                7.1           7.11.1      7.15.5

diff --git a/docs/options-in-versions b/docs/options-in-versions
@@ -119,6 +119,7 @@
 --max-redirs                         7.5
 --max-time (-m)                      4.0
 --metalink                           7.27.0
+--mime-escape                        7.80.0
 --negotiate                          7.10.6
 --netrc (-n)                         4.6
 --netrc-file                         7.21.5

diff --git a/include/curl/curl.h b/include/curl/curl.h
@@ -2126,6 +2126,9 @@ typedef enum {
   /* Data passed to the CURLOPT_PREREQFUNCTION callback */
   CURLOPT(CURLOPT_PREREQDATA, CURLOPTTYPE_CBPOINT, 313),
 
+  /* Use mime escaping in http form field and file names. */
+  CURLOPT(CURLOPT_FORM_ESCAPE_AS_MIME, CURLOPTTYPE_LONG, 314),
+
   CURLOPT_LASTENTRY /* the last unused */
 } CURLoption;
 

diff --git a/lib/easyoptions.c b/lib/easyoptions.c
@@ -94,6 +94,7 @@ struct curl_easyoption Curl_easyopts[] = {
   {"FNMATCH_FUNCTION", CURLOPT_FNMATCH_FUNCTION, CURLOT_FUNCTION, 0},
   {"FOLLOWLOCATION", CURLOPT_FOLLOWLOCATION, CURLOT_LONG, 0},
   {"FORBID_REUSE", CURLOPT_FORBID_REUSE, CURLOT_LONG, 0},
+  {"FORM_ESCAPE_AS_MIME", CURLOPT_FORM_ESCAPE_AS_MIME, CURLOT_LONG, 0},
   {"FRESH_CONNECT", CURLOPT_FRESH_CONNECT, CURLOT_LONG, 0},
   {"FTPAPPEND", CURLOPT_APPEND, CURLOT_LONG, CURLOT_FLAG_ALIAS},
   {"FTPLISTONLY", CURLOPT_DIRLISTONLY, CURLOT_LONG, CURLOT_FLAG_ALIAS},
@@ -358,6 +359,6 @@ struct curl_easyoption Curl_easyopts[] = {
  */
 int Curl_easyopts_check(void)
 {
-  return ((CURLOPT_LASTENTRY%10000) != (313 + 1));
+  return ((CURLOPT_LASTENTRY%10000) != (314 + 1));
 }
 #endif
diff --git a/lib/mime.c b/lib/mime.c
@@ -40,6 +40,7 @@
 #include "rand.h"
 #include "slist.h"
 #include "strcase.h"
+#include "dynbuf.h"
 /* The last 3 #include files should be in this order */
 #include "curl_printf.h"
 #include "curl_memory.h"
@@ -279,29 +280,66 @@ static void mimesetstate(struct mime_state *state,
 
 
 /* Escape header string into allocated memory. */
-static char *escape_string(const char *src)
+static CURLcode escape_mime_string(struct dynbuf *db, const char *src)
 {
-  size_t bytecount = 0;
-  size_t i;
-  char *dst;
+  CURLcode result = CURLE_OK;
 
-  for(i = 0; src[i]; i++)
-    if(src[i] == '"' || src[i] == '\\')
-      bytecount++;
+  for(; !result && *src; result = Curl_dyn_addn(db, src++, 1))
+    if(*src == '\\' || *src == '"') {
+      result = Curl_dyn_add(db, "\\");
+      if(result)
+        break;
+    }
+  return result;
+}
 
-  bytecount += i;
-  dst = malloc(bytecount + 1);
-  if(!dst)
-    return NULL;
+static CURLcode escape_form_string(struct Curl_easy *easy,
+                                   struct dynbuf *db, const char *src)
+{
+  CURLcode result = CURLE_OK;
+
+  (void) easy;
 
-  for(i = 0; *src; src++) {
-    if(*src == '"' || *src == '\\')
-      dst[i++] = '\\';
-    dst[i++] = *src;
+  for(; !result && *src; src++) {
+    if(*src != '"' && *src != '\r' && *src != '\n')
+      result = Curl_dyn_addn(db, src, 1);
+    else {
+      char ascchar = *src;
+
+#ifdef CURL_DOES_CONVERSIONS
+      result = Curl_convert_to_network(easy, &ascchar, 1);
+      if(result)
+        break;
+#endif
+      result = Curl_dyn_addf(db, "%%%02X", (unsigned char) ascchar);
+
+    }
   }
+  return result;
+}
+
+static char *escape_string(struct Curl_easy *easy,
+                           const char *src, enum mimestrategy strategy)
+{
+  CURLcode result;
+  struct dynbuf db;
 
-  dst[i] = '\0';
-  return dst;
+  Curl_dyn_init(&db, CURL_MAX_INPUT_LENGTH);
+  result = Curl_dyn_add(&db, "");
+
+  if(!result) {
+    if(strategy == MIMESTRATEGY_MAIL ||
+       (easy && easy->set.form_escape_as_mime))
+      result = escape_mime_string(&db, src);
+    else
+      result = escape_form_string(easy, &db, src);
+  }
+
+  if(!result)
+    return Curl_dyn_ptr(&db);
+
+  Curl_dyn_free(&db);
+  return NULL;
 }
 
 /* Check if header matches. */
@@ -1866,12 +1904,12 @@ CURLcode Curl_mime_prepare_headers(curl_mimepart *part,
       char *filename = NULL;
 
       if(part->name) {
-        name = escape_string(part->name);
+        name = escape_string(part->easy, part->name, strategy);
         if(!name)
           ret = CURLE_OUT_OF_MEMORY;
       }
       if(!ret && part->filename) {
-        filename = escape_string(part->filename);
+        filename = escape_string(part->easy, part->filename, strategy);
         if(!filename)
           ret = CURLE_OUT_OF_MEMORY;
       }

diff --git a/lib/setopt.c b/lib/setopt.c
@@ -930,6 +930,9 @@ CURLcode Curl_vsetopt(struct Curl_easy *data, CURLoption option, va_list param)
     data->set.http09_allowed = arg ? TRUE : FALSE;
 #endif
     break;
+  case CURLOPT_FORM_ESCAPE_AS_MIME:
+    data->set.form_escape_as_mime = va_arg(param, long) != 0;
+    break;
 #endif   /* CURL_DISABLE_HTTP */
 
   case CURLOPT_HTTPAUTH:

diff --git a/lib/urldata.h b/lib/urldata.h
@@ -1862,6 +1862,7 @@ struct UserDefined {
   BIT(http09_allowed); /* allow HTTP/0.9 responses */
   BIT(mail_rcpt_allowfails); /* allow RCPT TO command to fail for some
                                 recipients */
+  BIT(form_escape_as_mime); /* Backslash-escape form field/file names. */
 };
 
 struct Names {

diff --git a/packages/OS400/curl.inc.in b/packages/OS400/curl.inc.in
@@ -1583,6 +1583,8 @@
      d                 c                   40309
      d  CURLOPT_PROXY_CAINFO_BLOB...
      d                 c                   40310
+     d  CURLOPT_FORM_ESCAPE_AS_MIME...
+     d                 c                   00314
       *
       /if not defined(CURL_NO_OLDIES)
      d  CURLOPT_FILE   c                   10001

diff --git a/src/tool_cfgable.h b/src/tool_cfgable.h
@@ -248,6 +248,7 @@ struct OperationConfig {
   bool post301;
   bool post302;
   bool post303;
+  bool mimeescape;          /* form field/file names are mime-escaped */
   bool nokeepalive;         /* for keepalive needs */
   long alivetime;
   bool content_disposition; /* use Content-disposition filename */

diff --git a/src/tool_getparam.c b/src/tool_getparam.c
@@ -138,6 +138,7 @@ static const struct LongShort aliases[]= {
   {"$h", "retry-delay",              ARG_STRING},
   {"$i", "retry-max-time",           ARG_STRING},
   {"$k", "proxy-negotiate",          ARG_BOOL},
+  {"$l", "mime-escape",              ARG_BOOL},
   {"$m", "ftp-account",              ARG_STRING},
   {"$n", "proxy-anyauth",            ARG_BOOL},
   {"$o", "trace-time",               ARG_BOOL},
@@ -988,6 +989,10 @@ ParameterError getparameter(const char *flag, /* f or -long-flag */
           return PARAM_LIBCURL_DOESNT_SUPPORT;
         break;
 
+      case 'l': /* --mime-escape */
+        config->mimeescape = toggle;
+        break;
+
       case 'm': /* --ftp-account */
         GetStr(&config->ftp_account, nextarg);
         break;

diff --git a/src/tool_listhelp.c b/src/tool_listhelp.c
@@ -358,6 +358,9 @@ const struct helptxt helptext[] = {
   {"    --metalink",
    "Process given URLs as metalink XML file",
    CURLHELP_MISC},
+  {"    --mime-escape",
+   "Escape multipart form field/file names with the MIME algorithm",
+   CURLHELP_HTTP | CURLHELP_POST},
   {"    --negotiate",
    "Use HTTP Negotiate (SPNEGO) authentication",
    CURLHELP_AUTH | CURLHELP_HTTP},

diff --git a/src/tool_operate.c b/src/tool_operate.c
@@ -1376,6 +1376,10 @@ static CURLcode single_transfer(struct GlobalConfig *global,
             return result;
           }
 
+          /* new in libcurl 7.80.0 */
+          if(config->mimeescape)
+            my_setopt(curl, CURLOPT_FORM_ESCAPE_AS_MIME, 1L);
+
         } /* (built_in_protos & CURLPROTO_HTTP) */
 
         my_setopt_str(curl, CURLOPT_FTPPORT, config->ftpport);

diff --git a/tests/data/Makefile.inc b/tests/data/Makefile.inc
@@ -143,8 +143,7 @@ test1152 test1153 test1154 test1155 test1156 test1157 test1158 test1159 \
 test1160 test1161 test1162 test1163 test1164 test1165 test1166 test1167 \
 test1168 test1169 test1170 test1171 test1172 test1173 test1174 test1175 \
 test1176 test1177 test1178 test1179 test1180 test1181 test1182 test1183 \
-test1184 \
-test1188 \
+test1184 test1185 test1186 test1187 test1188 \
 \
 test1190 test1191 test1192 test1193 test1194 test1195 test1196 test1197 \
 test1198 test1199 \

diff --git a/tests/data/test1158 b/tests/data/test1158
@@ -50,11 +50,11 @@ POST /we/want/%TESTNUMBER HTTP/1.1
 Host: %HOSTIP:%HTTPPORT
 User-Agent: curl/%VERSION
 Accept: */*
-Content-Length: 954
+Content-Length: 958
 Content-Type: multipart/form-data; boundary=----------------------------24e78000bd32
 
 ------------------------------24e78000bd32
-Content-Disposition: form-data; name="file"; filename="test%TESTNUMBER\".txt"
+Content-Disposition: form-data; name="file"; filename="test%TESTNUMBER%22.txt"
 Content-Type: mo/foo
 
 foo bar
@@ -63,7 +63,7 @@ bar
 foo
 
 ------------------------------24e78000bd32
-Content-Disposition: form-data; name="file2"; filename="test%TESTNUMBER\".txt"
+Content-Disposition: form-data; name="file2"; filename="test%TESTNUMBER%22.txt"
 Content-Type: text/plain
 
 foo bar
@@ -75,15 +75,15 @@ foo
 Content-Disposition: form-data; name="file3"
 Content-Type: multipart/mixed; boundary=----------------------------7f0e85a48b0b
 
-Content-Disposition: attachment; filename="test%TESTNUMBER\".txt"
+Content-Disposition: attachment; filename="test%TESTNUMBER%22.txt"
 Content-Type: m/f
 
 foo bar
 This is a bar foo
 bar
 foo
 
-Content-Disposition: attachment; filename="test%TESTNUMBER\".txt"
+Content-Disposition: attachment; filename="test%TESTNUMBER%22.txt"
 Content-Type: text/plain
 
 foo bar