HTTP: implement trailing headers for chuncked transfers

This adds the CURLOPT_HTTP_TRAILER_DATA and CURLOPT_HTTP_TRAILER_FUNCTION
options that allow a callback based approach to sending trailing headers
with chunked transfers.

The test server (sws) was updated to take into account the detection of the
end of transfer in the case of trailing headers presence.

Test 1591 checks that trailing headers can be sent using libcurl.

docs/libcurl/curl_easy_setopt.3

@@ -327,6 +327,12 @@ Disable Content decoding. See \fICURLOPT_HTTP_CONTENT_DECODING(3)\fP
 Disable Transfer decoding. See \fICURLOPT_HTTP_TRANSFER_DECODING(3)\fP
 .IP CURLOPT_EXPECT_100_TIMEOUT_MS
 100-continue timeout. See \fICURLOPT_EXPECT_100_TIMEOUT_MS(3)\fP
+.IP CURLOPT_HTTP_TRAILER_FUNCTION
+Set a callback to send trailers after the HTTP body. See
+\fICURLOPT_HTTP_TRAILER_FUNCTION(3)\fP
+.IP CURLOPT_HTTP_TRAILER_DATA
+Set the data to be passed along to the HTTP trailer callback. See
+\fICURLOPT_HTTP_TRAILER_DATA(3)\fP
 .IP CURLOPT_PIPEWAIT
 Wait on connection to pipeline on it. See \fICURLOPT_PIPEWAIT(3)\fP
 .IP CURLOPT_STREAM_DEPENDS

docs/libcurl/opts/CURLOPT_HTTP_TRAILER_DATA.3

@@ -0,0 +1,58 @@
+.\" **************************************************************************
+.\" *                                  _   _ ____  _
+.\" *  Project                     ___| | | |  _ \| |
+.\" *                             / __| | | | |_) | |
+.\" *                            | (__| |_| |  _ <| |___
+.\" *                             \___|\___/|_| \_\_____|
+.\" *
+.\" * Copyright (C) 1998 - 2017, Daniel Stenberg, <daniel@haxx.se>, et al.
+.\" *
+.\" * This software is licensed as described in the file COPYING, which
+.\" * you should have received as part of this distribution. The terms
+.\" * are also available at https://curl.haxx.se/docs/copyright.html.
+.\" *
+.\" * You may opt to use, copy, modify, merge, publish, distribute and/or sell
+.\" * copies of the Software, and permit persons to whom the Software is
+.\" * furnished to do so, under the terms of the COPYING file.
+.\" *
+.\" * This software is distributed on an "AS IS" basis, WITHOUT WARRANTY OF ANY
+.\" * KIND, either express or implied.
+.\" *
+.\" **************************************************************************
+.\"
+.TH CURLOPT_HTTP_TRAILER_DATA 3 "14 Aug 2018" "libcurl 7.??.?" "curl_easy_setopt options"
+
+.SH NAME:
+CURLOPT_HTTP_TRAILER_DATA \- Set user data to be passed on to the trailing headers callback 
+if one set.
+
+.SH SYNOPSIS:
+#include <curl.h>
+
+CURLcode curl_easy_setopt(CURL *handle, CURLOPT_HTTP_TRAILER_DATA, void *userdata);
+
+.SH DESCRIPTION:
+Data pointer to be passed to the HTTP trailer callback function.
+
+.SH DEFAULT:
+NULL
+
+.SH PROTOCOLS:
+HTTP
+
+.SH EXAMPLE:
+.nf
+// Assuming we have a CURL handle in the hndl variable.
+
+struct MyData data;
+
+curl_easy_setopt(hndl, CURLOPT_HTTP_TRAILER_DATA, &data);
+
+.fi
+
+A more complete example can be found in examples/http_trailers.html
+.SH AVAILABILITY:
+This option was added in curl 7.63.0 and is present if HTTP support is enabled
+
+.SH "SEE ALSO"
+.BR CURLOPT_HTTP_TRAILER_FUNCTION "(3), "  

docs/libcurl/opts/CURLOPT_HTTP_TRAILER_FUNCTION.3

@@ -0,0 +1,110 @@
+.\" **************************************************************************
+.\" *                                  _   _ ____  _
+.\" *  Project                     ___| | | |  _ \| |
+.\" *                             / __| | | | |_) | |
+.\" *                            | (__| |_| |  _ <| |___
+.\" *                             \___|\___/|_| \_\_____|
+.\" *
+.\" * Copyright (C) 1998 - 2017, Daniel Stenberg, <daniel@haxx.se>, et al.
+.\" *
+.\" * This software is licensed as described in the file COPYING, which
+.\" * you should have received as part of this distribution. The terms
+.\" * are also available at https://curl.haxx.se/docs/copyright.html.
+.\" *
+.\" * You may opt to use, copy, modify, merge, publish, distribute and/or sell
+.\" * copies of the Software, and permit persons to whom the Software is
+.\" * furnished to do so, under the terms of the COPYING file.
+.\" *
+.\" * This software is distributed on an "AS IS" basis, WITHOUT WARRANTY OF ANY
+.\" * KIND, either express or implied.
+.\" *
+.\" **************************************************************************
+.\"
+.TH CURLOPT_HTTP_TRAILER_FUNCTION 3 "14 Aug 2018" "libcurl 7.63.0" "curl_easy_setopt options"
+
+.SH NAME:
+CURLOPT_HTTP_TRAILER_FUNCTION \- Callback to fill trailing headers list to be sent in a chunked transfer.
+
+.SH SYNOPSIS:
+#include <curl.h>
+
+void curl_trailer_callback(struct curl_slist ** list, void *userdata);
+
+CURLcode curl_easy_setopt(CURL *handle, CURLOPT_HTTP_TRAILER_FUNCTION, curl_trailer_callback *func);
+
+.SH DESCRIPTION:
+Pass a pointer to a callback function.
+
+This callback function will be called once right before sending the final
+CR LF in an HTTP chunked transfer to fill a list of trailing headers to be
+sent before finishing the HTTP transfer.
+
+You can set the userdata argument with the CURLOPT_HTTP_TRAILER_DATA option.
+
+The trailing headers included in the linked list must not be CRLF-terminated,
+because libcurl will add the appropriate line termination characters after
+each header item.
+
+If you use curl_slist_append to add trailing headers to the curl_slist then
+libcurl will duplicate the strings, and will free the curl_slist and the
+duplicates once the trailers have been sent.
+
+If one of the trailing headers is not formatted correctly
+(i.e. HeaderName: headerdata) it will be ignored and an info message
+will be emitted.
+
+If you set this option to NULL, then the HTTP transfer proceeds as usual
+without any interruptions.
+
+.SH DEFAULT:
+NULL
+
+.SH PROTOCOLS:
+HTTP
+
+.SH EXAMPLE:
+#include <curl/curl.h>
+
+static void trailer_cb(struct curl_slist **tr, void *data)
+{
+    /* libcurl will free the list as soon as it becomes not needed */
+	*tr = curl_slist_append(*tr, "My-super-awesome-trailer: trailer-stuff");
+	(void)data;
+}
+
+CURL *curl = curl_easy_init();
+if(curl) {
+
+    /* Set the URL of the request */
+    curl_easy_setopt(curl, CURLOPT_URL, "http://example.com/");
+    /* Now set it as a put */
+    curl_easy_setopt(curl, CURLOPT_PUT, 1L);
+
+    /* Assuming we have a function that will return the data to be pushed 
+       Let that function be read_cb */
+    curl_easy_setopt(curl, CURLOPT_READFUNCTION, read_cb);
+
+    /* We need to announce the trailers to come 
+       See CURLOPT_READFUNCTION for more on the matter */
+
+    struct curl_slist *headers = NULL;
+    headers = curl_slist_append(headers, "Trailer: My-super-awsome-trailer");
+    res = curl_easy_setopt(curl, CURLOPT_HTTPHEADER, headers);
+
+    /* Set the trailers filling callback */
+    curl_easy_setopt(curl, CURLOPT_HTTP_TRAILER_FUNCTION, (curl_trailer_callback)trailer_cb);
+
+    /* Perform the request, res will get the return code */ 
+    res = curl_easy_perform(curl);
+
+    curl_easy_cleanup(curl);
+
+    curl_slist_free_all(headers);
+}
+
+
+.SH AVAILABILITY:
+This option was added in curl 7.63.0 and is present if HTTP support is enabled
+
+.SH "SEE ALSO"
+.BR CURLOPT_HTTP_TRAILER_DATA "(3), " 

docs/libcurl/opts/Makefile.inc

@@ -163,6 +163,8 @@ man_MANS =                                      \
   CURLOPT_HTTP_CONTENT_DECODING.3               \
   CURLOPT_HTTP_TRANSFER_DECODING.3              \
   CURLOPT_HTTP_VERSION.3                        \
+  CURLOPT_HTTP_TRAILER_FUNCTION.3               \
+  CURLOPT_HTTP_TRAILER_DATA.3                   \
   CURLOPT_IGNORE_CONTENT_LENGTH.3               \
   CURLOPT_INFILESIZE.3                          \
   CURLOPT_INFILESIZE_LARGE.3                    \

docs/libcurl/symbols-in-versions

@@ -431,6 +431,8 @@ CURLOPT_HTTPREQUEST             7.1           -           7.15.5
 CURLOPT_HTTP_CONTENT_DECODING   7.16.2
 CURLOPT_HTTP_TRANSFER_DECODING  7.16.2
 CURLOPT_HTTP_VERSION            7.9.1
+CURLOPT_HTTP_TRAILER_FUNCTION   7.63.2
+CURLOPT_HTTP_TRAILER_DATA       7.63.2
 CURLOPT_IGNORE_CONTENT_LENGTH   7.14.1
 CURLOPT_INFILE                  7.1           7.9.7
 CURLOPT_INFILESIZE              7.1

include/curl/curl.h

@@ -360,6 +360,9 @@ typedef size_t (*curl_read_callback)(char *buffer,
                                       size_t nitems,
                                       void *instream);
 
+typedef void (*curl_trailer_callback)(struct curl_slist **list,
+                                      void *userdata);
+
 typedef enum {
   CURLSOCKTYPE_IPCXN,  /* socket created for a specific IP connection */
   CURLSOCKTYPE_ACCEPT, /* socket created by accept() call */
@@ -1875,6 +1878,12 @@ typedef enum {
   /* Specify URL using CURL URL API. */
   CINIT(CURLU, OBJECTPOINT, 282),
 
+  /* add trailing data just after no more data is available */
+  CINIT(HTTP_TRAILER_FUNCTION, FUNCTIONPOINT, 283),
+
+  /* pointer to be passed to HTTP_TRAILER_FUNCTION */
+  CINIT(HTTP_TRAILER_DATA, OBJECTPOINT, 284),
+
   CURLOPT_LASTENTRY /* the last unused */
 } CURLoption;
 

lib/http.c

@@ -1675,6 +1675,52 @@ enum proxy_use {
   HEADER_CONNECT  /* sending CONNECT to a proxy */
 };
 
+/* used to compile the provided trailers into one buffer
+   will return an error code if one of the headers is
+   not formatted correctly */
+CURLcode Curl_http_compile_trailers(struct curl_slist *trailers,
+                                    Curl_send_buffer *buffer,
+                                    struct Curl_easy *handle)
+{
+  char *ptr = NULL;
+  CURLcode result = CURLE_OK;
+  const char *endofline_native = NULL;
+  const char *endofline_network = NULL;
+
+  /* TODO: Maybe split Curl_add_custom_headers to make it reusable here */
+
+  if(
+#ifdef CURL_DO_LINEEND_CONV
+     (handle->set.prefer_ascii) ||
+#endif
+     (handle->set.crlf)) {
+    /* \n will become \r\n later on */
+    endofline_native  = "\n";
+    endofline_network = "\x0a";
+  }
+  else {
+    endofline_native  = "\r\n";
+    endofline_network = "\x0d\x0a";
+  }
+
+  while(trailers) {
+    /* only add correctly formatted trailers */
+    ptr = strchr(trailers->data, ':');
+    if(ptr && *(ptr + 1) == ' ') {
+      result = Curl_add_bufferf(&buffer, "%s%s", trailers->data,
+                                endofline_native);
+      if(result)
+        return result;
+    }
+    else
+      infof(handle, "Malformatted trailing header ! Skipping trailer.");
+    trailers = trailers->next;
+  }
+  result = Curl_add_buffer(&buffer, endofline_network,
+                           strlen(endofline_network));
+  return result;
+}
+
 CURLcode Curl_add_custom_headers(struct connectdata *conn,
                                  bool is_connect,
                                  Curl_send_buffer *req_buffer)

lib/http.h

@@ -74,6 +74,9 @@ CURLcode Curl_add_timecondition(struct Curl_easy *data,
 CURLcode Curl_add_custom_headers(struct connectdata *conn,
                                  bool is_connect,
                                  Curl_send_buffer *req_buffer);
+CURLcode Curl_http_compile_trailers(struct curl_slist *trailers,
+                                    Curl_send_buffer *buffer,
+                                    struct Curl_easy *handle);
 
 /* protocol-specific functions set up to be called by the main engine */
 CURLcode Curl_http(struct connectdata *conn, bool *done);

lib/setopt.c

@@ -2636,6 +2636,16 @@ CURLcode Curl_vsetopt(struct Curl_easy *data, CURLoption option,
       return CURLE_BAD_FUNCTION_ARGUMENT;
     data->set.upkeep_interval_ms = arg;
     break;
+  case CURLOPT_HTTP_TRAILER_FUNCTION:
+#ifndef CURL_DISABLE_HTTP
+    data->set.trailer_callback = va_arg(param, curl_trailer_callback);
+#endif
+    break;
+  case CURLOPT_HTTP_TRAILER_DATA:
+#ifndef CURL_DISABLE_HTTP
+    data->set.trailer_data = va_arg(param, void *);
+#endif
+    break;
   default:
     /* unknown tag and its companion, just ignore: */
     result = CURLE_UNKNOWN_OPTION;