header api: add curl_easy_header and curl_easy_nextheader

.cirrus.yml

@@ -84,14 +84,14 @@ windows_task:
         container_img: ghcr.io/mback2k/curl-docker-winbuildenv/msys2-mingw32:ltsc2019
         container_cmd: C:\msys64\usr\bin\sh
         prepare: pacman -S --needed --noconfirm --noprogressbar libssh2-devel mingw-w64-i686-libssh2 mingw-w64-i686-python-pip mingw-w64-i686-python-wheel mingw-w64-i686-python-pyopenssl && python3 -m pip install --prefer-binary impacket
-        configure: --host=i686-w64-mingw32 --build=i686-w64-mingw32 --prefix=/mingw32 --enable-werror --enable-sspi --with-schannel --with-winidn --with-libssh2
+        configure: --host=i686-w64-mingw32 --build=i686-w64-mingw32 --prefix=/mingw32 --enable-werror --enable-sspi --with-schannel --with-winidn --with-libssh2 --enable-headers-api
         tests: "~571"
     - name: Windows 32-bit static/release Schannel/SSPI/WinIDN/libssh2
       env:
         container_img: ghcr.io/mback2k/curl-docker-winbuildenv/msys2-mingw32:ltsc2019
         container_cmd: C:\msys64\usr\bin\sh
         prepare: pacman -S --needed --noconfirm --noprogressbar libssh2-devel mingw-w64-i686-libssh2 mingw-w64-i686-python-pip mingw-w64-i686-python-wheel mingw-w64-i686-python-pyopenssl && python3 -m pip install --prefer-binary impacket
-        configure: --host=i686-w64-mingw32 --build=i686-w64-mingw32 --prefix=/mingw32 --enable-werror --enable-sspi --with-schannel --with-winidn --with-libssh2 --disable-shared --enable-static
+        configure: --host=i686-w64-mingw32 --build=i686-w64-mingw32 --prefix=/mingw32 --enable-werror --enable-sspi --with-schannel --with-winidn --with-libssh2 --disable-shared --enable-static  --enable-headers-api
         tests: "~571"
         curl_LDFLAGS: -all-static
         PKG_CONFIG: pkg-config --static
@@ -100,14 +100,14 @@ windows_task:
         container_img: ghcr.io/mback2k/curl-docker-winbuildenv/msys2-mingw64:ltsc2019
         container_cmd: C:\msys64\usr\bin\sh
         prepare: pacman -S --needed --noconfirm --noprogressbar libssh2-devel mingw-w64-x86_64-libssh2 mingw-w64-x86_64-python-pip mingw-w64-x86_64-python-wheel mingw-w64-x86_64-python-pyopenssl && python3 -m pip install --prefer-binary impacket
-        configure: --host=x86_64-w64-mingw32 --build=x86_64-w64-mingw32 --prefix=/mingw64 --enable-werror --enable-sspi --with-schannel --with-winidn --with-libssh2
+        configure: --host=x86_64-w64-mingw32 --build=x86_64-w64-mingw32 --prefix=/mingw64 --enable-werror --enable-sspi --with-schannel --with-winidn --with-libssh2  --enable-headers-api
         tests: "~571"
     - name: Windows 64-bit static/release Schannel/SSPI/WinIDN/libssh2
       env:
         container_img: ghcr.io/mback2k/curl-docker-winbuildenv/msys2-mingw64:ltsc2019
         container_cmd: C:\msys64\usr\bin\sh
         prepare: pacman -S --needed --noconfirm --noprogressbar libssh2-devel mingw-w64-x86_64-libssh2 mingw-w64-x86_64-python-pip mingw-w64-x86_64-python-wheel mingw-w64-x86_64-python-pyopenssl && python3 -m pip install --prefer-binary impacket
-        configure: --host=x86_64-w64-mingw32 --build=x86_64-w64-mingw32 --prefix=/mingw64 --enable-werror --enable-sspi --with-schannel --with-winidn --with-libssh2 --disable-shared --enable-static
+        configure: --host=x86_64-w64-mingw32 --build=x86_64-w64-mingw32 --prefix=/mingw64 --enable-werror --enable-sspi --with-schannel --with-winidn --with-libssh2 --disable-shared --enable-static  --enable-headers-api
         tests: "~571"
         curl_LDFLAGS: -all-static
         PKG_CONFIG: pkg-config --static

.github/workflows/linux-hyper.yml

@@ -22,7 +22,7 @@ jobs:
         build:
         - name: hyper
           install:
-          configure: --with-openssl --with-hyper=$HOME/hyper
+          configure: --with-openssl --with-hyper=$HOME/hyper --enable-headers-api
 
     steps:
     - run: |

.github/workflows/macos.yml

@@ -85,7 +85,7 @@ jobs:
 
     - uses: actions/checkout@v2
 
-    - run: autoreconf -fi && ./configure --enable-warnings --enable-werror ${{ matrix.build.configure }}
+    - run: autoreconf -fi && ./configure --enable-warnings --enable-werror --enable-headers-api ${{ matrix.build.configure }}
       name: 'configure and build'
       env:
         # -Wvla is caused by brotli

.github/workflows/mbedtls.yml

@@ -42,7 +42,7 @@ jobs:
 
     - uses: actions/checkout@v2
 
-    - run: autoreconf -fi && LDFLAGS="-Wl,-rpath,$HOME/mbed/lib" ./configure --enable-warnings --enable-werror ${{ matrix.build.configure }} && make V=1
+    - run: autoreconf -fi && LDFLAGS="-Wl,-rpath,$HOME/mbed/lib" ./configure --enable-warnings --enable-werror --enable-headers-api ${{ matrix.build.configure }} && make V=1
       name: 'configure and build'
 
     - run: make V=1 test-ci

.github/workflows/nss.yml

@@ -22,7 +22,7 @@ jobs:
         build:
         - name: NSS
           install:
-          configure: --with-nss --enable-debug --enable-werror --with-nss-deprecated
+          configure: --with-nss --enable-debug --enable-werror --with-nss-deprecated --enable-headers-api
 
     steps:
     - run: |

.github/workflows/rustls.yml

@@ -22,7 +22,7 @@ jobs:
         build:
         - name: rustls
           install:
-          configure: --with-rustls=$HOME/rustls --enable-debug --enable-werror
+          configure: --with-rustls=$HOME/rustls --enable-debug --enable-werror --enable-headers-api
 
     steps:
     - run: |

.github/workflows/wolfssl.yml

@@ -46,7 +46,7 @@ jobs:
 
     - uses: actions/checkout@v2
 
-    - run: autoreconf -fi && LDFLAGS="-Wl,-rpath,$HOME/wssl/lib" ./configure --enable-warnings --enable-werror ${{ matrix.build.curl-configure }} && make V=1
+    - run: autoreconf -fi && LDFLAGS="-Wl,-rpath,$HOME/wssl/lib" ./configure --enable-warnings --enable-werror --enable-headers-api ${{ matrix.build.curl-configure }} && make V=1
       name: 'configure and build'
 
     - run: make V=1 test-ci

configure.ac

@@ -165,6 +165,7 @@ curl_verbose_msg="enabled (--disable-verbose)"
    curl_rtmp_msg="no      (--with-librtmp)"
     curl_psl_msg="no      (--with-libpsl)"
  curl_altsvc_msg="enabled (--disable-alt-svc)"
+ curl_headers_msg="no      (--enable-headers-api)"
    curl_hsts_msg="enabled (--disable-hsts)"
     ssl_backends=
      curl_h1_msg="enabled (internal)"
@@ -3931,6 +3932,25 @@ AS_HELP_STRING([--disable-alt-svc],[Disable alt-svc support]),
        AC_MSG_RESULT(yes)
 )
 
+dnl ************************************************************
+dnl switch on/off headers-api
+dnl
+AC_MSG_CHECKING([whether to support headers-api])
+AC_ARG_ENABLE(headers-api,
+AS_HELP_STRING([--enable-headers-api],[Enable headers-api support])
+AS_HELP_STRING([--disable-headers-api],[Disable headers-api support]),
+[ case "$enableval" in
+  yes)
+       AC_MSG_RESULT(yes)
+       AC_DEFINE(USE_HEADERS_API, 1, [enable headers-api])
+       curl_headers_msg="enabled";
+       ;;
+  *) AC_MSG_RESULT(no)
+       ;;
+  esac ],
+       AC_MSG_RESULT(no)
+)
+
 dnl only check for HSTS if there's SSL present
 if test -n "$SSL_ENABLED"; then
 
@@ -4345,6 +4365,7 @@ AC_MSG_NOTICE([Configured to build curl/libcurl:
   RTMP:             ${curl_rtmp_msg}
   PSL:              ${curl_psl_msg}
   Alt-svc:          ${curl_altsvc_msg}
+  Headers API:      ${curl_headers_msg}
   HSTS:             ${curl_hsts_msg}
   HTTP1:            ${curl_h1_msg}
   HTTP2:            ${curl_h2_msg}

docs/EXPERIMENTAL.md

@@ -20,4 +20,5 @@ Experimental support in curl means:
 
  - The Hyper HTTP backend
  - HTTP/3 support and options
- - CURLSSLOPT_NATIVE_CA (No configure option, feature built in when supported)
+ - `CURLSSLOPT_NATIVE_CA` (No configure option, feature built in when supported)
+ - The headers API: `curl_easy_header` and `curl_easy_nextheader`.

docs/cmdline-opts/write-out.d

@@ -22,6 +22,12 @@ output a newline by using \\n, a carriage return with \\r and a tab space with
 The output will be written to standard output, but this can be switched to
 standard error by using %{stderr}.
 
+EXPERIMENTAL feature: Output HTTP headers from the most recent request by
+using \fB%header{name}\fP where \fBname\fP is the case insensitive name of the
+header (without the trailing colon). The header contents are exactly as sent
+over the network, with leading and trailing whitespace trimmed. Added in curl
+7.83.0.
+
 .B NOTE:
 The %-symbol is a special symbol in the win32-environment, where all
 occurrences of % must be doubled when using this option.
@@ -48,6 +54,18 @@ option. (Added in 7.26.0)
 The initial path curl ended up in when logging on to the remote FTP
 server. (Added in 7.15.4)
 .TP
+.B header_json
+EXPERIMENTAL feature.
+
+A JSON object with all HTTP response headers from the recent transfer. Values
+are provided as arrays, since in the case of multiple headers there can be
+multiple values.
+
+The header names are listed in order of appearance over the wire using the
+same case as was used over the network. Except for duplicated headers. They
+are grouped on the first occurance of that header, using the casing of the
+first header, and then each value is presented in the JSON array.
+.TP
 .B http_code
 The numerical response code that was found in the last retrieved HTTP(S) or
 FTP(s) transfer.

docs/examples/Makefile.inc

@@ -44,6 +44,7 @@ check_PROGRAMS = \
   getinmemory \
   getredirect \
   getreferrer \
+  headerapi \
   http-post \
   http2-download \
   http2-pushinmemory \

docs/examples/headerapi.c

@@ -0,0 +1,79 @@
+/***************************************************************************
+ *                                  _   _ ____  _
+ *  Project                     ___| | | |  _ \| |
+ *                             / __| | | | |_) | |
+ *                            | (__| |_| |  _ <| |___
+ *                             \___|\___/|_| \_\_____|
+ *
+ * Copyright (C) 1998 - 2022, Daniel Stenberg, <daniel@haxx.se>, et al.
+ *
+ * This software is licensed as described in the file COPYING, which
+ * you should have received as part of this distribution. The terms
+ * are also available at https://curl.se/docs/copyright.html.
+ *
+ * You may opt to use, copy, modify, merge, publish, distribute and/or sell
+ * copies of the Software, and permit persons to whom the Software is
+ * furnished to do so, under the terms of the COPYING file.
+ *
+ * This software is distributed on an "AS IS" basis, WITHOUT WARRANTY OF ANY
+ * KIND, either express or implied.
+ *
+ ***************************************************************************/
+/* <DESC>
+ * Extract headers post transfer with the header API
+ * </DESC>
+ */
+#include <stdio.h>
+#include <curl/curl.h>
+
+static size_t write_cb(char *data, size_t n, size_t l, void *userp)
+{
+  /* take care of the data here, ignored in this example */
+  (void)data;
+  (void)userp;
+  return n*l;
+}
+
+int main(void)
+{
+  CURL *curl;
+
+  curl = curl_easy_init();
+  if(curl) {
+    CURLcode res;
+    struct curl_header *header;
+    curl_easy_setopt(curl, CURLOPT_URL, "https://example.com");
+    /* example.com is redirected, so we tell libcurl to follow redirection */
+    curl_easy_setopt(curl, CURLOPT_FOLLOWLOCATION, 1L);
+
+    /* this example just ignores the content */
+    curl_easy_setopt(curl, CURLOPT_WRITEFUNCTION, write_cb);
+
+    /* Perform the request, res will get the return code */
+    res = curl_easy_perform(curl);
+    /* Check for errors */
+    if(res != CURLE_OK)
+      fprintf(stderr, "curl_easy_perform() failed: %s\n",
+              curl_easy_strerror(res));
+
+    if(CURLHE_OK == curl_easy_header(curl, "Content-Type", 0, CURLH_HEADER,
+                                     -1, &header))
+      printf("Got content-type: %s\n", header->value);
+
+    printf("All server headers:\n");
+    {
+      struct curl_header *h;
+      struct curl_header *prev = NULL;
+      do {
+        h = curl_easy_nextheader(curl, CURLH_HEADER, -1, prev);
+        if(h)
+          printf(" %s: %s (%u)\n", h->name, h->value, (int)h->amount);
+        prev = h;
+      } while(h);
+
+    }
+    /* always cleanup */
+    curl_easy_cleanup(curl);
+  }
+  return 0;
+}

docs/libcurl/Makefile.inc

@@ -5,7 +5,7 @@
 #                            | (__| |_| |  _ <| |___
 #                             \___|\___/|_| \_\_____|
 #
-# Copyright (C) 2008 - 2021, Daniel Stenberg, <daniel@haxx.se>, et al.
+# Copyright (C) 2008 - 2022, Daniel Stenberg, <daniel@haxx.se>, et al.
 #
 # This software is licensed as described in the file COPYING, which
 # you should have received as part of this distribution. The terms
@@ -27,7 +27,9 @@ man_MANS = \
  curl_easy_duphandle.3 \
  curl_easy_escape.3 \
  curl_easy_getinfo.3 \
+ curl_easy_header.3 \
  curl_easy_init.3 \
+ curl_easy_nextheader.3 \
  curl_easy_option_by_id.3 \
  curl_easy_option_by_name.3 \
  curl_easy_option_next.3 \