http: add support to read and store the referrer header

- add CURLINFO_REFERER libcurl option - add --write-out '%{referer}' command-line option - extend --xattr command-line option to fill user.xdg.referrer.url extended attribute with the referrer (if there was any) Closes #xxxx
curl · Feb 18, 2021 · e54d5b2 · e54d5b2
1 parent bc928be
commit e54d5b2
Show file tree

Hide file tree

Showing 17 changed files with 153 additions and 11 deletions.
diff --git a/docs/cmdline-opts/write-out.d b/docs/cmdline-opts/write-out.d
@@ -93,6 +93,9 @@ When an HTTP request was made without --location to follow redirects (or when
 --max-redir is met), this variable will show the actual URL a redirect
 \fIwould\fP have gone to. (Added in 7.18.2)
 .TP
+.B referer
+The referrer header, if there was any. (Added in 7.76.0)
+.TP
 .B remote_ip
 The remote IP address of the most recently done connection - can be either
 IPv4 or IPv6 (Added in 7.29.0)

diff --git a/docs/examples/Makefile.inc b/docs/examples/Makefile.inc
@@ -43,6 +43,7 @@ check_PROGRAMS = \
   getinfo \
   getinmemory \
   getredirect \
+  getreferrer \
   http-post \
   http2-download \
   http2-pushinmemory \

diff --git a/docs/examples/getreferrer.c b/docs/examples/getreferrer.c
@@ -0,0 +1,57 @@
+/***************************************************************************
+ *                                  _   _ ____  _
+ *  Project                     ___| | | |  _ \| |
+ *                             / __| | | | |_) | |
+ *                            | (__| |_| |  _ <| |___
+ *                             \___|\___/|_| \_\_____|
+ *
+ * Copyright (C) 1998 - 2021, Daniel Stenberg, <daniel@haxx.se>, et al.
+ *
+ * This software is licensed as described in the file COPYING, which
+ * you should have received as part of this distribution. The terms
+ * are also available at https://curl.se/docs/copyright.html.
+ *
+ * You may opt to use, copy, modify, merge, publish, distribute and/or sell
+ * copies of the Software, and permit persons to whom the Software is
+ * furnished to do so, under the terms of the COPYING file.
+ *
+ * This software is distributed on an "AS IS" basis, WITHOUT WARRANTY OF ANY
+ * KIND, either express or implied.
+ *
+ ***************************************************************************/
+/* <DESC>
+ * Show how to extract referrer header.
+ * </DESC>
+ */
+#include <stdio.h>
+#include <curl/curl.h>
+
+int main(void)
+{
+  CURL *curl;
+
+  curl = curl_easy_init();
+  if(curl) {
+    CURLcode res;
+
+    curl_easy_setopt(curl, CURLOPT_URL, "https://example.com");
+    curl_easy_setopt(curl, CURLOPT_REFERER, "https://example.org/referrer");
+
+    /* Perform the request, res will get the return code */
+    res = curl_easy_perform(curl);
+    /* Check for errors */
+    if(res != CURLE_OK)
+      fprintf(stderr, "curl_easy_perform() failed: %s\n",
+              curl_easy_strerror(res));
+    else {
+      char *hdr;
+      res = curl_easy_getinfo(curl, CURLINFO_REFERER, &hdr);
+      if((res == CURLE_OK) && hdr)
+        printf("Referrer header: %s\n", hdr);
+    }
+
+    /* always cleanup */
+    curl_easy_cleanup(curl);
+  }
+  return 0;
+}
diff --git a/docs/libcurl/curl_easy_getinfo.3 b/docs/libcurl/curl_easy_getinfo.3
@@ -5,7 +5,7 @@
 .\" *                            | (__| |_| |  _ <| |___
 .\" *                             \___|\___/|_| \_\_____|
 .\" *
-.\" * Copyright (C) 1998 - 2020, Daniel Stenberg, <daniel@haxx.se>, et al.
+.\" * Copyright (C) 1998 - 2021, Daniel Stenberg, <daniel@haxx.se>, et al.
 .\" *
 .\" * This software is licensed as described in the file COPYING, which
 .\" * you should have received as part of this distribution. The terms
@@ -50,6 +50,9 @@ See \fICURLINFO_EFFECTIVE_URL(3)\fP
 .IP CURLINFO_RESPONSE_CODE
 Last received response code.
 See \fICURLINFO_RESPONSE_CODE(3)\fP
+.IP CURLINFO_REFERER
+Referrer header.
+See \fICURLINFO_REFERER(3)\fP
 .IP CURLINFO_HTTP_CONNECTCODE
 Last proxy CONNECT response code.
 See \fICURLINFO_HTTP_CONNECTCODE(3)\fP

diff --git a/docs/libcurl/opts/CURLINFO_REFERER.3 b/docs/libcurl/opts/CURLINFO_REFERER.3
@@ -0,0 +1,61 @@
+.\" **************************************************************************
+.\" *                                  _   _ ____  _
+.\" *  Project                     ___| | | |  _ \| |
+.\" *                             / __| | | | |_) | |
+.\" *                            | (__| |_| |  _ <| |___
+.\" *                             \___|\___/|_| \_\_____|
+.\" *
+.\" * Copyright (C) 1998 - 2021, Daniel Stenberg, <daniel@haxx.se>, et al.
+.\" *
+.\" * This software is licensed as described in the file COPYING, which
+.\" * you should have received as part of this distribution. The terms
+.\" * are also available at https://curl.se/docs/copyright.html.
+.\" *
+.\" * You may opt to use, copy, modify, merge, publish, distribute and/or sell
+.\" * copies of the Software, and permit persons to whom the Software is
+.\" * furnished to do so, under the terms of the COPYING file.
+.\" *
+.\" * This software is distributed on an "AS IS" basis, WITHOUT WARRANTY OF ANY
+.\" * KIND, either express or implied.
+.\" *
+.\" **************************************************************************
+.\"
+.TH CURLINFO_REFERER 3 "11 Feb 2021" "libcurl 7.76.0" "curl_easy_getinfo options"
+.SH NAME
+CURLINFO_REFERER \- get the referrer header
+.SH SYNOPSIS
+#include <curl/curl.h>
+
+CURLcode curl_easy_getinfo(CURL *handle, CURLINFO_REFERER, char **hdrp);
+.SH DESCRIPTION
+Pass in a pointer to a char pointer and get the referrer header.
+
+The \fBhdrp\fP pointer will be NULL or pointing to private memory you MUST NOT
+free - it gets freed when you call \fIcurl_easy_cleanup(3)\fP on the
+corresponding CURL handle.
+.SH PROTOCOLS
+HTTP(S)
+.SH EXAMPLE
+.nf
+CURL *curl = curl_easy_init();
+if(curl) {
+  CURLcode res;
+  curl_easy_setopt(curl, CURLOPT_URL, "https://example.com");
+  curl_easy_setopt(curl, CURLOPT_REFERER, "https://example.org/referrer");
+  res = curl_easy_perform(curl);
+  if(res == CURLE_OK) {
+    char *hdr = NULL;
+    curl_easy_getinfo(curl, CURLINFO_REFERER, &hdr);
+    if(hdr)
+      printf("Referrer header: %s\\n", hdr);
+  }
+  curl_easy_cleanup(curl);
+}
+.fi
+.SH AVAILABILITY
+Added in 7.76.0
+.SH RETURN VALUE
+Returns CURLE_OK if the option is supported, and CURLE_UNKNOWN_OPTION if not.
+.SH "SEE ALSO"
+.BR curl_easy_getinfo "(3), " curl_easy_setopt "(3), "
+.BR CURLOPT_REFERER "(3), "
diff --git a/docs/libcurl/opts/CURLOPT_REFERER.3 b/docs/libcurl/opts/CURLOPT_REFERER.3
@@ -5,7 +5,7 @@
 .\" *                            | (__| |_| |  _ <| |___
 .\" *                             \___|\___/|_| \_\_____|
 .\" *
-.\" * Copyright (C) 1998 - 2020, Daniel Stenberg, <daniel@haxx.se>, et al.
+.\" * Copyright (C) 1998 - 2021, Daniel Stenberg, <daniel@haxx.se>, et al.
 .\" *
 .\" * This software is licensed as described in the file COPYING, which
 .\" * you should have received as part of this distribution. The terms
@@ -58,3 +58,4 @@ Returns CURLE_OK if HTTP support is enabled, CURLE_UNKNOWN_OPTION if not, or
 CURLE_OUT_OF_MEMORY if there was insufficient heap space.
 .SH "SEE ALSO"
 .BR CURLOPT_USERAGENT "(3), " CURLOPT_HTTPHEADER "(3), "
+.BR CURLINFO_REFERER "(3), "
diff --git a/docs/libcurl/opts/Makefile.inc b/docs/libcurl/opts/Makefile.inc
@@ -5,7 +5,7 @@
 #                            | (__| |_| |  _ <| |___
 #                             \___|\___/|_| \_\_____|
 #
-# Copyright (C) 1998 - 2020, Daniel Stenberg, <daniel@haxx.se>, et al.
+# Copyright (C) 1998 - 2021, Daniel Stenberg, <daniel@haxx.se>, et al.
 #
 # This software is licensed as described in the file COPYING, which
 # you should have received as part of this distribution. The terms
@@ -64,6 +64,7 @@ man_MANS =                                      \
   CURLINFO_REDIRECT_TIME.3                      \
   CURLINFO_REDIRECT_TIME_T.3                    \
   CURLINFO_REDIRECT_URL.3                       \
+  CURLINFO_REFERER.3                            \
   CURLINFO_REQUEST_SIZE.3                       \
   CURLINFO_RESPONSE_CODE.3                      \
   CURLINFO_RETRY_AFTER.3                        \

diff --git a/docs/libcurl/symbols-in-versions b/docs/libcurl/symbols-in-versions
@@ -271,6 +271,7 @@ CURLINFO_REDIRECT_COUNT         7.9.7
 CURLINFO_REDIRECT_TIME          7.9.7
 CURLINFO_REDIRECT_TIME_T        7.61.0
 CURLINFO_REDIRECT_URL           7.18.2
+CURLINFO_REFERER                7.76.0
 CURLINFO_REQUEST_SIZE           7.4.1
 CURLINFO_RESPONSE_CODE          7.10.8
 CURLINFO_RETRY_AFTER            7.66.0

diff --git a/include/curl/curl.h b/include/curl/curl.h
@@ -2761,8 +2761,9 @@ typedef enum {
   CURLINFO_RETRY_AFTER      = CURLINFO_OFF_T + 57,
   CURLINFO_EFFECTIVE_METHOD = CURLINFO_STRING + 58,
   CURLINFO_PROXY_ERROR      = CURLINFO_LONG + 59,
+  CURLINFO_REFERER          = CURLINFO_STRING + 60,
 
-  CURLINFO_LASTONE          = 59
+  CURLINFO_LASTONE          = 60
 } CURLINFO;
 
 /* CURLINFO_RESPONSE_CODE is the new name for the option previously known as

diff --git a/lib/getinfo.c b/lib/getinfo.c
@@ -145,6 +145,10 @@ static CURLcode getinfo_char(struct Curl_easy *data, CURLINFO info,
        option had been enabled! */
     *param_charp = data->info.wouldredirect;
     break;
+  case CURLINFO_REFERER:
+    /* Return the referrer header for this request, or NULL if unset */
+    *param_charp = data->change.referer;
+    break;
   case CURLINFO_PRIMARY_IP:
     /* Return the ip address of the most recent (primary) connection */
     *param_charp = data->info.conn_primary_ip;

diff --git a/packages/OS400/README.OS400 b/packages/OS400/README.OS400
@@ -178,6 +178,7 @@ use:
         CURLINFO_CONTENT_TYPE
         CURLINFO_FTP_ENTRY_PATH
         CURLINFO_REDIRECT_URL
+        CURLINFO_REFERER
         CURLINFO_PRIMARY_IP
         CURLINFO_RTSP_SESSION_ID
         CURLINFO_LOCAL_IP

diff --git a/packages/OS400/curl.inc.in b/packages/OS400/curl.inc.in
@@ -1690,6 +1690,8 @@
      d                 c                   X'0010003A'
      d  CURLINFO_PROXY_ERROR...                                                 CURLINFO_LONG + 59
      d                 c                   X'0020003B'
+     d  CURLINFO_REFERER...                                                     CURLINFO_STRING + 60
+     d                 c                   X'0010003C'
       *
      d  CURLINFO_HTTP_CODE...                                                   Old ...RESPONSE_CODE
      d                 c                   X'00200002'

diff --git a/src/tool_writeout.c b/src/tool_writeout.c
@@ -83,6 +83,7 @@ static const struct writeoutvar variables[] = {
   {"proxy_ssl_verify_result", VAR_PROXY_SSL_VERIFY_RESULT,
    CURLINFO_PROXY_SSL_VERIFYRESULT, writeLong},
   {"redirect_url", VAR_REDIRECT_URL, CURLINFO_REDIRECT_URL, writeString},
+  {"referer", VAR_REFERER, CURLINFO_REFERER, writeString},
   {"remote_ip", VAR_PRIMARY_IP, CURLINFO_PRIMARY_IP, writeString},
   {"remote_port", VAR_PRIMARY_PORT, CURLINFO_PRIMARY_PORT, writeLong},
   {"response_code", VAR_HTTP_CODE, CURLINFO_RESPONSE_CODE, writeLong},

diff --git a/src/tool_writeout.h b/src/tool_writeout.h
@@ -7,7 +7,7 @@
  *                            | (__| |_| |  _ <| |___
  *                             \___|\___/|_| \_\_____|
  *
- * Copyright (C) 1998 - 2020, Daniel Stenberg, <daniel@haxx.se>, et al.
+ * Copyright (C) 1998 - 2021, Daniel Stenberg, <daniel@haxx.se>, et al.
  *
  * This software is licensed as described in the file COPYING, which
  * you should have received as part of this distribution. The terms
@@ -54,6 +54,7 @@ typedef enum {
   VAR_REDIRECT_COUNT,
   VAR_REDIRECT_TIME,
   VAR_REDIRECT_URL,
+  VAR_REFERER,
   VAR_REQUEST_SIZE,
   VAR_SCHEME,
   VAR_SIZE_DOWNLOAD,

diff --git a/src/tool_xattr.c b/src/tool_xattr.c
@@ -5,7 +5,7 @@
  *                            | (__| |_| |  _ <| |___
  *                             \___|\___/|_| \_\_____|
  *
- * Copyright (C) 1998 - 2020, Daniel Stenberg, <daniel@haxx.se>, et al.
+ * Copyright (C) 1998 - 2021, Daniel Stenberg, <daniel@haxx.se>, et al.
  *
  * This software is licensed as described in the file COPYING, which
  * you should have received as part of this distribution. The terms
@@ -44,9 +44,10 @@ static const struct xattr_mapping {
   /* mappings proposed by
    * https://freedesktop.org/wiki/CommonExtendedAttributes/
    */
-  { "user.xdg.origin.url", CURLINFO_EFFECTIVE_URL },
-  { "user.mime_type",      CURLINFO_CONTENT_TYPE },
-  { NULL,                  CURLINFO_NONE } /* last element, abort loop here */
+  { "user.xdg.referrer.url", CURLINFO_REFERER },
+  { "user.xdg.origin.url",   CURLINFO_EFFECTIVE_URL },
+  { "user.mime_type",        CURLINFO_CONTENT_TYPE },
+  { NULL,                    CURLINFO_NONE } /* last element, abort here */
 };
 
 /* returns TRUE if a new URL is returned, that then needs to be freed */

diff --git a/tests/data/test1067 b/tests/data/test1067
@@ -53,7 +53,7 @@ http
 HTTP Location: following with auto-referer
  </name>
  <command>
-http://%HOSTIP:%HTTPPORT/want/1067 -L --referer "firstone.html;auto"
+http://%HOSTIP:%HTTPPORT/want/1067 --silent --location --referer "firstone.html;auto" --write-out "%{stderr}|%{referer}|"
 </command>
 </client>
 
@@ -73,5 +73,8 @@ Accept: */*
 Referer: http://%HOSTIP:%HTTPPORT/want/1067
 
 </protocol>
+<stderr nonewline="yes">
+|http://%HOSTIP:%HTTPPORT/want/1067|
+</stderr>
 </verify>
 </testcase>
diff --git a/tests/data/test970 b/tests/data/test970
@@ -59,7 +59,7 @@ Accept: */*
 
 </protocol>
 <stdout nonewline="yes">
-{"content_type":"text/html","errormsg":null,"exitcode":0,"filename_effective":"log/out970","ftp_entry_path":null,"http_code":200,"http_connect":000,"http_version":"1.1","local_ip":"127.0.0.1","local_port":13,"method":"GET","num_connects":1,"num_headers":9,"num_redirects":0,"proxy_ssl_verify_result":0,"redirect_url":null,"remote_ip":"%HOSTIP","remote_port":%HTTPPORT,"response_code":200,"scheme":"HTTP","size_download":445,"size_header":4019,"size_request":4019,"size_upload":0,"speed_download":13,"speed_upload":13,"ssl_verify_result":0,"time_appconnect":0.000013,"time_connect":0.000013,"time_namelookup":0.000013,"time_pretransfer":0.000013,"time_redirect":0.000013,"time_starttransfer":0.000013,"time_total":0.000013,"url":"http://%HOSTIP:%HTTPPORT/970","url_effective":"http://%HOSTIP:%HTTPPORT/970","urlnum":0,"curl_version":"curl-unit-test-fake-version"}
+{"content_type":"text/html","errormsg":null,"exitcode":0,"filename_effective":"log/out970","ftp_entry_path":null,"http_code":200,"http_connect":000,"http_version":"1.1","local_ip":"127.0.0.1","local_port":13,"method":"GET","num_connects":1,"num_headers":9,"num_redirects":0,"proxy_ssl_verify_result":0,"redirect_url":null,"referer":null,"remote_ip":"%HOSTIP","remote_port":%HTTPPORT,"response_code":200,"scheme":"HTTP","size_download":445,"size_header":4019,"size_request":4019,"size_upload":0,"speed_download":13,"speed_upload":13,"ssl_verify_result":0,"time_appconnect":0.000013,"time_connect":0.000013,"time_namelookup":0.000013,"time_pretransfer":0.000013,"time_redirect":0.000013,"time_starttransfer":0.000013,"time_total":0.000013,"url":"http://%HOSTIP:%HTTPPORT/970","url_effective":"http://%HOSTIP:%HTTPPORT/970","urlnum":0,"curl_version":"curl-unit-test-fake-version"}
 </stdout>
 </verify>
 </testcase>