Permalink
Browse files

James Gallagher's Content-Encoding work

  • Loading branch information...
1 parent 2e8a941 commit 64bbe9dfafc6693a96b742f3133c636385835a19 @bagder bagder committed Sep 2, 2002
Showing with 433 additions and 27 deletions.
  1. +7 −0 CHANGES
  2. +1 −2 Makefile.am
  3. +34 −0 UPGRADE
  4. +3 −0 acconfig.h
  5. +15 −14 configure.in
  6. +1 −0 docs/THANKS
  7. +6 −0 include/curl/curl.h
  8. +4 −2 lib/Makefile.am
  9. +53 −0 lib/README.encoding
  10. +122 −0 lib/content_encoding.c
  11. +34 −0 lib/content_encoding.h
  12. +3 −0 lib/http.c
  13. +28 −1 lib/http_chunks.c
  14. +1 −0 lib/http_chunks.h
  15. +67 −3 lib/transfer.c
  16. +16 −1 lib/url.c
  17. +22 −0 lib/urldata.h
  18. +1 −1 src/Makefile.am
  19. +15 −3 src/main.c
View
@@ -6,6 +6,13 @@
Changelog
+Daniel (2 Sep 2002)
+- James Gallagher added Content-Encoding support to libcurl so now curl and
+ libcurl-using apps can request compressed contents using the 'deflate'
+ method. See the special file lib/README.encoding for details.
+
+ curl --compressed is now used to request compressed contents.
+
Daniel (30 Aug 2002)
- Applied an anonymous SOCKS5-proxy patch. Not properly working in all
situations though, as all getaddrinfo()-using libcurls will fail on this.
View
@@ -4,8 +4,7 @@
AUTOMAKE_OPTIONS = foreign
-EXTRA_DIST = \
- CHANGES LEGAL maketgz MITX.txt MPL-1.1.txt \
+EXTRA_DIST = CHANGES LEGAL maketgz MITX.txt MPL-1.1.txt UPGRADE \
reconf Makefile.dist curl-config.in build_vms.com curl-mode.el
bin_SCRIPTS = curl-config
View
34 UPGRADE
@@ -0,0 +1,34 @@
+Upgrading to curl/libcurl 7.10 from any previous version
+========================================================
+
+libcurl 7.10 performs peer SSL certificate verification by default. This is
+done by installing a default CA cert bundle on 'make install' (or similar),
+that is used by default on operations against SSL servers.
+
+Alas, if you use communicate with HTTPS servers using certifcates that are
+signed by CAs present in the bundle, you will not notice and changed behavior
+and you will seeminglessly get a higher security level on your SSL connections
+since you will make sure that the remote server really is who it claims to be.
+
+If the remote server uses a self-signed certificate, or if you don't install
+curl's CA cert bundle or if it uses a certificate signed by a CA that isn't
+included in the bundle, then you need to do one of the following:
+
+ 1. Tell libcurl to *not* verify the peer. With libcurl you disable with with
+ curl_easy_setopt(curl, CURLOPT_SSL_VERIFYPEER, FALSE);
+
+ With the curl command tool, you disable this with -k/--insecure.
+
+ 2. Get a CA certificate that can verify the remote server and use the proper
+ option to point out this CA cert for verification when connecting. For
+ libcurl hackers: curl_easy_setopt(curl, CURLOPT_CAPATH, capath);
+
+ With the curl command tool: --cacert [file]
+
+This upgrade procedure has been deemed The Right Thing even though it adds
+this extra trouble for some users, since it adds security to a majority of the
+SSL connections that previously weren't really secure.
+
+It turned out many people were using previous versions of curl/libcurl without
+realizing the need for the CA cert options to get truly secure SSL
+connections.
View
@@ -85,3 +85,6 @@
/* Define to disable TELNET */
#undef CURL_DISABLE_TELNET
+
+/* Define if you have zlib present */
+#undef HAVE_LIBZ
View
@@ -522,20 +522,21 @@ dnl NOTE: We *always* look for ZLIB headers & libraries, all this option
dnl does is change where we look (by adjusting LIBS and CPPFLAGS.)
dnl
-dnl AC_MSG_CHECKING(where to look for ZLIB)
-dnl if test X"$OPT_ZLIB" = Xno
-dnl then
-dnl AC_MSG_RESULT([defaults (or given in environment)])
-dnl else
-dnl test X"$OPT_ZLIB" = Xyes && OPT_ZLIB=/usr/local
-dnl LIBS="$LIBS -L$OPT_ZLIB/lib"
-dnl CPPFLAGS="$CPPFLAGS -I$OPT_ZLIB/include"
-dnl AC_MSG_RESULT([$OPT_ZLIB])
-dnl fi
-
-dnl z lib?
-dnl AC_CHECK_FUNC(gzread, , AC_CHECK_LIB(z, gzread))
+AC_MSG_CHECKING(where to look for ZLIB)
+if test X"$OPT_ZLIB" = Xno
+then
+ AC_MSG_RESULT([defaults (or given in environment)])
+else
+ test X"$OPT_ZLIB" = Xyes && OPT_ZLIB=/usr/local
+ LIBS="$LIBS -L$OPT_ZLIB/lib"
+ CPPFLAGS="$CPPFLAGS -I$OPT_ZLIB/include"
+ AC_MSG_RESULT([$OPT_ZLIB])
+fi
+dnl AC_CHECK_FUNC(gzread, , AC_CHECK_LIB(z, gzread))
+AC_CHECK_LIB(z, gzread, [AM_CONDITIONAL(CONTENT_ENCODING, true)
+ AC_DEFINE(HAVE_LIBZ)
+ LIBS="$LIBS -lz"])
dnl Default is to try the thread-safe versions of a few functions
OPT_THREAD=on
@@ -606,7 +607,7 @@ AC_CHECK_HEADERS( \
)
dnl Check for libz header
-dnl AC_CHECK_HEADERS(zlib.h)
+AC_CHECK_HEADERS(zlib.h)
dnl Checks for typedefs, structures, and compiler characteristics.
AC_C_CONST
View
@@ -81,3 +81,4 @@ that have contributed with non-trivial parts:
- G�tz Babin-Ebell <babin�ebell@trustcenter.de>
- Andreas Damm <andreas-sourceforge@radab.org>
- Jacky Lam <sylam@emsoftltd.com>
+ - James Gallagher <jgallagher@gso.uri.edu>
View
@@ -200,6 +200,7 @@ typedef enum {
CURLE_SSL_CERTPROBLEM, /* 58 - problem with the local certificate */
CURLE_SSL_CIPHER, /* 59 - couldn't use specified cipher */
CURLE_SSL_CACERT, /* 60 - problem with the CA cert (path?) */
+ CURLE_BAD_CONTENT_ENCODING, /* 61 - Unrecognized transfer encoding */
CURL_LAST /* never use! */
} CURLcode;
@@ -585,6 +586,11 @@ typedef enum {
CURLPROXY_SOCKS4 and CURLPROXY_SOCKS5. */
CINIT(PROXYTYPE, LONG, 101),
+ /* Set the Accept-Encoding string. Use this to tell a server you would like
+ the response to be compressed. */
+ CINIT(ENCODING, OBJECTPOINT, 102),
+
+
CURLOPT_LASTENTRY /* the last unused */
} CURLoption;
View
@@ -7,7 +7,7 @@ AUTOMAKE_OPTIONS = foreign nostdinc
EXTRA_DIST = getdate.y Makefile.b32 Makefile.b32.resp Makefile.m32 \
Makefile.vc6 Makefile.riscos libcurl.def dllinit.c curllib.dsp \
curllib.dsw config-vms.h config-win32.h config-riscos.h config-mac.h \
- config.h.in ca-bundle.crt
+ config.h.in ca-bundle.crt README.encoding
lib_LTLIBRARIES = libcurl.la
@@ -45,6 +45,8 @@ libcurl_la_LDFLAGS = -version-info 2:2:0
# set age to 0.
#
+# I could not get conditional compilation using LIBADD or LDADD and
+# EXTRA_..._SOURCES to work. 08/28/02 jhrg
libcurl_la_SOURCES = \
arpa_telnet.h file.c getpass.h netrc.h timeval.c \
base64.c file.h hostip.c progress.c timeval.h \
@@ -60,7 +62,7 @@ escape.h getpass.c netrc.c telnet.h \
getinfo.c getinfo.h transfer.c strequal.c strequal.h easy.c \
security.h security.c krb4.c krb4.h memdebug.c memdebug.h inet_ntoa_r.h \
http_chunks.c http_chunks.h strtok.c strtok.h connect.c connect.h \
-llist.c llist.h hash.c hash.h multi.c share.c share.h
+llist.c llist.h hash.c hash.h multi.c content_encoding.c
noinst_HEADERS = setup.h transfer.h
View
@@ -0,0 +1,53 @@
+
+ Content Encoding Support for libcurl
+
+* About content encodings:
+
+HTTP/1.1 [RFC 2616] specifies that a client may request that a server encode
+its response. This is usually used to compress a response using one of a set
+of commonly available compression techniques. These schemes are `deflate'
+(the zlib algorithm), `gzip' and `compress' [sec 3.5, RFC 2616]. A client
+requests that the sever perform an encoding by including an Accept-Encoding
+header in the request document. The value of the header should be one of the
+recognized tokens `deflate', ... (there's a way to register new
+schemes/tokens, see sec 3.5 of the spec). A server MAY honor the client's
+encoding request. When a response is encoded, the server includes a
+Content-Encoding header in the response. The value of the Content-Encoding
+header indicates which scheme was used to encode the data.
+
+A client may tell a server that it can understand several different encoding
+schemes. In this case the server may choose any one of those and use it to
+encode the response (indicating which one using the Content-Encoding header).
+It's also possible for a client to attach priorities to different schemes so
+that the server knows which it prefers. See sec 14.3 of RFC 2616 for more
+information on the Accept-Encoding header.
+
+* Current support for content encoding:
+
+I added support for the 'deflate' content encoding to both libcurl and curl.
+Both regular and chunked transfers should work although I've tested only the
+former. The library zlib is required for this feature. Places where I
+modified the source code are commented and typically include my initials and
+the date (e.g., 08/29/02 jhrg).
+
+* The libcurl interface:
+
+To cause libcurl to request a content encoding use:
+
+ curl_easy_setopt(curl, CURLOPT_ENCODING, <string>)
+
+where <string> is the intended value of the Accept-Encoding header.
+
+Currently, libcurl only understands how to process responses that use the
+`deflate' Content-Encoding, so the only value for CURLOPT_ENCODING that will
+work (besides "identity," which does nothing) is "deflate." If a response is
+encoded using either the `gzip' or `compress' methods, libcurl will return an
+error indicating that the response could not be decoded. If <string> is null
+or empty no Accept-Encoding header is generated.
+
+* The curl interface:
+
+Use the --compressed option with curl to cause it to ask servers to compress
+responses using deflate.
+
+James Gallagher <jgallagher@gso.uri.edu>
View
@@ -0,0 +1,122 @@
+/*****************************************************************************
+ * _ _ ____ _
+ * Project ___| | | | _ \| |
+ * / __| | | | |_) | |
+ * | (__| |_| | _ <| |___
+ * \___|\___/|_| \_\_____|
+ *
+ * Copyright (C) 1998 - 2002, Daniel Stenberg, <daniel@haxx.se>, et al.
+ *
+ * In order to be useful for every potential user, curl and libcurl are
+ * dual-licensed under the MPL and the MIT/X-derivate licenses.
+ *
+ * You may opt to use, copy, modify, merge, publish, distribute and/or sell
+ * copies of the Software, and permit persons to whom the Software is
+ * furnished to do so, under the terms of the MPL or the MIT/X-derivate
+ * licenses. You may pick one of these licenses.
+ *
+ * This software is distributed on an "AS IS" basis, WITHOUT WARRANTY OF ANY
+ * KIND, either express or implied.
+ *
+ * $Id$
+ *****************************************************************************/
+
+#include "setup.h"
+
+#ifdef HAVE_LIBZ
+
+#include "urldata.h"
+#include <curl/curl.h>
+#include <curl/types.h>
+#include "sendf.h"
+
+#define DSIZ 4096 /* buffer size for decompressed data */
+
+
+static CURLcode
+process_zlib_error(struct SessionHandle *data, z_stream *z)
+{
+ if (z->msg)
+ failf (data, "Error while processing content unencoding.\n%s",
+ z->msg);
+ else
+ failf (data, "Error while processing content unencoding.\n"
+ "Unknown failure within decompression software.");
+
+ return CURLE_BAD_CONTENT_ENCODING;
+}
+
+static CURLcode
+exit_zlib(z_stream *z, bool *zlib_init, CURLcode result)
+{
+ inflateEnd(z);
+ *zlib_init = 0;
+ return result;
+}
+
+CURLcode
+Curl_unencode_deflate_write(struct SessionHandle *data,
+ struct Curl_transfer_keeper *k,
+ ssize_t nread)
+{
+ int status; /* zlib status */
+ int result; /* Curl_client_write status */
+ char decomp[DSIZ]; /* Put the decompressed data here. */
+ z_stream *z = &k->z; /* zlib state structure */
+
+ /* Initialize zlib? */
+ if (!k->zlib_init) {
+ z->zalloc = (alloc_func)Z_NULL;
+ z->zfree = (free_func)Z_NULL;
+ z->opaque = 0; /* of dubious use 08/27/02 jhrg */
+ if (inflateInit(z) != Z_OK)
+ return process_zlib_error(data, z);
+ k->zlib_init = 1;
+ }
+
+ /* Set the compressed input when this fucntion is called */
+ z->next_in = (Bytef *)k->str;
+ z->avail_in = nread;
+
+ /* because the buffer size is fixed, iteratively decompress
+ and transfer to the client via client_write. */
+ for (;;) {
+ /* (re)set buffer for decompressed output for every iteration */
+ z->next_out = (Bytef *)&decomp[0];
+ z->avail_out = DSIZ;
+
+ status = inflate(z, Z_SYNC_FLUSH);
+ if (status == Z_OK || status == Z_STREAM_END) {
+ result = Curl_client_write(data, CLIENTWRITE_BODY, decomp,
+ DSIZ - z->avail_out);
+ /* if !CURLE_OK, clean up, return */
+ if (result) {
+ return exit_zlib(z, &k->zlib_init, result);
+ }
+
+ /* Done?; clean up, return */
+ if (status == Z_STREAM_END) {
+ if (inflateEnd(z) == Z_OK)
+ return exit_zlib(z, &k->zlib_init, result);
+ else
+ return exit_zlib(z, &k->zlib_init, process_zlib_error(data, z));
+ }
+
+ /* Done with these bytes, exit */
+ if (status == Z_OK && z->avail_in == 0 && z->avail_out > 0)
+ return result;
+ }
+ else { /* Error; exit loop, handle below */
+ return exit_zlib(z, &k->zlib_init, process_zlib_error(data, z));
+ }
+ }
+}
+#endif /* HAVE_LIBZ */
+
+/*
+ * local variables:
+ * eval: (load-file "../curl-mode.el")
+ * end:
+ * vim600: fdm=marker
+ * vim: et sw=2 ts=2 sts=2 tw=78
+ */
View
@@ -0,0 +1,34 @@
+/*****************************************************************************
+ * _ _ ____ _
+ * Project ___| | | | _ \| |
+ * / __| | | | |_) | |
+ * | (__| |_| | _ <| |___
+ * \___|\___/|_| \_\_____|
+ *
+ * Copyright (C) 1998 - 2002, Daniel Stenberg, <daniel@haxx.se>, et al.
+ *
+ * In order to be useful for every potential user, curl and libcurl are
+ * dual-licensed under the MPL and the MIT/X-derivate licenses.
+ *
+ * You may opt to use, copy, modify, merge, publish, distribute and/or sell
+ * copies of the Software, and permit persons to whom the Software is
+ * furnished to do so, under the terms of the MPL or the MIT/X-derivate
+ * licenses. You may pick one of these licenses.
+ *
+ * This software is distributed on an "AS IS" basis, WITHOUT WARRANTY OF ANY
+ * KIND, either express or implied.
+ *
+ * $Id$
+ *****************************************************************************/
+
+CURLcode Curl_unencode_deflate_write(struct SessionHandle *data,
+ struct Curl_transfer_keeper *k,
+ ssize_t nread);
+
+/*
+ * local variables:
+ * eval: (load-file "../curl-mode.el")
+ * end:
+ * vim600: fdm=marker
+ * vim: et sw=2 ts=2 sts=2 tw=78
+ */
View
@@ -707,6 +707,7 @@ CURLcode Curl_http(struct connectdata *conn)
"%s" /* host */
"%s" /* pragma */
"%s" /* accept */
+ "%s" /* accept-encoding */
"%s", /* referer */
data->set.customrequest?data->set.customrequest:
@@ -727,6 +728,8 @@ CURLcode Curl_http(struct connectdata *conn)
(conn->allocptr.host?conn->allocptr.host:""), /* Host: host */
http->p_pragma?http->p_pragma:"",
http->p_accept?http->p_accept:"",
+ (data->set.encoding && *data->set.encoding && conn->allocptr.accept_encoding)?
+ conn->allocptr.accept_encoding:"", /* 08/28/02 jhrg */
(data->change.referer && conn->allocptr.ref)?conn->allocptr.ref:"" /* Referer: <data> <CRLF> */
);
Oops, something went wrong.

0 comments on commit 64bbe9d

Please sign in to comment.