Add brotli decompression support for HTTP and HTTP2 dissectors.

Change-Id: I9c09f55673187f6fee723fcd72798fb6b9958b03
Reviewed-on: https://code.wireshark.org/review/32745
Petri-Dish: Peter Wu <peter@lekensteyn.nl>
Tested-by: Petri Dish Buildbot
Reviewed-by: Peter Wu <peter@lekensteyn.nl>

CMakeLists.txt

@@ -1104,6 +1104,9 @@ ws_find_package(CARES ENABLE_CARES HAVE_C_ARES "1.5.0")
 # Zlib compression
 ws_find_package(ZLIB ENABLE_ZLIB HAVE_ZLIB)
 
+# Brotli compression
+ws_find_package(BROTLI ENABLE_BROTLI HAVE_BROTLI)
+
 # LZ4 compression
 ws_find_package(LZ4 ENABLE_LZ4 HAVE_LZ4)
 
@@ -1809,6 +1812,11 @@ if(WIN32)
 		list (APPEND OPTIONAL_DLLS "${ZLIB_DLL_DIR}/${ZLIB_DLL}")
 		list (APPEND OPTIONAL_PDBS "${ZLIB_DLL_DIR}/${ZLIB_PDB}")
 	endif(ZLIB_FOUND)
+	if (BROTLI_FOUND)
+		foreach( _dll ${BROTLI_DLLS} )
+			list (APPEND OPTIONAL_DLLS "${BROTLI_DLL_DIR}/${_dll}")
+		endforeach(_dll)
+	endif(BROTLI_FOUND)
 
 	# With libs downloaded to c:/wireshark-win64-libs this currently
 	# (early 2018) expands to about 1900 characters.
@@ -2756,6 +2764,9 @@ if(RPMBUILD_EXECUTABLE)
 	if (SYSTEMD_FOUND)
 		list(APPEND _rpmbuild_with_args --with sdjournal)
 	endif()
+	if (BROTLI_FOUND)
+		list(APPEND _rpmbuild_with_args --with brotli)
+	endif()
 
 	execute_process(
 		COMMAND ${PERL_EXECUTABLE}

CMakeOptions.txt

@@ -65,6 +65,7 @@ option(ENABLE_PCAP_NG_DEFAULT   "Enable pcapng as default file format" ON)
 
 option(ENABLE_ZLIB       "Build with zlib compression support" ON)
 option(ENABLE_LZ4        "Build with LZ4 compression support" ON)
+option(ENABLE_BROTLI     "Build with brotli compression support" ON)
 option(ENABLE_SNAPPY     "Build with Snappy compression support" ON)
 option(ENABLE_NGHTTP2    "Build with HTTP/2 header decompression support" ON)
 option(ENABLE_LUA        "Build with Lua dissector support" ON)

INSTALL

@@ -153,6 +153,13 @@ README.windows for those instructions.
         the capture-file utilities that come in this package, use
         this switch.
 
+    -DENABLE_BROTLI=OFF
+        By default, if 'configure' finds brotli, the wiretap library
+        will be built so that it can read brotli compressed capture
+        files. If you have brotli but do not wish to build it into
+        the wiretap library, used by Wireshark, TShark, and the
+        capture-file utilities that come in this package, use this switch.
+
     -DENABLE_PLUGINS=OFF
         By default, if your system can support run-time loadable modules,
         the packet analyzers are build with support for plugins.

cmake/modules/FindBROTLI.cmake

@@ -0,0 +1,50 @@
+#
+# - Find brotli
+# Find brotli includes and libraries
+#
+#  BROTLI_INCLUDE_DIRS - where to find brotli header files
+#  BROTLI_LIBRARIES    - List of libraries when using brotli.
+#  BROTLI_FOUND        - True if brotli found.
+#  BROTLI_DLL_DIR      - (Windows) Path to the brotli DLLs
+#  BROTLI_DLLS         - (Windows) Name of the brotli DLLs
+
+include( FindWSWinLibs )
+FindWSWinLibs( "brotli-.*" "BROTLI_HINTS" )
+
+find_path(BROTLI_INCLUDE_DIR
+  NAMES "brotli/decode.h"
+  HINTS "${BROTLI_HINTS}/include"
+)
+
+find_library(BROTLIDEC_LIBRARY
+  NAMES brotlidec
+  HINTS "${BROTLI_HINTS}/lib"
+)
+
+include(FindPackageHandleStandardArgs)
+find_package_handle_standard_args( BROTLI DEFAULT_MSG BROTLIDEC_LIBRARY BROTLI_INCLUDE_DIR )
+
+if( BROTLI_FOUND )
+  set( BROTLI_INCLUDE_DIRS ${BROTLI_INCLUDE_DIR} )
+  set( BROTLI_LIBRARIES ${BROTLIDEC_LIBRARY} )
+
+  if (WIN32)
+    set ( BROTLI_DLL_DIR "${BROTLI_HINTS}/bin"
+      CACHE PATH "Path to the brotli DLLs"
+    )
+    file( GLOB _brotli_dlls RELATIVE "${BROTLI_DLL_DIR}"
+      "${BROTLI_DLL_DIR}/brotlicommon*.dll"
+      "${BROTLI_DLL_DIR}/brotlidec*.dll"
+    )
+    set ( BROTLI_DLLS ${_brotli_dlls}
+      # We're storing filenames only. Should we use STRING instead?
+      CACHE FILEPATH "brotli DLL list"
+    )
+    mark_as_advanced( BROTLI_DLL_DIR BROTLI_DLLS )
+  endif()
+else()
+  set( BROTLI_INCLUDE_DIRS )
+  set( BROTLI_LIBRARIES )
+endif()
+
+mark_as_advanced( BROTLI_LIBRARIES BROTLI_INCLUDE_DIRS )

cmakeconfig.h.in

@@ -136,6 +136,9 @@
 /* Define to use zlib library */
 #cmakedefine HAVE_ZLIB 1
 
+/* Define to use brotli library */
+#cmakedefine HAVE_BROTLI 1
+
 /* Define to use lz4 library */
 #cmakedefine HAVE_LZ4 1
 

debian/control

@@ -17,7 +17,7 @@ Build-Depends: lsb-release,
  libmaxminddb-dev, dpkg-dev (>= 1.16.1~), libsystemd-dev | libsystemd-journal-dev,
  libnl-genl-3-dev [linux-any], libnl-route-3-dev [linux-any], asciidoctor,
  cmake (>= 3.5) | cmake3, libsbc-dev, libnghttp2-dev, libssh-gcrypt-dev,
- liblz4-dev, libsnappy-dev, libspandsp-dev, libxml2-dev
+ liblz4-dev, libsnappy-dev, libspandsp-dev, libxml2-dev, libbrotli-dev
 Build-Conflicts: libsnmp4.2-dev, libsnmp-dev
 Vcs-Svn: svn://svn.debian.org/svn/collab-maint/ext-maint/wireshark/trunk
 Vcs-Browser: http://svn.debian.org/wsvn/collab-maint/ext-maint/wireshark/trunk/

debian/libwireshark0.symbols

@@ -1680,6 +1680,7 @@ libwireshark.so.0 libwireshark0 #MINVER#
  tvb_captured_length@Base 1.12.0~rc1
  tvb_captured_length_remaining@Base 1.12.0~rc1
  tvb_child_uncompress@Base 1.12.0~rc1
+ tvb_child_uncompress_brotli@Base 3.1.0
  tvb_clone@Base 1.12.0~rc1
  tvb_clone_offset_len@Base 1.12.0~rc1
  tvb_composite_append@Base 1.9.1
@@ -1801,6 +1802,7 @@ libwireshark.so.0 libwireshark0 #MINVER#
  tvb_strnlen@Base 1.9.1
  tvb_strsize@Base 1.9.1
  tvb_uncompress@Base 1.9.1
+ tvb_uncompress_brotli@Base 3.1.0
  tvb_unicode_strsize@Base 1.9.1
  tvb_ws_mempbrk_pattern_guint8@Base 1.99.3
  tvbparse_casestring@Base 1.9.1

docbook/release-notes.adoc

@@ -34,7 +34,7 @@ Features” section below for more details.
 The following features are new (or have been significantly updated)
 since version 3.0.0:
 
-* Watch this space.
+* Brotli decompression support in HTTP/HTTP2 (requires the brotli library).
 
 // === Removed Features and Support
 

epan/CMakeLists.txt

@@ -16,6 +16,7 @@ if (HAVE_HFI_SECTION_INIT)
 endif()
 
 include_directories(
+		${BROTLI_INCLUDE_DIRS}
 		${GLIB2_INCLUDE_DIRS}
 		${CARES_INCLUDE_DIRS}
 		${GCRYPT_INCLUDE_DIRS}
@@ -265,6 +266,7 @@ set(LIBWIRESHARK_NONGENERATED_FILES
 	tvbparse.c
 	tvbuff.c
 	tvbuff_base64.c
+	tvbuff_brotli.c
 	tvbuff_composite.c
 	tvbuff_real.c
 	tvbuff_subset.c
@@ -378,6 +380,7 @@ target_link_libraries(epan
 		wsutil
 		${GLIB2_LIBRARIES}
 	PRIVATE
+		${BROTLI_LIBRARIES}
 		${CARES_LIBRARIES}
 		${GCRYPT_LIBRARIES}
 		${GIO2_LIBRARIES}

epan/dissectors/packet-http.c

@@ -237,9 +237,9 @@ static gboolean http_desegment_body = TRUE;
 static gboolean http_dechunk_body = TRUE;
 
 /*
- * Decompression of zlib encoded entities.
+ * Decompression of zlib or brotli encoded entities.
  */
-#ifdef HAVE_ZLIB
+#if defined(HAVE_ZLIB) || defined(HAVE_BROTLI)
 static gboolean http_decompress_body = TRUE;
 #else
 static gboolean http_decompress_body = FALSE;
@@ -1704,6 +1704,7 @@ dissect_http_message(tvbuff_t *tvb, int offset, packet_info *pinfo,
 			proto_item *e_ti = NULL;
 			proto_tree *e_tree = NULL;
 
+#ifdef HAVE_ZLIB
 			if (http_decompress_body &&
 			    (g_ascii_strcasecmp(headers.content_encoding, "gzip") == 0 ||
 			     g_ascii_strcasecmp(headers.content_encoding, "deflate") == 0 ||
@@ -1713,6 +1714,16 @@ dissect_http_message(tvbuff_t *tvb, int offset, packet_info *pinfo,
 				uncomp_tvb = tvb_child_uncompress(tvb, next_tvb, 0,
 				    tvb_captured_length(next_tvb));
 			}
+#endif
+
+#ifdef HAVE_BROTLI
+			if (http_decompress_body &&
+			    g_ascii_strcasecmp(headers.content_encoding, "br") == 0)
+			{
+				uncomp_tvb = tvb_child_uncompress_brotli(tvb, next_tvb, 0,
+				    tvb_captured_length(next_tvb));
+			}
+#endif
 
 			/*
 			 * Add the encoded entity to the protocol tree
@@ -4017,7 +4028,7 @@ proto_register_http(void)
 	    "Whether to reassemble bodies of entities that are transferred "
 	    "using the \"Transfer-Encoding: chunked\" method",
 	    &http_dechunk_body);
-#ifdef HAVE_ZLIB
+#if defined(HAVE_ZLIB) || defined(HAVE_BROTLI)
 	prefs_register_bool_preference(http_module, "decompress_body",
 	    "Uncompress entity bodies",
 	    "Whether to uncompress entity bodies that are compressed "

epan/dissectors/packet-http2.c

@@ -65,9 +65,9 @@ VALUE_STRING_ENUM(http2_header_repr_type);
 VALUE_STRING_ARRAY(http2_header_repr_type);
 
 /*
- * Decompression of zlib encoded entities.
+ * Decompression of zlib or brotli encoded entities.
  */
-#ifdef HAVE_ZLIB
+#if defined(HAVE_ZLIB) || defined(HAVE_BROTLI)
 static gboolean http2_decompress_body = TRUE;
 #else
 static gboolean http2_decompress_body = FALSE;
@@ -1985,18 +1985,36 @@ dissect_frame_prio(tvbuff_t *tvb, proto_tree *http2_tree, guint offset, guint8 f
 }
 
 #ifdef HAVE_NGHTTP2
-static int
-can_uncompress_body(packet_info *pinfo)
+enum body_uncompression {
+    BODY_UNCOMPRESSION_NONE,
+    BODY_UNCOMPRESSION_ZLIB,
+    BODY_UNCOMPRESSION_BROTLI
+};
+
+static enum body_uncompression
+get_body_uncompression_info(packet_info *pinfo)
 {
     http2_data_stream_body_info_t *body_info = get_data_stream_body_info(pinfo);
     gchar *content_encoding = body_info->content_encoding;
 
     /* Check we have a content-encoding header appropriate as well as checking if this is partial content.
      * We can't decompress part of a gzip encoded entity */
-    return http2_decompress_body
-           && body_info->is_partial_content == FALSE
-           && content_encoding != NULL
-           && (strncmp(content_encoding, "gzip", 4) == 0 || strncmp(content_encoding, "deflate", 7) == 0);
+    if (!http2_decompress_body || body_info->is_partial_content == TRUE || content_encoding == NULL) {
+        return BODY_UNCOMPRESSION_NONE;
+    }
+
+#ifdef HAVE_ZLIB
+    if (strncmp(content_encoding, "gzip", 4) == 0 || strncmp(content_encoding, "deflate", 7) == 0) {
+        return BODY_UNCOMPRESSION_ZLIB;
+    }
+#endif
+#ifdef HAVE_BROTLI
+    if (strncmp(content_encoding, "br", 2) == 0) {
+        return BODY_UNCOMPRESSION_BROTLI;
+    }
+#endif
+
+    return BODY_UNCOMPRESSION_NONE;
 }
 
 /* Try to dissect reassembled http2.data.data according to content_type. */
@@ -2027,9 +2045,17 @@ dissect_http2_data_full_body(tvbuff_t *tvb, packet_info *pinfo, proto_tree *http
 
     gint datalen = tvb_reported_length(tvb);
 
-    if (can_uncompress_body(pinfo)) {
+    enum body_uncompression uncompression = get_body_uncompression_info(pinfo);
+    if (uncompression != BODY_UNCOMPRESSION_NONE) {
         proto_item *compressed_proto_item = NULL;
-        tvbuff_t *uncompressed_tvb = tvb_child_uncompress(tvb, tvb, 0, datalen);
+
+        tvbuff_t *uncompressed_tvb = NULL;
+        if (uncompression == BODY_UNCOMPRESSION_ZLIB) {
+            uncompressed_tvb = tvb_child_uncompress(tvb, tvb, 0, datalen);
+        } else if (uncompression == BODY_UNCOMPRESSION_BROTLI) {
+            uncompressed_tvb = tvb_child_uncompress_brotli(tvb, tvb, 0, datalen);
+        }
+
         http2_data_stream_body_info_t *body_info = get_data_stream_body_info(pinfo);
         gchar *compression_method = body_info->content_encoding;
 

epan/epan.c

@@ -80,6 +80,10 @@
 #include <nghttp2/nghttp2ver.h>
 #endif
 
+#ifdef HAVE_BROTLI
+#include <brotli/decode.h>
+#endif
+
 #ifdef HAVE_LIBXML2
 #include <libxml/xmlversion.h>
 #include <libxml/parser.h>
@@ -769,6 +773,14 @@ epan_get_compiled_version_info(GString *str)
 	g_string_append(str, "without nghttp2");
 #endif /* HAVE_NGHTTP2 */
 
+	/* brotli */
+	g_string_append(str, ", ");
+#ifdef HAVE_BROTLI
+	g_string_append(str, "with brotli");
+#else
+	g_string_append(str, "without brotli");
+#endif /* HAVE_BROTLI */
+
 	/* LZ4 */
 	g_string_append(str, ", ");
 #ifdef HAVE_LZ4
@@ -808,6 +820,12 @@ epan_get_runtime_version_info(GString *str)
 
 	/* Gcrypt */
 	g_string_append_printf(str, ", with Gcrypt %s", gcry_check_version(NULL));
+
+	/* brotli */
+#ifdef HAVE_BROTLI
+	g_string_append_printf(str, ", with brotli %d.%d.%d", BrotliDecoderVersion() >> 24,
+		(BrotliDecoderVersion() >> 12) & 0xFFF, BrotliDecoderVersion() & 0xFFF);
+#endif
 }
 
 /*

epan/tvbuff.h

@@ -907,6 +907,24 @@ WS_DLL_PUBLIC tvbuff_t *tvb_uncompress(tvbuff_t *tvb, const int offset,
 WS_DLL_PUBLIC tvbuff_t *tvb_child_uncompress(tvbuff_t *parent, tvbuff_t *tvb,
     const int offset, int comprlen);
 
+/* From tvbuff_brotli.c */
+
+/**
+ * Uncompresses a brotli compressed packet inside a tvbuff at offset with
+ * length comprlen.  Returns an uncompressed tvbuffer if uncompression
+ * succeeded or NULL if uncompression failed.
+ */
+WS_DLL_PUBLIC tvbuff_t *tvb_uncompress_brotli(tvbuff_t *tvb, const int offset,
+    int comprlen);
+
+/**
+ * Uncompresses a brotli compressed packet inside a tvbuff at offset with
+ * length comprlen.  Returns an uncompressed tvbuffer attached to tvb if
+ * uncompression succeeded or NULL if uncompression failed.
+ */
+WS_DLL_PUBLIC tvbuff_t *tvb_child_uncompress_brotli(tvbuff_t *parent, tvbuff_t *tvb,
+    const int offset, int comprlen);
+
 /* From tvbuff_base64.c */
 
 /** Return a tvb that contains the binary representation of a base64