compressor: Allow to optionally use zlib-ng

bazel/BUILD

@@ -243,6 +243,22 @@ config_setting(
     values = {"define": "boringssl=disabled"},
 )
 
+config_setting(
+    name = "zlib_ng",
+    constraint_values = [
+        "@bazel_tools//platforms:linux",
+    ],
+    values = {"define": "zlib=ng"},
+)
+
+config_setting(
+    name = "zlib_ng_with_optimization",
+    constraint_values = [
+        "@bazel_tools//platforms:linux",
+    ],
+    values = {"define": "zlib=ng-with-optimization"},
+)
+
 config_setting(
     name = "enable_quiche",
     values = {"define": "quiche=enabled"},
@@ -259,6 +275,28 @@ alias(
     }),
 )
 
+# Alias pointing to the selected version of zlib:
+# - zlib-ng from //bazel/foreign_cc:zlib_ng,
+# - zlib-ng with optimization from //bazel/foreign_cc:zlib_ng_with_optimization,
+# - zlib from //bazel/foreign_cc:zlib.
+alias(
+    name = "zlib",
+    actual = select({
+        "//bazel:zlib_ng": "//bazel/foreign_cc:zlib_ng",
+        "//bazel:zlib_ng_with_optimization": "//bazel/foreign_cc:zlib_ng_with_optimization",
+        "//conditions:default": "//bazel/foreign_cc:zlib",
+    }),
+)
+
+alias(
+    name = "curl",
+    actual = select({
+        "//bazel:zlib_ng": "//bazel/foreign_cc:curl_with_zlib_ng",
+        "//bazel:zlib_ng_with_optimization": "//bazel/foreign_cc:curl_with_zlib_ng_with_optimization",
+        "//conditions:default": "//bazel/foreign_cc:curl",
+    }),
+)
+
 config_setting(
     name = "linux_x86_64",
     values = {"cpu": "k8"},

bazel/envoy_build_system.bzl

@@ -78,6 +78,7 @@ def envoy_cmake_external(
         pdb_name = "",
         cmake_files_dir = "$BUILD_TMPDIR/CMakeFiles",
         generate_crosstool_file = False,
+        variants = [],
         **kwargs):
     cache_entries.update({"CMAKE_BUILD_TYPE": "Bazel"})
     cache_entries_debug = dict(cache_entries)
@@ -120,6 +121,34 @@ def envoy_cmake_external(
         **kwargs
     )
 
+    for variant in variants:
+        if "cache_entries" in variant:
+            cache_entries.update(variant["cache_entries"] or {})
+            cache_entries_debug = dict(cache_entries)
+            cache_entries_debug.update(debug_cache_entries)
+
+        if "lib_source" in variant:
+            lib_source = variant["lib_source"] or lib_source
+
+        cmake_external(
+            name = variant["name"],
+            cache_entries = select({
+                "@envoy//bazel:opt_build": cache_entries,
+                "//conditions:default": cache_entries_debug,
+            }),
+            cmake_options = cmake_options,
+            # TODO(lizan): Make this always true
+            generate_crosstool_file = select({
+                "@envoy//bazel:windows_x86_64": True,
+                "//conditions:default": generate_crosstool_file,
+            }),
+            lib_source = lib_source,
+            make_commands = make_commands,
+            postfix_script = pf,
+            static_libraries = static_libraries,
+            **kwargs
+        )
+
 # Used to select a dependency that has different implementations on POSIX vs Windows.
 # The platform-specific implementations should be specified with envoy_cc_posix_library
 # and envoy_cc_win32_library respectively

bazel/foreign_cc/BUILD

@@ -144,11 +144,27 @@ envoy_cmake_external(
         "//bazel:windows_x86_64": ["libcurl.lib"],
         "//conditions:default": ["libcurl.a"],
     }),
+    variants = [
+        {
+            "name": "curl_with_zlib_ng",
+            "cache_entries": {
+                "ZLIB_LIBRARY": "$EXT_BUILD_DEPS/zlib_ng",
+                "ZLIB_INCLUDE_DIR": "$EXT_BUILD_DEPS/zlib_ng/include",
+            },
+        },
+        {
+            "name": "curl_with_zlib_ng_with_optimization",
+            "cache_entries": {
+                "ZLIB_LIBRARY": "$EXT_BUILD_DEPS/zlib_ng_with_optimization",
+                "ZLIB_INCLUDE_DIR": "$EXT_BUILD_DEPS/zlib_ng_with_optimization/include",
+            },
+        },
+    ],
     deps = [
         ":ares",
         ":nghttp2",
-        ":zlib",
         "//external:ssl",
+        "//external:zlib",
     ],
 )
 
@@ -237,4 +253,42 @@ envoy_cmake_external(
         "//bazel:windows_x86_64": ["zlibstatic.lib"],
         "//conditions:default": ["libz.a"],
     }),
+    variants = [
+        {
+            "name": "zlib_ng",
+            "cache_entries": {
+                # zlib-ng build options. Reference: https://github.com/zlib-ng/zlib-ng#build-options.
+                "ZLIB_COMPAT": "on",
+                "ZLIB_ENABLE_TESTS": "off",
+
+                # Turning off WITH_OPTIM and WITH_NEW_STRATEGIES fixes checksum, but not sure if that is
+                # what we want.
+                "WITH_OPTIM": "off",
+                "WITH_NEW_STRATEGIES": "off",
+
+                # zlib-ng advance build options. Reference: https://github.com/zlib-ng/zlib-ng#advanced-build-options.
+                # Only allow aligned address.
+                "UNALIGNED_OK": "off",
+            },
+            "lib_source": "@com_github_zlib_ng_zlib_ng//:all",
+        },
+        {
+            "name": "zlib_ng_with_optimization",
+            "cache_entries": {
+                # zlib-ng build options. Reference: https://github.com/zlib-ng/zlib-ng#build-options.
+                "ZLIB_COMPAT": "on",
+                "ZLIB_ENABLE_TESTS": "off",
+
+                # Warning: Turning WITH_OPTIM to "on" doesn't pass ZlibCompressorImplTest.CallingChecksum.
+                "WITH_OPTIM": "on",
+                # Warning: Turning WITH_NEW_STRATEGIES to "on" doesn't pass gzip compressor fuzz test.
+                "WITH_NEW_STRATEGIES": "on",
+
+                # zlib-ng advance build options. Reference: https://github.com/zlib-ng/zlib-ng#advanced-build-options.
+                # Since UNALIGNED_OK is on by default, it is assumed to be one of the optimizations.
+                "UNALIGNED_OK": "on",
+            },
+            "lib_source": "@com_github_zlib_ng_zlib_ng//:all",
+        },
+    ],
 )

bazel/repositories.bzl

@@ -155,6 +155,23 @@ def envoy_dependencies(skip_targets = []):
         actual = "@envoy//bazel:boringssl",
     )
 
+    # Binding to an alias pointing to the selected version of zlib:
+    # - zlib-ng from @envoy//bazel/foreign_cc:zlib_ng,
+    # - zlib-ng with optimization from @envoy//bazel/foreign_cc:zlib_ng_with_optimization,
+    # - zlib from @envoy//bazel/foreign_cc:zlib.
+    _net_zlib()
+    _com_github_zlib_ng_zlib_ng()
+    native.bind(
+        name = "zlib",
+        actual = "@envoy//bazel:zlib",
+    )
+
+    # Bind for gRPC.
+    native.bind(
+        name = "madler_zlib",
+        actual = "@envoy//bazel:zlib",
+    )
+
     # The long repo names (`com_github_fmtlib_fmt` instead of `fmtlib`) are
     # semi-standard in the Bazel community, intended to avoid both duplicate
     # dependencies and name conflicts.
@@ -189,7 +206,6 @@ def envoy_dependencies(skip_targets = []):
     _com_googlesource_googleurl()
     _com_lightstep_tracer_cpp()
     _io_opentracing_cpp()
-    _net_zlib()
     _upb()
     _proxy_wasm_cpp_sdk()
     _proxy_wasm_cpp_host()
@@ -370,15 +386,10 @@ def _net_zlib():
         patches = ["@envoy//bazel/foreign_cc:zlib.patch"],
     )
 
-    native.bind(
-        name = "zlib",
-        actual = "@envoy//bazel/foreign_cc:zlib",
-    )
-
-    # Bind for grpc.
-    native.bind(
-        name = "madler_zlib",
-        actual = "@envoy//bazel/foreign_cc:zlib",
+def _com_github_zlib_ng_zlib_ng():
+    _repository_impl(
+        name = "com_github_zlib_ng_zlib_ng",
+        build_file_content = BUILD_ALL_CONTENT,
     )
 
 def _com_google_cel_cpp():
@@ -665,15 +676,15 @@ def _com_github_curl():
     http_archive(
         name = "com_github_curl",
         build_file_content = BUILD_ALL_CONTENT + """
-cc_library(name = "curl", visibility = ["//visibility:public"], deps = ["@envoy//bazel/foreign_cc:curl"])
+cc_library(name = "curl", visibility = ["//visibility:public"], deps = ["@envoy//bazel:curl"])
 """,
         patches = ["@envoy//bazel/foreign_cc:curl-revert-cmake-minreqver.patch"],
         patch_args = ["-p1"],
         **location
     )
     native.bind(
         name = "curl",
-        actual = "@envoy//bazel/foreign_cc:curl",
+        actual = "@envoy//bazel:curl",
     )
 
 def _com_googlesource_chromium_v8():

bazel/repository_locations.bzl

@@ -259,6 +259,14 @@ DEPENDENCY_REPOSITORIES = dict(
         use_category = ["dataplane"],
         cpe = "cpe:2.3:a:gnu:zlib:*",
     ),
+    com_github_zlib_ng_zlib_ng = dict(
+        sha256 = "9646501422faf7a0b3a1d82402a421c0d56c9f4e1926692b6ed0bd80042da209",
+        strip_prefix = "zlib-ng-c71ca170224f31a219513aa470f82aa50a3ded48",
+        # 2020-07-31 develop branch.
+        urls = ["https://github.com/zlib-ng/zlib-ng/archive/c71ca170224f31a219513aa470f82aa50a3ded48.tar.gz"],
+        use_category = ["dataplane"],
+        cpe = "N/A",
+    ),
     com_github_jbeder_yaml_cpp = dict(
         sha256 = "79ab7069ef1c7c3632e7ffe095f7185d4c77b64d8035db3c085c239d4fe96d5f",
         strip_prefix = "yaml-cpp-98acc5a8874faab28b82c28936f4b400b389f5d6",

docs/root/intro/arch_overview/other_features/compression.rst

@@ -0,0 +1,20 @@
+.. _arch_overview_compression:
+
+Compression
+===========
+
+Underlying implementation
+-------------------------
+
+Currently Envoy is written to use `zlib <http://zlib.net>`_ as the compression provider.
+
+.. note::
+
+  `zlib-ng <https://github.com/zlib-ng/zlib-ng>`_ is a fork that hosts several 3rd-party
+  contributions containing new optimizations. Those optimizations are considered useful for
+  `improving compression performance <https://github.com/envoyproxy/envoy/issues/8448#issuecomment-667152013>`_.
+  Envoy can be built with `zlib-ng <https://github.com/zlib-ng/zlib-ng>`_ support by using
+  ``--define zlib=ng`` Bazel option. In order to use `zlib-ng <https://github.com/zlib-ng/zlib-ng>`_
+  with optimization turned on, apply ``--define zlib=ng-with-optimization``. Please note that
+  building Envoy with ``ng-with-optimization`` means to have a different behavior on how it
+  accomplishes checksum. Currently, these options are only available on Linux.

docs/root/intro/arch_overview/other_features/other_features.rst

@@ -8,3 +8,4 @@ Other features
   global_rate_limiting
   scripting
   ip_transparency
+  compression

docs/root/version_history/current.rst

@@ -65,6 +65,7 @@ New Features
 * tcp: switched the TCP connection pool to the new "shared" connection pool, sharing a common code base with HTTP and HTTP/2. Any unexpected behavioral changes can be temporarily reverted by setting `envoy.reloadable_features.new_tcp_connection_pool` to false.
 * watchdog: support randomizing the watchdog's kill timeout to prevent synchronized kills via a maximium jitter parameter :ref:`max_kill_timeout_jitter<envoy_v3_api_field_config.bootstrap.v3.Watchdog.max_kill_timeout_jitter>`.
 * xds: added :ref:`extension config discovery<envoy_v3_api_msg_config.core.v3.ExtensionConfigSource>` support for HTTP filters.
+* zlib: added option to use `zlib-ng <https://github.com/zlib-ng/zlib-ng>`_ as zlib library.
 
 Deprecated
 ----------