compressor: Allow to optionally use zlib-ng

bazel/BUILD

@@ -243,6 +243,22 @@ config_setting(
     values = {"define": "boringssl=disabled"},
 )
 
+config_setting(
+    name = "zlib_ng",
+    constraint_values = [
+        "@bazel_tools//platforms:linux",
+    ],
+    values = {"define": "zlib=ng"},
+)
+
+config_setting(
+    name = "zlib_ng_with_optimizations",
+    constraint_values = [
+        "@bazel_tools//platforms:linux",
+    ],
+    values = {"define": "zlib=ng-with-optimizations"},
+)
+
 config_setting(
     name = "enable_quiche",
     values = {"define": "quiche=enabled"},
@@ -259,6 +275,28 @@ alias(
     }),
 )
 
+# Alias pointing to the selected version of zlib:
+# - zlib-ng from //bazel/foreign_cc:zlib_ng,
+# - zlib-ng with optimization from //bazel/foreign_cc:zlib_ng_with_optimizations,
+# - zlib from //bazel/foreign_cc:zlib.
+alias(
+    name = "zlib",
+    actual = select({
+        "//bazel:zlib_ng": "//bazel/foreign_cc:zlib_ng",
+        "//bazel:zlib_ng_with_optimizations": "//bazel/foreign_cc:zlib_ng_with_optimizations",
+        "//conditions:default": "//bazel/foreign_cc:zlib",
+    }),
+)
+
+alias(
+    name = "curl",
+    actual = select({
+        "//bazel:zlib_ng": "//bazel/foreign_cc:curl_with_zlib_ng",
+        "//bazel:zlib_ng_with_optimizations": "//bazel/foreign_cc:curl_with_zlib_ng_with_optimizations",
+        "//conditions:default": "//bazel/foreign_cc:curl",
+    }),
+)
+
 config_setting(
     name = "linux_x86_64",
     values = {"cpu": "k8"},

bazel/envoy_build_system.bzl

@@ -78,6 +78,7 @@ def envoy_cmake_external(
         pdb_name = "",
         cmake_files_dir = "$BUILD_TMPDIR/CMakeFiles",
         generate_crosstool_file = False,
+        variants = [],
         **kwargs):
     cache_entries.update({"CMAKE_BUILD_TYPE": "Bazel"})
     cache_entries_debug = dict(cache_entries)
@@ -120,6 +121,34 @@ def envoy_cmake_external(
         **kwargs
     )
 
+    for variant in variants:
+        if "cache_entries" in variant:
+            cache_entries.update(variant["cache_entries"] or {})
+            cache_entries_debug = dict(cache_entries)
+            cache_entries_debug.update(debug_cache_entries)
+
+        if "lib_source" in variant:
+            lib_source = variant["lib_source"] or lib_source
+
+        cmake_external(
+            name = variant["name"],
+            cache_entries = select({
+                "@envoy//bazel:opt_build": cache_entries,
+                "//conditions:default": cache_entries_debug,
+            }),
+            cmake_options = cmake_options,
+            # TODO(lizan): Make this always true
+            generate_crosstool_file = select({
+                "@envoy//bazel:windows_x86_64": True,
+                "//conditions:default": generate_crosstool_file,
+            }),
+            lib_source = lib_source,
+            make_commands = make_commands,
+            postfix_script = pf,
+            static_libraries = static_libraries,
+            **kwargs
+        )
+
 # Used to select a dependency that has different implementations on POSIX vs Windows.
 # The platform-specific implementations should be specified with envoy_cc_posix_library
 # and envoy_cc_win32_library respectively

bazel/foreign_cc/BUILD

@@ -146,11 +146,27 @@ envoy_cmake_external(
         "//bazel:windows_x86_64": ["libcurl.lib"],
         "//conditions:default": ["libcurl.a"],
     }),
+    variants = [
+        {
+            "name": "curl_with_zlib_ng",
+            "cache_entries": {
+                "ZLIB_LIBRARY": "$EXT_BUILD_DEPS/zlib_ng",
+                "ZLIB_INCLUDE_DIR": "$EXT_BUILD_DEPS/zlib_ng/include",
+            },
+        },
+        {
+            "name": "curl_with_zlib_ng_with_optimizations",
+            "cache_entries": {
+                "ZLIB_LIBRARY": "$EXT_BUILD_DEPS/zlib_ng_with_optimizations",
+                "ZLIB_INCLUDE_DIR": "$EXT_BUILD_DEPS/zlib_ng_with_optimizations/include",
+            },
+        },
+    ],
     deps = [
         ":ares",
         ":nghttp2",
-        ":zlib",
         "//external:ssl",
+        "//external:zlib",
     ],
 )
 
@@ -224,4 +240,46 @@ envoy_cmake_external(
         "//bazel:windows_x86_64": ["zlibstatic.lib"],
         "//conditions:default": ["libz.a"],
     }),
+    variants = [
+        {
+            "name": "zlib_ng",
+            "cache_entries": {
+                # zlib-ng build options. Reference: https://github.com/zlib-ng/zlib-ng#build-options.
+                "ZLIB_COMPAT": "on",
+                "ZLIB_ENABLE_TESTS": "off",
+
+                # Turning off WITH_OPTIM and WITH_NEW_STRATEGIES fixes checksum, but not sure if that is
+                # what we want.
+                "WITH_OPTIM": "off",
+                "WITH_NEW_STRATEGIES": "off",
+
+                # zlib-ng advance build options. Reference: https://github.com/zlib-ng/zlib-ng#advanced-build-options.
+                # Only allow aligned address.
+                "UNALIGNED_OK": "off",
+            },
+            "lib_source": "@com_github_zlib_ng_zlib_ng//:all",
+        },
+        {
+            "name": "zlib_ng_with_optimizations",
+            "cache_entries": {
+                # zlib-ng build options. Reference: https://github.com/zlib-ng/zlib-ng#build-options.
+                "ZLIB_COMPAT": "on",
+                "ZLIB_ENABLE_TESTS": "off",
+
+                # Warning: Turning WITH_OPTIM to "on" doesn't pass ZlibCompressorImplTest.CallingChecksum.
+                "WITH_OPTIM": "on",
+                # However turning off SSE4 fixes it.
+                "WITH_SSE4": "off",
+
+                # Warning: Turning WITH_NEW_STRATEGIES to "on" doesn't pass gzip compressor fuzz test.
+                # Turning this off means falling into NO_QUICK_STRATEGY route.
+                "WITH_NEW_STRATEGIES": "off",
+
+                # zlib-ng advance build options. Reference: https://github.com/zlib-ng/zlib-ng#advanced-build-options.
+                # Only allow aligned address.
+                "UNALIGNED_OK": "off",
+            },
+            "lib_source": "@com_github_zlib_ng_zlib_ng//:all",
+        },
+    ],
 )

bazel/repositories.bzl

@@ -155,6 +155,23 @@ def envoy_dependencies(skip_targets = []):
         actual = "@envoy//bazel:boringssl",
     )
 
+    # Binding to an alias pointing to the selected version of zlib:
+    # - zlib-ng from @envoy//bazel/foreign_cc:zlib_ng,
+    # - zlib-ng with optimization from @envoy//bazel/foreign_cc:zlib_ng_with_optimizations,
+    # - zlib from @envoy//bazel/foreign_cc:zlib.
+    _net_zlib()
+    _com_github_zlib_ng_zlib_ng()
+    native.bind(
+        name = "zlib",
+        actual = "@envoy//bazel:zlib",
+    )
+
+    # Bind for gRPC.
+    native.bind(
+        name = "madler_zlib",
+        actual = "@envoy//bazel:zlib",
+    )
+
     # The long repo names (`com_github_fmtlib_fmt` instead of `fmtlib`) are
     # semi-standard in the Bazel community, intended to avoid both duplicate
     # dependencies and name conflicts.
@@ -189,7 +206,6 @@ def envoy_dependencies(skip_targets = []):
     _com_googlesource_googleurl()
     _com_lightstep_tracer_cpp()
     _io_opentracing_cpp()
-    _net_zlib()
     _upb()
     _proxy_wasm_cpp_sdk()
     _proxy_wasm_cpp_host()
@@ -367,15 +383,10 @@ def _net_zlib():
         patches = ["@envoy//bazel/foreign_cc:zlib.patch"],
     )
 
-    native.bind(
-        name = "zlib",
-        actual = "@envoy//bazel/foreign_cc:zlib",
-    )
-
-    # Bind for grpc.
-    native.bind(
-        name = "madler_zlib",
-        actual = "@envoy//bazel/foreign_cc:zlib",
+def _com_github_zlib_ng_zlib_ng():
+    _repository_impl(
+        name = "com_github_zlib_ng_zlib_ng",
+        build_file_content = BUILD_ALL_CONTENT,
     )
 
 def _com_google_cel_cpp():
@@ -662,15 +673,15 @@ def _com_github_curl():
     http_archive(
         name = "com_github_curl",
         build_file_content = BUILD_ALL_CONTENT + """
-cc_library(name = "curl", visibility = ["//visibility:public"], deps = ["@envoy//bazel/foreign_cc:curl"])
+cc_library(name = "curl", visibility = ["//visibility:public"], deps = ["@envoy//bazel:curl"])
 """,
         patches = ["@envoy//bazel/foreign_cc:curl-revert-cmake-minreqver.patch"],
         patch_args = ["-p1"],
         **location
     )
     native.bind(
         name = "curl",
-        actual = "@envoy//bazel/foreign_cc:curl",
+        actual = "@envoy//bazel:curl",
     )
 
 def _com_googlesource_chromium_v8():

bazel/repository_locations.bzl

@@ -261,6 +261,14 @@ DEPENDENCY_REPOSITORIES = dict(
         use_category = ["dataplane"],
         cpe = "cpe:2.3:a:gnu:zlib:*",
     ),
+    com_github_zlib_ng_zlib_ng = dict(
+        sha256 = "eb1b3a33855e8994ee89901e74ca4317f3bff93d2e0f3fc5a11e655a78a73092",
+        strip_prefix = "zlib-ng-8832d7db7241194fa68509c96c092f3cf527ccce",
+        # 2020-08-03 develop branch.
+        urls = ["https://github.com/zlib-ng/zlib-ng/archive/8832d7db7241194fa68509c96c092f3cf527ccce.tar.gz"],
+        use_category = ["dataplane"],
+        cpe = "N/A",
+    ),
     com_github_jbeder_yaml_cpp = dict(
         sha256 = "79ab7069ef1c7c3632e7ffe095f7185d4c77b64d8035db3c085c239d4fe96d5f",
         strip_prefix = "yaml-cpp-98acc5a8874faab28b82c28936f4b400b389f5d6",

docs/root/extending/extending.rst

@@ -23,6 +23,7 @@ types including:
 * BoringSSL private key methods
 * :ref:`Watchdog action <envoy_v3_api_msg_config.bootstrap.v3.Watchdog.WatchdogAction>`
 * :ref:`Internal redirect policy <envoy_v3_api_field_config.route.v3.InternalRedirectPolicy.predicates>`
+* :ref:`Compression libraries <arch_overview_compression_libraries>`
 
 As of this writing there is no high level extension developer documentation. The
 :repo:`existing extensions <source/extensions>` are a good way to learn what is possible.

docs/root/intro/arch_overview/arch_overview.rst

@@ -11,6 +11,7 @@ Architecture overview
   observability/observability
   security/security
   operations/operations
+  compression/libraries
   other_features/other_features
   other_protocols/other_protocols
   advanced/advanced

docs/root/intro/arch_overview/compression/libraries.rst

@@ -0,0 +1,21 @@
+.. _arch_overview_compression_libraries:
+
+Compression Libraries
+=====================
+
+Underlying implementation
+-------------------------
+
+Currently Envoy uses `zlib <http://zlib.net>`_ as a compression library.
+
+.. note::
+
+  `zlib-ng <https://github.com/zlib-ng/zlib-ng>`_ is a fork that hosts several 3rd-party
+  contributions containing new optimizations. Those optimizations are considered useful for
+  `improving compression performance <https://github.com/envoyproxy/envoy/issues/8448#issuecomment-667152013>`_.
+  Envoy can be built to use `zlib-ng <https://github.com/zlib-ng/zlib-ng>`_ instead of regular
+  `zlib <http://zlib.net>`_ by using ``--define zlib=ng`` Bazel option. In order to use
+  `zlib-ng <https://github.com/zlib-ng/zlib-ng>`_ with optimizations turned on, apply ``--define
+  zlib=ng-with-optimization``. Please note that building Envoy with ``ng-with-optimization`` means
+  having a different behavior to generate checksums. Currently, these options are only
+  available on Linux.

docs/root/version_history/current.rst

@@ -80,6 +80,7 @@ New Features
 * watchdog: support randomizing the watchdog's kill timeout to prevent synchronized kills via a maximium jitter parameter :ref:`max_kill_timeout_jitter<envoy_v3_api_field_config.bootstrap.v3.Watchdog.max_kill_timeout_jitter>`.
 * watchdog: supports an extension point where actions can be registered to fire on watchdog events such as miss, megamiss, kill and multikill. See ref:`watchdog actions<envoy_v3_api_field_config.bootstrap.v3.Watchdog.actions>`.
 * xds: added :ref:`extension config discovery<envoy_v3_api_msg_config.core.v3.ExtensionConfigSource>` support for HTTP filters.
+* zlib: added option to use `zlib-ng <https://github.com/zlib-ng/zlib-ng>`_ as zlib library.
 
 Deprecated
 ----------