From 07f6cb54ece56691dbd2a94b0cbeec722ff6a631 Mon Sep 17 00:00:00 2001 From: Harish <140232061+perhapsmaple@users.noreply.github.com> Date: Thu, 29 Feb 2024 23:12:25 +0530 Subject: [PATCH] [EXPORTER] Gzip compression support for OTLP/HTTP and OTLP/gRPC exporter (#2530) --- CHANGELOG.md | 13 ++++ CMakeLists.txt | 17 +++++ api/CMakeLists.txt | 5 ++ bazel/repository.bzl | 13 ++++ bazel/zlib.BUILD | 74 +++++++++++++++++++ ci/do_ci.sh | 4 + docs/dependencies.md | 6 ++ .../exporters/otlp/otlp_environment.h | 4 + .../exporters/otlp/otlp_grpc_client_options.h | 3 + .../exporters/otlp/otlp_http_client.h | 5 ++ .../otlp/otlp_http_exporter_options.h | 3 + .../otlp_http_log_record_exporter_options.h | 3 + .../otlp/otlp_http_metric_exporter_options.h | 3 + exporters/otlp/src/otlp_environment.cc | 51 +++++++++++++ exporters/otlp/src/otlp_grpc_client.cc | 5 ++ .../otlp/src/otlp_grpc_exporter_options.cc | 2 + .../otlp_grpc_log_record_exporter_options.cc | 2 + .../src/otlp_grpc_metric_exporter_options.cc | 2 + exporters/otlp/src/otlp_http_client.cc | 5 ++ exporters/otlp/src/otlp_http_exporter.cc | 1 + .../otlp/src/otlp_http_exporter_options.cc | 2 + .../otlp/src/otlp_http_log_record_exporter.cc | 1 + .../otlp_http_log_record_exporter_options.cc | 2 + .../otlp/src/otlp_http_metric_exporter.cc | 1 + .../src/otlp_http_metric_exporter_options.cc | 2 + .../otlp/test/otlp_http_exporter_test.cc | 2 +- .../otlp_http_log_record_exporter_test.cc | 2 +- .../test/otlp_http_metric_exporter_test.cc | 2 +- exporters/zipkin/test/zipkin_exporter_test.cc | 10 ++- .../ext/http/client/curl/http_client_curl.h | 18 ++++- .../http/client/curl/http_operation_curl.h | 4 + .../ext/http/client/http_client.h | 25 +++++-- ext/src/http/client/curl/BUILD | 1 + ext/src/http/client/curl/CMakeLists.txt | 16 ++++ ext/src/http/client/curl/http_client_curl.cc | 52 ++++++++++++- .../http/client/curl/http_operation_curl.cc | 11 +++ ext/test/http/curl_http_test.cc | 8 +- functional/otlp/func_http_main.cc | 36 +++++++++ .../http/client/nosend/http_client_nosend.h | 8 ++ 39 files changed, 402 insertions(+), 22 deletions(-) create mode 100644 bazel/zlib.BUILD diff --git a/CHANGELOG.md b/CHANGELOG.md index d9e079783e..719def9844 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -15,6 +15,19 @@ Increment the: ## [Unreleased] +* [EXPORTER] Gzip compression support for OTLP/HTTP and OTLP/gRPC exporter + [#2530](https://github.com/open-telemetry/opentelemetry-cpp/pull/2530) + +Important changes: + +* [EXPORTER] Gzip compression support for OTLP/HTTP and OTLP/gRPC exporter + [#2530](https://github.com/open-telemetry/opentelemetry-cpp/pull/2530) + * In the `OtlpHttpExporterOptions` and `OtlpGrpcExporterOptions`, a new + field called compression has been introduced. This field can be set + to "gzip” to enable gzip compression. + * The CMake option `WITH_OTLP_HTTP_COMPRESSION` is introduced to enable + gzip compression support for the OTLP HTTP Exporter and includes a + dependency on zlib. * [SDK] Change OTLP HTTP content_type default to binary [#2558](https://github.com/open-telemetry/opentelemetry-cpp/pull/2558) diff --git a/CMakeLists.txt b/CMakeLists.txt index 5464f76d1d..cdb0f390a1 100644 --- a/CMakeLists.txt +++ b/CMakeLists.txt @@ -204,6 +204,11 @@ option(WITH_OTLP_GRPC "Whether to include the OTLP gRPC exporter in the SDK" option(WITH_OTLP_HTTP "Whether to include the OTLP http exporter in the SDK" OFF) +option( + WITH_OTLP_HTTP_COMPRESSION + "Whether to include gzip compression for the OTLP http exporter in the SDK" + OFF) + option(WITH_ZIPKIN "Whether to include the Zipkin exporter in the SDK" OFF) option(WITH_PROMETHEUS "Whether to include the Prometheus Client in the SDK" @@ -448,6 +453,18 @@ if((NOT WITH_API_ONLY) AND WITH_HTTP_CLIENT_CURL) message(STATUS "Found CURL: ${CURL_LIBRARIES}, version ${CURL_VERSION}") endif() +# +# Do we need ZLIB ? +# + +if((NOT WITH_API_ONLY) + AND WITH_HTTP_CLIENT_CURL + AND WITH_OTLP_HTTP_COMPRESSION) + # No specific version required. + find_package(ZLIB REQUIRED) + message(STATUS "Found ZLIB: ${ZLIB_LIBRARIES}, version ${ZLIB_VERSION}") +endif() + # # Do we need NLOHMANN_JSON ? # diff --git a/api/CMakeLists.txt b/api/CMakeLists.txt index 14332e295b..ac755549e3 100644 --- a/api/CMakeLists.txt +++ b/api/CMakeLists.txt @@ -120,6 +120,11 @@ if(WITH_METRICS_EXEMPLAR_PREVIEW) INTERFACE ENABLE_METRICS_EXEMPLAR_PREVIEW) endif() +if(WITH_OTLP_HTTP_COMPRESSION) + target_compile_definitions(opentelemetry_api + INTERFACE ENABLE_OTLP_COMPRESSION_PREVIEW) +endif() + include(${PROJECT_SOURCE_DIR}/cmake/pkgconfig.cmake) if(OPENTELEMETRY_INSTALL) diff --git a/bazel/repository.bzl b/bazel/repository.bzl index 0fde88dc5c..2abdca87ce 100644 --- a/bazel/repository.bzl +++ b/bazel/repository.bzl @@ -183,3 +183,16 @@ def opentelemetry_cpp_deps(): "https://github.com/opentracing/opentracing-cpp/archive/refs/tags/v1.6.0.tar.gz", ], ) + + # Zlib (optional) + maybe( + http_archive, + name = "zlib", + build_file = "@io_opentelemetry_cpp//bazel:zlib.BUILD", + sha256 = "d14c38e313afc35a9a8760dadf26042f51ea0f5d154b0630a31da0540107fb98", + strip_prefix = "zlib-1.2.13", + urls = [ + "https://github.com/madler/zlib/releases/download/v1.2.13/zlib-1.2.13.tar.xz", + "https://zlib.net/zlib-1.2.13.tar.xz", + ], + ) diff --git a/bazel/zlib.BUILD b/bazel/zlib.BUILD new file mode 100644 index 0000000000..676de88f28 --- /dev/null +++ b/bazel/zlib.BUILD @@ -0,0 +1,74 @@ +# Copyright The OpenTelemetry Authors +# SPDX-License-Identifier: Apache-2.0 + +# Builds ZLIB from a distribution. +# Copied from https://github.com/protocolbuffers/protobuf/blob/master/third_party/zlib.BUILD + +licenses(["notice"]) # BSD/MIT-like license (for zlib) + +exports_files(["zlib.BUILD"]) + +_ZLIB_HEADERS = [ + "crc32.h", + "deflate.h", + "gzguts.h", + "inffast.h", + "inffixed.h", + "inflate.h", + "inftrees.h", + "trees.h", + "zconf.h", + "zlib.h", + "zutil.h", +] + +_ZLIB_PREFIXED_HEADERS = ["zlib/include/" + hdr for hdr in _ZLIB_HEADERS] + +# In order to limit the damage from the `includes` propagation +# via `:zlib`, copy the public headers to a subdirectory and +# expose those. +genrule( + name = "copy_public_headers", + srcs = _ZLIB_HEADERS, + outs = _ZLIB_PREFIXED_HEADERS, + cmd_bash = "cp $(SRCS) $(@D)/zlib/include/", + cmd_bat = " && ".join( + ["@copy /Y $(location %s) $(@D)\\zlib\\include\\ >NUL" % + s for s in _ZLIB_HEADERS], + ), +) + +cc_library( + name = "zlib", + srcs = [ + "adler32.c", + "compress.c", + "crc32.c", + "deflate.c", + "gzclose.c", + "gzlib.c", + "gzread.c", + "gzwrite.c", + "infback.c", + "inffast.c", + "inflate.c", + "inftrees.c", + "trees.c", + "uncompr.c", + "zutil.c", + # Include the un-prefixed headers in srcs to work + # around the fact that zlib isn't consistent in its + # choice of <> or "" delimiter when including itself. + ] + _ZLIB_HEADERS, + hdrs = _ZLIB_PREFIXED_HEADERS, + copts = select({ + "@platforms//os:windows": [], + "//conditions:default": [ + "-Wno-deprecated-non-prototype", + "-Wno-unused-variable", + "-Wno-implicit-function-declaration", + ], + }), + includes = ["zlib/include/"], + visibility = ["//visibility:public"], +) \ No newline at end of file diff --git a/ci/do_ci.sh b/ci/do_ci.sh index 4a3b616c8a..9ea7ac6bc3 100755 --- a/ci/do_ci.sh +++ b/ci/do_ci.sh @@ -120,6 +120,7 @@ elif [[ "$1" == "cmake.maintainer.sync.test" ]]; then -DWITH_ASYNC_EXPORT_PREVIEW=OFF \ -DOTELCPP_MAINTAINER_MODE=ON \ -DWITH_NO_DEPRECATED_CODE=ON \ + -DWITH_OTLP_HTTP_COMPRESSION=ON \ ${IWYU} \ "${SRC_DIR}" eval "$MAKE_COMMAND" @@ -140,6 +141,7 @@ elif [[ "$1" == "cmake.maintainer.async.test" ]]; then -DWITH_ASYNC_EXPORT_PREVIEW=ON \ -DOTELCPP_MAINTAINER_MODE=ON \ -DWITH_NO_DEPRECATED_CODE=ON \ + -DWITH_OTLP_HTTP_COMPRESSION=ON \ ${IWYU} \ "${SRC_DIR}" eval "$MAKE_COMMAND" @@ -161,6 +163,7 @@ elif [[ "$1" == "cmake.maintainer.cpp11.async.test" ]]; then -DWITH_ASYNC_EXPORT_PREVIEW=ON \ -DOTELCPP_MAINTAINER_MODE=ON \ -DWITH_NO_DEPRECATED_CODE=ON \ + -DWITH_OTLP_HTTP_COMPRESSION=ON \ "${SRC_DIR}" make -k -j $(nproc) make test @@ -182,6 +185,7 @@ elif [[ "$1" == "cmake.maintainer.abiv2.test" ]]; then -DWITH_NO_DEPRECATED_CODE=ON \ -DWITH_ABI_VERSION_1=OFF \ -DWITH_ABI_VERSION_2=ON \ + -DWITH_OTLP_HTTP_COMPRESSION=ON \ ${IWYU} \ "${SRC_DIR}" eval "$MAKE_COMMAND" diff --git a/docs/dependencies.md b/docs/dependencies.md index da6c584fe8..dd495f4888 100644 --- a/docs/dependencies.md +++ b/docs/dependencies.md @@ -63,6 +63,12 @@ Both these dependencies are listed here: - protobuf serialized otlp messages are encoded in JSON format using this library. - License: `MIT License` + - [zlib](https://www.zlib.net/): A Massively Spiffy Yet Delicately + Unobtrusive Compression Library. + - The `http_client` utilizes zlib to compress the message body and send + it in gzip format. + - License: The library is licensed + [here](https://www.zlib.net/zlib_license.html) - [OTLP/gRPC](/exporters/otlp) exporter: diff --git a/exporters/otlp/include/opentelemetry/exporters/otlp/otlp_environment.h b/exporters/otlp/include/opentelemetry/exporters/otlp/otlp_environment.h index 306dff8f09..c2e0afb7bc 100644 --- a/exporters/otlp/include/opentelemetry/exporters/otlp/otlp_environment.h +++ b/exporters/otlp/include/opentelemetry/exporters/otlp/otlp_environment.h @@ -152,6 +152,10 @@ inline OtlpHeaders GetOtlpDefaultHeaders() return GetOtlpDefaultTracesHeaders(); } +std::string GetOtlpDefaultTracesCompression(); +std::string GetOtlpDefaultMetricsCompression(); +std::string GetOtlpDefaultLogsCompression(); + } // namespace otlp } // namespace exporter OPENTELEMETRY_END_NAMESPACE diff --git a/exporters/otlp/include/opentelemetry/exporters/otlp/otlp_grpc_client_options.h b/exporters/otlp/include/opentelemetry/exporters/otlp/otlp_grpc_client_options.h index 1191c2118a..45bd2e8896 100644 --- a/exporters/otlp/include/opentelemetry/exporters/otlp/otlp_grpc_client_options.h +++ b/exporters/otlp/include/opentelemetry/exporters/otlp/otlp_grpc_client_options.h @@ -55,6 +55,9 @@ struct OtlpGrpcClientOptions /** max number of threads that can be allocated from this */ std::size_t max_threads; + /** Compression type. */ + std::string compression; + #ifdef ENABLE_ASYNC_EXPORT // Concurrent requests std::size_t max_concurrent_requests; diff --git a/exporters/otlp/include/opentelemetry/exporters/otlp/otlp_http_client.h b/exporters/otlp/include/opentelemetry/exporters/otlp/otlp_http_client.h index 163cf7b57f..870ea88544 100644 --- a/exporters/otlp/include/opentelemetry/exporters/otlp/otlp_http_client.h +++ b/exporters/otlp/include/opentelemetry/exporters/otlp/otlp_http_client.h @@ -59,6 +59,9 @@ struct OtlpHttpClientOptions // This option is ignored if content_type is not kJson JsonBytesMappingKind json_bytes_mapping = JsonBytesMappingKind::kHexId; + // By default, do not compress data + std::string compression = "none"; + // If using the json name of protobuf field to set the key of json. By default, we will use the // field name just like proto files. bool use_json_name = false; @@ -94,6 +97,7 @@ struct OtlpHttpClientOptions nostd::string_view input_ssl_cipher_suite, HttpRequestContentType input_content_type, JsonBytesMappingKind input_json_bytes_mapping, + nostd::string_view input_compression, bool input_use_json_name, bool input_console_debug, std::chrono::system_clock::duration input_timeout, @@ -116,6 +120,7 @@ struct OtlpHttpClientOptions input_ssl_cipher_suite), content_type(input_content_type), json_bytes_mapping(input_json_bytes_mapping), + compression(input_compression), use_json_name(input_use_json_name), console_debug(input_console_debug), timeout(input_timeout), diff --git a/exporters/otlp/include/opentelemetry/exporters/otlp/otlp_http_exporter_options.h b/exporters/otlp/include/opentelemetry/exporters/otlp/otlp_http_exporter_options.h index f7ed0eb3b9..1be4bd8d95 100644 --- a/exporters/otlp/include/opentelemetry/exporters/otlp/otlp_http_exporter_options.h +++ b/exporters/otlp/include/opentelemetry/exporters/otlp/otlp_http_exporter_options.h @@ -100,6 +100,9 @@ struct OPENTELEMETRY_EXPORT OtlpHttpExporterOptions /** TLS cipher suite. */ std::string ssl_cipher_suite; + + /** Compression type. */ + std::string compression; }; } // namespace otlp diff --git a/exporters/otlp/include/opentelemetry/exporters/otlp/otlp_http_log_record_exporter_options.h b/exporters/otlp/include/opentelemetry/exporters/otlp/otlp_http_log_record_exporter_options.h index 1c34327a3c..7d60a28cf0 100644 --- a/exporters/otlp/include/opentelemetry/exporters/otlp/otlp_http_log_record_exporter_options.h +++ b/exporters/otlp/include/opentelemetry/exporters/otlp/otlp_http_log_record_exporter_options.h @@ -100,6 +100,9 @@ struct OPENTELEMETRY_EXPORT OtlpHttpLogRecordExporterOptions /** TLS cipher suite. */ std::string ssl_cipher_suite; + + /** Compression type. */ + std::string compression; }; } // namespace otlp diff --git a/exporters/otlp/include/opentelemetry/exporters/otlp/otlp_http_metric_exporter_options.h b/exporters/otlp/include/opentelemetry/exporters/otlp/otlp_http_metric_exporter_options.h index 4f30451387..d5cf3072f8 100644 --- a/exporters/otlp/include/opentelemetry/exporters/otlp/otlp_http_metric_exporter_options.h +++ b/exporters/otlp/include/opentelemetry/exporters/otlp/otlp_http_metric_exporter_options.h @@ -103,6 +103,9 @@ struct OPENTELEMETRY_EXPORT OtlpHttpMetricExporterOptions /** TLS cipher suite. */ std::string ssl_cipher_suite; + + /** Compression type. */ + std::string compression; }; } // namespace otlp diff --git a/exporters/otlp/src/otlp_environment.cc b/exporters/otlp/src/otlp_environment.cc index 0cb39cb68e..a7af30bb73 100644 --- a/exporters/otlp/src/otlp_environment.cc +++ b/exporters/otlp/src/otlp_environment.cc @@ -1069,6 +1069,57 @@ OtlpHeaders GetOtlpDefaultLogsHeaders() return GetHeaders(kSignalEnv, kGenericEnv); } +std::string GetOtlpDefaultTracesCompression() +{ + constexpr char kSignalEnv[] = "OTEL_EXPORTER_OTLP_TRACES_COMPRESSION"; + constexpr char kGenericEnv[] = "OTEL_EXPORTER_OTLP_COMPRESSION"; + + std::string value; + bool exists; + + exists = GetStringDualEnvVar(kSignalEnv, kGenericEnv, value); + if (exists) + { + return value; + } + + return std::string{"none"}; +} + +std::string GetOtlpDefaultMetricsCompression() +{ + constexpr char kSignalEnv[] = "OTEL_EXPORTER_OTLP_METRICS_COMPRESSION"; + constexpr char kGenericEnv[] = "OTEL_EXPORTER_OTLP_COMPRESSION"; + + std::string value; + bool exists; + + exists = GetStringDualEnvVar(kSignalEnv, kGenericEnv, value); + if (exists) + { + return value; + } + + return std::string{"none"}; +} + +std::string GetOtlpDefaultLogsCompression() +{ + constexpr char kSignalEnv[] = "OTEL_EXPORTER_OTLP_LOGS_COMPRESSION"; + constexpr char kGenericEnv[] = "OTEL_EXPORTER_OTLP_COMPRESSION"; + + std::string value; + bool exists; + + exists = GetStringDualEnvVar(kSignalEnv, kGenericEnv, value); + if (exists) + { + return value; + } + + return std::string{"none"}; +} + } // namespace otlp } // namespace exporter OPENTELEMETRY_END_NAMESPACE diff --git a/exporters/otlp/src/otlp_grpc_client.cc b/exporters/otlp/src/otlp_grpc_client.cc index fcd53985d4..f57e476955 100644 --- a/exporters/otlp/src/otlp_grpc_client.cc +++ b/exporters/otlp/src/otlp_grpc_client.cc @@ -297,6 +297,11 @@ std::shared_ptr OtlpGrpcClient::MakeChannel(const OtlpGrpcClientO grpc_arguments.SetResourceQuota(quota); } + if (options.compression == "gzip") + { + grpc_arguments.SetCompressionAlgorithm(GRPC_COMPRESS_GZIP); + } + if (options.use_ssl_credentials) { grpc::SslCredentialsOptions ssl_opts; diff --git a/exporters/otlp/src/otlp_grpc_exporter_options.cc b/exporters/otlp/src/otlp_grpc_exporter_options.cc index c5dc94ec9d..5fcc77fc2f 100644 --- a/exporters/otlp/src/otlp_grpc_exporter_options.cc +++ b/exporters/otlp/src/otlp_grpc_exporter_options.cc @@ -31,6 +31,8 @@ OtlpGrpcExporterOptions::OtlpGrpcExporterOptions() user_agent = GetOtlpDefaultUserAgent(); max_threads = 0; + + compression = GetOtlpDefaultTracesCompression(); #ifdef ENABLE_ASYNC_EXPORT max_concurrent_requests = 64; #endif diff --git a/exporters/otlp/src/otlp_grpc_log_record_exporter_options.cc b/exporters/otlp/src/otlp_grpc_log_record_exporter_options.cc index 5925808abe..b98f76a922 100644 --- a/exporters/otlp/src/otlp_grpc_log_record_exporter_options.cc +++ b/exporters/otlp/src/otlp_grpc_log_record_exporter_options.cc @@ -29,6 +29,8 @@ OtlpGrpcLogRecordExporterOptions::OtlpGrpcLogRecordExporterOptions() user_agent = GetOtlpDefaultUserAgent(); max_threads = 0; + + compression = GetOtlpDefaultLogsCompression(); #ifdef ENABLE_ASYNC_EXPORT max_concurrent_requests = 64; #endif diff --git a/exporters/otlp/src/otlp_grpc_metric_exporter_options.cc b/exporters/otlp/src/otlp_grpc_metric_exporter_options.cc index ff8466b0b2..983561424f 100644 --- a/exporters/otlp/src/otlp_grpc_metric_exporter_options.cc +++ b/exporters/otlp/src/otlp_grpc_metric_exporter_options.cc @@ -31,6 +31,8 @@ OtlpGrpcMetricExporterOptions::OtlpGrpcMetricExporterOptions() aggregation_temporality = PreferredAggregationTemporality::kCumulative; max_threads = 0; + + compression = GetOtlpDefaultMetricsCompression(); #ifdef ENABLE_ASYNC_EXPORT max_concurrent_requests = 64; #endif diff --git a/exporters/otlp/src/otlp_http_client.cc b/exporters/otlp/src/otlp_http_client.cc index 7814199ef2..151873c221 100644 --- a/exporters/otlp/src/otlp_http_client.cc +++ b/exporters/otlp/src/otlp_http_client.cc @@ -958,6 +958,11 @@ OtlpHttpClient::createSession( request->ReplaceHeader("Content-Type", content_type); request->ReplaceHeader("User-Agent", options_.user_agent); + if (options_.compression == "gzip") + { + request->SetCompression(opentelemetry::ext::http::client::Compression::kGzip); + } + // Returns the created session data return HttpSessionData{ std::move(session), diff --git a/exporters/otlp/src/otlp_http_exporter.cc b/exporters/otlp/src/otlp_http_exporter.cc index 60f037f07a..aa00c3f79a 100644 --- a/exporters/otlp/src/otlp_http_exporter.cc +++ b/exporters/otlp/src/otlp_http_exporter.cc @@ -40,6 +40,7 @@ OtlpHttpExporter::OtlpHttpExporter(const OtlpHttpExporterOptions &options) options.ssl_cipher_suite, options.content_type, options.json_bytes_mapping, + options.compression, options.use_json_name, options.console_debug, options.timeout, diff --git a/exporters/otlp/src/otlp_http_exporter_options.cc b/exporters/otlp/src/otlp_http_exporter_options.cc index ae768a0dc5..d88b8d2e3d 100644 --- a/exporters/otlp/src/otlp_http_exporter_options.cc +++ b/exporters/otlp/src/otlp_http_exporter_options.cc @@ -41,6 +41,8 @@ OtlpHttpExporterOptions::OtlpHttpExporterOptions() ssl_max_tls = GetOtlpDefaultTracesSslTlsMaxVersion(); ssl_cipher = GetOtlpDefaultTracesSslTlsCipher(); ssl_cipher_suite = GetOtlpDefaultTracesSslTlsCipherSuite(); + + compression = GetOtlpDefaultTracesCompression(); } OtlpHttpExporterOptions::~OtlpHttpExporterOptions() {} diff --git a/exporters/otlp/src/otlp_http_log_record_exporter.cc b/exporters/otlp/src/otlp_http_log_record_exporter.cc index 8530421f82..0d08c66a79 100644 --- a/exporters/otlp/src/otlp_http_log_record_exporter.cc +++ b/exporters/otlp/src/otlp_http_log_record_exporter.cc @@ -43,6 +43,7 @@ OtlpHttpLogRecordExporter::OtlpHttpLogRecordExporter( options.ssl_cipher_suite, options.content_type, options.json_bytes_mapping, + options.compression, options.use_json_name, options.console_debug, options.timeout, diff --git a/exporters/otlp/src/otlp_http_log_record_exporter_options.cc b/exporters/otlp/src/otlp_http_log_record_exporter_options.cc index 35b80f48ac..cf2227a8d5 100644 --- a/exporters/otlp/src/otlp_http_log_record_exporter_options.cc +++ b/exporters/otlp/src/otlp_http_log_record_exporter_options.cc @@ -41,6 +41,8 @@ OtlpHttpLogRecordExporterOptions::OtlpHttpLogRecordExporterOptions() ssl_max_tls = GetOtlpDefaultLogsSslTlsMaxVersion(); ssl_cipher = GetOtlpDefaultLogsSslTlsCipher(); ssl_cipher_suite = GetOtlpDefaultLogsSslTlsCipherSuite(); + + compression = GetOtlpDefaultLogsCompression(); } OtlpHttpLogRecordExporterOptions::~OtlpHttpLogRecordExporterOptions() {} diff --git a/exporters/otlp/src/otlp_http_metric_exporter.cc b/exporters/otlp/src/otlp_http_metric_exporter.cc index 00dbf2c6a3..1782bb6acb 100644 --- a/exporters/otlp/src/otlp_http_metric_exporter.cc +++ b/exporters/otlp/src/otlp_http_metric_exporter.cc @@ -43,6 +43,7 @@ OtlpHttpMetricExporter::OtlpHttpMetricExporter(const OtlpHttpMetricExporterOptio options.ssl_cipher_suite, options.content_type, options.json_bytes_mapping, + options.compression, options.use_json_name, options.console_debug, options.timeout, diff --git a/exporters/otlp/src/otlp_http_metric_exporter_options.cc b/exporters/otlp/src/otlp_http_metric_exporter_options.cc index ebed1c4c83..7e2c145639 100644 --- a/exporters/otlp/src/otlp_http_metric_exporter_options.cc +++ b/exporters/otlp/src/otlp_http_metric_exporter_options.cc @@ -42,6 +42,8 @@ OtlpHttpMetricExporterOptions::OtlpHttpMetricExporterOptions() ssl_max_tls = GetOtlpDefaultMetricsSslTlsMaxVersion(); ssl_cipher = GetOtlpDefaultMetricsSslTlsCipher(); ssl_cipher_suite = GetOtlpDefaultMetricsSslTlsCipherSuite(); + + compression = GetOtlpDefaultMetricsCompression(); } OtlpHttpMetricExporterOptions::~OtlpHttpMetricExporterOptions() {} diff --git a/exporters/otlp/test/otlp_http_exporter_test.cc b/exporters/otlp/test/otlp_http_exporter_test.cc index 26325142be..8f4b938b78 100644 --- a/exporters/otlp/test/otlp_http_exporter_test.cc +++ b/exporters/otlp/test/otlp_http_exporter_test.cc @@ -70,7 +70,7 @@ OtlpHttpClientOptions MakeOtlpHttpClientOptions(HttpRequestContentType content_t "", /* ssl_max_tls */ "", /* ssl_cipher */ "", /* ssl_cipher_suite */ - options.content_type, options.json_bytes_mapping, options.use_json_name, + options.content_type, options.json_bytes_mapping, options.compression, options.use_json_name, options.console_debug, options.timeout, options.http_headers); if (!async_mode) { diff --git a/exporters/otlp/test/otlp_http_log_record_exporter_test.cc b/exporters/otlp/test/otlp_http_log_record_exporter_test.cc index 3455b7cc36..8f2c2409eb 100644 --- a/exporters/otlp/test/otlp_http_log_record_exporter_test.cc +++ b/exporters/otlp/test/otlp_http_log_record_exporter_test.cc @@ -69,7 +69,7 @@ OtlpHttpClientOptions MakeOtlpHttpClientOptions(HttpRequestContentType content_t "", /* ssl_max_tls */ "", /* ssl_cipher */ "", /* ssl_cipher_suite */ - options.content_type, options.json_bytes_mapping, options.use_json_name, + options.content_type, options.json_bytes_mapping, options.compression, options.use_json_name, options.console_debug, options.timeout, options.http_headers); if (!async_mode) { diff --git a/exporters/otlp/test/otlp_http_metric_exporter_test.cc b/exporters/otlp/test/otlp_http_metric_exporter_test.cc index b230f68d19..1e5fe6469f 100644 --- a/exporters/otlp/test/otlp_http_metric_exporter_test.cc +++ b/exporters/otlp/test/otlp_http_metric_exporter_test.cc @@ -76,7 +76,7 @@ OtlpHttpClientOptions MakeOtlpHttpClientOptions(HttpRequestContentType content_t "", /* ssl_max_tls */ "", /* ssl_cipher */ "", /* ssl_cipher_suite */ - options.content_type, options.json_bytes_mapping, options.use_json_name, + options.content_type, options.json_bytes_mapping, options.compression, options.use_json_name, options.console_debug, options.timeout, options.http_headers); if (!async_mode) { diff --git a/exporters/zipkin/test/zipkin_exporter_test.cc b/exporters/zipkin/test/zipkin_exporter_test.cc index adc81c661b..18c47bcab5 100644 --- a/exporters/zipkin/test/zipkin_exporter_test.cc +++ b/exporters/zipkin/test/zipkin_exporter_test.cc @@ -65,14 +65,16 @@ class MockHttpClient : public opentelemetry::ext::http::client::HttpClientSync (const nostd::string_view &, const ext::http::client::HttpSslOptions &, const ext::http::client::Body &, - const ext::http::client::Headers &), + const ext::http::client::Headers &, + const ext::http::client::Compression &), (noexcept, override)); MOCK_METHOD(ext::http::client::Result, Get, (const nostd::string_view &, const ext::http::client::HttpSslOptions &, - const ext::http::client::Headers &), + const ext::http::client::Headers &, + const ext::http::client::Compression &), (noexcept, override)); }; @@ -153,7 +155,7 @@ TEST_F(ZipkinExporterTestPeer, ExportJsonIntegrationTest) auto expected_url = nostd::string_view{"http://localhost:9411/api/v2/spans"}; - EXPECT_CALL(*mock_http_client, Post(expected_url, _, IsValidMessage(report_trace_id), _)) + EXPECT_CALL(*mock_http_client, Post(expected_url, _, IsValidMessage(report_trace_id), _, _)) .Times(Exactly(1)) .WillOnce(Return(ByMove(ext::http::client::Result{ @@ -179,7 +181,7 @@ TEST_F(ZipkinExporterTestPeer, ShutdownTest) // exporter should not be shutdown by default nostd::span> batch_1(&recordable_1, 1); - EXPECT_CALL(*mock_http_client, Post(_, _, _, _)) + EXPECT_CALL(*mock_http_client, Post(_, _, _, _, _)) .Times(Exactly(1)) .WillOnce(Return(ByMove(ext::http::client::Result{ diff --git a/ext/include/opentelemetry/ext/http/client/curl/http_client_curl.h b/ext/include/opentelemetry/ext/http/client/curl/http_client_curl.h index 93d407f24a..63702be09a 100644 --- a/ext/include/opentelemetry/ext/http/client/curl/http_client_curl.h +++ b/ext/include/opentelemetry/ext/http/client/curl/http_client_curl.h @@ -89,6 +89,12 @@ class Request : public opentelemetry::ext::http::client::Request timeout_ms_ = timeout_ms; } + void SetCompression( + const opentelemetry::ext::http::client::Compression &compression) noexcept override + { + compression_ = compression; + } + public: opentelemetry::ext::http::client::Method method_; opentelemetry::ext::http::client::HttpSslOptions ssl_options_; @@ -96,6 +102,8 @@ class Request : public opentelemetry::ext::http::client::Request opentelemetry::ext::http::client::Headers headers_; std::string uri_; std::chrono::milliseconds timeout_ms_{5000}; // ms + opentelemetry::ext::http::client::Compression compression_{ + opentelemetry::ext::http::client::Compression::kNone}; }; class Response : public opentelemetry::ext::http::client::Response @@ -220,12 +228,13 @@ class HttpClientSync : public opentelemetry::ext::http::client::HttpClientSync opentelemetry::ext::http::client::Result Get( const nostd::string_view &url, const opentelemetry::ext::http::client::HttpSslOptions &ssl_options, - const opentelemetry::ext::http::client::Headers &headers) noexcept override + const opentelemetry::ext::http::client::Headers &headers, + const opentelemetry::ext::http::client::Compression &compression) noexcept override { opentelemetry::ext::http::client::Body body; HttpOperation curl_operation(opentelemetry::ext::http::client::Method::Get, url.data(), - ssl_options, nullptr, headers, body); + ssl_options, nullptr, headers, body, compression); curl_operation.SendSync(); auto session_state = curl_operation.GetSessionState(); @@ -249,10 +258,11 @@ class HttpClientSync : public opentelemetry::ext::http::client::HttpClientSync const nostd::string_view &url, const opentelemetry::ext::http::client::HttpSslOptions &ssl_options, const Body &body, - const opentelemetry::ext::http::client::Headers &headers) noexcept override + const opentelemetry::ext::http::client::Headers &headers, + const opentelemetry::ext::http::client::Compression &compression) noexcept override { HttpOperation curl_operation(opentelemetry::ext::http::client::Method::Post, url.data(), - ssl_options, nullptr, headers, body); + ssl_options, nullptr, headers, body, compression); curl_operation.SendSync(); auto session_state = curl_operation.GetSessionState(); if (curl_operation.WasAborted()) diff --git a/ext/include/opentelemetry/ext/http/client/curl/http_operation_curl.h b/ext/include/opentelemetry/ext/http/client/curl/http_operation_curl.h index 587e74a238..e32d590bdb 100644 --- a/ext/include/opentelemetry/ext/http/client/curl/http_operation_curl.h +++ b/ext/include/opentelemetry/ext/http/client/curl/http_operation_curl.h @@ -146,6 +146,8 @@ class HttpOperation opentelemetry::ext::http::client::Headers(), const opentelemetry::ext::http::client::Body &request_body = opentelemetry::ext::http::client::Body(), + const opentelemetry::ext::http::client::Compression &compression = + opentelemetry::ext::http::client::Compression::kNone, // Default connectivity and response size options bool is_raw_response = false, std::chrono::milliseconds http_conn_timeout = default_http_conn_timeout, @@ -295,6 +297,8 @@ class HttpOperation size_t request_nwrite_; opentelemetry::ext::http::client::SessionState session_state_; + const opentelemetry::ext::http::client::Compression &compression_; + // Processed response headers and body long response_code_; std::vector response_headers_; diff --git a/ext/include/opentelemetry/ext/http/client/http_client.h b/ext/include/opentelemetry/ext/http/client/http_client.h index 7fee1beb35..0615284836 100644 --- a/ext/include/opentelemetry/ext/http/client/http_client.h +++ b/ext/include/opentelemetry/ext/http/client/http_client.h @@ -95,6 +95,12 @@ enum class SessionState Cancelled // (manually) cancelled }; +enum class Compression +{ + kNone, + kGzip +}; + using Byte = uint8_t; using StatusCode = uint16_t; using Body = std::vector; @@ -239,6 +245,8 @@ class Request virtual void SetTimeoutMs(std::chrono::milliseconds timeout_ms) noexcept = 0; + virtual void SetCompression(const Compression &compression) noexcept = 0; + virtual ~Request() = default; }; @@ -352,28 +360,33 @@ class HttpClient class HttpClientSync { public: - Result GetNoSsl(const nostd::string_view &url, const Headers &headers = {{}}) noexcept + Result GetNoSsl(const nostd::string_view &url, + const Headers &headers = {{}}, + const Compression &compression = Compression::kNone) noexcept { static const HttpSslOptions no_ssl; - return Get(url, no_ssl, headers); + return Get(url, no_ssl, headers, compression); } virtual Result PostNoSsl(const nostd::string_view &url, const Body &body, - const Headers &headers = {{"content-type", "application/json"}}) noexcept + const Headers &headers = {{"content-type", "application/json"}}, + const Compression &compression = Compression::kNone) noexcept { static const HttpSslOptions no_ssl; - return Post(url, no_ssl, body, headers); + return Post(url, no_ssl, body, headers, compression); } virtual Result Get(const nostd::string_view &url, const HttpSslOptions &ssl_options, - const Headers & = {{}}) noexcept = 0; + const Headers & = {{}}, + const Compression &compression = Compression::kNone) noexcept = 0; virtual Result Post(const nostd::string_view &url, const HttpSslOptions &ssl_options, const Body &body, - const Headers & = {{"content-type", "application/json"}}) noexcept = 0; + const Headers & = {{"content-type", "application/json"}}, + const Compression &compression = Compression::kNone) noexcept = 0; virtual ~HttpClientSync() = default; }; diff --git a/ext/src/http/client/curl/BUILD b/ext/src/http/client/curl/BUILD index 8c2dda1ebe..de5a5aeb2c 100644 --- a/ext/src/http/client/curl/BUILD +++ b/ext/src/http/client/curl/BUILD @@ -29,5 +29,6 @@ cc_library( "//sdk:headers", "//sdk/src/common:random", "@curl", + "@zlib", ], ) diff --git a/ext/src/http/client/curl/CMakeLists.txt b/ext/src/http/client/curl/CMakeLists.txt index 7e2507bd75..6a69c7de51 100644 --- a/ext/src/http/client/curl/CMakeLists.txt +++ b/ext/src/http/client/curl/CMakeLists.txt @@ -24,6 +24,22 @@ else() PRIVATE ${CURL_LIBRARIES}) endif() +if(WITH_OTLP_HTTP_COMPRESSION) + if(TARGET ZLIB::ZLIB) + target_link_libraries( + opentelemetry_http_client_curl + PUBLIC opentelemetry_ext + PRIVATE ZLIB::ZLIB) + else() + target_include_directories(opentelemetry_http_client_curl + INTERFACE "${ZLIB_INCLUDE_DIRS}") + target_link_libraries( + opentelemetry_http_client_curl + PUBLIC opentelemetry_ext + PRIVATE ${ZLIB_LIBRARIES}) + endif() +endif() + if(OPENTELEMETRY_INSTALL) install( TARGETS opentelemetry_http_client_curl diff --git a/ext/src/http/client/curl/http_client_curl.cc b/ext/src/http/client/curl/http_client_curl.cc index 872519fb11..11c1435b1f 100644 --- a/ext/src/http/client/curl/http_client_curl.cc +++ b/ext/src/http/client/curl/http_client_curl.cc @@ -2,6 +2,11 @@ // SPDX-License-Identifier: Apache-2.0 #include "opentelemetry/ext/http/client/curl/http_client_curl.h" +#include "opentelemetry/sdk/common/global_log_handler.h" + +#ifdef ENABLE_OTLP_COMPRESSION_PREVIEW +# include +#endif #include @@ -48,10 +53,53 @@ void Session::SendRequest( reuse_connection = session_id_ % http_client_.GetMaxSessionsPerConnection() != 0; } + if (http_request_->compression_ == opentelemetry::ext::http::client::Compression::kGzip) + { +#ifdef ENABLE_OTLP_COMPRESSION_PREVIEW + http_request_->AddHeader("Content-Encoding", "gzip"); + + opentelemetry::ext::http::client::Body compressed_body(http_request_->body_.size()); + z_stream zs; + zs.zalloc = Z_NULL; + zs.zfree = Z_NULL; + zs.opaque = Z_NULL; + zs.avail_in = static_cast(http_request_->body_.size()); + zs.next_in = http_request_->body_.data(); + zs.avail_out = static_cast(compressed_body.size()); + zs.next_out = compressed_body.data(); + + // ZLIB: Have to maually specify 16 bits for the Gzip headers + const int window_bits = 15 + 16; + + int stream = + deflateInit2(&zs, Z_DEFAULT_COMPRESSION, Z_DEFLATED, window_bits, 8, Z_DEFAULT_STRATEGY); + + if (stream == Z_OK) + { + deflate(&zs, Z_FINISH); + deflateEnd(&zs); + compressed_body.resize(zs.total_out); + http_request_->SetBody(compressed_body); + } + else + { + if (callback) + { + callback->OnEvent(opentelemetry::ext::http::client::SessionState::CreateFailed, ""); + } + is_session_active_.store(false, std::memory_order_release); + } +#else + OTEL_INTERNAL_LOG_ERROR( + "[HTTP Client Curl] Set WITH_OTLP_HTTP_COMPRESSION=ON to use gzip compression with the " + "OTLP HTTP Exporter"); +#endif + } + curl_operation_.reset(new HttpOperation(http_request_->method_, url, http_request_->ssl_options_, callback_ptr, http_request_->headers_, - http_request_->body_, false, http_request_->timeout_ms_, - reuse_connection)); + http_request_->body_, http_request_->compression_, false, + http_request_->timeout_ms_, reuse_connection)); bool success = CURLE_OK == curl_operation_->SendAsync(this, [this, callback](HttpOperation &operation) { if (operation.WasAborted()) diff --git a/ext/src/http/client/curl/http_operation_curl.cc b/ext/src/http/client/curl/http_operation_curl.cc index 31de30fecf..25f43fcb2f 100644 --- a/ext/src/http/client/curl/http_operation_curl.cc +++ b/ext/src/http/client/curl/http_operation_curl.cc @@ -240,6 +240,7 @@ HttpOperation::HttpOperation(opentelemetry::ext::http::client::Method method, // Default empty headers and empty request body const opentelemetry::ext::http::client::Headers &request_headers, const opentelemetry::ext::http::client::Body &request_body, + const opentelemetry::ext::http::client::Compression &compression, // Default connectivity and response size options bool is_raw_response, std::chrono::milliseconds http_conn_timeout, @@ -262,6 +263,7 @@ HttpOperation::HttpOperation(opentelemetry::ext::http::client::Method method, request_body_(request_body), request_nwrite_(0), session_state_(opentelemetry::ext::http::client::SessionState::Created), + compression_(compression), response_code_(0) { /* get a curl handle */ @@ -865,6 +867,15 @@ CURLcode HttpOperation::Setup() } } + if (compression_ == opentelemetry::ext::http::client::Compression::kGzip) + { + rc = SetCurlStrOption(CURLOPT_ACCEPT_ENCODING, "gzip"); + if (rc != CURLE_OK) + { + return rc; + } + } + if (curl_resource_.headers_chunk != nullptr) { rc = SetCurlListOption(CURLOPT_HTTPHEADER, curl_resource_.headers_chunk); diff --git a/ext/test/http/curl_http_test.cc b/ext/test/http/curl_http_test.cc index c51da784df..6e30323eeb 100644 --- a/ext/test/http/curl_http_test.cc +++ b/ext/test/http/curl_http_test.cc @@ -300,16 +300,18 @@ TEST_F(BasicCurlHttpTests, CurlHttpOperations) http_client::Headers headers = { {"name1", "value1_1"}, {"name1", "value1_2"}, {"name2", "value3"}, {"name3", "value3"}}; + http_client::Compression compression = http_client::Compression::kNone; + curl::HttpOperation http_operations1(http_client::Method::Head, "/get", no_ssl, handler, headers, - body, true); + body, compression, true); http_operations1.Send(); curl::HttpOperation http_operations2(http_client::Method::Get, "/get", no_ssl, handler, headers, - body, true); + body, compression, true); http_operations2.Send(); curl::HttpOperation http_operations3(http_client::Method::Get, "/get", no_ssl, handler, headers, - body, false); + body, compression, false); http_operations3.Send(); delete handler; } diff --git a/functional/otlp/func_http_main.cc b/functional/otlp/func_http_main.cc index 98731f1d80..68638c97a9 100644 --- a/functional/otlp/func_http_main.cc +++ b/functional/otlp/func_http_main.cc @@ -348,6 +348,8 @@ int test_range_tls_13_10(); int test_range_tls_13_11(); int test_range_tls_13_12(); +int test_gzip_compression(); + static const test_case all_tests[] = {{"basic", test_basic}, {"cert-not-found", test_cert_not_found}, {"cert-invalid", test_cert_invalid}, @@ -387,6 +389,7 @@ static const test_case all_tests[] = {{"basic", test_basic}, {"range-tls-13-10", test_range_tls_13_10}, {"range-tls-13-11", test_range_tls_13_11}, {"range-tls-13-12", test_range_tls_13_12}, + {"gzip-compression", test_gzip_compression}, {"", nullptr}}; void list_test_cases() @@ -1794,3 +1797,36 @@ int test_range_tls_13_12() // Impossible return expect_connection_failed(); } + +int test_gzip_compression() +{ + otlp::OtlpHttpExporterOptions opts; + + set_common_opts(opts); + opts.compression = "gzip"; + + instrumented_payload(opts); + + if (opt_mode == MODE_NONE) + { + return expect_connection_failed(); + } + + if (!opt_secure && (opt_mode == MODE_HTTP)) + { + return expect_success(); + } + + if (!opt_secure && (opt_mode == MODE_HTTPS)) + { + return expect_export_failed(); + } + + if (opt_secure && (opt_mode == MODE_HTTP)) + { + return expect_connection_failed(); + } + + // Impossible + return expect_connection_failed(); +} diff --git a/test_common/include/opentelemetry/test_common/ext/http/client/nosend/http_client_nosend.h b/test_common/include/opentelemetry/test_common/ext/http/client/nosend/http_client_nosend.h index 50becac5dc..3207857743 100644 --- a/test_common/include/opentelemetry/test_common/ext/http/client/nosend/http_client_nosend.h +++ b/test_common/include/opentelemetry/test_common/ext/http/client/nosend/http_client_nosend.h @@ -61,6 +61,12 @@ class Request : public opentelemetry::ext::http::client::Request timeout_ms_ = timeout_ms; } + void SetCompression( + const opentelemetry::ext::http::client::Compression &compression) noexcept override + { + compression_ = compression; + } + public: opentelemetry::ext::http::client::Method method_; opentelemetry::ext::http::client::HttpSslOptions ssl_options_; @@ -68,6 +74,8 @@ class Request : public opentelemetry::ext::http::client::Request opentelemetry::ext::http::client::Headers headers_; std::string uri_; std::chrono::milliseconds timeout_ms_{5000}; // ms + opentelemetry::ext::http::client::Compression compression_{ + opentelemetry::ext::http::client::Compression::kNone}; }; class Response : public opentelemetry::ext::http::client::Response