Merge branch 'main' into metrics-benchmark

.github/workflows/ci.yml

@@ -41,9 +41,11 @@ jobs:
         CC: /usr/bin/gcc-12
         CXX: /usr/bin/g++-12
         GOOGLETEST_VERSION: 1.12.1
+        PROTOBUF_VERSION: 21.12
       run: |
         sudo -E ./ci/setup_cmake.sh
         sudo -E ./ci/setup_ci_environment.sh
+        sudo -E ./ci/install_protobuf.sh
     - name: run cmake gcc (maintainer mode)
       env:
         CC: /usr/bin/gcc-12
@@ -64,9 +66,11 @@ jobs:
         CC: /usr/bin/clang-14
         CXX: /usr/bin/clang++-14
         GOOGLETEST_VERSION: 1.12.1
+        PROTOBUF_VERSION: 21.12
       run: |
         sudo -E ./ci/setup_cmake.sh
         sudo -E ./ci/setup_ci_environment.sh
+        sudo -E ./ci/install_protobuf.sh
     - name: run cmake clang (maintainer mode)
       env:
         CC: /usr/bin/clang-14

CHANGELOG.md

@@ -15,6 +15,9 @@ Increment the:
 
 ## [Unreleased]
 
+* Convert Prometheus Exporter to Pull MetricReader [#1953](https://github.com/open-telemetry/opentelemetry-cpp/pull/1953)
+* Upgrade prometheus-cpp to v1.1.0 [#1954](https://github.com/open-telemetry/opentelemetry-cpp/pull/1954)
+
 ## [1.8.2] 2023-01-31
 
 * Remove redundant macro check in nostd::shared_ptr [#1939](https://github.com/open-telemetry/opentelemetry-cpp/pull/1939)
@@ -64,6 +67,8 @@ Increment the:
 * [MAINTAINER DOC] Define and document a deprecation process,
   [DEPRECATION] Deprecate the Jaeger exporter,
   implemented by [#1923](https://github.com/open-telemetry/opentelemetry-cpp/pull/1923)
+* [BUILD] OTLP HTTP Exporter has build warnings in maintainer mode
+  [#1943](https://github.com/open-telemetry/opentelemetry-cpp/pull/1943)
 
 Deprecations:
 

INSTALL.md

@@ -320,6 +320,10 @@ with `vcpkg install opentelemetry-cpp` and follow the then displayed
 descriptions. Please see the vcpkg project for any issues regarding the
 packaging.
 
+If you are using [alpine linux](https://www.alpinelinux.org/) you can install
+the [opentelemetry-cpp packages](https://pkgs.alpinelinux.org/packages?name=opentelemetry-cpp-*)
+with `apk add -X http://dl-cdn.alpinelinux.org/alpine/edge/testing opentelemetry-cpp-dev`.
+
 Please note, these packages are not officially provided and maintained by
 OpenTelemetry C++ project, and are just listed here to consolidate all such
 efforts for ease of developers.

bazel/repository.bzl

@@ -119,10 +119,10 @@ def opentelemetry_cpp_deps():
     maybe(
         http_archive,
         name = "com_github_jupp0r_prometheus_cpp",
-        sha256 = "07018db604ea3e61f5078583e87c80932ea10c300d979061490ee1b7dc8e3a41",
-        strip_prefix = "prometheus-cpp-1.0.0",
+        sha256 = "397544fe91e183029120b4eebcfab24ed9ec833d15850aae78fd5db19062d13a",
+        strip_prefix = "prometheus-cpp-1.1.0",
         urls = [
-            "https://github.com/jupp0r/prometheus-cpp/archive/refs/tags/v1.0.0.tar.gz",
+            "https://github.com/jupp0r/prometheus-cpp/archive/refs/tags/v1.1.0.tar.gz",
         ],
     )
 

ci/do_ci.sh

@@ -94,6 +94,8 @@ elif [[ "$1" == "cmake.maintainer.test" ]]; then
   cd "${BUILD_DIR}"
   rm -rf *
   cmake -DCMAKE_BUILD_TYPE=Debug  \
+        -DWITH_OTLP=ON \
+        -DWITH_OTLP_HTTP=ON \
         -DWITH_PROMETHEUS=ON \
         -DWITH_EXAMPLES=ON \
         -DWITH_EXAMPLES_HTTP=ON \
@@ -296,7 +298,9 @@ elif [[ "$1" == "bazel.asan" ]]; then
   //examples/metrics_simple:metrics_ostream_example > /dev/null
   exit 0
 elif [[ "$1" == "bazel.tsan" ]]; then
-  bazel $BAZEL_STARTUP_OPTIONS test --config=tsan $BAZEL_TEST_OPTIONS_ASYNC //...
+# TODO - potential race condition in Civetweb server used by prometheus-cpp during shutdown
+# https://github.com/civetweb/civetweb/issues/861, so removing prometheus from the test
+  bazel $BAZEL_STARTUP_OPTIONS test --config=tsan $BAZEL_TEST_OPTIONS_ASYNC  -- //... -//exporters/prometheus/...
   bazel $BAZEL_STARTUP_OPTIONS run --config=tsan $BAZEL_TEST_OPTIONS_ASYNC \
   //examples/metrics_simple:metrics_ostream_example > /dev/null
   exit 0

ci/install_protobuf.sh

@@ -5,12 +5,38 @@
 
 set -e
 
-[ -z "${PROTOBUF_VERSION}" ] && export PROTOBUF_VERSION="3.11.4"
+[ -z "${PROTOBUF_VERSION}" ] && export PROTOBUF_VERSION="21.12"
+
+#
+# Note
+#
+# protobuf uses two release number schemes,
+# for example 3.21.12 and 21.12,
+# and both tags corresponds to the same commit:
+#
+#   commit f0dc78d7e6e331b8c6bb2d5283e06aa26883ca7c (HEAD -> release-3.21.12, tag: v3.21.12, tag: v21.12)
+#   Author: Protobuf Team Bot <protobuf-team-bot@google.com>
+#   Date:   Mon Dec 12 16:03:12 2022 -0800
+#
+#       Updating version.json and repo version numbers to: 21.12
+#
+# tag v21.12 corresponds to the 'protoc version', or repo version
+# tag v3.21.12 corresponds to the 'cpp version'
+#
+# protobuf-cpp-3.21.12.tar.gz:
+# - is provided under releases/download/v21.12
+# - is no longer provided under releases/download/v3.21.12,
+#
+# Use the "repo version number" (PROTOBUF_VERSION=21.12)
+# when calling this script
+#
+
+export CPP_PROTOBUF_VERSION="3.${PROTOBUF_VERSION}"
 
 cd /
-wget https://github.com/google/protobuf/releases/download/v${PROTOBUF_VERSION}/protobuf-cpp-${PROTOBUF_VERSION}.tar.gz
-tar zxf protobuf-cpp-${PROTOBUF_VERSION}.tar.gz --no-same-owner
-cd protobuf-${PROTOBUF_VERSION}
+wget https://github.com/google/protobuf/releases/download/v${PROTOBUF_VERSION}/protobuf-cpp-${CPP_PROTOBUF_VERSION}.tar.gz
+tar zxf protobuf-cpp-${CPP_PROTOBUF_VERSION}.tar.gz --no-same-owner
+cd protobuf-${CPP_PROTOBUF_VERSION}
 ./configure
-make && make install
+make -j $(nproc) && make install
 ldconfig

examples/prometheus/main.cc

@@ -34,21 +34,16 @@ void InitMetrics(const std::string &name, const std::string &addr)
   }
   std::puts("PrometheusExporter example program running ...");
 
-  std::unique_ptr<metrics_sdk::PushMetricExporter> exporter{
-      new metrics_exporter::PrometheusExporter(opts)};
-
   std::string version{"1.2.0"};
   std::string schema{"https://opentelemetry.io/schemas/1.2.0"};
 
+  std::shared_ptr<metrics_sdk::MetricReader> prometheus_exporter(
+      new metrics_exporter::PrometheusExporter(opts));
+
   // Initialize and set the global MeterProvider
-  metrics_sdk::PeriodicExportingMetricReaderOptions options;
-  options.export_interval_millis = std::chrono::milliseconds(1000);
-  options.export_timeout_millis  = std::chrono::milliseconds(500);
-  std::unique_ptr<metrics_sdk::MetricReader> reader{
-      new metrics_sdk::PeriodicExportingMetricReader(std::move(exporter), options)};
   auto provider = std::shared_ptr<metrics_api::MeterProvider>(new metrics_sdk::MeterProvider());
   auto p        = std::static_pointer_cast<metrics_sdk::MeterProvider>(provider);
-  p->AddMetricReader(std::move(reader));
+  p->AddMetricReader(prometheus_exporter);
 
   // counter view
   std::string counter_name = name + "_counter";

exporters/jaeger/CMakeLists.txt

@@ -1,3 +1,6 @@
+# Copyright The OpenTelemetry Authors
+# SPDX-License-Identifier: Apache-2.0
+
 include_directories(thrift-gen)
 
 find_package(Thrift REQUIRED)

exporters/otlp/src/otlp_metric_utils.cc

@@ -266,7 +266,7 @@ sdk::metrics::AggregationTemporality OtlpMetricUtils::DeltaTemporalitySelector(
 }
 
 sdk::metrics::AggregationTemporality OtlpMetricUtils::CumulativeTemporalitySelector(
-    sdk::metrics::InstrumentType instrument_type) noexcept
+    sdk::metrics::InstrumentType /* instrument_type */) noexcept
 {
   return sdk::metrics::AggregationTemporality::kCumulative;
 }

exporters/prometheus/BUILD

@@ -96,6 +96,7 @@ cc_test(
     deps = [
         ":prometheus_exporter",
         ":prometheus_test_helper",
+        "//sdk/src/metrics",
         "@com_google_googletest//:gtest_main",
     ],
 )
@@ -112,6 +113,7 @@ cc_test(
     deps = [
         ":prometheus_collector",
         ":prometheus_test_helper",
+        "//sdk/src/metrics",
         "@com_google_googletest//:gtest_main",
     ],
 )

exporters/prometheus/include/opentelemetry/exporters/prometheus/collector.h

@@ -10,6 +10,7 @@
 #include <prometheus/collectable.h>
 #include <prometheus/metric_family.h>
 #include "opentelemetry/exporters/prometheus/exporter_utils.h"
+#include "opentelemetry/sdk/metrics/metric_reader.h"
 
 namespace prometheus_client = ::prometheus;
 
@@ -30,7 +31,7 @@ class PrometheusCollector : public prometheus_client::Collectable
    * This constructor initializes the collection for metrics to export
    * in this class with default capacity
    */
-  explicit PrometheusCollector(size_t max_collection_size = 2048);
+  explicit PrometheusCollector(sdk::metrics::MetricReader *reader);
 
   /**
    * Collects all metrics data from metricsToCollect collection.
@@ -39,41 +40,8 @@ class PrometheusCollector : public prometheus_client::Collectable
    */
   std::vector<prometheus_client::MetricFamily> Collect() const override;
 
-  /**
-   * This function is called by export() function and add the collection of
-   * records to the metricsToCollect collection
-   *
-   * @param records a collection of records to add to the metricsToCollect collection
-   */
-  void AddMetricData(const sdk::metrics::ResourceMetrics &data);
-
-  /**
-   * Get the current collection in the collector.
-   *
-   * @return the current metricsToCollect collection
-   */
-  std::vector<std::unique_ptr<sdk::metrics::ResourceMetrics>> &GetCollection();
-
-  /**
-   * Gets the maximum size of the collection.
-   *
-   * @return max collection size
-   */
-  int GetMaxCollectionSize() const;
-
 private:
-  /**
-   * Collection of metrics data from the export() function, and to be export
-   * to user when they send a pull request. This collection is a pointer
-   * to a collection so Collect() is able to clear the collection, even
-   * though it is a const function.
-   */
-  mutable std::vector<std::unique_ptr<sdk::metrics::ResourceMetrics>> metrics_to_collect_;
-
-  /**
-   * Maximum size of the metricsToCollect collection.
-   */
-  size_t max_collection_size_;
+  sdk::metrics::MetricReader *reader_;
 
   /*
    * Lock when operating the metricsToCollect collection

exporters/prometheus/include/opentelemetry/exporters/prometheus/exporter.h

@@ -12,7 +12,7 @@
 #include "opentelemetry/exporters/prometheus/collector.h"
 #include "opentelemetry/nostd/span.h"
 #include "opentelemetry/sdk/common/env_variables.h"
-#include "opentelemetry/sdk/metrics/push_metric_exporter.h"
+#include "opentelemetry/sdk/metrics/metric_reader.h"
 #include "opentelemetry/version.h"
 
 /**
@@ -46,7 +46,7 @@ struct PrometheusExporterOptions
   std::string url = GetPrometheusDefaultHttpEndpoint();
 };
 
-class PrometheusExporter : public sdk::metrics::PushMetricExporter
+class PrometheusExporter : public sdk::metrics::MetricReader
 {
 public:
   /**
@@ -56,54 +56,12 @@ class PrometheusExporter : public sdk::metrics::PushMetricExporter
    */
   PrometheusExporter(const PrometheusExporterOptions &options);
 
-  /**
-   * Get the AggregationTemporality for Prometheus exporter
-   *
-   * @return AggregationTemporality
-   */
   sdk::metrics::AggregationTemporality GetAggregationTemporality(
       sdk::metrics::InstrumentType instrument_type) const noexcept override;
 
-  /**
-   * Exports a batch of Metric Records.
-   * @param records: a collection of records to export
-   * @return: returns a ReturnCode detailing a success, or type of failure
-   */
-  sdk::common::ExportResult Export(const sdk::metrics::ResourceMetrics &data) noexcept override;
-
-  /**
-   * Force flush the exporter.
-   */
-  bool ForceFlush(
-      std::chrono::microseconds timeout = (std::chrono::microseconds::max)()) noexcept override;
-
-  /**
-   * Shuts down the exporter and does cleanup.
-   * Since Prometheus is a pull based interface,
-   * we cannot serve data remaining in the intermediate
-   * collection to to client an HTTP request being sent,
-   * so we flush the data.
-   */
-  bool Shutdown(std::chrono::microseconds timeout = std::chrono::microseconds(0)) noexcept override;
-
-  /**
-   * @return: returns a shared_ptr to
-   * the PrometheusCollector instance
-   */
-  std::shared_ptr<PrometheusCollector> &GetCollector();
-
-  /**
-   * @return: Gets the shutdown status of the exporter
-   */
-  bool IsShutdown() const;
-
 private:
   // The configuration options associated with this exporter.
   const PrometheusExporterOptions options_;
-  /**
-   * exporter shutdown status
-   */
-  bool is_shutdown_;
 
   /**
    * Pointer to a
@@ -117,16 +75,11 @@ class PrometheusExporter : public sdk::metrics::PushMetricExporter
    */
   std::unique_ptr<::prometheus::Exposer> exposer_;
 
-  /**
-   * friend class for testing
-   */
-  friend class PrometheusExporterTest;
+  bool OnForceFlush(std::chrono::microseconds timeout) noexcept override;
 
-  /**
-   * PrometheusExporter constructor with no parameters
-   * Used for testing only
-   */
-  PrometheusExporter();
+  bool OnShutDown(std::chrono::microseconds timeout) noexcept override;
+
+  void OnInitialized() noexcept override;
 };
 }  // namespace metrics
 }  // namespace exporter

exporters/prometheus/include/opentelemetry/exporters/prometheus/exporter_utils.h

@@ -29,7 +29,7 @@ class PrometheusExporterUtils
    * @return a collection of translated metrics that is acceptable by Prometheus
    */
   static std::vector<::prometheus::MetricFamily> TranslateToPrometheus(
-      const std::vector<std::unique_ptr<sdk::metrics::ResourceMetrics>> &data);
+      const sdk::metrics::ResourceMetrics &data);
 
 private:
   /**