chore: add opentelemetry-otlp integration example (#53)

## Motivation

* Provide specific example of how to set up various opentelemetry
configrations from tracing When connecting tracing and opentelemetry
ecosystems

## Solution

This example aims to demonstrate the following points
* tracing-opentelemetry configuration is done through the open telemetry
sdk
* telemetry export is done using a plugin (opentelemetry-otlp), and the
plugin provides the sdk construct helper methods

Cargo.toml

@@ -41,12 +41,17 @@ smallvec = { version = "1.0", optional = true }
 [dev-dependencies]
 async-trait = "0.1.56"
 criterion = { version = "0.4.0", default-features = false, features = ["html_reports"] }
+opentelemetry = { version = "0.20.0", features = ["trace", "metrics", "rt-tokio"] }
+opentelemetry_sdk = { version = "0.20.0", default-features = false, features = ["trace"] }
 opentelemetry-jaeger = "0.19.0"
-opentelemetry-stdout = { version = "0.1.0", features = ["trace"] }
+opentelemetry-stdout = { version = "0.1.0", features = ["trace", "metrics"] }
+opentelemetry-otlp = { version = "0.13.0", features = ["metrics"] }
+opentelemetry-semantic-conventions = "0.12.0"
 futures-util = { version = "0.3", default-features = false }
 tokio = { version = "1", features = ["full"] }
 tokio-stream = "0.1"
 tracing = { version = "0.1.35", default-features = false, features = ["std", "attributes"] }
+tracing-subscriber = { version = "0.3.0", default-features = false, features = ["registry", "std", "fmt"] }
 
 [target.'cfg(not(target_os = "windows"))'.dev-dependencies]
 pprof = { version = "0.11.1", features = ["flamegraph", "criterion"] }

examples/opentelemetry-otlp.rs

@@ -0,0 +1,161 @@
+use opentelemetry::{global, runtime, Key, KeyValue};
+use opentelemetry_otlp::TonicExporterBuilder;
+use opentelemetry_sdk::{
+    metrics::{
+        reader::{DefaultAggregationSelector, DefaultTemporalitySelector},
+        Aggregation, Instrument, MeterProvider, PeriodicReader, Stream,
+    },
+    trace::{BatchConfig, RandomIdGenerator, Sampler, Tracer},
+    Resource,
+};
+use opentelemetry_semantic_conventions::{
+    resource::{DEPLOYMENT_ENVIRONMENT, SERVICE_NAME, SERVICE_VERSION},
+    SCHEMA_URL,
+};
+use tracing_core::Level;
+use tracing_opentelemetry::{MetricsLayer, OpenTelemetryLayer};
+use tracing_subscriber::{layer::SubscriberExt, util::SubscriberInitExt};
+
+// Create a Resource that captures information about the entity for which telemetry is recorded.
+fn resource() -> Resource {
+    Resource::from_schema_url(
+        [
+            KeyValue::new(SERVICE_NAME, env!("CARGO_PKG_NAME")),
+            KeyValue::new(SERVICE_VERSION, env!("CARGO_PKG_VERSION")),
+            KeyValue::new(DEPLOYMENT_ENVIRONMENT, "develop"),
+        ],
+        SCHEMA_URL,
+    )
+}
+
+// Construct MeterProvider for MetricsLayer
+fn init_meter_provider() -> MeterProvider {
+    // Currently we could not access MeterProviderBuilder from opentelemetry_otlp
+    // However to customize View we need MeterBuilder, so manually construct.
+    let exporter = opentelemetry_otlp::MetricsExporter::new(
+        TonicExporterBuilder::default(),
+        Box::new(DefaultTemporalitySelector::new()),
+        Box::new(DefaultAggregationSelector::new()),
+    )
+    .unwrap();
+
+    let reader = PeriodicReader::builder(exporter, runtime::Tokio)
+        .with_interval(std::time::Duration::from_secs(30))
+        .build();
+
+    // For debugging in development
+    let stdout_reader = PeriodicReader::builder(
+        opentelemetry_stdout::MetricsExporter::default(),
+        runtime::Tokio,
+    )
+    .build();
+
+    // Rename foo metrics to foo_named and drop key_2 attribute
+    let view_foo = |instrument: &Instrument| -> Option<Stream> {
+        if instrument.name == "foo" {
+            Some(
+                Stream::new()
+                    .name("foo_named")
+                    .allowed_attribute_keys([Key::from("key_1")]),
+            )
+        } else {
+            None
+        }
+    };
+
+    // Set Custom histogram boundaries for baz metrics
+    let view_baz = |instrument: &Instrument| -> Option<Stream> {
+        if instrument.name == "baz" {
+            Some(
+                Stream::new()
+                    .name("baz")
+                    .aggregation(Aggregation::ExplicitBucketHistogram {
+                        boundaries: vec![0.0, 2.0, 4.0, 8.0, 16.0, 32.0, 64.0],
+                        record_min_max: true,
+                    }),
+            )
+        } else {
+            None
+        }
+    };
+
+    let meter_provider = MeterProvider::builder()
+        .with_resource(resource())
+        .with_reader(reader)
+        .with_reader(stdout_reader)
+        .with_view(view_foo)
+        .with_view(view_baz)
+        .build();
+
+    global::set_meter_provider(meter_provider.clone());
+
+    meter_provider
+}
+
+// Construct Tracer for OpenTelemetryLayer
+fn init_tracer() -> Tracer {
+    opentelemetry_otlp::new_pipeline()
+        .tracing()
+        .with_trace_config(
+            opentelemetry_sdk::trace::Config::default()
+                // Customize sampling strategy
+                .with_sampler(Sampler::ParentBased(Box::new(Sampler::TraceIdRatioBased(
+                    1.0,
+                ))))
+                // If export trace to AWS X-Ray, you can use XrayIdGenerator
+                .with_id_generator(RandomIdGenerator::default())
+                .with_resource(resource()),
+        )
+        .with_batch_config(BatchConfig::default())
+        .with_exporter(opentelemetry_otlp::new_exporter().tonic())
+        .install_batch(runtime::Tokio)
+        .unwrap()
+}
+
+// Initialize tracing-subscriber and return OtelGuard for opentelemetry-related termination processing
+fn init_tracing_subscriber() -> OtelGuard {
+    let meter_provider = init_meter_provider();
+
+    tracing_subscriber::registry()
+        .with(tracing_subscriber::filter::LevelFilter::from_level(
+            Level::INFO,
+        ))
+        .with(tracing_subscriber::fmt::layer())
+        .with(MetricsLayer::new(meter_provider.clone()))
+        .with(OpenTelemetryLayer::new(init_tracer()))
+        .init();
+
+    OtelGuard { meter_provider }
+}
+
+struct OtelGuard {
+    meter_provider: MeterProvider,
+}
+
+impl Drop for OtelGuard {
+    fn drop(&mut self) {
+        if let Err(err) = self.meter_provider.shutdown() {
+            eprintln!("{err:?}");
+        }
+        opentelemetry::global::shutdown_tracer_provider();
+    }
+}
+
+#[tokio::main]
+async fn main() {
+    let _guard = init_tracing_subscriber();
+
+    foo().await;
+}
+
+#[tracing::instrument]
+async fn foo() {
+    tracing::info!(
+        monotonic_counter.foo = 1_u64,
+        key_1 = "bar",
+        key_2 = 10,
+        "handle foo",
+    );
+
+    tracing::info!(histogram.baz = 10, "histogram example",);
+}