Add kafka reporter. (#61)

apache · Jun 19, 2023 · 84fbd00 · 84fbd00
1 parent 302526e
commit 84fbd00
Show file tree

Hide file tree

Showing 17 changed files with 934 additions and 71 deletions.
diff --git a/.github/workflows/ci.yaml b/.github/workflows/ci.yaml
@@ -91,6 +91,7 @@ jobs:
           - ""
           - "--features management"
           - "--features vendored"
+          - "--features kafka-reporter"
           - "--all-features"
     runs-on: ubuntu-20.04
     steps:

diff --git a/.github/workflows/e2e.yml b/.github/workflows/e2e.yml
@@ -40,3 +40,5 @@ jobs:
           pip3 install setuptools
           pip3 install -r requirements.txt
           python3 e2e/run_e2e.py --expected_file=e2e/data/expected_context.yaml --max_retry_times=3 --target_path=/ping
+          docker-compose -f docker-compose.e2e.yml logs producer consumer
+          docker-compose -f docker-compose.e2e.yml exec -T producer /build/target/release/e2e-kafka
diff --git a/Cargo.toml b/Cargo.toml
@@ -36,6 +36,7 @@ rust-version = "1.65"
 [features]
 management = ["hostname", "systemstat"]
 vendored = ["protobuf-src"]
+kafka-reporter = ["rdkafka"]
 
 mock = []  # For internal integration testing only, do not use.
 
@@ -46,11 +47,13 @@ cfg-if = "1.0.0"
 futures-core = "0.3.21"
 futures-util = "0.3.21"
 hostname = { version = "0.3.1", optional = true }
+libz-sys = "1.1.9"
 once_cell = "1.14.0"
 parking_lot = "0.12.1"
 portable-atomic = { version = "0.3.13", features = ["float"] }
 prost = "0.11.0"
 prost-derive = "0.11.0"
+rdkafka = { version = "0.32.2", optional = true }
 serde = { version = "1.0.143", features = ["derive"] }
 systemstat = { version = "0.2.0", optional = true }
 thiserror = "1.0.32"

diff --git a/README.md b/README.md
@@ -212,6 +212,42 @@ async fn handle(tracer: Tracer) {
 }
 ```
 
+# Advanced Reporter
+
+The advanced report provides an alternative way to submit the agent collected data to the backend.
+
+## kafka reporter
+
+The Kafka reporter plugin support report traces, metrics, logs, instance properties to Kafka cluster.
+
+Make sure the feature `kafka-reporter` is enabled.
+
+```rust
+#[cfg(feature = "kafka-reporter")]
+mod example {
+    use skywalking::reporter::Report;
+    use skywalking::reporter::kafka::{KafkaReportBuilder, KafkaReporter, RDKafkaClientConfig};
+
+    async fn do_something(reporter: &impl Report) {
+        // ....
+    }
+
+    async fn foo() {
+        let mut client_config = RDKafkaClientConfig::new();
+        client_config
+            .set("bootstrap.servers", "broker:9092")
+            .set("message.timeout.ms", "6000");
+
+        let (reporter, reporting) = KafkaReportBuilder::new(client_config).build().await.unwrap();
+        let handle = reporting.spawn();
+
+        do_something(&reporter);
+
+        handle.await.unwrap();
+    }
+}
+```
+
 # How to compile?
 
 If you have `skywalking-(VERSION).crate`, you can unpack it with the way as follows:

diff --git a/dist-material/LICENSE b/dist-material/LICENSE
@@ -220,9 +220,11 @@ The text of each license is also included in licenses/LICENSE-[project].txt.
     https://crates.io/crates/autocfg/1.1.0 1.1.0 Apache-2.0 OR MIT
     https://crates.io/crates/base64/0.13.0 0.13.0 Apache-2.0 OR MIT
     https://crates.io/crates/bitflags/1.3.1 1.3.1 Apache-2.0 OR MIT
-    https://crates.io/crates/cc/1.0.0 1.0.0 Apache-2.0 OR MIT
+    https://crates.io/crates/cc/1.0.18 1.0.18 Apache-2.0 OR MIT
     https://crates.io/crates/cfg-if/0.1.2 0.1.2 Apache-2.0 OR MIT
     https://crates.io/crates/cfg-if/1.0.0 1.0.0 Apache-2.0 OR MIT
+    https://crates.io/crates/derivative/2.0.0 2.0.0 Apache-2.0 OR MIT
+    https://crates.io/crates/dtoa/0.4.0 0.4.0 Apache-2.0 OR MIT
     https://crates.io/crates/either/1.0.0 1.0.0 Apache-2.0 OR MIT
     https://crates.io/crates/fixedbitset/0.4.0 0.4.0 Apache-2.0 OR MIT
     https://crates.io/crates/fnv/1.0.5 1.0.5 Apache-2.0 OR MIT
@@ -242,10 +244,12 @@ The text of each license is also included in licenses/LICENSE-[project].txt.
     https://crates.io/crates/hyper-timeout/0.4.0 0.4.0 Apache-2.0 OR MIT
     https://crates.io/crates/indexmap/1.6.2 1.6.2 Apache-2.0 OR MIT
     https://crates.io/crates/itertools/0.10.0 0.10.0 Apache-2.0 OR MIT
+    https://crates.io/crates/itoa/0.3.0 0.3.0 Apache-2.0 OR MIT
     https://crates.io/crates/itoa/0.4.1 0.4.1 Apache-2.0 OR MIT
     https://crates.io/crates/itoa/1.0.1 1.0.1 Apache-2.0 OR MIT
     https://crates.io/crates/lazy_static/1.4.0 1.4.0 Apache-2.0 OR MIT
     https://crates.io/crates/libc/0.2.114 0.2.114 Apache-2.0 OR MIT
+    https://crates.io/crates/libz-sys/1.1.9 1.1.9 Apache-2.0 OR MIT
     https://crates.io/crates/lock_api/0.4.6 0.4.6 Apache-2.0 OR MIT
     https://crates.io/crates/log/0.4.17 0.4.17 Apache-2.0 OR MIT
     https://crates.io/crates/match_cfg/0.1.0 0.1.0 Apache-2.0 OR MIT
@@ -254,6 +258,7 @@ The text of each license is also included in licenses/LICENSE-[project].txt.
     https://crates.io/crates/miow/0.3.6 0.3.6 Apache-2.0 OR MIT
     https://crates.io/crates/multimap/0.8.0 0.8.0 Apache-2.0 OR MIT
     https://crates.io/crates/ntapi/0.3.0 0.3.0 Apache-2.0 OR MIT
+    https://crates.io/crates/num-traits/0.1.32 0.1.32 Apache-2.0 OR MIT
     https://crates.io/crates/num_cpus/1.8.0 1.8.0 Apache-2.0 OR MIT
     https://crates.io/crates/num_threads/0.1.2 0.1.2 Apache-2.0 OR MIT
     https://crates.io/crates/once_cell/1.14.0 1.14.0 Apache-2.0 OR MIT
@@ -267,9 +272,11 @@ The text of each license is also included in licenses/LICENSE-[project].txt.
     https://crates.io/crates/pin-project-internal/1.0.2 1.0.2 Apache-2.0 OR MIT
     https://crates.io/crates/pin-project-lite/0.2.9 0.2.9 Apache-2.0 OR MIT
     https://crates.io/crates/pin-utils/0.1.0 0.1.0 Apache-2.0 OR MIT
+    https://crates.io/crates/pkg-config/0.3.9 0.3.9 Apache-2.0 OR MIT
     https://crates.io/crates/portable-atomic/0.3.13 0.3.13 Apache-2.0 OR MIT
     https://crates.io/crates/ppv-lite86/0.2.8 0.2.8 Apache-2.0 OR MIT
     https://crates.io/crates/prettyplease/0.1.0 0.1.0 Apache-2.0 OR MIT
+    https://crates.io/crates/proc-macro-crate/0.1.4 0.1.4 Apache-2.0 OR MIT
     https://crates.io/crates/proc-macro2/1.0.32 1.0.32 Apache-2.0 OR MIT
     https://crates.io/crates/quote/1.0.0 1.0.0 Apache-2.0 OR MIT
     https://crates.io/crates/rand/0.4.1 0.4.1 Apache-2.0 OR MIT
@@ -283,6 +290,7 @@ The text of each license is also included in licenses/LICENSE-[project].txt.
     https://crates.io/crates/scopeguard/1.1.0 1.1.0 Apache-2.0 OR MIT
     https://crates.io/crates/serde/1.0.143 1.0.143 Apache-2.0 OR MIT
     https://crates.io/crates/serde_derive/1.0.143 1.0.143 Apache-2.0 OR MIT
+    https://crates.io/crates/serde_json/1.0.0 1.0.0 Apache-2.0 OR MIT
     https://crates.io/crates/signal-hook-registry/1.1.1 1.1.1 Apache-2.0 OR MIT
     https://crates.io/crates/smallvec/1.6.1 1.6.1 Apache-2.0 OR MIT
     https://crates.io/crates/socket2/0.3.17 0.3.17 Apache-2.0 OR MIT
@@ -295,10 +303,12 @@ The text of each license is also included in licenses/LICENSE-[project].txt.
     https://crates.io/crates/thiserror-impl/1.0.32 1.0.32 Apache-2.0 OR MIT
     https://crates.io/crates/time/0.3.9 0.3.9 Apache-2.0 OR MIT
     https://crates.io/crates/tokio-io-timeout/1.0.1 1.0.1 Apache-2.0 OR MIT
+    https://crates.io/crates/toml/0.5.2 0.5.2 Apache-2.0 OR MIT
     https://crates.io/crates/unicode-segmentation/1.2.0 1.2.0 Apache-2.0 OR MIT
     https://crates.io/crates/unicode-width/0.1.4 0.1.4 Apache-2.0 OR MIT
     https://crates.io/crates/unicode-xid/0.2.0 0.2.0 Apache-2.0 OR MIT
     https://crates.io/crates/uuid/1.1.2 1.1.2 Apache-2.0 OR MIT
+    https://crates.io/crates/vcpkg/0.2.0 0.2.0 Apache-2.0 OR MIT
     https://crates.io/crates/vec_map/0.8.0 0.8.0 Apache-2.0 OR MIT
     https://crates.io/crates/version_check/0.9.0 0.9.0 Apache-2.0 OR MIT
     https://crates.io/crates/winapi/0.3.9 0.3.9 Apache-2.0 OR MIT
@@ -319,6 +329,8 @@ The text of each license is also included in licenses/LICENSE-[project].txt.
 
     https://crates.io/crates/fuchsia-zircon/0.3.1 0.3.1 BSD-3-Clause
     https://crates.io/crates/fuchsia-zircon-sys/0.3.1 0.3.1 BSD-3-Clause
+    https://crates.io/crates/num_enum/0.5.0 0.5.0 BSD-3-Clause
+    https://crates.io/crates/num_enum_derive/0.5.0 0.5.0 BSD-3-Clause
 
 ========================================================================
 MIT licenses
@@ -345,6 +357,8 @@ The text of each license is also included in licenses/LICENSE-[project].txt.
     https://crates.io/crates/mio/0.8.1 0.8.1 MIT
     https://crates.io/crates/nom/7.0.0 7.0.0 MIT
     https://crates.io/crates/proc-macro-error/0.2.0 0.2.0 MIT
+    https://crates.io/crates/rdkafka/0.32.2 0.32.2 MIT
+    https://crates.io/crates/rdkafka-sys/4.5.0+1.9.2 4.5.0+1.9.2 MIT
     https://crates.io/crates/redox_syscall/0.1.38 0.1.38 MIT
     https://crates.io/crates/redox_syscall/0.2.8 0.2.8 MIT
     https://crates.io/crates/slab/0.4.2 0.4.2 MIT

diff --git a/dist-material/licenses/LICENSE-rdkafka-sys.txt b/dist-material/licenses/LICENSE-rdkafka-sys.txt
@@ -0,0 +1,21 @@
+MIT License
+
+Copyright (c) 2016 Federico Giraud
+
+Permission is hereby granted, free of charge, to any person obtaining a copy
+of this software and associated documentation files (the "Software"), to deal
+in the Software without restriction, including without limitation the rights
+to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
+copies of the Software, and to permit persons to whom the Software is
+furnished to do so, subject to the following conditions:
+
+The above copyright notice and this permission notice shall be included in all
+copies or substantial portions of the Software.
+
+THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
+IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
+FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
+AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
+LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
+OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
+SOFTWARE.
diff --git a/dist-material/licenses/LICENSE-rdkafka.txt b/dist-material/licenses/LICENSE-rdkafka.txt
@@ -0,0 +1,21 @@
+MIT License
+
+Copyright (c) 2016 Federico Giraud
+
+Permission is hereby granted, free of charge, to any person obtaining a copy
+of this software and associated documentation files (the "Software"), to deal
+in the Software without restriction, including without limitation the rights
+to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
+copies of the Software, and to permit persons to whom the Software is
+furnished to do so, subject to the following conditions:
+
+The above copyright notice and this permission notice shall be included in all
+copies or substantial portions of the Software.
+
+THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
+IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
+FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
+AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
+LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
+OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
+SOFTWARE.
diff --git a/docker-compose.e2e.yml b/docker-compose.e2e.yml
@@ -37,6 +37,8 @@ services:
     depends_on:
       collector:
         condition: service_healthy
+      broker:
+        condition: service_healthy
     healthcheck:
       test: [ "CMD", "curl", "http://0.0.0.0:8082/healthCheck" ]
       interval: 5s
@@ -54,3 +56,15 @@ services:
         condition: service_healthy
       consumer:
         condition: service_healthy
+      broker:
+        condition: service_healthy
+
+  broker:
+    image: landoop/fast-data-dev:3.3
+    container_name: broker
+    ports:
+      - "9092:9092"
+    healthcheck:
+      test: [ "CMD", "nc", "-zv", "0.0.0.0", "9092"]
+      interval: 5s
+      timeout: 5s
diff --git a/e2e/Cargo.toml b/e2e/Cargo.toml
@@ -22,9 +22,16 @@ authors = ["Apache Software Foundation"]
 edition = "2021"
 publish = false
 license = "Apache-2.0"
+default-run = "e2e"
+
+[[bin]]
+name = "e2e-kafka"
+path = "src/e2e_kafka.rs"
 
 [dependencies]
-skywalking = { path = ".." }
 hyper = { version = "0.14", features = ["full"] }
-tokio = { version = "1", features = ["full"] }
+prost = "0.11.0"
+rdkafka = "0.32.2"
+skywalking = { path = "..", features = ["kafka-reporter"] }
 structopt = "0.3"
+tokio = { version = "1", features = ["full"] }
diff --git a/e2e/docker/Dockerfile b/e2e/docker/Dockerfile
@@ -20,4 +20,5 @@ RUN apt-get update && apt-get install -y cmake protobuf-compiler
 WORKDIR /build
 COPY . /build/
 RUN cargo build --release --workspace
+ENV RUST_BACKTRACE=1
 ENTRYPOINT ["/build/target/release/e2e"]