Merge pull request collectd#4180 from octo/open-telemetry

[collectd 6] Write OpenTelemetry: New plugin to export metrics via OTLP over gRPC.

.cirrus.yml

@@ -1,6 +1,7 @@
 env:
   LANG: C
   CIRRUS_CLONE_DEPTH: 1
+  CIRRUS_CLONE_SUBMODULES: true
   DEFAULT_CONFIG_OPTS: --enable-debug --without-libstatgrab --disable-dependency-tracking
 
 

.github/workflows/build.yml

@@ -47,7 +47,9 @@ jobs:
       # this env var picked up by valgrind during make check phase
       VALGRIND_OPTS: "--errors-for-leak-kinds=definite"
     steps:
-    - uses: actions/checkout@v2
+    - uses: actions/checkout@v4
+      with:
+        submodules: true
     - run: type pkg-config
     - run: pkg-config --list-all | sort -u
     - name: Generate configure script
@@ -106,7 +108,9 @@ jobs:
       CONFIGURE_FLAGS: ${{ matrix.configure_flags }}
       VALGRIND_OPTS: "--errors-for-leak-kinds=definite"
     steps:
-    - uses: actions/checkout@v2
+    - uses: actions/checkout@v4
+      with:
+        submodules: true
     - run: type pkg-config
     - run: pkg-config --list-all | sort -u
     - name: Generate configure script

.github/workflows/distcheck.yml

@@ -22,6 +22,8 @@ jobs:
       VALGRIND_OPTS: "--errors-for-leak-kinds=definite"
     steps:
     - uses: actions/checkout@v4
+      with:
+        submodules: true
     - name: Install bzip2
       run: apt install -y bzip2
     - name: Print available packages

.gitmodules

@@ -1,3 +1,6 @@
 [submodule "gnulib"]
 	path = gnulib
 	url = git://git.savannah.gnu.org/gnulib.git
+[submodule "opentelemetry-proto"]
+	path = opentelemetry-proto
+	url = https://github.com/open-telemetry/opentelemetry-proto/

Makefile.am

@@ -47,6 +47,11 @@ EXTRA_DIST = \
 	bindings/perl/lib/Collectd/Unixsock.pm \
 	bindings/perl/uninstall_mod.pl \
 	contrib \
+	opentelemetry-proto/LICENSE \
+	opentelemetry-proto/opentelemetry/proto/collector/metrics/v1/metrics_service.proto \
+	opentelemetry-proto/opentelemetry/proto/common/v1/common.proto \
+	opentelemetry-proto/opentelemetry/proto/metrics/v1/metrics.proto \
+	opentelemetry-proto/opentelemetry/proto/resource/v1/resource.proto \
 	proto/collectd.proto \
 	proto/types.proto \
 	src/collectd-email.pod \
@@ -454,10 +459,6 @@ libplugin_mock_la_SOURCES = \
 libplugin_mock_la_CPPFLAGS = $(AM_CPPFLAGS) -DMOCK_TIME
 libplugin_mock_la_LIBADD = libmetric.la liboconfig.la libcommon.la libignorelist.la $(COMMON_LIBS)
 
-libformat_influxdb_la_SOURCES = \
-	src/utils/format_influxdb/format_influxdb.c \
-	src/utils/format_influxdb/format_influxdb.h
-
 libformat_graphite_la_SOURCES = \
 	src/utils/format_graphite/format_graphite.c \
 	src/utils/format_graphite/format_graphite.h
@@ -472,6 +473,30 @@ test_format_graphite_LDADD = \
 	libstrbuf.la \
 	-lm
 
+libformat_influxdb_la_SOURCES = \
+	src/utils/format_influxdb/format_influxdb.c \
+	src/utils/format_influxdb/format_influxdb.h
+
+if BUILD_PLUGIN_WRITE_OPEN_TELEMETRY
+noinst_LTLIBRARIES += libformat_open_telemetry.la
+libformat_open_telemetry_la_SOURCES = \
+	src/utils/format_open_telemetry/format_open_telemetry.cc \
+	src/utils/format_open_telemetry/format_open_telemetry.h \
+	src/utils/resource_metrics/resource_metrics.c \
+	src/utils/resource_metrics/resource_metrics.h \
+	opentelemetry/proto/collector/metrics/v1/metrics_service.pb.cc \
+	opentelemetry/proto/collector/metrics/v1/metrics_service.pb.h \
+	opentelemetry/proto/common/v1/common.pb.cc \
+	opentelemetry/proto/common/v1/common.pb.h \
+	opentelemetry/proto/metrics/v1/metrics.pb.cc \
+	opentelemetry/proto/metrics/v1/metrics.pb.h \
+	opentelemetry/proto/resource/v1/resource.pb.cc \
+	opentelemetry/proto/resource/v1/resource.pb.h
+libformat_open_telemetry_la_CPPFLAGS = $(AM_CPPFLAGS) $(BUILD_WITH_LIBPROTOBUF_CPPFLAGS)
+libformat_open_telemetry_la_LDFLAGS = $(AM_LDFLAGS) $(BUILD_WITH_LIBPROTOBUF_LDFLAGS)
+libformat_open_telemetry_la_LIBADD = $(BUILD_WITH_LIBPROTOBUF_LIBS)
+endif
+
 if BUILD_WITH_LIBYAJL
 noinst_LTLIBRARIES += libformat_json.la
 libformat_json_la_SOURCES = \
@@ -2318,6 +2343,16 @@ write_mongodb_la_LDFLAGS = $(PLUGIN_LDFLAGS) $(BUILD_WITH_LIBMONGOC_LDFLAGS)
 write_mongodb_la_LIBADD = $(BUILD_WITH_LIBMONGOC_LIBS)
 endif
 
+if BUILD_PLUGIN_WRITE_OPEN_TELEMETRY
+pkglib_LTLIBRARIES += write_open_telemetry.la
+write_open_telemetry_la_SOURCES = src/write_open_telemetry.cc \
+				  opentelemetry/proto/collector/metrics/v1/metrics_service.grpc.pb.cc \
+				  opentelemetry/proto/collector/metrics/v1/metrics_service.grpc.pb.h
+write_open_telemetry_la_CPPFLAGS = $(AM_CPPFLAGS) $(BUILD_WITH_LIBGRPCPP_CPPFLAGS)
+write_open_telemetry_la_LDFLAGS = $(PLUGIN_LDFLAGS) $(BUILD_WITH_LIBGRPCPP_LDFLAGS)
+write_open_telemetry_la_LIBADD = $(BUILD_WITH_LIBGRPCPP_LIBS) libformat_open_telemetry.la
+endif
+
 if BUILD_PLUGIN_WRITE_PROMETHEUS
 pkglib_LTLIBRARIES += write_prometheus.la
 write_prometheus_la_SOURCES = src/write_prometheus.c
@@ -2445,20 +2480,50 @@ if HAVE_GRPC_CPP
 BUILT_SOURCES += collectd.grpc.pb.cc types.grpc.pb.cc collectd.pb.cc types.pb.cc
 
 collectd.grpc.pb.cc types.grpc.pb.cc: $(srcdir)/proto/collectd.proto $(srcdir)/proto/types.proto
-	$(V_PROTOC)$(PROTOC) -I$(srcdir)/proto \
+	$(V_PROTOC)$(PROTOC) $(PROTOC_FLAGS) -I$(srcdir)/proto \
 		--grpc_out=$(builddir) --plugin=protoc-gen-grpc=$(GRPC_CPP_PLUGIN) \
 	$(srcdir)/proto/collectd.proto $(srcdir)/proto/types.proto
 
 collectd.pb.cc: $(srcdir)/proto/collectd.proto $(srcdir)/proto/types.proto
-	$(V_PROTOC)$(PROTOC) -I$(srcdir)/proto --cpp_out=$(builddir) \
+	$(V_PROTOC)$(PROTOC) $(PROTOC_FLAGS) -I$(srcdir)/proto --cpp_out=$(builddir) \
 	$(srcdir)/proto/collectd.proto $(srcdir)/proto/types.proto
 
 types.pb.cc: $(srcdir)/proto/types.proto
-	$(V_PROTOC)$(PROTOC) -I$(srcdir)/proto --cpp_out=$(builddir) \
+	$(V_PROTOC)$(PROTOC) $(PROTOC_FLAGS) -I$(srcdir)/proto --cpp_out=$(builddir) \
 	$(srcdir)/proto/types.proto
 endif
 endif
 
+if BUILD_PLUGIN_WRITE_OPEN_TELEMETRY
+BUILT_SOURCES += \
+	opentelemetry/proto/collector/metrics/v1/metrics_service.grpc.pb.cc \
+	opentelemetry/proto/collector/metrics/v1/metrics_service.grpc.pb.h \
+	opentelemetry/proto/collector/metrics/v1/metrics_service.pb.cc \
+	opentelemetry/proto/collector/metrics/v1/metrics_service.pb.h \
+	opentelemetry/proto/common/v1/common.pb.cc \
+	opentelemetry/proto/common/v1/common.pb.h \
+	opentelemetry/proto/metrics/v1/metrics.pb.cc \
+	opentelemetry/proto/metrics/v1/metrics.pb.h \
+	opentelemetry/proto/resource/v1/resource.pb.cc \
+	opentelemetry/proto/resource/v1/resource.pb.h
+
+opentelemetry/proto/collector/metrics/v1/metrics_service.grpc.pb.cc opentelemetry/proto/collector/metrics/v1/metrics_service.grpc.pb.h: $(srcdir)/opentelemetry-proto/opentelemetry/proto/collector/metrics/v1/metrics_service.proto
+	$(V_PROTOC)$(PROTOC) $(PROTOC_FLAGS) -I$(srcdir)/opentelemetry-proto --grpc_out=$(builddir) \
+		--plugin="protoc-gen-grpc=$(GRPC_CPP_PLUGIN)" $<
+
+opentelemetry/proto/collector/metrics/v1/metrics_service.pb.cc opentelemetry/proto/collector/metrics/v1/metrics_service.pb.h: $(srcdir)/opentelemetry-proto/opentelemetry/proto/collector/metrics/v1/metrics_service.proto
+	$(V_PROTOC)$(PROTOC) $(PROTOC_FLAGS) -I$(srcdir)/opentelemetry-proto --cpp_out=$(builddir) $<
+
+opentelemetry/proto/common/v1/common.pb.cc opentelemetry/proto/common/v1/common.pb.h: $(srcdir)/opentelemetry-proto/opentelemetry/proto/common/v1/common.proto
+	$(V_PROTOC)$(PROTOC) $(PROTOC_FLAGS) -I$(srcdir)/opentelemetry-proto --cpp_out=$(builddir) $<
+
+opentelemetry/proto/metrics/v1/metrics.pb.cc opentelemetry/proto/metrics/v1/metrics.pb.h: $(srcdir)/opentelemetry-proto/opentelemetry/proto/metrics/v1/metrics.proto
+	$(V_PROTOC)$(PROTOC) $(PROTOC_FLAGS) -I$(srcdir)/opentelemetry-proto --cpp_out=$(builddir) $<
+
+opentelemetry/proto/resource/v1/resource.pb.cc opentelemetry/proto/resource/v1/resource.pb.h: $(srcdir)/opentelemetry-proto/opentelemetry/proto/resource/v1/resource.proto
+	$(V_PROTOC)$(PROTOC) $(PROTOC_FLAGS) -I$(srcdir)/opentelemetry-proto --cpp_out=$(builddir) $<
+endif
+
 install-exec-hook:
 	$(mkinstalldirs) $(DESTDIR)$(localstatedir)/run
 	$(mkinstalldirs) $(DESTDIR)$(localstatedir)/lib/$(PACKAGE_NAME)

configure.ac

@@ -4873,6 +4873,35 @@ if test "x$PROTOC" != "x"; then
 fi
 AM_CONDITIONAL([HAVE_PROTOC3], [test "x$have_protoc3" = "xyes"])
 
+protoc3_optional="no"
+if test "x$have_protoc3" = "xyes"; then
+  AC_MSG_CHECKING([whether protoc supports optional fields])
+  testfile=`mktemp`
+  cat <<EOF >"$testfile"
+syntax = "proto3";
+message Test {
+  optional bool support_optional = 1;
+}
+EOF
+  cpp_out=`mktemp -d`
+
+  protoc3_optional="no"
+  PROTOC_FLAGS=""
+  if $PROTOC -I`dirname $testfile` --cpp_out="$cpp_out" "$testfile"; then
+    protoc3_optional="yes"
+  elif $PROTOC -I`dirname $testfile` --cpp_out="$cpp_out" --experimental_allow_proto3_optional "$testfile"; then
+    protoc3_optional="yes (with --experimental_allow_proto3_optional)"
+    PROTOC_FLAGS="--experimental_allow_proto3_optional"
+  fi
+
+  # Clean up
+  rm "$testfile" "$cpp_out/`basename $testfile`.pb.cc" "$cpp_out/`basename $testfile`.pb.h"
+  rmdir "$cpp_out"
+
+  AC_MSG_RESULT([$protoc3_optional])
+  AC_SUBST(PROTOC_FLAGS)
+fi
+
 # --with-libprotobuf-c {{{
 AC_ARG_WITH([libprotobuf-c],
   [AS_HELP_STRING([--with-libprotobuf-c@<:@=PREFIX@:>@], [Path to libprotobuf-c.])],
@@ -6860,6 +6889,7 @@ plugin_write_influxdb_udp="yes"
 plugin_write_kafka="$with_librdkafka"
 plugin_write_log="no"
 plugin_write_mongodb="$with_libmongoc"
+plugin_write_open_telemetry="yes"
 plugin_write_prometheus="no"
 plugin_write_redis="$with_libhiredis"
 plugin_write_riemann="$with_libriemann_client"
@@ -7116,15 +7146,22 @@ fi
 
 if test "x$GRPC_CPP_PLUGIN" = "x"; then
   plugin_grpc="no (grpc_cpp_plugin not found)"
+  plugin_write_open_telemetry="no (grpc_cpp_plugin not found)"
 fi
 if test "x$have_protoc3" != "xyes"; then
   plugin_grpc="no (protoc3 not found)"
+  plugin_write_open_telemetry="no (protoc3 not found)"
 fi
 if test "x$with_libprotobuf" != "xyes"; then
   plugin_grpc="no (libprotobuf not found)"
+  plugin_write_open_telemetry="no (libprotobuf not found)"
 fi
 if test "x$with_libgrpcpp" != "xyes"; then
   plugin_grpc="no (libgrpc++ not found)"
+  plugin_write_open_telemetry="no (libgrpc++ not found)"
+fi
+if test "x$protoc3_optional" = "xno"; then
+  plugin_write_open_telemetry="no (protoc does not support optional fields)"
 fi
 
 if test "x$have_getifaddrs" = "xyes"; then
@@ -7457,6 +7494,7 @@ AC_PLUGIN([write_influxdb_udp],  [$plugin_write_influxdb_udp],[Influxdb udp outp
 AC_PLUGIN([write_kafka],         [$plugin_write_kafka],       [Kafka output plugin])
 AC_PLUGIN([write_log],           [$plugin_write_log],         [Log output plugin])
 AC_PLUGIN([write_mongodb],       [$plugin_write_mongodb],     [MongoDB output plugin])
+AC_PLUGIN([write_open_telemetry],[$plugin_write_open_telemetry],[Write OpenTelemetry plugin])
 AC_PLUGIN([write_prometheus],    [$plugin_write_prometheus],  [Prometheus write plugin])
 AC_PLUGIN([write_redis],         [$plugin_write_redis],       [Redis output plugin])
 AC_PLUGIN([write_riemann],       [$plugin_write_riemann],     [Riemann output plugin])
@@ -7903,6 +7941,7 @@ AC_MSG_RESULT([    write_influxdb_udp. . $enable_write_influxdb_udp])
 AC_MSG_RESULT([    write_kafka . . . . . $enable_write_kafka])
 AC_MSG_RESULT([    write_log . . . . . . $enable_write_log])
 AC_MSG_RESULT([    write_mongodb . . . . $enable_write_mongodb])
+AC_MSG_RESULT([    write_open_telemetry  $enable_write_open_telemetry])
 AC_MSG_RESULT([    write_prometheus. . . $enable_write_prometheus])
 AC_MSG_RESULT([    write_redis . . . . . $enable_write_redis])
 AC_MSG_RESULT([    write_riemann . . . . $enable_write_riemann])

src/collectd.conf.in

@@ -236,6 +236,7 @@
 #@BUILD_PLUGIN_WRITE_KAFKA_TRUE@LoadPlugin write_kafka
 #@BUILD_PLUGIN_WRITE_LOG_TRUE@LoadPlugin write_log
 #@BUILD_PLUGIN_WRITE_MONGODB_TRUE@LoadPlugin write_mongodb
+#@BUILD_PLUGIN_WRITE_OPEN_TELEMETRY_TRUE@LoadPlugin write_open_telemetry
 #@BUILD_PLUGIN_WRITE_PROMETHEUS_TRUE@LoadPlugin write_prometheus
 #@BUILD_PLUGIN_WRITE_REDIS_TRUE@LoadPlugin write_redis
 #@BUILD_PLUGIN_WRITE_RIEMANN_TRUE@LoadPlugin write_riemann
@@ -2067,6 +2068,13 @@
 #	</Node>
 #</Plugin>
 
+#<Plugin write_open_telemetry>
+#  <Node "example">
+#    Host "localhost"
+#    Port "4317"
+#  </Node>
+#</Plugin>
+
 #<Plugin write_prometheus>
 #	Port "9103"
 #</Plugin>

src/collectd.conf.pod

@@ -11138,6 +11138,35 @@ want to use authentication all three fields must be set.
 
 =back
 
+=head2 Plugin C<write_open_telemetry>
+
+The I<write_open_telemetry plugin> will export metrics to an I<OpenTelemetry
+collector> using the gRPC based OTLP protocol.
+
+B<Synopsis:>
+
+ <Plugin "write_open_telemetry">
+   <Node "default">
+     Host "localhost"
+     Port "4317"
+   </Node>
+ </Plugin>
+
+The plugin can export metrics to multiple collectors by specifying multiple
+B<Node> blocks. Within the B<Node> blocks, the following options are available:
+
+=over 4
+
+=item B<Host> I<Address>
+
+Hostname or address to connect to. Defaults to C<localhost>.
+
+=item B<Port> I<Service>
+
+Service name or port number to connect to. Defaults to C<4317>.
+
+=back
+
 =head2 Plugin C<write_prometheus>
 
 The I<write_prometheus plugin> implements a tiny webserver that can be scraped