Merge pull request #29 from fujiwara/otel-metric

Otel metric

README.md

@@ -15,7 +15,7 @@ maprobe agent works as below.
 1. For each hosts, execute probes (ping, tcp, http, command).
    - expand place holder in configuration `{{ .Host }}` as [Mackerel host struct](https://godoc.org/github.com/mackerelio/mackerel-client-go#Host).
    - `{{ .Host.IPAddress.eth0 }}` expand to e.g. `192.168.1.1`
-1. Posts host metrics to Mackerel.
+1. Posts host metrics to Mackerel (and/or OpenTelemetry metrics endpoint if configured).
 1. Iterates these processes each 60 sec.
 
 ### for aggregates
@@ -124,6 +124,9 @@ probes:
         - "mackerel-plugin-redis"
         - "-host={{ .Host.IPAddress.eth0 }}"
         - "-tempfile=/tmp/redis-{{ .Host.ID }}"
+    attributes: # supoort OpenTelemetry attributes
+      - service.namespaece: redis
+      - host.name: "{{ .Host.Name }}"
 
   - service: production
     service_metric: true # post metrics as service metrics
@@ -134,6 +137,47 @@ probes:
         Content-Type: application/json
       body: '{"hello":"world"}'
       expect_pattern: 'ok'
+
+destination:
+  mackerel:
+    enabled: true # default true
+  otel:
+    enabled: true # default false
+    endpoint: localhost:4317
+    insecure: true
+```
+
+#### OpenTelemetry metrics endpoint support
+
+`destination.otel.enabled: true` enables to post metrics to OpenTelemetry metrics endpoint.
+maprobe uses the gRPC protocol to send metrics.
+
+```yaml
+destination:
+  mackerel:
+    enabled: false # disable mackerel host/service metrics
+  otel:
+    enabled: true
+    endpoint: localhost:4317
+    insecure: true
+```
+
+Extra attributes can be added to metrics by `attributes` in probe configuration.
+
+By default, maprobe adds `service.name` and `host.id` attributes to metrics.
+
+```yaml
+probes:
+  - service: production
+    role: redis
+    command:
+      command:
+        - "mackerel-plugin-redis"
+        - "-host={{ .Host.IPAddress.eth0 }}"
+        - "-tempfile=/tmp/redis-{{ .Host.ID }}"
+    attributes: # extra attributes
+      - service.namespace: redis
+      - host.name: "{{ .Host.Name }}"
 ```
 
 #### Service metrics support in probes

channels.go

@@ -0,0 +1,52 @@
+package maprobe
+
+type Channels struct {
+	ServiceMetrics    chan ServiceMetric
+	HostMetrics       chan HostMetric
+	AggregatedMetrics chan ServiceMetric
+	OtelMetrics       chan Metric
+	Destination       *DestinationConfig
+}
+
+func NewChannels(dst *DestinationConfig) *Channels {
+	chs := Channels{
+		ServiceMetrics:    make(chan ServiceMetric, PostMetricBufferLength*10),
+		HostMetrics:       make(chan HostMetric, PostMetricBufferLength*10),
+		AggregatedMetrics: make(chan ServiceMetric, PostMetricBufferLength*10),
+		OtelMetrics:       make(chan Metric, PostMetricBufferLength*10),
+		Destination:       dst,
+	}
+	return &chs
+}
+
+func (ch *Channels) SendServiceMetric(m ServiceMetric) {
+	if ch.Destination.Mackerel.Enabled {
+		ch.ServiceMetrics <- m
+	}
+	if ch.Destination.Otel.Enabled {
+		ch.OtelMetrics <- m.Metric
+	}
+}
+
+func (ch *Channels) SendHostMetric(m HostMetric) {
+	if ch.Destination.Mackerel.Enabled {
+		ch.HostMetrics <- m
+	}
+	if ch.Destination.Otel.Enabled {
+		ch.OtelMetrics <- m.Metric
+	}
+}
+
+func (ch *Channels) SendAggregatedMetric(m ServiceMetric) {
+	if ch.Destination.Mackerel.Enabled {
+		ch.AggregatedMetrics <- m
+	}
+	// TODO: Otel Aggregated Metrics
+}
+
+func (ch *Channels) Close() {
+	close(ch.ServiceMetrics)
+	close(ch.HostMetrics)
+	close(ch.AggregatedMetrics)
+	close(ch.OtelMetrics)
+}

command.go

@@ -4,6 +4,7 @@ import (
 	"bufio"
 	"bytes"
 	"context"
+	"crypto/sha256"
 	"encoding/json"
 	"fmt"
 	"log"
@@ -114,7 +115,8 @@ func (p *CommandProbe) String() string {
 }
 
 func (p *CommandProbe) TempDir() string {
-	dir := filepath.Join(os.TempDir(), fmt.Sprintf("maprobe_command_%x", p))
+	s := sha256.Sum256([]byte(p.String()))
+	dir := filepath.Join(os.TempDir(), fmt.Sprintf("maprobe_command_%x", s))
 	err := os.Mkdir(dir, 0700)
 	if err != nil {
 		if os.IsExist(err) {

config.go

@@ -4,10 +4,11 @@ import (
 	"crypto/sha256"
 	"encoding/json"
 	"fmt"
-	"io/ioutil"
+	"io"
 	"log"
 	"net/http"
 	"net/url"
+	"os"
 	"strings"
 
 	"github.com/aws/aws-sdk-go/aws"
@@ -30,7 +31,23 @@ type Config struct {
 
 	ProbeOnly *bool `yaml:"probe_only"` // deprecated
 
-	Backup BackupConfig `yaml:"backup"`
+	Backup      *BackupConfig      `yaml:"backup"`
+	Destination *DestinationConfig `yaml:"destination"`
+}
+
+type MackerelConfig struct {
+	Enabled bool `yaml:"enabled"`
+}
+
+type OtelConfig struct {
+	Enabled  bool   `yaml:"enabled"`
+	Endpoint string `yaml:"endpoint"`
+	Insecure bool   `yaml:"insecure"`
+}
+
+type DestinationConfig struct {
+	Mackerel *MackerelConfig `yaml:"mackerel"`
+	Otel     *OtelConfig     `yaml:"otel"`
 }
 
 type exString struct {
@@ -71,6 +88,8 @@ type ProbeDefinition struct {
 	TCP     *TCPProbeConfig     `yaml:"tcp"`
 	HTTP    *HTTPProbeConfig    `yaml:"http"`
 	Command *CommandProbeConfig `yaml:"command"`
+
+	Attributes map[string]string `yaml:"attributes"`
 }
 
 func (pd *ProbeDefinition) Validate() error {
@@ -129,6 +148,15 @@ func LoadConfig(location string) (*Config, string, error) {
 		location:              location,
 		PostProbedMetrics:     true,
 		PostAggregatedMetrics: true,
+		Backup:                &BackupConfig{},
+		Destination: &DestinationConfig{
+			Mackerel: &MackerelConfig{
+				Enabled: true,
+			},
+			Otel: &OtelConfig{
+				Enabled: false,
+			},
+		},
 	}
 	b, err := c.fetch()
 	if err != nil {
@@ -205,7 +233,7 @@ func (c *Config) fetch() ([]byte, error) {
 	u, err := url.Parse(c.location)
 	if err != nil {
 		// file path
-		return ioutil.ReadFile(c.location)
+		return os.ReadFile(c.location)
 	}
 	switch u.Scheme {
 	case "http", "https":
@@ -214,7 +242,7 @@ func (c *Config) fetch() ([]byte, error) {
 		return fetchS3(u)
 	default:
 		// file
-		return ioutil.ReadFile(u.Path)
+		return os.ReadFile(u.Path)
 	}
 }
 
@@ -230,7 +258,7 @@ func fetchHTTP(u *url.URL) ([]byte, error) {
 		return nil, err
 	}
 	defer resp.Body.Close()
-	return ioutil.ReadAll(resp.Body)
+	return io.ReadAll(resp.Body)
 }
 
 func fetchS3(u *url.URL) ([]byte, error) {

docker/bullseye-slim/Dockerfile

@@ -1,4 +1,4 @@
-FROM --platform=${BUILDPLATFORM} golang:1.18.4-buster AS build-env
+FROM --platform=${BUILDPLATFORM} golang:1.21.5-bullseye AS build-env
 
 ARG TARGETOS
 ARG TARGETARCH

docker/buster-slim/Dockerfile

@@ -1,4 +1,4 @@
-FROM --platform=${BUILDPLATFORM} golang:1.18.4-buster AS build-env
+FROM --platform=${BUILDPLATFORM} golang:1.21.5-bullseye AS build-env
 
 ARG TARGETOS
 ARG TARGETARCH

docker/mackerel-plugins/Dockerfile

@@ -1,4 +1,4 @@
-FROM --platform=${BUILDPLATFORM} golang:1.18.4-buster AS build-env
+FROM --platform=${BUILDPLATFORM} golang:1.21.5-bullseye AS build-env
 
 ARG TARGETOS
 ARG TARGETARCH

docker/plain/Dockerfile

@@ -1,4 +1,4 @@
-FROM --platform=${BUILDPLATFORM} golang:1.18.4-buster AS build-env
+FROM --platform=${BUILDPLATFORM} golang:1.21.5-bullseye AS build-env
 
 ARG TARGETOS
 ARG TARGETARCH

go.mod

@@ -1,6 +1,6 @@
 module github.com/fujiwara/maprobe
 
-go 1.18
+go 1.20
 
 require (
 	github.com/aws/aws-lambda-go v1.39.1
@@ -12,17 +12,36 @@ require (
 	github.com/mackerelio/mackerel-client-go v0.21.1
 	github.com/pkg/errors v0.9.1
 	github.com/tatsushid/go-fastping v0.0.0-20160109021039-d7bb493dee3e
+	go.opentelemetry.io/otel v1.17.0
+	go.opentelemetry.io/otel/exporters/otlp/otlpmetric/otlpmetricgrpc v0.40.0
+	go.opentelemetry.io/otel/sdk v1.17.0
+	go.opentelemetry.io/otel/sdk/metric v0.40.0
 	gopkg.in/alecthomas/kingpin.v2 v2.2.6
 	gopkg.in/yaml.v2 v2.4.0
 )
 
 require (
 	github.com/alecthomas/template v0.0.0-20190718012654-fb15b899a751 // indirect
 	github.com/alecthomas/units v0.0.0-20190924025748-f65c72e2690d // indirect
+	github.com/cenkalti/backoff/v4 v4.2.1 // indirect
+	github.com/go-logr/logr v1.2.4 // indirect
+	github.com/go-logr/stdr v1.2.2 // indirect
+	github.com/golang/protobuf v1.5.3 // indirect
+	github.com/grpc-ecosystem/grpc-gateway/v2 v2.16.0 // indirect
 	github.com/jmespath/go-jmespath v0.4.0 // indirect
-	github.com/pires/go-proxyproto v0.6.1 // indirect
+	github.com/pires/go-proxyproto v0.7.0 // indirect
+	go.opentelemetry.io/otel/exporters/otlp/otlpmetric v0.40.0 // indirect
+	go.opentelemetry.io/otel/metric v1.17.0 // indirect
+	go.opentelemetry.io/otel/trace v1.17.0 // indirect
+	go.opentelemetry.io/proto/otlp v1.0.0 // indirect
 	golang.org/x/lint v0.0.0-20200302205851-738671d3881b // indirect
 	golang.org/x/net v0.17.0 // indirect
 	golang.org/x/sys v0.13.0 // indirect
-	golang.org/x/tools v0.0.0-20200612022331-742c5eb664c2 // indirect
+	golang.org/x/text v0.13.0 // indirect
+	golang.org/x/tools v0.6.0 // indirect
+	golang.org/x/tools/cmd/cover v0.1.0-deprecated // indirect
+	google.golang.org/genproto/googleapis/api v0.0.0-20230530153820-e85fd2cbaebc // indirect
+	google.golang.org/genproto/googleapis/rpc v0.0.0-20230530153820-e85fd2cbaebc // indirect
+	google.golang.org/grpc v1.57.0 // indirect
+	google.golang.org/protobuf v1.31.0 // indirect
 )