feat: export container metrics in Chaos Daemon for CRI-O and Docker runtime

CHANGELOG.md

@@ -23,6 +23,7 @@ For more information and how-to, see [RFC: Keep A Changelog](https://github.com/
 - Support for setting `loadBalancerSourceRanges` in chaos-dashboard service in helm chart [#4172](https://github.com/chaos-mesh/chaos-mesh/pull/4172)
 - Helm: allow templating of dashboard rootUrl [#4370](https://github.com/chaos-mesh/chaos-mesh/pull/4370)
 - Support for reading database connection string from secret [#4363](https://github.com/chaos-mesh/chaos-mesh/pull/4363)
+- Export container metrics in Chaos Daemon for containerd, Docker and CRI-O runtime [#4416](https://github.com/chaos-mesh/chaos-mesh/pull/4416) [#4422](https://github.com/chaos-mesh/chaos-mesh/pull/4422)
 
 ### Changed
 

cmd/chaos-daemon/main.go

@@ -46,6 +46,7 @@ func init() {
 	flag.IntVar(&conf.HTTPPort, "http-port", 31766, "the port which http server listens on")
 	flag.StringVar(&conf.CrClientConfig.Runtime, "runtime", "docker", "current container runtime")
 	flag.StringVar(&conf.CrClientConfig.SocketPath, "runtime-socket-path", "", "current container runtime socket path")
+	flag.StringVar(&conf.CrClientConfig.CriSocketPath, "cri-socket-path", "", "cri socket path for docker runtime")
 	flag.StringVar(&conf.CrClientConfig.ContainerdNS, "containerd-ns", "k8s.io", "namespace used for containerd")
 	flag.StringVar(&conf.CaCert, "ca", "", "ca certificate of grpc server")
 	flag.StringVar(&conf.Cert, "cert", "", "certificate of grpc server")

helm/chaos-mesh/templates/chaos-daemon-daemonset.yaml

@@ -95,6 +95,15 @@ spec:
           {{- else if eq .Values.chaosDaemon.runtime "crio" }}
             - /host-run/crio.sock
           {{- end }}
+        {{- end }}
+        {{- if eq .Values.chaosDaemon.runtime "docker" }}
+            - --cri-socket-path
+          {{- if .Values.chaosDaemon.criSocketPath }}
+            {{- $socketDir := include "chaos-daemon.socket-path" . }}
+            - /host-run/{{ .Values.chaosDaemon.criSocketPath | trimPrefix $socketDir | trimPrefix "/" }}
+          {{- else }}
+            - /host-run/cri-dockerd.sock
+          {{- end }}
         {{- end }}
           env:
             {{- if .Values.chaosDaemon.env }}
@@ -176,7 +185,7 @@ spec:
         {{- end }}
       volumes:
         - name: socket-path
-          hostPath: 
+          hostPath:
             path: {{template "chaos-daemon.socket-path" . }}
         - name: sys-path
           hostPath:

helm/chaos-mesh/values.yaml

@@ -205,6 +205,10 @@ chaosDaemon:
   runtime: docker
   # socketPath specifies the path of container runtime socket on the host.
   socketPath: /var/run/docker.sock
+  # criSocketPath specifies the path of CRI socket on the host for docker
+  # runtime. It is only used when runtime is docker. In Kubernetes 1.23
+  # and earlier, this value should be `/var/run/dockershim.sock`.
+  criSocketPath: /var/run/cri-dockerd.sock
 
   # If you are using Kind or using containerd as CRI, you can use the
   # config below to use containerd as the runtime in chaos-daemon.

install.sh

@@ -1628,6 +1628,8 @@ spec:
             - --pprof
             - --runtime-socket-path
             - /host-run/${socketName}
+            - --cri-socket-path
+            - /host-run/cri-dockerd.sock
           env:
             - name: TZ
               value: ${timezone}
@@ -1650,7 +1652,7 @@ spec:
               containerPort: 31766
       volumes:
         - name: socket-path
-          hostPath: 
+          hostPath:
             path: ${socketDir}
         - name: sys-path
           hostPath:

pkg/chaosdaemon/crclients/client.go

@@ -18,6 +18,8 @@ package crclients
 import (
 	"context"
 
+	"github.com/chaos-mesh/chaos-mesh/pkg/chaosdaemon/crclients/utils"
+
 	"github.com/pkg/errors"
 
 	"github.com/chaos-mesh/chaos-mesh/pkg/chaosdaemon/crclients/containerd"
@@ -31,6 +33,7 @@ const (
 	ContainerRuntimeCrio       = "crio"
 
 	defaultDockerSocket     = "unix:///var/run/docker.sock"
+	defaultDockerCriSocket  = "unix:///var/run/cri-dockerd.sock"
 	defaultContainerdSocket = "/run/containerd/containerd.sock"
 	defaultCrioSocket       = "/var/run/crio/crio.sock"
 	containerdDefaultNS     = "k8s.io"
@@ -39,18 +42,22 @@ const (
 // CrClientConfig contains the basic cr client configuration.
 type CrClientConfig struct {
 	// Support docker, containerd, crio for now
-	Runtime      string
-	SocketPath   string
-	ContainerdNS string
+	Runtime       string
+	SocketPath    string
+	CriSocketPath string
+	ContainerdNS  string
 }
 
+type ContainerStats = utils.ContainerStats
+
 // ContainerRuntimeInfoClient represents a struct which can give you information about container runtime
 type ContainerRuntimeInfoClient interface {
 	GetPidFromContainerID(ctx context.Context, containerID string) (uint32, error)
 	ContainerKillByContainerID(ctx context.Context, containerID string) error
 	FormatContainerID(ctx context.Context, containerID string) (string, error)
 	ListContainerIDs(ctx context.Context) ([]string, error)
 	GetLabelsFromContainerID(ctx context.Context, containerID string) (map[string]string, error)
+	StatsByContainerID(ctx context.Context, containerID string) (*ContainerStats, error)
 }
 
 // CreateContainerRuntimeInfoClient creates a container runtime information client.
@@ -67,7 +74,13 @@ func CreateContainerRuntimeInfoClient(clientConfig *CrClientConfig) (ContainerRu
 		} else {
 			socketPath = "unix://" + socketPath
 		}
-		cli, err = docker.New(socketPath, "", nil, nil)
+		criSocketPath := clientConfig.CriSocketPath
+		if criSocketPath == "" {
+			criSocketPath = defaultDockerCriSocket
+		} else {
+			criSocketPath = "unix://" + criSocketPath
+		}
+		cli, err = docker.New(socketPath, "", nil, nil, criSocketPath)
 		if err != nil {
 			return nil, err
 		}

pkg/chaosdaemon/crclients/containerd/client.go

@@ -20,6 +20,11 @@ import (
 	"fmt"
 	"syscall"
 
+	"google.golang.org/grpc"
+	runtimev1 "k8s.io/cri-api/pkg/apis/runtime/v1"
+
+	"github.com/chaos-mesh/chaos-mesh/pkg/chaosdaemon/crclients/utils"
+
 	"github.com/containerd/containerd"
 	"github.com/pkg/errors"
 
@@ -41,9 +46,15 @@ type ContainerdClientInterface interface {
 	Containers(ctx context.Context, filters ...string) ([]containerd.Container, error)
 }
 
+// ContainerdRuntimeServiceInterface represents the runtimev1.RuntimeServiceClient, it's used to simply unit test
+type ContainerdRuntimeServiceInterface interface {
+	ContainerStats(ctx context.Context, in *runtimev1.ContainerStatsRequest, opts ...grpc.CallOption) (*runtimev1.ContainerStatsResponse, error)
+}
+
 // ContainerdClient can get information from containerd
 type ContainerdClient struct {
-	client ContainerdClientInterface
+	client        ContainerdClientInterface
+	runtimeClient ContainerdRuntimeServiceInterface
 }
 
 // FormatContainerID strips protocol prefix from the container ID
@@ -133,6 +144,23 @@ func (c ContainerdClient) GetLabelsFromContainerID(ctx context.Context, containe
 	return labels, nil
 }
 
+// StatsByContainerID returns the stats according to container ID
+func (c ContainerdClient) StatsByContainerID(ctx context.Context, containerID string) (*utils.ContainerStats, error) {
+	id, err := c.FormatContainerID(ctx, containerID)
+	if err != nil {
+		return nil, err
+	}
+	req := &runtimev1.ContainerStatsRequest{
+		ContainerId: id,
+	}
+	resp, err := c.runtimeClient.ContainerStats(ctx, req)
+	if err != nil {
+		return nil, err
+	}
+
+	return utils.BuildContainerStatsFromCRIResponse(resp), nil
+}
+
 func New(address string, opts ...containerd.ClientOpt) (*ContainerdClient, error) {
 	// Mock point to return error in unit test
 	if err := mock.On("NewContainerdClientError"); err != nil {
@@ -141,16 +169,22 @@ func New(address string, opts ...containerd.ClientOpt) (*ContainerdClient, error
 	if client := mock.On("MockContainerdClient"); client != nil {
 		return &ContainerdClient{
 			client.(ContainerdClientInterface),
+			client.(ContainerdRuntimeServiceInterface),
 		}, nil
 	}
 
 	c, err := containerd.New(address, opts...)
 	if err != nil {
 		return nil, err
 	}
+	runtimeClient, err := utils.BuildRuntimeServiceClient(context.TODO(), address)
+	if err != nil {
+		return nil, err
+	}
 	// The real logic
 	return &ContainerdClient{
-		client: c,
+		client:        c,
+		runtimeClient: runtimeClient,
 	}, nil
 }
 

pkg/chaosdaemon/crclients/crio/client.go

@@ -24,8 +24,9 @@ import (
 	"syscall"
 	"time"
 
+	"github.com/chaos-mesh/chaos-mesh/pkg/chaosdaemon/crclients/utils"
+
 	"github.com/pkg/errors"
-	"google.golang.org/grpc"
 	v1 "k8s.io/cri-api/pkg/apis/runtime/v1"
 )
 
@@ -126,15 +127,21 @@ func (c CrioClient) GetLabelsFromContainerID(ctx context.Context, containerID st
 	return container.Status.Labels, nil
 }
 
-func buildRuntimeServiceClient(endpoint string) (v1.RuntimeServiceClient, error) {
-	addr := fmt.Sprintf("unix://%s", endpoint)
-	conn, err := grpc.Dial(addr, grpc.WithBlock(), grpc.WithInsecure())
+// StatsByContainerID returns the stats according to container ID
+func (c CrioClient) StatsByContainerID(ctx context.Context, containerID string) (*utils.ContainerStats, error) {
+	id, err := c.FormatContainerID(ctx, containerID)
+	if err != nil {
+		return nil, err
+	}
+	req := &v1.ContainerStatsRequest{
+		ContainerId: id,
+	}
+	resp, err := c.runtimeClient.ContainerStats(ctx, req)
 	if err != nil {
 		return nil, err
 	}
 
-	client := v1.NewRuntimeServiceClient(conn)
-	return client, err
+	return utils.BuildContainerStatsFromCRIResponse(resp), nil
 }
 
 func New(socketPath string) (*CrioClient, error) {
@@ -146,7 +153,7 @@ func New(socketPath string) (*CrioClient, error) {
 		Transport: tr,
 	}
 
-	runtimeClient, err := buildRuntimeServiceClient(socketPath)
+	runtimeClient, err := utils.BuildRuntimeServiceClient(context.TODO(), socketPath)
 	if err != nil {
 		return nil, err
 	}

pkg/chaosdaemon/crclients/docker/client.go

@@ -19,6 +19,12 @@ import (
 	"context"
 	"fmt"
 	"net/http"
+	"time"
+
+	"google.golang.org/grpc"
+	runtimev1 "k8s.io/cri-api/pkg/apis/runtime/v1"
+
+	"github.com/chaos-mesh/chaos-mesh/pkg/chaosdaemon/crclients/utils"
 
 	"github.com/docker/docker/api/types"
 	"github.com/docker/docker/api/types/filters"
@@ -44,9 +50,15 @@ type DockerClientInterface interface {
 	ContainerList(ctx context.Context, options types.ContainerListOptions) ([]types.Container, error)
 }
 
+// DockerRuntimeServiceInterface represents the runtimev1.RuntimeServiceClient, it's used to simply unit test
+type DockerRuntimeServiceInterface interface {
+	ContainerStats(ctx context.Context, in *runtimev1.ContainerStatsRequest, opts ...grpc.CallOption) (*runtimev1.ContainerStatsResponse, error)
+}
+
 // DockerClient can get information from docker
 type DockerClient struct {
-	client DockerClientInterface
+	client        DockerClientInterface
+	runtimeClient DockerRuntimeServiceInterface
 }
 
 // FormatContainerID strips protocol prefix from the container ID
@@ -122,14 +134,35 @@ func (c DockerClient) GetLabelsFromContainerID(ctx context.Context, containerID
 	return container.Config.Labels, nil
 }
 
-func New(host string, version string, client *http.Client, httpHeaders map[string]string) (*DockerClient, error) {
+// StatsByContainerID returns the stats according to container ID
+func (c DockerClient) StatsByContainerID(ctx context.Context, containerID string) (*utils.ContainerStats, error) {
+	if c.runtimeClient == nil {
+		return nil, errors.New("cri socket is not connected")
+	}
+	id, err := c.FormatContainerID(ctx, containerID)
+	if err != nil {
+		return nil, err
+	}
+	req := &runtimev1.ContainerStatsRequest{
+		ContainerId: id,
+	}
+	resp, err := c.runtimeClient.ContainerStats(ctx, req)
+	if err != nil {
+		return nil, err
+	}
+
+	return utils.BuildContainerStatsFromCRIResponse(resp), nil
+}
+
+func New(host string, version string, client *http.Client, httpHeaders map[string]string, criHost string) (*DockerClient, error) {
 	// Mock point to return error or mock client in unit test
 	if err := mock.On("NewDockerClientError"); err != nil {
 		return nil, err.(error)
 	}
 	if client := mock.On("MockDockerClient"); client != nil {
 		return &DockerClient{
-			client: client.(DockerClientInterface),
+			client:        client.(DockerClientInterface),
+			runtimeClient: client.(DockerRuntimeServiceInterface),
 		}, nil
 	}
 
@@ -142,8 +175,15 @@ func New(host string, version string, client *http.Client, httpHeaders map[strin
 	if err != nil {
 		return nil, err
 	}
+	ctx, cancel := context.WithTimeout(context.Background(), time.Second*10)
+	defer cancel()
+	runtimeClient, err := utils.BuildRuntimeServiceClient(ctx, criHost)
+	if err != nil {
+		runtimeClient = nil
+	}
 	// The real logic
 	return &DockerClient{
-		client: c,
+		client:        c,
+		runtimeClient: runtimeClient,
 	}, nil
 }