sysdump: Add kvstore data to dumps.

When running sysdump, if kvstore is detected then various kvstore
paths are added to the dump.

Signed-off-by: Tom Hadlaw <tom.hadlaw@isovalent.com>

.gitignore

@@ -21,3 +21,6 @@
 # Outputs of 'cilium sysdump'.
 cilium-sysdump-*/
 cilium-sysdump-*.zip
+
+# Editor metas
+.vscode/

sysdump/sysdump.go

@@ -23,6 +23,7 @@ import (
 	"k8s.io/apimachinery/pkg/runtime"
 	"k8s.io/cli-runtime/pkg/genericclioptions"
 
+	"github.com/cilium/cilium-cli/defaults"
 	"github.com/cilium/cilium-cli/internal/utils"
 	"github.com/cilium/cilium-cli/k8s"
 )
@@ -870,6 +871,37 @@ func (c *Collector) Run() error {
 				return nil
 			},
 		},
+		{
+			CreatesSubtasks: true,
+			Description:     "Collecting kvstore data",
+			Quick:           true,
+			Task: func(ctx context.Context) error {
+				cm, err := c.Client.GetConfigMap(ctx, c.Options.CiliumNamespace, ciliumConfigMapName, metav1.GetOptions{})
+				if err != nil {
+					return fmt.Errorf("failed to get cilium-config ConfigMap: %w", err)
+				}
+				// Check if kvstore is enabled, if not skip this dump.
+				if v, ok := cm.Data["kvstore"]; !ok || v != "etcd" {
+					c.logDebug("KVStore not enabled, skipping kvstore dump")
+					return nil
+				}
+				ps, err := c.Client.ListPods(ctx, c.Options.CiliumNamespace, metav1.ListOptions{
+					LabelSelector: c.Options.CiliumLabelSelector,
+				})
+				if err != nil {
+					return fmt.Errorf("failed to list Pods for gathering kvstore data: %w", err)
+				}
+				if len(ps.Items) == 0 {
+					return fmt.Errorf("could not find Cilium Agent Pod to run kvstore get, Cilium Pod list was empty")
+				}
+				for _, pod := range ps.Items {
+					if pod.Status.Phase == "Running" {
+						return c.submitKVStoreTasks(ctx, &pod)
+					}
+				}
+				return fmt.Errorf("could not find running Cilium Pod")
+			},
+		},
 	}
 	for _, selector := range c.Options.ExtraLabelSelectors {
 		tasks = append(tasks, Task{
@@ -1299,6 +1331,51 @@ func (c *Collector) submitFlavorSpecificTasks(ctx context.Context, f k8s.Flavor)
 	}
 }
 
+func (c *Collector) submitKVStoreTasks(ctx context.Context, pod *corev1.Pod) error {
+	for _, dump := range []struct {
+		name   string
+		prefix string
+	}{
+		{
+			name:   "identities",
+			prefix: "state/identities",
+		},
+		{
+			name:   "ips",
+			prefix: "state/ip",
+		},
+		{
+			name:   "nodes",
+			prefix: "state/nodes",
+		},
+		{
+			name:   "cnpstatuses",
+			prefix: "state/cnpstatuses",
+		},
+		{
+			name:   "heartbeat",
+			prefix: ".heartbeat",
+		},
+		{
+			name:   "services",
+			prefix: "state/services",
+		},
+	} {
+		stdout, stderr, err := c.Client.ExecInPodWithStderr(ctx, pod.Namespace, pod.Name, defaults.AgentContainerName, []string{
+			"cilium", "kvstore", "get", "cilium/" + dump.prefix, "--recursive", "-o", "json",
+		})
+		if err != nil {
+			return fmt.Errorf("failed to collect kvstore data in %q in namespace %q: %w: %s", pod.Name, pod.Namespace, err, stderr.String())
+		}
+		filename := "kvstore-" + dump.name + ".json"
+		if err := c.WriteString(filename, stdout.String()); err != nil {
+			return fmt.Errorf("failed writing kvstore dump for prefix %q to file %q: %w", dump.prefix, filename, err)
+		}
+	}
+
+	return nil
+}
+
 func buildNodeNameList(nodes *corev1.NodeList, filter string) ([]string, error) {
 	w := strings.Split(strings.TrimSpace(filter), ",")
 	r := make([]string, 0)

sysdump/sysdump_test.go

@@ -6,9 +6,12 @@ package sysdump
 import (
 	"bytes"
 	"context"
+	"fmt"
 	"io"
+	"io/ioutil"
 	"os"
 	"path"
+	"strings"
 	"testing"
 	"time"
 
@@ -22,7 +25,9 @@ import (
 
 	ciliumv2 "github.com/cilium/cilium/pkg/k8s/apis/cilium.io/v2"
 	ciliumv2alpha1 "github.com/cilium/cilium/pkg/k8s/apis/cilium.io/v2alpha1"
+	"github.com/stretchr/testify/assert"
 
+	"github.com/cilium/cilium-cli/defaults"
 	"github.com/cilium/cilium-cli/k8s"
 )
 
@@ -142,8 +147,57 @@ func (b *SysdumpSuite) TestExtractGopsPID(c *check.C) {
 
 }
 
+func TestKVStoreTask(t *testing.T) {
+	assert := assert.New(t)
+	client := &fakeClient{
+		nodeList: &corev1.NodeList{
+			Items: []corev1.Node{{ObjectMeta: metav1.ObjectMeta{Name: "node-a"}}},
+		},
+		execs: make(map[execRequest]execResult),
+	}
+	addKVStoreGet := func(c *fakeClient, ciliumPaths ...string) {
+		for _, path := range ciliumPaths {
+			c.expectExec("ns0", "pod0", defaults.AgentContainerName,
+				[]string{"cilium", "kvstore", "get", "cilium/" + path, "--recursive", "-o", "json"},
+				[]byte("{}"), nil, nil)
+		}
+	}
+	addKVStoreGet(client, "state/identities", "state/ip", "state/nodes", "state/cnpstatuses", ".heartbeat", "state/services")
+	options := Options{
+		OutputFileName: "my-sysdump-<ts>",
+		Writer:         io.Discard,
+	}
+	collector, err := NewCollector(client, options, time.Now(), "cilium-cli-version")
+	assert.NoError(err)
+	collector.submitKVStoreTasks(context.Background(), &corev1.Pod{
+		ObjectMeta: metav1.ObjectMeta{
+			Name:      "pod0",
+			Namespace: "ns0",
+		},
+	})
+	fd, err := os.Open(path.Join(collector.sysdumpDir, "kvstore-heartbeat.json"))
+	assert.NoError(err)
+	data, err := ioutil.ReadAll(fd)
+	assert.NoError(err)
+	assert.Equal([]byte("{}"), data)
+}
+
+type execRequest struct {
+	namespace string
+	pod       string
+	container string
+	command   string
+}
+
+type execResult struct {
+	stderr []byte
+	stdout []byte
+	err    error
+}
+
 type fakeClient struct {
 	nodeList *corev1.NodeList
+	execs    map[execRequest]execResult
 }
 
 func (c *fakeClient) ListCiliumClusterwideEnvoyConfigs(ctx context.Context, opts metav1.ListOptions) (*ciliumv2.CiliumClusterwideEnvoyConfigList, error) {
@@ -178,12 +232,27 @@ func (c *fakeClient) DeletePod(ctx context.Context, namespace, name string, opts
 	panic("implement me")
 }
 
+func (c *fakeClient) expectExec(namespace, pod, container string, command []string, expectedStdout []byte, expectedStderr []byte, expectedErr error) {
+	r := execRequest{namespace, pod, container, strings.Join(command, " ")}
+	c.execs[r] = execResult{
+		stdout: expectedStdout,
+		stderr: expectedStderr,
+		err:    expectedErr,
+	}
+}
+
 func (c *fakeClient) ExecInPod(ctx context.Context, namespace, pod, container string, command []string) (bytes.Buffer, error) {
-	panic("implement me")
+	stdout, _, err := c.ExecInPodWithStderr(ctx, namespace, pod, container, command)
+	return stdout, err
 }
 
 func (c *fakeClient) ExecInPodWithStderr(ctx context.Context, namespace, pod, container string, command []string) (bytes.Buffer, bytes.Buffer, error) {
-	panic("implement me")
+	r := execRequest{namespace, pod, container, strings.Join(command, " ")}
+	out, ok := c.execs[r]
+	if !ok {
+		panic(fmt.Sprintf("unexpected exec: %v", r))
+	}
+	return *bytes.NewBuffer(out.stdout), *bytes.NewBuffer(out.stderr), out.err
 }
 
 func (c *fakeClient) GetConfigMap(ctx context.Context, namespace, name string, opts metav1.GetOptions) (*corev1.ConfigMap, error) {