From 7435ebe48547a026150695485cb0039e1701b0e6 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Ricardo=20L=C3=BCders?= Date: Sat, 2 Dec 2023 02:29:10 +0100 Subject: [PATCH] adds helm information gather * feat: gather helm information * feat(gather/hemlchart_info): add resource counter * docs: adds helmchart_info sample data * test: adds some unit test for helmchart_gather * test: expanding gather_helm_info tests * feat: adding helmchart gatherer * docs: update gathered-data * Revert "feat: adding helmchart gatherer" This reverts commit e57a5c138ad6149e8db2c7755e113c516cc0b326. * refactor: revert helmcharts gatherer * style: fix lint errors * refactor(helm_info): remove if for label checking * refactor(gather_info): unexpose labelChartNameKey const * fix: wrong log message * test(helm_gather_info): invalid resources --- docs/gathered-data.md | 24 ++ .../config/helmchart_info.json | 19 ++ pkg/gather/gather.go | 2 +- pkg/gatherers/workloads/gather_helm_info.go | 167 ++++++++++ .../workloads/gather_helm_info_test.go | 307 ++++++++++++++++++ pkg/gatherers/workloads/types.go | 45 +++ pkg/gatherers/workloads/types_test.go | 105 ++++++ pkg/gatherers/workloads/workloads_gatherer.go | 9 +- .../workloads/workloads_gatherer_test.go | 2 +- .../workloads/workloads_info_test.go | 2 +- 10 files changed, 678 insertions(+), 4 deletions(-) create mode 100644 docs/insights-archive-sample/config/helmchart_info.json create mode 100644 pkg/gatherers/workloads/gather_helm_info.go create mode 100644 pkg/gatherers/workloads/gather_helm_info_test.go create mode 100644 pkg/gatherers/workloads/types_test.go diff --git a/docs/gathered-data.md b/docs/gathered-data.md index 7cfb99b0c..dd5d0ce09 100644 --- a/docs/gathered-data.md +++ b/docs/gathered-data.md @@ -404,6 +404,30 @@ filtered to only include those with a deployment_validation_operator_ prefix. - 4.10 +## HelmInfo + +Collects summarized info about the helm usage on a cluster +in a generic fashion + +### API Reference +None + +### Sample data +- [docs/insights-archive-sample/config/helmchart_info.json](./insights-archive-sample/config/helmchart_info.json) + +### Location in archive +- `config/helmchart_info.json` + +### Config ID +`workloads/helmchart_info` + +### Released version +- 4.15.0 + +### Backported versions +None + + ## HostSubnet collects HostSubnet information diff --git a/docs/insights-archive-sample/config/helmchart_info.json b/docs/insights-archive-sample/config/helmchart_info.json new file mode 100644 index 000000000..52de04c28 --- /dev/null +++ b/docs/insights-archive-sample/config/helmchart_info.json @@ -0,0 +1,19 @@ +{ + "6c7e03817b27fbe6cc67ae835381df521b8d847dd029fb2df483f1a327b63582": [ + { "name": "jenkins", "version": "0.0.3", "resources": { "services": 2 } } + ], + "a085ddf97d8c556fd4f965392f7d3446c4b11df0544848878e06b20c96523064": [ + { + "name": "nodejs", + "version": "", + "resources": { "deployments": 1, "replicasets": 1, "services": 1 } + } + ], + "e976fcc461dc1b1ad177c083135393e90ded18b3707987d4b6adf78e86e687ed": [ + { + "name": "nodejs", + "version": "", + "resources": { "deployments": 1, "replicasets": 1, "services": 1 } + } + ] +} diff --git a/pkg/gather/gather.go b/pkg/gather/gather.go index 8321c9a9c..672d03f7f 100644 --- a/pkg/gather/gather.go +++ b/pkg/gather/gather.go @@ -65,7 +65,7 @@ func CreateAllGatherers( gatherKubeConfig, gatherProtoKubeConfig, metricsGatherKubeConfig, alertsGatherKubeConfig, anonymizer, configObserver, ) - workloadsGatherer := workloads.New(gatherProtoKubeConfig) + workloadsGatherer := workloads.New(gatherKubeConfig, gatherProtoKubeConfig) conditionalGatherer := conditional.New( gatherProtoKubeConfig, metricsGatherKubeConfig, gatherKubeConfig, configObserver, insightsClient, ) diff --git a/pkg/gatherers/workloads/gather_helm_info.go b/pkg/gatherers/workloads/gather_helm_info.go new file mode 100644 index 000000000..13ab6f583 --- /dev/null +++ b/pkg/gatherers/workloads/gather_helm_info.go @@ -0,0 +1,167 @@ +package workloads + +import ( + "context" + "crypto/sha256" + "encoding/hex" + "fmt" + "strings" + + "k8s.io/apimachinery/pkg/runtime/schema" + + "k8s.io/apimachinery/pkg/api/errors" + "k8s.io/client-go/dynamic" + "k8s.io/klog/v2" + + metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" + + "github.com/openshift/insights-operator/pkg/record" +) + +const labelChartNameKey = "helm.sh/chart" + +// GatherHelmInfo Collects summarized info about the helm usage on a cluster +// in a generic fashion +// +// ### API Reference +// None +// +// ### Sample data +// - docs/insights-archive-sample/config/helmchart_info.json +// +// ### Location in archive +// - `config/helmchart_info.json` +// +// ### Config ID +// `workloads/helmchart_info` +// +// ### Released version +// - 4.15.0 +// +// ### Backported versions +// None +func (g *Gatherer) GatherHelmInfo(ctx context.Context) ([]record.Record, []error) { + dynamicClient, err := dynamic.NewForConfig(g.gatherKubeConfig) + if err != nil { + return nil, []error{err} + } + + return gatherHelmInfo(ctx, dynamicClient) +} + +func gatherHelmInfo( + ctx context.Context, + dynamicClient dynamic.Interface, +) ([]record.Record, []error) { + resources := []schema.GroupVersionResource{ + {Group: "apps", Version: "v1", Resource: "replicasets"}, + {Group: "apps", Version: "v1", Resource: "daemonsets"}, + {Group: "apps", Version: "v1", Resource: "statefulsets"}, + {Group: "", Version: "v1", Resource: "services"}, + {Group: "apps", Version: "v1", Resource: "deployments"}, + } + + var errs []error + var records []record.Record + helmList := newHelmChartInfoList() + + for _, resource := range resources { + listOptions := metav1.ListOptions{LabelSelector: "app.kubernetes.io/managed-by=Helm"} + + items, err := dynamicClient.Resource(resource).List(ctx, listOptions) + if errors.IsNotFound(err) { + return nil, nil + } + if err != nil { + klog.V(2).Infof("Unable to list %s resource due to: %s", resource, err) + errs = append(errs, err) + continue + } + + for _, item := range items.Items { + labels := item.GetLabels() + + // Anonymize the namespace to make it unique identifier + hash, err := createHash(item.GetNamespace()) + if err != nil { + klog.Errorf("unable to hash the namespace '%s': %v", labels[labelChartNameKey], err) + continue + } + + name, version := helmChartNameAndVersion(labels[labelChartNameKey]) + if name == "" && version == "" { + // some helm-maneged resource may not have reference to the chart + klog.Infof("unable to get HelmChart from %s on %s from %s.", resource.Resource, item.GetNamespace(), item.GetName()) + continue + } + + helmList.addItem(hash, resource.Resource, HelmChartInfo{ + Name: name, + Version: version, + }) + } + } + + if len(helmList.Namespaces) > 0 { + records = []record.Record{ + { + Name: "config/helmchart_info", + Item: record.JSONMarshaller{Object: &helmList.Namespaces}, + }, + } + } + + if len(errs) > 0 { + return records, errs + } + + return records, nil +} + +func createHash(chartName string) (string, error) { + h := sha256.New() + _, err := h.Write([]byte(chartName)) + if err != nil { + return "", err + } + + hashInBytes := h.Sum(nil) + hash := hex.EncodeToString(hashInBytes) + + return hash, nil +} + +func helmChartNameAndVersion(chart string) (name, version string) { + parts := strings.Split(chart, "-") + + // no version found + if len(parts) == 1 { + return chart, "" + } + + name = strings.Join(parts[:len(parts)-1], "-") + + // best guess to get the version + version = parts[len(parts)-1] + // check for standard version format + if !strings.Contains(version, ".") { + // maybe it is a string version + if !isStringVersion(version) { + // not a valid version, add to name and version should be empty + name = fmt.Sprintf("%s-%s", name, version) + version = "" + } + } + + return name, version +} + +func isStringVersion(version string) bool { + stringVersions := []string{"latest", "beta", "alpha"} + for _, v := range stringVersions { + if v == version { + return true + } + } + return false +} diff --git a/pkg/gatherers/workloads/gather_helm_info_test.go b/pkg/gatherers/workloads/gather_helm_info_test.go new file mode 100644 index 000000000..12d33246b --- /dev/null +++ b/pkg/gatherers/workloads/gather_helm_info_test.go @@ -0,0 +1,307 @@ +package workloads + +import ( + "context" + "testing" + + "github.com/openshift/insights-operator/pkg/record" + + "k8s.io/apimachinery/pkg/apis/meta/v1/unstructured" + "k8s.io/apimachinery/pkg/runtime" + "k8s.io/client-go/dynamic" + "k8s.io/client-go/dynamic/fake" + + "github.com/stretchr/testify/assert" +) + +func TestGatherHelmInfo(t *testing.T) { + ctx := context.TODO() + + hash, err := createHash("mynamespace") + assert.NoError(t, err, "failed to generate namespace hash") + + // create the data for testing here + helmChartInfoList := newHelmChartInfoList() + helmChartInfoList.Namespaces[hash] = []HelmChartInfo{ + { + Name: "postgres", + Version: "9.0.0", + Resources: map[string]int{ + "daemonsets": 1, + "deployments": 1, + "replicasets": 1, + "services": 1, + "statefulsets": 1, + }, + }, + } + + tests := []struct { + name string + fakeClientFunc func() dynamic.Interface + wantRecords []record.Record + wantErrors int + }{ + { + name: "valid helm resources", + fakeClientFunc: func() dynamic.Interface { + fakeClient := fake.NewSimpleDynamicClient(runtime.NewScheme(), []runtime.Object{ + &unstructured.Unstructured{ + Object: map[string]interface{}{ + "apiVersion": "apps/v1", + "kind": "Deployment", + "metadata": map[string]interface{}{ + "name": "mydeployment", + "namespace": "mynamespace", + "labels": map[string]interface{}{ + "app.kubernetes.io/managed-by": "Helm", + "helm.sh/chart": "postgres-9.0.0", + }, + }, + "spec": map[string]interface{}{}, + }, + }, + &unstructured.Unstructured{ + Object: map[string]interface{}{ + "apiVersion": "apps/v1", + "kind": "Replicaset", + "metadata": map[string]interface{}{ + "name": "myreplicaset", + "namespace": "mynamespace", + "labels": map[string]interface{}{ + "app.kubernetes.io/managed-by": "Helm", + "helm.sh/chart": "postgres-9.0.0", + }, + }, + "spec": map[string]interface{}{}, + }, + }, + &unstructured.Unstructured{ + Object: map[string]interface{}{ + "apiVersion": "apps/v1", + "kind": "Daemonset", + "metadata": map[string]interface{}{ + "name": "mydemonset", + "namespace": "mynamespace", + "labels": map[string]interface{}{ + "app.kubernetes.io/managed-by": "Helm", + "helm.sh/chart": "postgres-9.0.0", + }, + }, + "spec": map[string]interface{}{}, + }, + }, + &unstructured.Unstructured{ + Object: map[string]interface{}{ + "apiVersion": "apps/v1", + "kind": "Statefulset", + "metadata": map[string]interface{}{ + "name": "mystateful", + "namespace": "mynamespace", + "labels": map[string]interface{}{ + "app.kubernetes.io/managed-by": "Helm", + "helm.sh/chart": "postgres-9.0.0", + }, + }, + "spec": map[string]interface{}{}, + }, + }, + &unstructured.Unstructured{ + Object: map[string]interface{}{ + "apiVersion": "v1", + "kind": "Service", + "metadata": map[string]interface{}{ + "name": "myservice", + "namespace": "mynamespace", + "labels": map[string]interface{}{ + "app.kubernetes.io/managed-by": "Helm", + "helm.sh/chart": "postgres-9.0.0", + }, + }, + "spec": map[string]interface{}{}, + }, + }, + }...) + return fakeClient + }, + wantRecords: []record.Record{ + { + Name: "config/helmchart_info", + Item: record.JSONMarshaller{Object: &helmChartInfoList.Namespaces}, + }, + }, + wantErrors: 0, + }, + { + name: "invalid helm resources", + fakeClientFunc: func() dynamic.Interface { + fakeClient := fake.NewSimpleDynamicClient(runtime.NewScheme(), []runtime.Object{ + &unstructured.Unstructured{ + Object: map[string]interface{}{ + "apiVersion": "apps/v1", + "kind": "Deployment", + "metadata": map[string]interface{}{ + "name": "mydeployment", + "namespace": "mynamespace", + "labels": map[string]interface{}{}, + }, + "spec": map[string]interface{}{}, + }, + }, + &unstructured.Unstructured{ + Object: map[string]interface{}{ + "apiVersion": "apps/v1", + "kind": "Replicaset", + "metadata": map[string]interface{}{ + "name": "myreplicaset", + "namespace": "mynamespace", + "labels": nil, + }, + "spec": map[string]interface{}{}, + }, + }, + &unstructured.Unstructured{ + Object: map[string]interface{}{ + "apiVersion": "apps/v1", + "kind": "Daemonset", + "metadata": map[string]interface{}{ + "name": "mydemonset", + "namespace": "mynamespace", + "labels": map[string]interface{}{ + "app.kubernetes.io/managed-by": "Helm", + "app.kubernetes.io/version": "1.0.0", + }, + }, + "spec": map[string]interface{}{}, + }, + }, + &unstructured.Unstructured{ + Object: map[string]interface{}{ + "apiVersion": "apps/v1", + "kind": "Statefulset", + "metadata": map[string]interface{}{ + "name": "mystateful", + "namespace": "mynamespace", + "labels": map[string]interface{}{ + "helm.sh/chart": "postgres-9.0.0", + }, + }, + "spec": map[string]interface{}{}, + }, + }, + &unstructured.Unstructured{ + Object: map[string]interface{}{ + "apiVersion": "v1", + "kind": "Service", + "metadata": map[string]interface{}{ + "name": "myservice", + "namespace": "mynamespace", + "labels": map[string]interface{}{}, + }, + "spec": map[string]interface{}{}, + }, + }, + }...) + return fakeClient + }, + wantRecords: nil, + wantErrors: 0, + }, + } + + for _, testCase := range tests { + tt := testCase + t.Run(tt.name, func(t *testing.T) { + t.Parallel() + + dynamicClient := tt.fakeClientFunc() + records, errs := gatherHelmInfo(ctx, dynamicClient) + + assert.Equal(t, tt.wantRecords, records) + assert.Len(t, errs, tt.wantErrors) + }) + } +} + +func TestHelmChartNameAndVersion(t *testing.T) { + type args struct { + chart string + } + tests := []struct { + name string + args args + wantName string + wantVersion string + }{ + { + name: "Test with composed valid chart name and version", + args: args{chart: "nginx-server-1.2.3"}, + wantName: "nginx-server", + wantVersion: "1.2.3", + }, + { + name: "Test with simple valid chart name and version", + args: args{chart: "postgres-2.1.0"}, + wantName: "postgres", + wantVersion: "2.1.0", + }, + { + name: "Test with simple valid chart name but no version", + args: args{chart: "postgres"}, + wantName: "postgres", + wantVersion: "", + }, + { + name: "Test with composed valid chart name but no version", + args: args{chart: "postgres-alpine"}, + wantName: "postgres-alpine", + wantVersion: "", + }, + { + name: "Test with composed valid chart name and latest", + args: args{chart: "postgres-alpine-latest"}, + wantName: "postgres-alpine", + wantVersion: "latest", + }, + { + name: "Test with 3 parts valid chart name no version", + args: args{chart: "postgres-alpine-chart"}, + wantName: "postgres-alpine-chart", + wantVersion: "", + }, + } + for _, testCase := range tests { + tt := testCase + t.Run(tt.name, func(t *testing.T) { + t.Parallel() + + gotName, gotVersion := helmChartNameAndVersion(tt.args.chart) + assert.Equalf(t, tt.wantName, gotName, "expected name to be '%s', got '%s'", tt.wantName, gotName) + assert.Equalf(t, tt.wantVersion, gotVersion, "expected version to be '%s', got '%s'", tt.wantVersion, gotVersion) + }) + } +} + +func TestIsStringVersion(t *testing.T) { + tests := []struct { + version string + isValid bool + }{ + {"latest", true}, + {"beta", true}, + {"alpha", true}, + {"v1.2.3", false}, + {"1.2.3", false}, + {"", false}, + } + + for _, testCase := range tests { + tt := testCase + t.Run(tt.version, func(t *testing.T) { + t.Parallel() + + result := isStringVersion(tt.version) + assert.Equalf(t, tt.isValid, result, "Version '%s' expects to be '%v', got '%v'", tt.version, tt.isValid, result) + }) + } +} diff --git a/pkg/gatherers/workloads/types.go b/pkg/gatherers/workloads/types.go index 8bad0966f..e37464db9 100644 --- a/pkg/gatherers/workloads/types.go +++ b/pkg/gatherers/workloads/types.go @@ -104,3 +104,48 @@ type workloadImageInfo struct { count int images map[string]workloadImage } + +// HelmChartInfo describes the helm chart data collected by the gather +type HelmChartInfo struct { + Name string `json:"name"` + Version string `json:"version"` + Resources map[string]int `json:"resources"` +} + +// HelmChartInfoList encapsulate the logic to add items to the namespace map +type HelmChartInfoList struct { + Namespaces map[string][]HelmChartInfo +} + +func newHelmChartInfoList() HelmChartInfoList { + return HelmChartInfoList{ + Namespaces: make(map[string][]HelmChartInfo), + } +} + +func (h *HelmChartInfoList) addItem(ns, resourceType string, info HelmChartInfo) { + if _, ok := h.Namespaces[ns]; !ok { + h.Namespaces[ns] = make([]HelmChartInfo, 0) + } + + var helmIdx int + var found bool + for i, n := range h.Namespaces[ns] { + if n.Name == info.Name && n.Version == info.Version { + helmIdx = i + found = true + break + } + } + + if !found { + info.Resources = map[string]int{resourceType: 1} + h.Namespaces[ns] = append(h.Namespaces[ns], info) + return + } + + if h.Namespaces[ns][helmIdx].Resources == nil { + h.Namespaces[ns][helmIdx].Resources = make(map[string]int) + } + h.Namespaces[ns][helmIdx].Resources[resourceType]++ +} diff --git a/pkg/gatherers/workloads/types_test.go b/pkg/gatherers/workloads/types_test.go new file mode 100644 index 000000000..9360ff188 --- /dev/null +++ b/pkg/gatherers/workloads/types_test.go @@ -0,0 +1,105 @@ +package workloads + +import ( + "testing" + + "github.com/stretchr/testify/assert" +) + +func TestAddItem(t *testing.T) { + tests := []struct { + name string + info map[string]map[string][]HelmChartInfo + expectedList map[string][]HelmChartInfo + }{ + { + name: "two namespaces, two charts and multiple resources", + info: map[string]map[string][]HelmChartInfo{ + "mynamespace": { + "deployments": { + {Name: "mychart1", Version: "1.0.0"}, + }, + "statefulsets": { + {Name: "mychart2", Version: "2.0.0"}, + }, + }, + "mynamespace2": { + "deployments": { + {Name: "mychart1", Version: "1.0.0"}, + {Name: "mychart1", Version: "1.0.0"}, + }, + "statefulsets": { + {Name: "mychart1", Version: "1.0.0"}, + {Name: "mychart1", Version: "2.0.0"}, + }, + }, + }, + expectedList: map[string][]HelmChartInfo{ + "mynamespace": { + { + Name: "mychart1", + Version: "1.0.0", + Resources: map[string]int{"deployments": 1}, + }, + { + Name: "mychart2", + Version: "2.0.0", + Resources: map[string]int{"statefulsets": 1}, + }, + }, + "mynamespace2": { + { + Name: "mychart1", + Version: "1.0.0", + Resources: map[string]int{"deployments": 2, "statefulsets": 1}, + }, + { + Name: "mychart1", + Version: "2.0.0", + Resources: map[string]int{"statefulsets": 1}, + }, + }, + }, + }, + { + name: "one namespace, two resources for the same chart", + info: map[string]map[string][]HelmChartInfo{ + "mynamespace": { + "deployments": { + {Name: "mychart1", Version: "1.0.0"}, + }, + "statefulsets": { + {Name: "mychart1", Version: "1.0.0"}, + }, + }, + }, + expectedList: map[string][]HelmChartInfo{ + "mynamespace": { + { + Name: "mychart1", + Version: "1.0.0", + Resources: map[string]int{"deployments": 1, "statefulsets": 1}, + }, + }, + }, + }, + } + + for _, testCase := range tests { + tt := testCase + t.Run(tt.name, func(t *testing.T) { + t.Parallel() + + helmList := newHelmChartInfoList() + for namespace, resources := range tt.info { + for resource, charts := range resources { + for _, chartInfo := range charts { + helmList.addItem(namespace, resource, chartInfo) + } + } + } + + assert.Equal(t, tt.expectedList, helmList.Namespaces, "expected '%v', got '%v'", tt.expectedList, helmList.Namespaces) + }) + } +} diff --git a/pkg/gatherers/workloads/workloads_gatherer.go b/pkg/gatherers/workloads/workloads_gatherer.go index 3848b318e..1a93fec20 100644 --- a/pkg/gatherers/workloads/workloads_gatherer.go +++ b/pkg/gatherers/workloads/workloads_gatherer.go @@ -14,12 +14,14 @@ import ( var workloadsGathererPeriod = time.Hour * 12 type Gatherer struct { + gatherKubeConfig *rest.Config gatherProtoKubeConfig *rest.Config lastProcessingTime time.Time } -func New(gatherProtoKubeConfig *rest.Config) *Gatherer { +func New(gatherKubeConfig, gatherProtoKubeConfig *rest.Config) *Gatherer { return &Gatherer{ + gatherKubeConfig: gatherKubeConfig, gatherProtoKubeConfig: gatherProtoKubeConfig, lastProcessingTime: time.Unix(0, 0), } @@ -36,6 +38,11 @@ func (g *Gatherer) GetGatheringFunctions(context.Context) (map[string]gatherers. return g.GatherWorkloadInfo(ctx) }, }, + "helmchart_info": { + Run: func(ctx context.Context) ([]record.Record, []error) { + return g.GatherHelmInfo(ctx) + }, + }, }, nil } diff --git a/pkg/gatherers/workloads/workloads_gatherer_test.go b/pkg/gatherers/workloads/workloads_gatherer_test.go index e1314b4bc..c033550c5 100644 --- a/pkg/gatherers/workloads/workloads_gatherer_test.go +++ b/pkg/gatherers/workloads/workloads_gatherer_test.go @@ -11,7 +11,7 @@ import ( ) func Test_Gatherer_Basic(t *testing.T) { - gatherer := workloads.New(nil) + gatherer := workloads.New(nil, nil) assert.Equal(t, "workloads", gatherer.GetName()) gatheringFunctions, err := gatherer.GetGatheringFunctions(context.TODO()) assert.NoError(t, err) diff --git a/pkg/gatherers/workloads/workloads_info_test.go b/pkg/gatherers/workloads/workloads_info_test.go index 055c7d201..a8880c72e 100644 --- a/pkg/gatherers/workloads/workloads_info_test.go +++ b/pkg/gatherers/workloads/workloads_info_test.go @@ -32,7 +32,7 @@ func Test_gatherWorkloadInfo(t *testing.T) { config.AcceptContentTypes = "application/vnd.kubernetes.protobuf,application/json" config.ContentType = "application/vnd.kubernetes.protobuf" - g := New(config) + g := New(nil, config) ctx := context.TODO() start := time.Now() records, errs := g.GatherWorkloadInfo(ctx)