pkg/operator: refactor resource hierarchy discovery

pkg/operator/apis/monitoring/v1alpha1/types.go

@@ -4,6 +4,7 @@ import (
 	prom_v1 "github.com/prometheus-operator/prometheus-operator/pkg/apis/monitoring/v1"
 	v1 "k8s.io/api/core/v1"
 	metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
+	"sigs.k8s.io/controller-runtime/pkg/client"
 )
 
 // +kubebuilder:object:root=true
@@ -24,6 +25,7 @@ type GrafanaAgent struct {
 // MetricsInstanceSelector returns a selector to find MetricsInstances.
 func (a *GrafanaAgent) MetricsInstanceSelector() ObjectSelector {
 	return ObjectSelector{
+		ObjectType:        &MetricsInstance{},
 		ParentNamespace:   a.Namespace,
 		NamespaceSelector: a.Spec.Metrics.InstanceNamespaceSelector,
 		Labels:            a.Spec.Metrics.InstanceSelector,
@@ -33,6 +35,7 @@ func (a *GrafanaAgent) MetricsInstanceSelector() ObjectSelector {
 // LogsInstanceSelector returns a selector to find LogsInstances.
 func (a *GrafanaAgent) LogsInstanceSelector() ObjectSelector {
 	return ObjectSelector{
+		ObjectType:        &LogsInstance{},
 		ParentNamespace:   a.Namespace,
 		NamespaceSelector: a.Spec.Logs.InstanceNamespaceSelector,
 		Labels:            a.Spec.Logs.InstanceSelector,
@@ -153,6 +156,7 @@ type GrafanaAgentSpec struct {
 // resource hierarchy. When NamespaceSelector is nil, objects should be
 // searched directly in the ParentNamespace.
 type ObjectSelector struct {
+	ObjectType        client.Object
 	ParentNamespace   string
 	NamespaceSelector *metav1.LabelSelector
 	Labels            *metav1.LabelSelector

pkg/operator/apis/monitoring/v1alpha1/types_logs.go

@@ -100,9 +100,10 @@ type LogsInstance struct {
 	Spec LogsInstanceSpec `json:"spec,omitempty"`
 }
 
-// PodLogsInstanceSelector returns the selector to discover PodLogs.
-func (i *LogsInstance) PodLogsInstanceSelector() ObjectSelector {
+// PodLogsSelector returns the selector to discover PodLogs.
+func (i *LogsInstance) PodLogsSelector() ObjectSelector {
 	return ObjectSelector{
+		ObjectType:        &PodLogs{},
 		ParentNamespace:   i.Namespace,
 		NamespaceSelector: i.Spec.PodLogsNamespaceSelector,
 		Labels:            i.Spec.PodLogsSelector,

pkg/operator/apis/monitoring/v1alpha1/types_metrics.go

@@ -182,6 +182,7 @@ type MetricsInstance struct {
 // ServiceMonitorSelector returns a selector to find ServiceMonitors.
 func (p *MetricsInstance) ServiceMonitorSelector() ObjectSelector {
 	return ObjectSelector{
+		ObjectType:        &prom_v1.ServiceMonitor{},
 		ParentNamespace:   p.Namespace,
 		NamespaceSelector: p.Spec.ServiceMonitorNamespaceSelector,
 		Labels:            p.Spec.ServiceMonitorSelector,
@@ -191,6 +192,7 @@ func (p *MetricsInstance) ServiceMonitorSelector() ObjectSelector {
 // PodMonitorSelector returns a selector to find PodMonitors.
 func (p *MetricsInstance) PodMonitorSelector() ObjectSelector {
 	return ObjectSelector{
+		ObjectType:        &prom_v1.PodMonitor{},
 		ParentNamespace:   p.Namespace,
 		NamespaceSelector: p.Spec.PodMonitorNamespaceSelector,
 		Labels:            p.Spec.PodMonitorSelector,
@@ -200,6 +202,7 @@ func (p *MetricsInstance) PodMonitorSelector() ObjectSelector {
 // ProbeSelector returns a selector to find Probes.
 func (p *MetricsInstance) ProbeSelector() ObjectSelector {
 	return ObjectSelector{
+		ObjectType:        &prom_v1.Probe{},
 		ParentNamespace:   p.Namespace,
 		NamespaceSelector: p.Spec.ProbeNamespaceSelector,
 		Labels:            p.Spec.ProbeSelector,

pkg/operator/build_hierarchy.go

@@ -0,0 +1,282 @@
+package operator
+
+import (
+	"context"
+	"fmt"
+
+	"github.com/go-kit/log"
+	"github.com/go-kit/log/level"
+	grafana "github.com/grafana/agent/pkg/operator/apis/monitoring/v1alpha1"
+	"github.com/grafana/agent/pkg/operator/assets"
+	"github.com/grafana/agent/pkg/operator/config"
+	"github.com/grafana/agent/pkg/operator/hierarchy"
+	prom "github.com/prometheus-operator/prometheus-operator/pkg/apis/monitoring/v1"
+	corev1 "k8s.io/api/core/v1"
+	v1 "k8s.io/api/core/v1"
+	"k8s.io/apimachinery/pkg/api/meta"
+	metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
+	"k8s.io/apimachinery/pkg/runtime"
+	"sigs.k8s.io/controller-runtime/pkg/client"
+	"sigs.k8s.io/controller-runtime/pkg/client/apiutil"
+)
+
+// buildHierarchy constructs a resource hierarchy starting from root.
+func buildHierarchy(ctx context.Context, l log.Logger, cli client.Client, root *grafana.GrafanaAgent) (deployment config.Deployment, watchers []hierarchy.Watcher, err error) {
+	deployment.Agent = root
+
+	// search is used throughout BuildHierarchy, where it will perform a list for
+	// a set of objects in the hierarchy and populate the watchers return
+	// variable.
+	search := func(resources []hierarchyResource) error {
+		for _, res := range resources {
+			sel, err := res.Find(ctx, cli)
+			if err != nil {
+				gvk, _ := apiutil.GVKForObject(res.List, cli.Scheme())
+				return fmt.Errorf("failed to find %q resource: %w", gvk.String(), err)
+			}
+
+			watchers = append(watchers, hierarchy.Watcher{
+				Object:   res.Selector.ObjectType,
+				Owner:    client.ObjectKeyFromObject(root),
+				Selector: sel,
+			})
+		}
+		return nil
+	}
+
+	// Root resources
+	var (
+		metricInstances grafana.MetricsInstanceList
+		logsInstances   grafana.LogsInstanceList
+	)
+	var roots = []hierarchyResource{
+		{List: &metricInstances, Selector: root.MetricsInstanceSelector()},
+		{List: &logsInstances, Selector: root.LogsInstanceSelector()},
+	}
+	if err := search(roots); err != nil {
+		return deployment, nil, err
+	}
+
+	// Metrics resources
+	for _, metricsInst := range metricInstances.Items {
+		var (
+			serviceMonitors prom.ServiceMonitorList
+			podMonitors     prom.PodMonitorList
+			probes          prom.ProbeList
+		)
+		var children = []hierarchyResource{
+			{List: &serviceMonitors, Selector: metricsInst.ServiceMonitorSelector()},
+			{List: &podMonitors, Selector: metricsInst.PodMonitorSelector()},
+			{List: &probes, Selector: metricsInst.ProbeSelector()},
+		}
+		if err := search(children); err != nil {
+			return deployment, nil, err
+		}
+
+		deployment.Metrics = append(deployment.Metrics, config.MetricsInstance{
+			Instance:        metricsInst,
+			ServiceMonitors: filterServiceMonitors(l, root, &serviceMonitors).Items,
+			PodMonitors:     podMonitors.Items,
+			Probes:          probes.Items,
+		})
+	}
+
+	// Logs resources
+	for _, logsInst := range logsInstances.Items {
+		var (
+			podLogs grafana.PodLogsList
+		)
+		var children = []hierarchyResource{
+			{List: &podLogs, Selector: logsInst.PodLogsSelector()},
+		}
+		if err := search(children); err != nil {
+			return deployment, nil, err
+		}
+
+		deployment.Logs = append(deployment.Logs, config.LogInstance{
+			Instance: logsInst,
+			PodLogs:  podLogs.Items,
+		})
+	}
+
+	// Finally, find all referenced secrets
+	secrets, secretWatchers, err := buildSecrets(ctx, cli, deployment)
+	if err != nil {
+		return deployment, nil, fmt.Errorf("failed to discover secrets: %w", err)
+	}
+	deployment.Secrets = secrets
+	watchers = append(watchers, secretWatchers...)
+
+	return deployment, watchers, nil
+}
+
+type hierarchyResource struct {
+	List     client.ObjectList      // List to populate
+	Selector grafana.ObjectSelector // Raw selector to use for list
+}
+
+func (hr *hierarchyResource) Find(ctx context.Context, cli client.Client) (hierarchy.Selector, error) {
+	sel, err := toSelector(hr.Selector)
+	if err != nil {
+		return nil, fmt.Errorf("failed to build selector: %w", err)
+	}
+	err = hierarchy.List(ctx, cli, hr.List, sel)
+	if err != nil {
+		return nil, fmt.Errorf("failed to list resources: %w", err)
+	}
+	return sel, nil
+}
+
+func toSelector(os grafana.ObjectSelector) (hierarchy.Selector, error) {
+	var res hierarchy.LabelsSelector
+	res.NamespaceName = os.ParentNamespace
+
+	if os.NamespaceSelector != nil {
+		sel, err := metav1.LabelSelectorAsSelector(os.NamespaceSelector)
+		if err != nil {
+			return nil, fmt.Errorf("invalid namespace selector: %w", err)
+		}
+		res.NamespaceLabels = sel
+	}
+
+	sel, err := metav1.LabelSelectorAsSelector(os.Labels)
+	if err != nil {
+		return nil, fmt.Errorf("invalid label selector: %w", err)
+	}
+	res.Labels = sel
+	return &res, nil
+}
+
+func filterServiceMonitors(l log.Logger, root *grafana.GrafanaAgent, list *prom.ServiceMonitorList) *prom.ServiceMonitorList {
+	items := make([]*prom.ServiceMonitor, 0, len(list.Items))
+
+Item:
+	for _, item := range list.Items {
+		if root.Spec.Metrics.ArbitraryFSAccessThroughSMs.Deny {
+			for _, ep := range item.Spec.Endpoints {
+				err := testForArbitraryFSAccess(ep)
+				if err == nil {
+					continue
+				}
+
+				level.Warn(l).Log(
+					"msg", "skipping service monitor",
+					"agent", client.ObjectKeyFromObject(root),
+					"servicemonitor", client.ObjectKeyFromObject(item),
+					"err", err,
+				)
+				continue Item
+			}
+		}
+		items = append(items, item)
+	}
+
+	return &prom.ServiceMonitorList{
+		TypeMeta: list.TypeMeta,
+		ListMeta: *list.ListMeta.DeepCopy(),
+		Items:    items,
+	}
+}
+
+func testForArbitraryFSAccess(e prom.Endpoint) error {
+	if e.BearerTokenFile != "" {
+		return fmt.Errorf("it accesses file system via bearer token file which is disallowed via GrafanaAgent specification")
+	}
+
+	if e.TLSConfig == nil {
+		return nil
+	}
+
+	if e.TLSConfig.CAFile != "" || e.TLSConfig.CertFile != "" || e.TLSConfig.KeyFile != "" {
+		return fmt.Errorf("it accesses file system via TLS config which is disallowed via GrafanaAgent specification")
+	}
+
+	return nil
+}
+
+func buildSecrets(ctx context.Context, cli client.Client, deploy config.Deployment) (secrets assets.SecretStore, watchers []hierarchy.Watcher, err error) {
+	secrets = make(assets.SecretStore)
+
+	// KeySelector caches to make sure we don't create duplicate watchers.
+	var (
+		usedSecretSelectors    = map[hierarchy.KeySelector]struct{}{}
+		usedConfigMapSelectors = map[hierarchy.KeySelector]struct{}{}
+	)
+
+	for _, ref := range deploy.AssetReferences() {
+		var (
+			objectList client.ObjectList
+			sel        hierarchy.KeySelector
+		)
+
+		switch {
+		case ref.Reference.Secret != nil:
+			objectList = &corev1.SecretList{}
+			sel = hierarchy.KeySelector{
+				Namespace: ref.Namespace,
+				Name:      ref.Reference.Secret.Name,
+			}
+		case ref.Reference.ConfigMap != nil:
+			objectList = &corev1.ConfigMapList{}
+			sel = hierarchy.KeySelector{
+				Namespace: ref.Namespace,
+				Name:      ref.Reference.ConfigMap.Name,
+			}
+		}
+
+		gvk, _ := apiutil.GVKForObject(objectList, cli.Scheme())
+		if err := hierarchy.List(ctx, cli, objectList, &sel); err != nil {
+			return nil, nil, fmt.Errorf("failed to find %q resource: %w", gvk.String(), err)
+		}
+
+		err := meta.EachListItem(objectList, func(o runtime.Object) error {
+			var value string
+
+			switch o := o.(type) {
+			case *corev1.Secret:
+				rawValue, ok := o.Data[ref.Reference.Secret.Key]
+				if !ok {
+					return fmt.Errorf("no key %s in Secret %s", ref.Reference.ConfigMap.Key, o.Name)
+				}
+				value = string(rawValue)
+			case *corev1.ConfigMap:
+				rawValue, ok := o.BinaryData[ref.Reference.ConfigMap.Key]
+				if !ok {
+					return fmt.Errorf("no key %s in ConfigMap %s", ref.Reference.ConfigMap.Key, o.Name)
+				}
+				value = string(rawValue)
+			}
+
+			secrets[assets.KeyForSelector(ref.Namespace, &ref.Reference)] = value
+			return nil
+		})
+		if err != nil {
+			return nil, nil, fmt.Errorf("failed to iterate over %q list: %w", gvk.String(), err)
+		}
+
+		switch {
+		case ref.Reference.Secret != nil:
+			if _, used := usedSecretSelectors[sel]; used {
+				continue
+			}
+			watchers = append(watchers, hierarchy.Watcher{
+				Object:   &v1.Secret{},
+				Owner:    client.ObjectKeyFromObject(deploy.Agent),
+				Selector: &sel,
+			})
+			usedSecretSelectors[sel] = struct{}{}
+		case ref.Reference.ConfigMap != nil:
+			if _, used := usedConfigMapSelectors[sel]; used {
+				continue
+			}
+			watchers = append(watchers, hierarchy.Watcher{
+				Object:   &v1.ConfigMap{},
+				Owner:    client.ObjectKeyFromObject(deploy.Agent),
+				Selector: &sel,
+			})
+			usedConfigMapSelectors[sel] = struct{}{}
+		}
+	}
+
+	return secrets, watchers, nil
+}