Skip to content

Commit

Permalink
fix: Add validation method for controller and crd's
Browse files Browse the repository at this point in the history 
Move controller creation  pre-requisites to validation
method `Runnable()` to avoid creating controller object
incase of validation failure.

This is used for prometheus-agent controller as part of
this commit.
These methods can be re-used for future CRD validation
and controllers

Signed-off-by: Jayapriya Pai <slashpai9@gmail.com>
  • Loading branch information
@slashpai
slashpai committed Apr 19, 2023
1 parent 27943fe commit da926b9
Show file tree
Hide file tree
Showing 3 changed files with 123 additions and 68 deletions.
32 changes: 26 additions & 6 deletions cmd/operator/main.go
View file
Original file line number Diff line number Diff line change
Expand Up @@ -33,6 +33,7 @@ import (
logging "github.com/prometheus-operator/prometheus-operator/internal/log"
"github.com/prometheus-operator/prometheus-operator/pkg/admission"
alertmanagercontroller "github.com/prometheus-operator/prometheus-operator/pkg/alertmanager"
monitoringv1alpha1 "github.com/prometheus-operator/prometheus-operator/pkg/apis/monitoring/v1alpha1"
"github.com/prometheus-operator/prometheus-operator/pkg/k8sutil"
"github.com/prometheus-operator/prometheus-operator/pkg/operator"
prometheusagentcontroller "github.com/prometheus-operator/prometheus-operator/pkg/prometheus/agent"
Expand Down Expand Up @@ -266,11 +267,28 @@ func Main() int {
return 1
}

pao, err := prometheusagentcontroller.New(ctx, cfg, log.With(logger, "component", "prometheusagentoperator"), r)
if err != nil {
fmt.Fprint(os.Stderr, "instantiating prometheus-agent controller failed: ", err)
cancel()
return 1
var pao *prometheusagentcontroller.Operator
verbs := map[string][]string{
monitoringv1alpha1.PrometheusAgentName: {"get", "list", "watch"},
fmt.Sprintf("%s/status", monitoringv1alpha1.PrometheusAgentName): {"update"},
}
cr := k8sutil.NewCRConfig(cfg.Namespaces.AllowList,
verbs,
monitoringv1alpha1.SchemeGroupVersion.String(),
monitoringv1alpha1.PrometheusAgentsKind,
monitoringv1alpha1.PrometheusAgentName)
ok, err := cr.Runnable(ctx, cfg.Host, cfg.TLSInsecure, &cfg.TLSConfig, log.With(logger, "component", "prometheusagentoperator"))
if !ok && err != nil {
fmt.Fprint(os.Stderr, "pre-requistes for prometheus-agent controller failed: ", err)
}

if ok {
pao, err = prometheusagentcontroller.New(ctx, cfg, log.With(logger, "component", "prometheusagentoperator"), r)
if err != nil {
fmt.Fprint(os.Stderr, "instantiating prometheus-agent controller failed: ", err)
cancel()
return 1
}
}

ao, err := alertmanagercontroller.New(ctx, cfg, log.With(logger, "component", "alertmanageroperator"), r)
Expand Down Expand Up @@ -360,7 +378,9 @@ func Main() int {
}))

wg.Go(func() error { return po.Run(ctx) })
wg.Go(func() error { return pao.Run(ctx) })
if pao != nil {
wg.Go(func() error { return pao.Run(ctx) })
}
wg.Go(func() error { return ao.Run(ctx) })
wg.Go(func() error { return to.Run(ctx) })

Expand Down
98 changes: 97 additions & 1 deletion pkg/k8sutil/k8sutil.go
View file
Original file line number Diff line number Diff line change
Expand Up @@ -16,7 +16,6 @@ package k8sutil

import (
"context"
"errors"
"fmt"
"net/http"
"net/url"
Expand All @@ -25,18 +24,22 @@ import (
"strings"

"github.com/cespare/xxhash/v2"
"github.com/go-kit/log"
"github.com/pkg/errors"
monitoringv1 "github.com/prometheus-operator/prometheus-operator/pkg/apis/monitoring/v1"
monitoringv1alpha1 "github.com/prometheus-operator/prometheus-operator/pkg/apis/monitoring/v1alpha1"
monitoringv1beta1 "github.com/prometheus-operator/prometheus-operator/pkg/apis/monitoring/v1beta1"
promversion "github.com/prometheus/common/version"
appsv1 "k8s.io/api/apps/v1"
authv1 "k8s.io/api/authorization/v1"
v1 "k8s.io/api/core/v1"
apiequality "k8s.io/apimachinery/pkg/api/equality"
apierrors "k8s.io/apimachinery/pkg/api/errors"
metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
"k8s.io/apimachinery/pkg/runtime"
"k8s.io/apimachinery/pkg/util/validation"
"k8s.io/client-go/discovery"
"k8s.io/client-go/kubernetes"
clientappsv1 "k8s.io/client-go/kubernetes/typed/apps/v1"
clientv1 "k8s.io/client-go/kubernetes/typed/core/v1"
"k8s.io/client-go/rest"
Expand All @@ -57,6 +60,25 @@ func init() {
_ = monitoringv1beta1.SchemeBuilder.AddToScheme(scheme)
}

// CRConfig is CustomResource config
type CRConfig struct {
nsAllowList map[string]struct{}
verbs map[string][]string
schemeGroupVersion string
kind string
resourceName string
}

func NewCRConfig(nsList map[string]struct{}, verbs map[string][]string, sgv, kind, resource string) CRConfig {
return CRConfig{
nsAllowList: nsList,
verbs: verbs,
schemeGroupVersion: sgv,
kind: kind,
resourceName: resource,
}
}

// PodRunningAndReady returns whether a pod is running and each container has
// passed it's ready state.
func PodRunningAndReady(pod v1.Pod) (bool, error) {
Expand Down Expand Up @@ -114,6 +136,80 @@ func NewClusterConfig(host string, tlsInsecure bool, tlsConfig *rest.TLSClientCo
return cfg, nil
}

func (c CRConfig) Runnable(ctx context.Context, host string, tlsInsecure bool, tlsConfig *rest.TLSClientConfig, logger log.Logger) (bool, error) {
cfg, err := NewClusterConfig(host, tlsInsecure, tlsConfig)
if err != nil {
return false, errors.Wrap(err, "instantiating cluster config failed")
}

kclient, err := kubernetes.NewForConfig(cfg)
if err != nil {
return false, errors.Wrap(err, "instantiating kubernetes client failed")
}

crdInstalled, err := c.IsCRDInstalled(kclient)
if !crdInstalled {
return false, err
}

missingPermissions, err := c.GetMissingPermissions(ctx, kclient)
if err != nil {
return false, err
}
if len(missingPermissions) > 0 {
return false, errors.Wrapf(err, "controller disabled because it lacks the required permissions: missingpermissions %v", missingPermissions)
}

return true, nil
}

func (c CRConfig) IsCRDInstalled(kclient *kubernetes.Clientset) (bool, error) {
crdInstalled, err := IsAPIGroupVersionResourceSupported(kclient.Discovery(), c.schemeGroupVersion, c.resourceName)
if err != nil {
return false, errors.Wrapf(err, "failed to check if the API supports the %s CRD", c.kind)
}
if !crdInstalled {
return false, fmt.Errorf("%s controller disabled because the %s CRD isn't installed", strings.ToLower(c.kind), c.kind)
}
return true, nil
}

// GetMissingPermissions returns the RBAC permissions that the controller would need to be
// granted to fulfill its mission. An empty map means that everything is ok.
func (c CRConfig) GetMissingPermissions(ctx context.Context, kclient *kubernetes.Clientset) (map[string][]string, error) {
var ssar *authv1.SelfSubjectAccessReview
var ssarResponse *authv1.SelfSubjectAccessReview
var err error
missingPermissions := map[string][]string{}

for ns := range c.nsAllowList {
for resource, verbs := range c.verbs {
for _, verb := range verbs {
ssar = &authv1.SelfSubjectAccessReview{
Spec: authv1.SelfSubjectAccessReviewSpec{
ResourceAttributes: &authv1.ResourceAttributes{
Verb: verb,
Group: monitoringv1alpha1.SchemeGroupVersion.Group,
Resource: resource,
// If ns is empty string, it will check cluster-wide
Namespace: ns,
},
},
}
ssarResponse, err = kclient.AuthorizationV1().SelfSubjectAccessReviews().Create(ctx, ssar, metav1.CreateOptions{})
if err != nil {
return nil, err
}
if !ssarResponse.Status.Allowed {
missingPermissions[resource] = append(missingPermissions[resource], verb)
}
}
}
}

return missingPermissions, nil
}

func IsResourceNotFoundError(err error) bool {
se, ok := err.(*apierrors.StatusError)
if !ok {
Expand Down
61 changes: 0 additions & 61 deletions pkg/prometheus/agent/operator.go
View file
Original file line number Diff line number Diff line change
Expand Up @@ -39,7 +39,6 @@ import (
"github.com/pkg/errors"
"github.com/prometheus/client_golang/prometheus"
appsv1 "k8s.io/api/apps/v1"
authv1 "k8s.io/api/authorization/v1"
v1 "k8s.io/api/core/v1"
apierrors "k8s.io/apimachinery/pkg/api/errors"
metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
Expand Down Expand Up @@ -282,25 +281,6 @@ func New(ctx context.Context, conf operator.Config, logger log.Logger, r prometh

// Run the controller.
func (c *Operator) Run(ctx context.Context) error {
crdInstalled, err := k8sutil.IsAPIGroupVersionResourceSupported(c.kclient.Discovery(), monitoringv1alpha1.SchemeGroupVersion.String(), monitoringv1alpha1.PrometheusAgentName)
if err != nil {
level.Warn(c.logger).Log("msg", "failed to check if the API supports the PrometheusAgent CRD", "err ", err)
return nil
}
if !crdInstalled {
level.Info(c.logger).Log("msg", "Prometheus agent controller disabled because the PrometheusAgent CRD isn't installed")
return nil
}

missingPermissions, err := c.getMissingPermissions(ctx)
if err != nil {
return err
}
if len(missingPermissions) > 0 {
level.Warn(c.logger).Log("msg", "Prometheus agent controller disabled because it lacks the required permissions on PrometheusAgent objects", "missingpermissions", fmt.Sprintf("%v", missingPermissions))
return nil
}

errChan := make(chan error)
go func() {
v, err := c.kclient.Discovery().ServerVersion()
Expand Down Expand Up @@ -1332,44 +1312,3 @@ func (c *Operator) handleMonitorNamespaceUpdate(oldo, curo interface{}) {
)
}
}

// getMissingPermissions returns the RBAC permissions that the controller would need to be
// granted to fulfill its mission. An empty map means that everything is ok.
func (c *Operator) getMissingPermissions(ctx context.Context) (map[string][]string, error) {
verbs := map[string][]string{
monitoringv1alpha1.PrometheusAgentName: {"get", "list", "watch"},
fmt.Sprintf("%s/status", monitoringv1alpha1.PrometheusAgentName): {"update"},
}
var ssar *authv1.SelfSubjectAccessReview
var ssarResponse *authv1.SelfSubjectAccessReview
var err error

missingPermissions := map[string][]string{}

for ns := range c.config.Namespaces.PrometheusAllowList {
for resource, verbs := range verbs {
for _, verb := range verbs {
ssar = &authv1.SelfSubjectAccessReview{
Spec: authv1.SelfSubjectAccessReviewSpec{
ResourceAttributes: &authv1.ResourceAttributes{
Verb: verb,
Group: monitoringv1alpha1.SchemeGroupVersion.Group,
Resource: resource,
// If ns is empty string, it will check cluster-wide
Namespace: ns,
},
},
}
ssarResponse, err = c.kclient.AuthorizationV1().SelfSubjectAccessReviews().Create(ctx, ssar, metav1.CreateOptions{})
if err != nil {
return nil, err
}
if !ssarResponse.Status.Allowed {
missingPermissions[resource] = append(missingPermissions[resource], verb)
}
}
}
}

return missingPermissions, nil
}

0 comments on commit da926b9

Please sign in to comment.