Skip to content

Commit

Permalink
Merge pull request kubernetes#123611 from ritazh/authz-mcmetrics
Browse files Browse the repository at this point in the history
Add authz webhook matchcondition metrics
  • Loading branch information
k8s-ci-robot committed Mar 2, 2024
2 parents 65d7550 + e76fce7 commit 3e1da21
Show file tree
Hide file tree
Showing 12 changed files with 517 additions and 86 deletions.
11 changes: 11 additions & 0 deletions pkg/kubeapiserver/authorizer/reload.go
Original file line number Diff line number Diff line change
Expand Up @@ -32,11 +32,13 @@ import (
"k8s.io/apiserver/pkg/authentication/user"
"k8s.io/apiserver/pkg/authorization/authorizer"
"k8s.io/apiserver/pkg/authorization/authorizerfactory"
"k8s.io/apiserver/pkg/authorization/cel"
authorizationmetrics "k8s.io/apiserver/pkg/authorization/metrics"
"k8s.io/apiserver/pkg/authorization/union"
"k8s.io/apiserver/pkg/server/options/authorizationconfig/metrics"
webhookutil "k8s.io/apiserver/pkg/util/webhook"
"k8s.io/apiserver/plugin/pkg/authorizer/webhook"
webhookmetrics "k8s.io/apiserver/plugin/pkg/authorizer/webhook/metrics"
"k8s.io/klog/v2"
"k8s.io/kubernetes/pkg/auth/authorizer/abac"
"k8s.io/kubernetes/pkg/kubeapiserver/authorizer/modes"
Expand Down Expand Up @@ -142,6 +144,8 @@ func (r *reloadableAuthorizerResolver) newForConfig(authzConfig *authzconfig.Aut
*r.initialConfig.WebhookRetryBackoff,
decisionOnError,
configuredAuthorizer.Webhook.MatchConditions,
configuredAuthorizer.Name,
kubeapiserverWebhookMetrics{MatcherMetrics: cel.NewMatcherMetrics()},
)
if err != nil {
return nil, nil, err
Expand All @@ -162,6 +166,13 @@ func (r *reloadableAuthorizerResolver) newForConfig(authzConfig *authzconfig.Aut
return union.New(authorizers...), union.NewRuleResolvers(ruleResolvers...), nil
}

type kubeapiserverWebhookMetrics struct {
// kube-apiserver doesn't report request metrics
webhookmetrics.NoopRequestMetrics
// kube-apiserver does report matchCondition metrics
cel.MatcherMetrics
}

// runReload starts checking the config file for changes and reloads the authorizer when it changes.
// Blocks until ctx is complete.
func (r *reloadableAuthorizerResolver) runReload(ctx context.Context) {
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -26,7 +26,7 @@ import (
authorizationclient "k8s.io/client-go/kubernetes/typed/authorization/v1"
)

// DelegatingAuthorizerConfig is the minimal configuration needed to create an authenticator
// DelegatingAuthorizerConfig is the minimal configuration needed to create an authorizer
// built to delegate authorization to a kube API server
type DelegatingAuthorizerConfig struct {
SubjectAccessReviewClient authorizationclient.AuthorizationV1Interface
Expand Down Expand Up @@ -55,9 +55,6 @@ func (c DelegatingAuthorizerConfig) New() (authorizer.Authorizer, error) {
c.DenyCacheTTL,
*c.WebhookRetryBackoff,
authorizer.DecisionNoOpinion,
webhook.AuthorizerMetrics{
RecordRequestTotal: RecordRequestTotal,
RecordRequestLatency: RecordRequestLatency,
},
NewDelegatingAuthorizerMetrics(),
)
}
Original file line number Diff line number Diff line change
Expand Up @@ -18,18 +18,22 @@ package authorizerfactory

import (
"context"
"sync"

celmetrics "k8s.io/apiserver/pkg/authorization/cel"
webhookmetrics "k8s.io/apiserver/plugin/pkg/authorizer/webhook/metrics"
compbasemetrics "k8s.io/component-base/metrics"
"k8s.io/component-base/metrics/legacyregistry"
)

type registerables []compbasemetrics.Registerable
var registerMetrics sync.Once

// init registers all metrics
func init() {
for _, metric := range metrics {
legacyregistry.MustRegister(metric)
}
// RegisterMetrics registers authorizer metrics.
func RegisterMetrics() {
registerMetrics.Do(func() {
legacyregistry.MustRegister(requestTotal)
legacyregistry.MustRegister(requestLatency)
})
}

var (
Expand All @@ -51,19 +55,26 @@ var (
},
[]string{"code"},
)

metrics = registerables{
requestTotal,
requestLatency,
}
)

var _ = webhookmetrics.AuthorizerMetrics(delegatingAuthorizerMetrics{})

type delegatingAuthorizerMetrics struct {
// no-op for matchCondition metrics for now, delegating authorization doesn't configure match conditions
celmetrics.NoopMatcherMetrics
}

func NewDelegatingAuthorizerMetrics() delegatingAuthorizerMetrics {
RegisterMetrics()
return delegatingAuthorizerMetrics{}
}

// RecordRequestTotal increments the total number of requests for the delegated authorization.
func RecordRequestTotal(ctx context.Context, code string) {
func (delegatingAuthorizerMetrics) RecordRequestTotal(ctx context.Context, code string) {
requestTotal.WithContext(ctx).WithLabelValues(code).Add(1)
}

// RecordRequestLatency measures request latency in seconds for the delegated authorization. Broken down by status code.
func RecordRequestLatency(ctx context.Context, code string, latency float64) {
func (delegatingAuthorizerMetrics) RecordRequestLatency(ctx context.Context, code string, latency float64) {
requestLatency.WithContext(ctx).WithLabelValues(code).Observe(latency)
}
20 changes: 20 additions & 0 deletions staging/src/k8s.io/apiserver/pkg/authorization/cel/matcher.go
Original file line number Diff line number Diff line change
Expand Up @@ -19,6 +19,7 @@ package cel
import (
"context"
"fmt"
"time"

celgo "github.com/google/cel-go/cel"

Expand All @@ -28,11 +29,29 @@ import (

type CELMatcher struct {
CompilationResults []CompilationResult

// These are optional fields which can be populated if metrics reporting is desired
Metrics MatcherMetrics
AuthorizerType string
AuthorizerName string
}

// eval evaluates the given SubjectAccessReview against all cel matchCondition expression
func (c *CELMatcher) Eval(ctx context.Context, r *authorizationv1.SubjectAccessReview) (bool, error) {
var evalErrors []error

metrics := c.Metrics
if metrics == nil {
metrics = NoopMatcherMetrics{}
}
start := time.Now()
defer func() {
metrics.RecordAuthorizationMatchConditionEvaluation(ctx, c.AuthorizerType, c.AuthorizerName, time.Since(start))
if len(evalErrors) > 0 {
metrics.RecordAuthorizationMatchConditionEvaluationFailure(ctx, c.AuthorizerType, c.AuthorizerName)
}
}()

va := map[string]interface{}{
"request": convertObjectToUnstructured(&r.Spec),
}
Expand All @@ -54,6 +73,7 @@ func (c *CELMatcher) Eval(ctx context.Context, r *authorizationv1.SubjectAccessR
// If at least one matchCondition successfully evaluates to FALSE,
// return early
if !match {
metrics.RecordAuthorizationMatchConditionExclusion(ctx, c.AuthorizerType, c.AuthorizerName)
return false, nil
}
}
Expand Down
120 changes: 120 additions & 0 deletions staging/src/k8s.io/apiserver/pkg/authorization/cel/metrics.go
Original file line number Diff line number Diff line change
@@ -0,0 +1,120 @@
/*
Copyright 2024 The Kubernetes Authors.
Licensed under the Apache License, Version 2.0 (the "License");
you may not use this file except in compliance with the License.
You may obtain a copy of the License at
http://www.apache.org/licenses/LICENSE-2.0
Unless required by applicable law or agreed to in writing, software
distributed under the License is distributed on an "AS IS" BASIS,
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
See the License for the specific language governing permissions and
limitations under the License.
*/

package cel

import (
"context"
"sync"
"time"

"k8s.io/component-base/metrics"
"k8s.io/component-base/metrics/legacyregistry"
)

// MatcherMetrics defines methods for reporting matchCondition metrics
type MatcherMetrics interface {
// RecordAuthorizationMatchConditionEvaluation records the total time taken to evaluate matchConditions for an Authorize() call to the given authorizer
RecordAuthorizationMatchConditionEvaluation(ctx context.Context, authorizerType, authorizerName string, elapsed time.Duration)
// RecordAuthorizationMatchConditionEvaluationFailure increments if any evaluation error was encountered evaluating matchConditions for an Authorize() call to the given authorizer
RecordAuthorizationMatchConditionEvaluationFailure(ctx context.Context, authorizerType, authorizerName string)
// RecordAuthorizationMatchConditionExclusion records increments when at least one matchCondition evaluates to false and excludes an Authorize() call to the given authorizer
RecordAuthorizationMatchConditionExclusion(ctx context.Context, authorizerType, authorizerName string)
}

type NoopMatcherMetrics struct{}

func (NoopMatcherMetrics) RecordAuthorizationMatchConditionEvaluation(ctx context.Context, authorizerType, authorizerName string, elapsed time.Duration) {
}
func (NoopMatcherMetrics) RecordAuthorizationMatchConditionEvaluationFailure(ctx context.Context, authorizerType, authorizerName string) {
}
func (NoopMatcherMetrics) RecordAuthorizationMatchConditionExclusion(ctx context.Context, authorizerType, authorizerName string) {
}

type matcherMetrics struct{}

func NewMatcherMetrics() MatcherMetrics {
RegisterMetrics()
return matcherMetrics{}
}

const (
namespace = "apiserver"
subsystem = "authorization"
)

var (
authorizationMatchConditionEvaluationErrorsTotal = metrics.NewCounterVec(
&metrics.CounterOpts{
Namespace: namespace,
Subsystem: subsystem,
Name: "match_condition_evaluation_errors_total",
Help: "Total number of errors when an authorization webhook encounters a match condition error split by authorizer type and name.",
StabilityLevel: metrics.ALPHA,
},
[]string{"type", "name"},
)
authorizationMatchConditionExclusionsTotal = metrics.NewCounterVec(
&metrics.CounterOpts{
Namespace: namespace,
Subsystem: subsystem,
Name: "match_condition_exclusions_total",
Help: "Total number of exclusions when an authorization webhook is skipped because match conditions exclude it.",
StabilityLevel: metrics.ALPHA,
},
[]string{"type", "name"},
)
authorizationMatchConditionEvaluationSeconds = metrics.NewHistogramVec(
&metrics.HistogramOpts{
Namespace: namespace,
Subsystem: subsystem,
Name: "match_condition_evaluation_seconds",
Help: "Authorization match condition evaluation time in seconds, split by authorizer type and name.",
Buckets: []float64{0.001, 0.005, 0.01, 0.025, 0.1, 0.2, 0.25},
StabilityLevel: metrics.ALPHA,
},
[]string{"type", "name"},
)
)

var registerMetrics sync.Once

func RegisterMetrics() {
registerMetrics.Do(func() {
legacyregistry.MustRegister(authorizationMatchConditionEvaluationErrorsTotal)
legacyregistry.MustRegister(authorizationMatchConditionExclusionsTotal)
legacyregistry.MustRegister(authorizationMatchConditionEvaluationSeconds)
})
}

func ResetMetricsForTest() {
authorizationMatchConditionEvaluationErrorsTotal.Reset()
authorizationMatchConditionExclusionsTotal.Reset()
authorizationMatchConditionEvaluationSeconds.Reset()
}

func (matcherMetrics) RecordAuthorizationMatchConditionEvaluationFailure(ctx context.Context, authorizerType, authorizerName string) {
authorizationMatchConditionEvaluationErrorsTotal.WithContext(ctx).WithLabelValues(authorizerType, authorizerName).Inc()
}

func (matcherMetrics) RecordAuthorizationMatchConditionExclusion(ctx context.Context, authorizerType, authorizerName string) {
authorizationMatchConditionExclusionsTotal.WithContext(ctx).WithLabelValues(authorizerType, authorizerName).Inc()
}

func (matcherMetrics) RecordAuthorizationMatchConditionEvaluation(ctx context.Context, authorizerType, authorizerName string, elapsed time.Duration) {
elapsedSeconds := elapsed.Seconds()
authorizationMatchConditionEvaluationSeconds.WithContext(ctx).WithLabelValues(authorizerType, authorizerName).Observe(elapsedSeconds)
}
81 changes: 81 additions & 0 deletions staging/src/k8s.io/apiserver/pkg/authorization/cel/metrics_test.go
Original file line number Diff line number Diff line change
@@ -0,0 +1,81 @@
/*
Copyright 2024 The Kubernetes Authors.
Licensed under the Apache License, Version 2.0 (the "License");
you may not use this file except in compliance with the License.
You may obtain a copy of the License at
http://www.apache.org/licenses/LICENSE-2.0
Unless required by applicable law or agreed to in writing, software
distributed under the License is distributed on an "AS IS" BASIS,
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
See the License for the specific language governing permissions and
limitations under the License.
*/

package cel

import (
"context"
"strings"
"testing"
"time"

"k8s.io/component-base/metrics/legacyregistry"
"k8s.io/component-base/metrics/testutil"
)

func TestRecordAuthorizationMatchConditionEvaluationFailure(t *testing.T) {
testCases := []struct {
desc string
metrics []string
name string
authztype string
want string
}{
{
desc: "evaluation failure total",
metrics: []string{
"apiserver_authorization_match_condition_evaluation_errors_total",
"apiserver_authorization_match_condition_exclusions_total",
"apiserver_authorization_match_condition_evaluation_seconds",
},
name: "wh1.example.com",
authztype: "Webhook",
want: `
# HELP apiserver_authorization_match_condition_evaluation_errors_total [ALPHA] Total number of errors when an authorization webhook encounters a match condition error split by authorizer type and name.
# TYPE apiserver_authorization_match_condition_evaluation_errors_total counter
apiserver_authorization_match_condition_evaluation_errors_total{name="wh1.example.com",type="Webhook"} 1
# HELP apiserver_authorization_match_condition_evaluation_seconds [ALPHA] Authorization match condition evaluation time in seconds, split by authorizer type and name.
# TYPE apiserver_authorization_match_condition_evaluation_seconds histogram
apiserver_authorization_match_condition_evaluation_seconds_bucket{name="wh1.example.com",type="Webhook",le="0.001"} 0
apiserver_authorization_match_condition_evaluation_seconds_bucket{name="wh1.example.com",type="Webhook",le="0.005"} 0
apiserver_authorization_match_condition_evaluation_seconds_bucket{name="wh1.example.com",type="Webhook",le="0.01"} 0
apiserver_authorization_match_condition_evaluation_seconds_bucket{name="wh1.example.com",type="Webhook",le="0.025"} 0
apiserver_authorization_match_condition_evaluation_seconds_bucket{name="wh1.example.com",type="Webhook",le="0.1"} 0
apiserver_authorization_match_condition_evaluation_seconds_bucket{name="wh1.example.com",type="Webhook",le="0.2"} 0
apiserver_authorization_match_condition_evaluation_seconds_bucket{name="wh1.example.com",type="Webhook",le="0.25"} 0
apiserver_authorization_match_condition_evaluation_seconds_bucket{name="wh1.example.com",type="Webhook",le="+Inf"} 1
apiserver_authorization_match_condition_evaluation_seconds_sum{name="wh1.example.com",type="Webhook"} 1
apiserver_authorization_match_condition_evaluation_seconds_count{name="wh1.example.com",type="Webhook"} 1
# HELP apiserver_authorization_match_condition_exclusions_total [ALPHA] Total number of exclusions when an authorization webhook is skipped because match conditions exclude it.
# TYPE apiserver_authorization_match_condition_exclusions_total counter
apiserver_authorization_match_condition_exclusions_total{name="wh1.example.com",type="Webhook"} 1
`,
},
}

for _, tt := range testCases {
t.Run(tt.desc, func(t *testing.T) {
ResetMetricsForTest()
m := NewMatcherMetrics()
m.RecordAuthorizationMatchConditionEvaluationFailure(context.Background(), tt.authztype, tt.name)
m.RecordAuthorizationMatchConditionExclusion(context.Background(), tt.authztype, tt.name)
m.RecordAuthorizationMatchConditionEvaluation(context.Background(), tt.authztype, tt.name, time.Duration(1*time.Second))
if err := testutil.GatherAndCompare(legacyregistry.DefaultGatherer, strings.NewReader(tt.want), tt.metrics...); err != nil {
t.Fatal(err)
}
})
}
}

0 comments on commit 3e1da21

Please sign in to comment.