Skip to content

Commit

Permalink
added: admission_request_timestamp for kyverno_admission_review_laten…
Browse files Browse the repository at this point in the history 
…cy_milliseconds and a small fix (#1970)

Signed-off-by: Yashvardhan Kukreja <yash.kukreja.98@gmail.com>
  • Loading branch information
@yashvardhan-kukreja
yashvardhan-kukreja committed Jun 7, 2021
1 parent e227636 commit a931f8f
Show file tree
Hide file tree
Showing 5 changed files with 40 additions and 24 deletions.
25 changes: 14 additions & 11 deletions pkg/metrics/admissionreviewlatency/admissionReviewLatency.go
View file
Expand Up @@ -6,6 +6,7 @@ import (
"github.com/kyverno/kyverno/pkg/engine/response"
"github.com/kyverno/kyverno/pkg/metrics"
prom "github.com/prometheus/client_golang/prometheus"
"time"
)

func (pm PromMetrics) registerAdmissionReviewLatencyMetric(
Expand All @@ -14,22 +15,24 @@ func (pm PromMetrics) registerAdmissionReviewLatencyMetric(
resourceName, resourceKind, resourceNamespace string,
resourceRequestOperation metrics.ResourceRequestOperation,
admissionRequestLatency float64,
admissionRequestTimestamp int64,
) error {
pm.AdmissionReviewLatency.With(prom.Labels{
"cluster_policies_count": fmt.Sprintf("%d", clusterPoliciesCount),
"namespaced_policies_count": fmt.Sprintf("%d", namespacedPoliciesCount),
"validate_rules_count": fmt.Sprintf("%d", validateRulesCount),
"mutate_rules_count": fmt.Sprintf("%d", mutateRulesCount),
"generate_rules_count": fmt.Sprintf("%d", generateRulesCount),
"resource_name": resourceName,
"resource_kind": resourceKind,
"resource_namespace": resourceNamespace,
"resource_request_operation": string(resourceRequestOperation),
"cluster_policies_count": fmt.Sprintf("%d", clusterPoliciesCount),
"namespaced_policies_count": fmt.Sprintf("%d", namespacedPoliciesCount),
"validate_rules_count": fmt.Sprintf("%d", validateRulesCount),
"mutate_rules_count": fmt.Sprintf("%d", mutateRulesCount),
"generate_rules_count": fmt.Sprintf("%d", generateRulesCount),
"resource_name": resourceName,
"resource_kind": resourceKind,
"resource_namespace": resourceNamespace,
"resource_request_operation": string(resourceRequestOperation),
"admission_request_timestamp": fmt.Sprintf("%+v", time.Unix(admissionRequestTimestamp, 0)),
}).Set(admissionRequestLatency)
return nil
}

func (pm PromMetrics) ProcessEngineResponses(engineResponses []*response.EngineResponse, triggeredPolicies []kyverno.ClusterPolicy, admissionReviewLatencyDuration int64, resourceRequestOperation metrics.ResourceRequestOperation) error {
func (pm PromMetrics) ProcessEngineResponses(engineResponses []*response.EngineResponse, triggeredPolicies []kyverno.ClusterPolicy, admissionReviewLatencyDuration int64, resourceRequestOperation metrics.ResourceRequestOperation, admissionRequestTimestamp int64) error {
if len(engineResponses) == 0 {
return nil
}
Expand Down Expand Up @@ -64,5 +67,5 @@ func (pm PromMetrics) ProcessEngineResponses(engineResponses []*response.EngineR
return nil
}
admissionReviewLatencyDurationInMs := float64(admissionReviewLatencyDuration) / float64(1000*1000)
return pm.registerAdmissionReviewLatencyMetric(clusterPoliciesCount, namespacedPoliciesCount, totalValidateRulesCount, totalMutateRulesCount, totalGenerateRulesCount, resourceName, resourceKind, resourceNamespace, resourceRequestOperation, admissionReviewLatencyDurationInMs)
return pm.registerAdmissionReviewLatencyMetric(clusterPoliciesCount, namespacedPoliciesCount, totalValidateRulesCount, totalMutateRulesCount, totalGenerateRulesCount, resourceName, resourceKind, resourceNamespace, resourceRequestOperation, admissionReviewLatencyDurationInMs, admissionRequestTimestamp)
}
1 change: 1 addition & 0 deletions pkg/metrics/metrics.go
View file
Expand Up @@ -76,6 +76,7 @@ func NewPromConfig() *PromConfig {
"cluster_policies_count", "namespaced_policies_count",
"validate_rules_count", "mutate_rules_count", "generate_rules_count",
"resource_name", "resource_kind", "resource_namespace", "resource_request_operation",
"admission_request_timestamp",
}
admissionReviewLatencyMetric := prom.NewGaugeVec(
prom.GaugeOpts{
Expand Down
7 changes: 6 additions & 1 deletion pkg/webhooks/generation.go
View file
Expand Up @@ -36,10 +36,12 @@ import (
)

//HandleGenerate handles admission-requests for policies with generate rules
func (ws *WebhookServer) HandleGenerate(request *v1beta1.AdmissionRequest, policies []*kyverno.ClusterPolicy, ctx *context.Context, userRequestInfo kyverno.RequestInfo, dynamicConfig config.Interface, admissionRequestTimestamp int64, latencySender *chan int64) {
func (ws *WebhookServer) HandleGenerate(request *v1beta1.AdmissionRequest, policies []*kyverno.ClusterPolicy, ctx *context.Context, userRequestInfo kyverno.RequestInfo, dynamicConfig config.Interface, admissionRequestTimestamp int64, latencySender *chan int64, triggeredGeneratePoliciesSender *chan []kyverno.ClusterPolicy, generateEngineResponsesSender *chan []*response.EngineResponse) {
logger := ws.log.WithValues("action", "generation", "uid", request.UID, "kind", request.Kind, "namespace", request.Namespace, "name", request.Name, "operation", request.Operation, "gvk", request.Kind.String())
logger.V(4).Info("incoming request")

var engineResponses []*response.EngineResponse
var triggeredGeneratePolicies []kyverno.ClusterPolicy
if (request.Operation == v1beta1.Create || request.Operation == v1beta1.Update) && len(policies) != 0 {
// convert RAW to unstructured
new, old, err := kyvernoutils.ExtractResources(nil, request)
Expand Down Expand Up @@ -77,6 +79,7 @@ func (ws *WebhookServer) HandleGenerate(request *v1beta1.AdmissionRequest, polic
engineResponse.PolicyResponse.Rules = rules
// some generate rules do apply to the resource
engineResponses = append(engineResponses, engineResponse)
triggeredGeneratePolicies = append(triggeredGeneratePolicies, *policy)
ws.statusListener.Update(generateStats{
resp: engineResponse,
})
Expand Down Expand Up @@ -105,6 +108,8 @@ func (ws *WebhookServer) HandleGenerate(request *v1beta1.AdmissionRequest, polic
// sending the admission request latency to other goroutine (reporting the metrics) over the channel
admissionReviewLatencyDuration := int64(time.Since(time.Unix(admissionRequestTimestamp, 0)))
*latencySender <- admissionReviewLatencyDuration
*triggeredGeneratePoliciesSender <- triggeredGeneratePolicies
*generateEngineResponsesSender <- engineResponses
}

func (ws *WebhookServer) registerPolicyRuleResultsMetricGeneration(logger logr.Logger, resourceRequestOperation string, policy kyverno.ClusterPolicy, engineResponse response.EngineResponse, admissionRequestTimestamp int64) {
Expand Down
23 changes: 15 additions & 8 deletions pkg/webhooks/server.go
View file
Expand Up @@ -375,19 +375,20 @@ func (ws *WebhookServer) ResourceMutation(request *v1beta1.AdmissionRequest) *v1
logger.V(6).Info("", "patchedResource", string(patchedResource))
admissionReviewLatencyDuration := int64(time.Since(time.Unix(admissionRequestTimestamp, 0)))
// registering the kyverno_admission_review_latency_milliseconds metric concurrently
go registerAdmissionReviewLatencyMetricMutate(logger, *ws.promConfig.Metrics, string(request.Operation), mutateEngineResponses, triggeredMutatePolicies, admissionReviewLatencyDuration)
go registerAdmissionReviewLatencyMetricMutate(logger, *ws.promConfig.Metrics, string(request.Operation), mutateEngineResponses, triggeredMutatePolicies, admissionReviewLatencyDuration, admissionRequestTimestamp)

// GENERATE
newRequest := request.DeepCopy()
newRequest.Object.Raw = patchedResource

// this channel will be used to transmit the admissionReviewLatency from ws.HandleGenerate(..,) goroutine to registeGeneraterPolicyAdmissionReviewLatencyMetric(...) goroutine
admissionReviewCompletionLatencyChannel := make(chan int64, 1)
triggeredGeneratePoliciesChannel := make(chan []v1.ClusterPolicy, 1)
generateEngineResponsesChannel := make(chan []*response.EngineResponse, 1)

go ws.HandleGenerate(newRequest, generatePolicies, ctx, userRequestInfo, ws.configHandler, admissionRequestTimestamp, &admissionReviewCompletionLatencyChannel)

go ws.HandleGenerate(newRequest, generatePolicies, ctx, userRequestInfo, ws.configHandler, admissionRequestTimestamp, &admissionReviewCompletionLatencyChannel, &triggeredGeneratePoliciesChannel, &generateEngineResponsesChannel)
// registering the kyverno_admission_review_latency_milliseconds metric concurrently
go registerAdmissionReviewLatencyMetricGenerate(logger, *ws.promConfig.Metrics, string(request.Operation), mutateEngineResponses, triggeredMutatePolicies, &admissionReviewCompletionLatencyChannel)
go registerAdmissionReviewLatencyMetricGenerate(logger, *ws.promConfig.Metrics, string(newRequest.Operation), admissionRequestTimestamp, &admissionReviewCompletionLatencyChannel, &triggeredGeneratePoliciesChannel, &generateEngineResponsesChannel)
patchType := v1beta1.PatchTypeJSONPatch
return &v1beta1.AdmissionResponse{
Allowed: true,
Expand All @@ -399,25 +400,31 @@ func (ws *WebhookServer) ResourceMutation(request *v1beta1.AdmissionRequest) *v1
}
}

func registerAdmissionReviewLatencyMetricMutate(logger logr.Logger, promMetrics metrics.PromMetrics, requestOperation string, engineResponses []*response.EngineResponse, triggeredPolicies []v1.ClusterPolicy, admissionReviewLatencyDuration int64) {
func registerAdmissionReviewLatencyMetricMutate(logger logr.Logger, promMetrics metrics.PromMetrics, requestOperation string, engineResponses []*response.EngineResponse, triggeredPolicies []v1.ClusterPolicy, admissionReviewLatencyDuration int64, admissionRequestTimestamp int64) {
resourceRequestOperationPromAlias, err := admissionReviewLatency.ParseResourceRequestOperation(requestOperation)
if err != nil {
logger.Error(err, "error occurred while registering kyverno_admission_review_latency_milliseconds metrics")
}
if err := admissionReviewLatency.ParsePromMetrics(promMetrics).ProcessEngineResponses(engineResponses, triggeredPolicies, admissionReviewLatencyDuration, resourceRequestOperationPromAlias); err != nil {
if err := admissionReviewLatency.ParsePromMetrics(promMetrics).ProcessEngineResponses(engineResponses, triggeredPolicies, admissionReviewLatencyDuration, resourceRequestOperationPromAlias, admissionRequestTimestamp); err != nil {
logger.Error(err, "error occurred while registering kyverno_admission_review_latency_milliseconds metrics")
}
}

func registerAdmissionReviewLatencyMetricGenerate(logger logr.Logger, promMetrics metrics.PromMetrics, requestOperation string, engineResponses []*response.EngineResponse, triggeredPolicies []v1.ClusterPolicy, latencyReceiver *chan int64) {
func registerAdmissionReviewLatencyMetricGenerate(logger logr.Logger, promMetrics metrics.PromMetrics, requestOperation string, admissionRequestTimestamp int64, latencyReceiver *chan int64, triggeredGeneratePoliciesReceiver *chan []v1.ClusterPolicy, engineResponsesReceiver *chan []*response.EngineResponse) {
defer close(*latencyReceiver)
defer close(*triggeredGeneratePoliciesReceiver)
defer close(*engineResponsesReceiver)

triggeredPolicies := <-(*triggeredGeneratePoliciesReceiver)
engineResponses := <-(*engineResponsesReceiver)

resourceRequestOperationPromAlias, err := admissionReviewLatency.ParseResourceRequestOperation(requestOperation)
if err != nil {
logger.Error(err, "error occurred while registering kyverno_admission_review_latency_milliseconds metrics")
}
// this goroutine will keep on waiting here till it doesn't receive the admission review latency int64 from the other goroutine i.e. ws.HandleGenerate
admissionReviewLatencyDuration := <-(*latencyReceiver)
if err := admissionReviewLatency.ParsePromMetrics(promMetrics).ProcessEngineResponses(engineResponses, triggeredPolicies, admissionReviewLatencyDuration, resourceRequestOperationPromAlias); err != nil {
if err := admissionReviewLatency.ParsePromMetrics(promMetrics).ProcessEngineResponses(engineResponses, triggeredPolicies, admissionReviewLatencyDuration, resourceRequestOperationPromAlias, admissionRequestTimestamp); err != nil {
logger.Error(err, "error occurred while registering kyverno_admission_review_latency_milliseconds metrics")
}
}
Expand Down
8 changes: 4 additions & 4 deletions pkg/webhooks/validation.go
View file
Expand Up @@ -147,7 +147,7 @@ func HandleValidation(
logger.V(4).Info("resource blocked")
//registering the kyverno_admission_review_latency_milliseconds metric concurrently
admissionReviewLatencyDuration := int64(time.Since(time.Unix(admissionRequestTimestamp, 0)))
go registerAdmissionReviewLatencyMetricValidate(promConfig, logger, string(request.Operation), engineResponses, triggeredPolicies, admissionReviewLatencyDuration)
go registerAdmissionReviewLatencyMetricValidate(promConfig, logger, string(request.Operation), engineResponses, triggeredPolicies, admissionReviewLatencyDuration, admissionRequestTimestamp)
return false, getEnforceFailureErrorMsg(engineResponses)
}

Expand All @@ -161,7 +161,7 @@ func HandleValidation(

//registering the kyverno_admission_review_latency_milliseconds metric concurrently
admissionReviewLatencyDuration := int64(time.Since(time.Unix(admissionRequestTimestamp, 0)))
go registerAdmissionReviewLatencyMetricValidate(promConfig, logger, string(request.Operation), engineResponses, triggeredPolicies, admissionReviewLatencyDuration)
go registerAdmissionReviewLatencyMetricValidate(promConfig, logger, string(request.Operation), engineResponses, triggeredPolicies, admissionReviewLatencyDuration, admissionRequestTimestamp)

return true, ""
}
Expand All @@ -186,12 +186,12 @@ func registerPolicyRuleExecutionLatencyMetricValidate(promConfig *metrics.PromCo
}
}

func registerAdmissionReviewLatencyMetricValidate(promConfig *metrics.PromConfig, logger logr.Logger, requestOperation string, engineResponses []*response.EngineResponse, triggeredPolicies []kyverno.ClusterPolicy, admissionReviewLatencyDuration int64) {
func registerAdmissionReviewLatencyMetricValidate(promConfig *metrics.PromConfig, logger logr.Logger, requestOperation string, engineResponses []*response.EngineResponse, triggeredPolicies []kyverno.ClusterPolicy, admissionReviewLatencyDuration int64, admissionRequestTimestamp int64) {
resourceRequestOperationPromAlias, err := admissionReviewLatency.ParseResourceRequestOperation(requestOperation)
if err != nil {
logger.Error(err, "error occurred while registering kyverno_admission_review_latency_milliseconds metrics")
}
if err := admissionReviewLatency.ParsePromMetrics(*promConfig.Metrics).ProcessEngineResponses(engineResponses, triggeredPolicies, admissionReviewLatencyDuration, resourceRequestOperationPromAlias); err != nil {
if err := admissionReviewLatency.ParsePromMetrics(*promConfig.Metrics).ProcessEngineResponses(engineResponses, triggeredPolicies, admissionReviewLatencyDuration, resourceRequestOperationPromAlias, admissionRequestTimestamp); err != nil {
logger.Error(err, "error occurred while registering kyverno_admission_review_latency_milliseconds metrics")
}
}
Expand Down

0 comments on commit a931f8f

Please sign in to comment.