Skip to content
New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Add best-effort validation for prometheus scrape interval #11376

Merged
merged 4 commits into from Sep 26, 2023
Merged
Show file tree
Hide file tree
Changes from 3 commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Jump to
Jump to file
Failed to load files.
Diff view
Diff view
42 changes: 42 additions & 0 deletions viz/metrics-api/grpc_server.go
Expand Up @@ -15,6 +15,7 @@ import (
promv1 "github.com/prometheus/client_golang/api/prometheus/v1"
log "github.com/sirupsen/logrus"
"google.golang.org/grpc"
"gopkg.in/yaml.v2"
corev1 "k8s.io/api/core/v1"
"k8s.io/apimachinery/pkg/labels"
)
Expand Down Expand Up @@ -237,3 +238,44 @@ func (s *grpcServer) ListServices(ctx context.Context, req *pb.ListServicesReque

return &pb.ListServicesResponse{Services: svcs}, nil
}

// validateTimeWindow returns an error if the Prometheus scrape interval
// is longer than the query time window. This is an opportunistic, best-effort
// validation: if we cannot determine the Prometheus scrape interval for any
// reason, we do not return an error.
func (s *grpcServer) validateTimeWindow(ctx context.Context, window string) error {
config, err := s.prometheusAPI.Config(ctx)
if err != nil {
return nil
}

type PrometheusConfig struct {
Global map[string]string
}

var prom PrometheusConfig
err = yaml.Unmarshal([]byte(config.YAML), &prom)
if err != nil {
return nil
}

scrape_interval_str, found := prom.Global["scrape_interval"]
if !found {
return nil
}

scrape_interval, err := time.ParseDuration(scrape_interval_str)
if err != nil {
return nil
}

t, err := time.ParseDuration(window)
if err != nil {
return err
}

if t < scrape_interval {
return fmt.Errorf("Time window (%s) must be at least as long as the Prometheus scrape interval (%s)", window, scrape_interval)
Copy link
Member

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

Error strings should not be capitalized

Suggested change
return fmt.Errorf("Time window (%s) must be at least as long as the Prometheus scrape interval (%s)", window, scrape_interval)
return fmt.Errorf("time window (%s) must be at least as long as the Prometheus scrape interval (%s)", window, scrape_interval)

}
return nil
}
5 changes: 5 additions & 0 deletions viz/metrics-api/stat_summary.go
Expand Up @@ -102,6 +102,11 @@ func (s *grpcServer) StatSummary(ctx context.Context, req *pb.StatSummaryRequest
}
}

err := s.validateTimeWindow(ctx, req.TimeWindow)
if err != nil {
return statSummaryError(req, fmt.Sprintf("invalid time window: %s", err)), nil
}

statTables := make([]*pb.StatTable, 0)

var resourcesToQuery []string
Expand Down
5 changes: 5 additions & 0 deletions viz/metrics-api/top_routes.go
Expand Up @@ -63,6 +63,11 @@ func (s *grpcServer) TopRoutes(ctx context.Context, req *pb.TopRoutesRequest) (*
return topRoutesError(req, "Authority cannot be the target of a routes query; try using an authority in the --to flag instead"), nil
}

err = s.validateTimeWindow(ctx, req.TimeWindow)
if err != nil {
return topRoutesError(req, fmt.Sprintf("invalid time window: %s", err)), nil
}

// Non-authority resource
objects, err := s.k8sAPI.GetObjects(targetResource.Namespace, targetResource.Type, targetResource.Name, labelSelector)
if err != nil {
Expand Down