[YUNIKORN-333] rate limit predicate events (#240)

Reduce the number of events published to the K8s event system when a
predicate check fails.
The rate is now limited to 1 event per second.

Fixes: #240

go.mod

@@ -30,6 +30,7 @@ require (
 	github.com/onsi/ginkgo v1.11.0
 	github.com/onsi/gomega v1.7.0
 	go.uber.org/zap v1.13.0
+	golang.org/x/time v0.0.0-20190308202827-9d24e82272b4
 	gopkg.in/yaml.v2 v2.2.8
 	gotest.tools v2.2.0+incompatible
 	k8s.io/api v0.16.13

pkg/plugin/predicates/predictor.go

@@ -22,8 +22,10 @@ import (
 	"fmt"
 	"strings"
 	"sync"
+	"time"
 
 	"go.uber.org/zap"
+	"golang.org/x/time/rate"
 	v1 "k8s.io/api/core/v1"
 	"k8s.io/apimachinery/pkg/util/sets"
 	"k8s.io/kubernetes/pkg/scheduler/algorithm/predicates"
@@ -71,6 +73,9 @@ var reservationPredicates = []string{
 	predicates.MatchNodeSelectorPred,      // node selector check
 }
 
+var limit = rate.Every(time.Second)
+var limiter = rate.NewLimiter(limit, 1)
+
 type Predictor struct {
 	fitPredicateMap              map[string]factory.FitPredicateFactory
 	fitPredicateFunctions        map[string]predicates.FitPredicate
@@ -278,8 +283,10 @@ func (p *Predictor) predicatesAllocate(pod *v1.Pod, meta predicates.PredicateMet
 			}
 
 			if !fit {
-				events.GetRecorder().Eventf(pod, v1.EventTypeWarning,
-					"FailedScheduling", "%v", reasons)
+				if limiter.Allow() {
+					events.GetRecorder().Eventf(pod, v1.EventTypeWarning,
+						"FailedScheduling", "%v", reasons)
+				}
 				return fmt.Errorf("predicate %s cannot be satisfied, reason: %v", predicateKey, reasons)
 			}
 		}