From 0ba06cf37db53ddb1692e211d9733c44c1f45f3d Mon Sep 17 00:00:00 2001 From: Spencer Schrock Date: Tue, 7 May 2024 13:09:27 -0700 Subject: [PATCH] implement basic rate limiting for best practices worker. We are getting connection reset requests from bestpractices.dev and 429 errors from our GCS bucket for too many writes. The GCS limit (1000 QPS) is much higher, so just use the bestpractices.dev limit of 1 QPS. https://github.com/coreinfrastructure/best-practices-badge/blob/main/docs/api.md The construct was taken from https://go.dev/wiki/RateLimiting which "works well for rates up to tens of operations per second." Signed-off-by: Spencer Schrock --- cron/internal/cii/main.go | 5 +++++ 1 file changed, 5 insertions(+) diff --git a/cron/internal/cii/main.go b/cron/internal/cii/main.go index dc3739b7be9..f1048ceedbb 100644 --- a/cron/internal/cii/main.go +++ b/cron/internal/cii/main.go @@ -24,6 +24,7 @@ import ( "log" "net/http" "strings" + "time" "github.com/ossf/scorecard/v5/clients" "github.com/ossf/scorecard/v5/cron/config" @@ -95,9 +96,13 @@ func main() { panic(err) } + throttle := time.NewTicker(time.Second) // bestpractices.dev wants 1 QPS + defer throttle.Stop() + pageNum := 1 pageResp, err := getPage(ctx, pageNum) for err == nil && len(pageResp) > 0 { + <-throttle.C if err := writeToCIIDataBucket(ctx, pageResp, ciiDataBucket); err != nil { panic(err) }