feat: PagerDuty Rate Limiting (#7218)

In my tests (toy account with 1000 incidents), this fixed the rate-limit errors that prevented cloudquery from getting data. Fetching '*' took 5m13s unfortunately.

I used the `RateLimitHttpClient` instead of just calling `limiter.Wait` before every call, since some calls are paginated by the SDK (i.e. the `for` loop is in the `pagerduty-go` sdk).

mention: #6981

plugins/source/pagerduty/client/client.go

@@ -32,12 +32,17 @@ func Configure(ctx context.Context, logger zerolog.Logger, spec specs.Source, _
 		return nil, fmt.Errorf("failed to unmarshal pagerduty spec: %w", err)
 	}
 
+	pagerdutySpec.setDefaults()
+
 	authToken, err := getAuthToken()
 	if err != nil {
 		return nil, err
 	}
 
 	pagerdutyClient := pagerduty.NewClient(authToken)
+	pagerdutyClient.HTTPClient = newRateLimitedHttpClient(
+		pagerdutyClient.HTTPClient,
+		*pagerdutySpec.MaxRequestsPerSecond)
 
 	cqClient := Client{
 		PagerdutyClient: pagerdutyClient,

plugins/source/pagerduty/client/rate_limited_http_client.go

@@ -0,0 +1,34 @@
+package client
+
+import (
+	"net/http"
+
+	"github.com/PagerDuty/go-pagerduty"
+	"golang.org/x/time/rate"
+)
+
+// Note that this is an implementatioin of 'pagerduty.HTTPClient', not of 'http.Client'.
+func newRateLimitedHttpClient(underlyingHttpClient pagerduty.HTTPClient, maxRequestsPerSecond int) *RateLimitedHttpClient {
+	return &RateLimitedHttpClient{
+		underlyingHttpClient: underlyingHttpClient,
+		limiter: rate.NewLimiter(
+			/*r=*/ rate.Limit(maxRequestsPerSecond),
+			/*b=*/ 1,
+		),
+	}
+}
+
+// Note that this is an implementatioin of 'pagerduty.HTTPClient', not of 'http.Client'.
+type RateLimitedHttpClient struct {
+	underlyingHttpClient pagerduty.HTTPClient
+
+	limiter *rate.Limiter
+}
+
+func (c *RateLimitedHttpClient) Do(req *http.Request) (*http.Response, error) {
+	if err := c.limiter.Wait(req.Context()); err != nil {
+		return nil, err
+	}
+
+	return c.underlyingHttpClient.Do(req)
+}

plugins/source/pagerduty/client/spec.go

@@ -4,4 +4,18 @@ type Spec struct {
 	// Used in API requests to filter only resources related to these team ids.
 	// Used in the tables: ["escalation_policies", "incidents", "maintenance_windows", "services", "users"]
 	TeamIds []string `yaml:"team_ids,omitempty" json:"team_ids"`
+
+	MaxRequestsPerSecond *int `yaml:"max_requests_per_second,omitempty" json:"max_requests_per_second,omitempty"`
+}
+
+func (spec *Spec) setDefaults() {
+	// Calculated as 66% of 900 requests per minute.
+	// https://developer.pagerduty.com/docs/ZG9jOjExMDI5NTUz-rate-limiting#what-are-our-limits
+	// The 900 requests per minute is for the entire oragnization, so we don't actually want to come too
+	// close to it.
+	var defaultRateLimitPerSecond = 10
+
+	if spec.MaxRequestsPerSecond == nil || *spec.MaxRequestsPerSecond == 0 {
+		spec.MaxRequestsPerSecond = &defaultRateLimitPerSecond
+	}
 }

plugins/source/pagerduty/go.mod

@@ -7,6 +7,7 @@ require (
 	github.com/cloudquery/plugin-sdk v1.29.0
 	github.com/rs/zerolog v1.28.0
 	golang.org/x/exp v0.0.0-20221230185412-738e83a70c30
+	golang.org/x/time v0.0.0-20220922220347-f3bd1da661af
 	gopkg.in/yaml.v3 v3.0.1
 )
 

plugins/source/pagerduty/go.sum

@@ -336,6 +336,8 @@ golang.org/x/text v0.5.0/go.mod h1:mrYo+phRRbMaCq/xk9113O4dZlRixOauAjOtrjsXDZ8=
 golang.org/x/time v0.0.0-20181108054448-85acf8d2951c/go.mod h1:tRJNPiyCQ0inRvYxbN9jk5I+vvW/OXSQhTDSoE431IQ=
 golang.org/x/time v0.0.0-20190308202827-9d24e82272b4/go.mod h1:tRJNPiyCQ0inRvYxbN9jk5I+vvW/OXSQhTDSoE431IQ=
 golang.org/x/time v0.0.0-20191024005414-555d28b269f0/go.mod h1:tRJNPiyCQ0inRvYxbN9jk5I+vvW/OXSQhTDSoE431IQ=
+golang.org/x/time v0.0.0-20220922220347-f3bd1da661af h1:Yx9k8YCG3dvF87UAn2tu2HQLf2dt/eR1bXxpLMWeH+Y=
+golang.org/x/time v0.0.0-20220922220347-f3bd1da661af/go.mod h1:tRJNPiyCQ0inRvYxbN9jk5I+vvW/OXSQhTDSoE431IQ=
 golang.org/x/tools v0.0.0-20180221164845-07fd8470d635/go.mod h1:n7NCudcB/nEzxVGmLbDWY5pfWTLqBcC2KZ6jyYvM4mQ=
 golang.org/x/tools v0.0.0-20180917221912-90fa682c2a6e/go.mod h1:n7NCudcB/nEzxVGmLbDWY5pfWTLqBcC2KZ6jyYvM4mQ=
 golang.org/x/tools v0.0.0-20190114222345-bf090417da8b/go.mod h1:n7NCudcB/nEzxVGmLbDWY5pfWTLqBcC2KZ6jyYvM4mQ=

website/pages/docs/plugins/sources/pagerduty/configuration.md

@@ -26,4 +26,10 @@ This is the (nested) spec used by the PagerDuty source plugin.
 
 - `team_ids` ([]string) (default: empty. will sync data from all available teams)
 
-  If specified, limits the sync to only resources related to the specified teams.
+  If specified, limits the sync to only resources related to the specified teams.
+
+- `max_requests_per_second` (int) (default: 10)
+  PagerDuty API is heavily rate-limited (900 requests/min = 15 requests/sec, across the entire organization). 
+  This option allows you to control the rate at which the plugin will make requests to the API. 
+  You can reduce this parameter in case you are still seeing rate limit errors (status code 429), or increase
+  it if your PagerDuty API quota is higher. See https://developer.pagerduty.com/docs/ZG9jOjExMDI5NTUz-rate-limiting#what-are-our-limits for more info.