Merge pull request #309 from grosser/zdrve/rate-limit

Log / respect rate limits

lib/kennel/api.rb

@@ -5,6 +5,8 @@ module Kennel
   class Api
     CACHE_FILE = ENV.fetch("KENNEL_API_CACHE_FILE", "tmp/cache/details")
 
+    RateLimitParams = Data.define(:limit, :period, :remaining, :reset, :name)
+
     def self.tag(api_resource, reply)
       klass = Models::Record.api_resource_map[api_resource]
       return reply unless klass # do not blow up on unknown models
@@ -122,6 +124,23 @@ def request(method, path, body: nil, params: {}, ignore_404: false)
             end
           end
 
+          rate_limit = RateLimitParams.new(
+            limit: response.headers["x-ratelimit-limit"],
+            period: response.headers["x-ratelimit-period"],
+            remaining: response.headers["x-ratelimit-remaining"],
+            reset: response.headers["x-ratelimit-reset"],
+            name: response.headers["x-ratelimit-name"]
+          )
+
+          if response.status == 429
+            message = "Datadog rate limit #{rate_limit.name.inspect} hit"
+            message += " (#{rate_limit.limit} requests per #{rate_limit.period} seconds)"
+            message += "; sleeping #{rate_limit.reset} seconds before trying again"
+            Kennel.err.puts message
+            sleep rate_limit.reset.to_f
+            redo
+          end
+
           break if i == tries - 1 || method != :get || response.status < 500
           Kennel.err.puts "Retrying on server error #{response.status} for #{path}"
         end

test/kennel/api_test.rb

@@ -255,6 +255,28 @@ def tracking(id)
     end
   end
 
+  describe "rate limiting" do
+    capture_all
+
+    it "retries on a rate-limited response" do
+      request = stub_datadog_request(:get, "monitor/1234").to_return(
+        [
+          { status: 429, headers: {
+            "X-RateLimit-Name": "too many secrets",
+            "X-RateLimit-Limit": "1000",
+            "X-RateLimit-Period": "60",
+            "X-RateLimit-Remaining": "-5",
+            "X-RateLimit-Reset": "1.1"
+          } },
+          { status: 200, body: { foo: "bar" }.to_json }
+        ]
+      )
+      api.show("monitor", 1234).must_equal(foo: "bar", klass: Kennel::Models::Monitor, tracking_id: nil)
+      assert_requested request, times: 2
+      stderr.string.must_equal "Datadog rate limit \"too many secrets\" hit (1000 requests per 60 seconds); sleeping 1.1 seconds before trying again\n"
+    end
+  end
+
   describe "retries" do
     capture_all