respect rate limits during pagination by sleeping

CHANGELOG.md

@@ -1,5 +1,6 @@
 ### 0.9.2 (Next)
 
+* [#167](https://github.com/slack-ruby/slack-ruby-client/pull/167): Respect rate limits during pagination by sleeping. Also add optional `pause` parameter in order to proactively sleep between each paginated request - [@jmanian](https://github.com/jmanian).
 * [#163](https://github.com/slack-ruby/slack-ruby-client/pull/164): Use `OpenSSL::X509::DEFAULT_CERT_DIR` and `OpenSSL::X509::DEFAULT_CERT_FILE` for default ca_cert and ca_file - [@leifcr](https://github.com/leifcr).
 * [#161](https://github.com/slack-ruby/slack-ruby-client/pull/161): Added support for cursor pagination - [@dblock](https://github.com/dblock).
 * Your contribution here.

lib/slack/web/pagination/cursor.rb

@@ -7,23 +7,31 @@ class Cursor
 
           attr_reader :client
           attr_reader :verb
+          attr_reader :pause
           attr_reader :params
 
           def initialize(client, verb, params = {})
             @client = client
             @verb = verb
+            @pause = params.delete(:pause)
             @params = params
           end
 
           def each
             next_cursor = nil
             loop do
               query = { limit: client.default_page_size }.merge(params).merge(cursor: next_cursor)
-              response = client.send(verb, query)
+              begin
+                response = client.send(verb, query)
+              rescue Slack::Web::Api::Errors::TooManyRequestsError => e
+                sleep(e.retry_after.seconds)
+                next
+              end
               yield response
               break unless response.response_metadata
               next_cursor = response.response_metadata.next_cursor
               break if next_cursor.blank?
+              sleep(pause) if pause
             end
           end
         end

spec/slack/web/api/pagination/cursor_spec.rb

@@ -21,8 +21,20 @@
         Slack::Messages::Message.new(response_metadata: { next_cursor: 'next' }),
         Slack::Messages::Message.new
       )
+      expect(cursor).not_to receive(:sleep)
       cursor.to_a
     end
+    context 'with rate limiting' do
+      let(:error) { Slack::Web::Api::Errors::TooManyRequestsError.new(nil) }
+      it 'sleeps after a TooManyRequestsError' do
+        expect(client).to receive(:users_list).with(limit: 100, cursor: nil).ordered.and_return(Slack::Messages::Message.new(response_metadata: { next_cursor: 'next' }))
+        expect(client).to receive(:users_list).with(limit: 100, cursor: 'next').ordered.and_raise(error)
+        expect(error).to receive(:retry_after).once.ordered.and_return(9)
+        expect(cursor).to receive(:sleep).once.ordered.with(9)
+        expect(client).to receive(:users_list).with(limit: 100, cursor: 'next').ordered.and_return(Slack::Messages::Message.new)
+        cursor.to_a
+      end
+    end
   end
   context 'with a custom limit' do
     let(:cursor) { Slack::Web::Api::Pagination::Cursor.new(client, 'users_list', limit: 42) }
@@ -31,4 +43,16 @@
       cursor.first
     end
   end
+  context 'with a custom pause' do
+    let(:cursor) { Slack::Web::Api::Pagination::Cursor.new(client, 'users_list', pause: 3) }
+    it 'sleeps between requests' do
+      expect(client).to receive(:users_list).exactly(3).times.and_return(
+        Slack::Messages::Message.new(response_metadata: { next_cursor: 'next_a' }),
+        Slack::Messages::Message.new(response_metadata: { next_cursor: 'next_b' }),
+        Slack::Messages::Message.new
+      )
+      expect(cursor).to receive(:sleep).with(3).twice
+      cursor.to_a
+    end
+  end
 end