fix(Network): Throttle network calls

README.md

@@ -16,9 +16,10 @@ Supports IM/DM, channels and private groups now, support for multiparty direct m
 
 ## Options
   ```
-  Usage: slack-history-export [options]
-  Options:
+  Usage: slack-history-export.js [options]
+  Download message history from slack
 
+  Options:
     --help, -h      Show help text.                                      [boolean]
     --version, -v   Show version number                                  [boolean]
     --token, -t     Slack Token. You can generate it
@@ -29,6 +30,7 @@ Supports IM/DM, channels and private groups now, support for multiparty direct m
     --group, -g     Name of the group to download history
     --logLevel, -l  Enable and set log level
           [choices: "info", "silly", "verbose", "warn", "error"] [default: "info"]
+    --delay, -d     Milliseconds to delay between network calls    [default: 1000]
     --channel, -c   Name of the channel to download history
     --filepath, -f  Path to the json file to save the history  [default: "stdout"]
   ```

src/cli.js

@@ -57,6 +57,11 @@ Download message history from slack`,
       return level
     },
   })
+  .option('delay', {
+    alias: 'd',
+    default: 1000,
+    describe: 'Milliseconds to delay between network calls',
+  })
   .option('channel', {
     alias: 'c',
     describe: 'Name of the channel to download history',

src/main.js

@@ -7,11 +7,16 @@ import {
   getChannelInfo,
 } from './utils'
 
+const makeDelay = millis => data =>
+  /* eslint-disable no-unused-vars */
+  new Promise((resolve, _reject) => setTimeout(() => resolve(data), millis))
+
 export default class SlackHistoryExport {
   constructor (args, logger) {
     this.args = args
     this.logger = logger
     this.slack = new SlackApi(this.args.token)
+    this.delay = makeDelay(args.delay)
   }
   processIMs (outputStream) {
     return this.fetchUserDetail(this.args.username).then(
@@ -21,110 +26,137 @@ export default class SlackHistoryExport {
     )
   }
   processGroups (outputStream) {
-    return this.fetchGroupDetails(this.args.group).then(
+    return this.fetchGroupDetails(this.args.group).then(this.delay).then(
       groupObj => this.fetchGroupHistory(outputStream, groupObj.id),
     )
   }
   processChannels (outputStream) {
-    return this.fetchChannelDetails(this.args.channel).then(
+    return this.fetchChannelDetails(this.args.channel).then(this.delay).then(
       channelObj => this.fetchChannelHistory(outputStream, channelObj.id),
     )
   }
   fetchGroupDetails (groupName) {
-    return this.slack.groups().then(groups => getGroupInfo(groups, groupName))
+    return Promise.resolve()
+                  .then(this.delay)
+                  .then(() => this.slack.groups())
+                  .then(groups => getGroupInfo(groups, groupName))
   }
   fetchChannelDetails (chanName) {
-    return this.slack.channels().then(chans => getChannelInfo(chans, chanName))
+    return Promise.resolve()
+                  .then(this.delay)
+                  .then(() => this.slack.channels())
+                  .then(chans => getChannelInfo(chans, chanName))
   }
   fetchIMInfo (userObj) {
-    return this.slack.im().then(ims => getUserIMInfo(ims, userObj))
+    return Promise.resolve()
+                  .then(this.delay)
+                  .then(() => this.slack.im())
+                  .then(ims => getUserIMInfo(ims, userObj))
   }
   fetchUserDetail (username) {
-    return this.slack.users().then(users => getUserInfo(users, username))
+    return Promise.resolve()
+                  .then(this.delay)
+                  .then(() => this.slack.users())
+                  .then(users => getUserInfo(users, username))
   }
   fetchGroupHistory (outputStream, channel, latest, fnCalled) {
     if (!fnCalled)
       outputStream.write('[\n')
-    return this.slack.groupHistory(channel, latest).then((groupHistory) => {
-      _.each(groupHistory.messages, (message, index) => {
-        /* eslint-disable no-param-reassign */
-        message.timestamp = +(message.ts * 1e3).toString().split('.')[0]
-        message.isoDate = new Date(message.timestamp)
-        /* eslint-enable no-param-reassign */
-        outputStream.write(JSON.stringify(message, null, 2))
+    return Promise
+      .resolve()
+      .then(this.delay)
+      .then(() => this.slack.groupHistory(channel, latest))
+      .then((groupHistory) => {
+        _.each(groupHistory.messages, (message, index) => {
+          /* eslint-disable no-param-reassign */
+          message.timestamp = +(message.ts * 1e3).toString().split('.')[0]
+          message.isoDate = new Date(message.timestamp)
+          /* eslint-enable no-param-reassign */
+          outputStream.write(JSON.stringify(message, null, 2))
 
-        if (groupHistory.has_more || index !== groupHistory.messages.length - 1)
-          outputStream.write(',')
+          /* eslint-disable max-len */
+          if (groupHistory.has_more || index !== groupHistory.messages.length - 1)
+            outputStream.write(',')
+        })
+        if (groupHistory.has_more)
+          return this.fetchGroupHistory(
+            outputStream,
+            channel,
+            groupHistory.messages[groupHistory.messages.length - 1].ts,
+            true,
+          )
+        if (outputStream !== process.stdout)
+          outputStream.end(']\n')
+        else
+          outputStream.write(']\n')
+        return Promise.resolve()
       })
-      if (groupHistory.has_more)
-        return this.fetchGroupHistory(
-          outputStream,
-          channel,
-          groupHistory.messages[groupHistory.messages.length - 1].ts,
-          true,
-        )
-      if (outputStream !== process.stdout)
-        outputStream.end(']\n')
-      else
-        outputStream.write(']\n')
-      return Promise.resolve()
-    })
   }
   fetchChannelHistory (outputStream, channel, latest, fnCalled) {
     if (!fnCalled)
       outputStream.write('[\n')
-    return this.slack.channelHistory(channel, latest).then((chanHistory) => {
-      _.each(chanHistory.messages, (message, index) => {
-        /* eslint-disable no-param-reassign */
-        message.timestamp = +(message.ts * 1e3).toString().split('.')[0]
-        message.isoDate = new Date(message.timestamp)
-        /* eslint-enable no-param-reassign */
-        outputStream.write(JSON.stringify(message, null, 2))
+    return Promise
+      .resolve()
+      .then(this.delay)
+      .then(() => this.slack.channelHistory(channel, latest))
+      .then((chanHistory) => {
+        _.each(chanHistory.messages, (message, index) => {
+          /* eslint-disable no-param-reassign */
+          message.timestamp = +(message.ts * 1e3).toString().split('.')[0]
+          message.isoDate = new Date(message.timestamp)
+          /* eslint-enable no-param-reassign */
+          outputStream.write(JSON.stringify(message, null, 2))
 
-        if (chanHistory.has_more || index !== chanHistory.messages.length - 1)
-          outputStream.write(',')
+          if (chanHistory.has_more || index !== chanHistory.messages.length - 1)
+            outputStream.write(',')
+        })
+        if (chanHistory.has_more)
+          return Promise.resolve().then(this.delay).then(() =>
+            this.fetchChannelHistory(
+              outputStream,
+              channel,
+              chanHistory.messages[chanHistory.messages.length - 1].ts,
+              true,
+            ),
+          )
+        if (outputStream !== process.stdout)
+          outputStream.end(']\n')
+        else
+          outputStream.write(']\n')
+        return Promise.resolve()
       })
-      if (chanHistory.has_more)
-        return this.fetchChannelHistory(
-          outputStream,
-          channel,
-          chanHistory.messages[chanHistory.messages.length - 1].ts,
-          true,
-        )
-      if (outputStream !== process.stdout)
-        outputStream.end(']\n')
-      else
-        outputStream.write(']\n')
-      return Promise.resolve()
-    })
   }
   fetchIMHistory (outputStream, channel, latest, fnCalled) {
     if (!fnCalled)
       outputStream.write('[\n') // Use to detect the first call of the method
-    return this.slack.imHistory(channel, latest).then((imHistory) => {
-      _.each(imHistory.messages, (message, index) => {
-        /* eslint-disable no-param-reassign */
-        message.timestamp = +(message.ts * 1e3).toString().split('.')[0]
-        message.isoDate = new Date(message.timestamp)
-        /* eslint-enable no-param-reassign */
-        outputStream.write(JSON.stringify(message, null, 2))
+    return Promise
+      .resolve()
+      .then(this.delay)
+      .then(() => this.slack.imHistory(channel, latest))
+      .then((imHistory) => {
+        _.each(imHistory.messages, (message, index) => {
+          /* eslint-disable no-param-reassign */
+          message.timestamp = +(message.ts * 1e3).toString().split('.')[0]
+          message.isoDate = new Date(message.timestamp)
+          /* eslint-enable no-param-reassign */
+          outputStream.write(JSON.stringify(message, null, 2))
 
-        if (imHistory.has_more || index !== imHistory.messages.length - 1)
-          outputStream.write(',')
-      })
-      if (imHistory.has_more)
-        return this.fetchIMHistory(
-          outputStream,
-          channel,
-          imHistory.messages[imHistory.messages.length - 1].ts,
-          true,
-        )
-      if (outputStream !== process.stdout) // Process.stdout cannot be closed
-        outputStream.end(']\n')
-      else
-        outputStream.write(']\n')
+          if (imHistory.has_more || index !== imHistory.messages.length - 1)
+            outputStream.write(',')
+        })
+        if (imHistory.has_more)
+          return this.fetchIMHistory(
+            outputStream,
+            channel,
+            imHistory.messages[imHistory.messages.length - 1].ts,
+            true,
+          )
+        if (outputStream !== process.stdout) // Process.stdout cannot be closed
+          outputStream.end(']\n')
+        else
+          outputStream.write(']\n')
 
-      return Promise.resolve()
-    })
+        return Promise.resolve()
+      })
   }
 }