Skip to content
Permalink
Browse files

Log every request for the user list and implement a delay

Implement the worst-case rate limiting for users.list
  • Loading branch information
emedvedev committed Nov 21, 2019
1 parent 6419579 commit e5e51df0990d4af0d1114ee9a6e9168971902c87
Showing with 70 additions and 38 deletions.
  1. +1 −0 bin/slackin
  2. +3 −0 lib/index.js
  3. +62 −35 lib/slack.js
  4. +1 −0 readme.md
  5. +3 −3 test/index.js
@@ -100,6 +100,7 @@ flags.recaptcha = {
};

// Advanced parameters (env-only)
flags.pageDelay = process.env.SLACKIN_PAGE_DELAY;
flags.proxy = Boolean(process.env.SLACKIN_PROXY);
flags.redirectFQDN = process.env.SLACKIN_HTTPS_REDIRECT;
flags.letsencrypt = process.env.SLACKIN_LETSENCRYPT;
@@ -38,6 +38,7 @@ function slackin({
emails,
coc,
proxy,
pageDelay = 0,
redirectFQDN,
letsencrypt,
silent,
@@ -140,6 +141,8 @@ function slackin({
token,
interval,
org,
pageDelay,
fetchChannels: Boolean(channels),
logger: slackLog,
});
slack.setMaxListeners(Infinity);
@@ -1,14 +1,20 @@
const { EventEmitter } = require('events');
const request = require('superagent');

function sleep(ms) {
return new Promise((resolve) => setTimeout(resolve, ms));
}

class SlackData extends EventEmitter {
constructor({
token, interval, logger, org: host,
token, interval, logger, pageDelay, fetchChannels, org: host,
}) {
super();
this.host = host;
this.token = token;
this.interval = interval;
this.pageDelay = pageDelay;
this.fetchChannels = fetchChannels;
this.ready = false;
this.org = {};
this.users = {};
@@ -17,22 +23,10 @@ class SlackData extends EventEmitter {
this.bindLogs(logger);
}
this.init();
this.fetch();
this.fetchUserCount();
}

init() {
request
.get(`https://${this.host}.slack.com/api/channels.list`)
.query({ token: this.token })
.end((err, res) => {
if (err) {
throw err;
}
(res.body.channels || []).forEach((channel) => {
this.channelsByName[channel.name] = channel;
});
});

async init() {
request
.get(`https://${this.host}.slack.com/api/team.info`)
.query({ token: this.token })
@@ -47,29 +41,59 @@ class SlackData extends EventEmitter {
this.org.logo = team.icon.image_132;
}
});

if (this.fetchChannels) {
let cursor = '';
do {
let response = null;
response = await request // eslint-disable-line no-await-in-loop
.get(`https://${this.host}.slack.com/api/conversations.list`)
.query({
token: this.token, limit: 800, cursor,
});
if (response.ok === false && !response.body.channels) {
throw new Error(`Error: ${response.error} (status ${response.status})`);
}
(response.body.channels || []).forEach((channel) => {
this.channelsByName[channel.name] = channel;
});
cursor = response.body.response_metadata.next_cursor;
if (cursor && this.pageDelay) {
await sleep(this.pageDelay); // eslint-disable-line no-await-in-loop
}
}
while (cursor);
}
}

async fetch() {
async fetchUserCount() {
let users = [];
let cursor = '';

this.emit('fetch');

do {
let response = null;
try {
response = await request // eslint-disable-line no-await-in-loop
.get(`https://${this.host}.slack.com/api/users.list`)
.query({
token: this.token, limit: 200, cursor, presence: 1,
});
} catch (err) {
this.emit('error', err);
return this.retry();
}
if (response.status === 429) {
return this.retry(response.headers['retry-after'] * 1000);
let retryCurrentRequest = false;

do {
try {
this.emit('fetch');
response = await request // eslint-disable-line no-await-in-loop
.get(`https://${this.host}.slack.com/api/users.list`)
.query({
token: this.token, limit: 800, cursor, presence: 1,
});
} catch (err) {
this.emit('error', err);
if (err.response.status === 429) {
this.emit('error', `Rate limiting, retrying after ${err.response.headers['retry-after']}`);
await this.sleep(err.response.headers['retry-after'] * 1000); // eslint-disable-line no-await-in-loop
retryCurrentRequest = true;
} else {
return this.retry();
}
}
}
while (retryCurrentRequest);

if (response.ok === false) {
this.emit('error', new Error(`Slack API error: ${response.error}`));
return this.retry();
@@ -80,8 +104,11 @@ class SlackData extends EventEmitter {
}
users = users.concat(response.body.members);
cursor = response.body.response_metadata.next_cursor;
if (cursor && this.pageDelay) {
await sleep(this.pageDelay); // eslint-disable-line no-await-in-loop
}
}
while (cursor !== '');
while (cursor);

// remove slackbot and bots from users
// slackbot is not a bot, go figure!
@@ -107,7 +134,7 @@ class SlackData extends EventEmitter {
this.emit('ready');
}

setTimeout(this.fetch.bind(this), this.interval);
setTimeout(this.fetchUserCount.bind(this), this.interval);
return this.emit('data');
}

@@ -117,8 +144,8 @@ class SlackData extends EventEmitter {
}

retry(delay = this.interval * 2) {
setTimeout(this.fetch.bind(this), delay);
this.emit('retry');
setTimeout(this.fetchUserCount.bind(this), delay);
return this.emit('retry');
}

bindLogs(logger) {
@@ -88,6 +88,7 @@ Every CLI parameter, including mandatory arguments (workspace ID and token), can
| --accent | -A | `SLACKIN_ACCENT` | `#e01563` for `light`, `#9a0e44` for `dark` | Accent color to use instead of a theme default |
| --coc | -C | `SLACKIN_COC` | `''` | Full URL to a CoC that needs to be agreed to |
| --css | -S | `SLACKIN_CSS` | `''` | Full URL to a custom CSS file to use on the main page |
| | | `SLACKIN_PAGE_DELAY` | `0` | Delay (ms) between Slack API requests (may be required for large workspaces that hit the rate limit) |
| | | `SLACKIN_PROXY` | `false` | Trust proxy headers (only use if Slackin is served behind a reverse proxy) |
| | | `SLACKIN_HTTPS_REDIRECT` | `''` | If a domain name is specified in this parameter and `SLACKIN_PROXY` is set to `true`, Slackin will redirect requests with `x-forwarded-proto === 'http'` to `https://<SLACKIN_HTTPS_REDIRECT>/<original URL>` |
| | | `SLACKIN_LETSENCRYPT` | `''` | [Let's Encrypt](https://letsencrypt.org/) challenge response |
@@ -8,7 +8,7 @@ describe('slackin', () => {
nock('https://myorg.slack.com')
.get('/api/users.list')
.query({
token: 'mytoken', presence: '1', limit: 200, cursor: '',
token: 'mytoken', presence: '1', limit: 800, cursor: '',
})
.reply(200, {
ok: true,
@@ -18,15 +18,15 @@ describe('slackin', () => {

nock('https://myorg.slack.com')
.get('/api/users.list')
.query({ token: 'mytoken', limit: 200, cursor: '' })
.query({ token: 'mytoken', limit: 800, cursor: '' })
.reply(200, {
ok: true,
members: [{}],
response_metadata: { next_cursor: '' },
});

nock('https://myorg.slack.com')
.get('/api/channels.list?token=mytoken')
.get('/api/conversations.list?token=mytoken')
.reply(200, {
ok: true,
channels: [{}],

0 comments on commit e5e51df

Please sign in to comment.
You can’t perform that action at this time.