Twitter rate limit handling

.github/PULL_REQUEST_TEMPLATE.md

@@ -1,11 +1,10 @@
 # Changes proposed ✍️
-
 - ...
+
 - ### Screenshots (front-end changes only)
       - ...
 
 ## Checklist ✅
-
 - [ ] Label appropriately with `Feature`, `Enhancement`, or `Bug`.
 - [ ] Tests are passing.
 - [ ] New backend functionality has been unit-tested.
@@ -17,4 +16,4 @@
 - [ ] API documentation has been updated (if necessary) (see [docs on API documentation](https://docs.crowd.dev/docs/updating-api-documentation)).
 - [ ] [Quality standards](https://github.com/CrowdDotDev/crowd-github-test-public/blob/main/CONTRIBUTING.md#quality-standards) are met.
 - [ ] All changes have been tested in a staging site.
-- [ ] All changes are working locally running crowd.dev's Docker local environment.
+- [ ] All changes are working locally running crowd.dev's Docker local environment.

backend/src/serverless/integrations/services/integrations/hackerNewsIntegrationService.ts

@@ -174,7 +174,7 @@ export class HackerNewsIntegrationService extends IntegrationServiceBase {
     }
     return {
       ...activity,
-      member: member,
+      member,
     }
   }
 }

backend/src/serverless/integrations/usecases/hackerNews/getPostsByKeywords.ts

@@ -1,9 +1,7 @@
-import axios from 'axios'
 import { IServiceOptions } from '../../../../services/IServiceOptions'
 import { EagleEyeResponses, EagleEyeInput } from '../../types/hackerNewsTypes'
 import { Logger } from '../../../../utils/logging'
 import { timeout } from '../../../../utils/timing'
-import { IS_DEV_ENV } from '../../../../config'
 import EagleEyeContentService from '../../../../services/eagleEyeContentService'
 
 async function getPostsByKeyword(

backend/src/serverless/integrations/usecases/slack/errorHandler.ts

@@ -0,0 +1,35 @@
+import { AxiosError } from 'axios'
+import { RateLimitError } from '../../../../types/integration/rateLimitError'
+import { Logger } from '../../../../utils/logging'
+
+export const handleSlackError = (err: AxiosError, input: any, logger: Logger): any => {
+  let url = ''
+  if (err.config) {
+    const queryParams: string[] = []
+    if (err.config.params) {
+      for (const [key, value] of Object.entries(err.config.params)) {
+        queryParams.push(`${key}=${encodeURIComponent(value as any)}`)
+      }
+    }
+
+    url = `${err.config.url}?${queryParams.join('&')}`
+  } else if (err.request) {
+    url = `${err.request.host}${err.request.path}`
+  } else {
+    url = 'unknown-url'
+  }
+
+  // https://api.slack.com/docs/rate-limits
+  if (err && err.response && err.response.status === 429) {
+    logger.warn('Slack API rate limit exceeded')
+    let rateLimitResetSeconds = 60
+
+    if (err.response.headers['Retry-After']) {
+      rateLimitResetSeconds = parseInt(err.response.headers['Retry-After'], 10)
+    }
+
+    return new RateLimitError(rateLimitResetSeconds, url, err)
+  }
+  logger.error({ err, input }, `Error while calling Slack API URL: ${url}`)
+  return err
+}

backend/src/serverless/integrations/usecases/slack/getChannels.ts

@@ -1,16 +1,19 @@
-import axios from 'axios'
+import axios, { AxiosRequestConfig } from 'axios'
+import { handleSlackError } from './errorHandler'
 import { SlackChannels, SlackGetChannelsInput } from '../../types/slackTypes'
 import { Logger } from '../../../../utils/logging'
 import { timeout } from '../../../../utils/timing'
-import { RateLimitError } from '../../../../types/integration/rateLimitError'
 
 async function getChannels(input: SlackGetChannelsInput, logger: Logger): Promise<SlackChannels> {
   await timeout(2000)
 
   try {
-    const config = {
+    const config: AxiosRequestConfig<any> = {
       method: 'get',
-      url: 'https://slack.com/api/conversations.list?limit=100',
+      url: 'https://slack.com/api/conversations.list',
+      params: {
+        limit: 100,
+      },
       headers: {
         Authorization: `Bearer ${input.token}`,
       },
@@ -26,14 +29,8 @@ async function getChannels(input: SlackGetChannelsInput, logger: Logger): Promis
         id: c.id,
       }))
   } catch (err) {
-    if (err && err.response && err.response.status === 429 && err.response.headers['Retry-After']) {
-      logger.warn('Slack API rate limit exceeded')
-      const rateLimitResetSeconds = parseInt(err.response.headers['Retry-After'], 10)
-      throw new RateLimitError(rateLimitResetSeconds, '/conversations.list')
-    } else {
-      logger.error({ err, input }, 'Error while getting channels from Slack')
-      throw err
-    }
+    const newErr = handleSlackError(err, input, logger)
+    throw newErr
   }
 }
 

backend/src/serverless/integrations/usecases/slack/getMember.ts

@@ -1,8 +1,8 @@
-import axios from 'axios'
+import axios, { AxiosRequestConfig } from 'axios'
 import { SlackGetMemberInput, SlackGetMemberOutput } from '../../types/slackTypes'
 import { Logger } from '../../../../utils/logging'
 import { timeout } from '../../../../utils/timing'
-import { RateLimitError } from '../../../../types/integration/rateLimitError'
+import { handleSlackError } from './errorHandler'
 
 async function getMembers(
   input: SlackGetMemberInput,
@@ -11,9 +11,12 @@ async function getMembers(
   await timeout(2000)
 
   try {
-    const config = {
+    const config: AxiosRequestConfig<any> = {
       method: 'get',
-      url: `https://slack.com/api/users.info?user=${input.userId}`,
+      url: `https://slack.com/api/users.info`,
+      params: {
+        user: input.userId,
+      },
       headers: {
         Authorization: `Bearer ${input.token}`,
       },
@@ -27,14 +30,8 @@ async function getMembers(
       nextPage,
     }
   } catch (err) {
-    if (err && err.response && err.response.status === 429 && err.response.headers['Retry-After']) {
-      logger.warn('Slack API rate limit exceeded')
-      const rateLimitResetSeconds = parseInt(err.response.headers['Retry-After'], 10)
-      throw new RateLimitError(rateLimitResetSeconds, '/users.info')
-    } else {
-      logger.error({ err, input }, 'Error while getting members from Slack')
-      throw err
-    }
+    const newErr = handleSlackError(err, input, logger)
+    throw newErr
   }
 }
 

backend/src/serverless/integrations/usecases/slack/getMembers.ts

@@ -1,8 +1,8 @@
-import axios from 'axios'
-import { SlackGetMembersInput, SlackGetMembersOutput, SlackMembers } from '../../types/slackTypes'
+import axios, { AxiosRequestConfig } from 'axios'
 import { Logger } from '../../../../utils/logging'
 import { timeout } from '../../../../utils/timing'
-import { RateLimitError } from '../../../../types/integration/rateLimitError'
+import { SlackGetMembersInput, SlackGetMembersOutput, SlackMembers } from '../../types/slackTypes'
+import { handleSlackError } from './errorHandler'
 
 async function getMembers(
   input: SlackGetMembersInput,
@@ -11,14 +11,23 @@ async function getMembers(
   await timeout(2000)
 
   try {
-    const config = {
+    const config: AxiosRequestConfig<any> = {
       method: 'get',
-      url: `https://slack.com/api/users.list?cursor=${input.page}&limit=${input.perPage}`,
+      url: 'https://slack.com/api/users.list',
+      params: {},
       headers: {
         Authorization: `Bearer ${input.token}`,
       },
     }
 
+    if (input.page !== undefined && input.page !== '') {
+      config.params.cursor = input.page
+    }
+
+    if (input.perPage !== undefined && input.perPage > 0) {
+      config.params.limit = input.perPage
+    }
+
     const response = await axios(config)
     const records: SlackMembers = response.data.members
     const nextPage = response.data.response_metadata?.next_cursor || ''
@@ -27,14 +36,8 @@ async function getMembers(
       nextPage,
     }
   } catch (err) {
-    if (err && err.response && err.response.status === 429 && err.response.headers['Retry-After']) {
-      logger.warn('Slack API rate limit exceeded')
-      const rateLimitResetSeconds = parseInt(err.response.headers['Retry-After'], 10)
-      throw new RateLimitError(rateLimitResetSeconds, '/users.list')
-    } else {
-      logger.error({ err, input }, 'Error while getting members from Slack')
-      throw err
-    }
+    const newErr = handleSlackError(err, input, logger)
+    throw newErr
   }
 }
 

backend/src/serverless/integrations/usecases/slack/getMessages.ts

@@ -1,8 +1,8 @@
-import axios from 'axios'
-import { SlackMessages, SlackParsedResponse, SlackGetMessagesInput } from '../../types/slackTypes'
+import axios, { AxiosRequestConfig } from 'axios'
 import { Logger } from '../../../../utils/logging'
 import { timeout } from '../../../../utils/timing'
-import { RateLimitError } from '../../../../types/integration/rateLimitError'
+import { SlackGetMessagesInput, SlackMessages, SlackParsedResponse } from '../../types/slackTypes'
+import { handleSlackError } from './errorHandler'
 
 async function getMessages(
   input: SlackGetMessagesInput,
@@ -11,14 +11,25 @@ async function getMessages(
   await timeout(2000)
 
   try {
-    const config = {
+    const config: AxiosRequestConfig<any> = {
       method: 'get',
-      url: `https://slack.com/api/conversations.history?channel=${input.channelId}&cursor=${input.page}&limit=${input.perPage}`,
+      url: `https://slack.com/api/conversations.history`,
+      params: {
+        channel: input.channelId,
+      },
       headers: {
         Authorization: `Bearer ${input.token}`,
       },
     }
 
+    if (input.page !== undefined && input.page !== '') {
+      config.params.cursor = input.page
+    }
+
+    if (input.perPage !== undefined && input.perPage > 0) {
+      config.params.limit = input.perPage
+    }
+
     const response = await axios(config)
     const records: SlackMessages = response.data.messages
 
@@ -28,14 +39,8 @@ async function getMessages(
       nextPage,
     }
   } catch (err) {
-    if (err && err.response && err.response.status === 429 && err.response.headers['Retry-After']) {
-      logger.warn('Slack API rate limit exceeded')
-      const rateLimitResetSeconds = parseInt(err.response.headers['Retry-After'], 10)
-      throw new RateLimitError(rateLimitResetSeconds, '/conversation.history')
-    } else {
-      logger.error({ err, input }, 'Error while getting messages from Slack')
-      throw err
-    }
+    const newErr = handleSlackError(err, input, logger)
+    throw newErr
   }
 }
 

backend/src/serverless/integrations/usecases/slack/getMessagesInThreads.ts

@@ -1,12 +1,12 @@
-import axios from 'axios'
+import axios, { AxiosRequestConfig } from 'axios'
+import { Logger } from '../../../../utils/logging'
+import { timeout } from '../../../../utils/timing'
 import {
+  SlackGetMessagesInThreadsInput,
   SlackMessages,
   SlackParsedResponse,
-  SlackGetMessagesInThreadsInput,
 } from '../../types/slackTypes'
-import { Logger } from '../../../../utils/logging'
-import { timeout } from '../../../../utils/timing'
-import { RateLimitError } from '../../../../types/integration/rateLimitError'
+import { handleSlackError } from './errorHandler'
 
 async function getMessagesInThreads(
   input: SlackGetMessagesInThreadsInput,
@@ -15,14 +15,26 @@ async function getMessagesInThreads(
   await timeout(2000)
 
   try {
-    const config = {
+    const config: AxiosRequestConfig<any> = {
       method: 'get',
-      url: `https://slack.com/api/conversations.replies?channel=${input.channelId}&cursor=${input.page}&limit=${input.perPage}&ts=${input.threadId}`,
+      url: `https://slack.com/api/conversations.replies`,
+      params: {
+        channel: input.channelId,
+        ts: input.threadId,
+      },
       headers: {
         Authorization: `Bearer ${input.token}`,
       },
     }
 
+    if (input.page !== undefined && input.page !== '') {
+      config.params.cursor = input.page
+    }
+
+    if (input.perPage !== undefined && input.perPage > 0) {
+      config.params.limit = input.perPage
+    }
+
     const response = await axios(config)
     const records: SlackMessages = response.data.messages
     const nextPage = response.data.response_metadata?.next_cursor || ''
@@ -31,14 +43,8 @@ async function getMessagesInThreads(
       nextPage,
     }
   } catch (err) {
-    if (err && err.response && err.response.status === 429 && err.response.headers['Retry-After']) {
-      logger.warn('Slack API rate limit exceeded')
-      const rateLimitResetSeconds = parseInt(err.response.headers['Retry-After'], 10)
-      throw new RateLimitError(rateLimitResetSeconds, '/conversation.replies')
-    } else {
-      logger.error({ err, input }, 'Error while getting messages from Slack')
-      throw err
-    }
+    const newErr = handleSlackError(err, input, logger)
+    throw newErr
   }
 }
 

backend/src/serverless/integrations/usecases/slack/getTeam.ts

@@ -1,14 +1,14 @@
-import axios from 'axios'
-import { SlackGetChannelsInput, SlackTeam } from '../../types/slackTypes'
+import axios, { AxiosRequestConfig } from 'axios'
 import { Logger } from '../../../../utils/logging'
 import { timeout } from '../../../../utils/timing'
-import { RateLimitError } from '../../../../types/integration/rateLimitError'
+import { SlackGetChannelsInput, SlackTeam } from '../../types/slackTypes'
+import { handleSlackError } from './errorHandler'
 
 async function getChannels(input: SlackGetChannelsInput, logger: Logger): Promise<SlackTeam> {
   await timeout(2000)
 
   try {
-    const config = {
+    const config: AxiosRequestConfig<any> = {
       method: 'get',
       url: 'https://slack.com/api/team.info',
       headers: {
@@ -19,14 +19,8 @@ async function getChannels(input: SlackGetChannelsInput, logger: Logger): Promis
     const response = await axios(config)
     return response.data.team
   } catch (err) {
-    if (err && err.response && err.response.status === 429 && err.response.headers['Retry-After']) {
-      logger.warn('Slack API rate limit exceeded')
-      const rateLimitResetSeconds = parseInt(err.response.headers['Retry-After'], 10)
-      throw new RateLimitError(rateLimitResetSeconds, '/team.info')
-    } else {
-      logger.error({ err, input }, 'Error while getting channels from Slack')
-      throw err
-    }
+    const newErr = handleSlackError(err, input, logger)
+    throw newErr
   }
 }