From c8c6f69d6f467e4bb083efa250a3a053e7b1567c Mon Sep 17 00:00:00 2001 From: garrrikkotua Date: Thu, 13 Apr 2023 19:10:37 +0300 Subject: [PATCH 01/32] create and validation api routes --- .../helpers/discourseCreateOrUpdate.ts | 9 ++++ .../integration/helpers/discourseValidator.ts | 38 +++++++++++++++ backend/src/api/integration/index.ts | 10 ++++ backend/src/services/integrationService.ts | 46 +++++++++++++++++++ backend/src/types/integrationEnums.ts | 2 + 5 files changed, 105 insertions(+) create mode 100644 backend/src/api/integration/helpers/discourseCreateOrUpdate.ts create mode 100644 backend/src/api/integration/helpers/discourseValidator.ts diff --git a/backend/src/api/integration/helpers/discourseCreateOrUpdate.ts b/backend/src/api/integration/helpers/discourseCreateOrUpdate.ts new file mode 100644 index 0000000000..3ef779a16d --- /dev/null +++ b/backend/src/api/integration/helpers/discourseCreateOrUpdate.ts @@ -0,0 +1,9 @@ +import Permissions from '../../../security/permissions' +import IntegrationService from '../../../services/integrationService' +import PermissionChecker from '../../../services/user/permissionChecker' + +export default async (req, res) => { + new PermissionChecker(req).validateHas(Permissions.values.tenantEdit) + const payload = await new IntegrationService(req).discourseConnectOrUpdate(req.body) + await req.responseHandler.success(req, res, payload) +} \ No newline at end of file diff --git a/backend/src/api/integration/helpers/discourseValidator.ts b/backend/src/api/integration/helpers/discourseValidator.ts new file mode 100644 index 0000000000..469a92cdd8 --- /dev/null +++ b/backend/src/api/integration/helpers/discourseValidator.ts @@ -0,0 +1,38 @@ +import axios from 'axios' +import Error400 from '../../../errors/Error400' +import Permissions from '../../../security/permissions' +import PermissionChecker from '../../../services/user/permissionChecker' + + +export default async (req, res) => { + new PermissionChecker(req).validateHasAny([ + Permissions.values.integrationCreate, + Permissions.values.integrationEdit, + ]) + + const {apiKey, apiUsername, forumHostname} = req.body; + + if (apiKey && apiUsername && forumHostname) { + try { + const result = await axios.get( + `https://${forumHostname}/admin/users/list/active.json`, + { + headers: { + 'Api-Key': apiKey, + 'Api-Username': apiUsername, + }, + } + ) + if ( + result.status === 200 && + result.data && + result.data.length > 0 + ) { + return req.responseHandler.success(req, res, result.data) + } + } catch (e) { + return req.responseHandler.error(req, res, new Error400(req.language)) + } + } + return req.responseHandler.error(req, res, new Error400(req.language)) +} diff --git a/backend/src/api/integration/index.ts b/backend/src/api/integration/index.ts index 7e84330d1d..8b51c7f6d2 100644 --- a/backend/src/api/integration/index.ts +++ b/backend/src/api/integration/index.ts @@ -58,6 +58,16 @@ export default (app) => { safeWrap(require('./helpers/stackOverflowVolume').default), ) + app.post( + '/tenant/:tenantId/discourse-connect', + safeWrap(require('./helpers/discourseCreateOrUpdate').default), + ) + + app.post( + '/tenant/:tenantId/discourse-validate', + safeWrap(require('./helpers/discourseValidator').default), + ) + if (TWITTER_CONFIG.clientId) { /** * Using the passport.authenticate this endpoint forces a diff --git a/backend/src/services/integrationService.ts b/backend/src/services/integrationService.ts index e987aa6d62..ad7d711390 100644 --- a/backend/src/services/integrationService.ts +++ b/backend/src/services/integrationService.ts @@ -775,4 +775,50 @@ export default class IntegrationService { return integration } + + /** + * Adds/updates Discourse integration + * @param integrationData to create the integration object + * @returns integration object + */ + async discourseConnectOrUpdate(integrationData) { + const transaction = await SequelizeRepository.createTransaction(this.options) + let integration + let run + + try { + integration = await this.createOrUpdate( + { + platform: PlatformType.DISCOURSE, + settings: { + apiKey: integrationData.apiKey, + apiUserame: integrationData.apiUsername, + forumHostname: integrationData.forumHostname, + updateMemberAttributes: true, + }, + status: 'in-progress', + }, + transaction, + ) + + run = await new IntegrationRunRepository({ ...this.options, transaction }).create({ + integrationId: integration.id, + tenantId: integration.tenantId, + onboarding: true, + state: IntegrationRunState.PENDING, + }) + await SequelizeRepository.commitTransaction(transaction) + } catch (err) { + await SequelizeRepository.rollbackTransaction(transaction) + throw err + } + + await sendNodeWorkerMessage( + integration.tenantId, + new NodeWorkerIntegrationProcessMessage(run.id), + ) + + return integration + } } + diff --git a/backend/src/types/integrationEnums.ts b/backend/src/types/integrationEnums.ts index e16ff7e25b..648c96ead1 100644 --- a/backend/src/types/integrationEnums.ts +++ b/backend/src/types/integrationEnums.ts @@ -15,6 +15,7 @@ export enum PlatformType { PRODUCTHUNT = 'producthunt', YOUTUBE = 'youtube', STACKOVERFLOW = 'stackoverflow', + DISCOURSE = 'discourse', OTHER = 'other', } @@ -32,4 +33,5 @@ export enum IntegrationType { LINKEDIN = 'linkedin', CROWD = 'crowd', STACKOVERFLOW = 'stackoverflow', + DISCOURSE = 'discourse', } From 17a26ae344bfda287e039511c8b9f2e2f034630e Mon Sep 17 00:00:00 2001 From: garrrikkotua Date: Fri, 14 Apr 2023 17:18:11 +0300 Subject: [PATCH 02/32] discourse webhook endpoint --- backend/src/api/webhooks/discourse.ts | 73 ++++++++++++++++++++++ backend/src/api/webhooks/index.ts | 1 + backend/src/services/integrationService.ts | 3 + backend/src/types/webhooks.ts | 1 + backend/src/utils/crypto.ts | 28 +++++++++ 5 files changed, 106 insertions(+) create mode 100644 backend/src/api/webhooks/discourse.ts create mode 100644 backend/src/utils/crypto.ts diff --git a/backend/src/api/webhooks/discourse.ts b/backend/src/api/webhooks/discourse.ts new file mode 100644 index 0000000000..a641ed9cd2 --- /dev/null +++ b/backend/src/api/webhooks/discourse.ts @@ -0,0 +1,73 @@ +import IntegrationRepository from '../../database/repositories/integrationRepository' +import TenantRepository from '../../database/repositories/tenantRepository' +import SequelizeRepository from '../../database/repositories/sequelizeRepository' +import IncomingWebhookRepository from '../../database/repositories/incomingWebhookRepository' +import { WebhookType } from '../../types/webhooks' +import { sendNodeWorkerMessage } from '../../serverless/utils/nodeWorkerSQS' +import { NodeWorkerProcessWebhookMessage } from '../../types/mq/nodeWorkerProcessWebhookMessage' +import { verifyWebhookSignature } from '../../utils/crypto' + +export default async (req, res) => { + const signature = req.headers['x-discourse-event-signature'] + const eventId = req.headers['x-discourse-event-id'] + const eventType = req.headers['x-discourse-event-type'] + const data = req.body + + const integrationId = req.params.integrationId as string + const options = await SequelizeRepository.getDefaultIRepositoryOptions() + const tenant = await TenantRepository.findById(req.params.tenantId, options) + const optionsWithTenant = await SequelizeRepository.getDefaultIRepositoryOptions(null, tenant) + const integration = (await IntegrationRepository.findById( + integrationId, + optionsWithTenant, + )) as any + + + if (integration) { + + try { + if (!signature) { + req.log.error({ signature }, 'Discourse Webhook signature header missing!'); + await req.responseHandler.success(req, res, 'Discourse Webhook signature header missing!', 200); + return; + } + + if (!verifyWebhookSignature(JSON.stringify(data), integration.settings.webhookSecret, signature)) { + req.log.error({ signature }, 'Discourse Webhook signature verification failed!'); + await req.responseHandler.success(req, res, 'Discourse Webhook signature verification failed!', 200); + return; + } + } catch (error) { + req.log.error({ signature, error }, 'Internal error when verifying discourse webhook'); + await req.responseHandler.success(req, res, 'Internal error when verifying discourse webhook', 200); + return; + } + + + req.log.info({ integrationId: integration.id }, 'Incoming Discourse Webhook!') + const options = await SequelizeRepository.getDefaultIRepositoryOptions() + const repo = new IncomingWebhookRepository(options) + + const result = await repo.create({ + tenantId: integration.tenantId, + integrationId: integration.id, + type: WebhookType.DISCOURSE, + payload: { + signature, + eventId, + eventType, + data, + }, + }) + + await sendNodeWorkerMessage( + integration.tenantId, + new NodeWorkerProcessWebhookMessage(integration.tenantId, result.id), + ) + + await req.responseHandler.success(req, res, {}, 204) + } else { + req.log.error({ integrationId }, 'No integration found for incoming Discourse Webhook!') + await req.responseHandler.success(req, res, {}, 200) + } +} diff --git a/backend/src/api/webhooks/index.ts b/backend/src/api/webhooks/index.ts index 98012bbc49..f0bafbbd95 100644 --- a/backend/src/api/webhooks/index.ts +++ b/backend/src/api/webhooks/index.ts @@ -4,4 +4,5 @@ export default (app) => { app.post(`/github`, safeWrap(require('./github').default)) app.post(`/stripe`, safeWrap(require('./stripe').default)) app.post(`/sendgrid`, safeWrap(require('./sendgrid').default)) + app.post(`/discourse/:tenantId/:integrationId`, safeWrap(require('./discourse').default)) } diff --git a/backend/src/services/integrationService.ts b/backend/src/services/integrationService.ts index ad7d711390..f250a1caaf 100644 --- a/backend/src/services/integrationService.ts +++ b/backend/src/services/integrationService.ts @@ -20,6 +20,7 @@ import { getOrganizations } from '../serverless/integrations/usecases/linkedin/g import Error404 from '../errors/Error404' import IntegrationRunRepository from '../database/repositories/integrationRunRepository' import { IntegrationRunState } from '../types/integrationRunTypes' +import { generateWebhookSecret } from '../utils/crypto' const discordToken = DISCORD_CONFIG.token2 || DISCORD_CONFIG.token @@ -787,6 +788,7 @@ export default class IntegrationService { let run try { + const webhookSecret = generateWebhookSecret() integration = await this.createOrUpdate( { platform: PlatformType.DISCOURSE, @@ -794,6 +796,7 @@ export default class IntegrationService { apiKey: integrationData.apiKey, apiUserame: integrationData.apiUsername, forumHostname: integrationData.forumHostname, + webhookSecret, // we can do encryption here updateMemberAttributes: true, }, status: 'in-progress', diff --git a/backend/src/types/webhooks.ts b/backend/src/types/webhooks.ts index 33e32dc1bb..7257e6dcad 100644 --- a/backend/src/types/webhooks.ts +++ b/backend/src/types/webhooks.ts @@ -9,6 +9,7 @@ export enum WebhookState { export enum WebhookType { GITHUB = 'GITHUB', DISCORD = 'DISCORD', + DISCOURSE = 'DISCOURSE', } export enum DiscordWebsocketEvent { diff --git a/backend/src/utils/crypto.ts b/backend/src/utils/crypto.ts new file mode 100644 index 0000000000..14f2e51fcf --- /dev/null +++ b/backend/src/utils/crypto.ts @@ -0,0 +1,28 @@ +import * as crypto from 'crypto'; +import * as buffer from 'buffer'; + +export function generateWebhookSecret(length: number = 32): string { + return crypto.randomBytes(length).toString('hex'); +} + +export function verifyWebhookSignature( + payload: string, + secret: string, + signatureHeader: string +): boolean { + + console.log("payload", payload) + const hmac = crypto.createHmac('sha256', secret); + console.log("hmac", hmac) + hmac.update(payload); + console.log("hmac", hmac) + const expectedSignature = `sha256=${hmac.digest('hex')}`; + + console.log("expectedSignature", expectedSignature) + + return crypto.timingSafeEqual( + buffer.Buffer.from(signatureHeader), + buffer.Buffer.from(expectedSignature) + ); +} + From 07e5eef5b16cdfa46cb88a1ef54a366cd954181c Mon Sep 17 00:00:00 2001 From: garrrikkotua Date: Fri, 14 Apr 2023 17:35:37 +0300 Subject: [PATCH 03/32] endpoint to test that webhooks are received --- .../helpers/discourseTestWebhook.ts | 16 +++++++++++++ backend/src/api/integration/index.ts | 5 ++++ .../repositories/incomingWebhookRepository.ts | 24 +++++++++++++++++++ 3 files changed, 45 insertions(+) create mode 100644 backend/src/api/integration/helpers/discourseTestWebhook.ts diff --git a/backend/src/api/integration/helpers/discourseTestWebhook.ts b/backend/src/api/integration/helpers/discourseTestWebhook.ts new file mode 100644 index 0000000000..03e8c9d6ef --- /dev/null +++ b/backend/src/api/integration/helpers/discourseTestWebhook.ts @@ -0,0 +1,16 @@ +import Permissions from '../../../security/permissions' +import PermissionChecker from '../../../services/user/permissionChecker' +import IncomingWebhookRepository from '../../../database/repositories/incomingWebhookRepository' +import SequelizeRepository from '../../../database/repositories/sequelizeRepository' + +export default async (req, res) => { + new PermissionChecker(req).validateHas(Permissions.values.tenantEdit) + + const options = await SequelizeRepository.getDefaultIRepositoryOptions() + const repo = new IncomingWebhookRepository(options) + + const isWebhooksReceived = await repo.checkWebhooksExistForIntegration(req.body.integrationId) + + await req.responseHandler.success(req, res, {isWebhooksReceived}) + +} \ No newline at end of file diff --git a/backend/src/api/integration/index.ts b/backend/src/api/integration/index.ts index 8b51c7f6d2..d5c771d705 100644 --- a/backend/src/api/integration/index.ts +++ b/backend/src/api/integration/index.ts @@ -68,6 +68,11 @@ export default (app) => { safeWrap(require('./helpers/discourseValidator').default), ) + app.post( + '/tenant/:tenantId/discourse-test-webhook', + safeWrap(require('./helpers/discourseTestWebhook').default), + ) + if (TWITTER_CONFIG.clientId) { /** * Using the passport.authenticate this endpoint forces a diff --git a/backend/src/database/repositories/incomingWebhookRepository.ts b/backend/src/database/repositories/incomingWebhookRepository.ts index 9b8decb919..dcf477bfa4 100644 --- a/backend/src/database/repositories/incomingWebhookRepository.ts +++ b/backend/src/database/repositories/incomingWebhookRepository.ts @@ -160,4 +160,28 @@ export default class IncomingWebhookRepository extends RepositoryBase< throw new Error(`Failed to mark webhook '${id}' as error!`) } } + + async checkWebhooksExistForIntegration( + integrationId: string, + ): Promise { + const transaction = this.transaction + + const results = await this.seq.query( + ` + select count(*) as count + from "incomingWebhooks" + where "integrationId" = :integrationId + limit 1 + `, + { + replacements: { + integrationId, + }, + type: QueryTypes.SELECT, + transaction, + }, + ) + + return results[0].count > 0 + } } From 60112f2aeb8d8afe74585e29e5aa206d68bb96a2 Mon Sep 17 00:00:00 2001 From: garrrikkotua Date: Mon, 17 Apr 2023 12:16:35 +0300 Subject: [PATCH 04/32] wip --- .../services/integrations/discourseIntegrationService.ts | 0 1 file changed, 0 insertions(+), 0 deletions(-) create mode 100644 backend/src/serverless/integrations/services/integrations/discourseIntegrationService.ts diff --git a/backend/src/serverless/integrations/services/integrations/discourseIntegrationService.ts b/backend/src/serverless/integrations/services/integrations/discourseIntegrationService.ts new file mode 100644 index 0000000000..e69de29bb2 From 18c804beba4583e4f52eca43e7401fb4cd004c55 Mon Sep 17 00:00:00 2001 From: garrrikkotua Date: Mon, 24 Apr 2023 18:17:46 +0300 Subject: [PATCH 05/32] polling --- .../database/attributes/member/discourse.ts | 27 ++ .../integrations/grid/discourseGrid.ts | 13 + .../discourseIntegrationService.ts | 340 +++++++++++++++++ .../integrations/types/discourseTypes.ts | 354 ++++++++++++++++++ .../usecases/discourse/getCategories.ts | 28 ++ .../usecases/discourse/getPostsByIds.ts | 56 +++ .../usecases/discourse/getPostsFromTopic.ts | 34 ++ .../usecases/discourse/getTopics.ts | 34 ++ .../usecases/discourse/getUser.ts | 34 ++ backend/src/types/activityTypes.ts | 5 + 10 files changed, 925 insertions(+) create mode 100644 backend/src/database/attributes/member/discourse.ts create mode 100644 backend/src/serverless/integrations/grid/discourseGrid.ts create mode 100644 backend/src/serverless/integrations/types/discourseTypes.ts create mode 100644 backend/src/serverless/integrations/usecases/discourse/getCategories.ts create mode 100644 backend/src/serverless/integrations/usecases/discourse/getPostsByIds.ts create mode 100644 backend/src/serverless/integrations/usecases/discourse/getPostsFromTopic.ts create mode 100644 backend/src/serverless/integrations/usecases/discourse/getTopics.ts create mode 100644 backend/src/serverless/integrations/usecases/discourse/getUser.ts diff --git a/backend/src/database/attributes/member/discourse.ts b/backend/src/database/attributes/member/discourse.ts new file mode 100644 index 0000000000..79af834a05 --- /dev/null +++ b/backend/src/database/attributes/member/discourse.ts @@ -0,0 +1,27 @@ +import { Attribute } from '../attribute' +import { AttributeType } from '../types' +import { MemberAttributes, MemberAttributeName } from './enums' + +export const DiscourseMemberAttributes: Attribute[] = [ + { + name: MemberAttributes[MemberAttributeName.URL].name, + label: MemberAttributes[MemberAttributeName.URL].label, + type: AttributeType.URL, + canDelete: false, + show: true, + }, + { + name: MemberAttributes[MemberAttributeName.BIO].name, + label: MemberAttributes[MemberAttributeName.BIO].label, + type: AttributeType.STRING, + canDelete: false, + show: true, + }, + { + name: MemberAttributes[MemberAttributeName.AVATAR_URL].name, + label: MemberAttributes[MemberAttributeName.AVATAR_URL].label, + type: AttributeType.URL, + canDelete: false, + show: false, + }, +] diff --git a/backend/src/serverless/integrations/grid/discourseGrid.ts b/backend/src/serverless/integrations/grid/discourseGrid.ts new file mode 100644 index 0000000000..f14307b7ab --- /dev/null +++ b/backend/src/serverless/integrations/grid/discourseGrid.ts @@ -0,0 +1,13 @@ +import { gridEntry } from './grid' + +export class DiscourseGrid { + static post: gridEntry = { + score: 8, + isContribution: false, + } + + static reply: gridEntry = { + score: 6, + isContribution: false, + } +} diff --git a/backend/src/serverless/integrations/services/integrations/discourseIntegrationService.ts b/backend/src/serverless/integrations/services/integrations/discourseIntegrationService.ts index e69de29bb2..9b2e027b50 100644 --- a/backend/src/serverless/integrations/services/integrations/discourseIntegrationService.ts +++ b/backend/src/serverless/integrations/services/integrations/discourseIntegrationService.ts @@ -0,0 +1,340 @@ +import sanitizeHtml from 'sanitize-html' +import he from 'he' +import moment from 'moment/moment' +import { IntegrationServiceBase } from '../integrationServiceBase' +import { IntegrationType, PlatformType } from '../../../../types/integrationEnums' +import { + IIntegrationStream, + IPendingStream, + IProcessStreamResults, + IStepContext, +} from '../../../../types/integration/stepResult' +import { + DiscourseConnectionParams, + DiscourseCategoryResponse, + DiscourseTopicResponse, + DiscourseTopicsInput, + DiscoursePostsInput, + DiscoursePostsFromTopicResponse, + DiscoursePostsByIdsInput, + DiscoursePostsByIdsResponse +} from '../../types/discourseTypes' +import { getDiscourseCategories } from '../../usecases/discourse/getCategories' +import { getDiscourseTopics } from '../../usecases/discourse/getTopics' +import { getDiscoursePostsFromTopic } from '../../usecases/discourse/getPostsFromTopic' +import { getDiscoursePostsByIds } from '../../usecases/discourse/getPostsByIds' +import { getDiscourseUserByUsername } from '../../usecases/discourse/getUser' +import { AddActivitiesSingle, Member } from '../../types/messageTypes' +import Operations from '../../../dbOperations/operations' +import { DiscourseMemberAttributes } from '../../../../database/attributes/member/discourse' +import MemberAttributeSettingsService from '../../../../services/memberAttributeSettingsService' +import { MemberAttributeName } from '../../../../database/attributes/member/enums' +import { DiscourseActivityType } from '../../../../types/activityTypes' +import { DiscourseGrid } from '../../grid/discourseGrid' + +enum DiscourseStreamType { + CATEGORIES = 'categories', + TOPICS_FROM_CATEGORY = 'topicsFromCategory', + POSTS_FROM_TOPIC = 'postsFromTopic', + POSTS_BY_IDS = 'postsByIds', +} + +interface DiscourseProcessResult { + type: DiscourseStreamType + data: + | DiscourseCategoryResponse + | DiscourseTopicResponse + | DiscoursePostsFromTopicResponse + | DiscoursePostsByIdsResponse +} + +/* eslint class-methods-use-this: 0 */ + +/* eslint-disable @typescript-eslint/no-unused-vars */ + +/* eslint-disable no-case-declarations */ + +export class DiscourseIntegrationService extends IntegrationServiceBase { + constructor() { + // disable polling - new data will come trhough webhooks + super(IntegrationType.DISCOURSE, -1) + } + + async createMemberAttributes(context: IStepContext): Promise { + const service = new MemberAttributeSettingsService(context.repoContext) + await service.createPredefined(DiscourseMemberAttributes) + } + + /** + * Set up the pipeline data that will be needed throughout the processing. + * @param context context passed along worker messages + */ + async preprocess(context: IStepContext): Promise { + const settings = context.integration.settings + context.pipelineData = { + apiKey: settings.apiKey, + apiUsername: settings.apiUsername, + forumHostname: settings.forumHostname, + } + } + + async getStreams(context: IStepContext): Promise { + return [ + { + value: DiscourseStreamType.CATEGORIES, + metadata: { + page: '', + }, + }, + ] + } + + async processStream( + stream: IIntegrationStream, + context: IStepContext, + ): Promise { + const streamType = stream.value as DiscourseStreamType + let result: DiscourseProcessResult + switch (streamType) { + case DiscourseStreamType.CATEGORIES: + result = await DiscourseIntegrationService.processCategoriesStream(stream, context) + break + case DiscourseStreamType.TOPICS_FROM_CATEGORY: + result = await DiscourseIntegrationService.processTopicsFromCategoryStream(stream, context) + break + case DiscourseStreamType.POSTS_FROM_TOPIC: + result = await DiscourseIntegrationService.processPostsFromTopicStream(stream, context) + break + case DiscourseStreamType.POSTS_BY_IDS: + result = await DiscourseIntegrationService.processPostsByIds(stream, context) + break + default: + throw new Error(`Unknown stream type: ${streamType}`) + } + + const newStreams: IPendingStream[] = [] + const nextPageStream: IPendingStream | undefined = undefined + const activities: AddActivitiesSingle[] = [] + + // another switch statement to handle the different types of results, helps with type safety + switch (result.type) { + case DiscourseStreamType.CATEGORIES: + const data = result.data as DiscourseCategoryResponse + data.category_list.categories.forEach((category) => { + newStreams.push({ + value: DiscourseStreamType.TOPICS_FROM_CATEGORY, + metadata: { + category_id: category.id, + category_slug: category.slug, + page: 0, + }, + }) + }) + break + case DiscourseStreamType.TOPICS_FROM_CATEGORY: + const data2 = result.data as DiscourseTopicResponse + data2.topic_list.topics.forEach((topic) => { + newStreams.push({ + value: DiscourseStreamType.POSTS_FROM_TOPIC, + metadata: { + topicId: topic.id, + topic_slug: topic.slug, + page: 0, + }, + }) + }) + break + case DiscourseStreamType.POSTS_FROM_TOPIC: + const data3 = result.data as DiscoursePostsFromTopicResponse + const batchSize = 100 + const postBatches: number[][] = [] + + data3.post_stream.stream.forEach((postId, index) => { + if (index % batchSize === 0) { + postBatches.push([]) + } + postBatches[postBatches.length - 1].push(postId) + }) + + postBatches.forEach((postBatch, index) => { + newStreams.push({ + value: DiscourseStreamType.POSTS_BY_IDS, + metadata: { + topicId: data3.id, + topicSlug: data3.slug, + topicTitle: data3.title, + postIds: postBatch, + lastIdInPreviousBatch: + index === 0 ? undefined : postBatches[index - 1][postBatches[index - 1].length - 1], + }, + }) + }) + + break + case DiscourseStreamType.POSTS_BY_IDS: + // no new streams to launch + // just add the activities + const data4 = result.data as DiscoursePostsByIdsResponse + const { topicId, lastIdInPreviousBatch } = stream.metadata + const posts = data4.post_stream.posts + for (const post of posts) { + const user = await getDiscourseUserByUsername( + { + forumHostname: context.pipelineData.forumHostname, + apiKey: context.pipelineData.apiKey, + apiUsername: context.pipelineData.apiUsername, + }, + { username: post.username }, + context.logger, + ) + + const member: Member = { + username: { + [PlatformType.DISCOURSE]: post.username, + }, + displayName: user.user.name, + attributes: { + [MemberAttributeName.URL]: { + [PlatformType.DISCOURSE]: `https://${context.pipelineData.forumHostname}/u/${post.username}`, + }, + [MemberAttributeName.WEBSITE_URL]: { + [PlatformType.DISCOURSE]: user.user.website || '', + }, + [MemberAttributeName.LOCATION]: { + [PlatformType.DISCOURSE]: user.user.location || '', + }, + [MemberAttributeName.BIO]: { + [PlatformType.DISCOURSE]: sanitizeHtml(he.decode(user.user.bio_cooked)) || '', + }, + [MemberAttributeName.AVATAR_URL]: { + [PlatformType.DISCOURSE]: + `https://${context.pipelineData.forumHostname}${user.user.avatar_template.replace( + '{size}', + '200', + )}` || '', + }, + }, + emails: user.user.email ? [user.user.email] : [], + } + + const activity: AddActivitiesSingle = { + member, + platform: PlatformType.DISCOURSE, + tenant: context.integration.tenantId, + sourceId: `${post.id}`, + sourceParentId: lastIdInPreviousBatch || null, + type: post.post_number === 1 ? DiscourseActivityType.POST : DiscourseActivityType.REPLY, + timestamp: moment(post.created_at).utc().toDate(), + body: sanitizeHtml(he.decode(post.cooked)), + title: post.post_number === 1 ? stream.metadata.topicTitle : null, + url: `https://${context.pipelineData.forumHostname}/t/${stream.metadata.topicSlug}/${topicId}/${post.post_number}`, + score: post.post_number === 1 ? DiscourseGrid[DiscourseActivityType.POST].score : DiscourseGrid[DiscourseActivityType.REPLY].score, + isContribution: post.post_number === 1 ? DiscourseGrid[DiscourseActivityType.POST].isContribution : DiscourseGrid[DiscourseActivityType.REPLY].isContribution, + } + activities.push(activity) + } + break + default: + throw new Error(`Unknown stream type: ${streamType}`) + } + + return { + operations: [ + { + type: Operations.UPSERT_ACTIVITIES_WITH_MEMBERS, + records: activities, + }, + ], + newStreams, + nextPageStream, + } + } + + static async processCategoriesStream( + stream: IIntegrationStream, + context: IStepContext, + ): Promise { + const { forumHostname, apiKey, apiUsername } = context.pipelineData + const params: DiscourseConnectionParams = { + forumHostname, + apiKey, + apiUsername, + } + const discourseCategories = await getDiscourseCategories(params, context.logger) + return { + type: DiscourseStreamType.CATEGORIES, + data: discourseCategories, + } + } + + static async processTopicsFromCategoryStream( + stream: IIntegrationStream, + context: IStepContext, + ): Promise { + const { forumHostname, apiKey, apiUsername } = context.pipelineData + const params: DiscourseConnectionParams = { + forumHostname, + apiKey, + apiUsername, + } + + const input: DiscourseTopicsInput = { + category_id: stream.metadata.category_id, + category_slug: stream.metadata.category_slug, + page: stream.metadata.page, + } + + const discourseTopics = await getDiscourseTopics(params, input, context.logger) + return { + type: DiscourseStreamType.TOPICS_FROM_CATEGORY, + data: discourseTopics, + } + } + + static async processPostsFromTopicStream( + stream: IIntegrationStream, + context: IStepContext, + ): Promise { + const { forumHostname, apiKey, apiUsername } = context.pipelineData + const params: DiscourseConnectionParams = { + forumHostname, + apiKey, + apiUsername, + } + + const input: DiscoursePostsInput = { + topic_slug: stream.metadata.topic_slug, + topic_id: stream.metadata.topicId, + page: stream.metadata.page, + } + + const discourseTopics = await getDiscoursePostsFromTopic(params, input, context.logger) + return { + type: DiscourseStreamType.POSTS_FROM_TOPIC, + data: discourseTopics, + } + } + + static async processPostsByIds( + stream: IIntegrationStream, + context: IStepContext, + ): Promise { + const { forumHostname, apiKey, apiUsername } = context.pipelineData + const params: DiscourseConnectionParams = { + forumHostname, + apiKey, + apiUsername, + } + + const input: DiscoursePostsByIdsInput = { + topic_id: stream.metadata.topicId, + post_ids: stream.metadata.postIds, + } + + const discoursePosts = await getDiscoursePostsByIds(params, input, context.logger) + return { + type: DiscourseStreamType.POSTS_BY_IDS, + data: discoursePosts, + } + } +} diff --git a/backend/src/serverless/integrations/types/discourseTypes.ts b/backend/src/serverless/integrations/types/discourseTypes.ts new file mode 100644 index 0000000000..e84108a467 --- /dev/null +++ b/backend/src/serverless/integrations/types/discourseTypes.ts @@ -0,0 +1,354 @@ +export interface DiscourseConnectionParams { + apiKey: string + apiUsername: string + forumHostname: string +} + +export interface DiscourseCategoryResponse { + category_list: { + can_create_category: boolean + can_create_topic: boolean + categories: DiscourseCategory[] + } +} + +export interface DiscourseTopicsInput { + category_slug: string + category_id: number + page: number +} + +export interface DiscoursePostsInput { + topic_slug: string + topic_id: number + page: number +} + +export interface DisourseUserByUsernameInput { + username: string +} + +export interface DiscoursePostsByIdsInput { + topic_id: number + post_ids: number[] +} + +export interface DiscoursePostsByIdsResponse { + post_stream: { + posts: Post[] + } + id: number // this is the id of the topic +} + +export interface DiscourseCategory { + id: number + name: string + color: string + text_color: string + slug: string + topic_count: number + post_count: number + position: number + description: string + description_text: string + description_excerpt: string + topic_url: string + read_restricted: boolean + permission: number + notification_level: number + can_edit: boolean + topic_template: string + has_children: boolean + sort_order: string + sort_ascending: string + show_subcategory_list: boolean + num_featured_topics: number + default_view: string + subcategory_list_style: string + default_top_period: string + default_list_filter: string + minimum_required_tags: number + navigate_to_first_post_after_read: boolean + topics_day: number + topics_week: number + topics_month: number + topics_year: number + topics_all_time: number + is_uncategorized: boolean + subcategory_ids: (number | null)[] + subcategory_list: any[] + uploaded_logo: string + uploaded_logo_dark: string + uploaded_background: string +} + +export interface DiscourseTopicResponse { + users: User[] + primary_groups: any[] + topic_list: { + can_create_topic: boolean + per_page: number + top_tags: any[] + topics: Topic[] + } +} + +export interface DiscourseUserResponse { + user_badges: any[] + badges: any[] + badge_types: any[] + users: FullUser[] + user: FullUser +} + +export interface FullUser { + id: number + username: string + name: string + avatar_template: string + email?: string + bio_cooked?: string + bio_excerpt?: string + bio_raw?: string + location?: string + website?: string + secondary_emails: string[] | null[] + active: boolean + admin: boolean + moderator: boolean + last_seen_at: string + last_emailed_at: string + created_at: string + last_seen_age: number + last_emailed_age: number + created_at_age: number + trust_level: number + manual_locked_trust_level: string | null + flag_level: number + title: string | null + time_read: number + staged: boolean + days_visited: number + posts_read_count: number + topics_entered: number + post_count: number +} + +interface User { + id: number + username: string + name: string + avatar_template: string +} + +interface Topic { + id: number + title: string + fancy_title: string + slug: string + posts_count: number + reply_count: number + highest_post_number: number + image_url: string + created_at: string + last_posted_at: string + bumped: boolean + bumped_at: string + archetype: string + unseen: boolean + pinned: boolean + unpinned: string | null + excerpt: string + visible: boolean + closed: boolean + archived: boolean + bookmarked: string | null + liked: string | null + views: number + like_count: number + has_summary: boolean + last_poster_username: string + category_id: number + pinned_globally: boolean + featured_link: string + posters: Poster[] +} + +interface Poster { + extras: string | null + description: string + user_id: number + primary_group_id: string | null +} + +export interface DiscoursePostsFromTopicResponse { + post_stream: { + posts: Post[]; + stream: number[]; + }; + timeline_lookup: any[]; + suggested_topics: SuggestedTopic[]; + tags: any[]; + tags_descriptions: {}; + id: number; + title: string; + fancy_title: string; + posts_count: number; + created_at: string; + views: number; + reply_count: number; + like_count: number; + last_posted_at: string; + visible: boolean; + closed: boolean; + archived: boolean; + has_summary: boolean; + archetype: string; + slug: string; + category_id: number; + word_count: number; + deleted_at: string; + user_id: number; + featured_link: string; + pinned_globally: boolean; + pinned_at: string; + pinned_until: string; + image_url: string; + slow_mode_seconds: number; + draft: string; + draft_key: string; + draft_sequence: number; + unpinned: string | null; + pinned: boolean; + current_post_number: number; + highest_post_number: number; + deleted_by: string | null; + has_deleted: boolean; + actions_summary: ActionSummary[]; + chunk_size: number; + bookmarked: boolean; + bookmarks: any[]; + topic_timer: string | null; + message_bus_last_id: number; + participant_count: number; + show_read_indicator: boolean; + thumbnails: string | null; + slow_mode_enabled_until: string | null; + details: TopicDetails; +} + +interface Post { + id: number; // this id is unique accrss all posts + name: string; + username: string; + avatar_template: string; + created_at: string; + cooked: string; + post_number: number; // this shows the order of the post in the topic, 1 is the first post + post_type: number; + updated_at: string; + reply_count: number; + reply_to_post_number: string | null; + quote_count: number; + incoming_link_count: number; + reads: number; + readers_count: number; + score: number; + yours: boolean; + topic_id: number; + topic_slug: string; + display_username: string; + primary_group_name: string; + flair_name: string; + flair_url: string; + flair_bg_color: string; + flair_color: string; + version: number; + can_edit: boolean; + can_delete: boolean; + can_recover: boolean; + can_wiki: boolean; + link_counts: LinkCount[]; + read: boolean; + user_title: string; + bookmarked: boolean; + actions_summary: ActionSummary[]; + moderator: boolean; + admin: boolean; + staff: boolean; + user_id: number; + hidden: boolean; + trust_level: number; + deleted_at: string | null; + user_deleted: boolean; + edit_reason: string | null; + can_view_edit_history: boolean; + wiki: boolean; + reviewable_id: number; + reviewable_score_count: number; + reviewable_score_pending_count: number; +} + +interface LinkCount { + url: string; + internal: boolean; + reflection: boolean; + title: string; + clicks: number; +} + +interface ActionSummary { + id: number; + can_act: boolean; + count?: number; + hidden?: boolean; +} + +interface SuggestedTopic extends Omit { + tags: any[]; + posters: SuggestedTopicPoster[]; +} + +interface SuggestedTopicPoster { + extras: string | null; + description: string; + user: User +} + +interface TopicDetails { + can_edit: boolean + notification_level: number + can_move_posts: boolean + can_delete: boolean + can_remove_allowed_users: boolean + can_create_post: boolean + can_reply_as_new_topic: boolean + can_invite_to: boolean + can_invite_via_email: boolean + can_flag_topic: boolean + can_convert_topic: boolean + can_review_topic: boolean + can_close_topic: boolean + can_archive_topic: boolean + can_split_merge_topic: boolean + can_edit_staff_notes: boolean + can_toggle_topic_visibility: boolean + can_pin_unpin_topic: boolean + can_moderate_category: boolean + can_remove_self_id: number + participants: Participant[] + created_by: User + last_poster: User +} + +interface Participant extends User { + post_count: number + primary_group_name: string + flair_name: string + flair_url: string + flair_color: string + flair_bg_color: string + admin: boolean + moderator: boolean + trust_level: number +} \ No newline at end of file diff --git a/backend/src/serverless/integrations/usecases/discourse/getCategories.ts b/backend/src/serverless/integrations/usecases/discourse/getCategories.ts new file mode 100644 index 0000000000..e81e95388b --- /dev/null +++ b/backend/src/serverless/integrations/usecases/discourse/getCategories.ts @@ -0,0 +1,28 @@ +import axios, { AxiosRequestConfig } from 'axios' +import type { DiscourseConnectionParams } from '../../types/discourseTypes' +import { Logger } from '../../../../utils/logging' +import { DiscourseCategoryResponse } from '../../types/discourseTypes' + +export const getDiscourseCategories = async ( + params: DiscourseConnectionParams, + logger: Logger, +): Promise => { + + logger.info({ message: 'Fetching categories from Discourse', forumHostName: params.forumHostname }) + const config: AxiosRequestConfig = { + method: 'get', + url: `https://${params.forumHostname}/categories.json`, + headers: { + 'Api-Key': params.apiKey, + 'Api-Username': params.apiUsername, + }, + } + + try { + const response = await axios(config) + return response.data + } catch (err) { + logger.error({ err, params }, 'Error while getting Discourse categories') + throw err + } +} diff --git a/backend/src/serverless/integrations/usecases/discourse/getPostsByIds.ts b/backend/src/serverless/integrations/usecases/discourse/getPostsByIds.ts new file mode 100644 index 0000000000..a45383eb50 --- /dev/null +++ b/backend/src/serverless/integrations/usecases/discourse/getPostsByIds.ts @@ -0,0 +1,56 @@ +import axios, { AxiosRequestConfig } from 'axios' +import type { DiscourseConnectionParams } from '../../types/discourseTypes' +import { Logger } from '../../../../utils/logging' +import { + DiscoursePostsByIdsResponse, + DiscoursePostsByIdsInput, +} from '../../types/discourseTypes' + +const serializeArrayToQueryString = (params: Object) => Object.entries(params) + .map(([key, value]) => { + if (Array.isArray(value)) { + return value + .map((val) => `${encodeURIComponent(key)}[]=${encodeURIComponent(val)}`) + .join('&') + } + return `${encodeURIComponent(key)}=${encodeURIComponent(value)}` + }) + .join('&') + +// this methods returns ids of posts in a topic +// then we need to parse each topic individually (can be batched) +export const getDiscoursePostsByIds = async ( + params: DiscourseConnectionParams, + input: DiscoursePostsByIdsInput, + logger: Logger, +): Promise => { + logger.info({ + message: 'Fetching posts by ids from Discourse', + params, + input, + }) + + + const queryParameters = { + post_ids: input.post_ids, + } + + const queryString = serializeArrayToQueryString(queryParameters) + + const config: AxiosRequestConfig = { + method: 'get', + url: `https://${params.forumHostname}/t/${input.topic_id}/posts.json?${queryString}`, + headers: { + 'Api-Key': params.apiKey, + 'Api-Username': params.apiUsername, + }, + } + + try { + const response = await axios(config) + return response.data + } catch (err) { + logger.error({ err, params, input }, 'Error while getting posts by ids from Discourse ') + throw err + } +} diff --git a/backend/src/serverless/integrations/usecases/discourse/getPostsFromTopic.ts b/backend/src/serverless/integrations/usecases/discourse/getPostsFromTopic.ts new file mode 100644 index 0000000000..a024e0eb02 --- /dev/null +++ b/backend/src/serverless/integrations/usecases/discourse/getPostsFromTopic.ts @@ -0,0 +1,34 @@ +import axios, { AxiosRequestConfig } from 'axios' +import type { DiscourseConnectionParams } from '../../types/discourseTypes' +import { Logger } from '../../../../utils/logging' +import { DiscoursePostsFromTopicResponse, DiscoursePostsInput } from '../../types/discourseTypes' + +// this methods returns ids of posts in a topic +// then we need to parse each topic individually (can be batched) +export const getDiscoursePostsFromTopic = async ( + params: DiscourseConnectionParams, + input: DiscoursePostsInput, + logger: Logger, +): Promise => { + logger.info({ + message: 'Fetching posts from topic from Discourse', + params, + input, + }) + const config: AxiosRequestConfig = { + method: 'get', + url: `https://${params.forumHostname}/t/${input.topic_slug}/${input.topic_id}.json`, + headers: { + 'Api-Key': params.apiKey, + 'Api-Username': params.apiUsername, + }, + } + + try { + const response = await axios(config) + return response.data + } catch (err) { + logger.error({ err, params, input }, 'Error while getting posts from topic from Discourse ') + throw err + } +} diff --git a/backend/src/serverless/integrations/usecases/discourse/getTopics.ts b/backend/src/serverless/integrations/usecases/discourse/getTopics.ts new file mode 100644 index 0000000000..d4e2db2d13 --- /dev/null +++ b/backend/src/serverless/integrations/usecases/discourse/getTopics.ts @@ -0,0 +1,34 @@ +import axios, { AxiosRequestConfig } from 'axios' +import type { DiscourseConnectionParams } from '../../types/discourseTypes' +import { Logger } from '../../../../utils/logging' +import { DiscourseCategoryResponse, DiscourseTopicsInput } from '../../types/discourseTypes' + +export const getDiscourseTopics = async ( + params: DiscourseConnectionParams, + input: DiscourseTopicsInput, + logger: Logger, +): Promise => { + logger.info({ + message: 'Fetching categories from Discourse', + forumHostName: params.forumHostname, + }) + const config: AxiosRequestConfig = { + method: 'get', + url: `https://${params.forumHostname}/c/${input.category_slug}/${input.category_id}.json`, + headers: { + 'Api-Key': params.apiKey, + 'Api-Username': params.apiUsername, + }, + params: { + page: input.page, + } + } + + try { + const response = await axios(config) + return response.data + } catch (err) { + logger.error({ err, params }, 'Error while getting Discourse categories') + throw err + } +} diff --git a/backend/src/serverless/integrations/usecases/discourse/getUser.ts b/backend/src/serverless/integrations/usecases/discourse/getUser.ts new file mode 100644 index 0000000000..632b5c15e7 --- /dev/null +++ b/backend/src/serverless/integrations/usecases/discourse/getUser.ts @@ -0,0 +1,34 @@ +import axios, { AxiosRequestConfig } from 'axios' +import type { DiscourseConnectionParams } from '../../types/discourseTypes' +import { Logger } from '../../../../utils/logging' +import { DiscourseUserResponse, DisourseUserByUsernameInput } from '../../types/discourseTypes' + +// this methods returns ids of posts in a topic +// then we need to parse each topic individually (can be batched) +export const getDiscourseUserByUsername = async ( + params: DiscourseConnectionParams, + input: DisourseUserByUsernameInput, + logger: Logger, +): Promise => { + logger.info({ + message: 'Fetching user by username from Discourse', + params, + input, + }) + const config: AxiosRequestConfig = { + method: 'get', + url: `https://${params.forumHostname}/u/${input.username}.json`, + headers: { + 'Api-Key': params.apiKey, + 'Api-Username': params.apiUsername, + }, + } + + try { + const response = await axios(config) + return response.data + } catch (err) { + logger.error({ err, params, input }, 'Error while fetching user by username from Discourse ') + throw err + } +} diff --git a/backend/src/types/activityTypes.ts b/backend/src/types/activityTypes.ts index f0a727cd18..50d7aa3c09 100644 --- a/backend/src/types/activityTypes.ts +++ b/backend/src/types/activityTypes.ts @@ -97,6 +97,11 @@ export enum StackOverflowActivityType { ANSWER = 'answer', } +export enum DiscourseActivityType { + POST = 'post', + REPLY = 'reply', +} + const githubUrl = 'https://github.com' const defaultGithubChannelFormatter = (channel) => { From eb7869a5d9e6711f1ad33b5991f0a1db3933f3ae Mon Sep 17 00:00:00 2001 From: garrrikkotua Date: Mon, 24 Apr 2023 19:15:26 +0300 Subject: [PATCH 06/32] lint --- .../integration/helpers/discourseValidator.ts | 2 +- backend/src/api/webhooks/discourse.ts | 18 +++++++++--------- backend/src/utils/crypto.ts | 19 +++++++------------ 3 files changed, 17 insertions(+), 22 deletions(-) diff --git a/backend/src/api/integration/helpers/discourseValidator.ts b/backend/src/api/integration/helpers/discourseValidator.ts index 469a92cdd8..00f3a0e6ee 100644 --- a/backend/src/api/integration/helpers/discourseValidator.ts +++ b/backend/src/api/integration/helpers/discourseValidator.ts @@ -10,7 +10,7 @@ export default async (req, res) => { Permissions.values.integrationEdit, ]) - const {apiKey, apiUsername, forumHostname} = req.body; + const {apiKey, apiUsername, forumHostname} = req.body if (apiKey && apiUsername && forumHostname) { try { diff --git a/backend/src/api/webhooks/discourse.ts b/backend/src/api/webhooks/discourse.ts index a641ed9cd2..f7812b7def 100644 --- a/backend/src/api/webhooks/discourse.ts +++ b/backend/src/api/webhooks/discourse.ts @@ -27,20 +27,20 @@ export default async (req, res) => { try { if (!signature) { - req.log.error({ signature }, 'Discourse Webhook signature header missing!'); - await req.responseHandler.success(req, res, 'Discourse Webhook signature header missing!', 200); - return; + req.log.error({ signature }, 'Discourse Webhook signature header missing!') + await req.responseHandler.success(req, res, 'Discourse Webhook signature header missing!', 200) + return } if (!verifyWebhookSignature(JSON.stringify(data), integration.settings.webhookSecret, signature)) { - req.log.error({ signature }, 'Discourse Webhook signature verification failed!'); - await req.responseHandler.success(req, res, 'Discourse Webhook signature verification failed!', 200); - return; + req.log.error({ signature }, 'Discourse Webhook signature verification failed!') + await req.responseHandler.success(req, res, 'Discourse Webhook signature verification failed!', 200) + return } } catch (error) { - req.log.error({ signature, error }, 'Internal error when verifying discourse webhook'); - await req.responseHandler.success(req, res, 'Internal error when verifying discourse webhook', 200); - return; + req.log.error({ signature, error }, 'Internal error when verifying discourse webhook') + await req.responseHandler.success(req, res, 'Internal error when verifying discourse webhook', 200) + return } diff --git a/backend/src/utils/crypto.ts b/backend/src/utils/crypto.ts index 14f2e51fcf..1dd1211225 100644 --- a/backend/src/utils/crypto.ts +++ b/backend/src/utils/crypto.ts @@ -1,8 +1,8 @@ -import * as crypto from 'crypto'; -import * as buffer from 'buffer'; +import * as crypto from 'crypto' +import * as buffer from 'buffer' export function generateWebhookSecret(length: number = 32): string { - return crypto.randomBytes(length).toString('hex'); + return crypto.randomBytes(length).toString('hex') } export function verifyWebhookSignature( @@ -11,18 +11,13 @@ export function verifyWebhookSignature( signatureHeader: string ): boolean { - console.log("payload", payload) - const hmac = crypto.createHmac('sha256', secret); - console.log("hmac", hmac) - hmac.update(payload); - console.log("hmac", hmac) - const expectedSignature = `sha256=${hmac.digest('hex')}`; - - console.log("expectedSignature", expectedSignature) + const hmac = crypto.createHmac('sha256', secret) + hmac.update(payload) + const expectedSignature = `sha256=${hmac.digest('hex')}` return crypto.timingSafeEqual( buffer.Buffer.from(signatureHeader), buffer.Buffer.from(expectedSignature) - ); + ) } From bf171f435b69d48efebdffebae9c10d5704ad0c9 Mon Sep 17 00:00:00 2001 From: garrrikkotua Date: Thu, 27 Apr 2023 12:46:49 +0300 Subject: [PATCH 07/32] webhooks --- backend/src/api/webhooks/discourse.ts | 2 + .../integrations/grid/discourseGrid.ts | 16 +- .../discourseIntegrationService.ts | 271 +++++++++++++++--- .../integrations/types/discourseTypes.ts | 120 ++++++++ backend/src/types/activityTypes.ts | 6 +- 5 files changed, 374 insertions(+), 41 deletions(-) diff --git a/backend/src/api/webhooks/discourse.ts b/backend/src/api/webhooks/discourse.ts index f7812b7def..71047683fb 100644 --- a/backend/src/api/webhooks/discourse.ts +++ b/backend/src/api/webhooks/discourse.ts @@ -11,6 +11,7 @@ export default async (req, res) => { const signature = req.headers['x-discourse-event-signature'] const eventId = req.headers['x-discourse-event-id'] const eventType = req.headers['x-discourse-event-type'] + const event = req.headers['x-discourse-event'] const data = req.body const integrationId = req.params.integrationId as string @@ -56,6 +57,7 @@ export default async (req, res) => { signature, eventId, eventType, + event, data, }, }) diff --git a/backend/src/serverless/integrations/grid/discourseGrid.ts b/backend/src/serverless/integrations/grid/discourseGrid.ts index f14307b7ab..f2234a77af 100644 --- a/backend/src/serverless/integrations/grid/discourseGrid.ts +++ b/backend/src/serverless/integrations/grid/discourseGrid.ts @@ -1,13 +1,23 @@ import { gridEntry } from './grid' export class DiscourseGrid { - static post: gridEntry = { + static create_topic: gridEntry = { score: 8, - isContribution: false, + isContribution: true, } - static reply: gridEntry = { + static message_in_topic: gridEntry = { score: 6, + isContribution: true, + } + + static join: gridEntry = { + score: 3, + isContribution: false, + } + + static like: gridEntry = { + score: 1, isContribution: false, } } diff --git a/backend/src/serverless/integrations/services/integrations/discourseIntegrationService.ts b/backend/src/serverless/integrations/services/integrations/discourseIntegrationService.ts index 9b2e027b50..f32fb384d8 100644 --- a/backend/src/serverless/integrations/services/integrations/discourseIntegrationService.ts +++ b/backend/src/serverless/integrations/services/integrations/discourseIntegrationService.ts @@ -8,7 +8,7 @@ import { IPendingStream, IProcessStreamResults, IStepContext, -} from '../../../../types/integration/stepResult' + IProcessWebhookResults } from '../../../../types/integration/stepResult' import { DiscourseConnectionParams, DiscourseCategoryResponse, @@ -17,7 +17,11 @@ import { DiscoursePostsInput, DiscoursePostsFromTopicResponse, DiscoursePostsByIdsInput, - DiscoursePostsByIdsResponse + DiscoursePostsByIdsResponse, + DiscourseWebhookPost, + DiscourseWebhookUser, + DiscourseUserResponse, + DiscourseWebhookNotification } from '../../types/discourseTypes' import { getDiscourseCategories } from '../../usecases/discourse/getCategories' import { getDiscourseTopics } from '../../usecases/discourse/getTopics' @@ -39,6 +43,12 @@ enum DiscourseStreamType { POSTS_BY_IDS = 'postsByIds', } +enum DiscourseWebhookType { + POST_CREATED = 'post_created', + USER_CREATED = 'user_created', + LIKED_A_POST = 'notification_created', +} + interface DiscourseProcessResult { type: DiscourseStreamType data: @@ -188,48 +198,28 @@ export class DiscourseIntegrationService extends IntegrationServiceBase { context.logger, ) - const member: Member = { - username: { - [PlatformType.DISCOURSE]: post.username, - }, - displayName: user.user.name, - attributes: { - [MemberAttributeName.URL]: { - [PlatformType.DISCOURSE]: `https://${context.pipelineData.forumHostname}/u/${post.username}`, - }, - [MemberAttributeName.WEBSITE_URL]: { - [PlatformType.DISCOURSE]: user.user.website || '', - }, - [MemberAttributeName.LOCATION]: { - [PlatformType.DISCOURSE]: user.user.location || '', - }, - [MemberAttributeName.BIO]: { - [PlatformType.DISCOURSE]: sanitizeHtml(he.decode(user.user.bio_cooked)) || '', - }, - [MemberAttributeName.AVATAR_URL]: { - [PlatformType.DISCOURSE]: - `https://${context.pipelineData.forumHostname}${user.user.avatar_template.replace( - '{size}', - '200', - )}` || '', - }, - }, - emails: user.user.email ? [user.user.email] : [], - } - + const member = DiscourseIntegrationService.parseUserIntoMember(user, context.pipelineData.forumHostname) + const activity: AddActivitiesSingle = { member, platform: PlatformType.DISCOURSE, tenant: context.integration.tenantId, - sourceId: `${post.id}`, - sourceParentId: lastIdInPreviousBatch || null, - type: post.post_number === 1 ? DiscourseActivityType.POST : DiscourseActivityType.REPLY, + sourceId: `${topicId}-${post.post_number}`, + sourceParentId: post.post_number === 1 ? null : `${topicId}-${post.post_number - 1}`, + type: post.post_number === 1 ? DiscourseActivityType.CREATE_TOPIC : DiscourseActivityType.MESSAGE_IN_TOPIC, timestamp: moment(post.created_at).utc().toDate(), body: sanitizeHtml(he.decode(post.cooked)), title: post.post_number === 1 ? stream.metadata.topicTitle : null, url: `https://${context.pipelineData.forumHostname}/t/${stream.metadata.topicSlug}/${topicId}/${post.post_number}`, - score: post.post_number === 1 ? DiscourseGrid[DiscourseActivityType.POST].score : DiscourseGrid[DiscourseActivityType.REPLY].score, - isContribution: post.post_number === 1 ? DiscourseGrid[DiscourseActivityType.POST].isContribution : DiscourseGrid[DiscourseActivityType.REPLY].isContribution, + channel: stream.metadata.topicTitle, + score: + post.post_number === 1 + ? DiscourseGrid[DiscourseActivityType.CREATE_TOPIC].score + : DiscourseGrid[DiscourseActivityType.MESSAGE_IN_TOPIC].score, + isContribution: + post.post_number === 1 + ? DiscourseGrid[DiscourseActivityType.CREATE_TOPIC].isContribution + : DiscourseGrid[DiscourseActivityType.MESSAGE_IN_TOPIC].isContribution, } activities.push(activity) } @@ -337,4 +327,213 @@ export class DiscourseIntegrationService extends IntegrationServiceBase { data: discoursePosts, } } + + async processWebhook(webhook: any, context: IStepContext): Promise { + const { event, data } = webhook.payload + + switch (event) { + case DiscourseWebhookType.POST_CREATED: + return this.processPostCreatedWebhook(data as DiscourseWebhookPost, context) + case DiscourseWebhookType.USER_CREATED: + return this.processUserCreatedWebhook(data as DiscourseWebhookUser, context) + case DiscourseWebhookType.LIKED_A_POST: + const localData = data as DiscourseWebhookNotification + if (localData.notification.notification_type === 5) { + return this.processLikedAPostWebhook(localData, context) + } + break + default: + context.logger.warn( + { + event, + data, + }, + 'No record created for event!', + ) + + return { + operations: [], + } + } + + return { + operations: [], + } + } + + async processPostCreatedWebhook( + data: DiscourseWebhookPost, + context: IStepContext, + ): Promise { + const post = data.post + const user = await getDiscourseUserByUsername( + { + forumHostname: context.integration.settings.forumHostname, + apiKey: context.integration.settings.apiKey, + apiUsername: context.integration.settings.apiUsername, + }, + { username: post.username }, + context.logger, + ) + + const member = DiscourseIntegrationService.parseUserIntoMember( + user, + context.integration.settings.forumHostname, + ) + + const activity: AddActivitiesSingle = { + member, + platform: PlatformType.DISCOURSE, + tenant: context.integration.tenantId, + sourceId: `${post.id}`, + sourceParentId: post.post_number === 1 ? null : `${post.topic_id}-${post.post_number - 1}`, + type: post.post_number === 1 ? DiscourseActivityType.CREATE_TOPIC : DiscourseActivityType.MESSAGE_IN_TOPIC, + timestamp: moment(post.created_at).utc().toDate(), + body: sanitizeHtml(he.decode(post.cooked)), + title: post.post_number === 1 ? post.topic_title : null, + url: `https://${context.pipelineData.forumHostname}/t/${post.topic_slug}/${post.topic_id}/${post.post_number}`, + channel: post.topic_title, + score: + post.post_number === 1 + ? DiscourseGrid[DiscourseActivityType.CREATE_TOPIC].score + : DiscourseGrid[DiscourseActivityType.MESSAGE_IN_TOPIC].score, + isContribution: + post.post_number === 1 + ? DiscourseGrid[DiscourseActivityType.CREATE_TOPIC].isContribution + : DiscourseGrid[DiscourseActivityType.MESSAGE_IN_TOPIC].isContribution, + } + + return { + operations: [ + { + type: Operations.UPSERT_ACTIVITIES_WITH_MEMBERS, + records: [activity], + } + ] + } + + } + + async processUserCreatedWebhook( + data: DiscourseWebhookUser, + context: IStepContext, + ): Promise { + const user = data.user + const member = DiscourseIntegrationService.parseUserIntoMember( + { + user: user as any, + user_badges: [], + badges: [], + badge_types: [], + users: [], + }, + context.integration.settings.forumHostname, + ) + + const activity: AddActivitiesSingle = { + member, + platform: PlatformType.DISCOURSE, + tenant: context.integration.tenantId, + sourceId: `${user.id}`, + type: DiscourseActivityType.JOIN, + timestamp: moment(user.created_at).utc().toDate(), + body: null, + title: null, + url: `https://${context.pipelineData.forumHostname}/u/${user.username}`, + channel: null, + score: DiscourseGrid[DiscourseActivityType.JOIN].score, + isContribution: DiscourseGrid[DiscourseActivityType.JOIN].isContribution, + } + + return { + operations: [ + { + type: Operations.UPSERT_ACTIVITIES_WITH_MEMBERS, + records: [activity], + } + ] + } + } + + async processLikedAPostWebhook( + data: DiscourseWebhookNotification, + context: IStepContext, + ): Promise { + const notification = data.notification + const username = notification.data.username + const user = await getDiscourseUserByUsername( + { + forumHostname: context.integration.settings.forumHostname, + apiKey: context.integration.settings.apiKey, + apiUsername: context.integration.settings.apiUsername, + }, + { username }, + context.logger, + ) + + const member = DiscourseIntegrationService.parseUserIntoMember( + { + user: user as any, + user_badges: [], + badges: [], + badge_types: [], + users: [], + }, + context.integration.settings.forumHostname, + ) + + const activity: AddActivitiesSingle = { + member, + platform: PlatformType.DISCOURSE, + tenant: context.integration.tenantId, + sourceId: `${notification.id}`, + type: DiscourseActivityType.LIKE, + timestamp: moment(notification.created_at).utc().toDate(), + body: null, + title: null, + score: DiscourseGrid[DiscourseActivityType.LIKE].score, + isContribution: DiscourseGrid[DiscourseActivityType.LIKE].isContribution, + } + + return { + operations: [ + { + type: Operations.UPSERT_ACTIVITIES_WITH_MEMBERS, + records: [activity], + }, + ], + } + } + + + static parseUserIntoMember(user: DiscourseUserResponse, forumHostname: string): Member { + return { + username: { + [PlatformType.DISCOURSE]: user.user.username, + }, + displayName: user.user.name, + attributes: { + [MemberAttributeName.URL]: { + [PlatformType.DISCOURSE]: `https://${forumHostname}/u/${user.user.username}`, + }, + [MemberAttributeName.WEBSITE_URL]: { + [PlatformType.DISCOURSE]: user.user.website || '', + }, + [MemberAttributeName.LOCATION]: { + [PlatformType.DISCOURSE]: user.user.location || '', + }, + [MemberAttributeName.BIO]: { + [PlatformType.DISCOURSE]: sanitizeHtml(he.decode(user.user.bio_cooked)) || '', + }, + [MemberAttributeName.AVATAR_URL]: { + [PlatformType.DISCOURSE]: + `https://${forumHostname}${user.user.avatar_template.replace( + '{size}', + '200', + )}` || '', + }, + }, + emails: user.user.email ? [user.user.email] : [], + } +} } diff --git a/backend/src/serverless/integrations/types/discourseTypes.ts b/backend/src/serverless/integrations/types/discourseTypes.ts index e84108a467..e9ecf39770 100644 --- a/backend/src/serverless/integrations/types/discourseTypes.ts +++ b/backend/src/serverless/integrations/types/discourseTypes.ts @@ -40,6 +40,125 @@ export interface DiscoursePostsByIdsResponse { id: number // this is the id of the topic } +export interface DiscourseWebhookPost { + post: Post +} + +export interface DiscourseWebhookNotification { + notification: { + id: number; + user_id: number; + notification_type: number; + read: boolean; + created_at: string; + post_number: number; + topic_id: number; + slug: string; + data: { + group_id: number; + group_name: string; + inbox_count: number; + username: string; + } + } +} + +export interface DiscourseWebhookUser { + user: { + id: number; + username: string; + name: string; + avatar_template: string; + email: string; + secondary_emails: any; + last_posted_at: any; + last_seen_at: string; + created_at: string; + muted: boolean; + trust_level: number; + moderator: boolean; + admin: boolean; + title: any; + badge_count: number; + time_read: number; + recent_time_read: number; + primary_group_id: any; + primary_group_name: any; + primary_group_flair_url: any; + primary_group_flair_bg_color: any; + primary_group_flair_color: any; + featured_topic: any; + staged: boolean; + pending_count: number; + profile_view_count: number; + second_factor_enabled: boolean; + can_upload_profile_header: boolean; + can_upload_user_card_background: boolean; + post_count: number; + locale: any; + muted_category_ids: any; + regular_category_ids: any; + watched_tags: any; + watching_first_post_tags: any; + tracked_tags: any; + muted_tags: any; + tracked_category_ids: any; + watched_category_ids: any; + watched_first_post_category_ids: any; + system_avatar_template: string; + muted_usernames: any; + ignored_usernames: any; + allowed_pm_usernames: any; + mailing_list_posts_per_day: number; + featured_user_badge_ids: any; + invited_by: any; + groups: any; + user_option: any; + }; +} + +interface CreatedByLastPoster { + id: number; + username: string; + name: string; + avatar_template: string; +} + +export interface DiscourseWebhookTopic { + id: number; + title: string; + fancy_title: string; + posts_count: number; + created_at: string; + views: number; + reply_count: number; + like_count: number; + last_posted_at: string; + visible: boolean; + closed: boolean; + archived: boolean; + archetype: string; + slug: string; + category_id: number; + word_count: number; + deleted_at: null; + user_id: number; + featured_link: string; + pinned_globally: boolean; + pinned_at: string; + pinned_until: string; + unpinned: string; + pinned: boolean; + highest_post_number: number; + deleted_by: any; + has_deleted: boolean; + bookmarked: boolean; + participant_count: number; + thumbnails: any; + created_by: CreatedByLastPoster; + last_poster: CreatedByLastPoster; +} + export interface DiscourseCategory { id: number name: string @@ -257,6 +376,7 @@ interface Post { yours: boolean; topic_id: number; topic_slug: string; + topic_title?: string; display_username: string; primary_group_name: string; flair_name: string; diff --git a/backend/src/types/activityTypes.ts b/backend/src/types/activityTypes.ts index 50d7aa3c09..fa22a7f311 100644 --- a/backend/src/types/activityTypes.ts +++ b/backend/src/types/activityTypes.ts @@ -98,8 +98,10 @@ export enum StackOverflowActivityType { } export enum DiscourseActivityType { - POST = 'post', - REPLY = 'reply', + CREATE_TOPIC = 'create_topic', + MESSAGE_IN_TOPIC = 'message_in_topic', + JOIN = 'join', + LIKE = 'like', } const githubUrl = 'https://github.com' From 00d2ac1fd1513da9daa6e2977c2a6ce6982a7407 Mon Sep 17 00:00:00 2001 From: garrrikkotua Date: Thu, 27 Apr 2023 16:27:39 +0300 Subject: [PATCH 08/32] small bug fixed for backend --- .../services/integrationProcessor.ts | 2 ++ .../discourseIntegrationService.ts | 14 +++------ .../integrations/types/discourseTypes.ts | 31 ++++++++++++------- backend/src/services/integrationService.ts | 2 +- 4 files changed, 27 insertions(+), 22 deletions(-) diff --git a/backend/src/serverless/integrations/services/integrationProcessor.ts b/backend/src/serverless/integrations/services/integrationProcessor.ts index 1b9e2b47fa..23b74dad86 100644 --- a/backend/src/serverless/integrations/services/integrationProcessor.ts +++ b/backend/src/serverless/integrations/services/integrationProcessor.ts @@ -30,6 +30,7 @@ import { TwitterReachIntegrationService } from './integrations/twitterReachInteg import { SlackIntegrationService } from './integrations/slackIntegrationService' import { GithubIntegrationService } from './integrations/githubIntegrationService' import { StackOverlflowIntegrationService } from './integrations/stackOverflowIntegrationService' +import {DiscourseIntegrationService} from './integrations/discourseIntegrationService' import { LoggingBase } from '../../../services/loggingBase' import { API_CONFIG } from '../../../config' import EmailSender from '../../../services/emailSender' @@ -77,6 +78,7 @@ export class IntegrationProcessor extends LoggingBase { new SlackIntegrationService(), new GithubIntegrationService(), new StackOverlflowIntegrationService(), + new DiscourseIntegrationService(), ] // add premium integrations diff --git a/backend/src/serverless/integrations/services/integrations/discourseIntegrationService.ts b/backend/src/serverless/integrations/services/integrations/discourseIntegrationService.ts index f32fb384d8..46c3a13294 100644 --- a/backend/src/serverless/integrations/services/integrations/discourseIntegrationService.ts +++ b/backend/src/serverless/integrations/services/integrations/discourseIntegrationService.ts @@ -460,7 +460,8 @@ export class DiscourseIntegrationService extends IntegrationServiceBase { context: IStepContext, ): Promise { const notification = data.notification - const username = notification.data.username + const username = notification.data.username ? notification.data.username : notification.data.original_username + const channel = notification.fancy_title ? notification.fancy_title : notification.data.topic_title const user = await getDiscourseUserByUsername( { forumHostname: context.integration.settings.forumHostname, @@ -472,13 +473,7 @@ export class DiscourseIntegrationService extends IntegrationServiceBase { ) const member = DiscourseIntegrationService.parseUserIntoMember( - { - user: user as any, - user_badges: [], - badges: [], - badge_types: [], - users: [], - }, + user, context.integration.settings.forumHostname, ) @@ -491,6 +486,7 @@ export class DiscourseIntegrationService extends IntegrationServiceBase { timestamp: moment(notification.created_at).utc().toDate(), body: null, title: null, + channel, score: DiscourseGrid[DiscourseActivityType.LIKE].score, isContribution: DiscourseGrid[DiscourseActivityType.LIKE].isContribution, } @@ -523,7 +519,7 @@ export class DiscourseIntegrationService extends IntegrationServiceBase { [PlatformType.DISCOURSE]: user.user.location || '', }, [MemberAttributeName.BIO]: { - [PlatformType.DISCOURSE]: sanitizeHtml(he.decode(user.user.bio_cooked)) || '', + [PlatformType.DISCOURSE]: user.user.bio_cooked ? sanitizeHtml(he.decode(user.user.bio_cooked)) : '', }, [MemberAttributeName.AVATAR_URL]: { [PlatformType.DISCOURSE]: diff --git a/backend/src/serverless/integrations/types/discourseTypes.ts b/backend/src/serverless/integrations/types/discourseTypes.ts index e9ecf39770..492b0a021e 100644 --- a/backend/src/serverless/integrations/types/discourseTypes.ts +++ b/backend/src/serverless/integrations/types/discourseTypes.ts @@ -46,19 +46,26 @@ export interface DiscourseWebhookPost { export interface DiscourseWebhookNotification { notification: { - id: number; - user_id: number; - notification_type: number; - read: boolean; - created_at: string; - post_number: number; - topic_id: number; - slug: string; + id: number + user_id: number + notification_type: number + read: boolean + created_at: string + post_number: number + topic_id: number + slug: string + fancy_title?: string data: { - group_id: number; - group_name: string; - inbox_count: number; - username: string; + group_id: number + group_name: string + inbox_count: number + username?: string + topic_title?: string + original_post_id?: number + original_post_type?: number + original_username?: string + revision_number: any + display_username?: string } } } diff --git a/backend/src/services/integrationService.ts b/backend/src/services/integrationService.ts index f250a1caaf..ba487dbc81 100644 --- a/backend/src/services/integrationService.ts +++ b/backend/src/services/integrationService.ts @@ -794,7 +794,7 @@ export default class IntegrationService { platform: PlatformType.DISCOURSE, settings: { apiKey: integrationData.apiKey, - apiUserame: integrationData.apiUsername, + apiUsername: integrationData.apiUsername, forumHostname: integrationData.forumHostname, webhookSecret, // we can do encryption here updateMemberAttributes: true, From c0a357b98027af41c2d3bbf517299982ed561440 Mon Sep 17 00:00:00 2001 From: garrrikkotua Date: Mon, 15 May 2023 14:42:47 +0300 Subject: [PATCH 09/32] update member stricture --- .../discourseIntegrationService.ts | 28 +++++++++++++------ 1 file changed, 19 insertions(+), 9 deletions(-) diff --git a/backend/src/serverless/integrations/services/integrations/discourseIntegrationService.ts b/backend/src/serverless/integrations/services/integrations/discourseIntegrationService.ts index 46c3a13294..f9b9b4687f 100644 --- a/backend/src/serverless/integrations/services/integrations/discourseIntegrationService.ts +++ b/backend/src/serverless/integrations/services/integrations/discourseIntegrationService.ts @@ -35,6 +35,7 @@ import MemberAttributeSettingsService from '../../../../services/memberAttribute import { MemberAttributeName } from '../../../../database/attributes/member/enums' import { DiscourseActivityType } from '../../../../types/activityTypes' import { DiscourseGrid } from '../../grid/discourseGrid' +import type { PlatformIdentities } from '../../types/messageTypes' enum DiscourseStreamType { CATEGORIES = 'categories', @@ -198,10 +199,11 @@ export class DiscourseIntegrationService extends IntegrationServiceBase { context.logger, ) - const member = DiscourseIntegrationService.parseUserIntoMember(user, context.pipelineData.forumHostname) + const member = DiscourseIntegrationService.parseUserIntoMember(user, context.pipelineData.forumHostname, context) const activity: AddActivitiesSingle = { member, + username: member.username[PlatformType.DISCOURSE].username, platform: PlatformType.DISCOURSE, tenant: context.integration.tenantId, sourceId: `${topicId}-${post.post_number}`, @@ -379,10 +381,12 @@ export class DiscourseIntegrationService extends IntegrationServiceBase { const member = DiscourseIntegrationService.parseUserIntoMember( user, context.integration.settings.forumHostname, + context ) const activity: AddActivitiesSingle = { member, + username: member.username[PlatformType.DISCOURSE].username, platform: PlatformType.DISCOURSE, tenant: context.integration.tenantId, sourceId: `${post.id}`, @@ -428,10 +432,12 @@ export class DiscourseIntegrationService extends IntegrationServiceBase { users: [], }, context.integration.settings.forumHostname, + context ) const activity: AddActivitiesSingle = { member, + username: member.username[PlatformType.DISCOURSE].username, platform: PlatformType.DISCOURSE, tenant: context.integration.tenantId, sourceId: `${user.id}`, @@ -475,10 +481,12 @@ export class DiscourseIntegrationService extends IntegrationServiceBase { const member = DiscourseIntegrationService.parseUserIntoMember( user, context.integration.settings.forumHostname, + context ) const activity: AddActivitiesSingle = { member, + username: member.username[PlatformType.DISCOURSE].username, platform: PlatformType.DISCOURSE, tenant: context.integration.tenantId, sourceId: `${notification.id}`, @@ -502,11 +510,14 @@ export class DiscourseIntegrationService extends IntegrationServiceBase { } - static parseUserIntoMember(user: DiscourseUserResponse, forumHostname: string): Member { + static parseUserIntoMember(user: DiscourseUserResponse, forumHostname: string, context: IStepContext): Member { return { username: { - [PlatformType.DISCOURSE]: user.user.username, - }, + [PlatformType.DISCOURSE]: { + username: user.user.username, + integrationId: context.integration.id, + }, + } as PlatformIdentities, displayName: user.user.name, attributes: { [MemberAttributeName.URL]: { @@ -519,14 +530,13 @@ export class DiscourseIntegrationService extends IntegrationServiceBase { [PlatformType.DISCOURSE]: user.user.location || '', }, [MemberAttributeName.BIO]: { - [PlatformType.DISCOURSE]: user.user.bio_cooked ? sanitizeHtml(he.decode(user.user.bio_cooked)) : '', + [PlatformType.DISCOURSE]: user.user.bio_cooked + ? sanitizeHtml(he.decode(user.user.bio_cooked)) + : '', }, [MemberAttributeName.AVATAR_URL]: { [PlatformType.DISCOURSE]: - `https://${forumHostname}${user.user.avatar_template.replace( - '{size}', - '200', - )}` || '', + `https://${forumHostname}${user.user.avatar_template.replace('{size}', '200')}` || '', }, }, emails: user.user.email ? [user.user.email] : [], From 31e500132efcf24219aa71aef5125666e170618e Mon Sep 17 00:00:00 2001 From: garrrikkotua Date: Mon, 15 May 2023 20:38:19 +0300 Subject: [PATCH 10/32] discourse logo & integration card --- .../public/images/integrations/discourse.png | Bin 0 -> 5117 bytes frontend/src/integrations/discourse/config.js | 9 +++++++++ frontend/src/integrations/discourse/index.js | 3 +++ frontend/src/integrations/integrations-config.js | 3 ++- 4 files changed, 14 insertions(+), 1 deletion(-) create mode 100644 frontend/public/images/integrations/discourse.png create mode 100644 frontend/src/integrations/discourse/config.js create mode 100644 frontend/src/integrations/discourse/index.js diff --git a/frontend/public/images/integrations/discourse.png b/frontend/public/images/integrations/discourse.png new file mode 100644 index 0000000000000000000000000000000000000000..dd010b37fce7090925bebfea72066074b2a89f80 GIT binary patch literal 5117 zcmVuA^-pYCL$vMA0YqzvhrXezP`Ty|Fi(E_UaoW0JQf2q(}g% zO8}`s8Xq7ZA0QeP79GAEhybc}Vqjr0EiOhL zAe|i{yc{BZ9U)yFApF*l_oQXk8zb{`I5Hm~02d(ihfA;8i_hkn6?L8=Dlv!4c>}G~ z0jrS=lgR+B%5lVSaz2kZFFQcK~o`3r}{VUWrBk01_!lL_t(|oZX#?R})JX z#v8k#O+p}KX+jnVNPt98_B|+@;8WzWsVstQZ)TkRAh=Eb z_7bmmaH>2pAn-h&%@#8P4u%o`D^tv(-{l{c%H#8%LVAQZ)H*qVu`&p#MhF>%nVp#Q zjL`$Up{|*Ao-fu0(?J$_exTgt1)|${^W}#qXJ9fdG6->U(B-Rf6|c8zEyFuO)3W%m zJniBIxrx_0!}IWhte(X6$syOS$t}F;a>fNVDT;h)%AKon32$nko|-g=z-L`bVf}bR z<*aLo)1t_)xuG-l;!V%+Mc3KH5ctx3z<7fb9wg39znl;`ctdMm!%2V> z^*Etpys{r~65x#Uam^v#RMsOwv5E}u*ascq4VHNS;BfFIhc@U0Z^o~>)I#7VeZrd; ze5^|?vg=MXlp1fw=ekrw;LD!l%`>hJmvoV(??5`da!{$0Lf~6H#Ow82FwH?*pcHuX zS>FZI9N7|;$HVc)15ce8LWV{`!FYiwrYUG<7$9EnKp=wS2tZMl}zSxePZNtxnwHk3Yr)hdwr>mR}Wn(nd=LMQZHs-4nN-5n48EVoB5>%-n0`s#P)P}u_~F4ar5PNC|4&RKkg8(*TE2-4CP)f+M}t6 z&6_Vnef6b~uX!7yI^F>FoS6)r&cWFf!i^W9y4sMhwE1jjo($e@|V!pwI&hZ7?@_5}6dZ>jlAD|a2< z4D3`#Xs$M0_Y+5DsRvv_GWxq8DGkD84kX?XEO}`CQ;RMic~v)D6E4wbh>v!FF{Y2i zgC#FFtEV#>-D`W3gGF>~r~4D-R;aL66%UrY6w_RTF5j}`M(U)%3H(HvKYqFz9xQnV zSuvz;r^%(%NfG`1A3o77APni#@LGp4PdDmb>SVs-KYpUqPkr}kcx6cPQZMv$ zdR1|;Fx6oH^k*u*OxYR_l)TVlRl0m+%+3!T4bd#&)MAOlP8C7C?ktdf7@ z1s&o$j{o&1nvJ{cxD_7cmV#b2b#rFkWl_=Mq=WskPJzcT7I=e@+6--w$3F6i!ax+2JEF z-$Mf)WTB_)JY8ve;|L@l%d&|N`{4T2?iev()6=c;2)gc9Uek%<8DDL??g!#&lZ%8m zVPBi6ZR^r>f6lbx>pj1;aV`dJzjYFiQdqR)C)@4m4P^ah8h#%QHKr;f>qh6iUuoCC{(YL=nZ+ znYd{KmZ!2O!N6>xPpeHWdH$%T6ydixeOGN92OG|^Djwtz3!$6Zz|!scxI*Az?PL>2 zTx=i2CtCvpTI!Jlng6;Xg{YmMQTjmYB9GM!Xd=rETh->6pLEd+vHLibKFI7THy;CX zJz}v?2`G7D^RelVZCngCn<;pJwNRhddPL{>NhdcS8my5yK+-2U9|IXUQ%}{}RE5Ro z)lDA<76nh#7B-IXVk>~rMUe0SM?JKyceV8*ekV`dalzI34rT8aATHuL z0_Fi=2RLN1h5V@B$&;m1TsCmA^MXAdzSGBmR_bc&1u>5)anhkSA4FWlBc2T4ww~q- zYU{o6y*3r;6vt|gvtJj9V;KYS1$Cvav|a$Uso=2*whRo~k*>o*6bi2Cy7bXEH@NeV;dVc%pOynh&oT7{Er02`>^v@)SHa zAv(jYtu^5N2&QC9rxmA+=l?jZ$HR17tYC{bkMIU4@MK>I#Pw8Zk&KUv{bI+5S9~D6 zmwG(^R|{f96})#Mp6#juPh|{XqE>BP!3#uL6kAvDY_$dNR5K50@MPf(Bze+Spfa{_ zob8Sa6i?md90_n{QC18p-mPu+#({W@9?!?U_ZoOy!RE4yKs!YS`10q|O(c*+g1 zAn_Dsk(RR%ym=^|vSw%r4o?}V<+}_JylE(&GL}_>lNPo0q^%?fUN00+nOlOxQ%#*s z{L0D3V1rkLGgGVw`aAYLj( zNM`^m2Gtf+;j|s8#ru~)Jmtt-Q1?%?+DxCxc%01@i$`*ycye`)1hfyRPGst8&vrR0 zK5PTUleJT9zHa~NRNa)f0ReXz#Fr3&@%p4yLnM& z=M7WpYD zDm>{iDSwS!8>Y#5)85p(=<%e{(6@VP#qHVI@UXIsM7+Iy z8E(4J(Z#O_ILOl{x)&VN?X_%*iy<#vA$-Cb=$1)3`=_4u?>DBy+d{SJsVf*ng^jZ$ z9pu>){mav7@bJ0Lk%)(`5Ds6|q~idObQd2u#ly-{@z52byR1I+l?i%VOLp8b9#)o! zXRZ)FF{F-7-|12p+XuYNB=f*A9#WQ(h)1pv4qqDJ*u<)y4)Qd5h@!LnjX9_00h;vYBn!aa4b9Z*L8R81zs-MlUVB7uDVS;1R>`|K&I>G#{ zNE*DY1zT4@WV;u~drh{4=-=Sv(uz$?ossSU^!_J{Z)L&pQpxs_+OB|@_KxutcXV7_ zo`kD?{RF4?WcRHl8-iQ?V1SF=rV5Y?wE7qd#mR`-_FArhu6d7WBXsRwjxYC4n!?7| z+q*=6&XNtmU76}Kc31$>S$@g}j+Yx=sOky^HjNQeUK}64eS5ik`S$qu;zIg8>xd`V z8&h4vUX8?5`4ABtuWxRoiYp+tw+dT4o=zZ3j+;5FQfZkp!=cqVkj_F^sPIBtmaZVN zNpGu2ClYt>Kce3Gc$Y}mhB30G6RSsUX1LpP=qqupKY*J`ZCq(UuT7%=iJm>K^iwu($cGEM;clT2X}jWxPKc{a@fYbyZL;V zNF>r}geeF5HToTP^$hi?H5=|anip+be!h(g@9UUq0=pOJ@~%)liWetQafoVv<@}(( zr?aQ0^O}pPSBn48^3l!N+2_xzt9N&+tDoPW-F(EgTm=o+YL~a znC%}(A0-iyvURw>a(#Z#^Qg0@UsZzs{uKgHhSWtBFd42NUY9jLtP4IW{`Vz3W9|L} zi%1M#-66vIMM_q`#az+fvvM1&!B8FUcBO{~uI6Wvp?E+3GaB9d7cwstsZ{LIh6pM(S&T|+cX8Wb5hx%`*^~GoUV>|Rw=y`Ooa>&K1Gvfd2aOM0FJsXl0 z)Bc0ccqj_}o&D!4`?r#ZD4@!(kwl#rI>FUd_V*7_+@k;N zV@F}IMird?)aoCz_8Cu#0-{9GiK7QgZ=tRqM}F1iKH#|6@dQ3QQ+LbVQ#>KdPtdJb8wW4L^Al4p73KZH6A;>Zxz%O1`+z58vV68w?s7#a z?-8CL6tg_fm&#Lv?q^NKL;rSa@V-S>7%0t5P7QjKGXk%RB3fOD`T6<5p Date: Tue, 16 May 2023 18:57:12 +0300 Subject: [PATCH 11/32] wip frontend --- .../integration/helpers/discourseValidator.ts | 10 +- .../components/discourse-connect-drawer.vue | 247 ++++++++++++++++++ .../components/discourse-connect.vue | 20 ++ frontend/src/integrations/discourse/config.js | 5 +- .../integration/integration-service.js | 81 +++--- 5 files changed, 315 insertions(+), 48 deletions(-) create mode 100644 frontend/src/integrations/discourse/components/discourse-connect-drawer.vue create mode 100644 frontend/src/integrations/discourse/components/discourse-connect.vue diff --git a/backend/src/api/integration/helpers/discourseValidator.ts b/backend/src/api/integration/helpers/discourseValidator.ts index 00f3a0e6ee..a25b645d5d 100644 --- a/backend/src/api/integration/helpers/discourseValidator.ts +++ b/backend/src/api/integration/helpers/discourseValidator.ts @@ -12,10 +12,15 @@ export default async (req, res) => { const {apiKey, apiUsername, forumHostname} = req.body + console.log('apiKey', apiKey) + console.log('apiUsername', apiUsername) + console.log('forumHostname', forumHostname) + if (apiKey && apiUsername && forumHostname) { + console.log('here inside if') try { const result = await axios.get( - `https://${forumHostname}/admin/users/list/active.json`, + `${forumHostname}/admin/users/list/active.json`, { headers: { 'Api-Key': apiKey, @@ -28,11 +33,14 @@ export default async (req, res) => { result.data && result.data.length > 0 ) { + console.log('here inside if - request success') return req.responseHandler.success(req, res, result.data) } } catch (e) { + console.log('here inside catch') return req.responseHandler.error(req, res, new Error400(req.language)) } } + console.log('here inside else') return req.responseHandler.error(req, res, new Error400(req.language)) } diff --git a/frontend/src/integrations/discourse/components/discourse-connect-drawer.vue b/frontend/src/integrations/discourse/components/discourse-connect-drawer.vue new file mode 100644 index 0000000000..ad823092fc --- /dev/null +++ b/frontend/src/integrations/discourse/components/discourse-connect-drawer.vue @@ -0,0 +1,247 @@ +