From e2e1b10dcdf4ab247ce128e5ef5b581f684aebd5 Mon Sep 17 00:00:00 2001 From: Afrisal Yodi Purnama Date: Wed, 25 May 2022 13:54:58 +0700 Subject: [PATCH 1/9] #444 send message to kafka if token is not valid. --- config/default.js | 6 +++ index.js | 4 +- services/GithubService.js | 27 +++++++++---- services/GitlabService.js | 62 ++++++++++++++++++---------- utils/errors.js | 19 +++++++-- utils/git-helper.js | 14 +++---- utils/kafka-consumer.js | 85 +++++++++++++++++++++++++++++++++++++++ utils/kafka-sender.js | 61 ++++++++++++++++++++++++++++ utils/kafka.js | 81 +------------------------------------ utils/notification.js | 49 ++++++++++++++++++++++ 10 files changed, 285 insertions(+), 123 deletions(-) create mode 100644 utils/kafka-consumer.js create mode 100644 utils/kafka-sender.js create mode 100644 utils/notification.js diff --git a/config/default.js b/config/default.js index 2256a15..632c5ca 100644 --- a/config/default.js +++ b/config/default.js @@ -16,6 +16,7 @@ module.exports = { LOG_LEVEL: process.env.LOG_LEVEL || 'debug', PARTITION: process.env.PARTITION || 0, TOPIC: process.env.TOPIC || 'tc-x-events', + TOPIC_NOTIFICATION: process.env.TOPIC_NOTIFICATION || 'notifications.action.create', KAFKA_OPTIONS: { connectionString: process.env.KAFKA_URL || 'localhost:9092', groupId: process.env.KAFKA_GROUP_ID || 'topcoder-x-processor', @@ -25,6 +26,11 @@ module.exports = { passphrase: 'secret', // NOTE:* This configuration specifies the private key passphrase used while creating it. } }, + MAIL_NOTICIATION: { + type: 'tcx.mail_notification', + sendgridTemplateId: 'xxxxxx', + subject: 'Topcoder X Alert' + }, NEW_CHALLENGE_TEMPLATE: process.env.NEW_CHALLENGE_TEMPLATE || { status: 'Draft' }, diff --git a/index.js b/index.js index 3eed2a5..ee57023 100644 --- a/index.js +++ b/index.js @@ -5,7 +5,7 @@ const config = require('config'); const _ = require('lodash'); -const kafka = require('./utils/kafka'); +const kafkaConsumer = require('./utils/kafka-consumer'); const logger = require('./utils/logger'); process.on('uncaughtException', (err) => { @@ -55,4 +55,4 @@ dumpConfigs(config, 0); logger.debug('--- End of List of Configurations ---'); // run the server -kafka.run(); +kafkaConsumer.run(); diff --git a/services/GithubService.js b/services/GithubService.js index a53bc36..6f2eafe 100644 --- a/services/GithubService.js +++ b/services/GithubService.js @@ -53,7 +53,7 @@ async function _authenticate(accessToken) { }); return octokit.rest; } catch (err) { - throw errors.convertGitHubError(err, 'Failed to authenticate to Github using access token of copilot.'); + throw errors.handleGitHubError(err, 'Failed to authenticate to Github using access token of copilot.'); } } @@ -75,7 +75,7 @@ async function _removeAssignees(github, owner, repo, number, assignees) { assignees }); } catch (err) { - throw errors.convertGitHubError(err, 'Error occurred during remove assignees from issue.'); + throw errors.handleGitHubError(err, 'Error occurred during remove assignees from issue.'); } } @@ -93,6 +93,17 @@ async function _getUsernameById(id) { return user ? user.login : null; } +/** + * Get github issue url + * @param {String} repoPath the repo path + * @param {Number} number the issue number + * @returns {String} the url + * @private + */ +function _getIssueUrl(repoPath, number) { + return `https://github.com/${repoPath}/issues/${number}`; +} + /** * updates the title of github issue * @param {Object} copilot the copilot @@ -107,7 +118,7 @@ async function updateIssue(copilot, repoFullName, number, title) { try { await github.issues.update({owner, repo, issue_number: number, title}); } catch (err) { - throw errors.convertGitHubError(err, 'Error occurred during updating issue.'); + throw errors.handleGitHubError(err, 'Error occurred during updating issue.', copilot.topcoderUsername, _getIssueUrl(repoFullName, number)); } logger.debug(`Github issue title is updated for issue number ${number}`); } @@ -139,7 +150,7 @@ async function assignUser(copilot, repoFullName, number, user) { } await github.issues.addAssignees({owner, repo, issue_number: number, assignees: [user]}); } catch (err) { - throw errors.convertGitHubError(err, 'Error occurred during assigning issue user.'); + throw errors.handleGitHubError(err, 'Error occurred during assigning issue user.', copilot.topcoderUsername, _getIssueUrl(repoFullName, number)); } logger.debug(`Github issue with number ${number} is assigned to ${user}`); } @@ -184,7 +195,7 @@ async function createComment(copilot, repoFullName, number, body) { body = helper.prepareAutomatedComment(body, copilot); await github.issues.createComment({owner, repo, issue_number: number, body}); } catch (err) { - throw errors.convertGitHubError(err, 'Error occurred during creating comment on issue.'); + throw errors.handleGitHubError(err, 'Error occurred during creating comment on issue.', copilot.topcoderUsername, _getIssueUrl(repoFullName, number)); } logger.debug(`Github comment is added on issue with message: "${body}"`); } @@ -262,7 +273,7 @@ async function markIssueAsPaid(copilot, repoFullName, number, challengeUUID, exi const body = helper.prepareAutomatedComment(commentMessage, copilot); await github.issues.createComment({owner, repo, issue_number: number, body}); } catch (err) { - throw errors.convertGitHubError(err, 'Error occurred during updating issue as paid.'); + throw errors.handleGitHubError(err, 'Error occurred during updating issue as paid.', copilot.topcoderUsername, _getIssueUrl(repoFullName, number)); } logger.debug(`Github issue title is updated for as paid and fix accepted for ${number}`); } @@ -291,7 +302,7 @@ async function changeState(copilot, repoFullName, number, state) { try { await github.issues.update({owner, repo, issue_number: number, state}); } catch (err) { - throw errors.convertGitHubError(err, 'Error occurred during updating status of issue.'); + throw errors.handleGitHubError(err, 'Error occurred during updating status of issue.', copilot.topcoderUsername, _getIssueUrl(repoFullName, number)); } logger.debug(`Github issue state is updated to '${state}' for issue number ${number}`); } @@ -317,7 +328,7 @@ async function addLabels(copilot, repoFullName, number, labels) { try { await github.issues.update({owner, repo, issue_number: number, labels}); } catch (err) { - throw errors.convertGitHubError(err, 'Error occurred during adding label in issue.'); + throw errors.handleGitHubError(err, 'Error occurred during adding label in issue.', copilot.topcoderUsername, _getIssueUrl(repoFullName, number)); } logger.debug(`Github issue is updated with new labels for ${number}`); } diff --git a/services/GitlabService.js b/services/GitlabService.js index 24fa38a..542e9bb 100644 --- a/services/GitlabService.js +++ b/services/GitlabService.js @@ -37,7 +37,7 @@ async function _authenticate(accessToken) { }); return gitlab; } catch (err) { - throw errors.convertGitLabError(err, 'Failed to during authenticate to Github using access token of copilot.'); + throw errors.handleGitLabError(err, 'Failed to during authenticate to Github using access token of copilot.'); } } @@ -55,25 +55,37 @@ async function _removeAssignees(gitlab, projectId, issueId, assignees) { const oldAssignees = _.difference(issue.assignee_ids, assignees); await gitlab.projects.issues.edit(projectId, issueId, {assignee_ids: oldAssignees}); } catch (err) { - throw errors.convertGitLabError(err, 'Error occurred during remove assignees from issue.'); + throw errors.handleGitLabError(err, 'Error occurred during remove assignees from issue.'); } } +/** + * Get gitlab issue url + * @param {String} repoPath the repo path + * @param {Number} issueId the issue number + * @returns {String} the url + * @private + */ +function _getIssueUrl(repoPath, issueId) { + return `https://gitlab.com/${repoPath}/issues/${issueId}`; +} + /** * creates the comments on gitlab issue * @param {Object} copilot the copilot - * @param {Number} projectId the project id + * @param {Object} project the project object * @param {Number} issueId the issue number * @param {string} body the comment body text */ -async function createComment(copilot, projectId, issueId, body) { +async function createComment(copilot, project, issueId, body) { + const projectId = project.id; Joi.attempt({copilot, projectId, issueId, body}, createComment.schema); const gitlab = await _authenticate(copilot.accessToken); try { body = helper.prepareAutomatedComment(body, copilot); await gitlab.projects.issues.notes.create(projectId, issueId, {body}); } catch (err) { - throw errors.convertGitLabError(err, 'Error occurred during creating comment on issue.'); + throw errors.handleGitLabError(err, 'Error occurred during creating comment on issue.', copilot.topcoderUsername, _getIssueUrl(project.full_name, issueId)); } logger.debug(`Gitlab comment is added on issue with message: "${body}"`); } @@ -88,17 +100,18 @@ createComment.schema = { /** * updates the title of gitlab issue * @param {Object} copilot the copilot - * @param {Number} projectId the project id + * @param {Object} project the project object * @param {Number} issueId the issue number * @param {string} title new title */ -async function updateIssue(copilot, projectId, issueId, title) { +async function updateIssue(copilot, project, issueId, title) { + const projectId = project.id; Joi.attempt({copilot, projectId, issueId, title}, updateIssue.schema); const gitlab = await _authenticate(copilot.accessToken); try { await gitlab.projects.issues.edit(projectId, issueId, {title}); } catch (err) { - throw errors.convertGitLabError(err, 'Error occurred during updating issue.'); + throw errors.handleGitLabError(err, 'Error occurred during updating issue.', copilot.topcoderUsername, _getIssueUrl(project.full_name, issueId)); } logger.debug(`Gitlab issue title is updated for issue number ${issueId}`); } @@ -113,11 +126,12 @@ updateIssue.schema = { /** * Assigns the issue to user login * @param {Object} copilot the copilot - * @param {Number} projectId the project id + * @param {Object} project the project object * @param {Number} issueId the issue number * @param {Number} userId the user id of assignee */ -async function assignUser(copilot, projectId, issueId, userId) { +async function assignUser(copilot, project, issueId, userId) { + const projectId = project.id; Joi.attempt({copilot, projectId, issueId, userId}, assignUser.schema); const gitlab = await _authenticate(copilot.accessToken); try { @@ -128,7 +142,7 @@ async function assignUser(copilot, projectId, issueId, userId) { } await gitlab.projects.issues.edit(projectId, issueId, {assignee_ids: [userId]}); } catch (err) { - throw errors.convertGitLabError(err, 'Error occurred during assigning issue user.'); + throw errors.handleGitLabError(err, 'Error occurred during assigning issue user.', copilot.topcoderUsername, _getIssueUrl(project.full_name, issueId)); } logger.debug(`Gitlab issue with number ${issueId} is assigned to ${issueId}`); } @@ -143,11 +157,12 @@ assignUser.schema = { /** * Removes an assignee from the issue * @param {Object} copilot the copilot - * @param {Number} projectId the project id + * @param {Object} project the project object * @param {Number} issueId the issue number * @param {Number} userId the user id of assignee to remove */ -async function removeAssign(copilot, projectId, issueId, userId) { +async function removeAssign(copilot, project, issueId, userId) { + const projectId = project.id; Joi.attempt({copilot, projectId, issueId, userId}, removeAssign.schema); const gitlab = await _authenticate(copilot.accessToken); await _removeAssignees(gitlab, projectId, issueId, [userId]); @@ -195,14 +210,15 @@ getUserIdByLogin.schema = { /** * updates the gitlab issue as paid and fix accepted * @param {Object} copilot the copilot - * @param {Number} projectId the project id + * @param {Object} project the project object * @param {Number} issueId the issue number * @param {String} challengeUUID the challenge uuid * @param {Array} existLabels the issue labels * @param {String} winner the winner topcoder handle * @param {Boolean} createCopilotPayments the option to create copilot payments or not */ -async function markIssueAsPaid(copilot, projectId, issueId, challengeUUID, existLabels, winner, createCopilotPayments) { // eslint-disable-line max-params +async function markIssueAsPaid(copilot, project, issueId, challengeUUID, existLabels, winner, createCopilotPayments) { // eslint-disable-line max-params + const projectId = project.id; Joi.attempt({copilot, projectId, issueId, challengeUUID, existLabels, winner, createCopilotPayments}, markIssueAsPaid.schema); const gitlab = await _authenticate(copilot.accessToken); const labels = _(existLabels).filter((i) => i !== config.FIX_ACCEPTED_ISSUE_LABEL) @@ -222,7 +238,7 @@ async function markIssueAsPaid(copilot, projectId, issueId, challengeUUID, exist const body = helper.prepareAutomatedComment(commentMessage, copilot); await gitlab.projects.issues.notes.create(projectId, issueId, {body}); } catch (err) { - throw errors.convertGitLabError(err, 'Error occurred during updating issue as paid.'); + throw errors.handleGitLabError(err, 'Error occurred during updating issue as paid.', copilot.topcoderUsername, _getIssueUrl(project.full_name, issueId)); } logger.debug(`Gitlab issue is updated for as paid and fix accepted for ${issueId}`); } @@ -240,17 +256,18 @@ markIssueAsPaid.schema = { /** * change the state of gitlab issue * @param {Object} copilot the copilot - * @param {string} projectId the project id + * @param {Object} project the project object * @param {Number} issueId the issue issue id * @param {string} state new state */ -async function changeState(copilot, projectId, issueId, state) { +async function changeState(copilot, project, issueId, state) { + const projectId = project.id; Joi.attempt({copilot, projectId, issueId, state}, changeState.schema); const gitlab = await _authenticate(copilot.accessToken); try { await gitlab.projects.issues.edit(projectId, issueId, {state_event: state}); } catch (err) { - throw errors.convertGitLabError(err, 'Error occurred during updating status of issue.'); + throw errors.handleGitLabError(err, 'Error occurred during updating status of issue.', copilot.topcoderUsername, _getIssueUrl(project.full_name, issueId)); } logger.debug(`Gitlab issue state is updated to '${state}' for issue number ${issueId}`); } @@ -265,17 +282,18 @@ changeState.schema = { /** * updates the gitlab issue with new labels * @param {Object} copilot the copilot - * @param {string} projectId the project id + * @param {Object} project the project object * @param {Number} issueId the issue issue id * @param {Number} labels the labels */ -async function addLabels(copilot, projectId, issueId, labels) { +async function addLabels(copilot, project, issueId, labels) { + const projectId = project.id; Joi.attempt({copilot, projectId, issueId, labels}, addLabels.schema); const gitlab = await _authenticate(copilot.accessToken); try { await gitlab.projects.issues.edit(projectId, issueId, {labels: _.join(labels, ',')}); } catch (err) { - throw errors.convertGitLabError(err, 'Error occurred during adding label in issue.'); + throw errors.handleGitLabError(err, 'Error occurred during adding label in issue.', copilot.topcoderUsername, _getIssueUrl(project.full_name, issueId)); } logger.debug(`Gitlab issue is updated with new labels for ${issueId}`); } diff --git a/utils/errors.js b/utils/errors.js index 29592de..81f6750 100644 --- a/utils/errors.js +++ b/utils/errors.js @@ -12,6 +12,7 @@ const _ = require('lodash'); const constants = require('../constants'); +const notification = require('./notification'); // the error class wrapper class ProcessorError extends Error { @@ -27,12 +28,17 @@ class ProcessorError extends Error { const errors = {}; /** -* Convert github api error. +* Handle github api error. Return converted error. * @param {Error} err the github api error * @param {String} message the error message +* @param {String} copilotHandle the handle name of the copilot +* @param {String} repoPath the link to related github page * @returns {Error} converted error */ -errors.convertGitHubError = function convertGitHubError(err, message) { +errors.handleGitHubError = function handleGitHubError(err, message, copilotHandle, repoPath) { + if (err.statusCode === 401 && copilotHandle && repoPath) { // eslint-disable-line no-magic-numbers + notification.sendTokenExpiredAlert(copilotHandle, repoPath, 'Github'); + } let resMsg = `${message}. ${err.message}.`; const detail = _.get(err, 'response.body.message'); if (detail) { @@ -47,12 +53,17 @@ errors.convertGitHubError = function convertGitHubError(err, message) { }; /** - * Convert gitlab api error. + * Handle gitlab api error. Return converted error. * @param {Error} err the gitlab api error * @param {String} message the error message +* @param {String} copilotHandle the handle name of the copilot +* @param {String} repoPath the link to related gitlab page * @returns {Error} converted error */ -errors.convertGitLabError = function convertGitLabError(err, message) { +errors.handleGitLabError = function handleGitLabError(err, message, copilotHandle, repoPath) { + if (err.statusCode === 401 && copilotHandle && repoPath) { // eslint-disable-line no-magic-numbers + notification.sendTokenExpiredAlert(copilotHandle, repoPath, 'Gitlab'); + } let resMsg = `${message}. ${err.message}.`; const detail = _.get(err, 'response.body.message'); if (detail) { diff --git a/utils/git-helper.js b/utils/git-helper.js index 28791d0..cc53626 100644 --- a/utils/git-helper.js +++ b/utils/git-helper.js @@ -27,7 +27,7 @@ class GitHelper { if (event.provider === 'github') { await gitHubService.createComment(event.copilot, event.data.repository.full_name, issueNumber, comment); } else if (event.provider === 'gitlab') { - await gitlabService.createComment(event.copilot, event.data.repository.id, issueNumber, comment); + await gitlabService.createComment(event.copilot, event.data.repository, issueNumber, comment); } } @@ -41,7 +41,7 @@ class GitHelper { if (event.provider === 'github') { await gitHubService.addLabels(event.copilot, event.data.repository.full_name, issueNumber, labels); } else if (event.provider === 'gitlab') { - await gitlabService.addLabels(event.copilot, event.data.repository.id, issueNumber, labels); + await gitlabService.addLabels(event.copilot, event.data.repository, issueNumber, labels); } } @@ -54,7 +54,7 @@ class GitHelper { if (event.provider === 'github') { await gitHubService.changeState(event.copilot, event.data.repository.full_name, issue.number, 'open'); } else if (event.provider === 'gitlab') { - await gitlabService.changeState(event.copilot, event.data.repository.id, issue.number, 'reopen'); + await gitlabService.changeState(event.copilot, event.data.repository, issue.number, 'reopen'); } } @@ -84,7 +84,7 @@ class GitHelper { if (event.provider === 'github') { await gitHubService.removeAssign(event.copilot, event.data.repository.full_name, issueNumber, assigneeUsername); } else if (event.provider === 'gitlab') { - await gitlabService.removeAssign(event.copilot, event.data.repository.id, issueNumber, assigneeUserId); + await gitlabService.removeAssign(event.copilot, event.data.repository, issueNumber, assigneeUserId); } } @@ -98,7 +98,7 @@ class GitHelper { if (event.provider === 'github') { await gitHubService.updateIssue(event.copilot, event.data.repository.full_name, issueNumber, newTitle); } else if (event.provider === 'gitlab') { - await gitlabService.updateIssue(event.copilot, event.data.repository.id, issueNumber, newTitle); + await gitlabService.updateIssue(event.copilot, event.data.repository, issueNumber, newTitle); } } @@ -113,7 +113,7 @@ class GitHelper { await gitHubService.assignUser(event.copilot, event.data.repository.full_name, issueNumber, assignedUser); } else if (event.provider === 'gitlab') { const userId = await gitlabService.getUserIdByLogin(event.copilot, assignedUser); - await gitlabService.assignUser(event.copilot, event.data.repository.id, issueNumber, userId); + await gitlabService.assignUser(event.copilot, event.data.repository, issueNumber, userId); } } @@ -131,7 +131,7 @@ class GitHelper { await gitHubService.markIssueAsPaid(event.copilot, event.data.repository.full_name, issueNumber, challengeUUID, existLabels, winner, createCopilotPayments); } else if (event.provider === 'gitlab') { - await gitlabService.markIssueAsPaid(event.copilot, event.data.repository.id, issueNumber, challengeUUID, existLabels, winner, + await gitlabService.markIssueAsPaid(event.copilot, event.data.repository, issueNumber, challengeUUID, existLabels, winner, createCopilotPayments); } else if (event.provider === 'azure') { await azureService.markIssueAsPaid(event.copilot, event.data.repository.full_name, issueNumber, challengeUUID, existLabels); diff --git a/utils/kafka-consumer.js b/utils/kafka-consumer.js new file mode 100644 index 0000000..65fc9c4 --- /dev/null +++ b/utils/kafka-consumer.js @@ -0,0 +1,85 @@ +/** + * Module wrapper for consume kafka topic. + * + * @author TCSCODER + * @version 1.0 + */ +'use strict'; + +const config = require('config'); +const _ = require('lodash'); + +const healthcheck = require('topcoder-healthcheck-dropin'); +const IssueService = require('../services/IssueService'); +const CopilotPaymentService = require('../services/CopilotPaymentService'); +const logger = require('./logger'); +const kafka = require('./kafka'); + +/** + * Handle the message from kafka + * @param {Object} messageSet object to handle + */ +function messageHandler(messageSet) { + logger.debug(` topics ======= ${JSON.stringify(messageSet)}`); + messageSet.forEach((item) => { + // The event should be a JSON object + let event; + try { + const message = JSON.parse(item.message.value.toString('utf8')); + event = JSON.parse(message.payload.value); + message.payload.value = event; + logger.debug(`received message from kafka: ${JSON.stringify(_.omit(message, 'payload.value.data.issue.body'))}`); + } catch (err) { + logger.error(`"message" is not a valid JSON-formatted string: ${err.message}`); + return; + } + + if (event && _.includes(['issue.created', 'issue.updated', 'issue.closed', 'issue.recreated', + 'comment.created', 'comment.updated', 'issue.assigned', 'issue.labelUpdated', 'issue.unassigned'] + , event.event)) { + IssueService + .process(event) + .catch(logger.error); + } + if (event && _.includes(['copilotPayment.add', 'copilotPayment.update', 'copilotPayment.delete', 'copilotPayment.checkUpdates'] + , event.event)) { + CopilotPaymentService + .process(event) + .catch(logger.error); + } + }); +} + +/** + * check if there is kafka connection alive + * @returns {Boolean} true + */ +function check() { + // if (!this.consumer.client.initialBrokers && !this.consumer.client.initialBrokers.length) { + // logger.info(`Brokers Exist Check Failed ${this.consumer.client.initialBrokers} ${this.consumer.client.initialBrokers.length}`) + // return false; + // } + // let connected = true; + // this.consumer.client.initialBrokers.forEach((conn) => { + // logger.info(`Brokers Check Failed ${conn.connected}`) + // connected = conn.connected && connected; + // }); + + // return connected; + return true; +} + +/** + * run the consumer + */ +function run() { + kafka.consumer.init().then(() => { + logger.info('kafka consumer is ready'); + healthcheck.init([check]); + kafka.consumer.subscribe(config.TOPIC, {}, messageHandler); + }).catch((err) => { + logger.error(`kafka consumer is not connected. ${err.stack}`); + }); +} + +module.exports = {run}; diff --git a/utils/kafka-sender.js b/utils/kafka-sender.js new file mode 100644 index 0000000..a053456 --- /dev/null +++ b/utils/kafka-sender.js @@ -0,0 +1,61 @@ +/** + * Module wrapper for sending messages to kafka. + * + * @author TCSCODER + * @version 1.0 + */ +'use strict'; + +const config = require('config'); +const kafka = require('./kafka'); + +/** + * Send message to general topic in kafka. + * @param {String} message the message to send + * @returns {Object} Result from kafka + */ +function send(message) { + const data = JSON.stringify({ + topic: config.TOPIC, + originator: 'topcoder-x-processor', + timestamp: (new Date()).toISOString(), + 'mime-type': 'application/json', + payload: { + value: message + } + }); + return kafka.producer.send({ + topic: config.TOPIC, + message: { + value: data + } + }); +} + +/** + * Send message to notification topic in kafka. + * @param {String} notification the message to send + * @returns {Object} Result from kafka + */ +function sendNotification(notification) { + const data = JSON.stringify({ + topic: config.TOPIC_NOTIFICATION, + originator: 'topcoder-x-processor', + timestamp: (new Date()).toISOString(), + 'mime-type': 'application/json', + payload: { + notifications: [notification] + } + }); + return kafka.producer.send({ + topic: config.TOPIC_NOTIFICATION, + message: { + value: data + } + }); +} + +module.exports = { + send, + sendNotification +}; diff --git a/utils/kafka.js b/utils/kafka.js index b731664..3056fba 100644 --- a/utils/kafka.js +++ b/utils/kafka.js @@ -12,11 +12,7 @@ 'use strict'; const config = require('config'); -const _ = require('lodash'); const kafka = require('no-kafka'); -const healthcheck = require('topcoder-healthcheck-dropin'); -const IssueService = require('../services/IssueService'); -const CopilotPaymentService = require('../services/CopilotPaymentService'); const logger = require('./logger'); class Kafka { @@ -29,82 +25,7 @@ class Kafka { }).catch((err) => { logger.error(`kafka producer is not connected. ${err.stack}`); }); - this.check = this.check.bind(this); - } - - messageHandler(messageSet) { - logger.debug(` topics ======= ${JSON.stringify(messageSet)}`); - messageSet.forEach((item) => { - // The event should be a JSON object - let event; - try { - const message = JSON.parse(item.message.value.toString('utf8')); - event = JSON.parse(message.payload.value); - message.payload.value = event; - logger.debug(`received message from kafka: ${JSON.stringify(_.omit(message, 'payload.value.data.issue.body'))}`); - } catch (err) { - logger.error(`"message" is not a valid JSON-formatted string: ${err.message}`); - return; - } - - if (event && _.includes(['issue.created', 'issue.updated', 'issue.closed', 'issue.recreated', - 'comment.created', 'comment.updated', 'issue.assigned', 'issue.labelUpdated', 'issue.unassigned'] - , event.event)) { - IssueService - .process(event) - .catch(logger.error); - } - if (event && _.includes(['copilotPayment.add', 'copilotPayment.update', 'copilotPayment.delete', 'copilotPayment.checkUpdates'] - , event.event)) { - CopilotPaymentService - .process(event) - .catch(logger.error); - } - }); - } - - // check if there is kafka connection alive - check() { - // if (!this.consumer.client.initialBrokers && !this.consumer.client.initialBrokers.length) { - // logger.info(`Brokers Exist Check Failed ${this.consumer.client.initialBrokers} ${this.consumer.client.initialBrokers.length}`) - // return false; - // } - // let connected = true; - // this.consumer.client.initialBrokers.forEach((conn) => { - // logger.info(`Brokers Check Failed ${conn.connected}`) - // connected = conn.connected && connected; - // }); - - // return connected; - return true; - } - - run() { - this.consumer.init().then(() => { - logger.info('kafka consumer is ready'); - healthcheck.init([this.check]); - this.consumer.subscribe(config.TOPIC, {}, this.messageHandler); - }).catch((err) => { - logger.error(`kafka consumer is not connected. ${err.stack}`); - }); - } - - send(message) { - const data = JSON.stringify({ - topic: config.TOPIC, - originator: 'topcoder-x-processor', - timestamp: (new Date()).toISOString(), - 'mime-type': 'application/json', - payload: { - value: message - } - }); - return this.producer.send({ - topic: config.TOPIC, - message: { - value: data - } - }); + // this.check = this.check.bind(this); } } diff --git a/utils/notification.js b/utils/notification.js new file mode 100644 index 0000000..ad9a8fa --- /dev/null +++ b/utils/notification.js @@ -0,0 +1,49 @@ +/** + * This module contains the helper methods + * for sending notification action to kafka service. + * + * @author TCSCODER + * @version 1.0 + */ +'use strict'; + +const config = require('config'); + +const kafkaSender = require('./kafka-sender'); +const topcoderApiHelper = require('./topcoder-api-helper'); +const logger = require('./logger'); + +const notification = {}; + +const content = `Hi {handle}, +You made an update to ticket {link}, but Topcoder-X couldn't process it properly because your {provider} token has expired. To fix this, please login to x.topcoder.com, click your handle in the upper right and then "Settings" to refresh your token. You will need to redo the action that failed in {provider}.`; // eslint-disable-line max-len + +notification.sendTokenExpiredAlert = async function sendTokenExpiredAlert(copilotHandle, repoPath, provider) { + const copilotId = await topcoderApiHelper.getTopcoderMemberId(copilotHandle); + const notificationConfigs = config.MAIL_NOTICIATION; + logger.debug(`Sending mail notification to copilot ${copilotHandle} Repo: ${repoPath} Provider: ${provider}`); + await kafkaSender.sendNotification({ + serviceId: 'email', + type: notificationConfigs.type, + details: { + from: 'noreply@topcoder.com', + recipients: [ + { + userId: copilotId + } + ], + cc: [], + data: { + subject: notificationConfigs.subject, + body: content + .replace(/{handle}/g, copilotHandle) + .replace(/{link}/g, repoPath) + .replace(/{provider}/g, provider) + }, + sendgridTemplateId: notificationConfigs.sendgridTemplateId, + version: 'v3' + } + }); +}; + +module.exports = notification; From 5e46a7f47f2bcc5eaaa281e6ad4e134f2abcb9a7 Mon Sep 17 00:00:00 2001 From: gets0ul Date: Wed, 25 May 2022 19:33:55 +0700 Subject: [PATCH 2/9] Refresh copilot Gitlab access token automatically when needed. --- config/default.js | 7 ++++- configuration.md | 3 ++ index.js | 4 ++- services/GitlabService.js | 65 +++++++++++++++++++++++++++++++++------ services/UserService.js | 2 ++ 5 files changed, 70 insertions(+), 11 deletions(-) diff --git a/config/default.js b/config/default.js index 632c5ca..e17d3f4 100644 --- a/config/default.js +++ b/config/default.js @@ -74,5 +74,10 @@ module.exports = { ROLE_ID_SUBMITTER: process.env.ROLE_ID_SUBMITTER || '732339e7-8e30-49d7-9198-cccf9451e221', TYPE_ID_TASK: process.env.TYPE_ID_TASK || 'ecd58c69-238f-43a4-a4bb-d172719b9f31', DEFAULT_TIMELINE_TEMPLATE_ID: process.env.DEFAULT_TIMELINE_TEMPLATE_ID || '53a307ce-b4b3-4d6f-b9a1-3741a58f77e6', - DEFAULT_TRACK_ID: process.env.DEFAULT_TRACK_ID || '9b6fc876-f4d9-4ccb-9dfd-419247628825' + DEFAULT_TRACK_ID: process.env.DEFAULT_TRACK_ID || '9b6fc876-f4d9-4ccb-9dfd-419247628825', + GITLAB_ACCESS_TOKEN_DEFAULT_EXPIRATION: 3600 * 2, + GITLAB_REFRESH_TOKEN_BEFORE_EXPIRATION: 300, + GITLAB_CLIENT_ID: process.env.GITLAB_CLIENT_ID, + GITLAB_CLIENT_SECRET: process.env.GITLAB_CLIENT_SECRET, + GITLAB_OWNER_USER_CALLBACK_URL: process.env.GITLAB_OWNER_USER_CALLBACK_URL }; diff --git a/configuration.md b/configuration.md index 5ca0e4f..a8f5ddb 100644 --- a/configuration.md +++ b/configuration.md @@ -16,6 +16,9 @@ The following config parameters are supported, they are defined in `config/defau | NEW_CHALLENGE_DURATION_IN_DAYS | the duration of new challenge | 5 | |TC_URL| the base URL of topcoder to get the challenge URL| defaults to `https://www.topcoder-dev.com`| |GITLAB_API_BASE_URL| the URL for gitlab host| defaults to `https://gitlab.com`| +| GITLAB_CLIENT_ID | the GitLab client id | No default - needs to be set up with same value found in topcoder-x-ui | +| GITLAB_CLIENT_SECRET | the GitLab client secret | No default - needs to be set up with same value found in topcoder-x-ui | +| GITLAB_OWNER_USER_CALLBACK_URL | the GitLab callback redirect uri for refreshing copilot token | No default - needs to be set up with same owner user callback value in topcoder-x-ui | |PAID_ISSUE_LABEL|the label name for paid, should be one of the label configured in topcoder x ui|'tcx_Paid'| |FIX_ACCEPTED_ISSUE_LABEL|the label name for fix accepted, should be one of the label configured in topcoder x ui|'tcx_FixAccepted'| |ASSIGNED_ISSUE_LABEL| the label name for assigned, should be one of the label configured in topcoder x ui| 'tcx_Assigned'| diff --git a/index.js b/index.js index ee57023..ef15b99 100644 --- a/index.js +++ b/index.js @@ -22,7 +22,9 @@ process.on('unhandledRejection', (err) => { }); // dump the configuration to logger -const ignoreConfigLog = ['cert', 'key', 'AWS_ACCESS_KEY_ID', 'AWS_SECRET_ACCESS_KEY', 'AUTH0_CLIENT_ID', 'AUTH0_CLIENT_SECRET']; +const ignoreConfigLog = ['cert', 'key', 'AWS_ACCESS_KEY_ID', 'AWS_SECRET_ACCESS_KEY', 'AUTH0_CLIENT_ID', 'AUTH0_CLIENT_SECRET', + 'GITLAB_CLIENT_ID', 'GITLAB_CLIENT_SECRET']; + /** * Print configs to logger * @param {Object} params the config params diff --git a/services/GitlabService.js b/services/GitlabService.js index 542e9bb..559b31c 100644 --- a/services/GitlabService.js +++ b/services/GitlabService.js @@ -16,9 +16,18 @@ const GitlabAPI = require('node-gitlab-api'); const logger = require('../utils/logger'); const errors = require('../utils/errors'); const helper = require('../utils/helper'); +const dbHelper = require('../utils/db-helper'); +const superagent = require('superagent'); +const superagentPromise = require('superagent-promise'); + +const request = superagentPromise(superagent, Promise); +// milliseconds per second +const MS_PER_SECOND = 1000; const copilotUserSchema = Joi.object().keys({ accessToken: Joi.string().required(), + accessTokenExpiration: Joi.date().required(), + refreshToken: Joi.string().required(), userProviderId: Joi.number().required(), topcoderUsername: Joi.string() }).required(); @@ -80,7 +89,8 @@ function _getIssueUrl(repoPath, issueId) { async function createComment(copilot, project, issueId, body) { const projectId = project.id; Joi.attempt({copilot, projectId, issueId, body}, createComment.schema); - const gitlab = await _authenticate(copilot.accessToken); + const refreshedCopilot = await _refreshGitlabUserAccessToken(copilot); + const gitlab = await _authenticate(refreshedCopilot.accessToken); try { body = helper.prepareAutomatedComment(body, copilot); await gitlab.projects.issues.notes.create(projectId, issueId, {body}); @@ -107,7 +117,8 @@ createComment.schema = { async function updateIssue(copilot, project, issueId, title) { const projectId = project.id; Joi.attempt({copilot, projectId, issueId, title}, updateIssue.schema); - const gitlab = await _authenticate(copilot.accessToken); + const refreshedCopilot = await _refreshGitlabUserAccessToken(copilot); + const gitlab = await _authenticate(refreshedCopilot.accessToken); try { await gitlab.projects.issues.edit(projectId, issueId, {title}); } catch (err) { @@ -133,7 +144,8 @@ updateIssue.schema = { async function assignUser(copilot, project, issueId, userId) { const projectId = project.id; Joi.attempt({copilot, projectId, issueId, userId}, assignUser.schema); - const gitlab = await _authenticate(copilot.accessToken); + const refreshedCopilot = await _refreshGitlabUserAccessToken(copilot); + const gitlab = await _authenticate(refreshedCopilot.accessToken); try { const issue = await gitlab.projects.issues.show(projectId, issueId); const oldAssignees = _.without(issue.assignee_ids, userId); @@ -164,7 +176,8 @@ assignUser.schema = { async function removeAssign(copilot, project, issueId, userId) { const projectId = project.id; Joi.attempt({copilot, projectId, issueId, userId}, removeAssign.schema); - const gitlab = await _authenticate(copilot.accessToken); + const refreshedCopilot = await _refreshGitlabUserAccessToken(copilot); + const gitlab = await _authenticate(refreshedCopilot.accessToken); await _removeAssignees(gitlab, projectId, issueId, [userId]); logger.debug(`Gitlab user ${userId} is unassigned from issue number ${issueId}`); } @@ -179,7 +192,8 @@ removeAssign.schema = assignUser.schema; */ async function getUsernameById(copilot, userId) { Joi.attempt({copilot, userId}, getUsernameById.schema); - const gitlab = await _authenticate(copilot.accessToken); + const refreshedCopilot = await _refreshGitlabUserAccessToken(copilot); + const gitlab = await _authenticate(refreshedCopilot.accessToken); const user = await gitlab.users.show(userId); return user ? user.username : null; } @@ -197,7 +211,8 @@ getUsernameById.schema = { */ async function getUserIdByLogin(copilot, login) { Joi.attempt({copilot, login}, getUserIdByLogin.schema); - const gitlab = await _authenticate(copilot.accessToken); + const refreshedCopilot = await _refreshGitlabUserAccessToken(copilot); + const gitlab = await _authenticate(refreshedCopilot.accessToken); const user = await gitlab.users.all({username: login}); return user.length ? user[0].id : null; } @@ -220,7 +235,8 @@ getUserIdByLogin.schema = { async function markIssueAsPaid(copilot, project, issueId, challengeUUID, existLabels, winner, createCopilotPayments) { // eslint-disable-line max-params const projectId = project.id; Joi.attempt({copilot, projectId, issueId, challengeUUID, existLabels, winner, createCopilotPayments}, markIssueAsPaid.schema); - const gitlab = await _authenticate(copilot.accessToken); + const refreshedCopilot = await _refreshGitlabUserAccessToken(copilot); + const gitlab = await _authenticate(refreshedCopilot.accessToken); const labels = _(existLabels).filter((i) => i !== config.FIX_ACCEPTED_ISSUE_LABEL) .push(config.FIX_ACCEPTED_ISSUE_LABEL, config.PAID_ISSUE_LABEL).value(); try { @@ -263,7 +279,8 @@ markIssueAsPaid.schema = { async function changeState(copilot, project, issueId, state) { const projectId = project.id; Joi.attempt({copilot, projectId, issueId, state}, changeState.schema); - const gitlab = await _authenticate(copilot.accessToken); + const refreshedCopilot = await _refreshGitlabUserAccessToken(copilot); + const gitlab = await _authenticate(refreshedCopilot.accessToken); try { await gitlab.projects.issues.edit(projectId, issueId, {state_event: state}); } catch (err) { @@ -289,7 +306,8 @@ changeState.schema = { async function addLabels(copilot, project, issueId, labels) { const projectId = project.id; Joi.attempt({copilot, projectId, issueId, labels}, addLabels.schema); - const gitlab = await _authenticate(copilot.accessToken); + const refreshedCopilot = await _refreshGitlabUserAccessToken(copilot); + const gitlab = await _authenticate(refreshedCopilot.accessToken); try { await gitlab.projects.issues.edit(projectId, issueId, {labels: _.join(labels, ',')}); } catch (err) { @@ -305,6 +323,35 @@ addLabels.schema = { labels: Joi.array().items(Joi.string()).required() }; +/** + * Refresh the copilot access token if token is needed + * @param {Object} copilot the copilot + * @returns {Promise} the promise result of copilot with refreshed token + */ +async function _refreshGitlabUserAccessToken(copilot) { + if (copilot.accessTokenExpiration && new Date().getTime() > copilot.accessTokenExpiration.getTime() - + (config.GITLAB_REFRESH_TOKEN_BEFORE_EXPIRATION * MS_PER_SECOND)) { + const refreshTokenResult = await request + .post(`${config.GITLAB_API_BASE_URL}/oauth/token`) + .query({ + client_id: config.GITLAB_CLIENT_ID, + client_secret: config.GITLAB_CLIENT_SECRET, + refresh_token: copilot.refreshToken, + grant_type: 'refresh_token', + redirect_uri: config.GITLAB_OWNER_USER_CALLBACK_URL, + }) + .end(); + // save user token data + const expiresIn = refreshTokenResult.body.expires_in || config.GITLAB_ACCESS_TOKEN_DEFAULT_EXPIRATION; + return await dbHelper.update(User, copilot.id, { + accessToken: refreshTokenResult.body.access_token, + accessTokenExpiration: new Date(new Date().getTime() + expiresIn * MS_PER_SECOND), + refreshToken: refreshTokenResult.body.refresh_token, + }); + } + return copilot; +} + module.exports = { createComment, diff --git a/services/UserService.js b/services/UserService.js index 3b1b342..71379e7 100755 --- a/services/UserService.js +++ b/services/UserService.js @@ -97,6 +97,8 @@ async function getRepositoryCopilotOrOwner(provider, repoFullName) { return { accessToken: user.accessToken, + accessTokenExpiration: user.accessTokenExpiration, + refreshToken: user.refreshToken, userProviderId: user.userProviderId, topcoderUsername: userMapping.topcoderUsername }; From af42897d3076cc704f494924b1de9dcf6ed63706 Mon Sep 17 00:00:00 2001 From: 52code Date: Thu, 2 Jun 2022 12:15:45 +0800 Subject: [PATCH 3/9] https://github.com/topcoder-platform/topcoder-x-ui/issues/453 --- models/Project.js | 5 +++ services/ChallengeService.js | 65 +++++++++++++++++++++++++++++++ services/CopilotPaymentService.js | 1 + services/IssueService.js | 1 + utils/db-helper.js | 19 +++++++++ utils/kafka-consumer.js | 7 ++++ 6 files changed, 98 insertions(+) create mode 100644 services/ChallengeService.js diff --git a/models/Project.js b/models/Project.js index 08a7837..c27edc1 100755 --- a/models/Project.js +++ b/models/Project.js @@ -23,6 +23,11 @@ const schema = new Schema({ type: Number, required: true }, + tags: { + type: Array, + required: true, + default: [] + }, rocketChatWebhook: {type: String, required: false}, rocketChatChannelName: {type: String, required: false}, archived: {type: String, required: true}, diff --git a/services/ChallengeService.js b/services/ChallengeService.js new file mode 100644 index 0000000..f11f47f --- /dev/null +++ b/services/ChallengeService.js @@ -0,0 +1,65 @@ +/* + * Copyright (c) 2022 TopCoder, Inc. All rights reserved. + */ +'use strict'; + +/** + * This service processes incoming pure challenge events. + * + * @author TCSCODER + * @version 1.0 + */ +const _ = require('lodash'); +const Joi = require('joi'); +const logger = require('../utils/logger'); +const topcoderApiHelper = require('../utils/topcoder-api-helper'); +const dbHelper = require('../utils/db-helper'); + +/** + * Update challenge tags + * @param {Object} event the event + */ +async handleChallengeTagsUpdate(event) { + const tags = event.data.tags; + try { + _.each(event.data.challengeUUIDsList, challengeUUIDs => { + if (_.isString(challengeUUIDs)) { // repoUrl + challengeUUIDs = await dbHelper.queryChallengeUUIDsByRepoUrl(challengeUUIDs); + } + _.each(challengeUUIDs, challengeUUID => await topcoderApiHelper.updateChallenge(challengeUUID, {tags})); + }); + } catch (err) { + logger.error(`handleChallengeTagsUpdate failed. Internal Error: ${err}`); + throw new Error(`handleChallengeTagsUpdate failed. Internal Error: ${err}`); + } +} + +/** + * Process pure challenge event. + * @param {Object} event the event + */ +async function process(event) { + Joi.attempt(event, process.schema); + + if (event.event === 'challengeTags.update') { + await handleChallengeTagsUpdate(event); + } +} + +process.schema = Joi.object().keys({ + event: Joi.string().valid('challengeUUIDTags.update').required(), + data: Joi.object().keys({ + challengeUUIDsList: Joi.array().items( + Joi.alternatives().try(Joi.string(), Joi.array().items(Joi.string())) + ).required(), + tags: Joi.array().items(Joi.string().required()).min(1).required(), + }).required(), + retryCount: Joi.number().integer().default(0).optional(), +}); + + +module.exports = { + process +}; + +logger.buildService(module.exports); diff --git a/services/CopilotPaymentService.js b/services/CopilotPaymentService.js index edfe5a4..6e54afe 100644 --- a/services/CopilotPaymentService.js +++ b/services/CopilotPaymentService.js @@ -194,6 +194,7 @@ async function handlePaymentAdd(event, payment) { const newChallenge = { name: challengeTitle, projectId: project.tcDirectId, + tags: project.tags, detailedRequirements: challengeRequirements, prizes: [payment.amount], reviewType: 'INTERNAL' diff --git a/services/IssueService.js b/services/IssueService.js index e1d1cb7..0c309f3 100755 --- a/services/IssueService.js +++ b/services/IssueService.js @@ -636,6 +636,7 @@ async function handleIssueCreate(event, issue, forceAssign = false) { issue.challengeUUID = await topcoderApiHelper.createChallenge({ name: issue.title, projectId, + tags: project.tags, detailedRequirements: issue.body, prizes: issue.prizes }); diff --git a/utils/db-helper.js b/utils/db-helper.js index 38425b5..8775177 100644 --- a/utils/db-helper.js +++ b/utils/db-helper.js @@ -342,6 +342,24 @@ async function queryOneActiveRepository(model, url) { }); } +/** + * Get Issue's challengeUUID by repoUrl + * @param {String} repoUrl The repo url + * @returns {Promise} + */ +async function queryChallengeUUIDsByRepoUrl(repoUrl) { + return await new Promise((resolve, reject) => { + models.Issue.scan('repoUrl').eq(repoUrl) + .attributes(['challengeUUID']) + .exec((err, results) => { + if (err) { + return reject(err); + } + return resolve(results.map({challengeUUID} => challengeUUID)); + }); + }); +} + module.exports = { getById, scan, @@ -357,6 +375,7 @@ module.exports = { queryOneUserMappingByGithubUsername, queryOneUserMappingByGitlabUsername, queryOneUserMappingByTCUsername, + queryChallengeUUIDsByRepoUrl, removeCopilotPayment, removeIssue }; diff --git a/utils/kafka-consumer.js b/utils/kafka-consumer.js index 65fc9c4..d9eda8d 100644 --- a/utils/kafka-consumer.js +++ b/utils/kafka-consumer.js @@ -12,6 +12,7 @@ const _ = require('lodash'); const healthcheck = require('topcoder-healthcheck-dropin'); const IssueService = require('../services/IssueService'); const CopilotPaymentService = require('../services/CopilotPaymentService'); +const ChallengeService = require('../services/ChallengeService'); const logger = require('./logger'); const kafka = require('./kafka'); @@ -47,6 +48,12 @@ function messageHandler(messageSet) { .process(event) .catch(logger.error); } + if (event && _.includes(['challengeUUIDTags.update'] + , event.event)) { + ChallengeService + .process(event) + .catch(logger.error); + } }); } From c15cd1f8fdafc60ef17934b7fd1aeb5c5abbc450 Mon Sep 17 00:00:00 2001 From: 52code Date: Thu, 2 Jun 2022 15:33:36 +0800 Subject: [PATCH 4/9] fix lint of PR#93 for topcoder-x-ui/Issue#453 --- services/ChallengeService.js | 20 +++++++++++--------- services/GitlabService.js | 7 ++++--- utils/db-helper.js | 9 ++++++--- 3 files changed, 21 insertions(+), 15 deletions(-) diff --git a/services/ChallengeService.js b/services/ChallengeService.js index f11f47f..7ecaa9d 100644 --- a/services/ChallengeService.js +++ b/services/ChallengeService.js @@ -19,19 +19,21 @@ const dbHelper = require('../utils/db-helper'); * Update challenge tags * @param {Object} event the event */ -async handleChallengeTagsUpdate(event) { +async function handleChallengeTagsUpdate(event) { const tags = event.data.tags; - try { - _.each(event.data.challengeUUIDsList, challengeUUIDs => { + await Promise.all( + event.data.challengeUUIDsList.map(async (challengeUUIDs) => { if (_.isString(challengeUUIDs)) { // repoUrl challengeUUIDs = await dbHelper.queryChallengeUUIDsByRepoUrl(challengeUUIDs); } - _.each(challengeUUIDs, challengeUUID => await topcoderApiHelper.updateChallenge(challengeUUID, {tags})); - }); - } catch (err) { + return challengeUUIDs.map(async (challengeUUID) => await topcoderApiHelper.updateChallenge(challengeUUID, {tags})); + }).reduce((a, b) => _.concat(a, b), []) + ).then((resps) => { + logger.debug(`handleChallengeTagsUpdate updated ${_.size(resps)} challenges successfully.`); + }).catch((err) => { logger.error(`handleChallengeTagsUpdate failed. Internal Error: ${err}`); throw new Error(`handleChallengeTagsUpdate failed. Internal Error: ${err}`); - } + }); } /** @@ -52,9 +54,9 @@ process.schema = Joi.object().keys({ challengeUUIDsList: Joi.array().items( Joi.alternatives().try(Joi.string(), Joi.array().items(Joi.string())) ).required(), - tags: Joi.array().items(Joi.string().required()).min(1).required(), + tags: Joi.array().items(Joi.string().required()).min(1).required() }).required(), - retryCount: Joi.number().integer().default(0).optional(), + retryCount: Joi.number().integer().default(0).optional() }); diff --git a/services/GitlabService.js b/services/GitlabService.js index 559b31c..9e100f3 100644 --- a/services/GitlabService.js +++ b/services/GitlabService.js @@ -15,6 +15,7 @@ const Joi = require('joi'); const GitlabAPI = require('node-gitlab-api'); const logger = require('../utils/logger'); const errors = require('../utils/errors'); +const models = require('../models'); const helper = require('../utils/helper'); const dbHelper = require('../utils/db-helper'); const superagent = require('superagent'); @@ -338,15 +339,15 @@ async function _refreshGitlabUserAccessToken(copilot) { client_secret: config.GITLAB_CLIENT_SECRET, refresh_token: copilot.refreshToken, grant_type: 'refresh_token', - redirect_uri: config.GITLAB_OWNER_USER_CALLBACK_URL, + redirect_uri: config.GITLAB_OWNER_USER_CALLBACK_URL }) .end(); // save user token data const expiresIn = refreshTokenResult.body.expires_in || config.GITLAB_ACCESS_TOKEN_DEFAULT_EXPIRATION; - return await dbHelper.update(User, copilot.id, { + return await dbHelper.update(models.User, copilot.id, { accessToken: refreshTokenResult.body.access_token, accessTokenExpiration: new Date(new Date().getTime() + expiresIn * MS_PER_SECOND), - refreshToken: refreshTokenResult.body.refresh_token, + refreshToken: refreshTokenResult.body.refresh_token }); } return copilot; diff --git a/utils/db-helper.js b/utils/db-helper.js index 8775177..29cc2e4 100644 --- a/utils/db-helper.js +++ b/utils/db-helper.js @@ -83,8 +83,10 @@ async function queryOneIssue(model, repositoryId, number, provider) { async function queryOneActiveProject(model, repoUrl) { return await new Promise((resolve, reject) => { queryOneActiveRepository(models.Repository, repoUrl).then((repo) => { - if (!repo) resolve(null); - else model.queryOne('id').eq(repo.projectId).consistent() + if (!repo) { + resolve(null); + } else { + model.queryOne('id').eq(repo.projectId).consistent() .exec((err, result) => { if (err) { logger.debug(`queryOneActiveProject. Error. ${err}`); @@ -92,6 +94,7 @@ async function queryOneActiveProject(model, repoUrl) { } return resolve(result); }); + } }); }); } @@ -355,7 +358,7 @@ async function queryChallengeUUIDsByRepoUrl(repoUrl) { if (err) { return reject(err); } - return resolve(results.map({challengeUUID} => challengeUUID)); + return resolve(results.map(({challengeUUID}) => challengeUUID)); }); }); } From 6997d3c781fb08ff5528356e98bc66e05f493718 Mon Sep 17 00:00:00 2001 From: 52code Date: Thu, 2 Jun 2022 22:50:58 +0800 Subject: [PATCH 5/9] https://github.com/topcoder-platform/topcoder-x-ui/issues/453 --- models/Project.js | 4 ++-- services/ChallengeService.js | 4 ++-- services/CopilotPaymentService.js | 2 +- services/IssueService.js | 2 +- 4 files changed, 6 insertions(+), 6 deletions(-) diff --git a/models/Project.js b/models/Project.js index c27edc1..df53b8d 100755 --- a/models/Project.js +++ b/models/Project.js @@ -24,9 +24,9 @@ const schema = new Schema({ required: true }, tags: { - type: Array, + type: String, required: true, - default: [] + default: '' }, rocketChatWebhook: {type: String, required: false}, rocketChatChannelName: {type: String, required: false}, diff --git a/services/ChallengeService.js b/services/ChallengeService.js index 7ecaa9d..6bcf787 100644 --- a/services/ChallengeService.js +++ b/services/ChallengeService.js @@ -20,7 +20,7 @@ const dbHelper = require('../utils/db-helper'); * @param {Object} event the event */ async function handleChallengeTagsUpdate(event) { - const tags = event.data.tags; + const tags = event.data.tags.split(','); await Promise.all( event.data.challengeUUIDsList.map(async (challengeUUIDs) => { if (_.isString(challengeUUIDs)) { // repoUrl @@ -54,7 +54,7 @@ process.schema = Joi.object().keys({ challengeUUIDsList: Joi.array().items( Joi.alternatives().try(Joi.string(), Joi.array().items(Joi.string())) ).required(), - tags: Joi.array().items(Joi.string().required()).min(1).required() + tags: Joi.string().required() }).required(), retryCount: Joi.number().integer().default(0).optional() }); diff --git a/services/CopilotPaymentService.js b/services/CopilotPaymentService.js index 6e54afe..7189510 100644 --- a/services/CopilotPaymentService.js +++ b/services/CopilotPaymentService.js @@ -194,7 +194,7 @@ async function handlePaymentAdd(event, payment) { const newChallenge = { name: challengeTitle, projectId: project.tcDirectId, - tags: project.tags, + tags: project.tags.split(','), detailedRequirements: challengeRequirements, prizes: [payment.amount], reviewType: 'INTERNAL' diff --git a/services/IssueService.js b/services/IssueService.js index 0c309f3..7413f11 100755 --- a/services/IssueService.js +++ b/services/IssueService.js @@ -636,7 +636,7 @@ async function handleIssueCreate(event, issue, forceAssign = false) { issue.challengeUUID = await topcoderApiHelper.createChallenge({ name: issue.title, projectId, - tags: project.tags, + tags: project.tags.split(','), detailedRequirements: issue.body, prizes: issue.prizes }); From 3736d024190be06d5a2c231b95d3ac7b3656b88d Mon Sep 17 00:00:00 2001 From: 52code Date: Fri, 3 Jun 2022 08:22:10 +0800 Subject: [PATCH 6/9] fix-empty-tags --- services/CopilotPaymentService.js | 2 +- services/IssueService.js | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/services/CopilotPaymentService.js b/services/CopilotPaymentService.js index 7189510..f828244 100644 --- a/services/CopilotPaymentService.js +++ b/services/CopilotPaymentService.js @@ -194,7 +194,7 @@ async function handlePaymentAdd(event, payment) { const newChallenge = { name: challengeTitle, projectId: project.tcDirectId, - tags: project.tags.split(','), + tags: !!project.tags ? project.tags.split(',') : [], detailedRequirements: challengeRequirements, prizes: [payment.amount], reviewType: 'INTERNAL' diff --git a/services/IssueService.js b/services/IssueService.js index 7413f11..0d24826 100755 --- a/services/IssueService.js +++ b/services/IssueService.js @@ -636,7 +636,7 @@ async function handleIssueCreate(event, issue, forceAssign = false) { issue.challengeUUID = await topcoderApiHelper.createChallenge({ name: issue.title, projectId, - tags: project.tags.split(','), + tags: !!project.tags ? project.tags.split(',') : [], detailedRequirements: issue.body, prizes: issue.prizes }); From 0d14af4c30db0e0f2ec199b492f310845c1f7bd1 Mon Sep 17 00:00:00 2001 From: 52code Date: Fri, 3 Jun 2022 09:55:10 +0800 Subject: [PATCH 7/9] add tags param --- utils/topcoder-api-helper.js | 1 + 1 file changed, 1 insertion(+) diff --git a/utils/topcoder-api-helper.js b/utils/topcoder-api-helper.js index 94d18b1..ade67d0 100644 --- a/utils/topcoder-api-helper.js +++ b/utils/topcoder-api-helper.js @@ -75,6 +75,7 @@ async function createChallenge(challenge) { }], timelineTemplateId: config.DEFAULT_TIMELINE_TEMPLATE_ID, projectId: challenge.projectId, + tags: challenge.tags, trackId: config.DEFAULT_TRACK_ID, legacy: { pureV5Task: true From f254181f0bb1a7b31b00f6100d1fab3da64c0662 Mon Sep 17 00:00:00 2001 From: 52code Date: Fri, 3 Jun 2022 10:15:48 +0800 Subject: [PATCH 8/9] fix name of challenge event --- services/ChallengeService.js | 2 +- utils/kafka-consumer.js | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/services/ChallengeService.js b/services/ChallengeService.js index 6bcf787..ae4dd22 100644 --- a/services/ChallengeService.js +++ b/services/ChallengeService.js @@ -49,7 +49,7 @@ async function process(event) { } process.schema = Joi.object().keys({ - event: Joi.string().valid('challengeUUIDTags.update').required(), + event: Joi.string().valid('challengeTags.update').required(), data: Joi.object().keys({ challengeUUIDsList: Joi.array().items( Joi.alternatives().try(Joi.string(), Joi.array().items(Joi.string())) diff --git a/utils/kafka-consumer.js b/utils/kafka-consumer.js index d9eda8d..c0e6fc5 100644 --- a/utils/kafka-consumer.js +++ b/utils/kafka-consumer.js @@ -48,7 +48,7 @@ function messageHandler(messageSet) { .process(event) .catch(logger.error); } - if (event && _.includes(['challengeUUIDTags.update'] + if (event && _.includes(['challengeTags.update'] , event.event)) { ChallengeService .process(event) From a902dd7e660de5bdfbdcc21d27ff797790804fe7 Mon Sep 17 00:00:00 2001 From: 52code Date: Tue, 14 Jun 2022 09:58:54 +0800 Subject: [PATCH 9/9] https://github.com/topcoder-platform/topcoder-x-ui/issues/459 --- services/NotificationService.js | 56 +++++++++++++++++++++++++++++++++ utils/kafka-consumer.js | 7 +++++ utils/notification.js | 17 ++++++++-- 3 files changed, 78 insertions(+), 2 deletions(-) create mode 100644 services/NotificationService.js diff --git a/services/NotificationService.js b/services/NotificationService.js new file mode 100644 index 0000000..76d2c9d --- /dev/null +++ b/services/NotificationService.js @@ -0,0 +1,56 @@ +/* + * Copyright (c) 2022 TopCoder, Inc. All rights reserved. + */ +'use strict'; + +/** + * This service processes incoming pure challenge events. + * + * @author TCSCODER + * @version 1.0 + */ +const Joi = require('joi'); +const logger = require('../utils/logger'); +const notification = require('../utils/notification'); + +/** + * Send token expired notification + * @param {Object} event the event + */ +async function handleTokenExpired(event) { + try { + const {copilotHandle, provider} = event.data; + await notification.sendTokenExpiredAlert(copilotHandle, '', provider); + logger.debug('Send token expired notification success'); + } catch (err) { + logger.debug(`Send token expired notification failed. Internal Error: ${err}`); + } +} + +/** + * Process notification event. + * @param {Object} event the event + */ +async function process(event) { + Joi.attempt(event, process.schema); + + if (event.event === 'notification.tokenExpired') { + await handleTokenExpired(event); + } +} + +process.schema = Joi.object().keys({ + event: Joi.string().valid('notification.tokenExpired').required(), + data: Joi.object().keys({ + copilotHandle: Joi.string().required(), + provider: Joi.string().required() + }).required(), + retryCount: Joi.number().integer().default(0).optional() +}); + + +module.exports = { + process +}; + +logger.buildService(module.exports); diff --git a/utils/kafka-consumer.js b/utils/kafka-consumer.js index c0e6fc5..30655a7 100644 --- a/utils/kafka-consumer.js +++ b/utils/kafka-consumer.js @@ -13,6 +13,7 @@ const healthcheck = require('topcoder-healthcheck-dropin'); const IssueService = require('../services/IssueService'); const CopilotPaymentService = require('../services/CopilotPaymentService'); const ChallengeService = require('../services/ChallengeService'); +const NotificationService = require('../services/NotificationService'); const logger = require('./logger'); const kafka = require('./kafka'); @@ -54,6 +55,12 @@ function messageHandler(messageSet) { .process(event) .catch(logger.error); } + if (event && _.includes(['notification.tokenExpired'] + , event.event)) { + NotificationService + .process(event) + .catch(logger.error); + } }); } diff --git a/utils/notification.js b/utils/notification.js index ad9a8fa..f294275 100644 --- a/utils/notification.js +++ b/utils/notification.js @@ -15,12 +15,25 @@ const logger = require('./logger'); const notification = {}; -const content = `Hi {handle}, -You made an update to ticket {link}, but Topcoder-X couldn't process it properly because your {provider} token has expired. To fix this, please login to x.topcoder.com, click your handle in the upper right and then "Settings" to refresh your token. You will need to redo the action that failed in {provider}.`; // eslint-disable-line max-len +/** + * get content template to send + * @param {String} repoPath the repo path + * @returns {String} + */ +function getContent(repoPath) { + return repoPath + ? + `Hi {handle}, + You made an update to ticket {link}, but Topcoder-X couldn't process it properly because your {provider} token has expired. To fix this, please login to x.topcoder.com, click your handle in the upper right and then "Settings" to refresh your token. You will need to redo the action that failed in {provider}.` // eslint-disable-line max-len + : + `Hi {handle}, + You made an operation on {provider}, but Topcoder-X couldn't process it properly because your {provider} token has expired. To fix this, please login to x.topcoder.com, click your handle in the upper right and then "Settings" to refresh your token. You will need to redo the action that failed in {provider}.`; // eslint-disable-line max-len +} notification.sendTokenExpiredAlert = async function sendTokenExpiredAlert(copilotHandle, repoPath, provider) { const copilotId = await topcoderApiHelper.getTopcoderMemberId(copilotHandle); const notificationConfigs = config.MAIL_NOTICIATION; + const content = getContent(repoPath); logger.debug(`Sending mail notification to copilot ${copilotHandle} Repo: ${repoPath} Provider: ${provider}`); await kafkaSender.sendNotification({ serviceId: 'email',