From ce7474e835e2401d3d71d295a029b4129df9ba1a Mon Sep 17 00:00:00 2001 From: Joan Reyero Date: Fri, 17 Mar 2023 10:38:28 +0100 Subject: [PATCH 1/6] First draft --- .../config/custom-environment-variables.json | 3 ++ backend/config/default.json | 3 ++ backend/src/bin/jobs/index.ts | 2 ++ backend/src/bin/jobs/refreshSampleData.ts | 21 +++++++++++ backend/src/config/configTypes.ts | 4 +++ backend/src/config/index.ts | 7 ++++ .../refreshSampleDataWorker.ts | 36 +++++++++++++++++++ .../microservices/nodejs/workerFactory.ts | 4 +++ 8 files changed, 80 insertions(+) create mode 100644 backend/src/bin/jobs/refreshSampleData.ts create mode 100644 backend/src/serverless/microservices/nodejs/integration-data-checker/refreshSampleDataWorker.ts diff --git a/backend/config/custom-environment-variables.json b/backend/config/custom-environment-variables.json index a73b1730c5..c81d436228 100644 --- a/backend/config/custom-environment-variables.json +++ b/backend/config/custom-environment-variables.json @@ -145,6 +145,9 @@ "slackAlerting": { "url": "CROWD_SLACK_ALERTING_URL" }, + "sampleData": { + "tenantId": "CROWD_SAMPLE_DATA_TENANT_ID" + }, "unleash": { "url": "CROWD_UNLEASH_URL", "adminApiKey": "CROWD_UNLEASH_ADMIN_API_KEY", diff --git a/backend/config/default.json b/backend/config/default.json index 71039aaade..3309cb8657 100644 --- a/backend/config/default.json +++ b/backend/config/default.json @@ -40,5 +40,8 @@ }, "slackAlerting": { "url": "" + }, + "sampleData": { + "tenantId": "" } } diff --git a/backend/src/bin/jobs/index.ts b/backend/src/bin/jobs/index.ts index 7ab1cc8021..12bfaca2f3 100644 --- a/backend/src/bin/jobs/index.ts +++ b/backend/src/bin/jobs/index.ts @@ -7,6 +7,7 @@ import refreshMaterializedViews from './refreshMaterializedViews' import downgradeExpiredPlans from './downgradeExpiredPlans' import eagleEyeEmailDigestTicks from './eagleEyeEmailDigestTicks' import integrationDataChecker from './integrationDataChecker' +import refreshSampleData from './refreshSampleData' const jobs: CrowdJob[] = [ weeklyAnalyticsEmailsCoordinator, @@ -17,6 +18,7 @@ const jobs: CrowdJob[] = [ downgradeExpiredPlans, eagleEyeEmailDigestTicks, integrationDataChecker, + refreshSampleData ] export default jobs diff --git a/backend/src/bin/jobs/refreshSampleData.ts b/backend/src/bin/jobs/refreshSampleData.ts new file mode 100644 index 0000000000..6cf033c1c8 --- /dev/null +++ b/backend/src/bin/jobs/refreshSampleData.ts @@ -0,0 +1,21 @@ +import { CrowdJob } from '../../types/jobTypes' +import { sendNodeWorkerMessage } from '../../serverless/utils/nodeWorkerSQS' +import { NodeWorkerMessageType } from '../../serverless/types/workerTypes' +import { NodeWorkerMessageBase } from '../../types/mq/nodeWorkerMessageBase' + +const job: CrowdJob = { + name: 'Integration Data Checker', + // every day + // cronTime: '0 0 * * *', + // every minute + cronTime: '* * * * *', + onTrigger: async () => { + console.log('triggering!') + await sendNodeWorkerMessage('refresh-sample-data', { + type: NodeWorkerMessageType.NODE_MICROSERVICE, + service: 'refresh-sample-data', + } as NodeWorkerMessageBase) + }, +} + +export default job diff --git a/backend/src/config/configTypes.ts b/backend/src/config/configTypes.ts index 835967698e..09bf39a076 100644 --- a/backend/src/config/configTypes.ts +++ b/backend/src/config/configTypes.ts @@ -204,3 +204,7 @@ export interface UnleashConfiguration { export interface SlackAlertingConfiguration { url: string } + +export interface SampleDataConfiguration { + tenantId: string +} diff --git a/backend/src/config/index.ts b/backend/src/config/index.ts index 8346ad25be..fe910a065f 100644 --- a/backend/src/config/index.ts +++ b/backend/src/config/index.ts @@ -26,6 +26,7 @@ import { EagleEyeConfiguration, UnleashConfiguration, SlackAlertingConfiguration, + SampleDataConfiguration, } from './configTypes' // TODO-kube @@ -240,3 +241,9 @@ export const SLACK_ALERTING_CONFIG: SlackAlertingConfiguration = KUBE_MODE : { url: process.env.SLACK_ALERTING_URL, } + +export const SAMPLE_DATA_CONFIG: SampleDataConfiguration = KUBE_MODE + ? config.get('sampleData') + : { + tenantId: process.env.SAMPLE_DATA_TENANT_ID, + } diff --git a/backend/src/serverless/microservices/nodejs/integration-data-checker/refreshSampleDataWorker.ts b/backend/src/serverless/microservices/nodejs/integration-data-checker/refreshSampleDataWorker.ts new file mode 100644 index 0000000000..5cacde6b78 --- /dev/null +++ b/backend/src/serverless/microservices/nodejs/integration-data-checker/refreshSampleDataWorker.ts @@ -0,0 +1,36 @@ +import { Sequelize } from 'sequelize/types' +import { QueryTypes } from 'sequelize' + +import { API_CONFIG, SAMPLE_DATA_CONFIG } from '../../../../config' + +const sequelize = new Sequelize(/* your sequelize configuration */) + +async function refreshSampleDataWorker(): Promise { + if (API_CONFIG.edition === 'crowd-hosted') { + const tenantId = SAMPLE_DATA_CONFIG.tenantId + const updateDays = 1 // Or any number of days you want to add + + const tables = [ + { name: 'activities', columns: ['createdAt', 'timestamp'] }, + { name: 'members', columns: ['joinedAt', 'createdAt'] }, + { name: 'notes', columns: ['createdAt'] }, + { name: 'conversations', columns: ['createdAt'] }, + { name: 'organizations', columns: ['createdAt'] }, + { name: 'tags', columns: ['createdAt'] }, + ] + + for (const table of tables) { + for (const column of table.columns) { + const query = ` + UPDATE ${table.name} + SET ${column} = ${column} + INTERVAL '${updateDays} days' + WHERE tenantId = ${tenantId}; + ` + + await sequelize.query(query, { type: QueryTypes.UPDATE }) + } + } + } +} + +export { refreshSampleDataWorker } diff --git a/backend/src/serverless/microservices/nodejs/workerFactory.ts b/backend/src/serverless/microservices/nodejs/workerFactory.ts index fac185fe3e..b2bb6103b3 100644 --- a/backend/src/serverless/microservices/nodejs/workerFactory.ts +++ b/backend/src/serverless/microservices/nodejs/workerFactory.ts @@ -22,6 +22,7 @@ import { processSendgridWebhook } from '../../integrations/workers/sendgridWebho import { bulkEnrichmentWorker } from './bulk-enrichment/bulkEnrichmentWorker' import { eagleEyeEmailDigestWorker } from './eagle-eye-email-digest/eagleEyeEmailDigestWorker' import { integrationDataCheckerWorker } from './integration-data-checker/integrationDataCheckerWorker' +import { refreshSampleDataWorker } from './integration-data-checker/refreshSampleDataWorker' /** * Worker factory for spawning different microservices @@ -49,6 +50,9 @@ async function workerFactory(event: NodeMicroserviceMessage): Promise { integrationDataCheckerMessage.integrationId, integrationDataCheckerMessage.tenantId, ) + case 'refresh-sample-data': + return refreshSampleDataWorker() + case 'csv-export': const csvExportMessage = event as CsvExportMessage return csvExportWorker( From 9ecaf13ba9dffdd84b75d8ca331e21b185d7311d Mon Sep 17 00:00:00 2001 From: Joan Reyero Date: Fri, 17 Mar 2023 11:02:12 +0100 Subject: [PATCH 2/6] Fixed query --- backend/src/bin/jobs/refreshSampleData.ts | 6 +-- .../refreshSampleDataWorker.ts | 43 +++++++++---------- 2 files changed, 24 insertions(+), 25 deletions(-) diff --git a/backend/src/bin/jobs/refreshSampleData.ts b/backend/src/bin/jobs/refreshSampleData.ts index 6cf033c1c8..9aeba5f0f7 100644 --- a/backend/src/bin/jobs/refreshSampleData.ts +++ b/backend/src/bin/jobs/refreshSampleData.ts @@ -4,11 +4,11 @@ import { NodeWorkerMessageType } from '../../serverless/types/workerTypes' import { NodeWorkerMessageBase } from '../../types/mq/nodeWorkerMessageBase' const job: CrowdJob = { - name: 'Integration Data Checker', + name: 'Refresh sample data', // every day // cronTime: '0 0 * * *', - // every minute - cronTime: '* * * * *', + // every hour + cronTime: '0 * * * *', onTrigger: async () => { console.log('triggering!') await sendNodeWorkerMessage('refresh-sample-data', { diff --git a/backend/src/serverless/microservices/nodejs/integration-data-checker/refreshSampleDataWorker.ts b/backend/src/serverless/microservices/nodejs/integration-data-checker/refreshSampleDataWorker.ts index 5cacde6b78..f21ccdba35 100644 --- a/backend/src/serverless/microservices/nodejs/integration-data-checker/refreshSampleDataWorker.ts +++ b/backend/src/serverless/microservices/nodejs/integration-data-checker/refreshSampleDataWorker.ts @@ -1,36 +1,35 @@ -import { Sequelize } from 'sequelize/types' import { QueryTypes } from 'sequelize' - import { API_CONFIG, SAMPLE_DATA_CONFIG } from '../../../../config' - -const sequelize = new Sequelize(/* your sequelize configuration */) +import getUserContext from '../../../../database/utils/getUserContext' async function refreshSampleDataWorker(): Promise { - if (API_CONFIG.edition === 'crowd-hosted') { - const tenantId = SAMPLE_DATA_CONFIG.tenantId - const updateDays = 1 // Or any number of days you want to add + // if (API_CONFIG.edition === 'crowd-hosted') { + const tenantId = SAMPLE_DATA_CONFIG.tenantId + const userContext = await getUserContext(SAMPLE_DATA_CONFIG.tenantId) + const updateDays = 1 - const tables = [ - { name: 'activities', columns: ['createdAt', 'timestamp'] }, - { name: 'members', columns: ['joinedAt', 'createdAt'] }, - { name: 'notes', columns: ['createdAt'] }, - { name: 'conversations', columns: ['createdAt'] }, - { name: 'organizations', columns: ['createdAt'] }, - { name: 'tags', columns: ['createdAt'] }, - ] + const tables = [ + { name: 'activities', columns: ['createdAt', 'timestamp'] }, + { name: 'members', columns: ['joinedAt', 'createdAt'] }, + { name: 'notes', columns: ['createdAt'] }, + { name: 'conversations', columns: ['createdAt'] }, + { name: 'organizations', columns: ['createdAt'] }, + { name: 'tags', columns: ['createdAt'] }, + ] - for (const table of tables) { - for (const column of table.columns) { - const query = ` + for (const table of tables) { + for (const column of table.columns) { + const query = ` UPDATE ${table.name} - SET ${column} = ${column} + INTERVAL '${updateDays} days' - WHERE tenantId = ${tenantId}; + SET "${column}" = "${column}" + INTERVAL '${updateDays} days' + WHERE "tenantId" = '${tenantId}'; ` - await sequelize.query(query, { type: QueryTypes.UPDATE }) - } + console.log(query) + await userContext.database.sequelize.query(query, { type: QueryTypes.UPDATE }) } } + // } } export { refreshSampleDataWorker } From d9cbd0f6ef67a7e4ea796ed7d8a824e97a8d2199 Mon Sep 17 00:00:00 2001 From: Joan Reyero Date: Fri, 17 Mar 2023 11:10:02 +0100 Subject: [PATCH 3/6] Added comments --- .../refreshSampleDataWorker.ts | 40 ++++++++++--------- 1 file changed, 21 insertions(+), 19 deletions(-) diff --git a/backend/src/serverless/microservices/nodejs/integration-data-checker/refreshSampleDataWorker.ts b/backend/src/serverless/microservices/nodejs/integration-data-checker/refreshSampleDataWorker.ts index f21ccdba35..6bf43fa529 100644 --- a/backend/src/serverless/microservices/nodejs/integration-data-checker/refreshSampleDataWorker.ts +++ b/backend/src/serverless/microservices/nodejs/integration-data-checker/refreshSampleDataWorker.ts @@ -3,33 +3,35 @@ import { API_CONFIG, SAMPLE_DATA_CONFIG } from '../../../../config' import getUserContext from '../../../../database/utils/getUserContext' async function refreshSampleDataWorker(): Promise { - // if (API_CONFIG.edition === 'crowd-hosted') { - const tenantId = SAMPLE_DATA_CONFIG.tenantId - const userContext = await getUserContext(SAMPLE_DATA_CONFIG.tenantId) - const updateDays = 1 + // This is only needed for hosted edition + if (API_CONFIG.edition === 'crowd-hosted') { + const tenantId = SAMPLE_DATA_CONFIG.tenantId + const userContext = await getUserContext(SAMPLE_DATA_CONFIG.tenantId) + const updateDays = 1 // Every day we need to refresh - const tables = [ - { name: 'activities', columns: ['createdAt', 'timestamp'] }, - { name: 'members', columns: ['joinedAt', 'createdAt'] }, - { name: 'notes', columns: ['createdAt'] }, - { name: 'conversations', columns: ['createdAt'] }, - { name: 'organizations', columns: ['createdAt'] }, - { name: 'tags', columns: ['createdAt'] }, - ] + // These are all the tables that have columns that need to be updated + const tables = [ + { name: 'activities', columns: ['createdAt', 'timestamp'] }, + { name: 'members', columns: ['joinedAt', 'createdAt'] }, + { name: 'notes', columns: ['createdAt'] }, + { name: 'conversations', columns: ['createdAt'] }, + { name: 'organizations', columns: ['createdAt'] }, + { name: 'tags', columns: ['createdAt'] }, + ] - for (const table of tables) { - for (const column of table.columns) { - const query = ` + // We are using a direct query because it is very specific functionality. + // There is no point creating repository methods. + for (const table of tables) { + for (const column of table.columns) { + const query = ` UPDATE ${table.name} SET "${column}" = "${column}" + INTERVAL '${updateDays} days' WHERE "tenantId" = '${tenantId}'; ` - - console.log(query) - await userContext.database.sequelize.query(query, { type: QueryTypes.UPDATE }) + await userContext.database.sequelize.query(query, { type: QueryTypes.UPDATE }) + } } } - // } } export { refreshSampleDataWorker } From b691520630c67e13ededbea0ba157b496eaaf06b Mon Sep 17 00:00:00 2001 From: Joan Reyero Date: Mon, 20 Mar 2023 13:20:20 +0100 Subject: [PATCH 4/6] Update to 8 days --- backend/src/bin/jobs/index.ts | 2 +- backend/src/bin/jobs/refreshSampleData.ts | 5 +---- .../integration-data-checker/refreshSampleDataWorker.ts | 2 +- 3 files changed, 3 insertions(+), 6 deletions(-) diff --git a/backend/src/bin/jobs/index.ts b/backend/src/bin/jobs/index.ts index 12bfaca2f3..52ea31e0dc 100644 --- a/backend/src/bin/jobs/index.ts +++ b/backend/src/bin/jobs/index.ts @@ -18,7 +18,7 @@ const jobs: CrowdJob[] = [ downgradeExpiredPlans, eagleEyeEmailDigestTicks, integrationDataChecker, - refreshSampleData + refreshSampleData, ] export default jobs diff --git a/backend/src/bin/jobs/refreshSampleData.ts b/backend/src/bin/jobs/refreshSampleData.ts index 9aeba5f0f7..efed37c975 100644 --- a/backend/src/bin/jobs/refreshSampleData.ts +++ b/backend/src/bin/jobs/refreshSampleData.ts @@ -6,11 +6,8 @@ import { NodeWorkerMessageBase } from '../../types/mq/nodeWorkerMessageBase' const job: CrowdJob = { name: 'Refresh sample data', // every day - // cronTime: '0 0 * * *', - // every hour - cronTime: '0 * * * *', + cronTime: '0 0 * * *', onTrigger: async () => { - console.log('triggering!') await sendNodeWorkerMessage('refresh-sample-data', { type: NodeWorkerMessageType.NODE_MICROSERVICE, service: 'refresh-sample-data', diff --git a/backend/src/serverless/microservices/nodejs/integration-data-checker/refreshSampleDataWorker.ts b/backend/src/serverless/microservices/nodejs/integration-data-checker/refreshSampleDataWorker.ts index 6bf43fa529..69b969c42f 100644 --- a/backend/src/serverless/microservices/nodejs/integration-data-checker/refreshSampleDataWorker.ts +++ b/backend/src/serverless/microservices/nodejs/integration-data-checker/refreshSampleDataWorker.ts @@ -7,7 +7,7 @@ async function refreshSampleDataWorker(): Promise { if (API_CONFIG.edition === 'crowd-hosted') { const tenantId = SAMPLE_DATA_CONFIG.tenantId const userContext = await getUserContext(SAMPLE_DATA_CONFIG.tenantId) - const updateDays = 1 // Every day we need to refresh + const updateDays = 8 // Every day we need to refresh // These are all the tables that have columns that need to be updated const tables = [ From 4bf521f5ae440d87a2c27b0f186a75dddd7dc1bb Mon Sep 17 00:00:00 2001 From: Joan Reyero Date: Mon, 20 Mar 2023 13:22:08 +0100 Subject: [PATCH 5/6] Back to 1 day --- backend/src/bin/jobs/refreshSampleData.ts | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) diff --git a/backend/src/bin/jobs/refreshSampleData.ts b/backend/src/bin/jobs/refreshSampleData.ts index efed37c975..21383c6ab4 100644 --- a/backend/src/bin/jobs/refreshSampleData.ts +++ b/backend/src/bin/jobs/refreshSampleData.ts @@ -5,8 +5,9 @@ import { NodeWorkerMessageBase } from '../../types/mq/nodeWorkerMessageBase' const job: CrowdJob = { name: 'Refresh sample data', - // every day - cronTime: '0 0 * * *', + // every hour + cronTime: '0 * * * *', + // cronTime: '0 0 * * *', onTrigger: async () => { await sendNodeWorkerMessage('refresh-sample-data', { type: NodeWorkerMessageType.NODE_MICROSERVICE, From 16580fc97732f438dd517df8351cc519bee55020 Mon Sep 17 00:00:00 2001 From: Joan Reyero Date: Tue, 21 Mar 2023 16:38:03 +0100 Subject: [PATCH 6/6] Update refreshSampleDataWorker.ts --- .../nodejs/integration-data-checker/refreshSampleDataWorker.ts | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/backend/src/serverless/microservices/nodejs/integration-data-checker/refreshSampleDataWorker.ts b/backend/src/serverless/microservices/nodejs/integration-data-checker/refreshSampleDataWorker.ts index 69b969c42f..e370c9adb0 100644 --- a/backend/src/serverless/microservices/nodejs/integration-data-checker/refreshSampleDataWorker.ts +++ b/backend/src/serverless/microservices/nodejs/integration-data-checker/refreshSampleDataWorker.ts @@ -7,7 +7,7 @@ async function refreshSampleDataWorker(): Promise { if (API_CONFIG.edition === 'crowd-hosted') { const tenantId = SAMPLE_DATA_CONFIG.tenantId const userContext = await getUserContext(SAMPLE_DATA_CONFIG.tenantId) - const updateDays = 8 // Every day we need to refresh + const updateDays = 9 // Every day we need to refresh // These are all the tables that have columns that need to be updated const tables = [