Brought merge suggestions to Node.js

backend/src/bin/jobs/index.ts

@@ -7,6 +7,7 @@ import refreshMaterializedViews from './refreshMaterializedViews'
 import downgradeExpiredPlans from './downgradeExpiredPlans'
 import eagleEyeEmailDigestTicks from './eagleEyeEmailDigestTicks'
 import integrationDataChecker from './integrationDataChecker'
+import mergeSuggestions from './mergeSuggestions'
 import refreshSampleData from './refreshSampleData'
 import cleanUpIntegrationRuns from './cleanUpIntegrationRuns'
 import checkStuckIntegrationRuns from './checkStuckIntegrationRuns'
@@ -20,6 +21,7 @@ const jobs: CrowdJob[] = [
   downgradeExpiredPlans,
   eagleEyeEmailDigestTicks,
   integrationDataChecker,
+  mergeSuggestions,
   refreshSampleData,
   cleanUpIntegrationRuns,
   checkStuckIntegrationRuns,

backend/src/bin/jobs/mergeSuggestions.ts

@@ -0,0 +1,27 @@
+import TenantService from '../../services/tenantService'
+import { CrowdJob } from '../../types/jobTypes'
+import { sendNodeWorkerMessage } from '../../serverless/utils/nodeWorkerSQS'
+import { NodeWorkerMessageType } from '../../serverless/types/workerTypes'
+import { NodeWorkerMessageBase } from '../../types/mq/nodeWorkerMessageBase'
+import { timeout } from '../../utils/timing'
+
+const job: CrowdJob = {
+  name: 'Merge suggestions',
+  // every hour
+  cronTime: '0 * * * *',
+  onTrigger: async () => {
+    const tenants = await TenantService._findAndCountAllForEveryUser({})
+    for (const tenant of tenants.rows) {
+      await sendNodeWorkerMessage(tenant.id, {
+        type: NodeWorkerMessageType.NODE_MICROSERVICE,
+        tenant: tenant.id,
+        service: 'merge-suggestions',
+      } as NodeWorkerMessageBase)
+
+      // Wait 1 second between messages to potentially reduce spike load on cube between each tenant runs
+      await timeout(1000)
+    }
+  },
+}
+
+export default job

backend/src/database/initializers/entities/2022-04-05-add-microservices.ts

@@ -15,15 +15,6 @@ export default async () => {
     const ws = new WidgetService(userContext)
     const is = new IntegrationService(userContext)
 
-    // add check_merge microservice
-    const checkMergeMicroservice = {
-      init: true,
-      running: false,
-      type: microserviceTypes.checkMerge,
-      variant: 'default',
-    }
-    await ms.create(checkMergeMicroservice)
-
     // add members_score microservice
     const membersScoreMicroservice = {
       init: true,

backend/src/database/migrations/U1682100889__members-to-merge-similarity.sql

@@ -0,0 +1,2 @@
+-- Remove the 'similarity' column from the memberToMerge table
+ALTER TABLE "memberToMerge" DROP COLUMN similarity;

backend/src/database/migrations/V1682098472__fuzzy-postgres.sql

@@ -0,0 +1 @@
+CREATE EXTENSION IF NOT EXISTS pg_trgm;

backend/src/database/migrations/V1682100889__members-to-merge-similarity.sql

@@ -0,0 +1,2 @@
+-- Add the 'similarity' column to the memberToMerge table (allow NULL values)
+ALTER TABLE "memberToMerge" ADD COLUMN similarity double precision;

backend/src/database/repositories/__tests__/memberRepository.test.ts

@@ -3169,107 +3169,6 @@ describe('MemberRepository tests', () => {
     })
   })
 
-  describe('addToMerge method', () => {
-    it('Should add a member to other members toMerge list', async () => {
-      const mockIRepositoryOptions = await SequelizeTestUtils.getTestIRepositoryOptions(db)
-
-      const member1 = {
-        username: {
-          [PlatformType.DISCORD]: {
-            username: 'anil',
-            integrationId: generateUUIDv1(),
-          },
-        },
-        displayName: 'Member 1',
-        joinedAt: '2020-05-27T15:13:30Z',
-      }
-
-      const member2 = {
-        username: {
-          [PlatformType.DISCORD]: {
-            username: 'anil2',
-            integrationId: generateUUIDv1(),
-          },
-        },
-        displayName: 'Member 2',
-        joinedAt: '2020-05-27T15:13:30Z',
-      }
-
-      const memberCreated1 = await MemberRepository.create(member1, mockIRepositoryOptions)
-      const memberCreated2 = await MemberRepository.create(member2, mockIRepositoryOptions)
-
-      const memberUpdated1 = await MemberRepository.addToMerge(
-        memberCreated1.id,
-        memberCreated2.id,
-        mockIRepositoryOptions,
-      )
-      const memberUpdated2 = await MemberRepository.addToMerge(
-        memberCreated2.id,
-        memberCreated1.id,
-        mockIRepositoryOptions,
-      )
-
-      expect(memberUpdated1.toMerge[0]).toBe(memberUpdated2.id)
-      expect(memberUpdated2.toMerge[0]).toBe(memberUpdated1.id)
-    })
-
-    it('Should return same result for multiple addToMerge calls for the same member', async () => {
-      const mockIRepositoryOptions = await SequelizeTestUtils.getTestIRepositoryOptions(db)
-
-      const member1 = {
-        username: {
-          [PlatformType.DISCORD]: {
-            username: 'anil',
-            integrationId: generateUUIDv1(),
-          },
-        },
-        displayName: 'Member 1',
-        joinedAt: '2020-05-27T15:13:30Z',
-      }
-
-      const member2 = {
-        username: {
-          [PlatformType.DISCORD]: {
-            username: 'anil2',
-            integrationId: generateUUIDv1(),
-          },
-        },
-        displayName: 'Member 2',
-        joinedAt: '2020-05-27T15:13:30Z',
-      }
-
-      const memberCreated1 = await MemberRepository.create(member1, mockIRepositoryOptions)
-      const memberCreated2 = await MemberRepository.create(member2, mockIRepositoryOptions)
-
-      const memberUpdated1 = await MemberRepository.addToMerge(
-        memberCreated1.id,
-        memberCreated2.id,
-        mockIRepositoryOptions,
-      )
-      const memberUpdated2 = await MemberRepository.addToMerge(
-        memberCreated2.id,
-        memberCreated1.id,
-        mockIRepositoryOptions,
-      )
-
-      // multiple calls to for same (member, mergeMember) should result in no change
-      await MemberRepository.addToMerge(
-        memberCreated2.id,
-        memberCreated1.id,
-        mockIRepositoryOptions,
-      )
-      await MemberRepository.addToMerge(
-        memberCreated2.id,
-        memberCreated1.id,
-        mockIRepositoryOptions,
-      )
-
-      expect(memberUpdated1.toMerge.length).toBe(1)
-      expect(memberUpdated1.toMerge[0]).toBe(memberUpdated2.id)
-      expect(memberUpdated2.toMerge[0]).toBe(memberUpdated1.id)
-    })
-  })
-
   describe('removeToMerge method', () => {
     it('Should remove a member from other members toMerge list', async () => {
       const mockIRepositoryOptions = await SequelizeTestUtils.getTestIRepositoryOptions(db)
@@ -3299,28 +3198,28 @@ describe('MemberRepository tests', () => {
       const memberCreated1 = await MemberRepository.create(member1, mockIRepositoryOptions)
       const memberCreated2 = await MemberRepository.create(member2, mockIRepositoryOptions)
 
-      let memberUpdated1 = await MemberRepository.addToMerge(
-        memberCreated1.id,
-        memberCreated2.id,
+      await MemberRepository.addToMerge(
+        [{ members: [memberCreated1.id, memberCreated2.id], similarity: null }],
         mockIRepositoryOptions,
       )
-      const memberUpdated2 = await MemberRepository.addToMerge(
-        memberCreated2.id,
-        memberCreated1.id,
+      await MemberRepository.addToMerge(
+        [{ members: [memberCreated2.id, memberCreated1.id], similarity: null }],
         mockIRepositoryOptions,
       )
 
-      memberUpdated1 = await MemberRepository.removeToMerge(
+      let m1 = await MemberRepository.findById(memberCreated1.id, mockIRepositoryOptions)
+      const m2 = await MemberRepository.findById(memberCreated2.id, mockIRepositoryOptions)
+      m1 = await MemberRepository.removeToMerge(
         memberCreated1.id,
         memberCreated2.id,
         mockIRepositoryOptions,
       )
 
       // Member2 should be removed from Member1.toMerge
-      expect(memberUpdated1.toMerge.length).toBe(0)
+      expect(m1.toMerge.length).toBe(0)
 
       // Member1 is still in member2.toMerge list
-      expect(memberUpdated2.toMerge[0]).toBe(memberUpdated1.id)
+      expect(m2.toMerge[0]).toBe(m1.id)
     })
   })
 

backend/src/database/repositories/__tests__/microserviceRepository.test.ts

@@ -19,7 +19,7 @@ describe('MicroserviceRepository tests', () => {
     it('Should create a microservice succesfully with default values', async () => {
       const mockIRepositoryOptions = await SequelizeTestUtils.getTestIRepositoryOptions(db)
 
-      const microservice2Add = { type: 'check_merge' }
+      const microservice2Add = { type: 'members_score' }
 
       const microserviceCreated = await MicroserviceRepository.create(
         microservice2Add,
@@ -53,7 +53,7 @@ describe('MicroserviceRepository tests', () => {
       const microservice2Add = {
         init: true,
         running: true,
-        type: 'check_merge',
+        type: 'members_score',
         variant: 'premium',
         settings: { testSettingsField: 'test' },
       }
@@ -90,15 +90,15 @@ describe('MicroserviceRepository tests', () => {
       const microservice1 = {
         init: true,
         running: true,
-        type: 'check_merge',
+        type: 'members_score',
         variant: 'premium',
         settings: { testSettingsField: 'test' },
       }
 
       await MicroserviceRepository.create(microservice1, mockIRepositoryOptions)
 
       await expect(() =>
-        MicroserviceRepository.create({ type: 'check_merge' }, mockIRepositoryOptions),
+        MicroserviceRepository.create({ type: 'members_score' }, mockIRepositoryOptions),
       ).rejects.toThrow()
     })
 
@@ -122,7 +122,7 @@ describe('MicroserviceRepository tests', () => {
     it('Should successfully find created microservice by id', async () => {
       const mockIRepositoryOptions = await SequelizeTestUtils.getTestIRepositoryOptions(db)
 
-      const microservice2Add = { type: 'check_merge' }
+      const microservice2Add = { type: 'members_score' }
 
       const microserviceCreated = await MicroserviceRepository.create(
         microservice2Add,
@@ -170,11 +170,11 @@ describe('MicroserviceRepository tests', () => {
       const mockIRepositoryOptions = await SequelizeTestUtils.getTestIRepositoryOptions(db)
 
       const microservice1Created = await MicroserviceRepository.create(
-        { type: 'check_merge' },
+        { type: 'members_score' },
         mockIRepositoryOptions,
       )
       const microservice2Created = await MicroserviceRepository.create(
-        { type: 'members_score' },
+        { type: 'second' },
         mockIRepositoryOptions,
       )
 
@@ -190,7 +190,7 @@ describe('MicroserviceRepository tests', () => {
       const mockIRepositoryOptions = await SequelizeTestUtils.getTestIRepositoryOptions(db)
 
       const microserviceCreated = await MicroserviceRepository.create(
-        { type: 'check_merge' },
+        { type: 'members_score' },
         mockIRepositoryOptions,
       )
 
@@ -208,7 +208,7 @@ describe('MicroserviceRepository tests', () => {
       let mockIRepositoryOptions = await SequelizeTestUtils.getTestIRepositoryOptions(db)
 
       const microserviceCreated = await MicroserviceRepository.create(
-        { type: 'check_merge' },
+        { type: 'members_score' },
         mockIRepositoryOptions,
       )
 
@@ -229,18 +229,18 @@ describe('MicroserviceRepository tests', () => {
       const mockIRepositoryOptions = await SequelizeTestUtils.getTestIRepositoryOptions(db)
 
       const microservice1Created = await MicroserviceRepository.create(
-        { type: 'check_merge', variant: 'premium' },
+        { type: 'members_score', variant: 'premium' },
         mockIRepositoryOptions,
       )
 
       const microservice2Created = await MicroserviceRepository.create(
-        { type: 'members_score', variant: 'premium' },
+        { type: 'second', variant: 'premium' },
         mockIRepositoryOptions,
       )
 
       // Filter by type
       let microservices = await MicroserviceRepository.findAndCountAll(
-        { filter: { type: 'check_merge' } },
+        { filter: { type: 'members_score' } },
         mockIRepositoryOptions,
       )
 
@@ -329,7 +329,7 @@ describe('MicroserviceRepository tests', () => {
       const mockIRepositoryOptions = await SequelizeTestUtils.getTestIRepositoryOptions(db)
 
       const microserviceCreated = await MicroserviceRepository.create(
-        { type: 'check_merge', variant: 'premium' },
+        { type: 'members_score', variant: 'premium' },
         mockIRepositoryOptions,
       )
 
@@ -435,7 +435,7 @@ describe('MicroserviceRepository tests', () => {
       }
 
       const ms2 = {
-        type: 'check_merge',
+        type: 'members_score',
         running: false,
         init: false,
         variant: 'default',