Twitter follows duplicate activities bug

backend/src/database/initializers/twitterSourceIdsFixedTimestamps.ts

@@ -0,0 +1,61 @@
+/**
+ * This script is responsible for regenerating
+ * sourceIds for twitter follow activities that have timestamp > 1970-01-01
+ */
+
+import dotenv from 'dotenv'
+import dotenvExpand from 'dotenv-expand'
+import BaseIterator from '../../serverless/integrations/iterators/baseIterator'
+import ActivityService from '../../services/activityService'
+import IntegrationService from '../../services/integrationService'
+import TenantService from '../../services/tenantService'
+import { PlatformType } from '../../utils/platforms'
+import getUserContext from '../utils/getUserContext'
+
+const path = require('path')
+
+const env = dotenv.config({
+  path: path.resolve(__dirname, `../../../.env.staging`),
+})
+
+dotenvExpand.expand(env)
+
+async function twitterFollowsFixSourceIdsWithTimestamp() {
+  const tenants = await TenantService._findAndCountAllForEveryUser({})
+
+  // for each tenant
+  for (const t of tenants.rows) {
+    const tenantId = t.id
+    // get user context
+    const userContext = await getUserContext(tenantId)
+    const integrationService = new IntegrationService(userContext)
+
+    const twitterIntegration = (
+      await integrationService.findAndCountAll({ filter: { platform: PlatformType.TWITTER } })
+    ).rows[0]
+
+    if (twitterIntegration) {
+      const actService = new ActivityService(userContext)
+
+      // get activities where timestamp != 1970-01-01, we can query by > 2000-01-01
+      const activities = await actService.findAndCountAll({
+        filter: { type: 'follow', timestampRange: ['2000-01-01'] },
+      })
+
+      for (const activity of activities.rows) {
+        console.log(activity)
+        console.log(activity.communityMember)
+        // calculate sourceId with fixed timestamps
+        const sourceIdRegenerated = BaseIterator.generateSourceIdHash(
+          activity.communityMember.username.twitter,
+          'follow',
+          '1970-01-01T00:00:00+00:00',
+          'twitter',
+        )
+        await actService.update(activity.id, { sourceId: sourceIdRegenerated })
+      }
+    }
+  }
+}
+
+twitterFollowsFixSourceIdsWithTimestamp()

backend/src/serverless/integrations/iterators/__tests__/twitterIterator.test.ts

@@ -378,7 +378,7 @@ describe('Integrations worker static tests', () => {
           sourceId: BaseIterator.generateSourceIdHash(
             'michael_scott',
             'follow',
-            moment(activities[0].timestamp).unix().toString(),
+            '0',
             PlatformType.TWITTER,
           ),
           timestamp: moment('1970-01-01T00:00:00+00:00').utc().toDate(),
@@ -405,7 +405,7 @@ describe('Integrations worker static tests', () => {
           sourceId: BaseIterator.generateSourceIdHash(
             'dwight_schrute',
             'follow',
-            moment(activities[1].timestamp).unix().toString(),
+            '0',
             PlatformType.TWITTER,
           ),
           timestamp: moment('1970-01-01T00:00:00+00:00').utc().toDate(),
@@ -456,7 +456,7 @@ describe('Integrations worker static tests', () => {
           sourceId: BaseIterator.generateSourceIdHash(
             'michael_scott',
             'follow',
-            moment(activities[0].timestamp).unix().toString(),
+            '0',
             PlatformType.TWITTER,
           ),
           timestamp: moment('1970-01-01T00:00:00+00:00').utc().toDate(),
@@ -523,7 +523,7 @@ describe('Integrations worker static tests', () => {
           sourceId: BaseIterator.generateSourceIdHash(
             'michael_scott',
             'follow',
-            moment(activities[0].timestamp).unix().toString(),
+            '0',
             PlatformType.TWITTER,
           ),
           timestamp: moment('1970-01-01T00:00:00+00:00').utc().toDate(),
@@ -550,7 +550,7 @@ describe('Integrations worker static tests', () => {
           sourceId: BaseIterator.generateSourceIdHash(
             'dwight_schrute',
             'follow',
-            moment(activities[1].timestamp).unix().toString(),
+            '0',
             PlatformType.TWITTER,
           ),
           timestamp: moment('1970-01-01T00:00:00+00:00').utc().toDate(),

backend/src/serverless/integrations/iterators/twitterIterator.ts

@@ -279,7 +279,7 @@ export default class TwitterIterator extends BaseIterator {
       sourceId: BaseIterator.generateSourceIdHash(
         record.username,
         'follow',
-        timestampObj.unix().toString(),
+        moment('1970-01-01T00:00:00+00:00').utc().unix().toString(),
         PlatformType.TWITTER,
       ),
       // When onboarding we need a super old date. Otherwise we can place it in a 2h window