Fixing a few issues with unleash and new services (#927)

backend/src/api/member/memberExport.ts

@@ -1,5 +1,5 @@
 import { RedisCache } from '@crowd/redis'
-import { getSecondsTillEndOfMonth } from '@crowd/common'
+import { getSecondsTillEndOfMonth } from '../../utils/timing'
 import Permissions from '../../security/permissions'
 import identifyTenant from '../../segment/identifyTenant'
 import track from '../../segment/track'

backend/src/api/premium/enrichment/memberEnrich.ts

@@ -1,6 +1,6 @@
 import { RedisCache } from '@crowd/redis'
 import { getServiceLogger } from '@crowd/logging'
-import { getSecondsTillEndOfMonth } from '@crowd/common'
+import { getSecondsTillEndOfMonth } from '../../../utils/timing'
 import Permissions from '../../../security/permissions'
 import identifyTenant from '../../../segment/identifyTenant'
 import MemberEnrichmentService from '../../../services/premium/enrichment/memberEnrichmentService'

backend/src/api/premium/enrichment/memberEnrichBulk.ts

@@ -1,6 +1,6 @@
 import { RedisCache } from '@crowd/redis'
 import { getServiceLogger } from '@crowd/logging'
-import { getSecondsTillEndOfMonth } from '@crowd/common'
+import { getSecondsTillEndOfMonth } from '../../../utils/timing'
 import Error403 from '../../../errors/Error403'
 import Permissions from '../../../security/permissions'
 import identifyTenant from '../../../segment/identifyTenant'

backend/src/bin/scripts/unleash-init.ts

@@ -153,6 +153,42 @@ const constaintConfiguration = {
       },
     ],
   ],
+  [FeatureFlag.ORGANIZATION_ENRICHMENT]: [
+    [
+      {
+        values: [Plans.values.growth],
+        inverted: false,
+        operator: 'IN',
+        contextName: 'plan',
+        caseInsensitive: false,
+      },
+      {
+        value: PLAN_LIMITS[Plans.values.growth][FeatureFlag.ORGANIZATION_ENRICHMENT].toString(),
+        values: [],
+        inverted: false,
+        operator: 'NUM_LT',
+        contextName: 'organizationEnrichmentCount',
+        caseInsensitive: false,
+      },
+    ],
+    [
+      {
+        values: [Plans.values.essential],
+        inverted: false,
+        operator: 'IN',
+        contextName: 'plan',
+        caseInsensitive: false,
+      },
+      {
+        value: PLAN_LIMITS[Plans.values.essential][FeatureFlag.ORGANIZATION_ENRICHMENT].toString(),
+        values: [],
+        inverted: false,
+        operator: 'NUM_LT',
+        contextName: 'organizationEnrichmentCount',
+        caseInsensitive: false,
+      },
+    ],
+  ],
 }
 
 let seq: any

backend/src/feature-flags/getFeatureFlagTenantContext.ts

@@ -1,6 +1,6 @@
 import { RedisCache, RedisClient } from '@crowd/redis'
 import { Logger } from '@crowd/logging'
-import { getSecondsTillEndOfMonth } from '@crowd/common'
+import { getSecondsTillEndOfMonth } from '../utils/timing'
 import AutomationRepository from '../database/repositories/automationRepository'
 import { FeatureFlagRedisKey } from '../types/common'
 

backend/src/feature-flags/isFeatureEnabled.ts

@@ -9,6 +9,7 @@ export const PLAN_LIMITS = {
     [FeatureFlag.AUTOMATIONS]: 2,
     [FeatureFlag.CSV_EXPORT]: 2,
     [FeatureFlag.MEMBER_ENRICHMENT]: 5,
+    [FeatureFlag.ORGANIZATION_ENRICHMENT]: 5,
   },
   [Plans.values.growth]: {
     [FeatureFlag.AUTOMATIONS]: 10,

backend/src/serverless/microservices/nodejs/bulk-enrichment/bulkEnrichmentWorker.ts

@@ -1,5 +1,5 @@
 import { getRedisClient, RedisCache } from '@crowd/redis'
-import { getSecondsTillEndOfMonth } from '@crowd/common'
+import { getSecondsTillEndOfMonth } from '../../../../utils/timing'
 import { REDIS_CONFIG } from '../../../../conf'
 import getUserContext from '../../../../database/utils/getUserContext'
 import MemberEnrichmentService from '../../../../services/premium/enrichment/memberEnrichmentService'

backend/src/serverless/microservices/nodejs/bulk-enrichment/bulkOrganizationEnrichmentWorker.ts

@@ -1,5 +1,5 @@
 import { getRedisClient, RedisCache } from '@crowd/redis'
-import { getSecondsTillEndOfMonth } from '@crowd/common'
+import { getSecondsTillEndOfMonth } from '../../../../utils/timing'
 import { ORGANIZATION_ENRICHMENT_CONFIG, REDIS_CONFIG } from '../../../../conf'
 import getUserContext from '../../../../database/utils/getUserContext'
 import { PLAN_LIMITS } from '../../../../feature-flags/isFeatureEnabled'

backend/src/utils/timing.ts

@@ -0,0 +1,8 @@
+import moment from 'moment'
+
+export const getSecondsTillEndOfMonth = () => {
+  const endTime = moment().endOf('month')
+  const startTime = moment()
+
+  return endTime.diff(startTime, 'seconds')
+}

scripts/cli

@@ -335,6 +335,7 @@ function migrate_test() {
 
 function source_edition() {
     __CROWD_EDITION=$(source $CLI_HOME/../backend/.env.dist.local && source $CLI_HOME/../backend/.env.override.local && echo $CROWD_EDITION)
+    echo "Crowd edition detected: $__CROWD_EDITION"
 }
 
 function check_init_premium() {
@@ -345,7 +346,12 @@ function check_init_premium() {
     fi
 }
 
+function install_libs() {
+    (cd $CLI_HOME/../services/scripts && ./install_lib_packages.sh && ./build_libs.sh)
+}
+
 function init_unleash() {
+    install_libs
     (cd $CLI_HOME/../backend && source source-local.sh && npm i && npm run script:unleash-init)
 }
 

scripts/scaffold/sqs/queue.conf

@@ -32,6 +32,34 @@ queues {
     delay = 0 seconds
     defaultVisibilityTimeout = 0 seconds
   }
+  "integration-run-worker.fifo" {
+    fifo = true
+    contentBasedDeduplication = false
+    receiveMessageWait = 30 seconds
+    delay = 0 seconds
+    defaultVisibilityTimeout = 0 seconds
+  }
+  "integration-data-worker.fifo" {
+    fifo = true
+    contentBasedDeduplication = false
+    receiveMessageWait = 30 seconds
+    delay = 0 seconds
+    defaultVisibilityTimeout = 0 seconds
+  }
+  "integration-stream-worker.fifo" {
+    fifo = true
+    contentBasedDeduplication = false
+    receiveMessageWait = 30 seconds
+    delay = 0 seconds
+    defaultVisibilityTimeout = 0 seconds
+  }
+  "data-sink-worker.fifo" {
+    fifo = true
+    contentBasedDeduplication = false
+    receiveMessageWait = 30 seconds
+    delay = 0 seconds
+    defaultVisibilityTimeout = 0 seconds
+  }
 }
 
 aws {

services/apps/data_sink_worker/src/bin/restart-failed-results.ts

@@ -1,10 +1,8 @@
 import { DB_CONFIG, SQS_CONFIG } from '@/conf'
-import { DataSinkWorkerEmitter } from '@crowd/sqs'
 import DataSinkRepository from '@/repo/dataSink.repo'
-import { processPaginated } from '@crowd/common'
 import { DbStore, getDbConnection } from '@crowd/database'
 import { getServiceLogger } from '@crowd/logging'
-import { getSqsClient } from '@crowd/sqs'
+import { DataSinkWorkerEmitter, getSqsClient } from '@crowd/sqs'
 import { ProcessIntegrationResultQueueMessage } from '@crowd/types'
 
 const log = getServiceLogger()

services/libs/common/src/timing.ts

@@ -8,12 +8,3 @@ export const addSeconds = (date: Date, seconds: number): Date => {
   newDate.setSeconds(newDate.getSeconds() + seconds)
   return newDate
 }
-
-export const getSecondsTillEndOfMonth = (): number => {
-  const now = new Date()
-  const endOfMonth = new Date(now.getFullYear(), now.getMonth() + 1, 0)
-
-  const diffInSeconds = (endOfMonth.getTime() - now.getTime()) / 1000
-
-  return diffInSeconds
-}