From c308d578e3a77477d79e13f057eebd7dd722ca7d Mon Sep 17 00:00:00 2001 From: Gosha Date: Fri, 15 Dec 2023 12:58:06 +0200 Subject: [PATCH 1/5] refactor: move inbound mail parser to worker --- apps/api/src/app/auth/usecases/login/login.usecase.ts | 3 +-- .../auth/usecases/register/user-register.usecase.ts | 3 +-- apps/api/src/app/inbound-parse/inbound-parse.module.ts | 4 +--- apps/api/src/app/inbound-parse/usecases/index.ts | 4 ++-- apps/api/src/app/shared/helpers/hmac.service.ts | 8 -------- .../usecases/chat-oauth/chat-oauth.usecase.ts | 2 +- .../usecases/get-my-profile/get-my-profile.usecase.ts | 4 +++- .../initialize-session/initialize-session.usecase.ts | 2 +- apps/worker/src/app/shared/utils/hmac.ts | 8 -------- apps/worker/src/app/shared/utils/index.ts | 1 - .../inbound-email-parse/inbound-email-parse.command.ts | 0 .../inbound-email-parse/inbound-email-parse.usecase.ts | 3 +-- .../workflow/workers}/inbound-parse.worker.service.ts | 6 +++--- apps/worker/src/app/workflow/workflow.module.ts | 2 ++ apps/worker/src/config/worker-init.config.ts | 10 ++++++++-- packages/application-generic/src/index.ts | 1 + 16 files changed, 25 insertions(+), 36 deletions(-) delete mode 100644 apps/api/src/app/shared/helpers/hmac.service.ts delete mode 100644 apps/worker/src/app/shared/utils/hmac.ts rename apps/{api/src/app/inbound-parse => worker/src/app/workflow}/usecases/inbound-email-parse/inbound-email-parse.command.ts (100%) rename apps/{api/src/app/inbound-parse => worker/src/app/workflow}/usecases/inbound-email-parse/inbound-email-parse.usecase.ts (96%) rename apps/{api/src/app/inbound-parse/services => worker/src/app/workflow/workers}/inbound-parse.worker.service.ts (83%) diff --git a/apps/api/src/app/auth/usecases/login/login.usecase.ts b/apps/api/src/app/auth/usecases/login/login.usecase.ts index 13c472cfb10..4d4d1a9e220 100644 --- a/apps/api/src/app/auth/usecases/login/login.usecase.ts +++ b/apps/api/src/app/auth/usecases/login/login.usecase.ts @@ -2,12 +2,11 @@ import * as bcrypt from 'bcrypt'; import { Injectable, UnauthorizedException } from '@nestjs/common'; import { differenceInMinutes, parseISO } from 'date-fns'; import { UserRepository, UserEntity, OrganizationRepository } from '@novu/dal'; -import { AnalyticsService, AuthService } from '@novu/application-generic'; +import { AnalyticsService, AuthService, createHash } from '@novu/application-generic'; import { LoginCommand } from './login.command'; import { ApiException } from '../../../shared/exceptions/api.exception'; import { normalizeEmail } from '../../../shared/helpers/email-normalization.service'; -import { createHash } from '../../../shared/helpers/hmac.service'; @Injectable() export class Login { diff --git a/apps/api/src/app/auth/usecases/register/user-register.usecase.ts b/apps/api/src/app/auth/usecases/register/user-register.usecase.ts index b6b5df585c5..60008337c9e 100644 --- a/apps/api/src/app/auth/usecases/register/user-register.usecase.ts +++ b/apps/api/src/app/auth/usecases/register/user-register.usecase.ts @@ -2,14 +2,13 @@ import { Inject, Injectable } from '@nestjs/common'; import { OrganizationEntity, UserRepository } from '@novu/dal'; import * as bcrypt from 'bcrypt'; import { SignUpOriginEnum } from '@novu/shared'; -import { AnalyticsService, AuthService } from '@novu/application-generic'; +import { AnalyticsService, AuthService, createHash } from '@novu/application-generic'; import { UserRegisterCommand } from './user-register.command'; import { normalizeEmail } from '../../../shared/helpers/email-normalization.service'; import { ApiException } from '../../../shared/exceptions/api.exception'; import { CreateOrganization } from '../../../organization/usecases/create-organization/create-organization.usecase'; import { CreateOrganizationCommand } from '../../../organization/usecases/create-organization/create-organization.command'; -import { createHash } from '../../../shared/helpers/hmac.service'; @Injectable() export class UserRegister { diff --git a/apps/api/src/app/inbound-parse/inbound-parse.module.ts b/apps/api/src/app/inbound-parse/inbound-parse.module.ts index e35c36838e1..edb12f76771 100644 --- a/apps/api/src/app/inbound-parse/inbound-parse.module.ts +++ b/apps/api/src/app/inbound-parse/inbound-parse.module.ts @@ -3,13 +3,11 @@ import { CompileTemplate, WorkflowInMemoryProviderService } from '@novu/applicat import { USE_CASES } from './usecases'; import { InboundParseController } from './inbound-parse.controller'; -import { GetMxRecord } from './usecases/get-mx-record/get-mx-record.usecase'; import { SharedModule } from '../shared/shared.module'; import { AuthModule } from '../auth/auth.module'; -import { InboundParseWorkerService } from './services/inbound-parse.worker.service'; -const PROVIDERS = [GetMxRecord, CompileTemplate, InboundParseWorkerService]; +const PROVIDERS = [CompileTemplate]; const memoryQueueService = { provide: WorkflowInMemoryProviderService, diff --git a/apps/api/src/app/inbound-parse/usecases/index.ts b/apps/api/src/app/inbound-parse/usecases/index.ts index bc67ec9189a..b68274fc879 100644 --- a/apps/api/src/app/inbound-parse/usecases/index.ts +++ b/apps/api/src/app/inbound-parse/usecases/index.ts @@ -1,3 +1,3 @@ -import { InboundEmailParse } from './inbound-email-parse/inbound-email-parse.usecase'; +import { GetMxRecord } from './get-mx-record/get-mx-record.usecase'; -export const USE_CASES = [InboundEmailParse]; +export const USE_CASES = [GetMxRecord]; diff --git a/apps/api/src/app/shared/helpers/hmac.service.ts b/apps/api/src/app/shared/helpers/hmac.service.ts deleted file mode 100644 index cfeb5da9f1c..00000000000 --- a/apps/api/src/app/shared/helpers/hmac.service.ts +++ /dev/null @@ -1,8 +0,0 @@ -import { Logger } from '@nestjs/common'; -import { createHmac } from 'crypto'; - -export function createHash(key: string, valueToHash: string) { - Logger.verbose('Creating Hmac'); - - return createHmac('sha256', key).update(valueToHash).digest('hex'); -} diff --git a/apps/api/src/app/subscribers/usecases/chat-oauth/chat-oauth.usecase.ts b/apps/api/src/app/subscribers/usecases/chat-oauth/chat-oauth.usecase.ts index b0744ad3dec..c9c5455b7ee 100644 --- a/apps/api/src/app/subscribers/usecases/chat-oauth/chat-oauth.usecase.ts +++ b/apps/api/src/app/subscribers/usecases/chat-oauth/chat-oauth.usecase.ts @@ -2,9 +2,9 @@ import { Injectable, NotFoundException } from '@nestjs/common'; import { ChannelTypeEnum } from '@novu/stateless'; import { IntegrationEntity, IntegrationRepository, EnvironmentRepository, ICredentialsEntity } from '@novu/dal'; +import { createHash } from '@novu/application-generic'; import { ChatOauthCommand } from './chat-oauth.command'; -import { createHash } from '../../../shared/helpers/hmac.service'; import { ApiException } from '../../../shared/exceptions/api.exception'; @Injectable() diff --git a/apps/api/src/app/user/usecases/get-my-profile/get-my-profile.usecase.ts b/apps/api/src/app/user/usecases/get-my-profile/get-my-profile.usecase.ts index 074953a1a2e..498c9c5b56d 100644 --- a/apps/api/src/app/user/usecases/get-my-profile/get-my-profile.usecase.ts +++ b/apps/api/src/app/user/usecases/get-my-profile/get-my-profile.usecase.ts @@ -1,7 +1,9 @@ import { Injectable, NotFoundException, Logger } from '@nestjs/common'; + import { UserRepository } from '@novu/dal'; +import { createHash } from '@novu/application-generic'; + import { GetMyProfileCommand } from './get-my-profile.dto'; -import { createHash } from '../../../shared/helpers/hmac.service'; @Injectable() export class GetMyProfileUsecase { constructor(private readonly userRepository: UserRepository) {} diff --git a/apps/api/src/app/widgets/usecases/initialize-session/initialize-session.usecase.ts b/apps/api/src/app/widgets/usecases/initialize-session/initialize-session.usecase.ts index 80027dabef8..b90bc2df6a2 100644 --- a/apps/api/src/app/widgets/usecases/initialize-session/initialize-session.usecase.ts +++ b/apps/api/src/app/widgets/usecases/initialize-session/initialize-session.usecase.ts @@ -9,13 +9,13 @@ import { SelectIntegrationCommand, SelectIntegration, AuthService, + createHash, } from '@novu/application-generic'; import { ApiException } from '../../../shared/exceptions/api.exception'; import { InitializeSessionCommand } from './initialize-session.command'; import { SessionInitializeResponseDto } from '../../dtos/session-initialize-response.dto'; -import { createHash } from '../../../shared/helpers/hmac.service'; @Injectable() export class InitializeSession { diff --git a/apps/worker/src/app/shared/utils/hmac.ts b/apps/worker/src/app/shared/utils/hmac.ts deleted file mode 100644 index cfeb5da9f1c..00000000000 --- a/apps/worker/src/app/shared/utils/hmac.ts +++ /dev/null @@ -1,8 +0,0 @@ -import { Logger } from '@nestjs/common'; -import { createHmac } from 'crypto'; - -export function createHash(key: string, valueToHash: string) { - Logger.verbose('Creating Hmac'); - - return createHmac('sha256', key).update(valueToHash).digest('hex'); -} diff --git a/apps/worker/src/app/shared/utils/index.ts b/apps/worker/src/app/shared/utils/index.ts index 42d722103bc..daf676373fd 100644 --- a/apps/worker/src/app/shared/utils/index.ts +++ b/apps/worker/src/app/shared/utils/index.ts @@ -1,4 +1,3 @@ export * from './constants'; export * from './exceptions'; -export * from './hmac'; export * from './cron-health'; diff --git a/apps/api/src/app/inbound-parse/usecases/inbound-email-parse/inbound-email-parse.command.ts b/apps/worker/src/app/workflow/usecases/inbound-email-parse/inbound-email-parse.command.ts similarity index 100% rename from apps/api/src/app/inbound-parse/usecases/inbound-email-parse/inbound-email-parse.command.ts rename to apps/worker/src/app/workflow/usecases/inbound-email-parse/inbound-email-parse.command.ts diff --git a/apps/api/src/app/inbound-parse/usecases/inbound-email-parse/inbound-email-parse.usecase.ts b/apps/worker/src/app/workflow/usecases/inbound-email-parse/inbound-email-parse.usecase.ts similarity index 96% rename from apps/api/src/app/inbound-parse/usecases/inbound-email-parse/inbound-email-parse.usecase.ts rename to apps/worker/src/app/workflow/usecases/inbound-email-parse/inbound-email-parse.usecase.ts index 682dc6abbd0..bcfb22ce747 100644 --- a/apps/api/src/app/inbound-parse/usecases/inbound-email-parse/inbound-email-parse.usecase.ts +++ b/apps/worker/src/app/workflow/usecases/inbound-email-parse/inbound-email-parse.usecase.ts @@ -9,8 +9,7 @@ import { NotificationTemplateEntity, } from '@novu/dal'; import axios from 'axios'; -import { createHash } from '../../../shared/helpers/hmac.service'; -import { CompileTemplate, CompileTemplateCommand } from '@novu/application-generic'; +import { CompileTemplate, CompileTemplateCommand, createHash } from '@novu/application-generic'; const LOG_CONTEXT = 'InboundEmailParse'; diff --git a/apps/api/src/app/inbound-parse/services/inbound-parse.worker.service.ts b/apps/worker/src/app/workflow/workers/inbound-parse.worker.service.ts similarity index 83% rename from apps/api/src/app/inbound-parse/services/inbound-parse.worker.service.ts rename to apps/worker/src/app/workflow/workers/inbound-parse.worker.service.ts index 38ba9e4fc79..4608a0f2b03 100644 --- a/apps/api/src/app/inbound-parse/services/inbound-parse.worker.service.ts +++ b/apps/worker/src/app/workflow/workers/inbound-parse.worker.service.ts @@ -14,9 +14,9 @@ import { InboundEmailParseCommand } from '../usecases/inbound-email-parse/inboun const LOG_CONTEXT = 'InboundParseQueueService'; @Injectable() -export class InboundParseWorkerService extends WorkerBaseService { +export class InboundParseWorker extends WorkerBaseService { constructor( - private emailParseUsecase: InboundEmailParse, + private inboundEmailParseUsecase: InboundEmailParse, public workflowInMemoryProviderService: WorkflowInMemoryProviderService ) { super(JobTopicNameEnum.INBOUND_PARSE_MAIL, new BullMqService(workflowInMemoryProviderService)); @@ -31,7 +31,7 @@ export class InboundParseWorkerService extends WorkerBaseService { public getWorkerProcessor() { return async ({ data }: { data: InboundEmailParseCommand }) => { Logger.verbose({ data }, 'Processing the inbound parsed email', LOG_CONTEXT); - await this.emailParseUsecase.execute(InboundEmailParseCommand.create({ ...data })); + await this.inboundEmailParseUsecase.execute(InboundEmailParseCommand.create({ ...data })); }; } } diff --git a/apps/worker/src/app/workflow/workflow.module.ts b/apps/worker/src/app/workflow/workflow.module.ts index 2468195446f..5e744751233 100644 --- a/apps/worker/src/app/workflow/workflow.module.ts +++ b/apps/worker/src/app/workflow/workflow.module.ts @@ -55,6 +55,7 @@ import { import { SharedModule } from '../shared/shared.module'; import { ACTIVE_WORKERS } from '../../config/worker-init.config'; +import { InboundEmailParse } from './usecases/inbound-email-parse/inbound-email-parse.usecase'; const REPOSITORIES = [JobRepository]; @@ -105,6 +106,7 @@ const USE_CASES = [ SubscriberJobBound, TriggerBroadcast, TriggerMulticast, + InboundEmailParse, ]; const PROVIDERS: Provider[] = []; diff --git a/apps/worker/src/config/worker-init.config.ts b/apps/worker/src/config/worker-init.config.ts index bf245487ef0..8e8ae2ca20e 100644 --- a/apps/worker/src/config/worker-init.config.ts +++ b/apps/worker/src/config/worker-init.config.ts @@ -4,12 +4,14 @@ import { JobTopicNameEnum } from '@novu/shared'; import { ExecutionLogWorker, StandardWorker, WorkflowWorker } from '../app/workflow/services'; import { SubscriberProcessWorker } from '../app/workflow/services/subscriber-process.worker'; +import { InboundParseWorker } from '../app/workflow/workers/inbound-parse.worker.service'; type WorkerClass = | typeof StandardWorker | typeof WorkflowWorker | typeof ExecutionLogWorker - | typeof SubscriberProcessWorker; + | typeof SubscriberProcessWorker + | typeof InboundParseWorker; type WorkerModuleTree = { workerClass: WorkerClass; queueDependencies: JobTopicNameEnum[] }; @@ -52,6 +54,10 @@ export const WORKER_MAPPING: WorkerDepTree = { JobTopicNameEnum.PROCESS_SUBSCRIBER, ], }, + [JobTopicNameEnum.INBOUND_PARSE_MAIL]: { + workerClass: InboundParseWorker, + queueDependencies: [], + }, }; const validQueueEntries = Object.keys(JobTopicNameEnum).map((key) => JobTopicNameEnum[key]); @@ -82,7 +88,7 @@ export const UNIQUE_WORKER_DEPENDENCIES = [...new Set(WORKER_DEPENDENCIES)]; export const ACTIVE_WORKERS: Provider[] | any[] = []; if (!workersToProcess.length) { - ACTIVE_WORKERS.push(StandardWorker, WorkflowWorker, ExecutionLogWorker, SubscriberProcessWorker); + ACTIVE_WORKERS.push(StandardWorker, WorkflowWorker, ExecutionLogWorker, SubscriberProcessWorker, InboundParseWorker); } else { workersToProcess.forEach((queue) => { const workerClass = WORKER_MAPPING[queue]?.workerClass; diff --git a/packages/application-generic/src/index.ts b/packages/application-generic/src/index.ts index 7f2cceea977..b14143a746b 100644 --- a/packages/application-generic/src/index.ts +++ b/packages/application-generic/src/index.ts @@ -17,5 +17,6 @@ export * from './utils/exceptions'; export * from './utils/email-normalization'; export * from './utils/digest'; export * from './utils/object'; +export * from './utils/hmac'; export * from './decorators/external-api.decorator'; export * from './decorators/user-session.decorator'; From 7eafaf4c42071ec3680e954c46e66259b9cd4b5a Mon Sep 17 00:00:00 2001 From: Gosha Date: Sun, 17 Dec 2023 13:06:15 +0200 Subject: [PATCH 2/5] refactor: moved the inbound mail parser e2e to worker and changed it to spec --- .../e2e/inbound-email-parse.e2e.ts | 176 --------- .../specs/inbound-email-parse.spec.ts | 347 ++++++++++++++++++ .../inbound-email-parse.usecase.ts | 2 +- libs/testing/src/index.ts | 1 + 4 files changed, 349 insertions(+), 177 deletions(-) delete mode 100644 apps/api/src/app/inbound-parse/e2e/inbound-email-parse.e2e.ts create mode 100644 apps/worker/src/app/workflow/specs/inbound-email-parse.spec.ts diff --git a/apps/api/src/app/inbound-parse/e2e/inbound-email-parse.e2e.ts b/apps/api/src/app/inbound-parse/e2e/inbound-email-parse.e2e.ts deleted file mode 100644 index ef8f8a974d2..00000000000 --- a/apps/api/src/app/inbound-parse/e2e/inbound-email-parse.e2e.ts +++ /dev/null @@ -1,176 +0,0 @@ -import { Test, TestingModule } from '@nestjs/testing'; -import { InboundEmailParse, IUserWebhookPayload } from '../usecases/inbound-email-parse/inbound-email-parse.usecase'; -import { EnvironmentRepository, JobRepository, MessageRepository, SubscriberEntity } from '@novu/dal'; -import { expect } from 'chai'; -import { InboundEmailParseCommand } from '../usecases/inbound-email-parse/inbound-email-parse.command'; -import { ChannelTypeEnum, EmailBlockTypeEnum, StepTypeEnum } from '@novu/shared'; -import { sendTrigger } from '../../events/e2e/trigger-event.e2e'; -import { UserSession } from '@novu/testing'; -import { CreateTemplatePayload } from '@novu/testing/src/create-notification-template.interface'; -import { SubscribersService } from '@novu/testing/src'; -import axios from 'axios'; -import * as sinon from 'sinon'; -import { CompileTemplate } from '@novu/application-generic'; - -const USER_MAIL_DOMAIN = 'mail.domain.com'; -const USER_PARSE_WEBHOOK = 'user-parse.com/webhook/{{compiledVariable}}'; - -describe('Should handle the new arrived mail', () => { - let inboundEmailParseUsecase: InboundEmailParse; - let session: UserSession; - - const environmentRepository = new EnvironmentRepository(); - const messageRepository = new MessageRepository(); - - let subscriber: SubscriberEntity; - let subscriberService: SubscribersService; - - let sandbox; - - beforeEach(async () => { - sandbox = sinon.createSandbox(); - - session = new UserSession(); - await session.initialize(); - subscriberService = new SubscribersService(session.organization._id, session.environment._id); - subscriber = await subscriberService.createSubscriber(); - - const module: TestingModule = await Test.createTestingModule({ - providers: [InboundEmailParse, JobRepository, MessageRepository, CompileTemplate], - }).compile(); - - inboundEmailParseUsecase = module.get(InboundEmailParse); - }); - - afterEach(async () => { - sandbox.restore(); - }); - - it('should send webhook request to the users webhook', async () => { - const message = await triggerEmail(); - - const mail = getMailData(message); - - const getStub = sandbox.stub(axios, 'post').resolves(); - - await inboundEmailParseUsecase.execute(InboundEmailParseCommand.create(mail)); - - sinon.assert.calledOnce(getStub); - getStub.calledWith(sinon.match.array); - const args = getStub.getCall(0).args; - - const webhook: string = args[0]; - const payload: IUserWebhookPayload = args[1]; - - // Should compile the payload variables - expect(webhook).to.equal(USER_PARSE_WEBHOOK.replace('{{compiledVariable}}', 'test-env')); - expect(payload.mail).to.be.ok; - expect(payload.payload).to.ok; - expect(payload.template).to.ok; - expect(payload.message).to.ok; - expect(payload.transactionId).to.ok; - expect(payload.hmac).to.ok; - expect(payload.notification).to.ok; - expect(payload.templateIdentifier).to.ok; - }); - - it('should not send webhook request with missing transactionId', async () => { - try { - const message = await triggerEmail(); - const mail = getMailData(message, false); - - await inboundEmailParseUsecase.execute(InboundEmailParseCommand.create(mail)); - } catch (e) { - expect(e.message).to.contains('Missing transactionId on address'); - } - }); - - it('should not send webhook request with when domain white list', async () => { - try { - const message = await triggerEmail(true, false); - - const mail = getMailData(message); - - await inboundEmailParseUsecase.execute(InboundEmailParseCommand.create(mail)); - } catch (e) { - expect(e.message).to.equal('Domain is not in environment white list'); - } - }); - - it('should not send webhook request when missing replay callback url', async () => { - try { - const message = await triggerEmail(true, true, true, false); - - const mail = getMailData(message); - - await inboundEmailParseUsecase.execute(InboundEmailParseCommand.create(mail)); - } catch (e) { - expect(e.message).to.contains('Missing parse webhook on template'); - } - }); - - async function triggerEmail( - mxRecordConfigured = true, - inboundParseDomain = true, - replyCallbackActive = true, - replyCallbackUrl = true - ) { - await environmentRepository.update( - { _id: session.environment._id, _organizationId: session.organization._id }, - { - $set: { - dns: { - mxRecordConfigured: mxRecordConfigured, - inboundParseDomain: inboundParseDomain ? USER_MAIL_DOMAIN : undefined, - }, - }, - } - ); - - const template = await createTemplate(session, replyCallbackActive, replyCallbackUrl); - - await sendTrigger(session, template, subscriber.subscriberId); - - await session.awaitRunningJobs(template._id); - - return await messageRepository.findOne({ - _environmentId: session.environment._id, - _templateId: template._id, - _subscriberId: subscriber._id, - channel: ChannelTypeEnum.EMAIL, - }); - } - - function getMailData(message, transactionId = true, environmentId = true) { - const mail = JSON.parse(mailData) as InboundEmailParseCommand; - mail.to[0].address = `parse+${transactionId ? message.transactionId : ''}-nv-e=${ - environmentId ? message._environmentId : '' - }@${USER_MAIL_DOMAIN}`; - - return mail; - } -}); - -async function createTemplate(session: UserSession, replyCallbackActive = true, replyCallbackUrl = true) { - const test: Partial = { - steps: [ - { - replyCallback: { active: replyCallbackActive, url: replyCallbackUrl ? USER_PARSE_WEBHOOK : '' }, - type: StepTypeEnum.EMAIL, - name: 'Message Name', - subject: 'Test email {{nested.subject}}', - content: [ - { - type: EmailBlockTypeEnum.TEXT, - content: 'Hello {{subscriber.lastName}}, Welcome to {{organizationName}}' as string, - }, - ], - }, - ], - }; - - return await session.createTemplate(test); -} - -const mailData = - '{"html":"This is a test email sent to a local SMTP server.","text":"This is a test email sent to a local SMTP server.","headers":{"content-type":"multipart/alternative; boundary=\\"--_NmP-f7fda3731bcaef89-Part_1\\"","from":"sender@example.com","to":"parse+c50420f2-6aef-48f5-9a41-3c9dd1a81ba5-nv-e=63945d20068f12be94e79cb0@local-demo.com","subject":"Test email","message-id":"<705c2187-b2ad-2b1e-e3fc-9f40a840e736@example.com>","date":"Wed, 25 Jan 2023 20:37:24 +0000","mime-version":"1.0"},"subject":"Test email","messageId":"705c2187-b2ad-2b1e-e3fc-9f40a840e736@example.com","priority":"normal","from":[{"address":"sender@example.com","name":""}],"to":[{"address":"parse+c50420f2-6aef-48f5-9a41-3c9dd1a81ba5-nv-e=63945d20068f12be94e79cb0@local-demo.com","name":""}],"date":"2023-01-25T20:37:24.000Z","dkim":"failed","spf":"failed","spamScore":0,"language":"english","cc":[],"connection":{"id":"bb49053e-a142-4492-9459-61d7960b0857","remoteAddress":"127.0.0.1","remotePort":55722,"clientHostname":"[127.0.0.1]","openingCommand":"HELLO","hostNameAppearsAs":"[127.0.0.1]","xClient":{},"xForward":{},"transmissionType":"ESMTPS","tlsOptions":{"name":"TLS_AES_256_GCM_SHA384","standardName":"TLS_AES_256_GCM_SHA384","version":"TLSv1.3"},"envelope":{"mailFrom":{"address":"sender@example.com","args":false},"rcptTo":[{"address":"parse+c50420f2-6aef-48f5-9a41-3c9dd1a81ba5@local-demo.com","args":false}]},"transaction":1,"mailPath":".tmp/bb49053e-a142-4492-9459-61d7960b0857"},"envelopeFrom":{"address":"sender@example.com","args":false},"envelopeTo":[{"address":"parse+c50420f2-6aef-48f5-9a41-3c9dd1a81ba5-nv-e=63945d20068f12be94e79cb0@local-demo.com","args":false}]}\n'; diff --git a/apps/worker/src/app/workflow/specs/inbound-email-parse.spec.ts b/apps/worker/src/app/workflow/specs/inbound-email-parse.spec.ts new file mode 100644 index 00000000000..21342118f4c --- /dev/null +++ b/apps/worker/src/app/workflow/specs/inbound-email-parse.spec.ts @@ -0,0 +1,347 @@ +import axios, { AxiosResponse } from 'axios'; +import { Test, TestingModule } from '@nestjs/testing'; +import { expect } from 'chai'; +import * as sinon from 'sinon'; + +import { JobRepository, MessageRepository } from '@novu/dal'; +import { CompileTemplate } from '@novu/application-generic'; + +import { InboundEmailParse, IUserWebhookPayload } from '../usecases/inbound-email-parse/inbound-email-parse.usecase'; +import { InboundEmailParseCommand } from '../usecases/inbound-email-parse/inbound-email-parse.command'; +const axiosInstance = axios.create(); + +const eventTriggerPath = '/v1/events/trigger'; +const USER_MAIL_DOMAIN = 'mail.domain.com'; +const USER_PARSE_WEBHOOK = 'user-parse.com/webhook/{{compiledVariable}}'; + +describe('Should handle the new arrived mail', () => { + let inboundEmailParseUsecase: InboundEmailParse; + + let sandbox; + + beforeEach(async () => { + sandbox = sinon.createSandbox(); + + const module: TestingModule = await Test.createTestingModule({ + providers: [InboundEmailParse, JobRepository, MessageRepository, CompileTemplate], + }).compile(); + + inboundEmailParseUsecase = module.get(InboundEmailParse); + }); + + afterEach(async () => { + sandbox.restore(); + }); + + it('should send webhook request to the users webhook', async () => { + const mail = getMailData(); + + const axiosPostStub = sandbox.stub(axios, 'post').resolves(); + const getEntitiesStub = sandbox.stub(inboundEmailParseUsecase, 'getEntities').resolves(getEntitiesStubObject); + + await inboundEmailParseUsecase.execute(InboundEmailParseCommand.create(mail)); + + sinon.assert.calledOnce(axiosPostStub); + axiosPostStub.calledWith(sinon.match.array); + const args = axiosPostStub.getCall(0).args; + + const webhook: string = args[0]; + const payload: IUserWebhookPayload = args[1]; + + // Should compile the payload variables + expect(webhook).to.equal(USER_PARSE_WEBHOOK.replace('{{compiledVariable}}', 'test-env')); + expect(payload.mail).to.be.ok; + expect(payload.payload).to.ok; + expect(payload.template).to.ok; + expect(payload.message).to.ok; + expect(payload.transactionId).to.ok; + expect(payload.hmac).to.ok; + expect(payload.notification).to.ok; + expect(payload.templateIdentifier).to.ok; + }); + + it('should not send webhook request with missing transactionId', async () => { + try { + // const message = await triggerEmail(); + const mail = getMailData({ skipTransactionId: true }); + + await inboundEmailParseUsecase.execute(InboundEmailParseCommand.create(mail)); + + throw new Error('Should not reach here, en error should be thrown'); + } catch (e) { + expect(e.message).to.contains('Missing transactionId on address'); + } + }); + + it('should not send webhook request with when domain white list', async () => { + try { + const mail = getMailData({ userDomain: 'invalid-domain.com' }); + const getEntitiesStub = sandbox.stub(inboundEmailParseUsecase, 'getEntities').resolves(getEntitiesStubObject); + + await inboundEmailParseUsecase.execute(InboundEmailParseCommand.create(mail)); + + throw new Error('Should not reach here, en error should be thrown'); + } catch (e) { + expect(e.message).to.equal('Domain is not in environment white list'); + } + }); + + it('should not send webhook request when missing replay callback url', async () => { + try { + const entitiesWithMissingParseWebhook = getEntitiesStubObject; + entitiesWithMissingParseWebhook.template.steps[0].replyCallback = {} as any; + + const mail = getMailData(); + const getEntitiesStub = sandbox + .stub(inboundEmailParseUsecase, 'getEntities') + .resolves(entitiesWithMissingParseWebhook); + + await inboundEmailParseUsecase.execute(InboundEmailParseCommand.create(mail)); + + throw new Error('Should not reach here, en error should be thrown'); + } catch (e) { + expect(e.message).to.contains('Missing parse webhook on template'); + } + }); + + interface IMailData { + message?: any; + transactionId?: string; + environmentId?: string; + userDomain?: string; + skipTransactionId?: boolean; + } + + function getMailData({ transactionId, environmentId, userDomain, skipTransactionId }: IMailData = {}) { + const mail = JSON.parse(mailData) as InboundEmailParseCommand; + + const userNameDelimiter = '-nv-e='; + + const [user, domain] = mail.to[0].address.split('@'); + const toMetaIds = user.split('+')[1]; + const [mailTransactionId, mailEnvironmentId] = toMetaIds.split(userNameDelimiter); + + const parsedTransactionId = skipTransactionId ? '' : transactionId ? transactionId : mailTransactionId; + + mail.to[0].address = `parse+${parsedTransactionId}-nv-e=${environmentId ? environmentId : mailTransactionId}@${ + userDomain ? userDomain : USER_MAIL_DOMAIN + }`; + + return mail; + } +}); + +const mailData = + '{"html":"This is a test email sent to a local SMTP server.","text":"This is a test email sent to a local SMTP server.","headers":{"content-type":"multipart/alternative; boundary=\\"--_NmP-f7fda3731bcaef89-Part_1\\"","from":"sender@example.com","to":"parse+c50420f2-6aef-48f5-9a41-3c9dd1a81ba5-nv-e=63945d20068f12be94e79cb0@local-demo.com","subject":"Test email","message-id":"<705c2187-b2ad-2b1e-e3fc-9f40a840e736@example.com>","date":"Wed, 25 Jan 2023 20:37:24 +0000","mime-version":"1.0"},"subject":"Test email","messageId":"705c2187-b2ad-2b1e-e3fc-9f40a840e736@example.com","priority":"normal","from":[{"address":"sender@example.com","name":""}],"to":[{"address":"parse+c50420f2-6aef-48f5-9a41-3c9dd1a81ba5-nv-e=63945d20068f12be94e79cb0@local-demo.com","name":""}],"date":"2023-01-25T20:37:24.000Z","dkim":"failed","spf":"failed","spamScore":0,"language":"english","cc":[],"connection":{"id":"bb49053e-a142-4492-9459-61d7960b0857","remoteAddress":"127.0.0.1","remotePort":55722,"clientHostname":"[127.0.0.1]","openingCommand":"HELLO","hostNameAppearsAs":"[127.0.0.1]","xClient":{},"xForward":{},"transmissionType":"ESMTPS","tlsOptions":{"name":"TLS_AES_256_GCM_SHA384","standardName":"TLS_AES_256_GCM_SHA384","version":"TLSv1.3"},"envelope":{"mailFrom":{"address":"sender@example.com","args":false},"rcptTo":[{"address":"parse+c50420f2-6aef-48f5-9a41-3c9dd1a81ba5@local-demo.com","args":false}]},"transaction":1,"mailPath":".tmp/bb49053e-a142-4492-9459-61d7960b0857"},"envelopeFrom":{"address":"sender@example.com","args":false},"envelopeTo":[{"address":"parse+c50420f2-6aef-48f5-9a41-3c9dd1a81ba5-nv-e=63945d20068f12be94e79cb0@local-demo.com","args":false}]}\n'; + +const getEntitiesStubObject = { + template: { + _id: '657ec2402c5ac81fb1e0f007', + steps: [ + { + active: true, + replyCallback: { + active: true, + url: 'user-parse.com/webhook/{{compiledVariable}}', + }, + shouldStopOnFail: false, + filters: [], + _templateId: '657ec2402c5ac81fb1e0f005', + metadata: { + timed: { + weekDays: [], + monthDays: [], + }, + }, + variants: [], + _id: '657ec2402c5ac81fb1e0f00c', + }, + ], + }, + notification: { + _id: '657ec24013bdfd2ae0785f3f', + _templateId: '657ec2402c5ac81fb1e0f007', + _environmentId: '657ec2402c5ac81fb1e0efbc', + _organizationId: '657ec2402c5ac81fb1e0efb6', + _subscriberId: '657ec2402c5ac81fb1e0efff', + transactionId: 'ec7d3f9b-ede7-4287-8761-0b192d473f7c', + channels: ['email'], + to: { + subscriberId: '657ec2402c5ac81fb1e0effe', + lastName: 'Smith', + email: 'test@email.novu', + }, + payload: { + organizationName: 'Umbrella Corp', + compiledVariable: 'test-env', + }, + expireAt: '2024-01-16T09:41:20.863Z', + createdAt: '2023-12-17T09:41:20.863Z', + updatedAt: '2023-12-17T09:41:20.863Z', + __v: 0, + }, + subscriber: { + _id: '657ec2402c5ac81fb1e0efff', + subscriberId: '657ec2402c5ac81fb1e0effe', + }, + environment: { + _id: '657ec2402c5ac81fb1e0efbc', + apiKeys: [ + { + key: 'e088ccce-d18c-42d6-9acb-a40b232b846f', + _userId: '657ec2402c5ac81fb1e0efb4', + _id: '657ec2402c5ac81fb1e0efbd', + }, + ], + dns: { + mxRecordConfigured: true, + inboundParseDomain: 'mail.domain.com', + }, + }, + job: { + _id: '657ec24013bdfd2ae0785f41', + identifier: 'test-event-6f1b2973-d4bd-44fc-889e-4b9024eb2bea', + status: 'completed', + payload: { + organizationName: 'Umbrella Corp', + compiledVariable: 'test-env', + }, + tenant: null, + step: { + replyCallback: { + active: true, + url: 'user-parse.com/webhook/{{compiledVariable}}', + }, + metadata: { + timed: { + weekDays: [], + monthDays: [], + }, + }, + active: true, + shouldStopOnFail: false, + filters: [], + _templateId: '657ec2402c5ac81fb1e0f005', + variants: [], + _id: '657ec2402c5ac81fb1e0f00c', + id: '657ec2402c5ac81fb1e0f00c', + template: { + _id: '657ec2402c5ac81fb1e0f005', + type: 'email', + active: true, + name: 'Message Name', + subject: 'Test email {{nested.subject}}', + variables: [], + content: [ + { + type: 'text', + content: 'Hello {{subscriber.lastName}}, Welcome to {{organizationName}}', + }, + ], + _environmentId: '657ec2402c5ac81fb1e0efbc', + _organizationId: '657ec2402c5ac81fb1e0efb6', + _creatorId: '657ec2402c5ac81fb1e0efb4', + _feedId: '657ec2402c5ac81fb1e0efeb', + _layoutId: '657ec2402c5ac81fb1e0efc1', + deleted: false, + createdAt: '2023-12-17T09:41:20.768Z', + updatedAt: '2023-12-17T09:41:20.768Z', + __v: 0, + id: '657ec2402c5ac81fb1e0f005', + }, + }, + _templateId: '657ec2402c5ac81fb1e0f007', + transactionId: 'ec7d3f9b-ede7-4287-8761-0b192d473f7c', + _notificationId: '657ec24013bdfd2ae0785f3f', + subscriberId: '657ec2402c5ac81fb1e0effe', + _subscriberId: '657ec2402c5ac81fb1e0efff', + _userId: '657ec2402c5ac81fb1e0efb4', + _organizationId: '657ec2402c5ac81fb1e0efb6', + _environmentId: '657ec2402c5ac81fb1e0efbc', + digest: { + events: [], + timed: { + weekDays: [], + monthDays: [], + }, + }, + type: 'email', + providerId: 'sendgrid', + expireAt: '2024-01-16T09:41:20.863Z', + createdAt: '2023-12-17T09:41:20.866Z', + __v: 0, + updatedAt: '2023-12-17T09:41:20.978Z', + }, + message: { + cta: { + action: { + buttons: [], + }, + }, + _id: '657ec24013bdfd2ae0785f54', + _templateId: '657ec2402c5ac81fb1e0f007', + _environmentId: '657ec2402c5ac81fb1e0efbc', + _messageTemplateId: '657ec2402c5ac81fb1e0f005', + _notificationId: '657ec24013bdfd2ae0785f3f', + _organizationId: '657ec2402c5ac81fb1e0efb6', + _subscriberId: '657ec2402c5ac81fb1e0efff', + _jobId: '657ec24013bdfd2ae0785f41', + templateIdentifier: 'test-event-6f1b2973-d4bd-44fc-889e-4b9024eb2bea', + email: 'test@email.novu', + subject: 'Test email', + channel: 'email', + providerId: 'sendgrid', + deviceTokens: [], + seen: false, + read: false, + status: 'sent', + transactionId: 'ec7d3f9b-ede7-4287-8761-0b192d473f7c', + payload: { + organizationName: 'Umbrella Corp', + compiledVariable: 'test-env', + }, + expireAt: '2024-01-16T09:41:20.940Z', + deleted: false, + createdAt: '2023-12-17T09:41:20.940Z', + updatedAt: '2023-12-17T09:41:20.970Z', + __v: 0, + content: [ + { + type: 'text', + content: 'Hello Smith, Welcome to Umbrella Corp', + url: '', + }, + ], + id: '657ec24013bdfd2ae0785f54', + }, +}; +export async function sendTrigger( + session, + template, + newSubscriberIdInAppNotification: string, + payload: Record = {}, + overrides: Record = {}, + tenant?: string, + actor?: string +): Promise { + return await axiosInstance.post( + `${session.serverUrl}${eventTriggerPath}`, + { + name: template.triggers[0].identifier, + to: [{ subscriberId: newSubscriberIdInAppNotification, lastName: 'Smith', email: 'test@email.novu' }], + payload: { + organizationName: 'Umbrella Corp', + compiledVariable: 'test-env', + ...payload, + }, + overrides, + tenant, + actor, + }, + { + headers: { + authorization: `ApiKey ${session.apiKey}`, + }, + } + ); +} diff --git a/apps/worker/src/app/workflow/usecases/inbound-email-parse/inbound-email-parse.usecase.ts b/apps/worker/src/app/workflow/usecases/inbound-email-parse/inbound-email-parse.usecase.ts index bcfb22ce747..438405bf737 100644 --- a/apps/worker/src/app/workflow/usecases/inbound-email-parse/inbound-email-parse.usecase.ts +++ b/apps/worker/src/app/workflow/usecases/inbound-email-parse/inbound-email-parse.usecase.ts @@ -109,7 +109,7 @@ export class InboundEmailParse { _subscriberId: subscriber._id, }); - return { transactionId, template, notification, subscriber, environment, job, message }; + return { template, notification, subscriber, environment, job, message }; } } diff --git a/libs/testing/src/index.ts b/libs/testing/src/index.ts index 3eff5e387b9..2d857b02d56 100644 --- a/libs/testing/src/index.ts +++ b/libs/testing/src/index.ts @@ -10,3 +10,4 @@ export * from './user.service'; export * from './jobs.service'; export * from './testing-queue.service'; export * from './workflow-override.service'; +export * from './create-notification-template.interface'; From 0c2cd2bf03920f579946a30f31734a4c5cb1e052 Mon Sep 17 00:00:00 2001 From: Gosha Date: Sun, 17 Dec 2023 15:35:19 +0200 Subject: [PATCH 3/5] fix: create hash import --- apps/api/src/app/subscribers/e2e/chat-oauth-callback.e2e.ts | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/apps/api/src/app/subscribers/e2e/chat-oauth-callback.e2e.ts b/apps/api/src/app/subscribers/e2e/chat-oauth-callback.e2e.ts index 781b0c372a9..a48197a5b18 100644 --- a/apps/api/src/app/subscribers/e2e/chat-oauth-callback.e2e.ts +++ b/apps/api/src/app/subscribers/e2e/chat-oauth-callback.e2e.ts @@ -5,7 +5,7 @@ import * as sinon from 'sinon'; import { UserSession } from '@novu/testing'; import { ChannelTypeEnum, ChatProviderIdEnum } from '@novu/shared'; import { IntegrationRepository, SubscriberRepository } from '@novu/dal'; -import { createHash } from '../../shared/helpers/hmac.service'; +import { createHash } from '@novu/application-generic'; const axiosInstance = axios.create(); From 96de7965068ac5889796a7c6f972d672662b8893 Mon Sep 17 00:00:00 2001 From: Gosha Date: Mon, 18 Dec 2023 10:26:16 +0200 Subject: [PATCH 4/5] fix: import --- apps/api/src/app/subscribers/e2e/chat-oauth.e2e.ts | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/apps/api/src/app/subscribers/e2e/chat-oauth.e2e.ts b/apps/api/src/app/subscribers/e2e/chat-oauth.e2e.ts index 1684c58328e..c392490f2cd 100644 --- a/apps/api/src/app/subscribers/e2e/chat-oauth.e2e.ts +++ b/apps/api/src/app/subscribers/e2e/chat-oauth.e2e.ts @@ -4,7 +4,7 @@ import axios from 'axios'; import { UserSession } from '@novu/testing'; import { ChannelTypeEnum, ChatProviderIdEnum } from '@novu/shared'; import { IntegrationRepository, SubscriberRepository } from '@novu/dal'; -import { createHash } from '../../shared/helpers/hmac.service'; +import { createHash } from '@novu/application-generic'; const axiosInstance = axios.create(); From 241a2b6f2948a97419bc99577d8a7a4248455ef2 Mon Sep 17 00:00:00 2001 From: Gosha Date: Tue, 23 Jan 2024 18:22:22 +0200 Subject: [PATCH 5/5] test(widget): fix create hash import --- apps/api/src/app/widgets/e2e/initialize-widget-session.e2e.ts | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/apps/api/src/app/widgets/e2e/initialize-widget-session.e2e.ts b/apps/api/src/app/widgets/e2e/initialize-widget-session.e2e.ts index e2f5b0292e5..b9411df88f1 100644 --- a/apps/api/src/app/widgets/e2e/initialize-widget-session.e2e.ts +++ b/apps/api/src/app/widgets/e2e/initialize-widget-session.e2e.ts @@ -2,11 +2,11 @@ import { IntegrationRepository } from '@novu/dal'; import { ChannelTypeEnum, InAppProviderIdEnum } from '@novu/shared'; import { UserSession } from '@novu/testing'; import { expect } from 'chai'; -import { createHash } from '../../shared/helpers/hmac.service'; import { buildIntegrationKey, CacheInMemoryProviderService, CacheService, + createHash, InvalidateCacheService, } from '@novu/application-generic';