From dc83466790b45e1cdd34ff6cb9bd67f6d11e8e0e Mon Sep 17 00:00:00 2001 From: rostaklein Date: Wed, 24 Apr 2024 22:24:41 +0200 Subject: [PATCH 1/9] fetch and parse full gmail message --- .vscode/launch.json | 17 +++ package.json | 4 +- .../fetch-messages-by-batches.service.ts | 94 +++++++++--- ...mat-address-object-as-participants.util.ts | 27 ++-- .../types/gmail-message-parsed-response.ts | 12 +- .../modules/messaging/types/gmail-message.ts | 9 +- .../create-queries-from-message-ids.util.ts | 2 +- yarn.lock | 136 ++++-------------- 8 files changed, 148 insertions(+), 153 deletions(-) diff --git a/.vscode/launch.json b/.vscode/launch.json index 8c8d67934e0d..ea0b7ba27ec4 100644 --- a/.vscode/launch.json +++ b/.vscode/launch.json @@ -33,6 +33,23 @@ "internalConsoleOptions": "openOnSessionStart", "console": "internalConsole", "cwd": "${workspaceFolder}/packages/twenty-server/" + }, + { + "name": "twenty-server - gmail fetch debug", + "type": "node", + "request": "launch", + "runtimeExecutable": "yarn", + "runtimeVersion": "18", + "runtimeArgs": [ + "nx", + "run", + "twenty-server:command", + "cron:messaging:gmail-fetch-messages-from-cache", + ], + "outputCapture": "std", + "internalConsoleOptions": "openOnSessionStart", + "console": "internalConsole", + "cwd": "${workspaceFolder}/packages/twenty-server/" } ] } \ No newline at end of file diff --git a/package.json b/package.json index fa285253be7d..9db97da269b0 100644 --- a/package.json +++ b/package.json @@ -58,10 +58,10 @@ "@types/lodash.camelcase": "^4.3.7", "@types/lodash.merge": "^4.6.7", "@types/lodash.pick": "^4.3.7", - "@types/mailparser": "^3.4.4", "@types/nodemailer": "^6.4.14", "@types/passport-microsoft": "^1.0.3", "add": "^2.0.6", + "addressparser": "^1.0.1", "afterframe": "^1.0.2", "apollo-server-express": "^3.12.0", "apollo-upload-client": "^17.0.0", @@ -123,7 +123,6 @@ "lodash.snakecase": "^4.1.1", "lodash.upperfirst": "^4.3.1", "luxon": "^3.3.0", - "mailparser": "^3.6.5", "microdiff": "^1.3.2", "nest-commander": "^3.12.0", "next": "14.0.4", @@ -228,6 +227,7 @@ "@swc/helpers": "~0.5.2", "@testing-library/jest-dom": "^6.1.5", "@testing-library/react": "14.0.0", + "@types/addressparser": "^1.0.3", "@types/apollo-upload-client": "^17.0.2", "@types/bcrypt": "^5.0.0", "@types/better-sqlite3": "^7.6.8", diff --git a/packages/twenty-server/src/modules/messaging/services/fetch-messages-by-batches/fetch-messages-by-batches.service.ts b/packages/twenty-server/src/modules/messaging/services/fetch-messages-by-batches/fetch-messages-by-batches.service.ts index cde2d7dccd99..ef8d8001d3e0 100644 --- a/packages/twenty-server/src/modules/messaging/services/fetch-messages-by-batches/fetch-messages-by-batches.service.ts +++ b/packages/twenty-server/src/modules/messaging/services/fetch-messages-by-batches/fetch-messages-by-batches.service.ts @@ -1,14 +1,15 @@ import { Injectable, Logger } from '@nestjs/common'; import { AxiosResponse } from 'axios'; -import { simpleParser } from 'mailparser'; import planer from 'planer'; +import addressparser from 'addressparser'; import { GmailMessage } from 'src/modules/messaging/types/gmail-message'; import { MessageQuery } from 'src/modules/messaging/types/message-or-thread-query'; import { GmailMessageParsedResponse } from 'src/modules/messaging/types/gmail-message-parsed-response'; import { FetchByBatchesService } from 'src/modules/messaging/services/fetch-by-batch/fetch-by-batch.service'; import { formatAddressObjectAsParticipants } from 'src/modules/messaging/services/utils/format-address-object-as-participants.util'; +import { assert } from 'src/utils/assert'; @Injectable() export class FetchMessagesByBatchesService { @@ -73,28 +74,25 @@ export class FetchMessagesByBatchesService { return; } - const { historyId, id, threadId, internalDate, raw } = message; - - const body = atob(raw?.replace(/-/g, '+').replace(/_/g, '/')); - try { - const parsed = await simpleParser(body, { - skipHtmlToText: true, - skipImageLinks: true, - skipTextToHtml: true, - maxHtmlLengthToParse: 0, - }); - - const { subject, messageId, from, to, cc, bcc, text, attachments } = - parsed; + const { + historyId, + id, + threadId, + internalDate, + subject, + from, + to, + headerMessageId, + text, + attachments, + } = this.parseGmailMessage(message); if (!from) throw new Error('From value is missing'); const participants = [ ...formatAddressObjectAsParticipants(from, 'from'), ...formatAddressObjectAsParticipants(to, 'to'), - ...formatAddressObjectAsParticipants(cc, 'cc'), - ...formatAddressObjectAsParticipants(bcc, 'bcc'), ]; let textWithoutReplyQuotations = text; @@ -115,12 +113,12 @@ export class FetchMessagesByBatchesService { const messageFromGmail: GmailMessage = { historyId, externalId: id, - headerMessageId: messageId || '', + headerMessageId, subject: subject || '', messageThreadExternalId: threadId, internalDate, - fromHandle: from.value[0].address || '', - fromDisplayName: from.value[0].name || '', + fromHandle: from[0].address || '', + fromDisplayName: from[0].name || '', participants, text: sanitizeString(textWithoutReplyQuotations || ''), attachments, @@ -157,4 +155,62 @@ export class FetchMessagesByBatchesService { return { messages, errors }; } + + private parseGmailMessage(message: GmailMessageParsedResponse) { + const subject = this.getPropertyFromHeaders(message, 'Subject'); + const rawFrom = this.getPropertyFromHeaders(message, 'From'); + const rawTo = this.getPropertyFromHeaders(message, 'To'); + const messageId = this.getPropertyFromHeaders(message, 'Message-ID'); + const id = message.id; + const threadId = message.threadId; + const historyId = message.historyId; + const internalDate = message.internalDate; + + assert(id); + assert(threadId); + assert(historyId); + assert(internalDate); + + const bodyData = this.getBodyData(message); + const text = bodyData ? Buffer.from(bodyData, 'base64').toString() : ''; + + return { + id, + headerMessageId: messageId, + threadId, + historyId, + internalDate, + subject, + from: addressparser(rawFrom), + to: addressparser(rawTo), + text, + attachments: [], + }; + } + + private getBodyData(message: GmailMessageParsedResponse) { + const firstPart = message.payload?.parts?.[0]; + + if (firstPart?.mimeType === 'text/plain') { + return firstPart?.body?.data; + } + + return firstPart?.parts?.find((part) => part.mimeType === 'text/plain') + ?.body?.data; + } + + private getPropertyFromHeaders( + message: GmailMessageParsedResponse, + property: string, + ) { + const value = message.payload?.headers?.find( + (header) => header.name === property, + )?.value; + + if (value === undefined || value === null) { + throw new Error(`Cannot find property "${property}" in message headers`); + } + + return value; + } } diff --git a/packages/twenty-server/src/modules/messaging/services/utils/format-address-object-as-participants.util.ts b/packages/twenty-server/src/modules/messaging/services/utils/format-address-object-as-participants.util.ts index 674bfa2c7d44..113a9c1d2b3b 100644 --- a/packages/twenty-server/src/modules/messaging/services/utils/format-address-object-as-participants.util.ts +++ b/packages/twenty-server/src/modules/messaging/services/utils/format-address-object-as-participants.util.ts @@ -1,10 +1,10 @@ -import { AddressObject } from 'mailparser'; +import addressparser from 'addressparser'; import { Participant } from 'src/modules/messaging/types/gmail-message'; const formatAddressObjectAsArray = ( - addressObject: AddressObject | AddressObject[], -): AddressObject[] => { + addressObject: addressparser.EmailAddress | addressparser.EmailAddress[], +): addressparser.EmailAddress[] => { return Array.isArray(addressObject) ? addressObject : [addressObject]; }; @@ -13,24 +13,23 @@ const removeSpacesAndLowerCase = (email: string): string => { }; export const formatAddressObjectAsParticipants = ( - addressObject: AddressObject | AddressObject[] | undefined, + addressObject: + | addressparser.EmailAddress + | addressparser.EmailAddress[] + | undefined, role: 'from' | 'to' | 'cc' | 'bcc', ): Participant[] => { if (!addressObject) return []; const addressObjects = formatAddressObjectAsArray(addressObject); const participants = addressObjects.map((addressObject) => { - const emailAdresses = addressObject.value; + const address = addressObject.address; - return emailAdresses.map((emailAddress) => { - const { name, address } = emailAddress; - - return { - role, - handle: address ? removeSpacesAndLowerCase(address) : '', - displayName: name || '', - }; - }); + return { + role, + handle: address ? removeSpacesAndLowerCase(address) : '', + displayName: addressObject.name || '', + }; }); return participants.flat(); diff --git a/packages/twenty-server/src/modules/messaging/types/gmail-message-parsed-response.ts b/packages/twenty-server/src/modules/messaging/types/gmail-message-parsed-response.ts index ef888321b20e..c84c52440a98 100644 --- a/packages/twenty-server/src/modules/messaging/types/gmail-message-parsed-response.ts +++ b/packages/twenty-server/src/modules/messaging/types/gmail-message-parsed-response.ts @@ -1,12 +1,6 @@ -export type GmailMessageParsedResponse = { - id: string; - threadId: string; - labelIds: string[]; - snippet: string; - sizeEstimate: number; - raw: string; - historyId: string; - internalDate: string; +import { gmail_v1 } from 'googleapis'; + +export type GmailMessageParsedResponse = gmail_v1.Schema$Message & { error?: { code: number; message: string; diff --git a/packages/twenty-server/src/modules/messaging/types/gmail-message.ts b/packages/twenty-server/src/modules/messaging/types/gmail-message.ts index fe9b693e058b..8e50595d1c44 100644 --- a/packages/twenty-server/src/modules/messaging/types/gmail-message.ts +++ b/packages/twenty-server/src/modules/messaging/types/gmail-message.ts @@ -1,5 +1,3 @@ -import { Attachment } from 'mailparser'; - export type GmailMessage = { historyId: string; externalId: string; @@ -25,3 +23,10 @@ export type ParticipantWithMessageId = Participant & { messageId: string }; export type ParticipantWithId = Participant & { id: string; }; + +export type Attachment = { + id: string; + filename: string; + size: number; + mimeType: string; +}; diff --git a/packages/twenty-server/src/modules/messaging/utils/create-queries-from-message-ids.util.ts b/packages/twenty-server/src/modules/messaging/utils/create-queries-from-message-ids.util.ts index 304454488f15..fbf654934a36 100644 --- a/packages/twenty-server/src/modules/messaging/utils/create-queries-from-message-ids.util.ts +++ b/packages/twenty-server/src/modules/messaging/utils/create-queries-from-message-ids.util.ts @@ -4,6 +4,6 @@ export const createQueriesFromMessageIds = ( messageExternalIds: string[], ): MessageQuery[] => { return messageExternalIds.map((messageId) => ({ - uri: '/gmail/v1/users/me/messages/' + messageId + '?format=RAW', + uri: '/gmail/v1/users/me/messages/' + messageId + '?format=FULL', })); }; diff --git a/yarn.lock b/yarn.lock index 8ba667ee181b..7755935baee4 100644 --- a/yarn.lock +++ b/yarn.lock @@ -15757,6 +15757,15 @@ __metadata: languageName: node linkType: hard +"@types/addressparser@npm:^1.0.3": + version: 1.0.3 + resolution: "@types/addressparser@npm:1.0.3" + dependencies: + "@types/node": "npm:*" + checksum: ed8dfe05271eca6b4904a870fc3fd9b5d0e72c7e7aa36254dee9ede496bbbc16acd3630bc072b7e5d84fabb9257e4ed4a82c363b2f4a99c877e3a1accc271fc6 + languageName: node + linkType: hard + "@types/apollo-upload-client@npm:^17.0.2": version: 17.0.5 resolution: "@types/apollo-upload-client@npm:17.0.5" @@ -16729,16 +16738,6 @@ __metadata: languageName: node linkType: hard -"@types/mailparser@npm:^3.4.4": - version: 3.4.4 - resolution: "@types/mailparser@npm:3.4.4" - dependencies: - "@types/node": "npm:*" - iconv-lite: "npm:^0.6.3" - checksum: 5d16e87cebff438f9e725ebb4f4cea4e6c55dfa1d5cdda3c56f3f91b915a0801a84675fee2a8d20b6de20ca8be79678a4e99fb5956104e2eb3344dfac387691c - languageName: node - linkType: hard - "@types/mdast@npm:^3.0.0": version: 3.0.15 resolution: "@types/mdast@npm:3.0.15" @@ -18508,6 +18507,13 @@ __metadata: languageName: node linkType: hard +"addressparser@npm:^1.0.1": + version: 1.0.1 + resolution: "addressparser@npm:1.0.1" + checksum: 15a6b149c643e3fb0888bcad89aa385e7718714a33049b5f357063b64b84a2febd6a0775011783c25e6b161982663d38a84fd4522de69adb4779971b92b4ddb3 + languageName: node + linkType: hard + "adm-zip@npm:0.5.10": version: 0.5.10 resolution: "adm-zip@npm:0.5.10" @@ -25322,13 +25328,6 @@ __metadata: languageName: node linkType: hard -"encoding-japanese@npm:2.0.0": - version: 2.0.0 - resolution: "encoding-japanese@npm:2.0.0" - checksum: 453bbca71d3666213a9bc873d5a69441b379f158a2992aa5cd1fc124c915b518e19fce7654f973d1334234f870e8053443a464c8f73ff9d7efe66bbc1ce1f4f6 - languageName: node - linkType: hard - "encoding@npm:^0.1.12, encoding@npm:^0.1.13": version: 0.1.13 resolution: "encoding@npm:0.1.13" @@ -29737,7 +29736,7 @@ __metadata: languageName: node linkType: hard -"he@npm:1.2.0, he@npm:^1.2.0": +"he@npm:^1.2.0": version: 1.2.0 resolution: "he@npm:1.2.0" bin: @@ -30363,7 +30362,7 @@ __metadata: languageName: node linkType: hard -"iconv-lite@npm:0.6.3, iconv-lite@npm:^0.6.2, iconv-lite@npm:^0.6.3": +"iconv-lite@npm:0.6.3, iconv-lite@npm:^0.6.2": version: 0.6.3 resolution: "iconv-lite@npm:0.6.3" dependencies: @@ -33430,37 +33429,6 @@ __metadata: languageName: node linkType: hard -"libbase64@npm:1.2.1": - version: 1.2.1 - resolution: "libbase64@npm:1.2.1" - checksum: 908db9dc88cbcd9e1b9355c78b9fefde5034d933a50e823bbbb6008a56908de1e5183e25bf648e9e7fe907f53e10e11676d5ac89fd624a300c46a705556182a5 - languageName: node - linkType: hard - -"libmime@npm:5.2.0": - version: 5.2.0 - resolution: "libmime@npm:5.2.0" - dependencies: - encoding-japanese: "npm:2.0.0" - iconv-lite: "npm:0.6.3" - libbase64: "npm:1.2.1" - libqp: "npm:2.0.1" - checksum: 22a75d7aad8f01bed7d9b32270a40a32c4d4e44070edda1067ea5229df99a09f34aedf3481693394aa998fa8375b6c90d1c651b491655692cb313561c5a48762 - languageName: node - linkType: hard - -"libmime@npm:5.2.1": - version: 5.2.1 - resolution: "libmime@npm:5.2.1" - dependencies: - encoding-japanese: "npm:2.0.0" - iconv-lite: "npm:0.6.3" - libbase64: "npm:1.2.1" - libqp: "npm:2.0.1" - checksum: cf91c78a05824f160e45b36850d52eee9e18073bfd4561ace3b3af8e52a8d551eccc0dcef428505e44d1f2146c16cec84e91e17d9489244451c38572862a857a - languageName: node - linkType: hard - "libphonenumber-js@npm:^1.10.14, libphonenumber-js@npm:^1.10.26, libphonenumber-js@npm:^1.10.53": version: 1.10.53 resolution: "libphonenumber-js@npm:1.10.53" @@ -33468,13 +33436,6 @@ __metadata: languageName: node linkType: hard -"libqp@npm:2.0.1": - version: 2.0.1 - resolution: "libqp@npm:2.0.1" - checksum: c52e51c70180fbf0b000036de33ed976da1f8355fd63feffbbf5a9653a816e9169917b1ce28b289a5006b28e44b2d84d234fdedbdfefc0de4802867aa03537df - languageName: node - linkType: hard - "lilconfig@npm:^2.0.3": version: 2.1.0 resolution: "lilconfig@npm:2.1.0" @@ -33503,15 +33464,6 @@ __metadata: languageName: node linkType: hard -"linkify-it@npm:5.0.0, linkify-it@npm:^5.0.0": - version: 5.0.0 - resolution: "linkify-it@npm:5.0.0" - dependencies: - uc.micro: "npm:^2.0.0" - checksum: ff4abbcdfa2003472fc3eb4b8e60905ec97718e11e33cca52059919a4c80cc0e0c2a14d23e23d8c00e5402bc5a885cdba8ca053a11483ab3cc8b3c7a52f88e2d - languageName: node - linkType: hard - "linkify-it@npm:^3.0.1": version: 3.0.3 resolution: "linkify-it@npm:3.0.3" @@ -33521,6 +33473,15 @@ __metadata: languageName: node linkType: hard +"linkify-it@npm:^5.0.0": + version: 5.0.0 + resolution: "linkify-it@npm:5.0.0" + dependencies: + uc.micro: "npm:^2.0.0" + checksum: ff4abbcdfa2003472fc3eb4b8e60905ec97718e11e33cca52059919a4c80cc0e0c2a14d23e23d8c00e5402bc5a885cdba8ca053a11483ab3cc8b3c7a52f88e2d + languageName: node + linkType: hard + "linkifyjs@npm:^4.1.0": version: 4.1.3 resolution: "linkifyjs@npm:4.1.3" @@ -34216,34 +34177,6 @@ __metadata: languageName: node linkType: hard -"mailparser@npm:^3.6.5": - version: 3.6.6 - resolution: "mailparser@npm:3.6.6" - dependencies: - encoding-japanese: "npm:2.0.0" - he: "npm:1.2.0" - html-to-text: "npm:9.0.5" - iconv-lite: "npm:0.6.3" - libmime: "npm:5.2.1" - linkify-it: "npm:5.0.0" - mailsplit: "npm:5.4.0" - nodemailer: "npm:6.9.8" - tlds: "npm:1.248.0" - checksum: 5cf6f3f3d457b7564aa96d3e682f1f4cb27ffcdb86219138568b4e2b6bb8ebea6d0b14cd9f6c75b9ff4b33c6f3151d04fbb30b4d5d3a7ba688d8e04e83c3bd5a - languageName: node - linkType: hard - -"mailsplit@npm:5.4.0": - version: 5.4.0 - resolution: "mailsplit@npm:5.4.0" - dependencies: - libbase64: "npm:1.2.1" - libmime: "npm:5.2.0" - libqp: "npm:2.0.1" - checksum: b0e1ce1866ea44413ca0ee8b7291afb671cb3f7ced2a53c644e3097b64b74079a4cb1ec02c9aaaef6a9927a71187304ac1a809852503aba2f829b67ce2d41496 - languageName: node - linkType: hard - "make-dir@npm:^2.0.0, make-dir@npm:^2.1.0": version: 2.1.0 resolution: "make-dir@npm:2.1.0" @@ -37465,7 +37398,7 @@ __metadata: languageName: node linkType: hard -"nodemailer@npm:6.9.8, nodemailer@npm:^6.9.8": +"nodemailer@npm:^6.9.8": version: 6.9.8 resolution: "nodemailer@npm:6.9.8" checksum: 9332587975240ac648e1295b1df15e339fcace3f7fab8af0382e7f2dd10e48296344dfa698d58f1667f220f7fe13c779d55d39144c9cd9ed6f5f559714183c75 @@ -45581,15 +45514,6 @@ __metadata: languageName: node linkType: hard -"tlds@npm:1.248.0": - version: 1.248.0 - resolution: "tlds@npm:1.248.0" - bin: - tlds: bin.js - checksum: 640cb52fa15c116ef6b76ca81173c37314b482c3f2e70424fd8684ae172f4a007be40f6e3b47afcd530c6586b877ddd6142640fc18a0b9c0ae5ce5ffd9f27a0f - languageName: node - linkType: hard - "tmp@npm:0.2.1, tmp@npm:~0.2.1": version: 0.2.1 resolution: "tmp@npm:0.2.1" @@ -46435,6 +46359,7 @@ __metadata: "@tabler/icons-react": "npm:^2.44.0" "@testing-library/jest-dom": "npm:^6.1.5" "@testing-library/react": "npm:14.0.0" + "@types/addressparser": "npm:^1.0.3" "@types/apollo-upload-client": "npm:^17.0.2" "@types/bcrypt": "npm:^5.0.0" "@types/better-sqlite3": "npm:^7.6.8" @@ -46462,7 +46387,6 @@ __metadata: "@types/lodash.snakecase": "npm:^4.1.7" "@types/lodash.upperfirst": "npm:^4.3.7" "@types/luxon": "npm:^3.3.0" - "@types/mailparser": "npm:^3.4.4" "@types/ms": "npm:^0.7.31" "@types/node": "npm:18.19.26" "@types/nodemailer": "npm:^6.4.14" @@ -46482,6 +46406,7 @@ __metadata: "@vitejs/plugin-react-swc": "npm:^3.5.0" "@vitest/ui": "npm:1.4.0" add: "npm:^2.0.6" + addressparser: "npm:^1.0.1" afterframe: "npm:^1.0.2" apollo-server-express: "npm:^3.12.0" apollo-upload-client: "npm:^17.0.0" @@ -46569,7 +46494,6 @@ __metadata: lodash.snakecase: "npm:^4.1.1" lodash.upperfirst: "npm:^4.3.1" luxon: "npm:^3.3.0" - mailparser: "npm:^3.6.5" microdiff: "npm:^1.3.2" msw: "npm:^2.0.11" msw-storybook-addon: "npm:2.0.0--canary.122.b3ed3b1.0" From c2ec26027023d79438930d1bd8be65eac68a70ac Mon Sep 17 00:00:00 2001 From: rostaklein Date: Tue, 30 Apr 2024 22:48:55 +0200 Subject: [PATCH 2/9] revert launch json --- .vscode/launch.json | 17 ----------------- 1 file changed, 17 deletions(-) diff --git a/.vscode/launch.json b/.vscode/launch.json index ea0b7ba27ec4..8c8d67934e0d 100644 --- a/.vscode/launch.json +++ b/.vscode/launch.json @@ -33,23 +33,6 @@ "internalConsoleOptions": "openOnSessionStart", "console": "internalConsole", "cwd": "${workspaceFolder}/packages/twenty-server/" - }, - { - "name": "twenty-server - gmail fetch debug", - "type": "node", - "request": "launch", - "runtimeExecutable": "yarn", - "runtimeVersion": "18", - "runtimeArgs": [ - "nx", - "run", - "twenty-server:command", - "cron:messaging:gmail-fetch-messages-from-cache", - ], - "outputCapture": "std", - "internalConsoleOptions": "openOnSessionStart", - "console": "internalConsole", - "cwd": "${workspaceFolder}/packages/twenty-server/" } ] } \ No newline at end of file From d4831b01bd88257dc088dee7a11f2c4acee09cd5 Mon Sep 17 00:00:00 2001 From: Charles Bochet Date: Wed, 8 May 2024 11:56:06 +0200 Subject: [PATCH 3/9] Authorize case mismatch in mail headers parsing --- .../gmail-users-messages-get-batch-size.constant.ts | 2 +- .../fetch-messages-by-batches.service.ts | 8 +------- .../messaging/types/gmail-message-parsed-response.ts | 8 +------- 3 files changed, 3 insertions(+), 15 deletions(-) diff --git a/packages/twenty-server/src/modules/messaging/constants/gmail-users-messages-get-batch-size.constant.ts b/packages/twenty-server/src/modules/messaging/constants/gmail-users-messages-get-batch-size.constant.ts index 5c3803f4abe3..a883ab0d4cc5 100644 --- a/packages/twenty-server/src/modules/messaging/constants/gmail-users-messages-get-batch-size.constant.ts +++ b/packages/twenty-server/src/modules/messaging/constants/gmail-users-messages-get-batch-size.constant.ts @@ -1 +1 @@ -export const GMAIL_USERS_MESSAGES_GET_BATCH_SIZE = 10; +export const GMAIL_USERS_MESSAGES_GET_BATCH_SIZE = 50; diff --git a/packages/twenty-server/src/modules/messaging/services/fetch-messages-by-batches/fetch-messages-by-batches.service.ts b/packages/twenty-server/src/modules/messaging/services/fetch-messages-by-batches/fetch-messages-by-batches.service.ts index ef8d8001d3e0..18a27cc408f8 100644 --- a/packages/twenty-server/src/modules/messaging/services/fetch-messages-by-batches/fetch-messages-by-batches.service.ts +++ b/packages/twenty-server/src/modules/messaging/services/fetch-messages-by-batches/fetch-messages-by-batches.service.ts @@ -68,12 +68,6 @@ export class FetchMessagesByBatchesService { const formattedResponse = Promise.all( parsedResponses.map(async (message: GmailMessageParsedResponse) => { - if (message.error) { - errors.push(message.error); - - return; - } - try { const { historyId, @@ -204,7 +198,7 @@ export class FetchMessagesByBatchesService { property: string, ) { const value = message.payload?.headers?.find( - (header) => header.name === property, + (header) => header.name?.toLowerCase() === property.toLowerCase(), )?.value; if (value === undefined || value === null) { diff --git a/packages/twenty-server/src/modules/messaging/types/gmail-message-parsed-response.ts b/packages/twenty-server/src/modules/messaging/types/gmail-message-parsed-response.ts index c84c52440a98..8037a382884e 100644 --- a/packages/twenty-server/src/modules/messaging/types/gmail-message-parsed-response.ts +++ b/packages/twenty-server/src/modules/messaging/types/gmail-message-parsed-response.ts @@ -1,9 +1,3 @@ import { gmail_v1 } from 'googleapis'; -export type GmailMessageParsedResponse = gmail_v1.Schema$Message & { - error?: { - code: number; - message: string; - status: string; - }; -}; +export type GmailMessageParsedResponse = gmail_v1.Schema$Message; From 7a4446dc73699d91b7c652f65e7d7783c0246228 Mon Sep 17 00:00:00 2001 From: Charles Bochet Date: Wed, 8 May 2024 12:27:12 +0200 Subject: [PATCH 4/9] Reduce batch size --- .../constants/gmail-users-messages-get-batch-size.constant.ts | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/packages/twenty-server/src/modules/messaging/constants/gmail-users-messages-get-batch-size.constant.ts b/packages/twenty-server/src/modules/messaging/constants/gmail-users-messages-get-batch-size.constant.ts index a883ab0d4cc5..ccc1d42df096 100644 --- a/packages/twenty-server/src/modules/messaging/constants/gmail-users-messages-get-batch-size.constant.ts +++ b/packages/twenty-server/src/modules/messaging/constants/gmail-users-messages-get-batch-size.constant.ts @@ -1 +1 @@ -export const GMAIL_USERS_MESSAGES_GET_BATCH_SIZE = 50; +export const GMAIL_USERS_MESSAGES_GET_BATCH_SIZE = 30; From bc3063993b219ffcce59806bf583412f8fbcd3a9 Mon Sep 17 00:00:00 2001 From: rostaklein Date: Thu, 9 May 2024 21:35:54 +0200 Subject: [PATCH 5/9] stop storing errors in an array --- .../fetch-messages-by-batches.service.ts | 154 +++++++++--------- ...etch-message-content-from-cache.service.ts | 18 +- 2 files changed, 78 insertions(+), 94 deletions(-) diff --git a/packages/twenty-server/src/modules/messaging/services/fetch-messages-by-batches/fetch-messages-by-batches.service.ts b/packages/twenty-server/src/modules/messaging/services/fetch-messages-by-batches/fetch-messages-by-batches.service.ts index 18a27cc408f8..6cc824b7e7e1 100644 --- a/packages/twenty-server/src/modules/messaging/services/fetch-messages-by-batches/fetch-messages-by-batches.service.ts +++ b/packages/twenty-server/src/modules/messaging/services/fetch-messages-by-batches/fetch-messages-by-batches.service.ts @@ -6,10 +6,10 @@ import addressparser from 'addressparser'; import { GmailMessage } from 'src/modules/messaging/types/gmail-message'; import { MessageQuery } from 'src/modules/messaging/types/message-or-thread-query'; -import { GmailMessageParsedResponse } from 'src/modules/messaging/types/gmail-message-parsed-response'; import { FetchByBatchesService } from 'src/modules/messaging/services/fetch-by-batch/fetch-by-batch.service'; import { formatAddressObjectAsParticipants } from 'src/modules/messaging/services/utils/format-address-object-as-participants.util'; -import { assert } from 'src/utils/assert'; +import { assert, assertNotNull } from 'src/utils/assert'; +import { GmailMessageParsedResponse } from 'src/modules/messaging/types/gmail-message-parsed-response'; @Injectable() export class FetchMessagesByBatchesService { @@ -22,7 +22,7 @@ export class FetchMessagesByBatchesService { accessToken: string, workspaceId?: string, connectedAccountId?: string, - ): Promise<{ messages: GmailMessage[]; errors: any[] }> { + ): Promise { let startTime = Date.now(); const batchResponses = await this.fetchByBatchesService.fetchAllByBatches( queries, @@ -55,99 +55,99 @@ export class FetchMessagesByBatchesService { async formatBatchResponseAsGmailMessage( responseCollection: AxiosResponse, - ): Promise<{ messages: GmailMessage[]; errors: any[] }> { - const parsedResponses = this.fetchByBatchesService.parseBatch( - responseCollection, - ) as GmailMessageParsedResponse[]; - - const errors: any = []; + ): Promise { + const parsedResponses = + this.fetchByBatchesService.parseBatch(responseCollection); const sanitizeString = (str: string) => { return str.replace(/\0/g, ''); }; - const formattedResponse = Promise.all( - parsedResponses.map(async (message: GmailMessageParsedResponse) => { - try { - const { - historyId, - id, - threadId, - internalDate, - subject, - from, - to, - headerMessageId, - text, - attachments, - } = this.parseGmailMessage(message); - - if (!from) throw new Error('From value is missing'); - - const participants = [ - ...formatAddressObjectAsParticipants(from, 'from'), - ...formatAddressObjectAsParticipants(to, 'to'), - ]; - - let textWithoutReplyQuotations = text; - - if (text) - try { - textWithoutReplyQuotations = planer.extractFrom( - text, - 'text/plain', - ); - } catch (error) { - console.log( - 'Error while trying to remove reply quotations', - error, - ); + const formattedResponse = await Promise.all( + parsedResponses.map( + async ( + message: GmailMessageParsedResponse, + ): Promise => { + try { + const { + historyId, + id, + threadId, + internalDate, + subject, + from, + to, + headerMessageId, + text, + attachments, + } = this.parseGmailMessage(message); + + if (!from) throw new Error('From value is missing'); + + const participants = [ + ...formatAddressObjectAsParticipants(from, 'from'), + ...formatAddressObjectAsParticipants(to, 'to'), + ]; + + let textWithoutReplyQuotations = text; + + if (text) + try { + textWithoutReplyQuotations = planer.extractFrom( + text, + 'text/plain', + ); + } catch (error) { + console.log( + 'Error while trying to remove reply quotations', + error, + ); + } + + const messageFromGmail: GmailMessage = { + historyId, + externalId: id, + headerMessageId, + subject: subject || '', + messageThreadExternalId: threadId, + internalDate, + fromHandle: from[0].address || '', + fromDisplayName: from[0].name || '', + participants, + text: sanitizeString(textWithoutReplyQuotations || ''), + attachments, + }; + + return messageFromGmail; + } catch (error) { + // if message was not found, silenty ignore it + if (error?.code === 404) { + return null; } - const messageFromGmail: GmailMessage = { - historyId, - externalId: id, - headerMessageId, - subject: subject || '', - messageThreadExternalId: threadId, - internalDate, - fromHandle: from[0].address || '', - fromDisplayName: from[0].name || '', - participants, - text: sanitizeString(textWithoutReplyQuotations || ''), - attachments, - }; - - return messageFromGmail; - } catch (error) { - console.log('Error', error); - - errors.push(error); - } - }), + throw error; + } + }, + ), ); - const filteredMessages = (await formattedResponse).filter( - (message) => message, + const filteredMessages = formattedResponse.filter((message) => + assertNotNull(message), ) as GmailMessage[]; - return { messages: filteredMessages, errors }; + return filteredMessages; } async formatBatchResponsesAsGmailMessages( batchResponses: AxiosResponse[], - ): Promise<{ messages: GmailMessage[]; errors: any[] }> { - const messagesAndErrors = await Promise.all( + ): Promise { + const messageBatches = await Promise.all( batchResponses.map(async (response) => { return this.formatBatchResponseAsGmailMessage(response); }), ); - const messages = messagesAndErrors.map((item) => item.messages).flat(); - - const errors = messagesAndErrors.map((item) => item.errors).flat(); - - return { messages, errors }; + return messageBatches.flat(); } private parseGmailMessage(message: GmailMessageParsedResponse) { @@ -198,7 +198,7 @@ export class FetchMessagesByBatchesService { property: string, ) { const value = message.payload?.headers?.find( - (header) => header.name?.toLowerCase() === property.toLowerCase(), + (header) => header.name === property, )?.value; if (value === undefined || value === null) { diff --git a/packages/twenty-server/src/modules/messaging/services/gmail-fetch-message-content-from-cache/gmail-fetch-message-content-from-cache.service.ts b/packages/twenty-server/src/modules/messaging/services/gmail-fetch-message-content-from-cache/gmail-fetch-message-content-from-cache.service.ts index 35556a7d1091..80def19b9f27 100644 --- a/packages/twenty-server/src/modules/messaging/services/gmail-fetch-message-content-from-cache/gmail-fetch-message-content-from-cache.service.ts +++ b/packages/twenty-server/src/modules/messaging/services/gmail-fetch-message-content-from-cache/gmail-fetch-message-content-from-cache.service.ts @@ -174,7 +174,7 @@ export class GmailFetchMessageContentFromCacheService { const messageQueries = createQueriesFromMessageIds(messageIdsToFetch); try { - const { messages: messagesToSave, errors } = + const messagesToSave = await this.fetchMessagesByBatchesService.fetchAllMessages( messageQueries, accessToken, @@ -194,22 +194,6 @@ export class GmailFetchMessageContentFromCacheService { return []; } - if (errors.length) { - const errorsCanBeIgnored = errors.every( - (error) => error.code === 404, - ); - - if (!errorsCanBeIgnored) { - throw new Error( - `Error fetching messages for ${connectedAccountId} in workspace ${workspaceId}: ${JSON.stringify( - errors, - null, - 2, - )}`, - ); - } - } - const messageExternalIdsAndIdsMap = await this.messageService.saveMessagesWithinTransaction( messagesToSave, From 5632a668a1f7e55e65ca4010d62a0f212f2eee75 Mon Sep 17 00:00:00 2001 From: rostaklein Date: Thu, 9 May 2024 21:37:22 +0200 Subject: [PATCH 6/9] cc and bcc back --- .../fetch-messages-by-batches.service.ts | 8 ++++++++ 1 file changed, 8 insertions(+) diff --git a/packages/twenty-server/src/modules/messaging/services/fetch-messages-by-batches/fetch-messages-by-batches.service.ts b/packages/twenty-server/src/modules/messaging/services/fetch-messages-by-batches/fetch-messages-by-batches.service.ts index 6cc824b7e7e1..5618814453c4 100644 --- a/packages/twenty-server/src/modules/messaging/services/fetch-messages-by-batches/fetch-messages-by-batches.service.ts +++ b/packages/twenty-server/src/modules/messaging/services/fetch-messages-by-batches/fetch-messages-by-batches.service.ts @@ -77,6 +77,8 @@ export class FetchMessagesByBatchesService { subject, from, to, + cc, + bcc, headerMessageId, text, attachments, @@ -87,6 +89,8 @@ export class FetchMessagesByBatchesService { const participants = [ ...formatAddressObjectAsParticipants(from, 'from'), ...formatAddressObjectAsParticipants(to, 'to'), + ...formatAddressObjectAsParticipants(cc, 'cc'), + ...formatAddressObjectAsParticipants(bcc, 'bcc'), ]; let textWithoutReplyQuotations = text; @@ -154,6 +158,8 @@ export class FetchMessagesByBatchesService { const subject = this.getPropertyFromHeaders(message, 'Subject'); const rawFrom = this.getPropertyFromHeaders(message, 'From'); const rawTo = this.getPropertyFromHeaders(message, 'To'); + const rawCc = this.getPropertyFromHeaders(message, 'Cc'); + const rawBcc = this.getPropertyFromHeaders(message, 'Bcc'); const messageId = this.getPropertyFromHeaders(message, 'Message-ID'); const id = message.id; const threadId = message.threadId; @@ -177,6 +183,8 @@ export class FetchMessagesByBatchesService { subject, from: addressparser(rawFrom), to: addressparser(rawTo), + cc: addressparser(rawCc), + bcc: addressparser(rawBcc), text, attachments: [], }; From daa91caedcfac080d9cfe01a425b05be43cd9586 Mon Sep 17 00:00:00 2001 From: rostaklein Date: Thu, 9 May 2024 22:43:27 +0200 Subject: [PATCH 7/9] remove unnecessary promises, working with GmailMessageError --- .../fetch-messages-by-batches.service.ts | 180 +++++++++--------- .../types/gmail-message-parsed-response.ts | 12 +- 2 files changed, 100 insertions(+), 92 deletions(-) diff --git a/packages/twenty-server/src/modules/messaging/services/fetch-messages-by-batches/fetch-messages-by-batches.service.ts b/packages/twenty-server/src/modules/messaging/services/fetch-messages-by-batches/fetch-messages-by-batches.service.ts index 5618814453c4..a94b74d448bf 100644 --- a/packages/twenty-server/src/modules/messaging/services/fetch-messages-by-batches/fetch-messages-by-batches.service.ts +++ b/packages/twenty-server/src/modules/messaging/services/fetch-messages-by-batches/fetch-messages-by-batches.service.ts @@ -3,13 +3,13 @@ import { Injectable, Logger } from '@nestjs/common'; import { AxiosResponse } from 'axios'; import planer from 'planer'; import addressparser from 'addressparser'; +import { gmail_v1 } from 'googleapis'; import { GmailMessage } from 'src/modules/messaging/types/gmail-message'; import { MessageQuery } from 'src/modules/messaging/types/message-or-thread-query'; import { FetchByBatchesService } from 'src/modules/messaging/services/fetch-by-batch/fetch-by-batch.service'; import { formatAddressObjectAsParticipants } from 'src/modules/messaging/services/utils/format-address-object-as-participants.util'; import { assert, assertNotNull } from 'src/utils/assert'; -import { GmailMessageParsedResponse } from 'src/modules/messaging/types/gmail-message-parsed-response'; @Injectable() export class FetchMessagesByBatchesService { @@ -40,7 +40,7 @@ export class FetchMessagesByBatchesService { startTime = Date.now(); const formattedResponse = - await this.formatBatchResponsesAsGmailMessages(batchResponses); + this.formatBatchResponsesAsGmailMessages(batchResponses); endTime = Date.now(); @@ -53,9 +53,9 @@ export class FetchMessagesByBatchesService { return formattedResponse; } - async formatBatchResponseAsGmailMessage( + private formatBatchResponseAsGmailMessage( responseCollection: AxiosResponse, - ): Promise { + ): GmailMessage[] { const parsedResponses = this.fetchByBatchesService.parseBatch(responseCollection); @@ -63,76 +63,77 @@ export class FetchMessagesByBatchesService { return str.replace(/\0/g, ''); }; - const formattedResponse = await Promise.all( - parsedResponses.map( - async ( - message: GmailMessageParsedResponse, - ): Promise => { - try { - const { - historyId, - id, - threadId, - internalDate, - subject, - from, - to, - cc, - bcc, - headerMessageId, - text, - attachments, - } = this.parseGmailMessage(message); - - if (!from) throw new Error('From value is missing'); - - const participants = [ - ...formatAddressObjectAsParticipants(from, 'from'), - ...formatAddressObjectAsParticipants(to, 'to'), - ...formatAddressObjectAsParticipants(cc, 'cc'), - ...formatAddressObjectAsParticipants(bcc, 'bcc'), - ]; - - let textWithoutReplyQuotations = text; - - if (text) - try { - textWithoutReplyQuotations = planer.extractFrom( - text, - 'text/plain', - ); - } catch (error) { - console.log( - 'Error while trying to remove reply quotations', - error, - ); - } - - const messageFromGmail: GmailMessage = { - historyId, - externalId: id, - headerMessageId, - subject: subject || '', - messageThreadExternalId: threadId, - internalDate, - fromHandle: from[0].address || '', - fromDisplayName: from[0].name || '', - participants, - text: sanitizeString(textWithoutReplyQuotations || ''), - attachments, - }; - - return messageFromGmail; - } catch (error) { - // if message was not found, silenty ignore it - if (error?.code === 404) { - return null; + const formattedResponse = parsedResponses.map( + (response): GmailMessage | null => { + if ('error' in response) { + if (response.error.code === 404) { + return null; + } + + throw response.error; + } + + try { + const { + historyId, + id, + threadId, + internalDate, + subject, + from, + to, + cc, + bcc, + headerMessageId, + text, + attachments, + } = this.parseGmailMessage(response); + + if (!from) throw new Error('From value is missing'); + + const participants = [ + ...formatAddressObjectAsParticipants(from, 'from'), + ...formatAddressObjectAsParticipants(to, 'to'), + ...formatAddressObjectAsParticipants(cc, 'cc'), + ...formatAddressObjectAsParticipants(bcc, 'bcc'), + ]; + + let textWithoutReplyQuotations = text; + + if (text) + try { + textWithoutReplyQuotations = planer.extractFrom( + text, + 'text/plain', + ); + } catch (error) { + console.log( + 'Error while trying to remove reply quotations', + error, + ); } - throw error; - } - }, - ), + const messageFromGmail: GmailMessage = { + historyId, + externalId: id, + headerMessageId, + subject: subject || '', + messageThreadExternalId: threadId, + internalDate, + fromHandle: from[0].address || '', + fromDisplayName: from[0].name || '', + participants, + text: sanitizeString(textWithoutReplyQuotations || ''), + attachments, + }; + + return messageFromGmail; + } catch (error) { + console.log('Error while trying to parse a message', response, error); + + return null; + } + }, ); const filteredMessages = formattedResponse.filter((message) => @@ -142,19 +143,17 @@ export class FetchMessagesByBatchesService { return filteredMessages; } - async formatBatchResponsesAsGmailMessages( + private formatBatchResponsesAsGmailMessages( batchResponses: AxiosResponse[], - ): Promise { - const messageBatches = await Promise.all( - batchResponses.map(async (response) => { - return this.formatBatchResponseAsGmailMessage(response); - }), - ); + ): GmailMessage[] { + const messageBatches = batchResponses.map((response) => { + return this.formatBatchResponseAsGmailMessage(response); + }); return messageBatches.flat(); } - private parseGmailMessage(message: GmailMessageParsedResponse) { + private parseGmailMessage(message: gmail_v1.Schema$Message) { const subject = this.getPropertyFromHeaders(message, 'Subject'); const rawFrom = this.getPropertyFromHeaders(message, 'From'); const rawTo = this.getPropertyFromHeaders(message, 'To'); @@ -167,6 +166,9 @@ export class FetchMessagesByBatchesService { const internalDate = message.internalDate; assert(id); + assert(messageId); + assert(rawFrom); + assert(rawTo); assert(threadId); assert(historyId); assert(internalDate); @@ -183,14 +185,14 @@ export class FetchMessagesByBatchesService { subject, from: addressparser(rawFrom), to: addressparser(rawTo), - cc: addressparser(rawCc), - bcc: addressparser(rawBcc), + cc: rawCc ? addressparser(rawCc) : undefined, + bcc: rawBcc ? addressparser(rawBcc) : undefined, text, attachments: [], }; } - private getBodyData(message: GmailMessageParsedResponse) { + private getBodyData(message: gmail_v1.Schema$Message) { const firstPart = message.payload?.parts?.[0]; if (firstPart?.mimeType === 'text/plain') { @@ -202,17 +204,13 @@ export class FetchMessagesByBatchesService { } private getPropertyFromHeaders( - message: GmailMessageParsedResponse, + message: gmail_v1.Schema$Message, property: string, ) { - const value = message.payload?.headers?.find( + const header = message.payload?.headers?.find( (header) => header.name === property, - )?.value; - - if (value === undefined || value === null) { - throw new Error(`Cannot find property "${property}" in message headers`); - } + ); - return value; + return header?.value; } } diff --git a/packages/twenty-server/src/modules/messaging/types/gmail-message-parsed-response.ts b/packages/twenty-server/src/modules/messaging/types/gmail-message-parsed-response.ts index 8037a382884e..9938220bb216 100644 --- a/packages/twenty-server/src/modules/messaging/types/gmail-message-parsed-response.ts +++ b/packages/twenty-server/src/modules/messaging/types/gmail-message-parsed-response.ts @@ -1,3 +1,13 @@ import { gmail_v1 } from 'googleapis'; -export type GmailMessageParsedResponse = gmail_v1.Schema$Message; +type GmailMessageError = { + error: { + code: number; + message: string; + status: string; + }; +}; + +export type GmailMessageParsedResponse = + | gmail_v1.Schema$Message + | GmailMessageError; From fb344187f1e36fc9df94a5ff1ba8ad00d5728ba5 Mon Sep 17 00:00:00 2001 From: rostaklein Date: Thu, 9 May 2024 22:43:40 +0200 Subject: [PATCH 8/9] handling all errors as failed status --- ...etch-message-content-from-cache.service.ts | 24 +++++++++---------- 1 file changed, 11 insertions(+), 13 deletions(-) diff --git a/packages/twenty-server/src/modules/messaging/services/gmail-fetch-message-content-from-cache/gmail-fetch-message-content-from-cache.service.ts b/packages/twenty-server/src/modules/messaging/services/gmail-fetch-message-content-from-cache/gmail-fetch-message-content-from-cache.service.ts index 80def19b9f27..2abeb32677e6 100644 --- a/packages/twenty-server/src/modules/messaging/services/gmail-fetch-message-content-from-cache/gmail-fetch-message-content-from-cache.service.ts +++ b/packages/twenty-server/src/modules/messaging/services/gmail-fetch-message-content-from-cache/gmail-fetch-message-content-from-cache.service.ts @@ -276,21 +276,19 @@ export class GmailFetchMessageContentFromCacheService { messageIdsToFetch, ); - if (error?.message?.code === 429) { - this.logger.error( - `Error fetching messages for ${connectedAccountId} in workspace ${workspaceId}: Resource has been exhausted, locking for ${GMAIL_ONGOING_SYNC_TIMEOUT}ms...`, - ); + await this.messageChannelRepository.updateSyncStatus( + gmailMessageChannelId, + MessageChannelSyncStatus.FAILED, + workspaceId, + ); - await this.messageChannelRepository.updateSyncStatus( - gmailMessageChannelId, - MessageChannelSyncStatus.FAILED, - workspaceId, - ); + this.logger.error( + `Error fetching messages for ${connectedAccountId} in workspace ${workspaceId}: locking for ${GMAIL_ONGOING_SYNC_TIMEOUT}ms...`, + ); - throw new Error( - `Error fetching messages for ${connectedAccountId} in workspace ${workspaceId}: ${error.message}`, - ); - } + throw new Error( + `Error fetching messages for ${connectedAccountId} in workspace ${workspaceId}: ${error.message}`, + ); } } From e492f7a07524679d42cc876d3777cb350e7a22ea Mon Sep 17 00:00:00 2001 From: Charles Bochet Date: Mon, 20 May 2024 17:24:41 +0200 Subject: [PATCH 9/9] Fixes --- ...-users-messages-get-batch-size.constant.ts | 2 +- .../fetch-messages-by-batches.service.ts | 145 ++++++++++-------- 2 files changed, 79 insertions(+), 68 deletions(-) diff --git a/packages/twenty-server/src/modules/messaging/constants/gmail-users-messages-get-batch-size.constant.ts b/packages/twenty-server/src/modules/messaging/constants/gmail-users-messages-get-batch-size.constant.ts index ccc1d42df096..b8ca7bb59c4b 100644 --- a/packages/twenty-server/src/modules/messaging/constants/gmail-users-messages-get-batch-size.constant.ts +++ b/packages/twenty-server/src/modules/messaging/constants/gmail-users-messages-get-batch-size.constant.ts @@ -1 +1 @@ -export const GMAIL_USERS_MESSAGES_GET_BATCH_SIZE = 30; +export const GMAIL_USERS_MESSAGES_GET_BATCH_SIZE = 20; diff --git a/packages/twenty-server/src/modules/messaging/services/fetch-messages-by-batches/fetch-messages-by-batches.service.ts b/packages/twenty-server/src/modules/messaging/services/fetch-messages-by-batches/fetch-messages-by-batches.service.ts index a94b74d448bf..24a3d87858b2 100644 --- a/packages/twenty-server/src/modules/messaging/services/fetch-messages-by-batches/fetch-messages-by-batches.service.ts +++ b/packages/twenty-server/src/modules/messaging/services/fetch-messages-by-batches/fetch-messages-by-batches.service.ts @@ -20,8 +20,8 @@ export class FetchMessagesByBatchesService { async fetchAllMessages( queries: MessageQuery[], accessToken: string, - workspaceId?: string, - connectedAccountId?: string, + workspaceId: string, + connectedAccountId: string, ): Promise { let startTime = Date.now(); const batchResponses = await this.fetchByBatchesService.fetchAllByBatches( @@ -39,8 +39,11 @@ export class FetchMessagesByBatchesService { startTime = Date.now(); - const formattedResponse = - this.formatBatchResponsesAsGmailMessages(batchResponses); + const formattedResponse = this.formatBatchResponsesAsGmailMessages( + batchResponses, + workspaceId, + connectedAccountId, + ); endTime = Date.now(); @@ -55,6 +58,8 @@ export class FetchMessagesByBatchesService { private formatBatchResponseAsGmailMessage( responseCollection: AxiosResponse, + workspaceId: string, + connectedAccountId: string, ): GmailMessage[] { const parsedResponses = this.fetchByBatchesService.parseBatch(responseCollection); @@ -73,66 +78,66 @@ export class FetchMessagesByBatchesService { throw response.error; } - try { - const { - historyId, - id, - threadId, - internalDate, - subject, - from, - to, - cc, - bcc, - headerMessageId, - text, - attachments, - } = this.parseGmailMessage(response); - - if (!from) throw new Error('From value is missing'); - - const participants = [ - ...formatAddressObjectAsParticipants(from, 'from'), - ...formatAddressObjectAsParticipants(to, 'to'), - ...formatAddressObjectAsParticipants(cc, 'cc'), - ...formatAddressObjectAsParticipants(bcc, 'bcc'), - ]; - - let textWithoutReplyQuotations = text; - - if (text) - try { - textWithoutReplyQuotations = planer.extractFrom( - text, - 'text/plain', - ); - } catch (error) { - console.log( - 'Error while trying to remove reply quotations', - error, - ); - } - - const messageFromGmail: GmailMessage = { - historyId, - externalId: id, - headerMessageId, - subject: subject || '', - messageThreadExternalId: threadId, - internalDate, - fromHandle: from[0].address || '', - fromDisplayName: from[0].name || '', - participants, - text: sanitizeString(textWithoutReplyQuotations || ''), - attachments, - }; - - return messageFromGmail; - } catch (error) { - console.log('Error while trying to parse a message', response, error); + const { + historyId, + id, + threadId, + internalDate, + subject, + from, + to, + cc, + bcc, + headerMessageId, + text, + attachments, + deliveredTo, + } = this.parseGmailMessage(response); + + if (!from) { + this.logger.log( + `From value is missing while importing message in workspace ${workspaceId} and account ${connectedAccountId}`, + ); + + return null; + } + + if (!to && !deliveredTo && !bcc && !cc) { + this.logger.log( + `To, Delivered-To, Bcc or Cc value is missing while importing message in workspace ${workspaceId} and account ${connectedAccountId}`, + ); return null; } + + const participants = [ + ...formatAddressObjectAsParticipants(from, 'from'), + ...formatAddressObjectAsParticipants(to ?? deliveredTo, 'to'), + ...formatAddressObjectAsParticipants(cc, 'cc'), + ...formatAddressObjectAsParticipants(bcc, 'bcc'), + ]; + + let textWithoutReplyQuotations = text; + + if (text) { + textWithoutReplyQuotations = planer.extractFrom(text, 'text/plain'); + } + + const messageFromGmail: GmailMessage = { + historyId, + externalId: id, + headerMessageId, + subject: subject || '', + messageThreadExternalId: threadId, + internalDate, + fromHandle: from[0].address || '', + fromDisplayName: from[0].name || '', + participants, + text: sanitizeString(textWithoutReplyQuotations || ''), + attachments, + }; + + return messageFromGmail; }, ); @@ -145,9 +150,15 @@ export class FetchMessagesByBatchesService { private formatBatchResponsesAsGmailMessages( batchResponses: AxiosResponse[], + workspaceId: string, + connectedAccountId: string, ): GmailMessage[] { const messageBatches = batchResponses.map((response) => { - return this.formatBatchResponseAsGmailMessage(response); + return this.formatBatchResponseAsGmailMessage( + response, + workspaceId, + connectedAccountId, + ); }); return messageBatches.flat(); @@ -157,6 +168,7 @@ export class FetchMessagesByBatchesService { const subject = this.getPropertyFromHeaders(message, 'Subject'); const rawFrom = this.getPropertyFromHeaders(message, 'From'); const rawTo = this.getPropertyFromHeaders(message, 'To'); + const rawDeliveredTo = this.getPropertyFromHeaders(message, 'Delivered-To'); const rawCc = this.getPropertyFromHeaders(message, 'Cc'); const rawBcc = this.getPropertyFromHeaders(message, 'Bcc'); const messageId = this.getPropertyFromHeaders(message, 'Message-ID'); @@ -167,8 +179,6 @@ export class FetchMessagesByBatchesService { assert(id); assert(messageId); - assert(rawFrom); - assert(rawTo); assert(threadId); assert(historyId); assert(internalDate); @@ -183,8 +193,9 @@ export class FetchMessagesByBatchesService { historyId, internalDate, subject, - from: addressparser(rawFrom), - to: addressparser(rawTo), + from: rawFrom ? addressparser(rawFrom) : undefined, + deliveredTo: rawDeliveredTo ? addressparser(rawDeliveredTo) : undefined, + to: rawTo ? addressparser(rawTo) : undefined, cc: rawCc ? addressparser(rawCc) : undefined, bcc: rawBcc ? addressparser(rawBcc) : undefined, text, @@ -208,7 +219,7 @@ export class FetchMessagesByBatchesService { property: string, ) { const header = message.payload?.headers?.find( - (header) => header.name === property, + (header) => header.name?.toLowerCase() === property.toLowerCase(), ); return header?.value;