From 215a0e99fd432f75c56ad7eb210faf5c07bd9b25 Mon Sep 17 00:00:00 2001 From: Roman Shevchenko Date: Mon, 4 Apr 2022 09:44:29 +0000 Subject: [PATCH 01/45] wip --- .../compose-modules/compose-draft-module.ts | 2 +- .../compose-send-btn-module.ts | 31 +++++++------ .../encrypted-mail-msg-formatter.ts | 43 ++++++++++++++----- .../formatters/general-mail-formatter.ts | 8 ++-- extension/js/common/api/account-server.ts | 8 +++- .../api/account-servers/enterprise-server.ts | 6 ++- test/source/mock/fes/fes-endpoints.ts | 36 +++++++++++++++- .../strategies/send-message-strategy.ts | 18 ++++++-- 8 files changed, 119 insertions(+), 33 deletions(-) diff --git a/extension/chrome/elements/compose-modules/compose-draft-module.ts b/extension/chrome/elements/compose-modules/compose-draft-module.ts index 5cf2b6023cd..130ef4c3b4f 100644 --- a/extension/chrome/elements/compose-modules/compose-draft-module.ts +++ b/extension/chrome/elements/compose-modules/compose-draft-module.ts @@ -120,7 +120,7 @@ export class ComposeDraftModule extends ViewModule { const primaryKi = await this.view.storageModule.getKey(msgData.from); const pubkeys = [{ isMine: true, email: msgData.from, pubkey: await KeyUtil.parse(primaryKi.public) }]; msgData.pwd = undefined; // not needed for drafts - const sendable = await new EncryptedMsgMailFormatter(this.view, true).sendableMsg(msgData, pubkeys); + const [sendable] = await new EncryptedMsgMailFormatter(this.view, true).sendableMsgs(msgData, pubkeys); if (this.view.replyParams?.inReplyTo) { sendable.headers.References = this.view.replyParams.inReplyTo; sendable.headers['In-Reply-To'] = this.view.replyParams.inReplyTo; diff --git a/extension/chrome/elements/compose-modules/compose-send-btn-module.ts b/extension/chrome/elements/compose-modules/compose-send-btn-module.ts index 1c418ed1bb9..5fe36dd9863 100644 --- a/extension/chrome/elements/compose-modules/compose-send-btn-module.ts +++ b/extension/chrome/elements/compose-modules/compose-send-btn-module.ts @@ -116,8 +116,10 @@ export class ComposeSendBtnModule extends ViewModule { await ContactStore.update(undefined, emails, { lastUse: Date.now() }); const msgObj = await GeneralMailFormatter.processNewMsg(this.view, newMsgData); if (msgObj) { - await this.finalizeSendableMsg(msgObj); - await this.doSendMsg(msgObj.msg); + for (const msg of msgObj.msgs) { + await this.finalizeSendableMsg({ msg, senderKi: msgObj.senderKi }); + } + await this.doSendMsgs(msgObj.msgs); } } catch (e) { await this.view.errModule.handleSendErr(e); @@ -197,21 +199,26 @@ export class ComposeSendBtnModule extends ViewModule { }; - private doSendMsg = async (msg: SendableMsg) => { + private doSendMsgs = async (msgs: SendableMsg[]) => { // if this is a password-encrypted message, then we've already shown progress for uploading to backend // and this requests represents second half of uploadable effort. Else this represents all (no previous heavy requests) const progressRepresents = this.view.pwdOrPubkeyContainerModule.isVisible() ? 'SECOND-HALF' : 'EVERYTHING'; let msgSentRes: GmailRes.GmailMsgSend; - try { - this.isSendMessageInProgress = true; - msgSentRes = await this.view.emailProvider.msgSend(msg, (p) => this.renderUploadProgress(p, progressRepresents)); - } catch (e) { - if (msg.thread && ApiErr.isNotFound(e) && this.view.threadId) { // cannot send msg because threadId not found - eg user since deleted it - msg.thread = undefined; + for (const msg of msgs) { + try { + this.isSendMessageInProgress = true; msgSentRes = await this.view.emailProvider.msgSend(msg, (p) => this.renderUploadProgress(p, progressRepresents)); - } else { - this.isSendMessageInProgress = false; - throw e; + } catch (e) { + if (msg.thread && ApiErr.isNotFound(e) && this.view.threadId) { // cannot send msg because threadId not found - eg user since deleted it + msg.thread = undefined; + msgSentRes = await this.view.emailProvider.msgSend(msg, (p) => this.renderUploadProgress(p, progressRepresents)); + } else { + this.isSendMessageInProgress = false; + throw e; + } + } + if (msg.externalId) { + this.view.acctServer.messageGatewayUpdate(msg.externalId, msgSentRes.id).catch(Catch.reportErr); } } BrowserMsg.send.notificationShow(this.view.parentTabId, { notification: `Your ${this.view.isReplyBox ? 'reply' : 'message'} has been sent.` }); diff --git a/extension/chrome/elements/compose-modules/formatters/encrypted-mail-msg-formatter.ts b/extension/chrome/elements/compose-modules/formatters/encrypted-mail-msg-formatter.ts index cfb60249de6..c860f48d235 100644 --- a/extension/chrome/elements/compose-modules/formatters/encrypted-mail-msg-formatter.ts +++ b/extension/chrome/elements/compose-modules/formatters/encrypted-mail-msg-formatter.ts @@ -22,28 +22,41 @@ import { AcctStore } from '../../../../js/common/platform/store/acct-store.js'; import { FcUuidAuth } from '../../../../js/common/api/account-servers/flowcrypt-com-api.js'; import { SmimeKey } from '../../../../js/common/core/crypto/smime/smime-key.js'; import { PgpHash } from '../../../../js/common/core/crypto/pgp/pgp-hash.js'; +import { UploadedMessageData } from '../../../../js/common/api/account-server.js'; export class EncryptedMsgMailFormatter extends BaseMailFormatter { - public sendableMsg = async (newMsg: NewMsgData, pubkeys: PubkeyResult[], signingPrv?: Key): Promise => { + public sendableMsgs = async (newMsg: NewMsgData, pubkeys: PubkeyResult[], signingPrv?: Key): Promise => { if (newMsg.pwd && !this.isDraft) { // password-protected message, temporarily uploaded (already encrypted) to: // - flowcrypt.com/api (consumers and customers without on-prem setup), or // - FlowCrypt Enterprise Server (enterprise customers with on-prem setup) // It will be served to recipient through web - const { url: msgUrl, externalId } = await this.prepareAndUploadPwdEncryptedMsg(newMsg); // encrypted for pwd only, pubkeys ignored + const uploadedMessageData = await this.prepareAndUploadPwdEncryptedMsg(newMsg); // encrypted for pwd only, pubkeys ignored newMsg.pwd = undefined; - return await this.sendablePwdMsg(newMsg, pubkeys, { msgUrl, externalId }, signingPrv); // encrypted for pubkeys only, pwd ignored + const entries = Object.entries(uploadedMessageData.emailToExternalIdAndUrl ?? {}); + if (entries.length) { + const msgs: SendableMsg[] = []; + for (const entry of entries) { + const recipientEmail = entry[0]; + const msgUrl = entry[1].url; + const externalId = entry[1].externalId; + msgs.push(await this.sendablePwdMsg(newMsg, pubkeys, { recipientEmail, msgUrl, externalId }, signingPrv)); + } + return msgs; + } else { + return [await this.sendablePwdMsg(newMsg, pubkeys, { msgUrl: uploadedMessageData.url, externalId: uploadedMessageData.externalId }, signingPrv)]; + } } else if (this.richtext) { // rich text: PGP/MIME - https://tools.ietf.org/html/rfc3156#section-4 // or S/MIME - return await this.sendableRichTextMsg(newMsg, pubkeys, signingPrv); + return [await this.sendableRichTextMsg(newMsg, pubkeys, signingPrv)]; } else { // simple text: PGP or S/MIME Inline with attachments in separate files // todo: #4046 check attachments for S/MIME - return await this.sendableSimpleTextMsg(newMsg, pubkeys, signingPrv); + return [await this.sendableSimpleTextMsg(newMsg, pubkeys, signingPrv)]; } }; - private prepareAndUploadPwdEncryptedMsg = async (newMsg: NewMsgData): Promise<{ url: string; externalId?: string }> => { + private prepareAndUploadPwdEncryptedMsg = async (newMsg: NewMsgData): Promise => { // PGP/MIME + included attachments (encrypted for password only) if (!newMsg.pwd) { throw new Error('password unexpectedly missing'); @@ -77,7 +90,7 @@ export class EncryptedMsgMailFormatter extends BaseMailFormatter { const { bodyWithReplyToken, replyToken } = await this.getPwdMsgSendableBodyWithOnlineReplyMsgToken(authInfo, newMsg); const pgpMimeWithAttachments = await Mime.encode(bodyWithReplyToken, { Subject: newMsg.subject }, await this.view.attachmentsModule.attachment.collectAttachments()); const { data: pwdEncryptedWithAttachments } = await this.encryptDataArmor(Buf.fromUtfStr(pgpMimeWithAttachments), newMsg.pwd, []); // encrypted only for pwd, not signed - const { url, externalId } = await this.view.acctServer.messageUpload( + return await this.view.acctServer.messageUpload( authInfo.uuid ? authInfo : undefined, pwdEncryptedWithAttachments, replyToken, @@ -85,10 +98,13 @@ export class EncryptedMsgMailFormatter extends BaseMailFormatter { newMsg.recipients, (p) => this.view.sendBtnModule.renderUploadProgress(p, 'FIRST-HALF'), // still need to upload to Gmail later, this request represents first half of progress ); - return { url, externalId }; }; - private sendablePwdMsg = async (newMsg: NewMsgData, pubs: PubkeyResult[], { msgUrl, externalId }: { msgUrl: string, externalId?: string }, signingPrv?: Key) => { + private sendablePwdMsg = async ( + newMsg: NewMsgData, + pubs: PubkeyResult[], + { msgUrl, externalId, recipientEmail }: { msgUrl: string, externalId?: string, recipientEmail?: string }, + signingPrv?: Key) => { // encoded as: PGP/MIME-like structure but with attachments as external files due to email size limit (encrypted for pubkeys only) const msgBody = this.richtext ? { 'text/plain': newMsg.plaintext, 'text/html': newMsg.plainhtml } : { 'text/plain': newMsg.plaintext }; const pgpMimeNoAttachments = await Mime.encode(msgBody, { Subject: newMsg.subject }, []); // no attachments, attached to email separately @@ -96,7 +112,14 @@ export class EncryptedMsgMailFormatter extends BaseMailFormatter { const attachments = this.createPgpMimeAttachments(pubEncryptedNoAttachments). concat(await this.view.attachmentsModule.attachment.collectEncryptAttachments(pubs)); // encrypted only for pubs const emailIntroAndLinkBody = await this.formatPwdEncryptedMsgBodyLink(msgUrl); - return await SendableMsg.createPwdMsg(this.acctEmail, this.headers(newMsg), emailIntroAndLinkBody, attachments, { isDraft: this.isDraft, externalId }); + const headers = this.headers(newMsg); + if (recipientEmail) { + // we will send this copy of the message only to this recipient + const foundParsedRecipient = (headers.recipients.to ?? []).concat(headers.recipients.cc ?? []).concat(headers.recipients.bcc ?? []). + find(r => r.email.toLowerCase() === recipientEmail.toLowerCase()); + headers.recipients = { to: [foundParsedRecipient ?? { email: recipientEmail }] }; + } + return await SendableMsg.createPwdMsg(this.acctEmail, headers, emailIntroAndLinkBody, attachments, { isDraft: this.isDraft, externalId }); }; private sendableSimpleTextMsg = async (newMsg: NewMsgData, pubs: PubkeyResult[], signingPrv?: Key): Promise => { diff --git a/extension/chrome/elements/compose-modules/formatters/general-mail-formatter.ts b/extension/chrome/elements/compose-modules/formatters/general-mail-formatter.ts index 14fb1eeee2e..ab3078e03bc 100644 --- a/extension/chrome/elements/compose-modules/formatters/general-mail-formatter.ts +++ b/extension/chrome/elements/compose-modules/formatters/general-mail-formatter.ts @@ -13,11 +13,11 @@ import { ComposeView } from '../../compose.js'; export class GeneralMailFormatter { // returns undefined in case user cancelled decryption of the signing key - public static processNewMsg = async (view: ComposeView, newMsgData: NewMsgData): Promise<{ msg: SendableMsg, senderKi: KeyInfo | undefined } | undefined> => { + public static processNewMsg = async (view: ComposeView, newMsgData: NewMsgData): Promise<{ msgs: SendableMsg[], senderKi: KeyInfo | undefined } | undefined> => { const choices = view.sendBtnModule.popover.choices; const recipientsEmails = getUniqueRecipientEmails(newMsgData.recipients); if (!choices.encrypt && !choices.sign) { // plain - return { senderKi: undefined, msg: await new PlainMsgMailFormatter(view).sendableMsg(newMsgData) }; + return { senderKi: undefined, msgs: [await new PlainMsgMailFormatter(view).sendableMsg(newMsgData)] }; } let signingPrv: Key | undefined; if (!choices.encrypt && choices.sign) { // sign only @@ -27,7 +27,7 @@ export class GeneralMailFormatter { if (!signingPrv) { return undefined; } - return { senderKi, msg: await new SignedMsgMailFormatter(view).sendableMsg(newMsgData, signingPrv) }; + return { senderKi, msgs: [await new SignedMsgMailFormatter(view).sendableMsg(newMsgData, signingPrv)] }; } // encrypt (optionally sign) const result = await view.storageModule.collectSingleFamilyKeys(recipientsEmails, newMsgData.from, choices.sign); @@ -41,7 +41,7 @@ export class GeneralMailFormatter { await view.errModule.throwIfEncryptionPasswordInvalid(newMsgData); } view.S.now('send_btn_text').text('Encrypting...'); - return { senderKi: result.senderKi, msg: await new EncryptedMsgMailFormatter(view).sendableMsg(newMsgData, result.pubkeys, signingPrv) }; + return { senderKi: result.senderKi, msgs: await new EncryptedMsgMailFormatter(view).sendableMsgs(newMsgData, result.pubkeys, signingPrv) }; }; } diff --git a/extension/js/common/api/account-server.ts b/extension/js/common/api/account-server.ts index 84bc01e0a4e..389fc543ec4 100644 --- a/extension/js/common/api/account-server.ts +++ b/extension/js/common/api/account-server.ts @@ -8,6 +8,12 @@ import { BackendRes, FcUuidAuth, FlowCryptComApi, ProfileUpdate } from './accoun import { ParsedRecipients } from './email-provider/email-provider-api.js'; import { Api, ProgressCb } from './shared/api.js'; +export type UploadedMessageData = { + url: string, // both FES and FlowCryptComApi + externalId?: string, // legacy FES + emailToExternalIdAndUrl?: { [email: string]: { url: string, externalId: string } } // FES only +}; + /** * This may be calling to FlowCryptComApi or Enterprise Server (FES, customer on-prem) depending on * whether FES is deployed on the customer domain or not. @@ -58,7 +64,7 @@ export class AccountServer extends Api { from: string, recipients: ParsedRecipients, progressCb: ProgressCb - ): Promise<{ url: string, externalId?: string }> => { + ): Promise => { if (await this.isFesUsed()) { const fes = new EnterpriseServer(this.acctEmail); // Recipients are used to later cross-check replies from the web diff --git a/extension/js/common/api/account-servers/enterprise-server.ts b/extension/js/common/api/account-servers/enterprise-server.ts index 5bc83a6a16c..cc640af103b 100644 --- a/extension/js/common/api/account-servers/enterprise-server.ts +++ b/extension/js/common/api/account-servers/enterprise-server.ts @@ -23,7 +23,11 @@ type EventTag = 'compose' | 'decrypt' | 'setup' | 'settings' | 'import-pub' | 'i export namespace FesRes { export type ReplyToken = { replyToken: string }; - export type MessageUpload = { url: string; externalId: string }; + export type MessageUpload = { + url: string, // LEGACY + externalId: string, // LEGACY + emailToExternalIdAndUrl?: { [email: string]: { url: string, externalId: string } } + }; export type ServiceInfo = { vendor: string, service: string, orgId: string, version: string, apiVersion: string }; export type ClientConfiguration = { clientConfiguration: DomainRulesJson }; } diff --git a/test/source/mock/fes/fes-endpoints.ts b/test/source/mock/fes/fes-endpoints.ts index 459836638cf..6534c69baf4 100644 --- a/test/source/mock/fes/fes-endpoints.ts +++ b/test/source/mock/fes/fes-endpoints.ts @@ -63,11 +63,45 @@ export const mockFesEndpoints: HandlersDefinition = { expect(body).to.contain('"cc":[]'); expect(body).to.contain('"bcc":["Mr Bcc "]'); expect(body).to.contain('"from":"user@standardsubdomainfes.test:8001"'); - return { 'url': `http://${standardFesUrl}/message/FES-MOCK-MESSAGE-ID`, 'externalId': 'FES-MOCK-EXTERNAL-ID' }; + const response = + { + // todo: do we need to support and test legacy? + 'url': `http://${standardFesUrl}/message/FES-MOCK-MESSAGE-ID`, + 'externalId': 'FES-MOCK-EXTERNAL-ID', + emailToExternalIdAndUrl: {} as { [email: string]: { url: string, externalId: string } } + }; + response.emailToExternalIdAndUrl['to@example.com'] = { + url: `http://${standardFesUrl}/message/FES-MOCK-MESSAGE-FOR-TO@EXAMPLE.COM-ID`, + externalId: 'FES-MOCK-EXTERNAL-FOR-TO@EXAMPLE.COM-ID' + }; + response.emailToExternalIdAndUrl['bcc@example.com'] = { + url: `http://${standardFesUrl}/message/FES-MOCK-MESSAGE-FOR-BCC@EXAMPLE.COM-ID`, + externalId: 'FES-MOCK-EXTERNAL-FOR-BCC@EXAMPLE.COM-ID' + }; + return response; } throw new HttpClientErr('Not Found', 404); }, '/api/v1/message/FES-MOCK-EXTERNAL-ID/gateway': async ({ body }, req) => { + if (req.headers.host === standardFesUrl && req.method === 'POST') { + // todo: remove legacy endpoint? + // test: `compose - user@standardsubdomainfes.test:8001 - PWD encrypted message with FES web portal` + authenticate(req, 'oidc'); + expect(body).to.match(/{"emailGatewayMessageId":"<(.+)@standardsubdomainfes.test:8001>"}/); + return {}; + } + throw new HttpClientErr('Not Found', 404); + }, + '/api/v1/message/FES-MOCK-EXTERNAL-FOR-TO@EXAMPLE.COM-ID/gateway': async ({ body }, req) => { + if (req.headers.host === standardFesUrl && req.method === 'POST') { + // test: `compose - user@standardsubdomainfes.test:8001 - PWD encrypted message with FES web portal` + authenticate(req, 'oidc'); + expect(body).to.match(/{"emailGatewayMessageId":"<(.+)@standardsubdomainfes.test:8001>"}/); + return {}; + } + throw new HttpClientErr('Not Found', 404); + }, + '/api/v1/message/FES-MOCK-EXTERNAL-FOR-BCC@EXAMPLE.COM-ID/gateway': async ({ body }, req) => { if (req.headers.host === standardFesUrl && req.method === 'POST') { // test: `compose - user@standardsubdomainfes.test:8001 - PWD encrypted message with FES web portal` authenticate(req, 'oidc'); diff --git a/test/source/mock/google/strategies/send-message-strategy.ts b/test/source/mock/google/strategies/send-message-strategy.ts index 3786c14a220..4c62ee6fe68 100644 --- a/test/source/mock/google/strategies/send-message-strategy.ts +++ b/test/source/mock/google/strategies/send-message-strategy.ts @@ -42,12 +42,24 @@ class PwdEncryptedMessageWithFesIdTokenTestStrategy implements ITestMsgStrategy public test = async (mimeMsg: ParsedMail, base64Msg: string, id: string) => { const expectedSenderEmail = 'user@standardsubdomainfes.test:8001'; expect(mimeMsg.from!.text).to.equal(`First Last <${expectedSenderEmail}>`); - expect((mimeMsg.to as AddressObject).text).to.equal('Mr To '); - expect((mimeMsg.bcc as AddressObject).text).to.equal('Mr Bcc '); if (!mimeMsg.text?.includes(`${expectedSenderEmail} has sent you a password-encrypted email`)) { throw new HttpClientErr(`Error checking sent text in:\n\n${mimeMsg.text}`); } - if (!mimeMsg.text?.includes('http://fes.standardsubdomainfes.test:8001/message/FES-MOCK-MESSAGE-ID')) { + if (mimeMsg.text?.includes('http://fes.standardsubdomainfes.test:8001/message/FES-MOCK-MESSAGE-ID')) { + // legacy + // todo: remove this test? + expect((mimeMsg.to as AddressObject).text).to.equal('Mr To '); + expect(mimeMsg.cc).to.be.an.undefined; + expect((mimeMsg.bcc as AddressObject).text).to.equal('Mr Bcc '); + } else if (mimeMsg.text?.includes('http://fes.standardsubdomainfes.test:8001/message/FES-MOCK-MESSAGE-FOR-TO@EXAMPLE.COM-ID')) { + expect((mimeMsg.to as AddressObject).text).to.equal('Mr To '); + expect(mimeMsg.cc).to.be.an.undefined; + expect(mimeMsg.bcc).to.be.an.undefined; + } else if (mimeMsg.text?.includes('http://fes.standardsubdomainfes.test:8001/message/FES-MOCK-MESSAGE-FOR-BCC@EXAMPLE.COM-ID')) { + expect((mimeMsg.to as AddressObject).text).to.equal('Mr Bcc '); + expect(mimeMsg.cc).to.be.an.undefined; + expect(mimeMsg.bcc).to.be.an.undefined; + } else { throw new HttpClientErr(`Error: cannot find pwd encrypted FES link in:\n\n${mimeMsg.text}`); } if (!mimeMsg.text?.includes('Follow this link to open it')) { From 6e1a9ec685761f78a51098215215133ce159877f Mon Sep 17 00:00:00 2001 From: Roman Shevchenko Date: Thu, 14 Apr 2022 09:37:42 +0000 Subject: [PATCH 02/45] wip --- .../compose-modules/compose-render-module.ts | 17 ++++++++--------- .../compose-modules/compose-send-btn-module.ts | 9 +++++++-- .../formatters/general-mail-formatter.ts | 2 +- extension/chrome/elements/compose.htm | 6 +++--- extension/js/common/core/common.ts | 4 ++++ test/source/tests/compose.ts | 1 + 6 files changed, 24 insertions(+), 15 deletions(-) diff --git a/extension/chrome/elements/compose-modules/compose-render-module.ts b/extension/chrome/elements/compose-modules/compose-render-module.ts index e2683e1b49c..d35ef0718e2 100644 --- a/extension/chrome/elements/compose-modules/compose-render-module.ts +++ b/extension/chrome/elements/compose-modules/compose-render-module.ts @@ -8,8 +8,7 @@ import { BrowserMsg } from '../../../js/common/browser/browser-msg.js'; import { Catch } from '../../../js/common/platform/catch.js'; import { KeyImportUi } from '../../../js/common/ui/key-import-ui.js'; import { Lang } from '../../../js/common/lang.js'; -import { Recipients } from '../../../js/common/api/email-provider/email-provider-api.js'; -import { SendableMsg } from '../../../js/common/api/email-provider/sendable-msg.js'; +import { ParsedRecipients, Recipients } from '../../../js/common/api/email-provider/email-provider-api.js'; import { Str } from '../../../js/common/core/common.js'; import { Ui } from '../../../js/common/browser/ui.js'; import { Xss } from '../../../js/common/platform/xss.js'; @@ -114,7 +113,7 @@ export class ComposeRenderModule extends ViewModule { } }; - public renderReplySuccess = (msg: SendableMsg, msgId: string) => { + public renderReplySuccess = (attachments: Attachment[], recipients: ParsedRecipients, msgId: string) => { this.view.renderModule.renderReinsertReplyBox(msgId); if (!this.view.sendBtnModule.popover.choices.encrypt) { this.view.S.cached('replied_body').removeClass('pgp_secure'); @@ -125,13 +124,13 @@ export class ComposeRenderModule extends ViewModule { this.view.S.cached('replied_body').css('width', ($('table#compose').width() || 500) - 30); this.view.S.cached('compose_table').css('display', 'none'); this.view.S.cached('reply_msg_successful').find('div.replied_from').text(this.view.senderModule.getSender()); - this.view.S.cached('reply_msg_successful').find('div.replied_to span').text(msg.headers.To.replace(/,/g, ', ')); - if (msg.recipients.cc !== undefined && msg.recipients.cc.length > 0) { - this.view.S.cached('reply_msg_successful').find('div.replied_cc span').text(msg.recipients.cc.join(', ')); + this.view.S.cached('reply_msg_successful').find('div.replied_to span').text(Str.formatEmailList(recipients.to || [])); + if (recipients.cc !== undefined && recipients.cc.length > 0) { + this.view.S.cached('reply_msg_successful').find('div.replied_cc span').text(Str.formatEmailList(recipients.cc)); $('.replied_cc').show(); } - if (msg.recipients.bcc !== undefined && msg.recipients.bcc.length > 0) { - this.view.S.cached('reply_msg_successful').find('div.replied_bcc span').text(msg.recipients.bcc.join(', ')); + if (recipients.bcc !== undefined && recipients.bcc.length > 0) { + this.view.S.cached('reply_msg_successful').find('div.replied_bcc span').text(Str.formatEmailList(recipients.bcc)); $('.replied_bcc').show(); } const repliedBodyEl = this.view.S.cached('reply_msg_successful').find('div.replied_body'); @@ -141,7 +140,7 @@ export class ComposeRenderModule extends ViewModule { this.renderReplySuccessMimeAttachments(this.view.inputModule.extractAttachments()); } else { Xss.sanitizeRender(repliedBodyEl, Str.escapeTextAsRenderableHtml(this.view.inputModule.extract('text', 'input_text', 'SKIP-ADDONS'))); - this.renderReplySuccessAttachments(msg.attachments, msgId, this.view.sendBtnModule.popover.choices.encrypt); + this.renderReplySuccessAttachments(attachments, msgId, this.view.sendBtnModule.popover.choices.encrypt); } const t = new Date(); const time = ((t.getHours() !== 12) ? (t.getHours() % 12) : 12) + ':' + (t.getMinutes() < 10 ? '0' : '') + t.getMinutes() + ((t.getHours() >= 12) ? ' PM ' : ' AM ') + '(0 minutes ago)'; diff --git a/extension/chrome/elements/compose-modules/compose-send-btn-module.ts b/extension/chrome/elements/compose-modules/compose-send-btn-module.ts index 6e2351c5bad..ca6ecfe4302 100644 --- a/extension/chrome/elements/compose-modules/compose-send-btn-module.ts +++ b/extension/chrome/elements/compose-modules/compose-send-btn-module.ts @@ -201,9 +201,10 @@ export class ComposeSendBtnModule extends ViewModule { // if this is a password-encrypted message, then we've already shown progress for uploading to backend // and this requests represents second half of uploadable effort. Else this represents all (no previous heavy requests) const progressRepresents = this.view.pwdOrPubkeyContainerModule.isVisible() ? 'SECOND-HALF' : 'EVERYTHING'; - let msgSentRes: GmailRes.GmailMsgSend; + const sentIds: string[] = []; const operations = [this.view.draftModule.draftDelete()]; for (const msg of msgs) { + let msgSentRes: GmailRes.GmailMsgSend; try { this.isSendMessageInProgress = true; msgSentRes = await this.view.emailProvider.msgSend(msg, (p) => this.renderUploadProgress(p, progressRepresents)); @@ -216,6 +217,7 @@ export class ComposeSendBtnModule extends ViewModule { throw e; } } + sentIds.push(msgSentRes.id); if (msg.externalId) { operations.push((async (externalId, id) => { const gmailMsg = await this.view.emailProvider.msgGet(id, 'metadata'); @@ -233,7 +235,10 @@ export class ComposeSendBtnModule extends ViewModule { await Promise.all(operations); this.isSendMessageInProgress = false; if (this.view.isReplyBox) { - this.view.renderModule.renderReplySuccess(msg, msgSentRes.id); + // todo: collect recipients correctly + // todo: collect attachments correctly + const attachments = msgs.map(m => m.attachments).reduce((a, b) => a.concat(b), []); + this.view.renderModule.renderReplySuccess(attachments, msgs[0].recipients, sentIds[0]); // todo: } else { this.view.renderModule.closeMsg(); } diff --git a/extension/chrome/elements/compose-modules/formatters/general-mail-formatter.ts b/extension/chrome/elements/compose-modules/formatters/general-mail-formatter.ts index c18a816fde8..3f8fb398a0f 100644 --- a/extension/chrome/elements/compose-modules/formatters/general-mail-formatter.ts +++ b/extension/chrome/elements/compose-modules/formatters/general-mail-formatter.ts @@ -47,7 +47,7 @@ export class GeneralMailFormatter { } view.S.now('send_btn_text').text('Encrypting...'); return { senderKi: signingKey?.keyInfo, msgs: await new EncryptedMsgMailFormatter(view).sendableMsgs(newMsgData, singleFamilyKeys.pubkeys, signingKey?.key) }; - } + }; private static chooseSigningKeyAndDecryptIt = async ( view: ComposeView, diff --git a/extension/chrome/elements/compose.htm b/extension/chrome/elements/compose.htm index c6c043ece48..853c2224281 100644 --- a/extension/chrome/elements/compose.htm +++ b/extension/chrome/elements/compose.htm @@ -166,9 +166,9 @@

New Secure Message

-
to:
-
cc:
-
bcc:
+
to:
+
cc:
+
bcc:
diff --git a/extension/js/common/core/common.ts b/extension/js/common/core/common.ts index 0884a817a5d..fd3f93496f7 100644 --- a/extension/js/common/core/common.ts +++ b/extension/js/common/core/common.ts @@ -52,6 +52,10 @@ export class Str { return name ? `${Str.rmSpecialCharsKeepUtf(name, 'ALLOW-SOME')} <${email}>` : email; }; + public static formatEmailList = (list: EmailParts[]): string => { + return list.map(Str.formatEmailWithOptionalName).join(', '); + }; + public static prettyPrint = (obj: any) => { return (typeof obj === 'object') ? JSON.stringify(obj, undefined, 2).replace(/ /g, ' ').replace(/\n/g, '
') : String(obj); }; diff --git a/test/source/tests/compose.ts b/test/source/tests/compose.ts index 312494b105e..4bf697b3d89 100644 --- a/test/source/tests/compose.ts +++ b/test/source/tests/compose.ts @@ -965,6 +965,7 @@ export const defineComposeTests = (testVariant: TestVariant, testWithBrowser: Te await ComposePageRecipe.fillMsg(composePage, {}, undefined, bodyWithLeadingTabs); await composePage.click('@action-send'); await composePage.waitForContent('@container-reply-msg-successful', bodyWithLeadingTabs); + await composePage.waitForContent('@replied-to', 'to: First Last '); })); ava.default('compose - RTL subject', testWithBrowser('compatibility', async (t, browser) => { From 6e1c4173bee3ab66343824f09f2f4c34f8862efc Mon Sep 17 00:00:00 2001 From: Roman Shevchenko Date: Thu, 14 Apr 2022 10:39:12 +0000 Subject: [PATCH 03/45] eslint fix --- .../google/strategies/send-message-strategy.ts | 15 ++++++++++----- 1 file changed, 10 insertions(+), 5 deletions(-) diff --git a/test/source/mock/google/strategies/send-message-strategy.ts b/test/source/mock/google/strategies/send-message-strategy.ts index 4c62ee6fe68..282866c9fba 100644 --- a/test/source/mock/google/strategies/send-message-strategy.ts +++ b/test/source/mock/google/strategies/send-message-strategy.ts @@ -49,16 +49,21 @@ class PwdEncryptedMessageWithFesIdTokenTestStrategy implements ITestMsgStrategy // legacy // todo: remove this test? expect((mimeMsg.to as AddressObject).text).to.equal('Mr To '); - expect(mimeMsg.cc).to.be.an.undefined; + // tslint:disable-next-line:no-unused-expression + expect(mimeMsg.cc).to.be.an.undefined; // eslint-disable-line no-unused-expressions expect((mimeMsg.bcc as AddressObject).text).to.equal('Mr Bcc '); } else if (mimeMsg.text?.includes('http://fes.standardsubdomainfes.test:8001/message/FES-MOCK-MESSAGE-FOR-TO@EXAMPLE.COM-ID')) { expect((mimeMsg.to as AddressObject).text).to.equal('Mr To '); - expect(mimeMsg.cc).to.be.an.undefined; - expect(mimeMsg.bcc).to.be.an.undefined; + // tslint:disable-next-line:no-unused-expression + expect(mimeMsg.cc).to.be.an.undefined; // eslint-disable-line no-unused-expressions + // tslint:disable-next-line:no-unused-expression + expect(mimeMsg.bcc).to.be.an.undefined; // eslint-disable-line no-unused-expressions } else if (mimeMsg.text?.includes('http://fes.standardsubdomainfes.test:8001/message/FES-MOCK-MESSAGE-FOR-BCC@EXAMPLE.COM-ID')) { expect((mimeMsg.to as AddressObject).text).to.equal('Mr Bcc '); - expect(mimeMsg.cc).to.be.an.undefined; - expect(mimeMsg.bcc).to.be.an.undefined; + // tslint:disable-next-line:no-unused-expression + expect(mimeMsg.cc).to.be.an.undefined; // eslint-disable-line no-unused-expressions + // tslint:disable-next-line:no-unused-expression + expect(mimeMsg.bcc).to.be.an.undefined; // eslint-disable-line no-unused-expressions } else { throw new HttpClientErr(`Error: cannot find pwd encrypted FES link in:\n\n${mimeMsg.text}`); } From 25f1bff683cac40a56344da3dd4f8b8a68971344 Mon Sep 17 00:00:00 2001 From: Roman Shevchenko Date: Sat, 16 Apr 2022 16:56:26 +0000 Subject: [PATCH 04/45] test rendering of recipients after successful sending --- test/source/mock/google/google-endpoints.ts | 3 ++- .../source/mock/google/strategies/send-message-strategy.ts | 6 ++++++ test/source/tests/compose.ts | 7 +++++++ 3 files changed, 15 insertions(+), 1 deletion(-) diff --git a/test/source/mock/google/google-endpoints.ts b/test/source/mock/google/google-endpoints.ts index 84b8e55731f..c9edd76a2a4 100644 --- a/test/source/mock/google/google-endpoints.ts +++ b/test/source/mock/google/google-endpoints.ts @@ -17,7 +17,8 @@ const allowedRecipients: Array = ['flowcrypt.compatibility@gmail.com', ' 'censored@email.com', 'test@email.com', 'human@flowcrypt.com', 'human+nopgp@flowcrypt.com', 'expired.on.attester@domain.com', 'ci.tests.gmail@flowcrypt.test', 'smime1@recipient.com', 'smime2@recipient.com', 'smime@recipient.com', 'smime.attachment@recipient.com', 'auto.refresh.expired.key@recipient.com', 'to@example.com', 'cc@example.com', 'bcc@example.com', - 'flowcrypt.test.key.multiple.inbox1@gmail.com', 'flowcrypt.test.key.multiple.inbox2@gmail.com', 'mock.only.pubkey@flowcrypt.com']; + 'flowcrypt.test.key.multiple.inbox1@gmail.com', 'flowcrypt.test.key.multiple.inbox2@gmail.com', 'mock.only.pubkey@flowcrypt.com', + 'vladimir@flowcrypt.com', 'limon.monte@gmail.com', 'sweetalert2@gmail.com']; export const mockGoogleEndpoints: HandlersDefinition = { '/o/oauth2/auth': async ({ query: { client_id, response_type, access_type, state, redirect_uri, scope, login_hint, proceed } }, req) => { diff --git a/test/source/mock/google/strategies/send-message-strategy.ts b/test/source/mock/google/strategies/send-message-strategy.ts index 282866c9fba..20ad2b2cc11 100644 --- a/test/source/mock/google/strategies/send-message-strategy.ts +++ b/test/source/mock/google/strategies/send-message-strategy.ts @@ -123,6 +123,10 @@ class PlainTextMessageTestStrategy implements ITestMsgStrategy { }; } +class NoopTestStrategy implements ITestMsgStrategy { + public test = async () => { }; +} + class IncludeQuotedPartTestStrategy implements ITestMsgStrategy { private readonly quotedContent: string = [ 'On 2019-06-14 at 23:24, flowcrypt.compatibility@gmail.com wrote:', @@ -252,6 +256,8 @@ export class TestBySubjectStrategyContext { this.strategy = new SmimeSignedMessageStrategy(); } else if (GMAIL_RECOVERY_EMAIL_SUBJECTS.includes(subject)) { this.strategy = new SaveMessageInStorageStrategy(); + } else if (subject.includes('Re: FROM: flowcrypt.compatibility@gmail.com, TO: flowcrypt.compatibility@gmail.com + vladimir@flowcrypt.com')) { + this.strategy = new NoopTestStrategy(); } else { throw new UnsuportableStrategyError(`There isn't any strategy for this subject: ${subject}`); } diff --git a/test/source/tests/compose.ts b/test/source/tests/compose.ts index 4bf697b3d89..419e1ba0c21 100644 --- a/test/source/tests/compose.ts +++ b/test/source/tests/compose.ts @@ -815,6 +815,7 @@ export const defineComposeTests = (testVariant: TestVariant, testWithBrowser: Te const fileInput = await composePage.target.$('input[type=file]'); await fileInput!.uploadFile(`test/samples/${attachmentFilename}`); await composePage.waitAndClick('@action-send', { delay: 1 }); + await composePage.waitForContent('@replied-to', 'to: censored@email.com'); const attachment = await composePage.getFrame(['attachment.htm', `name=${attachmentFilename}`]); await attachment.waitForSelTestState('ready'); const fileText = await composePage.awaitDownloadTriggeredByClicking(async () => { @@ -1550,6 +1551,12 @@ export const defineComposeTests = (testVariant: TestVariant, testWithBrowser: Te to: [{ email: 'flowcrypt.compatibility@gmail.com', name: 'First Last' }, { email: 'vladimir@flowcrypt.com' }], cc: [{ email: 'limon.monte@gmail.com' }], bcc: [{ email: 'sweetalert2@gmail.com' }] }); + await composePage.waitAndType('@input-password', 'gO0d-pwd'); + await composePage.waitAndClick('@action-send', { delay: 1 }); + // test rendering of recipients after successful sending + await composePage.waitForContent('@replied-to', 'to: First Last , vladimir@flowcrypt.com'); + await composePage.waitForContent('@replied-cc', 'cc: limon.monte@gmail.com'); + await composePage.waitForContent('@replied-bcc', 'bcc: sweetalert2@gmail.com'); })); ava.default('compose - reply all - from !== acctEmail', testWithBrowser('compatibility', async (t, browser) => { From 1f7daf5f1b9d13230dab4bb3b6ca454ab1b42718 Mon Sep 17 00:00:00 2001 From: Roman Shevchenko Date: Mon, 18 Apr 2022 08:19:34 +0000 Subject: [PATCH 05/45] lint fix --- test/source/mock/google/strategies/send-message-strategy.ts | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/test/source/mock/google/strategies/send-message-strategy.ts b/test/source/mock/google/strategies/send-message-strategy.ts index 20ad2b2cc11..d4630c0a6b2 100644 --- a/test/source/mock/google/strategies/send-message-strategy.ts +++ b/test/source/mock/google/strategies/send-message-strategy.ts @@ -124,7 +124,7 @@ class PlainTextMessageTestStrategy implements ITestMsgStrategy { } class NoopTestStrategy implements ITestMsgStrategy { - public test = async () => { }; + public test = async () => { }; // tslint:disable-line:no-empty } class IncludeQuotedPartTestStrategy implements ITestMsgStrategy { From 4460fb388f9801fd61297026263f27fa5dcccc51 Mon Sep 17 00:00:00 2001 From: Roman Shevchenko Date: Tue, 19 Apr 2022 12:15:27 +0000 Subject: [PATCH 06/45] testing PWD-encrypted message in reply thread --- test/source/mock/fes/fes-endpoints.ts | 85 +++++++++++----- .../message-export-1803be3182d1937b.json | 97 +++++++++++++++++++ test/source/mock/google/google-endpoints.ts | 2 +- .../strategies/send-message-strategy.ts | 2 + test/source/tests/compose.ts | 20 ++++ 5 files changed, 182 insertions(+), 24 deletions(-) create mode 100644 test/source/mock/google/exported-messages/message-export-1803be3182d1937b.json diff --git a/test/source/mock/fes/fes-endpoints.ts b/test/source/mock/fes/fes-endpoints.ts index 6534c69baf4..292c0cc39cb 100644 --- a/test/source/mock/fes/fes-endpoints.ts +++ b/test/source/mock/fes/fes-endpoints.ts @@ -9,6 +9,51 @@ import { MockJwt } from '../lib/oauth'; const standardFesUrl = 'fes.standardsubdomainfes.test:8001'; const issuedAccessTokens: string[] = []; +const processMessageFromUser = (body: string) => { + expect(body).to.contain('-----BEGIN PGP MESSAGE-----'); + expect(body).to.contain('"associateReplyToken":"mock-fes-reply-token"'); + expect(body).to.contain('"to":["Mr To "]'); + expect(body).to.contain('"cc":[]'); + expect(body).to.contain('"bcc":["Mr Bcc "]'); + const response = + { + // todo: do we need to support and test legacy? + 'url': `http://${standardFesUrl}/message/FES-MOCK-MESSAGE-ID`, + 'externalId': 'FES-MOCK-EXTERNAL-ID', + emailToExternalIdAndUrl: {} as { [email: string]: { url: string, externalId: string } } + }; + response.emailToExternalIdAndUrl['to@example.com'] = { + url: `http://${standardFesUrl}/message/FES-MOCK-MESSAGE-FOR-TO@EXAMPLE.COM-ID`, + externalId: 'FES-MOCK-EXTERNAL-FOR-TO@EXAMPLE.COM-ID' + }; + response.emailToExternalIdAndUrl['bcc@example.com'] = { + url: `http://${standardFesUrl}/message/FES-MOCK-MESSAGE-FOR-BCC@EXAMPLE.COM-ID`, + externalId: 'FES-MOCK-EXTERNAL-FOR-BCC@EXAMPLE.COM-ID' + }; + return response; +} + +const processMessageFromUser2 = (body: string) => { + expect(body).to.contain('-----BEGIN PGP MESSAGE-----'); + expect(body).to.contain('"associateReplyToken":"mock-fes-reply-token"'); + expect(body).to.contain('"to":["sender@domain.com","flowcrypt.compatibility@gmail.com","to@example.com","mock.only.pubkey@flowcrypt.com"]'); + expect(body).to.contain('"cc":[]'); + expect(body).to.contain('"bcc":[]'); + const response = + { + emailToExternalIdAndUrl: {} as { [email: string]: { url: string, externalId: string } } + }; + response.emailToExternalIdAndUrl['to@example.com'] = { + url: `http://${standardFesUrl}/message/FES-MOCK-MESSAGE-FOR-TO@EXAMPLE.COM-ID`, + externalId: 'FES-MOCK-EXTERNAL-FOR-TO@EXAMPLE.COM-ID' + }; + response.emailToExternalIdAndUrl['sender@domain.com'] = { + url: `http://${standardFesUrl}/message/FES-MOCK-MESSAGE-FOR-SENDER@DOMAIN.COM-ID`, + externalId: 'FES-MOCK-EXTERNAL-FOR-SENDER@DOMAIN.COM-ID' + }; + return response; +} + export const mockFesEndpoints: HandlersDefinition = { // standard fes location at https://fes.domain.com '/api/': async ({ }, req) => { @@ -54,31 +99,15 @@ export const mockFesEndpoints: HandlersDefinition = { }, '/api/v1/message': async ({ body }, req) => { // body is a mime-multipart string, we're doing a few smoke checks here without parsing it - if (req.headers.host === standardFesUrl && req.method === 'POST') { + if (req.headers.host === standardFesUrl && req.method === 'POST' && typeof body === 'string') { // test: `compose - user@standardsubdomainfes.test:8001 - PWD encrypted message with FES web portal` authenticate(req, 'oidc'); - expect(body).to.contain('-----BEGIN PGP MESSAGE-----'); - expect(body).to.contain('"associateReplyToken":"mock-fes-reply-token"'); - expect(body).to.contain('"to":["Mr To "]'); - expect(body).to.contain('"cc":[]'); - expect(body).to.contain('"bcc":["Mr Bcc "]'); - expect(body).to.contain('"from":"user@standardsubdomainfes.test:8001"'); - const response = - { - // todo: do we need to support and test legacy? - 'url': `http://${standardFesUrl}/message/FES-MOCK-MESSAGE-ID`, - 'externalId': 'FES-MOCK-EXTERNAL-ID', - emailToExternalIdAndUrl: {} as { [email: string]: { url: string, externalId: string } } - }; - response.emailToExternalIdAndUrl['to@example.com'] = { - url: `http://${standardFesUrl}/message/FES-MOCK-MESSAGE-FOR-TO@EXAMPLE.COM-ID`, - externalId: 'FES-MOCK-EXTERNAL-FOR-TO@EXAMPLE.COM-ID' - }; - response.emailToExternalIdAndUrl['bcc@example.com'] = { - url: `http://${standardFesUrl}/message/FES-MOCK-MESSAGE-FOR-BCC@EXAMPLE.COM-ID`, - externalId: 'FES-MOCK-EXTERNAL-FOR-BCC@EXAMPLE.COM-ID' - }; - return response; + if (body.includes('"from":"user@standardsubdomainfes.test:8001"')) { + return processMessageFromUser(body); + } + if (body.includes('"from":"user2@standardsubdomainfes.test:8001"')) { + return processMessageFromUser2(body); + } } throw new HttpClientErr('Not Found', 404); }, @@ -92,9 +121,19 @@ export const mockFesEndpoints: HandlersDefinition = { } throw new HttpClientErr('Not Found', 404); }, + '/api/v1/message/FES-MOCK-EXTERNAL-FOR-SENDER@DOMAIN.COM-ID/gateway': async ({ body }, req) => { + if (req.headers.host === standardFesUrl && req.method === 'POST') { + // test: `compose - user2@standardsubdomainfes.test:8001 - PWD encrypted message with FES - Reply rendering` + authenticate(req, 'oidc'); + expect(body).to.match(/{"emailGatewayMessageId":"<(.+)@standardsubdomainfes.test:8001>"}/); + return {}; + } + throw new HttpClientErr('Not Found', 404); + }, '/api/v1/message/FES-MOCK-EXTERNAL-FOR-TO@EXAMPLE.COM-ID/gateway': async ({ body }, req) => { if (req.headers.host === standardFesUrl && req.method === 'POST') { // test: `compose - user@standardsubdomainfes.test:8001 - PWD encrypted message with FES web portal` + // test: `compose - user2@standardsubdomainfes.test:8001 - PWD encrypted message with FES - Reply rendering` authenticate(req, 'oidc'); expect(body).to.match(/{"emailGatewayMessageId":"<(.+)@standardsubdomainfes.test:8001>"}/); return {}; diff --git a/test/source/mock/google/exported-messages/message-export-1803be3182d1937b.json b/test/source/mock/google/exported-messages/message-export-1803be3182d1937b.json new file mode 100644 index 00000000000..8f4d7b511e6 --- /dev/null +++ b/test/source/mock/google/exported-messages/message-export-1803be3182d1937b.json @@ -0,0 +1,97 @@ +{ + "acctEmail": "user2@standardsubdomainfes.test:8001", + "full": { + "id": "1803be3182d1937b", + "threadId": "1803be2e506153d2", + "labelIds": [ + "IMPORTANT", + "SENT", + "INBOX" + ], + "snippet": "some dummy text", + "payload": { + "partId": "", + "mimeType": "multipart/alternative", + "filename": "", + "headers": [ + { + "name": "MIME-Version", + "value": "1.0" + }, + { + "name": "Date", + "value": "Mon, 18 Apr 2022 11:56:18 +0300" + }, + { + "name": "Subject", + "value": "PWD encrypted message with FES - Reply rendering" + }, + { + "name": "From", + "value": "sender@domain.com" + }, + { + "name": "To", + "value": "flowcrypt.compatibility@gmail.com, to@example.com, mock.only.pubkey@flowcrypt.com" + }, + { + "name": "Content-Type", + "value": "multipart/alternative; boundary=\"0000000000001c389105dce9ef23\"" + } + ], + "body": { + "size": 0 + }, + "parts": [ + { + "partId": "0", + "mimeType": "text/plain", + "filename": "", + "headers": [ + { + "name": "Content-Type", + "value": "text/plain; charset=\"UTF-8\"" + } + ], + "body": { + "size": 17, + "data": "c29tZSBkdW1teSB0ZXh0DQo=" + } + }, + { + "partId": "1", + "mimeType": "text/html", + "filename": "", + "headers": [ + { + "name": "Content-Type", + "value": "text/html; charset=\"UTF-8\"" + } + ], + "body": { + "size": 38, + "data": "PGRpdiBkaXI9Imx0ciI-c29tZSBkdW1teSB0ZXh0PC9kaXY-DQo=" + } + } + ] + }, + "sizeEstimate": 595, + "historyId": "2271300", + "internalDate": "1650272178000" + }, + "attachments": {}, + "raw": { + "id": "1803be3182d1937b", + "threadId": "1803be2e506153d2", + "labelIds": [ + "IMPORTANT", + "SENT", + "INBOX" + ], + "snippet": "some dummy text", + "sizeEstimate": 595, + "raw": "TUlNRS1WZXJzaW9uOiAxLjANCkRhdGU6IE1vbiwgMTggQXByIDIwMjIgMTE6NTY6MTggKzAzMDANCk1lc3NhZ2UtSUQ6IDxDQU85Rlk5dnE5ejRqVXE3WnAzUUFuRWtNdC04Y05yQmlxS0hSQngxMTRWYmNMcHN5ZVFAbWFpbC5nbWFpbC5jb20-DQpTdWJqZWN0OiBzb21lIGR1bW15IG1lc3NhZ2UNCkZyb206IEdtYWlsIENJIFRlc3QgPGNpLnRlc3RzLmdtYWlsQGZsb3djcnlwdC5kZXY-DQpUbzogR21haWwgQ0kgVGVzdCA8Y2kudGVzdHMuZ21haWxAZmxvd2NyeXB0LmRldj4NCkNvbnRlbnQtVHlwZTogbXVsdGlwYXJ0L2FsdGVybmF0aXZlOyBib3VuZGFyeT0iMDAwMDAwMDAwMDAwMWMzODkxMDVkY2U5ZWYyMyINCg0KLS0wMDAwMDAwMDAwMDAxYzM4OTEwNWRjZTllZjIzDQpDb250ZW50LVR5cGU6IHRleHQvcGxhaW47IGNoYXJzZXQ9IlVURi04Ig0KDQpzb21lIGR1bW15IHRleHQNCg0KLS0wMDAwMDAwMDAwMDAxYzM4OTEwNWRjZTllZjIzDQpDb250ZW50LVR5cGU6IHRleHQvaHRtbDsgY2hhcnNldD0iVVRGLTgiDQoNCjxkaXYgZGlyPSJsdHIiPnNvbWUgZHVtbXkgdGV4dDwvZGl2Pg0KDQotLTAwMDAwMDAwMDAwMDFjMzg5MTA1ZGNlOWVmMjMtLQ==", + "historyId": "2271300", + "internalDate": "1650272178000" + } +} \ No newline at end of file diff --git a/test/source/mock/google/google-endpoints.ts b/test/source/mock/google/google-endpoints.ts index c9edd76a2a4..53ae483bf1d 100644 --- a/test/source/mock/google/google-endpoints.ts +++ b/test/source/mock/google/google-endpoints.ts @@ -18,7 +18,7 @@ const allowedRecipients: Array = ['flowcrypt.compatibility@gmail.com', ' 'ci.tests.gmail@flowcrypt.test', 'smime1@recipient.com', 'smime2@recipient.com', 'smime@recipient.com', 'smime.attachment@recipient.com', 'auto.refresh.expired.key@recipient.com', 'to@example.com', 'cc@example.com', 'bcc@example.com', 'flowcrypt.test.key.multiple.inbox1@gmail.com', 'flowcrypt.test.key.multiple.inbox2@gmail.com', 'mock.only.pubkey@flowcrypt.com', - 'vladimir@flowcrypt.com', 'limon.monte@gmail.com', 'sweetalert2@gmail.com']; + 'vladimir@flowcrypt.com', 'limon.monte@gmail.com', 'sweetalert2@gmail.com', 'sender@domain.com']; export const mockGoogleEndpoints: HandlersDefinition = { '/o/oauth2/auth': async ({ query: { client_id, response_type, access_type, state, redirect_uri, scope, login_hint, proceed } }, req) => { diff --git a/test/source/mock/google/strategies/send-message-strategy.ts b/test/source/mock/google/strategies/send-message-strategy.ts index d4630c0a6b2..30ac91768b0 100644 --- a/test/source/mock/google/strategies/send-message-strategy.ts +++ b/test/source/mock/google/strategies/send-message-strategy.ts @@ -242,6 +242,8 @@ export class TestBySubjectStrategyContext { this.strategy = new PwdEncryptedMessageWithFlowCryptComApiTestStrategy(); } else if (subject.includes('PWD encrypted message with FES - ID TOKEN')) { this.strategy = new PwdEncryptedMessageWithFesIdTokenTestStrategy(); + } else if (subject.includes('PWD encrypted message with FES - Reply rendering')) { + this.strategy = new SaveMessageInStorageStrategy(); } else if (subject.includes('Message With Image')) { this.strategy = new SaveMessageInStorageStrategy(); } else if (subject.includes('Message With Test Text')) { diff --git a/test/source/tests/compose.ts b/test/source/tests/compose.ts index 419e1ba0c21..75201e9be3a 100644 --- a/test/source/tests/compose.ts +++ b/test/source/tests/compose.ts @@ -1604,6 +1604,26 @@ export const defineComposeTests = (testVariant: TestVariant, testWithBrowser: Te // also see '/api/v1/message' in fes-endpoints.ts mock })); + /** + * You need the following lines in /etc/hosts: + * 127.0.0.1 standardsubdomainfes.test + * 127.0.0.1 fes.standardsubdomainfes.test + */ + ava.default('user2@standardsubdomainfes.test:8001 - PWD encrypted message with FES - Reply rendering', testWithBrowser(undefined, async (t, browser) => { + const acct = 'user2@standardsubdomainfes.test:8001'; // added port to trick extension into calling the mock + const settingsPage = await BrowserRecipe.openSettingsLoginApprove(t, browser, acct); + await SetupPageRecipe.manualEnter(settingsPage, 'flowcrypt.test.key.used.pgp', { submitPubkey: false, usedPgpBefore: false }, + { isSavePassphraseChecked: false, isSavePassphraseHidden: false }); + const appendUrl = 'threadId=1803be2e506153d2&skipClickPrompt=___cu_false___&ignoreDraft=___cu_false___&replyMsgId=1803be3182d1937b'; + const composePage = await ComposePageRecipe.openStandalone(t, browser, 'user2@standardsubdomainfes.test:8001', { appendUrl, hasReplyPrompt: true }); + await composePage.waitAndClick('@action-accept-reply-all-prompt', { delay: 2 }); + const fileInput = await composePage.target.$('input[type=file]'); + await fileInput!.uploadFile('test/samples/small.txt'); + await composePage.waitAndType('@input-password', 'gO0d-pwd'); + await composePage.waitAndClick('@action-send', { delay: 1 }); + // todo: + })); + ava.default('first.key.revoked@key-manager-autoimport-no-prv-create.flowcrypt.test - selects valid own key when saving draft or sending', testWithBrowser(undefined, async (t, browser) => { const acct = 'first.key.revoked@key-manager-autoimport-no-prv-create.flowcrypt.test'; const settingsPage = await BrowserRecipe.openSettingsLoginApprove(t, browser, acct); From a4c9c8e08cfdf623e43dc0b37fce76ce9e741f96 Mon Sep 17 00:00:00 2001 From: Roman Shevchenko Date: Wed, 20 Apr 2022 16:09:00 +0000 Subject: [PATCH 07/45] lint fix --- test/source/mock/fes/fes-endpoints.ts | 5 +---- 1 file changed, 1 insertion(+), 4 deletions(-) diff --git a/test/source/mock/fes/fes-endpoints.ts b/test/source/mock/fes/fes-endpoints.ts index 292c0cc39cb..4be8d886db4 100644 --- a/test/source/mock/fes/fes-endpoints.ts +++ b/test/source/mock/fes/fes-endpoints.ts @@ -39,10 +39,7 @@ const processMessageFromUser2 = (body: string) => { expect(body).to.contain('"to":["sender@domain.com","flowcrypt.compatibility@gmail.com","to@example.com","mock.only.pubkey@flowcrypt.com"]'); expect(body).to.contain('"cc":[]'); expect(body).to.contain('"bcc":[]'); - const response = - { - emailToExternalIdAndUrl: {} as { [email: string]: { url: string, externalId: string } } - }; + const response = { emailToExternalIdAndUrl: {} as { [email: string]: { url: string, externalId: string } } }; response.emailToExternalIdAndUrl['to@example.com'] = { url: `http://${standardFesUrl}/message/FES-MOCK-MESSAGE-FOR-TO@EXAMPLE.COM-ID`, externalId: 'FES-MOCK-EXTERNAL-FOR-TO@EXAMPLE.COM-ID' From 3f8c32583c277cdad79c21d861cd2dbc4ba294ef Mon Sep 17 00:00:00 2001 From: Roman Shevchenko Date: Thu, 21 Apr 2022 08:36:51 +0000 Subject: [PATCH 08/45] lint fix --- test/source/mock/fes/fes-endpoints.ts | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/test/source/mock/fes/fes-endpoints.ts b/test/source/mock/fes/fes-endpoints.ts index 4be8d886db4..44029c5e9b2 100644 --- a/test/source/mock/fes/fes-endpoints.ts +++ b/test/source/mock/fes/fes-endpoints.ts @@ -31,7 +31,7 @@ const processMessageFromUser = (body: string) => { externalId: 'FES-MOCK-EXTERNAL-FOR-BCC@EXAMPLE.COM-ID' }; return response; -} +}; const processMessageFromUser2 = (body: string) => { expect(body).to.contain('-----BEGIN PGP MESSAGE-----'); @@ -49,7 +49,7 @@ const processMessageFromUser2 = (body: string) => { externalId: 'FES-MOCK-EXTERNAL-FOR-SENDER@DOMAIN.COM-ID' }; return response; -} +}; export const mockFesEndpoints: HandlersDefinition = { // standard fes location at https://fes.domain.com From 8b9108467456f3e7b8aa12e70dfd4f85c7b9821c Mon Sep 17 00:00:00 2001 From: Roman Shevchenko Date: Thu, 21 Apr 2022 15:21:12 +0000 Subject: [PATCH 09/45] wip --- extension/chrome/elements/compose.htm | 2 +- test/source/tests/compose.ts | 19 ++++++++++++++++++- 2 files changed, 19 insertions(+), 2 deletions(-) diff --git a/extension/chrome/elements/compose.htm b/extension/chrome/elements/compose.htm index 853c2224281..8ff4144760c 100644 --- a/extension/chrome/elements/compose.htm +++ b/extension/chrome/elements/compose.htm @@ -181,7 +181,7 @@

New Secure Message

-
+
diff --git a/test/source/tests/compose.ts b/test/source/tests/compose.ts index 4ca964442df..2c3742b7336 100644 --- a/test/source/tests/compose.ts +++ b/test/source/tests/compose.ts @@ -24,6 +24,7 @@ import { testConstants } from './tooling/consts'; import { MsgUtil } from '../core/crypto/pgp/msg-util'; import { Buf } from '../core/buf'; import { PubkeyInfoWithLastCheck } from '../core/crypto/key'; +import { Page } from 'puppeteer'; // tslint:disable:no-blank-lines-func // tslint:disable:no-unused-expression @@ -1617,11 +1618,27 @@ export const defineComposeTests = (testVariant: TestVariant, testWithBrowser: Te const appendUrl = 'threadId=1803be2e506153d2&skipClickPrompt=___cu_false___&ignoreDraft=___cu_false___&replyMsgId=1803be3182d1937b'; const composePage = await ComposePageRecipe.openStandalone(t, browser, 'user2@standardsubdomainfes.test:8001', { appendUrl, hasReplyPrompt: true }); await composePage.waitAndClick('@action-accept-reply-all-prompt', { delay: 2 }); + // we should have 4 recipients, 2 green and 2 gray + const container = (await composePage.waitAny('@container-to'))!; + const recipients = await container.$$('.recipients > span'); + expect(recipients.length).to.equal(4); + expect(await PageRecipe.getElementPropertyJson(recipients[0], 'textContent')).to.equal('sender@domain.com '); + expect(await PageRecipe.getElementPropertyJson(recipients[0], 'className')).to.equal('no_pgp'); + expect(await PageRecipe.getElementPropertyJson(recipients[1], 'textContent')).to.equal('flowcrypt.compatibility@gmail.com '); + expect(await PageRecipe.getElementPropertyJson(recipients[1], 'className')).to.equal('has_pgp'); + expect(await PageRecipe.getElementPropertyJson(recipients[2], 'textContent')).to.equal('to@example.com '); + expect(await PageRecipe.getElementPropertyJson(recipients[2], 'className')).to.equal('no_pgp'); + expect(await PageRecipe.getElementPropertyJson(recipients[3], 'textContent')).to.equal('mock.only.pubkey@flowcrypt.com '); + expect(await PageRecipe.getElementPropertyJson(recipients[3], 'className')).to.equal('has_pgp'); const fileInput = await composePage.target.$('input[type=file]'); await fileInput!.uploadFile('test/samples/small.txt'); await composePage.waitAndType('@input-password', 'gO0d-pwd'); await composePage.waitAndClick('@action-send', { delay: 1 }); - // todo: + const attachmentsContainer = (await composePage.waitAny('@replied-attachments'))!; + const attachments = await attachmentsContainer.$$('.pgp_attachment'); + expect(attachments.length).to.equal(6); // todo: + const attachmentFrames = (composePage.target as Page).frames(); + expect(attachmentFrames.length).to.equal(7); // todo: })); ava.default('first.key.revoked@key-manager-autoimport-no-prv-create.flowcrypt.test - selects valid own key when saving draft or sending', testWithBrowser(undefined, async (t, browser) => { From 56c69f3cd0ffaa0dd13f1b25ef03964d7aabb26a Mon Sep 17 00:00:00 2001 From: Roman Shevchenko Date: Tue, 26 Apr 2022 11:37:10 +0000 Subject: [PATCH 10/45] fixes --- extension/chrome/elements/attachment.htm | 2 +- .../compose-modules/compose-draft-module.ts | 2 +- .../compose-send-btn-module.ts | 13 ++--- .../encrypted-mail-msg-formatter.ts | 56 ++++++++++++------- .../formatters/general-mail-formatter.ts | 19 +++++-- test/source/mock/fes/fes-endpoints.ts | 21 +++++-- .../message-export-1803be3182d1937b.json | 2 +- test/source/mock/google/google-data.ts | 8 ++- test/source/mock/google/google-endpoints.ts | 2 +- .../strategies/send-message-strategy.ts | 42 ++++++++------ .../mock/google/strategies/strategy-base.ts | 4 +- test/source/tests/compose.ts | 13 ++++- 12 files changed, 120 insertions(+), 64 deletions(-) diff --git a/extension/chrome/elements/attachment.htm b/extension/chrome/elements/attachment.htm index b3f7e564dce..b5783e26766 100644 --- a/extension/chrome/elements/attachment.htm +++ b/extension/chrome/elements/attachment.htm @@ -20,7 +20,7 @@ -
+
diff --git a/extension/chrome/elements/compose-modules/compose-draft-module.ts b/extension/chrome/elements/compose-modules/compose-draft-module.ts index 6faf717b984..d0e914c5038 100644 --- a/extension/chrome/elements/compose-modules/compose-draft-module.ts +++ b/extension/chrome/elements/compose-modules/compose-draft-module.ts @@ -133,7 +133,7 @@ export class ComposeDraftModule extends ViewModule { throw new UnreportableError('Your account keys are not usable for encryption'); } msgData.pwd = undefined; // not needed for drafts - const [sendable] = await new EncryptedMsgMailFormatter(this.view, true).sendableMsgs(msgData, pubkeys); + const sendable = await new EncryptedMsgMailFormatter(this.view, true).sendableNonPwdMsg(msgData, pubkeys); if (this.view.replyParams?.inReplyTo) { sendable.headers.References = this.view.replyParams.inReplyTo; sendable.headers['In-Reply-To'] = this.view.replyParams.inReplyTo; diff --git a/extension/chrome/elements/compose-modules/compose-send-btn-module.ts b/extension/chrome/elements/compose-modules/compose-send-btn-module.ts index ca6ecfe4302..c6d27b86d07 100644 --- a/extension/chrome/elements/compose-modules/compose-send-btn-module.ts +++ b/extension/chrome/elements/compose-modules/compose-send-btn-module.ts @@ -11,7 +11,7 @@ import { Buf } from '../../../js/common/core/buf.js'; import { Catch } from '../../../js/common/platform/catch.js'; import { ComposerUserError } from './compose-err-module.js'; import { ComposeSendBtnPopoverModule } from './compose-send-btn-popover-module.js'; -import { GeneralMailFormatter } from './formatters/general-mail-formatter.js'; +import { GeneralMailFormatter, MultipleMessages } from './formatters/general-mail-formatter.js'; import { GmailParser, GmailRes } from '../../../js/common/api/email-provider/gmail/gmail-parser.js'; import { KeyInfoWithIdentity } from '../../../js/common/core/crypto/key.js'; import { getUniqueRecipientEmails, SendBtnTexts } from './compose-types.js'; @@ -118,7 +118,7 @@ export class ComposeSendBtnModule extends ViewModule { for (const msg of msgObj.msgs) { await this.finalizeSendableMsg({ msg, senderKi: msgObj.senderKi }); } - await this.doSendMsgs(msgObj.msgs); + await this.doSendMsgs(msgObj); } catch (e) { await this.view.errModule.handleSendErr(e); } finally { @@ -197,13 +197,13 @@ export class ComposeSendBtnModule extends ViewModule { }; - private doSendMsgs = async (msgs: SendableMsg[]) => { + private doSendMsgs = async (msgObj: MultipleMessages) => { // if this is a password-encrypted message, then we've already shown progress for uploading to backend // and this requests represents second half of uploadable effort. Else this represents all (no previous heavy requests) const progressRepresents = this.view.pwdOrPubkeyContainerModule.isVisible() ? 'SECOND-HALF' : 'EVERYTHING'; const sentIds: string[] = []; const operations = [this.view.draftModule.draftDelete()]; - for (const msg of msgs) { + for (const msg of msgObj.msgs) { let msgSentRes: GmailRes.GmailMsgSend; try { this.isSendMessageInProgress = true; @@ -235,10 +235,7 @@ export class ComposeSendBtnModule extends ViewModule { await Promise.all(operations); this.isSendMessageInProgress = false; if (this.view.isReplyBox) { - // todo: collect recipients correctly - // todo: collect attachments correctly - const attachments = msgs.map(m => m.attachments).reduce((a, b) => a.concat(b), []); - this.view.renderModule.renderReplySuccess(attachments, msgs[0].recipients, sentIds[0]); // todo: + this.view.renderModule.renderReplySuccess(msgObj.attachments, msgObj.recipients, sentIds[0]); } else { this.view.renderModule.closeMsg(); } diff --git a/extension/chrome/elements/compose-modules/formatters/encrypted-mail-msg-formatter.ts b/extension/chrome/elements/compose-modules/formatters/encrypted-mail-msg-formatter.ts index 1b0e1691ee7..2570bb47d83 100644 --- a/extension/chrome/elements/compose-modules/formatters/encrypted-mail-msg-formatter.ts +++ b/extension/chrome/elements/compose-modules/formatters/encrypted-mail-msg-formatter.ts @@ -23,10 +23,12 @@ import { FcUuidAuth } from '../../../../js/common/api/account-servers/flowcrypt- import { SmimeKey } from '../../../../js/common/core/crypto/smime/smime-key.js'; import { PgpHash } from '../../../../js/common/core/crypto/pgp/pgp-hash.js'; import { UploadedMessageData } from '../../../../js/common/api/account-server.js'; +import { ParsedKeyInfo } from '../../../../js/common/core/crypto/key-store-util.js'; +import { MultipleMessages } from './general-mail-formatter.js'; export class EncryptedMsgMailFormatter extends BaseMailFormatter { - public sendableMsgs = async (newMsg: NewMsgData, pubkeys: PubkeyResult[], signingPrv?: Key): Promise => { + public sendableMsgs = async (newMsg: NewMsgData, pubkeys: PubkeyResult[], signingKey?: ParsedKeyInfo): Promise => { if (newMsg.pwd && !this.isDraft) { // password-protected message, temporarily uploaded (already encrypted) to: // - flowcrypt.com/api (consumers and customers without on-prem setup), or @@ -34,25 +36,40 @@ export class EncryptedMsgMailFormatter extends BaseMailFormatter { // It will be served to recipient through web const uploadedMessageData = await this.prepareAndUploadPwdEncryptedMsg(newMsg); // encrypted for pwd only, pubkeys ignored newMsg.pwd = undefined; - const entries = Object.entries(uploadedMessageData.emailToExternalIdAndUrl ?? {}); - if (entries.length) { - const msgs: SendableMsg[] = []; - for (const entry of entries) { - const recipientEmail = entry[0]; - const msgUrl = entry[1].url; - const externalId = entry[1].externalId; - msgs.push(await this.sendablePwdMsg(newMsg, pubkeys, { recipientEmail, msgUrl, externalId }, signingPrv)); - } - return msgs; - } else { - return [await this.sendablePwdMsg(newMsg, pubkeys, { msgUrl: uploadedMessageData.url, externalId: uploadedMessageData.externalId }, signingPrv)]; + const collectedAttachments = await this.view.attachmentsModule.attachment.collectEncryptAttachments(pubkeys); + // we always have at least one pubkey, this is the main message, encrypted for pubkeys + const msgs: SendableMsg[] = [ + await this.sendablePwdMsg( + newMsg, + pubkeys, + { msgUrl: uploadedMessageData.url, externalId: uploadedMessageData.externalId }, + collectedAttachments, + signingKey?.key) + ]; + // adding individual messages for each recipient that doesn't have a pubkey + for (const recipientEmail of Object.keys(uploadedMessageData.emailToExternalIdAndUrl ?? {}).filter(email => !pubkeys.some(p => p.email === email))) { + const { url, externalId } = uploadedMessageData.emailToExternalIdAndUrl![recipientEmail]; + msgs.push(await this.sendablePwdMsg(newMsg, pubkeys, { recipientEmail, msgUrl: url, externalId }, [], signingKey?.key)); } - } else if (this.richtext) { // rich text: PGP/MIME - https://tools.ietf.org/html/rfc3156#section-4 + return { senderKi: signingKey?.keyInfo, msgs, recipients: newMsg.recipients, attachments: collectedAttachments }; + } else { + const msg = await this.sendableNonPwdMsg(newMsg, pubkeys, signingKey?.key); + return { + senderKi: signingKey?.keyInfo, + msgs: [msg], + recipients: msg.recipients, + attachments: msg.attachments // todo: perhaps, we should hide technical attachments, like `encrypted.asc` and use collectedAttachments too? + }; + } + }; + + public sendableNonPwdMsg = async (newMsg: NewMsgData, pubkeys: PubkeyResult[], signingPrv?: Key): Promise => { + if (this.richtext) { // rich text: PGP/MIME - https://tools.ietf.org/html/rfc3156#section-4 // or S/MIME - return [await this.sendableRichTextMsg(newMsg, pubkeys, signingPrv)]; + return await this.sendableRichTextMsg(newMsg, pubkeys, signingPrv); } else { // simple text: PGP or S/MIME Inline with attachments in separate files // todo: #4046 check attachments for S/MIME - return [await this.sendableSimpleTextMsg(newMsg, pubkeys, signingPrv)]; + return await this.sendableSimpleTextMsg(newMsg, pubkeys, signingPrv); } }; @@ -104,13 +121,12 @@ export class EncryptedMsgMailFormatter extends BaseMailFormatter { newMsg: NewMsgData, pubs: PubkeyResult[], { msgUrl, externalId, recipientEmail }: { msgUrl: string, externalId?: string, recipientEmail?: string }, + attachments: Attachment[], signingPrv?: Key) => { // encoded as: PGP/MIME-like structure but with attachments as external files due to email size limit (encrypted for pubkeys only) const msgBody = this.richtext ? { 'text/plain': newMsg.plaintext, 'text/html': newMsg.plainhtml } : { 'text/plain': newMsg.plaintext }; const pgpMimeNoAttachments = await Mime.encode(msgBody, { Subject: newMsg.subject }, []); // no attachments, attached to email separately const { data: pubEncryptedNoAttachments } = await this.encryptDataArmor(Buf.fromUtfStr(pgpMimeNoAttachments), undefined, pubs, signingPrv); // encrypted only for pubs - const attachments = this.createPgpMimeAttachments(pubEncryptedNoAttachments). - concat(await this.view.attachmentsModule.attachment.collectEncryptAttachments(pubs)); // encrypted only for pubs const emailIntroAndLinkBody = await this.formatPwdEncryptedMsgBodyLink(msgUrl); const headers = this.headers(newMsg); if (recipientEmail) { @@ -119,7 +135,9 @@ export class EncryptedMsgMailFormatter extends BaseMailFormatter { find(r => r.email.toLowerCase() === recipientEmail.toLowerCase()); headers.recipients = { to: [foundParsedRecipient ?? { email: recipientEmail }] }; } - return await SendableMsg.createPwdMsg(this.acctEmail, headers, emailIntroAndLinkBody, attachments, { isDraft: this.isDraft, externalId }); + return await SendableMsg.createPwdMsg(this.acctEmail, headers, emailIntroAndLinkBody, + this.createPgpMimeAttachments(pubEncryptedNoAttachments).concat(attachments), + { isDraft: this.isDraft, externalId }); }; private sendableSimpleTextMsg = async (newMsg: NewMsgData, pubs: PubkeyResult[], signingPrv?: Key): Promise => { diff --git a/extension/chrome/elements/compose-modules/formatters/general-mail-formatter.ts b/extension/chrome/elements/compose-modules/formatters/general-mail-formatter.ts index 3f8fb398a0f..673c436643d 100644 --- a/extension/chrome/elements/compose-modules/formatters/general-mail-formatter.ts +++ b/extension/chrome/elements/compose-modules/formatters/general-mail-formatter.ts @@ -11,16 +11,26 @@ import { SignedMsgMailFormatter } from './signed-msg-mail-formatter.js'; import { ComposeView } from '../../compose.js'; import { KeyStoreUtil, ParsedKeyInfo } from "../../../../js/common/core/crypto/key-store-util.js"; import { UnreportableError } from '../../../../js/common/platform/catch.js'; +import { ParsedRecipients } from '../../../../js/common/api/email-provider/email-provider-api.js'; +import { Attachment } from '../../../../js/common/core/attachment.js'; + +export type MultipleMessages = { + msgs: SendableMsg[]; + senderKi: KeyInfoWithIdentity | undefined; + recipients: ParsedRecipients; + attachments: Attachment[]; +}; export class GeneralMailFormatter { // returns undefined in case user cancelled decryption of the signing key - public static processNewMsg = async (view: ComposeView, newMsgData: NewMsgData): Promise<{ msgs: SendableMsg[], senderKi: KeyInfoWithIdentity | undefined }> => { + public static processNewMsg = async (view: ComposeView, newMsgData: NewMsgData): Promise => { const choices = view.sendBtnModule.popover.choices; const recipientsEmails = getUniqueRecipientEmails(newMsgData.recipients); if (!choices.encrypt && !choices.sign) { // plain view.S.now('send_btn_text').text('Formatting...'); - return { senderKi: undefined, msgs: [await new PlainMsgMailFormatter(view).sendableMsg(newMsgData)] }; + const msg = await new PlainMsgMailFormatter(view).sendableMsg(newMsgData); + return { senderKi: undefined, msgs: [msg], recipients: msg.recipients, attachments: msg.attachments }; } if (!choices.encrypt && choices.sign) { // sign only view.S.now('send_btn_text').text('Signing...'); @@ -29,7 +39,8 @@ export class GeneralMailFormatter { if (!signingKey) { throw new UnreportableError('Could not find account key usable for signing this plain text message'); } - return { senderKi: signingKey!.keyInfo, msgs: [await new SignedMsgMailFormatter(view).sendableMsg(newMsgData, signingKey!.key)] }; + const msg = await new SignedMsgMailFormatter(view).sendableMsg(newMsgData, signingKey!.key); + return { senderKi: signingKey!.keyInfo, msgs: [msg], recipients: msg.recipients, attachments: msg.attachments }; } // encrypt (optionally sign) const singleFamilyKeys = await view.storageModule.collectSingleFamilyKeys(recipientsEmails, newMsgData.from, choices.sign); @@ -46,7 +57,7 @@ export class GeneralMailFormatter { } } view.S.now('send_btn_text').text('Encrypting...'); - return { senderKi: signingKey?.keyInfo, msgs: await new EncryptedMsgMailFormatter(view).sendableMsgs(newMsgData, singleFamilyKeys.pubkeys, signingKey?.key) }; + return await new EncryptedMsgMailFormatter(view).sendableMsgs(newMsgData, singleFamilyKeys.pubkeys, signingKey); }; private static chooseSigningKeyAndDecryptIt = async ( diff --git a/test/source/mock/fes/fes-endpoints.ts b/test/source/mock/fes/fes-endpoints.ts index 44029c5e9b2..9f90d214bef 100644 --- a/test/source/mock/fes/fes-endpoints.ts +++ b/test/source/mock/fes/fes-endpoints.ts @@ -17,9 +17,9 @@ const processMessageFromUser = (body: string) => { expect(body).to.contain('"bcc":["Mr Bcc "]'); const response = { - // todo: do we need to support and test legacy? - 'url': `http://${standardFesUrl}/message/FES-MOCK-MESSAGE-ID`, - 'externalId': 'FES-MOCK-EXTERNAL-ID', + // this url is required for pubkey encrypted message + url: `http://${standardFesUrl}/message/FES-MOCK-MESSAGE-ID`, + externalId: 'FES-MOCK-EXTERNAL-ID', emailToExternalIdAndUrl: {} as { [email: string]: { url: string, externalId: string } } }; response.emailToExternalIdAndUrl['to@example.com'] = { @@ -39,7 +39,12 @@ const processMessageFromUser2 = (body: string) => { expect(body).to.contain('"to":["sender@domain.com","flowcrypt.compatibility@gmail.com","to@example.com","mock.only.pubkey@flowcrypt.com"]'); expect(body).to.contain('"cc":[]'); expect(body).to.contain('"bcc":[]'); - const response = { emailToExternalIdAndUrl: {} as { [email: string]: { url: string, externalId: string } } }; + const response = { + // this url is required for pubkey encrypted message + url: `http://${standardFesUrl}/message/FES-MOCK-MESSAGE-ID`, + externalId: 'FES-MOCK-EXTERNAL-ID', + emailToExternalIdAndUrl: {} as { [email: string]: { url: string, externalId: string } } + }; response.emailToExternalIdAndUrl['to@example.com'] = { url: `http://${standardFesUrl}/message/FES-MOCK-MESSAGE-FOR-TO@EXAMPLE.COM-ID`, externalId: 'FES-MOCK-EXTERNAL-FOR-TO@EXAMPLE.COM-ID' @@ -48,6 +53,14 @@ const processMessageFromUser2 = (body: string) => { url: `http://${standardFesUrl}/message/FES-MOCK-MESSAGE-FOR-SENDER@DOMAIN.COM-ID`, externalId: 'FES-MOCK-EXTERNAL-FOR-SENDER@DOMAIN.COM-ID' }; + response.emailToExternalIdAndUrl['flowcrypt.compatibility@gmail.com'] = { + url: `http://${standardFesUrl}/message/FES-MOCK-MESSAGE-FOR-FLOWCRYPT.COMPATIBILITY@GMAIL.COM-ID`, + externalId: 'FES-MOCK-EXTERNAL-FOR-FLOWCRYPT.COMPATIBILITY@GMAIL.COM-ID' + }; + response.emailToExternalIdAndUrl['mock.only.pubkey@flowcrypt.com'] = { + url: `http://${standardFesUrl}/message/FES-MOCK-MESSAGE-FOR-MOCK.ONLY.PUBKEY@FLOWCRYPT.COM-ID`, + externalId: 'FES-MOCK-EXTERNAL-FOR-MOCK.ONLY.PUBKEY@FLOWCRYPT.COM-ID' + }; return response; }; diff --git a/test/source/mock/google/exported-messages/message-export-1803be3182d1937b.json b/test/source/mock/google/exported-messages/message-export-1803be3182d1937b.json index 8f4d7b511e6..474a54226ed 100644 --- a/test/source/mock/google/exported-messages/message-export-1803be3182d1937b.json +++ b/test/source/mock/google/exported-messages/message-export-1803be3182d1937b.json @@ -32,7 +32,7 @@ }, { "name": "To", - "value": "flowcrypt.compatibility@gmail.com, to@example.com, mock.only.pubkey@flowcrypt.com" + "value": "flowcrypt.Compatibility@gmail.com, to@example.com, mock.only.puBkey@flowcrypt.com" }, { "name": "Content-Type", diff --git a/test/source/mock/google/google-data.ts b/test/source/mock/google/google-data.ts index ddddb3e7615..1d609d9002d 100644 --- a/test/source/mock/google/google-data.ts +++ b/test/source/mock/google/google-data.ts @@ -4,6 +4,7 @@ import { AddressObject, ParsedMail, StructuredHeader } from 'mailparser'; import { Util } from '../../util/index'; import { readFile, readdir } from 'fs'; +import { ParseMsgResult } from '../../util/parse'; type GmailMsg$header = { name: string, value: string }; type GmailMsg$payload$body = { attachmentId?: string, size: number, data?: string }; @@ -177,8 +178,9 @@ export class GoogleData { } } - public storeSentMessage = (parsedMail: ParsedMail, base64Msg: string, id: string): string => { + public storeSentMessage = (parseResult: ParseMsgResult, id: string): string => { let bodyContentAtt: { data: string; size: number; filename?: string; id: string } | undefined; + const parsedMail = parseResult.mimeMsg; for (const attachment of parsedMail.attachments || []) { const attId = Util.lousyRandom(); const gmailAtt = { data: attachment.content.toString('base64'), size: attachment.size, filename: attachment.filename, id: attId }; @@ -198,7 +200,7 @@ export class GoogleData { } const barebonesGmailMsg: GmailMsg = { // todo - could be improved - very barebones id, - threadId: null, // tslint:disable-line:no-null-keyword + threadId: parseResult.threadId ?? null, // tslint:disable-line:no-null-keyword historyId: '', labelIds: ['SENT' as GmailMsg$labelId], payload: { @@ -208,7 +210,7 @@ export class GoogleData { ], body }, - raw: base64Msg + raw: parseResult.base64 }; DATA[this.acct].messages.push(barebonesGmailMsg); return barebonesGmailMsg.id; diff --git a/test/source/mock/google/google-endpoints.ts b/test/source/mock/google/google-endpoints.ts index 53ae483bf1d..13803cc59f0 100644 --- a/test/source/mock/google/google-endpoints.ts +++ b/test/source/mock/google/google-endpoints.ts @@ -222,7 +222,7 @@ export const mockGoogleEndpoints: HandlersDefinition = { const id = `msg_id_${Util.lousyRandom()}`; try { const testingStrategyContext = new TestBySubjectStrategyContext(parseResult.mimeMsg.subject || ''); - await testingStrategyContext.test(parseResult.mimeMsg, parseResult.base64, id); + await testingStrategyContext.test(parseResult, id); } catch (e) { if (!(e instanceof UnsuportableStrategyError)) { // No such strategy for test throw e; // todo - should start throwing unsupported test strategies too, else changing subject will cause incomplete testing diff --git a/test/source/mock/google/strategies/send-message-strategy.ts b/test/source/mock/google/strategies/send-message-strategy.ts index 30ac91768b0..a6aa924e1ce 100644 --- a/test/source/mock/google/strategies/send-message-strategy.ts +++ b/test/source/mock/google/strategies/send-message-strategy.ts @@ -1,7 +1,7 @@ /* ©️ 2016 - present FlowCrypt a.s. Limitations apply. Contact human@flowcrypt.com */ import * as forge from 'node-forge'; -import { AddressObject, ParsedMail, StructuredHeader } from 'mailparser'; +import { AddressObject, StructuredHeader } from 'mailparser'; import { ITestMsgStrategy, UnsuportableStrategyError } from './strategy-base.js'; import { Buf } from '../../../core/buf'; import { Config } from '../../../util'; @@ -9,7 +9,7 @@ import { expect } from 'chai'; import { GoogleData } from '../google-data'; import { HttpClientErr } from '../../lib/api'; import { MsgUtil } from '../../../core/crypto/pgp/msg-util'; -import Parse from '../../../util/parse'; +import Parse, { ParseMsgResult } from '../../../util/parse'; import { parsedMailAddressObjectAsArray } from '../google-endpoints.js'; import { Str } from '../../../core/common.js'; import { GMAIL_RECOVERY_EMAIL_SUBJECTS } from '../../../core/const.js'; @@ -18,13 +18,14 @@ import { testConstants } from '../../../tests/tooling/consts.js'; // TODO: Make a better structure of ITestMsgStrategy. Because this class doesn't test anything, it only saves message in the Mock class SaveMessageInStorageStrategy implements ITestMsgStrategy { - public test = async (mimeMsg: ParsedMail, base64Msg: string, id: string) => { - (await GoogleData.withInitializedData(mimeMsg.from!.value[0].address!)).storeSentMessage(mimeMsg, base64Msg, id); + public test = async (parseResult: ParseMsgResult, id: string) => { + (await GoogleData.withInitializedData(parseResult.mimeMsg.from!.value[0].address!)).storeSentMessage(parseResult, id); }; } class PwdEncryptedMessageWithFlowCryptComApiTestStrategy implements ITestMsgStrategy { - public test = async (mimeMsg: ParsedMail) => { + public test = async (parseResult: ParseMsgResult) => { + const mimeMsg = parseResult.mimeMsg; const senderEmail = Str.parseEmail(mimeMsg.from!.text).email; if (!mimeMsg.text?.includes(`${senderEmail} has sent you a password-encrypted email`)) { throw new HttpClientErr(`Error checking sent text in:\n\n${mimeMsg.text}`); @@ -39,7 +40,8 @@ class PwdEncryptedMessageWithFlowCryptComApiTestStrategy implements ITestMsgStra } class PwdEncryptedMessageWithFesIdTokenTestStrategy implements ITestMsgStrategy { - public test = async (mimeMsg: ParsedMail, base64Msg: string, id: string) => { + public test = async (parseResult: ParseMsgResult, id: string) => { + const mimeMsg = parseResult.mimeMsg; const expectedSenderEmail = 'user@standardsubdomainfes.test:8001'; expect(mimeMsg.from!.text).to.equal(`First Last <${expectedSenderEmail}>`); if (!mimeMsg.text?.includes(`${expectedSenderEmail} has sent you a password-encrypted email`)) { @@ -70,14 +72,15 @@ class PwdEncryptedMessageWithFesIdTokenTestStrategy implements ITestMsgStrategy if (!mimeMsg.text?.includes('Follow this link to open it')) { throw new HttpClientErr(`Error: cannot find pwd encrypted open link prompt in ${mimeMsg.text}`); } - await new SaveMessageInStorageStrategy().test(mimeMsg, base64Msg, id); + await new SaveMessageInStorageStrategy().test(parseResult, id); }; } class MessageWithFooterTestStrategy implements ITestMsgStrategy { private readonly footer = 'flowcrypt.compatibility test footer with an img'; - public test = async (mimeMsg: ParsedMail) => { + public test = async (parseResult: ParseMsgResult) => { + const mimeMsg = parseResult.mimeMsg; const keyInfo = await Config.getKeyInfo(["flowcrypt.compatibility.1pp1", "flowcrypt.compatibility.2pp1"]); const decrypted = await MsgUtil.decryptMessage({ kisWithPp: keyInfo!, encryptedData: Buf.fromUtfStr(mimeMsg.text || ''), verificationPubs: [] }); if (!decrypted.success) { @@ -94,7 +97,8 @@ class SignedMessageTestStrategy implements ITestMsgStrategy { private readonly expectedText = 'New Signed Message (Mock Test)'; private readonly signedBy = 'B6BE3C4293DDCF66'; // could potentially grab this from test-secrets.json file - public test = async (mimeMsg: ParsedMail) => { + public test = async (parseResult: ParseMsgResult) => { + const mimeMsg = parseResult.mimeMsg; const keyInfo = await Config.getKeyInfo(["flowcrypt.compatibility.1pp1", "flowcrypt.compatibility.2pp1"]); const decrypted = await MsgUtil.decryptMessage({ kisWithPp: keyInfo!, encryptedData: Buf.fromUtfStr(mimeMsg.text!), verificationPubs: [] }); if (!decrypted.success) { @@ -116,7 +120,8 @@ class SignedMessageTestStrategy implements ITestMsgStrategy { class PlainTextMessageTestStrategy implements ITestMsgStrategy { private readonly expectedText = 'New Plain Message'; - public test = async (mimeMsg: ParsedMail) => { + public test = async (parseResult: ParseMsgResult) => { + const mimeMsg = parseResult.mimeMsg; if (!mimeMsg.text?.includes(this.expectedText)) { throw new HttpClientErr(`Error: Msg Text is not matching expected. Current: '${mimeMsg.text}', expected: '${this.expectedText}'`); } @@ -141,9 +146,9 @@ class IncludeQuotedPartTestStrategy implements ITestMsgStrategy { '> >> again double quote' ].join('\n'); - public test = async (mimeMsg: ParsedMail) => { + public test = async (parseResult: ParseMsgResult) => { const keyInfo = await Config.getKeyInfo(["flowcrypt.compatibility.1pp1", "flowcrypt.compatibility.2pp1"]); - const decrypted = await MsgUtil.decryptMessage({ kisWithPp: keyInfo!, encryptedData: Buf.fromUtfStr(mimeMsg.text!), verificationPubs: [] }); + const decrypted = await MsgUtil.decryptMessage({ kisWithPp: keyInfo!, encryptedData: Buf.fromUtfStr(parseResult.mimeMsg.text!), verificationPubs: [] }); if (!decrypted.success) { throw new HttpClientErr(`Error: can't decrypt message`); } @@ -155,7 +160,8 @@ class IncludeQuotedPartTestStrategy implements ITestMsgStrategy { } class NewMessageCCAndBCCTestStrategy implements ITestMsgStrategy { - public test = async (mimeMsg: ParsedMail) => { + public test = async (parseResult: ParseMsgResult) => { + const mimeMsg = parseResult.mimeMsg; const hasAtLeastOneRecipient = (ao: AddressObject[]) => ao && ao.length && ao[0].value && ao[0].value.length && ao[0].value[0].address; if (!hasAtLeastOneRecipient(parsedMailAddressObjectAsArray(mimeMsg.to))) { throw new HttpClientErr(`Error: There is no 'To' header.`, 400); @@ -170,7 +176,8 @@ class NewMessageCCAndBCCTestStrategy implements ITestMsgStrategy { } class SmimeEncryptedMessageStrategy implements ITestMsgStrategy { - public test = async (mimeMsg: ParsedMail) => { + public test = async (parseResult: ParseMsgResult) => { + const mimeMsg = parseResult.mimeMsg; expect((mimeMsg.headers.get('content-type') as StructuredHeader).value).to.equal('application/pkcs7-mime'); expect((mimeMsg.headers.get('content-type') as StructuredHeader).params.name).to.equal('smime.p7m'); expect((mimeMsg.headers.get('content-type') as StructuredHeader).params['smime-type']).to.equal('enveloped-data'); @@ -207,7 +214,8 @@ class SmimeEncryptedMessageStrategy implements ITestMsgStrategy { } class SmimeSignedMessageStrategy implements ITestMsgStrategy { - public test = async (mimeMsg: ParsedMail) => { + public test = async (parseResult: ParseMsgResult) => { + const mimeMsg = parseResult.mimeMsg; expect((mimeMsg.headers.get('content-type') as StructuredHeader).value).to.equal('application/pkcs7-mime'); expect((mimeMsg.headers.get('content-type') as StructuredHeader).params.name).to.equal('smime.p7m'); expect((mimeMsg.headers.get('content-type') as StructuredHeader).params['smime-type']).to.equal('signed-data'); @@ -265,7 +273,7 @@ export class TestBySubjectStrategyContext { } } - public test = async (mimeMsg: ParsedMail, base64Msg: string, id: string) => { - await this.strategy.test(mimeMsg, base64Msg, id); + public test = async (parseResult: ParseMsgResult, id: string) => { + await this.strategy.test(parseResult, id); }; } diff --git a/test/source/mock/google/strategies/strategy-base.ts b/test/source/mock/google/strategies/strategy-base.ts index 70abc7e3bab..9dfc61dbc5a 100644 --- a/test/source/mock/google/strategies/strategy-base.ts +++ b/test/source/mock/google/strategies/strategy-base.ts @@ -1,9 +1,9 @@ /* ©️ 2016 - present FlowCrypt a.s. Limitations apply. Contact human@flowcrypt.com */ -import { ParsedMail } from 'mailparser'; +import { ParseMsgResult } from '../../../util/parse'; export interface ITestMsgStrategy { - test(mimeMsg: ParsedMail, base64Msg: string, id: string): Promise; + test(parseResult: ParseMsgResult, id: string): Promise; } export class UnsuportableStrategyError extends Error { } diff --git a/test/source/tests/compose.ts b/test/source/tests/compose.ts index 2c3742b7336..ed52c335c8b 100644 --- a/test/source/tests/compose.ts +++ b/test/source/tests/compose.ts @@ -2,7 +2,7 @@ import * as ava from 'ava'; -import { BrowserHandle, Controllable, ControllablePage } from './../browser'; +import { BrowserHandle, Controllable, ControllableFrame, ControllablePage } from './../browser'; import { Config, Util } from './../util'; import { writeFileSync } from 'fs'; import { AvaContext } from './tooling'; @@ -1632,13 +1632,20 @@ export const defineComposeTests = (testVariant: TestVariant, testWithBrowser: Te expect(await PageRecipe.getElementPropertyJson(recipients[3], 'className')).to.equal('has_pgp'); const fileInput = await composePage.target.$('input[type=file]'); await fileInput!.uploadFile('test/samples/small.txt'); + await fileInput!.uploadFile('test/samples/small.pdf'); await composePage.waitAndType('@input-password', 'gO0d-pwd'); await composePage.waitAndClick('@action-send', { delay: 1 }); const attachmentsContainer = (await composePage.waitAny('@replied-attachments'))!; const attachments = await attachmentsContainer.$$('.pgp_attachment'); - expect(attachments.length).to.equal(6); // todo: + expect(attachments.length).to.equal(2); + await composePage.waitForContent('@replied-to', 'to: sender@domain.com, flowcrypt.compatibility@gmail.com, to@example.com, mock.only.pubkey@flowcrypt.com'); + const sentMsgs = (await GoogleData.withInitializedData(acct)).getMessagesByThread('1803be2e506153d2'); + expect(sentMsgs.length).to.equal(4); // 1 original + 3 newly sent const attachmentFrames = (composePage.target as Page).frames(); - expect(attachmentFrames.length).to.equal(7); // todo: + expect(attachmentFrames.length).to.equal(3); // 1 pgp block + 2 attachments + expect(await Promise.all(attachmentFrames.filter(f => f.url().includes('attachment.htm')).map(async (frame) => + await PageRecipe.getElementPropertyJson(await new ControllableFrame(frame).waitAny("@attachment-name"), 'textContent')))). + to.eql(['small.txt.pgp', 'small.pdf.pgp']); })); ava.default('first.key.revoked@key-manager-autoimport-no-prv-create.flowcrypt.test - selects valid own key when saving draft or sending', testWithBrowser(undefined, async (t, browser) => { From 4ae00f2c4b64ea9550261b49d00996b250d2220d Mon Sep 17 00:00:00 2001 From: Roman Shevchenko Date: Wed, 27 Apr 2022 12:58:31 +0000 Subject: [PATCH 11/45] removed extra test code --- test/source/mock/google/google-data.ts | 24 ++++++------------------ test/source/tests/compose.ts | 8 ++++---- test/source/tests/settings.ts | 4 ++-- 3 files changed, 12 insertions(+), 24 deletions(-) diff --git a/test/source/mock/google/google-data.ts b/test/source/mock/google/google-data.ts index 1d609d9002d..84726f2db4a 100644 --- a/test/source/mock/google/google-data.ts +++ b/test/source/mock/google/google-data.ts @@ -220,18 +220,6 @@ export class GoogleData { return DATA[this.acct].messages.find(m => m.id === id); }; - public getMessageBySubject = (subject: string): GmailMsg | undefined => { - return DATA[this.acct].messages.find(m => { - if (m.payload?.headers) { - const subjectHeader = m.payload.headers.find(x => x.name === 'Subject'); - if (subjectHeader) { - return subjectHeader.value.includes(subject); - } - } - return false; - }); - }; - public getMessagesAndDraftsByThread = (threadId: string) => { return this.getMessagesAndDrafts().filter(m => m.threadId === threadId); }; @@ -295,17 +283,17 @@ export class GoogleData { return threads; }; - // returns ordinary messages and drafts - private getMessagesAndDrafts = () => { - return DATA[this.acct].messages.concat(DATA[this.acct].drafts); - }; - - private searchMessagesBySubject = (subject: string) => { + public searchMessagesBySubject = (subject: string) => { subject = subject.trim().toLowerCase(); const messages = DATA[this.acct].messages.filter(m => GoogleData.msgSubject(m).toLowerCase().includes(subject)); return messages; }; + // returns ordinary messages and drafts + private getMessagesAndDrafts = () => { + return DATA[this.acct].messages.concat(DATA[this.acct].drafts); + }; + private searchMessagesByPeople = (includePeople: string[], excludePeople: string[]) => { includePeople = includePeople.map(person => person.trim().toLowerCase()); excludePeople = excludePeople.map(person => person.trim().toLowerCase()); diff --git a/test/source/tests/compose.ts b/test/source/tests/compose.ts index ed52c335c8b..2ad2ef658c7 100644 --- a/test/source/tests/compose.ts +++ b/test/source/tests/compose.ts @@ -1020,7 +1020,7 @@ export const defineComposeTests = (testVariant: TestVariant, testWithBrowser: Te await ComposePageRecipe.fillMsg(composePage, { to: 'flowcrypt.compatibility@gmail.com' }, subject, text, { sign: true, encrypt: true }); await ComposePageRecipe.sendAndClose(composePage); // get sent msg from mock - const sentMsg = (await GoogleData.withInitializedData('flowcrypt.compatibility@gmail.com')).getMessageBySubject(subject)!; + const sentMsg = (await GoogleData.withInitializedData('flowcrypt.compatibility@gmail.com')).searchMessagesBySubject(subject)[0]!; const message = sentMsg.payload!.body!.data!; const encrypted = message.match(/\-\-\-\-\-BEGIN PGP MESSAGE\-\-\-\-\-.*\-\-\-\-\-END PGP MESSAGE\-\-\-\-\-/s)![0]; const encryptedData = Buf.fromUtfStr(encrypted); @@ -1052,7 +1052,7 @@ export const defineComposeTests = (testVariant: TestVariant, testWithBrowser: Te await ComposePageRecipe.fillMsg(composePage, { to: 'human@flowcrypt.com' }, subject, undefined, { sign: true }); await ComposePageRecipe.sendAndClose(composePage); // get sent msg from mock - const sentMsg = (await GoogleData.withInitializedData(acct)).getMessageBySubject(subject)!; + const sentMsg = (await GoogleData.withInitializedData(acct)).searchMessagesBySubject(subject)[0]!; const message = sentMsg.payload!.body!.data!; expect(message).to.include('-----BEGIN PGP MESSAGE-----'); expect(message).to.include('-----END PGP MESSAGE-----'); @@ -1695,7 +1695,7 @@ const sendImgAndVerifyPresentInSentMsg = async (t: AvaContext, browser: BrowserH await composePage.page.evaluate((src: string) => { $('[data-test=action-insert-image]').val(src).click(); }, imgBase64); await ComposePageRecipe.sendAndClose(composePage); // get sent msg id from mock - const sentMsg = (await GoogleData.withInitializedData('flowcrypt.compatibility@gmail.com')).getMessageBySubject(subject)!; + const sentMsg = (await GoogleData.withInitializedData('flowcrypt.compatibility@gmail.com')).searchMessagesBySubject(subject)[0]!; if (sendingType === 'plain') { expect(sentMsg.payload?.body?.data).to.match(/Test Sending Plain Message With Image/); return; @@ -1727,7 +1727,7 @@ const sendTextAndVerifyPresentInSentMsg = async (t: AvaContext, await ComposePageRecipe.fillMsg(composePage, { to: 'human@flowcrypt.com' }, subject, text, sendingOpt); await ComposePageRecipe.sendAndClose(composePage); // get sent msg from mock - const sentMsg = (await GoogleData.withInitializedData('flowcrypt.compatibility@gmail.com')).getMessageBySubject(subject)!; + const sentMsg = (await GoogleData.withInitializedData('flowcrypt.compatibility@gmail.com')).searchMessagesBySubject(subject)[0]!; const message = encodeURIComponent(sentMsg.payload!.body!.data!); await BrowserRecipe.pgpBlockVerifyDecryptedContent(t, browser, { content: [text], diff --git a/test/source/tests/settings.ts b/test/source/tests/settings.ts index a29f9eaa212..002ae26fd34 100644 --- a/test/source/tests/settings.ts +++ b/test/source/tests/settings.ts @@ -633,7 +633,7 @@ export const defineSettingsTests = (testVariant: TestVariant, testWithBrowser: T expect(await backupPage.isDisabled('[data-id="515431151DDD3EA232B37A4C98ACFA1EADAB5B92"]')).to.equal(false); await backupPage.waitAndClick('@action-backup-step3manual-continue'); await backupPage.waitAndRespondToModal('info', 'confirm', 'Your private keys have been successfully backed up'); - const sentMsg = (await GoogleData.withInitializedData(acctEmail)).getMessageBySubject('Your FlowCrypt Backup')!; + const sentMsg = (await GoogleData.withInitializedData(acctEmail)).searchMessagesBySubject('Your FlowCrypt Backup')[0]!; const mimeMsg = await Parse.convertBase64ToMimeMsg(sentMsg.raw!); const { keys } = await KeyUtil.readMany(Buf.concat(mimeMsg.attachments.map(a => a.content))); expect(keys.length).to.equal(2); @@ -814,7 +814,7 @@ export const defineSettingsTests = (testVariant: TestVariant, testWithBrowser: T '&type=openpgp&id=515431151DDD3EA232B37A4C98ACFA1EADAB5B92&idToken=fakeheader.01')); await backupPage.waitAndClick('@action-backup-step3manual-continue'); await backupPage.waitAll('@action-step4done-account-settings'); - const sentMsg = (await GoogleData.withInitializedData(acctEmail)).getMessageBySubject('Your FlowCrypt Backup')!; + const sentMsg = (await GoogleData.withInitializedData(acctEmail)).searchMessagesBySubject('Your FlowCrypt Backup')[0]!; const mimeMsg = await Parse.convertBase64ToMimeMsg(sentMsg.raw!); const { keys } = await KeyUtil.readMany(new Buf(mimeMsg.attachments[0]!.content!)); expect(keys.length).to.equal(1); From 6d6e241a23be00699b854492d71424ac8f955f96 Mon Sep 17 00:00:00 2001 From: Roman Shevchenko Date: Tue, 10 May 2022 07:55:20 +0000 Subject: [PATCH 12/45] use Reply-To to reply to PWD recipients from pubkey-encrypted message --- .../compose-modules/compose-draft-module.ts | 4 +- .../compose-modules/compose-err-module.ts | 2 +- .../compose-modules/compose-input-module.ts | 4 +- .../compose-send-btn-module.ts | 25 +---------- .../compose-modules/compose-sender-module.ts | 24 +++++++++++ .../elements/compose-modules/compose-types.ts | 2 +- .../formatters/base-mail-formatter.ts | 3 +- .../encrypted-mail-msg-formatter.ts | 43 +++++++++++++------ .../formatters/general-mail-formatter.ts | 4 +- .../common/api/email-provider/sendable-msg.ts | 15 +++++-- extension/js/common/core/common.ts | 15 +++++-- .../message-export-1803be3182d1937b.json | 2 +- .../strategies/send-message-strategy.ts | 41 +++++++++++++++++- test/source/tests/compose.ts | 6 +++ 14 files changed, 136 insertions(+), 54 deletions(-) diff --git a/extension/chrome/elements/compose-modules/compose-draft-module.ts b/extension/chrome/elements/compose-modules/compose-draft-module.ts index d0e914c5038..57a8b43b54a 100644 --- a/extension/chrome/elements/compose-modules/compose-draft-module.ts +++ b/extension/chrome/elements/compose-modules/compose-draft-module.ts @@ -119,8 +119,8 @@ export class ComposeDraftModule extends ViewModule { if (this.hasBodyChanged(this.view.inputModule.squire.getHTML()) || this.hasSubjectChanged(String(this.view.S.cached('input_subject').val())) || forceSave) { this.currentlySavingDraft = true; try { - const msgData = this.view.inputModule.extractAll(); - const { pubkeys } = await this.view.storageModule.collectSingleFamilyKeys([], msgData.from, true); + const msgData = await this.view.inputModule.extractAll(); + const { pubkeys } = await this.view.storageModule.collectSingleFamilyKeys([], msgData.from.email, true); // collectSingleFamilyKeys filters out bad keys, but only if there are any good keys available // if no good keys available, it leaves bad keys so we can explain the issue here if (pubkeys.some(pub => pub.pubkey.expiration && pub.pubkey.expiration < Date.now())) { diff --git a/extension/chrome/elements/compose-modules/compose-err-module.ts b/extension/chrome/elements/compose-modules/compose-err-module.ts index 2973a6ea15f..a644ed5e78a 100644 --- a/extension/chrome/elements/compose-modules/compose-err-module.ts +++ b/extension/chrome/elements/compose-modules/compose-err-module.ts @@ -126,7 +126,7 @@ export class ComposeErrModule extends ViewModule { if (!subject && ! await Ui.modal.confirm('Send without a subject?')) { throw new ComposerResetBtnTrigger(); } - let footer = await this.view.footerModule.getFooterFromStorage(from); + let footer = await this.view.footerModule.getFooterFromStorage(from.email); if (footer) { // format footer the way it would be in outgoing plaintext footer = Xss.htmlUnescape(Xss.htmlSanitizeAndStripAllTags(this.view.footerModule.createFooterHtml(footer), '\n')).trim(); } diff --git a/extension/chrome/elements/compose-modules/compose-input-module.ts b/extension/chrome/elements/compose-modules/compose-input-module.ts index 4e297fe7192..fb56e960400 100644 --- a/extension/chrome/elements/compose-modules/compose-input-module.ts +++ b/extension/chrome/elements/compose-modules/compose-input-module.ts @@ -60,14 +60,14 @@ export class ComposeInputModule extends ViewModule { return this.view.S.cached('fineuploader').find('.qq-upload-file').toArray().map((el) => $(el).text().trim()); }; - public extractAll = (): NewMsgData => { + public extractAll = async (): Promise => { const recipients = this.mapRecipients(this.view.recipientsModule.getValidRecipients()); const subject = this.view.isReplyBox && this.view.replyParams ? this.view.replyParams.subject : String($('#input_subject').val() || ''); const plaintext = this.view.inputModule.extract('text', 'input_text'); const plainhtml = this.view.inputModule.extract('html', 'input_text'); const password = this.view.S.cached('input_password').val(); const pwd = typeof password === 'string' && password ? password : undefined; - const from = this.view.senderModule.getSender(); + const from = await this.view.senderModule.getEmailWithOptionalName(this.view.senderModule.getSender()); return { recipients, subject, plaintext, plainhtml, pwd, from }; }; diff --git a/extension/chrome/elements/compose-modules/compose-send-btn-module.ts b/extension/chrome/elements/compose-modules/compose-send-btn-module.ts index c6d27b86d07..f39f9be86a1 100644 --- a/extension/chrome/elements/compose-modules/compose-send-btn-module.ts +++ b/extension/chrome/elements/compose-modules/compose-send-btn-module.ts @@ -21,8 +21,6 @@ import { Xss } from '../../../js/common/platform/xss.js'; import { ViewModule } from '../../../js/common/view-module.js'; import { ComposeView } from '../compose.js'; import { ContactStore } from '../../../js/common/platform/store/contact-store.js'; -import { AcctStore } from '../../../js/common/platform/store/acct-store.js'; -import { Str } from '../../../js/common/core/common.js'; export class ComposeSendBtnModule extends ViewModule { @@ -110,7 +108,7 @@ export class ComposeSendBtnModule extends ViewModule { this.view.S.now('send_btn_text').text('Loading...'); Xss.sanitizeRender(this.view.S.now('send_btn_i'), Ui.spinner('white')); this.view.S.cached('send_btn_note').text(''); - const newMsgData = this.view.inputModule.extractAll(); + const newMsgData = await this.view.inputModule.extractAll(); await this.view.errModule.throwIfFormValsInvalid(newMsgData); const emails = getUniqueRecipientEmails(newMsgData.recipients); await ContactStore.update(undefined, emails, { lastUse: Date.now() }); @@ -147,7 +145,6 @@ export class ComposeSendBtnModule extends ViewModule { if (this.view.myPubkeyModule.shouldAttach() && senderKi) { // todo: report on undefined? msg.attachments.push(Attachment.keyinfoAsPubkeyAttachment(senderKi)); } - msg.from = await this.formatSenderEmailAsMimeString(msg.from); }; private extractInlineImagesToAttachments = (html: string) => { @@ -241,24 +238,4 @@ export class ComposeSendBtnModule extends ViewModule { } }; - private formatSenderEmailAsMimeString = async (email: string): Promise => { - const parsedEmail = Str.parseEmail(email); - if (!parsedEmail.email) { - throw new Error(`Recipient email ${email} is not valid`); - } - if (parsedEmail.name) { - return Str.formatEmailWithOptionalName({ email: parsedEmail.email, name: parsedEmail.name }); - } - const { sendAs } = await AcctStore.get(this.view.acctEmail, ['sendAs']); - let name: string | undefined; - if (sendAs && sendAs[email]?.name) { - name = sendAs[email].name!; - } else { - const contactWithPubKeys = await ContactStore.getOneWithAllPubkeys(undefined, email); - if (contactWithPubKeys && contactWithPubKeys.info.name) { - name = contactWithPubKeys.info.name; - } - } - return Str.formatEmailWithOptionalName({ email: parsedEmail.email, name }); - }; } diff --git a/extension/chrome/elements/compose-modules/compose-sender-module.ts b/extension/chrome/elements/compose-modules/compose-sender-module.ts index 0390b61bde9..e2e7af21d21 100644 --- a/extension/chrome/elements/compose-modules/compose-sender-module.ts +++ b/extension/chrome/elements/compose-modules/compose-sender-module.ts @@ -9,6 +9,8 @@ import { Xss } from '../../../js/common/platform/xss.js'; import { ViewModule } from '../../../js/common/view-module.js'; import { ComposeView } from '../compose.js'; import { AcctStore } from '../../../js/common/platform/store/acct-store.js'; +import { EmailParts, Str } from '../../../js/common/core/common.js'; +import { ContactStore } from '../../../js/common/platform/store/contact-store.js'; export class ComposeSenderModule extends ViewModule { @@ -22,6 +24,28 @@ export class ComposeSenderModule extends ViewModule { return this.view.acctEmail; }; + // searches in AcctStore and ContactStore to find name for this email (if it is missing in MIME string) + public getEmailWithOptionalName = async (emailInMimeFormat: string): Promise => { + const parsedEmail = Str.parseEmail(emailInMimeFormat); + if (!parsedEmail.email) { + throw new Error(`Recipient email ${emailInMimeFormat} is not valid`); + } + if (parsedEmail.name) { + return { email: parsedEmail.email, name: parsedEmail.name }; + } + const { sendAs } = await AcctStore.get(this.view.acctEmail, ['sendAs']); + let name: string | undefined; + if (sendAs && sendAs[parsedEmail.email]?.name) { + name = sendAs[parsedEmail.email].name!; + } else { + const contactWithPubKeys = await ContactStore.getOneWithAllPubkeys(undefined, parsedEmail.email); + if (contactWithPubKeys && contactWithPubKeys.info.name) { + name = contactWithPubKeys.info.name; + } + } + return { email: parsedEmail.email, name }; + }; + public renderSendFromOrChevron = async () => { if (this.view.isReplyBox) { const { sendAs } = await AcctStore.get(this.view.acctEmail, ['sendAs']); diff --git a/extension/chrome/elements/compose-modules/compose-types.ts b/extension/chrome/elements/compose-modules/compose-types.ts index 49a1b57935a..3ac37d8ddb7 100644 --- a/extension/chrome/elements/compose-modules/compose-types.ts +++ b/extension/chrome/elements/compose-modules/compose-types.ts @@ -51,7 +51,7 @@ export type CollectKeysResult = { pubkeys: PubkeyResult[], emailsWithoutPubkeys: export type PopoverOpt = 'encrypt' | 'sign' | 'richtext'; export type PopoverChoices = { [key in PopoverOpt]: boolean }; -export type NewMsgData = { recipients: ParsedRecipients, subject: string, plaintext: string, plainhtml: string, pwd: string | undefined, from: string }; +export type NewMsgData = { recipients: ParsedRecipients, subject: string, plaintext: string, plainhtml: string, pwd: string | undefined, from: EmailParts, replyTo?: string }; export class SendBtnTexts { public static readonly BTN_ENCRYPT_AND_SEND: string = "Encrypt and Send"; diff --git a/extension/chrome/elements/compose-modules/formatters/base-mail-formatter.ts b/extension/chrome/elements/compose-modules/formatters/base-mail-formatter.ts index a485b1e9ca5..8448bccfe20 100644 --- a/extension/chrome/elements/compose-modules/formatters/base-mail-formatter.ts +++ b/extension/chrome/elements/compose-modules/formatters/base-mail-formatter.ts @@ -8,6 +8,7 @@ import { Key } from '../../../../js/common/core/crypto/key.js'; import { SmimeKey } from '../../../../js/common/core/crypto/smime/smime-key.js'; import { SendableMsg } from '../../../../js/common/api/email-provider/sendable-msg.js'; import { Buf } from '../../../../js/common/core/buf.js'; +import { Str } from '../../../../js/common/core/common.js'; export class BaseMailFormatter { @@ -22,7 +23,7 @@ export class BaseMailFormatter { } protected headers = (newMsg: NewMsgData) => { - return { from: newMsg.from, recipients: newMsg.recipients, subject: newMsg.subject, thread: this.view.threadId }; + return { from: Str.formatEmailWithOptionalName(newMsg.from), replyTo: newMsg.replyTo, recipients: newMsg.recipients, subject: newMsg.subject, thread: this.view.threadId }; }; protected signMimeMessage = async (signingPrv: Key, mimeEncodedMessage: string, newMsg: NewMsgData) => { diff --git a/extension/chrome/elements/compose-modules/formatters/encrypted-mail-msg-formatter.ts b/extension/chrome/elements/compose-modules/formatters/encrypted-mail-msg-formatter.ts index 2570bb47d83..91d28ad4bd9 100644 --- a/extension/chrome/elements/compose-modules/formatters/encrypted-mail-msg-formatter.ts +++ b/extension/chrome/elements/compose-modules/formatters/encrypted-mail-msg-formatter.ts @@ -6,7 +6,7 @@ import { BaseMailFormatter } from './base-mail-formatter.js'; import { ComposerResetBtnTrigger } from '../compose-err-module.js'; import { Mime, SendableMsgBody } from '../../../../js/common/core/mime.js'; import { getUniqueRecipientEmails, NewMsgData } from '../compose-types.js'; -import { Str, Value } from '../../../../js/common/core/common.js'; +import { EmailParts, Str, Value } from '../../../../js/common/core/common.js'; import { ApiErr } from '../../../../js/common/api/shared/api-error.js'; import { Attachment } from '../../../../js/common/core/attachment.js'; import { Buf } from '../../../../js/common/core/buf.js'; @@ -25,6 +25,7 @@ import { PgpHash } from '../../../../js/common/core/crypto/pgp/pgp-hash.js'; import { UploadedMessageData } from '../../../../js/common/api/account-server.js'; import { ParsedKeyInfo } from '../../../../js/common/core/crypto/key-store-util.js'; import { MultipleMessages } from './general-mail-formatter.js'; +import { RecipientType } from '../../../../js/common/api/shared/api.js'; export class EncryptedMsgMailFormatter extends BaseMailFormatter { @@ -38,9 +39,30 @@ export class EncryptedMsgMailFormatter extends BaseMailFormatter { newMsg.pwd = undefined; const collectedAttachments = await this.view.attachmentsModule.attachment.collectEncryptAttachments(pubkeys); // we always have at least one pubkey, this is the main message, encrypted for pubkeys + const pubkeyRecipients: { [type in RecipientType]?: EmailParts[] } = {}; + for (const [key, value] of Object.entries(newMsg.recipients)) { + if (['to', 'cc', 'bcc'].includes(key)) { + const sendingType = key as RecipientType; + pubkeyRecipients[sendingType] = value?.filter(emailPart => pubkeys.some(p => p.email === emailPart.email)); + } + } + const uniquePubkeyRecipientEmails = Value.arr.unique((pubkeyRecipients.to || []).concat(pubkeyRecipients.cc || []) + .map(recipient => recipient.email.toLowerCase())); + // pubkey recipients should be able to reply to "to" and "cc" pwd recipients + const replyTo = (newMsg.recipients.to ?? []).concat(newMsg.recipients.cc ?? []) + .filter(recipient => !uniquePubkeyRecipientEmails.includes(recipient.email.toLowerCase())); + const pubkeyMsgData = { + ...newMsg, + recipients: pubkeyRecipients, + // brackets are required for test emails like '@test:8001' + replyTo: replyTo.length ? `${Str.formatEmailList([newMsg.from, ...replyTo], true)}` : undefined + }; + if (!uniquePubkeyRecipientEmails.length) { + // todo: add myself? + } const msgs: SendableMsg[] = [ await this.sendablePwdMsg( - newMsg, + pubkeyMsgData, pubkeys, { msgUrl: uploadedMessageData.url, externalId: uploadedMessageData.externalId }, collectedAttachments, @@ -49,7 +71,10 @@ export class EncryptedMsgMailFormatter extends BaseMailFormatter { // adding individual messages for each recipient that doesn't have a pubkey for (const recipientEmail of Object.keys(uploadedMessageData.emailToExternalIdAndUrl ?? {}).filter(email => !pubkeys.some(p => p.email === email))) { const { url, externalId } = uploadedMessageData.emailToExternalIdAndUrl![recipientEmail]; - msgs.push(await this.sendablePwdMsg(newMsg, pubkeys, { recipientEmail, msgUrl: url, externalId }, [], signingKey?.key)); + const foundParsedRecipient = (newMsg.recipients.to ?? []).concat(newMsg.recipients.cc ?? []).concat(newMsg.recipients.bcc ?? []). + find(r => r.email.toLowerCase() === recipientEmail.toLowerCase()); + const individualMsgData = { ...newMsg, recipients: { to: [foundParsedRecipient ?? { email: recipientEmail }] } }; + msgs.push(await this.sendablePwdMsg(individualMsgData, pubkeys, { msgUrl: url, externalId }, [], signingKey?.key)); } return { senderKi: signingKey?.keyInfo, msgs, recipients: newMsg.recipients, attachments: collectedAttachments }; } else { @@ -111,7 +136,7 @@ export class EncryptedMsgMailFormatter extends BaseMailFormatter { authInfo.uuid ? authInfo : undefined, pwdEncryptedWithAttachments, replyToken, - newMsg.from, + newMsg.from.email, // todo: Str.formatEmailWithOptionalName? newMsg.recipients, (p) => this.view.sendBtnModule.renderUploadProgress(p, 'FIRST-HALF'), // still need to upload to Gmail later, this request represents first half of progress ); @@ -120,7 +145,7 @@ export class EncryptedMsgMailFormatter extends BaseMailFormatter { private sendablePwdMsg = async ( newMsg: NewMsgData, pubs: PubkeyResult[], - { msgUrl, externalId, recipientEmail }: { msgUrl: string, externalId?: string, recipientEmail?: string }, + { msgUrl, externalId }: { msgUrl: string, externalId?: string }, attachments: Attachment[], signingPrv?: Key) => { // encoded as: PGP/MIME-like structure but with attachments as external files due to email size limit (encrypted for pubkeys only) @@ -129,12 +154,6 @@ export class EncryptedMsgMailFormatter extends BaseMailFormatter { const { data: pubEncryptedNoAttachments } = await this.encryptDataArmor(Buf.fromUtfStr(pgpMimeNoAttachments), undefined, pubs, signingPrv); // encrypted only for pubs const emailIntroAndLinkBody = await this.formatPwdEncryptedMsgBodyLink(msgUrl); const headers = this.headers(newMsg); - if (recipientEmail) { - // we will send this copy of the message only to this recipient - const foundParsedRecipient = (headers.recipients.to ?? []).concat(headers.recipients.cc ?? []).concat(headers.recipients.bcc ?? []). - find(r => r.email.toLowerCase() === recipientEmail.toLowerCase()); - headers.recipients = { to: [foundParsedRecipient ?? { email: recipientEmail }] }; - } return await SendableMsg.createPwdMsg(this.acctEmail, headers, emailIntroAndLinkBody, this.createPgpMimeAttachments(pubEncryptedNoAttachments).concat(attachments), { isDraft: this.isDraft, externalId }); @@ -206,7 +225,7 @@ export class EncryptedMsgMailFormatter extends BaseMailFormatter { 'class': 'cryptup_reply', 'cryptup-data': Str.htmlAttrEncode({ sender: newMsgData.from, - recipient: Value.arr.withoutVal(Value.arr.withoutVal(recipients, newMsgData.from), this.acctEmail), + recipient: Value.arr.withoutVal(Value.arr.withoutVal(recipients, newMsgData.from.email), this.acctEmail), subject: newMsgData.subject, token: response.replyToken, }) diff --git a/extension/chrome/elements/compose-modules/formatters/general-mail-formatter.ts b/extension/chrome/elements/compose-modules/formatters/general-mail-formatter.ts index 673c436643d..7ab3736621d 100644 --- a/extension/chrome/elements/compose-modules/formatters/general-mail-formatter.ts +++ b/extension/chrome/elements/compose-modules/formatters/general-mail-formatter.ts @@ -34,7 +34,7 @@ export class GeneralMailFormatter { } if (!choices.encrypt && choices.sign) { // sign only view.S.now('send_btn_text').text('Signing...'); - const senderKis = await view.storageModule.getAccountKeys(newMsgData.from); + const senderKis = await view.storageModule.getAccountKeys(newMsgData.from.email); const signingKey = await GeneralMailFormatter.chooseSigningKeyAndDecryptIt(view, senderKis); if (!signingKey) { throw new UnreportableError('Could not find account key usable for signing this plain text message'); @@ -43,7 +43,7 @@ export class GeneralMailFormatter { return { senderKi: signingKey!.keyInfo, msgs: [msg], recipients: msg.recipients, attachments: msg.attachments }; } // encrypt (optionally sign) - const singleFamilyKeys = await view.storageModule.collectSingleFamilyKeys(recipientsEmails, newMsgData.from, choices.sign); + const singleFamilyKeys = await view.storageModule.collectSingleFamilyKeys(recipientsEmails, newMsgData.from.email, choices.sign); if (singleFamilyKeys.emailsWithoutPubkeys.length) { await view.errModule.throwIfEncryptionPasswordInvalid(newMsgData); } diff --git a/extension/js/common/api/email-provider/sendable-msg.ts b/extension/js/common/api/email-provider/sendable-msg.ts index 3db6e0ce1ec..8eb0fd25922 100644 --- a/extension/js/common/api/email-provider/sendable-msg.ts +++ b/extension/js/common/api/email-provider/sendable-msg.ts @@ -13,6 +13,7 @@ import { ParsedRecipients } from './email-provider-api.js'; type SendableMsgHeaders = { headers?: Dict; from: string; + replyTo?: string; recipients: ParsedRecipients; subject: string; thread?: string; @@ -88,12 +89,15 @@ export class SendableMsg { attachments: Attachment[], options: SendableMsgOptions ): Promise => { - const { from, recipients, subject, thread } = headers; + const { from, replyTo, recipients, subject, thread } = headers; const { type, isDraft, externalId } = options; - return await SendableMsg.create(acctEmail, { from, recipients, subject, thread, body, attachments, type, isDraft, externalId }); + return await SendableMsg.create(acctEmail, { from, replyTo, recipients, subject, thread, body, attachments, type, isDraft, externalId }); }; - private static create = async (acctEmail: string, { from, recipients, subject, thread, body, attachments, type, isDraft, externalId }: SendableMsgDefinition): Promise => { + private static create = async ( + acctEmail: string, + { from, replyTo, recipients, subject, thread, body, attachments, type, isDraft, externalId }: SendableMsgDefinition + ): Promise => { const mostUsefulPrv = KeyStoreUtil.chooseMostUseful( await KeyStoreUtil.parse(await KeyStore.getRequired(acctEmail)), 'EVEN-IF-UNUSABLE' @@ -107,6 +111,7 @@ export class SendableMsg { headers, isDraft === true, from, + replyTo, recipients, subject, body || {}, @@ -122,6 +127,7 @@ export class SendableMsg { public headers: Dict, isDraft: boolean, public from: string, + public replyTo: string | undefined, public recipients: ParsedRecipients, public subject: string, public body: SendableMsgBody, @@ -142,6 +148,9 @@ export class SendableMsg { public toMime = async () => { this.headers.From = this.from; + if (this.replyTo) { + this.headers['Reply-To'] = this.replyTo; + } for (const [recipientType, value] of Object.entries(this.recipients)) { if (value && value!.length) { // todo - properly escape/encode this header using emailjs diff --git a/extension/js/common/core/common.ts b/extension/js/common/core/common.ts index fd3f93496f7..a3f0e68f5f7 100644 --- a/extension/js/common/core/common.ts +++ b/extension/js/common/core/common.ts @@ -48,12 +48,12 @@ export class Str { return str.replace(/[.~!$%^*=?]/gi, ''); }; - public static formatEmailWithOptionalName = ({ email, name }: EmailParts): string => { - return name ? `${Str.rmSpecialCharsKeepUtf(name, 'ALLOW-SOME')} <${email}>` : email; + public static formatEmailWithOptionalName = (emailParts: EmailParts): string => { + return Str.formatEmailWithOptionalNameEx(emailParts); }; - public static formatEmailList = (list: EmailParts[]): string => { - return list.map(Str.formatEmailWithOptionalName).join(', '); + public static formatEmailList = (list: EmailParts[], forceBrackets?: boolean): string => { + return list.map(x => Str.formatEmailWithOptionalNameEx(x, forceBrackets)).join(', '); }; public static prettyPrint = (obj: any) => { @@ -174,6 +174,13 @@ export class Str { return rtlCount > lrtCount; }; + private static formatEmailWithOptionalNameEx = ({ email, name }: EmailParts, forceBrackets?: boolean): string => { + if (name) { + return `${Str.rmSpecialCharsKeepUtf(name, 'ALLOW-SOME')} <${email}>`; + } + return forceBrackets ? `<${email}>` : email; + }; + private static base64urlUtfEncode = (str: string) => { // https://stackoverflow.com/questions/30106476/using-javascripts-atob-to-decode-base64-doesnt-properly-decode-utf-8-strings if (typeof str === 'undefined') { diff --git a/test/source/mock/google/exported-messages/message-export-1803be3182d1937b.json b/test/source/mock/google/exported-messages/message-export-1803be3182d1937b.json index 474a54226ed..1f85a37dd1f 100644 --- a/test/source/mock/google/exported-messages/message-export-1803be3182d1937b.json +++ b/test/source/mock/google/exported-messages/message-export-1803be3182d1937b.json @@ -32,7 +32,7 @@ }, { "name": "To", - "value": "flowcrypt.Compatibility@gmail.com, to@example.com, mock.only.puBkey@flowcrypt.com" + "value": "flowcrypt.Compatibility@gmail.com, Mr To , mock.only.puBkey@flowcrypt.com" }, { "name": "Content-Type", diff --git a/test/source/mock/google/strategies/send-message-strategy.ts b/test/source/mock/google/strategies/send-message-strategy.ts index a6aa924e1ce..6407f336f81 100644 --- a/test/source/mock/google/strategies/send-message-strategy.ts +++ b/test/source/mock/google/strategies/send-message-strategy.ts @@ -76,6 +76,45 @@ class PwdEncryptedMessageWithFesIdTokenTestStrategy implements ITestMsgStrategy }; } +class PwdEncryptedMessageWithFesIdReplyRenderingTestStrategy implements ITestMsgStrategy { + public test = async (parseResult: ParseMsgResult, id: string) => { + const mimeMsg = parseResult.mimeMsg; + const expectedSenderEmail = 'user2@standardsubdomainfes.test:8001'; + expect(mimeMsg.from!.text).to.equal(`First Last <${expectedSenderEmail}>`); + if (!mimeMsg.text?.includes(`${expectedSenderEmail} has sent you a password-encrypted email`)) { + throw new HttpClientErr(`Error checking sent text in:\n\n${mimeMsg.text}`); + } + if (mimeMsg.text?.includes('http://fes.standardsubdomainfes.test:8001/message/FES-MOCK-MESSAGE-ID')) { + // this is a message to pubkey recipients + expect((mimeMsg.to as AddressObject).text).to.equal('flowcrypt.compatibility@gmail.com, mock.only.pubkey@flowcrypt.com'); + // tslint:disable-next-line:no-unused-expression + expect(mimeMsg.cc).to.be.an.undefined; // eslint-disable-line no-unused-expressions + expect(mimeMsg.bcc).to.be.an.undefined; // eslint-disable-line no-unused-expressions + expect((mimeMsg.headers.get('reply-to') as AddressObject).text).to.equal('First Last , sender@domain.com, to@example.com'); + } else if (mimeMsg.text?.includes('http://fes.standardsubdomainfes.test:8001/message/FES-MOCK-MESSAGE-FOR-SENDER@DOMAIN.COM-ID')) { + expect((mimeMsg.to as AddressObject).text).to.equal('sender@domain.com'); + // tslint:disable-next-line:no-unused-expression + expect(mimeMsg.cc).to.be.an.undefined; // eslint-disable-line no-unused-expressions + // tslint:disable-next-line:no-unused-expression + expect(mimeMsg.bcc).to.be.an.undefined; // eslint-disable-line no-unused-expressions + expect(mimeMsg.headers.get('reply-to')).to.be.an.undefined; // eslint-disable-line no-unused-expressions + } else if (mimeMsg.text?.includes('http://fes.standardsubdomainfes.test:8001/message/FES-MOCK-MESSAGE-FOR-TO@EXAMPLE.COM-ID')) { + expect((mimeMsg.to as AddressObject).text).to.equal('to@example.com'); + // tslint:disable-next-line:no-unused-expression + expect(mimeMsg.cc).to.be.an.undefined; // eslint-disable-line no-unused-expressions + // tslint:disable-next-line:no-unused-expression + expect(mimeMsg.bcc).to.be.an.undefined; // eslint-disable-line no-unused-expressions + expect(mimeMsg.headers.get('reply-to')).to.be.an.undefined; // eslint-disable-line no-unused-expressions + } else { + throw new HttpClientErr(`Error: cannot find pwd encrypted FES link in:\n\n${mimeMsg.text}`); + } + if (!mimeMsg.text?.includes('Follow this link to open it')) { + throw new HttpClientErr(`Error: cannot find pwd encrypted open link prompt in ${mimeMsg.text}`); + } + await new SaveMessageInStorageStrategy().test(parseResult, id); + }; +} + class MessageWithFooterTestStrategy implements ITestMsgStrategy { private readonly footer = 'flowcrypt.compatibility test footer with an img'; @@ -251,7 +290,7 @@ export class TestBySubjectStrategyContext { } else if (subject.includes('PWD encrypted message with FES - ID TOKEN')) { this.strategy = new PwdEncryptedMessageWithFesIdTokenTestStrategy(); } else if (subject.includes('PWD encrypted message with FES - Reply rendering')) { - this.strategy = new SaveMessageInStorageStrategy(); + this.strategy = new PwdEncryptedMessageWithFesIdReplyRenderingTestStrategy(); } else if (subject.includes('Message With Image')) { this.strategy = new SaveMessageInStorageStrategy(); } else if (subject.includes('Message With Test Text')) { diff --git a/test/source/tests/compose.ts b/test/source/tests/compose.ts index 2ad2ef658c7..9fceb81a4ce 100644 --- a/test/source/tests/compose.ts +++ b/test/source/tests/compose.ts @@ -1601,6 +1601,12 @@ export const defineComposeTests = (testVariant: TestVariant, testWithBrowser: Te await composePage.waitAndType('@input-password', 'gO0d-pwd'); await composePage.waitAndClick('@action-send', { delay: 1 }); await ComposePageRecipe.closed(composePage); + const sentMsgs = (await GoogleData.withInitializedData(acct)).searchMessagesBySubject(subject); + for (const msg of sentMsgs) { + const message = msg.payload!.body!.data!; + console.log(message); + } + expect(sentMsgs.length).to.equal(2); // this test is using PwdEncryptedMessageWithFesIdTokenTestStrategy to check sent result based on subject "PWD encrypted message with flowcrypt.com/api" // also see '/api/v1/message' in fes-endpoints.ts mock })); From dc88b3e5fe7195b6c0bb44633d98553e163e0e9c Mon Sep 17 00:00:00 2001 From: Roman Shevchenko Date: Tue, 10 May 2022 17:12:49 +0000 Subject: [PATCH 13/45] lint fix --- test/source/mock/google/strategies/send-message-strategy.ts | 3 +++ 1 file changed, 3 insertions(+) diff --git a/test/source/mock/google/strategies/send-message-strategy.ts b/test/source/mock/google/strategies/send-message-strategy.ts index 6407f336f81..23005ac9d06 100644 --- a/test/source/mock/google/strategies/send-message-strategy.ts +++ b/test/source/mock/google/strategies/send-message-strategy.ts @@ -89,6 +89,7 @@ class PwdEncryptedMessageWithFesIdReplyRenderingTestStrategy implements ITestMsg expect((mimeMsg.to as AddressObject).text).to.equal('flowcrypt.compatibility@gmail.com, mock.only.pubkey@flowcrypt.com'); // tslint:disable-next-line:no-unused-expression expect(mimeMsg.cc).to.be.an.undefined; // eslint-disable-line no-unused-expressions + // tslint:disable-next-line:no-unused-expression expect(mimeMsg.bcc).to.be.an.undefined; // eslint-disable-line no-unused-expressions expect((mimeMsg.headers.get('reply-to') as AddressObject).text).to.equal('First Last , sender@domain.com, to@example.com'); } else if (mimeMsg.text?.includes('http://fes.standardsubdomainfes.test:8001/message/FES-MOCK-MESSAGE-FOR-SENDER@DOMAIN.COM-ID')) { @@ -97,6 +98,7 @@ class PwdEncryptedMessageWithFesIdReplyRenderingTestStrategy implements ITestMsg expect(mimeMsg.cc).to.be.an.undefined; // eslint-disable-line no-unused-expressions // tslint:disable-next-line:no-unused-expression expect(mimeMsg.bcc).to.be.an.undefined; // eslint-disable-line no-unused-expressions + // tslint:disable-next-line:no-unused-expression expect(mimeMsg.headers.get('reply-to')).to.be.an.undefined; // eslint-disable-line no-unused-expressions } else if (mimeMsg.text?.includes('http://fes.standardsubdomainfes.test:8001/message/FES-MOCK-MESSAGE-FOR-TO@EXAMPLE.COM-ID')) { expect((mimeMsg.to as AddressObject).text).to.equal('to@example.com'); @@ -104,6 +106,7 @@ class PwdEncryptedMessageWithFesIdReplyRenderingTestStrategy implements ITestMsg expect(mimeMsg.cc).to.be.an.undefined; // eslint-disable-line no-unused-expressions // tslint:disable-next-line:no-unused-expression expect(mimeMsg.bcc).to.be.an.undefined; // eslint-disable-line no-unused-expressions + // tslint:disable-next-line:no-unused-expression expect(mimeMsg.headers.get('reply-to')).to.be.an.undefined; // eslint-disable-line no-unused-expressions } else { throw new HttpClientErr(`Error: cannot find pwd encrypted FES link in:\n\n${mimeMsg.text}`); From 66d0ebb90c09f7fdd7782806f87f2e8c49294c80 Mon Sep 17 00:00:00 2001 From: Roman Shevchenko Date: Sat, 14 May 2022 13:50:19 +0000 Subject: [PATCH 14/45] support for pwd-encrypted messages with no pubkey recipients --- .../encrypted-mail-msg-formatter.ts | 29 +++++++------- .../common/api/email-provider/sendable-msg.ts | 3 +- test/source/mock/fes/fes-endpoints.ts | 27 +++++++++++++ test/source/mock/google/google-endpoints.ts | 6 +-- .../strategies/send-message-strategy.ts | 40 ++++++++++++++++++- test/source/tests/compose.ts | 30 +++++++++++++- 6 files changed, 112 insertions(+), 23 deletions(-) diff --git a/extension/chrome/elements/compose-modules/formatters/encrypted-mail-msg-formatter.ts b/extension/chrome/elements/compose-modules/formatters/encrypted-mail-msg-formatter.ts index 91d28ad4bd9..febf1e8228d 100644 --- a/extension/chrome/elements/compose-modules/formatters/encrypted-mail-msg-formatter.ts +++ b/extension/chrome/elements/compose-modules/formatters/encrypted-mail-msg-formatter.ts @@ -38,7 +38,6 @@ export class EncryptedMsgMailFormatter extends BaseMailFormatter { const uploadedMessageData = await this.prepareAndUploadPwdEncryptedMsg(newMsg); // encrypted for pwd only, pubkeys ignored newMsg.pwd = undefined; const collectedAttachments = await this.view.attachmentsModule.attachment.collectEncryptAttachments(pubkeys); - // we always have at least one pubkey, this is the main message, encrypted for pubkeys const pubkeyRecipients: { [type in RecipientType]?: EmailParts[] } = {}; for (const [key, value] of Object.entries(newMsg.recipients)) { if (['to', 'cc', 'bcc'].includes(key)) { @@ -46,33 +45,33 @@ export class EncryptedMsgMailFormatter extends BaseMailFormatter { pubkeyRecipients[sendingType] = value?.filter(emailPart => pubkeys.some(p => p.email === emailPart.email)); } } - const uniquePubkeyRecipientEmails = Value.arr.unique((pubkeyRecipients.to || []).concat(pubkeyRecipients.cc || []) + const uniquePubkeyRecipientToAndCCs = Value.arr.unique((pubkeyRecipients.to || []).concat(pubkeyRecipients.cc || []) .map(recipient => recipient.email.toLowerCase())); // pubkey recipients should be able to reply to "to" and "cc" pwd recipients const replyTo = (newMsg.recipients.to ?? []).concat(newMsg.recipients.cc ?? []) - .filter(recipient => !uniquePubkeyRecipientEmails.includes(recipient.email.toLowerCase())); - const pubkeyMsgData = { - ...newMsg, - recipients: pubkeyRecipients, - // brackets are required for test emails like '@test:8001' - replyTo: replyTo.length ? `${Str.formatEmailList([newMsg.from, ...replyTo], true)}` : undefined - }; - if (!uniquePubkeyRecipientEmails.length) { - // todo: add myself? - } - const msgs: SendableMsg[] = [ - await this.sendablePwdMsg( + .filter(recipient => !uniquePubkeyRecipientToAndCCs.includes(recipient.email.toLowerCase())); + const msgs: SendableMsg[] = []; + if (pubkeyRecipients.to?.length || pubkeyRecipients.cc?.length || pubkeyRecipients.bcc?.length) { + const pubkeyMsgData = { + ...newMsg, + recipients: pubkeyRecipients, + // brackets are required for test emails like '@test:8001' + replyTo: replyTo.length ? `${Str.formatEmailList([newMsg.from, ...replyTo], true)}` : undefined + }; + msgs.push(await this.sendablePwdMsg( pubkeyMsgData, pubkeys, { msgUrl: uploadedMessageData.url, externalId: uploadedMessageData.externalId }, collectedAttachments, signingKey?.key) - ]; + ); + } // adding individual messages for each recipient that doesn't have a pubkey for (const recipientEmail of Object.keys(uploadedMessageData.emailToExternalIdAndUrl ?? {}).filter(email => !pubkeys.some(p => p.email === email))) { const { url, externalId } = uploadedMessageData.emailToExternalIdAndUrl![recipientEmail]; const foundParsedRecipient = (newMsg.recipients.to ?? []).concat(newMsg.recipients.cc ?? []).concat(newMsg.recipients.bcc ?? []). find(r => r.email.toLowerCase() === recipientEmail.toLowerCase()); + // todo: since a message is allowed to have only `cc` or `bcc` without `to`, should we preserve the original placement(s) of the recipient? const individualMsgData = { ...newMsg, recipients: { to: [foundParsedRecipient ?? { email: recipientEmail }] } }; msgs.push(await this.sendablePwdMsg(individualMsgData, pubkeys, { msgUrl: url, externalId }, [], signingKey?.key)); } diff --git a/extension/js/common/api/email-provider/sendable-msg.ts b/extension/js/common/api/email-provider/sendable-msg.ts index 8eb0fd25922..4628add2d62 100644 --- a/extension/js/common/api/email-provider/sendable-msg.ts +++ b/extension/js/common/api/email-provider/sendable-msg.ts @@ -138,9 +138,10 @@ export class SendableMsg { ) { const allEmails = [...recipients.to || [], ...recipients.cc || [], ...recipients.bcc || []]; if (!allEmails.length && !isDraft) { - throw new Error('The To: field is empty. Please add recipients and try again'); + throw new Error('The To:, Cc: and Bcc: fields are empty. Please add recipients and try again'); } const invalidEmails = allEmails.filter(email => !Str.isEmailValid(email.email)); + // todo: distinguish To:, Cc: and Bcc: in error report? if (invalidEmails.length) { throw new InvalidRecipientError(`The To: field contains invalid emails: ${invalidEmails.join(', ')}\n\nPlease check recipients and try again.`); } diff --git a/test/source/mock/fes/fes-endpoints.ts b/test/source/mock/fes/fes-endpoints.ts index 9f90d214bef..1969b57e5ca 100644 --- a/test/source/mock/fes/fes-endpoints.ts +++ b/test/source/mock/fes/fes-endpoints.ts @@ -64,6 +64,30 @@ const processMessageFromUser2 = (body: string) => { return response; }; +const processMessageFromUser3 = (body: string) => { + expect(body).to.contain('-----BEGIN PGP MESSAGE-----'); + expect(body).to.contain('"associateReplyToken":"mock-fes-reply-token"'); + expect(body).to.contain('"to":["to@example.com"]'); + expect(body).to.contain('"cc":[]'); + expect(body).to.contain('"bcc":["flowcrypt.compatibility@gmail.com"]'); + const response = + { + // this url is required for pubkey encrypted message + url: `http://${standardFesUrl}/message/FES-MOCK-MESSAGE-ID`, + externalId: 'FES-MOCK-EXTERNAL-ID', + emailToExternalIdAndUrl: {} as { [email: string]: { url: string, externalId: string } } + }; + response.emailToExternalIdAndUrl['to@example.com'] = { + url: `http://${standardFesUrl}/message/FES-MOCK-MESSAGE-FOR-TO@EXAMPLE.COM-ID`, + externalId: 'FES-MOCK-EXTERNAL-FOR-TO@EXAMPLE.COM-ID' + }; + response.emailToExternalIdAndUrl['flowcrypt.compatibility@gmail.com'] = { + url: `http://${standardFesUrl}/message/FES-MOCK-MESSAGE-FOR-FLOWCRYPT.COMPATIBILITY@GMAIL.COM-ID`, + externalId: 'FES-MOCK-EXTERNAL-FOR-FLOWCRYPT.COMPATIBILITY@GMAIL.COM-ID' + }; + return response; +}; + export const mockFesEndpoints: HandlersDefinition = { // standard fes location at https://fes.domain.com '/api/': async ({ }, req) => { @@ -118,6 +142,9 @@ export const mockFesEndpoints: HandlersDefinition = { if (body.includes('"from":"user2@standardsubdomainfes.test:8001"')) { return processMessageFromUser2(body); } + if (body.includes('"from":"user3@standardsubdomainfes.test:8001"')) { + return processMessageFromUser3(body); + } } throw new HttpClientErr('Not Found', 404); }, diff --git a/test/source/mock/google/google-endpoints.ts b/test/source/mock/google/google-endpoints.ts index b3e39037578..c39b0392c9f 100644 --- a/test/source/mock/google/google-endpoints.ts +++ b/test/source/mock/google/google-endpoints.ts @@ -337,10 +337,8 @@ const validateMimeMsg = async (acct: string, mimeMsg: ParsedMail, threadId?: str if (!mimeMsg.text && !mimeMsg.attachments?.length) { throw new HttpClientErr('Error: Message body cannot be empty', 400); } - if ( - !parsedMailAddressObjectAsArray(mimeMsg.to).length && parsedMailAddressObjectAsArray(mimeMsg.to)[0].value.length - || parsedMailAddressObjectAsArray(mimeMsg.to)[0].value.find(em => !allowedRecipients.includes(em.address!)) - ) { + const recipients = parsedMailAddressObjectAsArray(mimeMsg.to).concat(parsedMailAddressObjectAsArray(mimeMsg.cc)).concat(parsedMailAddressObjectAsArray(mimeMsg.bcc)); + if (!recipients.length || recipients.some(addr => addr.value.some(em => !allowedRecipients.includes(em.address!)))) { throw new HttpClientErr('Error: You can\'t send a message to unexisting email address(es)'); } const aliases = [acct]; diff --git a/test/source/mock/google/strategies/send-message-strategy.ts b/test/source/mock/google/strategies/send-message-strategy.ts index 23005ac9d06..20c1aac0f04 100644 --- a/test/source/mock/google/strategies/send-message-strategy.ts +++ b/test/source/mock/google/strategies/send-message-strategy.ts @@ -76,7 +76,41 @@ class PwdEncryptedMessageWithFesIdTokenTestStrategy implements ITestMsgStrategy }; } -class PwdEncryptedMessageWithFesIdReplyRenderingTestStrategy implements ITestMsgStrategy { +class PwdEncryptedMessageWithFesPubkeyRecipientInBccTestStrategy implements ITestMsgStrategy { + public test = async (parseResult: ParseMsgResult, id: string) => { + const mimeMsg = parseResult.mimeMsg; + const expectedSenderEmail = 'user3@standardsubdomainfes.test:8001'; + expect(mimeMsg.from!.text).to.equal(`First Last <${expectedSenderEmail}>`); + if (!mimeMsg.text?.includes(`${expectedSenderEmail} has sent you a password-encrypted email`)) { + throw new HttpClientErr(`Error checking sent text in:\n\n${mimeMsg.text}`); + } + if (mimeMsg.text?.includes('http://fes.standardsubdomainfes.test:8001/message/FES-MOCK-MESSAGE-ID')) { + // this is a message to pubkey recipients + expect((mimeMsg.bcc as AddressObject).text).to.equal('flowcrypt.compatibility@gmail.com'); + // tslint:disable-next-line:no-unused-expression + expect(mimeMsg.cc).to.be.an.undefined; // eslint-disable-line no-unused-expressions + // tslint:disable-next-line:no-unused-expression + expect(mimeMsg.to).to.be.an.undefined; // eslint-disable-line no-unused-expressions + expect((mimeMsg.headers.get('reply-to') as AddressObject).text).to.equal('First Last , to@example.com'); + } else if (mimeMsg.text?.includes('http://fes.standardsubdomainfes.test:8001/message/FES-MOCK-MESSAGE-FOR-TO@EXAMPLE.COM-ID')) { + expect((mimeMsg.to as AddressObject).text).to.equal('to@example.com'); + // tslint:disable-next-line:no-unused-expression + expect(mimeMsg.cc).to.be.an.undefined; // eslint-disable-line no-unused-expressions + // tslint:disable-next-line:no-unused-expression + expect(mimeMsg.bcc).to.be.an.undefined; // eslint-disable-line no-unused-expressions + // tslint:disable-next-line:no-unused-expression + expect(mimeMsg.headers.get('reply-to')).to.be.an.undefined; // eslint-disable-line no-unused-expressions + } else { + throw new HttpClientErr(`Error: cannot find pwd encrypted FES link in:\n\n${mimeMsg.text}`); + } + if (!mimeMsg.text?.includes('Follow this link to open it')) { + throw new HttpClientErr(`Error: cannot find pwd encrypted open link prompt in ${mimeMsg.text}`); + } + await new SaveMessageInStorageStrategy().test(parseResult, id); + }; +} + +class PwdEncryptedMessageWithFesReplyRenderingTestStrategy implements ITestMsgStrategy { public test = async (parseResult: ParseMsgResult, id: string) => { const mimeMsg = parseResult.mimeMsg; const expectedSenderEmail = 'user2@standardsubdomainfes.test:8001'; @@ -293,7 +327,9 @@ export class TestBySubjectStrategyContext { } else if (subject.includes('PWD encrypted message with FES - ID TOKEN')) { this.strategy = new PwdEncryptedMessageWithFesIdTokenTestStrategy(); } else if (subject.includes('PWD encrypted message with FES - Reply rendering')) { - this.strategy = new PwdEncryptedMessageWithFesIdReplyRenderingTestStrategy(); + this.strategy = new PwdEncryptedMessageWithFesReplyRenderingTestStrategy(); + } else if (subject.includes('PWD encrypted message with FES - pubkey recipient in bcc')) { + this.strategy = new PwdEncryptedMessageWithFesPubkeyRecipientInBccTestStrategy(); } else if (subject.includes('Message With Image')) { this.strategy = new SaveMessageInStorageStrategy(); } else if (subject.includes('Message With Test Text')) { diff --git a/test/source/tests/compose.ts b/test/source/tests/compose.ts index 9fceb81a4ce..ec0c2a8ee8f 100644 --- a/test/source/tests/compose.ts +++ b/test/source/tests/compose.ts @@ -1607,7 +1607,7 @@ export const defineComposeTests = (testVariant: TestVariant, testWithBrowser: Te console.log(message); } expect(sentMsgs.length).to.equal(2); - // this test is using PwdEncryptedMessageWithFesIdTokenTestStrategy to check sent result based on subject "PWD encrypted message with flowcrypt.com/api" + // this test is using PwdEncryptedMessageWithFesIdTokenTestStrategy to check sent result based on subject "PWD encrypted message with FES - ID TOKEN" // also see '/api/v1/message' in fes-endpoints.ts mock })); @@ -1641,6 +1641,8 @@ export const defineComposeTests = (testVariant: TestVariant, testWithBrowser: Te await fileInput!.uploadFile('test/samples/small.pdf'); await composePage.waitAndType('@input-password', 'gO0d-pwd'); await composePage.waitAndClick('@action-send', { delay: 1 }); + // this test is using PwdEncryptedMessageWithFesReplyRenderingTestStrategy to check sent result based on subject "PWD encrypted message with FES - Reply rendering" + // also see '/api/v1/message' in fes-endpoints.ts mock const attachmentsContainer = (await composePage.waitAny('@replied-attachments'))!; const attachments = await attachmentsContainer.$$('.pgp_attachment'); expect(attachments.length).to.equal(2); @@ -1654,6 +1656,32 @@ export const defineComposeTests = (testVariant: TestVariant, testWithBrowser: Te to.eql(['small.txt.pgp', 'small.pdf.pgp']); })); + /** + * You need the following lines in /etc/hosts: + * 127.0.0.1 standardsubdomainfes.test + * 127.0.0.1 fes.standardsubdomainfes.test + */ + ava.default('user3@standardsubdomainfes.test:8001 - PWD encrypted message with FES web portal - pubkey recipient in bcc', testWithBrowser(undefined, async (t, browser) => { + const acct = 'user3@standardsubdomainfes.test:8001'; // added port to trick extension into calling the mock + const settingsPage = await BrowserRecipe.openSettingsLoginApprove(t, browser, acct); + await SetupPageRecipe.manualEnter(settingsPage, 'flowcrypt.test.key.used.pgp', { submitPubkey: false, usedPgpBefore: false }, + { isSavePassphraseChecked: false, isSavePassphraseHidden: false }); + const subject = 'PWD encrypted message with FES - pubkey recipient in bcc'; + const composePage = await ComposePageRecipe.openStandalone(t, browser, 'user3@standardsubdomainfes.test:8001'); + await ComposePageRecipe.fillMsg(composePage, { to: 'to@example.com', bcc: 'flowcrypt.compatibility@gmail.com' }, subject); + await composePage.waitAndType('@input-password', 'gO0d-pwd'); + await composePage.waitAndClick('@action-send', { delay: 1 }); + await ComposePageRecipe.closed(composePage); + const sentMsgs = (await GoogleData.withInitializedData(acct)).searchMessagesBySubject(subject); + for (const msg of sentMsgs) { + const message = msg.payload!.body!.data!; + console.log(message); + } + expect(sentMsgs.length).to.equal(2); + // this test is using PwdEncryptedMessageWithFesPubkeyRecipientInBccTestStrategy to check sent result based on subject "PWD encrypted message with FES - pubkey recipient in bcc" + // also see '/api/v1/message' in fes-endpoints.ts mock + })); + ava.default('first.key.revoked@key-manager-autoimport-no-prv-create.flowcrypt.test - selects valid own key when saving draft or sending', testWithBrowser(undefined, async (t, browser) => { const acct = 'first.key.revoked@key-manager-autoimport-no-prv-create.flowcrypt.test'; const settingsPage = await BrowserRecipe.openSettingsLoginApprove(t, browser, acct); From e6acd9e90bdf707445e34500f9c0d38caae41f64 Mon Sep 17 00:00:00 2001 From: Roman Shevchenko Date: Sun, 15 May 2022 06:50:58 +0000 Subject: [PATCH 15/45] added tests for content of uploaded pwd-protected message --- test/source/mock/fes/fes-endpoints.ts | 48 +++++++++++++++++++++++---- 1 file changed, 42 insertions(+), 6 deletions(-) diff --git a/test/source/mock/fes/fes-endpoints.ts b/test/source/mock/fes/fes-endpoints.ts index 1969b57e5ca..089d151d365 100644 --- a/test/source/mock/fes/fes-endpoints.ts +++ b/test/source/mock/fes/fes-endpoints.ts @@ -2,6 +2,8 @@ import { expect } from 'chai'; import { IncomingMessage } from 'http'; +import { Buf } from '../../core/buf'; +import { MsgUtil } from '../../core/crypto/pgp/msg-util'; import { HandlersDefinition } from '../all-apis-mock'; import { HttpClientErr } from '../lib/api'; import { MockJwt } from '../lib/oauth'; @@ -9,12 +11,24 @@ import { MockJwt } from '../lib/oauth'; const standardFesUrl = 'fes.standardsubdomainfes.test:8001'; const issuedAccessTokens: string[] = []; -const processMessageFromUser = (body: string) => { +const processMessageFromUser = async (body: string) => { expect(body).to.contain('-----BEGIN PGP MESSAGE-----'); expect(body).to.contain('"associateReplyToken":"mock-fes-reply-token"'); expect(body).to.contain('"to":["Mr To "]'); expect(body).to.contain('"cc":[]'); expect(body).to.contain('"bcc":["Mr Bcc "]'); + const encryptedData = Buf.fromUtfStr(body.match(/-----BEGIN PGP MESSAGE-----.*-----END PGP MESSAGE-----/s)![0]); + const decrypted = await MsgUtil.decryptMessage({ kisWithPp: [], msgPwd: 'lousy pwdgO0d-pwd', encryptedData, verificationPubs: [] }); + expect(decrypted.success).to.equal(true); + const decryptedMimeMsg = decrypted.content!.toUtfStr(); + expect(decryptedMimeMsg).to.contain('Content-Type: text/plain\r\n' + + 'Content-Transfer-Encoding: quoted-printable\r\n\r\n' + + 'PWD encrypted message with FES - ID TOKEN'); + // small.txt + expect(decryptedMimeMsg).to.contain('Content-Type: text/plain; name=small.txt\r\n' + + 'Content-Disposition: attachment; filename=small.txt'); + expect(decryptedMimeMsg).to.contain('Content-Transfer-Encoding: base64\r\n\r\n' + + 'c21hbGwgdGV4dCBmaWxlCm5vdCBtdWNoIGhlcmUKdGhpcyB3b3JrZWQK'); const response = { // this url is required for pubkey encrypted message @@ -33,12 +47,26 @@ const processMessageFromUser = (body: string) => { return response; }; -const processMessageFromUser2 = (body: string) => { +const processMessageFromUser2 = async (body: string) => { expect(body).to.contain('-----BEGIN PGP MESSAGE-----'); expect(body).to.contain('"associateReplyToken":"mock-fes-reply-token"'); expect(body).to.contain('"to":["sender@domain.com","flowcrypt.compatibility@gmail.com","to@example.com","mock.only.pubkey@flowcrypt.com"]'); expect(body).to.contain('"cc":[]'); expect(body).to.contain('"bcc":[]'); + const encryptedData = Buf.fromUtfStr(body.match(/-----BEGIN PGP MESSAGE-----.*-----END PGP MESSAGE-----/s)![0]); + const decrypted = await MsgUtil.decryptMessage({ kisWithPp: [], msgPwd: 'gO0d-pwd', encryptedData, verificationPubs: [] }); + expect(decrypted.success).to.equal(true); + const decryptedMimeMsg = decrypted.content!.toUtfStr(); + // small.txt + expect(decryptedMimeMsg).to.contain('Content-Type: text/plain; name=small.txt\r\n' + + 'Content-Disposition: attachment; filename=small.txt'); + expect(decryptedMimeMsg).to.contain('Content-Transfer-Encoding: base64\r\n\r\n' + + 'c21hbGwgdGV4dCBmaWxlCm5vdCBtdWNoIGhlcmUKdGhpcyB3b3JrZWQK'); + // small.pdf + expect(decryptedMimeMsg).to.contain('Content-Type: application/pdf; name=small.pdf\r\n' + + 'Content-Disposition: attachment; filename=small.pdf'); + expect(decryptedMimeMsg).to.contain('Content-Transfer-Encoding: base64\r\n\r\n' + + 'JVBERi0xLjQKJcOkw7zDtsOfCjIgMCBvYmoKPDwvTGVuZ3RoIDMgMCBSL0ZpbHRlci9GbGF0ZURl'); const response = { // this url is required for pubkey encrypted message url: `http://${standardFesUrl}/message/FES-MOCK-MESSAGE-ID`, @@ -64,12 +92,20 @@ const processMessageFromUser2 = (body: string) => { return response; }; -const processMessageFromUser3 = (body: string) => { +const processMessageFromUser3 = async (body: string) => { expect(body).to.contain('-----BEGIN PGP MESSAGE-----'); expect(body).to.contain('"associateReplyToken":"mock-fes-reply-token"'); expect(body).to.contain('"to":["to@example.com"]'); expect(body).to.contain('"cc":[]'); expect(body).to.contain('"bcc":["flowcrypt.compatibility@gmail.com"]'); + const encryptedData = Buf.fromUtfStr(body.match(/-----BEGIN PGP MESSAGE-----.*-----END PGP MESSAGE-----/s)![0]); + const decrypted = await MsgUtil.decryptMessage({ kisWithPp: [], msgPwd: 'gO0d-pwd', encryptedData, verificationPubs: [] }); + expect(decrypted.success).to.equal(true); + const decryptedMimeMsg = decrypted.content!.toUtfStr(); + // small.txt + expect(decryptedMimeMsg).to.contain('Content-Type: text/plain\r\n' + + 'Content-Transfer-Encoding: quoted-printable\r\n\r\n' + + 'PWD encrypted message with FES - pubkey recipient in bcc'); const response = { // this url is required for pubkey encrypted message @@ -137,13 +173,13 @@ export const mockFesEndpoints: HandlersDefinition = { // test: `compose - user@standardsubdomainfes.test:8001 - PWD encrypted message with FES web portal` authenticate(req, 'oidc'); if (body.includes('"from":"user@standardsubdomainfes.test:8001"')) { - return processMessageFromUser(body); + return await processMessageFromUser(body); } if (body.includes('"from":"user2@standardsubdomainfes.test:8001"')) { - return processMessageFromUser2(body); + return await processMessageFromUser2(body); } if (body.includes('"from":"user3@standardsubdomainfes.test:8001"')) { - return processMessageFromUser3(body); + return await processMessageFromUser3(body); } } throw new HttpClientErr('Not Found', 404); From c280407248850f0a185e687958e45f0a359631b3 Mon Sep 17 00:00:00 2001 From: Roman Shevchenko Date: Tue, 17 May 2022 08:52:16 +0000 Subject: [PATCH 16/45] returned legacy mode --- .../formatters/encrypted-mail-msg-formatter.ts | 14 ++++++++++++++ 1 file changed, 14 insertions(+) diff --git a/extension/chrome/elements/compose-modules/formatters/encrypted-mail-msg-formatter.ts b/extension/chrome/elements/compose-modules/formatters/encrypted-mail-msg-formatter.ts index febf1e8228d..959491065c2 100644 --- a/extension/chrome/elements/compose-modules/formatters/encrypted-mail-msg-formatter.ts +++ b/extension/chrome/elements/compose-modules/formatters/encrypted-mail-msg-formatter.ts @@ -38,6 +38,20 @@ export class EncryptedMsgMailFormatter extends BaseMailFormatter { const uploadedMessageData = await this.prepareAndUploadPwdEncryptedMsg(newMsg); // encrypted for pwd only, pubkeys ignored newMsg.pwd = undefined; const collectedAttachments = await this.view.attachmentsModule.attachment.collectEncryptAttachments(pubkeys); + if (!Object.keys(uploadedMessageData.emailToExternalIdAndUrl ?? {}).length) { + const legacyMsg = await this.sendablePwdMsg( + newMsg, + pubkeys, + { msgUrl: uploadedMessageData.url, externalId: uploadedMessageData.externalId }, + collectedAttachments, + signingKey?.key); + return { + senderKi: signingKey?.keyInfo, + msgs: [legacyMsg], + recipients: legacyMsg.recipients, + attachments: collectedAttachments + }; + } const pubkeyRecipients: { [type in RecipientType]?: EmailParts[] } = {}; for (const [key, value] of Object.entries(newMsg.recipients)) { if (['to', 'cc', 'bcc'].includes(key)) { From b26cde5bc474edebbfdf62522115886dd9787a5d Mon Sep 17 00:00:00 2001 From: Roman Shevchenko Date: Tue, 17 May 2022 11:19:53 +0000 Subject: [PATCH 17/45] lint fix --- .../formatters/encrypted-mail-msg-formatter.ts | 7 +------ 1 file changed, 1 insertion(+), 6 deletions(-) diff --git a/extension/chrome/elements/compose-modules/formatters/encrypted-mail-msg-formatter.ts b/extension/chrome/elements/compose-modules/formatters/encrypted-mail-msg-formatter.ts index 959491065c2..18d5b4fe00d 100644 --- a/extension/chrome/elements/compose-modules/formatters/encrypted-mail-msg-formatter.ts +++ b/extension/chrome/elements/compose-modules/formatters/encrypted-mail-msg-formatter.ts @@ -45,12 +45,7 @@ export class EncryptedMsgMailFormatter extends BaseMailFormatter { { msgUrl: uploadedMessageData.url, externalId: uploadedMessageData.externalId }, collectedAttachments, signingKey?.key); - return { - senderKi: signingKey?.keyInfo, - msgs: [legacyMsg], - recipients: legacyMsg.recipients, - attachments: collectedAttachments - }; + return { senderKi: signingKey?.keyInfo, msgs: [legacyMsg], recipients: legacyMsg.recipients, attachments: collectedAttachments }; } const pubkeyRecipients: { [type in RecipientType]?: EmailParts[] } = {}; for (const [key, value] of Object.entries(newMsg.recipients)) { From 1dc81eb3e74c9fd77c272d659b7a99630b5e0a9a Mon Sep 17 00:00:00 2001 From: Roman Shevchenko Date: Wed, 25 May 2022 09:23:16 +0000 Subject: [PATCH 18/45] error handling when sending multiple messages --- .../compose-modules/compose-err-module.ts | 30 ++++- .../compose-send-btn-module.ts | 85 ++++++++------ .../elements/compose-modules/compose-types.ts | 6 + .../encrypted-mail-msg-formatter.ts | 17 +-- .../common/api/email-provider/sendable-msg.ts | 4 +- test/source/mock/fes/fes-endpoints.ts | 38 ++++++ test/source/mock/google/google-endpoints.ts | 2 +- .../strategies/send-message-strategy.ts | 26 ++++- test/source/tests/compose.ts | 108 ++++++++++++++++-- 9 files changed, 256 insertions(+), 60 deletions(-) diff --git a/extension/chrome/elements/compose-modules/compose-err-module.ts b/extension/chrome/elements/compose-modules/compose-err-module.ts index 8811366bf12..349973ec224 100644 --- a/extension/chrome/elements/compose-modules/compose-err-module.ts +++ b/extension/chrome/elements/compose-modules/compose-err-module.ts @@ -5,7 +5,7 @@ import { Browser } from '../../../js/common/browser/browser.js'; import { BrowserEventErrHandler, Ui } from '../../../js/common/browser/ui.js'; import { Catch } from '../../../js/common/platform/catch.js'; -import { NewMsgData, SendBtnTexts } from './compose-types.js'; +import { NewMsgData, SendBtnTexts, SendMsgsResult } from './compose-types.js'; import { ApiErr } from '../../../js/common/api/shared/api-error.js'; import { BrowserExtension } from '../../../js/common/browser/browser-extension.js'; import { BrowserMsg } from '../../../js/common/browser/browser-msg.js'; @@ -61,11 +61,19 @@ export class ComposeErrModule extends ViewModule { } }; - public handleSendErr = async (e: any) => { + public handleSendErr = async (e: any, sendMsgsResult?: SendMsgsResult) => { + // this.view.errModule.debug(`handleSendErr: ${String(e)}`); if (ApiErr.isNetErr(e)) { - let netErrMsg = 'Could not send message due to network error. Please check your internet connection and try again.\n'; - netErrMsg += '(This may also be caused by missing extension permissions).'; + let netErrMsg: string | undefined; + if (sendMsgsResult?.success.length) { + // there were some successful sends + netErrMsg = ComposeErrModule.getErrSayingSomeMessagesHaveBeenSent(sendMsgsResult) + + 'network errors. Please check your internet connection and try again.'; + } else { + netErrMsg = 'Could not send message due to network error. Please check your internet connection and try again.\n' + + '(This may also be caused by missing extension permissions).'; + } await Ui.modal.error(netErrMsg, true); } else if (ApiErr.isAuthErr(e)) { BrowserMsg.send.notificationShowAuthPopupNeeded(this.view.parentTabId, { acctEmail: this.view.acctEmail }); @@ -73,10 +81,15 @@ export class ComposeErrModule extends ViewModule { } else if (ApiErr.isReqTooLarge(e)) { await Ui.modal.error(`Could not send: message or attachments too large.`); } else if (ApiErr.isBadReq(e)) { + let gmailErrMsg: string | undefined; + if (sendMsgsResult?.success.length) { + gmailErrMsg = ComposeErrModule.getErrSayingSomeMessagesHaveBeenSent(sendMsgsResult) + 'error(s) from Gmail'; + } if (e.resMsg === AjaxErrMsgs.GOOGLE_INVALID_TO_HEADER || e.resMsg === AjaxErrMsgs.GOOGLE_RECIPIENT_ADDRESS_REQUIRED) { - await Ui.modal.error('Error from google: Invalid recipients\n\nPlease remove recipients, add them back and re-send the message.'); + await Ui.modal.error((gmailErrMsg || 'Error from google') + ': Invalid recipients\n\nPlease remove recipients, add them back and re-send the message.'); } else { - if (await Ui.modal.confirm(`Google returned an error when sending message. Please help us improve FlowCrypt by reporting the error to us.`)) { + if (await Ui.modal.confirm((gmailErrMsg || 'Google returned an error when sending message') + + `. Please help us improve FlowCrypt by reporting the error to us.`)) { const page = '/chrome/settings/modules/help.htm'; const pageUrlParams = { bugReport: BrowserExtension.prepareBugReport(`composer: send: bad request (errMsg: ${e.resMsg})`, {}, e) }; await Browser.openSettingsPage('index.htm', this.view.acctEmail, page, pageUrlParams); @@ -162,4 +175,9 @@ export class ComposeErrModule extends ViewModule { } }; + private static getErrSayingSomeMessagesHaveBeenSent = (sendMsgsResult: SendMsgsResult) => { + return 'Messages to some recipients were sent successfully, while messages to ' + + Str.formatEmailList(sendMsgsResult.failures.map(el => el.recipient)) + ' encountered '; + } + } diff --git a/extension/chrome/elements/compose-modules/compose-send-btn-module.ts b/extension/chrome/elements/compose-modules/compose-send-btn-module.ts index f39f9be86a1..90ae7d7a537 100644 --- a/extension/chrome/elements/compose-modules/compose-send-btn-module.ts +++ b/extension/chrome/elements/compose-modules/compose-send-btn-module.ts @@ -14,13 +14,14 @@ import { ComposeSendBtnPopoverModule } from './compose-send-btn-popover-module.j import { GeneralMailFormatter, MultipleMessages } from './formatters/general-mail-formatter.js'; import { GmailParser, GmailRes } from '../../../js/common/api/email-provider/gmail/gmail-parser.js'; import { KeyInfoWithIdentity } from '../../../js/common/core/crypto/key.js'; -import { getUniqueRecipientEmails, SendBtnTexts } from './compose-types.js'; +import { getUniqueRecipientEmails, SendBtnTexts, SendMsgsResult } from './compose-types.js'; import { SendableMsg } from '../../../js/common/api/email-provider/sendable-msg.js'; import { Ui } from '../../../js/common/browser/ui.js'; import { Xss } from '../../../js/common/platform/xss.js'; import { ViewModule } from '../../../js/common/view-module.js'; import { ComposeView } from '../compose.js'; import { ContactStore } from '../../../js/common/platform/store/contact-store.js'; +import { EmailParts } from '../../../js/common/core/common.js'; export class ComposeSendBtnModule extends ViewModule { @@ -105,6 +106,7 @@ export class ComposeSendBtnModule extends ViewModule { this.view.S.cached('toggle_send_options').hide(); try { this.view.errModule.throwIfFormNotReady(); + this.isSendMessageInProgress = true; this.view.S.now('send_btn_text').text('Loading...'); Xss.sanitizeRender(this.view.S.now('send_btn_i'), Ui.spinner('white')); this.view.S.cached('send_btn_note').text(''); @@ -116,10 +118,15 @@ export class ComposeSendBtnModule extends ViewModule { for (const msg of msgObj.msgs) { await this.finalizeSendableMsg({ msg, senderKi: msgObj.senderKi }); } - await this.doSendMsgs(msgObj); + const result = await this.doSendMsgs(msgObj); + if (result.failures.length) { + await this.view.errModule.handleSendErr(result.failures[0].e, result); + } + // todo: supplementary operation errors? } catch (e) { - await this.view.errModule.handleSendErr(e); + await this.view.errModule.handleSendErr(e, undefined); } finally { + this.isSendMessageInProgress = false; this.view.sendBtnModule.enableBtn(); this.view.S.cached('toggle_send_options').show(); } @@ -193,49 +200,63 @@ export class ComposeSendBtnModule extends ViewModule { return { mimeType, data }; }; - - private doSendMsgs = async (msgObj: MultipleMessages) => { + private doSendMsgs = async (msgObj: MultipleMessages): Promise => { // if this is a password-encrypted message, then we've already shown progress for uploading to backend // and this requests represents second half of uploadable effort. Else this represents all (no previous heavy requests) const progressRepresents = this.view.pwdOrPubkeyContainerModule.isVisible() ? 'SECOND-HALF' : 'EVERYTHING'; const sentIds: string[] = []; - const operations = [this.view.draftModule.draftDelete()]; + const operations = [this.view.draftModule.draftDelete()]; // todo: don't delete on error + const success: EmailParts[] = []; + const failures: { recipient: EmailParts, e: any }[] = []; for (const msg of msgObj.msgs) { let msgSentRes: GmailRes.GmailMsgSend; - try { - this.isSendMessageInProgress = true; - msgSentRes = await this.view.emailProvider.msgSend(msg, (p) => this.renderUploadProgress(p, progressRepresents)); - } catch (e) { - if (msg.thread && ApiErr.isNotFound(e) && this.view.threadId) { // cannot send msg because threadId not found - eg user since deleted it - msg.thread = undefined; + const msgRecipients = msg.getAllRecipients(); + while (true) { // first try with msg.thread, and then possibly try again without it + try { msgSentRes = await this.view.emailProvider.msgSend(msg, (p) => this.renderUploadProgress(p, progressRepresents)); - } else { - this.isSendMessageInProgress = false; - throw e; - } - } - sentIds.push(msgSentRes.id); - if (msg.externalId) { - operations.push((async (externalId, id) => { - const gmailMsg = await this.view.emailProvider.msgGet(id, 'metadata'); - const messageId = GmailParser.findHeader(gmailMsg, 'message-id'); - if (messageId) { - await this.view.acctServer.messageGatewayUpdate(externalId, messageId); + success.push(...msgRecipients); + sentIds.push(msgSentRes.id); + if (msg.externalId) { + operations.push((async (externalId, id) => { + const gmailMsg = await this.view.emailProvider.msgGet(id, 'metadata'); + const messageId = GmailParser.findHeader(gmailMsg, 'message-id'); + if (messageId) { + await this.view.acctServer.messageGatewayUpdate(externalId, messageId); + } else { + Catch.report('Failed to extract Message-ID of sent message'); + } + })(msg.externalId, msgSentRes.id)); + } + } catch (e) { + if (msg.thread && ApiErr.isNotFound(e) && this.view.threadId) { // cannot send msg because threadId not found - eg user since deleted it + msg.thread = undefined; + continue; + // give it another try, this time without msg.thread } else { - Catch.report('Failed to extract Message-ID of sent message'); + failures.push(...msgRecipients.map(recipient => { return { recipient, e }; })); } - })(msg.externalId, msgSentRes.id)); - } + } + break; + } // while loop for thread retry } BrowserMsg.send.notificationShow(this.view.parentTabId, { notification: `Your ${this.view.isReplyBox ? 'reply' : 'message'} has been sent.` }); BrowserMsg.send.focusBody(this.view.parentTabId); // Bring focus back to body so Gmails shortcuts will work - await Promise.all(operations); + try { + await Promise.all(operations); + } catch (e) { + // todo: bindings or draft delete failed, not really critical, add to supplementary errors + } this.isSendMessageInProgress = false; - if (this.view.isReplyBox) { - this.view.renderModule.renderReplySuccess(msgObj.attachments, msgObj.recipients, sentIds[0]); - } else { - this.view.renderModule.closeMsg(); + if (!failures.length) { + // todo: supplementary errors + // closing the composer on successful send + if (this.view.isReplyBox) { + this.view.renderModule.renderReplySuccess(msgObj.attachments, msgObj.recipients, sentIds[0]); + } else { + this.view.renderModule.closeMsg(); + } } + return { success, failures, supplementaryOperationsError: undefined }; // todo: }; } diff --git a/extension/chrome/elements/compose-modules/compose-types.ts b/extension/chrome/elements/compose-modules/compose-types.ts index 3ac37d8ddb7..6c1732c74e2 100644 --- a/extension/chrome/elements/compose-modules/compose-types.ts +++ b/extension/chrome/elements/compose-modules/compose-types.ts @@ -65,3 +65,9 @@ export class SendBtnTexts { export const getUniqueRecipientEmails = (recipients: ParsedRecipients) => { return Value.arr.unique(Object.values(recipients).reduce((a, b) => a.concat(b), []).filter(x => x.email).map(x => x.email)); }; + +export type SendMsgsResult = { + success: EmailParts[], + failures: { recipient: EmailParts, e: any }[], + supplementaryOperationsError: any +}; diff --git a/extension/chrome/elements/compose-modules/formatters/encrypted-mail-msg-formatter.ts b/extension/chrome/elements/compose-modules/formatters/encrypted-mail-msg-formatter.ts index 18d5b4fe00d..cbe50c66e64 100644 --- a/extension/chrome/elements/compose-modules/formatters/encrypted-mail-msg-formatter.ts +++ b/extension/chrome/elements/compose-modules/formatters/encrypted-mail-msg-formatter.ts @@ -36,22 +36,17 @@ export class EncryptedMsgMailFormatter extends BaseMailFormatter { // - FlowCrypt Enterprise Server (enterprise customers with on-prem setup) // It will be served to recipient through web const uploadedMessageData = await this.prepareAndUploadPwdEncryptedMsg(newMsg); // encrypted for pwd only, pubkeys ignored + // pwdRecipients that have their personal link + const pwdRecipients = Object.keys(uploadedMessageData.emailToExternalIdAndUrl ?? {}).filter(email => !pubkeys.some(p => p.email === email)); newMsg.pwd = undefined; const collectedAttachments = await this.view.attachmentsModule.attachment.collectEncryptAttachments(pubkeys); - if (!Object.keys(uploadedMessageData.emailToExternalIdAndUrl ?? {}).length) { - const legacyMsg = await this.sendablePwdMsg( - newMsg, - pubkeys, - { msgUrl: uploadedMessageData.url, externalId: uploadedMessageData.externalId }, - collectedAttachments, - signingKey?.key); - return { senderKi: signingKey?.keyInfo, msgs: [legacyMsg], recipients: legacyMsg.recipients, attachments: collectedAttachments }; - } const pubkeyRecipients: { [type in RecipientType]?: EmailParts[] } = {}; for (const [key, value] of Object.entries(newMsg.recipients)) { if (['to', 'cc', 'bcc'].includes(key)) { const sendingType = key as RecipientType; - pubkeyRecipients[sendingType] = value?.filter(emailPart => pubkeys.some(p => p.email === emailPart.email)); + pubkeyRecipients[sendingType] = value?.filter(emailPart => pubkeys.some(p => p.email === emailPart.email) + // pwd recipients that have no personal links go to legacy message + || (uploadedMessageData.emailToExternalIdAndUrl || {})[emailPart.email] === undefined); } } const uniquePubkeyRecipientToAndCCs = Value.arr.unique((pubkeyRecipients.to || []).concat(pubkeyRecipients.cc || []) @@ -76,7 +71,7 @@ export class EncryptedMsgMailFormatter extends BaseMailFormatter { ); } // adding individual messages for each recipient that doesn't have a pubkey - for (const recipientEmail of Object.keys(uploadedMessageData.emailToExternalIdAndUrl ?? {}).filter(email => !pubkeys.some(p => p.email === email))) { + for (const recipientEmail of pwdRecipients) { const { url, externalId } = uploadedMessageData.emailToExternalIdAndUrl![recipientEmail]; const foundParsedRecipient = (newMsg.recipients.to ?? []).concat(newMsg.recipients.cc ?? []).concat(newMsg.recipients.bcc ?? []). find(r => r.email.toLowerCase() === recipientEmail.toLowerCase()); diff --git a/extension/js/common/api/email-provider/sendable-msg.ts b/extension/js/common/api/email-provider/sendable-msg.ts index 4628add2d62..445c07c8eae 100644 --- a/extension/js/common/api/email-provider/sendable-msg.ts +++ b/extension/js/common/api/email-provider/sendable-msg.ts @@ -136,7 +136,7 @@ export class SendableMsg { public type: MimeEncodeType, public externalId?: string, // for binding a password-protected message ) { - const allEmails = [...recipients.to || [], ...recipients.cc || [], ...recipients.bcc || []]; + const allEmails = this.getAllRecipients(); if (!allEmails.length && !isDraft) { throw new Error('The To:, Cc: and Bcc: fields are empty. Please add recipients and try again'); } @@ -147,6 +147,8 @@ export class SendableMsg { } } + public getAllRecipients = () => [...this.recipients.to || [], ...this.recipients.cc || [], ...this.recipients.bcc || []]; + public toMime = async () => { this.headers.From = this.from; if (this.replyTo) { diff --git a/test/source/mock/fes/fes-endpoints.ts b/test/source/mock/fes/fes-endpoints.ts index 089d151d365..6b6346d6c82 100644 --- a/test/source/mock/fes/fes-endpoints.ts +++ b/test/source/mock/fes/fes-endpoints.ts @@ -124,6 +124,41 @@ const processMessageFromUser3 = async (body: string) => { return response; }; +const processMessageFromUser4 = async (body: string) => { + const response = + { + // this url is required for pubkey encrypted message + url: `http://${standardFesUrl}/message/FES-MOCK-MESSAGE-ID`, + externalId: 'FES-MOCK-EXTERNAL-ID', + emailToExternalIdAndUrl: {} as { [email: string]: { url: string, externalId: string } } + }; + if (body.includes("to@example.com")) { + response.emailToExternalIdAndUrl['to@example.com'] = { + url: `http://${standardFesUrl}/message/FES-MOCK-MESSAGE-FOR-TO@EXAMPLE.COM-ID`, + externalId: 'FES-MOCK-EXTERNAL-FOR-TO@EXAMPLE.COM-ID' + }; + } + if (body.includes("invalid@example.com")) { + response.emailToExternalIdAndUrl['invalid@example.com'] = { + url: `http://${standardFesUrl}/message/FES-MOCK-MESSAGE-FOR-INVALID@EXAMPLE.COM-ID`, + externalId: 'FES-MOCK-EXTERNAL-FOR-INVALID@EXAMPLE.COM-ID' + }; + } + if (body.includes("Mr Cc ")) { + response.emailToExternalIdAndUrl['cc@example.com'] = { + url: `http://${standardFesUrl}/message/FES-MOCK-MESSAGE-FOR-CC@EXAMPLE.COM-ID`, + externalId: 'FES-MOCK-EXTERNAL-FOR-CC@EXAMPLE.COM-ID' + }; + } + if (body.includes("First Last ")) { + response.emailToExternalIdAndUrl['flowcrypt.compatibility@gmail.com'] = { + url: `http://${standardFesUrl}/message/FES-MOCK-MESSAGE-FOR-FLOWCRYPT.COMPATIBILITY@GMAIL.COM-ID`, + externalId: 'FES-MOCK-EXTERNAL-FOR-FLOWCRYPT.COMPATIBILITY@GMAIL.COM-ID' + }; + } + return response; +}; + export const mockFesEndpoints: HandlersDefinition = { // standard fes location at https://fes.domain.com '/api/': async ({ }, req) => { @@ -181,6 +216,9 @@ export const mockFesEndpoints: HandlersDefinition = { if (body.includes('"from":"user3@standardsubdomainfes.test:8001"')) { return await processMessageFromUser3(body); } + if (body.includes('"from":"user4@standardsubdomainfes.test:8001"')) { + return await processMessageFromUser4(body); + } } throw new HttpClientErr('Not Found', 404); }, diff --git a/test/source/mock/google/google-endpoints.ts b/test/source/mock/google/google-endpoints.ts index c39b0392c9f..a7f69d35e8a 100644 --- a/test/source/mock/google/google-endpoints.ts +++ b/test/source/mock/google/google-endpoints.ts @@ -18,7 +18,7 @@ const allowedRecipients: Array = ['flowcrypt.compatibility@gmail.com', ' 'ci.tests.gmail@flowcrypt.test', 'smime1@recipient.com', 'smime2@recipient.com', 'smime@recipient.com', 'smime.attachment@recipient.com', 'auto.refresh.expired.key@recipient.com', 'to@example.com', 'cc@example.com', 'bcc@example.com', 'flowcrypt.test.key.multiple.inbox1@gmail.com', 'flowcrypt.test.key.multiple.inbox2@gmail.com', 'mock.only.pubkey@flowcrypt.com', - 'vladimir@flowcrypt.com', 'limon.monte@gmail.com', 'sweetalert2@gmail.com', 'sender@domain.com']; + 'vladimir@flowcrypt.com', 'limon.monte@gmail.com', 'sweetalert2@gmail.com', 'sender@domain.com', 'invalid@example.com', 'timeout@example.com']; export const mockGoogleEndpoints: HandlersDefinition = { '/o/oauth2/auth': async ({ query: { client_id, response_type, access_type, state, redirect_uri, scope, login_hint, proceed } }, req) => { diff --git a/test/source/mock/google/strategies/send-message-strategy.ts b/test/source/mock/google/strategies/send-message-strategy.ts index 20c1aac0f04..bd7cc88ed42 100644 --- a/test/source/mock/google/strategies/send-message-strategy.ts +++ b/test/source/mock/google/strategies/send-message-strategy.ts @@ -7,7 +7,7 @@ import { Buf } from '../../../core/buf'; import { Config } from '../../../util'; import { expect } from 'chai'; import { GoogleData } from '../google-data'; -import { HttpClientErr } from '../../lib/api'; +import { HttpClientErr, Status } from '../../lib/api'; import { MsgUtil } from '../../../core/crypto/pgp/msg-util'; import Parse, { ParseMsgResult } from '../../../util/parse'; import { parsedMailAddressObjectAsArray } from '../google-endpoints.js'; @@ -110,6 +110,28 @@ class PwdEncryptedMessageWithFesPubkeyRecipientInBccTestStrategy implements ITes }; } +class PwdEncryptedMessageWithFesReplyBadRequestTestStrategy implements ITestMsgStrategy { + public test = async (parseResult: ParseMsgResult, id: string) => { + const mimeMsg = parseResult.mimeMsg; + const expectedSenderEmail = 'user4@standardsubdomainfes.test:8001'; + expect(mimeMsg.from!.text).to.equal(`First Last <${expectedSenderEmail}>`); + const to = parsedMailAddressObjectAsArray(mimeMsg.to).concat(parsedMailAddressObjectAsArray(mimeMsg.cc)).concat(parsedMailAddressObjectAsArray(mimeMsg.bcc)); + expect(to.length).to.equal(1); + const recipientEmail = to[0].text; + if (recipientEmail === 'to@example.com') { + // success + await new SaveMessageInStorageStrategy().test(parseResult, id); + return; + } else if (recipientEmail === 'invalid@example.com') { + throw new HttpClientErr('Invalid to header', Status.BAD_REQUEST); + } else if (recipientEmail === 'timeout@example.com') { + throw new HttpClientErr('RequestTimeout', Status.BAD_REQUEST); + } else { + throw new HttpClientErr(`Vague failure for ${recipientEmail}`, Status.BAD_REQUEST); + } + }; +} + class PwdEncryptedMessageWithFesReplyRenderingTestStrategy implements ITestMsgStrategy { public test = async (parseResult: ParseMsgResult, id: string) => { const mimeMsg = parseResult.mimeMsg; @@ -330,6 +352,8 @@ export class TestBySubjectStrategyContext { this.strategy = new PwdEncryptedMessageWithFesReplyRenderingTestStrategy(); } else if (subject.includes('PWD encrypted message with FES - pubkey recipient in bcc')) { this.strategy = new PwdEncryptedMessageWithFesPubkeyRecipientInBccTestStrategy(); + } else if (subject.includes('PWD encrypted message with FES web portal - some sends fail with BadRequest error')) { + this.strategy = new PwdEncryptedMessageWithFesReplyBadRequestTestStrategy(); } else if (subject.includes('Message With Image')) { this.strategy = new SaveMessageInStorageStrategy(); } else if (subject.includes('Message With Test Text')) { diff --git a/test/source/tests/compose.ts b/test/source/tests/compose.ts index ec0c2a8ee8f..c1812944b54 100644 --- a/test/source/tests/compose.ts +++ b/test/source/tests/compose.ts @@ -1602,10 +1602,6 @@ export const defineComposeTests = (testVariant: TestVariant, testWithBrowser: Te await composePage.waitAndClick('@action-send', { delay: 1 }); await ComposePageRecipe.closed(composePage); const sentMsgs = (await GoogleData.withInitializedData(acct)).searchMessagesBySubject(subject); - for (const msg of sentMsgs) { - const message = msg.payload!.body!.data!; - console.log(message); - } expect(sentMsgs.length).to.equal(2); // this test is using PwdEncryptedMessageWithFesIdTokenTestStrategy to check sent result based on subject "PWD encrypted message with FES - ID TOKEN" // also see '/api/v1/message' in fes-endpoints.ts mock @@ -1673,15 +1669,111 @@ export const defineComposeTests = (testVariant: TestVariant, testWithBrowser: Te await composePage.waitAndClick('@action-send', { delay: 1 }); await ComposePageRecipe.closed(composePage); const sentMsgs = (await GoogleData.withInitializedData(acct)).searchMessagesBySubject(subject); - for (const msg of sentMsgs) { - const message = msg.payload!.body!.data!; - console.log(message); - } expect(sentMsgs.length).to.equal(2); // this test is using PwdEncryptedMessageWithFesPubkeyRecipientInBccTestStrategy to check sent result based on subject "PWD encrypted message with FES - pubkey recipient in bcc" // also see '/api/v1/message' in fes-endpoints.ts mock })); + /** + * You need the following lines in /etc/hosts: + * 127.0.0.1 standardsubdomainfes.test + * 127.0.0.1 fes.standardsubdomainfes.test + */ + ava.default('user4@standardsubdomainfes.test:8001 - PWD encrypted message with FES web portal - some sends fail with BadRequest error', testWithBrowser(undefined, async (t, browser) => { + const acct = 'user4@standardsubdomainfes.test:8001'; // added port to trick extension into calling the mock + const settingsPage = await BrowserRecipe.openSettingsLoginApprove(t, browser, acct); + await SetupPageRecipe.manualEnter(settingsPage, 'flowcrypt.test.key.used.pgp', { submitPubkey: false, usedPgpBefore: false }, + { isSavePassphraseChecked: false, isSavePassphraseHidden: false }); + // add a name to one of the contacts + const dbPage = await browser.newPage(t, TestUrls.extension('chrome/dev/ci_unit_test.htm')); + await dbPage.page.evaluate(async () => { + const db = await (window as any).ContactStore.dbOpen(); + await (window as any).ContactStore.update(db, 'cc@example.com', { name: 'Mr Cc' }); + }); + await dbPage.close(); + const subject = 'PWD encrypted message with FES web portal - some sends fail with BadRequest error'; + // 1. vague Gmail error with partial success + let composePage = await ComposePageRecipe.openStandalone(t, browser, 'user4@standardsubdomainfes.test:8001'); + await ComposePageRecipe.fillMsg(composePage, { + to: 'to@example.com', + cc: 'cc@example.com', + bcc: 'flowcrypt.compatibility@gmail.com' + }, subject); + await composePage.waitAndType('@input-password', 'gO0d-pwd'); + await composePage.waitAndClick('@action-send', { delay: 1 }); + await composePage.waitAndRespondToModal('confirm', 'cancel', + 'Messages to some recipients were sent successfully, while messages to flowcrypt.compatibility@gmail.com, Mr Cc ' + + 'encountered error(s) from Gmail. Please help us improve FlowCrypt by reporting the error to us.'); + await composePage.close(); + expect((await GoogleData.withInitializedData(acct)).searchMessagesBySubject(subject).length).to.equal(1); + // 2. vague Gmail error with all failures + composePage = await ComposePageRecipe.openStandalone(t, browser, 'user4@standardsubdomainfes.test:8001'); + await ComposePageRecipe.fillMsg(composePage, { + cc: 'cc@example.com', + bcc: 'flowcrypt.compatibility@gmail.com' + }, subject); + await composePage.waitAndType('@input-password', 'gO0d-pwd'); + await composePage.waitAndClick('@action-send', { delay: 1 }); + await composePage.waitAndRespondToModal('confirm', 'cancel', + 'Google returned an error when sending message. ' + + 'Please help us improve FlowCrypt by reporting the error to us.'); + await composePage.close(); + expect((await GoogleData.withInitializedData(acct)).searchMessagesBySubject(subject).length).to.equal(1); // + 0 messages + // 3. "invalid To" Gmail error with partial success + composePage = await ComposePageRecipe.openStandalone(t, browser, 'user4@standardsubdomainfes.test:8001'); + await ComposePageRecipe.fillMsg(composePage, { + to: 'invalid@example.com', + cc: 'to@example.com' + }, subject); + await composePage.waitAndType('@input-password', 'gO0d-pwd'); + await composePage.waitAndClick('@action-send', { delay: 1 }); + await composePage.waitAndRespondToModal('error', 'confirm', + 'Messages to some recipients were sent successfully, while messages to invalid@example.com ' + + 'encountered error(s) from Gmail: Invalid recipients\n\nPlease remove recipients, add them back and re-send the message.'); + await composePage.close(); + expect((await GoogleData.withInitializedData(acct)).searchMessagesBySubject(subject).length).to.equal(2); // + 1 message + // 4. "invalid To" Gmail error with all failures + composePage = await ComposePageRecipe.openStandalone(t, browser, 'user4@standardsubdomainfes.test:8001'); + await ComposePageRecipe.fillMsg(composePage, { + to: 'invalid@example.com', + cc: 'cc@example.com' + }, subject); + await composePage.waitAndType('@input-password', 'gO0d-pwd'); + await composePage.waitAndClick('@action-send', { delay: 1 }); + await composePage.waitAndRespondToModal('error', 'confirm', + 'Error from google: Invalid recipients\n\nPlease remove recipients, add them back and re-send the message.'); + await composePage.close(); + expect((await GoogleData.withInitializedData(acct)).searchMessagesBySubject(subject).length).to.equal(2); // + 0 messages + // 5. "RequestTimeout" error with partial success + composePage = await ComposePageRecipe.openStandalone(t, browser, 'user4@standardsubdomainfes.test:8001'); + await ComposePageRecipe.fillMsg(composePage, { + to: 'timeout@example.com', + cc: 'to@example.com' + }, subject); + await composePage.waitAndType('@input-password', 'gO0d-pwd'); + await composePage.waitAndClick('@action-send', { delay: 1 }); + await composePage.waitAndRespondToModal('error', 'confirm', + 'Messages to some recipients were sent successfully, while messages to timeout@example.com ' + + 'encountered network errors. Please check your internet connection and try again.'); + await composePage.close(); + expect((await GoogleData.withInitializedData(acct)).searchMessagesBySubject(subject).length).to.equal(3); // + 1 message + // 6. "RequestTimeout" error with all failures + composePage = await ComposePageRecipe.openStandalone(t, browser, 'user4@standardsubdomainfes.test:8001'); + await ComposePageRecipe.fillMsg(composePage, { + to: 'timeout@example.com', + cc: 'cc@example.com' + }, subject); + await composePage.waitAndType('@input-password', 'gO0d-pwd'); + await composePage.waitAndClick('@action-send', { delay: 1 }); + await composePage.waitAndRespondToModal('error', 'confirm', + 'Could not send message due to network error. Please check your internet connection and try again. ' + + '(This may also be caused by missing extension permissions).'); + await composePage.close(); + expect((await GoogleData.withInitializedData(acct)).searchMessagesBySubject(subject).length).to.equal(3); // + 0 messages + // this test is using PwdEncryptedMessageWithFesReplyBadRequestTestStrategy to check sent result based on subject "PWD encrypted message with FES web portal - some sends fail with BadRequest error" + // also see '/api/v1/message' in fes-endpoints.ts mock + })); + ava.default('first.key.revoked@key-manager-autoimport-no-prv-create.flowcrypt.test - selects valid own key when saving draft or sending', testWithBrowser(undefined, async (t, browser) => { const acct = 'first.key.revoked@key-manager-autoimport-no-prv-create.flowcrypt.test'; const settingsPage = await BrowserRecipe.openSettingsLoginApprove(t, browser, acct); From 553612d205cf3d2aca5bfc026036430d99b46b23 Mon Sep 17 00:00:00 2001 From: Roman Shevchenko Date: Wed, 25 May 2022 15:52:50 +0000 Subject: [PATCH 19/45] test fix / lint fix --- .../compose-modules/compose-err-module.ts | 11 +++++------ test/source/tests/compose.ts | 15 ++++++++------- 2 files changed, 13 insertions(+), 13 deletions(-) diff --git a/extension/chrome/elements/compose-modules/compose-err-module.ts b/extension/chrome/elements/compose-modules/compose-err-module.ts index 349973ec224..a225c439ca3 100644 --- a/extension/chrome/elements/compose-modules/compose-err-module.ts +++ b/extension/chrome/elements/compose-modules/compose-err-module.ts @@ -27,6 +27,11 @@ export class ComposeErrModule extends ViewModule { private debugId = Str.sloppyRandom(); + private static getErrSayingSomeMessagesHaveBeenSent = (sendMsgsResult: SendMsgsResult) => { + return 'Messages to some recipients were sent successfully, while messages to ' + + Str.formatEmailList(sendMsgsResult.failures.map(el => el.recipient)) + ' encountered '; + } + public handle = (couldNotDoWhat: string): BrowserEventErrHandler => { return { network: async () => await Ui.modal.info(`Could not ${couldNotDoWhat} (network error). Please try again.`), @@ -62,7 +67,6 @@ export class ComposeErrModule extends ViewModule { }; public handleSendErr = async (e: any, sendMsgsResult?: SendMsgsResult) => { - // this.view.errModule.debug(`handleSendErr: ${String(e)}`); if (ApiErr.isNetErr(e)) { let netErrMsg: string | undefined; @@ -175,9 +179,4 @@ export class ComposeErrModule extends ViewModule { } }; - private static getErrSayingSomeMessagesHaveBeenSent = (sendMsgsResult: SendMsgsResult) => { - return 'Messages to some recipients were sent successfully, while messages to ' + - Str.formatEmailList(sendMsgsResult.failures.map(el => el.recipient)) + ' encountered '; - } - } diff --git a/test/source/tests/compose.ts b/test/source/tests/compose.ts index c1812944b54..182ced3f2d7 100644 --- a/test/source/tests/compose.ts +++ b/test/source/tests/compose.ts @@ -1691,7 +1691,8 @@ export const defineComposeTests = (testVariant: TestVariant, testWithBrowser: Te await (window as any).ContactStore.update(db, 'cc@example.com', { name: 'Mr Cc' }); }); await dbPage.close(); - const subject = 'PWD encrypted message with FES web portal - some sends fail with BadRequest error'; + const subject = 'PWD encrypted message with FES web portal - some sends fail with BadRequest error - ' + testVariant; + let expectedNumberOfPassedMessages = (await GoogleData.withInitializedData(acct)).searchMessagesBySubject(subject).length; // 1. vague Gmail error with partial success let composePage = await ComposePageRecipe.openStandalone(t, browser, 'user4@standardsubdomainfes.test:8001'); await ComposePageRecipe.fillMsg(composePage, { @@ -1705,7 +1706,7 @@ export const defineComposeTests = (testVariant: TestVariant, testWithBrowser: Te 'Messages to some recipients were sent successfully, while messages to flowcrypt.compatibility@gmail.com, Mr Cc ' + 'encountered error(s) from Gmail. Please help us improve FlowCrypt by reporting the error to us.'); await composePage.close(); - expect((await GoogleData.withInitializedData(acct)).searchMessagesBySubject(subject).length).to.equal(1); + expect((await GoogleData.withInitializedData(acct)).searchMessagesBySubject(subject).length).to.equal(++expectedNumberOfPassedMessages); // 2. vague Gmail error with all failures composePage = await ComposePageRecipe.openStandalone(t, browser, 'user4@standardsubdomainfes.test:8001'); await ComposePageRecipe.fillMsg(composePage, { @@ -1718,7 +1719,7 @@ export const defineComposeTests = (testVariant: TestVariant, testWithBrowser: Te 'Google returned an error when sending message. ' + 'Please help us improve FlowCrypt by reporting the error to us.'); await composePage.close(); - expect((await GoogleData.withInitializedData(acct)).searchMessagesBySubject(subject).length).to.equal(1); // + 0 messages + expect((await GoogleData.withInitializedData(acct)).searchMessagesBySubject(subject).length).to.equal(expectedNumberOfPassedMessages); // + 0 messages // 3. "invalid To" Gmail error with partial success composePage = await ComposePageRecipe.openStandalone(t, browser, 'user4@standardsubdomainfes.test:8001'); await ComposePageRecipe.fillMsg(composePage, { @@ -1731,7 +1732,7 @@ export const defineComposeTests = (testVariant: TestVariant, testWithBrowser: Te 'Messages to some recipients were sent successfully, while messages to invalid@example.com ' + 'encountered error(s) from Gmail: Invalid recipients\n\nPlease remove recipients, add them back and re-send the message.'); await composePage.close(); - expect((await GoogleData.withInitializedData(acct)).searchMessagesBySubject(subject).length).to.equal(2); // + 1 message + expect((await GoogleData.withInitializedData(acct)).searchMessagesBySubject(subject).length).to.equal(++expectedNumberOfPassedMessages); // 4. "invalid To" Gmail error with all failures composePage = await ComposePageRecipe.openStandalone(t, browser, 'user4@standardsubdomainfes.test:8001'); await ComposePageRecipe.fillMsg(composePage, { @@ -1743,7 +1744,7 @@ export const defineComposeTests = (testVariant: TestVariant, testWithBrowser: Te await composePage.waitAndRespondToModal('error', 'confirm', 'Error from google: Invalid recipients\n\nPlease remove recipients, add them back and re-send the message.'); await composePage.close(); - expect((await GoogleData.withInitializedData(acct)).searchMessagesBySubject(subject).length).to.equal(2); // + 0 messages + expect((await GoogleData.withInitializedData(acct)).searchMessagesBySubject(subject).length).to.equal(expectedNumberOfPassedMessages); // + 0 messages // 5. "RequestTimeout" error with partial success composePage = await ComposePageRecipe.openStandalone(t, browser, 'user4@standardsubdomainfes.test:8001'); await ComposePageRecipe.fillMsg(composePage, { @@ -1756,7 +1757,7 @@ export const defineComposeTests = (testVariant: TestVariant, testWithBrowser: Te 'Messages to some recipients were sent successfully, while messages to timeout@example.com ' + 'encountered network errors. Please check your internet connection and try again.'); await composePage.close(); - expect((await GoogleData.withInitializedData(acct)).searchMessagesBySubject(subject).length).to.equal(3); // + 1 message + expect((await GoogleData.withInitializedData(acct)).searchMessagesBySubject(subject).length).to.equal(++expectedNumberOfPassedMessages); // 6. "RequestTimeout" error with all failures composePage = await ComposePageRecipe.openStandalone(t, browser, 'user4@standardsubdomainfes.test:8001'); await ComposePageRecipe.fillMsg(composePage, { @@ -1769,7 +1770,7 @@ export const defineComposeTests = (testVariant: TestVariant, testWithBrowser: Te 'Could not send message due to network error. Please check your internet connection and try again. ' + '(This may also be caused by missing extension permissions).'); await composePage.close(); - expect((await GoogleData.withInitializedData(acct)).searchMessagesBySubject(subject).length).to.equal(3); // + 0 messages + expect((await GoogleData.withInitializedData(acct)).searchMessagesBySubject(subject).length).to.equal(expectedNumberOfPassedMessages); // + 0 messages // this test is using PwdEncryptedMessageWithFesReplyBadRequestTestStrategy to check sent result based on subject "PWD encrypted message with FES web portal - some sends fail with BadRequest error" // also see '/api/v1/message' in fes-endpoints.ts mock })); From 3da55d4bd92b201e7c4db7181199b231c82511c2 Mon Sep 17 00:00:00 2001 From: Roman Shevchenko Date: Wed, 25 May 2022 16:36:30 +0000 Subject: [PATCH 20/45] lint fix --- test/source/tests/compose.ts | 31 ++++++------------------------- 1 file changed, 6 insertions(+), 25 deletions(-) diff --git a/test/source/tests/compose.ts b/test/source/tests/compose.ts index 182ced3f2d7..841e78a44a2 100644 --- a/test/source/tests/compose.ts +++ b/test/source/tests/compose.ts @@ -1695,11 +1695,7 @@ export const defineComposeTests = (testVariant: TestVariant, testWithBrowser: Te let expectedNumberOfPassedMessages = (await GoogleData.withInitializedData(acct)).searchMessagesBySubject(subject).length; // 1. vague Gmail error with partial success let composePage = await ComposePageRecipe.openStandalone(t, browser, 'user4@standardsubdomainfes.test:8001'); - await ComposePageRecipe.fillMsg(composePage, { - to: 'to@example.com', - cc: 'cc@example.com', - bcc: 'flowcrypt.compatibility@gmail.com' - }, subject); + await ComposePageRecipe.fillMsg(composePage, { to: 'to@example.com', cc: 'cc@example.com', bcc: 'flowcrypt.compatibility@gmail.com' }, subject); await composePage.waitAndType('@input-password', 'gO0d-pwd'); await composePage.waitAndClick('@action-send', { delay: 1 }); await composePage.waitAndRespondToModal('confirm', 'cancel', @@ -1709,10 +1705,7 @@ export const defineComposeTests = (testVariant: TestVariant, testWithBrowser: Te expect((await GoogleData.withInitializedData(acct)).searchMessagesBySubject(subject).length).to.equal(++expectedNumberOfPassedMessages); // 2. vague Gmail error with all failures composePage = await ComposePageRecipe.openStandalone(t, browser, 'user4@standardsubdomainfes.test:8001'); - await ComposePageRecipe.fillMsg(composePage, { - cc: 'cc@example.com', - bcc: 'flowcrypt.compatibility@gmail.com' - }, subject); + await ComposePageRecipe.fillMsg(composePage, { cc: 'cc@example.com', bcc: 'flowcrypt.compatibility@gmail.com' }, subject); await composePage.waitAndType('@input-password', 'gO0d-pwd'); await composePage.waitAndClick('@action-send', { delay: 1 }); await composePage.waitAndRespondToModal('confirm', 'cancel', @@ -1722,10 +1715,7 @@ export const defineComposeTests = (testVariant: TestVariant, testWithBrowser: Te expect((await GoogleData.withInitializedData(acct)).searchMessagesBySubject(subject).length).to.equal(expectedNumberOfPassedMessages); // + 0 messages // 3. "invalid To" Gmail error with partial success composePage = await ComposePageRecipe.openStandalone(t, browser, 'user4@standardsubdomainfes.test:8001'); - await ComposePageRecipe.fillMsg(composePage, { - to: 'invalid@example.com', - cc: 'to@example.com' - }, subject); + await ComposePageRecipe.fillMsg(composePage, { to: 'invalid@example.com', cc: 'to@example.com' }, subject); await composePage.waitAndType('@input-password', 'gO0d-pwd'); await composePage.waitAndClick('@action-send', { delay: 1 }); await composePage.waitAndRespondToModal('error', 'confirm', @@ -1735,10 +1725,7 @@ export const defineComposeTests = (testVariant: TestVariant, testWithBrowser: Te expect((await GoogleData.withInitializedData(acct)).searchMessagesBySubject(subject).length).to.equal(++expectedNumberOfPassedMessages); // 4. "invalid To" Gmail error with all failures composePage = await ComposePageRecipe.openStandalone(t, browser, 'user4@standardsubdomainfes.test:8001'); - await ComposePageRecipe.fillMsg(composePage, { - to: 'invalid@example.com', - cc: 'cc@example.com' - }, subject); + await ComposePageRecipe.fillMsg(composePage, { to: 'invalid@example.com', cc: 'cc@example.com' }, subject); await composePage.waitAndType('@input-password', 'gO0d-pwd'); await composePage.waitAndClick('@action-send', { delay: 1 }); await composePage.waitAndRespondToModal('error', 'confirm', @@ -1747,10 +1734,7 @@ export const defineComposeTests = (testVariant: TestVariant, testWithBrowser: Te expect((await GoogleData.withInitializedData(acct)).searchMessagesBySubject(subject).length).to.equal(expectedNumberOfPassedMessages); // + 0 messages // 5. "RequestTimeout" error with partial success composePage = await ComposePageRecipe.openStandalone(t, browser, 'user4@standardsubdomainfes.test:8001'); - await ComposePageRecipe.fillMsg(composePage, { - to: 'timeout@example.com', - cc: 'to@example.com' - }, subject); + await ComposePageRecipe.fillMsg(composePage, { to: 'timeout@example.com', cc: 'to@example.com' }, subject); await composePage.waitAndType('@input-password', 'gO0d-pwd'); await composePage.waitAndClick('@action-send', { delay: 1 }); await composePage.waitAndRespondToModal('error', 'confirm', @@ -1760,10 +1744,7 @@ export const defineComposeTests = (testVariant: TestVariant, testWithBrowser: Te expect((await GoogleData.withInitializedData(acct)).searchMessagesBySubject(subject).length).to.equal(++expectedNumberOfPassedMessages); // 6. "RequestTimeout" error with all failures composePage = await ComposePageRecipe.openStandalone(t, browser, 'user4@standardsubdomainfes.test:8001'); - await ComposePageRecipe.fillMsg(composePage, { - to: 'timeout@example.com', - cc: 'cc@example.com' - }, subject); + await ComposePageRecipe.fillMsg(composePage, { to: 'timeout@example.com', cc: 'cc@example.com' }, subject); await composePage.waitAndType('@input-password', 'gO0d-pwd'); await composePage.waitAndClick('@action-send', { delay: 1 }); await composePage.waitAndRespondToModal('error', 'confirm', From 6d3df57d47df2c2420ccfd6f601285434f8f6ab4 Mon Sep 17 00:00:00 2001 From: Roman Shevchenko Date: Wed, 25 May 2022 16:57:11 +0000 Subject: [PATCH 21/45] lint fix --- extension/chrome/elements/compose-modules/compose-err-module.ts | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/extension/chrome/elements/compose-modules/compose-err-module.ts b/extension/chrome/elements/compose-modules/compose-err-module.ts index a225c439ca3..8b493005f1e 100644 --- a/extension/chrome/elements/compose-modules/compose-err-module.ts +++ b/extension/chrome/elements/compose-modules/compose-err-module.ts @@ -30,7 +30,7 @@ export class ComposeErrModule extends ViewModule { private static getErrSayingSomeMessagesHaveBeenSent = (sendMsgsResult: SendMsgsResult) => { return 'Messages to some recipients were sent successfully, while messages to ' + Str.formatEmailList(sendMsgsResult.failures.map(el => el.recipient)) + ' encountered '; - } + }; public handle = (couldNotDoWhat: string): BrowserEventErrHandler => { return { From 9f74a85a5a34c18f5f3d94c96fbf4466fe5774e5 Mon Sep 17 00:00:00 2001 From: Roman Shevchenko Date: Thu, 2 Jun 2022 11:41:51 +0000 Subject: [PATCH 22/45] display a supplementary operation error as a toast --- .../compose-send-btn-module.ts | 56 ++++++++++++------- .../elements/compose-modules/compose-types.ts | 3 +- test/source/mock/fes/fes-endpoints.ts | 19 ++++++- test/source/mock/google/google-endpoints.ts | 4 +- .../strategies/send-message-strategy.ts | 2 + test/source/tests/compose.ts | 23 ++++++++ 6 files changed, 80 insertions(+), 27 deletions(-) diff --git a/extension/chrome/elements/compose-modules/compose-send-btn-module.ts b/extension/chrome/elements/compose-modules/compose-send-btn-module.ts index 90ae7d7a537..2492828cd67 100644 --- a/extension/chrome/elements/compose-modules/compose-send-btn-module.ts +++ b/extension/chrome/elements/compose-modules/compose-send-btn-module.ts @@ -119,10 +119,24 @@ export class ComposeSendBtnModule extends ViewModule { await this.finalizeSendableMsg({ msg, senderKi: msgObj.senderKi }); } const result = await this.doSendMsgs(msgObj); - if (result.failures.length) { + if (!result.failures.length) { + // toast isn't supported together with a confirmation/alert popup + if (result.supplementaryOperationsErrors.length) { + console.error(result.supplementaryOperationsErrors); + Catch.setHandledTimeout(() => { + Ui.toast(result.supplementaryOperationsErrors[0]); + }, 0); + } + BrowserMsg.send.notificationShow(this.view.parentTabId, { notification: `Your ${this.view.isReplyBox ? 'reply' : 'message'} has been sent.` }); + BrowserMsg.send.focusBody(this.view.parentTabId); // Bring focus back to body so Gmails shortcuts will work + if (this.view.isReplyBox) { + this.view.renderModule.renderReplySuccess(msgObj.attachments, msgObj.recipients, result.sentIds[0]); + } else { + this.view.renderModule.closeMsg(); + } + } else { await this.view.errModule.handleSendErr(result.failures[0].e, result); } - // todo: supplementary operation errors? } catch (e) { await this.view.errModule.handleSendErr(e, undefined); } finally { @@ -205,7 +219,8 @@ export class ComposeSendBtnModule extends ViewModule { // and this requests represents second half of uploadable effort. Else this represents all (no previous heavy requests) const progressRepresents = this.view.pwdOrPubkeyContainerModule.isVisible() ? 'SECOND-HALF' : 'EVERYTHING'; const sentIds: string[] = []; - const operations = [this.view.draftModule.draftDelete()]; // todo: don't delete on error + const supplementaryOperations: Promise[] = []; + const supplementaryOperationsErrors: any[] = []; // tslint:disable-line:no-unsafe-any const success: EmailParts[] = []; const failures: { recipient: EmailParts, e: any }[] = []; for (const msg of msgObj.msgs) { @@ -217,15 +232,21 @@ export class ComposeSendBtnModule extends ViewModule { success.push(...msgRecipients); sentIds.push(msgSentRes.id); if (msg.externalId) { - operations.push((async (externalId, id) => { + const operation = (async (externalId, id) => { const gmailMsg = await this.view.emailProvider.msgGet(id, 'metadata'); const messageId = GmailParser.findHeader(gmailMsg, 'message-id'); if (messageId) { await this.view.acctServer.messageGatewayUpdate(externalId, messageId); } else { - Catch.report('Failed to extract Message-ID of sent message'); + throw new Error('Failed to extract Message-ID of sent message'); + } + })(msg.externalId, msgSentRes.id); + supplementaryOperations.push(operation.catch( + e => { + supplementaryOperationsErrors.push(`Failed to bind Gateway ID of the message: ${e}`); + Catch.reportErr(e); } - })(msg.externalId, msgSentRes.id)); + )); } } catch (e) { if (msg.thread && ApiErr.isNotFound(e) && this.view.threadId) { // cannot send msg because threadId not found - eg user since deleted it @@ -239,24 +260,17 @@ export class ComposeSendBtnModule extends ViewModule { break; } // while loop for thread retry } - BrowserMsg.send.notificationShow(this.view.parentTabId, { notification: `Your ${this.view.isReplyBox ? 'reply' : 'message'} has been sent.` }); - BrowserMsg.send.focusBody(this.view.parentTabId); // Bring focus back to body so Gmails shortcuts will work + const isSentToAllRecipients = !failures.length; try { - await Promise.all(operations); - } catch (e) { - // todo: bindings or draft delete failed, not really critical, add to supplementary errors - } - this.isSendMessageInProgress = false; - if (!failures.length) { - // todo: supplementary errors - // closing the composer on successful send - if (this.view.isReplyBox) { - this.view.renderModule.renderReplySuccess(msgObj.attachments, msgObj.recipients, sentIds[0]); - } else { - this.view.renderModule.closeMsg(); + if (isSentToAllRecipients) { + supplementaryOperations.push(this.view.draftModule.draftDelete()); } + await Promise.all(supplementaryOperations); + } catch (e) { + Catch.reportErr(e); + supplementaryOperationsErrors.push(e); } - return { success, failures, supplementaryOperationsError: undefined }; // todo: + return { success, failures, supplementaryOperationsErrors, sentIds }; }; } diff --git a/extension/chrome/elements/compose-modules/compose-types.ts b/extension/chrome/elements/compose-modules/compose-types.ts index 6c1732c74e2..e8498d414e6 100644 --- a/extension/chrome/elements/compose-modules/compose-types.ts +++ b/extension/chrome/elements/compose-modules/compose-types.ts @@ -69,5 +69,6 @@ export const getUniqueRecipientEmails = (recipients: ParsedRecipients) => { export type SendMsgsResult = { success: EmailParts[], failures: { recipient: EmailParts, e: any }[], - supplementaryOperationsError: any + supplementaryOperationsErrors: any[], + sentIds: string[] }; diff --git a/test/source/mock/fes/fes-endpoints.ts b/test/source/mock/fes/fes-endpoints.ts index 6b6346d6c82..70083458d38 100644 --- a/test/source/mock/fes/fes-endpoints.ts +++ b/test/source/mock/fes/fes-endpoints.ts @@ -5,7 +5,7 @@ import { IncomingMessage } from 'http'; import { Buf } from '../../core/buf'; import { MsgUtil } from '../../core/crypto/pgp/msg-util'; import { HandlersDefinition } from '../all-apis-mock'; -import { HttpClientErr } from '../lib/api'; +import { HttpClientErr, Status } from '../lib/api'; import { MockJwt } from '../lib/oauth'; const standardFesUrl = 'fes.standardsubdomainfes.test:8001'; @@ -144,6 +144,8 @@ const processMessageFromUser4 = async (body: string) => { externalId: 'FES-MOCK-EXTERNAL-FOR-INVALID@EXAMPLE.COM-ID' }; } + // we can add a clause for timeout@example.com here, but it's not necessary as without it the recipient goes to the legacy clause + // and the test is still valid if (body.includes("Mr Cc ")) { response.emailToExternalIdAndUrl['cc@example.com'] = { url: `http://${standardFesUrl}/message/FES-MOCK-MESSAGE-FOR-CC@EXAMPLE.COM-ID`, @@ -156,6 +158,12 @@ const processMessageFromUser4 = async (body: string) => { externalId: 'FES-MOCK-EXTERNAL-FOR-FLOWCRYPT.COMPATIBILITY@GMAIL.COM-ID' }; } + if (body.includes("gatewayfailure@example.com")) { + response.emailToExternalIdAndUrl['gatewayfailure@example.com'] = { + url: `http://${standardFesUrl}/message/FES-MOCK-MESSAGE-FOR-GATEWAYFAILURE@EXAMPLE.COM-ID`, + externalId: 'FES-MOCK-EXTERNAL-FOR-GATEWAYFAILURE@EXAMPLE.COM-ID' + }; + } return response; }; @@ -224,7 +232,6 @@ export const mockFesEndpoints: HandlersDefinition = { }, '/api/v1/message/FES-MOCK-EXTERNAL-ID/gateway': async ({ body }, req) => { if (req.headers.host === standardFesUrl && req.method === 'POST') { - // todo: remove legacy endpoint? // test: `compose - user@standardsubdomainfes.test:8001 - PWD encrypted message with FES web portal` authenticate(req, 'oidc'); expect(body).to.match(/{"emailGatewayMessageId":"<(.+)@standardsubdomainfes.test:8001>"}/); @@ -245,6 +252,8 @@ export const mockFesEndpoints: HandlersDefinition = { if (req.headers.host === standardFesUrl && req.method === 'POST') { // test: `compose - user@standardsubdomainfes.test:8001 - PWD encrypted message with FES web portal` // test: `compose - user2@standardsubdomainfes.test:8001 - PWD encrypted message with FES - Reply rendering` + // test: `compose - user3@standardsubdomainfes.test:8001 - PWD encrypted message with FES web portal - pubkey recipient in bcc` + // test: `compose - user4@standardsubdomainfes.test:8001 - PWD encrypted message with FES web portal - some sends fail with BadRequest error` authenticate(req, 'oidc'); expect(body).to.match(/{"emailGatewayMessageId":"<(.+)@standardsubdomainfes.test:8001>"}/); return {}; @@ -259,7 +268,11 @@ export const mockFesEndpoints: HandlersDefinition = { return {}; } throw new HttpClientErr('Not Found', 404); - } + }, + '/api/v1/message/FES-MOCK-EXTERNAL-FOR-GATEWAYFAILURE@EXAMPLE.COM-ID/gateway': async () => { + // test: `user4@standardsubdomainfes.test:8001 - PWD encrypted message with FES web portal - a send fails with gateway update error` + throw new HttpClientErr(`Test error`, Status.BAD_REQUEST); + }, }; const authenticate = (req: IncomingMessage, type: 'oidc' | 'fes'): string => { diff --git a/test/source/mock/google/google-endpoints.ts b/test/source/mock/google/google-endpoints.ts index a7f69d35e8a..41eab9ad714 100644 --- a/test/source/mock/google/google-endpoints.ts +++ b/test/source/mock/google/google-endpoints.ts @@ -15,8 +15,8 @@ type DraftSaveModel = { message: { raw: string, threadId: string } }; const allowedRecipients: Array = ['flowcrypt.compatibility@gmail.com', 'manualcopypgp@flowcrypt.com', 'censored@email.com', 'test@email.com', 'human@flowcrypt.com', 'human+nopgp@flowcrypt.com', 'expired.on.attester@domain.com', - 'ci.tests.gmail@flowcrypt.test', 'smime1@recipient.com', 'smime2@recipient.com', 'smime@recipient.com', - 'smime.attachment@recipient.com', 'auto.refresh.expired.key@recipient.com', 'to@example.com', 'cc@example.com', 'bcc@example.com', + 'ci.tests.gmail@flowcrypt.test', 'smime1@recipient.com', 'smime2@recipient.com', 'smime@recipient.com', 'smime.attachment@recipient.com', + 'auto.refresh.expired.key@recipient.com', 'to@example.com', 'cc@example.com', 'bcc@example.com', 'gatewayfailure@example.com', 'flowcrypt.test.key.multiple.inbox1@gmail.com', 'flowcrypt.test.key.multiple.inbox2@gmail.com', 'mock.only.pubkey@flowcrypt.com', 'vladimir@flowcrypt.com', 'limon.monte@gmail.com', 'sweetalert2@gmail.com', 'sender@domain.com', 'invalid@example.com', 'timeout@example.com']; diff --git a/test/source/mock/google/strategies/send-message-strategy.ts b/test/source/mock/google/strategies/send-message-strategy.ts index bd7cc88ed42..6e281b4724a 100644 --- a/test/source/mock/google/strategies/send-message-strategy.ts +++ b/test/source/mock/google/strategies/send-message-strategy.ts @@ -354,6 +354,8 @@ export class TestBySubjectStrategyContext { this.strategy = new PwdEncryptedMessageWithFesPubkeyRecipientInBccTestStrategy(); } else if (subject.includes('PWD encrypted message with FES web portal - some sends fail with BadRequest error')) { this.strategy = new PwdEncryptedMessageWithFesReplyBadRequestTestStrategy(); + } else if (subject.includes('PWD encrypted message with FES web portal - a send fails with gateway update error')) { + this.strategy = new SaveMessageInStorageStrategy(); } else if (subject.includes('Message With Image')) { this.strategy = new SaveMessageInStorageStrategy(); } else if (subject.includes('Message With Test Text')) { diff --git a/test/source/tests/compose.ts b/test/source/tests/compose.ts index 841e78a44a2..6f4349b6210 100644 --- a/test/source/tests/compose.ts +++ b/test/source/tests/compose.ts @@ -1756,6 +1756,29 @@ export const defineComposeTests = (testVariant: TestVariant, testWithBrowser: Te // also see '/api/v1/message' in fes-endpoints.ts mock })); + /** + * You need the following lines in /etc/hosts: + * 127.0.0.1 standardsubdomainfes.test + * 127.0.0.1 fes.standardsubdomainfes.test + */ + ava.default('user4@standardsubdomainfes.test:8001 - PWD encrypted message with FES web portal - a send fails with gateway update error', testWithBrowser(undefined, async (t, browser) => { + const acct = 'user4@standardsubdomainfes.test:8001'; // added port to trick extension into calling the mock + const settingsPage = await BrowserRecipe.openSettingsLoginApprove(t, browser, acct); + await SetupPageRecipe.manualEnter(settingsPage, 'flowcrypt.test.key.used.pgp', { submitPubkey: false, usedPgpBefore: false }, + { isSavePassphraseChecked: false, isSavePassphraseHidden: false }); + const subject = 'PWD encrypted message with FES web portal - a send fails with gateway update error - ' + testVariant; + const expectedNumberOfPassedMessages = (await GoogleData.withInitializedData(acct)).searchMessagesBySubject(subject).length; + const composePage = await ComposePageRecipe.openStandalone(t, browser, 'user4@standardsubdomainfes.test:8001'); + await ComposePageRecipe.fillMsg(composePage, { to: 'gatewayfailure@example.com' }, subject); + await composePage.waitAndType('@input-password', 'gO0d-pwd'); + await composePage.waitAndClick('@action-send', { delay: 1 }); + await composePage.waitForContent('.ui-toast-title', 'Failed to bind Gateway ID of the message:'); + await composePage.close(); + expect((await GoogleData.withInitializedData(acct)).searchMessagesBySubject(subject).length).to.equal(expectedNumberOfPassedMessages + 1); + // this test is using PwdEncryptedMessageWithFesReplyGatewayErrorTestStrategy to check sent result based on subject "PWD encrypted message with FES web portal - a send fails with gateway update error" + // also see '/api/v1/message' in fes-endpoints.ts mock + })); + ava.default('first.key.revoked@key-manager-autoimport-no-prv-create.flowcrypt.test - selects valid own key when saving draft or sending', testWithBrowser(undefined, async (t, browser) => { const acct = 'first.key.revoked@key-manager-autoimport-no-prv-create.flowcrypt.test'; const settingsPage = await BrowserRecipe.openSettingsLoginApprove(t, browser, acct); From 4ea2be60372b864192e65230aa2e8446f3cd0de6 Mon Sep 17 00:00:00 2001 From: Roman Shevchenko Date: Thu, 2 Jun 2022 13:55:17 +0000 Subject: [PATCH 23/45] lint fix --- .../chrome/elements/compose-modules/compose-send-btn-module.ts | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/extension/chrome/elements/compose-modules/compose-send-btn-module.ts b/extension/chrome/elements/compose-modules/compose-send-btn-module.ts index 2492828cd67..66a7f04763d 100644 --- a/extension/chrome/elements/compose-modules/compose-send-btn-module.ts +++ b/extension/chrome/elements/compose-modules/compose-send-btn-module.ts @@ -124,7 +124,7 @@ export class ComposeSendBtnModule extends ViewModule { if (result.supplementaryOperationsErrors.length) { console.error(result.supplementaryOperationsErrors); Catch.setHandledTimeout(() => { - Ui.toast(result.supplementaryOperationsErrors[0]); + Ui.toast(result.supplementaryOperationsErrors[0]); // tslint:disable-line:no-unsafe-any }, 0); } BrowserMsg.send.notificationShow(this.view.parentTabId, { notification: `Your ${this.view.isReplyBox ? 'reply' : 'message'} has been sent.` }); From c10df8f6ca54c65b9ed81be4697a9371e6c77ea6 Mon Sep 17 00:00:00 2001 From: Roman Shevchenko Date: Thu, 2 Jun 2022 14:01:56 +0000 Subject: [PATCH 24/45] lint fix --- test/source/mock/fes/fes-endpoints.ts | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/test/source/mock/fes/fes-endpoints.ts b/test/source/mock/fes/fes-endpoints.ts index 70083458d38..29143f68981 100644 --- a/test/source/mock/fes/fes-endpoints.ts +++ b/test/source/mock/fes/fes-endpoints.ts @@ -145,7 +145,7 @@ const processMessageFromUser4 = async (body: string) => { }; } // we can add a clause for timeout@example.com here, but it's not necessary as without it the recipient goes to the legacy clause - // and the test is still valid + // and the test is still valid if (body.includes("Mr Cc ")) { response.emailToExternalIdAndUrl['cc@example.com'] = { url: `http://${standardFesUrl}/message/FES-MOCK-MESSAGE-FOR-CC@EXAMPLE.COM-ID`, From 14a2d45f8ab929d9981b16a8050c67294ce925fe Mon Sep 17 00:00:00 2001 From: Roman Shevchenko Date: Thu, 2 Jun 2022 14:12:01 +0000 Subject: [PATCH 25/45] don't fail the test on reported 'Test error' --- test/source/test.ts | 2 ++ 1 file changed, 2 insertions(+) diff --git a/test/source/test.ts b/test/source/test.ts index 815bf11b106..6c07a2f452e 100644 --- a/test/source/test.ts +++ b/test/source/test.ts @@ -122,6 +122,8 @@ ava.default.after.always('evaluate Catch.reportErr errors', async t => { // our S/MIME implementation is still early so it throws "reportable" errors like this during tests const usefulErrors = mockBackendData.reportedErrors .filter(e => e.message !== 'Too few bytes to read ASN.1 value.') + // below for test "user4@standardsubdomainfes.test:8001 - PWD encrypted message with FES web portal - a send fails with gateway update error" + .filter(e => !e.message.includes('Test error')) // below for test "no.fes@example.com - skip FES on consumer, show friendly message on enterprise" .filter(e => !e.trace.includes('-1 when GET-ing https://fes.example.com')) // todo - ideally mock tests would never call this. But we do tests with human@flowcrypt.com so it's calling here From 44f54d3c849ff67e69faa416464dca5a9931d1e4 Mon Sep 17 00:00:00 2001 From: Roman Shevchenko Date: Thu, 2 Jun 2022 14:15:45 +0000 Subject: [PATCH 26/45] simplify --- .../chrome/elements/compose-modules/compose-send-btn-module.ts | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/extension/chrome/elements/compose-modules/compose-send-btn-module.ts b/extension/chrome/elements/compose-modules/compose-send-btn-module.ts index 66a7f04763d..0816dd3978d 100644 --- a/extension/chrome/elements/compose-modules/compose-send-btn-module.ts +++ b/extension/chrome/elements/compose-modules/compose-send-btn-module.ts @@ -260,9 +260,8 @@ export class ComposeSendBtnModule extends ViewModule { break; } // while loop for thread retry } - const isSentToAllRecipients = !failures.length; try { - if (isSentToAllRecipients) { + if (!failures.length) { supplementaryOperations.push(this.view.draftModule.draftDelete()); } await Promise.all(supplementaryOperations); From 90d2f603df96d60f595aee3a59543eb8a0483cfe Mon Sep 17 00:00:00 2001 From: Roman Shevchenko Date: Thu, 9 Jun 2022 12:42:13 +0000 Subject: [PATCH 27/45] Refactored attempSendMsg and bindMessage into separate functions --- .../compose-send-btn-module.ts | 73 ++++++++++--------- 1 file changed, 39 insertions(+), 34 deletions(-) diff --git a/extension/chrome/elements/compose-modules/compose-send-btn-module.ts b/extension/chrome/elements/compose-modules/compose-send-btn-module.ts index 0816dd3978d..5936bd9e21e 100644 --- a/extension/chrome/elements/compose-modules/compose-send-btn-module.ts +++ b/extension/chrome/elements/compose-modules/compose-send-btn-module.ts @@ -214,51 +214,56 @@ export class ComposeSendBtnModule extends ViewModule { return { mimeType, data }; }; + private sendAttempt = async (msg: SendableMsg): Promise => { + const progressRepresents = this.view.pwdOrPubkeyContainerModule.isVisible() ? 'SECOND-HALF' : 'EVERYTHING'; + try { + return await this.view.emailProvider.msgSend(msg, (p) => this.renderUploadProgress(p, progressRepresents)); + } catch (e) { + if (msg.thread && ApiErr.isNotFound(e) && this.view.threadId) { // cannot send msg because threadId not found - eg user since deleted it + msg.thread = undefined; + // give it another try, this time without msg.thread + return await this.sendAttempt(msg); + } else { + throw e; + } + } + }; + + private bindMessageId = async (externalId: string, id: string, supplementaryOperationsErrors: any[]) => { + try { + const gmailMsg = await this.view.emailProvider.msgGet(id, 'metadata'); + const messageId = GmailParser.findHeader(gmailMsg, 'message-id'); + if (messageId) { + await this.view.acctServer.messageGatewayUpdate(externalId, messageId); + } else { + throw new Error('Failed to extract Message-ID of sent message'); + } + } catch (e) { + supplementaryOperationsErrors.push(`Failed to bind Gateway ID of the message: ${e}`); + Catch.reportErr(e); + } + }; + private doSendMsgs = async (msgObj: MultipleMessages): Promise => { // if this is a password-encrypted message, then we've already shown progress for uploading to backend // and this requests represents second half of uploadable effort. Else this represents all (no previous heavy requests) - const progressRepresents = this.view.pwdOrPubkeyContainerModule.isVisible() ? 'SECOND-HALF' : 'EVERYTHING'; const sentIds: string[] = []; const supplementaryOperations: Promise[] = []; const supplementaryOperationsErrors: any[] = []; // tslint:disable-line:no-unsafe-any const success: EmailParts[] = []; const failures: { recipient: EmailParts, e: any }[] = []; for (const msg of msgObj.msgs) { - let msgSentRes: GmailRes.GmailMsgSend; const msgRecipients = msg.getAllRecipients(); - while (true) { // first try with msg.thread, and then possibly try again without it - try { - msgSentRes = await this.view.emailProvider.msgSend(msg, (p) => this.renderUploadProgress(p, progressRepresents)); - success.push(...msgRecipients); - sentIds.push(msgSentRes.id); - if (msg.externalId) { - const operation = (async (externalId, id) => { - const gmailMsg = await this.view.emailProvider.msgGet(id, 'metadata'); - const messageId = GmailParser.findHeader(gmailMsg, 'message-id'); - if (messageId) { - await this.view.acctServer.messageGatewayUpdate(externalId, messageId); - } else { - throw new Error('Failed to extract Message-ID of sent message'); - } - })(msg.externalId, msgSentRes.id); - supplementaryOperations.push(operation.catch( - e => { - supplementaryOperationsErrors.push(`Failed to bind Gateway ID of the message: ${e}`); - Catch.reportErr(e); - } - )); - } - } catch (e) { - if (msg.thread && ApiErr.isNotFound(e) && this.view.threadId) { // cannot send msg because threadId not found - eg user since deleted it - msg.thread = undefined; - continue; - // give it another try, this time without msg.thread - } else { - failures.push(...msgRecipients.map(recipient => { return { recipient, e }; })); - } + try { + const msgSentRes = await this.sendAttempt(msg); + success.push(...msgRecipients); + sentIds.push(msgSentRes.id); + if (msg.externalId) { + supplementaryOperations.push(this.bindMessageId(msg.externalId, msgSentRes.id, supplementaryOperationsErrors)); } - break; - } // while loop for thread retry + } catch (e) { + failures.push(...msgRecipients.map(recipient => { return { recipient, e }; })); + } } try { if (!failures.length) { From 7f28d8179bcc08db4c6d52369af208437eeda5f8 Mon Sep 17 00:00:00 2001 From: Roman Shevchenko Date: Thu, 9 Jun 2022 13:00:43 +0000 Subject: [PATCH 28/45] rename --- .../elements/compose-modules/compose-send-btn-module.ts | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/extension/chrome/elements/compose-modules/compose-send-btn-module.ts b/extension/chrome/elements/compose-modules/compose-send-btn-module.ts index 5936bd9e21e..63864535390 100644 --- a/extension/chrome/elements/compose-modules/compose-send-btn-module.ts +++ b/extension/chrome/elements/compose-modules/compose-send-btn-module.ts @@ -214,7 +214,7 @@ export class ComposeSendBtnModule extends ViewModule { return { mimeType, data }; }; - private sendAttempt = async (msg: SendableMsg): Promise => { + private attemptSendMsg = async (msg: SendableMsg): Promise => { const progressRepresents = this.view.pwdOrPubkeyContainerModule.isVisible() ? 'SECOND-HALF' : 'EVERYTHING'; try { return await this.view.emailProvider.msgSend(msg, (p) => this.renderUploadProgress(p, progressRepresents)); @@ -222,7 +222,7 @@ export class ComposeSendBtnModule extends ViewModule { if (msg.thread && ApiErr.isNotFound(e) && this.view.threadId) { // cannot send msg because threadId not found - eg user since deleted it msg.thread = undefined; // give it another try, this time without msg.thread - return await this.sendAttempt(msg); + return await this.attemptSendMsg(msg); } else { throw e; } @@ -255,7 +255,7 @@ export class ComposeSendBtnModule extends ViewModule { for (const msg of msgObj.msgs) { const msgRecipients = msg.getAllRecipients(); try { - const msgSentRes = await this.sendAttempt(msg); + const msgSentRes = await this.attemptSendMsg(msg); success.push(...msgRecipients); sentIds.push(msgSentRes.id); if (msg.externalId) { From 7e24a101542161c6b19caeebe7be695ef165ede0 Mon Sep 17 00:00:00 2001 From: Roman Shevchenko Date: Thu, 9 Jun 2022 13:01:57 +0000 Subject: [PATCH 29/45] moved getEmailWithOptionalName to storage module --- .../compose-modules/compose-input-module.ts | 2 +- .../compose-modules/compose-sender-module.ts | 24 ------------------- .../compose-modules/compose-storage-module.ts | 24 +++++++++++++++++++ 3 files changed, 25 insertions(+), 25 deletions(-) diff --git a/extension/chrome/elements/compose-modules/compose-input-module.ts b/extension/chrome/elements/compose-modules/compose-input-module.ts index fb56e960400..1b7fd55fb8b 100644 --- a/extension/chrome/elements/compose-modules/compose-input-module.ts +++ b/extension/chrome/elements/compose-modules/compose-input-module.ts @@ -67,7 +67,7 @@ export class ComposeInputModule extends ViewModule { const plainhtml = this.view.inputModule.extract('html', 'input_text'); const password = this.view.S.cached('input_password').val(); const pwd = typeof password === 'string' && password ? password : undefined; - const from = await this.view.senderModule.getEmailWithOptionalName(this.view.senderModule.getSender()); + const from = await this.view.storageModule.getEmailWithOptionalName(this.view.senderModule.getSender()); return { recipients, subject, plaintext, plainhtml, pwd, from }; }; diff --git a/extension/chrome/elements/compose-modules/compose-sender-module.ts b/extension/chrome/elements/compose-modules/compose-sender-module.ts index e2e7af21d21..0390b61bde9 100644 --- a/extension/chrome/elements/compose-modules/compose-sender-module.ts +++ b/extension/chrome/elements/compose-modules/compose-sender-module.ts @@ -9,8 +9,6 @@ import { Xss } from '../../../js/common/platform/xss.js'; import { ViewModule } from '../../../js/common/view-module.js'; import { ComposeView } from '../compose.js'; import { AcctStore } from '../../../js/common/platform/store/acct-store.js'; -import { EmailParts, Str } from '../../../js/common/core/common.js'; -import { ContactStore } from '../../../js/common/platform/store/contact-store.js'; export class ComposeSenderModule extends ViewModule { @@ -24,28 +22,6 @@ export class ComposeSenderModule extends ViewModule { return this.view.acctEmail; }; - // searches in AcctStore and ContactStore to find name for this email (if it is missing in MIME string) - public getEmailWithOptionalName = async (emailInMimeFormat: string): Promise => { - const parsedEmail = Str.parseEmail(emailInMimeFormat); - if (!parsedEmail.email) { - throw new Error(`Recipient email ${emailInMimeFormat} is not valid`); - } - if (parsedEmail.name) { - return { email: parsedEmail.email, name: parsedEmail.name }; - } - const { sendAs } = await AcctStore.get(this.view.acctEmail, ['sendAs']); - let name: string | undefined; - if (sendAs && sendAs[parsedEmail.email]?.name) { - name = sendAs[parsedEmail.email].name!; - } else { - const contactWithPubKeys = await ContactStore.getOneWithAllPubkeys(undefined, parsedEmail.email); - if (contactWithPubKeys && contactWithPubKeys.info.name) { - name = contactWithPubKeys.info.name; - } - } - return { email: parsedEmail.email, name }; - }; - public renderSendFromOrChevron = async () => { if (this.view.isReplyBox) { const { sendAs } = await AcctStore.get(this.view.acctEmail, ['sendAs']); diff --git a/extension/chrome/elements/compose-modules/compose-storage-module.ts b/extension/chrome/elements/compose-modules/compose-storage-module.ts index 40a6f71858b..59ec8ae14ee 100644 --- a/extension/chrome/elements/compose-modules/compose-storage-module.ts +++ b/extension/chrome/elements/compose-modules/compose-storage-module.ts @@ -17,6 +17,8 @@ import { PassphraseStore } from '../../../js/common/platform/store/passphrase-st import { compareAndSavePubkeysToStorage } from '../../../js/common/shared.js'; import { KeyFamily } from '../../../js/common/core/crypto/key.js'; import { ParsedKeyInfo } from '../../../js/common/core/crypto/key-store-util.js'; +import { EmailParts, Str } from '../../../js/common/core/common.js'; +import { AcctStore } from '../../../js/common/platform/store/acct-store.js'; export class ComposeStorageModule extends ViewModule { @@ -176,6 +178,28 @@ export class ComposeStorageModule extends ViewModule { } }; + // searches in AcctStore and ContactStore to find name for this email (if it is missing in MIME string) + public getEmailWithOptionalName = async (emailInMimeFormat: string): Promise => { + const parsedEmail = Str.parseEmail(emailInMimeFormat); + if (!parsedEmail.email) { + throw new Error(`Recipient email ${emailInMimeFormat} is not valid`); + } + if (parsedEmail.name) { + return { email: parsedEmail.email, name: parsedEmail.name }; + } + const { sendAs } = await AcctStore.get(this.view.acctEmail, ['sendAs']); + let name: string | undefined; + if (sendAs && sendAs[parsedEmail.email]?.name) { + name = sendAs[parsedEmail.email].name!; + } else { + const contactWithPubKeys = await ContactStore.getOneWithAllPubkeys(undefined, parsedEmail.email); + if (contactWithPubKeys && contactWithPubKeys.info.name) { + name = contactWithPubKeys.info.name; + } + } + return { email: parsedEmail.email, name }; + }; + private collectSingleFamilyKeysInternal = async ( family: KeyFamily, senderEmail: string, From a0189ac6e611aa25aa8e112643f12c8ccfce9a58 Mon Sep 17 00:00:00 2001 From: Roman Shevchenko Date: Fri, 10 Jun 2022 12:21:46 +0000 Subject: [PATCH 30/45] using a type guard for RecipientType --- .../elements/compose-modules/compose-recipients-module.ts | 7 +++---- .../formatters/encrypted-mail-msg-formatter.ts | 7 +++---- extension/js/common/api/shared/api.ts | 4 ++++ 3 files changed, 10 insertions(+), 8 deletions(-) diff --git a/extension/chrome/elements/compose-modules/compose-recipients-module.ts b/extension/chrome/elements/compose-modules/compose-recipients-module.ts index 990915b8e43..4ed8d045094 100644 --- a/extension/chrome/elements/compose-modules/compose-recipients-module.ts +++ b/extension/chrome/elements/compose-modules/compose-recipients-module.ts @@ -2,7 +2,7 @@ 'use strict'; -import { ChunkedCb, EmailProviderContact, RecipientType } from '../../../js/common/api/shared/api.js'; +import { Api, ChunkedCb, EmailProviderContact, RecipientType } from '../../../js/common/api/shared/api.js'; import { ContactInfoWithSortedPubkeys, KeyUtil, PubkeyInfo } from '../../../js/common/core/crypto/key.js'; import { PUBKEY_LOOKUP_RESULT_FAIL } from './compose-err-module.js'; import { ProviderContactsQuery, Recipients } from '../../../js/common/api/email-provider/email-provider-api.js'; @@ -160,9 +160,8 @@ export class ComposeRecipientsModule extends ViewModule { public addRecipients = async (recipients: Recipients, triggerCallback: boolean = true) => { const newRecipients: ValidRecipientElement[] = []; - for (const [key, value] of Object.entries(recipients)) { - if (['to', 'cc', 'bcc'].includes(key)) { - const sendingType = key as RecipientType; + for (const [sendingType, value] of Object.entries(recipients)) { + if (Api.isRecipientHeaderNameType(sendingType)) { if (value?.length) { const recipientsContainer = this.view.S.cached('input_addresses_container_outer').find(`#input-container-${sendingType}`); for (const email of value) { diff --git a/extension/chrome/elements/compose-modules/formatters/encrypted-mail-msg-formatter.ts b/extension/chrome/elements/compose-modules/formatters/encrypted-mail-msg-formatter.ts index cbe50c66e64..3985690427e 100644 --- a/extension/chrome/elements/compose-modules/formatters/encrypted-mail-msg-formatter.ts +++ b/extension/chrome/elements/compose-modules/formatters/encrypted-mail-msg-formatter.ts @@ -25,7 +25,7 @@ import { PgpHash } from '../../../../js/common/core/crypto/pgp/pgp-hash.js'; import { UploadedMessageData } from '../../../../js/common/api/account-server.js'; import { ParsedKeyInfo } from '../../../../js/common/core/crypto/key-store-util.js'; import { MultipleMessages } from './general-mail-formatter.js'; -import { RecipientType } from '../../../../js/common/api/shared/api.js'; +import { Api, RecipientType } from '../../../../js/common/api/shared/api.js'; export class EncryptedMsgMailFormatter extends BaseMailFormatter { @@ -41,9 +41,8 @@ export class EncryptedMsgMailFormatter extends BaseMailFormatter { newMsg.pwd = undefined; const collectedAttachments = await this.view.attachmentsModule.attachment.collectEncryptAttachments(pubkeys); const pubkeyRecipients: { [type in RecipientType]?: EmailParts[] } = {}; - for (const [key, value] of Object.entries(newMsg.recipients)) { - if (['to', 'cc', 'bcc'].includes(key)) { - const sendingType = key as RecipientType; + for (const [sendingType, value] of Object.entries(newMsg.recipients)) { + if (Api.isRecipientHeaderNameType(sendingType)) { pubkeyRecipients[sendingType] = value?.filter(emailPart => pubkeys.some(p => p.email === emailPart.email) // pwd recipients that have no personal links go to legacy message || (uploadedMessageData.emailToExternalIdAndUrl || {})[emailPart.email] === undefined); diff --git a/extension/js/common/api/shared/api.ts b/extension/js/common/api/shared/api.ts index 4eb3a70c044..666ee44a3a9 100644 --- a/extension/js/common/api/shared/api.ts +++ b/extension/js/common/api/shared/api.ts @@ -143,6 +143,10 @@ export class Api { return bytes.map(b => ('0' + (b & 0xFF).toString(16)).slice(-2)).join(''); }; + public static isRecipientHeaderNameType = (value: string): value is "to" | "cc" | "bcc" => { + return ['to', 'cc', 'bcc'].includes(value); + } + protected static apiCall = async ( url: string, path: string, From c2c518813e187f17c6b4b7ad1f73d92d0a65e061 Mon Sep 17 00:00:00 2001 From: Roman Shevchenko Date: Fri, 10 Jun 2022 18:03:34 +0000 Subject: [PATCH 31/45] added comments --- .../formatters/encrypted-mail-msg-formatter.ts | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/extension/chrome/elements/compose-modules/formatters/encrypted-mail-msg-formatter.ts b/extension/chrome/elements/compose-modules/formatters/encrypted-mail-msg-formatter.ts index 3985690427e..186ba4efb4e 100644 --- a/extension/chrome/elements/compose-modules/formatters/encrypted-mail-msg-formatter.ts +++ b/extension/chrome/elements/compose-modules/formatters/encrypted-mail-msg-formatter.ts @@ -44,7 +44,8 @@ export class EncryptedMsgMailFormatter extends BaseMailFormatter { for (const [sendingType, value] of Object.entries(newMsg.recipients)) { if (Api.isRecipientHeaderNameType(sendingType)) { pubkeyRecipients[sendingType] = value?.filter(emailPart => pubkeys.some(p => p.email === emailPart.email) - // pwd recipients that have no personal links go to legacy message + // flowcrypt.com/api doesn't return individual links unlike FES + // so pwd recipients without individual links will go to legacy message || (uploadedMessageData.emailToExternalIdAndUrl || {})[emailPart.email] === undefined); } } @@ -54,6 +55,7 @@ export class EncryptedMsgMailFormatter extends BaseMailFormatter { const replyTo = (newMsg.recipients.to ?? []).concat(newMsg.recipients.cc ?? []) .filter(recipient => !uniquePubkeyRecipientToAndCCs.includes(recipient.email.toLowerCase())); const msgs: SendableMsg[] = []; + // pubkey recipients get one combined message. If there are not pubkey recpients, only password - protected messages will be sent if (pubkeyRecipients.to?.length || pubkeyRecipients.cc?.length || pubkeyRecipients.bcc?.length) { const pubkeyMsgData = { ...newMsg, From 2199b85bf654a443d0a4ea32a1bd071e8411f805 Mon Sep 17 00:00:00 2001 From: Roman Shevchenko Date: Fri, 10 Jun 2022 18:13:41 +0000 Subject: [PATCH 32/45] optimizing refactoring --- .../formatters/encrypted-mail-msg-formatter.ts | 13 +++++++------ 1 file changed, 7 insertions(+), 6 deletions(-) diff --git a/extension/chrome/elements/compose-modules/formatters/encrypted-mail-msg-formatter.ts b/extension/chrome/elements/compose-modules/formatters/encrypted-mail-msg-formatter.ts index 186ba4efb4e..3d21530dda3 100644 --- a/extension/chrome/elements/compose-modules/formatters/encrypted-mail-msg-formatter.ts +++ b/extension/chrome/elements/compose-modules/formatters/encrypted-mail-msg-formatter.ts @@ -49,19 +49,20 @@ export class EncryptedMsgMailFormatter extends BaseMailFormatter { || (uploadedMessageData.emailToExternalIdAndUrl || {})[emailPart.email] === undefined); } } - const uniquePubkeyRecipientToAndCCs = Value.arr.unique((pubkeyRecipients.to || []).concat(pubkeyRecipients.cc || []) - .map(recipient => recipient.email.toLowerCase())); - // pubkey recipients should be able to reply to "to" and "cc" pwd recipients - const replyTo = (newMsg.recipients.to ?? []).concat(newMsg.recipients.cc ?? []) - .filter(recipient => !uniquePubkeyRecipientToAndCCs.includes(recipient.email.toLowerCase())); const msgs: SendableMsg[] = []; // pubkey recipients get one combined message. If there are not pubkey recpients, only password - protected messages will be sent if (pubkeyRecipients.to?.length || pubkeyRecipients.cc?.length || pubkeyRecipients.bcc?.length) { + const uniquePubkeyRecipientToAndCCs = Value.arr.unique((pubkeyRecipients.to || []).concat(pubkeyRecipients.cc || []) + .map(recipient => recipient.email.toLowerCase())); + // pubkey recipients should be able to reply to "to" and "cc" pwd recipients + const replyToForMessageSentToPubkeyRecipients = (newMsg.recipients.to ?? []).concat(newMsg.recipients.cc ?? []) + .filter(recipient => !uniquePubkeyRecipientToAndCCs.includes(recipient.email.toLowerCase())); const pubkeyMsgData = { ...newMsg, recipients: pubkeyRecipients, // brackets are required for test emails like '@test:8001' - replyTo: replyTo.length ? `${Str.formatEmailList([newMsg.from, ...replyTo], true)}` : undefined + replyTo: replyToForMessageSentToPubkeyRecipients.length ? `${Str.formatEmailList([newMsg.from, ...replyToForMessageSentToPubkeyRecipients], true)}` + : undefined }; msgs.push(await this.sendablePwdMsg( pubkeyMsgData, From a327facbdef3681ac2a2e94fd4b65d740cb196ba Mon Sep 17 00:00:00 2001 From: Roman Shevchenko Date: Sat, 11 Jun 2022 16:24:54 +0000 Subject: [PATCH 33/45] tslint fix --- extension/js/common/api/shared/api.ts | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/extension/js/common/api/shared/api.ts b/extension/js/common/api/shared/api.ts index 666ee44a3a9..64d085b481e 100644 --- a/extension/js/common/api/shared/api.ts +++ b/extension/js/common/api/shared/api.ts @@ -145,7 +145,7 @@ export class Api { public static isRecipientHeaderNameType = (value: string): value is "to" | "cc" | "bcc" => { return ['to', 'cc', 'bcc'].includes(value); - } + }; protected static apiCall = async ( url: string, From d38ffddde93ecf36c749f1f10617714618e05c14 Mon Sep 17 00:00:00 2001 From: Roman Shevchenko Date: Sat, 11 Jun 2022 16:37:23 +0000 Subject: [PATCH 34/45] refactored msg formatter --- .../encrypted-mail-msg-formatter.ts | 103 +++++++++--------- 1 file changed, 50 insertions(+), 53 deletions(-) diff --git a/extension/chrome/elements/compose-modules/formatters/encrypted-mail-msg-formatter.ts b/extension/chrome/elements/compose-modules/formatters/encrypted-mail-msg-formatter.ts index 3d21530dda3..c868602221f 100644 --- a/extension/chrome/elements/compose-modules/formatters/encrypted-mail-msg-formatter.ts +++ b/extension/chrome/elements/compose-modules/formatters/encrypted-mail-msg-formatter.ts @@ -39,7 +39,7 @@ export class EncryptedMsgMailFormatter extends BaseMailFormatter { // pwdRecipients that have their personal link const pwdRecipients = Object.keys(uploadedMessageData.emailToExternalIdAndUrl ?? {}).filter(email => !pubkeys.some(p => p.email === email)); newMsg.pwd = undefined; - const collectedAttachments = await this.view.attachmentsModule.attachment.collectEncryptAttachments(pubkeys); + const encryptedAttachments = await this.view.attachmentsModule.attachment.collectEncryptAttachments(pubkeys); const pubkeyRecipients: { [type in RecipientType]?: EmailParts[] } = {}; for (const [sendingType, value] of Object.entries(newMsg.recipients)) { if (Api.isRecipientHeaderNameType(sendingType)) { @@ -64,11 +64,11 @@ export class EncryptedMsgMailFormatter extends BaseMailFormatter { replyTo: replyToForMessageSentToPubkeyRecipients.length ? `${Str.formatEmailList([newMsg.from, ...replyToForMessageSentToPubkeyRecipients], true)}` : undefined }; - msgs.push(await this.sendablePwdMsg( + msgs.push(await this.sendablePubkeyMsgWithPwdLink( pubkeyMsgData, pubkeys, { msgUrl: uploadedMessageData.url, externalId: uploadedMessageData.externalId }, - collectedAttachments, + encryptedAttachments, signingKey?.key) ); } @@ -79,9 +79,9 @@ export class EncryptedMsgMailFormatter extends BaseMailFormatter { find(r => r.email.toLowerCase() === recipientEmail.toLowerCase()); // todo: since a message is allowed to have only `cc` or `bcc` without `to`, should we preserve the original placement(s) of the recipient? const individualMsgData = { ...newMsg, recipients: { to: [foundParsedRecipient ?? { email: recipientEmail }] } }; - msgs.push(await this.sendablePwdMsg(individualMsgData, pubkeys, { msgUrl: url, externalId }, [], signingKey?.key)); + msgs.push(await this.sendablePwdMsg(individualMsgData, pubkeys, { msgUrl: url, externalId }, signingKey?.key)); } - return { senderKi: signingKey?.keyInfo, msgs, recipients: newMsg.recipients, attachments: collectedAttachments }; + return { senderKi: signingKey?.keyInfo, msgs, recipients: newMsg.recipients, attachments: encryptedAttachments }; } else { const msg = await this.sendableNonPwdMsg(newMsg, pubkeys, signingKey?.key); return { @@ -94,13 +94,40 @@ export class EncryptedMsgMailFormatter extends BaseMailFormatter { }; public sendableNonPwdMsg = async (newMsg: NewMsgData, pubkeys: PubkeyResult[], signingPrv?: Key): Promise => { - if (this.richtext) { // rich text: PGP/MIME - https://tools.ietf.org/html/rfc3156#section-4 - // or S/MIME - return await this.sendableRichTextMsg(newMsg, pubkeys, signingPrv); - } else { // simple text: PGP or S/MIME Inline with attachments in separate files - // todo: #4046 check attachments for S/MIME - return await this.sendableSimpleTextMsg(newMsg, pubkeys, signingPrv); + const x509certs = pubkeys.map(entry => entry.pubkey).filter(pub => pub.family === 'x509'); + if (x509certs.length) { // s/mime + return await this.sendableSmimeMsg(newMsg, x509certs, signingPrv); + } + const textToEncrypt = this.richtext + ? await Mime.encode({ 'text/plain': newMsg.plaintext, 'text/html': newMsg.plainhtml }, { Subject: newMsg.subject }, + this.isDraft ? [] : await this.view.attachmentsModule.attachment.collectAttachments()) + : newMsg.plaintext; + const { data: encrypted } = await this.encryptDataArmor(Buf.fromUtfStr(textToEncrypt), undefined, pubkeys, signingPrv); + if (!this.richtext || this.isDraft) { + // draft richtext messages go inline as gmail makes it hard (or impossible) to render messages saved as https://tools.ietf.org/html/rfc3156 + return await SendableMsg.createInlineArmored(this.acctEmail, this.headers(newMsg), Buf.fromUint8(encrypted).toUtfStr(), + this.isDraft ? [] : await this.view.attachmentsModule.attachment.collectEncryptAttachments(pubkeys), + { isDraft: this.isDraft }); } + // rich text: PGP/MIME - https://tools.ietf.org/html/rfc3156#section-4 + const attachments = this.createPgpMimeAttachments(encrypted); + return await SendableMsg.createPgpMime(this.acctEmail, this.headers(newMsg), attachments, { isDraft: this.isDraft }); + }; + + private sendablePubkeyMsgWithPwdLink = async ( + newMsg: NewMsgData, + pubs: PubkeyResult[], + { msgUrl, externalId }: { msgUrl: string, externalId?: string }, + encryptedAttachments: Attachment[], + signingPrv?: Key) => { + // encoded as: PGP/MIME-like structure but with attachments as external files due to email size limit (encrypted for pubkeys only) + const msgBody = this.richtext ? { 'text/plain': newMsg.plaintext, 'text/html': newMsg.plainhtml } : { 'text/plain': newMsg.plaintext }; + const pgpMimeNoAttachments = await Mime.encode(msgBody, { Subject: newMsg.subject }, []); // no attachments, attached to email separately + const { data: pubEncryptedNoAttachments } = await this.encryptDataArmor(Buf.fromUtfStr(pgpMimeNoAttachments), undefined, pubs, signingPrv); // encrypted only for pubs + const emailIntroAndLinkBody = await this.formatPwdEncryptedMsgBodyLink(msgUrl); + return await SendableMsg.createPwdMsg(this.acctEmail, this.headers(newMsg), emailIntroAndLinkBody, + this.createPgpMimeAttachments(pubEncryptedNoAttachments).concat(encryptedAttachments), + { isDraft: this.isDraft, externalId }); }; private prepareAndUploadPwdEncryptedMsg = async (newMsg: NewMsgData): Promise => { @@ -151,58 +178,28 @@ export class EncryptedMsgMailFormatter extends BaseMailFormatter { newMsg: NewMsgData, pubs: PubkeyResult[], { msgUrl, externalId }: { msgUrl: string, externalId?: string }, - attachments: Attachment[], signingPrv?: Key) => { // encoded as: PGP/MIME-like structure but with attachments as external files due to email size limit (encrypted for pubkeys only) const msgBody = this.richtext ? { 'text/plain': newMsg.plaintext, 'text/html': newMsg.plainhtml } : { 'text/plain': newMsg.plaintext }; const pgpMimeNoAttachments = await Mime.encode(msgBody, { Subject: newMsg.subject }, []); // no attachments, attached to email separately const { data: pubEncryptedNoAttachments } = await this.encryptDataArmor(Buf.fromUtfStr(pgpMimeNoAttachments), undefined, pubs, signingPrv); // encrypted only for pubs const emailIntroAndLinkBody = await this.formatPwdEncryptedMsgBodyLink(msgUrl); - const headers = this.headers(newMsg); - return await SendableMsg.createPwdMsg(this.acctEmail, headers, emailIntroAndLinkBody, - this.createPgpMimeAttachments(pubEncryptedNoAttachments).concat(attachments), + return await SendableMsg.createPwdMsg(this.acctEmail, this.headers(newMsg), emailIntroAndLinkBody, + this.createPgpMimeAttachments(pubEncryptedNoAttachments), { isDraft: this.isDraft, externalId }); }; - private sendableSimpleTextMsg = async (newMsg: NewMsgData, pubs: PubkeyResult[], signingPrv?: Key): Promise => { - const pubsForEncryption = pubs.map(entry => entry.pubkey); - if (this.isDraft) { - const { data: encrypted } = await this.encryptDataArmor(Buf.fromUtfStr(newMsg.plaintext), undefined, pubs, signingPrv); - return await SendableMsg.createInlineArmored(this.acctEmail, this.headers(newMsg), Buf.fromUint8(encrypted).toUtfStr(), [], { isDraft: this.isDraft }); - } - const x509certs = pubsForEncryption.filter(pub => pub.family === 'x509'); - if (x509certs.length) { // s/mime - const attachments: Attachment[] = this.isDraft ? [] : await this.view.attachmentsModule.attachment.collectAttachments(); // collects attachments - const msgBody = { 'text/plain': newMsg.plaintext }; - const mimeEncodedPlainMessage = await Mime.encode(msgBody, { Subject: newMsg.subject }, attachments); - let mimeData = Buf.fromUtfStr(mimeEncodedPlainMessage); - if (signingPrv) { - const signedMessage = await this.signMimeMessage(signingPrv, mimeEncodedPlainMessage, newMsg); - mimeData = Buf.fromUtfStr(await signedMessage.toMime()); - } - const encryptedMessage = await SmimeKey.encryptMessage({ pubkeys: x509certs, data: mimeData, armor: false }); - const data = encryptedMessage.data; - return await SendableMsg.createSMimeEncrypted(this.acctEmail, this.headers(newMsg), data, { isDraft: this.isDraft }); - } else { // openpgp - const attachments: Attachment[] = this.isDraft ? [] : await this.view.attachmentsModule.attachment.collectEncryptAttachments(pubs); - const encrypted = await this.encryptDataArmor(Buf.fromUtfStr(newMsg.plaintext), undefined, pubs, signingPrv); - return await SendableMsg.createInlineArmored(this.acctEmail, this.headers(newMsg), Buf.fromUint8(encrypted.data).toUtfStr(), attachments, { isDraft: this.isDraft }); - } - }; - - private sendableRichTextMsg = async (newMsg: NewMsgData, pubs: PubkeyResult[], signingPrv?: Key) => { - // todo: pubs.type === 'x509' #4047 - const plainAttachments = this.isDraft ? [] : await this.view.attachmentsModule.attachment.collectAttachments(); - if (this.isDraft) { // this patch is needed as gmail makes it hard (or impossible) to render messages saved as https://tools.ietf.org/html/rfc3156 - const pgpMimeToEncrypt = await Mime.encode({ 'text/plain': newMsg.plaintext, 'text/html': newMsg.plainhtml }, { Subject: newMsg.subject }, plainAttachments); - const { data: encrypted } = await this.encryptDataArmor(Buf.fromUtfStr(pgpMimeToEncrypt), undefined, pubs, signingPrv); - return await SendableMsg.createInlineArmored(this.acctEmail, this.headers(newMsg), Buf.fromUint8(encrypted).toUtfStr(), plainAttachments, { isDraft: this.isDraft }); + private sendableSmimeMsg = async (newMsg: NewMsgData, x509certs: Key[], signingPrv?: Key): Promise => { + const plainAttachments: Attachment[] = this.isDraft ? [] : await this.view.attachmentsModule.attachment.collectAttachments(); + const msgBody = { 'text/plain': newMsg.plaintext }; // todo: richtext #4047 + const mimeEncodedPlainMessage = await Mime.encode(msgBody, { Subject: newMsg.subject }, plainAttachments); + let mimeData = Buf.fromUtfStr(mimeEncodedPlainMessage); + if (signingPrv) { + const signedMessage = await this.signMimeMessage(signingPrv, mimeEncodedPlainMessage, newMsg); + mimeData = Buf.fromUtfStr(await signedMessage.toMime()); } - const pgpMimeToEncrypt = await Mime.encode({ 'text/plain': newMsg.plaintext, 'text/html': newMsg.plainhtml }, { Subject: newMsg.subject }, plainAttachments); - // todo: don't armor S/MIME and decide what to do with attachments #4046 and #4047 - const { data: encrypted } = await this.encryptDataArmor(Buf.fromUtfStr(pgpMimeToEncrypt), undefined, pubs, signingPrv); - const attachments = this.createPgpMimeAttachments(encrypted); - return await SendableMsg.createPgpMime(this.acctEmail, this.headers(newMsg), attachments, { isDraft: this.isDraft }); + const encryptedMessage = await SmimeKey.encryptMessage({ pubkeys: x509certs, data: mimeData, armor: false }); + return await SendableMsg.createSMimeEncrypted(this.acctEmail, this.headers(newMsg), encryptedMessage.data, { isDraft: this.isDraft }); }; private createPgpMimeAttachments = (data: Uint8Array) => { From 08fc01001968c9ebcd5d98b66069e50cc3dc5506 Mon Sep 17 00:00:00 2001 From: Roman Shevchenko Date: Sun, 12 Jun 2022 10:09:27 +0000 Subject: [PATCH 35/45] fix S/MIME drafts --- .../formatters/encrypted-mail-msg-formatter.ts | 9 ++++++--- 1 file changed, 6 insertions(+), 3 deletions(-) diff --git a/extension/chrome/elements/compose-modules/formatters/encrypted-mail-msg-formatter.ts b/extension/chrome/elements/compose-modules/formatters/encrypted-mail-msg-formatter.ts index c868602221f..c84f10f737c 100644 --- a/extension/chrome/elements/compose-modules/formatters/encrypted-mail-msg-formatter.ts +++ b/extension/chrome/elements/compose-modules/formatters/encrypted-mail-msg-formatter.ts @@ -94,9 +94,12 @@ export class EncryptedMsgMailFormatter extends BaseMailFormatter { }; public sendableNonPwdMsg = async (newMsg: NewMsgData, pubkeys: PubkeyResult[], signingPrv?: Key): Promise => { - const x509certs = pubkeys.map(entry => entry.pubkey).filter(pub => pub.family === 'x509'); - if (x509certs.length) { // s/mime - return await this.sendableSmimeMsg(newMsg, x509certs, signingPrv); + if (!this.isDraft) { + // S/MIME drafts are currently formatted with inline armored data + const x509certs = pubkeys.map(entry => entry.pubkey).filter(pub => pub.family === 'x509'); + if (x509certs.length) { // s/mime + return await this.sendableSmimeMsg(newMsg, x509certs, signingPrv); + } } const textToEncrypt = this.richtext ? await Mime.encode({ 'text/plain': newMsg.plaintext, 'text/html': newMsg.plainhtml }, { Subject: newMsg.subject }, From f1b2ca8740c075fdb12016ebd82dd16d8b2cc15f Mon Sep 17 00:00:00 2001 From: Roman Shevchenko Date: Sun, 12 Jun 2022 11:28:31 +0000 Subject: [PATCH 36/45] updated remarks --- .../elements/compose-modules/compose-send-btn-module.ts | 6 ++++-- 1 file changed, 4 insertions(+), 2 deletions(-) diff --git a/extension/chrome/elements/compose-modules/compose-send-btn-module.ts b/extension/chrome/elements/compose-modules/compose-send-btn-module.ts index 63864535390..ba66489404d 100644 --- a/extension/chrome/elements/compose-modules/compose-send-btn-module.ts +++ b/extension/chrome/elements/compose-modules/compose-send-btn-module.ts @@ -215,6 +215,9 @@ export class ComposeSendBtnModule extends ViewModule { }; private attemptSendMsg = async (msg: SendableMsg): Promise => { + // if this is a password-encrypted message, then we've already shown progress for uploading to backend + // and this requests represents second half of uploadable effort. Else this represents all (no previous heavy requests) + // todo: this isn't correct when we're sending multiple messages const progressRepresents = this.view.pwdOrPubkeyContainerModule.isVisible() ? 'SECOND-HALF' : 'EVERYTHING'; try { return await this.view.emailProvider.msgSend(msg, (p) => this.renderUploadProgress(p, progressRepresents)); @@ -222,6 +225,7 @@ export class ComposeSendBtnModule extends ViewModule { if (msg.thread && ApiErr.isNotFound(e) && this.view.threadId) { // cannot send msg because threadId not found - eg user since deleted it msg.thread = undefined; // give it another try, this time without msg.thread + // todo: progressRepresents? return await this.attemptSendMsg(msg); } else { throw e; @@ -245,8 +249,6 @@ export class ComposeSendBtnModule extends ViewModule { }; private doSendMsgs = async (msgObj: MultipleMessages): Promise => { - // if this is a password-encrypted message, then we've already shown progress for uploading to backend - // and this requests represents second half of uploadable effort. Else this represents all (no previous heavy requests) const sentIds: string[] = []; const supplementaryOperations: Promise[] = []; const supplementaryOperationsErrors: any[] = []; // tslint:disable-line:no-unsafe-any From cdf57d6b3cf99558a341d30aaa1479efea5887b2 Mon Sep 17 00:00:00 2001 From: Roman Shevchenko Date: Sun, 12 Jun 2022 11:47:14 +0000 Subject: [PATCH 37/45] refactored pwd-related code to a separate method formatttSendablePwdMsgs --- .../encrypted-mail-msg-formatter.ts | 106 +++++++++--------- 1 file changed, 55 insertions(+), 51 deletions(-) diff --git a/extension/chrome/elements/compose-modules/formatters/encrypted-mail-msg-formatter.ts b/extension/chrome/elements/compose-modules/formatters/encrypted-mail-msg-formatter.ts index c84f10f737c..0b12a13bf46 100644 --- a/extension/chrome/elements/compose-modules/formatters/encrypted-mail-msg-formatter.ts +++ b/extension/chrome/elements/compose-modules/formatters/encrypted-mail-msg-formatter.ts @@ -31,57 +31,7 @@ export class EncryptedMsgMailFormatter extends BaseMailFormatter { public sendableMsgs = async (newMsg: NewMsgData, pubkeys: PubkeyResult[], signingKey?: ParsedKeyInfo): Promise => { if (newMsg.pwd && !this.isDraft) { - // password-protected message, temporarily uploaded (already encrypted) to: - // - flowcrypt.com/api (consumers and customers without on-prem setup), or - // - FlowCrypt Enterprise Server (enterprise customers with on-prem setup) - // It will be served to recipient through web - const uploadedMessageData = await this.prepareAndUploadPwdEncryptedMsg(newMsg); // encrypted for pwd only, pubkeys ignored - // pwdRecipients that have their personal link - const pwdRecipients = Object.keys(uploadedMessageData.emailToExternalIdAndUrl ?? {}).filter(email => !pubkeys.some(p => p.email === email)); - newMsg.pwd = undefined; - const encryptedAttachments = await this.view.attachmentsModule.attachment.collectEncryptAttachments(pubkeys); - const pubkeyRecipients: { [type in RecipientType]?: EmailParts[] } = {}; - for (const [sendingType, value] of Object.entries(newMsg.recipients)) { - if (Api.isRecipientHeaderNameType(sendingType)) { - pubkeyRecipients[sendingType] = value?.filter(emailPart => pubkeys.some(p => p.email === emailPart.email) - // flowcrypt.com/api doesn't return individual links unlike FES - // so pwd recipients without individual links will go to legacy message - || (uploadedMessageData.emailToExternalIdAndUrl || {})[emailPart.email] === undefined); - } - } - const msgs: SendableMsg[] = []; - // pubkey recipients get one combined message. If there are not pubkey recpients, only password - protected messages will be sent - if (pubkeyRecipients.to?.length || pubkeyRecipients.cc?.length || pubkeyRecipients.bcc?.length) { - const uniquePubkeyRecipientToAndCCs = Value.arr.unique((pubkeyRecipients.to || []).concat(pubkeyRecipients.cc || []) - .map(recipient => recipient.email.toLowerCase())); - // pubkey recipients should be able to reply to "to" and "cc" pwd recipients - const replyToForMessageSentToPubkeyRecipients = (newMsg.recipients.to ?? []).concat(newMsg.recipients.cc ?? []) - .filter(recipient => !uniquePubkeyRecipientToAndCCs.includes(recipient.email.toLowerCase())); - const pubkeyMsgData = { - ...newMsg, - recipients: pubkeyRecipients, - // brackets are required for test emails like '@test:8001' - replyTo: replyToForMessageSentToPubkeyRecipients.length ? `${Str.formatEmailList([newMsg.from, ...replyToForMessageSentToPubkeyRecipients], true)}` - : undefined - }; - msgs.push(await this.sendablePubkeyMsgWithPwdLink( - pubkeyMsgData, - pubkeys, - { msgUrl: uploadedMessageData.url, externalId: uploadedMessageData.externalId }, - encryptedAttachments, - signingKey?.key) - ); - } - // adding individual messages for each recipient that doesn't have a pubkey - for (const recipientEmail of pwdRecipients) { - const { url, externalId } = uploadedMessageData.emailToExternalIdAndUrl![recipientEmail]; - const foundParsedRecipient = (newMsg.recipients.to ?? []).concat(newMsg.recipients.cc ?? []).concat(newMsg.recipients.bcc ?? []). - find(r => r.email.toLowerCase() === recipientEmail.toLowerCase()); - // todo: since a message is allowed to have only `cc` or `bcc` without `to`, should we preserve the original placement(s) of the recipient? - const individualMsgData = { ...newMsg, recipients: { to: [foundParsedRecipient ?? { email: recipientEmail }] } }; - msgs.push(await this.sendablePwdMsg(individualMsgData, pubkeys, { msgUrl: url, externalId }, signingKey?.key)); - } - return { senderKi: signingKey?.keyInfo, msgs, recipients: newMsg.recipients, attachments: encryptedAttachments }; + return await this.formatSendablePwdMsgs(newMsg, pubkeys, signingKey); } else { const msg = await this.sendableNonPwdMsg(newMsg, pubkeys, signingKey?.key); return { @@ -133,6 +83,60 @@ export class EncryptedMsgMailFormatter extends BaseMailFormatter { { isDraft: this.isDraft, externalId }); }; + private formatSendablePwdMsgs = async (newMsg: NewMsgData, pubkeys: PubkeyResult[], signingKey?: ParsedKeyInfo) => { + // password-protected message, temporarily uploaded (already encrypted) to: + // - flowcrypt.com/api (consumers and customers without on-prem setup), or + // - FlowCrypt Enterprise Server (enterprise customers with on-prem setup) + // It will be served to recipient through web + const uploadedMessageData = await this.prepareAndUploadPwdEncryptedMsg(newMsg); // encrypted for pwd only, pubkeys ignored + // pwdRecipients that have their personal link + const pwdRecipients = Object.keys(uploadedMessageData.emailToExternalIdAndUrl ?? {}).filter(email => !pubkeys.some(p => p.email === email)); + newMsg.pwd = undefined; + const encryptedAttachments = await this.view.attachmentsModule.attachment.collectEncryptAttachments(pubkeys); + const pubkeyRecipients: { [type in RecipientType]?: EmailParts[] } = {}; + for (const [sendingType, value] of Object.entries(newMsg.recipients)) { + if (Api.isRecipientHeaderNameType(sendingType)) { + pubkeyRecipients[sendingType] = value?.filter(emailPart => pubkeys.some(p => p.email === emailPart.email) + // flowcrypt.com/api doesn't return individual links unlike FES + // so pwd recipients without individual links will go to legacy message + || (uploadedMessageData.emailToExternalIdAndUrl || {})[emailPart.email] === undefined); + } + } + const msgs: SendableMsg[] = []; + // pubkey recipients get one combined message. If there are not pubkey recpients, only password - protected messages will be sent + if (pubkeyRecipients.to?.length || pubkeyRecipients.cc?.length || pubkeyRecipients.bcc?.length) { + const uniquePubkeyRecipientToAndCCs = Value.arr.unique((pubkeyRecipients.to || []).concat(pubkeyRecipients.cc || []) + .map(recipient => recipient.email.toLowerCase())); + // pubkey recipients should be able to reply to "to" and "cc" pwd recipients + const replyToForMessageSentToPubkeyRecipients = (newMsg.recipients.to ?? []).concat(newMsg.recipients.cc ?? []) + .filter(recipient => !uniquePubkeyRecipientToAndCCs.includes(recipient.email.toLowerCase())); + const pubkeyMsgData = { + ...newMsg, + recipients: pubkeyRecipients, + // brackets are required for test emails like '@test:8001' + replyTo: replyToForMessageSentToPubkeyRecipients.length ? `${Str.formatEmailList([newMsg.from, ...replyToForMessageSentToPubkeyRecipients], true)}` + : undefined + }; + msgs.push(await this.sendablePubkeyMsgWithPwdLink( + pubkeyMsgData, + pubkeys, + { msgUrl: uploadedMessageData.url, externalId: uploadedMessageData.externalId }, + encryptedAttachments, + signingKey?.key) + ); + } + // adding individual messages for each recipient that doesn't have a pubkey + for (const recipientEmail of pwdRecipients) { + const { url, externalId } = uploadedMessageData.emailToExternalIdAndUrl![recipientEmail]; + const foundParsedRecipient = (newMsg.recipients.to ?? []).concat(newMsg.recipients.cc ?? []).concat(newMsg.recipients.bcc ?? []). + find(r => r.email.toLowerCase() === recipientEmail.toLowerCase()); + // todo: since a message is allowed to have only `cc` or `bcc` without `to`, should we preserve the original placement(s) of the recipient? + const individualMsgData = { ...newMsg, recipients: { to: [foundParsedRecipient ?? { email: recipientEmail }] } }; + msgs.push(await this.sendablePwdMsg(individualMsgData, pubkeys, { msgUrl: url, externalId }, signingKey?.key)); + } + return { senderKi: signingKey?.keyInfo, msgs, recipients: newMsg.recipients, attachments: encryptedAttachments }; + }; + private prepareAndUploadPwdEncryptedMsg = async (newMsg: NewMsgData): Promise => { // PGP/MIME + included attachments (encrypted for password only) if (!newMsg.pwd) { From 00d330cb0c211a69c79bab578c7fffb68f551cc1 Mon Sep 17 00:00:00 2001 From: Roman Shevchenko Date: Sat, 25 Jun 2022 04:53:42 -0400 Subject: [PATCH 38/45] separate recipients and attachments to render into a separate renderSentMessage object --- .../elements/compose-modules/compose-send-btn-module.ts | 2 +- .../formatters/encrypted-mail-msg-formatter.ts | 8 +++++--- .../compose-modules/formatters/general-mail-formatter.ts | 7 +++---- 3 files changed, 9 insertions(+), 8 deletions(-) diff --git a/extension/chrome/elements/compose-modules/compose-send-btn-module.ts b/extension/chrome/elements/compose-modules/compose-send-btn-module.ts index ba66489404d..e334b5536c0 100644 --- a/extension/chrome/elements/compose-modules/compose-send-btn-module.ts +++ b/extension/chrome/elements/compose-modules/compose-send-btn-module.ts @@ -130,7 +130,7 @@ export class ComposeSendBtnModule extends ViewModule { BrowserMsg.send.notificationShow(this.view.parentTabId, { notification: `Your ${this.view.isReplyBox ? 'reply' : 'message'} has been sent.` }); BrowserMsg.send.focusBody(this.view.parentTabId); // Bring focus back to body so Gmails shortcuts will work if (this.view.isReplyBox) { - this.view.renderModule.renderReplySuccess(msgObj.attachments, msgObj.recipients, result.sentIds[0]); + this.view.renderModule.renderReplySuccess(msgObj.renderSentMessage.attachments, msgObj.renderSentMessage.recipients, result.sentIds[0]); } else { this.view.renderModule.closeMsg(); } diff --git a/extension/chrome/elements/compose-modules/formatters/encrypted-mail-msg-formatter.ts b/extension/chrome/elements/compose-modules/formatters/encrypted-mail-msg-formatter.ts index 0b12a13bf46..6ac1e044cbd 100644 --- a/extension/chrome/elements/compose-modules/formatters/encrypted-mail-msg-formatter.ts +++ b/extension/chrome/elements/compose-modules/formatters/encrypted-mail-msg-formatter.ts @@ -37,8 +37,10 @@ export class EncryptedMsgMailFormatter extends BaseMailFormatter { return { senderKi: signingKey?.keyInfo, msgs: [msg], - recipients: msg.recipients, - attachments: msg.attachments // todo: perhaps, we should hide technical attachments, like `encrypted.asc` and use collectedAttachments too? + renderSentMessage: { + recipients: msg.recipients, + attachments: msg.attachments // todo: perhaps, we should hide technical attachments, like `encrypted.asc` and use collectedAttachments too? + } }; } }; @@ -134,7 +136,7 @@ export class EncryptedMsgMailFormatter extends BaseMailFormatter { const individualMsgData = { ...newMsg, recipients: { to: [foundParsedRecipient ?? { email: recipientEmail }] } }; msgs.push(await this.sendablePwdMsg(individualMsgData, pubkeys, { msgUrl: url, externalId }, signingKey?.key)); } - return { senderKi: signingKey?.keyInfo, msgs, recipients: newMsg.recipients, attachments: encryptedAttachments }; + return { senderKi: signingKey?.keyInfo, msgs, renderSentMessage: { recipients: newMsg.recipients, attachments: encryptedAttachments } }; }; private prepareAndUploadPwdEncryptedMsg = async (newMsg: NewMsgData): Promise => { diff --git a/extension/chrome/elements/compose-modules/formatters/general-mail-formatter.ts b/extension/chrome/elements/compose-modules/formatters/general-mail-formatter.ts index 7ab3736621d..ec260ac5df0 100644 --- a/extension/chrome/elements/compose-modules/formatters/general-mail-formatter.ts +++ b/extension/chrome/elements/compose-modules/formatters/general-mail-formatter.ts @@ -17,8 +17,7 @@ import { Attachment } from '../../../../js/common/core/attachment.js'; export type MultipleMessages = { msgs: SendableMsg[]; senderKi: KeyInfoWithIdentity | undefined; - recipients: ParsedRecipients; - attachments: Attachment[]; + renderSentMessage: { recipients: ParsedRecipients, attachments: Attachment[] }; }; export class GeneralMailFormatter { @@ -30,7 +29,7 @@ export class GeneralMailFormatter { if (!choices.encrypt && !choices.sign) { // plain view.S.now('send_btn_text').text('Formatting...'); const msg = await new PlainMsgMailFormatter(view).sendableMsg(newMsgData); - return { senderKi: undefined, msgs: [msg], recipients: msg.recipients, attachments: msg.attachments }; + return { senderKi: undefined, msgs: [msg], renderSentMessage: { recipients: msg.recipients, attachments: msg.attachments } }; } if (!choices.encrypt && choices.sign) { // sign only view.S.now('send_btn_text').text('Signing...'); @@ -40,7 +39,7 @@ export class GeneralMailFormatter { throw new UnreportableError('Could not find account key usable for signing this plain text message'); } const msg = await new SignedMsgMailFormatter(view).sendableMsg(newMsgData, signingKey!.key); - return { senderKi: signingKey!.keyInfo, msgs: [msg], recipients: msg.recipients, attachments: msg.attachments }; + return { senderKi: signingKey!.keyInfo, msgs: [msg], renderSentMessage: { recipients: msg.recipients, attachments: msg.attachments } }; } // encrypt (optionally sign) const singleFamilyKeys = await view.storageModule.collectSingleFamilyKeys(recipientsEmails, newMsgData.from.email, choices.sign); From 62a826fad6281f440c4d16dc7232268f0261c46b Mon Sep 17 00:00:00 2001 From: Roman Shevchenko Date: Sat, 25 Jun 2022 05:03:15 -0400 Subject: [PATCH 39/45] renamed createPgpMimeAttachments to formatEncryptedMimeDataAsPgpMimeMetaAttachments --- .../formatters/encrypted-mail-msg-formatter.ts | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/extension/chrome/elements/compose-modules/formatters/encrypted-mail-msg-formatter.ts b/extension/chrome/elements/compose-modules/formatters/encrypted-mail-msg-formatter.ts index 6ac1e044cbd..2e1d1c12bfb 100644 --- a/extension/chrome/elements/compose-modules/formatters/encrypted-mail-msg-formatter.ts +++ b/extension/chrome/elements/compose-modules/formatters/encrypted-mail-msg-formatter.ts @@ -65,7 +65,7 @@ export class EncryptedMsgMailFormatter extends BaseMailFormatter { { isDraft: this.isDraft }); } // rich text: PGP/MIME - https://tools.ietf.org/html/rfc3156#section-4 - const attachments = this.createPgpMimeAttachments(encrypted); + const attachments = this.formatEncryptedMimeDataAsPgpMimeMetaAttachments(encrypted); return await SendableMsg.createPgpMime(this.acctEmail, this.headers(newMsg), attachments, { isDraft: this.isDraft }); }; @@ -81,7 +81,7 @@ export class EncryptedMsgMailFormatter extends BaseMailFormatter { const { data: pubEncryptedNoAttachments } = await this.encryptDataArmor(Buf.fromUtfStr(pgpMimeNoAttachments), undefined, pubs, signingPrv); // encrypted only for pubs const emailIntroAndLinkBody = await this.formatPwdEncryptedMsgBodyLink(msgUrl); return await SendableMsg.createPwdMsg(this.acctEmail, this.headers(newMsg), emailIntroAndLinkBody, - this.createPgpMimeAttachments(pubEncryptedNoAttachments).concat(encryptedAttachments), + this.formatEncryptedMimeDataAsPgpMimeMetaAttachments(pubEncryptedNoAttachments).concat(encryptedAttachments), { isDraft: this.isDraft, externalId }); }; @@ -194,7 +194,7 @@ export class EncryptedMsgMailFormatter extends BaseMailFormatter { const { data: pubEncryptedNoAttachments } = await this.encryptDataArmor(Buf.fromUtfStr(pgpMimeNoAttachments), undefined, pubs, signingPrv); // encrypted only for pubs const emailIntroAndLinkBody = await this.formatPwdEncryptedMsgBodyLink(msgUrl); return await SendableMsg.createPwdMsg(this.acctEmail, this.headers(newMsg), emailIntroAndLinkBody, - this.createPgpMimeAttachments(pubEncryptedNoAttachments), + this.formatEncryptedMimeDataAsPgpMimeMetaAttachments(pubEncryptedNoAttachments), { isDraft: this.isDraft, externalId }); }; @@ -211,7 +211,7 @@ export class EncryptedMsgMailFormatter extends BaseMailFormatter { return await SendableMsg.createSMimeEncrypted(this.acctEmail, this.headers(newMsg), encryptedMessage.data, { isDraft: this.isDraft }); }; - private createPgpMimeAttachments = (data: Uint8Array) => { + private formatEncryptedMimeDataAsPgpMimeMetaAttachments = (data: Uint8Array) => { const attachments: Attachment[] = []; attachments.push(new Attachment({ data: Buf.fromUtfStr('Version: 1'), type: 'application/pgp-encrypted', contentDescription: 'PGP/MIME version identification' })); attachments.push(new Attachment({ data, type: 'application/octet-stream', contentDescription: 'OpenPGP encrypted message', name: 'encrypted.asc', inline: true })); From 5cf0652977f06ef63f9d6558fcda53235fb70967 Mon Sep 17 00:00:00 2001 From: Roman Shevchenko Date: Sat, 25 Jun 2022 12:01:52 -0400 Subject: [PATCH 40/45] Use plain sendableNonPwdMsg for pubkey recipients --- .../encrypted-mail-msg-formatter.ts | 24 +----- .../strategies/send-message-strategy.ts | 86 +++++++++---------- 2 files changed, 41 insertions(+), 69 deletions(-) diff --git a/extension/chrome/elements/compose-modules/formatters/encrypted-mail-msg-formatter.ts b/extension/chrome/elements/compose-modules/formatters/encrypted-mail-msg-formatter.ts index 2e1d1c12bfb..67e3337c223 100644 --- a/extension/chrome/elements/compose-modules/formatters/encrypted-mail-msg-formatter.ts +++ b/extension/chrome/elements/compose-modules/formatters/encrypted-mail-msg-formatter.ts @@ -69,22 +69,6 @@ export class EncryptedMsgMailFormatter extends BaseMailFormatter { return await SendableMsg.createPgpMime(this.acctEmail, this.headers(newMsg), attachments, { isDraft: this.isDraft }); }; - private sendablePubkeyMsgWithPwdLink = async ( - newMsg: NewMsgData, - pubs: PubkeyResult[], - { msgUrl, externalId }: { msgUrl: string, externalId?: string }, - encryptedAttachments: Attachment[], - signingPrv?: Key) => { - // encoded as: PGP/MIME-like structure but with attachments as external files due to email size limit (encrypted for pubkeys only) - const msgBody = this.richtext ? { 'text/plain': newMsg.plaintext, 'text/html': newMsg.plainhtml } : { 'text/plain': newMsg.plaintext }; - const pgpMimeNoAttachments = await Mime.encode(msgBody, { Subject: newMsg.subject }, []); // no attachments, attached to email separately - const { data: pubEncryptedNoAttachments } = await this.encryptDataArmor(Buf.fromUtfStr(pgpMimeNoAttachments), undefined, pubs, signingPrv); // encrypted only for pubs - const emailIntroAndLinkBody = await this.formatPwdEncryptedMsgBodyLink(msgUrl); - return await SendableMsg.createPwdMsg(this.acctEmail, this.headers(newMsg), emailIntroAndLinkBody, - this.formatEncryptedMimeDataAsPgpMimeMetaAttachments(pubEncryptedNoAttachments).concat(encryptedAttachments), - { isDraft: this.isDraft, externalId }); - }; - private formatSendablePwdMsgs = async (newMsg: NewMsgData, pubkeys: PubkeyResult[], signingKey?: ParsedKeyInfo) => { // password-protected message, temporarily uploaded (already encrypted) to: // - flowcrypt.com/api (consumers and customers without on-prem setup), or @@ -119,13 +103,7 @@ export class EncryptedMsgMailFormatter extends BaseMailFormatter { replyTo: replyToForMessageSentToPubkeyRecipients.length ? `${Str.formatEmailList([newMsg.from, ...replyToForMessageSentToPubkeyRecipients], true)}` : undefined }; - msgs.push(await this.sendablePubkeyMsgWithPwdLink( - pubkeyMsgData, - pubkeys, - { msgUrl: uploadedMessageData.url, externalId: uploadedMessageData.externalId }, - encryptedAttachments, - signingKey?.key) - ); + msgs.push(await this.sendableNonPwdMsg(pubkeyMsgData, pubkeys, signingKey?.key)); } // adding individual messages for each recipient that doesn't have a pubkey for (const recipientEmail of pwdRecipients) { diff --git a/test/source/mock/google/strategies/send-message-strategy.ts b/test/source/mock/google/strategies/send-message-strategy.ts index 6e281b4724a..a4e43bad0bc 100644 --- a/test/source/mock/google/strategies/send-message-strategy.ts +++ b/test/source/mock/google/strategies/send-message-strategy.ts @@ -44,17 +44,7 @@ class PwdEncryptedMessageWithFesIdTokenTestStrategy implements ITestMsgStrategy const mimeMsg = parseResult.mimeMsg; const expectedSenderEmail = 'user@standardsubdomainfes.test:8001'; expect(mimeMsg.from!.text).to.equal(`First Last <${expectedSenderEmail}>`); - if (!mimeMsg.text?.includes(`${expectedSenderEmail} has sent you a password-encrypted email`)) { - throw new HttpClientErr(`Error checking sent text in:\n\n${mimeMsg.text}`); - } - if (mimeMsg.text?.includes('http://fes.standardsubdomainfes.test:8001/message/FES-MOCK-MESSAGE-ID')) { - // legacy - // todo: remove this test? - expect((mimeMsg.to as AddressObject).text).to.equal('Mr To '); - // tslint:disable-next-line:no-unused-expression - expect(mimeMsg.cc).to.be.an.undefined; // eslint-disable-line no-unused-expressions - expect((mimeMsg.bcc as AddressObject).text).to.equal('Mr Bcc '); - } else if (mimeMsg.text?.includes('http://fes.standardsubdomainfes.test:8001/message/FES-MOCK-MESSAGE-FOR-TO@EXAMPLE.COM-ID')) { + if (mimeMsg.text?.includes('http://fes.standardsubdomainfes.test:8001/message/FES-MOCK-MESSAGE-FOR-TO@EXAMPLE.COM-ID')) { expect((mimeMsg.to as AddressObject).text).to.equal('Mr To '); // tslint:disable-next-line:no-unused-expression expect(mimeMsg.cc).to.be.an.undefined; // eslint-disable-line no-unused-expressions @@ -67,11 +57,11 @@ class PwdEncryptedMessageWithFesIdTokenTestStrategy implements ITestMsgStrategy // tslint:disable-next-line:no-unused-expression expect(mimeMsg.bcc).to.be.an.undefined; // eslint-disable-line no-unused-expressions } else { + // no pubkey recipients in this test throw new HttpClientErr(`Error: cannot find pwd encrypted FES link in:\n\n${mimeMsg.text}`); } - if (!mimeMsg.text?.includes('Follow this link to open it')) { - throw new HttpClientErr(`Error: cannot find pwd encrypted open link prompt in ${mimeMsg.text}`); - } + expect(mimeMsg.text!).to.include(`${expectedSenderEmail} has sent you a password-encrypted email`); + expect(mimeMsg.text!).to.include('Follow this link to open it'); await new SaveMessageInStorageStrategy().test(parseResult, id); }; } @@ -81,18 +71,9 @@ class PwdEncryptedMessageWithFesPubkeyRecipientInBccTestStrategy implements ITes const mimeMsg = parseResult.mimeMsg; const expectedSenderEmail = 'user3@standardsubdomainfes.test:8001'; expect(mimeMsg.from!.text).to.equal(`First Last <${expectedSenderEmail}>`); - if (!mimeMsg.text?.includes(`${expectedSenderEmail} has sent you a password-encrypted email`)) { - throw new HttpClientErr(`Error checking sent text in:\n\n${mimeMsg.text}`); - } - if (mimeMsg.text?.includes('http://fes.standardsubdomainfes.test:8001/message/FES-MOCK-MESSAGE-ID')) { - // this is a message to pubkey recipients - expect((mimeMsg.bcc as AddressObject).text).to.equal('flowcrypt.compatibility@gmail.com'); - // tslint:disable-next-line:no-unused-expression - expect(mimeMsg.cc).to.be.an.undefined; // eslint-disable-line no-unused-expressions - // tslint:disable-next-line:no-unused-expression - expect(mimeMsg.to).to.be.an.undefined; // eslint-disable-line no-unused-expressions - expect((mimeMsg.headers.get('reply-to') as AddressObject).text).to.equal('First Last , to@example.com'); - } else if (mimeMsg.text?.includes('http://fes.standardsubdomainfes.test:8001/message/FES-MOCK-MESSAGE-FOR-TO@EXAMPLE.COM-ID')) { + if (mimeMsg.text?.includes('http://fes.standardsubdomainfes.test:8001/message/FES-MOCK-MESSAGE-FOR-TO@EXAMPLE.COM-ID')) { + expect(mimeMsg.text!).to.include(`${expectedSenderEmail} has sent you a password-encrypted email`); + expect(mimeMsg.text!).to.include('Follow this link to open it'); expect((mimeMsg.to as AddressObject).text).to.equal('to@example.com'); // tslint:disable-next-line:no-unused-expression expect(mimeMsg.cc).to.be.an.undefined; // eslint-disable-line no-unused-expressions @@ -101,10 +82,20 @@ class PwdEncryptedMessageWithFesPubkeyRecipientInBccTestStrategy implements ITes // tslint:disable-next-line:no-unused-expression expect(mimeMsg.headers.get('reply-to')).to.be.an.undefined; // eslint-disable-line no-unused-expressions } else { - throw new HttpClientErr(`Error: cannot find pwd encrypted FES link in:\n\n${mimeMsg.text}`); - } - if (!mimeMsg.text?.includes('Follow this link to open it')) { - throw new HttpClientErr(`Error: cannot find pwd encrypted open link prompt in ${mimeMsg.text}`); + // this is a message to pubkey recipients + expect(mimeMsg.text!).to.not.include('has sent you a password-encrypted email'); + expect(mimeMsg.text!).to.not.include('Follow this link to open it'); + const kisWithPp = await Config.getKeyInfo(['flowcrypt.test.key.used.pgp']); + const encryptedData = Buf.fromUtfStr(mimeMsg.text!); + const decrypted = await MsgUtil.decryptMessage({ kisWithPp, encryptedData, verificationPubs: [] }); + expect(decrypted.success).to.be.true; + expect(decrypted.content!.toUtfStr()).to.equal('PWD encrypted message with FES - pubkey recipient in bcc'); + expect((mimeMsg.bcc as AddressObject).text).to.equal('flowcrypt.compatibility@gmail.com'); + // tslint:disable-next-line:no-unused-expression + expect(mimeMsg.cc).to.be.an.undefined; // eslint-disable-line no-unused-expressions + // tslint:disable-next-line:no-unused-expression + expect(mimeMsg.to).to.be.an.undefined; // eslint-disable-line no-unused-expressions + expect((mimeMsg.headers.get('reply-to') as AddressObject).text).to.equal('First Last , to@example.com'); } await new SaveMessageInStorageStrategy().test(parseResult, id); }; @@ -137,18 +128,9 @@ class PwdEncryptedMessageWithFesReplyRenderingTestStrategy implements ITestMsgSt const mimeMsg = parseResult.mimeMsg; const expectedSenderEmail = 'user2@standardsubdomainfes.test:8001'; expect(mimeMsg.from!.text).to.equal(`First Last <${expectedSenderEmail}>`); - if (!mimeMsg.text?.includes(`${expectedSenderEmail} has sent you a password-encrypted email`)) { - throw new HttpClientErr(`Error checking sent text in:\n\n${mimeMsg.text}`); - } - if (mimeMsg.text?.includes('http://fes.standardsubdomainfes.test:8001/message/FES-MOCK-MESSAGE-ID')) { - // this is a message to pubkey recipients - expect((mimeMsg.to as AddressObject).text).to.equal('flowcrypt.compatibility@gmail.com, mock.only.pubkey@flowcrypt.com'); - // tslint:disable-next-line:no-unused-expression - expect(mimeMsg.cc).to.be.an.undefined; // eslint-disable-line no-unused-expressions - // tslint:disable-next-line:no-unused-expression - expect(mimeMsg.bcc).to.be.an.undefined; // eslint-disable-line no-unused-expressions - expect((mimeMsg.headers.get('reply-to') as AddressObject).text).to.equal('First Last , sender@domain.com, to@example.com'); - } else if (mimeMsg.text?.includes('http://fes.standardsubdomainfes.test:8001/message/FES-MOCK-MESSAGE-FOR-SENDER@DOMAIN.COM-ID')) { + if (mimeMsg.text?.includes('http://fes.standardsubdomainfes.test:8001/message/FES-MOCK-MESSAGE-FOR-SENDER@DOMAIN.COM-ID')) { + expect(mimeMsg.text!).to.include(`${expectedSenderEmail} has sent you a password-encrypted email`); + expect(mimeMsg.text!).to.include('Follow this link to open it'); expect((mimeMsg.to as AddressObject).text).to.equal('sender@domain.com'); // tslint:disable-next-line:no-unused-expression expect(mimeMsg.cc).to.be.an.undefined; // eslint-disable-line no-unused-expressions @@ -157,6 +139,8 @@ class PwdEncryptedMessageWithFesReplyRenderingTestStrategy implements ITestMsgSt // tslint:disable-next-line:no-unused-expression expect(mimeMsg.headers.get('reply-to')).to.be.an.undefined; // eslint-disable-line no-unused-expressions } else if (mimeMsg.text?.includes('http://fes.standardsubdomainfes.test:8001/message/FES-MOCK-MESSAGE-FOR-TO@EXAMPLE.COM-ID')) { + expect(mimeMsg.text!).to.include(`${expectedSenderEmail} has sent you a password-encrypted email`); + expect(mimeMsg.text!).to.include('Follow this link to open it'); expect((mimeMsg.to as AddressObject).text).to.equal('to@example.com'); // tslint:disable-next-line:no-unused-expression expect(mimeMsg.cc).to.be.an.undefined; // eslint-disable-line no-unused-expressions @@ -165,10 +149,20 @@ class PwdEncryptedMessageWithFesReplyRenderingTestStrategy implements ITestMsgSt // tslint:disable-next-line:no-unused-expression expect(mimeMsg.headers.get('reply-to')).to.be.an.undefined; // eslint-disable-line no-unused-expressions } else { - throw new HttpClientErr(`Error: cannot find pwd encrypted FES link in:\n\n${mimeMsg.text}`); - } - if (!mimeMsg.text?.includes('Follow this link to open it')) { - throw new HttpClientErr(`Error: cannot find pwd encrypted open link prompt in ${mimeMsg.text}`); + // this is a message to pubkey recipients + expect(mimeMsg.text!).to.not.include('has sent you a password-encrypted email'); + expect(mimeMsg.text!).to.not.include('Follow this link to open it'); + const kisWithPp = await Config.getKeyInfo(['flowcrypt.test.key.used.pgp']); + const encryptedData = Buf.fromUtfStr(mimeMsg.text!); + const decrypted = await MsgUtil.decryptMessage({ kisWithPp, encryptedData, verificationPubs: [] }); + expect(decrypted.success).to.be.true; + expect(decrypted.content!.toUtfStr()).to.include('> some dummy text'); + expect((mimeMsg.to as AddressObject).text).to.equal('flowcrypt.compatibility@gmail.com, mock.only.pubkey@flowcrypt.com'); + // tslint:disable-next-line:no-unused-expression + expect(mimeMsg.cc).to.be.an.undefined; // eslint-disable-line no-unused-expressions + // tslint:disable-next-line:no-unused-expression + expect(mimeMsg.bcc).to.be.an.undefined; // eslint-disable-line no-unused-expressions + expect((mimeMsg.headers.get('reply-to') as AddressObject).text).to.equal('First Last , sender@domain.com, to@example.com'); } await new SaveMessageInStorageStrategy().test(parseResult, id); }; From f86d1497589da26847ca047dffe056bccf482b78 Mon Sep 17 00:00:00 2001 From: Roman Shevchenko Date: Sun, 26 Jun 2022 10:43:01 -0400 Subject: [PATCH 41/45] lint fix --- test/source/mock/google/strategies/send-message-strategy.ts | 6 ++++-- 1 file changed, 4 insertions(+), 2 deletions(-) diff --git a/test/source/mock/google/strategies/send-message-strategy.ts b/test/source/mock/google/strategies/send-message-strategy.ts index a4e43bad0bc..616021f80b0 100644 --- a/test/source/mock/google/strategies/send-message-strategy.ts +++ b/test/source/mock/google/strategies/send-message-strategy.ts @@ -88,7 +88,8 @@ class PwdEncryptedMessageWithFesPubkeyRecipientInBccTestStrategy implements ITes const kisWithPp = await Config.getKeyInfo(['flowcrypt.test.key.used.pgp']); const encryptedData = Buf.fromUtfStr(mimeMsg.text!); const decrypted = await MsgUtil.decryptMessage({ kisWithPp, encryptedData, verificationPubs: [] }); - expect(decrypted.success).to.be.true; + // tslint:disable-next-line:no-unused-expression + expect(decrypted.success).to.be.true; // eslint-disable-line no-unused-expressions expect(decrypted.content!.toUtfStr()).to.equal('PWD encrypted message with FES - pubkey recipient in bcc'); expect((mimeMsg.bcc as AddressObject).text).to.equal('flowcrypt.compatibility@gmail.com'); // tslint:disable-next-line:no-unused-expression @@ -155,7 +156,8 @@ class PwdEncryptedMessageWithFesReplyRenderingTestStrategy implements ITestMsgSt const kisWithPp = await Config.getKeyInfo(['flowcrypt.test.key.used.pgp']); const encryptedData = Buf.fromUtfStr(mimeMsg.text!); const decrypted = await MsgUtil.decryptMessage({ kisWithPp, encryptedData, verificationPubs: [] }); - expect(decrypted.success).to.be.true; + // tslint:disable-next-line:no-unused-expression + expect(decrypted.success).to.be.true; // eslint-disable-line no-unused-expressions expect(decrypted.content!.toUtfStr()).to.include('> some dummy text'); expect((mimeMsg.to as AddressObject).text).to.equal('flowcrypt.compatibility@gmail.com, mock.only.pubkey@flowcrypt.com'); // tslint:disable-next-line:no-unused-expression From 49fa0f898865b0df4e1b80f953fdebf557982c84 Mon Sep 17 00:00:00 2001 From: Roman Shevchenko Date: Sun, 26 Jun 2022 11:25:33 -0400 Subject: [PATCH 42/45] corrected legacy pwd message --- .../formatters/encrypted-mail-msg-formatter.ts | 17 +++++++++++------ 1 file changed, 11 insertions(+), 6 deletions(-) diff --git a/extension/chrome/elements/compose-modules/formatters/encrypted-mail-msg-formatter.ts b/extension/chrome/elements/compose-modules/formatters/encrypted-mail-msg-formatter.ts index 67e3337c223..d35f408a5e9 100644 --- a/extension/chrome/elements/compose-modules/formatters/encrypted-mail-msg-formatter.ts +++ b/extension/chrome/elements/compose-modules/formatters/encrypted-mail-msg-formatter.ts @@ -76,16 +76,17 @@ export class EncryptedMsgMailFormatter extends BaseMailFormatter { // It will be served to recipient through web const uploadedMessageData = await this.prepareAndUploadPwdEncryptedMsg(newMsg); // encrypted for pwd only, pubkeys ignored // pwdRecipients that have their personal link - const pwdRecipients = Object.keys(uploadedMessageData.emailToExternalIdAndUrl ?? {}).filter(email => !pubkeys.some(p => p.email === email)); + const individualPwdRecipients = Object.keys(uploadedMessageData.emailToExternalIdAndUrl ?? {}).filter(email => !pubkeys.some(p => p.email === email)); + const legacyPwdRecipients: { [type in RecipientType]?: EmailParts[] } = {}; newMsg.pwd = undefined; const encryptedAttachments = await this.view.attachmentsModule.attachment.collectEncryptAttachments(pubkeys); const pubkeyRecipients: { [type in RecipientType]?: EmailParts[] } = {}; for (const [sendingType, value] of Object.entries(newMsg.recipients)) { if (Api.isRecipientHeaderNameType(sendingType)) { - pubkeyRecipients[sendingType] = value?.filter(emailPart => pubkeys.some(p => p.email === emailPart.email) - // flowcrypt.com/api doesn't return individual links unlike FES - // so pwd recipients without individual links will go to legacy message - || (uploadedMessageData.emailToExternalIdAndUrl || {})[emailPart.email] === undefined); + pubkeyRecipients[sendingType] = value?.filter(emailPart => pubkeys.some(p => p.email === emailPart.email)); + legacyPwdRecipients[sendingType] = value?.filter( + emailPart => !pubkeys.some(p => p.email === emailPart.email) + && !individualPwdRecipients.includes(emailPart.email)); } } const msgs: SendableMsg[] = []; @@ -106,7 +107,7 @@ export class EncryptedMsgMailFormatter extends BaseMailFormatter { msgs.push(await this.sendableNonPwdMsg(pubkeyMsgData, pubkeys, signingKey?.key)); } // adding individual messages for each recipient that doesn't have a pubkey - for (const recipientEmail of pwdRecipients) { + for (const recipientEmail of individualPwdRecipients) { const { url, externalId } = uploadedMessageData.emailToExternalIdAndUrl![recipientEmail]; const foundParsedRecipient = (newMsg.recipients.to ?? []).concat(newMsg.recipients.cc ?? []).concat(newMsg.recipients.bcc ?? []). find(r => r.email.toLowerCase() === recipientEmail.toLowerCase()); @@ -114,6 +115,10 @@ export class EncryptedMsgMailFormatter extends BaseMailFormatter { const individualMsgData = { ...newMsg, recipients: { to: [foundParsedRecipient ?? { email: recipientEmail }] } }; msgs.push(await this.sendablePwdMsg(individualMsgData, pubkeys, { msgUrl: url, externalId }, signingKey?.key)); } + if (legacyPwdRecipients.to?.length || legacyPwdRecipients.cc?.length || legacyPwdRecipients.bcc?.length) { + const legacyPwdMsgData = { ...newMsg, recipients: legacyPwdRecipients }; + msgs.push(await this.sendablePwdMsg(legacyPwdMsgData, pubkeys, { msgUrl: uploadedMessageData.url }, signingKey?.key)); + } return { senderKi: signingKey?.keyInfo, msgs, renderSentMessage: { recipients: newMsg.recipients, attachments: encryptedAttachments } }; }; From 5d77514131c786c431a881ec23928388c52338ec Mon Sep 17 00:00:00 2001 From: Roman Shevchenko Date: Mon, 27 Jun 2022 05:12:55 -0400 Subject: [PATCH 43/45] fixed test setup --- test/source/mock/fes/fes-endpoints.ts | 8 ++++++-- 1 file changed, 6 insertions(+), 2 deletions(-) diff --git a/test/source/mock/fes/fes-endpoints.ts b/test/source/mock/fes/fes-endpoints.ts index b768d7bafb6..4e7bca40547 100644 --- a/test/source/mock/fes/fes-endpoints.ts +++ b/test/source/mock/fes/fes-endpoints.ts @@ -144,8 +144,12 @@ const processMessageFromUser4 = async (body: string) => { externalId: 'FES-MOCK-EXTERNAL-FOR-INVALID@EXAMPLE.COM-ID' }; } - // we can add a clause for timeout@example.com here, but it's not necessary as without it the recipient goes to the legacy clause - // and the test is still valid + if (body.includes("timeout@example.com")) { + response.emailToExternalIdAndUrl['timeout@example.com'] = { + url: `http://${standardFesUrl}/message/FES-MOCK-MESSAGE-FOR-TIMEOUT@EXAMPLE.COM-ID`, + externalId: 'FES-MOCK-EXTERNAL-FOR-TIMEOUT@EXAMPLE.COM-ID' + }; + } if (body.includes("Mr Cc ")) { response.emailToExternalIdAndUrl['cc@example.com'] = { url: `http://${standardFesUrl}/message/FES-MOCK-MESSAGE-FOR-CC@EXAMPLE.COM-ID`, From 35ef46a3e52a4c655ec1074d33675a2c44cdb060 Mon Sep 17 00:00:00 2001 From: Roman Shevchenko Date: Mon, 27 Jun 2022 09:41:07 -0400 Subject: [PATCH 44/45] Added a test for one legacy pwd recipient and one pubkey recipient of a message --- .../strategies/send-message-strategy.ts | 32 +++++++++++++++++++ test/source/tests/compose.ts | 13 ++++++++ 2 files changed, 45 insertions(+) diff --git a/test/source/mock/google/strategies/send-message-strategy.ts b/test/source/mock/google/strategies/send-message-strategy.ts index 616021f80b0..7cdd21b62b0 100644 --- a/test/source/mock/google/strategies/send-message-strategy.ts +++ b/test/source/mock/google/strategies/send-message-strategy.ts @@ -23,6 +23,36 @@ class SaveMessageInStorageStrategy implements ITestMsgStrategy { }; } +class PwdAndPubkeyEncryptedMessagesWithFlowCryptComApiTestStrategy implements ITestMsgStrategy { + public test = async (parseResult: ParseMsgResult, id: string) => { + const mimeMsg = parseResult.mimeMsg; + const senderEmail = Str.parseEmail(mimeMsg.from!.text).email; + (new SaveMessageInStorageStrategy()).test(parseResult, id); + if (mimeMsg.cc) { + // this is a message to the pubkey recipient + expect((mimeMsg.cc as AddressObject).text!).to.include('flowcrypt.compatibility@gmail.com'); + expect(mimeMsg.text!).to.not.include('has sent you a password-encrypted email'); + expect(mimeMsg.text!).to.not.include('Follow this link to open it'); + const kisWithPp = await Config.getKeyInfo(['flowcrypt.compatibility.1pp1', 'flowcrypt.compatibility.2pp1']); + const encryptedData = Buf.fromUtfStr(mimeMsg.text!); + const decrypted = await MsgUtil.decryptMessage({ kisWithPp, encryptedData, verificationPubs: [] }); + // tslint:disable-next-line:no-unused-expression + expect(decrypted.success).to.be.true; // eslint-disable-line no-unused-expressions + expect(decrypted.content!.toUtfStr()).to.contain('PWD and pubkey encrypted messages with flowcrypt.com/api'); + // tslint:disable-next-line:no-unused-expression + expect(mimeMsg.bcc).to.be.an.undefined; // eslint-disable-line no-unused-expressions + // tslint:disable-next-line:no-unused-expression + expect(mimeMsg.to).to.be.an.undefined; // eslint-disable-line no-unused-expressions + expect((mimeMsg.headers.get('reply-to') as AddressObject).text).to.equal('First Last , test@email.com'); + } else { + expect(mimeMsg.text!).to.contain(`${senderEmail} has sent you a password-encrypted email`); + expect(mimeMsg.text!).to.contain('Follow this link to open it'); + if (!mimeMsg.text?.match(/https:\/\/flowcrypt.com\/[a-z0-9A-Z]{10}/)) { + throw new HttpClientErr(`Error: cannot find pwd encrypted flowcrypt.com/api link in:\n\n${mimeMsg.text}`); + } + } + }; +} class PwdEncryptedMessageWithFlowCryptComApiTestStrategy implements ITestMsgStrategy { public test = async (parseResult: ParseMsgResult) => { const mimeMsg = parseResult.mimeMsg; @@ -342,6 +372,8 @@ export class TestBySubjectStrategyContext { this.strategy = new MessageWithFooterTestStrategy(); } else if (subject.includes('PWD encrypted message with flowcrypt.com/api')) { this.strategy = new PwdEncryptedMessageWithFlowCryptComApiTestStrategy(); + } else if (subject.includes('PWD and pubkey encrypted messages with flowcrypt.com/api')) { + this.strategy = new PwdAndPubkeyEncryptedMessagesWithFlowCryptComApiTestStrategy(); } else if (subject.includes('PWD encrypted message with FES - ID TOKEN')) { this.strategy = new PwdEncryptedMessageWithFesIdTokenTestStrategy(); } else if (subject.includes('PWD encrypted message with FES - Reply rendering')) { diff --git a/test/source/tests/compose.ts b/test/source/tests/compose.ts index 3b3e1de2a4b..ee6882c6e68 100644 --- a/test/source/tests/compose.ts +++ b/test/source/tests/compose.ts @@ -34,6 +34,19 @@ export const defineComposeTests = (testVariant: TestVariant, testWithBrowser: Te if (testVariant !== 'CONSUMER-LIVE-GMAIL') { + ava.default('compose - send an encrypted message to a legacy pwd recipient and a pubkey recipient', testWithBrowser('compatibility', async (t, browser) => { + const acct = 'flowcrypt.compatibility@gmail.com'; + const msgPwd = 'super hard password for the message'; + const subject = 'PWD and pubkey encrypted messages with flowcrypt.com/api'; + const expectedNumberOfPassedMessages = (await GoogleData.withInitializedData(acct)).searchMessagesBySubject(subject).length + 2; + const composePage = await ComposePageRecipe.openStandalone(t, browser, 'compatibility'); + await composePage.selectOption('@input-from', acct); + await ComposePageRecipe.fillMsg(composePage, { to: 'test@email.com', cc: 'flowcrypt.compatibility@gmail.com' }, subject); + await ComposePageRecipe.sendAndClose(composePage, { password: msgPwd }); + expect((await GoogleData.withInitializedData(acct)).searchMessagesBySubject(subject).length).to.equal(expectedNumberOfPassedMessages); + // this test is using PwdAndPubkeyEncryptedMessagesWithFlowCryptComApiTestStrategy to check sent result based on subject "PWD and pubkey encrypted messages with flowcrypt.com/api" + })); + ava.default('compose - check for sender [flowcrypt.compatibility@gmail.com] from a password-protected email', testWithBrowser('compatibility', async (t, browser) => { const senderEmail = 'flowcrypt.compatibility@gmail.com'; const msgPwd = 'super hard password for the message'; From a9978c5a4e84c2aba74561ef5aa97b32569e544e Mon Sep 17 00:00:00 2001 From: Roman Shevchenko Date: Mon, 27 Jun 2022 09:45:39 -0400 Subject: [PATCH 45/45] fix --- test/source/mock/google/strategies/send-message-strategy.ts | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/test/source/mock/google/strategies/send-message-strategy.ts b/test/source/mock/google/strategies/send-message-strategy.ts index 7cdd21b62b0..347343b791e 100644 --- a/test/source/mock/google/strategies/send-message-strategy.ts +++ b/test/source/mock/google/strategies/send-message-strategy.ts @@ -27,7 +27,7 @@ class PwdAndPubkeyEncryptedMessagesWithFlowCryptComApiTestStrategy implements IT public test = async (parseResult: ParseMsgResult, id: string) => { const mimeMsg = parseResult.mimeMsg; const senderEmail = Str.parseEmail(mimeMsg.from!.text).email; - (new SaveMessageInStorageStrategy()).test(parseResult, id); + await (new SaveMessageInStorageStrategy()).test(parseResult, id); if (mimeMsg.cc) { // this is a message to the pubkey recipient expect((mimeMsg.cc as AddressObject).text!).to.include('flowcrypt.compatibility@gmail.com');