[FIX] Support the whole Matrix Markdown spec

apps/meteor/app/federation-v2/server/application/RoomServiceListener.ts

@@ -1,5 +1,5 @@
 import { RoomType } from '@rocket.chat/apps-engine/definition/rooms';
-import { isDirectMessageRoom } from '@rocket.chat/core-typings';
+import { isDirectMessageRoom, isQuoteAttachment } from '@rocket.chat/core-typings';
 
 import { DirectMessageFederatedRoom, FederatedRoom } from '../domain/FederatedRoom';
 import { FederatedUser } from '../domain/FederatedUser';
@@ -222,8 +222,8 @@ export class FederationRoomServiceListener extends FederationService {
 	}
 
 	public async onExternalMessageReceived(roomReceiveExternalMessageInput: FederationRoomReceiveExternalMessageDto): Promise<void> {
-		const { externalRoomId, externalSenderId, messageText, externalEventId, replyToEventId } = roomReceiveExternalMessageInput;
-
+		const { externalRoomId, externalSenderId, rawMessage, externalFormattedText, externalEventId, replyToEventId } =
+			roomReceiveExternalMessageInput;
 		const federatedRoom = await this.internalRoomAdapter.getFederatedRoomByExternalId(externalRoomId);
 		if (!federatedRoom) {
 			return;
@@ -246,19 +246,27 @@ export class FederationRoomServiceListener extends FederationService {
 			await this.internalMessageAdapter.sendQuoteMessage(
 				senderUser,
 				federatedRoom,
-				messageText,
+				externalFormattedText,
+				rawMessage,
 				externalEventId,
 				messageToReplyTo,
 				this.internalHomeServerDomain,
 			);
 			return;
 		}
 
-		await this.internalMessageAdapter.sendMessage(senderUser, federatedRoom, messageText, externalEventId);
+		await this.internalMessageAdapter.sendMessage(
+			senderUser,
+			federatedRoom,
+			rawMessage,
+			externalFormattedText,
+			externalEventId,
+			this.internalHomeServerDomain,
+		);
 	}
 
 	public async onExternalMessageEditedReceived(roomEditExternalMessageInput: FederationRoomEditExternalMessageDto): Promise<void> {
-		const { externalRoomId, externalSenderId, editsEvent, newMessageText } = roomEditExternalMessageInput;
+		const { externalRoomId, externalSenderId, editsEvent, newExternalFormattedText, newRawMessage } = roomEditExternalMessageInput;
 
 		const federatedRoom = await this.internalRoomAdapter.getFederatedRoomByExternalId(externalRoomId);
 		if (!federatedRoom) {
@@ -274,12 +282,46 @@ export class FederationRoomServiceListener extends FederationService {
 		if (!message) {
 			return;
 		}
-		// TODO: create an entity to abstract all the message logic
-		if (!FederatedRoom.shouldUpdateMessage(newMessageText, message)) {
+
+		// TODO: leaked business logic, move this to its proper place
+		const isAQuotedMessage = message.attachments?.some((attachment) => isQuoteAttachment(attachment) && Boolean(attachment.message_link));
+		if (isAQuotedMessage) {
+			const wasGeneratedLocally = FederatedUser.isOriginalFromTheProxyServer(
+				this.bridge.extractHomeserverOrigin(externalSenderId),
+				this.internalHomeServerDomain,
+			);
+			if (wasGeneratedLocally) {
+				return;
+			}
+			const internalFormattedMessageToBeEdited = await this.internalMessageAdapter.getMessageToEditWhenReplyAndQuote(
+				message,
+				newExternalFormattedText,
+				newRawMessage,
+				this.internalHomeServerDomain,
+			);
+			// TODO: create an entity to abstract all the message logic
+			if (!FederatedRoom.shouldUpdateMessage(internalFormattedMessageToBeEdited, message)) {
+				return;
+			}
+			await this.internalMessageAdapter.editQuotedMessage(
+				senderUser,
+				newRawMessage,
+				newExternalFormattedText,
+				message,
+				this.internalHomeServerDomain,
+			);
+			return;
+		}
+		if (!FederatedRoom.shouldUpdateMessage(newRawMessage, message)) {
 			return;
 		}
-
-		await this.internalMessageAdapter.editMessage(senderUser, newMessageText, message);
+		await this.internalMessageAdapter.editMessage(
+			senderUser,
+			newRawMessage,
+			newExternalFormattedText,
+			message,
+			this.internalHomeServerDomain,
+		);
 	}
 
 	public async onExternalFileMessageReceived(roomReceiveExternalMessageInput: FederationRoomReceiveExternalFileMessageDto): Promise<void> {

apps/meteor/app/federation-v2/server/application/input/RoomReceiverDto.ts

@@ -40,7 +40,8 @@ export interface IFederationChangeMembershipInputDto extends IFederationReceiver
 export interface IFederationSendInternalMessageInputDto extends IFederationReceiverBaseRoomInputDto {
 	externalSenderId: string;
 	normalizedSenderId: string;
-	messageText: string;
+	rawMessage: string;
+	externalFormattedText: string;
 	replyToEventId?: string;
 }
 
@@ -194,14 +195,16 @@ export class FederationRoomReceiveExternalMessageDto extends ExternalMessageBase
 		normalizedRoomId,
 		externalSenderId,
 		normalizedSenderId,
-		messageText,
+		externalFormattedText,
+		rawMessage,
 		externalEventId,
 		replyToEventId,
 	}: IFederationSendInternalMessageInputDto) {
 		super({ externalRoomId, normalizedRoomId });
 		this.externalSenderId = externalSenderId;
 		this.normalizedSenderId = normalizedSenderId;
-		this.messageText = messageText;
+		this.externalFormattedText = externalFormattedText;
+		this.rawMessage = rawMessage;
 		this.replyToEventId = replyToEventId;
 		this.externalEventId = externalEventId;
 	}
@@ -210,7 +213,9 @@ export class FederationRoomReceiveExternalMessageDto extends ExternalMessageBase
 
 	normalizedSenderId: string;
 
-	messageText: string;
+	externalFormattedText: string;
+
+	rawMessage: string;
 
 	replyToEventId?: string;
 }
@@ -221,22 +226,30 @@ export class FederationRoomEditExternalMessageDto extends ExternalMessageBaseDto
 		normalizedRoomId,
 		externalSenderId,
 		normalizedSenderId,
-		newMessageText,
+		newRawMessage,
+		newExternalFormattedText,
 		editsEvent,
 		externalEventId,
-	}: IFederationSendInternalMessageBaseInputDto & { newMessageText: string; editsEvent: string }) {
+	}: IFederationSendInternalMessageBaseInputDto & {
+		newRawMessage: string;
+		newExternalFormattedText: string;
+		editsEvent: string;
+	}) {
 		super({ externalRoomId, normalizedRoomId, externalEventId });
 		this.externalSenderId = externalSenderId;
 		this.normalizedSenderId = normalizedSenderId;
-		this.newMessageText = newMessageText;
+		this.newRawMessage = newRawMessage;
+		this.newExternalFormattedText = newExternalFormattedText;
 		this.editsEvent = editsEvent;
 	}
 
 	externalSenderId: string;
 
 	normalizedSenderId: string;
 
-	newMessageText: string;
+	newExternalFormattedText: string;
+
+	newRawMessage: string;
 
 	editsEvent: string;
 }

apps/meteor/app/federation-v2/server/infrastructure/Factory.ts

@@ -199,7 +199,7 @@ export class FederationFactory {
 		return [
 			new MatrixRoomCreatedHandler(roomServiceReceiver),
 			new MatrixRoomMembershipChangedHandler(roomServiceReceiver, rocketSettingsAdapter),
-			new MatrixRoomMessageSentHandler(roomServiceReceiver, rocketSettingsAdapter),
+			new MatrixRoomMessageSentHandler(roomServiceReceiver),
 			new MatrixRoomJoinRulesChangedHandler(roomServiceReceiver),
 			new MatrixRoomNameChangedHandler(roomServiceReceiver),
 			new MatrixRoomTopicChangedHandler(roomServiceReceiver),

apps/meteor/app/federation-v2/server/infrastructure/matrix/Bridge.ts

@@ -5,7 +5,7 @@ import { fetch } from '../../../../../server/lib/http/fetch';
 import type { IExternalUserProfileInformation, IFederationBridge, IFederationBridgeRegistrationFile } from '../../domain/IFederationBridge';
 import { federationBridgeLogger } from '../rocket-chat/adapters/logger';
 import { toExternalMessageFormat, toExternalQuoteMessageFormat } from './converters/MessageTextParser';
-import { convertEmojisRCFormatToMatrixFormat } from './converters/MessageReceiver';
+import { convertEmojisFromRCFormatToMatrixFormat } from './converters/MessageReceiver';
 import type { AbstractMatrixEvent } from './definitions/AbstractMatrixEvent';
 import { MatrixEnumRelatesToRelType, MatrixEnumSendMessageType } from './definitions/events/RoomMessageSent';
 import { MatrixEventType } from './definitions/MatrixEventType';
@@ -215,7 +215,7 @@ export class MatrixBridge implements IFederationBridge {
 	}
 
 	private escapeEmojis(text: string): string {
-		return convertEmojisRCFormatToMatrixFormat(text);
+		return convertEmojisFromRCFormatToMatrixFormat(text);
 	}
 
 	public async getReadStreamForFileFromUrl(externalUserId: string, fileUrl: string): Promise<ReadableStream> {
@@ -281,7 +281,7 @@ export class MatrixBridge implements IFederationBridge {
 			.matrixClient.sendEvent(externalRoomId, MatrixEventType.MESSAGE_REACTED, {
 				'm.relates_to': {
 					event_id: externalEventId,
-					key: convertEmojisRCFormatToMatrixFormat(reaction),
+					key: convertEmojisFromRCFormatToMatrixFormat(reaction),
 					rel_type: 'm.annotation',
 				},
 			});
@@ -304,7 +304,7 @@ export class MatrixBridge implements IFederationBridge {
 			'format': 'org.matrix.custom.html',
 			'formatted_body': messageInExternalFormat,
 			'm.new_content': {
-				body: messageInExternalFormat,
+				body: newMessageText,
 				format: 'org.matrix.custom.html',
 				formatted_body: messageInExternalFormat,
 				msgtype: MatrixEnumSendMessageType.TEXT,

apps/meteor/app/federation-v2/server/infrastructure/matrix/converters/MessageReceiver.ts

@@ -5,7 +5,7 @@ import { FederationMessageReactionEventDto } from '../../../application/input/Me
 import { convertExternalRoomIdToInternalRoomIdFormat } from './RoomReceiver';
 
 const convertEmojisMatrixFormatToRCFormat = (emoji: string): string => emojione.toShort(emoji);
-export const convertEmojisRCFormatToMatrixFormat = (emoji: string): string => emojione.shortnameToUnicode(emoji);
+export const convertEmojisFromRCFormatToMatrixFormat = (emoji: string): string => emojione.shortnameToUnicode(emoji);
 
 export class MatrixMessageReceiverConverter {
 	public static toMessageReactionDto(externalEvent: MatrixEventMessageReact): FederationMessageReactionEventDto {

apps/meteor/app/federation-v2/server/infrastructure/matrix/converters/MessageTextParser.ts

@@ -1,69 +1,57 @@
 import type { MentionPill as MentionPillType } from '@rocket.chat/forked-matrix-bot-sdk';
+import { marked } from 'marked';
 
-const INTERNAL_MENTIONS_FOR_EXTERNAL_USERS_REGEX = new RegExp(`@([0-9a-zA-Z-_.]+(@([0-9a-zA-Z-_.]+))?):+([0-9a-zA-Z-_.]+)`, 'gm'); // @username:server.com
-const INTERNAL_MENTIONS_FOR_INTERNAL_USERS_REGEX = new RegExp(`@([0-9a-zA-Z-_.]+(@([0-9a-zA-Z-_.]+))?)$`, 'gm'); // @username, @username.name
+const INTERNAL_MENTIONS_FOR_EXTERNAL_USERS_REGEX = /@([0-9a-zA-Z-_.]+(@([0-9a-zA-Z-_.]+))?):+([0-9a-zA-Z-_.]+)(?=[^<>]*(?:<\w|$))/gm; // @username:server.com excluding any <a> tags
+const INTERNAL_MENTIONS_FOR_INTERNAL_USERS_REGEX = /(?:(?!\S*@\S*\s).)@([0-9a-zA-Z-_.]+(@([0-9a-zA-Z-_.]+))?)(?=[^<>]*(?:<\w|$))/gm; // @username, @username.name excluding any <a> tags and emails
 const INTERNAL_GENERAL_REGEX = /(@all)|(@here)/gm;
 
-const replaceInternalUserMentionsForExternal = async (message: string): Promise<string> => {
-	const { MentionPill } = await import('@rocket.chat/forked-matrix-bot-sdk');
+const replaceMessageMentions = async (
+	message: string,
+	mentionRegex: RegExp,
+	parseMatchFn: (match: string) => Promise<MentionPillType>,
+): Promise<string> => {
 	const promises: Promise<MentionPillType>[] = [];
 
-	message
-		.split(' ')
-		.forEach((word) =>
-			word.replace(INTERNAL_MENTIONS_FOR_EXTERNAL_USERS_REGEX, (match): any => promises.push(MentionPill.forUser(match.trimStart()))),
-		);
+	message.replace(mentionRegex, (match: string): any => promises.push(parseMatchFn(match)));
 
-	const externalUserMentions = await Promise.all(promises);
+	const mentions = await Promise.all(promises);
 
-	return message
-		.split(' ')
-		.map((word) => word.replace(INTERNAL_MENTIONS_FOR_EXTERNAL_USERS_REGEX, () => ` ${externalUserMentions.shift()?.html}`))
-		.join(' ');
+	return message.replace(mentionRegex, () => ` ${mentions.shift()?.html}`);
 };
 
-const replaceInternalUserExternalMentionsForExternal = async (message: string, homeServerDomain: string): Promise<string> => {
+const replaceMentionsFromLocalExternalUsersForExternalFormat = async (message: string): Promise<string> => {
 	const { MentionPill } = await import('@rocket.chat/forked-matrix-bot-sdk');
-	const promises: Promise<MentionPillType>[] = [];
-
-	message
-		.split(' ')
-		.forEach((word) =>
-			word.replace(INTERNAL_MENTIONS_FOR_INTERNAL_USERS_REGEX, (match): any =>
-				promises.push(MentionPill.forUser(`${match.trimStart()}:${homeServerDomain}`)),
-			),
-		);
-
-	const externalUserMentions = await Promise.all(promises);
 
-	return message
-		.split(' ')
-		.map((word) => word.replace(INTERNAL_MENTIONS_FOR_INTERNAL_USERS_REGEX, () => ` ${externalUserMentions.shift()?.html}`))
-		.join(' ');
+	return replaceMessageMentions(message, INTERNAL_MENTIONS_FOR_EXTERNAL_USERS_REGEX, (match: string) =>
+		MentionPill.forUser(match.trimStart()),
+	);
 };
 
-const replaceInternalGeneralMentionsForExternal = async (message: string, externalRoomId: string): Promise<string> => {
+const replaceInternalUsersMentionsForExternalFormat = async (message: string, homeServerDomain: string): Promise<string> => {
 	const { MentionPill } = await import('@rocket.chat/forked-matrix-bot-sdk');
-	const promises: Promise<MentionPillType>[] = [];
 
-	message.replace(INTERNAL_GENERAL_REGEX, (): any => promises.push(MentionPill.forRoom(externalRoomId)));
+	return replaceMessageMentions(message, INTERNAL_MENTIONS_FOR_INTERNAL_USERS_REGEX, (match: string) =>
+		MentionPill.forUser(`${match.trimStart()}:${homeServerDomain}`),
+	);
+};
 
-	const externalMentions = await Promise.all(promises);
+const replaceInternalGeneralMentionsForExternalFormat = async (message: string, externalRoomId: string): Promise<string> => {
+	const { MentionPill } = await import('@rocket.chat/forked-matrix-bot-sdk');
 
-	return message.replace(INTERNAL_GENERAL_REGEX, () => ` ${externalMentions.shift()?.html}`);
+	return replaceMessageMentions(message, INTERNAL_GENERAL_REGEX, () => MentionPill.forRoom(externalRoomId));
 };
 
 const removeAllExtraBlankSpacesForASingleOne = (message: string): string => message.replace(/\s+/g, ' ').trim();
 
 const replaceInternalWithExternalMentions = async (message: string, externalRoomId: string, homeServerDomain: string): Promise<string> =>
-	removeAllExtraBlankSpacesForASingleOne(
-		await replaceInternalUserExternalMentionsForExternal(
-			await replaceInternalUserMentionsForExternal(await replaceInternalGeneralMentionsForExternal(message, externalRoomId)),
-			homeServerDomain,
+	replaceInternalUsersMentionsForExternalFormat(
+		await replaceMentionsFromLocalExternalUsersForExternalFormat(
+			await replaceInternalGeneralMentionsForExternalFormat(message, externalRoomId),
 		),
+		homeServerDomain,
 	);
 
-const removeMarkdownFromMessage = (message: string): string => message.replace(/\[(.*?)\]\(.*?\)/g, '').trim();
+const convertMarkdownToHTML = (message: string): string => marked.parse(message);
 
 export const toExternalMessageFormat = async ({
 	externalRoomId,
@@ -73,7 +61,10 @@ export const toExternalMessageFormat = async ({
 	message: string;
 	externalRoomId: string;
 	homeServerDomain: string;
-}): Promise<string> => replaceInternalWithExternalMentions(removeMarkdownFromMessage(message), externalRoomId, homeServerDomain);
+}): Promise<string> =>
+	removeAllExtraBlankSpacesForASingleOne(
+		convertMarkdownToHTML(await replaceInternalWithExternalMentions(message, externalRoomId, homeServerDomain)),
+	);
 
 export const toExternalQuoteMessageFormat = async ({
 	message,
@@ -90,19 +81,28 @@ export const toExternalQuoteMessageFormat = async ({
 }): Promise<{ message: string; formattedMessage: string }> => {
 	const { RichReply } = await import('@rocket.chat/forked-matrix-bot-sdk');
 
-	const formattedMessage = removeMarkdownFromMessage(message);
-
-	const { body, formatted_body: formattedBody } = RichReply.createFor(
-		externalRoomId,
-		{ event_id: eventToReplyTo, sender: originalEventSender },
-		formattedMessage,
+	const formattedMessage = convertMarkdownToHTML(message);
+	const finalFormattedMessage = convertMarkdownToHTML(
 		await toExternalMessageFormat({
-			message: formattedMessage,
+			message,
 			externalRoomId,
 			homeServerDomain,
 		}),
 	);
 
+	const { formatted_body: formattedBody } = RichReply.createFor(
+		externalRoomId,
+		{ event_id: eventToReplyTo, sender: originalEventSender },
+		formattedMessage,
+		finalFormattedMessage,
+	);
+	const { body } = RichReply.createFor(
+		externalRoomId,
+		{ event_id: eventToReplyTo, sender: originalEventSender },
+		message,
+		finalFormattedMessage,
+	);
+
 	return {
 		message: body,
 		formattedMessage: formattedBody,