From d9782bb38114be1e0b7ed33e03d4da745397d642 Mon Sep 17 00:00:00 2001 From: Khaled Ferjani Date: Thu, 2 Jun 2022 13:04:18 +0100 Subject: [PATCH] =?UTF-8?q?=F0=9F=8C=9F=20Added=20link=20preview=20service?= =?UTF-8?q?=20(#2204)?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit * 🌟 added link preview service * unit tests * moved links preview callback handler to message service * added publish message in realtime helper * publish message in realtime after preview generation * publish message only to channels --- twake/backend/node/package.json | 4 + .../node/src/@types/get-website-favicon.ts | 1 + .../node/src/services/global-resolver.ts | 19 +- .../services/messages/entities/messages.ts | 13 ++ .../messages/services/engine/index.ts | 7 + .../engine/processors/channel-view/index.ts | 20 +- .../services/engine/processors/links/index.ts | 73 ++++++++ .../engine/processors/user-inbox/index.ts | 20 +- .../services/messages/services/messages.ts | 16 +- .../src/services/messages/services/utils.ts | 56 +++++- .../services/{ => files}/engine/clear.ts | 8 +- .../services/{ => files}/engine/index.ts | 4 +- .../services/{ => files}/engine/service.ts | 12 +- .../services/{ => files}/processing/image.ts | 6 +- .../services/{ => files}/processing/mime.ts | 0 .../services/{ => files}/processing/office.ts | 4 +- .../services/{ => files}/processing/pdf.ts | 2 +- .../{ => files}/processing/service.ts | 5 +- .../services/{ => files}/processing/video.ts | 4 +- .../previews/services/links/engine/index.ts | 11 ++ .../previews/services/links/engine/service.ts | 74 ++++++++ .../services/links/processing/link.ts | 96 ++++++++++ .../services/links/processing/service.ts | 25 +++ .../node/src/services/previews/types.ts | 25 +++ .../node/src/services/previews/utils.ts | 2 +- .../{ => files}/processing/video.test.ts | 6 +- .../services/links/processing/links.test.ts | 176 ++++++++++++++++++ 27 files changed, 618 insertions(+), 71 deletions(-) create mode 100644 twake/backend/node/src/@types/get-website-favicon.ts create mode 100644 twake/backend/node/src/services/messages/services/engine/processors/links/index.ts rename twake/backend/node/src/services/previews/services/{ => files}/engine/clear.ts (86%) rename twake/backend/node/src/services/previews/services/{ => files}/engine/index.ts (79%) rename twake/backend/node/src/services/previews/services/{ => files}/engine/service.ts (90%) rename twake/backend/node/src/services/previews/services/{ => files}/processing/image.ts (88%) rename twake/backend/node/src/services/previews/services/{ => files}/processing/mime.ts (100%) rename twake/backend/node/src/services/previews/services/{ => files}/processing/office.ts (87%) rename twake/backend/node/src/services/previews/services/{ => files}/processing/pdf.ts (94%) rename twake/backend/node/src/services/previews/services/{ => files}/processing/service.ts (93%) rename twake/backend/node/src/services/previews/services/{ => files}/processing/video.ts (97%) create mode 100644 twake/backend/node/src/services/previews/services/links/engine/index.ts create mode 100644 twake/backend/node/src/services/previews/services/links/engine/service.ts create mode 100644 twake/backend/node/src/services/previews/services/links/processing/link.ts create mode 100644 twake/backend/node/src/services/previews/services/links/processing/service.ts rename twake/backend/node/test/unit/services/previews/services/{ => files}/processing/video.test.ts (94%) create mode 100644 twake/backend/node/test/unit/services/previews/services/links/processing/links.test.ts diff --git a/twake/backend/node/package.json b/twake/backend/node/package.json index 2afc3be69d..f6b0b00b6f 100644 --- a/twake/backend/node/package.json +++ b/twake/backend/node/package.json @@ -71,6 +71,7 @@ "@types/node-uuid": "^0.0.28", "@types/pdf-image": "^2.0.1", "@types/pino": "^6.3.2", + "@types/probe-image-size": "^7.0.1", "@types/pump": "^1.1.1", "@types/socket.io-client": "^1.4.34", "@types/supertest": "2.0.4", @@ -134,6 +135,8 @@ "fluent-ffmpeg": "^2.1.2", "fold-to-ascii": "^5.0.0", "generate-password": "^1.6.0", + "get-website-favicon": "^0.0.7", + "html-metadata-parser": "^2.0.4", "jsonwebtoken": "^8.5.1", "lodash": "^4.17.21", "match-all": "^1.2.6", @@ -148,6 +151,7 @@ "ora": "^5.4.0", "pdf2pic": "^2.1.4", "pino": "^6.8.0", + "probe-image-size": "^7.2.3", "pump": "^3.0.0", "redis": "3", "reflect-metadata": "^0.1.13", diff --git a/twake/backend/node/src/@types/get-website-favicon.ts b/twake/backend/node/src/@types/get-website-favicon.ts new file mode 100644 index 0000000000..0e02735f4f --- /dev/null +++ b/twake/backend/node/src/@types/get-website-favicon.ts @@ -0,0 +1 @@ +declare module "get-website-favicon"; diff --git a/twake/backend/node/src/services/global-resolver.ts b/twake/backend/node/src/services/global-resolver.ts index 7cf36a4ac6..50136fecd1 100644 --- a/twake/backend/node/src/services/global-resolver.ts +++ b/twake/backend/node/src/services/global-resolver.ts @@ -64,17 +64,19 @@ import { MobilePushService } from "./notifications/services/mobile-push"; import { ChannelMemberPreferencesServiceImpl } from "./notifications/services/channel-preferences"; import { ChannelThreadUsersServiceImpl } from "./notifications/services/channel-thread-users"; import { PushServiceAPI } from "../core/platform/services/push/api"; -import { PreviewProcessService } from "./previews/services/processing/service"; -import { PreviewServiceAPI } from "./previews/types"; +import { PreviewProcessService } from "./previews/services/files/processing/service"; +import { LinkPreviewServiceAPI, PreviewServiceAPI } from "./previews/types"; import { CronAPI } from "../core/platform/services/cron/api"; import WebSocketAPI from "../core/platform/services/websocket/provider"; import TrackerAPI from "../core/platform/services/tracker/provider"; import { ApplicationHooksService } from "./applications/services/hooks"; import { OnlineServiceAPI } from "./online/api"; import OnlineServiceImpl from "./online/service"; -import { PreviewEngine } from "./previews/services/engine"; +import { PreviewEngine } from "./previews/services/files/engine"; import KnowledgeGraphService from "../core/platform/services/knowledge-graph"; import { ChannelsPubsubListener } from "./channels/services/pubsub"; +import { LinkPreviewProcessService } from "./previews/services/links/processing/service"; +import { LinkPreviewEngine } from "./previews/services/links/engine"; type PlatformServices = { auth: AuthServiceAPI; @@ -105,7 +107,10 @@ type TwakeServices = { preferences: UserNotificationPreferencesAPI; mobilePush: MobilePushService; }; - preview: PreviewServiceAPI; + preview: { + files: PreviewServiceAPI; + links: LinkPreviewServiceAPI; + }; messages: { messages: MessageThreadMessagesServiceAPI; threads: MessageThreadsServiceAPI; @@ -167,6 +172,7 @@ class GlobalResolver { }); await new PreviewEngine().init(); + await new LinkPreviewEngine().init(); this.services = { workspaces: await new WorkspaceServiceImpl().init(), @@ -183,7 +189,10 @@ class GlobalResolver { preferences: await new NotificationPreferencesService().init(), mobilePush: await new MobilePushService().init(), }, - preview: await new PreviewProcessService().init(), + preview: { + files: await new PreviewProcessService().init(), + links: await new LinkPreviewProcessService().init(), + }, messages: { messages: await new ThreadMessagesService().init(platform), threads: await new ThreadsService().init(platform), diff --git a/twake/backend/node/src/services/messages/entities/messages.ts b/twake/backend/node/src/services/messages/entities/messages.ts index 11b97abeb6..3e7e007dd5 100644 --- a/twake/backend/node/src/services/messages/entities/messages.ts +++ b/twake/backend/node/src/services/messages/entities/messages.ts @@ -89,6 +89,9 @@ export class Message { workspace_id: string; channel_id: string; }; + + @Column("links", "encoded_json") + links: null | MessageLinks[]; } export type MessageReaction = { count: number; name: string; users: string[] }; @@ -132,3 +135,13 @@ export type MessageWithUsers = Message & { users?: UserObject[]; application?: Partial; }; + +export type MessageLinks = { + title: string; + description: string | null; + domain: string; + img: string | null; + favicon: string | null; + img_width: number | null; + img_height: number | null; +}; diff --git a/twake/backend/node/src/services/messages/services/engine/index.ts b/twake/backend/node/src/services/messages/services/engine/index.ts index 61db4983fd..25cddd6de1 100644 --- a/twake/backend/node/src/services/messages/services/engine/index.ts +++ b/twake/backend/node/src/services/messages/services/engine/index.ts @@ -15,6 +15,8 @@ import _ from "lodash"; import { StatisticsMessageProcessor } from "../../../statistics/pubsub/messages"; import { MessageToHooksProcessor } from "./processors/message-to-hooks"; import gr from "../../../global-resolver"; +import { MessageLinksPreviewFinishedProcessor } from "./processors/links"; +import { Message } from "../../entities/messages"; export class MessagesEngine implements Initializable { private channelViewProcessor: ChannelViewProcessor; @@ -26,6 +28,7 @@ export class MessagesEngine implements Initializable { private messageToHooks: MessageToHooksProcessor; private threadRepository: Repository; + private messageRepository: Repository; constructor() { this.channelViewProcessor = new ChannelViewProcessor(); @@ -68,6 +71,7 @@ export class MessagesEngine implements Initializable { async init(): Promise { this.threadRepository = await gr.database.getRepository("threads", Thread); + this.messageRepository = await gr.database.getRepository("messages", Message); await this.channelViewProcessor.init(); await this.channelMarkedViewProcessor.init(); @@ -77,6 +81,9 @@ export class MessagesEngine implements Initializable { gr.platformServices.pubsub.processor.addHandler(new ChannelSystemActivityMessageProcessor()); gr.platformServices.pubsub.processor.addHandler(new StatisticsMessageProcessor()); + gr.platformServices.pubsub.processor.addHandler( + new MessageLinksPreviewFinishedProcessor(this.messageRepository, this.threadRepository), + ); localEventBus.subscribe("message:saved", async (e: MessageLocalEvent) => { this.dispatchMessage(e); diff --git a/twake/backend/node/src/services/messages/services/engine/processors/channel-view/index.ts b/twake/backend/node/src/services/messages/services/engine/processors/channel-view/index.ts index 9bc2f2aedc..963f348369 100644 --- a/twake/backend/node/src/services/messages/services/engine/processors/channel-view/index.ts +++ b/twake/backend/node/src/services/messages/services/engine/processors/channel-view/index.ts @@ -19,6 +19,7 @@ import { } from "../../../../../../core/platform/framework/api/crud-service"; import { getThreadMessagePath } from "../../../../web/realtime"; import gr from "../../../../../global-resolver"; +import { publishMessageInRealtime } from "../../../utils"; export class ChannelViewProcessor { repository: Repository; @@ -111,24 +112,7 @@ export class ChannelViewProcessor { } //Publish message in realtime - const room = `/companies/${participant.company_id}/workspaces/${participant.workspace_id}/channels/${participant.id}/feed`; - const type = "message"; - const entity = message.resource; - const context = message.context; - localEventBus.publish("realtime:publish", { - topic: message.created - ? RealtimeEntityActionType.Created - : RealtimeEntityActionType.Updated, - event: { - type: type, - room: ResourcePath.get(room), - resourcePath: getThreadMessagePath(context as ThreadExecutionContext) + "/" + entity.id, - entity: entity, - result: message.created - ? new CreateResult(type, entity) - : new UpdateResult(type, entity), - }, - } as RealtimeLocalBusEvent); + publishMessageInRealtime(message, participant); } } } diff --git a/twake/backend/node/src/services/messages/services/engine/processors/links/index.ts b/twake/backend/node/src/services/messages/services/engine/processors/links/index.ts new file mode 100644 index 0000000000..5a9c9f3bd4 --- /dev/null +++ b/twake/backend/node/src/services/messages/services/engine/processors/links/index.ts @@ -0,0 +1,73 @@ +import { logger, TwakeContext } from "../../../../../../core/platform/framework"; +import { PubsubHandler } from "../../../../../../core/platform/services/pubsub/api"; +import { Message } from "../../../../entities/messages"; +import Repository from "../../../../../../core/platform/services/database/services/orm/repository/repository"; +import { LinkPreviewPubsubCallback } from "../../../../../previews/types"; +import { Thread } from "../../../../entities/threads"; +import { publishMessageInRealtime } from "../../../utils"; + +export class MessageLinksPreviewFinishedProcessor + implements PubsubHandler +{ + constructor( + private MessageRepository: Repository, + private ThreadRepository: Repository, + ) {} + readonly name = "MessageLinksPreviewFinishedProcessor"; + readonly topics = { + in: "services:preview:links:callback", + }; + + readonly options = { + unique: true, + ack: true, + }; + + init?(context?: TwakeContext): Promise { + throw new Error("Method not implemented."); + } + + validate(message: LinkPreviewPubsubCallback): boolean { + return !!(message && message.previews && message.previews.length); + } + + async process(localMessage: LinkPreviewPubsubCallback): Promise { + logger.info( + `${this.name} - updating message links with generated previews: ${localMessage.previews.length}`, + ); + + const entity = await this.MessageRepository.findOne({ + thread_id: localMessage.message.resource.thread_id, + id: localMessage.message.resource.thread_id, + }); + + if (!entity) { + logger.error(`${this.name} - message not found`); + return ""; + } + + entity.links = localMessage.previews; + + await this.MessageRepository.save(entity); + + const thread: Thread = await this.ThreadRepository.findOne({ + id: localMessage.message.resource.thread_id, + }); + + if (!thread) { + logger.error(`${this.name} - thread not found`); + return ""; + } + + const updatedMessage = { + ...localMessage.message, + resource: entity, + }; + + for (const participant of thread.participants.filter(p => p.type === "channel")) { + publishMessageInRealtime(updatedMessage, participant); + } + + return "done"; + } +} diff --git a/twake/backend/node/src/services/messages/services/engine/processors/user-inbox/index.ts b/twake/backend/node/src/services/messages/services/engine/processors/user-inbox/index.ts index 76472609f1..a9cc763c7d 100644 --- a/twake/backend/node/src/services/messages/services/engine/processors/user-inbox/index.ts +++ b/twake/backend/node/src/services/messages/services/engine/processors/user-inbox/index.ts @@ -22,6 +22,7 @@ import { } from "../../../../../../core/platform/framework/api/crud-service"; import { Message } from "../../../../entities/messages"; import gr from "../../../../../global-resolver"; +import { publishMessageInRealtime } from "../../../utils"; export class UserInboxViewProcessor { repositoryRef: Repository; @@ -87,24 +88,7 @@ export class UserInboxViewProcessor { //Publish message in realtime //TODO send a thread object instead of a message object - const room = `/companies/${channelParticipant.company_id}/users/${userParticipant.id}/inbox`; - const type = "message"; - const entity = message.resource; - const context = message.context; - localEventBus.publish("realtime:publish", { - topic: message.created - ? RealtimeEntityActionType.Created - : RealtimeEntityActionType.Updated, - event: { - type: type, - room: ResourcePath.get(room), - resourcePath: getThreadMessagePath(context as ThreadExecutionContext) + "/" + entity.id, - entity: entity, - result: message.created - ? new CreateResult(type, entity) - : new UpdateResult(type, entity), - }, - } as RealtimeLocalBusEvent); + publishMessageInRealtime(message, channelParticipant); } } } diff --git a/twake/backend/node/src/services/messages/services/messages.ts b/twake/backend/node/src/services/messages/services/messages.ts index 9d1d1ef1e7..0c78526451 100644 --- a/twake/backend/node/src/services/messages/services/messages.ts +++ b/twake/backend/node/src/services/messages/services/messages.ts @@ -35,7 +35,7 @@ import { UserObject } from "../../user/web/types"; import { formatUser } from "../../../utils/users"; import gr from "../../global-resolver"; import { getDefaultMessageInstance } from "../../../utils/messages"; -import { buildMessageListPagination, getMentions } from "./utils"; +import { buildMessageListPagination, getLinks, getMentions } from "./utils"; import { localEventBus } from "../../../core/platform/framework/pubsub"; import { KnowledgeGraphEvents, @@ -43,6 +43,7 @@ import { } from "../../../core/platform/services/knowledge-graph/types"; import { MessageUserInboxRef } from "../entities/message-user-inbox-refs"; import { MessageUserInboxRefReversed } from "../entities/message-user-inbox-refs-reversed"; +import { LinkPreviewPubsubRequest } from "../../../services/previews/types"; export class ThreadMessagesService implements MessageThreadMessagesServiceAPI { version: "1"; @@ -545,6 +546,19 @@ export class ThreadMessagesService implements MessageThreadMessagesServiceAPI { }, ]) async onSaved(message: Message, options: { created?: boolean }, context: ThreadExecutionContext) { + const messageLinks = getLinks(message); + + gr.platformServices.pubsub.publish("services:preview:links", { + data: { + links: messageLinks, + message: { + context, + resource: message, + created: options?.created, + }, + }, + }); + if (options.created && !message.ephemeral) { await gr.services.messages.threads.addReply(message.thread_id); } diff --git a/twake/backend/node/src/services/messages/services/utils.ts b/twake/backend/node/src/services/messages/services/utils.ts index cff695814c..1798c2d4b6 100644 --- a/twake/backend/node/src/services/messages/services/utils.ts +++ b/twake/backend/node/src/services/messages/services/utils.ts @@ -1,8 +1,18 @@ import { FindOptions } from "../../../core/platform/services/database/services/orm/repository/repository"; -import { Pagination } from "../../../core/platform/framework/api/crud-service"; +import { + CreateResult, + Pagination, + UpdateResult, +} from "../../../core/platform/framework/api/crud-service"; import { Message } from "../entities/messages"; -import { specialMention } from "../types"; +import { MessageLocalEvent, specialMention, ThreadExecutionContext } from "../types"; import User from "../../../services/user/entities/user"; +import { RealtimeEntityActionType } from "../../../core/platform/services/realtime/types"; +import { getThreadMessagePath } from "../web/realtime"; +import { ResourcePath } from "../../../core/platform/services/realtime/types"; +import { RealtimeLocalBusEvent } from "../../../core/platform/services/realtime/types"; +import { localEventBus } from "../../../core/platform/framework/pubsub"; +import { ParticipantObject } from "../entities/threads"; export const buildMessageListPagination = ( pagination: Pagination, @@ -48,3 +58,45 @@ export const getMentions = async ( specials: (globalOutput || []).map(g => (g || "").trim().split("@").pop()) as specialMention[], }; }; + +/** + * extracts the links from a message + * + * @param {Message} messageResource - The message to be parsed + * @returns {String} - links found in the message + */ +export const getLinks = (messageResource: Message): string[] => { + const links = (messageResource.text || "").match(/https?:\/\/[^ ]+/gm); + return links || []; +}; + +/** + * Publish a message to the realtime bus + * + * @param {MessageLocalEvent} message - The event to be published + * @param {ParticipantObject} participant - The participant + */ +export const publishMessageInRealtime = ( + message: MessageLocalEvent, + participant: ParticipantObject, +): void => { + if (participant.type !== "channel") return; + + const room = `/companies/${participant.company_id}/workspaces/${participant.workspace_id}/channels/${participant.id}/feed`; + const type = "message"; + const entity = message.resource; + const context = message.context; + + localEventBus.publish("realtime:publish", { + topic: message.created ? RealtimeEntityActionType.Created : RealtimeEntityActionType.Updated, + event: { + type, + room: ResourcePath.get(room), + resourcePath: getThreadMessagePath(context as ThreadExecutionContext) + "/" + entity.id, + entity, + result: message.created + ? new CreateResult(type, entity) + : new UpdateResult(type, entity), + }, + } as RealtimeLocalBusEvent); +}; diff --git a/twake/backend/node/src/services/previews/services/engine/clear.ts b/twake/backend/node/src/services/previews/services/files/engine/clear.ts similarity index 86% rename from twake/backend/node/src/services/previews/services/engine/clear.ts rename to twake/backend/node/src/services/previews/services/files/engine/clear.ts index a2b375a579..dbce78a06a 100644 --- a/twake/backend/node/src/services/previews/services/engine/clear.ts +++ b/twake/backend/node/src/services/previews/services/files/engine/clear.ts @@ -1,7 +1,7 @@ -import { PreviewPubsubHandler } from "../../api"; -import { logger, TwakeContext } from "../../../../core/platform/framework"; -import { PreviewClearPubsubRequest, PreviewPubsubCallback } from "../../types"; -import gr from "../../../global-resolver"; +import { PreviewPubsubHandler } from "../../../api"; +import { logger, TwakeContext } from "../../../../../core/platform/framework"; +import { PreviewClearPubsubRequest, PreviewPubsubCallback } from "../../../types"; +import gr from "../../../../global-resolver"; /** * Clear thumbnails when the delete task is called diff --git a/twake/backend/node/src/services/previews/services/engine/index.ts b/twake/backend/node/src/services/previews/services/files/engine/index.ts similarity index 79% rename from twake/backend/node/src/services/previews/services/engine/index.ts rename to twake/backend/node/src/services/previews/services/files/engine/index.ts index 9fd551fe0c..ccdb8a9fa7 100644 --- a/twake/backend/node/src/services/previews/services/engine/index.ts +++ b/twake/backend/node/src/services/previews/services/files/engine/index.ts @@ -1,7 +1,7 @@ -import { Initializable } from "../../../../core/platform/framework"; +import { Initializable } from "../../../../../core/platform/framework"; import { ClearProcessor } from "./clear"; import { PreviewProcessor } from "./service"; -import gr from "../../../global-resolver"; +import gr from "../../../../global-resolver"; /** * The notification engine is in charge of processing data and delivering user notifications on the right place diff --git a/twake/backend/node/src/services/previews/services/engine/service.ts b/twake/backend/node/src/services/previews/services/files/engine/service.ts similarity index 90% rename from twake/backend/node/src/services/previews/services/engine/service.ts rename to twake/backend/node/src/services/previews/services/files/engine/service.ts index 2e988828d5..c5388ee3b7 100644 --- a/twake/backend/node/src/services/previews/services/engine/service.ts +++ b/twake/backend/node/src/services/previews/services/files/engine/service.ts @@ -1,9 +1,9 @@ import fs, { promises as fsPromise } from "fs"; -import { PreviewPubsubHandler } from "../../api"; -import { logger, TwakeContext } from "../../../../core/platform/framework"; -import { PreviewPubsubCallback, PreviewPubsubRequest, ThumbnailResult } from "../../types"; -import { getTmpFile } from "../../utils"; -import gr from "../../../global-resolver"; +import { PreviewPubsubHandler } from "../../../api"; +import { logger, TwakeContext } from "../../../../../core/platform/framework"; +import { PreviewPubsubCallback, PreviewPubsubRequest, ThumbnailResult } from "../../../types"; +import { getTmpFile } from "../../../utils"; +import gr from "../../../../global-resolver"; const { unlink } = fsPromise; /** @@ -77,7 +77,7 @@ export class PreviewProcessor let localThumbnails: ThumbnailResult[] = []; try { - localThumbnails = await gr.services.preview.generateThumbnails( + localThumbnails = await gr.services.preview.files.generateThumbnails( { path: inputPath, mime: message.document.mime, filename: message.document.filename }, message.output, true, diff --git a/twake/backend/node/src/services/previews/services/processing/image.ts b/twake/backend/node/src/services/previews/services/files/processing/image.ts similarity index 88% rename from twake/backend/node/src/services/previews/services/processing/image.ts rename to twake/backend/node/src/services/previews/services/files/processing/image.ts index fe651064d2..9b945371de 100644 --- a/twake/backend/node/src/services/previews/services/processing/image.ts +++ b/twake/backend/node/src/services/previews/services/files/processing/image.ts @@ -1,7 +1,7 @@ import sharp from "sharp"; -import { cleanFiles, getTmpFile } from "../../utils"; -import { PreviewPubsubRequest, ThumbnailResult } from "../../types"; -import { logger } from "../../../../core/platform/framework/logger"; +import { cleanFiles, getTmpFile } from "../../../utils"; +import { PreviewPubsubRequest, ThumbnailResult } from "../../../types"; +import { logger } from "../../../../../core/platform/framework/logger"; export async function generatePreview( inputPaths: string[], diff --git a/twake/backend/node/src/services/previews/services/processing/mime.ts b/twake/backend/node/src/services/previews/services/files/processing/mime.ts similarity index 100% rename from twake/backend/node/src/services/previews/services/processing/mime.ts rename to twake/backend/node/src/services/previews/services/files/processing/mime.ts diff --git a/twake/backend/node/src/services/previews/services/processing/office.ts b/twake/backend/node/src/services/previews/services/files/processing/office.ts similarity index 87% rename from twake/backend/node/src/services/previews/services/processing/office.ts rename to twake/backend/node/src/services/previews/services/files/processing/office.ts index 08907d8047..1ea108c113 100644 --- a/twake/backend/node/src/services/previews/services/processing/office.ts +++ b/twake/backend/node/src/services/previews/services/files/processing/office.ts @@ -1,7 +1,7 @@ // eslint-disable-next-line @typescript-eslint/no-var-requires const unoconv = require("unoconv-promise"); -import { logger } from "../../../../core/platform/framework/logger"; -import { cleanFiles } from "../../utils"; +import { logger } from "../../../../../core/platform/framework/logger"; +import { cleanFiles } from "../../../utils"; export async function convertFromOffice( path: string, diff --git a/twake/backend/node/src/services/previews/services/processing/pdf.ts b/twake/backend/node/src/services/previews/services/files/processing/pdf.ts similarity index 94% rename from twake/backend/node/src/services/previews/services/processing/pdf.ts rename to twake/backend/node/src/services/previews/services/files/processing/pdf.ts index 8ca650f7db..9f3f3c961a 100644 --- a/twake/backend/node/src/services/previews/services/processing/pdf.ts +++ b/twake/backend/node/src/services/previews/services/files/processing/pdf.ts @@ -1,6 +1,6 @@ import { fromPath } from "pdf2pic"; import { mkdirSync } from "fs"; -import { cleanFiles, getTmpFile } from "../../utils"; +import { cleanFiles, getTmpFile } from "../../../utils"; export async function convertFromPdf( inputPath: string, diff --git a/twake/backend/node/src/services/previews/services/processing/service.ts b/twake/backend/node/src/services/previews/services/files/processing/service.ts similarity index 93% rename from twake/backend/node/src/services/previews/services/processing/service.ts rename to twake/backend/node/src/services/previews/services/files/processing/service.ts index 39fdc5d34f..9d72f6547b 100644 --- a/twake/backend/node/src/services/previews/services/processing/service.ts +++ b/twake/backend/node/src/services/previews/services/files/processing/service.ts @@ -1,10 +1,9 @@ import { generatePreview as thumbnailsFromImages } from "./image"; import { convertFromOffice } from "./office"; import { convertFromPdf } from "./pdf"; -import { cleanFiles, isFileType } from "../../utils"; +import { cleanFiles, isFileType } from "../../../utils"; import { imageExtensions, officeExtensions, pdfExtensions, videoExtensions } from "./mime"; -import StorageAPI from "../../../../core/platform/services/storage/provider"; -import { PreviewPubsubRequest, PreviewServiceAPI, ThumbnailResult } from "../../types"; +import { PreviewPubsubRequest, PreviewServiceAPI, ThumbnailResult } from "../../../types"; import { generateVideoPreview } from "./video"; export class PreviewProcessService implements PreviewServiceAPI { diff --git a/twake/backend/node/src/services/previews/services/processing/video.ts b/twake/backend/node/src/services/previews/services/files/processing/video.ts similarity index 97% rename from twake/backend/node/src/services/previews/services/processing/video.ts rename to twake/backend/node/src/services/previews/services/files/processing/video.ts index 023c64ff02..bce0d36b44 100644 --- a/twake/backend/node/src/services/previews/services/processing/video.ts +++ b/twake/backend/node/src/services/previews/services/files/processing/video.ts @@ -1,6 +1,6 @@ import ffmpeg from "fluent-ffmpeg"; -import { temporaryThumbnailFile, ThumbnailResult } from "../../types"; -import { cleanFiles, getTmpFile } from "../../utils"; +import { temporaryThumbnailFile, ThumbnailResult } from "../../../types"; +import { cleanFiles, getTmpFile } from "../../../utils"; import fs from "fs"; /** diff --git a/twake/backend/node/src/services/previews/services/links/engine/index.ts b/twake/backend/node/src/services/previews/services/links/engine/index.ts new file mode 100644 index 0000000000..90d0461ddd --- /dev/null +++ b/twake/backend/node/src/services/previews/services/links/engine/index.ts @@ -0,0 +1,11 @@ +import { Initializable } from "../../../../../core/platform/framework"; +import { LinkPreviewProcessor } from "./service"; +import gr from "../../../../global-resolver"; + +export class LinkPreviewEngine implements Initializable { + async init(): Promise { + gr.platformServices.pubsub.processor.addHandler(new LinkPreviewProcessor()); + + return this; + } +} diff --git a/twake/backend/node/src/services/previews/services/links/engine/service.ts b/twake/backend/node/src/services/previews/services/links/engine/service.ts new file mode 100644 index 0000000000..92723ff67f --- /dev/null +++ b/twake/backend/node/src/services/previews/services/links/engine/service.ts @@ -0,0 +1,74 @@ +import { LinkPreview, LinkPreviewPubsubCallback, LinkPreviewPubsubRequest } from "../../../types"; +import { PreviewPubsubHandler } from "../../../api"; +import { logger, TwakeContext } from "./../../../../../core/platform/framework"; +import gr from "../../../../global-resolver"; + +export class LinkPreviewProcessor + implements PreviewPubsubHandler +{ + readonly name = "LinkPreviewProcessor"; + + readonly topics = { + in: "services:preview:links", + out: "services:preview:links:callback", + }; + + readonly options = { + unique: true, + ack: true, + }; + + init?(context?: TwakeContext): Promise { + throw new Error("Method not implemented."); + } + + /** + * Checks if the message is valid + * + * @param {LinkPreviewPubsubRequest} message - The message to check + * @returns {Boolean} - true if the message is valid + */ + validate(message: LinkPreviewPubsubRequest): boolean { + return !!(message && message.links && message.links.length); + } + + /** + * process the links preview generation message + * + * @param {LinkPreviewPubsubRequest} message - The message to process + * @returns {Promise} - links preview callback + */ + async process(message: LinkPreviewPubsubRequest): Promise { + logger.info(`${this.name} - Processing preview generation for ${message.links.length} links`); + + let res: LinkPreviewPubsubCallback = { previews: [], message: message.message }; + + try { + res = await this.generate(message); + } catch (err) { + logger.error(`${this.name} - Can't generate link previews ${err}`); + } + + logger.info(`${this.name} - Generated ${res.previews.length} link previews`); + + return res; + } + + /** + * Generate previews for links + * + * @param {LinkPreviewPubsubRequest} message - The message to process + * @returns {Promise} - links preview callback + */ + async generate(message: LinkPreviewPubsubRequest): Promise { + let previews: LinkPreview[] = []; + try { + previews = await gr.services.preview.links.generatePreviews(message.links); + } catch (err) { + logger.error(`${this.name} - Can't generate link previews ${err}`); + throw Error(`cannot generate link previews: ${err}`); + } + + return { previews, message: message.message }; + } +} diff --git a/twake/backend/node/src/services/previews/services/links/processing/link.ts b/twake/backend/node/src/services/previews/services/links/processing/link.ts new file mode 100644 index 0000000000..248a6578d3 --- /dev/null +++ b/twake/backend/node/src/services/previews/services/links/processing/link.ts @@ -0,0 +1,96 @@ +import { parser } from "html-metadata-parser"; +import getFavicons from "get-website-favicon"; +import { LinkPreview } from "../../../types"; +import { logger } from "../../../../../core/platform/framework"; +import imageProbe from "probe-image-size"; + +/** + * Generate a thumbnail for a given url. + * + * @param {String[]} urls - the input urls + * @returns {Promise} - resolves when the thumbnails are generated + */ +export async function generateLinksPreviews(urls: string[]): Promise { + const output: LinkPreview[] = []; + + for (const url of urls) { + try { + output.push(await getUrlInformation(url)); + } catch (error) { + logger.error(`failed to generate link preview: ${error}`); + } + } + + return output; +} + +/** + * Get domain from a given url. + * + * @param {String} url - the input url + * @returns {String} - resolves when the domain is retrieved + */ +const getDomain = (url: string): string => { + try { + const domain = new URL(url).hostname; + return domain; + } catch (error) { + throw Error(`failed to get domain: ${error}`); + } +}; + +/** + * get url information + * + * @param {String} url - the input url + * @returns {Promise} - resolves when the information is retrieved + */ +const getUrlInformation = async (url: string): Promise => { + try { + const parsedPage = await parser(url); + const title = parsedPage.og?.title || parsedPage.meta?.title || null; + const description = parsedPage.og?.description || parsedPage.meta?.description || null; + const img = parsedPage.og?.image || parsedPage.meta?.image || parsedPage.images?.[0] || null; + const favicon = (await getUrlFavicon(url)) || null; + const domain = getDomain(url); + let img_height: number | null = null, + img_width: number | null = null; + + if (img) { + const dimensions = await imageProbe(img); + img_height = dimensions.height; + img_width = dimensions.width; + } + + return { + title, + domain, + description, + favicon, + img, + img_height, + img_width, + }; + } catch (error) { + throw Error(`failed to get url information: ${error}`); + } +}; + +/** + * get url favicon + * + * @param {String} url - the input url + * @returns {Promise} - resolves when the favicon is retrieved + */ +const getUrlFavicon = async (url: string): Promise => { + try { + const result = await getFavicons(url); + if (!result.icons || !result.icons.length) { + return; + } + + return result.icons[0].src; + } catch (error) { + logger.error(`failed to get url favicon: ${error}`); + } +}; diff --git a/twake/backend/node/src/services/previews/services/links/processing/service.ts b/twake/backend/node/src/services/previews/services/links/processing/service.ts new file mode 100644 index 0000000000..ff820855c0 --- /dev/null +++ b/twake/backend/node/src/services/previews/services/links/processing/service.ts @@ -0,0 +1,25 @@ +import { LinkPreviewServiceAPI, LinkPreview, LinkPreviewPubsubRequest } from "../../../types"; +import { generateLinksPreviews } from "./link"; + +export class LinkPreviewProcessService implements LinkPreviewServiceAPI { + name: "LinkPreviewProcessService"; + version: "1"; + + async init(): Promise { + return this; + } + + /** + * Generate previews for links + * + * @param {LinkPreviewPubsubRequest["links"]} links - input urls + * @returns {Promise} - The generated url previews + */ + async generatePreviews(links: LinkPreviewPubsubRequest["links"]): Promise { + try { + return await generateLinksPreviews(links); + } catch (error) { + throw Error(`cannot process: failed to generate links previews: ${error}`); + } + } +} diff --git a/twake/backend/node/src/services/previews/types.ts b/twake/backend/node/src/services/previews/types.ts index d199fa9e49..784b906a11 100644 --- a/twake/backend/node/src/services/previews/types.ts +++ b/twake/backend/node/src/services/previews/types.ts @@ -1,4 +1,5 @@ import { Initializable, TwakeServiceProvider } from "../../core/platform/framework"; +import { MessageLocalEvent } from "../messages/types"; export interface PreviewServiceAPI extends TwakeServiceProvider, Initializable { generateThumbnails( @@ -8,6 +9,10 @@ export interface PreviewServiceAPI extends TwakeServiceProvider, Initializable { ): Promise; } +export interface LinkPreviewServiceAPI extends TwakeServiceProvider, Initializable { + generatePreviews(links: LinkPreviewPubsubRequest["links"]): Promise; +} + export type PreviewPubsubRequest = { document: { id: string; @@ -69,3 +74,23 @@ export type temporaryThumbnailFile = { fileName: string; folder: string; }; + +export type LinkPreview = { + title: string; + description: string | null; + domain: string; + favicon: string | null; + img: string | null; + img_height: number | null; + img_width: number | null; +}; + +export type LinkPreviewPubsubRequest = { + links: string[]; + message: MessageLocalEvent; +}; + +export type LinkPreviewPubsubCallback = { + message: MessageLocalEvent; + previews: LinkPreview[]; +}; diff --git a/twake/backend/node/src/services/previews/utils.ts b/twake/backend/node/src/services/previews/utils.ts index b1dac62fac..d9f105063e 100644 --- a/twake/backend/node/src/services/previews/utils.ts +++ b/twake/backend/node/src/services/previews/utils.ts @@ -1,5 +1,5 @@ import { v4 as uuidv4 } from "uuid"; -import mimes from "./services/processing/mime"; +import mimes from "./services/files/processing/mime"; import fs, { existsSync, promises as fsPromise } from "fs"; const { unlink } = fsPromise; diff --git a/twake/backend/node/test/unit/services/previews/services/processing/video.test.ts b/twake/backend/node/test/unit/services/previews/services/files/processing/video.test.ts similarity index 94% rename from twake/backend/node/test/unit/services/previews/services/processing/video.test.ts rename to twake/backend/node/test/unit/services/previews/services/files/processing/video.test.ts index a9d020eeb6..d4009439a3 100644 --- a/twake/backend/node/test/unit/services/previews/services/processing/video.test.ts +++ b/twake/backend/node/test/unit/services/previews/services/files/processing/video.test.ts @@ -1,11 +1,11 @@ import { describe, expect, it, jest, beforeEach, afterEach, afterAll } from "@jest/globals"; -import { generateVideoPreview } from "../../../../../../src/services/previews/services/processing/video"; +import { generateVideoPreview } from "../../../../../../../src/services/previews/services/files/processing/video"; import ffmpeg, { ffprobe } from "fluent-ffmpeg"; -import { cleanFiles, getTmpFile } from "../../../../../../src/services/previews/utils"; +import { cleanFiles, getTmpFile } from "../../../../../../../src/services/previews/utils"; import fs from "fs"; jest.mock("fluent-ffmpeg"); -jest.mock("../../../../../../src/services/previews/utils"); +jest.mock("../../../../../../../src/services/previews/utils"); const ffmpegMock = { screenshot: jest.fn().mockReturnValue({ diff --git a/twake/backend/node/test/unit/services/previews/services/links/processing/links.test.ts b/twake/backend/node/test/unit/services/previews/services/links/processing/links.test.ts new file mode 100644 index 0000000000..6e38df9aa1 --- /dev/null +++ b/twake/backend/node/test/unit/services/previews/services/links/processing/links.test.ts @@ -0,0 +1,176 @@ +import { describe, expect, it, jest, beforeEach, afterEach, afterAll } from "@jest/globals"; +import { generateLinksPreviews } from "../../../../../../../src/services/previews/services/links/processing/link"; +import { parser } from "html-metadata-parser"; +import getFavicons from "get-website-favicon"; +import imageProbe from "probe-image-size"; + +jest.mock("html-metadata-parser"); +jest.mock("get-website-favicon"); +jest.mock("probe-image-size"); + +beforeEach(() => { + (imageProbe as any).mockImplementation(() => ({ + width: 320, + height: 240, + })); + + (getFavicons as any).mockImplementation(() => ({ + icons: [ + { + src: "http://foo.bar/favicon.ico", + }, + ], + })); + + (parser as any).mockImplementation(() => ({ + og: { + title: "Foo", + description: "Bar", + image: "http://foo.bar/image.jpg", + }, + meta: { + title: "Foo", + description: "Bar", + image: "http://foo.bar/image1.jpg", + }, + images: ["http://foo.bar/image2.jpg"], + })); +}); + +afterEach(() => { + jest.clearAllMocks(); +}); + +afterAll(() => { + jest.restoreAllMocks(); +}); + +describe("the generateLinksPreviews service", () => { + it("should return a promise", () => { + const result = generateLinksPreviews([]); + expect(result).toBeInstanceOf(Promise); + }); + + it("should return a promise that resolves to an array of previews", async () => { + const result = await generateLinksPreviews(["https://foo.bar"]); + expect(result).toEqual([ + { + title: "Foo", + description: "Bar", + img: "http://foo.bar/image.jpg", + favicon: "http://foo.bar/favicon.ico", + img_width: 320, + img_height: 240, + domain: "foo.bar", + }, + ]); + }); + + it("should return a promise that resolves to an empty array if no previews are found", async () => { + (parser as any).mockImplementation(() => { + throw new Error("failed to parse"); + }); + (getFavicons as any).mockImplementation(() => []); + (imageProbe as any).mockImplementation(() => ({})); + + const result = await generateLinksPreviews(["https://foo.bar"]); + expect(result).toEqual([]); + }); + + it("should use og information as first choice", async () => { + (parser as any).mockImplementation(() => ({ + og: { + title: "test", + description: "test", + image: "http://foo.bar/test.jpg", + }, + meta: { + title: "test2", + description: "test2", + image: "http://foo.bar/test2.jpg", + }, + images: ["http://foo.bar/test3.jpg"], + })); + + const result = await generateLinksPreviews(["https://foo.bar"]); + expect(result).toEqual([ + { + title: "test", + description: "test", + img: "http://foo.bar/test.jpg", + favicon: "http://foo.bar/favicon.ico", + img_width: 320, + img_height: 240, + domain: "foo.bar", + }, + ]); + }); + + it("should use meta information as second choice", async () => { + (parser as any).mockImplementation(() => ({ + meta: { + title: "test2", + description: "test2", + image: "http://foo.bar/test2.jpg", + }, + images: [], + })); + + const result = await generateLinksPreviews(["https://foo.bar"]); + expect(result).toEqual([ + { + title: "test2", + description: "test2", + img: "http://foo.bar/test2.jpg", + favicon: "http://foo.bar/favicon.ico", + img_width: 320, + img_height: 240, + domain: "foo.bar", + }, + ]); + }); + + it("should use the first image found in the url when none are present in the og or meta information", async () => { + (parser as any).mockImplementation(() => ({ + og: { + title: "test", + description: "test", + }, + meta: { + title: "test2", + description: "test2", + }, + images: ["http://foo.bar/test3.jpg", "http://foo.bar/test4.jpg"], + })); + + const result = await generateLinksPreviews(["https://foo.bar"]); + expect(result).toEqual([ + { + title: "test", + description: "test", + img: "http://foo.bar/test3.jpg", + favicon: "http://foo.bar/favicon.ico", + img_width: 320, + img_height: 240, + domain: "foo.bar", + }, + ]); + }); + + it("shouldn't attempt to probe for image size when none are found", async () => { + (parser as any).mockImplementation(() => ({ + og: { + title: "test", + description: "test", + }, + meta: { + title: "test2", + description: "test2", + }, + images: [], + })); + + await generateLinksPreviews(["https://foo.bar"]); + expect(imageProbe).not.toHaveBeenCalled(); + }); +});