Add support for Thumbhash MSC2448

package.json

@@ -127,6 +127,7 @@
         "sanitize-filename": "^1.6.3",
         "sanitize-html": "2.11.0",
         "tar-js": "^0.3.0",
+        "thumbhash": "^0.1.1",
         "ua-parser-js": "^1.0.2",
         "uuid": "^9.0.0",
         "what-input": "^5.2.10"

src/ThumbhashEncoder.ts

@@ -0,0 +1,43 @@
+/*
+Copyright 2021 The Matrix.org Foundation C.I.C.
+
+Licensed under the Apache License, Version 2.0 (the "License");
+you may not use this file except in compliance with the License.
+You may obtain a copy of the License at
+
+    http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+See the License for the specific language governing permissions and
+limitations under the License.
+*/
+
+// @ts-ignore - `.ts` is needed here to make TS happy
+import { Request, Response } from "./workers/thumbhash.worker.ts";
+import { WorkerManager } from "./WorkerManager";
+import blurhashWorkerFactory from "./workers/thumbhashWorkerFactory";
+
+export class ThumbhashEncoder {
+    private static internalInstance = new ThumbhashEncoder();
+
+    public static get instance(): ThumbhashEncoder {
+        return ThumbhashEncoder.internalInstance;
+    }
+
+    private readonly worker = new WorkerManager<Request, Response>(blurhashWorkerFactory());
+
+    public getThumbhash(imageData: ImageData): Promise<string> {
+        return this.worker.call({ imageData }).then((resp) => resp.thumbhash);
+    }
+}
+
+export function base64ToArrayBuffer(base64: string): Uint8Array {
+    const binaryString = atob(base64);
+    const bytes = new Uint8Array(binaryString.length);
+    for (let i = 0; i < binaryString.length; i++) {
+        bytes[i] = binaryString.charCodeAt(i);
+    }
+    return bytes;
+}

src/components/views/messages/MImageBody.tsx

@@ -22,14 +22,15 @@ import { CSSTransition, SwitchTransition } from "react-transition-group";
 import { logger } from "matrix-js-sdk/src/logger";
 import { ClientEvent, ClientEventHandlerMap } from "matrix-js-sdk/src/matrix";
 import { Tooltip } from "@vector-im/compound-web";
+import { thumbHashToDataURL } from "thumbhash";
 
 import MFileBody from "./MFileBody";
 import Modal from "../../../Modal";
 import { _t } from "../../../languageHandler";
 import SettingsStore from "../../../settings/SettingsStore";
 import Spinner from "../elements/Spinner";
 import { Media, mediaFromContent } from "../../../customisations/Media";
-import { BLURHASH_FIELD, createThumbnail } from "../../../utils/image-media";
+import { BLURHASH_FIELD, createThumbnail, THUMBHASH_FIELD } from "../../../utils/image-media";
 import { ImageContent } from "../../../customisations/models/IMediaEventContent";
 import ImageView from "../elements/ImageView";
 import { IBodyProps } from "./IBodyProps";
@@ -41,6 +42,7 @@ import { presentableTextForFile } from "../../../utils/FileUtils";
 import { createReconnectedListener } from "../../../utils/connection";
 import MediaProcessingError from "./shared/MediaProcessingError";
 import { DecryptError, DownloadError } from "../../../utils/DecryptFile";
+import { base64ToArrayBuffer } from "../../../ThumbhashEncoder";
 
 enum Placeholder {
     NoImage,
@@ -576,12 +578,25 @@ export default class MImageBody extends React.Component<IBodyProps, IState> {
 
     // Overridden by MStickerBody
     protected getPlaceholder(width: number, height: number): ReactNode {
+        const thumbhash = this.props.mxEvent.getContent().info?.[THUMBHASH_FIELD];
         const blurhash = this.props.mxEvent.getContent().info?.[BLURHASH_FIELD];
 
-        if (blurhash) {
+        if (thumbhash || blurhash) {
             if (this.state.placeholder === Placeholder.NoImage) {
                 return null;
             } else if (this.state.placeholder === Placeholder.Blurhash) {
+                if (thumbhash) {
+                    return (
+                        <img
+                            className="mx_Blurhash"
+                            src={thumbHashToDataURL(base64ToArrayBuffer(thumbhash))}
+                            width={width}
+                            height={height}
+                            role="presentation"
+                            alt=""
+                        />
+                    );
+                }
                 return <Blurhash className="mx_Blurhash" hash={blurhash} width={width} height={height} />;
             }
         }

src/components/views/messages/MVideoBody.tsx

@@ -16,19 +16,21 @@ limitations under the License.
 
 import React, { ReactNode } from "react";
 import { decode } from "blurhash";
+import { thumbHashToDataURL } from "thumbhash";
 import { logger } from "matrix-js-sdk/src/logger";
 
 import { _t } from "../../../languageHandler";
 import SettingsStore from "../../../settings/SettingsStore";
 import InlineSpinner from "../elements/InlineSpinner";
 import { mediaFromContent } from "../../../customisations/Media";
-import { BLURHASH_FIELD } from "../../../utils/image-media";
+import { BLURHASH_FIELD, THUMBHASH_FIELD } from "../../../utils/image-media";
 import { IMediaEventContent } from "../../../customisations/models/IMediaEventContent";
 import { IBodyProps } from "./IBodyProps";
 import MFileBody from "./MFileBody";
 import { ImageSize, suggestedSize as suggestedVideoSize } from "../../../settings/enums/ImageSize";
 import RoomContext, { TimelineRenderingType } from "../../../contexts/RoomContext";
 import MediaProcessingError from "./shared/MediaProcessingError";
+import { base64ToArrayBuffer } from "../../../ThumbhashEncoder";
 
 interface IState {
     decryptedUrl: string | null;
@@ -98,26 +100,35 @@ export default class MVideoBody extends React.PureComponent<IBodyProps, IState>
 
     private loadBlurhash(): void {
         const info = this.props.mxEvent.getContent()?.info;
-        if (!info[BLURHASH_FIELD]) return;
 
-        const canvas = document.createElement("canvas");
+        // We prefer thumbhash over blurhash due to alpha channel support
+        let blurhashUrl: string | undefined;
+        if (info[THUMBHASH_FIELD]) {
+            blurhashUrl = thumbHashToDataURL(base64ToArrayBuffer(info[THUMBHASH_FIELD]));
+        } else if (info[BLURHASH_FIELD]) {
+            const canvas = document.createElement("canvas");
 
-        const { w: width, h: height } = suggestedVideoSize(SettingsStore.getValue("Images.size") as ImageSize, {
-            w: info.w,
-            h: info.h,
-        });
+            const { w: width, h: height } = suggestedVideoSize(SettingsStore.getValue("Images.size") as ImageSize, {
+                w: info.w,
+                h: info.h,
+            });
+
+            canvas.width = width;
+            canvas.height = height;
 
-        canvas.width = width;
-        canvas.height = height;
+            const pixels = decode(info[BLURHASH_FIELD], width, height);
+            const ctx = canvas.getContext("2d")!;
+            const imgData = ctx.createImageData(width, height);
+            imgData.data.set(pixels);
+            ctx.putImageData(imgData, 0, 0);
+
+            blurhashUrl = canvas.toDataURL();
+        }
 
-        const pixels = decode(info[BLURHASH_FIELD], width, height);
-        const ctx = canvas.getContext("2d")!;
-        const imgData = ctx.createImageData(width, height);
-        imgData.data.set(pixels);
-        ctx.putImageData(imgData, 0, 0);
+        if (!blurhashUrl) return;
 
         this.setState({
-            blurhashUrl: canvas.toDataURL(),
+            blurhashUrl,
             posterLoading: true,
         });
 

src/utils/image-media.ts

@@ -16,10 +16,12 @@ limitations under the License.
 
 import { BlurhashEncoder } from "../BlurhashEncoder";
 import { EncryptedFile } from "../customisations/models/IMediaEventContent";
+import { ThumbhashEncoder } from "../ThumbhashEncoder";
 
 type ThumbnailableElement = HTMLImageElement | HTMLVideoElement;
 
-export const BLURHASH_FIELD = "xyz.amorgan.blurhash"; // MSC2448
+export const BLURHASH_FIELD = "xyz.amorgan.blurhash"; // MSC2448 legacy
+export const THUMBHASH_FIELD = "xyz.amorgan.thumbhash"; // MSC2448
 
 interface IThumbnail {
     info: {
@@ -32,6 +34,7 @@ interface IThumbnail {
         w: number;
         h: number;
         [BLURHASH_FIELD]?: string;
+        [THUMBHASH_FIELD]?: string;
         thumbnail_url?: string;
         thumbnail_file?: EncryptedFile;
     };
@@ -103,8 +106,17 @@ export async function createThumbnail(
     }
 
     const imageData = context.getImageData(0, 0, targetWidth, targetHeight);
-    // thumbnailPromise and blurhash promise are being awaited concurrently
-    const blurhash = calculateBlurhash ? await BlurhashEncoder.instance.getBlurhash(imageData) : undefined;
+
+    // thumbnailPromise and blurhash/thumbhash promises are being executed concurrently
+    let blurhash: string | undefined;
+    let thumbhash: string | undefined;
+    if (calculateBlurhash) {
+        [blurhash, thumbhash] = await Promise.all([
+            BlurhashEncoder.instance.getBlurhash(imageData),
+            ThumbhashEncoder.instance.getThumbhash(imageData),
+        ]);
+    }
+
     const thumbnail = await thumbnailPromise;
 
     return {
@@ -118,6 +130,7 @@ export async function createThumbnail(
             w: inputWidth,
             h: inputHeight,
             [BLURHASH_FIELD]: blurhash,
+            [THUMBHASH_FIELD]: thumbhash,
         },
         thumbnail,
     };

src/workers/thumbhash.worker.ts

@@ -0,0 +1,62 @@
+/*
+Copyright 2023 The Matrix.org Foundation C.I.C.
+
+Licensed under the Apache License, Version 2.0 (the "License");
+you may not use this file except in compliance with the License.
+You may obtain a copy of the License at
+
+    http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+See the License for the specific language governing permissions and
+limitations under the License.
+*/
+
+import { rgbaToThumbHash } from "thumbhash";
+
+import { WorkerPayload } from "./worker";
+
+const ctx: Worker = self as any;
+
+export interface Request {
+    imageData: ImageData;
+}
+
+export interface Response {
+    thumbhash: string;
+}
+
+function arrayBufferToBase64(buffer: ArrayBuffer): string {
+    let binary = "";
+    const bytes = new Uint8Array(buffer);
+    const len = bytes.byteLength;
+    for (let i = 0; i < len; i++) {
+        binary += String.fromCharCode(bytes[i]);
+    }
+    return btoa(binary);
+}
+
+ctx.addEventListener("message", async (event: MessageEvent<Request & WorkerPayload>): Promise<void> => {
+    const { seq, imageData } = event.data;
+
+    // We need to resize it to a max of 100px square in order for the lib to process it
+    const maxSize = 100;
+    const width = imageData.width;
+    const height = imageData.height;
+
+    const scale = Math.min(maxSize / width, maxSize / height);
+    const resizedWidth = Math.floor(width * scale);
+    const resizedHeight = Math.floor(height * scale);
+
+    const canvas = new OffscreenCanvas(resizedWidth, resizedHeight);
+    const canvasCtx = canvas.getContext("2d")!;
+    const bitmap = await createImageBitmap(imageData);
+    canvasCtx.drawImage(bitmap, 0, 0, resizedWidth, resizedHeight);
+
+    const rgba = new Uint8Array(canvasCtx.getImageData(0, 0, resizedWidth, resizedHeight).data.buffer);
+    const thumbhash = arrayBufferToBase64(rgbaToThumbHash(resizedWidth, resizedHeight, rgba));
+
+    ctx.postMessage({ seq, thumbhash });
+});

src/workers/thumbhashWorkerFactory.ts

@@ -0,0 +1,22 @@
+/*
+Copyright 2023 The Matrix.org Foundation C.I.C.
+
+Licensed under the Apache License, Version 2.0 (the "License");
+you may not use this file except in compliance with the License.
+You may obtain a copy of the License at
+
+    http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+See the License for the specific language governing permissions and
+limitations under the License.
+*/
+
+export default function factory(options?: WorkerOptions | undefined): Worker {
+    return new Worker(
+        /* webpackChunkName: "thumbhash.worker" */ new URL("./thumbhash.worker.ts", import.meta.url),
+        options,
+    );
+}

yarn.lock

@@ -8833,6 +8833,11 @@ text-table@^0.2.0:
   resolved "https://registry.yarnpkg.com/text-table/-/text-table-0.2.0.tgz#7f5ee823ae805207c00af2df4a84ec3fcfa570b4"
   integrity sha512-N+8UisAXDGk8PFXP4HAzVR9nbfmVJ3zYLAWiTIoqC5v5isinhr+r5uaO8+7r3BMfuNIufIsA7RdpVgacC2cSpw==
 
+thumbhash@^0.1.1:
+  version "0.1.1"
+  resolved "https://registry.yarnpkg.com/thumbhash/-/thumbhash-0.1.1.tgz#bd2b8616fc043f2b17151dfce0cce1408e0ebbeb"
+  integrity sha512-kH5pKeIIBPQXAOni2AiY/Cu/NKdkFREdpH+TLdM0g6WA7RriCv0kPLgP731ady67MhTAqrVG/4mnEeibVuCJcg==
+
 timers-ext@^0.1.7:
   version "0.1.7"
   resolved "https://registry.yarnpkg.com/timers-ext/-/timers-ext-0.1.7.tgz#6f57ad8578e07a3fb9f91d9387d65647555e25c6"