enhance: Proxy custom emojis to reduce image size and accelerate the frontend

packages/backend/package.json

@@ -105,7 +105,7 @@
 		"sanitize-html": "2.8.1",
 		"seedrandom": "^3.0.5",
 		"semver": "7.3.8",
-		"sharp": "0.29.3",
+		"sharp": "0.31.3",
 		"speakeasy": "2.0.0",
 		"strict-event-emitter-types": "2.0.0",
 		"stringz": "2.1.0",

packages/backend/src/core/CustomEmojiService.ts

@@ -2,12 +2,10 @@ import { Inject, Injectable } from '@nestjs/common';
 import { DataSource, In, IsNull } from 'typeorm';
 import { GlobalEventService } from '@/core/GlobalEventService.js';
 import { DI } from '@/di-symbols.js';
-import type { Config } from '@/config.js';
 import { IdService } from '@/core/IdService.js';
 import type { DriveFile } from '@/models/entities/DriveFile.js';
 import type { Emoji } from '@/models/entities/Emoji.js';
 import { Cache } from '@/misc/cache.js';
-import { query } from '@/misc/prelude/url.js';
 import type { Note } from '@/models/entities/Note.js';
 import type { EmojisRepository } from '@/models/index.js';
 import { UtilityService } from '@/core/UtilityService.js';
@@ -27,9 +25,6 @@ export class CustomEmojiService {
 	private cache: Cache<Emoji | null>;
 
 	constructor(
-		@Inject(DI.config)
-		private config: Config,
-
 		@Inject(DI.db)
 		private db: DataSource,
 
@@ -117,7 +112,7 @@ export class CustomEmojiService {
 
 		const isLocal = emoji.host == null;
 		const emojiUrl = emoji.publicUrl || emoji.originalUrl; // || emoji.originalUrl してるのは後方互換性のため
-		const url = isLocal ? emojiUrl : `${this.config.url}/proxy/${encodeURIComponent((new URL(emojiUrl)).pathname)}?${query({ url: emojiUrl })}`;
+		const url = emojiUrl;
 
 		return {
 			name: emojiName,

packages/backend/src/core/DownloadService.ts

@@ -33,7 +33,7 @@ export class DownloadService {
 
 	@bindThis
 	public async downloadUrl(url: string, path: string): Promise<void> {
-		this.logger.info(`Downloading ${chalk.cyan(url)} ...`);
+		this.logger.info(`Downloading ${chalk.cyan(url)} to ${chalk.cyanBright(path)} ...`);
 
 		const timeout = 30 * 1000;
 		const operationTimeout = 60 * 1000;

packages/backend/src/core/ImageProcessingService.ts

@@ -8,6 +8,16 @@ export type IImage = {
 	ext: string | null;
 	type: string;
 };
+
+export const webpDefault: sharp.WebpOptions = {
+	quality: 85,
+	alphaQuality: 95,
+	lossless: false,
+	nearLossless: false,
+	smartSubsample: true,
+	mixed: true,
+};
+
 import { bindThis } from '@/decorators.js';
 
 @Injectable()
@@ -53,21 +63,19 @@ export class ImageProcessingService {
 	 *   with resize, remove metadata, resolve orientation, stop animation
 	 */
 	@bindThis
-	public async convertToWebp(path: string, width: number, height: number, quality = 85): Promise<IImage> {
-		return this.convertSharpToWebp(await sharp(path), width, height, quality);
+	public async convertToWebp(path: string, width: number, height: number, options: sharp.WebpOptions = webpDefault): Promise<IImage> {
+		return this.convertSharpToWebp(await sharp(path), width, height, options);
 	}
 
 	@bindThis
-	public async convertSharpToWebp(sharp: sharp.Sharp, width: number, height: number, quality = 85): Promise<IImage> {
+	public async convertSharpToWebp(sharp: sharp.Sharp, width: number, height: number, options: sharp.WebpOptions = webpDefault): Promise<IImage> {
 		const data = await sharp
 			.resize(width, height, {
 				fit: 'inside',
 				withoutEnlargement: true,
 			})
 			.rotate()
-			.webp({
-				quality,
-			})
+			.webp(options)
 			.toBuffer();
 
 		return {

packages/backend/src/misc/create-temp.ts

@@ -4,7 +4,7 @@ export function createTemp(): Promise<[string, () => void]> {
 	return new Promise<[string, () => void]>((res, rej) => {
 		tmp.file((e, path, fd, cleanup) => {
 			if (e) return rej(e);
-			res([path, cleanup]);
+			res([path, process.env.NODE_ENV === 'production' ? cleanup : () => {}]);
 		});
 	});
 }
@@ -17,7 +17,7 @@ export function createTempDir(): Promise<[string, () => void]> {
 			},
 			(e, path, cleanup) => {
 				if (e) return rej(e);
-				res([path, cleanup]);
+				res([path, process.env.NODE_ENV === 'production' ? cleanup : () => {}]);
 			},
 		);
 	});

packages/backend/src/misc/prelude/url.ts

@@ -1,3 +1,8 @@
+/* objを検査して
+ * 1. 配列に何も入っていない時はクエリを付けない
+ * 2. プロパティがundefinedの時はクエリを付けない
+ * （new URLSearchParams(obj)ではそこまで丁寧なことをしてくれない）
+ */ 
 export function query(obj: Record<string, unknown>): string {
 	const params = Object.entries(obj)
 		.filter(([, v]) => Array.isArray(v) ? v.length : v !== undefined)

packages/backend/src/server/MediaProxyServerService.ts

@@ -9,7 +9,7 @@ import type { Config } from '@/config.js';
 import { isMimeImage } from '@/misc/is-mime-image.js';
 import { createTemp } from '@/misc/create-temp.js';
 import { DownloadService } from '@/core/DownloadService.js';
-import { ImageProcessingService } from '@/core/ImageProcessingService.js';
+import { ImageProcessingService, webpDefault } from '@/core/ImageProcessingService.js';
 import type { IImage } from '@/core/ImageProcessingService.js';
 import { FILE_TYPE_BROWSERSAFE } from '@/const.js';
 import { StatusError } from '@/misc/status-error.js';
@@ -81,8 +81,21 @@ export class MediaProxyServerService {
 			const isConvertibleImage = isMimeImage(mime, 'sharp-convertible-image');
 
 			let image: IImage;
-
-			if ('static' in request.query && isConvertibleImage) {
+			if ('emoji' in request.query && isConvertibleImage) {
+				const data = await sharp(path, { animated: !('static' in request.query) })
+					.resize({
+						height: 128,
+						withoutEnlargement: true,
+					})
+					.webp(webpDefault)
+					.toBuffer();
+
+				image = {
+					data,
+					ext: 'webp',
+					type: 'image/webp',
+				};
+			} else if ('static' in request.query && isConvertibleImage) {
 				image = await this.imageProcessingService.convertToWebp(path, 498, 280);
 			} else if ('preview' in request.query && isConvertibleImage) {
 				image = await this.imageProcessingService.convertToWebp(path, 200, 200);
@@ -91,7 +104,7 @@ export class MediaProxyServerService {
 					// 画像でないなら404でお茶を濁す
 					throw new StatusError('Unexpected mime', 404);
 				}
-	
+
 				const mask = sharp(path)
 					.resize(96, 96, {
 						fit: 'inside',
@@ -121,8 +134,8 @@ export class MediaProxyServerService {
 					ext: 'png',
 					type: 'image/png',
 				};
-			}	else if (mime === 'image/svg+xml') {
-				image = await this.imageProcessingService.convertToWebp(path, 2048, 2048, 1);
+			} else if (mime === 'image/svg+xml') {
+				image = await this.imageProcessingService.convertToWebp(path, 2048, 2048, webpDefault);
 			} else if (!mime.startsWith('image/') || !FILE_TYPE_BROWSERSAFE.includes(mime)) {
 				throw new StatusError('Rejected type', 403, 'Rejected type');
 			} else {

packages/backend/src/server/web/ClientServerService.ts

@@ -220,7 +220,7 @@ export class ClientServerService {
 			return reply.sendFile('/apple-touch-icon.png', staticAssets);
 		});
 
-		fastify.get<{ Params: { path: string } }>('/emoji/:path(.*)', async (request, reply) => {
+		fastify.get<{ Params: { path: string }; Querystring: { static?: any; }; }>('/emoji/:path(.*)', async (request, reply) => {
 			const path = request.params.path;
 
 			if (!path.match(/^[a-zA-Z0-9\-_@\.]+?\.webp$/)) {
@@ -244,8 +244,15 @@ export class ClientServerService {
 
 			reply.header('Content-Security-Policy', 'default-src \'none\'; style-src \'unsafe-inline\'');
 
-			// ?? emoji.originalUrl してるのは後方互換性のため
-			return await reply.redirect(301, emoji.publicUrl ?? emoji.originalUrl);
+			const url = new URL("/proxy/emoji.webp", this.config.url);
+			url.searchParams.set('url', emoji.publicUrl ?? emoji.originalUrl); // ?? emoji.originalUrl してるのは後方互換性のため
+			url.searchParams.set('emoji', '1');
+			if ('static' in request.query) url.searchParams.set('static', '1');
+
+			return await reply.redirect(
+				301,
+				url.toString(),
+			);
 		});
 
 		fastify.get<{ Params: { path: string } }>('/fluent-emoji/:path(.*)', async (request, reply) => {

packages/frontend/src/components/MkAutocomplete.vue

@@ -16,12 +16,12 @@
 		</li>
 	</ol>
 	<ol v-else-if="emojis.length > 0" ref="suggests" class="emojis">
-		<li v-for="emoji in emojis" tabindex="-1" @click="complete(type, emoji.emoji)" @keydown="onKeydown">
-			<span v-if="emoji.isCustomEmoji" class="emoji"><img :src="`/emoji/${emoji.name}.webp`" :alt="emoji.emoji"/></span>
-			<span v-else-if="defaultStore.state.emojiStyle != 'native'" class="emoji"><img :src="emoji.url" :alt="emoji.emoji"/></span>
-			<span v-else class="emoji">{{ emoji.emoji }}</span>
+		<li v-for="emoji in emojis" tabindex="-1" :key="emoji.emoji" @click="complete(type, emoji.emoji)" @keydown="onKeydown">
+			<div class="emoji">
+				<MkEmoji :emoji="emoji.emoji" />
+			</div>
 			<!-- eslint-disable-next-line vue/no-v-html -->
-			<span class="name" v-html="emoji.name.replace(q, `<b>${q}</b>`)"></span>
+			<span class="name" v-html="emoji.name.replace(q ?? '', `<b>${q}</b>`)"></span>
 			<span v-if="emoji.aliasOf" class="alias">({{ emoji.aliasOf }})</span>
 		</li>
 	</ol>
@@ -37,7 +37,6 @@
 import { markRaw, ref, onUpdated, onMounted, onBeforeUnmount, nextTick, watch } from 'vue';
 import contains from '@/scripts/contains';
 import { char2twemojiFilePath, char2fluentEmojiFilePath } from '@/scripts/emoji-base';
-import { getStaticImageUrl } from '@/scripts/get-static-image-url';
 import { acct } from '@/filters/user';
 import * as os from '@/os';
 import { MFM_TAGS } from '@/scripts/mfm-tags';
@@ -49,9 +48,13 @@ import { i18n } from '@/i18n';
 type EmojiDef = {
 	emoji: string;
 	name: string;
+	url: string;
 	aliasOf?: string;
-	url?: string;
-	isCustomEmoji?: boolean;
+} | {
+	emoji: string;
+	name: string;
+	aliasOf?: string;
+	isCustomEmoji?: true;
 };
 
 const lib = emojilist.filter(x => x.category !== 'flags');
@@ -87,7 +90,6 @@ for (const x of customEmojis) {
 	emojiDefinitions.push({
 		name: x.name,
 		emoji: `:${x.name}:`,
-		url: x.url,
 		isCustomEmoji: true,
 	});
 
@@ -97,7 +99,6 @@ for (const x of customEmojis) {
 				name: alias,
 				aliasOf: x.name,
 				emoji: `:${x.name}:`,
-				url: x.url,
 				isCustomEmoji: true,
 			});
 		}
@@ -452,14 +453,20 @@ onBeforeUnmount(() => {
 	> .emojis > li {
 
 		.emoji {
-			display: inline-block;
+			display: flex;
 			margin: 0 4px 0 0;
+			height: 24px;
 			width: 24px;
+			justify-content: center;
+			align-items: center;
+			font-size: 20px;
 
 			> img {
+				height: 24px;
 				width: 24px;
-				vertical-align: bottom;
+				object-fit: scale-down;
 			}
+
 		}
 
 		.alias {

packages/frontend/src/components/MkEmojiPicker.vue

@@ -81,7 +81,6 @@ import { ref, computed, watch, onMounted } from 'vue';
 import * as Misskey from 'misskey-js';
 import XSection from '@/components/MkEmojiPicker.section.vue';
 import { emojilist, UnicodeEmojiDef, unicodeEmojiCategories as categories } from '@/scripts/emojilist';
-import { getStaticImageUrl } from '@/scripts/get-static-image-url';
 import Ripple from '@/components/MkRipple.vue';
 import * as os from '@/os';
 import { isTouchUsing } from '@/scripts/touch';

packages/frontend/src/components/MkMediaImage.vue

@@ -23,7 +23,7 @@
 <script lang="ts" setup>
 import { watch } from 'vue';
 import * as misskey from 'misskey-js';
-import { getStaticImageUrl } from '@/scripts/get-static-image-url';
+import { getStaticImageUrl } from '@/scripts/media-proxy';
 import ImgWithBlurhash from '@/components/MkImgWithBlurhash.vue';
 import { defaultStore } from '@/store';
 

packages/frontend/src/components/global/MkAvatar.vue

@@ -12,7 +12,7 @@
 <script lang="ts" setup>
 import { onMounted, watch } from 'vue';
 import * as misskey from 'misskey-js';
-import { getStaticImageUrl } from '@/scripts/get-static-image-url';
+import { getStaticImageUrl } from '@/scripts/media-proxy';
 import { extractAvgColorFromBlurhash } from '@/scripts/extract-avg-color-from-blurhash';
 import { acct, userPage } from '@/filters/user';
 import MkUserOnlineIndicator from '@/components/MkUserOnlineIndicator.vue';

packages/frontend/src/components/global/MkEmoji.vue

@@ -7,7 +7,7 @@
 
 <script lang="ts" setup>
 import { computed } from 'vue';
-import { getStaticImageUrl } from '@/scripts/get-static-image-url';
+import { getStaticImageUrl } from '@/scripts/media-proxy';
 import { char2twemojiFilePath, char2fluentEmojiFilePath } from '@/scripts/emoji-base';
 import { defaultStore } from '@/store';
 import { getEmojiName } from '@/scripts/emojilist';

packages/frontend/src/pages/user/index.photos.vue

@@ -21,7 +21,7 @@
 <script lang="ts" setup>
 import { onMounted } from 'vue';
 import * as misskey from 'misskey-js';
-import { getStaticImageUrl } from '@/scripts/get-static-image-url';
+import { getStaticImageUrl } from '@/scripts/media-proxy';
 import { notePage } from '@/filters/note';
 import * as os from '@/os';
 import MkContainer from '@/components/MkContainer.vue';

packages/frontend/src/scripts/media-proxy.ts

@@ -1,7 +1,15 @@
-import { query } from '@/scripts/url';
+import { query, appendQuery } from '@/scripts/url';
 import { url } from '@/config';
 
 export function getProxiedImageUrl(imageUrl: string, type?: 'preview'): string {
+	if (imageUrl.startsWith(`${url}/proxy/`) || imageUrl.startsWith('/proxy/')) {
+		// もう既にproxyっぽそうだったらsearchParams付けるだけ
+		return appendQuery(imageUrl, query({
+			fallback: '1',
+			...(type ? { [type]: '1' } : {}),
+		}));
+	}
+
 	return `${url}/proxy/image.webp?${query({
 		url: imageUrl,
 		fallback: '1',
@@ -13,3 +21,27 @@ export function getProxiedImageUrlNullable(imageUrl: string | null | undefined,
 	if (imageUrl == null) return null;
 	return getProxiedImageUrl(imageUrl, type);
 }
+
+export function getStaticImageUrl(baseUrl: string): string {
+	const u = baseUrl.startsWith('http') ? new URL(baseUrl) : new URL(baseUrl, url);
+
+	if (u.href.startsWith(`${url}/proxy/`)) {
+		// もう既にproxyっぽそうだったらsearchParams付けるだけ
+		u.searchParams.set('static', '1');
+		return u.href;
+	}
+
+	if (u.href.startsWith(`${url}/emoji/`)) {
+		// もう既にemojiっぽそうだったらsearchParams付けるだけ
+		u.searchParams.set('static', '1');
+		return u.href;
+	}
+
+	// 拡張子がないとキャッシュしてくれないCDNがあるのでダミーの名前を指定する
+	const dummy = `${encodeURIComponent(`${u.host}${u.pathname}`)}.webp`;
+
+	return `${url}/proxy/${dummy}?${query({
+		url: u.href,
+		static: '1',
+	})}`;
+}

packages/frontend/src/scripts/url.ts

@@ -1,3 +1,8 @@
+/* objを検査して
+ * 1. 配列に何も入っていない時はクエリを付けない
+ * 2. プロパティがundefinedの時はクエリを付けない
+ * （new URLSearchParams(obj)ではそこまで丁寧なことをしてくれない）
+ */ 
 export function query(obj: Record<string, any>): string {
 	const params = Object.entries(obj)
 		.filter(([, v]) => Array.isArray(v) ? v.length : v !== undefined)