assets: rework mime-type detection to be consistent/centralized; add …

…support for webp/webm, apng, avif (#3730)

As I started working on image LOD stuff and wrapping my head around the
codebase, this was bothering me.
- there are missing popular types, especially WebP
- there are places where we're copy/pasting the same list of types but
they can get out-of-date with each other (also, one place described
supporting webm but we didn't actually do that)

This adds animated apng/avif detection as well (alongside our animated
gif detection). Furthermore, it moves the gif logic to be alongside the
png logic (they were in separate packages unnecessarily)

### Change Type

<!-- ❗ Please select a 'Scope' label ❗️ -->

- [x] `sdk` — Changes the tldraw SDK
- [ ] `dotcom` — Changes the tldraw.com web app
- [ ] `docs` — Changes to the documentation, examples, or templates.
- [ ] `vs code` — Changes to the vscode plugin
- [ ] `internal` — Does not affect user-facing stuff

<!-- ❗ Please select a 'Type' label ❗️ -->

- [ ] `bugfix` — Bug fix
- [ ] `feature` — New feature
- [x] `improvement` — Improving existing features
- [ ] `chore` — Updating dependencies, other boring stuff
- [ ] `galaxy brain` — Architectural changes
- [ ] `tests` — Changes to any test code
- [ ] `tools` — Changes to infrastructure, CI, internal scripts,
debugging tools, etc.
- [ ] `dunno` — I don't know


### Release Notes

- Images: unify list of acceptable types and expand to include webp,
webm, apng, avif

apps/dotcom/src/hooks/useMultiplayerAssets.ts

@@ -1,7 +1,6 @@
 import { useCallback } from 'react'
 import {
 	AssetRecordType,
-	DEFAULT_ACCEPTED_IMG_TYPE,
 	MediaHelpers,
 	TLAsset,
 	TLAssetId,
@@ -25,7 +24,7 @@ export function useMultiplayerAssets(assetUploaderUrl: string) {
 
 			const assetId: TLAssetId = AssetRecordType.createId(getHashForString(url))
 
-			const isImageType = DEFAULT_ACCEPTED_IMG_TYPE.includes(file.type)
+			const isImageType = MediaHelpers.isImageType(file.type)
 
 			let size: {
 				w: number
@@ -35,7 +34,7 @@ export function useMultiplayerAssets(assetUploaderUrl: string) {
 
 			if (isImageType) {
 				size = await MediaHelpers.getImageSize(file)
-				if (file.type === 'image/gif') {
+				if (MediaHelpers.isAnimatedImageType(file.type)) {
 					isAnimated = true // await getIsGifAnimated(file) todo export me from editor
 				} else {
 					isAnimated = false

apps/dotcom/src/utils/createAssetFromFile.ts

@@ -1,6 +1,5 @@
 import {
 	AssetRecordType,
-	DEFAULT_ACCEPTED_IMG_TYPE,
 	MediaHelpers,
 	TLAsset,
 	TLAssetId,
@@ -23,7 +22,7 @@ export async function createAssetFromFile({ file }: { type: 'file'; file: File }
 
 	const assetId: TLAssetId = AssetRecordType.createId(getHashForString(url))
 
-	const isImageType = DEFAULT_ACCEPTED_IMG_TYPE.includes(file.type)
+	const isImageType = MediaHelpers.isImageType(file.type)
 
 	let size: {
 		w: number
@@ -33,7 +32,7 @@ export async function createAssetFromFile({ file }: { type: 'file'; file: File }
 
 	if (isImageType) {
 		size = await MediaHelpers.getImageSize(file)
-		if (file.type === 'image/gif') {
+		if (MediaHelpers.isAnimatedImageType(file.type)) {
 			isAnimated = true // await getIsGifAnimated(file) todo export me from editor
 		} else {
 			isAnimated = false

apps/examples/src/examples/hosted-images/HostedImagesExample.tsx

@@ -7,7 +7,6 @@ import {
 	TLAssetId,
 	Tldraw,
 	getHashForString,
-	isGifAnimated,
 	uniqueId,
 } from 'tldraw'
 import 'tldraw/tldraw.css'
@@ -40,10 +39,10 @@ export default function HostedImagesExample() {
 			let shapeType: 'image' | 'video'
 
 			//[c]
-			if (['image/jpeg', 'image/png', 'image/gif', 'image/svg+xml'].includes(file.type)) {
+			if (MediaHelpers.isImageType(file.type)) {
 				shapeType = 'image'
 				size = await MediaHelpers.getImageSize(file)
-				isAnimated = file.type === 'image/gif' && (await isGifAnimated(file))
+				isAnimated = await MediaHelpers.isAnimated(file)
 			} else {
 				shapeType = 'video'
 				isAnimated = true

apps/examples/src/examples/image-annotator/ImagePicker.tsx

@@ -1,5 +1,5 @@
 import { useState } from 'react'
-import { FileHelpers, MediaHelpers } from 'tldraw'
+import { DEFAULT_SUPPORTED_MEDIA_TYPE_LIST, FileHelpers, MediaHelpers } from 'tldraw'
 import anakin from './assets/anakin.jpeg'
 import distractedBf from './assets/distracted-bf.jpeg'
 import expandingBrain from './assets/expanding-brain.png'
@@ -13,7 +13,7 @@ export function ImagePicker({
 	function onClickChooseImage() {
 		const input = window.document.createElement('input')
 		input.type = 'file'
-		input.accept = 'image/jpeg,image/png,image/gif,image/svg+xml,video/mp4,video/quicktime'
+		input.accept = DEFAULT_SUPPORTED_MEDIA_TYPE_LIST
 		input.addEventListener('change', async (e) => {
 			const fileList = (e.target as HTMLInputElement).files
 			if (!fileList || fileList.length === 0) return

packages/tldraw/api-report.md

@@ -328,12 +328,6 @@ export function CutMenuItem(): JSX_2.Element;
 // @public (undocumented)
 export function DebugFlags(): JSX_2.Element | null;
 
-// @public (undocumented)
-export const DEFAULT_ACCEPTED_IMG_TYPE: string[];
-
-// @public (undocumented)
-export const DEFAULT_ACCEPTED_VID_TYPE: string[];
-
 // @public (undocumented)
 export const DefaultActionsMenu: NamedExoticComponent<TLUiActionsMenuProps>;
 
@@ -586,7 +580,7 @@ export function ExportFileContentSubMenu(): JSX_2.Element;
 // @public
 export function exportToBlob({ editor, ids, format, opts, }: {
     editor: Editor;
-    format: 'jpeg' | 'json' | 'png' | 'svg' | 'webp';
+    format: TLExportType;
     ids: TLShapeId[];
     opts?: Partial<TLSvgOptions>;
 }): Promise<Blob>;
@@ -951,6 +945,8 @@ export class ImageShapeUtil extends BaseBoxShapeUtil<TLImageShape> {
     // (undocumented)
     indicator(shape: TLImageShape): JSX_2.Element | null;
     // (undocumented)
+    isAnimated(shape: TLImageShape): boolean;
+    // (undocumented)
     isAspectRatioLocked: () => boolean;
     // (undocumented)
     static migrations: TLPropsMigrations;
@@ -976,9 +972,6 @@ export class ImageShapeUtil extends BaseBoxShapeUtil<TLImageShape> {
     static type: "image";
 }
 
-// @public (undocumented)
-export function isGifAnimated(file: Blob): Promise<boolean>;
-
 // @public (undocumented)
 export function KeyboardShortcutsMenuItem(): JSX_2.Element | null;
 

packages/tldraw/src/index.ts

@@ -115,13 +115,7 @@ export {
 } from './lib/ui/hooks/useTranslation/useTranslation'
 export { type TLUiIconType } from './lib/ui/icon-types'
 export { useDefaultHelpers, type TLUiOverrides } from './lib/ui/overrides'
-export {
-	DEFAULT_ACCEPTED_IMG_TYPE,
-	DEFAULT_ACCEPTED_VID_TYPE,
-	containBoxSize,
-	downsizeImage,
-	isGifAnimated,
-} from './lib/utils/assets/assets'
+export { containBoxSize, downsizeImage } from './lib/utils/assets/assets'
 export { getEmbedInfo } from './lib/utils/embeds/embeds'
 export { copyAs } from './lib/utils/export/copyAs'
 export { exportToBlob, getSvgAsImage } from './lib/utils/export/export'

packages/tldraw/src/lib/Tldraw.tsx

@@ -1,4 +1,6 @@
 import {
+	DEFAULT_SUPPORTED_IMAGE_TYPES,
+	DEFAULT_SUPPORT_VIDEO_TYPES,
 	Editor,
 	ErrorScreen,
 	Expand,
@@ -148,21 +150,12 @@ export function Tldraw(props: TldrawProps) {
 	)
 }
 
-const defaultAcceptedImageMimeTypes = Object.freeze([
-	'image/jpeg',
-	'image/png',
-	'image/gif',
-	'image/svg+xml',
-])
-
-const defaultAcceptedVideoMimeTypes = Object.freeze(['video/mp4', 'video/quicktime'])
-
 // We put these hooks into a component here so that they can run inside of the context provided by TldrawEditor and TldrawUi.
 function InsideOfEditorAndUiContext({
 	maxImageDimension = 1000,
 	maxAssetSize = 10 * 1024 * 1024, // 10mb
-	acceptedImageMimeTypes = defaultAcceptedImageMimeTypes,
-	acceptedVideoMimeTypes = defaultAcceptedVideoMimeTypes,
+	acceptedImageMimeTypes = DEFAULT_SUPPORTED_IMAGE_TYPES,
+	acceptedVideoMimeTypes = DEFAULT_SUPPORT_VIDEO_TYPES,
 	onMount,
 }: Partial<TLExternalContentProps & { onMount: TLOnMountHandler }>) {
 	const editor = useEditor()

packages/tldraw/src/lib/defaultExternalContentHandlers.ts

@@ -22,7 +22,7 @@ import {
 import { FONT_FAMILIES, FONT_SIZES, TEXT_PROPS } from './shapes/shared/default-shape-constants'
 import { TLUiToastsContextType } from './ui/context/toasts'
 import { useTranslation } from './ui/hooks/useTranslation/useTranslation'
-import { containBoxSize, downsizeImage, isGifAnimated } from './utils/assets/assets'
+import { containBoxSize, downsizeImage } from './utils/assets/assets'
 import { getEmbedInfo } from './utils/embeds/embeds'
 import { cleanupText, isRightToLeftLanguage, truncateStringWithEllipsis } from './utils/text/text'
 
@@ -32,9 +32,9 @@ export type TLExternalContentProps = {
 	maxImageDimension: number
 	// The maximum size (in bytes) of an asset. Assets larger than this will be rejected. Defaults to 10mb (10 * 1024 * 1024).
 	maxAssetSize: number
-	// The mime types of images that are allowed to be handled. Defaults to ['image/jpeg', 'image/png', 'image/gif', 'image/svg+xml'].
+	// The mime types of images that are allowed to be handled. Defaults to DEFAULT_SUPPORTED_IMAGE_TYPES.
 	acceptedImageMimeTypes: readonly string[]
-	// The mime types of videos that are allowed to be handled. Defaults to ['video/mp4', 'video/webm', 'video/quicktime'].
+	// The mime types of videos that are allowed to be handled. Defaults to DEFAULT_SUPPORT_VIDEO_TYPES.
 	acceptedVideoMimeTypes: readonly string[]
 }
 
@@ -70,19 +70,19 @@ export function registerDefaultExternalContentHandlers(
 			? await MediaHelpers.getImageSize(file)
 			: await MediaHelpers.getVideoSize(file)
 
-		const isAnimated = file.type === 'image/gif' ? await isGifAnimated(file) : isVideoType
+		const isAnimated = (await MediaHelpers.isAnimated(file)) || isVideoType
 
 		const hash = await getHashForBuffer(await file.arrayBuffer())
 
 		if (isFinite(maxImageDimension)) {
 			const resizedSize = containBoxSize(size, { w: maxImageDimension, h: maxImageDimension })
-			if (size !== resizedSize && (file.type === 'image/jpeg' || file.type === 'image/png')) {
+			if (size !== resizedSize && MediaHelpers.isStaticImageType(file.type)) {
 				size = resizedSize
 			}
 		}
 
 		// Always rescale the image
-		if (file.type === 'image/jpeg' || file.type === 'image/png') {
+		if (!isAnimated && MediaHelpers.isStaticImageType(file.type)) {
 			file = await downsizeImage(file, size.w, size.h, {
 				type: file.type,
 				quality: 0.92,

packages/tldraw/src/lib/shapes/image/ImageShapeUtil.tsx

@@ -3,6 +3,7 @@ import {
 	BaseBoxShapeUtil,
 	FileHelpers,
 	HTMLContainer,
+	MediaHelpers,
 	TLImageShape,
 	TLOnDoubleClickHandler,
 	TLShapePartial,
@@ -43,6 +44,17 @@ export class ImageShapeUtil extends BaseBoxShapeUtil<TLImageShape> {
 		}
 	}
 
+	isAnimated(shape: TLImageShape) {
+		const asset = shape.props.assetId ? this.editor.getAsset(shape.props.assetId) : undefined
+
+		if (!asset) return false
+
+		return (
+			('mimeType' in asset.props && MediaHelpers.isAnimatedImageType(asset?.props.mimeType)) ||
+			('isAnimated' in asset.props && asset.props.isAnimated)
+		)
+	}
+
 	component(shape: TLImageShape) {
 		const isCropping = this.editor.getCroppingShapeId() === shape.id
 		const prefersReducedMotion = usePrefersReducedMotion()
@@ -53,7 +65,7 @@ export class ImageShapeUtil extends BaseBoxShapeUtil<TLImageShape> {
 		const isSelected = shape.id === this.editor.getOnlySelectedShapeId()
 
 		useEffect(() => {
-			if (asset?.props.src && 'mimeType' in asset.props && asset?.props.mimeType === 'image/gif') {
+			if (asset?.props.src && this.isAnimated(shape)) {
 				let cancelled = false
 				const url = asset.props.src
 				if (!url) return
@@ -79,7 +91,7 @@ export class ImageShapeUtil extends BaseBoxShapeUtil<TLImageShape> {
 					cancelled = true
 				}
 			}
-		}, [prefersReducedMotion, asset?.props])
+		}, [prefersReducedMotion, asset?.props, shape])
 
 		if (asset?.type === 'bookmark') {
 			throw Error("Bookmark assets can't be rendered as images")
@@ -92,8 +104,7 @@ export class ImageShapeUtil extends BaseBoxShapeUtil<TLImageShape> {
 
 		// We only want to reduce motion for mimeTypes that have motion
 		const reduceMotion =
-			prefersReducedMotion &&
-			(asset?.props.mimeType?.includes('video') || asset?.props.mimeType?.includes('gif'))
+			prefersReducedMotion && (asset?.props.mimeType?.includes('video') || this.isAnimated(shape))
 
 		const containerStyle = getCroppedContainerStyle(shape)
 
@@ -151,7 +162,7 @@ export class ImageShapeUtil extends BaseBoxShapeUtil<TLImageShape> {
 							}}
 							draggable={false}
 						/>
-						{asset.props.isAnimated && !shape.props.playing && (
+						{this.isAnimated(shape) && !shape.props.playing && (
 							<div className="tl-image__tg">GIF</div>
 						)}
 					</div>
@@ -218,8 +229,7 @@ export class ImageShapeUtil extends BaseBoxShapeUtil<TLImageShape> {
 
 		if (!asset) return
 
-		const canPlay =
-			asset.props.src && 'mimeType' in asset.props && asset.props.mimeType === 'image/gif'
+		const canPlay = asset.props.src && this.isAnimated(shape)
 
 		if (!canPlay) return
 

packages/tldraw/src/lib/tools/SelectTool/childStates/Crop/children/Idle.ts

@@ -133,7 +133,13 @@ export class Idle extends StateNode {
 
 		if (info.target === 'selection') {
 			util.onDoubleClickEdge?.(shape)
+			return
 		}
+
+		// If the user double clicks the canvas, we want to cancel cropping,
+		// especially if it's an animated image, we want the image to continue playing.
+		this.cancel()
+		this.editor.root.handleEvent(info)
 	}
 
 	override onKeyDown: TLEventHandlers['onKeyDown'] = () => {

packages/tldraw/src/lib/ui/hooks/useInsertMedia.ts

@@ -1,4 +1,4 @@
-import { useEditor } from '@tldraw/editor'
+import { DEFAULT_SUPPORTED_MEDIA_TYPE_LIST, useEditor } from '@tldraw/editor'
 import { useCallback, useEffect, useRef } from 'react'
 
 export function useInsertMedia() {
@@ -8,7 +8,7 @@ export function useInsertMedia() {
 	useEffect(() => {
 		const input = window.document.createElement('input')
 		input.type = 'file'
-		input.accept = 'image/jpeg,image/png,image/gif,image/svg+xml,video/mp4,video/quicktime'
+		input.accept = DEFAULT_SUPPORTED_MEDIA_TYPE_LIST
 		input.multiple = true
 		inputRef.current = input
 		async function onchange(e: Event) {

packages/tldraw/src/lib/utils/assets/assets.ts

@@ -1,6 +1,5 @@
 import { MediaHelpers, assertExists } from '@tldraw/editor'
 import { clampToBrowserMaxCanvasSize } from '../../shapes/shared/getBrowserCanvasMaxSize'
-import { isAnimated } from './is-gif-animated'
 
 type BoxWidthHeight = {
 	w: number
@@ -91,13 +90,3 @@ export async function downsizeImage(
 		)
 	})
 }
-
-/** @public */
-export const DEFAULT_ACCEPTED_IMG_TYPE = ['image/jpeg', 'image/png', 'image/gif', 'image/svg+xml']
-/** @public */
-export const DEFAULT_ACCEPTED_VID_TYPE = ['video/mp4', 'video/quicktime']
-
-/** @public */
-export async function isGifAnimated(file: Blob): Promise<boolean> {
-	return isAnimated(await file.arrayBuffer())
-}

packages/tldraw/src/lib/utils/export/export.ts

@@ -7,6 +7,7 @@ import {
 	exhaustiveSwitchError,
 } from '@tldraw/editor'
 import { clampToBrowserMaxCanvasSize } from '../../shapes/shared/getBrowserCanvasMaxSize'
+import { TLExportType } from './exportAs'
 
 /** @public */
 export async function getSvgAsImage(
@@ -143,7 +144,7 @@ export async function exportToBlob({
 }: {
 	editor: Editor
 	ids: TLShapeId[]
-	format: 'svg' | 'png' | 'jpeg' | 'webp' | 'json'
+	format: TLExportType
 	opts?: Partial<TLSvgOptions>
 }): Promise<Blob> {
 	switch (format) {
@@ -185,7 +186,7 @@ const mimeTypeByFormat = {
 export function exportToBlobPromise(
 	editor: Editor,
 	ids: TLShapeId[],
-	format: 'svg' | 'png' | 'jpeg' | 'webp' | 'json',
+	format: TLExportType,
 	opts = {} as Partial<TLSvgOptions>
 ): { blobPromise: Promise<Blob>; mimeType: string } {
 	return {

packages/utils/api-report.md

@@ -37,6 +37,15 @@ export function debounce<T extends unknown[], U>(callback: (...args: T) => Promi
 // @public
 export function dedupe<T>(input: T[], equals?: (a: any, b: any) => boolean): T[];
 
+// @public (undocumented)
+export const DEFAULT_SUPPORT_VIDEO_TYPES: readonly string[];
+
+// @public (undocumented)
+export const DEFAULT_SUPPORTED_IMAGE_TYPES: readonly string[];
+
+// @public (undocumented)
+export const DEFAULT_SUPPORTED_MEDIA_TYPE_LIST: string;
+
 // @internal
 export function deleteFromLocalStorage(key: string): void;
 
@@ -195,6 +204,14 @@ export class MediaHelpers {
         h: number;
         w: number;
     }>;
+    // (undocumented)
+    static isAnimated(file: Blob): Promise<boolean>;
+    // (undocumented)
+    static isAnimatedImageType(mimeType: null | string): boolean;
+    // (undocumented)
+    static isImageType(mimeType: string): boolean;
+    // (undocumented)
+    static isStaticImageType(mimeType: null | string): boolean;
     static loadImage(src: string): Promise<HTMLImageElement>;
     static loadVideo(src: string): Promise<HTMLVideoElement>;
     // (undocumented)