From 85d27dad48d013d5ebce185e64c0226f6e7da49f Mon Sep 17 00:00:00 2001 From: Arnei Date: Wed, 3 Jun 2026 17:02:12 +0200 Subject: [PATCH] Thumbnails in backend Currently when generating a thumbnail, we create it here in the frontend and then send it to the backend. With this patch, we instead send the timestamp it was generated from to the backend. The goal is to let the backend generate the actual thumbnail, instead of the frontend. This avoids quality issues when generating thumbnails from low resolution videos in the frontend. It also allows for respecting institituion specific thumbnail settings (e.g. encoding profiles). The downside is that this relies on workflow properties. As such, admins will have to adapt their workflows or thumbnail generation will be broken for them. This change should not impact the user experience of using the editor frontend. --- src/main/ThumbnailGeneration.tsx | 3 ++ src/main/ThumbnailSelect.tsx | 63 +++++++++++++++++++++++++++++++- src/main/VideoPlayers.tsx | 4 ++ src/redux/videoSlice.ts | 11 ++++++ src/types.ts | 1 + 5 files changed, 81 insertions(+), 1 deletion(-) diff --git a/src/main/ThumbnailGeneration.tsx b/src/main/ThumbnailGeneration.tsx index 3a31768c8..33abd4bae 100644 --- a/src/main/ThumbnailGeneration.tsx +++ b/src/main/ThumbnailGeneration.tsx @@ -22,6 +22,7 @@ import { setJumpTriggered, setAspectRatio, selectPrimaryThumbnailTrack, + setThumbnailTime, } from "../redux/videoSlice"; import { Track } from "../types"; import Timeline from "./Timeline"; @@ -210,8 +211,10 @@ const ThumbnailActions: React.FC<{ // *track: Generate to // *index: Generate from const generate = (track: Track, index: number) => { + const time = generateRefs.current[index]?.getCurrentTime(); const uri = generateRefs.current[index]?.captureVideo(); dispatch(setThumbnail({ id: track.id, uri: uri })); + dispatch(setThumbnailTime({ id: track.id, time: time?.toString() })); dispatch(setHasChanges(true)); }; diff --git a/src/main/ThumbnailSelect.tsx b/src/main/ThumbnailSelect.tsx index 857a927fd..fbecbec89 100644 --- a/src/main/ThumbnailSelect.tsx +++ b/src/main/ThumbnailSelect.tsx @@ -1,7 +1,7 @@ import { css, SerializedStyles } from "@emotion/react"; import { IconType } from "react-icons"; import { LuCamera, LuCopy, LuCircleX, LuUpload } from "react-icons/lu"; -import React from "react"; +import React, { useEffect, useRef, useState } from "react"; import { useTranslation } from "react-i18next"; import { useAppDispatch, useAppSelector } from "../redux/store"; import { @@ -16,11 +16,13 @@ import { setHasChanges, setThumbnail, setThumbnails, + setThumbnailTime, } from "../redux/videoSlice"; import { Track } from "../types"; import { ThemedTooltip } from "./Tooltip"; import { ProtoButton } from "@opencast/appkit"; import { setIndex, setIsDisplayEditView } from "../redux/thumbnailSlice"; +import ReactPlayer from "react-player"; /** * Choose between various thumbnail actions for the available tracks. @@ -98,6 +100,7 @@ const ThumbnailSelector: React.FC<{ track={track} trackIndex={trackIndex} /> + ); }; @@ -265,6 +268,7 @@ export const UploadButton: React.FC<{ if (e.target && e.target.result) { const uri = e.target.result as string; // We know this must be string because we use "readAsDataURL" dispatch(setThumbnail({ id: track.id, uri: uri })); + dispatch(setThumbnailTime({ id: track.id, time: undefined })); dispatch(setHasChanges(true)); } }; @@ -454,6 +458,63 @@ export const ThumbnailButton: React.FC<{ ); }; +/** + * Generates a temporary thumbnail from a timestamp + * + * Workaround for the backend being unable to send us thumbnails from + * publications. This way, we can at least show a thumbnail to a user + * if they previously generated one via timestamp. + */ +const WorkaroundThumbnailGenerator: React.FC<{ + track: Track, +}> = ({ track }) => { + const dispatch = useAppDispatch(); + + const ref = useRef(null); + const [ready, setReady] = useState(false); + const [seeked, setSeeked] = useState(false); + + useEffect(() => { + if (ref.current && ready && track && track.thumbnailTime && !track.thumbnailUri) { + ref.current.seekTo(parseFloat(track.thumbnailTime), "seconds"); + } + }, [dispatch, ready, track]); + + useEffect(() => { + if (seeked) { + const videoElement = ref.current?.getInternalPlayer() as HTMLVideoElement; + const canvas = document.createElement("canvas"); + canvas.width = videoElement.videoWidth; + canvas.height = videoElement.videoHeight; + const canvasContext = canvas.getContext("2d"); + if (canvasContext !== null) { + canvasContext.drawImage(videoElement, 0, 0); + const uri = canvas.toDataURL("image/png"); + + if (uri) { + dispatch(setThumbnail({ id: track.id, uri: uri })); + } + } + } + }, [dispatch, seeked, track]); + + const playerStyle = css({ + display: "none", + }); + + return ( + setReady(true)} + onSeek={() => setSeeked(true)} + /> + ); +}; + /** * Shared CSS */ diff --git a/src/main/VideoPlayers.tsx b/src/main/VideoPlayers.tsx index d5eee634e..83ec2bba6 100644 --- a/src/main/VideoPlayers.tsx +++ b/src/main/VideoPlayers.tsx @@ -116,6 +116,7 @@ const VideoPlayers: React.FC<{ export interface VideoPlayerForwardRef { captureVideo: () => string | undefined, getWidth: () => number, + getCurrentTime: () => number, } interface VideoPlayerProps { @@ -390,6 +391,9 @@ export const VideoPlayer = React.forwardRef) => { + setThumbnailTimeHelper(state, action.payload.id, action.payload.time); + }, removeThumbnail: (state, action: PayloadAction) => { const index = state.tracks.findIndex(t => t.id === action.payload); state.tracks[index].thumbnailUri = undefined; @@ -573,6 +576,13 @@ const setThumbnailHelper = (state: video, id: Track["id"], uri: Track["thumbnail } }; +const setThumbnailTimeHelper = (state: video, id: Track["id"], time: Track["thumbnailTime"]) => { + const index = state.tracks.findIndex(t => t.id === id); + if (index >= 0) { + state.tracks[index].thumbnailTime = time; + } +}; + export const { addSegment, cut, @@ -603,6 +613,7 @@ export const { setSelectedWorkflowIndex, setThumbnail, setThumbnails, + setThumbnailTime, setVideoEnabled, setVolume, setWaveformImages, diff --git a/src/types.ts b/src/types.ts index ef3f9fa7c..1230dc4e2 100644 --- a/src/types.ts +++ b/src/types.ts @@ -16,6 +16,7 @@ export interface Track { video_stream: {available: boolean, enabled: boolean, thumbnail_uri: string}, thumbnailUri: string | undefined, thumbnailPriority: number, + thumbnailTime?: string, } export interface Flavor {