diff --git a/ash/webui/camera_app_ui/resources/js/device/mode/video.ts b/ash/webui/camera_app_ui/resources/js/device/mode/video.ts index 4427f3afa83f7..58014eb180f1a 100644 --- a/ash/webui/camera_app_ui/resources/js/device/mode/video.ts +++ b/ash/webui/camera_app_ui/resources/js/device/mode/video.ts @@ -18,6 +18,7 @@ import {Filenamer} from '../../models/file_namer.js'; import * as loadTimeData from '../../models/load_time_data.js'; import { GifSaver, + TimeLapseSaver, VideoSaver, } from '../../models/video_saver.js'; import {ChromeHelper} from '../../mojo/chrome_helper.js'; @@ -82,6 +83,18 @@ const MAX_GIF_DURATION_MS = 5000; */ const GRAB_GIF_FRAME_RATIO = 2; +/** + * Initial speed for time-lapse recording. + */ +const TIME_LAPSE_INITIAL_SPEED = 5; + +/** + * Number of maximum frames recorded in a specific speed time-lapse video. If + * the current number of frames exceeds, the speed must be increasd. + */ +const TIME_LAPSE_MAX_FRAMES = 5 * 30; + + /** * Sets avc1 parameter used in video recording. */ @@ -89,21 +102,46 @@ export function setAvc1Parameters(params: h264.EncoderParameters|null): void { avc1Parameters = params; } +/** + * Generate AVC suffix string from given h264 params. + * + * @return Suffix string for AVC codecs. + */ +function getAVCSuffix(param: h264.EncoderParameters) { + const {profile, level} = param; + return '.' + profile.toString(16).padStart(2, '0') + + level.toString(16).padStart(4, '0'); +} + /** * Gets video recording MIME type. Mkv with AVC1 is the only preferred format. * * @return Video recording MIME type. */ function getVideoMimeType(param: h264.EncoderParameters|null): string { - let suffix = ''; - if (param !== null) { - const {profile, level} = param; - suffix = '.' + profile.toString(16).padStart(2, '0') + - level.toString(16).padStart(4, '0'); - } + const suffix = param !== null ? getAVCSuffix(param) : ''; return `video/x-matroska;codecs=avc1${suffix},pcm`; } +/** + * Gets VideoEncoder's config from current h264 params and resolutions. + * + * @return VideoEncoderConfig. + */ +function getVideoEncoderConfig( + param: h264.EncoderParameters, resolution: Resolution): VideoEncoderConfig { + const suffix = getAVCSuffix(param); + return { + codec: `avc1${suffix}`, + width: resolution.width, + height: resolution.height, + bitrate: param.bitrate, + bitrateMode: 'constant', + avc: {format: 'annexb'}, + }; +} + + /** * The 'beforeunload' listener which will show confirm dialog when trying to * close window. @@ -128,6 +166,13 @@ export interface GifResult { duration: number; } +export interface TimeLapseResult { + duration: number; + resolution: Resolution; + speed: number; + timeLapseSaver: TimeLapseSaver; +} + /** * Provides functions with external dependency used by video mode and handles * the captured result video. @@ -151,6 +196,8 @@ export interface VideoHandler { onGifCaptureDone(gifResult: GifResult): Promise; onVideoCaptureDone(videoResult: VideoResult): Promise; + + onTimeLapseCaptureDone(timeLapseResult: TimeLapseResult): Promise; } // This is used as an enum. @@ -158,6 +205,7 @@ export interface VideoHandler { const RecordType = { NORMAL: state.State.RECORD_TYPE_NORMAL, GIF: state.State.RECORD_TYPE_GIF, + TIME_LAPSE: state.State.RECORD_TYPE_TIME_LAPSE, } as const; /* eslint-enable @typescript-eslint/naming-convention */ @@ -490,8 +538,9 @@ export class Video extends ModeBase { this.recordingImageCapture = new CrosImageCapture(this.getVideoTrack()); } + let param: h264.EncoderParameters|null = null; try { - const param = this.getEncoderParameters(); + param = this.getEncoderParameters(); const mimeType = getVideoMimeType(param); if (!MediaRecorder.isTypeSupported(mimeType)) { throw new Error( @@ -545,6 +594,25 @@ export class Video extends ModeBase { resolution: this.captureResolution, duration: this.gifRecordTime.inMilliseconds(), })]; + } else if (this.recordingType === RecordType.TIME_LAPSE) { + this.recordTime.start({resume: false}); + let timeLapseSaver: TimeLapseSaver|null = null; + try { + assert(param !== null); + timeLapseSaver = await this.captureTimeLapse(param); + } finally { + // TODO(b/236800499): Handle pause. + // TODO(b/236800499): Handle video too short. + state.set(state.State.RECORDING, false); + this.recordTime.stop({pause: false}); + } + + return [this.handler.onTimeLapseCaptureDone({ + duration: this.recordTime.inMilliseconds(), + resolution: this.captureResolution, + speed: timeLapseSaver.getSpeed(), + timeLapseSaver, + })]; } else { this.recordTime.start({resume: false}); let videoSaver: VideoSaver|null = null; @@ -599,7 +667,8 @@ export class Video extends ModeBase { if (!state.get(state.State.RECORDING)) { return; } - if (this.recordingType === RecordType.GIF) { + if (this.recordingType === RecordType.GIF || + this.recordingType === RecordType.TIME_LAPSE) { state.set(state.State.RECORDING, false); } else { sound.cancel(dom.get('#sound-rec-start', HTMLAudioElement)); @@ -616,8 +685,6 @@ export class Video extends ModeBase { /** * Starts recording gif animation and waits for stop recording event triggered * by stop shutter or time out over 5 seconds. - * - * @return Saves recorded video. */ private async captureGif(): Promise { // TODO(b/191950622): Grab frames from capture stream when multistream @@ -668,11 +735,75 @@ export class Video extends ModeBase { return gifSaver; } + /** + * Initial time-lapse saver with specified encoder parameters. Then, Starts + * recording time-lapse and waits for stop recording event. + */ + private async captureTimeLapse(param: h264.EncoderParameters): + Promise { + const encoderConfig = getVideoEncoderConfig(param, this.captureResolution); + const saver = await TimeLapseSaver.create( + encoderConfig, this.captureResolution, TIME_LAPSE_INITIAL_SPEED); + const video = this.video.video; + + // Handles time-lapse speed adjustment. + let speed = TIME_LAPSE_INITIAL_SPEED; + let speedCheckpoint = speed * TIME_LAPSE_MAX_FRAMES; + function updateSpeed() { + speed = speed * 2; + speedCheckpoint = speed * TIME_LAPSE_MAX_FRAMES; + saver.updateSpeed(speed); + } + + // Creates a frame reader from track processor. + const track = this.getVideoTrack() as MediaStreamVideoTrack; + const trackProcessor = new MediaStreamTrackProcessor({track}); + const reader = trackProcessor.readable.getReader(); + + state.set(state.State.RECORDING, true); + const frames = await new Promise((resolve, reject) => { + let frameCount = 0; + let writtenFrameCount = 0; + // TODO(b/236800499): Investigate whether we should use async, or use + // reader.read().then() instead. + async function updateFrame(): Promise { + if (!state.get(state.State.RECORDING)) { + resolve(writtenFrameCount); + return; + } + if (frameCount % speed === 0) { + try { + const {done, value: frame} = await reader.read(); + if (done) { + resolve(writtenFrameCount); + return; + } + saver.write(frame, frameCount); + writtenFrameCount++; + frame.close(); + if (frameCount >= speedCheckpoint) { + updateSpeed(); + } + } catch (e) { + reject(e); + } + } + frameCount++; + video.requestVideoFrameCallback(updateFrame); + } + video.requestVideoFrameCallback(updateFrame); + }); + + if (frames === 0) { + throw new NoFrameError(); + } + + return saver; + } + /** * Starts recording and waits for stop recording event triggered by stop * shutter. - * - * @return Saves recorded video. */ private async captureVideo(): Promise { const saver = await this.handler.createVideoSaver(); diff --git a/ash/webui/camera_app_ui/resources/js/gallerybutton.ts b/ash/webui/camera_app_ui/resources/js/gallerybutton.ts index 7b16bfc4ef0a5..ce2e441a9f641 100644 --- a/ash/webui/camera_app_ui/resources/js/gallerybutton.ts +++ b/ash/webui/camera_app_ui/resources/js/gallerybutton.ts @@ -13,7 +13,7 @@ import { FileAccessEntry, } from './models/file_system_access_entry.js'; import {ResultSaver} from './models/result_saver.js'; -import {VideoSaver} from './models/video_saver.js'; +import {TimeLapseSaver, VideoSaver} from './models/video_saver.js'; import {ChromeHelper} from './mojo/chrome_helper.js'; import {extractImageFromBlob} from './thumbnailer.js'; import { @@ -247,7 +247,7 @@ export class GalleryButton implements ResultSaver { return VideoSaver.create(videoRotation); } - async finishSaveVideo(video: VideoSaver): Promise { + async finishSaveVideo(video: TimeLapseSaver|VideoSaver): Promise { const file = await video.endWrite(); assert(file !== null); diff --git a/ash/webui/camera_app_ui/resources/js/models/ffmpeg/video_processor_args.ts b/ash/webui/camera_app_ui/resources/js/models/ffmpeg/video_processor_args.ts index 9d69130d7e89a..96377b3658921 100644 --- a/ash/webui/camera_app_ui/resources/js/models/ffmpeg/video_processor_args.ts +++ b/ash/webui/camera_app_ui/resources/js/models/ffmpeg/video_processor_args.ts @@ -62,3 +62,30 @@ export function createGifArgs({width, height}: Resolution): VideoProcessorArgs { return {decoderArgs, encoderArgs, outputExtension: 'gif'}; } + +/** + * Creates the command line arguments to ffmpeg for time-lapse recording. + */ +export function createTimeLapseArgs({width, height}: Resolution): + VideoProcessorArgs { + // clang-format off + const decoderArgs = [ + // input format + '-f', 'h264', + // force input framerate + '-r', '30', + // specify video size + '-s', `${width}x${height}`, + ]; + + // clang-format formats one argument per line, which makes the list harder + // to read with comments. + // clang-format off + const encoderArgs = [ + // disable audio and copy the video stream + '-an', '-c:v', 'copy', + ]; + // clang-format on + + return {decoderArgs, encoderArgs, outputExtension: 'mp4'}; +} diff --git a/ash/webui/camera_app_ui/resources/js/models/result_saver.ts b/ash/webui/camera_app_ui/resources/js/models/result_saver.ts index 7b72f0c5dba7b..1b648016fe22a 100644 --- a/ash/webui/camera_app_ui/resources/js/models/result_saver.ts +++ b/ash/webui/camera_app_ui/resources/js/models/result_saver.ts @@ -4,7 +4,7 @@ import {Metadata} from '../type.js'; -import {VideoSaver} from './video_saver.js'; +import {TimeLapseSaver, VideoSaver} from './video_saver.js'; /** * Handles captured result photos and video. @@ -41,5 +41,5 @@ export interface ResultSaver { * * @param video Contains the video result to be saved. */ - finishSaveVideo(video: VideoSaver): Promise; + finishSaveVideo(video: TimeLapseSaver|VideoSaver): Promise; } diff --git a/ash/webui/camera_app_ui/resources/js/models/video_saver.ts b/ash/webui/camera_app_ui/resources/js/models/video_saver.ts index 290700784bb3f..a2a28f777d29d 100644 --- a/ash/webui/camera_app_ui/resources/js/models/video_saver.ts +++ b/ash/webui/camera_app_ui/resources/js/models/video_saver.ts @@ -2,6 +2,7 @@ // Use of this source code is governed by a BSD-style license that can be // found in the LICENSE file. +import {assert} from '../assert.js'; import {Intent} from '../intent.js'; import * as Comlink from '../lib/comlink.js'; import { @@ -18,6 +19,7 @@ import { import { createGifArgs, createMp4Args, + createTimeLapseArgs, } from './ffmpeg/video_processor_args.js'; import {createPrivateTempVideoFile} from './file_system.js'; import {FileAccessEntry} from './file_system_access_entry.js'; @@ -52,6 +54,16 @@ async function createGifVideoProcessor( Comlink.proxy(output), createGifArgs(resolution)); } +/* + * Creates a VideoProcessor instance for recording time-lapse. + */ +async function createTimeLapseProcessor( + output: AsyncWriter, + resolution: Resolution): Promise> { + return new (await FFMpegVideoProcessor)( + Comlink.proxy(output), createTimeLapseArgs(resolution)); +} + /** * Creates an AsyncWriter that writes to the given intent. */ @@ -159,3 +171,110 @@ export class GifSaver { return new GifSaver(blobs, processor); } } + +interface FrameInfo { + chunk: Blob; + frameNo: number; +} + +/** + * Used to save time-lapse video. + */ +export class TimeLapseSaver { + /** + * Video encoder used to encode frame. + */ + private readonly encoder: VideoEncoder; + + /** + * Map a frame's timestamp with frameNo, only store frame that's being + * encoded. + */ + private readonly frameNoMap = new Map(); + + /** + * Store all encoded frames with its frame numbers. + * TODO(b/236800499): Investigate if it is OK to store number of blobs in + * memory. + */ + private readonly frames: FrameInfo[] = []; + + private speed: number; + + constructor( + encoderConfig: VideoEncoderConfig, + private readonly resolution: Resolution, initialSpeed: number) { + this.speed = initialSpeed; + this.encoder = new VideoEncoder({ + error: (error) => { + throw error; + }, + output: (chunk) => this.onFrameEncoded(chunk), + }); + this.encoder.configure(encoderConfig); + } + + onFrameEncoded(chunk: EncodedVideoChunk): void { + const frameNo = this.frameNoMap.get(chunk.timestamp); + assert(frameNo !== undefined); + const chunkData = new Uint8Array(chunk.byteLength); + chunk.copyTo(chunkData); + this.frames.push({ + chunk: new Blob([chunkData]), + frameNo, + }); + this.frameNoMap.delete(chunk.timestamp); + } + + write(frame: VideoFrame, frameNo: number): void { + if (!frame.timestamp) { + return; + } + this.frameNoMap.set(frame.timestamp, frameNo); + this.encoder.encode(frame, {keyFrame: true}); + } + + updateSpeed(newSpeed: number): void { + this.speed = newSpeed; + } + + getSpeed(): number { + return this.speed; + } + + /** + * Finishes the write of video data parts and returns result video file. + * + * @return Result video file. + */ + async endWrite(): Promise { + // TODO(b/236800499): Optimize file writing mechanism to make it faster. + const file = await createPrivateTempVideoFile(); + const writer = await file.getWriter(); + const processor = await createTimeLapseProcessor(writer, this.resolution); + + const filteredChunk = + this.frames.filter(({frameNo}) => frameNo % this.speed === 0); + for (const {chunk} of filteredChunk) { + processor.write(chunk); + } + await processor.close(); + this.encoder.close(); + + return file; + } + + /** + * Creates video saver with encoder using provided |encoderConfig|. + */ + static async create( + encoderConfig: VideoEncoderConfig, resolution: Resolution, + initialSpeed: number): Promise { + const encoderSupport = await VideoEncoder.isConfigSupported(encoderConfig); + if (!encoderSupport.supported) { + throw new Error('Video encoder is not supported.'); + } + + return new TimeLapseSaver(encoderConfig, resolution, initialSpeed); + } +} diff --git a/ash/webui/camera_app_ui/resources/js/state.ts b/ash/webui/camera_app_ui/resources/js/state.ts index 89cb14f9f0ff8..2101c1eb8e5c1 100644 --- a/ash/webui/camera_app_ui/resources/js/state.ts +++ b/ash/webui/camera_app_ui/resources/js/state.ts @@ -40,6 +40,7 @@ export enum State { PLAYING_RESULT_VIDEO = 'playing-result-video', RECORD_TYPE_GIF = 'record-type-gif', RECORD_TYPE_NORMAL = 'record-type-normal', + RECORD_TYPE_TIME_LAPSE = 'record-type-time-lapse', // Starts/Ends when start/stop event of MediaRecorder is triggered. RECORDING = 'recording', // Binds with paused state of MediaRecorder. diff --git a/ash/webui/camera_app_ui/resources/js/views/camera.ts b/ash/webui/camera_app_ui/resources/js/views/camera.ts index 983c133e46811..e514fba4401b1 100644 --- a/ash/webui/camera_app_ui/resources/js/views/camera.ts +++ b/ash/webui/camera_app_ui/resources/js/views/camera.ts @@ -18,6 +18,7 @@ import { setAvc1Parameters, VideoResult, } from '../device/index.js'; +import {TimeLapseResult} from '../device/mode/video'; import * as dom from '../dom.js'; import * as error from '../error.js'; import * as expert from '../expert.js'; @@ -835,6 +836,14 @@ export class Camera extends View implements CameraViewUI { ChromeHelper.getInstance().maybeTriggerSurvey(); } + async onTimeLapseCaptureDone({timeLapseSaver}: TimeLapseResult): + Promise { + // TODO(b/236800499): Send perf metrics, trigger survey. + nav.open(ViewName.FLASH); + await this.resultSaver.finishSaveVideo(timeLapseSaver); + nav.close(ViewName.FLASH); + } + override layout(): void { this.layoutHandler.update(); } diff --git a/ash/webui/camera_app_ui/resources/js/views/camera_intent.ts b/ash/webui/camera_app_ui/resources/js/views/camera_intent.ts index c8b02357aab84..4ca43d491ac33 100644 --- a/ash/webui/camera_app_ui/resources/js/views/camera_intent.ts +++ b/ash/webui/camera_app_ui/resources/js/views/camera_intent.ts @@ -59,6 +59,7 @@ export class CameraIntent extends Camera { return VideoSaver.createForIntent(intent, outputVideoRotation); }, finishSaveVideo: async (video) => { + assert(video instanceof VideoSaver); this.videoResultFile = await video.endWrite(); }, saveGif: () => { diff --git a/ash/webui/camera_app_ui/resources/tsconfig_base.json b/ash/webui/camera_app_ui/resources/tsconfig_base.json index e3c184ab6a4a6..4c26c10ca7876 100644 --- a/ash/webui/camera_app_ui/resources/tsconfig_base.json +++ b/ash/webui/camera_app_ui/resources/tsconfig_base.json @@ -3,6 +3,8 @@ "compilerOptions": { "typeRoots": ["../../../../third_party/node/node_modules/@types"], "types": [ + "dom-mediacapture-transform", + "dom-webcodecs", "google.analytics", "offscreencanvas", "trusted-types", diff --git a/ash/webui/camera_app_ui/resources/views/main.html b/ash/webui/camera_app_ui/resources/views/main.html index d998f181bb820..303efa5e553f9 100644 --- a/ash/webui/camera_app_ui/resources/views/main.html +++ b/ash/webui/camera_app_ui/resources/views/main.html @@ -173,6 +173,12 @@ +