AV1 record

flutter/lib/common/widgets/toolbar.dart

@@ -224,11 +224,8 @@ List<TTextMenu> toolbarControls(BuildContext context, String id, FFI ffi) {
     ));
   }
   // record
-  var codecFormat = ffi.qualityMonitorModel.data.codecFormat;
   if (!isDesktop &&
-      (ffi.recordingModel.start ||
-          (perms["recording"] != false &&
-              (codecFormat == "VP8" || codecFormat == "VP9")))) {
+      (ffi.recordingModel.start || (perms["recording"] != false))) {
     v.add(TTextMenu(
         child: Row(
           children: [

flutter/lib/desktop/pages/remote_page.dart

@@ -677,7 +677,8 @@ class _ImagePaintState extends State<ImagePaint> {
     } else {
       final key = cache.updateGetKey(scale);
       if (!cursor.cachedKeys.contains(key)) {
-        debugPrint("Register custom cursor with key $key (${cache.hotx},${cache.hoty})");
+        debugPrint(
+            "Register custom cursor with key $key (${cache.hotx},${cache.hoty})");
         // [Safety]
         // It's ok to call async registerCursor in current synchronous context,
         // because activating the cursor is also an async call and will always

flutter/lib/desktop/widgets/remote_toolbar.dart

@@ -478,7 +478,7 @@ class _RemoteToolbarState extends State<RemoteToolbar> {
       toolbarItems.add(_ChatMenu(id: widget.id, ffi: widget.ffi));
       toolbarItems.add(_VoiceCallMenu(id: widget.id, ffi: widget.ffi));
     }
-    toolbarItems.add(_RecordMenu(ffi: widget.ffi));
+    toolbarItems.add(_RecordMenu());
     toolbarItems.add(_CloseMenu(id: widget.id, ffi: widget.ffi));
     return Column(
       mainAxisSize: MainAxisSize.min,
@@ -1646,17 +1646,17 @@ class _VoiceCallMenu extends StatelessWidget {
 }
 
 class _RecordMenu extends StatelessWidget {
-  final FFI ffi;
-  const _RecordMenu({Key? key, required this.ffi}) : super(key: key);
+  const _RecordMenu({Key? key}) : super(key: key);
 
   @override
   Widget build(BuildContext context) {
-    var ffiModel = Provider.of<FfiModel>(context);
+    var ffi = Provider.of<FfiModel>(context);
     var recordingModel = Provider.of<RecordingModel>(context);
     final visible =
-        recordingModel.start || ffiModel.permissions['recording'] != false;
+        (recordingModel.start || ffi.permissions['recording'] != false) &&
+            ffi.pi.currentDisplay != kAllDisplayValue;
     if (!visible) return Offstage();
-    final menuButton = _IconMenuButton(
+    return _IconMenuButton(
       assetName: 'assets/rec.svg',
       tooltip: recordingModel.start
           ? 'Stop session recording'
@@ -1669,14 +1669,6 @@ class _RecordMenu extends StatelessWidget {
           ? _ToolbarTheme.hoverRedColor
           : _ToolbarTheme.hoverBlueColor,
     );
-    return ChangeNotifierProvider.value(
-        value: ffi.qualityMonitorModel,
-        child: Consumer<QualityMonitorModel>(
-            builder: (context, model, child) => Offstage(
-                  // If already started, AV1->Hidden/Stop, Other->Start, same as actual
-                  offstage: model.data.codecFormat == 'AV1',
-                  child: menuButton,
-                )));
   }
 }
 

flutter/lib/mobile/pages/remote_page.dart

@@ -763,7 +763,9 @@ void showOptions(
       children.add(InkWell(
           onTap: () {
             if (i == cur) return;
-            bind.sessionSwitchDisplay(sessionId: gFFI.sessionId, value: Int32List.fromList([i]));
+            gFFI.recordingModel.onClose();
+            bind.sessionSwitchDisplay(
+                sessionId: gFFI.sessionId, value: Int32List.fromList([i]));
             gFFI.dialogManager.dismissAll();
           },
           child: Ink(

flutter/lib/models/model.dart

@@ -860,6 +860,8 @@ class FfiModel with ChangeNotifier {
 
   // Directly switch to the new display without waiting for the response.
   switchToNewDisplay(int display, SessionID sessionId, String peerId) {
+    // VideoHandler creation is upon when video frames are received, so either caching commands(don't know next width/height) or stopping recording when switching displays.
+    parent.target?.recordingModel.onClose();
     // no need to wait for the response
     pi.currentDisplay = display;
     updateCurDisplay(sessionId);
@@ -868,7 +870,6 @@ class FfiModel with ChangeNotifier {
     } catch (e) {
       //
     }
-    parent.target?.recordingModel.onSwitchDisplay();
   }
 
   updateBlockInputState(Map<String, dynamic> evt, String peerId) {
@@ -1850,56 +1851,66 @@ class RecordingModel with ChangeNotifier {
     int? width = parent.target?.canvasModel.getDisplayWidth();
     int? height = parent.target?.canvasModel.getDisplayHeight();
     if (sessionId == null || width == null || height == null) return;
-    final currentDisplay = parent.target?.ffiModel.pi.currentDisplay;
-    if (currentDisplay != kAllDisplayValue) {
-      bind.sessionRecordScreen(
-          sessionId: sessionId,
-          start: true,
-          display: currentDisplay!,
-          width: width,
-          height: height);
-    }
+    final pi = parent.target?.ffiModel.pi;
+    if (pi == null) return;
+    final currentDisplay = pi.currentDisplay;
+    if (currentDisplay == kAllDisplayValue) return;
+    bind.sessionRecordScreen(
+        sessionId: sessionId,
+        start: true,
+        display: currentDisplay,
+        width: width,
+        height: height);
   }
 
   toggle() async {
     if (isIOS) return;
     final sessionId = parent.target?.sessionId;
     if (sessionId == null) return;
+    final pi = parent.target?.ffiModel.pi;
+    if (pi == null) return;
+    final currentDisplay = pi.currentDisplay;
+    if (currentDisplay == kAllDisplayValue) return;
     _start = !_start;
     notifyListeners();
-    await bind.sessionRecordStatus(sessionId: sessionId, status: _start);
+    await _sendStatusMessage(sessionId, pi, _start);
     if (_start) {
-      final pi = parent.target?.ffiModel.pi;
-      if (pi != null) {
-        sessionRefreshVideo(sessionId, pi);
+      sessionRefreshVideo(sessionId, pi);
+      if (versionCmp(pi.version, '1.2.4') >= 0) {
+        // will not receive SwitchDisplay since 1.2.4
+        onSwitchDisplay();
       }
     } else {
-      final currentDisplay = parent.target?.ffiModel.pi.currentDisplay;
-      if (currentDisplay != kAllDisplayValue) {
-        bind.sessionRecordScreen(
-            sessionId: sessionId,
-            start: false,
-            display: currentDisplay!,
-            width: 0,
-            height: 0);
-      }
+      bind.sessionRecordScreen(
+          sessionId: sessionId,
+          start: false,
+          display: currentDisplay,
+          width: 0,
+          height: 0);
     }
   }
 
-  onClose() {
+  onClose() async {
     if (isIOS) return;
     final sessionId = parent.target?.sessionId;
     if (sessionId == null) return;
+    if (!_start) return;
     _start = false;
-    final currentDisplay = parent.target?.ffiModel.pi.currentDisplay;
-    if (currentDisplay != kAllDisplayValue) {
-      bind.sessionRecordScreen(
-          sessionId: sessionId,
-          start: false,
-          display: currentDisplay!,
-          width: 0,
-          height: 0);
-    }
+    final pi = parent.target?.ffiModel.pi;
+    if (pi == null) return;
+    final currentDisplay = pi.currentDisplay;
+    if (currentDisplay == kAllDisplayValue) return;
+    await _sendStatusMessage(sessionId, pi, false);
+    bind.sessionRecordScreen(
+        sessionId: sessionId,
+        start: false,
+        display: currentDisplay,
+        width: 0,
+        height: 0);
+  }
+
+  _sendStatusMessage(SessionID sessionId, PeerInfo pi, bool status) async {
+    await bind.sessionRecordStatus(sessionId: sessionId, status: status);
   }
 }
 

libs/scrap/Cargo.toml

@@ -19,7 +19,7 @@ cfg-if = "1.0"
 num_cpus = "1.15"
 lazy_static = "1.4"
 hbb_common = { path = "../hbb_common" }
-webm = "1.0"
+webm = { git = "https://github.com/21pages/rust-webm" }
 
 [dependencies.winapi]
 version = "0.3"

libs/scrap/src/common/record.rs

@@ -49,9 +49,12 @@
         }
         let file = if self.server { "s" } else { "c" }.to_string()
             + &self.id.clone()
-            + &chrono::Local::now().format("_%Y%m%d%H%M%S_").to_string()
-            + &self.format.to_string()
-            + if self.format == CodecFormat::VP9 || self.format == CodecFormat::VP8 {
+            + &chrono::Local::now().format("_%Y%m%d%H%M%S%3f_").to_string()
+            + &self.format.to_string().to_lowercase()
+            + if self.format == CodecFormat::VP9
+                || self.format == CodecFormat::VP8
+                || self.format == CodecFormat::AV1
+            {
                 ".webm"
             } else {
                 ".mp4"
@@ -83,6 +86,7 @@
 pub struct Recorder {
     pub inner: Box<dyn RecorderApi>,
     ctx: RecorderContext,
+    pts: Option<i64>,
 }
 
 impl Deref for Recorder {
@@ -101,19 +105,18 @@
 
 impl Recorder {
     pub fn new(mut ctx: RecorderContext) -> ResultType<Self> {
-        if ctx.format == CodecFormat::AV1 {
-            bail!("not support av1 recording");
-        }
         ctx.set_filename()?;
         let recorder = match ctx.format {
-            CodecFormat::VP8 | CodecFormat::VP9 => Recorder {
+            CodecFormat::VP8 | CodecFormat::VP9 | CodecFormat::AV1 => Recorder {
                 inner: Box::new(WebmRecorder::new(ctx.clone())?),
                 ctx,
+                pts: None,
             },
             #[cfg(feature = "hwcodec")]
             _ => Recorder {
                 inner: Box::new(HwRecorder::new(ctx.clone())?),
                 ctx,
+                pts: None,
             },
             #[cfg(not(feature = "hwcodec"))]
             _ => bail!("unsupported codec type"),
@@ -125,13 +128,16 @@
     fn change(&mut self, mut ctx: RecorderContext) -> ResultType<()> {
         ctx.set_filename()?;
         self.inner = match ctx.format {
-            CodecFormat::VP8 | CodecFormat::VP9 => Box::new(WebmRecorder::new(ctx.clone())?),
+            CodecFormat::VP8 | CodecFormat::VP9 | CodecFormat::AV1 => {
+                Box::new(WebmRecorder::new(ctx.clone())?)
+            }
             #[cfg(feature = "hwcodec")]
             _ => Box::new(HwRecorder::new(ctx.clone())?),
             #[cfg(not(feature = "hwcodec"))]
             _ => bail!("unsupported codec type"),
         };
         self.ctx = ctx;
+        self.pts = None;
         self.send_state(RecordState::NewFile(self.ctx.filename.clone()));
         Ok(())
     }
@@ -153,7 +159,10 @@
                         ..self.ctx.clone()
                     })?;
                 }
-                vp8s.frames.iter().map(|f| self.write_video(f)).count();
+                for f in vp8s.frames.iter() {
+                    self.check_pts(f.pts)?;
+                    self.write_video(f);
+                }
             }
             video_frame::Union::Vp9s(vp9s) => {
                 if self.ctx.format != CodecFormat::VP9 {
@@ -162,7 +171,22 @@
                         ..self.ctx.clone()
                     })?;
                 }
-                vp9s.frames.iter().map(|f| self.write_video(f)).count();
+                for f in vp9s.frames.iter() {
+                    self.check_pts(f.pts)?;
+                    self.write_video(f);
+                }
+            }
+            video_frame::Union::Av1s(av1s) => {
+                if self.ctx.format != CodecFormat::AV1 {
+                    self.change(RecorderContext {
+                        format: CodecFormat::AV1,
+                        ..self.ctx.clone()
+                    })?;
+                }
+                for f in av1s.frames.iter() {
+                    self.check_pts(f.pts)?;
+                    self.write_video(f);
+                }
             }
             #[cfg(feature = "hwcodec")]
             video_frame::Union::H264s(h264s) => {
@@ -172,8 +196,9 @@
                         ..self.ctx.clone()
                     })?;
                 }
-                if self.ctx.format == CodecFormat::H264 {
-                    h264s.frames.iter().map(|f| self.write_video(f)).count();
+                for f in h264s.frames.iter() {
+                    self.check_pts(f.pts)?;
+                    self.write_video(f);
                 }
             }
             #[cfg(feature = "hwcodec")]
@@ -184,8 +209,9 @@
                         ..self.ctx.clone()
                     })?;
                 }
-                if self.ctx.format == CodecFormat::H265 {
-                    h265s.frames.iter().map(|f| self.write_video(f)).count();
+                for f in h265s.frames.iter() {
+                    self.check_pts(f.pts)?;
+                    self.write_video(f);
                 }
             }
             _ => bail!("unsupported frame type"),
@@ -194,6 +220,17 @@
         Ok(())
     }
 
+    fn check_pts(&mut self, pts: i64) -> ResultType<()> {
+        // https://stackoverflow.com/questions/76379101/how-to-create-one-playable-webm-file-from-two-different-video-tracks-with-same-c
+        let old_pts = self.pts;
+        self.pts = Some(pts);
+        if old_pts.clone().unwrap_or_default() > pts {
+            log::info!("pts {:?}->{}, change record filename", old_pts, pts);
+            self.change(self.ctx.clone())?;
+        }
+        Ok(())
+    }
+
     fn send_state(&self, state: RecordState) {
         self.ctx.tx.as_ref().map(|tx| tx.send(state));
     }
@@ -230,10 +267,19 @@
             None,
             if ctx.format == CodecFormat::VP9 {
                 mux::VideoCodecId::VP9
-            } else {
+            } else if ctx.format == CodecFormat::VP8 {
                 mux::VideoCodecId::VP8
+            } else {
+                mux::VideoCodecId::AV1
             },
         );
+        if ctx.format == CodecFormat::AV1 {
+            // [129, 8, 12, 0] in 3.6.0, but zero works
+            let codec_private = vec![0, 0, 0, 0];
+            if !webm.set_codec_private(vt.track_number(), &codec_private) {
+                bail!("Failed to set codec private");
+            }
+        }
         Ok(WebmRecorder {
             vt,
             webm: Some(webm),
@@ -264,7 +310,7 @@
 
 impl Drop for WebmRecorder {
     fn drop(&mut self) {
         std::mem::replace(&mut self.webm, None).map_or(false, |webm| webm.finalize(None));
         let mut state = RecordState::WriteTail;
         if !self.written || self.start.elapsed().as_secs() < MIN_SECS {
             std::fs::remove_file(&self.ctx.filename).ok();