AV1 のサイマルキャストがちゃんと動くようになった

shiguredo · Apr 20, 2024 · b681e57 · b681e57
1 parent fc7741e
commit b681e57
Show file tree

Hide file tree

Showing 6 changed files with 21 additions and 7 deletions.
diff --git a/.vscode/settings.json b/.vscode/settings.json
@@ -103,6 +103,8 @@
         "recorder.h": "c",
         "__mutex_base": "cpp",
         "sorac.h": "c",
-        "soracp.json.c.h": "c"
+        "soracp.json.c.h": "c",
+        "complex": "cpp",
+        "cfenv": "cpp"
     }
 }
diff --git a/examples/sumomo/fake_capturer.cpp b/examples/sumomo/fake_capturer.cpp
@@ -48,16 +48,17 @@ class FakeCapturer : public SumomoCapturer {
       std::uniform_int_distribution<int> dist(0, width_ * height_ - 1);
       sorac::VideoFrame frame;
       frame.timestamp = timestamp;
+      frame.frame_number = ++frame_number_;
       if (format_ == SUMOMO_FAKE_CAPTURER_FORMAT_I420) {
         frame.i420_buffer =
             sorac::VideoFrameBufferI420::Create(width_, height_);
-        for (int i = 0; i < width_ / 100; i++) {
+        for (int i = 0; i < width_ / 10; i++) {
           frame.i420_buffer->y[dist(*engine_)] = 0xff;
         }
       } else if (format_ == SUMOMO_FAKE_CAPTURER_FORMAT_NV12) {
         frame.nv12_buffer =
             sorac::VideoFrameBufferNV12::Create(width_, height_);
-        for (int i = 0; i < width_ / 100; i++) {
+        for (int i = 0; i < width_ / 10; i++) {
           frame.nv12_buffer->y[dist(*engine_)] = 0xff;
         }
       }
@@ -73,6 +74,7 @@ class FakeCapturer : public SumomoCapturer {
   int width_;
   int height_;
   int fps_;
+  int frame_number_ = 0;
   SumomoFakeCapturerFormat format_;
   std::function<void(const sorac::VideoFrame& frame)> callback_;
   SteadyFrameThread th_;

diff --git a/examples/sumomo/mac_capturer.mm b/examples/sumomo/mac_capturer.mm
@@ -48,6 +48,7 @@ @implementation SumomoMacCapturer {
   std::function<void(const sorac::VideoFrame&)> _callback;
   BOOL _willBeRunning;
   dispatch_queue_t _frameQueue;
+  int _frameNumber;
 }
 
 - (instancetype)initWithCallback:
@@ -64,6 +65,7 @@ - (instancetype)initWithCallback:
     _videoDataOutput = [[AVCaptureVideoDataOutput alloc] init];
     _willBeRunning = NO;
     _frameQueue = nil;
+    _frameNumber = 0;
 
     NSSet<NSNumber*>* supportedPixelFormats = [NSSet
         setWithObjects:@(kCVPixelFormatType_420YpCbCr8BiPlanarFullRange),
@@ -283,6 +285,7 @@ - (void)captureOutput:(AVCaptureOutput*)captureOutput
       (int64_t)(CMTimeGetSeconds(
                     CMSampleBufferGetPresentationTimeStamp(sampleBuffer)) *
                 kMicrosecondsPerSecond));
+  frame.frame_number = ++_frameNumber;
   frame.nv12_buffer = sorac::VideoFrameBufferNV12::Create(width, height);
   frame.base_width = width;
   frame.base_height = height;

diff --git a/examples/sumomo/v4l2_capturer.cpp b/examples/sumomo/v4l2_capturer.cpp
@@ -229,6 +229,7 @@ class V4L2Capturer : public SumomoCapturer {
         frame.timestamp = sorac::get_current_time();
         frame.base_width = width_;
         frame.base_height = height_;
+        frame.frame_number = ++frame_number_;
         callback_(frame);
 
         if (ioctl(device_fd_, VIDIOC_QBUF, &buf) < 0) {
@@ -267,6 +268,7 @@ class V4L2Capturer : public SumomoCapturer {
   int width_;
   int height_;
   int fps_;
+  int frame_number_ = 0;
 
   int device_fd_ = -1;
   std::atomic<bool> quit_;

diff --git a/include/sorac/types.hpp b/include/sorac/types.hpp
@@ -45,6 +45,9 @@ struct VideoFrame {
   int height() const {
     return i420_buffer != nullptr ? i420_buffer->height : nv12_buffer->height;
   }
+  // サイマルキャストで DD を利用する時にフレーム番号を全体で同じにする必要があるため
+  // ここにフレーム番号を持たせる
+  int frame_number;
 };
 
 struct EncodedImage {

diff --git a/src/aom_av1_video_encoder.cpp b/src/aom_av1_video_encoder.cpp
@@ -172,8 +172,6 @@ class AomAv1VideoEncoder : public VideoEncoder {
     SET_PARAM(AV1E_SET_ENABLE_TX64, 0);
     SET_PARAM(AV1E_SET_MAX_REFERENCE_FRAMES, 3);
 
-    frame_number_ = 0;
-
     return true;
   }
 
@@ -272,17 +270,21 @@ class AomAv1VideoEncoder : public VideoEncoder {
     delta_frame_template.frameDiffs = {1};
     ctx.structure.templates = {key_frame_template, delta_frame_template};
     ctx.activeChains[0] = true;
-    ctx.descriptor.frameNumber = ++frame_number_;
+    ctx.descriptor.frameNumber = frame.frame_number;
     if (is_key_frame) {
       ctx.descriptor.dependencyTemplate = key_frame_template;
     } else {
       ctx.descriptor.dependencyTemplate = delta_frame_template;
+      ctx.descriptor.dependencyTemplate.frameDiffs = {frame.frame_number -
+                                                      prev_frame_number_};
     }
     ctx.descriptor.structureAttached = is_key_frame;
 
     encoded.dependency_descriptor_context = std::make_shared<
         rtc::RtpPacketizationConfig::DependencyDescriptorContext>(ctx);
 
+    prev_frame_number_ = frame.frame_number;
+
     callback_(encoded);
   }
 
@@ -341,7 +343,7 @@ class AomAv1VideoEncoder : public VideoEncoder {
   aom_codec_enc_cfg_t cfg_;
   aom_image_t* frame_for_encode_ = nullptr;
   int64_t timestamp_ = 0;
-  int frame_number_ = 0;
+  int prev_frame_number_ = 0;
 
   std::function<void(const EncodedImage&)> callback_;