ちゃんとビットレートとフレームレートを考慮する

CMakeLists.txt

@@ -95,6 +95,7 @@ target_sources(sorac
     src/aom_av1_video_encoder.cpp
     src/current_time.cpp
     src/data_channel.cpp
+    src/default_encoder_adapter.cpp
     src/open_h264_video_encoder.cpp
     src/opus_audio_encoder.cpp
     src/signaling.cpp

examples/sumomo/fake_capturer.cpp

@@ -13,7 +13,8 @@ namespace sumomo {
 
 class FakeCapturer : public SumomoCapturer {
  public:
-  FakeCapturer() {
+  FakeCapturer(int width, int height, int fps)
+      : width_(width), height_(height), fps_(fps) {
     this->destroy = [](SumomoCapturer* p) { delete (sumomo::FakeCapturer*)p; };
     this->set_frame_callback = [](SumomoCapturer* p,
                                   sumomo_capturer_on_frame_func on_frame,
@@ -42,26 +43,29 @@ class FakeCapturer : public SumomoCapturer {
       engine_ = std::make_unique<std::mt19937>(seed_gen());
       return nullptr;
     });
-    th_.Start(30, [this](std::chrono::microseconds timestamp,
-                         std::chrono::microseconds prev) {
-      std::uniform_int_distribution<int> dist(0, 640 * 480 - 1);
+    th_.Start(fps_, [this](std::chrono::microseconds timestamp,
+                           std::chrono::microseconds prev) {
+      std::uniform_int_distribution<int> dist(0, width_ * height_ - 1);
       sorac::VideoFrame frame;
       frame.timestamp = timestamp;
-      frame.i420_buffer = sorac::VideoFrameBufferI420::Create(640, 480);
+      frame.i420_buffer = sorac::VideoFrameBufferI420::Create(width_, height_);
       frame.i420_buffer->y[dist(*engine_)] = 0xff;
       frame.i420_buffer->y[dist(*engine_)] = 0xff;
       frame.i420_buffer->y[dist(*engine_)] = 0xff;
       frame.i420_buffer->y[dist(*engine_)] = 0xff;
       frame.i420_buffer->y[dist(*engine_)] = 0xff;
-      frame.base_width = 640;
-      frame.base_height = 480;
+      frame.base_width = width_;
+      frame.base_height = height_;
       callback_(frame);
     });
     return 0;
   }
   void Stop() { th_.Stop(); }
 
  private:
+  int width_;
+  int height_;
+  int fps_;
   std::function<void(const sorac::VideoFrame& frame)> callback_;
   SteadyFrameThread th_;
   std::unique_ptr<std::mt19937> engine_;
@@ -71,7 +75,7 @@ class FakeCapturer : public SumomoCapturer {
 
 extern "C" {
 
-SumomoCapturer* sumomo_fake_capturer_create() {
-  return new sumomo::FakeCapturer();
+SumomoCapturer* sumomo_fake_capturer_create(int width, int height, int fps) {
+  return new sumomo::FakeCapturer(width, height, fps);
 }
 }

examples/sumomo/fake_capturer.h

@@ -9,7 +9,9 @@
 extern "C" {
 #endif
 
-extern SumomoCapturer* sumomo_fake_capturer_create();
+extern SumomoCapturer* sumomo_fake_capturer_create(int width,
+                                                   int height,
+                                                   int fps);
 
 #ifdef __cplusplus
 }

examples/sumomo/mac_capturer.h

@@ -11,7 +11,8 @@ extern "C" {
 
 extern SumomoCapturer* sumomo_mac_capturer_create(const char* device,
                                                   int width,
-                                                  int height);
+                                                  int height,
+                                                  int fps);
 
 #ifdef __cplusplus
 }

examples/sumomo/mac_capturer.mm

@@ -368,10 +368,11 @@ - (dispatch_queue_t)frameQueue {
 
 class MacCapturer : public SumomoCapturer {
  public:
-  MacCapturer(const char* device, int width, int height) {
+  MacCapturer(const char* device, int width, int height, int fps) {
     this->device_ = device;
     this->width_ = width;
     this->height_ = height;
+    this->fps_ = fps;
     this->destroy = [](SumomoCapturer* p) { delete (sumomo::MacCapturer*)p; };
     this->set_frame_callback = [](SumomoCapturer* p,
                                   sumomo_capturer_on_frame_func on_frame,
@@ -385,7 +386,7 @@ - (dispatch_queue_t)frameQueue {
     };
     this->start = [](SumomoCapturer* p) {
       auto q = (sumomo::MacCapturer*)p;
-      return q->Start(q->device_.c_str(), q->width_, q->height_);
+      return q->Start(q->device_.c_str(), q->width_, q->height_, q->fps_);
     };
     this->stop = [](SumomoCapturer* p) { ((sumomo::MacCapturer*)p)->Stop(); };
   }
@@ -396,14 +397,14 @@ void SetFrameCallback(
     callback_ = callback;
   }
 
-  int Start(const char* device, int width, int height) {
+  int Start(const char* device, int width, int height, int fps) {
     Stop();
 
     capturer_ = [[SumomoMacCapturer alloc] initWithCallback:callback_];
     [capturer_ startCaptureWithDeviceName:device
                                     width:width
                                    height:height
-                                      fps:30
+                                      fps:fps
                         completionHandler:[](NSError* _Nullable error) {
                           if (error) {
                             fprintf(stderr, "Failed to start capture: %s\n",
@@ -422,6 +423,7 @@ void Stop() {
   std::function<void(const sorac::VideoFrame& frame)> callback_;
   int width_;
   int height_;
+  int fps_;
 
   SumomoMacCapturer* capturer_;
 };
@@ -432,7 +434,8 @@ void Stop() {
 
 SumomoCapturer* sumomo_mac_capturer_create(const char* device,
                                            int width,
-                                           int height) {
-  return new sumomo::MacCapturer(device, width, height);
+                                           int height,
+                                           int fps) {
+  return new sumomo::MacCapturer(device, width, height, fps);
 }
 }

examples/sumomo/option.c

@@ -22,6 +22,7 @@ static struct option long_opts[] = {
     {"capture-device-name", required_argument, 0, 0},
     {"capture-device-width", required_argument, 0, 0},
     {"capture-device-height", required_argument, 0, 0},
+    {"capture-device-fps", required_argument, 0, 0},
     {"audio-type", required_argument, 0, 0},
     {"h264-encoder-type", required_argument, 0, 0},
     {"h265-encoder-type", required_argument, 0, 0},
@@ -51,6 +52,7 @@ int sumomo_option_parse(SumomoOption* option,
 #endif
   option->capture_device_width = 640;
   option->capture_device_height = 480;
+  option->capture_device_fps = 30;
   option->audio_type = SUMOMO_OPTION_AUDIO_TYPE_FAKE;
   option->video_codec_type = "H264";
   option->cacert = "/etc/ssl/certs/ca-certificates.crt";
@@ -134,6 +136,8 @@ int sumomo_option_parse(SumomoOption* option,
           option->capture_device_width = atoi(optarg);
         } else if (OPT_IS("capture-device-height")) {
           option->capture_device_height = atoi(optarg);
+        } else if (OPT_IS("capture-device-fps")) {
+          option->capture_device_fps = atoi(optarg);
         } else if (OPT_IS("audio-type")) {
           if (strcmp(optarg, "fake") == 0) {
             option->audio_type = SUMOMO_OPTION_AUDIO_TYPE_FAKE;
@@ -200,6 +204,7 @@ int sumomo_option_parse(SumomoOption* option,
       fprintf(stdout, "  --capture-device-name=NAME\n");
       fprintf(stdout, "  --capture-device-width=WIDTH\n");
       fprintf(stdout, "  --capture-device-height=HEIGHT\n");
+      fprintf(stdout, "  --capture-device-fps=FPS\n");
       fprintf(stdout, "  --audio-type=fake,pulse,macos\n");
       fprintf(stdout, "  --h264-encoder-type=openh264,videotoolbox\n");
       fprintf(stdout, "  --h265-encoder-type=videotoolbox\n");

examples/sumomo/option.h

@@ -36,6 +36,7 @@ typedef struct SumomoOption {
   const char* capture_device_name;
   int capture_device_width;
   int capture_device_height;
+  int capture_device_fps;
   SumomoOptionAudioType audio_type;
   soracp_H264EncoderType h264_encoder_type;
   soracp_H265EncoderType h265_encoder_type;

examples/sumomo/sumomo.c

@@ -81,7 +81,7 @@ void on_track(SoracTrack* track, void* userdata) {
 #if defined(__linux__)
       state->capturer = sumomo_v4l2_capturer_create(
           state->opt->capture_device_name, state->opt->capture_device_width,
-          state->opt->capture_device_height);
+          state->opt->capture_device_height, state->opt->capture_device_fps);
 #else
       fprintf(stderr,
               "V4L2 capturer cannot be used on environments other than Linux");
@@ -91,14 +91,16 @@ void on_track(SoracTrack* track, void* userdata) {
 #if defined(__APPLE__)
       state->capturer = sumomo_mac_capturer_create(
           state->opt->capture_device_name, state->opt->capture_device_width,
-          state->opt->capture_device_height);
+          state->opt->capture_device_height, state->opt->capture_device_fps);
 #else
       fprintf(stderr,
               "V4L2 capturer cannot be used on environments other than Linux");
       exit(1);
 #endif
     } else {
-      state->capturer = sumomo_fake_capturer_create();
+      state->capturer = sumomo_fake_capturer_create(
+          state->opt->capture_device_width, state->opt->capture_device_height,
+          state->opt->capture_device_fps);
     }
     sumomo_capturer_set_frame_callback(state->capturer, on_capture_frame,
                                        state);
@@ -199,8 +201,6 @@ int main(int argc, char* argv[]) {
   soracp_SignalingConfig_set_h264_encoder_type(&config, opt.h264_encoder_type);
   soracp_SignalingConfig_set_h265_encoder_type(&config, opt.h265_encoder_type);
   soracp_SignalingConfig_set_av1_encoder_type(&config, opt.av1_encoder_type);
-  soracp_SignalingConfig_set_video_encoder_initial_bitrate_kbps(
-      &config, opt.video_bit_rate == 0 ? 500 : opt.video_bit_rate);
   SoracSignaling* signaling = sorac_signaling_create(&config);
   state.signaling = signaling;
 

examples/sumomo/v4l2_capturer.cpp

@@ -32,10 +32,11 @@ namespace sumomo {
 
 class V4L2Capturer : public SumomoCapturer {
  public:
-  V4L2Capturer(const char* device, int width, int height) {
+  V4L2Capturer(const char* device, int width, int height, int fps) {
     this->device_ = device;
     this->width_ = width;
     this->height_ = height;
+    this->fps_ = fps;
     this->destroy = [](SumomoCapturer* p) { delete (sumomo::V4L2Capturer*)p; };
     this->set_frame_callback = [](SumomoCapturer* p,
                                   sumomo_capturer_on_frame_func on_frame,
@@ -49,7 +50,7 @@ class V4L2Capturer : public SumomoCapturer {
     };
     this->start = [](SumomoCapturer* p) {
       auto q = (sumomo::V4L2Capturer*)p;
-      return q->Start(q->device_.c_str(), q->width_, q->height_);
+      return q->Start(q->device_.c_str(), q->width_, q->height_, q->fps_);
     };
     this->stop = [](SumomoCapturer* p) { ((sumomo::V4L2Capturer*)p)->Stop(); };
   }
@@ -59,7 +60,7 @@ class V4L2Capturer : public SumomoCapturer {
     callback_ = callback;
   }
 
-  int Start(const char* device, int width, int height) {
+  int Start(const char* device, int width, int height, int fps) {
     Stop();
 
     device_fd_ = open(device, O_RDWR | O_NONBLOCK, 0);
@@ -102,6 +103,24 @@ class V4L2Capturer : public SumomoCapturer {
     width_ = fmt.fmt.pix.width;
     height_ = fmt.fmt.pix.height;
 
+    // フレームレートの設定
+    struct v4l2_streamparm sp;
+    memset(&sp, 0, sizeof(sp));
+    sp.type = V4L2_BUF_TYPE_VIDEO_CAPTURE;
+    if (ioctl(device_fd_, VIDIOC_G_PARM, &sp) < 0) {
+      fprintf(stderr, "Failed to VIDIOC_G_PARM: %s\n", strerror(errno));
+    } else {
+      if ((sp.parm.capture.capability & V4L2_CAP_TIMEPERFRAME) != 0) {
+        memset(&sp, 0, sizeof(sp));
+        sp.type = V4L2_BUF_TYPE_VIDEO_CAPTURE;
+        sp.parm.capture.timeperframe.numerator = 1;
+        sp.parm.capture.timeperframe.denominator = fps;
+        if (ioctl(device_fd_, VIDIOC_S_PARM, &sp) < 0) {
+          fprintf(stderr, "Failed to set the framerate: %s\n", strerror(errno));
+        }
+      }
+    }
+
     // ビデオバッファの設定
     const int V4L2_BUFFER_COUNT = 4;
     {
@@ -247,6 +266,7 @@ class V4L2Capturer : public SumomoCapturer {
   std::function<void(const sorac::VideoFrame& frame)> callback_;
   int width_;
   int height_;
+  int fps_;
 
   int device_fd_ = -1;
   std::atomic<bool> quit_;
@@ -265,7 +285,8 @@ extern "C" {
 
 SumomoCapturer* sumomo_v4l2_capturer_create(const char* device,
                                             int width,
-                                            int height) {
-  return new sumomo::V4L2Capturer(device, width, height);
+                                            int height,
+                                            int fps) {
+  return new sumomo::V4L2Capturer(device, width, height, fps);
 }
 }

examples/sumomo/v4l2_capturer.h

@@ -11,7 +11,8 @@ extern "C" {
 
 extern SumomoCapturer* sumomo_v4l2_capturer_create(const char* device,
                                                    int width,
-                                                   int height);
+                                                   int height,
+                                                   int fps);
 
 #ifdef __cplusplus
 }

include/sorac/default_encoder_adapter.hpp

@@ -0,0 +1,20 @@
+#ifndef SORAC_DEFAULT_ENCODER_ADAPTER_HPP_
+#define SORAC_DEFAULT_ENCODER_ADAPTER_HPP_
+
+#include <functional>
+#include <memory>
+#include <optional>
+#include <vector>
+
+#include "soracp.json.h"
+#include "types.hpp"
+#include "video_encoder.hpp"
+
+namespace sorac {
+
+std::shared_ptr<VideoEncoder> CreateDefaultEncoderAdapter(
+    std::shared_ptr<VideoEncoder> encoder);
+
+}
+
+#endif

include/sorac/video_encoder.hpp

@@ -15,6 +15,7 @@ class VideoEncoder {
     int width;
     int height;
     Bps bitrate;
+    int fps;
   };
 
   virtual ~VideoEncoder() {}

proto/soracp.proto

@@ -54,7 +54,6 @@ message SignalingConfig {
     string proxy_username = 45;
     string proxy_password = 46;
     string proxy_agent = 47;
-    int32 video_encoder_initial_bitrate_kbps = 4;
 }
 
 message SoraConnectConfig {

src/aom_av1_video_encoder.cpp

@@ -62,6 +62,8 @@ class AomAv1VideoEncoder : public VideoEncoder {
 
     PLOG_INFO << "AOM InitEncode";
 
+    settings_ = settings;
+
     // https://source.chromium.org/chromium/chromium/src/+/main:third_party/webrtc/modules/video_coding/codecs/av1/libaom_av1_encoder.cc
     // を参考に初期化やエンコードを行う
 
@@ -214,7 +216,7 @@ class AomAv1VideoEncoder : public VideoEncoder {
       frame_for_encode_->stride[AOM_PLANE_V] = 0;
     }
 
-    const uint32_t duration = 90000 / 30;
+    const uint32_t duration = 90000 / settings_.fps;
     timestamp_ += duration;
 
     aom_enc_frame_flags_t flags = 0;
@@ -333,6 +335,7 @@ class AomAv1VideoEncoder : public VideoEncoder {
   }
 
  private:
+  Settings settings_;
   bool init_ctx_ = false;
   aom_codec_ctx_t ctx_;
   aom_codec_enc_cfg_t cfg_;