enalbing low latency mode for RTC

BUILD.gn

@@ -53,7 +53,7 @@ if (!build_with_chromium) {
         "modules/audio_processing:audio_processing_tests",
         "modules/remote_bitrate_estimator:rtp_to_text",
         "modules/rtp_rtcp:test_packet_masks_metrics",
-        "modules/video_capture:video_capture_internal_impl",
+        "modules/video_capture:video_capture",
         "pc:peerconnection_unittests",
         "pc:rtc_pc_unittests",
         "rtc_tools:rtp_generator",
@@ -470,7 +470,7 @@ if (!build_with_chromium) {
       "logging:rtc_event_log_api",
       "media",
       "modules",
-      "modules/video_capture:video_capture_internal_impl",
+      "modules/video_capture:video_capture",
       "p2p:rtc_p2p",
       "pc:libjingle_peerconnection",
       "pc:peerconnection",

api/audio_codecs/opus/audio_decoder_opus.cc

@@ -44,7 +44,7 @@ absl::optional<AudioDecoderOpus::Config> AudioDecoderOpus::SdpToConfig(
         return absl::nullopt;  // Bad stereo parameter.
       }
     }
-    return 1;  // Default to mono.
+    return 2;  // Default to mono.
   }();
   if (absl::EqualsIgnoreCase(format.name, "opus") &&
       format.clockrate_hz == 48000 && format.num_channels == 2 &&

api/rtp_headers.h

@@ -159,6 +159,9 @@ struct RTPHeaderExtension {
   std::string mid;
 
   absl::optional<ColorSpace> color_space;
+
+  // Required for Temporal scalability support
+  uint16_t picture_id;
 };
 
 enum { kRtpCsrcSize = 15 };  // RFC 3550 page 13

api/rtp_parameters.cc

@@ -116,6 +116,7 @@ constexpr char RtpExtension::kColorSpaceUri[];
 constexpr char RtpExtension::kMidUri[];
 constexpr char RtpExtension::kRidUri[];
 constexpr char RtpExtension::kRepairedRidUri[];
+constexpr char RtpExtension::kPictureIdUri[];
 
 constexpr int RtpExtension::kMinId;
 constexpr int RtpExtension::kMaxId;
@@ -151,7 +152,8 @@ bool RtpExtension::IsSupportedForVideo(absl::string_view uri) {
          uri == webrtc::RtpExtension::kDependencyDescriptorUri ||
          uri == webrtc::RtpExtension::kColorSpaceUri ||
          uri == webrtc::RtpExtension::kRidUri ||
-         uri == webrtc::RtpExtension::kRepairedRidUri;
+         uri == webrtc::RtpExtension::kRepairedRidUri ||
+         uri == webrtc::RtpExtension::kPictureIdUri;
 }
 
 bool RtpExtension::IsEncryptionSupported(absl::string_view uri) {

api/rtp_parameters.h

@@ -360,6 +360,10 @@ struct RTC_EXPORT RtpExtension {
   static constexpr char kRepairedRidUri[] =
       "urn:ietf:params:rtp-hdrext:sdes:repaired-rtp-stream-id";
 
+  // Picture ID for temporal scalability support
+  static constexpr char kPictureIdUri[] = "urn:intel:picture-id";
+  static constexpr int kPictureIdDefaultId = 12;
+
   // Inclusive min and max IDs for two-byte header extensions and one-byte
   // header extensions, per RFC8285 Section 4.2-4.3.
   static constexpr int kMinId = 1;

api/transport/bitrate_settings.h

@@ -42,7 +42,7 @@ struct BitrateConstraints {
   int max_bitrate_bps = -1;
 
  private:
-  static constexpr int kDefaultStartBitrateBps = 300000;
+  static constexpr int kDefaultStartBitrateBps = 5000000;  // Update from 300k to 5MB for low latency mode
 };
 
 }  // namespace webrtc

api/transport/network_control.h

@@ -93,7 +93,7 @@ class NetworkControllerInterface {
       ABSL_MUST_USE_RESULT = 0;
   // Called with per packet feedback regarding receive time.
   virtual NetworkControlUpdate OnTransportPacketsFeedback(
-      TransportPacketsFeedback) ABSL_MUST_USE_RESULT = 0;
+      TransportPacketsFeedback, int64_t) ABSL_MUST_USE_RESULT = 0;
   // Called with network state estimate updates.
   virtual NetworkControlUpdate OnNetworkStateEstimate(NetworkStateEstimate)
       ABSL_MUST_USE_RESULT = 0;

call/call_config.h

@@ -30,7 +30,7 @@ struct CallConfig {
   CallConfig(const CallConfig&);
   ~CallConfig();
 
-  RTC_DEPRECATED static constexpr int kDefaultStartBitrateBps = 300000;
+  RTC_DEPRECATED static constexpr int kDefaultStartBitrateBps = 5000000;  //Updated from 300k to 5MB for low latency
 
   // Bitrate config used until valid bitrate estimates are calculated. Also
   // used to cap total bitrate used. This comes from the remote connection.

call/rtp_payload_params.cc

@@ -19,6 +19,9 @@
 #include "absl/types/variant.h"
 #include "api/video/video_timing.h"
 #include "modules/video_coding/codecs/h264/include/h264_globals.h"
+#ifndef DISABLE_H265
+#include "modules/video_coding/codecs/h265/include/h265_globals.h"
+#endif
 #include "modules/video_coding/codecs/interface/common_constants.h"
 #include "modules/video_coding/codecs/vp8/include/vp8_globals.h"
 #include "modules/video_coding/codecs/vp9/include/vp9_globals.h"
@@ -103,13 +106,15 @@ void PopulateRtpWithCodecSpecifics(const CodecSpecificInfo& info,
         rtp->frame_marking.base_layer_sync =
             info.codecSpecific.H264.base_layer_sync;
       }
+      h264_header.picture_id = info.codecSpecific.H264.picture_id;
       return;
     }
 #ifndef DISABLE_H265
     case kVideoCodecH265: {
       auto& h265_header = rtp->video_type_header.emplace<RTPVideoHeaderH265>();
       h265_header.packetization_mode =
           info.codecSpecific.H265.packetization_mode;
+      h265_header.picture_id = info.codecSpecific.H265.picture_id;
     }
     return;
 #endif

call/rtp_transport_controller_send.cc

@@ -27,13 +27,15 @@
 #include "rtc_base/checks.h"
 #include "rtc_base/logging.h"
 #include "rtc_base/rate_limiter.h"
+#include "system_wrappers/include/field_trial.h"
 
 namespace webrtc {
 namespace {
 static const int64_t kRetransmitWindowSizeMs = 500;
 static const size_t kMaxOverheadBytes = 500;
 
 constexpr TimeDelta kPacerQueueUpdateInterval = TimeDelta::Millis(25);
+const char kLowLatencyStreaming[] = "OWT-LowLatencyMode";
 
 TargetRateConstraints ConvertConstraints(int min_bitrate_bps,
                                          int max_bitrate_bps,
@@ -52,6 +54,11 @@ TargetRateConstraints ConvertConstraints(int min_bitrate_bps,
   return msg;
 }
 
+bool LowLatencyStreamingEnabled() {
+  std::string trial = webrtc::field_trial::FindFullName(kLowLatencyStreaming);
+  return trial.find("Enabled") == 0;
+}
+
 TargetRateConstraints ConvertConstraints(const BitrateConstraints& contraints,
                                          Clock* clock) {
   return ConvertConstraints(contraints.min_bitrate_bps,
@@ -128,7 +135,11 @@ RtpTransportControllerSend::RtpTransportControllerSend(
       DataRate::BitsPerSec(bitrate_config.start_bitrate_bps), DataRate::Zero());
 
   if (!use_task_queue_pacer_) {
-    process_thread_->Start();
+    if (!LowLatencyStreamingEnabled()) {
+      process_thread_->Start();
+    } else {
+      process_thread_->StartWithHighPriority();
+    }
   }
 }
 
@@ -543,7 +554,12 @@ void RtpTransportControllerSend::OnTransportFeedback(
         transport_feedback_adapter_.ProcessTransportFeedback(feedback,
                                                              feedback_time);
     if (feedback_msg && controller_) {
-      PostUpdates(controller_->OnTransportPacketsFeedback(*feedback_msg));
+#ifdef INTEL_GPRA
+      PostUpdates(controller_->OnTransportPacketsFeedback(
+          *feedback_msg, transport_feedback_adapter_.GetCurrentOffsetMs()));
+#else
+      PostUpdates(controller_->OnTransportPacketsFeedback(*feedback_msg, 0));
+#endif
     }
     pacer()->UpdateOutstandingData(
         transport_feedback_adapter_.GetOutstandingData());

call/rtp_video_sender.cc

@@ -528,12 +528,20 @@ EncodedImageCallback::Result RtpVideoSender::OnEncodedImage(
             ? &*codec_specific_info->template_structure
             : nullptr);
   }
-
+  RTPVideoHeader rtp_video_header = params_[stream_index].GetRtpVideoHeader(
+      encoded_image, codec_specific_info, shared_frame_id_);
+  if (codec_specific_info->codecSpecific.H264.last_fragment_in_frame)
+    absl::get<RTPVideoHeaderH264>(rtp_video_header.video_type_header)
+        .has_last_fragement = true;
+#ifndef DISABLE_H265
+  else if (codec_specific_info->codecSpecific.H265.last_fragment_in_frame)
+    absl::get<RTPVideoHeaderH265>(rtp_video_header.video_type_header)
+        .has_last_fragement = true;
+#endif
   bool send_result = rtp_streams_[stream_index].sender_video->SendEncodedImage(
       rtp_config_.payload_type, codec_type_, rtp_timestamp, encoded_image,
       fragmentation,
-      params_[stream_index].GetRtpVideoHeader(
-          encoded_image, codec_specific_info, shared_frame_id_),
+      rtp_video_header,
       expected_retransmission_time_ms);
   if (frame_count_observer_) {
     FrameCounts& counts = frame_counts_[stream_index];

media/BUILD.gn

@@ -356,7 +356,7 @@ rtc_library("rtc_audio_video") {
   public_configs = []
   if (!build_with_chromium) {
     public_configs += [ ":rtc_media_defines_config" ]
-    deps += [ "../modules/video_capture:video_capture_internal_impl" ]
+    deps += [ "../modules/video_capture:video_capture" ]
   }
   if (rtc_enable_protobuf) {
     deps += [ "../modules/audio_processing/aec_dump:aec_dump_impl" ]

media/engine/adm_helpers.cc

@@ -58,6 +58,7 @@ void Init(AudioDeviceModule* adm) {
     }
   }
 
+#if defined(WEBRTC_INCLUDE_INTERNAL_AUDIO_DEVICE)
   // Recording device.
   {
     if (adm->SetRecordingDevice(AUDIO_DEVICE_ID) != 0) {
@@ -77,6 +78,7 @@ void Init(AudioDeviceModule* adm) {
       RTC_LOG(LS_ERROR) << "Failed to set stereo recording mode.";
     }
   }
+#endif
 }
 }  // namespace adm_helpers
 }  // namespace webrtc

media/engine/internal_decoder_factory.cc

@@ -48,6 +48,9 @@ std::vector<SdpVideoFormat> InternalDecoderFactory::GetSupportedFormats()
     formats.push_back(format);
   for (const SdpVideoFormat& h264_format : SupportedH264Codecs())
     formats.push_back(h264_format);
+#ifndef DISABLE_H265
+  formats.push_back(SdpVideoFormat(cricket::kH265CodecName));
+#endif
   if (kIsLibaomAv1DecoderSupported)
     formats.push_back(SdpVideoFormat(cricket::kAv1CodecName));
   return formats;

media/engine/webrtc_video_engine.cc

@@ -43,6 +43,8 @@
 #include "rtc_base/trace_event.h"
 #include "system_wrappers/include/field_trial.h"
 
+static const int kRtpBufferSizeIncreased = 4 * 1024 * 1024;
+
 namespace cricket {
 
 namespace {
@@ -52,7 +54,8 @@ const int kMinLayerSize = 16;
 // If this field trial is enabled, we will enable sending FlexFEC and disable
 // sending ULPFEC whenever the former has been negotiated in the SDPs.
 bool IsFlexfecFieldTrialEnabled() {
-  return webrtc::field_trial::IsEnabled("WebRTC-FlexFEC-03");
+  //return webrtc::field_trial::IsEnabled("WebRTC-FlexFEC-03");
+  return true;
 }
 
 // If this field trial is enabled, the "flexfec-03" codec will be advertised
@@ -62,7 +65,8 @@ bool IsFlexfecFieldTrialEnabled() {
 // MediaSession and added as "a=ssrc:" and "a=ssrc-group:" lines in the local
 // SDP.
 bool IsFlexfecAdvertisedFieldTrialEnabled() {
-  return webrtc::field_trial::IsEnabled("WebRTC-FlexFEC-03-Advertised");
+  //return webrtc::field_trial::IsEnabled("WebRTC-FlexFEC-03-Advertised");
+  return true;
 }
 
 void AddDefaultFeedbackParams(VideoCodec* codec) {
@@ -591,7 +595,8 @@ WebRtcVideoEngine::GetRtpHeaderExtensions() const {
         webrtc::RtpExtension::kVideoTimingUri,
         webrtc::RtpExtension::kFrameMarkingUri,
         webrtc::RtpExtension::kColorSpaceUri, webrtc::RtpExtension::kMidUri,
-        webrtc::RtpExtension::kRidUri, webrtc::RtpExtension::kRepairedRidUri}) {
+        webrtc::RtpExtension::kRidUri, webrtc::RtpExtension::kRepairedRidUri,
+        webrtc::RtpExtension::kPictureIdUri}) {
     result.emplace_back(uri, id++, webrtc::RtpTransceiverDirection::kSendRecv);
   }
   result.emplace_back(
@@ -1741,13 +1746,23 @@ void WebRtcVideoChannel::SetInterface(
   const std::string group_name =
       webrtc::field_trial::FindFullName("WebRTC-IncreasedReceivebuffers");
   int recv_buffer_size = kVideoRtpRecvBufferSize;
+  int send_buffer_size = kVideoRtpSendBufferSize;
+
   if (!group_name.empty() &&
       (sscanf(group_name.c_str(), "%d", &recv_buffer_size) != 1 ||
        recv_buffer_size <= 0)) {
     RTC_LOG(LS_WARNING) << "Invalid receive buffer size: " << group_name;
     recv_buffer_size = kVideoRtpRecvBufferSize;
   }
 
+  // If in low latency mode and doing push-mode streaming, enlarge the buffer to
+  // 4MB.
+  bool is_low_latency_mode =
+      webrtc::field_trial::IsEnabled("OWT-LowLatencyMode");
+  if (is_low_latency_mode) {
+    recv_buffer_size = kRtpBufferSizeIncreased;
+    send_buffer_size = kRtpBufferSizeIncreased;
+  }
   MediaChannel::SetOption(NetworkInterface::ST_RTP, rtc::Socket::OPT_RCVBUF,
                           recv_buffer_size);
 
@@ -1756,7 +1771,7 @@ void WebRtcVideoChannel::SetInterface(
   // due to lack of socket buffer space, although it's not yet clear what the
   // ideal value should be.
   MediaChannel::SetOption(NetworkInterface::ST_RTP, rtc::Socket::OPT_SNDBUF,
-                          kVideoRtpSendBufferSize);
+                          send_buffer_size);
 }
 
 void WebRtcVideoChannel::SetFrameDecryptor(
@@ -1931,8 +1946,18 @@ WebRtcVideoChannel::WebRtcVideoSendStream::WebRtcVideoSendStream(
   // Maximum packet size may come in RtpConfig from external transport, for
   // example from QuicTransportInterface implementation, so do not exceed
   // given max_packet_size.
-  parameters_.config.rtp.max_packet_size =
-      std::min<size_t>(parameters_.config.rtp.max_packet_size, kVideoMtu);
+  std::string experiment_string =
+      webrtc::field_trial::FindFullName("OWT-LinkMTU");
+  if (!experiment_string.empty()) {
+    double link_mtu = ::strtod(experiment_string.c_str(), nullptr);
+    if (link_mtu > 0) {
+      parameters_.config.rtp.max_packet_size = link_mtu;
+    } else {
+      parameters_.config.rtp.max_packet_size = kVideoMtu;
+    }
+  } else {
+    parameters_.config.rtp.max_packet_size = kVideoMtu;
+  }
   parameters_.conference_mode = send_params.conference_mode;
 
   sp.GetPrimarySsrcs(&parameters_.config.rtp.ssrcs);

media/engine/webrtc_voice_engine.cc

@@ -288,6 +288,15 @@ void WebRtcVoiceEngine::Init() {
     options.experimental_agc = false;
     options.experimental_ns = false;
     options.residual_echo_detector = true;
+    // Overrided for low latency mode
+    if (webrtc::field_trial::IsEnabled("OWT-LowLatencyMode")) {
+      options.echo_cancellation = false;
+      options.auto_gain_control = false;
+      options.noise_suppression = false;
+      options.highpass_filter = false;
+      options.typing_detection = false;
+      options.residual_echo_detector = false;
+    }
     bool error = ApplyOptions(options);
     RTC_DCHECK(error);
   }
@@ -549,7 +558,8 @@ WebRtcVoiceEngine::GetRtpHeaderExtensions() const {
         webrtc::RtpExtension::kAbsSendTimeUri,
         webrtc::RtpExtension::kTransportSequenceNumberUri,
         webrtc::RtpExtension::kMidUri, webrtc::RtpExtension::kRidUri,
-        webrtc::RtpExtension::kRepairedRidUri}) {
+        webrtc::RtpExtension::kRepairedRidUri,
+        webrtc::RtpExtension::kPictureIdUri}) {
     result.emplace_back(uri, id++, webrtc::RtpTransceiverDirection::kSendRecv);
   }
   return result;

modules/audio_coding/codecs/opus/audio_encoder_opus.cc

@@ -130,6 +130,13 @@ int CalculateBitrate(int max_playback_rate_hz,
   const int default_bitrate =
       CalculateDefaultBitrate(max_playback_rate_hz, num_channels);
 
+  // For lown latency mode we fix the audio bitrate.
+  if (field_trial::IsEnabled("OWT-LowLatencyMode")) {
+    if (default_bitrate != -1)
+      return 256000;
+    return default_bitrate;
+  }
+
   if (bitrate_param) {
     const auto bitrate = rtc::StringToNumber<int>(*bitrate_param);
     if (bitrate) {
@@ -154,7 +161,7 @@ int GetChannelCount(const SdpAudioFormat& format) {
   if (param == "1") {
     return 2;
   } else {
-    return 1;
+    return 2;
   }
 }
 

modules/audio_processing/audio_processing_impl.cc

@@ -840,7 +840,8 @@ int AudioProcessingImpl::ProcessStream(const float* const* src,
     capture_.capture_fullband_audio->CopyFrom(
         src, formats_.api_format.input_stream());
   }
-  RETURN_ON_ERR(ProcessCaptureStreamLocked());
+  // Bypass processing
+  //RETURN_ON_ERR(ProcessCaptureStreamLocked());
   if (capture_.capture_fullband_audio) {
     capture_.capture_fullband_audio->CopyTo(formats_.api_format.output_stream(),
                                             dest);