diff --git a/BUILD.gn b/BUILD.gn index 5fe6f849f33..e12c051c723 100644 --- a/BUILD.gn +++ b/BUILD.gn @@ -53,7 +53,7 @@ if (!build_with_chromium) { "modules/audio_processing:audio_processing_tests", "modules/remote_bitrate_estimator:rtp_to_text", "modules/rtp_rtcp:test_packet_masks_metrics", - "modules/video_capture:video_capture_internal_impl", + "modules/video_capture:video_capture", "pc:peerconnection_unittests", "pc:rtc_pc_unittests", "rtc_tools:rtp_generator", @@ -470,7 +470,7 @@ if (!build_with_chromium) { "logging:rtc_event_log_api", "media", "modules", - "modules/video_capture:video_capture_internal_impl", + "modules/video_capture:video_capture", "p2p:rtc_p2p", "pc:libjingle_peerconnection", "pc:peerconnection", diff --git a/api/audio_codecs/opus/audio_decoder_opus.cc b/api/audio_codecs/opus/audio_decoder_opus.cc index 6b4e0d3797e..9840668c420 100644 --- a/api/audio_codecs/opus/audio_decoder_opus.cc +++ b/api/audio_codecs/opus/audio_decoder_opus.cc @@ -44,7 +44,7 @@ absl::optional AudioDecoderOpus::SdpToConfig( return absl::nullopt; // Bad stereo parameter. } } - return 1; // Default to mono. + return 2; // Default to mono. }(); if (absl::EqualsIgnoreCase(format.name, "opus") && format.clockrate_hz == 48000 && format.num_channels == 2 && diff --git a/api/rtp_headers.h b/api/rtp_headers.h index 163347f6758..acdf16efc7e 100644 --- a/api/rtp_headers.h +++ b/api/rtp_headers.h @@ -159,6 +159,9 @@ struct RTPHeaderExtension { std::string mid; absl::optional color_space; + + // Required for Temporal scalability support + uint16_t picture_id; }; enum { kRtpCsrcSize = 15 }; // RFC 3550 page 13 diff --git a/api/rtp_parameters.cc b/api/rtp_parameters.cc index 16b053ea058..b4b7ef82021 100644 --- a/api/rtp_parameters.cc +++ b/api/rtp_parameters.cc @@ -116,6 +116,7 @@ constexpr char RtpExtension::kColorSpaceUri[]; constexpr char RtpExtension::kMidUri[]; constexpr char RtpExtension::kRidUri[]; constexpr char RtpExtension::kRepairedRidUri[]; +constexpr char RtpExtension::kPictureIdUri[]; constexpr int RtpExtension::kMinId; constexpr int RtpExtension::kMaxId; @@ -151,7 +152,8 @@ bool RtpExtension::IsSupportedForVideo(absl::string_view uri) { uri == webrtc::RtpExtension::kDependencyDescriptorUri || uri == webrtc::RtpExtension::kColorSpaceUri || uri == webrtc::RtpExtension::kRidUri || - uri == webrtc::RtpExtension::kRepairedRidUri; + uri == webrtc::RtpExtension::kRepairedRidUri || + uri == webrtc::RtpExtension::kPictureIdUri; } bool RtpExtension::IsEncryptionSupported(absl::string_view uri) { diff --git a/api/rtp_parameters.h b/api/rtp_parameters.h index 543592240f1..7c81ebc5550 100644 --- a/api/rtp_parameters.h +++ b/api/rtp_parameters.h @@ -360,6 +360,10 @@ struct RTC_EXPORT RtpExtension { static constexpr char kRepairedRidUri[] = "urn:ietf:params:rtp-hdrext:sdes:repaired-rtp-stream-id"; + // Picture ID for temporal scalability support + static constexpr char kPictureIdUri[] = "urn:intel:picture-id"; + static constexpr int kPictureIdDefaultId = 12; + // Inclusive min and max IDs for two-byte header extensions and one-byte // header extensions, per RFC8285 Section 4.2-4.3. static constexpr int kMinId = 1; diff --git a/api/transport/bitrate_settings.h b/api/transport/bitrate_settings.h index b6c022dd112..9fcbe5541f3 100644 --- a/api/transport/bitrate_settings.h +++ b/api/transport/bitrate_settings.h @@ -42,7 +42,7 @@ struct BitrateConstraints { int max_bitrate_bps = -1; private: - static constexpr int kDefaultStartBitrateBps = 300000; + static constexpr int kDefaultStartBitrateBps = 5000000; // Update from 300k to 5MB for low latency mode }; } // namespace webrtc diff --git a/api/transport/network_control.h b/api/transport/network_control.h index 6fc1f7c0d15..9cd3ff48243 100644 --- a/api/transport/network_control.h +++ b/api/transport/network_control.h @@ -93,7 +93,7 @@ class NetworkControllerInterface { ABSL_MUST_USE_RESULT = 0; // Called with per packet feedback regarding receive time. virtual NetworkControlUpdate OnTransportPacketsFeedback( - TransportPacketsFeedback) ABSL_MUST_USE_RESULT = 0; + TransportPacketsFeedback, int64_t) ABSL_MUST_USE_RESULT = 0; // Called with network state estimate updates. virtual NetworkControlUpdate OnNetworkStateEstimate(NetworkStateEstimate) ABSL_MUST_USE_RESULT = 0; diff --git a/call/call_config.h b/call/call_config.h index cd0ef3352e8..77fc5ab7c95 100644 --- a/call/call_config.h +++ b/call/call_config.h @@ -30,7 +30,7 @@ struct CallConfig { CallConfig(const CallConfig&); ~CallConfig(); - RTC_DEPRECATED static constexpr int kDefaultStartBitrateBps = 300000; + RTC_DEPRECATED static constexpr int kDefaultStartBitrateBps = 5000000; //Updated from 300k to 5MB for low latency // Bitrate config used until valid bitrate estimates are calculated. Also // used to cap total bitrate used. This comes from the remote connection. diff --git a/call/rtp_payload_params.cc b/call/rtp_payload_params.cc index d58e7031441..e72488afbed 100644 --- a/call/rtp_payload_params.cc +++ b/call/rtp_payload_params.cc @@ -19,6 +19,9 @@ #include "absl/types/variant.h" #include "api/video/video_timing.h" #include "modules/video_coding/codecs/h264/include/h264_globals.h" +#ifndef DISABLE_H265 +#include "modules/video_coding/codecs/h265/include/h265_globals.h" +#endif #include "modules/video_coding/codecs/interface/common_constants.h" #include "modules/video_coding/codecs/vp8/include/vp8_globals.h" #include "modules/video_coding/codecs/vp9/include/vp9_globals.h" @@ -103,6 +106,7 @@ void PopulateRtpWithCodecSpecifics(const CodecSpecificInfo& info, rtp->frame_marking.base_layer_sync = info.codecSpecific.H264.base_layer_sync; } + h264_header.picture_id = info.codecSpecific.H264.picture_id; return; } #ifndef DISABLE_H265 @@ -110,6 +114,7 @@ void PopulateRtpWithCodecSpecifics(const CodecSpecificInfo& info, auto& h265_header = rtp->video_type_header.emplace(); h265_header.packetization_mode = info.codecSpecific.H265.packetization_mode; + h265_header.picture_id = info.codecSpecific.H265.picture_id; } return; #endif diff --git a/call/rtp_transport_controller_send.cc b/call/rtp_transport_controller_send.cc index 56c5e55ca17..ec165c92315 100644 --- a/call/rtp_transport_controller_send.cc +++ b/call/rtp_transport_controller_send.cc @@ -27,6 +27,7 @@ #include "rtc_base/checks.h" #include "rtc_base/logging.h" #include "rtc_base/rate_limiter.h" +#include "system_wrappers/include/field_trial.h" namespace webrtc { namespace { @@ -34,6 +35,7 @@ static const int64_t kRetransmitWindowSizeMs = 500; static const size_t kMaxOverheadBytes = 500; constexpr TimeDelta kPacerQueueUpdateInterval = TimeDelta::Millis(25); +const char kLowLatencyStreaming[] = "OWT-LowLatencyMode"; TargetRateConstraints ConvertConstraints(int min_bitrate_bps, int max_bitrate_bps, @@ -52,6 +54,11 @@ TargetRateConstraints ConvertConstraints(int min_bitrate_bps, return msg; } +bool LowLatencyStreamingEnabled() { + std::string trial = webrtc::field_trial::FindFullName(kLowLatencyStreaming); + return trial.find("Enabled") == 0; +} + TargetRateConstraints ConvertConstraints(const BitrateConstraints& contraints, Clock* clock) { return ConvertConstraints(contraints.min_bitrate_bps, @@ -128,7 +135,11 @@ RtpTransportControllerSend::RtpTransportControllerSend( DataRate::BitsPerSec(bitrate_config.start_bitrate_bps), DataRate::Zero()); if (!use_task_queue_pacer_) { - process_thread_->Start(); + if (!LowLatencyStreamingEnabled()) { + process_thread_->Start(); + } else { + process_thread_->StartWithHighPriority(); + } } } @@ -543,7 +554,12 @@ void RtpTransportControllerSend::OnTransportFeedback( transport_feedback_adapter_.ProcessTransportFeedback(feedback, feedback_time); if (feedback_msg && controller_) { - PostUpdates(controller_->OnTransportPacketsFeedback(*feedback_msg)); +#ifdef INTEL_GPRA + PostUpdates(controller_->OnTransportPacketsFeedback( + *feedback_msg, transport_feedback_adapter_.GetCurrentOffsetMs())); +#else + PostUpdates(controller_->OnTransportPacketsFeedback(*feedback_msg, 0)); +#endif } pacer()->UpdateOutstandingData( transport_feedback_adapter_.GetOutstandingData()); diff --git a/call/rtp_video_sender.cc b/call/rtp_video_sender.cc index cb32085d357..54d7a16fe03 100644 --- a/call/rtp_video_sender.cc +++ b/call/rtp_video_sender.cc @@ -528,12 +528,20 @@ EncodedImageCallback::Result RtpVideoSender::OnEncodedImage( ? &*codec_specific_info->template_structure : nullptr); } - + RTPVideoHeader rtp_video_header = params_[stream_index].GetRtpVideoHeader( + encoded_image, codec_specific_info, shared_frame_id_); + if (codec_specific_info->codecSpecific.H264.last_fragment_in_frame) + absl::get(rtp_video_header.video_type_header) + .has_last_fragement = true; +#ifndef DISABLE_H265 + else if (codec_specific_info->codecSpecific.H265.last_fragment_in_frame) + absl::get(rtp_video_header.video_type_header) + .has_last_fragement = true; +#endif bool send_result = rtp_streams_[stream_index].sender_video->SendEncodedImage( rtp_config_.payload_type, codec_type_, rtp_timestamp, encoded_image, fragmentation, - params_[stream_index].GetRtpVideoHeader( - encoded_image, codec_specific_info, shared_frame_id_), + rtp_video_header, expected_retransmission_time_ms); if (frame_count_observer_) { FrameCounts& counts = frame_counts_[stream_index]; diff --git a/media/BUILD.gn b/media/BUILD.gn index 550f9f9fa03..06dbad3e9b7 100644 --- a/media/BUILD.gn +++ b/media/BUILD.gn @@ -356,7 +356,7 @@ rtc_library("rtc_audio_video") { public_configs = [] if (!build_with_chromium) { public_configs += [ ":rtc_media_defines_config" ] - deps += [ "../modules/video_capture:video_capture_internal_impl" ] + deps += [ "../modules/video_capture:video_capture" ] } if (rtc_enable_protobuf) { deps += [ "../modules/audio_processing/aec_dump:aec_dump_impl" ] diff --git a/media/engine/adm_helpers.cc b/media/engine/adm_helpers.cc index c349b7ce067..66a86dcaf1d 100644 --- a/media/engine/adm_helpers.cc +++ b/media/engine/adm_helpers.cc @@ -58,6 +58,7 @@ void Init(AudioDeviceModule* adm) { } } +#if defined(WEBRTC_INCLUDE_INTERNAL_AUDIO_DEVICE) // Recording device. { if (adm->SetRecordingDevice(AUDIO_DEVICE_ID) != 0) { @@ -77,6 +78,7 @@ void Init(AudioDeviceModule* adm) { RTC_LOG(LS_ERROR) << "Failed to set stereo recording mode."; } } +#endif } } // namespace adm_helpers } // namespace webrtc diff --git a/media/engine/internal_decoder_factory.cc b/media/engine/internal_decoder_factory.cc index e68bb369b55..006e4394bd9 100644 --- a/media/engine/internal_decoder_factory.cc +++ b/media/engine/internal_decoder_factory.cc @@ -48,6 +48,9 @@ std::vector InternalDecoderFactory::GetSupportedFormats() formats.push_back(format); for (const SdpVideoFormat& h264_format : SupportedH264Codecs()) formats.push_back(h264_format); +#ifndef DISABLE_H265 + formats.push_back(SdpVideoFormat(cricket::kH265CodecName)); +#endif if (kIsLibaomAv1DecoderSupported) formats.push_back(SdpVideoFormat(cricket::kAv1CodecName)); return formats; diff --git a/media/engine/webrtc_video_engine.cc b/media/engine/webrtc_video_engine.cc index a19d444579b..288268e6dbe 100644 --- a/media/engine/webrtc_video_engine.cc +++ b/media/engine/webrtc_video_engine.cc @@ -43,6 +43,8 @@ #include "rtc_base/trace_event.h" #include "system_wrappers/include/field_trial.h" +static const int kRtpBufferSizeIncreased = 4 * 1024 * 1024; + namespace cricket { namespace { @@ -52,7 +54,8 @@ const int kMinLayerSize = 16; // If this field trial is enabled, we will enable sending FlexFEC and disable // sending ULPFEC whenever the former has been negotiated in the SDPs. bool IsFlexfecFieldTrialEnabled() { - return webrtc::field_trial::IsEnabled("WebRTC-FlexFEC-03"); + //return webrtc::field_trial::IsEnabled("WebRTC-FlexFEC-03"); + return true; } // If this field trial is enabled, the "flexfec-03" codec will be advertised @@ -62,7 +65,8 @@ bool IsFlexfecFieldTrialEnabled() { // MediaSession and added as "a=ssrc:" and "a=ssrc-group:" lines in the local // SDP. bool IsFlexfecAdvertisedFieldTrialEnabled() { - return webrtc::field_trial::IsEnabled("WebRTC-FlexFEC-03-Advertised"); + //return webrtc::field_trial::IsEnabled("WebRTC-FlexFEC-03-Advertised"); + return true; } void AddDefaultFeedbackParams(VideoCodec* codec) { @@ -591,7 +595,8 @@ WebRtcVideoEngine::GetRtpHeaderExtensions() const { webrtc::RtpExtension::kVideoTimingUri, webrtc::RtpExtension::kFrameMarkingUri, webrtc::RtpExtension::kColorSpaceUri, webrtc::RtpExtension::kMidUri, - webrtc::RtpExtension::kRidUri, webrtc::RtpExtension::kRepairedRidUri}) { + webrtc::RtpExtension::kRidUri, webrtc::RtpExtension::kRepairedRidUri, + webrtc::RtpExtension::kPictureIdUri}) { result.emplace_back(uri, id++, webrtc::RtpTransceiverDirection::kSendRecv); } result.emplace_back( @@ -1741,6 +1746,8 @@ void WebRtcVideoChannel::SetInterface( const std::string group_name = webrtc::field_trial::FindFullName("WebRTC-IncreasedReceivebuffers"); int recv_buffer_size = kVideoRtpRecvBufferSize; + int send_buffer_size = kVideoRtpSendBufferSize; + if (!group_name.empty() && (sscanf(group_name.c_str(), "%d", &recv_buffer_size) != 1 || recv_buffer_size <= 0)) { @@ -1748,6 +1755,14 @@ void WebRtcVideoChannel::SetInterface( recv_buffer_size = kVideoRtpRecvBufferSize; } + // If in low latency mode and doing push-mode streaming, enlarge the buffer to + // 4MB. + bool is_low_latency_mode = + webrtc::field_trial::IsEnabled("OWT-LowLatencyMode"); + if (is_low_latency_mode) { + recv_buffer_size = kRtpBufferSizeIncreased; + send_buffer_size = kRtpBufferSizeIncreased; + } MediaChannel::SetOption(NetworkInterface::ST_RTP, rtc::Socket::OPT_RCVBUF, recv_buffer_size); @@ -1756,7 +1771,7 @@ void WebRtcVideoChannel::SetInterface( // due to lack of socket buffer space, although it's not yet clear what the // ideal value should be. MediaChannel::SetOption(NetworkInterface::ST_RTP, rtc::Socket::OPT_SNDBUF, - kVideoRtpSendBufferSize); + send_buffer_size); } void WebRtcVideoChannel::SetFrameDecryptor( @@ -1931,8 +1946,18 @@ WebRtcVideoChannel::WebRtcVideoSendStream::WebRtcVideoSendStream( // Maximum packet size may come in RtpConfig from external transport, for // example from QuicTransportInterface implementation, so do not exceed // given max_packet_size. - parameters_.config.rtp.max_packet_size = - std::min(parameters_.config.rtp.max_packet_size, kVideoMtu); + std::string experiment_string = + webrtc::field_trial::FindFullName("OWT-LinkMTU"); + if (!experiment_string.empty()) { + double link_mtu = ::strtod(experiment_string.c_str(), nullptr); + if (link_mtu > 0) { + parameters_.config.rtp.max_packet_size = link_mtu; + } else { + parameters_.config.rtp.max_packet_size = kVideoMtu; + } + } else { + parameters_.config.rtp.max_packet_size = kVideoMtu; + } parameters_.conference_mode = send_params.conference_mode; sp.GetPrimarySsrcs(¶meters_.config.rtp.ssrcs); diff --git a/media/engine/webrtc_voice_engine.cc b/media/engine/webrtc_voice_engine.cc index 42109e86a84..8e64100c6e9 100644 --- a/media/engine/webrtc_voice_engine.cc +++ b/media/engine/webrtc_voice_engine.cc @@ -288,6 +288,15 @@ void WebRtcVoiceEngine::Init() { options.experimental_agc = false; options.experimental_ns = false; options.residual_echo_detector = true; + // Overrided for low latency mode + if (webrtc::field_trial::IsEnabled("OWT-LowLatencyMode")) { + options.echo_cancellation = false; + options.auto_gain_control = false; + options.noise_suppression = false; + options.highpass_filter = false; + options.typing_detection = false; + options.residual_echo_detector = false; + } bool error = ApplyOptions(options); RTC_DCHECK(error); } @@ -549,7 +558,8 @@ WebRtcVoiceEngine::GetRtpHeaderExtensions() const { webrtc::RtpExtension::kAbsSendTimeUri, webrtc::RtpExtension::kTransportSequenceNumberUri, webrtc::RtpExtension::kMidUri, webrtc::RtpExtension::kRidUri, - webrtc::RtpExtension::kRepairedRidUri}) { + webrtc::RtpExtension::kRepairedRidUri, + webrtc::RtpExtension::kPictureIdUri}) { result.emplace_back(uri, id++, webrtc::RtpTransceiverDirection::kSendRecv); } return result; diff --git a/modules/audio_coding/codecs/opus/audio_encoder_opus.cc b/modules/audio_coding/codecs/opus/audio_encoder_opus.cc index 768031b0fb3..6067ea38ea7 100644 --- a/modules/audio_coding/codecs/opus/audio_encoder_opus.cc +++ b/modules/audio_coding/codecs/opus/audio_encoder_opus.cc @@ -130,6 +130,13 @@ int CalculateBitrate(int max_playback_rate_hz, const int default_bitrate = CalculateDefaultBitrate(max_playback_rate_hz, num_channels); + // For lown latency mode we fix the audio bitrate. + if (field_trial::IsEnabled("OWT-LowLatencyMode")) { + if (default_bitrate != -1) + return 256000; + return default_bitrate; + } + if (bitrate_param) { const auto bitrate = rtc::StringToNumber(*bitrate_param); if (bitrate) { @@ -154,7 +161,7 @@ int GetChannelCount(const SdpAudioFormat& format) { if (param == "1") { return 2; } else { - return 1; + return 2; } } diff --git a/modules/audio_processing/audio_processing_impl.cc b/modules/audio_processing/audio_processing_impl.cc index e5e6ccef60f..8ecfe1fb594 100644 --- a/modules/audio_processing/audio_processing_impl.cc +++ b/modules/audio_processing/audio_processing_impl.cc @@ -840,7 +840,8 @@ int AudioProcessingImpl::ProcessStream(const float* const* src, capture_.capture_fullband_audio->CopyFrom( src, formats_.api_format.input_stream()); } - RETURN_ON_ERR(ProcessCaptureStreamLocked()); + // Bypass processing + //RETURN_ON_ERR(ProcessCaptureStreamLocked()); if (capture_.capture_fullband_audio) { capture_.capture_fullband_audio->CopyTo(formats_.api_format.output_stream(), dest); diff --git a/modules/congestion_controller/BUILD.gn b/modules/congestion_controller/BUILD.gn index f65cde43c34..eebf952427c 100644 --- a/modules/congestion_controller/BUILD.gn +++ b/modules/congestion_controller/BUILD.gn @@ -8,11 +8,13 @@ import("../../webrtc.gni") +defines = [] + config("bwe_test_logging") { if (rtc_enable_bwe_test_logging) { - defines = [ "BWE_TEST_LOGGING_COMPILE_TIME_ENABLE=1" ] + defines += [ "BWE_TEST_LOGGING_COMPILE_TIME_ENABLE=1" ] } else { - defines = [ "BWE_TEST_LOGGING_COMPILE_TIME_ENABLE=0" ] + defines += [ "BWE_TEST_LOGGING_COMPILE_TIME_ENABLE=0" ] } } @@ -24,6 +26,14 @@ rtc_library("congestion_controller") { "receive_side_congestion_controller.cc", ] + if (owt_use_gpra) { + # use Intel Goodput Rate Adapatation BWE + include_dirs = [ owt_gpra_header_root ] + libs = [ "gpra_lib.lib" ] + lib_dirs = [ owt_gpra_lib_root ] + defines += [ "INTEL_GPRA" ] + } + deps = [ "..:module_api", "../../api/transport:field_trial_based_config", diff --git a/modules/congestion_controller/goog_cc/goog_cc_network_control.cc b/modules/congestion_controller/goog_cc/goog_cc_network_control.cc index e29a6704ba1..53a279773ff 100644 --- a/modules/congestion_controller/goog_cc/goog_cc_network_control.cc +++ b/modules/congestion_controller/goog_cc/goog_cc_network_control.cc @@ -30,6 +30,9 @@ #include "modules/remote_bitrate_estimator/test/bwe_test_logging.h" #include "rtc_base/checks.h" #include "rtc_base/logging.h" +#ifdef INTEL_GPRA +#include "gpra_bwe.h" +#endif namespace webrtc { @@ -94,9 +97,13 @@ GoogCcNetworkController::GoogCcNetworkController(NetworkControllerConfig config, network_estimator_(std::move(goog_cc_config.network_state_estimator)), network_state_predictor_( std::move(goog_cc_config.network_state_predictor)), +#ifdef INTEL_GPRA + delay_based_bwe_(new GPRABwe()), +#else delay_based_bwe_(new DelayBasedBwe(key_value_config_, event_log_, network_state_predictor_.get())), +#endif acknowledged_bitrate_estimator_( AcknowledgedBitrateEstimatorInterface::Create(key_value_config_)), initial_config_(config), @@ -117,6 +124,11 @@ GoogCcNetworkController::GoogCcNetworkController(NetworkControllerConfig config, key_value_config_->Lookup("WebRTC-Bwe-SafeResetOnRouteChange")); if (delay_based_bwe_) delay_based_bwe_->SetMinBitrate(congestion_controller::GetMinBitrate()); +#ifdef INTEL_GPRA + RTC_LOG(LS_ERROR) << "DebugP Use GPRA BWE!!!"; +#else + RTC_LOG(LS_ERROR) << "DebugP Use TransportCC BWE"; +#endif } GoogCcNetworkController::~GoogCcNetworkController() {} @@ -154,8 +166,12 @@ NetworkControlUpdate GoogCcNetworkController::OnNetworkRouteChange( probe_bitrate_estimator_.reset(new ProbeBitrateEstimator(event_log_)); if (network_estimator_) network_estimator_->OnRouteChange(msg); +#ifdef INTEL_GPRA + delay_based_bwe_.reset(new GPRABwe()); +#else delay_based_bwe_.reset(new DelayBasedBwe(key_value_config_, event_log_, network_state_predictor_.get())); +#endif bandwidth_estimation_->OnRouteChange(); probe_controller_->Reset(msg.at_time.ms()); NetworkControlUpdate update; @@ -394,7 +410,7 @@ void GoogCcNetworkController::UpdateCongestionWindowSize() { } NetworkControlUpdate GoogCcNetworkController::OnTransportPacketsFeedback( - TransportPacketsFeedback report) { + TransportPacketsFeedback report, int64_t current_offset_ms) { if (report.packet_feedbacks.empty()) { // TODO(bugs.webrtc.org/10125): Design a better mechanism to safe-guard // against building very large network queues. @@ -514,9 +530,16 @@ NetworkControlUpdate GoogCcNetworkController::OnTransportPacketsFeedback( bool backoff_in_alr = false; DelayBasedBwe::Result result; +#ifdef INTEL_GPRA + delay_based_bwe_->SetCurrentOffsetMs(current_offset_ms); result = delay_based_bwe_->IncomingPacketFeedbackVector( report, acknowledged_bitrate, probe_bitrate, estimate_, alr_start_time.has_value()); +#else + result = delay_based_bwe_->IncomingPacketFeedbackVector( + report, acknowledged_bitrate, probe_bitrate, estimate_, + alr_start_time.has_value()); +#endif if (result.updated) { if (result.probe) { @@ -639,6 +662,10 @@ void GoogCcNetworkController::MaybeTriggerOnNetworkChanged( TimeDelta bwe_period = delay_based_bwe_->GetExpectedBwePeriod(); +#ifdef INTEL_GPRA + delay_based_bwe_->SetCurrentPacketLossRate(fraction_loss); +#endif + TargetTransferRate target_rate_msg; target_rate_msg.at_time = at_time; if (rate_control_settings_.UseCongestionWindowDropFrameOnly()) { diff --git a/modules/congestion_controller/goog_cc/goog_cc_network_control.h b/modules/congestion_controller/goog_cc/goog_cc_network_control.h index 5af8a3f3822..d1cc8008280 100644 --- a/modules/congestion_controller/goog_cc/goog_cc_network_control.h +++ b/modules/congestion_controller/goog_cc/goog_cc_network_control.h @@ -43,6 +43,7 @@ struct GoogCcConfig { std::unique_ptr network_state_predictor = nullptr; bool feedback_only = false; }; +class GPRABwe; class GoogCcNetworkController : public NetworkControllerInterface { public: @@ -63,12 +64,17 @@ class GoogCcNetworkController : public NetworkControllerInterface { TargetRateConstraints msg) override; NetworkControlUpdate OnTransportLossReport(TransportLossReport msg) override; NetworkControlUpdate OnTransportPacketsFeedback( - TransportPacketsFeedback msg) override; + TransportPacketsFeedback msg, int64_t current_offset_ms) override; NetworkControlUpdate OnNetworkStateEstimate( NetworkStateEstimate msg) override; NetworkControlUpdate GetNetworkState(Timestamp at_time) const; +#if defined(INTEL_GPRA) + // Not implemented here? + void SetBitrateEstimationWindowSize(int init_window, int rate_window); +#endif + private: friend class GoogCcStatePrinter; std::vector ResetConstraints( @@ -99,7 +105,11 @@ class GoogCcNetworkController : public NetworkControllerInterface { std::unique_ptr probe_bitrate_estimator_; std::unique_ptr network_estimator_; std::unique_ptr network_state_predictor_; +#ifdef INTEL_GPRA + std::unique_ptr delay_based_bwe_; +#else std::unique_ptr delay_based_bwe_; +#endif std::unique_ptr acknowledged_bitrate_estimator_; diff --git a/modules/congestion_controller/goog_cc/goog_cc_network_control_unittest.cc b/modules/congestion_controller/goog_cc/goog_cc_network_control_unittest.cc index 1083214fa5c..496fd570a51 100644 --- a/modules/congestion_controller/goog_cc/goog_cc_network_control_unittest.cc +++ b/modules/congestion_controller/goog_cc/goog_cc_network_control_unittest.cc @@ -223,7 +223,7 @@ class GoogCcNetworkControllerTest : public ::testing::Test { TransportPacketsFeedback feedback; feedback.feedback_time = packet.receive_time; feedback.packet_feedbacks.push_back(packet); - OnUpdate(controller_->OnTransportPacketsFeedback(feedback)); + OnUpdate(controller_->OnTransportPacketsFeedback(feedback, 0)); AdvanceTimeMilliseconds(50); OnUpdate(controller_->OnProcessInterval(DefaultInterval())); } diff --git a/modules/congestion_controller/goog_cc/send_side_bandwidth_estimation.cc b/modules/congestion_controller/goog_cc/send_side_bandwidth_estimation.cc index d2ae5284040..e7142211307 100644 --- a/modules/congestion_controller/goog_cc/send_side_bandwidth_estimation.cc +++ b/modules/congestion_controller/goog_cc/send_side_bandwidth_estimation.cc @@ -309,6 +309,10 @@ void SendSideBandwidthEstimation::UpdateReceiverEstimate(Timestamp at_time, DataRate bandwidth) { // TODO(srte): Ensure caller passes PlusInfinity, not zero, to represent no // limitation. + if (field_trial::IsEnabled("OWT-LowLatencyMode")) { + // For low latency mode, we will always ignore REMB result. + return; + } receiver_limit_ = bandwidth.IsZero() ? DataRate::PlusInfinity() : bandwidth; ApplyTargetLimits(at_time); } @@ -580,13 +584,23 @@ DataRate SendSideBandwidthEstimation::MaybeRampupOrBackoff(DataRate new_bitrate, } DataRate SendSideBandwidthEstimation::GetUpperLimit() const { - DataRate upper_limit = std::min(delay_based_limit_, receiver_limit_); - upper_limit = std::min(upper_limit, max_bitrate_configured_); - if (loss_based_bandwidth_estimation_.Enabled() && - loss_based_bandwidth_estimation_.GetEstimate() > DataRate::Zero()) { - upper_limit = - std::min(upper_limit, loss_based_bandwidth_estimation_.GetEstimate()); + std::string experiment_string = + webrtc::field_trial::FindFullName("OWT-DelayBweWeight"); + double delay_weight = ::strtod(experiment_string.c_str(), nullptr); + double delay_fraction = delay_weight / 100.0; + double lost_fraction = 1 - delay_fraction; + + // Calculate new weighted BWE using delay/loss limit. + DataRate weighted_limit = delay_based_limit_; + if (delay_weight < 100 && delay_weight >= 0) { + if (loss_based_bandwidth_estimation_.Enabled() && + loss_based_bandwidth_estimation_.GetEstimate() > DataRate::Zero()) { + weighted_limit.BitsPerSec(delay_based_limit_.bps_or(0) * delay_fraction + + loss_based_bandwidth_estimation_.GetEstimate().bps_or(0) * lost_fraction); + } } + DataRate upper_limit = std::min(weighted_limit, receiver_limit_); + upper_limit = std::min(upper_limit, max_bitrate_configured_); return upper_limit; } diff --git a/modules/congestion_controller/rtp/BUILD.gn b/modules/congestion_controller/rtp/BUILD.gn index b444f5495b9..0a4d7449e28 100644 --- a/modules/congestion_controller/rtp/BUILD.gn +++ b/modules/congestion_controller/rtp/BUILD.gn @@ -49,6 +49,10 @@ rtc_library("transport_feedback") { "transport_feedback_demuxer.h", ] + if (owt_use_gpra) { + defines += [ "INTEL_GPRA" ] + } + deps = [ "../..:module_api_public", "../../../api/transport:network_control", diff --git a/modules/congestion_controller/rtp/transport_feedback_adapter.cc b/modules/congestion_controller/rtp/transport_feedback_adapter.cc index 87691bf263a..99d1db0aa3a 100644 --- a/modules/congestion_controller/rtp/transport_feedback_adapter.cc +++ b/modules/congestion_controller/rtp/transport_feedback_adapter.cc @@ -62,6 +62,12 @@ DataSize InFlightBytesTracker::GetOutstandingData( } } +#ifdef INTEL_GPRA +int64_t TransportFeedbackAdapter::GetCurrentOffsetMs() { + return current_offset_ms_; +} +#endif + // Comparator for consistent map with NetworkRoute as key. bool InFlightBytesTracker::NetworkRouteComparator::operator()( const rtc::NetworkRoute& a, diff --git a/modules/congestion_controller/rtp/transport_feedback_adapter.h b/modules/congestion_controller/rtp/transport_feedback_adapter.h index b8148a252f1..578ea61d26f 100644 --- a/modules/congestion_controller/rtp/transport_feedback_adapter.h +++ b/modules/congestion_controller/rtp/transport_feedback_adapter.h @@ -73,6 +73,10 @@ class TransportFeedbackAdapter { DataSize GetOutstandingData() const; +#ifdef INTEL_GPRA + int64_t GetCurrentOffsetMs(); +#endif + private: enum class SendTimeHistoryStatus { kNotAdded, kOk, kDuplicate }; diff --git a/modules/pacing/paced_sender.cc b/modules/pacing/paced_sender.cc index cd298f9b0bf..813d808ee63 100644 --- a/modules/pacing/paced_sender.cc +++ b/modules/pacing/paced_sender.cc @@ -23,29 +23,42 @@ #include "rtc_base/logging.h" #include "rtc_base/time_utils.h" #include "system_wrappers/include/clock.h" +#include "system_wrappers/include/field_trial.h" namespace webrtc { const int64_t PacedSender::kMaxQueueLengthMs = 2000; const float PacedSender::kDefaultPaceMultiplier = 2.5f; -PacedSender::PacedSender(Clock* clock, PacketRouter* packet_router, +bool PacedSender::IsLowLatencyMode() const { + return low_latency_mode_; +} + +PacedSender::PacedSender(Clock* clock, + PacketRouter* packet_router, RtcEventLog* event_log, const WebRtcKeyValueConfig* field_trials, ProcessThread* process_thread) - : process_mode_( + : low_latency_mode_(field_trial::IsEnabled("OWT-LowLatencyMode")), + process_mode_( (field_trials != nullptr && - absl::StartsWith(field_trials->Lookup("WebRTC-Pacer-DynamicProcess"), + absl::StartsWith(field_trials->Lookup("OWT-LowLatencyMode"), "Enabled")) - ? PacingController::ProcessMode::kDynamic + ? PacingController::ProcessMode::kRealtime : PacingController::ProcessMode::kPeriodic), pacing_controller_(clock, static_cast(this), - event_log, field_trials, process_mode_), + event_log, + field_trials, + process_mode_), clock_(clock), packet_router_(packet_router), process_thread_(process_thread) { if (process_thread_) process_thread_->RegisterModule(&module_proxy_, RTC_FROM_HERE); + + if (low_latency_mode_) { + pacing_controller_.SetProbingEnabled(false); + } } PacedSender::~PacedSender() { @@ -56,13 +69,17 @@ PacedSender::~PacedSender() { void PacedSender::CreateProbeCluster(DataRate bitrate, int cluster_id) { rtc::CritScope cs(&critsect_); - return pacing_controller_.CreateProbeCluster(bitrate, cluster_id); + if (!low_latency_mode_) { // Do not create probe cluster on low latency mode. + return pacing_controller_.CreateProbeCluster(bitrate, cluster_id); + } } void PacedSender::Pause() { { rtc::CritScope cs(&critsect_); - pacing_controller_.Pause(); + if (!low_latency_mode_) { + pacing_controller_.Pause(); + } } // Tell the process thread to call our TimeUntilNextProcess() method to get @@ -75,7 +92,9 @@ void PacedSender::Pause() { void PacedSender::Resume() { { rtc::CritScope cs(&critsect_); - pacing_controller_.Resume(); + if (!low_latency_mode_) { + pacing_controller_.Resume(); + } } // Tell the process thread to call our TimeUntilNextProcess() method to @@ -88,7 +107,11 @@ void PacedSender::Resume() { void PacedSender::SetCongestionWindow(DataSize congestion_window_size) { { rtc::CritScope cs(&critsect_); - pacing_controller_.SetCongestionWindow(congestion_window_size); + if (low_latency_mode_) { // Ignore congestion window for realtime pacing + pacing_controller_.SetCongestionWindow(DataSize::PlusInfinity()); + } else { + pacing_controller_.SetCongestionWindow(congestion_window_size); + } } MaybeWakupProcessThread(); } @@ -157,14 +180,17 @@ TimeDelta PacedSender::OldestPacketWaitTime() const { int64_t PacedSender::TimeUntilNextProcess() { rtc::CritScope cs(&critsect_); - - Timestamp next_send_time = pacing_controller_.NextSendTime(); - TimeDelta sleep_time = - std::max(TimeDelta::Zero(), next_send_time - clock_->CurrentTime()); - if (process_mode_ == PacingController::ProcessMode::kDynamic) { - return std::max(sleep_time, PacingController::kMinSleepTime).ms(); + if (process_mode_ == PacingController::ProcessMode::kRealtime) + return PacingController::kMinSleepTime.ms(); + else { + Timestamp next_send_time = pacing_controller_.NextSendTime(); + TimeDelta sleep_time = + std::max(TimeDelta::Zero(), next_send_time - clock_->CurrentTime()); + if (process_mode_ == PacingController::ProcessMode::kDynamic) { + return std::max(sleep_time, PacingController::kMinSleepTime).ms(); + } + return sleep_time.ms(); } - return sleep_time.ms(); } void PacedSender::Process() { @@ -181,7 +207,8 @@ void PacedSender::MaybeWakupProcessThread() { // Tell the process thread to call our TimeUntilNextProcess() method to get // a new time for when to call Process(). if (process_thread_ && - process_mode_ == PacingController::ProcessMode::kDynamic) { + (process_mode_ == PacingController::ProcessMode::kDynamic|| + process_mode_ == PacingController::ProcessMode::kRealtime)) { process_thread_->WakeUp(&module_proxy_); } } @@ -196,9 +223,10 @@ void PacedSender::SetQueueTimeLimit(TimeDelta limit) { void PacedSender::SendRtpPacket(std::unique_ptr packet, const PacedPacketInfo& cluster_info) { - critsect_.Leave(); + // Since we don't generate padding for realtime mode, no lock required. + //critsect_.Leave(); packet_router_->SendPacket(std::move(packet), cluster_info); - critsect_.Enter(); + //critsect_.Enter(); } std::vector> PacedSender::GeneratePadding( diff --git a/modules/pacing/paced_sender.h b/modules/pacing/paced_sender.h index 16137dfcd69..00063da02e5 100644 --- a/modules/pacing/paced_sender.h +++ b/modules/pacing/paced_sender.h @@ -137,6 +137,8 @@ class PacedSender : public Module, // Called when the prober is associated with a process thread. void ProcessThreadAttached(ProcessThread* process_thread) override; + bool IsLowLatencyMode() const; + // In dynamic process mode, refreshes the next process time. void MaybeWakupProcessThread(); @@ -166,6 +168,7 @@ class PacedSender : public Module, PacedSender* const delegate_; } module_proxy_{this}; + const bool low_latency_mode_; rtc::CriticalSection critsect_; const PacingController::ProcessMode process_mode_; PacingController pacing_controller_ RTC_GUARDED_BY(critsect_); diff --git a/modules/pacing/pacing_controller.cc b/modules/pacing/pacing_controller.cc index f9ca408eee0..55b331b283f 100644 --- a/modules/pacing/pacing_controller.cc +++ b/modules/pacing/pacing_controller.cc @@ -28,6 +28,7 @@ namespace webrtc { namespace { // Time limit in milliseconds between packet bursts. constexpr TimeDelta kDefaultMinPacketLimit = TimeDelta::Millis(5); +constexpr TimeDelta kDefaultMinPacketLimitLowLatency = TimeDelta::Millis(1); constexpr TimeDelta kCongestedPacketInterval = TimeDelta::Millis(500); // TODO(sprang): Consider dropping this limit. // The maximum debt level, in terms of time, capped when sending packets. @@ -132,6 +133,9 @@ PacingController::PacingController(Clock* clock, ParseFieldTrial({&min_packet_limit_ms}, field_trials_->Lookup("WebRTC-Pacer-MinPacketLimitMs")); min_packet_limit_ = TimeDelta::Millis(min_packet_limit_ms.Get()); + if (mode_ == ProcessMode::kRealtime) { + min_packet_limit_ = kDefaultMinPacketLimitLowLatency; + } UpdateBudgetWithElapsedTime(min_packet_limit_); } @@ -198,6 +202,11 @@ Timestamp PacingController::CurrentTime() const { } void PacingController::SetProbingEnabled(bool enabled) { + // For low latency mode we will always disable prober. + if (mode_ == ProcessMode::kRealtime) { + prober_.SetEnabled(false); + return; + } RTC_CHECK_EQ(0, packet_counter_); prober_.SetEnabled(enabled); } @@ -324,6 +333,11 @@ bool PacingController::ShouldSendKeepalive(Timestamp now) const { Timestamp PacingController::NextSendTime() const { Timestamp now = CurrentTime(); + // Realtime pacer works at RT mode. + if (mode_ == ProcessMode::kRealtime) { + return now; + } + if (paused_) { return last_send_time_ + kPausedProcessInterval; } @@ -402,12 +416,19 @@ void PacingController::ProcessPackets() { UpdateBudgetWithElapsedTime(last_process_time_ - target_send_time); target_send_time = last_process_time_; } + } else if (mode_ == ProcessMode::kRealtime) { + target_send_time = NextSendTime(); + if (target_send_time < last_process_time_) { + UpdateBudgetWithElapsedTime(last_process_time_ - target_send_time); + target_send_time = last_process_time_; + } } Timestamp previous_process_time = last_process_time_; TimeDelta elapsed_time = UpdateTimeAndGetElapsed(now); - if (ShouldSendKeepalive(now)) { + // Disable padding for realtime mode + if (ShouldSendKeepalive(now) && mode_ != ProcessMode::kRealtime) { // We can not send padding unless a normal packet has first been sent. If // we do, timestamps get messed up. if (packet_counter_ == 0) { @@ -425,7 +446,7 @@ void PacingController::ProcessPackets() { } } - if (paused_) { + if (paused_ && mode_ != ProcessMode::kRealtime) { return; } @@ -476,7 +497,9 @@ void PacingController::ProcessPackets() { // The paused state is checked in the loop since it leaves the critical // section allowing the paused state to be changed from other code. while (!paused_) { - if (small_first_probe_packet_ && first_packet_in_probe) { + // No padding for realtime mode + if (small_first_probe_packet_ && first_packet_in_probe && + mode_ != ProcessMode::kRealtime) { // If first packet in probe, insert a small padding packet so we have a // more reliable start window for the rate estimation. auto padding = packet_sender_->GeneratePadding(DataSize::Bytes(1)); @@ -492,7 +515,7 @@ void PacingController::ProcessPackets() { first_packet_in_probe = false; } - if (mode_ == ProcessMode::kDynamic && + if ((mode_ == ProcessMode::kDynamic || mode_ == ProcessMode::kRealtime) && previous_process_time < target_send_time) { // Reduce buffer levels with amount corresponding to time between last // process and target send time for the next packet. @@ -509,19 +532,22 @@ void PacingController::ProcessPackets() { if (rtp_packet == nullptr) { // No packet available to send, check if we should send padding. - DataSize padding_to_add = PaddingToAdd(recommended_probe_size, data_sent); - if (padding_to_add > DataSize::Zero()) { - std::vector> padding_packets = - packet_sender_->GeneratePadding(padding_to_add); - if (padding_packets.empty()) { - // No padding packets were generated, quite send loop. - break; - } - for (auto& packet : padding_packets) { - EnqueuePacket(std::move(packet)); + if (mode_ != ProcessMode::kRealtime) { + DataSize padding_to_add = + PaddingToAdd(recommended_probe_size, data_sent); + if (padding_to_add > DataSize::Zero()) { + std::vector> padding_packets = + packet_sender_->GeneratePadding(padding_to_add); + if (padding_packets.empty()) { + // No padding packets were generated, quite send loop. + break; + } + for (auto& packet : padding_packets) { + EnqueuePacket(std::move(packet)); + } + // Continue loop to send the padding that was just added. + continue; } - // Continue loop to send the padding that was just added. - continue; } // Can't fetch new packet and no padding to send, exit send loop. @@ -547,7 +573,7 @@ void PacingController::ProcessPackets() { if (recommended_probe_size && data_sent > *recommended_probe_size) break; - if (mode_ == ProcessMode::kDynamic) { + if (mode_ == ProcessMode::kDynamic || mode_ == ProcessMode::kRealtime) { // Update target send time in case that are more packets that we are late // in processing. Timestamp next_send_time = NextSendTime(); @@ -559,7 +585,7 @@ void PacingController::ProcessPackets() { } } - if (is_probing) { + if (is_probing && mode_ != ProcessMode::kRealtime) { probing_send_failure_ = data_sent == DataSize::Zero(); if (!probing_send_failure_) { prober_.ProbeSent(CurrentTime(), data_sent.bytes()); @@ -615,27 +641,29 @@ std::unique_ptr PacingController::GetPendingPacket( // Unpaced audio packets and probes are exempted from send checks. bool unpaced_audio_packet = !pace_audio_ && packet_queue_.NextPacketIsAudio(); bool is_probe = pacing_info.probe_cluster_id != PacedPacketInfo::kNotAProbe; - if (!unpaced_audio_packet && !is_probe) { - if (Congested()) { - // Don't send anything if congested. - return nullptr; - } - - if (mode_ == ProcessMode::kPeriodic) { - if (media_budget_.bytes_remaining() <= 0) { - // Not enough budget. + if (mode_ != ProcessMode::kRealtime) { + if (!unpaced_audio_packet && !is_probe) { + if (Congested()) { + // Don't send anything if congested. return nullptr; } - } else { - // Dynamic processing mode. - if (now <= target_send_time) { - // We allow sending slightly early if we think that we would actually - // had been able to, had we been right on time - i.e. the current debt - // is not more than would be reduced to zero at the target sent time. - TimeDelta flush_time = media_debt_ / media_rate_; - if (now + flush_time > target_send_time) { + + if (mode_ == ProcessMode::kPeriodic) { + if (media_budget_.bytes_remaining() <= 0) { + // Not enough budget. return nullptr; } + } else { + // Dynamic processing mode. + if (now <= target_send_time) { + // We allow sending slightly early if we think that we would actually + // had been able to, had we been right on time - i.e. the current debt + // is not more than would be reduced to zero at the target sent time. + TimeDelta flush_time = media_debt_ / media_rate_; + if (now + flush_time > target_send_time) { + return nullptr; + } + } } } } diff --git a/modules/pacing/pacing_controller.h b/modules/pacing/pacing_controller.h index 4ffcbd3afcb..49add988a76 100644 --- a/modules/pacing/pacing_controller.h +++ b/modules/pacing/pacing_controller.h @@ -50,7 +50,7 @@ class PacingController { // e.g. every 5ms as implemented by PacedSender. // Dynamic mode allows for arbitrary time delta between calls to // ProcessPackets. - enum class ProcessMode { kPeriodic, kDynamic }; + enum class ProcessMode { kPeriodic, kDynamic, kRealtime}; class PacketSender { public: diff --git a/modules/rtp_rtcp/include/rtp_rtcp_defines.h b/modules/rtp_rtcp/include/rtp_rtcp_defines.h index a95ac996a25..b41db65adde 100644 --- a/modules/rtp_rtcp/include/rtp_rtcp_defines.h +++ b/modules/rtp_rtcp/include/rtp_rtcp_defines.h @@ -73,6 +73,7 @@ enum RTPExtensionType : int { kRtpExtensionGenericFrameDescriptor01, kRtpExtensionGenericFrameDescriptor02, kRtpExtensionColorSpace, + kRtpExtensionPictureId, // Required by temporal scalability support kRtpExtensionNumberOfExtensions // Must be the last entity in the enum. }; diff --git a/modules/rtp_rtcp/source/rtp_format.cc b/modules/rtp_rtcp/source/rtp_format.cc index d3a507c1854..3d614a085c2 100644 --- a/modules/rtp_rtcp/source/rtp_format.cc +++ b/modules/rtp_rtcp/source/rtp_format.cc @@ -37,7 +37,8 @@ std::unique_ptr RtpPacketizer::Create( PayloadSizeLimits limits, // Codec-specific details. const RTPVideoHeader& rtp_video_header, - const RTPFragmentationHeader* fragmentation) { + const RTPFragmentationHeader* fragmentation, + bool end_of_frame) { if (!type) { // Use raw packetizer. return std::make_unique(payload, limits); diff --git a/modules/rtp_rtcp/source/rtp_format.h b/modules/rtp_rtcp/source/rtp_format.h index dca8285b62c..ec2b950e609 100644 --- a/modules/rtp_rtcp/source/rtp_format.h +++ b/modules/rtp_rtcp/source/rtp_format.h @@ -42,7 +42,8 @@ class RtpPacketizer { PayloadSizeLimits limits, // Codec-specific details. const RTPVideoHeader& rtp_video_header, - const RTPFragmentationHeader* fragmentation); + const RTPFragmentationHeader* fragmentation, + bool end_of_frame); virtual ~RtpPacketizer() = default; diff --git a/modules/rtp_rtcp/source/rtp_format_h264.cc b/modules/rtp_rtcp/source/rtp_format_h264.cc index 6f19e386294..664a53cf3f0 100644 --- a/modules/rtp_rtcp/source/rtp_format_h264.cc +++ b/modules/rtp_rtcp/source/rtp_format_h264.cc @@ -50,12 +50,15 @@ RtpPacketizerH264::RtpPacketizerH264( rtc::ArrayView payload, PayloadSizeLimits limits, H264PacketizationMode packetization_mode, - const RTPFragmentationHeader& fragmentation) + const RTPFragmentationHeader& fragmentation, + bool end_of_frame) : limits_(limits), num_packets_left_(0) { // Guard against uninitialized memory in packetization_mode. RTC_CHECK(packetization_mode == H264PacketizationMode::NonInterleaved || packetization_mode == H264PacketizationMode::SingleNalUnit); + end_of_frame_ = end_of_frame; + for (size_t i = 0; i < fragmentation.fragmentationVectorSize; ++i) { input_fragments_.push_back( payload.subview(fragmentation.Offset(i), fragmentation.Length(i))); @@ -255,7 +258,7 @@ bool RtpPacketizerH264::NextPacket(RtpPacketToSend* rtp_packet) { } else { NextFragmentPacket(rtp_packet); } - rtp_packet->SetMarker(packets_.empty()); + rtp_packet->SetMarker(packets_.empty() && end_of_frame_); --num_packets_left_; return true; } diff --git a/modules/rtp_rtcp/source/rtp_format_h264.h b/modules/rtp_rtcp/source/rtp_format_h264.h index 4661dc21636..8c63c13faf7 100644 --- a/modules/rtp_rtcp/source/rtp_format_h264.h +++ b/modules/rtp_rtcp/source/rtp_format_h264.h @@ -35,7 +35,8 @@ class RtpPacketizerH264 : public RtpPacketizer { RtpPacketizerH264(rtc::ArrayView payload, PayloadSizeLimits limits, H264PacketizationMode packetization_mode, - const RTPFragmentationHeader& fragmentation); + const RTPFragmentationHeader& fragmentation, + bool end_of_frame = true); ~RtpPacketizerH264() override; @@ -84,6 +85,7 @@ class RtpPacketizerH264 : public RtpPacketizer { size_t num_packets_left_; std::deque> input_fragments_; std::queue packets_; + bool end_of_frame_ = true; RTC_DISALLOW_COPY_AND_ASSIGN(RtpPacketizerH264); }; diff --git a/modules/rtp_rtcp/source/rtp_format_h264_unittest.cc b/modules/rtp_rtcp/source/rtp_format_h264_unittest.cc index bf9771ab9f2..a59948937f1 100644 --- a/modules/rtp_rtcp/source/rtp_format_h264_unittest.cc +++ b/modules/rtp_rtcp/source/rtp_format_h264_unittest.cc @@ -120,7 +120,7 @@ TEST_P(RtpPacketizerH264ModeTest, SingleNalu) { const uint8_t frame[2] = {kIdr, 0xFF}; RtpPacketizerH264 packetizer(frame, kNoLimits, GetParam(), - NoFragmentation(frame)); + NoFragmentation(frame), true); std::vector packets = FetchAllPackets(&packetizer); ASSERT_THAT(packets, SizeIs(1)); diff --git a/modules/rtp_rtcp/source/rtp_header_extension_map.cc b/modules/rtp_rtcp/source/rtp_header_extension_map.cc index 06f2e928f91..c770f39e7ee 100644 --- a/modules/rtp_rtcp/source/rtp_header_extension_map.cc +++ b/modules/rtp_rtcp/source/rtp_header_extension_map.cc @@ -50,6 +50,7 @@ constexpr ExtensionInfo kExtensions[] = { CreateExtensionInfo(), CreateExtensionInfo(), CreateExtensionInfo(), + CreateExtensionInfo(), }; // Because of kRtpExtensionNone, NumberOfExtension is 1 bigger than the actual diff --git a/modules/rtp_rtcp/source/rtp_header_extensions.cc b/modules/rtp_rtcp/source/rtp_header_extensions.cc index fefe6c618f6..d973294faee 100644 --- a/modules/rtp_rtcp/source/rtp_header_extensions.cc +++ b/modules/rtp_rtcp/source/rtp_header_extensions.cc @@ -317,6 +317,22 @@ bool TransportSequenceNumberV2::Write( return true; } +constexpr RTPExtensionType PictureId::kId; +constexpr uint8_t PictureId::kValueSizeBytes; +constexpr const char PictureId::kUri[]; + +bool PictureId::Parse(rtc::ArrayView data, uint16_t* value) { + if (data.size() != 2) + return false; + *value = ByteReader::ReadBigEndian(data.data()); + return true; +} + +bool PictureId::Write(rtc::ArrayView data, uint16_t value) { + ByteWriter::WriteBigEndian(data.data(), value); + return true; +} + // Coordination of Video Orientation in RTP streams. // // Coordination of Video Orientation consists in signaling of the current diff --git a/modules/rtp_rtcp/source/rtp_header_extensions.h b/modules/rtp_rtcp/source/rtp_header_extensions.h index f4517bb513e..3fa5bfa60c1 100644 --- a/modules/rtp_rtcp/source/rtp_header_extensions.h +++ b/modules/rtp_rtcp/source/rtp_header_extensions.h @@ -132,6 +132,16 @@ class TransportSequenceNumberV2 { static constexpr uint16_t kIncludeTimestampsBit = 1 << 15; }; +class PictureId { + public: + static constexpr RTPExtensionType kId = kRtpExtensionPictureId; + static constexpr uint8_t kValueSizeBytes = 2; + static constexpr const char kUri[] = "urn:intel:picture-id"; + static bool Parse(rtc::ArrayView data, uint16_t* value); + static size_t ValueSize(uint16_t value) { return kValueSizeBytes; } + static bool Write(rtc::ArrayView data, uint16_t value); +}; + class VideoOrientation { public: using value_type = VideoRotation; diff --git a/modules/rtp_rtcp/source/rtp_rtcp_impl.cc b/modules/rtp_rtcp/source/rtp_rtcp_impl.cc index 25fa545213b..77217788fef 100644 --- a/modules/rtp_rtcp/source/rtp_rtcp_impl.cc +++ b/modules/rtp_rtcp/source/rtp_rtcp_impl.cc @@ -24,6 +24,7 @@ #include "modules/rtp_rtcp/source/rtp_rtcp_config.h" #include "rtc_base/checks.h" #include "rtc_base/logging.h" +#include "system_wrappers/include/field_trial.h" #ifdef _WIN32 // Disable warning C4355: 'this' : used in base member initializer list. @@ -81,7 +82,18 @@ ModuleRtpRtcpImpl::ModuleRtpRtcpImpl(const Configuration& configuration) // TODO(nisse): Kind-of duplicates // webrtc::VideoSendStream::Config::Rtp::kDefaultMaxPacketSize. const size_t kTcpOverIpv4HeaderSize = 40; - SetMaxRtpPacketSize(IP_PACKET_SIZE - kTcpOverIpv4HeaderSize); + std::string experiment_string = + webrtc::field_trial::FindFullName("OWT-LinkMTU"); + if (!experiment_string.empty()) { + double link_mtu = ::strtod(experiment_string.c_str(), nullptr); + if (link_mtu > 0) { + SetMaxRtpPacketSize(link_mtu - kTcpOverIpv4HeaderSize); + } else { + SetMaxRtpPacketSize(IP_PACKET_SIZE - kTcpOverIpv4HeaderSize); + } + } else { + SetMaxRtpPacketSize(IP_PACKET_SIZE - kTcpOverIpv4HeaderSize); + } } ModuleRtpRtcpImpl::~ModuleRtpRtcpImpl() = default; diff --git a/modules/rtp_rtcp/source/rtp_sender.cc b/modules/rtp_rtcp/source/rtp_sender.cc index 584f89c8ce3..6d80de48299 100644 --- a/modules/rtp_rtcp/source/rtp_sender.cc +++ b/modules/rtp_rtcp/source/rtp_sender.cc @@ -83,6 +83,7 @@ constexpr RtpExtensionSize kVideoExtensionSizes[] = { RtpGenericFrameDescriptorExtension00::kMaxSizeBytes}, {RtpGenericFrameDescriptorExtension01::kId, RtpGenericFrameDescriptorExtension01::kMaxSizeBytes}, + CreateExtensionSize(), }; bool HasBweExtension(const RtpHeaderExtensionMap& extensions_map) { diff --git a/modules/rtp_rtcp/source/rtp_sender_video.cc b/modules/rtp_rtcp/source/rtp_sender_video.cc index 0a9ab3741bb..eb3a097b77b 100644 --- a/modules/rtp_rtcp/source/rtp_sender_video.cc +++ b/modules/rtp_rtcp/source/rtp_sender_video.cc @@ -160,7 +160,8 @@ RTPSenderVideo::RTPSenderVideo(const Config& config) config.frame_transformer ? new rtc::RefCountedObject< RTPSenderVideoFrameTransformerDelegate>( - this, std::move(config.frame_transformer)) + this, + std::move(config.frame_transformer)) : nullptr) { if (frame_transformer_delegate_) frame_transformer_delegate_->Init(); @@ -408,6 +409,18 @@ void RTPSenderVideo::AddRtpHeaderExtensions( packet->SetExtension( generic_descriptor); } + if (video_header.codec == kVideoCodecH264 && last_packet) { + packet->SetExtension( + absl::get(video_header.video_type_header) + .picture_id); + } +#ifndef DISABLE_H265 + else if (video_header.codec == kVideoCodecH265 && last_packet) { + packet->SetExtension( + absl::get(video_header.video_type_header) + .picture_id); + } +#endif } } } @@ -434,11 +447,23 @@ bool RTPSenderVideo::SendVideo( return false; int32_t retransmission_settings = retransmission_settings_; + bool frame_completed = true; if (codec_type == VideoCodecType::kVideoCodecH264) { // Backward compatibility for older receivers without temporal layer logic. retransmission_settings = kRetransmitBaseLayer | kRetransmitHigherLayers; + if (!absl::get(video_header.video_type_header) + .has_last_fragement) { + frame_completed = false; + } } - +#ifndef DISABLE_H265 + else if (codec_type == VideoCodecType::kVideoCodecH265) { + if (!absl::get(video_header.video_type_header) + .has_last_fragement) { + frame_completed = false; + } + } +#endif MaybeUpdateCurrentPlayoutDelay(video_header); if (video_header.frame_type == VideoFrameType::kVideoFrameKey && !IsNoopDelay(current_playout_delay_)) { @@ -469,19 +494,21 @@ bool RTPSenderVideo::SendVideo( auto first_packet = std::make_unique(*single_packet); auto middle_packet = std::make_unique(*single_packet); auto last_packet = std::make_unique(*single_packet); - // Simplest way to estimate how much extensions would occupy is to set them. - AddRtpHeaderExtensions(video_header, absolute_capture_time, - /*first_packet=*/true, /*last_packet=*/true, - single_packet.get()); - AddRtpHeaderExtensions(video_header, absolute_capture_time, - /*first_packet=*/true, /*last_packet=*/false, - first_packet.get()); - AddRtpHeaderExtensions(video_header, absolute_capture_time, - /*first_packet=*/false, /*last_packet=*/false, - middle_packet.get()); - AddRtpHeaderExtensions(video_header, absolute_capture_time, - /*first_packet=*/false, /*last_packet=*/true, - last_packet.get()); + if (frame_completed) {// Jianlin: Not adding extension if frame not completed yet. + // Simplest way to estimate how much extensions would occupy is to set them. + AddRtpHeaderExtensions(video_header, absolute_capture_time, + /*first_packet=*/true, /*last_packet=*/true, + single_packet.get()); + AddRtpHeaderExtensions(video_header, absolute_capture_time, + /*first_packet=*/true, /*last_packet=*/false, + first_packet.get()); + AddRtpHeaderExtensions(video_header, absolute_capture_time, + /*first_packet=*/false, /*last_packet=*/false, + middle_packet.get()); + AddRtpHeaderExtensions(video_header, absolute_capture_time, + /*first_packet=*/false, /*last_packet=*/true, + last_packet.get()); + } RTC_DCHECK_GT(packet_capacity, single_packet->headers_size()); RTC_DCHECK_GT(packet_capacity, first_packet->headers_size()); @@ -558,7 +585,7 @@ bool RTPSenderVideo::SendVideo( } std::unique_ptr packetizer = RtpPacketizer::Create( - codec_type, payload, limits, video_header, fragmentation); + codec_type, payload, limits, video_header, fragmentation, frame_completed); // TODO(bugs.webrtc.org/10714): retransmission_settings_ should generally be // replaced by expected_retransmission_time_ms.has_value(). For now, though, diff --git a/modules/rtp_rtcp/source/rtp_utility.cc b/modules/rtp_rtcp/source/rtp_utility.cc index 75ee052b7c7..2692b8901bd 100644 --- a/modules/rtp_rtcp/source/rtp_utility.cc +++ b/modules/rtp_rtcp/source/rtp_utility.cc @@ -450,6 +450,16 @@ void RtpHeaderParser::ParseOneByteExtensionHeader( RTC_LOG(WARNING) << "TransportSequenceNumberV2 unsupported by rtp " "header parser."; break; + case kRtpExtensionPictureId: { + if (len != 1) { + RTC_LOG(LS_WARNING) << "Incorrect picture ID len: " << len; + return; + } + uint16_t picture_id = ptr[0] << 8; + picture_id += ptr[1]; + header->extension.picture_id = picture_id; + break; + } case kRtpExtensionPlayoutDelay: { if (len != 2) { RTC_LOG(LS_WARNING) << "Incorrect playout delay len: " << len; diff --git a/modules/rtp_rtcp/source/rtp_video_header.h b/modules/rtp_rtcp/source/rtp_video_header.h index 2441cb26865..cd420d38dab 100644 --- a/modules/rtp_rtcp/source/rtp_video_header.h +++ b/modules/rtp_rtcp/source/rtp_video_header.h @@ -84,6 +84,8 @@ struct RTPVideoHeader { bool is_last_packet_in_frame = false; uint8_t simulcastIdx = 0; VideoCodecType codec = VideoCodecType::kVideoCodecGeneric; + // TODO: should be removed as it is in codec specific header. + uint16_t picture_id; PlayoutDelay playout_delay = {-1, -1}; VideoSendTiming video_timing; diff --git a/modules/utility/include/mock/mock_process_thread.h b/modules/utility/include/mock/mock_process_thread.h index e356bca99ff..6dfa4adae97 100644 --- a/modules/utility/include/mock/mock_process_thread.h +++ b/modules/utility/include/mock/mock_process_thread.h @@ -22,6 +22,7 @@ namespace webrtc { class MockProcessThread : public ProcessThread { public: MOCK_METHOD(void, Start, (), (override)); + MOCK_METHOD0(StartWithHighPriority, void()); MOCK_METHOD(void, Stop, (), (override)); MOCK_METHOD(void, Delete, (), (override)); MOCK_METHOD(void, WakeUp, (Module*), (override)); diff --git a/modules/utility/include/process_thread.h b/modules/utility/include/process_thread.h index 7786dacf942..e0a3440c177 100644 --- a/modules/utility/include/process_thread.h +++ b/modules/utility/include/process_thread.h @@ -36,6 +36,10 @@ class ProcessThread : public TaskQueueBase { // Starts the worker thread. Must be called from the construction thread. virtual void Start() = 0; + // Starts the worker thread at high priority. Special startup mode used at + // low latency mode. Used by congestion controller only. + virtual void StartWithHighPriority() = 0; + // Stops the worker thread. Must be called from the construction thread. virtual void Stop() = 0; diff --git a/modules/utility/source/process_thread_impl.cc b/modules/utility/source/process_thread_impl.cc index 37093069258..26345f91e43 100644 --- a/modules/utility/source/process_thread_impl.cc +++ b/modules/utility/source/process_thread_impl.cc @@ -85,6 +85,23 @@ void ProcessThreadImpl::Start() { thread_->Start(); } +void ProcessThreadImpl::StartWithHighPriority() { + RTC_DCHECK(thread_checker_.CalledOnValidThread()); + RTC_DCHECK(!thread_.get()); + if (thread_.get()) + return; + + RTC_DCHECK(!stop_); + + for (ModuleCallback& m : modules_) + m.module->ProcessThreadAttached(this); + + thread_.reset( + new rtc::PlatformThread(&ProcessThreadImpl::Run, this, thread_name_)); + thread_->Start(); + thread_->SetPriority(rtc::kRealtimePriority); +} + void ProcessThreadImpl::Stop() { RTC_DCHECK(thread_checker_.IsCurrent()); if (!thread_.get()) diff --git a/modules/utility/source/process_thread_impl.h b/modules/utility/source/process_thread_impl.h index 24a72d91f5a..987e3290176 100644 --- a/modules/utility/source/process_thread_impl.h +++ b/modules/utility/source/process_thread_impl.h @@ -34,6 +34,7 @@ class ProcessThreadImpl : public ProcessThread { ~ProcessThreadImpl() override; void Start() override; + void StartWithHighPriority() override; void Stop() override; void WakeUp(Module* module) override; diff --git a/modules/video_capture/BUILD.gn b/modules/video_capture/BUILD.gn index 1c024122642..18ff9e44b52 100644 --- a/modules/video_capture/BUILD.gn +++ b/modules/video_capture/BUILD.gn @@ -102,6 +102,27 @@ if (!build_with_chromium) { } } + rtc_static_library("video_capture") { + visibility = [ "*" ] + sources = [ + "external/device_info_external.cc", + "external/video_capture_external.cc", + ] + + deps = [ + ":video_capture_module", + "../../api:scoped_refptr", + "../../rtc_base:checks", + "../../rtc_base:rtc_base_approved", + "../../system_wrappers", + ] + + if (!build_with_chromium && is_clang) { + # Suppress warnings from the Chromium Clang plugin (bugs.webrtc.org/163). + suppressed_configs += [ "//build/config/clang:find_bad_constructs" ] + } + } + if (!is_android && rtc_include_tests) { rtc_test("video_capture_tests") { sources = [ "test/video_capture_unittest.cc" ] diff --git a/modules/video_capture/external/device_info_external.cc b/modules/video_capture/external/device_info_external.cc new file mode 100644 index 00000000000..cd660e2e0ba --- /dev/null +++ b/modules/video_capture/external/device_info_external.cc @@ -0,0 +1,52 @@ +/* + * Copyright (c) 2012 The WebRTC project authors. All Rights Reserved. + * + * Use of this source code is governed by a BSD-style license + * that can be found in the LICENSE file in the root of the source + * tree. An additional intellectual property rights grant can be found + * in the file PATENTS. All contributing project authors may + * be found in the AUTHORS file in the root of the source tree. + */ + +#include "modules/video_capture/device_info_impl.h" +#include "modules/video_capture/video_capture_impl.h" + +namespace webrtc { + +namespace videocapturemodule { + +class ExternalDeviceInfo : public DeviceInfoImpl { + public: + ExternalDeviceInfo() {} + virtual ~ExternalDeviceInfo() {} + virtual uint32_t NumberOfDevices() { return 0; } + virtual int32_t DisplayCaptureSettingsDialogBox( + const char* /*deviceUniqueIdUTF8*/, + const char* /*dialogTitleUTF8*/, + void* /*parentWindow*/, + uint32_t /*positionX*/, + uint32_t /*positionY*/) { + return -1; + } + virtual int32_t GetDeviceName(uint32_t deviceNumber, + char* deviceNameUTF8, + uint32_t deviceNameLength, + char* deviceUniqueIdUTF8, + uint32_t deviceUniqueIdUTF8Length, + char* productUniqueIdUTF8 = 0, + uint32_t productUniqueIdUTF8Length = 0) { + return -1; + } + virtual int32_t CreateCapabilityMap(const char* deviceUniqueIdUTF8) { + return 0; + } + virtual int32_t Init() { return 0; } +}; + +VideoCaptureModule::DeviceInfo* VideoCaptureImpl::CreateDeviceInfo() { + return new ExternalDeviceInfo(); +} + +} // namespace videocapturemodule + +} // namespace webrtc diff --git a/modules/video_capture/external/video_capture_external.cc b/modules/video_capture/external/video_capture_external.cc new file mode 100644 index 00000000000..b652a675746 --- /dev/null +++ b/modules/video_capture/external/video_capture_external.cc @@ -0,0 +1,26 @@ +/* + * Copyright (c) 2012 The WebRTC project authors. All Rights Reserved. + * + * Use of this source code is governed by a BSD-style license + * that can be found in the LICENSE file in the root of the source + * tree. An additional intellectual property rights grant can be found + * in the file PATENTS. All contributing project authors may + * be found in the AUTHORS file in the root of the source tree. + */ + +#include "modules/video_capture/video_capture_impl.h" +#include "rtc_base/ref_count.h" +#include "rtc_base/ref_counted_object.h" + +namespace webrtc { + +namespace videocapturemodule { + +rtc::scoped_refptr VideoCaptureImpl::Create( + const char* deviceUniqueIdUTF8) { + return new rtc::RefCountedObject(); +} + +} // namespace videocapturemodule + +} // namespace webrtc diff --git a/modules/video_coding/codecs/h264/include/h264_globals.h b/modules/video_coding/codecs/h264/include/h264_globals.h index 073d8f9a81e..f4d73bcd588 100644 --- a/modules/video_coding/codecs/h264/include/h264_globals.h +++ b/modules/video_coding/codecs/h264/include/h264_globals.h @@ -78,6 +78,17 @@ struct RTPVideoHeaderH264 { // The packetization mode of this transport. Packetization mode // determines which packetization types are allowed when packetizing. H264PacketizationMode packetization_mode; + // Running cuter for every frame to determin frame decodable + // depending along with Temporal ID (obtained from RTP header extn). + // '0' if PictureID does not exist. + uint16_t picture_id; + // For support slice-based transmission, mark end of a frame so that + // the H.264 packetizer will not set marker bit for the last fragment of + // current outgoing data if it does not contain last fragment of the frame; + // and will treat the first fragment of the frame as continuous playload, so + // that it will not create FU header or STAP-A header on first fragment if + // contains last fragment of the frame. + bool has_last_fragement; }; } // namespace webrtc diff --git a/modules/video_coding/codecs/h265/include/h265_globals.h b/modules/video_coding/codecs/h265/include/h265_globals.h index bc0eef236cf..91b71cac8e9 100644 --- a/modules/video_coding/codecs/h265/include/h265_globals.h +++ b/modules/video_coding/codecs/h265/include/h265_globals.h @@ -53,6 +53,17 @@ struct RTPVideoHeaderH265 { size_t nalus_length; // The packetization type of this buffer - single, aggregated or fragmented. H265PacketizationMode packetization_mode; + // Running cuter for every frame to determin frame decodable + // depending along with Temporal ID (obtained from RTP header extn). + // '0' if PictureID does not exist. + uint16_t picture_id; + // For support slice-based transmission, mark end of a frame so that + // the H.265 packetizer will not set marker bit for the last fragment of + // current outgoing data if it does not contain last fragment of the frame; + // and will treat the first fragment of the frame as continuous playload, so + // that it will not create FU header or STAP-A header on first fragment if + // contains last fragment of the frame. + bool has_last_fragement; }; } // namespace webrtc diff --git a/modules/video_coding/include/video_codec_interface.h b/modules/video_coding/include/video_codec_interface.h index fbf6cefbd65..6cc9c6a613a 100644 --- a/modules/video_coding/include/video_codec_interface.h +++ b/modules/video_coding/include/video_codec_interface.h @@ -92,12 +92,16 @@ struct CodecSpecificInfoH264 { uint8_t temporal_idx; bool base_layer_sync; bool idr_frame; + int16_t picture_id; // Required by temporal scalability + bool last_fragment_in_frame; }; #ifndef DISABLE_H265 struct CodecSpecificInfoH265 { H265PacketizationMode packetization_mode; bool idr_frame; + bool last_fragment_in_frame; + int16_t picture_id; }; #endif diff --git a/modules/video_coding/timing.cc b/modules/video_coding/timing.cc index c62c848c09a..1c1e69f830a 100644 --- a/modules/video_coding/timing.cc +++ b/modules/video_coding/timing.cc @@ -16,6 +16,7 @@ #include "rtc_base/time/timestamp_extrapolator.h" #include "system_wrappers/include/clock.h" +#include "system_wrappers/include/field_trial.h" namespace webrtc { @@ -172,6 +173,9 @@ void VCMTiming::IncomingTimestamp(uint32_t time_stamp, int64_t now_ms) { int64_t VCMTiming::RenderTimeMs(uint32_t frame_timestamp, int64_t now_ms) const { rtc::CritScope cs(&crit_sect_); + if (field_trial::IsEnabled("OWT-LowLatencyMode")) { + return 0; + } return RenderTimeMsInternal(frame_timestamp, now_ms); } diff --git a/pc/used_ids.h b/pc/used_ids.h index 78e64caa413..ddcbc8f45b0 100644 --- a/pc/used_ids.h +++ b/pc/used_ids.h @@ -154,8 +154,8 @@ class UsedRtpHeaderExtensionIds : public UsedIds { } } } - RTC_DCHECK(next_extension_id_ >= min_allowed_id_); - RTC_DCHECK(next_extension_id_ <= max_allowed_id_); + //RTC_DCHECK(next_extension_id_ >= min_allowed_id_); + //RTC_DCHECK(next_extension_id_ <= max_allowed_id_); return next_extension_id_; } diff --git a/rtc_base/platform_thread.h b/rtc_base/platform_thread.h index 4968de9ee5b..370cefc8449 100644 --- a/rtc_base/platform_thread.h +++ b/rtc_base/platform_thread.h @@ -68,6 +68,7 @@ class PlatformThread { // Stops (joins) the spawned thread. void Stop(); + bool SetPriority(ThreadPriority priority); protected: #if defined(WEBRTC_WIN) // Exposed to derived classes to allow for special cases specific to Windows. @@ -76,7 +77,6 @@ class PlatformThread { private: void Run(); - bool SetPriority(ThreadPriority priority); ThreadRunFunction const run_function_ = nullptr; const ThreadPriority priority_ = kNormalPriority; diff --git a/rtc_tools/rtc_event_log_visualizer/analyzer.cc b/rtc_tools/rtc_event_log_visualizer/analyzer.cc index 9fcb510adcb..22deb17ed1e 100644 --- a/rtc_tools/rtc_event_log_visualizer/analyzer.cc +++ b/rtc_tools/rtc_event_log_visualizer/analyzer.cc @@ -1405,7 +1405,7 @@ void EventLogAnalyzer::CreateSendSideBweSimulationGraph(Plot* plot) { Timestamp::Millis(clock.TimeInMilliseconds())); absl::optional bitrate_bps; if (feedback_msg) { - observer.Update(goog_cc->OnTransportPacketsFeedback(*feedback_msg)); + observer.Update(goog_cc->OnTransportPacketsFeedback(*feedback_msg), 0); std::vector feedback = feedback_msg->SortedByReceiveTime(); if (!feedback.empty()) { diff --git a/rtc_tools/rtc_event_log_visualizer/log_simulation.cc b/rtc_tools/rtc_event_log_visualizer/log_simulation.cc index 0e5b5d04a9f..8fd4e0e80ac 100644 --- a/rtc_tools/rtc_event_log_visualizer/log_simulation.cc +++ b/rtc_tools/rtc_event_log_visualizer/log_simulation.cc @@ -111,7 +111,7 @@ void LogBasedNetworkControllerSimulation::OnFeedback( auto msg = transport_feedback_.ProcessTransportFeedback( feedback.transport_feedback, feedback_time); if (msg) - HandleStateUpdate(controller_->OnTransportPacketsFeedback(*msg)); + HandleStateUpdate(controller_->OnTransportPacketsFeedback(*msg, 0)); } void LogBasedNetworkControllerSimulation::OnReceiverReport( diff --git a/test/scenario/call_client.cc b/test/scenario/call_client.cc index fb888df6946..595f6d26aac 100644 --- a/test/scenario/call_client.cc +++ b/test/scenario/call_client.cc @@ -128,8 +128,8 @@ NetworkControlUpdate NetworkControleUpdateCache::OnTransportLossReport( return Update(controller_->OnTransportLossReport(msg)); } NetworkControlUpdate NetworkControleUpdateCache::OnTransportPacketsFeedback( - TransportPacketsFeedback msg) { - return Update(controller_->OnTransportPacketsFeedback(msg)); + TransportPacketsFeedback msg, int64_t current_time_ms) { + return Update(controller_->OnTransportPacketsFeedback(msg, current_time_ms)); } NetworkControlUpdate NetworkControleUpdateCache::OnNetworkStateEstimate( NetworkStateEstimate msg) { diff --git a/test/scenario/call_client.h b/test/scenario/call_client.h index 803b4a83138..3a5165056da 100644 --- a/test/scenario/call_client.h +++ b/test/scenario/call_client.h @@ -51,7 +51,7 @@ class NetworkControleUpdateCache : public NetworkControllerInterface { TargetRateConstraints msg) override; NetworkControlUpdate OnTransportLossReport(TransportLossReport msg) override; NetworkControlUpdate OnTransportPacketsFeedback( - TransportPacketsFeedback msg) override; + TransportPacketsFeedback msg, int64_t current_offset_ms) override; NetworkControlUpdate OnNetworkStateEstimate( NetworkStateEstimate msg) override; diff --git a/test/time_controller/external_time_controller.cc b/test/time_controller/external_time_controller.cc index ca8b5ac687b..7f268227c00 100644 --- a/test/time_controller/external_time_controller.cc +++ b/test/time_controller/external_time_controller.cc @@ -43,6 +43,12 @@ class ExternalTimeController::ProcessThreadWrapper : public ProcessThread { parent_->ScheduleNext(); } + void StartWithHighPriority() override { + parent_->UpdateTime(); + thread_->StartWithHighPriority(); + parent_->ScheduleNext(); + } + void Stop() override { parent_->UpdateTime(); thread_->Stop(); diff --git a/test/time_controller/simulated_process_thread.cc b/test/time_controller/simulated_process_thread.cc index df90f54ed6b..620c1a1d534 100644 --- a/test/time_controller/simulated_process_thread.cc +++ b/test/time_controller/simulated_process_thread.cc @@ -104,6 +104,33 @@ void SimulatedProcessThread::Start() { } } +void SimulatedProcessThread::StartWithHighPriority() { + // same implementation as Start(); + std::vector starting; + { + rtc::CritScope lock(&lock_); + if (process_thread_running_) + return; + process_thread_running_ = true; + starting.swap(stopped_modules_); + } + for (auto& module : starting) + module->ProcessThreadAttached(this); + + Timestamp at_time = handler_->CurrentTime(); + rtc::CritScope lock(&lock_); + for (auto& module : starting) + delayed_modules_[GetNextTime(module, at_time)].push_back(module); + + if (!queue_.empty()) { + next_run_time_ = Timestamp::MinusInfinity(); + } else if (!delayed_modules_.empty()) { + next_run_time_ = delayed_modules_.begin()->first; + } else { + next_run_time_ = Timestamp::PlusInfinity(); + } +} + void SimulatedProcessThread::Stop() { std::vector stopping; { diff --git a/test/time_controller/simulated_process_thread.h b/test/time_controller/simulated_process_thread.h index 6026826d2f5..63e3cc496f6 100644 --- a/test/time_controller/simulated_process_thread.h +++ b/test/time_controller/simulated_process_thread.h @@ -37,6 +37,7 @@ class SimulatedProcessThread : public ProcessThread, // ProcessThread interface void Start() override; + void StartWithHighPriority() override; void Stop() override; void WakeUp(Module* module) override; void RegisterModule(Module* module, const rtc::Location& from) override; diff --git a/video/video_stream_encoder.cc b/video/video_stream_encoder.cc index d079d613c6a..c1dfc616d16 100644 --- a/video/video_stream_encoder.cc +++ b/video/video_stream_encoder.cc @@ -752,7 +752,10 @@ void VideoStreamEncoder::OnFrame(const VideoFrame& video_frame) { incoming_frame.set_timestamp_us(current_time_us); // Capture time may come from clock with an offset and drift from clock_. + // Jianlin: in case of slice-based encoding, the capturer will not set the + // ntp_time_ms and render_tim_ms. int64_t capture_ntp_time_ms; +#if 0 if (video_frame.ntp_time_ms() > 0) { capture_ntp_time_ms = video_frame.ntp_time_ms(); } else if (video_frame.render_time_ms() != 0) { @@ -760,6 +763,8 @@ void VideoStreamEncoder::OnFrame(const VideoFrame& video_frame) { } else { capture_ntp_time_ms = current_time_ms + delta_ntp_internal_ms_; } +#endif + capture_ntp_time_ms = current_time_ms + delta_ntp_internal_ms_; incoming_frame.set_ntp_time_ms(capture_ntp_time_ms); // Convert NTP time, in ms, to RTP timestamp. @@ -769,6 +774,11 @@ void VideoStreamEncoder::OnFrame(const VideoFrame& video_frame) { if (incoming_frame.ntp_time_ms() <= last_captured_timestamp_) { // We don't allow the same capture time for two frames, drop this one. + // Jianlin(implementation deviation: For push-mode, dropping frame due + // to same timestamp is not allowed. + // So make sure dropping is enabled for pull mode while disabled for + // push mode. +#if 0 RTC_LOG(LS_WARNING) << "Same/old NTP timestamp (" << incoming_frame.ntp_time_ms() << " <= " << last_captured_timestamp_ @@ -779,6 +789,7 @@ void VideoStreamEncoder::OnFrame(const VideoFrame& video_frame) { accumulated_update_rect_is_valid_ &= incoming_frame.has_update_rect(); }); return; +#endif } bool log_stats = false; @@ -805,7 +816,8 @@ void VideoStreamEncoder::OnFrame(const VideoFrame& video_frame) { bool cwnd_frame_drop = cwnd_frame_drop_interval_ && (cwnd_frame_counter_++ % cwnd_frame_drop_interval_.value() == 0); - if (posted_frames_waiting_for_encode == 1 && !cwnd_frame_drop) { + if (field_trial::IsEnabled("OWT-LowLatencyMode") || + (posted_frames_waiting_for_encode == 1 && !cwnd_frame_drop)) { MaybeEncodeVideoFrame(incoming_frame, post_time_us); } else { if (cwnd_frame_drop) { @@ -1083,7 +1095,9 @@ void VideoStreamEncoder::MaybeEncodeVideoFrame(const VideoFrame& video_frame, !force_disable_frame_dropper_ && !encoder_info_.has_trusted_rate_controller; frame_dropper_.Enable(frame_dropping_enabled); - if (frame_dropping_enabled && frame_dropper_.DropFrame()) { + // For low latency mode we don't drop frame + if (frame_dropping_enabled && frame_dropper_.DropFrame() && + !field_trial::IsEnabled("OWT-LowLatencyMode")) { RTC_LOG(LS_VERBOSE) << "Drop Frame: " "target bitrate "