From 30d364e0b5f65440e157b1adc589355432a54dd3 Mon Sep 17 00:00:00 2001 From: Jianjun Zhu Date: Tue, 13 Dec 2022 10:08:51 +0800 Subject: [PATCH] Fix build issues after merging changes for low latency mode. --- BUILD.gn | 8 +- api/audio_codecs/opus/audio_decoder_opus.cc | 2 +- api/stats/rtc_stats_report.h | 5 +- api/transport/bitrate_settings.h | 2 +- api/video_codecs/video_codec.cc | 16 - api/video_codecs/video_codec.h | 19 - call/call_config.h | 1 - call/rtp_payload_params.cc | 21 +- call/rtp_transport_controller_send.cc | 6 - call/rtp_video_sender.cc | 10 +- media/BUILD.gn | 2 +- media/base/media_constants.cc | 3 - media/base/media_constants.h | 2 - media/engine/encoder_simulcast_proxy.h | 2 - media/engine/webrtc_video_engine.cc | 19 +- media/engine/webrtc_voice_engine.cc | 3 +- .../codecs/opus/audio_encoder_opus.cc | 2 +- .../audio_device/win/audio_device_core_win.cc | 2 - .../aec3/clockdrift_detector.h | 1 - .../audio_processing/audio_processing_impl.cc | 3 +- modules/congestion_controller/BUILD.gn | 10 - .../congestion_controller/goog_cc/BUILD.gn | 12 - .../goog_cc/goog_cc_network_control.cc | 22 +- .../goog_cc/goog_cc_network_control.h | 7 - .../goog_cc/send_side_bandwidth_estimation.cc | 28 +- modules/pacing/pacing_controller.cc | 23 +- modules/rtp_rtcp/BUILD.gn | 13 - modules/rtp_rtcp/include/rtp_rtcp_defines.h | 2 +- .../source/create_video_rtp_depacketizer.cc | 5 +- modules/rtp_rtcp/source/rtp_format.cc | 5 +- modules/rtp_rtcp/source/rtp_format_h264.cc | 9 +- modules/rtp_rtcp/source/rtp_format_h264.h | 1 - .../source/rtp_header_extension_map.cc | 3 +- .../rtp_rtcp/source/rtp_header_extensions.h | 3 + modules/rtp_rtcp/source/rtp_sender.cc | 2 + modules/rtp_rtcp/source/rtp_sender_video.cc | 25 +- modules/rtp_rtcp/source/rtp_video_header.h | 10 +- modules/rtp_rtcp/source/ulpfec_generator.cc | 5 - .../include/mock/mock_process_thread.h | 42 - modules/utility/include/process_thread.h | 64 -- modules/utility/source/process_thread_impl.cc | 280 ------ modules/utility/source/process_thread_impl.h | 112 --- modules/video_capture/BUILD.gn | 16 - .../external/device_info_external.cc | 52 -- .../external/video_capture_external.cc | 26 - modules/video_coding/BUILD.gn | 10 +- modules/video_coding/codecs/h264/h264.cc | 2 +- modules/video_coding/encoded_frame.cc | 9 +- modules/video_coding/frame_object.cc | 5 +- modules/video_coding/frame_object.h | 4 +- .../include/video_codec_interface.h | 5 - modules/video_coding/packet_buffer.cc | 3 +- modules/video_coding/timing.cc | 253 ------ p2p/base/port_allocator.h | 1 - pc/peer_connection.cc | 2 +- pc/used_ids.h | 4 +- .../min_video_bitrate_experiment.cc | 2 - rtc_base/ptp_clock_sync.h | 58 ++ sdk/BUILD.gn | 1 - .../src/jni/android_media_codec_common.h | 90 -- test/scenario/video_stream.cc | 16 - .../simulated_process_thread.cc | 215 ----- .../simulated_process_thread.h | 73 -- video/config/video_encoder_config.cc | 11 - video/rtp_video_stream_receiver.h | 379 --------- video/rtp_video_stream_receiver2.cc | 10 +- video/rtp_video_stream_receiver2.h | 16 +- video/rtp_video_stream_receiver2_unittest.cc | 4 +- video/video_receive_stream.cc | 801 ------------------ video/video_stream_encoder.cc | 11 +- 70 files changed, 151 insertions(+), 2740 deletions(-) delete mode 100644 modules/utility/include/mock/mock_process_thread.h delete mode 100644 modules/utility/include/process_thread.h delete mode 100644 modules/utility/source/process_thread_impl.cc delete mode 100644 modules/utility/source/process_thread_impl.h delete mode 100644 modules/video_capture/external/device_info_external.cc delete mode 100644 modules/video_capture/external/video_capture_external.cc delete mode 100644 modules/video_coding/timing.cc create mode 100644 rtc_base/ptp_clock_sync.h delete mode 100644 sdk/android/src/jni/android_media_codec_common.h delete mode 100644 test/time_controller/simulated_process_thread.cc delete mode 100644 test/time_controller/simulated_process_thread.h delete mode 100644 video/rtp_video_stream_receiver.h delete mode 100644 video/video_receive_stream.cc diff --git a/BUILD.gn b/BUILD.gn index a6abaea23b8..56342d4353a 100644 --- a/BUILD.gn +++ b/BUILD.gn @@ -62,7 +62,7 @@ if (!build_with_chromium) { "modules/audio_processing:audio_processing_tests", "modules/remote_bitrate_estimator:rtp_to_text", "modules/rtp_rtcp:test_packet_masks_metrics", - "modules/video_capture:video_capture", + "modules/video_capture:video_capture_internal_impl", "net/dcsctp:dcsctp_unittests", "pc:peerconnection_unittests", "pc:rtc_pc_unittests", @@ -293,10 +293,6 @@ config("common_config") { defines += [ "WEBRTC_ENABLE_PROTOBUF=0" ] } - if (owt_use_gpra) { - defines += [ "INTEL_GPRA" ] - } - if (rtc_include_internal_audio_device) { defines += [ "WEBRTC_INCLUDE_INTERNAL_AUDIO_DEVICE" ] } @@ -517,7 +513,7 @@ if (!build_with_chromium) { "logging:rtc_event_log_api", "media", "modules", - "modules/video_capture:video_capture", + "modules/video_capture:video_capture_internal_impl", "p2p:rtc_p2p", "pc:libjingle_peerconnection", "pc:rtc_pc", diff --git a/api/audio_codecs/opus/audio_decoder_opus.cc b/api/audio_codecs/opus/audio_decoder_opus.cc index e1e4463df7c..efc9a73546e 100644 --- a/api/audio_codecs/opus/audio_decoder_opus.cc +++ b/api/audio_codecs/opus/audio_decoder_opus.cc @@ -44,7 +44,7 @@ absl::optional AudioDecoderOpus::SdpToConfig( return absl::nullopt; // Bad stereo parameter. } } - return 2; // Default to mono. + return 1; // Default to mono. }(); if (absl::EqualsIgnoreCase(format.name, "opus") && format.clockrate_hz == 48000 && format.num_channels == 2 && diff --git a/api/stats/rtc_stats_report.h b/api/stats/rtc_stats_report.h index a78f6f3bd23..f84272cb81f 100644 --- a/api/stats/rtc_stats_report.h +++ b/api/stats/rtc_stats_report.h @@ -23,7 +23,6 @@ #include "api/ref_counted_base.h" #include "api/scoped_refptr.h" #include "api/stats/rtc_stats.h" -#include "rtc_base/constructor_magic.h" // TODO(tommi): Remove this include after fixing iwyu issue in chromium. // See: third_party/blink/renderer/platform/peerconnection/rtc_stats.cc #include "rtc_base/ref_counted_object.h" @@ -58,7 +57,6 @@ class RTC_EXPORT RTCStatsReport final // Reference report to make sure it is kept alive. rtc::scoped_refptr report_; StatsMap::const_iterator it_; - RTC_DISALLOW_COPY_AND_ASSIGN(ConstIterator); }; // TODO(hbos): Remove "= 0" once Chromium unittest has been updated to call @@ -66,7 +64,7 @@ class RTC_EXPORT RTCStatsReport final static rtc::scoped_refptr Create(int64_t timestamp_us = 0); explicit RTCStatsReport(int64_t timestamp_us); - //RTCStatsReport(const RTCStatsReport& other) = delete; + RTCStatsReport(const RTCStatsReport& other) = delete; rtc::scoped_refptr Copy() const; int64_t timestamp_us() const { return timestamp_us_; } @@ -132,7 +130,6 @@ class RTC_EXPORT RTCStatsReport final private: int64_t timestamp_us_; StatsMap stats_; - RTC_DISALLOW_COPY_AND_ASSIGN(RTCStatsReport); }; } // namespace webrtc diff --git a/api/transport/bitrate_settings.h b/api/transport/bitrate_settings.h index 233f2f5409d..562309a46ce 100644 --- a/api/transport/bitrate_settings.h +++ b/api/transport/bitrate_settings.h @@ -42,7 +42,7 @@ struct BitrateConstraints { int max_bitrate_bps = -1; private: - static constexpr int kDefaultStartBitrateBps = 5000000; // Update from 300k to 5MB for low latency mode + static constexpr int kDefaultStartBitrateBps = 300000; }; } // namespace webrtc diff --git a/api/video_codecs/video_codec.cc b/api/video_codecs/video_codec.cc index e93ec7c18a9..65783689b2a 100644 --- a/api/video_codecs/video_codec.cc +++ b/api/video_codecs/video_codec.cc @@ -53,7 +53,6 @@ bool VideoCodecH264::operator==(const VideoCodecH264& other) const { numberOfTemporalLayers == other.numberOfTemporalLayers); } -#ifndef DISABLE_H265 bool VideoCodecH265::operator==(const VideoCodecH265& other) const { return (frameDroppingOn == other.frameDroppingOn && keyFrameInterval == other.keyFrameInterval && @@ -62,18 +61,6 @@ bool VideoCodecH265::operator==(const VideoCodecH265& other) const { (spsLen == 0 || memcmp(spsData, other.spsData, spsLen) == 0) && (ppsLen == 0 || memcmp(ppsData, other.ppsData, ppsLen) == 0)); } -#endif - -#ifdef WEBRTC_USE_H265 -bool VideoCodecH265::operator==(const VideoCodecH265& other) const { - return (frameDroppingOn == other.frameDroppingOn && - keyFrameInterval == other.keyFrameInterval && - vpsLen == other.vpsLen && spsLen == other.spsLen && - ppsLen == other.ppsLen && - (spsLen == 0 || memcmp(spsData, other.spsData, spsLen) == 0) && - (ppsLen == 0 || memcmp(ppsData, other.ppsData, ppsLen) == 0)); -} -#endif VideoCodec::VideoCodec() : codecType(kVideoCodecGeneric), @@ -134,7 +121,6 @@ const VideoCodecH265& VideoCodec::H265() const { RTC_DCHECK_EQ(codecType, kVideoCodecH265); return codec_specific_.H265; } -#endif const char* CodecTypeToPayloadString(VideoCodecType type) { switch (type) { @@ -148,7 +134,6 @@ const char* CodecTypeToPayloadString(VideoCodecType type) { return kPayloadNameH264; case kVideoCodecH265: return kPayloadNameH265; -#endif case kVideoCodecMultiplex: return kPayloadNameMultiplex; case kVideoCodecGeneric: @@ -172,7 +157,6 @@ VideoCodecType PayloadStringToCodecType(const std::string& name) { return kVideoCodecMultiplex; if (absl::EqualsIgnoreCase(name, kPayloadNameH265)) return kVideoCodecH265; -#endif return kVideoCodecGeneric; } diff --git a/api/video_codecs/video_codec.h b/api/video_codecs/video_codec.h index 080425a310a..074e0fecdbb 100644 --- a/api/video_codecs/video_codec.h +++ b/api/video_codecs/video_codec.h @@ -112,23 +112,6 @@ struct VideoCodecH265 { size_t ppsLen; }; -#ifndef DISABLE_H265 -struct VideoCodecH265 { - bool operator==(const VideoCodecH265& other) const; - bool operator!=(const VideoCodecH265& other) const { - return !(*this == other); - } - bool frameDroppingOn; - int keyFrameInterval; - const uint8_t* vpsData; - size_t vpsLen; - const uint8_t* spsData; - size_t spsLen; - const uint8_t* ppsData; - size_t ppsLen; -}; -#endif - // Translates from name of codec to codec type and vice versa. RTC_EXPORT const char* CodecTypeToPayloadString(VideoCodecType type); RTC_EXPORT VideoCodecType PayloadStringToCodecType(const std::string& name); @@ -137,9 +120,7 @@ union VideoCodecUnion { VideoCodecVP8 VP8; VideoCodecVP9 VP9; VideoCodecH264 H264; -#ifndef DISABLE_H265 VideoCodecH265 H265; -#endif }; enum class VideoCodecMode { kRealtimeVideo, kScreensharing }; diff --git a/call/call_config.h b/call/call_config.h index 58f1570e284..3072fa452f0 100644 --- a/call/call_config.h +++ b/call/call_config.h @@ -38,7 +38,6 @@ struct CallConfig { RtpTransportConfig ExtractTransportConfig() const; ~CallConfig(); - RTC_DEPRECATED static constexpr int kDefaultStartBitrateBps = 300000; // Bitrate config used until valid bitrate estimates are calculated. Also // used to cap total bitrate used. This comes from the remote connection. BitrateConstraints bitrate_config; diff --git a/call/rtp_payload_params.cc b/call/rtp_payload_params.cc index d928da3651c..8f18703b909 100644 --- a/call/rtp_payload_params.cc +++ b/call/rtp_payload_params.cc @@ -19,9 +19,7 @@ #include "absl/types/variant.h" #include "api/video/video_timing.h" #include "modules/video_coding/codecs/h264/include/h264_globals.h" -#ifndef DISABLE_H265 #include "modules/video_coding/codecs/h265/include/h265_globals.h" -#endif #include "modules/video_coding/codecs/interface/common_constants.h" #include "modules/video_coding/codecs/vp8/include/vp8_globals.h" #include "modules/video_coding/codecs/vp9/include/vp9_globals.h" @@ -317,16 +315,12 @@ void RtpPayloadParams::SetGeneric(const CodecSpecificInfo* codec_specific_info, int64_t frame_id, bool is_keyframe, RTPVideoHeader* rtp_video_header) { - -#ifdef WEBRTC_USE_H265 if (codec_specific_info->codecType == webrtc::kVideoCodecH265 && codec_specific_info->codecSpecific.H265.picture_id > 0) { // H265ToGeneric implementation. Only set it when picture id is valid. rtp_video_header->generic->frame_id = codec_specific_info->codecSpecific.H265.picture_id; - rtp_video_header->generic->discardable = - codec_specific_info->codecSpecific.H265.discardable; - rtp_video_header->generic->spatial_index = 0; // Not enabled at present. + rtp_video_header->generic->spatial_index = 0; // Not enabled at present. rtp_video_header->generic->temporal_index = 0; // Not enabled at present. for (int dep_idx = 0; dep_idx < 5; dep_idx++) { if (codec_specific_info->codecSpecific.H265.dependencies[dep_idx] <= 0) @@ -334,15 +328,12 @@ void RtpPayloadParams::SetGeneric(const CodecSpecificInfo* codec_specific_info, rtp_video_header->generic->dependencies[dep_idx] = codec_specific_info->codecSpecific.H265.dependencies[dep_idx]; } - // Not filling DTIs at present. - rtp_video_header->generic->discardable = - codec_specific_info->codecSpecific.H265.discardable; return; } -#else - if (codec_specific_info && codec_specific_info->generic_frame_info && + if (codec_specific_info && codec_specific_info->generic_frame_info && !codec_specific_info->generic_frame_info->encoder_buffers.empty()) { - // If generic frame info is provided for other codecs, use generic frame info. + // If generic frame info is provided for other codecs, use generic frame + // info. if (is_keyframe) { // Key frame resets all chains it is in. chains_calculator_.Reset( @@ -352,8 +343,6 @@ void RtpPayloadParams::SetGeneric(const CodecSpecificInfo* codec_specific_info, *codec_specific_info->generic_frame_info, frame_id); return; } -#endif - switch (rtp_video_header->codec) { case VideoCodecType::kVideoCodecGeneric: @@ -380,10 +369,8 @@ void RtpPayloadParams::SetGeneric(const CodecSpecificInfo* codec_specific_info, is_keyframe, rtp_video_header); } return; -#ifndef DISABLE_H265 // No further special handling for H.265 case VideoCodecType::kVideoCodecH265: -#endif case VideoCodecType::kVideoCodecMultiplex: return; } diff --git a/call/rtp_transport_controller_send.cc b/call/rtp_transport_controller_send.cc index 9d04a667bdb..24c28e8c259 100644 --- a/call/rtp_transport_controller_send.cc +++ b/call/rtp_transport_controller_send.cc @@ -36,7 +36,6 @@ static const int64_t kRetransmitWindowSizeMs = 500; static const size_t kMaxOverheadBytes = 500; constexpr TimeDelta kPacerQueueUpdateInterval = TimeDelta::Millis(25); -const char kLowLatencyStreaming[] = "OWT-LowLatencyMode"; TargetRateConstraints ConvertConstraints(int min_bitrate_bps, int max_bitrate_bps, @@ -55,11 +54,6 @@ TargetRateConstraints ConvertConstraints(int min_bitrate_bps, return msg; } -bool LowLatencyStreamingEnabled() { - std::string trial = webrtc::field_trial::FindFullName(kLowLatencyStreaming); - return trial.find("Enabled") == 0; -} - TargetRateConstraints ConvertConstraints(const BitrateConstraints& contraints, Clock* clock) { return ConvertConstraints(contraints.min_bitrate_bps, diff --git a/call/rtp_video_sender.cc b/call/rtp_video_sender.cc index 285ca20d3c4..805f5ec7813 100644 --- a/call/rtp_video_sender.cc +++ b/call/rtp_video_sender.cc @@ -613,17 +613,13 @@ EncodedImageCallback::Result RtpVideoSender::OnEncodedImage( codec_specific_info->codecSpecific.H264.last_fragment_in_frame) absl::get(rtp_video_header.video_type_header) .has_last_fragement = true; -#ifndef DISABLE_H265 - else if (codec_specific_info->codecType == - kVideoCodecH265 && codec_specific_info->codecSpecific.H265 - .last_fragment_in_frame) + else if (codec_specific_info->codecType == kVideoCodecH265 && + codec_specific_info->codecSpecific.H265.last_fragment_in_frame) absl::get(rtp_video_header.video_type_header) .has_last_fragement = true; -#endif bool send_result = rtp_streams_[stream_index].sender_video->SendEncodedImage( rtp_config_.payload_type, codec_type_, rtp_timestamp, encoded_image, - rtp_video_header, - expected_retransmission_time_ms); + rtp_video_header, expected_retransmission_time_ms); if (frame_count_observer_) { FrameCounts& counts = frame_counts_[stream_index]; if (encoded_image._frameType == VideoFrameType::kVideoFrameKey) { diff --git a/media/BUILD.gn b/media/BUILD.gn index c258e29f963..d4afad91f6f 100644 --- a/media/BUILD.gn +++ b/media/BUILD.gn @@ -357,7 +357,7 @@ rtc_library("rtc_audio_video") { public_configs = [] if (!build_with_chromium) { public_configs += [ ":rtc_media_defines_config" ] - deps += [ "../modules/video_capture:video_capture" ] + deps += [ "../modules/video_capture:video_capture_internal_impl" ] } if (rtc_enable_protobuf) { deps += [ diff --git a/media/base/media_constants.cc b/media/base/media_constants.cc index 319fa3f3f33..07464b8a1d5 100644 --- a/media/base/media_constants.cc +++ b/media/base/media_constants.cc @@ -105,9 +105,7 @@ const char kVp8CodecName[] = "VP8"; const char kVp9CodecName[] = "VP9"; const char kAv1CodecName[] = "AV1"; const char kH264CodecName[] = "H264"; -#ifdef WEBRTC_USE_H265 const char kH265CodecName[] = "H265"; -#endif // RFC 6184 RTP Payload Format for H.264 video const char kH264FmtpProfileLevelId[] = "profile-level-id"; @@ -122,7 +120,6 @@ const char kH265FmtpProfileSpace[] = "profile-space"; const char kH265FmtpProfileId[] = "profile-id"; const char kH265FmtpTierFlag[] = "tier-flag"; const char kH265FmtpLevelId[] = "level-id"; -#endif const char kVP9ProfileId[] = "profile-id"; diff --git a/media/base/media_constants.h b/media/base/media_constants.h index 9e05cd792b7..0459e0c848d 100644 --- a/media/base/media_constants.h +++ b/media/base/media_constants.h @@ -127,9 +127,7 @@ RTC_EXPORT extern const char kVp8CodecName[]; RTC_EXPORT extern const char kVp9CodecName[]; RTC_EXPORT extern const char kAv1CodecName[]; RTC_EXPORT extern const char kH264CodecName[]; -#ifdef WEBRTC_USE_H265 RTC_EXPORT extern const char kH265CodecName[]; -#endif // RFC 6184 RTP Payload Format for H.264 video RTC_EXPORT extern const char kH264FmtpProfileLevelId[]; diff --git a/media/engine/encoder_simulcast_proxy.h b/media/engine/encoder_simulcast_proxy.h index 17310ffd071..a8c28add64e 100644 --- a/media/engine/encoder_simulcast_proxy.h +++ b/media/engine/encoder_simulcast_proxy.h @@ -25,7 +25,6 @@ #include "api/video_codecs/video_encoder.h" #include "api/video_codecs/video_encoder_factory.h" #include "modules/video_coding/include/video_codec_interface.h" -#include "rtc_base/constructor_magic.h" #include "rtc_base/system/rtc_export.h" namespace webrtc { @@ -58,7 +57,6 @@ class RTC_EXPORT EncoderSimulcastProxy : public VideoEncoder { SdpVideoFormat video_format_; std::unique_ptr encoder_; EncodedImageCallback* callback_; - RTC_DISALLOW_COPY_AND_ASSIGN(EncoderSimulcastProxy); }; } // namespace webrtc diff --git a/media/engine/webrtc_video_engine.cc b/media/engine/webrtc_video_engine.cc index b2f0932fe97..6ca0a08cd32 100644 --- a/media/engine/webrtc_video_engine.cc +++ b/media/engine/webrtc_video_engine.cc @@ -40,8 +40,7 @@ #include "rtc_base/strings/string_builder.h" #include "rtc_base/time_utils.h" #include "rtc_base/trace_event.h" - -static const int kRtpBufferSizeIncreased = 4 * 1024 * 1024; +#include "system_wrappers/include/field_trial.h" namespace cricket { @@ -68,15 +67,6 @@ const char* StreamTypeToString( } return nullptr; } -// If this field trial is enabled, the "flexfec-03" codec will be advertised -// as being supported. This means that "flexfec-03" will appear in the default -// SDP offer, and we therefore need to be ready to receive FlexFEC packets from -// the remote. It also means that FlexFEC SSRCs will be generated by -// MediaSession and added as "a=ssrc:" and "a=ssrc-group:" lines in the local -// SDP. -bool IsFlexfecAdvertisedFieldTrialEnabled() { - return webrtc::field_trial::IsEnabled("OWT-FlexFEC"); -} bool IsEnabled(const webrtc::FieldTrialsView& trials, absl::string_view name) { return absl::StartsWith(trials.Lookup(name), "Enabled"); @@ -102,11 +92,10 @@ void AddDefaultFeedbackParams(VideoCodec* codec, codec->AddFeedbackParam(FeedbackParam(kRtcpFbParamNack, kRtcpFbNackParamPli)); codec->AddFeedbackParam( FeedbackParam(kRtcpFbParamNack, kRtcpFbNackParamRpsi)); - if ((codec->name == kVp8CodecName || codec->name == kH264CodecName - || codec->name == kH265CodecName - ) && + if ((codec->name == kVp8CodecName || codec->name == kH264CodecName || + codec->name == kH265CodecName) && (webrtc::field_trial::IsEnabled("WebRTC-RtcpLossNotification") || - (webrtc::field_trial::IsEnabled("OWT-LowLatencyMode")))) { + (webrtc::field_trial::IsEnabled("OWT-LowLatencyMode")))) { codec->AddFeedbackParam(FeedbackParam(kRtcpFbParamLntf, kParamValueEmpty)); } } diff --git a/media/engine/webrtc_voice_engine.cc b/media/engine/webrtc_voice_engine.cc index 1649a649121..e4ddf0c2658 100644 --- a/media/engine/webrtc_voice_engine.cc +++ b/media/engine/webrtc_voice_engine.cc @@ -52,6 +52,7 @@ #include "rtc_base/third_party/base64/base64.h" #include "rtc_base/trace_event.h" #include "system_wrappers/include/metrics.h" +#include "system_wrappers/include/field_trial.h" #if WEBRTC_ENABLE_PROTOBUF RTC_PUSH_IGNORING_WUNDEF() @@ -403,8 +404,6 @@ void WebRtcVoiceEngine::Init() { options.auto_gain_control = false; options.noise_suppression = false; options.highpass_filter = false; - options.typing_detection = false; - options.residual_echo_detector = false; } ApplyOptions(options); } diff --git a/modules/audio_coding/codecs/opus/audio_encoder_opus.cc b/modules/audio_coding/codecs/opus/audio_encoder_opus.cc index 69243013d48..dba00b0cf45 100644 --- a/modules/audio_coding/codecs/opus/audio_encoder_opus.cc +++ b/modules/audio_coding/codecs/opus/audio_encoder_opus.cc @@ -123,7 +123,7 @@ int GetChannelCount(const SdpAudioFormat& format) { if (param == "1") { return 2; } else { - return 2; + return 1; } } diff --git a/modules/audio_device/win/audio_device_core_win.cc b/modules/audio_device/win/audio_device_core_win.cc index 7960cb14cb3..9da0ad326f9 100644 --- a/modules/audio_device/win/audio_device_core_win.cc +++ b/modules/audio_device/win/audio_device_core_win.cc @@ -184,13 +184,11 @@ HRESULT __stdcall AudioDeviceWindowsCore::OnDefaultDeviceChanged( ULONG AudioDeviceWindowsCore::AddRef() { ULONG new_ref = InterlockedIncrement(&ref_count_); - // RTC_DLOG(INFO) << "__AddRef => " << new_ref; return new_ref; } ULONG AudioDeviceWindowsCore::Release() { ULONG new_ref = InterlockedDecrement(&ref_count_); - // RTC_DLOG(INFO) << "__Release => " << new_ref; return new_ref; } diff --git a/modules/audio_processing/aec3/clockdrift_detector.h b/modules/audio_processing/aec3/clockdrift_detector.h index 732cd2279c8..2ba90bb8890 100644 --- a/modules/audio_processing/aec3/clockdrift_detector.h +++ b/modules/audio_processing/aec3/clockdrift_detector.h @@ -14,7 +14,6 @@ #include #include -#include namespace webrtc { diff --git a/modules/audio_processing/audio_processing_impl.cc b/modules/audio_processing/audio_processing_impl.cc index 535f527de6c..9405e14fce5 100644 --- a/modules/audio_processing/audio_processing_impl.cc +++ b/modules/audio_processing/audio_processing_impl.cc @@ -770,8 +770,7 @@ int AudioProcessingImpl::ProcessStream(const float* const* src, capture_.capture_fullband_audio->CopyFrom( src, formats_.api_format.input_stream()); } - // Bypass processing - //RETURN_ON_ERR(ProcessCaptureStreamLocked()); + RETURN_ON_ERR(ProcessCaptureStreamLocked()); if (capture_.capture_fullband_audio) { capture_.capture_fullband_audio->CopyTo(formats_.api_format.output_stream(), dest); diff --git a/modules/congestion_controller/BUILD.gn b/modules/congestion_controller/BUILD.gn index 66b437f3fec..35256eb05b4 100644 --- a/modules/congestion_controller/BUILD.gn +++ b/modules/congestion_controller/BUILD.gn @@ -28,16 +28,6 @@ rtc_library("congestion_controller") { "remb_throttler.h", ] - if (owt_use_gpra) { - # use Intel Goodput Rate Adapatation BWE - include_dirs = [ owt_gpra_header_root ] - if (is_win && owt_gpra_lib_root != "") { - libs = [ "gpra_lib.lib" ] - lib_dirs = [ owt_gpra_lib_root ] - } - defines += [ "INTEL_GPRA" ] - } - deps = [ "../../api/transport:field_trial_based_config", "../../api/transport:network_control", diff --git a/modules/congestion_controller/goog_cc/BUILD.gn b/modules/congestion_controller/goog_cc/BUILD.gn index 79869326291..1335a1ebc79 100644 --- a/modules/congestion_controller/goog_cc/BUILD.gn +++ b/modules/congestion_controller/goog_cc/BUILD.gn @@ -8,8 +8,6 @@ import("../../../webrtc.gni") -defines = [] - config("bwe_test_logging") { if (rtc_enable_bwe_test_logging) { defines = [ "BWE_TEST_LOGGING_COMPILE_TIME_ENABLE=1" ] @@ -25,16 +23,6 @@ rtc_library("goog_cc") { "goog_cc_network_control.h", ] - if (owt_use_gpra) { - # use Intel Goodput Rate Adapatation BWE - include_dirs = [ owt_gpra_header_root ] - if (is_win && owt_gpra_lib_root != "") { - libs = [ "gpra_lib.lib" ] - lib_dirs = [ owt_gpra_lib_root ] - } - defines += [ "INTEL_GPRA" ] - } - deps = [ ":alr_detector", ":delay_based_bwe", diff --git a/modules/congestion_controller/goog_cc/goog_cc_network_control.cc b/modules/congestion_controller/goog_cc/goog_cc_network_control.cc index 5064a2f5e8c..54f6240e2ef 100644 --- a/modules/congestion_controller/goog_cc/goog_cc_network_control.cc +++ b/modules/congestion_controller/goog_cc/goog_cc_network_control.cc @@ -33,9 +33,6 @@ #include "rtc_base/checks.h" #include "rtc_base/logging.h" #include "system_wrappers/include/field_trial.h" -#ifdef INTEL_GPRA -#include "gpra_bwe.h" -#endif namespace webrtc { @@ -76,8 +73,13 @@ bool GetExternalBweRateLimits(int* start_bitrate_kbps, if (expr_str.empty()) return false; int parsed_values = +#if defined(WEBRTC_WIN) sscanf_s(expr_str.c_str(), "Enabled-%u,%u,%u", start_bitrate_kbps, min_bitrate_kbps, max_bitrate_kbps); +#else + sscanf(expr_str.c_str(), "Enabled-%u,%u,%u", start_bitrate_kbps, + min_bitrate_kbps, max_bitrate_kbps); +#endif if (parsed_values == 3) { RTC_CHECK_GE(*start_bitrate_kbps, 0) << "start_bitrate_kbps must not be smaller than 0."; @@ -88,6 +90,7 @@ bool GetExternalBweRateLimits(int* start_bitrate_kbps, return true; } return false; +} bool IsEnabled(const FieldTrialsView* config, absl::string_view key) { return absl::StartsWith(config->Lookup(key), "Enabled"); @@ -96,7 +99,6 @@ bool IsEnabled(const FieldTrialsView* config, absl::string_view key) { bool IsNotDisabled(const FieldTrialsView* config, absl::string_view key) { return !absl::StartsWith(config->Lookup(key), "Disabled"); } -} // namespace GoogCcNetworkController::GoogCcNetworkController(NetworkControllerConfig config, GoogCcConfig goog_cc_config) @@ -139,19 +141,9 @@ GoogCcNetworkController::GoogCcNetworkController(NetworkControllerConfig config, network_estimator_(std::move(goog_cc_config.network_state_estimator)), network_state_predictor_( std::move(goog_cc_config.network_state_predictor)), -#ifdef INTEL_GPRA - delay_based_bwe_(AllowExternalBwe() ? new GPRABwe() : nullptr), - delay_based_bwe_gcc_( - AllowExternalBwe() - ? nullptr - : new DelayBasedBwe(key_value_config_, - event_log_, - network_state_predictor_.get())), -#else delay_based_bwe_(new DelayBasedBwe(key_value_config_, event_log_, network_state_predictor_.get())), -#endif acknowledged_bitrate_estimator_( AcknowledgedBitrateEstimatorInterface::Create(key_value_config_)), initial_config_(config), @@ -175,8 +167,6 @@ GoogCcNetworkController::GoogCcNetworkController(NetworkControllerConfig config, key_value_config_->Lookup("WebRTC-Bwe-SafeResetOnRouteChange")); if (delay_based_bwe_) delay_based_bwe_->SetMinBitrate(kCongestionControllerMinBitrate); - if (delay_based_bwe_gcc_) - delay_based_bwe_gcc_->SetMinBitrate(congestion_controller::GetMinBitrate()); if (AllowExternalBwe()) { ParseFieldTrial({&external_start_bitrate_kbps_, &external_min_bitrate_kbps_, &external_max_bitrate_kbps_}, diff --git a/modules/congestion_controller/goog_cc/goog_cc_network_control.h b/modules/congestion_controller/goog_cc/goog_cc_network_control.h index 8909e4aef1c..3b3ca9e5085 100644 --- a/modules/congestion_controller/goog_cc/goog_cc_network_control.h +++ b/modules/congestion_controller/goog_cc/goog_cc_network_control.h @@ -42,7 +42,6 @@ struct GoogCcConfig { std::unique_ptr network_state_predictor = nullptr; bool feedback_only = false; }; -class GPRABwe; class GoogCcNetworkController : public NetworkControllerInterface { public: @@ -112,12 +111,7 @@ class GoogCcNetworkController : public NetworkControllerInterface { std::unique_ptr probe_bitrate_estimator_; std::unique_ptr network_estimator_; std::unique_ptr network_state_predictor_; -#ifdef INTEL_GPRA - std::unique_ptr delay_based_bwe_; - std::unique_ptr delay_based_bwe_gcc_; -#else std::unique_ptr delay_based_bwe_; -#endif std::unique_ptr acknowledged_bitrate_estimator_; @@ -157,7 +151,6 @@ class GoogCcNetworkController : public NetworkControllerInterface { FieldTrialParameter external_start_bitrate_kbps_; FieldTrialParameter external_min_bitrate_kbps_; FieldTrialParameter external_max_bitrate_kbps_; - RTC_DISALLOW_IMPLICIT_CONSTRUCTORS(GoogCcNetworkController); }; } // namespace webrtc diff --git a/modules/congestion_controller/goog_cc/send_side_bandwidth_estimation.cc b/modules/congestion_controller/goog_cc/send_side_bandwidth_estimation.cc index bc50607f6f5..ed3170b2d03 100644 --- a/modules/congestion_controller/goog_cc/send_side_bandwidth_estimation.cc +++ b/modules/congestion_controller/goog_cc/send_side_bandwidth_estimation.cc @@ -333,10 +333,6 @@ void SendSideBandwidthEstimation::UpdateReceiverEstimate(Timestamp at_time, DataRate bandwidth) { // TODO(srte): Ensure caller passes PlusInfinity, not zero, to represent no // limitation. - if (is_using_external_bwe_) { - // Ignore remb if using GPRA. - return; - } receiver_limit_ = bandwidth.IsZero() ? DataRate::PlusInfinity() : bandwidth; ApplyTargetLimits(at_time); } @@ -627,19 +623,10 @@ void SendSideBandwidthEstimation::UpdateMinHistory(Timestamp at_time) { } DataRate SendSideBandwidthEstimation::GetUpperLimit() const { - std::string experiment_string = - webrtc::field_trial::FindFullName("OWT-DelayBweWeight"); - DataRate upper_limit = std::min(delay_based_limit_, receiver_limit_); - double delay_fraction = delay_weight / 100.0; - - // Calculate new weighted BWE using delay/loss limit. - upper_limit = std::min(upper_limit, max_bitrate_configured_); - if (delay_weight < 100 && delay_weight >= 0) { - upper_limit = - std::min(upper_limit, loss_based_bandwidth_estimation_.GetEstimate()); - } - DataRate upper_limit = std::min(weighted_limit, receiver_limit_); - upper_limit = std::min(upper_limit, max_bitrate_configured_); + DataRate upper_limit = delay_based_limit_; + if (disable_receiver_limit_caps_only_) + upper_limit = std::min(upper_limit, receiver_limit_); + return std::min(upper_limit, max_bitrate_configured_); } void SendSideBandwidthEstimation::MaybeLogLowBitrateWarning(DataRate bitrate, @@ -667,11 +654,8 @@ void SendSideBandwidthEstimation::MaybeLogLossBasedEvent(Timestamp at_time) { void SendSideBandwidthEstimation::UpdateTargetBitrate(DataRate new_bitrate, Timestamp at_time) { - if (is_using_external_bwe_) - new_bitrate = GetUpperLimit(); - else - new_bitrate = std::min(new_bitrate, GetUpperLimit()); - if (!is_using_external_bwe_ && new_bitrate < min_bitrate_configured_) { + new_bitrate = std::min(new_bitrate, GetUpperLimit()); + if (new_bitrate < min_bitrate_configured_) { MaybeLogLowBitrateWarning(new_bitrate, at_time); new_bitrate = min_bitrate_configured_; } diff --git a/modules/pacing/pacing_controller.cc b/modules/pacing/pacing_controller.cc index e6b5f8c484e..e9db974d4f6 100644 --- a/modules/pacing/pacing_controller.cc +++ b/modules/pacing/pacing_controller.cc @@ -28,7 +28,6 @@ namespace webrtc { namespace { // Time limit in milliseconds between packet bursts. constexpr TimeDelta kDefaultMinPacketLimit = TimeDelta::Millis(5); -constexpr TimeDelta kDefaultMinPacketLimitLowLatency = TimeDelta::Millis(1); constexpr TimeDelta kCongestedPacketInterval = TimeDelta::Millis(500); // TODO(sprang): Consider dropping this limit. // The maximum debt level, in terms of time, capped when sending packets. @@ -97,9 +96,6 @@ PacingController::PacingController(Clock* clock, ParseFieldTrial({&min_packet_limit_ms}, field_trials_.Lookup("WebRTC-Pacer-MinPacketLimitMs")); min_packet_limit_ = TimeDelta::Millis(min_packet_limit_ms.Get()); - if (mode_ == ProcessMode::kRealtime) { - min_packet_limit_ = kDefaultMinPacketLimitLowLatency; - } UpdateBudgetWithElapsedTime(min_packet_limit_); } @@ -163,11 +159,6 @@ Timestamp PacingController::CurrentTime() const { } void PacingController::SetProbingEnabled(bool enabled) { - // For low latency mode we will always disable prober. - if (mode_ == ProcessMode::kRealtime) { - prober_.SetEnabled(false); - return; - } RTC_CHECK(!seen_first_packet_); prober_.SetEnabled(enabled); } @@ -311,11 +302,6 @@ Timestamp PacingController::NextSendTime() const { const Timestamp now = CurrentTime(); Timestamp next_send_time = Timestamp::PlusInfinity(); - // Realtime pacer works at RT mode. - if (mode_ == ProcessMode::kRealtime) { - return now; - } - if (paused_) { return last_send_time_ + kPausedProcessInterval; } @@ -381,7 +367,7 @@ void PacingController::ProcessPackets() { Timestamp target_send_time = now; // Disable padding for realtime mode - if (ShouldSendKeepalive(now) && mode_ != ProcessMode::kRealtime) { + if (ShouldSendKeepalive(now)) { DataSize keepalive_data_sent = DataSize::Zero(); // We can not send padding unless a normal packet has first been sent. If // we do, timestamps get messed up. @@ -400,7 +386,7 @@ void PacingController::ProcessPackets() { OnPacketSent(RtpPacketMediaType::kPadding, keepalive_data_sent, now); } - if (paused_ && mode_ != ProcessMode::kRealtime) { + if (paused_) { return; } @@ -450,7 +436,6 @@ void PacingController::ProcessPackets() { GetPendingPacket(pacing_info, target_send_time, now); if (rtp_packet == nullptr) { // No packet available to send, check if we should send padding. - if (mode_ != ProcessMode::kRealtime) { DataSize padding_to_add = PaddingToAdd(recommended_probe_size, data_sent); if (padding_to_add > DataSize::Zero()) { @@ -529,7 +514,7 @@ void PacingController::ProcessPackets() { return; } - if (is_probing && mode_ != ProcessMode::kRealtime) { + if (is_probing) { probing_send_failure_ = data_sent == DataSize::Zero(); if (!probing_send_failure_) { prober_.ProbeSent(CurrentTime(), data_sent); @@ -597,7 +582,6 @@ std::unique_ptr PacingController::GetPendingPacket( return nullptr; } - if (mode_ != ProcessMode::kRealtime) { // First, check if there is any reason _not_ to send the next queued packet. // Unpaced packets and probes are exempted from send checks. if (NextUnpacedSendTime().IsInfinite() && !is_probe) { @@ -618,7 +602,6 @@ std::unique_ptr PacingController::GetPendingPacket( } } } - } return packet_queue_.Pop(); } diff --git a/modules/rtp_rtcp/BUILD.gn b/modules/rtp_rtcp/BUILD.gn index 171956f3e75..af3d8330683 100644 --- a/modules/rtp_rtcp/BUILD.gn +++ b/modules/rtp_rtcp/BUILD.gn @@ -257,19 +257,6 @@ rtc_library("rtp_rtcp") { defines = [ "BWE_TEST_LOGGING_COMPILE_TIME_ENABLE=0" ] } - if (rtc_use_h265) { - sources += [ - "source/rtp_format_h265.cc", - "source/rtp_format_h265.h", - "source/video_rtp_depacketizer_h265.cc", - "source/video_rtp_depacketizer_h265.h", - ] - } - - if (!rtc_use_h265) { - defines += ["DISABLE_H265"] - } - deps = [ ":rtp_rtcp_format", ":rtp_video_header", diff --git a/modules/rtp_rtcp/include/rtp_rtcp_defines.h b/modules/rtp_rtcp/include/rtp_rtcp_defines.h index af04da2c437..20c2aea6f35 100644 --- a/modules/rtp_rtcp/include/rtp_rtcp_defines.h +++ b/modules/rtp_rtcp/include/rtp_rtcp_defines.h @@ -76,8 +76,8 @@ enum RTPExtensionType : int { kRtpExtensionGenericFrameDescriptor02, kRtpExtensionColorSpace, kRtpExtensionVideoFrameTrackingId, - kRtpExtensionNumberOfExtensions // Must be the last entity in the enum. kRtpExtensionPictureId, // Required by temporal scalability support + kRtpExtensionNumberOfExtensions, // Must be the last entity in the enum. }; enum RTCPAppSubTypes { kAppSubtypeBwe = 0x00 }; diff --git a/modules/rtp_rtcp/source/create_video_rtp_depacketizer.cc b/modules/rtp_rtcp/source/create_video_rtp_depacketizer.cc index 4c6e525ed89..aaa0ba3f57f 100644 --- a/modules/rtp_rtcp/source/create_video_rtp_depacketizer.cc +++ b/modules/rtp_rtcp/source/create_video_rtp_depacketizer.cc @@ -19,9 +19,8 @@ #include "modules/rtp_rtcp/source/video_rtp_depacketizer_h264.h" #include "modules/rtp_rtcp/source/video_rtp_depacketizer_vp8.h" #include "modules/rtp_rtcp/source/video_rtp_depacketizer_vp9.h" -#ifndef DISABLE_H265 #include "modules/rtp_rtcp/source/video_rtp_depacketizer_h265.h" -#endif + namespace webrtc { std::unique_ptr CreateVideoRtpDepacketizer( @@ -33,10 +32,8 @@ std::unique_ptr CreateVideoRtpDepacketizer( return std::make_unique(); case kVideoCodecVP9: return std::make_unique(); -#ifndef DISABLE_H265 case kVideoCodecH265: return std::make_unique(); -#endif case kVideoCodecAV1: return std::make_unique(); case kVideoCodecGeneric: diff --git a/modules/rtp_rtcp/source/rtp_format.cc b/modules/rtp_rtcp/source/rtp_format.cc index b7504edb5af..d2bc2e535b1 100644 --- a/modules/rtp_rtcp/source/rtp_format.cc +++ b/modules/rtp_rtcp/source/rtp_format.cc @@ -43,9 +43,8 @@ std::unique_ptr RtpPacketizer::Create( case kVideoCodecH264: { const auto& h264 = absl::get(rtp_video_header.video_type_header); - return std::make_unique(payload, limits, - h264.packetization_mode, - end_of_frame); + return std::make_unique( + payload, limits, h264.packetization_mode, end_of_frame); } case kVideoCodecH265: { const auto& h265 = diff --git a/modules/rtp_rtcp/source/rtp_format_h264.cc b/modules/rtp_rtcp/source/rtp_format_h264.cc index 95cc305e976..68743adf399 100644 --- a/modules/rtp_rtcp/source/rtp_format_h264.cc +++ b/modules/rtp_rtcp/source/rtp_format_h264.cc @@ -42,22 +42,17 @@ static const size_t kLengthFieldSize = 2; RtpPacketizerH264::RtpPacketizerH264(rtc::ArrayView payload, PayloadSizeLimits limits, H264PacketizationMode packetization_mode, - const RTPFragmentationHeader& fragmentation) - bool end_of_frame) - : limits_(limits), num_packets_left_(0) { + bool end_of_frame) + : limits_(limits), num_packets_left_(0), end_of_frame_(end_of_frame) { // Guard against uninitialized memory in packetization_mode. RTC_CHECK(packetization_mode == H264PacketizationMode::NonInterleaved || packetization_mode == H264PacketizationMode::SingleNalUnit); - end_of_frame_ = end_of_frame; - for (const auto& nalu : H264::FindNaluIndices(payload.data(), payload.size())) { - for (size_t i = 0; i < fragmentation.fragmentationVectorSize; ++i) { input_fragments_.push_back( payload.subview(nalu.payload_start_offset, nalu.payload_size)); } - } if (!GeneratePackets(packetization_mode)) { // If failed to generate all the packets, discard already generated diff --git a/modules/rtp_rtcp/source/rtp_format_h264.h b/modules/rtp_rtcp/source/rtp_format_h264.h index 971ba37f267..289cebfec11 100644 --- a/modules/rtp_rtcp/source/rtp_format_h264.h +++ b/modules/rtp_rtcp/source/rtp_format_h264.h @@ -43,7 +43,6 @@ class RtpPacketizerH264 : public RtpPacketizer { RtpPacketizerH264(rtc::ArrayView payload, PayloadSizeLimits limits, H264PacketizationMode packetization_mode, - const RTPFragmentationHeader& fragmentation, bool end_of_frame = true); ~RtpPacketizerH264() override; diff --git a/modules/rtp_rtcp/source/rtp_header_extension_map.cc b/modules/rtp_rtcp/source/rtp_header_extension_map.cc index 46b57b8ec41..15ae3706c0d 100644 --- a/modules/rtp_rtcp/source/rtp_header_extension_map.cc +++ b/modules/rtp_rtcp/source/rtp_header_extension_map.cc @@ -52,7 +52,8 @@ constexpr ExtensionInfo kExtensions[] = { CreateExtensionInfo(), CreateExtensionInfo(), CreateExtensionInfo(), - CreateExtensionInfo(), + CreateExtensionInfo(), + CreateExtensionInfo(), }; // Because of kRtpExtensionNone, NumberOfExtension is 1 bigger than the actual diff --git a/modules/rtp_rtcp/source/rtp_header_extensions.h b/modules/rtp_rtcp/source/rtp_header_extensions.h index 37f53baf352..5538e398816 100644 --- a/modules/rtp_rtcp/source/rtp_header_extensions.h +++ b/modules/rtp_rtcp/source/rtp_header_extensions.h @@ -167,6 +167,9 @@ class PictureId { static constexpr RTPExtensionType kId = kRtpExtensionPictureId; static constexpr uint8_t kValueSizeBytes = 2; static constexpr const char kUri[] = "urn:intel:picture-id"; + static constexpr absl::string_view Uri() { + return kUri; + } static bool Parse(rtc::ArrayView data, uint16_t* value); static size_t ValueSize(uint16_t value) { return kValueSizeBytes; } static bool Write(rtc::ArrayView data, uint16_t value); diff --git a/modules/rtp_rtcp/source/rtp_sender.cc b/modules/rtp_rtcp/source/rtp_sender.cc index c063e0ebc93..57b5e3005ba 100644 --- a/modules/rtp_rtcp/source/rtp_sender.cc +++ b/modules/rtp_rtcp/source/rtp_sender.cc @@ -22,6 +22,7 @@ #include "api/rtc_event_log/rtc_event_log.h" #include "logging/rtc_event_log/events/rtc_event_rtp_packet_outgoing.h" #include "modules/rtp_rtcp/include/rtp_cvo.h" +#include "modules/rtp_rtcp/include/rtp_rtcp_defines.h" #include "modules/rtp_rtcp/source/byte_io.h" #include "modules/rtp_rtcp/source/rtp_generic_frame_descriptor_extension.h" #include "modules/rtp_rtcp/source/rtp_header_extensions.h" @@ -117,6 +118,7 @@ bool IsNonVolatile(RTPExtensionType type) { case kRtpExtensionMid: case kRtpExtensionGenericFrameDescriptor00: case kRtpExtensionGenericFrameDescriptor02: + case kRtpExtensionPictureId: return true; case kRtpExtensionInbandComfortNoise: case kRtpExtensionAbsoluteCaptureTime: diff --git a/modules/rtp_rtcp/source/rtp_sender_video.cc b/modules/rtp_rtcp/source/rtp_sender_video.cc index 00e1e0fd738..3da3ec2136a 100644 --- a/modules/rtp_rtcp/source/rtp_sender_video.cc +++ b/modules/rtp_rtcp/source/rtp_sender_video.cc @@ -448,18 +448,15 @@ void RTPSenderVideo::AddRtpHeaderExtensions(const RTPVideoHeader& video_header, packet->SetExtension( generic_descriptor); } - if (video_header.codec == kVideoCodecH264 && last_packet) { - packet->SetExtension( - absl::get(video_header.video_type_header) - .picture_id); - } -#ifdef WEBRTC_USE_H265 - else if (video_header.codec == kVideoCodecH265 && last_packet) { - packet->SetExtension( - absl::get(video_header.video_type_header) - .picture_id); - } -#endif + if (video_header.codec == kVideoCodecH264 && last_packet) { + packet->SetExtension( + absl::get(video_header.video_type_header) + .picture_id); + } else if (video_header.codec == kVideoCodecH265 && last_packet) { + packet->SetExtension( + absl::get(video_header.video_type_header) + .picture_id); + } } if (packet->IsRegistered() && @@ -715,8 +712,6 @@ bool RTPSenderVideo::SendVideo( packet->set_is_key_frame(video_header.frame_type == VideoFrameType::kVideoFrameKey); - packet->set_is_key_frame(video_header.frame_type == VideoFrameType::kVideoFrameKey); - // Put packetization finish timestamp into extension. if (packet->HasExtension()) { packet->set_packetization_finish_time(clock_->CurrentTime()); @@ -843,9 +838,7 @@ uint8_t RTPSenderVideo::GetTemporalId(const RTPVideoHeader& header) { uint8_t operator()(const RTPVideoHeaderLegacyGeneric&) { return kNoTemporalIdx; } -#ifndef DISABLE_H265 uint8_t operator()(const RTPVideoHeaderH265&) { return kNoTemporalIdx; } -#endif uint8_t operator()(const absl::monostate&) { return kNoTemporalIdx; } }; return absl::visit(TemporalIdGetter(), header.video_type_header); diff --git a/modules/rtp_rtcp/source/rtp_video_header.h b/modules/rtp_rtcp/source/rtp_video_header.h index cdeee56a37e..3b1c0543392 100644 --- a/modules/rtp_rtcp/source/rtp_video_header.h +++ b/modules/rtp_rtcp/source/rtp_video_header.h @@ -37,20 +37,12 @@ struct RTPVideoHeaderLegacyGeneric { uint16_t picture_id; }; -#ifndef DISABLE_H265 using RTPVideoTypeHeader = absl::variant; -#else -using RTPVideoTypeHeader = absl::variant; -#endif + RTPVideoHeaderLegacyGeneric>; struct RTPVideoHeader { struct GenericDescriptorInfo { diff --git a/modules/rtp_rtcp/source/ulpfec_generator.cc b/modules/rtp_rtcp/source/ulpfec_generator.cc index 758d2ab6a12..20402fc4d37 100644 --- a/modules/rtp_rtcp/source/ulpfec_generator.cc +++ b/modules/rtp_rtcp/source/ulpfec_generator.cc @@ -110,11 +110,6 @@ void UlpfecGenerator::AddPacketAndGenerateFec(const RtpPacketToSend& packet) { RTC_DCHECK_RUNS_SERIALIZED(&race_checker_); RTC_DCHECK(generated_fec_packets_.empty()); - // reset state upon async keyframe request - if (keyframe_in_process_ != packet.is_key_frame()) { - ResetState(); - } - { MutexLock lock(&mutex_); if (pending_params_) { diff --git a/modules/utility/include/mock/mock_process_thread.h b/modules/utility/include/mock/mock_process_thread.h deleted file mode 100644 index 6dfa4adae97..00000000000 --- a/modules/utility/include/mock/mock_process_thread.h +++ /dev/null @@ -1,42 +0,0 @@ -/* - * Copyright (c) 2014 The WebRTC project authors. All Rights Reserved. - * - * Use of this source code is governed by a BSD-style license - * that can be found in the LICENSE file in the root of the source - * tree. An additional intellectual property rights grant can be found - * in the file PATENTS. All contributing project authors may - * be found in the AUTHORS file in the root of the source tree. - */ - -#ifndef MODULES_UTILITY_INCLUDE_MOCK_MOCK_PROCESS_THREAD_H_ -#define MODULES_UTILITY_INCLUDE_MOCK_MOCK_PROCESS_THREAD_H_ - -#include - -#include "modules/utility/include/process_thread.h" -#include "rtc_base/location.h" -#include "test/gmock.h" - -namespace webrtc { - -class MockProcessThread : public ProcessThread { - public: - MOCK_METHOD(void, Start, (), (override)); - MOCK_METHOD0(StartWithHighPriority, void()); - MOCK_METHOD(void, Stop, (), (override)); - MOCK_METHOD(void, Delete, (), (override)); - MOCK_METHOD(void, WakeUp, (Module*), (override)); - MOCK_METHOD(void, PostTask, (std::unique_ptr), (override)); - MOCK_METHOD(void, - PostDelayedTask, - (std::unique_ptr, uint32_t), - (override)); - MOCK_METHOD(void, - RegisterModule, - (Module*, const rtc::Location&), - (override)); - MOCK_METHOD(void, DeRegisterModule, (Module*), (override)); -}; - -} // namespace webrtc -#endif // MODULES_UTILITY_INCLUDE_MOCK_MOCK_PROCESS_THREAD_H_ diff --git a/modules/utility/include/process_thread.h b/modules/utility/include/process_thread.h deleted file mode 100644 index e0a3440c177..00000000000 --- a/modules/utility/include/process_thread.h +++ /dev/null @@ -1,64 +0,0 @@ -/* - * Copyright (c) 2011 The WebRTC project authors. All Rights Reserved. - * - * Use of this source code is governed by a BSD-style license - * that can be found in the LICENSE file in the root of the source - * tree. An additional intellectual property rights grant can be found - * in the file PATENTS. All contributing project authors may - * be found in the AUTHORS file in the root of the source tree. - */ - -#ifndef MODULES_UTILITY_INCLUDE_PROCESS_THREAD_H_ -#define MODULES_UTILITY_INCLUDE_PROCESS_THREAD_H_ - -#include - -#include "api/task_queue/queued_task.h" -#include "api/task_queue/task_queue_base.h" - -namespace rtc { -class Location; -} - -namespace webrtc { -class Module; - -// TODO(tommi): ProcessThread probably doesn't need to be a virtual -// interface. There exists one override besides ProcessThreadImpl, -// MockProcessThread, but when looking at how it is used, it seems -// a nullptr might suffice (or simply an actual ProcessThread instance). -class ProcessThread : public TaskQueueBase { - public: - ~ProcessThread() override; - - static std::unique_ptr Create(const char* thread_name); - - // Starts the worker thread. Must be called from the construction thread. - virtual void Start() = 0; - - // Starts the worker thread at high priority. Special startup mode used at - // low latency mode. Used by congestion controller only. - virtual void StartWithHighPriority() = 0; - - // Stops the worker thread. Must be called from the construction thread. - virtual void Stop() = 0; - - // Wakes the thread up to give a module a chance to do processing right - // away. This causes the worker thread to wake up and requery the specified - // module for when it should be called back. (Typically the module should - // return 0 from TimeUntilNextProcess on the worker thread at that point). - // Can be called on any thread. - virtual void WakeUp(Module* module) = 0; - - // Adds a module that will start to receive callbacks on the worker thread. - // Can be called from any thread. - virtual void RegisterModule(Module* module, const rtc::Location& from) = 0; - - // Removes a previously registered module. - // Can be called from any thread. - virtual void DeRegisterModule(Module* module) = 0; -}; - -} // namespace webrtc - -#endif // MODULES_UTILITY_INCLUDE_PROCESS_THREAD_H_ diff --git a/modules/utility/source/process_thread_impl.cc b/modules/utility/source/process_thread_impl.cc deleted file mode 100644 index 4257ce330fa..00000000000 --- a/modules/utility/source/process_thread_impl.cc +++ /dev/null @@ -1,280 +0,0 @@ -/* - * Copyright (c) 2012 The WebRTC project authors. All Rights Reserved. - * - * Use of this source code is governed by a BSD-style license - * that can be found in the LICENSE file in the root of the source - * tree. An additional intellectual property rights grant can be found - * in the file PATENTS. All contributing project authors may - * be found in the AUTHORS file in the root of the source tree. - */ - -#include "modules/utility/source/process_thread_impl.h" - -#include - -#include "modules/include/module.h" -#include "rtc_base/checks.h" -#include "rtc_base/logging.h" -#include "rtc_base/time_utils.h" -#include "rtc_base/trace_event.h" - -namespace webrtc { -namespace { - -// We use this constant internally to signal that a module has requested -// a callback right away. When this is set, no call to TimeUntilNextProcess -// should be made, but Process() should be called directly. -const int64_t kCallProcessImmediately = -1; - -int64_t GetNextCallbackTime(Module* module, int64_t time_now) { - int64_t interval = module->TimeUntilNextProcess(); - if (interval < 0) { - // Falling behind, we should call the callback now. - return time_now; - } - return time_now + interval; -} -} // namespace - -ProcessThread::~ProcessThread() {} - -// static -std::unique_ptr ProcessThread::Create(const char* thread_name) { - return std::unique_ptr(new ProcessThreadImpl(thread_name)); -} - -ProcessThreadImpl::ProcessThreadImpl(const char* thread_name) - : stop_(false), thread_name_(thread_name) {} - -ProcessThreadImpl::~ProcessThreadImpl() { - RTC_DCHECK(thread_checker_.IsCurrent()); - RTC_DCHECK(!thread_.get()); - RTC_DCHECK(!stop_); - - while (!delayed_tasks_.empty()) { - delete delayed_tasks_.top().task; - delayed_tasks_.pop(); - } - - while (!queue_.empty()) { - delete queue_.front(); - queue_.pop(); - } -} - -void ProcessThreadImpl::Delete() { - RTC_LOG(LS_WARNING) << "Process thread " << thread_name_ - << " is destroyed as a TaskQueue."; - Stop(); - delete this; -} - -void ProcessThreadImpl::Start() { - RTC_DCHECK(thread_checker_.IsCurrent()); - RTC_DCHECK(!thread_.get()); - if (thread_.get()) - return; - - RTC_DCHECK(!stop_); - - for (ModuleCallback& m : modules_) - m.module->ProcessThreadAttached(this); - - thread_.reset( - new rtc::PlatformThread(&ProcessThreadImpl::Run, this, thread_name_)); - thread_->Start(); -} - -void ProcessThreadImpl::StartWithHighPriority() { - RTC_DCHECK(!thread_.get()); - if (thread_.get()) - return; - - RTC_DCHECK(!stop_); - - for (ModuleCallback& m : modules_) - m.module->ProcessThreadAttached(this); - - thread_.reset(new rtc::PlatformThread(&ProcessThreadImpl::Run, this, - thread_name_, rtc::kRealtimePriority)); - thread_->Start(); -} - -void ProcessThreadImpl::Stop() { - RTC_DCHECK(thread_checker_.IsCurrent()); - if (!thread_.get()) - return; - - { - rtc::CritScope lock(&lock_); - stop_ = true; - } - - wake_up_.Set(); - - thread_->Stop(); - stop_ = false; - - thread_.reset(); - for (ModuleCallback& m : modules_) - m.module->ProcessThreadAttached(nullptr); -} - -void ProcessThreadImpl::WakeUp(Module* module) { - // Allowed to be called on any thread. - { - rtc::CritScope lock(&lock_); - for (ModuleCallback& m : modules_) { - if (m.module == module) - m.next_callback = kCallProcessImmediately; - } - } - wake_up_.Set(); -} - -void ProcessThreadImpl::PostTask(std::unique_ptr task) { - // Allowed to be called on any thread. - { - rtc::CritScope lock(&lock_); - queue_.push(task.release()); - } - wake_up_.Set(); -} - -void ProcessThreadImpl::PostDelayedTask(std::unique_ptr task, - uint32_t milliseconds) { - int64_t run_at_ms = rtc::TimeMillis() + milliseconds; - bool recalculate_wakeup_time; - { - rtc::CritScope lock(&lock_); - recalculate_wakeup_time = - delayed_tasks_.empty() || run_at_ms < delayed_tasks_.top().run_at_ms; - delayed_tasks_.emplace(run_at_ms, std::move(task)); - } - if (recalculate_wakeup_time) { - wake_up_.Set(); - } -} - -void ProcessThreadImpl::RegisterModule(Module* module, - const rtc::Location& from) { - RTC_DCHECK(thread_checker_.IsCurrent()); - RTC_DCHECK(module) << from.ToString(); - -#if RTC_DCHECK_IS_ON - { - // Catch programmer error. - rtc::CritScope lock(&lock_); - for (const ModuleCallback& mc : modules_) { - RTC_DCHECK(mc.module != module) - << "Already registered here: " << mc.location.ToString() - << "\n" - "Now attempting from here: " - << from.ToString(); - } - } -#endif - - // Now that we know the module isn't in the list, we'll call out to notify - // the module that it's attached to the worker thread. We don't hold - // the lock while we make this call. - if (thread_.get()) - module->ProcessThreadAttached(this); - - { - rtc::CritScope lock(&lock_); - modules_.push_back(ModuleCallback(module, from)); - } - - // Wake the thread calling ProcessThreadImpl::Process() to update the - // waiting time. The waiting time for the just registered module may be - // shorter than all other registered modules. - wake_up_.Set(); -} - -void ProcessThreadImpl::DeRegisterModule(Module* module) { - RTC_DCHECK(thread_checker_.IsCurrent()); - RTC_DCHECK(module); - - { - rtc::CritScope lock(&lock_); - modules_.remove_if( - [&module](const ModuleCallback& m) { return m.module == module; }); - } - - // Notify the module that it's been detached. - module->ProcessThreadAttached(nullptr); -} - -// static -void ProcessThreadImpl::Run(void* obj) { - ProcessThreadImpl* impl = static_cast(obj); - CurrentTaskQueueSetter set_current(impl); - while (impl->Process()) { - } -} - -bool ProcessThreadImpl::Process() { - TRACE_EVENT1("webrtc", "ProcessThreadImpl", "name", thread_name_); - int64_t now = rtc::TimeMillis(); - int64_t next_checkpoint = now + (1000 * 60); - - { - rtc::CritScope lock(&lock_); - if (stop_) - return false; - for (ModuleCallback& m : modules_) { - // TODO(tommi): Would be good to measure the time TimeUntilNextProcess - // takes and dcheck if it takes too long (e.g. >=10ms). Ideally this - // operation should not require taking a lock, so querying all modules - // should run in a matter of nanoseconds. - if (m.next_callback == 0) - m.next_callback = GetNextCallbackTime(m.module, now); - - if (m.next_callback <= now || - m.next_callback == kCallProcessImmediately) { - { - TRACE_EVENT2("webrtc", "ModuleProcess", "function", - m.location.function_name(), "file", - m.location.file_name()); - m.module->Process(); - } - // Use a new 'now' reference to calculate when the next callback - // should occur. We'll continue to use 'now' above for the baseline - // of calculating how long we should wait, to reduce variance. - int64_t new_now = rtc::TimeMillis(); - m.next_callback = GetNextCallbackTime(m.module, new_now); - } - - if (m.next_callback < next_checkpoint) - next_checkpoint = m.next_callback; - } - - while (!delayed_tasks_.empty() && delayed_tasks_.top().run_at_ms <= now) { - queue_.push(delayed_tasks_.top().task); - delayed_tasks_.pop(); - } - - if (!delayed_tasks_.empty()) { - next_checkpoint = - std::min(next_checkpoint, delayed_tasks_.top().run_at_ms); - } - - while (!queue_.empty()) { - QueuedTask* task = queue_.front(); - queue_.pop(); - lock_.Leave(); - if (task->Run()) { - delete task; - } - lock_.Enter(); - } - } - - int64_t time_to_wait = next_checkpoint - rtc::TimeMillis(); - if (time_to_wait > 0) - wake_up_.Wait(static_cast(time_to_wait)); - - return true; -} -} // namespace webrtc diff --git a/modules/utility/source/process_thread_impl.h b/modules/utility/source/process_thread_impl.h deleted file mode 100644 index 987e3290176..00000000000 --- a/modules/utility/source/process_thread_impl.h +++ /dev/null @@ -1,112 +0,0 @@ -/* - * Copyright (c) 2011 The WebRTC project authors. All Rights Reserved. - * - * Use of this source code is governed by a BSD-style license - * that can be found in the LICENSE file in the root of the source - * tree. An additional intellectual property rights grant can be found - * in the file PATENTS. All contributing project authors may - * be found in the AUTHORS file in the root of the source tree. - */ - -#ifndef MODULES_UTILITY_SOURCE_PROCESS_THREAD_IMPL_H_ -#define MODULES_UTILITY_SOURCE_PROCESS_THREAD_IMPL_H_ - -#include - -#include -#include -#include - -#include "api/task_queue/queued_task.h" -#include "modules/include/module.h" -#include "modules/utility/include/process_thread.h" -#include "rtc_base/critical_section.h" -#include "rtc_base/event.h" -#include "rtc_base/location.h" -#include "rtc_base/platform_thread.h" -#include "rtc_base/thread_checker.h" - -namespace webrtc { - -class ProcessThreadImpl : public ProcessThread { - public: - explicit ProcessThreadImpl(const char* thread_name); - ~ProcessThreadImpl() override; - - void Start() override; - void StartWithHighPriority() override; - void Stop() override; - - void WakeUp(Module* module) override; - void PostTask(std::unique_ptr task) override; - void PostDelayedTask(std::unique_ptr task, - uint32_t milliseconds) override; - - void RegisterModule(Module* module, const rtc::Location& from) override; - void DeRegisterModule(Module* module) override; - - protected: - static void Run(void* obj); - bool Process(); - - private: - struct ModuleCallback { - ModuleCallback() = delete; - ModuleCallback(ModuleCallback&& cb) = default; - ModuleCallback(const ModuleCallback& cb) = default; - ModuleCallback(Module* module, const rtc::Location& location) - : module(module), location(location) {} - bool operator==(const ModuleCallback& cb) const { - return cb.module == module; - } - - Module* const module; - int64_t next_callback = 0; // Absolute timestamp. - const rtc::Location location; - - private: - ModuleCallback& operator=(ModuleCallback&); - }; - struct DelayedTask { - DelayedTask(int64_t run_at_ms, std::unique_ptr task) - : run_at_ms(run_at_ms), task(task.release()) {} - friend bool operator<(const DelayedTask& lhs, const DelayedTask& rhs) { - // Earliest DelayedTask should be at the top of the priority queue. - return lhs.run_at_ms > rhs.run_at_ms; - } - - int64_t run_at_ms; - // DelayedTask owns the |task|, but some delayed tasks must be removed from - // the std::priority_queue, but mustn't be deleted. std::priority_queue does - // not give non-const access to the values, so storing unique_ptr would - // delete the task as soon as it is remove from the priority queue. - // Thus lifetime of the |task| is managed manually. - QueuedTask* task; - }; - typedef std::list ModuleList; - - void Delete() override; - - // Warning: For some reason, if |lock_| comes immediately before |modules_| - // with the current class layout, we will start to have mysterious crashes - // on Mac 10.9 debug. I (Tommi) suspect we're hitting some obscure alignemnt - // issues, but I haven't figured out what they are, if there are alignment - // requirements for mutexes on Mac or if there's something else to it. - // So be careful with changing the layout. - rtc::CriticalSection lock_; // Used to guard modules_, tasks_ and stop_. - - rtc::ThreadChecker thread_checker_; - rtc::Event wake_up_; - // TODO(pbos): Remove unique_ptr and stop recreating the thread. - std::unique_ptr thread_; - - ModuleList modules_; - std::queue queue_; - std::priority_queue delayed_tasks_ RTC_GUARDED_BY(lock_); - bool stop_; - const char* thread_name_; -}; - -} // namespace webrtc - -#endif // MODULES_UTILITY_SOURCE_PROCESS_THREAD_IMPL_H_ diff --git a/modules/video_capture/BUILD.gn b/modules/video_capture/BUILD.gn index e282b10217d..ad2d85e3fe8 100644 --- a/modules/video_capture/BUILD.gn +++ b/modules/video_capture/BUILD.gn @@ -120,22 +120,6 @@ if (!build_with_chromium) { } } - rtc_static_library("video_capture") { - visibility = [ "*" ] - sources = [ - "external/device_info_external.cc", - "external/video_capture_external.cc", - ] - - deps = [ - ":video_capture_module", - "../../api:scoped_refptr", - "../../rtc_base:checks", - "../../rtc_base:rtc_base_approved", - "../../system_wrappers", - ] - } - if (!is_android && rtc_include_tests) { rtc_test("video_capture_tests") { sources = [ "test/video_capture_unittest.cc" ] diff --git a/modules/video_capture/external/device_info_external.cc b/modules/video_capture/external/device_info_external.cc deleted file mode 100644 index cd660e2e0ba..00000000000 --- a/modules/video_capture/external/device_info_external.cc +++ /dev/null @@ -1,52 +0,0 @@ -/* - * Copyright (c) 2012 The WebRTC project authors. All Rights Reserved. - * - * Use of this source code is governed by a BSD-style license - * that can be found in the LICENSE file in the root of the source - * tree. An additional intellectual property rights grant can be found - * in the file PATENTS. All contributing project authors may - * be found in the AUTHORS file in the root of the source tree. - */ - -#include "modules/video_capture/device_info_impl.h" -#include "modules/video_capture/video_capture_impl.h" - -namespace webrtc { - -namespace videocapturemodule { - -class ExternalDeviceInfo : public DeviceInfoImpl { - public: - ExternalDeviceInfo() {} - virtual ~ExternalDeviceInfo() {} - virtual uint32_t NumberOfDevices() { return 0; } - virtual int32_t DisplayCaptureSettingsDialogBox( - const char* /*deviceUniqueIdUTF8*/, - const char* /*dialogTitleUTF8*/, - void* /*parentWindow*/, - uint32_t /*positionX*/, - uint32_t /*positionY*/) { - return -1; - } - virtual int32_t GetDeviceName(uint32_t deviceNumber, - char* deviceNameUTF8, - uint32_t deviceNameLength, - char* deviceUniqueIdUTF8, - uint32_t deviceUniqueIdUTF8Length, - char* productUniqueIdUTF8 = 0, - uint32_t productUniqueIdUTF8Length = 0) { - return -1; - } - virtual int32_t CreateCapabilityMap(const char* deviceUniqueIdUTF8) { - return 0; - } - virtual int32_t Init() { return 0; } -}; - -VideoCaptureModule::DeviceInfo* VideoCaptureImpl::CreateDeviceInfo() { - return new ExternalDeviceInfo(); -} - -} // namespace videocapturemodule - -} // namespace webrtc diff --git a/modules/video_capture/external/video_capture_external.cc b/modules/video_capture/external/video_capture_external.cc deleted file mode 100644 index b652a675746..00000000000 --- a/modules/video_capture/external/video_capture_external.cc +++ /dev/null @@ -1,26 +0,0 @@ -/* - * Copyright (c) 2012 The WebRTC project authors. All Rights Reserved. - * - * Use of this source code is governed by a BSD-style license - * that can be found in the LICENSE file in the root of the source - * tree. An additional intellectual property rights grant can be found - * in the file PATENTS. All contributing project authors may - * be found in the AUTHORS file in the root of the source tree. - */ - -#include "modules/video_capture/video_capture_impl.h" -#include "rtc_base/ref_count.h" -#include "rtc_base/ref_counted_object.h" - -namespace webrtc { - -namespace videocapturemodule { - -rtc::scoped_refptr VideoCaptureImpl::Create( - const char* deviceUniqueIdUTF8) { - return new rtc::RefCountedObject(); -} - -} // namespace videocapturemodule - -} // namespace webrtc diff --git a/modules/video_coding/BUILD.gn b/modules/video_coding/BUILD.gn index f59ca1693ff..d72b28e4b22 100644 --- a/modules/video_coding/BUILD.gn +++ b/modules/video_coding/BUILD.gn @@ -6,6 +6,7 @@ # in the file PATENTS. All contributing project authors may # be found in the AUTHORS file in the root of the source tree. +import("//third_party/ffmpeg/ffmpeg_options.gni") import("//third_party/libaom/options.gni") import("../../webrtc.gni") @@ -212,6 +213,8 @@ rtc_library("video_coding") { "h264_sprop_parameter_sets.h", "h264_sps_pps_tracker.cc", "h264_sps_pps_tracker.h", + "h265_vps_sps_pps_tracker.cc", + "h265_vps_sps_pps_tracker.h", "include/video_codec_initializer.h", "internal_defines.h", "loss_notification_controller.cc", @@ -235,13 +238,6 @@ rtc_library("video_coding") { "video_receiver2.h", ] - if (rtc_use_h265) { - sources += [ - "h265_vps_sps_pps_tracker.cc", - "h265_vps_sps_pps_tracker.h", - ] - } - deps = [ ":codec_globals_headers", ":encoded_frame", diff --git a/modules/video_coding/codecs/h264/h264.cc b/modules/video_coding/codecs/h264/h264.cc index ad2d2774676..9a885284ab6 100644 --- a/modules/video_coding/codecs/h264/h264.cc +++ b/modules/video_coding/codecs/h264/h264.cc @@ -161,7 +161,7 @@ std::unique_ptr H264Decoder::Create() { return nullptr; #endif #else - RTC_NOTREACHED(); + RTC_DCHECK_NOTREACHED(); return nullptr; #endif } diff --git a/modules/video_coding/encoded_frame.cc b/modules/video_coding/encoded_frame.cc index b72cb98d738..565a1278197 100644 --- a/modules/video_coding/encoded_frame.cc +++ b/modules/video_coding/encoded_frame.cc @@ -136,21 +136,14 @@ void VCMEncodedFrame::CopyCodecSpecific(const RTPVideoHeader* header) { _codecSpecificInfo.codecType = kVideoCodecH264; break; } - case kVideoCodecAV1: { + case kVideoCodecAV1: { _codecSpecificInfo.codecType = kVideoCodecAV1; break; } -#ifdef WEBRTC_USE_H265 case kVideoCodecH265: { _codecSpecificInfo.codecType = kVideoCodecH265; break; } -#ifndef DISABLE_H265 - case kVideoCodecH265: { - _codecSpecificInfo.codecType = kVideoCodecH265; - break; - } -#endif default: { _codecSpecificInfo.codecType = kVideoCodecGeneric; break; diff --git a/modules/video_coding/frame_object.cc b/modules/video_coding/frame_object.cc index d226dcd0138..48364454b10 100644 --- a/modules/video_coding/frame_object.cc +++ b/modules/video_coding/frame_object.cc @@ -37,8 +37,7 @@ RtpFrameObject::RtpFrameObject( const absl::optional& color_space, RtpPacketInfos packet_infos, rtc::scoped_refptr image_buffer) - : image_buffer_(image_buffer), - first_seq_num_(first_seq_num), + : first_seq_num_(first_seq_num), last_seq_num_(last_seq_num), last_packet_received_time_(last_packet_received_time), times_nacked_(times_nacked) { @@ -59,7 +58,7 @@ RtpFrameObject::RtpFrameObject( // as of the first packet's. SetPlayoutDelay(rtp_video_header_.playout_delay); - SetEncodedData(image_buffer_); + SetEncodedData(image_buffer); _encodedWidth = rtp_video_header_.width; _encodedHeight = rtp_video_header_.height; diff --git a/modules/video_coding/frame_object.h b/modules/video_coding/frame_object.h index f3011fe2eee..4c3d733be27 100644 --- a/modules/video_coding/frame_object.h +++ b/modules/video_coding/frame_object.h @@ -46,7 +46,7 @@ class RtpFrameObject : public EncodedFrame { int64_t RenderTime() const override; bool delayed_by_retransmission() const override; const RTPVideoHeader& GetRtpVideoHeader() const; - const FrameMarking& GetFrameMarking() const; + uint8_t* mutable_data() { return image_buffer_->data(); } #if defined(WEBRTC_WIN) void SetBWETiming(double start_duration, double last_duration, @@ -57,6 +57,8 @@ class RtpFrameObject : public EncodedFrame { } #endif private: + // Reference for mutable access. + rtc::scoped_refptr image_buffer_; RTPVideoHeader rtp_video_header_; VideoCodecType codec_type_; uint16_t first_seq_num_; diff --git a/modules/video_coding/include/video_codec_interface.h b/modules/video_coding/include/video_codec_interface.h index b10a67d783e..b827b03d197 100644 --- a/modules/video_coding/include/video_codec_interface.h +++ b/modules/video_coding/include/video_codec_interface.h @@ -21,7 +21,6 @@ #include "common_video/generic_frame_descriptor/generic_frame_info.h" #include "modules/video_coding/codecs/h264/include/h264_globals.h" #include "modules/video_coding/codecs/h265/include/h265_globals.h" -#endif #include "modules/video_coding/codecs/vp9/include/vp9_globals.h" #include "modules/video_coding/include/video_error_codes.h" #include "rtc_base/system/rtc_export.h" @@ -102,9 +101,7 @@ struct CodecSpecificInfoH265 { int16_t picture_id; int dependencies[5]; int dtis[10]; - bool discardable; }; -#endif static_assert(std::is_pod::value, ""); @@ -112,9 +109,7 @@ union CodecSpecificInfoUnion { CodecSpecificInfoVP8 VP8; CodecSpecificInfoVP9 VP9; CodecSpecificInfoH264 H264; -#ifdef WEBRTC_USE_H265 CodecSpecificInfoH265 H265; -#endif }; static_assert(std::is_pod::value, ""); diff --git a/modules/video_coding/packet_buffer.cc b/modules/video_coding/packet_buffer.cc index 5ca546c3192..bdc8b46cb17 100644 --- a/modules/video_coding/packet_buffer.cc +++ b/modules/video_coding/packet_buffer.cc @@ -271,7 +271,7 @@ std::vector> PacketBuffer::FindFrames( bool has_h264_pps = false; bool has_h264_idr = false; bool is_h264_keyframe = false; - + bool is_h265 = false; #ifdef WEBRTC_USE_H265 is_h265 = buffer_[start_index]->codec() == kVideoCodecH265; @@ -447,6 +447,7 @@ std::vector> PacketBuffer::FindFrames( } } #endif + if (is_h264 || full_frame_found) { const uint16_t end_seq_num = seq_num + 1; // Use uint16_t type to handle sequence number wrap around case. uint16_t num_packets = end_seq_num - start_seq_num; diff --git a/modules/video_coding/timing.cc b/modules/video_coding/timing.cc deleted file mode 100644 index 1c1e69f830a..00000000000 --- a/modules/video_coding/timing.cc +++ /dev/null @@ -1,253 +0,0 @@ -/* - * Copyright (c) 2011 The WebRTC project authors. All Rights Reserved. - * - * Use of this source code is governed by a BSD-style license - * that can be found in the LICENSE file in the root of the source - * tree. An additional intellectual property rights grant can be found - * in the file PATENTS. All contributing project authors may - * be found in the AUTHORS file in the root of the source tree. - */ - -#include "modules/video_coding/timing.h" - -#include - -#include - -#include "rtc_base/time/timestamp_extrapolator.h" -#include "system_wrappers/include/clock.h" -#include "system_wrappers/include/field_trial.h" - -namespace webrtc { - -VCMTiming::VCMTiming(Clock* clock, VCMTiming* master_timing) - : clock_(clock), - master_(false), - ts_extrapolator_(), - codec_timer_(new VCMCodecTimer()), - render_delay_ms_(kDefaultRenderDelayMs), - min_playout_delay_ms_(0), - max_playout_delay_ms_(10000), - jitter_delay_ms_(0), - current_delay_ms_(0), - prev_frame_timestamp_(0), - timing_frame_info_(), - num_decoded_frames_(0) { - if (master_timing == NULL) { - master_ = true; - ts_extrapolator_ = new TimestampExtrapolator(clock_->TimeInMilliseconds()); - } else { - ts_extrapolator_ = master_timing->ts_extrapolator_; - } -} - -VCMTiming::~VCMTiming() { - if (master_) { - delete ts_extrapolator_; - } -} - -void VCMTiming::Reset() { - rtc::CritScope cs(&crit_sect_); - ts_extrapolator_->Reset(clock_->TimeInMilliseconds()); - codec_timer_.reset(new VCMCodecTimer()); - render_delay_ms_ = kDefaultRenderDelayMs; - min_playout_delay_ms_ = 0; - jitter_delay_ms_ = 0; - current_delay_ms_ = 0; - prev_frame_timestamp_ = 0; -} - -void VCMTiming::set_render_delay(int render_delay_ms) { - rtc::CritScope cs(&crit_sect_); - render_delay_ms_ = render_delay_ms; -} - -void VCMTiming::set_min_playout_delay(int min_playout_delay_ms) { - rtc::CritScope cs(&crit_sect_); - min_playout_delay_ms_ = min_playout_delay_ms; -} - -int VCMTiming::min_playout_delay() { - rtc::CritScope cs(&crit_sect_); - return min_playout_delay_ms_; -} - -void VCMTiming::set_max_playout_delay(int max_playout_delay_ms) { - rtc::CritScope cs(&crit_sect_); - max_playout_delay_ms_ = max_playout_delay_ms; -} - -int VCMTiming::max_playout_delay() { - rtc::CritScope cs(&crit_sect_); - return max_playout_delay_ms_; -} - -void VCMTiming::SetJitterDelay(int jitter_delay_ms) { - rtc::CritScope cs(&crit_sect_); - if (jitter_delay_ms != jitter_delay_ms_) { - jitter_delay_ms_ = jitter_delay_ms; - // When in initial state, set current delay to minimum delay. - if (current_delay_ms_ == 0) { - current_delay_ms_ = jitter_delay_ms_; - } - } -} - -void VCMTiming::UpdateCurrentDelay(uint32_t frame_timestamp) { - rtc::CritScope cs(&crit_sect_); - int target_delay_ms = TargetDelayInternal(); - - if (current_delay_ms_ == 0) { - // Not initialized, set current delay to target. - current_delay_ms_ = target_delay_ms; - } else if (target_delay_ms != current_delay_ms_) { - int64_t delay_diff_ms = - static_cast(target_delay_ms) - current_delay_ms_; - // Never change the delay with more than 100 ms every second. If we're - // changing the delay in too large steps we will get noticeable freezes. By - // limiting the change we can increase the delay in smaller steps, which - // will be experienced as the video is played in slow motion. When lowering - // the delay the video will be played at a faster pace. - int64_t max_change_ms = 0; - if (frame_timestamp < 0x0000ffff && prev_frame_timestamp_ > 0xffff0000) { - // wrap - max_change_ms = kDelayMaxChangeMsPerS * - (frame_timestamp + (static_cast(1) << 32) - - prev_frame_timestamp_) / - 90000; - } else { - max_change_ms = kDelayMaxChangeMsPerS * - (frame_timestamp - prev_frame_timestamp_) / 90000; - } - - if (max_change_ms <= 0) { - // Any changes less than 1 ms are truncated and will be postponed. - // Negative change will be due to reordering and should be ignored. - return; - } - delay_diff_ms = std::max(delay_diff_ms, -max_change_ms); - delay_diff_ms = std::min(delay_diff_ms, max_change_ms); - - current_delay_ms_ = current_delay_ms_ + delay_diff_ms; - } - prev_frame_timestamp_ = frame_timestamp; -} - -void VCMTiming::UpdateCurrentDelay(int64_t render_time_ms, - int64_t actual_decode_time_ms) { - rtc::CritScope cs(&crit_sect_); - uint32_t target_delay_ms = TargetDelayInternal(); - int64_t delayed_ms = - actual_decode_time_ms - - (render_time_ms - RequiredDecodeTimeMs() - render_delay_ms_); - if (delayed_ms < 0) { - return; - } - if (current_delay_ms_ + delayed_ms <= target_delay_ms) { - current_delay_ms_ += delayed_ms; - } else { - current_delay_ms_ = target_delay_ms; - } -} - -void VCMTiming::StopDecodeTimer(uint32_t /*time_stamp*/, - int32_t decode_time_ms, - int64_t now_ms, - int64_t /*render_time_ms*/) { - StopDecodeTimer(decode_time_ms, now_ms); -} - -void VCMTiming::StopDecodeTimer(int32_t decode_time_ms, int64_t now_ms) { - rtc::CritScope cs(&crit_sect_); - codec_timer_->AddTiming(decode_time_ms, now_ms); - assert(decode_time_ms >= 0); - ++num_decoded_frames_; -} - -void VCMTiming::IncomingTimestamp(uint32_t time_stamp, int64_t now_ms) { - rtc::CritScope cs(&crit_sect_); - ts_extrapolator_->Update(now_ms, time_stamp); -} - -int64_t VCMTiming::RenderTimeMs(uint32_t frame_timestamp, - int64_t now_ms) const { - rtc::CritScope cs(&crit_sect_); - if (field_trial::IsEnabled("OWT-LowLatencyMode")) { - return 0; - } - return RenderTimeMsInternal(frame_timestamp, now_ms); -} - -int64_t VCMTiming::RenderTimeMsInternal(uint32_t frame_timestamp, - int64_t now_ms) const { - if (min_playout_delay_ms_ == 0 && max_playout_delay_ms_ == 0) { - // Render as soon as possible. - return 0; - } - int64_t estimated_complete_time_ms = - ts_extrapolator_->ExtrapolateLocalTime(frame_timestamp); - if (estimated_complete_time_ms == -1) { - estimated_complete_time_ms = now_ms; - } - - // Make sure the actual delay stays in the range of |min_playout_delay_ms_| - // and |max_playout_delay_ms_|. - int actual_delay = std::max(current_delay_ms_, min_playout_delay_ms_); - actual_delay = std::min(actual_delay, max_playout_delay_ms_); - return estimated_complete_time_ms + actual_delay; -} - -int VCMTiming::RequiredDecodeTimeMs() const { - const int decode_time_ms = codec_timer_->RequiredDecodeTimeMs(); - assert(decode_time_ms >= 0); - return decode_time_ms; -} - -int64_t VCMTiming::MaxWaitingTime(int64_t render_time_ms, - int64_t now_ms) const { - rtc::CritScope cs(&crit_sect_); - - const int64_t max_wait_time_ms = - render_time_ms - now_ms - RequiredDecodeTimeMs() - render_delay_ms_; - - return max_wait_time_ms; -} - -int VCMTiming::TargetVideoDelay() const { - rtc::CritScope cs(&crit_sect_); - return TargetDelayInternal(); -} - -int VCMTiming::TargetDelayInternal() const { - return std::max(min_playout_delay_ms_, - jitter_delay_ms_ + RequiredDecodeTimeMs() + render_delay_ms_); -} - -bool VCMTiming::GetTimings(int* max_decode_ms, - int* current_delay_ms, - int* target_delay_ms, - int* jitter_buffer_ms, - int* min_playout_delay_ms, - int* render_delay_ms) const { - rtc::CritScope cs(&crit_sect_); - *max_decode_ms = RequiredDecodeTimeMs(); - *current_delay_ms = current_delay_ms_; - *target_delay_ms = TargetDelayInternal(); - *jitter_buffer_ms = jitter_delay_ms_; - *min_playout_delay_ms = min_playout_delay_ms_; - *render_delay_ms = render_delay_ms_; - return (num_decoded_frames_ > 0); -} - -void VCMTiming::SetTimingFrameInfo(const TimingFrameInfo& info) { - rtc::CritScope cs(&crit_sect_); - timing_frame_info_.emplace(info); -} - -absl::optional VCMTiming::GetTimingFrameInfo() { - rtc::CritScope cs(&crit_sect_); - return timing_frame_info_; -} - -} // namespace webrtc diff --git a/p2p/base/port_allocator.h b/p2p/base/port_allocator.h index 5b35743869f..420560ccbbf 100644 --- a/p2p/base/port_allocator.h +++ b/p2p/base/port_allocator.h @@ -665,7 +665,6 @@ class RTC_EXPORT PortAllocator : public sigslot::has_slots<> { // if ice_credentials is nullptr. std::vector>::const_iterator FindPooledSession(const IceParameters* ice_credentials = nullptr) const; - RTC_DISALLOW_COPY_AND_ASSIGN(PortAllocator); }; } // namespace cricket diff --git a/pc/peer_connection.cc b/pc/peer_connection.cc index 2dc4e5c4e17..e05a2b730ea 100644 --- a/pc/peer_connection.cc +++ b/pc/peer_connection.cc @@ -1808,7 +1808,7 @@ void PeerConnection::Close() { } // Update stats here so that we have the most recent stats for tracks and // streams before the channels are closed. - stats_->UpdateStats(kStatsOutputLevelStandard); + legacy_stats_->UpdateStats(kStatsOutputLevelStandard); ice_connection_state_ = PeerConnectionInterface::kIceConnectionClosed; Observer()->OnIceConnectionChange(ice_connection_state_); diff --git a/pc/used_ids.h b/pc/used_ids.h index 9f65c525ec9..1236a786df6 100644 --- a/pc/used_ids.h +++ b/pc/used_ids.h @@ -168,8 +168,8 @@ class UsedRtpHeaderExtensionIds : public UsedIds { } } } - //RTC_DCHECK(next_extension_id_ >= min_allowed_id_); - //RTC_DCHECK(next_extension_id_ <= max_allowed_id_); + RTC_DCHECK(next_extension_id_ >= min_allowed_id_); + RTC_DCHECK(next_extension_id_ <= max_allowed_id_); return next_extension_id_; } diff --git a/rtc_base/experiments/min_video_bitrate_experiment.cc b/rtc_base/experiments/min_video_bitrate_experiment.cc index 88797828373..bb43e6df152 100644 --- a/rtc_base/experiments/min_video_bitrate_experiment.cc +++ b/rtc_base/experiments/min_video_bitrate_experiment.cc @@ -100,9 +100,7 @@ absl::optional GetExperimentalMinVideoBitrate(VideoCodecType type) { return min_bitrate_av1.GetOptional(); case kVideoCodecH264: return min_bitrate_h264.GetOptional(); -#ifndef DISABLE_H265 case kVideoCodecH265: -#endif case kVideoCodecGeneric: case kVideoCodecMultiplex: return absl::nullopt; diff --git a/rtc_base/ptp_clock_sync.h b/rtc_base/ptp_clock_sync.h new file mode 100644 index 00000000000..3298f3fe79e --- /dev/null +++ b/rtc_base/ptp_clock_sync.h @@ -0,0 +1,58 @@ +// Copyright (C) <2021> Intel Corporation +// +// SPDX-License-Identifier: Apache-2.0 + +#ifndef RTC_BASE_TIME_PTP_CLOCK_SYNC_H_ +#define RTC_BASE_TIME_PTP_CLOCK_SYNC_H_ + +#include + +#include + +#define MICROSECONDS_FACTOR 1000000.0 +#define OFFSET_FACTOR 200000 +#define SERVER_FREQUENCY 0.09 // RTP/NTP timestamp runs at 90KHz clock + +// A Windows implementation for PTP using timestamp from RTP and local +// timestamp. We may need to implement something like QueryPerformanceFrequency +// for this to work on Linux platforms. +namespace webrtc { +class PTPClockSync { + public: + PTPClockSync() + : m_server_point(0), m_server_freq(0), m_client_point(0), m_last_ts(0) { + uint64_t freq; // Performance counter frequency in a second. + QueryPerformanceFrequency((LARGE_INTEGER*)&freq); + m_client_freq = (double)freq / MICROSECONDS_FACTOR; + m_server_freq = SERVER_FREQUENCY; + } + ~PTPClockSync() {} + + void Sync(uint32_t ts, uint64_t tc) { + if (GetDuration(ts, tc) < 0 || + (ts - m_last_ts) > OFFSET_FACTOR * m_server_freq) { + UpdateSync(ts, tc); + } + m_last_ts = ts; + } + + double GetDuration(uint32_t ts, uint64_t tc) { + int ds = (int)(ts - m_server_point); + int dc = (int)(tc - m_client_point); + return (double)dc / m_client_freq - (double)ds / m_server_freq; + } + + protected: + uint32_t m_server_point; + double m_server_freq; // count per us + uint64_t m_client_point; + double m_client_freq; // count per us + uint32_t m_last_ts; + + void UpdateSync(uint32_t ts, uint64_t tc) { + m_client_point = tc; + m_server_point = ts; + } +}; +} // namespace webrtc +#endif \ No newline at end of file diff --git a/sdk/BUILD.gn b/sdk/BUILD.gn index a0ddaa6307a..8db640564f0 100644 --- a/sdk/BUILD.gn +++ b/sdk/BUILD.gn @@ -1406,7 +1406,6 @@ if (is_ios || is_mac) { ":videocodec_objc", ":videotoolbox_objc", "../api:create_peerconnection_factory", - "../api:create_peerconnection_factory", ] if (rtc_ios_macos_use_opengl_rendering) { deps += [ ":opengl_ui_objc" ] diff --git a/sdk/android/src/jni/android_media_codec_common.h b/sdk/android/src/jni/android_media_codec_common.h deleted file mode 100644 index 1f024ef78e5..00000000000 --- a/sdk/android/src/jni/android_media_codec_common.h +++ /dev/null @@ -1,90 +0,0 @@ -/* - * Copyright 2015 The WebRTC project authors. All Rights Reserved. - * - * Use of this source code is governed by a BSD-style license - * that can be found in the LICENSE file in the root of the source - * tree. An additional intellectual property rights grant can be found - * in the file PATENTS. All contributing project authors may - * be found in the AUTHORS file in the root of the source tree. - */ - -#ifndef SDK_ANDROID_SRC_JNI_ANDROID_MEDIA_CODEC_COMMON_H_ -#define SDK_ANDROID_SRC_JNI_ANDROID_MEDIA_CODEC_COMMON_H_ - -#include - -#include "rtc_base/logging.h" -#include "rtc_base/thread.h" -#include "sdk/android/src/jni/jni_helpers.h" - -namespace webrtc { -namespace jni { - -// Uncomment this define to enable verbose logging for every encoded/decoded -// video frame. -//#define TRACK_BUFFER_TIMING - -#define TAG_COMMON "MediaCodecVideo" - -// Color formats supported by encoder or decoder - should include all -// colors from supportedColorList in MediaCodecVideoEncoder.java and -// MediaCodecVideoDecoder.java. Supported color format set in encoder -// and decoder could be different. -enum COLOR_FORMATTYPE { - COLOR_FormatYUV420Planar = 0x13, - COLOR_FormatYUV420SemiPlanar = 0x15, - COLOR_QCOM_FormatYUV420SemiPlanar = 0x7FA30C00, - // NV12 color format supported by QCOM codec, but not declared in MediaCodec - - // see /hardware/qcom/media/mm-core/inc/OMX_QCOMExtns.h - // This format is presumably similar to COLOR_FormatYUV420SemiPlanar, - // but requires some (16, 32?) byte alignment. - COLOR_QCOM_FORMATYVU420PackedSemiPlanar32m4ka = 0x7FA30C01, - COLOR_QCOM_FORMATYVU420PackedSemiPlanar16m4ka = 0x7FA30C02, - COLOR_QCOM_FORMATYVU420PackedSemiPlanar64x32Tile2m8ka = 0x7FA30C03, - COLOR_QCOM_FORMATYUV420PackedSemiPlanar32m = 0x7FA30C04 -}; - -// Arbitrary interval to poll the codec for new outputs. -enum { kMediaCodecPollMs = 10 }; -// Arbitrary interval to poll at when there should be no more frames. -enum { kMediaCodecPollNoFramesMs = 100 }; -// Media codec maximum output buffer ready timeout. -enum { kMediaCodecTimeoutMs = 1000 }; -// Interval to print codec statistics (bitrate, fps, encoding/decoding time). -enum { kMediaCodecStatisticsIntervalMs = 3000 }; -// Maximum amount of pending frames for VP8 decoder. -enum { kMaxPendingFramesVp8 = 1 }; -// Maximum amount of pending frames for VP9 decoder. -enum { kMaxPendingFramesVp9 = 1 }; -// Maximum amount of pending frames for H.264 decoder. -enum { kMaxPendingFramesH264 = 4 }; -// TODO(zhanghe): update the value after checking more devices if needed. -// Maximum amount of pending frames for H.265 decoder. -enum { kMaxPendingFramesH265 = 4 }; -// Maximum amount of decoded frames for which per-frame logging is enabled. -enum { kMaxDecodedLogFrames = 10 }; -// Maximum amount of encoded frames for which per-frame logging is enabled. -enum { kMaxEncodedLogFrames = 10 }; - -static inline void AllowBlockingCalls() { - rtc::Thread* current_thread = rtc::Thread::Current(); - if (current_thread != NULL) - current_thread->DEPRECATED_AllowBlockingCalls(); -} - -// Checks for any Java exception, prints stack backtrace and clears -// currently thrown exception. -static inline bool CheckException(JNIEnv* jni) { - if (jni->ExceptionCheck()) { - RTC_LOG_TAG(rtc::LS_ERROR, TAG_COMMON) << "Java JNI exception."; - jni->ExceptionDescribe(); - jni->ExceptionClear(); - return true; - } - return false; -} - -} // namespace jni -} // namespace webrtc - -#endif // SDK_ANDROID_SRC_JNI_ANDROID_MEDIA_CODEC_COMMON_H_ diff --git a/test/scenario/video_stream.cc b/test/scenario/video_stream.cc index d16b3a80f8c..d6d1e45626d 100644 --- a/test/scenario/video_stream.cc +++ b/test/scenario/video_stream.cc @@ -204,21 +204,6 @@ CreateH264SpecificSettings(VideoStreamConfig config) { return nullptr; } -#ifdef WEBRTC_USE_H265 -rtc::scoped_refptr -CreateH265SpecificSettings(VideoStreamConfig config) { - RTC_DCHECK_EQ(config.encoder.layers.temporal, 1); - RTC_DCHECK_EQ(config.encoder.layers.spatial, 1); - - VideoCodecH265 h265_settings = VideoEncoder::GetDefaultH265Settings(); - h265_settings.frameDroppingOn = config.encoder.frame_dropping; - h265_settings.keyFrameInterval = - config.encoder.key_frame_interval.value_or(0); - return new rtc::RefCountedObject< - VideoEncoderConfig::H264EncoderSpecificSettings>(h264_settings); -} - -#ifdef WEBRTC_USE_H265 rtc::scoped_refptr CreateH265SpecificSettings(VideoStreamConfig config) { RTC_DCHECK_EQ(config.encoder.layers.temporal, 1); @@ -231,7 +216,6 @@ CreateH265SpecificSettings(VideoStreamConfig config) { return new rtc::RefCountedObject< VideoEncoderConfig::H265EncoderSpecificSettings>(h265_settings); } -#endif rtc::scoped_refptr CreateEncoderSpecificSettings(VideoStreamConfig config) { diff --git a/test/time_controller/simulated_process_thread.cc b/test/time_controller/simulated_process_thread.cc deleted file mode 100644 index 620c1a1d534..00000000000 --- a/test/time_controller/simulated_process_thread.cc +++ /dev/null @@ -1,215 +0,0 @@ -/* - * Copyright (c) 2020 The WebRTC project authors. All Rights Reserved. - * - * Use of this source code is governed by a BSD-style license - * that can be found in the LICENSE file in the root of the source - * tree. An additional intellectual property rights grant can be found - * in the file PATENTS. All contributing project authors may - * be found in the AUTHORS file in the root of the source tree. - */ - -#include "test/time_controller/simulated_process_thread.h" - -#include -#include - -namespace webrtc { -namespace { -// Helper function to remove from a std container by value. -template -bool RemoveByValue(C* vec, typename C::value_type val) { - auto it = std::find(vec->begin(), vec->end(), val); - if (it == vec->end()) - return false; - vec->erase(it); - return true; -} -} // namespace -SimulatedProcessThread::SimulatedProcessThread( - sim_time_impl::SimulatedTimeControllerImpl* handler, - absl::string_view name) - : handler_(handler), name_(new char[name.size()]) { - std::copy_n(name.begin(), name.size(), name_); -} - -SimulatedProcessThread::~SimulatedProcessThread() { - handler_->Unregister(this); - delete[] name_; -} - -void SimulatedProcessThread::RunReady(Timestamp at_time) { - CurrentTaskQueueSetter set_current(this); - rtc::CritScope lock(&lock_); - std::vector ready_modules; - for (auto it = delayed_modules_.begin(); - it != delayed_modules_.end() && it->first <= at_time; - it = delayed_modules_.erase(it)) { - for (auto module : it->second) { - ready_modules.push_back(module); - } - } - for (auto* module : ready_modules) { - module->Process(); - delayed_modules_[GetNextTime(module, at_time)].push_back(module); - } - - for (auto it = delayed_tasks_.begin(); - it != delayed_tasks_.end() && it->first <= at_time; - it = delayed_tasks_.erase(it)) { - for (auto& task : it->second) { - queue_.push_back(std::move(task)); - } - } - while (!queue_.empty()) { - std::unique_ptr task = std::move(queue_.front()); - queue_.pop_front(); - lock_.Leave(); - bool should_delete = task->Run(); - RTC_CHECK(should_delete); - lock_.Enter(); - } - RTC_DCHECK(queue_.empty()); - if (!delayed_modules_.empty()) { - next_run_time_ = delayed_modules_.begin()->first; - } else { - next_run_time_ = Timestamp::PlusInfinity(); - } - if (!delayed_tasks_.empty()) { - next_run_time_ = std::min(next_run_time_, delayed_tasks_.begin()->first); - } -} -void SimulatedProcessThread::Start() { - std::vector starting; - { - rtc::CritScope lock(&lock_); - if (process_thread_running_) - return; - process_thread_running_ = true; - starting.swap(stopped_modules_); - } - for (auto& module : starting) - module->ProcessThreadAttached(this); - - Timestamp at_time = handler_->CurrentTime(); - rtc::CritScope lock(&lock_); - for (auto& module : starting) - delayed_modules_[GetNextTime(module, at_time)].push_back(module); - - if (!queue_.empty()) { - next_run_time_ = Timestamp::MinusInfinity(); - } else if (!delayed_modules_.empty()) { - next_run_time_ = delayed_modules_.begin()->first; - } else { - next_run_time_ = Timestamp::PlusInfinity(); - } -} - -void SimulatedProcessThread::StartWithHighPriority() { - // same implementation as Start(); - std::vector starting; - { - rtc::CritScope lock(&lock_); - if (process_thread_running_) - return; - process_thread_running_ = true; - starting.swap(stopped_modules_); - } - for (auto& module : starting) - module->ProcessThreadAttached(this); - - Timestamp at_time = handler_->CurrentTime(); - rtc::CritScope lock(&lock_); - for (auto& module : starting) - delayed_modules_[GetNextTime(module, at_time)].push_back(module); - - if (!queue_.empty()) { - next_run_time_ = Timestamp::MinusInfinity(); - } else if (!delayed_modules_.empty()) { - next_run_time_ = delayed_modules_.begin()->first; - } else { - next_run_time_ = Timestamp::PlusInfinity(); - } -} - -void SimulatedProcessThread::Stop() { - std::vector stopping; - { - rtc::CritScope lock(&lock_); - process_thread_running_ = false; - - for (auto& delayed : delayed_modules_) { - for (auto mod : delayed.second) - stopped_modules_.push_back(mod); - } - delayed_modules_.clear(); - - stopping = stopped_modules_; - } - for (auto& module : stopping) - module->ProcessThreadAttached(nullptr); -} - -void SimulatedProcessThread::WakeUp(Module* module) { - rtc::CritScope lock(&lock_); - for (auto it = delayed_modules_.begin(); it != delayed_modules_.end(); ++it) { - if (RemoveByValue(&it->second, module)) - break; - } - Timestamp next_time = GetNextTime(module, handler_->CurrentTime()); - delayed_modules_[next_time].push_back(module); - next_run_time_ = std::min(next_run_time_, next_time); -} - -void SimulatedProcessThread::RegisterModule(Module* module, - const rtc::Location& from) { - module->ProcessThreadAttached(this); - rtc::CritScope lock(&lock_); - if (!process_thread_running_) { - stopped_modules_.push_back(module); - } else { - Timestamp next_time = GetNextTime(module, handler_->CurrentTime()); - delayed_modules_[next_time].push_back(module); - next_run_time_ = std::min(next_run_time_, next_time); - } -} - -void SimulatedProcessThread::DeRegisterModule(Module* module) { - bool modules_running; - { - rtc::CritScope lock(&lock_); - if (!process_thread_running_) { - RemoveByValue(&stopped_modules_, module); - } else { - for (auto& pair : delayed_modules_) { - if (RemoveByValue(&pair.second, module)) - break; - } - } - modules_running = process_thread_running_; - } - if (modules_running) - module->ProcessThreadAttached(nullptr); -} - -void SimulatedProcessThread::PostTask(std::unique_ptr task) { - rtc::CritScope lock(&lock_); - queue_.emplace_back(std::move(task)); - next_run_time_ = Timestamp::MinusInfinity(); -} - -void SimulatedProcessThread::PostDelayedTask(std::unique_ptr task, - uint32_t milliseconds) { - rtc::CritScope lock(&lock_); - Timestamp target_time = - handler_->CurrentTime() + TimeDelta::Millis(milliseconds); - delayed_tasks_[target_time].push_back(std::move(task)); - next_run_time_ = std::min(next_run_time_, target_time); -} - -Timestamp SimulatedProcessThread::GetNextTime(Module* module, - Timestamp at_time) { - CurrentTaskQueueSetter set_current(this); - return at_time + TimeDelta::Millis(module->TimeUntilNextProcess()); -} - -} // namespace webrtc diff --git a/test/time_controller/simulated_process_thread.h b/test/time_controller/simulated_process_thread.h deleted file mode 100644 index 63e3cc496f6..00000000000 --- a/test/time_controller/simulated_process_thread.h +++ /dev/null @@ -1,73 +0,0 @@ -/* - * Copyright (c) 2020 The WebRTC project authors. All Rights Reserved. - * - * Use of this source code is governed by a BSD-style license - * that can be found in the LICENSE file in the root of the source - * tree. An additional intellectual property rights grant can be found - * in the file PATENTS. All contributing project authors may - * be found in the AUTHORS file in the root of the source tree. - */ -#ifndef TEST_TIME_CONTROLLER_SIMULATED_PROCESS_THREAD_H_ -#define TEST_TIME_CONTROLLER_SIMULATED_PROCESS_THREAD_H_ - -#include -#include -#include -#include -#include - -#include "test/time_controller/simulated_time_controller.h" - -namespace webrtc { - -class SimulatedProcessThread : public ProcessThread, - public sim_time_impl::SimulatedSequenceRunner { - public: - SimulatedProcessThread(sim_time_impl::SimulatedTimeControllerImpl* handler, - absl::string_view name); - virtual ~SimulatedProcessThread(); - void RunReady(Timestamp at_time) override; - - Timestamp GetNextRunTime() const override { - rtc::CritScope lock(&lock_); - return next_run_time_; - } - - TaskQueueBase* GetAsTaskQueue() override { return this; } - - // ProcessThread interface - void Start() override; - void StartWithHighPriority() override; - void Stop() override; - void WakeUp(Module* module) override; - void RegisterModule(Module* module, const rtc::Location& from) override; - void DeRegisterModule(Module* module) override; - void PostTask(std::unique_ptr task) override; - void PostDelayedTask(std::unique_ptr task, - uint32_t milliseconds) override; - - private: - void Delete() override { - // ProcessThread shouldn't be deleted as a TaskQueue. - RTC_NOTREACHED(); - } - Timestamp GetNextTime(Module* module, Timestamp at_time); - - sim_time_impl::SimulatedTimeControllerImpl* const handler_; - // Using char* to be debugger friendly. - char* name_; - rtc::CriticalSection lock_; - Timestamp next_run_time_ RTC_GUARDED_BY(lock_) = Timestamp::PlusInfinity(); - - std::deque> queue_; - std::map>> delayed_tasks_ - RTC_GUARDED_BY(lock_); - - bool process_thread_running_ RTC_GUARDED_BY(lock_) = false; - std::vector stopped_modules_ RTC_GUARDED_BY(lock_); - std::map> delayed_modules_ - RTC_GUARDED_BY(lock_); -}; -} // namespace webrtc - -#endif // TEST_TIME_CONTROLLER_SIMULATED_PROCESS_THREAD_H_ diff --git a/video/config/video_encoder_config.cc b/video/config/video_encoder_config.cc index cc4d67e5621..add489ef03a 100644 --- a/video/config/video_encoder_config.cc +++ b/video/config/video_encoder_config.cc @@ -126,17 +126,6 @@ void VideoEncoderConfig::H265EncoderSpecificSettings::FillVideoCodecH265( *h265_settings = specifics_; } -#ifndef DISABLE_H265 -VideoEncoderConfig::H265EncoderSpecificSettings::H265EncoderSpecificSettings( - const VideoCodecH265& specifics) - : specifics_(specifics) {} - -void VideoEncoderConfig::H265EncoderSpecificSettings::FillVideoCodecH265( - VideoCodecH265* h265_settings) const { - *h265_settings = specifics_; -} -#endif - VideoEncoderConfig::Vp8EncoderSpecificSettings::Vp8EncoderSpecificSettings( const VideoCodecVP8& specifics) : specifics_(specifics) {} diff --git a/video/rtp_video_stream_receiver.h b/video/rtp_video_stream_receiver.h deleted file mode 100644 index 0b7f9e7a47f..00000000000 --- a/video/rtp_video_stream_receiver.h +++ /dev/null @@ -1,379 +0,0 @@ -/* - * Copyright (c) 2012 The WebRTC project authors. All Rights Reserved. - * - * Use of this source code is governed by a BSD-style license - * that can be found in the LICENSE file in the root of the source - * tree. An additional intellectual property rights grant can be found - * in the file PATENTS. All contributing project authors may - * be found in the AUTHORS file in the root of the source tree. - */ - -#ifndef VIDEO_RTP_VIDEO_STREAM_RECEIVER_H_ -#define VIDEO_RTP_VIDEO_STREAM_RECEIVER_H_ - -#include -#include -#include -#include -#include -#include - -#include "absl/types/optional.h" -#include "api/array_view.h" -#include "api/crypto/frame_decryptor_interface.h" -#include "api/video/color_space.h" -#include "api/video_codecs/video_codec.h" -#include "call/rtp_packet_sink_interface.h" -#include "call/syncable.h" -#include "call/video_receive_stream.h" -#include "modules/rtp_rtcp/include/receive_statistics.h" -#include "modules/rtp_rtcp/include/remote_ntp_time_estimator.h" -#include "modules/rtp_rtcp/include/rtp_header_extension_map.h" -#include "modules/rtp_rtcp/include/rtp_rtcp.h" -#include "modules/rtp_rtcp/include/rtp_rtcp_defines.h" -#include "modules/rtp_rtcp/source/absolute_capture_time_receiver.h" -#include "modules/rtp_rtcp/source/rtp_dependency_descriptor_extension.h" -#include "modules/rtp_rtcp/source/rtp_packet_received.h" -#include "modules/rtp_rtcp/source/rtp_video_header.h" -#include "modules/rtp_rtcp/source/video_rtp_depacketizer.h" -#include "modules/video_coding/h264_sps_pps_tracker.h" -#include "modules/video_coding/loss_notification_controller.h" -#ifndef DISABLE_H265 -#include "modules/video_coding/h265_vps_sps_pps_tracker.h" -#endif -#include "modules/video_coding/packet_buffer.h" -#include "modules/video_coding/rtp_frame_reference_finder.h" -#include "modules/video_coding/unique_timestamp_counter.h" -#include "rtc_base/constructor_magic.h" -#include "rtc_base/critical_section.h" -#include "rtc_base/numerics/sequence_number_util.h" -#if defined(WEBRTC_WIN) -#include "rtc_base/time/ptp_clock_sync.h" -#endif -#include "rtc_base/synchronization/sequence_checker.h" -#include "rtc_base/thread_annotations.h" -#include "rtc_base/thread_checker.h" -#include "video/buffered_frame_decryptor.h" -#include "video/rtp_video_stream_receiver_frame_transformer_delegate.h" - -namespace webrtc { - -class NackModule; -class PacketRouter; -class ProcessThread; -class ReceiveStatistics; -class ReceiveStatisticsProxy; -class RtcpRttStats; -class RtpPacketReceived; -class Transport; -class UlpfecReceiver; - -class RtpVideoStreamReceiver : public LossNotificationSender, - public RecoveredPacketReceiver, - public RtpPacketSinkInterface, - public KeyFrameRequestSender, - public video_coding::OnCompleteFrameCallback, - public OnDecryptedFrameCallback, - public OnDecryptionStatusChangeCallback { - public: - RtpVideoStreamReceiver( - Clock* clock, - Transport* transport, - RtcpRttStats* rtt_stats, - // The packet router is optional; if provided, the RtpRtcp module for this - // stream is registered as a candidate for sending REMB and transport - // feedback. - PacketRouter* packet_router, - const VideoReceiveStream::Config* config, - ReceiveStatistics* rtp_receive_statistics, - ReceiveStatisticsProxy* receive_stats_proxy, - ProcessThread* process_thread, - NackSender* nack_sender, - // The KeyFrameRequestSender is optional; if not provided, key frame - // requests are sent via the internal RtpRtcp module. - KeyFrameRequestSender* keyframe_request_sender, - video_coding::OnCompleteFrameCallback* complete_frame_callback, - rtc::scoped_refptr frame_decryptor, - rtc::scoped_refptr frame_transformer); - ~RtpVideoStreamReceiver() override; - - void AddReceiveCodec(const VideoCodec& video_codec, - const std::map& codec_params, - bool raw_payload); - - void StartReceive(); - void StopReceive(); - - // Produces the transport-related timestamps; current_delay_ms is left unset. - absl::optional GetSyncInfo() const; - - bool DeliverRtcp(const uint8_t* rtcp_packet, size_t rtcp_packet_length); - - void FrameContinuous(int64_t seq_num); - - void FrameDecoded(int64_t seq_num); - - void SignalNetworkState(NetworkState state); - - // Returns number of different frames seen. - int GetUniqueFramesSeen() const { - RTC_DCHECK_RUN_ON(&worker_task_checker_); - return frame_counter_.GetUniqueSeen(); - } - - // Implements RtpPacketSinkInterface. - void OnRtpPacket(const RtpPacketReceived& packet) override; - - // TODO(philipel): Stop using VCMPacket in the new jitter buffer and then - // remove this function. Public only for tests. - void OnReceivedPayloadData(rtc::CopyOnWriteBuffer codec_payload, - const RtpPacketReceived& rtp_packet, - const RTPVideoHeader& video); - - // Implements RecoveredPacketReceiver. - void OnRecoveredPacket(const uint8_t* packet, size_t packet_length) override; - - // Send an RTCP keyframe request. - void RequestKeyFrame() override; - - // Implements LossNotificationSender. - void SendLossNotification(uint16_t last_decoded_seq_num, - uint16_t last_received_seq_num, - bool decodability_flag, - bool buffering_allowed) override; - - bool IsUlpfecEnabled() const; - bool IsRetransmissionsEnabled() const; - - // Returns true if a decryptor is attached and frames can be decrypted. - // Updated by OnDecryptionStatusChangeCallback. Note this refers to Frame - // Decryption not SRTP. - bool IsDecryptable() const; - - // Don't use, still experimental. - void RequestPacketRetransmit(const std::vector& sequence_numbers); - - // Implements OnCompleteFrameCallback. - void OnCompleteFrame( - std::unique_ptr frame) override; - - // Implements OnDecryptedFrameCallback. - void OnDecryptedFrame( - std::unique_ptr frame) override; - - // Implements OnDecryptionStatusChangeCallback. - void OnDecryptionStatusChange( - FrameDecryptorInterface::Status status) override; - - // Optionally set a frame decryptor after a stream has started. This will not - // reset the decoder state. - void SetFrameDecryptor( - rtc::scoped_refptr frame_decryptor); - - // Sets a frame transformer after a stream has started, if no transformer - // has previously been set. Does not reset the decoder state. - void SetDepacketizerToDecoderFrameTransformer( - rtc::scoped_refptr frame_transformer); - - // Called by VideoReceiveStream when stats are updated. - void UpdateRtt(int64_t max_rtt_ms); - - absl::optional LastReceivedPacketMs() const; - absl::optional LastReceivedKeyframePacketMs() const; - - // RtpDemuxer only forwards a given RTP packet to one sink. However, some - // sinks, such as FlexFEC, might wish to be informed of all of the packets - // a given sink receives (or any set of sinks). They may do so by registering - // themselves as secondary sinks. - void AddSecondarySink(RtpPacketSinkInterface* sink); - void RemoveSecondarySink(const RtpPacketSinkInterface* sink); - - virtual void ManageFrame(std::unique_ptr frame); - - private: - // Used for buffering RTCP feedback messages and sending them all together. - // Note: - // 1. Key frame requests and NACKs are mutually exclusive, with the - // former taking precedence over the latter. - // 2. Loss notifications are orthogonal to either. (That is, may be sent - // alongside either.) - class RtcpFeedbackBuffer : public KeyFrameRequestSender, - public NackSender, - public LossNotificationSender { - public: - RtcpFeedbackBuffer(KeyFrameRequestSender* key_frame_request_sender, - NackSender* nack_sender, - LossNotificationSender* loss_notification_sender); - - ~RtcpFeedbackBuffer() override = default; - - // KeyFrameRequestSender implementation. - void RequestKeyFrame() override; - - // NackSender implementation. - void SendNack(const std::vector& sequence_numbers, - bool buffering_allowed) override; - - // LossNotificationSender implementation. - void SendLossNotification(uint16_t last_decoded_seq_num, - uint16_t last_received_seq_num, - bool decodability_flag, - bool buffering_allowed) override; - - // Send all RTCP feedback messages buffered thus far. - void SendBufferedRtcpFeedback(); - - private: - KeyFrameRequestSender* const key_frame_request_sender_; - NackSender* const nack_sender_; - LossNotificationSender* const loss_notification_sender_; - - // NACKs are accessible from two threads due to nack_module_ being a module. - rtc::CriticalSection cs_; - - // Key-frame-request-related state. - bool request_key_frame_ RTC_GUARDED_BY(cs_); - - // NACK-related state. - std::vector nack_sequence_numbers_ RTC_GUARDED_BY(cs_); - - // LNTF-related state. - struct LossNotificationState { - LossNotificationState(uint16_t last_decoded_seq_num, - uint16_t last_received_seq_num, - bool decodability_flag) - : last_decoded_seq_num(last_decoded_seq_num), - last_received_seq_num(last_received_seq_num), - decodability_flag(decodability_flag) {} - - uint16_t last_decoded_seq_num; - uint16_t last_received_seq_num; - bool decodability_flag; - }; - absl::optional lntf_state_ RTC_GUARDED_BY(cs_); - }; - enum ParseGenericDependenciesResult { - kDropPacket, - kHasGenericDescriptor, - kNoGenericDescriptor - }; - - // Entry point doing non-stats work for a received packet. Called - // for the same packet both before and after RED decapsulation. - void ReceivePacket(const RtpPacketReceived& packet); - // Parses and handles RED headers. - // This function assumes that it's being called from only one thread. - void ParseAndHandleEncapsulatingHeader(const RtpPacketReceived& packet); - void NotifyReceiverOfEmptyPacket(uint16_t seq_num); - void UpdateHistograms(); - bool IsRedEnabled() const; - void InsertSpsPpsIntoTracker(uint8_t payload_type); - void OnInsertedPacket(video_coding::PacketBuffer::InsertResult result); - ParseGenericDependenciesResult ParseGenericDependenciesExtension( - const RtpPacketReceived& rtp_packet, - RTPVideoHeader* video_header) RTC_RUN_ON(worker_task_checker_); - void OnAssembledFrame(std::unique_ptr frame); - - Clock* const clock_; -#if defined(WEBRTC_WIN) - PTPClockSync clock_sync_; -#endif - // Ownership of this object lies with VideoReceiveStream, which owns |this|. - const VideoReceiveStream::Config& config_; - PacketRouter* const packet_router_; - ProcessThread* const process_thread_; - - RemoteNtpTimeEstimator ntp_estimator_; - - RtpHeaderExtensionMap rtp_header_extensions_; - ReceiveStatistics* const rtp_receive_statistics_; - std::unique_ptr ulpfec_receiver_; - - SequenceChecker worker_task_checker_; - bool receiving_ RTC_GUARDED_BY(worker_task_checker_); - int64_t last_packet_log_ms_ RTC_GUARDED_BY(worker_task_checker_); - - const std::unique_ptr rtp_rtcp_; - - video_coding::OnCompleteFrameCallback* complete_frame_callback_; - KeyFrameRequestSender* const keyframe_request_sender_; - - RtcpFeedbackBuffer rtcp_feedback_buffer_; - std::unique_ptr nack_module_; - std::unique_ptr loss_notification_controller_; - - video_coding::PacketBuffer packet_buffer_; - UniqueTimestampCounter frame_counter_ RTC_GUARDED_BY(worker_task_checker_); - SeqNumUnwrapper frame_id_unwrapper_ - RTC_GUARDED_BY(worker_task_checker_); - - // Video structure provided in the dependency descriptor in a first packet - // of a key frame. It is required to parse dependency descriptor in the - // following delta packets. - std::unique_ptr video_structure_ - RTC_GUARDED_BY(worker_task_checker_); - // Frame id of the last frame with the attached video structure. - // absl::nullopt when `video_structure_ == nullptr`; - absl::optional video_structure_frame_id_ - RTC_GUARDED_BY(worker_task_checker_); - - rtc::CriticalSection reference_finder_lock_; - std::unique_ptr reference_finder_ - RTC_GUARDED_BY(reference_finder_lock_); - absl::optional current_codec_; - uint32_t last_assembled_frame_rtp_timestamp_; - - rtc::CriticalSection last_seq_num_cs_; - std::map last_seq_num_for_pic_id_ - RTC_GUARDED_BY(last_seq_num_cs_); - video_coding::H264SpsPpsTracker tracker_; - - // Maps payload id to the depacketizer. - std::map> payload_type_map_; - - -#ifndef DISABLE_H265 - video_coding::H265VpsSpsPpsTracker h265_tracker_; -#endif - - // TODO(johan): Remove pt_codec_params_ once - // https://bugs.chromium.org/p/webrtc/issues/detail?id=6883 is resolved. - // Maps a payload type to a map of out-of-band supplied codec parameters. - std::map> pt_codec_params_; - int16_t last_payload_type_ = -1; - - bool has_received_frame_; - - std::vector secondary_sinks_ - RTC_GUARDED_BY(worker_task_checker_); - - // Info for GetSyncInfo is updated on network or worker thread, and queried on - // the worker thread. - rtc::CriticalSection sync_info_lock_; - absl::optional last_received_rtp_timestamp_ - RTC_GUARDED_BY(sync_info_lock_); - absl::optional last_received_rtp_system_time_ms_ - RTC_GUARDED_BY(sync_info_lock_); - - // Used to validate the buffered frame decryptor is always run on the correct - // thread. - rtc::ThreadChecker network_tc_; - // Handles incoming encrypted frames and forwards them to the - // rtp_reference_finder if they are decryptable. - std::unique_ptr buffered_frame_decryptor_ - RTC_PT_GUARDED_BY(network_tc_); - std::atomic frames_decryptable_; - absl::optional last_color_space_; - - AbsoluteCaptureTimeReceiver absolute_capture_time_receiver_ - RTC_GUARDED_BY(worker_task_checker_); - - int64_t last_completed_picture_id_ = 0; - - rtc::scoped_refptr - frame_transformer_delegate_; -}; - -} // namespace webrtc - -#endif // VIDEO_RTP_VIDEO_STREAM_RECEIVER_H_ diff --git a/video/rtp_video_stream_receiver2.cc b/video/rtp_video_stream_receiver2.cc index c1216e5b7c6..415170800d2 100644 --- a/video/rtp_video_stream_receiver2.cc +++ b/video/rtp_video_stream_receiver2.cc @@ -39,6 +39,7 @@ #include "modules/video_coding/frame_object.h" #include "modules/video_coding/h264_sprop_parameter_sets.h" #include "modules/video_coding/h264_sps_pps_tracker.h" +#include "modules/video_coding/h265_vps_sps_pps_tracker.h" #include "modules/video_coding/nack_requester.h" #include "modules/video_coding/packet_buffer.h" #include "rtc_base/checks.h" @@ -687,7 +688,6 @@ void RtpVideoStreamReceiver2::OnReceivedPayloadData( packet->video_payload = std::move(fixed.bitstream); break; } -#ifdef WEBRTC_USE_H265 } else if (packet->codec() == kVideoCodecH265) { // Only when we start to receive packets will we know what payload type // that will be used. When we know the payload type insert the correct @@ -713,9 +713,7 @@ void RtpVideoStreamReceiver2::OnReceivedPayloadData( packet->video_payload = std::move(fixed.bitstream); break; } - } -#endif - else { + } else { packet->video_payload = std::move(codec_payload); } @@ -865,8 +863,8 @@ void RtpVideoStreamReceiver2::OnInsertedPacket( } const video_coding::PacketBuffer::Packet& last_packet = *packet; - std::unique_ptr frame = - std::make_unique( + std::unique_ptr frame = + std::make_unique( first_packet->seq_num, // last_packet.seq_num, // last_packet.marker_bit, // diff --git a/video/rtp_video_stream_receiver2.h b/video/rtp_video_stream_receiver2.h index 7413ef650ad..6a89bf7e3e8 100644 --- a/video/rtp_video_stream_receiver2.h +++ b/video/rtp_video_stream_receiver2.h @@ -38,14 +38,15 @@ #include "modules/rtp_rtcp/source/rtp_video_header.h" #include "modules/rtp_rtcp/source/video_rtp_depacketizer.h" #include "modules/video_coding/h264_sps_pps_tracker.h" -#include "modules/video_coding/loss_notification_controller.h" -#ifdef WEBRTC_USE_H265 #include "modules/video_coding/h265_vps_sps_pps_tracker.h" -#endif +#include "modules/video_coding/loss_notification_controller.h" #include "modules/video_coding/nack_requester.h" #include "modules/video_coding/packet_buffer.h" #include "modules/video_coding/rtp_frame_reference_finder.h" #include "rtc_base/experiments/field_trial_parser.h" +#if defined(WEBRTC_WIN) +#include "rtc_base/ptp_clock_sync.h" +#endif #include "rtc_base/numerics/sequence_number_util.h" #include "rtc_base/system/no_unique_address.h" #include "rtc_base/thread_annotations.h" @@ -319,6 +320,9 @@ class RtpVideoStreamReceiver2 : public LossNotificationSender, const FieldTrialsView& field_trials_; TaskQueueBase* const worker_queue_; Clock* const clock_; +#if defined(WEBRTC_WIN) + PTPClockSync clock_sync_; +#endif // Ownership of this object lies with VideoReceiveStreamInterface, which owns // `this`. const VideoReceiveStreamInterface::Config& config_; @@ -392,15 +396,13 @@ class RtpVideoStreamReceiver2 : public LossNotificationSender, RTC_GUARDED_BY(packet_sequence_checker_); video_coding::H264SpsPpsTracker tracker_ RTC_GUARDED_BY(packet_sequence_checker_); + video_coding::H265VpsSpsPpsTracker h265_tracker_ + RTC_GUARDED_BY(packet_sequence_checker_); // Maps payload id to the depacketizer. std::map> payload_type_map_ RTC_GUARDED_BY(packet_sequence_checker_); -#ifdef WEBRTC_USE_H265 - video_coding::H265VpsSpsPpsTracker h265_tracker_; -#endif - // TODO(johan): Remove pt_codec_params_ once // https://bugs.chromium.org/p/webrtc/issues/detail?id=6883 is resolved. // Maps a payload type to a map of out-of-band supplied codec parameters. diff --git a/video/rtp_video_stream_receiver2_unittest.cc b/video/rtp_video_stream_receiver2_unittest.cc index b9e834fbc6f..325188e93c1 100644 --- a/video/rtp_video_stream_receiver2_unittest.cc +++ b/video/rtp_video_stream_receiver2_unittest.cc @@ -450,10 +450,10 @@ TEST_F(RtpVideoStreamReceiver2Test, NoInfiniteRecursionOnEncapsulatedRedPacket) { const std::vector data({ 0x80, // RTP version. - static_cast(kRedPayloadType), // Payload type. + kRedPayloadType, // Payload type. 0, 0, 0, 0, 0, 0, // Don't care. 0, 0, 0x4, 0x57, // SSRC - static_cast(kRedPayloadType), // RED header. + kRedPayloadType, // RED header. 0, 0, 0, 0, 0 // Don't care. }); RtpPacketReceived packet; diff --git a/video/video_receive_stream.cc b/video/video_receive_stream.cc deleted file mode 100644 index f812c8d467d..00000000000 --- a/video/video_receive_stream.cc +++ /dev/null @@ -1,801 +0,0 @@ -/* - * Copyright (c) 2013 The WebRTC project authors. All Rights Reserved. - * - * Use of this source code is governed by a BSD-style license - * that can be found in the LICENSE file in the root of the source - * tree. An additional intellectual property rights grant can be found - * in the file PATENTS. All contributing project authors may - * be found in the AUTHORS file in the root of the source tree. - */ - -#include "video/video_receive_stream.h" - -#include -#include - -#include -#include -#include -#include -#include - -#include "absl/algorithm/container.h" -#include "absl/types/optional.h" -#include "api/array_view.h" -#include "api/crypto/frame_decryptor_interface.h" -#include "api/video/encoded_image.h" -#include "api/video_codecs/sdp_video_format.h" -#include "api/video_codecs/video_codec.h" -#include "api/video_codecs/video_decoder_factory.h" -#include "api/video_codecs/video_encoder.h" -#include "call/rtp_stream_receiver_controller_interface.h" -#include "call/rtx_receive_stream.h" -#include "common_video/include/incoming_video_stream.h" -#include "media/base/h264_profile_level_id.h" -#include "modules/utility/include/process_thread.h" -#include "modules/video_coding/include/video_codec_interface.h" -#include "modules/video_coding/include/video_coding_defines.h" -#include "modules/video_coding/include/video_error_codes.h" -#include "modules/video_coding/timing.h" -#include "modules/video_coding/utility/vp8_header_parser.h" -#include "rtc_base/checks.h" -#include "rtc_base/experiments/keyframe_interval_settings.h" -#include "rtc_base/location.h" -#include "rtc_base/logging.h" -#include "rtc_base/strings/string_builder.h" -#include "rtc_base/system/thread_registry.h" -#include "rtc_base/time_utils.h" -#include "rtc_base/trace_event.h" -#include "system_wrappers/include/clock.h" -#include "system_wrappers/include/field_trial.h" -#include "video/call_stats.h" -#include "video/frame_dumping_decoder.h" -#include "video/receive_statistics_proxy.h" - -namespace webrtc { - -namespace internal { -constexpr int VideoReceiveStream::kMaxWaitForKeyFrameMs; -} // namespace internal - -namespace { - -using video_coding::EncodedFrame; -using ReturnReason = video_coding::FrameBuffer::ReturnReason; - -constexpr int kMinBaseMinimumDelayMs = 0; -constexpr int kMaxBaseMinimumDelayMs = 10000; - -constexpr int kMaxWaitForFrameMs = 3000; - -// Concrete instance of RecordableEncodedFrame wrapping needed content -// from video_coding::EncodedFrame. -class WebRtcRecordableEncodedFrame : public RecordableEncodedFrame { - public: - explicit WebRtcRecordableEncodedFrame(const EncodedFrame& frame) - : buffer_(frame.GetEncodedData()), - render_time_ms_(frame.RenderTime()), - codec_(frame.CodecSpecific()->codecType), - is_key_frame_(frame.FrameType() == VideoFrameType::kVideoFrameKey), - resolution_{frame.EncodedImage()._encodedWidth, - frame.EncodedImage()._encodedHeight} { - if (frame.ColorSpace()) { - color_space_ = *frame.ColorSpace(); - } - } - - // VideoEncodedSinkInterface::FrameBuffer - rtc::scoped_refptr encoded_buffer() - const override { - return buffer_; - } - - absl::optional color_space() const override { - return color_space_; - } - - VideoCodecType codec() const override { return codec_; } - - bool is_key_frame() const override { return is_key_frame_; } - - EncodedResolution resolution() const override { return resolution_; } - - Timestamp render_time() const override { - return Timestamp::Millis(render_time_ms_); - } - - private: - rtc::scoped_refptr buffer_; - int64_t render_time_ms_; - VideoCodecType codec_; - bool is_key_frame_; - EncodedResolution resolution_; - absl::optional color_space_; -}; - -VideoCodec CreateDecoderVideoCodec(const VideoReceiveStream::Decoder& decoder) { - VideoCodec codec; - memset(&codec, 0, sizeof(codec)); - - codec.plType = decoder.payload_type; - codec.codecType = PayloadStringToCodecType(decoder.video_format.name); - - if (codec.codecType == kVideoCodecVP8) { - *(codec.VP8()) = VideoEncoder::GetDefaultVp8Settings(); - } else if (codec.codecType == kVideoCodecVP9) { - *(codec.VP9()) = VideoEncoder::GetDefaultVp9Settings(); - } else if (codec.codecType == kVideoCodecH264) { - *(codec.H264()) = VideoEncoder::GetDefaultH264Settings(); - } else if (codec.codecType == kVideoCodecMultiplex) { - VideoReceiveStream::Decoder associated_decoder = decoder; - associated_decoder.video_format = - SdpVideoFormat(CodecTypeToPayloadString(kVideoCodecVP9)); - VideoCodec associated_codec = CreateDecoderVideoCodec(associated_decoder); - associated_codec.codecType = kVideoCodecMultiplex; - return associated_codec; - } -#ifndef DISABLE_H265 - else if (codec.codecType == kVideoCodecH265) { - *(codec.H265()) = VideoEncoder::GetDefaultH265Settings(); - } -#endif - - codec.width = 320; - codec.height = 180; - const int kDefaultStartBitrate = 300; - codec.startBitrate = codec.minBitrate = codec.maxBitrate = - kDefaultStartBitrate; - - return codec; -} - -// Video decoder class to be used for unknown codecs. Doesn't support decoding -// but logs messages to LS_ERROR. -class NullVideoDecoder : public webrtc::VideoDecoder { - public: - int32_t InitDecode(const webrtc::VideoCodec* codec_settings, - int32_t number_of_cores) override { - RTC_LOG(LS_ERROR) << "Can't initialize NullVideoDecoder."; - return WEBRTC_VIDEO_CODEC_OK; - } - - int32_t Decode(const webrtc::EncodedImage& input_image, - bool missing_frames, - int64_t render_time_ms) override { - RTC_LOG(LS_ERROR) << "The NullVideoDecoder doesn't support decoding."; - return WEBRTC_VIDEO_CODEC_OK; - } - - int32_t RegisterDecodeCompleteCallback( - webrtc::DecodedImageCallback* callback) override { - RTC_LOG(LS_ERROR) - << "Can't register decode complete callback on NullVideoDecoder."; - return WEBRTC_VIDEO_CODEC_OK; - } - - int32_t Release() override { return WEBRTC_VIDEO_CODEC_OK; } - - const char* ImplementationName() const override { return "NullVideoDecoder"; } -}; - -// TODO(https://bugs.webrtc.org/9974): Consider removing this workaround. -// Maximum time between frames before resetting the FrameBuffer to avoid RTP -// timestamps wraparound to affect FrameBuffer. -constexpr int kInactiveStreamThresholdMs = 600000; // 10 minutes. - -} // namespace - -namespace internal { - -VideoReceiveStream::VideoReceiveStream( - TaskQueueFactory* task_queue_factory, - RtpStreamReceiverControllerInterface* receiver_controller, - int num_cpu_cores, - PacketRouter* packet_router, - VideoReceiveStream::Config config, - ProcessThread* process_thread, - CallStats* call_stats, - Clock* clock, - VCMTiming* timing) - : task_queue_factory_(task_queue_factory), - transport_adapter_(config.rtcp_send_transport), - config_(std::move(config)), - num_cpu_cores_(num_cpu_cores), - process_thread_(process_thread), - clock_(clock), - call_stats_(call_stats), - source_tracker_(clock_), - stats_proxy_(&config_, clock_), - rtp_receive_statistics_(ReceiveStatistics::Create(clock_)), - timing_(timing), - video_receiver_(clock_, timing_.get()), - rtp_video_stream_receiver_(clock_, - &transport_adapter_, - call_stats, - packet_router, - &config_, - rtp_receive_statistics_.get(), - &stats_proxy_, - process_thread_, - this, // NackSender - nullptr, // Use default KeyFrameRequestSender - this, // OnCompleteFrameCallback - config_.frame_decryptor, - config_.frame_transformer), - rtp_stream_sync_(this), - max_wait_for_keyframe_ms_(KeyframeIntervalSettings::ParseFromFieldTrials() - .MaxWaitForKeyframeMs() - .value_or(kMaxWaitForKeyFrameMs)), - max_wait_for_frame_ms_(KeyframeIntervalSettings::ParseFromFieldTrials() - .MaxWaitForFrameMs() - .value_or(kMaxWaitForFrameMs)), - decode_queue_(task_queue_factory_->CreateTaskQueue( - "DecodingQueue", - TaskQueueFactory::Priority::HIGH)) { - RTC_LOG(LS_INFO) << "VideoReceiveStream: " << config_.ToString(); - - RTC_DCHECK(config_.renderer); - RTC_DCHECK(process_thread_); - RTC_DCHECK(call_stats_); - - module_process_sequence_checker_.Detach(); - network_sequence_checker_.Detach(); - - RTC_DCHECK(!config_.decoders.empty()); - std::set decoder_payload_types; - for (const Decoder& decoder : config_.decoders) { - RTC_CHECK(decoder.decoder_factory); - RTC_CHECK(decoder_payload_types.find(decoder.payload_type) == - decoder_payload_types.end()) - << "Duplicate payload type (" << decoder.payload_type - << ") for different decoders."; - decoder_payload_types.insert(decoder.payload_type); - } - - timing_->set_render_delay(config_.render_delay_ms); - - frame_buffer_.reset( - new video_coding::FrameBuffer(clock_, timing_.get(), &stats_proxy_)); - - process_thread_->RegisterModule(&rtp_stream_sync_, RTC_FROM_HERE); - // Register with RtpStreamReceiverController. - media_receiver_ = receiver_controller->CreateReceiver( - config_.rtp.remote_ssrc, &rtp_video_stream_receiver_); - if (config_.rtp.rtx_ssrc) { - rtx_receive_stream_ = std::make_unique( - &rtp_video_stream_receiver_, config.rtp.rtx_associated_payload_types, - config_.rtp.remote_ssrc, rtp_receive_statistics_.get()); - rtx_receiver_ = receiver_controller->CreateReceiver( - config_.rtp.rtx_ssrc, rtx_receive_stream_.get()); - } else { - rtp_receive_statistics_->EnableRetransmitDetection(config.rtp.remote_ssrc, - true); - } -} - -VideoReceiveStream::VideoReceiveStream( - TaskQueueFactory* task_queue_factory, - RtpStreamReceiverControllerInterface* receiver_controller, - int num_cpu_cores, - PacketRouter* packet_router, - VideoReceiveStream::Config config, - ProcessThread* process_thread, - CallStats* call_stats, - Clock* clock) - : VideoReceiveStream(task_queue_factory, - receiver_controller, - num_cpu_cores, - packet_router, - std::move(config), - process_thread, - call_stats, - clock, - new VCMTiming(clock)) {} - -VideoReceiveStream::~VideoReceiveStream() { - RTC_DCHECK_RUN_ON(&worker_sequence_checker_); - RTC_LOG(LS_INFO) << "~VideoReceiveStream: " << config_.ToString(); - Stop(); - process_thread_->DeRegisterModule(&rtp_stream_sync_); -} - -void VideoReceiveStream::SignalNetworkState(NetworkState state) { - RTC_DCHECK_RUN_ON(&worker_sequence_checker_); - rtp_video_stream_receiver_.SignalNetworkState(state); -} - -bool VideoReceiveStream::DeliverRtcp(const uint8_t* packet, size_t length) { - return rtp_video_stream_receiver_.DeliverRtcp(packet, length); -} - -void VideoReceiveStream::SetSync(Syncable* audio_syncable) { - RTC_DCHECK_RUN_ON(&worker_sequence_checker_); - rtp_stream_sync_.ConfigureSync(audio_syncable); -} - -void VideoReceiveStream::Start() { - RTC_DCHECK_RUN_ON(&worker_sequence_checker_); - - if (decoder_running_) { - return; - } - - const bool protected_by_fec = config_.rtp.protected_by_flexfec || - rtp_video_stream_receiver_.IsUlpfecEnabled(); - - frame_buffer_->Start(); - - if (rtp_video_stream_receiver_.IsRetransmissionsEnabled() && - protected_by_fec) { - frame_buffer_->SetProtectionMode(kProtectionNackFEC); - } - - transport_adapter_.Enable(); - rtc::VideoSinkInterface* renderer = nullptr; - if (config_.enable_prerenderer_smoothing) { - incoming_video_stream_.reset(new IncomingVideoStream( - task_queue_factory_, config_.render_delay_ms, this)); - renderer = incoming_video_stream_.get(); - } else { - renderer = this; - } - - for (const Decoder& decoder : config_.decoders) { - std::unique_ptr video_decoder = - decoder.decoder_factory->LegacyCreateVideoDecoder(decoder.video_format, - config_.stream_id); - // If we still have no valid decoder, we have to create a "Null" decoder - // that ignores all calls. The reason we can get into this state is that the - // old decoder factory interface doesn't have a way to query supported - // codecs. - if (!video_decoder) { - video_decoder = std::make_unique(); - } - - std::string decoded_output_file = - field_trial::FindFullName("WebRTC-DecoderDataDumpDirectory"); - // Because '/' can't be used inside a field trial parameter, we use ';' - // instead. - // This is only relevant to WebRTC-DecoderDataDumpDirectory - // field trial. ';' is chosen arbitrary. Even though it's a legal character - // in some file systems, we can sacrifice ability to use it in the path to - // dumped video, since it's developers-only feature for debugging. - absl::c_replace(decoded_output_file, ';', '/'); - if (!decoded_output_file.empty()) { - char filename_buffer[256]; - rtc::SimpleStringBuilder ssb(filename_buffer); - ssb << decoded_output_file << "/webrtc_receive_stream_" - << this->config_.rtp.remote_ssrc << "-" << rtc::TimeMicros() - << ".ivf"; - video_decoder = CreateFrameDumpingDecoderWrapper( - std::move(video_decoder), FileWrapper::OpenWriteOnly(ssb.str())); - } - - video_decoders_.push_back(std::move(video_decoder)); - - video_receiver_.RegisterExternalDecoder(video_decoders_.back().get(), - decoder.payload_type); - VideoCodec codec = CreateDecoderVideoCodec(decoder); - - const bool raw_payload = - config_.rtp.raw_payload_types.count(codec.plType) > 0; - rtp_video_stream_receiver_.AddReceiveCodec( - codec, decoder.video_format.parameters, raw_payload); - RTC_CHECK_EQ(VCM_OK, video_receiver_.RegisterReceiveCodec( - &codec, num_cpu_cores_, false)); - } - - RTC_DCHECK(renderer != nullptr); - video_stream_decoder_.reset( - new VideoStreamDecoder(&video_receiver_, &stats_proxy_, renderer)); - - // Make sure we register as a stats observer *after* we've prepared the - // |video_stream_decoder_|. - call_stats_->RegisterStatsObserver(this); - - // Start decoding on task queue. - video_receiver_.DecoderThreadStarting(); - stats_proxy_.DecoderThreadStarting(); - decode_queue_.PostTask([this] { - RTC_DCHECK_RUN_ON(&decode_queue_); - decoder_stopped_ = false; - StartNextDecode(); - }); - decoder_running_ = true; - rtp_video_stream_receiver_.StartReceive(); -} - -void VideoReceiveStream::Stop() { - RTC_DCHECK_RUN_ON(&worker_sequence_checker_); - rtp_video_stream_receiver_.StopReceive(); - - stats_proxy_.OnUniqueFramesCounted( - rtp_video_stream_receiver_.GetUniqueFramesSeen()); - - decode_queue_.PostTask([this] { frame_buffer_->Stop(); }); - - call_stats_->DeregisterStatsObserver(this); - - if (decoder_running_) { - rtc::Event done; - decode_queue_.PostTask([this, &done] { - RTC_DCHECK_RUN_ON(&decode_queue_); - decoder_stopped_ = true; - done.Set(); - }); - done.Wait(rtc::Event::kForever); - - decoder_running_ = false; - video_receiver_.DecoderThreadStopped(); - stats_proxy_.DecoderThreadStopped(); - // Deregister external decoders so they are no longer running during - // destruction. This effectively stops the VCM since the decoder thread is - // stopped, the VCM is deregistered and no asynchronous decoder threads are - // running. - for (const Decoder& decoder : config_.decoders) - video_receiver_.RegisterExternalDecoder(nullptr, decoder.payload_type); - - UpdateHistograms(); - } - - video_stream_decoder_.reset(); - incoming_video_stream_.reset(); - transport_adapter_.Disable(); -} - -VideoReceiveStream::Stats VideoReceiveStream::GetStats() const { - VideoReceiveStream::Stats stats = stats_proxy_.GetStats(); - stats.total_bitrate_bps = 0; - StreamStatistician* statistician = - rtp_receive_statistics_->GetStatistician(stats.ssrc); - if (statistician) { - stats.rtp_stats = statistician->GetStats(); - stats.total_bitrate_bps = statistician->BitrateReceived(); - } - if (config_.rtp.rtx_ssrc) { - StreamStatistician* rtx_statistician = - rtp_receive_statistics_->GetStatistician(config_.rtp.rtx_ssrc); - if (rtx_statistician) - stats.total_bitrate_bps += rtx_statistician->BitrateReceived(); - } - return stats; -} - -void VideoReceiveStream::UpdateHistograms() { - absl::optional fraction_lost; - StreamDataCounters rtp_stats; - StreamStatistician* statistician = - rtp_receive_statistics_->GetStatistician(config_.rtp.remote_ssrc); - if (statistician) { - fraction_lost = statistician->GetFractionLostInPercent(); - rtp_stats = statistician->GetReceiveStreamDataCounters(); - } - if (config_.rtp.rtx_ssrc) { - StreamStatistician* rtx_statistician = - rtp_receive_statistics_->GetStatistician(config_.rtp.rtx_ssrc); - if (rtx_statistician) { - StreamDataCounters rtx_stats = - rtx_statistician->GetReceiveStreamDataCounters(); - stats_proxy_.UpdateHistograms(fraction_lost, rtp_stats, &rtx_stats); - return; - } - } - stats_proxy_.UpdateHistograms(fraction_lost, rtp_stats, nullptr); -} - -void VideoReceiveStream::AddSecondarySink(RtpPacketSinkInterface* sink) { - rtp_video_stream_receiver_.AddSecondarySink(sink); -} - -void VideoReceiveStream::RemoveSecondarySink( - const RtpPacketSinkInterface* sink) { - rtp_video_stream_receiver_.RemoveSecondarySink(sink); -} - -bool VideoReceiveStream::SetBaseMinimumPlayoutDelayMs(int delay_ms) { - RTC_DCHECK_RUN_ON(&worker_sequence_checker_); - if (delay_ms < kMinBaseMinimumDelayMs || delay_ms > kMaxBaseMinimumDelayMs) { - return false; - } - - rtc::CritScope cs(&playout_delay_lock_); - base_minimum_playout_delay_ms_ = delay_ms; - UpdatePlayoutDelays(); - return true; -} - -int VideoReceiveStream::GetBaseMinimumPlayoutDelayMs() const { - RTC_DCHECK_RUN_ON(&worker_sequence_checker_); - - rtc::CritScope cs(&playout_delay_lock_); - return base_minimum_playout_delay_ms_; -} - -// TODO(tommi): This method grabs a lock 6 times. -void VideoReceiveStream::OnFrame(const VideoFrame& video_frame) { - int64_t video_playout_ntp_ms; - int64_t sync_offset_ms; - double estimated_freq_khz; - // TODO(tommi): GetStreamSyncOffsetInMs grabs three locks. One inside the - // function itself, another in GetChannel() and a third in - // GetPlayoutTimestamp. Seems excessive. Anyhow, I'm assuming the function - // succeeds most of the time, which leads to grabbing a fourth lock. - if (rtp_stream_sync_.GetStreamSyncOffsetInMs( - video_frame.timestamp(), video_frame.render_time_ms(), - &video_playout_ntp_ms, &sync_offset_ms, &estimated_freq_khz)) { - // TODO(tommi): OnSyncOffsetUpdated grabs a lock. - stats_proxy_.OnSyncOffsetUpdated(video_playout_ntp_ms, sync_offset_ms, - estimated_freq_khz); - } - source_tracker_.OnFrameDelivered(video_frame.packet_infos()); - - config_.renderer->OnFrame(video_frame); - - // TODO(tommi): OnRenderFrame grabs a lock too. - stats_proxy_.OnRenderedFrame(video_frame); -} - -void VideoReceiveStream::SetFrameDecryptor( - rtc::scoped_refptr frame_decryptor) { - rtp_video_stream_receiver_.SetFrameDecryptor(std::move(frame_decryptor)); -} - -void VideoReceiveStream::SetDepacketizerToDecoderFrameTransformer( - rtc::scoped_refptr frame_transformer) { - rtp_video_stream_receiver_.SetDepacketizerToDecoderFrameTransformer( - std::move(frame_transformer)); -} - -void VideoReceiveStream::SendNack(const std::vector& sequence_numbers, - bool buffering_allowed) { - RTC_DCHECK(buffering_allowed); - rtp_video_stream_receiver_.RequestPacketRetransmit(sequence_numbers); -} - -void VideoReceiveStream::RequestKeyFrame(int64_t timestamp_ms) { - rtp_video_stream_receiver_.RequestKeyFrame(); - last_keyframe_request_ms_ = timestamp_ms; -} - -void VideoReceiveStream::OnCompleteFrame( - std::unique_ptr frame) { - RTC_DCHECK_RUN_ON(&network_sequence_checker_); - // TODO(https://bugs.webrtc.org/9974): Consider removing this workaround. - int64_t time_now_ms = clock_->TimeInMilliseconds(); - if (last_complete_frame_time_ms_ > 0 && - time_now_ms - last_complete_frame_time_ms_ > kInactiveStreamThresholdMs) { - frame_buffer_->Clear(); - } - last_complete_frame_time_ms_ = time_now_ms; - - const PlayoutDelay& playout_delay = frame->EncodedImage().playout_delay_; - if (playout_delay.min_ms >= 0) { - rtc::CritScope cs(&playout_delay_lock_); - frame_minimum_playout_delay_ms_ = playout_delay.min_ms; - UpdatePlayoutDelays(); - } - - if (playout_delay.max_ms >= 0) { - rtc::CritScope cs(&playout_delay_lock_); - frame_maximum_playout_delay_ms_ = playout_delay.max_ms; - UpdatePlayoutDelays(); - } - - int64_t last_continuous_pid = frame_buffer_->InsertFrame(std::move(frame)); - if (last_continuous_pid != -1) - rtp_video_stream_receiver_.FrameContinuous(last_continuous_pid); -} - -void VideoReceiveStream::OnRttUpdate(int64_t avg_rtt_ms, int64_t max_rtt_ms) { - RTC_DCHECK_RUN_ON(&module_process_sequence_checker_); - frame_buffer_->UpdateRtt(max_rtt_ms); - rtp_video_stream_receiver_.UpdateRtt(max_rtt_ms); -} - -uint32_t VideoReceiveStream::id() const { - RTC_DCHECK_RUN_ON(&worker_sequence_checker_); - return config_.rtp.remote_ssrc; -} - -absl::optional VideoReceiveStream::GetInfo() const { - RTC_DCHECK_RUN_ON(&module_process_sequence_checker_); - absl::optional info = - rtp_video_stream_receiver_.GetSyncInfo(); - - if (!info) - return absl::nullopt; - - info->current_delay_ms = timing_->TargetVideoDelay(); - return info; -} - -bool VideoReceiveStream::GetPlayoutRtpTimestamp(uint32_t* rtp_timestamp, - int64_t* time_ms) const { - RTC_NOTREACHED(); - return 0; -} - -void VideoReceiveStream::SetEstimatedPlayoutNtpTimestampMs( - int64_t ntp_timestamp_ms, - int64_t time_ms) { - RTC_NOTREACHED(); -} - -void VideoReceiveStream::SetMinimumPlayoutDelay(int delay_ms) { - RTC_DCHECK_RUN_ON(&module_process_sequence_checker_); - rtc::CritScope cs(&playout_delay_lock_); - syncable_minimum_playout_delay_ms_ = delay_ms; - UpdatePlayoutDelays(); -} - -int64_t VideoReceiveStream::GetWaitMs() const { - return keyframe_required_ ? max_wait_for_keyframe_ms_ - : max_wait_for_frame_ms_; -} - -void VideoReceiveStream::StartNextDecode() { - TRACE_EVENT0("webrtc", "VideoReceiveStream::StartNextDecode"); - frame_buffer_->NextFrame( - GetWaitMs(), keyframe_required_, &decode_queue_, - /* encoded frame handler */ - [this](std::unique_ptr frame, ReturnReason res) { - RTC_DCHECK_EQ(frame == nullptr, res == ReturnReason::kTimeout); - RTC_DCHECK_EQ(frame != nullptr, res == ReturnReason::kFrameFound); - decode_queue_.PostTask([this, frame = std::move(frame)]() mutable { - RTC_DCHECK_RUN_ON(&decode_queue_); - if (decoder_stopped_) - return; - if (frame) { - HandleEncodedFrame(std::move(frame)); - } else { - HandleFrameBufferTimeout(); - } - StartNextDecode(); - }); - }); -} - -void VideoReceiveStream::HandleEncodedFrame( - std::unique_ptr frame) { - int64_t now_ms = clock_->TimeInMilliseconds(); - - // Current OnPreDecode only cares about QP for VP8. - int qp = -1; - if (frame->CodecSpecific()->codecType == kVideoCodecVP8) { - if (!vp8::GetQp(frame->data(), frame->size(), &qp)) { - RTC_LOG(LS_WARNING) << "Failed to extract QP from VP8 video frame"; - } - } - stats_proxy_.OnPreDecode(frame->CodecSpecific()->codecType, qp); - HandleKeyFrameGeneration(frame->FrameType() == VideoFrameType::kVideoFrameKey, - now_ms); - int decode_result = video_receiver_.Decode(frame.get()); - if (decode_result == WEBRTC_VIDEO_CODEC_OK || - decode_result == WEBRTC_VIDEO_CODEC_OK_REQUEST_KEYFRAME) { - keyframe_required_ = false; - frame_decoded_ = true; - rtp_video_stream_receiver_.FrameDecoded(frame->id.picture_id); - - if (decode_result == WEBRTC_VIDEO_CODEC_OK_REQUEST_KEYFRAME) - RequestKeyFrame(now_ms); - } else if (!frame_decoded_ || !keyframe_required_ || - (last_keyframe_request_ms_ + max_wait_for_keyframe_ms_ < now_ms)) { - keyframe_required_ = true; - // TODO(philipel): Remove this keyframe request when downstream project - // has been fixed. - RequestKeyFrame(now_ms); - } - - if (encoded_frame_buffer_function_) { - frame->Retain(); - encoded_frame_buffer_function_(WebRtcRecordableEncodedFrame(*frame)); - } -} - -void VideoReceiveStream::HandleKeyFrameGeneration( - bool received_frame_is_keyframe, - int64_t now_ms) { - // Repeat sending keyframe requests if we've requested a keyframe. - if (!keyframe_generation_requested_) { - return; - } - if (received_frame_is_keyframe) { - keyframe_generation_requested_ = false; - } else if (last_keyframe_request_ms_ + max_wait_for_keyframe_ms_ <= now_ms) { - if (!IsReceivingKeyFrame(now_ms)) { - RequestKeyFrame(now_ms); - } - } else { - // It hasn't been long enough since the last keyframe request, do nothing. - } -} - -void VideoReceiveStream::HandleFrameBufferTimeout() { - int64_t now_ms = clock_->TimeInMilliseconds(); - absl::optional last_packet_ms = - rtp_video_stream_receiver_.LastReceivedPacketMs(); - - // To avoid spamming keyframe requests for a stream that is not active we - // check if we have received a packet within the last 5 seconds. - bool stream_is_active = last_packet_ms && now_ms - *last_packet_ms < 5000; - if (!stream_is_active) - stats_proxy_.OnStreamInactive(); - - if (stream_is_active && !IsReceivingKeyFrame(now_ms) && - (!config_.crypto_options.sframe.require_frame_encryption || - rtp_video_stream_receiver_.IsDecryptable())) { - RTC_LOG(LS_WARNING) << "No decodable frame in " << GetWaitMs() - << " ms, requesting keyframe."; - RequestKeyFrame(now_ms); - } -} - -bool VideoReceiveStream::IsReceivingKeyFrame(int64_t timestamp_ms) const { - absl::optional last_keyframe_packet_ms = - rtp_video_stream_receiver_.LastReceivedKeyframePacketMs(); - - // If we recently have been receiving packets belonging to a keyframe then - // we assume a keyframe is currently being received. - bool receiving_keyframe = - last_keyframe_packet_ms && - timestamp_ms - *last_keyframe_packet_ms < max_wait_for_keyframe_ms_; - return receiving_keyframe; -} - -void VideoReceiveStream::UpdatePlayoutDelays() const { - const int minimum_delay_ms = - std::max({frame_minimum_playout_delay_ms_, base_minimum_playout_delay_ms_, - syncable_minimum_playout_delay_ms_}); - if (minimum_delay_ms >= 0) { - timing_->set_min_playout_delay(minimum_delay_ms); - } - - const int maximum_delay_ms = frame_maximum_playout_delay_ms_; - if (maximum_delay_ms >= 0) { - timing_->set_max_playout_delay(maximum_delay_ms); - } -} - -std::vector VideoReceiveStream::GetSources() const { - return source_tracker_.GetSources(); -} - -VideoReceiveStream::RecordingState VideoReceiveStream::SetAndGetRecordingState( - RecordingState state, - bool generate_key_frame) { - RTC_DCHECK_RUN_ON(&worker_sequence_checker_); - rtc::Event event; - RecordingState old_state; - decode_queue_.PostTask([this, &event, &old_state, generate_key_frame, - state = std::move(state)] { - RTC_DCHECK_RUN_ON(&decode_queue_); - // Save old state. - old_state.callback = std::move(encoded_frame_buffer_function_); - old_state.keyframe_needed = keyframe_generation_requested_; - old_state.last_keyframe_request_ms = last_keyframe_request_ms_; - - // Set new state. - encoded_frame_buffer_function_ = std::move(state.callback); - if (generate_key_frame) { - RequestKeyFrame(clock_->TimeInMilliseconds()); - keyframe_generation_requested_ = true; - } else { - keyframe_generation_requested_ = state.keyframe_needed; - last_keyframe_request_ms_ = state.last_keyframe_request_ms.value_or(0); - } - event.Set(); - }); - event.Wait(rtc::Event::kForever); - return old_state; -} - -void VideoReceiveStream::GenerateKeyFrame() { - decode_queue_.PostTask([this]() { - RTC_DCHECK_RUN_ON(&decode_queue_); - RequestKeyFrame(clock_->TimeInMilliseconds()); - keyframe_generation_requested_ = true; - }); -} - -} // namespace internal -} // namespace webrtc diff --git a/video/video_stream_encoder.cc b/video/video_stream_encoder.cc index 581e129d33f..21603b522c9 100644 --- a/video/video_stream_encoder.cc +++ b/video/video_stream_encoder.cc @@ -50,6 +50,7 @@ #include "rtc_base/thread_annotations.h" #include "rtc_base/trace_event.h" #include "system_wrappers/include/metrics.h" +#include "system_wrappers/include/field_trial.h" #include "video/adaptation/video_stream_encoder_resource_manager.h" #include "video/alignment_adjuster.h" #include "video/config/encoder_stream_factory.h" @@ -1454,7 +1455,7 @@ void VideoStreamEncoder::OnFrame(Timestamp post_time, // Jianlin: in case of slice-based encoding, the capturer will not set the // ntp_time_ms and render_tim_ms. int64_t capture_ntp_time_ms; -#if 0 + // TODO: Check latency mode. if (video_frame.ntp_time_ms() > 0) { capture_ntp_time_ms = video_frame.ntp_time_ms(); } else if (video_frame.render_time_ms() != 0) { @@ -1462,8 +1463,6 @@ void VideoStreamEncoder::OnFrame(Timestamp post_time, } else { capture_ntp_time_ms = post_time.ms() + delta_ntp_internal_ms_; } -#endif - capture_ntp_time_ms = current_time_ms + delta_ntp_internal_ms_; incoming_frame.set_ntp_time_ms(capture_ntp_time_ms); // Convert NTP time, in ms, to RTP timestamp. @@ -1472,12 +1471,12 @@ void VideoStreamEncoder::OnFrame(Timestamp post_time, kMsToRtpTimestamp * static_cast(incoming_frame.ntp_time_ms())); if (incoming_frame.ntp_time_ms() <= last_captured_timestamp_) { + // TODO: Check latency mode. // We don't allow the same capture time for two frames, drop this one. // Jianlin(implementation deviation: For push-mode, dropping frame due // to same timestamp is not allowed. // So make sure dropping is enabled for pull mode while disabled for // push mode. -#if 0 RTC_LOG(LS_WARNING) << "Same/old NTP timestamp (" << incoming_frame.ntp_time_ms() << " <= " << last_captured_timestamp_ @@ -1488,7 +1487,6 @@ void VideoStreamEncoder::OnFrame(Timestamp post_time, accumulated_update_rect_is_valid_ &= incoming_frame.has_update_rect(); }); return; -#endif } bool log_stats = false; @@ -1506,7 +1504,8 @@ void VideoStreamEncoder::OnFrame(Timestamp post_time, bool cwnd_frame_drop = cwnd_frame_drop_interval_ && (cwnd_frame_counter_++ % cwnd_frame_drop_interval_.value() == 0); - if (field_trial::IsEnabled("OWT-LowLatencyMode") || posted_frames_waiting_for_encode == 1 && !cwnd_frame_drop) { + if (webrtc::field_trial::IsEnabled("OWT-LowLatencyMode") || + (frames_scheduled_for_processing == 1 && !cwnd_frame_drop)) { MaybeEncodeVideoFrame(incoming_frame, post_time.us()); } else { if (cwnd_frame_drop) {