diff --git a/trunk/conf/full.conf b/trunk/conf/full.conf index 7733f04585..81e98f1e2d 100644 --- a/trunk/conf/full.conf +++ b/trunk/conf/full.conf @@ -562,6 +562,12 @@ vhost rtc.vhost.srs.com { # Overwrite by env SRS_VHOST_RTC_KEEP_BFRAME for all vhosts. # default: off keep_bframe off; + # Whether keep the h.264 SEI type NALU packet. + # DJI drone M30T will send many SEI type NALU packet, while + # iOS hardware decoder (Video Toolbox) dislike to feed it + # so many SEI NALU between NonIDR and IDR NALU packets. + # @see https://github.com/ossrs/srs/issues/4052 + keep_avc_nalu_sei on; # The transcode audio bitrate, for RTMP to RTC. # Overwrite by env SRS_VHOST_RTC_OPUS_BITRATE for all vhosts. # [8000, 320000] diff --git a/trunk/conf/rtmp2rtc.conf b/trunk/conf/rtmp2rtc.conf index daa50e2a74..efb070f655 100644 --- a/trunk/conf/rtmp2rtc.conf +++ b/trunk/conf/rtmp2rtc.conf @@ -31,6 +31,7 @@ vhost __defaultVhost__ { rtmp_to_rtc on; # @see https://ossrs.net/lts/zh-cn/docs/v4/doc/webrtc#rtc-to-rtmp rtc_to_rtmp on; + keep_avc_nalu_sei off; } http_remux { enabled on; diff --git a/trunk/src/app/srs_app_config.cpp b/trunk/src/app/srs_app_config.cpp index b7efd889d9..00bd735951 100644 --- a/trunk/src/app/srs_app_config.cpp +++ b/trunk/src/app/srs_app_config.cpp @@ -2726,7 +2726,7 @@ srs_error_t SrsConfig::check_normal_config() && m != "bframe" && m != "aac" && m != "stun_timeout" && m != "stun_strict_check" && m != "dtls_role" && m != "dtls_version" && m != "drop_for_pt" && m != "rtc_to_rtmp" && m != "pli_for_rtmp" && m != "rtmp_to_rtc" && m != "keep_bframe" && m != "opus_bitrate" - && m != "aac_bitrate") { + && m != "aac_bitrate" && m != "keep_avc_nalu_sei") { return srs_error_new(ERROR_SYSTEM_CONFIG_INVALID, "illegal vhost.rtc.%s of %s", m.c_str(), vhost->arg0().c_str()); } } @@ -4474,6 +4474,26 @@ bool SrsConfig::get_rtc_keep_bframe(string vhost) return SRS_CONF_PERFER_FALSE(conf->arg0()); } +bool SrsConfig::get_rtc_keep_avc_nalu_sei(std::string vhost) +{ + SRS_OVERWRITE_BY_ENV_BOOL("srs.vhost.rtc.keep_avc_nalu_sei"); // SRS_VHOST_RTC_KEEP_AVC_NALU_SEI + + static bool DEFAULT = true; + + SrsConfDirective* conf = get_rtc(vhost); + + if (!conf) { + return DEFAULT; + } + + conf = conf->get("keep_avc_nalu_sei"); + if (!conf || conf->arg0().empty()) { + return DEFAULT; + } + + return SRS_CONF_PERFER_TRUE(conf->arg0()); +} + bool SrsConfig::get_rtc_from_rtmp(string vhost) { SRS_OVERWRITE_BY_ENV_BOOL("srs.vhost.rtc.rtmp_to_rtc"); // SRS_VHOST_RTC_RTMP_TO_RTC diff --git a/trunk/src/app/srs_app_config.hpp b/trunk/src/app/srs_app_config.hpp index c6906e5fdf..afa700a0c8 100644 --- a/trunk/src/app/srs_app_config.hpp +++ b/trunk/src/app/srs_app_config.hpp @@ -533,6 +533,7 @@ class SrsConfig SrsConfDirective* get_rtc(std::string vhost); bool get_rtc_enabled(std::string vhost); bool get_rtc_keep_bframe(std::string vhost); + bool get_rtc_keep_avc_nalu_sei(std::string vhost); bool get_rtc_from_rtmp(std::string vhost); srs_utime_t get_rtc_stun_timeout(std::string vhost); bool get_rtc_stun_strict_check(std::string vhost); diff --git a/trunk/src/app/srs_app_http_stream.cpp b/trunk/src/app/srs_app_http_stream.cpp index 181ee14c09..e32472844c 100755 --- a/trunk/src/app/srs_app_http_stream.cpp +++ b/trunk/src/app/srs_app_http_stream.cpp @@ -855,6 +855,7 @@ srs_error_t SrsLiveStream::streaming_send_messages(ISrsBufferEncoder* enc, SrsSh if (msg->is_audio()) { err = enc->write_audio(msg->timestamp, msg->payload, msg->size); } else if (msg->is_video()) { + // TODO: drop h.264 NALU SEI here. @see https://github.com/ossrs/srs/issues/4052 err = enc->write_video(msg->timestamp, msg->payload, msg->size); } else { err = enc->write_metadata(msg->timestamp, msg->payload, msg->size); diff --git a/trunk/src/app/srs_app_rtc_source.cpp b/trunk/src/app/srs_app_rtc_source.cpp index c74b59e9e4..e541d35f53 100644 --- a/trunk/src/app/srs_app_rtc_source.cpp +++ b/trunk/src/app/srs_app_rtc_source.cpp @@ -739,6 +739,7 @@ SrsRtcRtpBuilder::SrsRtcRtpBuilder(SrsFrameToRtcBridge* bridge, uint32_t assrc, codec_ = new SrsAudioTranscoder(); latest_codec_ = SrsAudioCodecIdForbidden; keep_bframe = false; + keep_avc_nalu_sei = true; merge_nalus = false; meta = new SrsMetaCache(); audio_sequence = 0; @@ -771,8 +772,9 @@ srs_error_t SrsRtcRtpBuilder::initialize(SrsRequest* r) format->try_annexb_first = _srs_config->try_annexb_first(r->vhost); keep_bframe = _srs_config->get_rtc_keep_bframe(req->vhost); + keep_avc_nalu_sei = _srs_config->get_rtc_keep_avc_nalu_sei(req->vhost); merge_nalus = _srs_config->get_rtc_server_merge_nalus(); - srs_trace("RTC bridge from RTMP, keep_bframe=%d, merge_nalus=%d", keep_bframe, merge_nalus); + srs_trace("RTC bridge from RTMP, keep_bframe=%d, keep_avc_nalu_sei=%d, merge_nalus=%d", keep_bframe, keep_avc_nalu_sei, merge_nalus); return err; } @@ -1050,6 +1052,15 @@ srs_error_t SrsRtcRtpBuilder::filter(SrsSharedPtrMessage* msg, SrsFormat* format // Update samples to shared frame. for (int i = 0; i < format->video->nb_samples; ++i) { SrsSample* sample = &format->video->samples[i]; + + if (!keep_avc_nalu_sei) { + if ((err = sample->parse_nalu_type()) != srs_success) { + return srs_error_wrap(err, "parse nalu_type"); + } + if (sample->nalu_type == SrsAvcNaluTypeSEI) { + continue; + } + } // Because RTC does not support B-frame, so we will drop them. // TODO: Drop B-frame in better way, which not cause picture corruption. diff --git a/trunk/src/app/srs_app_rtc_source.hpp b/trunk/src/app/srs_app_rtc_source.hpp index 02e786cf28..75777a38f3 100644 --- a/trunk/src/app/srs_app_rtc_source.hpp +++ b/trunk/src/app/srs_app_rtc_source.hpp @@ -262,6 +262,7 @@ class SrsRtcRtpBuilder SrsAudioCodecId latest_codec_; SrsAudioTranscoder* codec_; bool keep_bframe; + bool keep_avc_nalu_sei; bool merge_nalus; uint16_t audio_sequence; uint16_t video_sequence; diff --git a/trunk/src/kernel/srs_kernel_codec.cpp b/trunk/src/kernel/srs_kernel_codec.cpp index 9afe07867e..d4fa6839e8 100644 --- a/trunk/src/kernel/srs_kernel_codec.cpp +++ b/trunk/src/kernel/srs_kernel_codec.cpp @@ -505,6 +505,7 @@ SrsSample::SrsSample() size = 0; bytes = NULL; bframe = false; + nalu_type = SrsAvcNaluTypeReserved; } SrsSample::SrsSample(char* b, int s) @@ -512,6 +513,7 @@ SrsSample::SrsSample(char* b, int s) size = s; bytes = b; bframe = false; + nalu_type = SrsAvcNaluTypeReserved; } SrsSample::~SrsSample() @@ -523,9 +525,9 @@ srs_error_t SrsSample::parse_bframe() srs_error_t err = srs_success; uint8_t header = bytes[0]; - SrsAvcNaluType nal_type = (SrsAvcNaluType)(header & kNalTypeMask); - - if (nal_type != SrsAvcNaluTypeNonIDR && nal_type != SrsAvcNaluTypeDataPartitionA && nal_type != SrsAvcNaluTypeIDR) { + nalu_type = (SrsAvcNaluType)(header & kNalTypeMask); + + if (nalu_type != SrsAvcNaluTypeNonIDR && nalu_type != SrsAvcNaluTypeDataPartitionA && nalu_type != SrsAvcNaluTypeIDR) { return err; } @@ -549,18 +551,33 @@ srs_error_t SrsSample::parse_bframe() if (slice_type == SrsAvcSliceTypeB || slice_type == SrsAvcSliceTypeB1) { bframe = true; - srs_verbose("nal_type=%d, slice type=%d", nal_type, slice_type); + srs_verbose("nalu_type=%d, slice type=%d", nalu_type, slice_type); } return err; } +srs_error_t SrsSample::parse_nalu_type() +{ + srs_error_t err = srs_success; + + if (size < 1) { + return srs_error_new(ERROR_AVC_NALU_EMPTY, "empty nalu"); + } + + uint8_t header = bytes[0]; + nalu_type = (SrsAvcNaluType)(header & kNalTypeMask); + + return err; +} + SrsSample* SrsSample::copy() { SrsSample* p = new SrsSample(); p->bytes = bytes; p->size = size; p->bframe = bframe; + p->nalu_type = nalu_type; return p; } @@ -656,6 +673,7 @@ srs_error_t SrsFrame::add_sample(char* bytes, int size) sample->bytes = bytes; sample->size = size; sample->bframe = false; + sample->nalu_type = SrsAvcNaluTypeReserved; return err; } diff --git a/trunk/src/kernel/srs_kernel_codec.hpp b/trunk/src/kernel/srs_kernel_codec.hpp index 5f8fc934af..4467f2fa4a 100644 --- a/trunk/src/kernel/srs_kernel_codec.hpp +++ b/trunk/src/kernel/srs_kernel_codec.hpp @@ -1133,13 +1133,18 @@ class SrsSample char* bytes; // Whether is B frame. bool bframe; + // h.264 nalu type + SrsAvcNaluType nalu_type; public: SrsSample(); SrsSample(char* b, int s); ~SrsSample(); public: // If we need to know whether sample is bframe, we have to parse the NALU payload. + // TODO: FIXME: SrsSample can also be used as H.265(HEVC), the following two method + // need to be refactor to let SrsSample aware the format of NALU(avc or hevc). srs_error_t parse_bframe(); + srs_error_t parse_nalu_type(); // Copy sample, share the bytes pointer. SrsSample* copy(); }; diff --git a/trunk/src/kernel/srs_kernel_error.hpp b/trunk/src/kernel/srs_kernel_error.hpp index f24895b466..5c4f982d59 100644 --- a/trunk/src/kernel/srs_kernel_error.hpp +++ b/trunk/src/kernel/srs_kernel_error.hpp @@ -276,7 +276,8 @@ XX(ERROR_HEVC_DISABLED , 3098, "HevcDisabled", "HEVC is disabled") \ XX(ERROR_HEVC_DECODE_ERROR , 3099, "HevcDecode", "HEVC decode av stream failed") \ XX(ERROR_MP4_HVCC_CHANGE , 3100, "Mp4HvcCChange", "MP4 does not support video HvcC change") \ - XX(ERROR_HEVC_API_NO_PREFIXED , 3101, "HevcAnnexbPrefix", "No annexb prefix for HEVC decoder") + XX(ERROR_HEVC_API_NO_PREFIXED , 3101, "HevcAnnexbPrefix", "No annexb prefix for HEVC decoder") \ + XX(ERROR_AVC_NALU_EMPTY , 3102, "AvcNalu", "AVC NALU is empty") /**************************************************/ /* HTTP/StreamConverter protocol error. */