Skip to content

Commit

Permalink
issue ossrs#4052: support h.264 nalu type SEI filter.
Browse files Browse the repository at this point in the history
  • Loading branch information
suzp1984 committed May 17, 2024
1 parent 427104f commit e8a4d0f
Show file tree
Hide file tree
Showing 11 changed files with 76 additions and 7 deletions.
6 changes: 6 additions & 0 deletions trunk/conf/full.conf
Original file line number Diff line number Diff line change
Expand Up @@ -562,6 +562,12 @@ vhost rtc.vhost.srs.com {
# Overwrite by env SRS_VHOST_RTC_KEEP_BFRAME for all vhosts.
# default: off
keep_bframe off;
# Whether keep the h.264 SEI type NALU packet.
# DJI drone M30T will send many SEI type NALU packet, while
# iOS hardware decoder (Video Toolbox) dislike to feed it
# so many SEI NALU between NonIDR and IDR NALU packets.
# @see https://github.com/ossrs/srs/issues/4052
keep_avc_nalu_sei on;
# The transcode audio bitrate, for RTMP to RTC.
# Overwrite by env SRS_VHOST_RTC_OPUS_BITRATE for all vhosts.
# [8000, 320000]
Expand Down
1 change: 1 addition & 0 deletions trunk/conf/rtmp2rtc.conf
Original file line number Diff line number Diff line change
Expand Up @@ -31,6 +31,7 @@ vhost __defaultVhost__ {
rtmp_to_rtc on;
# @see https://ossrs.net/lts/zh-cn/docs/v4/doc/webrtc#rtc-to-rtmp
rtc_to_rtmp on;
keep_avc_nalu_sei off;
}
http_remux {
enabled on;
Expand Down
22 changes: 21 additions & 1 deletion trunk/src/app/srs_app_config.cpp
Original file line number Diff line number Diff line change
Expand Up @@ -2726,7 +2726,7 @@ srs_error_t SrsConfig::check_normal_config()
&& m != "bframe" && m != "aac" && m != "stun_timeout" && m != "stun_strict_check"
&& m != "dtls_role" && m != "dtls_version" && m != "drop_for_pt" && m != "rtc_to_rtmp"
&& m != "pli_for_rtmp" && m != "rtmp_to_rtc" && m != "keep_bframe" && m != "opus_bitrate"
&& m != "aac_bitrate") {
&& m != "aac_bitrate" && m != "keep_avc_nalu_sei") {
return srs_error_new(ERROR_SYSTEM_CONFIG_INVALID, "illegal vhost.rtc.%s of %s", m.c_str(), vhost->arg0().c_str());
}
}
Expand Down Expand Up @@ -4474,6 +4474,26 @@ bool SrsConfig::get_rtc_keep_bframe(string vhost)
return SRS_CONF_PERFER_FALSE(conf->arg0());
}

bool SrsConfig::get_rtc_keep_avc_nalu_sei(std::string vhost)
{
SRS_OVERWRITE_BY_ENV_BOOL("srs.vhost.rtc.keep_avc_nalu_sei"); // SRS_VHOST_RTC_KEEP_AVC_NALU_SEI

static bool DEFAULT = true;

SrsConfDirective* conf = get_rtc(vhost);

if (!conf) {
return DEFAULT;
}

conf = conf->get("keep_avc_nalu_sei");
if (!conf || conf->arg0().empty()) {
return DEFAULT;
}

return SRS_CONF_PERFER_TRUE(conf->arg0());
}

bool SrsConfig::get_rtc_from_rtmp(string vhost)
{
SRS_OVERWRITE_BY_ENV_BOOL("srs.vhost.rtc.rtmp_to_rtc"); // SRS_VHOST_RTC_RTMP_TO_RTC
Expand Down
1 change: 1 addition & 0 deletions trunk/src/app/srs_app_config.hpp
Original file line number Diff line number Diff line change
Expand Up @@ -533,6 +533,7 @@ class SrsConfig
SrsConfDirective* get_rtc(std::string vhost);
bool get_rtc_enabled(std::string vhost);
bool get_rtc_keep_bframe(std::string vhost);
bool get_rtc_keep_avc_nalu_sei(std::string vhost);
bool get_rtc_from_rtmp(std::string vhost);
srs_utime_t get_rtc_stun_timeout(std::string vhost);
bool get_rtc_stun_strict_check(std::string vhost);
Expand Down
2 changes: 2 additions & 0 deletions trunk/src/app/srs_app_http_stream.cpp
Original file line number Diff line number Diff line change
Expand Up @@ -855,6 +855,8 @@ srs_error_t SrsLiveStream::streaming_send_messages(ISrsBufferEncoder* enc, SrsSh
if (msg->is_audio()) {
err = enc->write_audio(msg->timestamp, msg->payload, msg->size);
} else if (msg->is_video()) {
// TODO: drop h.264 flv video tags with NALU SEI here to fix http-flv play error in safari mac.
// @see https://github.com/ossrs/srs/issues/4052
err = enc->write_video(msg->timestamp, msg->payload, msg->size);
} else {
err = enc->write_metadata(msg->timestamp, msg->payload, msg->size);
Expand Down
13 changes: 12 additions & 1 deletion trunk/src/app/srs_app_rtc_source.cpp
Original file line number Diff line number Diff line change
Expand Up @@ -739,6 +739,7 @@ SrsRtcRtpBuilder::SrsRtcRtpBuilder(SrsFrameToRtcBridge* bridge, uint32_t assrc,
codec_ = new SrsAudioTranscoder();
latest_codec_ = SrsAudioCodecIdForbidden;
keep_bframe = false;
keep_avc_nalu_sei = true;
merge_nalus = false;
meta = new SrsMetaCache();
audio_sequence = 0;
Expand Down Expand Up @@ -771,8 +772,9 @@ srs_error_t SrsRtcRtpBuilder::initialize(SrsRequest* r)
format->try_annexb_first = _srs_config->try_annexb_first(r->vhost);

keep_bframe = _srs_config->get_rtc_keep_bframe(req->vhost);
keep_avc_nalu_sei = _srs_config->get_rtc_keep_avc_nalu_sei(req->vhost);
merge_nalus = _srs_config->get_rtc_server_merge_nalus();
srs_trace("RTC bridge from RTMP, keep_bframe=%d, merge_nalus=%d", keep_bframe, merge_nalus);
srs_trace("RTC bridge from RTMP, keep_bframe=%d, keep_avc_nalu_sei=%d, merge_nalus=%d", keep_bframe, keep_avc_nalu_sei, merge_nalus);

return err;
}
Expand Down Expand Up @@ -1050,6 +1052,15 @@ srs_error_t SrsRtcRtpBuilder::filter(SrsSharedPtrMessage* msg, SrsFormat* format
// Update samples to shared frame.
for (int i = 0; i < format->video->nb_samples; ++i) {
SrsSample* sample = &format->video->samples[i];

if (!keep_avc_nalu_sei) {
if ((err = sample->parse_nalu_type()) != srs_success) {
return srs_error_wrap(err, "parse nalu_type");
}
if (sample->nalu_type == SrsAvcNaluTypeSEI) {
continue;
}
}

// Because RTC does not support B-frame, so we will drop them.
// TODO: Drop B-frame in better way, which not cause picture corruption.
Expand Down
1 change: 1 addition & 0 deletions trunk/src/app/srs_app_rtc_source.hpp
Original file line number Diff line number Diff line change
Expand Up @@ -262,6 +262,7 @@ class SrsRtcRtpBuilder
SrsAudioCodecId latest_codec_;
SrsAudioTranscoder* codec_;
bool keep_bframe;
bool keep_avc_nalu_sei;
bool merge_nalus;
uint16_t audio_sequence;
uint16_t video_sequence;
Expand Down
26 changes: 22 additions & 4 deletions trunk/src/kernel/srs_kernel_codec.cpp
Original file line number Diff line number Diff line change
Expand Up @@ -505,13 +505,15 @@ SrsSample::SrsSample()
size = 0;
bytes = NULL;
bframe = false;
nalu_type = SrsAvcNaluTypeReserved;
}

SrsSample::SrsSample(char* b, int s)
{
size = s;
bytes = b;
bframe = false;
nalu_type = SrsAvcNaluTypeReserved;
}

SrsSample::~SrsSample()
Expand All @@ -523,9 +525,9 @@ srs_error_t SrsSample::parse_bframe()
srs_error_t err = srs_success;

uint8_t header = bytes[0];
SrsAvcNaluType nal_type = (SrsAvcNaluType)(header & kNalTypeMask);

if (nal_type != SrsAvcNaluTypeNonIDR && nal_type != SrsAvcNaluTypeDataPartitionA && nal_type != SrsAvcNaluTypeIDR) {
nalu_type = (SrsAvcNaluType)(header & kNalTypeMask);
if (nalu_type != SrsAvcNaluTypeNonIDR && nalu_type != SrsAvcNaluTypeDataPartitionA && nalu_type != SrsAvcNaluTypeIDR) {
return err;
}

Expand All @@ -549,18 +551,33 @@ srs_error_t SrsSample::parse_bframe()

if (slice_type == SrsAvcSliceTypeB || slice_type == SrsAvcSliceTypeB1) {
bframe = true;
srs_verbose("nal_type=%d, slice type=%d", nal_type, slice_type);
srs_verbose("nalu_type=%d, slice type=%d", nalu_type, slice_type);
}

return err;
}

srs_error_t SrsSample::parse_nalu_type()
{
srs_error_t err = srs_success;

if (size < 1) {
return srs_error_new(ERROR_AVC_NALU_EMPTY, "empty nalu");
}

uint8_t header = bytes[0];
nalu_type = (SrsAvcNaluType)(header & kNalTypeMask);

return err;
}

SrsSample* SrsSample::copy()
{
SrsSample* p = new SrsSample();
p->bytes = bytes;
p->size = size;
p->bframe = bframe;
p->nalu_type = nalu_type;
return p;
}

Expand Down Expand Up @@ -656,6 +673,7 @@ srs_error_t SrsFrame::add_sample(char* bytes, int size)
sample->bytes = bytes;
sample->size = size;
sample->bframe = false;
sample->nalu_type = SrsAvcNaluTypeReserved;

return err;
}
Expand Down
5 changes: 5 additions & 0 deletions trunk/src/kernel/srs_kernel_codec.hpp
Original file line number Diff line number Diff line change
Expand Up @@ -1133,13 +1133,18 @@ class SrsSample
char* bytes;
// Whether is B frame.
bool bframe;
// h.264 nalu type
SrsAvcNaluType nalu_type;
public:
SrsSample();
SrsSample(char* b, int s);
~SrsSample();
public:
// If we need to know whether sample is bframe, we have to parse the NALU payload.
// TODO: FIXME: SrsSample can also be used as H.265(HEVC), the following two method
// need to be refactor to let SrsSample aware the format of NALU(avc or hevc).
srs_error_t parse_bframe();
srs_error_t parse_nalu_type();
// Copy sample, share the bytes pointer.
SrsSample* copy();
};
Expand Down
3 changes: 2 additions & 1 deletion trunk/src/kernel/srs_kernel_error.hpp
Original file line number Diff line number Diff line change
Expand Up @@ -276,7 +276,8 @@
XX(ERROR_HEVC_DISABLED , 3098, "HevcDisabled", "HEVC is disabled") \
XX(ERROR_HEVC_DECODE_ERROR , 3099, "HevcDecode", "HEVC decode av stream failed") \
XX(ERROR_MP4_HVCC_CHANGE , 3100, "Mp4HvcCChange", "MP4 does not support video HvcC change") \
XX(ERROR_HEVC_API_NO_PREFIXED , 3101, "HevcAnnexbPrefix", "No annexb prefix for HEVC decoder")
XX(ERROR_HEVC_API_NO_PREFIXED , 3101, "HevcAnnexbPrefix", "No annexb prefix for HEVC decoder") \
XX(ERROR_AVC_NALU_EMPTY , 3102, "AvcNalu", "AVC NALU is empty")

/**************************************************/
/* HTTP/StreamConverter protocol error. */
Expand Down
3 changes: 3 additions & 0 deletions trunk/src/utest/srs_utest_config.cpp
Original file line number Diff line number Diff line change
Expand Up @@ -4405,6 +4405,9 @@ VOID TEST(ConfigEnvTest, CheckEnvValuesVhostRtc)

SrsSetEnvConfig(rtc_keep_bframe, "SRS_VHOST_RTC_KEEP_BFRAME", "on");
EXPECT_TRUE(conf.get_rtc_keep_bframe("__defaultVhost__"));

SrsSetEnvConfig(rtc_keep_avc_nalu_sei, "SRS_VHOST_RTC_KEEP_AVC_NALU_SEI", "off");
EXPECT_FALSE(conf.get_rtc_keep_avc_nalu_sei("__defaultVhost__"));
}

if (true) {
Expand Down

0 comments on commit e8a4d0f

Please sign in to comment.