From 91330b7d116fdbcea68d6cbd48da044d36fbcd1e Mon Sep 17 00:00:00 2001 From: runner365 Date: Wed, 23 Nov 2022 16:05:09 +0800 Subject: [PATCH] H265: Support HEVC over HTTP-TS. v6.0.4 1. Update TS video codec to HEVC during streaming. 2. Return error when HEVC is disabled. 3. Parse HEVC NALU type by SrsHevcNaluTypeParse. 4. Show message when codec change for TS. --- trunk/doc/CHANGELOG.md | 1 + trunk/src/app/srs_app_hls.cpp | 12 ++- trunk/src/app/srs_app_hls.hpp | 2 +- trunk/src/core/srs_core_version6.hpp | 2 +- trunk/src/kernel/srs_kernel_codec.cpp | 9 +- trunk/src/kernel/srs_kernel_codec.hpp | 1 + trunk/src/kernel/srs_kernel_error.hpp | 3 +- trunk/src/kernel/srs_kernel_ts.cpp | 140 ++++++++++++++++++++++---- trunk/src/kernel/srs_kernel_ts.hpp | 10 +- trunk/src/utest/srs_utest_kernel.cpp | 31 +++++- 10 files changed, 178 insertions(+), 33 deletions(-) diff --git a/trunk/doc/CHANGELOG.md b/trunk/doc/CHANGELOG.md index bafd482123..f46e277442 100644 --- a/trunk/doc/CHANGELOG.md +++ b/trunk/doc/CHANGELOG.md @@ -8,6 +8,7 @@ The changelog for SRS. ## SRS 6.0 Changelog +* v6.0, 2022-11-23, Merge [#3275](https://github.com/ossrs/srs/pull/3275): H265: Support HEVC over HTTP-TS. v6.0.4 * v6.0, 2022-11-23, Merge [#3274](https://github.com/ossrs/srs/pull/3274): H265: Support parse multiple NALUs in a frame. v6.0.3 * v6.0, 2022-11-22, Merge [#3272](https://github.com/ossrs/srs/pull/3272): H265: Support HEVC over RTMP or HTTP-FLV. v6.0.2 * v6.0, 2022-11-22, Merge [#3268](https://github.com/ossrs/srs/pull/3268): H265: Update mpegts.js to play HEVC over HTTP-TS/FLV. v6.0.1 diff --git a/trunk/src/app/srs_app_hls.cpp b/trunk/src/app/srs_app_hls.cpp index 0183d16261..a1a5446315 100644 --- a/trunk/src/app/srs_app_hls.cpp +++ b/trunk/src/app/srs_app_hls.cpp @@ -555,7 +555,7 @@ srs_error_t SrsHlsMuxer::flush_audio(SrsTsMessageCache* cache) return err; } -srs_error_t SrsHlsMuxer::flush_video(SrsTsMessageCache* cache) +srs_error_t SrsHlsMuxer::flush_video(SrsTsMessageCache* cache, SrsVideoFrame* frame) { srs_error_t err = srs_success; @@ -573,6 +573,12 @@ srs_error_t SrsHlsMuxer::flush_video(SrsTsMessageCache* cache) // update the duration of segment. current->append(cache->video->dts / 90); + + // The video codec might change during streaming. Note that the frame might be NULL, when reap segment. + if (frame && frame->vcodec()) { + SrsTsContextWriter* tscw = current->tscw; + tscw->update_video_codec(frame->vcodec()->id); + } if ((err = current->tscw->write_video(cache->video)) != srs_success) { return srs_error_wrap(err, "hls: write video"); @@ -1025,7 +1031,7 @@ srs_error_t SrsHlsController::write_video(SrsVideoFrame* frame, int64_t dts) } // flush video when got one - if ((err = muxer->flush_video(tsmc)) != srs_success) { + if ((err = muxer->flush_video(tsmc, frame)) != srs_success) { return srs_error_wrap(err, "hls: flush video"); } @@ -1057,7 +1063,7 @@ srs_error_t SrsHlsController::reap_segment() } // segment open, flush video first. - if ((err = muxer->flush_video(tsmc)) != srs_success) { + if ((err = muxer->flush_video(tsmc, NULL)) != srs_success) { return srs_error_wrap(err, "hls: flush video"); } diff --git a/trunk/src/app/srs_app_hls.hpp b/trunk/src/app/srs_app_hls.hpp index cd1eb7438e..b5dd8a82a6 100644 --- a/trunk/src/app/srs_app_hls.hpp +++ b/trunk/src/app/srs_app_hls.hpp @@ -193,7 +193,7 @@ class SrsHlsMuxer // Whether current hls muxer is pure audio mode. virtual bool pure_audio(); virtual srs_error_t flush_audio(SrsTsMessageCache* cache); - virtual srs_error_t flush_video(SrsTsMessageCache* cache); + virtual srs_error_t flush_video(SrsTsMessageCache* cache, SrsVideoFrame* frame); // Close segment(ts). virtual srs_error_t segment_close(); private: diff --git a/trunk/src/core/srs_core_version6.hpp b/trunk/src/core/srs_core_version6.hpp index e18455ca33..2ec200fc54 100644 --- a/trunk/src/core/srs_core_version6.hpp +++ b/trunk/src/core/srs_core_version6.hpp @@ -9,6 +9,6 @@ #define VERSION_MAJOR 6 #define VERSION_MINOR 0 -#define VERSION_REVISION 3 +#define VERSION_REVISION 4 #endif diff --git a/trunk/src/kernel/srs_kernel_codec.cpp b/trunk/src/kernel/srs_kernel_codec.cpp index df35dd54a9..ef15211dcc 100644 --- a/trunk/src/kernel/srs_kernel_codec.cpp +++ b/trunk/src/kernel/srs_kernel_codec.cpp @@ -620,11 +620,11 @@ srs_error_t SrsVideoFrame::add_sample(char* bytes, int size) // For HEVC(H.265), try to parse the IDR from NALUs. if (c && c->id == SrsVideoCodecIdHEVC) { #ifdef SRS_H265 - SrsHevcNaluType nalu_type = (SrsHevcNaluType)(uint8_t)((bytes[0] & 0x3f) >> 1); + SrsHevcNaluType nalu_type = SrsHevcNaluTypeParse(bytes[0]); has_idr = (SrsHevcNaluType_CODED_SLICE_BLA <= nalu_type) && (nalu_type <= SrsHevcNaluType_RESERVED_23); return err; #else - return srs_error_new(ERROR_HLS_DECODE_ERROR, "H.265 is disabled"); + return srs_error_new(ERROR_HEVC_DISABLED, "H.265 is disabled"); #endif } @@ -854,7 +854,7 @@ srs_error_t SrsFormat::video_avc_demux(SrsBuffer* stream, int64_t timestamp) } return err; #else - return srs_error_new(ERROR_HLS_DECODE_ERROR, "H.265 is disabled"); + return srs_error_new(ERROR_HEVC_DISABLED, "H.265 is disabled"); #endif } @@ -1373,7 +1373,7 @@ srs_error_t SrsFormat::video_nalu_demux(SrsBuffer* stream) // TODO: FIXME: Might need to guess format? return do_avc_demux_ibmf_format(stream); #else - return srs_error_new(ERROR_HLS_DECODE_ERROR, "H.265 is disabled"); + return srs_error_new(ERROR_HEVC_DISABLED, "H.265 is disabled"); #endif } @@ -1500,6 +1500,7 @@ srs_error_t SrsFormat::do_avc_demux_ibmf_format(SrsBuffer* stream) // 5.3.4.2.1 Syntax, ISO_IEC_14496-15-AVC-format-2012.pdf, page 20 for (int i = 0; i < PictureLength;) { // unsigned int((NAL_unit_length+1)*8) NALUnitLength; + // TODO: FIXME: Should ignore error? See https://github.com/ossrs/srs-gb28181/commit/a13b9b54938a14796abb9011e7a8ee779439a452 if (!stream->require(vcodec->NAL_unit_length + 1)) { return srs_error_new(ERROR_HLS_DECODE_ERROR, "PictureLength:%d, i:%d, NaluLength:%d, left:%d", PictureLength, i, vcodec->NAL_unit_length, stream->left()); diff --git a/trunk/src/kernel/srs_kernel_codec.hpp b/trunk/src/kernel/srs_kernel_codec.hpp index aa7b32c51d..682d4bca91 100644 --- a/trunk/src/kernel/srs_kernel_codec.hpp +++ b/trunk/src/kernel/srs_kernel_codec.hpp @@ -468,6 +468,7 @@ enum SrsHevcNaluType { SrsHevcNaluType_UNSPECIFIED_63, SrsHevcNaluType_INVALID, }; +#define SrsHevcNaluTypeParse(code) (SrsHevcNaluType)((code & 0x7E) >> 1) struct SrsHevcNalData { uint16_t nal_unit_length; diff --git a/trunk/src/kernel/srs_kernel_error.hpp b/trunk/src/kernel/srs_kernel_error.hpp index fee130c792..4a5553e293 100644 --- a/trunk/src/kernel/srs_kernel_error.hpp +++ b/trunk/src/kernel/srs_kernel_error.hpp @@ -266,7 +266,8 @@ XX(ERROR_INOTIFY_OPENFD , 3094, "InotifyOpenFd", "Failed to open inotify fd for config listener") \ XX(ERROR_INOTIFY_WATCH , 3095, "InotfyWatch", "Failed to watch inotify for config listener") \ XX(ERROR_HTTP_URL_UNESCAPE , 3096, "HttpUrlUnescape", "Failed to unescape URL for HTTP") \ - XX(ERROR_HTTP_WITH_BODY , 3097, "HttpWithBody", "Failed for HTTP body") + XX(ERROR_HTTP_WITH_BODY , 3097, "HttpWithBody", "Failed for HTTP body") \ + XX(ERROR_HEVC_DISABLED , 3098, "HevcDisabled", "HEVC is disabled") /**************************************************/ /* HTTP/StreamConverter protocol error. */ diff --git a/trunk/src/kernel/srs_kernel_ts.cpp b/trunk/src/kernel/srs_kernel_ts.cpp index 2b3e43ab53..7dd659cc16 100644 --- a/trunk/src/kernel/srs_kernel_ts.cpp +++ b/trunk/src/kernel/srs_kernel_ts.cpp @@ -43,6 +43,9 @@ string srs_ts_stream2string(SrsTsStream stream) case SrsTsStreamAudioAC3: return "AC3"; case SrsTsStreamAudioDTS: return "AudioDTS"; case SrsTsStreamVideoH264: return "H.264"; +#ifdef SRS_H265 + case SrsTsStreamVideoHEVC: return "H.265"; +#endif case SrsTsStreamVideoMpeg4: return "MP4"; case SrsTsStreamAudioMpeg4: return "MP4A"; default: return "Other"; @@ -285,6 +288,14 @@ srs_error_t SrsTsContext::encode(ISrsStreamWriter* writer, SrsTsMessage* msg, Sr vs = SrsTsStreamVideoH264; video_pid = TS_VIDEO_AVC_PID; break; + case SrsVideoCodecIdHEVC: +#ifdef SRS_H265 + vs = SrsTsStreamVideoHEVC; + video_pid = TS_VIDEO_AVC_PID; + break; +#else + return srs_error_new(ERROR_HEVC_DISABLED, "H.265 is disabled"); +#endif case SrsVideoCodecIdDisabled: vs = SrsTsStreamReserved; break; @@ -296,7 +307,6 @@ srs_error_t SrsTsContext::encode(ISrsStreamWriter* writer, SrsTsMessage* msg, Sr case SrsVideoCodecIdOn2VP6: case SrsVideoCodecIdOn2VP6WithAlphaChannel: case SrsVideoCodecIdScreenVideoVersion2: - case SrsVideoCodecIdHEVC: case SrsVideoCodecIdAV1: vs = SrsTsStreamReserved; break; @@ -335,10 +345,13 @@ srs_error_t SrsTsContext::encode(ISrsStreamWriter* writer, SrsTsMessage* msg, Sr return srs_error_new(ERROR_HLS_NO_STREAM, "ts: no a/v stream, vcodec=%d, acodec=%d", vc, ac); } - // when any codec changed, write PAT/PMT table. + // When any codec changed, write PAT/PMT table. if (vcodec != vc || acodec != ac) { - vcodec = vc; - acodec = ac; + if (vcodec != SrsVideoCodecIdReserved || acodec != SrsAudioCodecIdReserved1) { + srs_trace("TS: Refresh PMT when vcodec=%d=>%d, acodec=%d=>%d", vcodec, vc, acodec, ac); + } + vcodec = vc; acodec = ac; + if ((err = encode_pat_pmt(writer, video_pid, vs, audio_pid, as)) != srs_success) { return srs_error_wrap(err, "ts: encode PAT/PMT"); } @@ -355,8 +368,12 @@ srs_error_t SrsTsContext::encode(ISrsStreamWriter* writer, SrsTsMessage* msg, Sr srs_error_t SrsTsContext::encode_pat_pmt(ISrsStreamWriter* writer, int16_t vpid, SrsTsStream vs, int16_t apid, SrsTsStream as) { srs_error_t err = srs_success; - - if (vs != SrsTsStreamVideoH264 && as != SrsTsStreamAudioAAC && as != SrsTsStreamAudioMp3) { + + bool codec_ok = (vs == SrsTsStreamVideoH264 || as == SrsTsStreamAudioAAC || as == SrsTsStreamAudioMp3); +#ifdef SRS_H265 + codec_ok = codec_ok ? : (vs == SrsTsStreamVideoHEVC); +#endif + if (!codec_ok) { return srs_error_new(ERROR_HLS_NO_STREAM, "ts: no PID, vs=%d, as=%d", vs, as); } @@ -425,8 +442,12 @@ srs_error_t SrsTsContext::encode_pes(ISrsStreamWriter* writer, SrsTsMessage* msg if (msg->payload->length() == 0) { return err; } - - if (sid != SrsTsStreamVideoH264 && sid != SrsTsStreamAudioMp3 && sid != SrsTsStreamAudioAAC) { + + bool codec_ok = (sid == SrsTsStreamVideoH264 || sid == SrsTsStreamAudioAAC || sid == SrsTsStreamAudioMp3); +#ifdef SRS_H265 + codec_ok = codec_ok ? : (sid == SrsTsStreamVideoHEVC); +#endif + if (!codec_ok) { srs_info("ts: ignore the unknown stream, sid=%d", sid); return err; } @@ -750,10 +771,14 @@ SrsTsPacket* SrsTsPacket::create_pmt(SrsTsContext* context, pmt->current_next_indicator = 1; pmt->section_number = 0; pmt->last_section_number = 0; - - // must got one valid codec. - srs_assert(vs == SrsTsStreamVideoH264 || as == SrsTsStreamAudioAAC || as == SrsTsStreamAudioMp3); - + + // Here we must get the correct codec. + bool codec_ok = (vs == SrsTsStreamVideoH264 || as == SrsTsStreamAudioAAC || as == SrsTsStreamAudioMp3); +#ifdef SRS_H265 + codec_ok = codec_ok ? : (vs == SrsTsStreamVideoHEVC); +#endif + srs_assert(codec_ok); + // if mp3 or aac specified, use audio to carry pcr. if (as == SrsTsStreamAudioAAC || as == SrsTsStreamAudioMp3) { // use audio to carray pcr by default. @@ -762,8 +787,12 @@ SrsTsPacket* SrsTsPacket::create_pmt(SrsTsContext* context, pmt->infos.push_back(new SrsTsPayloadPMTESInfo(as, apid)); } - // if h.264 specified, use video to carry pcr. - if (vs == SrsTsStreamVideoH264) { + // If h.264/h.265 specified, use video to carry pcr. + codec_ok = (vs == SrsTsStreamVideoH264); +#ifdef SRS_H265 + codec_ok = codec_ok ? : (vs == SrsTsStreamVideoHEVC); +#endif + if (codec_ok) { pmt->PCR_PID = vpid; pmt->infos.push_back(new SrsTsPayloadPMTESInfo(vs, vpid)); } @@ -2533,6 +2562,9 @@ srs_error_t SrsTsPayloadPMT::psi_decode(SrsBuffer* stream) // update the apply pid table switch (info->stream_type) { case SrsTsStreamVideoH264: +#ifdef SRS_H265 + case SrsTsStreamVideoHEVC: +#endif case SrsTsStreamVideoMpeg4: packet->context->set(info->elementary_PID, SrsTsPidApplyVideo, info->stream_type); break; @@ -2616,6 +2648,9 @@ srs_error_t SrsTsPayloadPMT::psi_encode(SrsBuffer* stream) // update the apply pid table switch (info->stream_type) { case SrsTsStreamVideoH264: +#ifdef SRS_H265 + case SrsTsStreamVideoHEVC: +#endif case SrsTsStreamVideoMpeg4: packet->context->set(info->elementary_PID, SrsTsPidApplyVideo, info->stream_type); break; @@ -2685,6 +2720,11 @@ SrsVideoCodecId SrsTsContextWriter::video_codec() return vcodec; } +void SrsTsContextWriter::update_video_codec(SrsVideoCodecId v) +{ + vcodec = v; +} + SrsEncFileWriter::SrsEncFileWriter() { memset(iv,0,16); @@ -2832,8 +2872,17 @@ srs_error_t SrsTsMessageCache::cache_video(SrsVideoFrame* frame, int64_t dts) video->dts = dts; video->pts = video->dts + frame->cts * 90; video->sid = SrsTsPESStreamIdVideoCommon; - - // write video to cache. + + // Write H.265 video frame to cache. + if (frame && frame->vcodec()->id == SrsVideoCodecIdHEVC) { +#ifdef SRS_H265 + return do_cache_hevc(frame); +#else + return srs_error_new(ERROR_HEVC_DISABLED, "H.265 is disabled"); +#endif + } + + // Write H.264 video frame to cache. if ((err = do_cache_avc(frame)) != srs_success) { return srs_error_wrap(err, "ts: cache avc"); } @@ -2924,7 +2973,7 @@ srs_error_t SrsTsMessageCache::do_cache_aac(SrsAudioFrame* frame) return err; } -void srs_avc_insert_aud(SrsSimpleStream* payload, bool& aud_inserted) +void srs_avc_insert_aud(SrsSimpleStream* payload, bool aud_inserted) { // mux the samples in annexb format, // ISO_IEC_14496-10-AVC-2012.pdf, page 324. @@ -3064,6 +3113,52 @@ srs_error_t SrsTsMessageCache::do_cache_avc(SrsVideoFrame* frame) return err; } +#ifdef SRS_H265 +srs_error_t SrsTsMessageCache::do_cache_hevc(SrsVideoFrame* frame) +{ + srs_error_t err = srs_success; + + // Whether aud inserted. + bool aud_inserted = false; + + SrsVideoCodecConfig* codec = frame->vcodec(); + srs_assert(codec); + + bool is_sps_pps_appended = false; + + // all sample use cont nalu header, except the sps-pps before IDR frame. + for (int i = 0; i < frame->nb_samples; i++) { + SrsSample* sample = &frame->samples[i]; + int32_t size = sample->size; + + if (!sample->bytes || size <= 0) { + return srs_error_new(ERROR_HLS_AVC_SAMPLE_SIZE, "ts: invalid avc sample length=%d", size); + } + + // Insert aud before NALU for HEVC. + SrsHevcNaluType nalu_type = (SrsHevcNaluType)SrsHevcNaluTypeParse(sample->bytes[0]); + bool is_idr = (SrsHevcNaluType_CODED_SLICE_BLA <= nalu_type) && (nalu_type <= SrsHevcNaluType_RESERVED_23); + if (is_idr && !frame->has_sps_pps && !is_sps_pps_appended) { + for (size_t i = 0; i < codec->hevc_dec_conf_record_.nalu_vec.size(); i++) { + const SrsHevcHvccNalu& nalu = codec->hevc_dec_conf_record_.nalu_vec[i]; + if (nalu.num_nalus <= 0 || nalu.nal_data_vec.empty()) continue; + + srs_avc_insert_aud(video->payload, aud_inserted); + const SrsHevcNalData& data = nalu.nal_data_vec.at(0); + video->payload->append((char*)&data.nal_unit_data[0], (int)data.nal_unit_data.size()); + is_sps_pps_appended = true; + } + } + + // Insert the NALU to video in annexb. + srs_avc_insert_aud(video->payload, aud_inserted); + video->payload->append(sample->bytes, sample->size); + } + + return err; +} +#endif + SrsTsTransmuxer::SrsTsTransmuxer() { writer = NULL; @@ -3158,10 +3253,17 @@ srs_error_t SrsTsTransmuxer::write_video(int64_t timestamp, char* data, int size if (format->video->frame_type == SrsVideoAvcFrameTypeVideoInfoFrame) { return err; } - - if (format->vcodec->id != SrsVideoCodecIdAVC) { + + bool codec_ok = (format->vcodec->id != SrsVideoCodecIdAVC); +#ifdef SRS_H265 + codec_ok = codec_ok ? : (format->vcodec->id != SrsVideoCodecIdHEVC); +#endif + if (!codec_ok) { return err; } + + // The video codec might change during streaming. + tscw->update_video_codec(format->vcodec->id); // ignore sequence header if (format->video->frame_type == SrsVideoAvcFrameTypeKeyFrame && format->video->avc_packet_type == SrsVideoAvcFrameTraitSequenceHeader) { diff --git a/trunk/src/kernel/srs_kernel_ts.hpp b/trunk/src/kernel/srs_kernel_ts.hpp index dd7bb83fd4..0fd7acea2b 100644 --- a/trunk/src/kernel/srs_kernel_ts.hpp +++ b/trunk/src/kernel/srs_kernel_ts.hpp @@ -131,6 +131,10 @@ enum SrsTsStream // ITU-T Rec. H.222.0 | ISO/IEC 13818-1 Reserved // 0x15-0x7F SrsTsStreamVideoH264 = 0x1b, +#ifdef SRS_H265 + // For HEVC(H.265). + SrsTsStreamVideoHEVC = 0x24, +#endif // User Private // 0x80-0xFF SrsTsStreamAudioAC3 = 0x81, @@ -1272,8 +1276,9 @@ class SrsTsContextWriter // Write a video frame to ts, virtual srs_error_t write_video(SrsTsMessage* video); public: - // get the video codec of ts muxer. + // Get or update the video codec of ts muxer. virtual SrsVideoCodecId video_codec(); + virtual void update_video_codec(SrsVideoCodecId v); }; // Used for HLS Encryption @@ -1315,6 +1320,9 @@ class SrsTsMessageCache virtual srs_error_t do_cache_mp3(SrsAudioFrame* frame); virtual srs_error_t do_cache_aac(SrsAudioFrame* frame); virtual srs_error_t do_cache_avc(SrsVideoFrame* frame); +#ifdef SRS_H265 + virtual srs_error_t do_cache_hevc(SrsVideoFrame* frame); +#endif }; // Transmux the RTMP stream to HTTP-TS stream. diff --git a/trunk/src/utest/srs_utest_kernel.cpp b/trunk/src/utest/srs_utest_kernel.cpp index a469a61069..6ba29930d4 100644 --- a/trunk/src/utest/srs_utest_kernel.cpp +++ b/trunk/src/utest/srs_utest_kernel.cpp @@ -4921,9 +4921,6 @@ VOID TEST(KernelTSTest, CoverContextEncode) srs_error_t err = ctx.encode(&f, &m, SrsVideoCodecIdDisabled, SrsAudioCodecIdDisabled); HELPER_EXPECT_FAILED(err); - - err = ctx.encode(&f, &m, SrsVideoCodecIdHEVC, SrsAudioCodecIdOpus); - HELPER_EXPECT_FAILED(err); err = ctx.encode(&f, &m, SrsVideoCodecIdAV1, SrsAudioCodecIdOpus); HELPER_EXPECT_FAILED(err); @@ -4953,6 +4950,34 @@ VOID TEST(KernelTSTest, CoverContextEncode) } } +VOID TEST(KernelTSTest, CoverContextEncodeHEVC) +{ + srs_error_t err; + + SrsTsContext ctx; + MockTsHandler h; + +#ifndef SRS_H265 + if (true) { + MockSrsFileWriter f; + SrsTsMessage m; + + err = ctx.encode(&f, &m, SrsVideoCodecIdHEVC, SrsAudioCodecIdOpus); + HELPER_EXPECT_FAILED(err); + } +#endif + +#ifdef SRS_H265 + if (true) { + MockSrsFileWriter f; + SrsTsMessage m; + + err = ctx.encode(&f, &m, SrsVideoCodecIdHEVC, SrsAudioCodecIdOpus); + HELPER_EXPECT_SUCCESS(err); + } +#endif +} + VOID TEST(KernelTSTest, CoverContextDecode) { srs_error_t err;