support reading H265 tracks with HLS (#1342)

* support reading H265 tracks with HLS

* update README

README.md

@@ -22,7 +22,7 @@ And can be read from the server with:
 |--------|--------|------|
 |RTSP|UDP, UDP-Multicast, TCP, RTSPS|H264, H265, VP8, VP9, AV1, MPEG2, M-JPEG, MP3, MPEG4 Audio (AAC), Opus, G711, G722, LPCM and any RTP-compatible codec|
 |RTMP|RTMP, RTMPS|H264, MPEG4 Audio (AAC)|
-|HLS|Low-Latency HLS, MP4-based HLS, legacy HLS|H264, MPEG4 Audio (AAC)|
+|HLS|Low-Latency HLS, MP4-based HLS, legacy HLS|H264, H265, MPEG4 Audio (AAC)|
 |WebRTC||H264, VP8, VP9, Opus, G711, G722|
 
 Features:
@@ -89,6 +89,7 @@ Features:
   * [Encryption](#encryption-1)
 * [HLS protocol](#hls-protocol)
   * [General usage](#general-usage-2)
+  * [Browser support](#browser-support)
   * [Embedding](#embedding)
   * [Low-Latency variant](#low-latency-variant)
   * [Decreasing latency](#decreasing-latency)
@@ -903,7 +904,13 @@ http://localhost:8888/mystream
 
 where `mystream` is the name of a stream that is being published.
 
-Please be aware that HLS only supports a single H264 video track and a single AAC audio track due to limitations of most browsers. If you want to use HLS with streams that use other codecs, you have to re-encode them, for instance by using _FFmpeg_:
+### Browser support
+
+Although the server can produce HLS with a variety of video and audio codecs (that are listed at the beginningo of the README), not all browsers can read all codecs. You can check what codecs your browser can read by visiting this page:
+
+https://jsfiddle.net/7nwxmLto
+
+If you want to increase the compatibility of the stream in order to support most browsers, you have to re-encode it by using the H264 and AAC codecs, for instance by using _FFmpeg_:
 
 ```
 ffmpeg -i rtsp://original-source -pix_fmt yuv420p -c:v libx264 -preset ultrafast -b:v 600k -c:a aac -b:a 160k -f rtsp rtsp://localhost:8554/mystream

go.mod

@@ -3,9 +3,9 @@ module github.com/aler9/rtsp-simple-server
 go 1.18
 
 require (
-	code.cloudfoundry.org/bytefmt v0.0.0-20211005130812-5bb3c17173e5
-	github.com/abema/go-mp4 v0.8.0
-	github.com/aler9/gortsplib/v2 v2.0.0-20221228192116-da21f946e562
+	code.cloudfoundry.org/bytefmt v0.0.0
+	github.com/abema/go-mp4 v0.0.0
+	github.com/aler9/gortsplib/v2 v2.0.0-20221229123705-ce25207cb823
 	github.com/asticode/go-astits v1.10.1-0.20220319093903-4abe66a9b757
 	github.com/fsnotify/fsnotify v1.4.9
 	github.com/gin-gonic/gin v1.8.1
@@ -68,3 +68,5 @@ require (
 replace github.com/orcaman/writerseeker => github.com/aler9/writerseeker v0.0.0-20220601075008-6f0e685b9c82
 
 replace code.cloudfoundry.org/bytefmt => github.com/cloudfoundry/bytefmt v0.0.0-20211005130812-5bb3c17173e5
+
+replace github.com/abema/go-mp4 => github.com/aler9/go-mp4 v0.0.0-20221229152535-34c82c552218

go.sum

@@ -1,11 +1,11 @@
-github.com/abema/go-mp4 v0.8.0 h1:JHYkOvTfBpTnqJHiFFOXe8d6wiFy5MtDnA10fgccNqY=
-github.com/abema/go-mp4 v0.8.0/go.mod h1:vPl9t5ZK7K0x68jh12/+ECWBCXoWuIDtNgPtU2f04ws=
 github.com/alecthomas/template v0.0.0-20190718012654-fb15b899a751 h1:JYp7IbQjafoB+tBA3gMyHYHrpOtNuDiK/uB5uXxq5wM=
 github.com/alecthomas/template v0.0.0-20190718012654-fb15b899a751/go.mod h1:LOuyumcjzFXgccqObfd/Ljyb9UuFJ6TxHnclSeseNhc=
 github.com/alecthomas/units v0.0.0-20190924025748-f65c72e2690d h1:UQZhZ2O0vMHr2cI+DC1Mbh0TJxzA3RcLoMsFw+aXw7E=
 github.com/alecthomas/units v0.0.0-20190924025748-f65c72e2690d/go.mod h1:rBZYJk541a8SKzHPHnH3zbiI+7dagKZ0cgpgrD7Fyho=
-github.com/aler9/gortsplib/v2 v2.0.0-20221228192116-da21f946e562 h1://BJIsHw2vYKdPL6sKbxZEnlGPpj2PTznNzRpou87ds=
-github.com/aler9/gortsplib/v2 v2.0.0-20221228192116-da21f946e562/go.mod h1:lMdAxc6daduSzVwh75yQkvH9UHCYHpng/vJ8uXKFzdA=
+github.com/aler9/go-mp4 v0.0.0-20221229152535-34c82c552218 h1:Zak89uY+y0q/gL7jaKbl2XeyMOLT/5qVuW6TIJphEJY=
+github.com/aler9/go-mp4 v0.0.0-20221229152535-34c82c552218/go.mod h1:vPl9t5ZK7K0x68jh12/+ECWBCXoWuIDtNgPtU2f04ws=
+github.com/aler9/gortsplib/v2 v2.0.0-20221229123705-ce25207cb823 h1:EFq9LqgA15drNgXj3hNlmAouxjMYb9jyyBq6hmjDO8U=
+github.com/aler9/gortsplib/v2 v2.0.0-20221229123705-ce25207cb823/go.mod h1:lMdAxc6daduSzVwh75yQkvH9UHCYHpng/vJ8uXKFzdA=
 github.com/aler9/writerseeker v0.0.0-20220601075008-6f0e685b9c82 h1:9WgSzBLo3a9ToSVV7sRTBYZ1GGOZUpq4+5H3SN0UZq4=
 github.com/aler9/writerseeker v0.0.0-20220601075008-6f0e685b9c82/go.mod h1:qsMrZCbeBf/mCLOeF16KDkPu4gktn/pOWyaq1aYQE7U=
 github.com/asticode/go-astikit v0.20.0 h1:+7N+J4E4lWx2QOkRdOf6DafWJMv6O4RRfgClwQokrH8=

internal/core/formatprocessor_h264.go

@@ -71,7 +71,7 @@ type dataH264 struct {
 	rtpPackets []*rtp.Packet
 	ntp        time.Time
 	pts        time.Duration
-	nalus      [][]byte
+	au         [][]byte
 }
 
 func (d *dataH264) getRTPPackets() []*rtp.Packet {
@@ -134,22 +134,23 @@ func (t *formatProcessorH264) updateTrackParametersFromNALUs(nalus [][]byte) {
 	}
 }
 
-// remux is needed to fix corrupted streams and make streams
-// compatible with all protocols.
-func (t *formatProcessorH264) remuxNALUs(nalus [][]byte) [][]byte {
-	addSPSPPS := false
+func (t *formatProcessorH264) remuxAccessUnit(nalus [][]byte) [][]byte {
+	addParameters := false
 	n := 0
+
 	for _, nalu := range nalus {
 		typ := h264.NALUType(nalu[0] & 0x1F)
+
 		switch typ {
-		case h264.NALUTypeSPS, h264.NALUTypePPS:
+		case h264.NALUTypeSPS, h264.NALUTypePPS: // remove parameters
 			continue
-		case h264.NALUTypeAccessUnitDelimiter:
+
+		case h264.NALUTypeAccessUnitDelimiter: // remove AUDs
 			continue
-		case h264.NALUTypeIDR:
-			// prepend SPS and PPS to the group if there's at least an IDR
-			if !addSPSPPS {
-				addSPSPPS = true
+
+		case h264.NALUTypeIDR: // prepend parameters if there's at least an IDR
+			if !addParameters {
+				addParameters = true
 				n += 2
 			}
 		}
@@ -163,21 +164,20 @@ func (t *formatProcessorH264) remuxNALUs(nalus [][]byte) [][]byte {
 	filteredNALUs := make([][]byte, n)
 	i := 0
 
-	if addSPSPPS {
+	if addParameters {
 		filteredNALUs[0] = t.format.SafeSPS()
 		filteredNALUs[1] = t.format.SafePPS()
 		i = 2
 	}
 
 	for _, nalu := range nalus {
 		typ := h264.NALUType(nalu[0] & 0x1F)
+
 		switch typ {
 		case h264.NALUTypeSPS, h264.NALUTypePPS:
-			// remove since they're automatically added
 			continue
 
 		case h264.NALUTypeAccessUnitDelimiter:
-			// remove since it is not needed
 			continue
 		}
 
@@ -227,30 +227,29 @@ func (t *formatProcessorH264) process(dat data, hasNonRTSPReaders bool) error {
 			}
 
 			// DecodeUntilMarker() is necessary, otherwise Encode() generates partial groups
-			nalus, pts, err := t.decoder.DecodeUntilMarker(pkt)
+			au, pts, err := t.decoder.DecodeUntilMarker(pkt)
 			if err != nil {
 				if err == rtph264.ErrNonStartingPacketAndNoPrevious || err == rtph264.ErrMorePacketsNeeded {
 					return nil
 				}
 				return err
 			}
 
-			tdata.nalus = nalus
+			tdata.au = au
 			tdata.pts = pts
-
-			tdata.nalus = t.remuxNALUs(tdata.nalus)
+			tdata.au = t.remuxAccessUnit(tdata.au)
 		}
 
 		// route packet as is
 		if t.encoder == nil {
 			return nil
 		}
 	} else {
-		t.updateTrackParametersFromNALUs(tdata.nalus)
-		tdata.nalus = t.remuxNALUs(tdata.nalus)
+		t.updateTrackParametersFromNALUs(tdata.au)
+		tdata.au = t.remuxAccessUnit(tdata.au)
 	}
 
-	pkts, err := t.encoder.Encode(tdata.nalus, tdata.pts)
+	pkts, err := t.encoder.Encode(tdata.au, tdata.pts)
 	if err != nil {
 		return err
 	}

internal/core/formatprocessor_h265.go

@@ -78,7 +78,7 @@ type dataH265 struct {
 	rtpPackets []*rtp.Packet
 	ntp        time.Time
 	pts        time.Duration
-	nalus      [][]byte
+	au         [][]byte
 }
 
 func (d *dataH265) getRTPPackets() []*rtp.Packet {
@@ -128,12 +128,82 @@ func (t *formatProcessorH265) updateTrackParametersFromRTPPacket(pkt *rtp.Packet
 }
 
 func (t *formatProcessorH265) updateTrackParametersFromNALUs(nalus [][]byte) {
-	// TODO: extract VPS, SPS, PPS and set them into the track
+	for _, nalu := range nalus {
+		typ := h265.NALUType((nalu[0] >> 1) & 0b111111)
+
+		switch typ {
+		case h265.NALUType_VPS_NUT:
+			if !bytes.Equal(nalu, t.format.SafeVPS()) {
+				t.format.SafeSetVPS(nalu)
+			}
+
+		case h265.NALUType_SPS_NUT:
+			if !bytes.Equal(nalu, t.format.SafePPS()) {
+				t.format.SafeSetSPS(nalu)
+			}
+
+		case h265.NALUType_PPS_NUT:
+			if !bytes.Equal(nalu, t.format.SafePPS()) {
+				t.format.SafeSetPPS(nalu)
+			}
+		}
+	}
 }
 
-func (t *formatProcessorH265) remuxNALUs(nalus [][]byte) [][]byte {
-	// TODO: add VPS, SPS, PPS before IDRs
-	return nalus
+func (t *formatProcessorH265) remuxAccessUnit(nalus [][]byte) [][]byte {
+	addParameters := false
+	n := 0
+
+	for _, nalu := range nalus {
+		typ := h265.NALUType((nalu[0] >> 1) & 0b111111)
+
+		switch typ {
+		case h265.NALUType_VPS_NUT, h265.NALUType_SPS_NUT, h265.NALUType_PPS_NUT: // remove parameters
+			continue
+
+		case h265.NALUType_AUD_NUT: // remove AUDs
+			continue
+
+		// prepend parameters if there's at least a random access unit
+		case h265.NALUType_IDR_W_RADL, h265.NALUType_IDR_N_LP, h265.NALUType_CRA_NUT:
+			if !addParameters {
+				addParameters = true
+				n += 3
+			}
+		}
+		n++
+	}
+
+	if n == 0 {
+		return nil
+	}
+
+	filteredNALUs := make([][]byte, n)
+	i := 0
+
+	if addParameters {
+		filteredNALUs[0] = t.format.SafeVPS()
+		filteredNALUs[1] = t.format.SafeSPS()
+		filteredNALUs[2] = t.format.SafePPS()
+		i = 3
+	}
+
+	for _, nalu := range nalus {
+		typ := h265.NALUType((nalu[0] >> 1) & 0b111111)
+
+		switch typ {
+		case h265.NALUType_VPS_NUT, h265.NALUType_SPS_NUT, h265.NALUType_PPS_NUT:
+			continue
+
+		case h265.NALUType_AUD_NUT:
+			continue
+		}
+
+		filteredNALUs[i] = nalu
+		i++
+	}
+
+	return filteredNALUs
 }
 
 func (t *formatProcessorH265) process(dat data, hasNonRTSPReaders bool) error { //nolint:dupl
@@ -175,30 +245,29 @@ func (t *formatProcessorH265) process(dat data, hasNonRTSPReaders bool) error {
 			}
 
 			// DecodeUntilMarker() is necessary, otherwise Encode() generates partial groups
-			nalus, pts, err := t.decoder.DecodeUntilMarker(pkt)
+			au, pts, err := t.decoder.DecodeUntilMarker(pkt)
 			if err != nil {
 				if err == rtph265.ErrNonStartingPacketAndNoPrevious || err == rtph265.ErrMorePacketsNeeded {
 					return nil
 				}
 				return err
 			}
 
-			tdata.nalus = nalus
+			tdata.au = au
 			tdata.pts = pts
-
-			tdata.nalus = t.remuxNALUs(tdata.nalus)
+			tdata.au = t.remuxAccessUnit(tdata.au)
 		}
 
 		// route packet as is
 		if t.encoder == nil {
 			return nil
 		}
 	} else {
-		t.updateTrackParametersFromNALUs(tdata.nalus)
-		tdata.nalus = t.remuxNALUs(tdata.nalus)
+		t.updateTrackParametersFromNALUs(tdata.au)
+		tdata.au = t.remuxAccessUnit(tdata.au)
 	}
 
-	pkts, err := t.encoder.Encode(tdata.nalus, tdata.pts)
+	pkts, err := t.encoder.Encode(tdata.au, tdata.pts)
 	if err != nil {
 		return err
 	}