Add vp9 svc support by Dependency Descriptor (livekit#1586)

* Add VP9 SVC support * Fix preferred fps does not work * Fix forwarder test
Stark-Industries-Ltd · Jul 21, 2023 · 1be3e66 · 1be3e66
1 parent 4e5cef5
commit 1be3e66
Show file tree

Hide file tree

Showing 6 changed files with 44 additions and 18 deletions.
diff --git a/pkg/rtc/mediaengine.go b/pkg/rtc/mediaengine.go
@@ -40,16 +40,14 @@ func registerCodecs(me *webrtc.MediaEngine, codecs []*livekit.Codec, rtcpFeedbac
 			RTPCodecCapability: webrtc.RTPCodecCapability{MimeType: webrtc.MimeTypeVP8, ClockRate: 90000, RTCPFeedback: rtcpFeedback.Video},
 			PayloadType:        96,
 		},
-		/*
-			{
-				RTPCodecCapability: webrtc.RTPCodecCapability{MimeType: webrtc.MimeTypeVP9, ClockRate: 90000, SDPFmtpLine: "profile-id=0", RTCPFeedback: rtcpFeedback.Video},
-				PayloadType:        98,
-			},
-			{
-				RTPCodecCapability: webrtc.RTPCodecCapability{MimeType: webrtc.MimeTypeVP9, ClockRate: 90000, SDPFmtpLine: "profile-id=1", RTCPFeedback: rtcpFeedback.Video},
-				PayloadType:        100,
-			},
-		*/
+		{
+			RTPCodecCapability: webrtc.RTPCodecCapability{MimeType: webrtc.MimeTypeVP9, ClockRate: 90000, SDPFmtpLine: "profile-id=0", RTCPFeedback: rtcpFeedback.Video},
+			PayloadType:        98,
+		},
+		{
+			RTPCodecCapability: webrtc.RTPCodecCapability{MimeType: webrtc.MimeTypeVP9, ClockRate: 90000, SDPFmtpLine: "profile-id=1", RTCPFeedback: rtcpFeedback.Video},
+			PayloadType:        100,
+		},
 		{
 			RTPCodecCapability: webrtc.RTPCodecCapability{MimeType: webrtc.MimeTypeH264, ClockRate: 90000, SDPFmtpLine: "level-asymmetry-allowed=1;packetization-mode=1;profile-level-id=42e01f", RTCPFeedback: rtcpFeedback.Video},
 			PayloadType:        125,

diff --git a/pkg/rtc/participant_sdp.go b/pkg/rtc/participant_sdp.go
@@ -132,7 +132,7 @@ func (p *ParticipantImpl) setCodecPreferencesVideoForPublisher(offer webrtc.Sess
 
 		mime = strings.ToUpper(mime)
 		// remove dd extension if av1 not preferred
-		if !strings.Contains(mime, "AV1") {
+		if !strings.Contains(mime, "AV1") && !strings.Contains(mime, "VP9") {
 			for i, attr := range unmatchVideo.Attributes {
 				if strings.Contains(attr.Value, dd.ExtensionUrl) {
 					unmatchVideo.Attributes[i] = unmatchVideo.Attributes[len(unmatchVideo.Attributes)-1]

diff --git a/pkg/sfu/buffer/buffer.go b/pkg/sfu/buffer/buffer.go
@@ -564,6 +564,8 @@ func (b *Buffer) getExtPacket(rtpPacket *rtp.Packet, arrivalTime int64) *ExtPack
 		ep.KeyFrame = IsH264Keyframe(rtpPacket.Payload)
 	case "video/av1":
 		ep.KeyFrame = IsAV1Keyframe(rtpPacket.Payload)
+	case "video/vp9":
+		ep.KeyFrame = IsVP9Keyframe(rtpPacket.Payload)
 	}
 
 	if ep.KeyFrame {

diff --git a/pkg/sfu/buffer/helpers.go b/pkg/sfu/buffer/helpers.go
@@ -4,6 +4,8 @@ import (
 	"encoding/binary"
 	"errors"
 
+	"github.com/pion/rtp/codecs"
+
 	"github.com/livekit/protocol/logger"
 )
 
@@ -351,4 +353,28 @@ func IsAV1Keyframe(payload []byte) bool {
 	}
 }
 
+// IsVP9Keyframe detects if vp9 payload is a keyframe
+// taken from https://github.com/jech/galene/blob/master/codecs/codecs.go
+// all credits belongs to Juliusz Chroboczek @jech and the awesome Galene SFU
+func IsVP9Keyframe(payload []byte) bool {
+	var vp9 codecs.VP9Packet
+	_, err := vp9.Unmarshal(payload)
+	if err != nil || len(vp9.Payload) < 1 {
+		return false
+	}
+	if !vp9.B {
+		return false
+	}
+
+	if (vp9.Payload[0] & 0xc0) != 0x80 {
+		return false
+	}
+
+	profile := (vp9.Payload[0] >> 4) & 0x3
+	if profile != 3 {
+		return (vp9.Payload[0] & 0xC) == 0
+	}
+	return (vp9.Payload[0] & 0x6) == 0
+}
+
 // -------------------------------------
diff --git a/pkg/sfu/forwarder.go b/pkg/sfu/forwarder.go
@@ -273,8 +273,8 @@ func (f *Forwarder) DetermineCodec(codec webrtc.RTPCodecCapability) {
 	case "video/vp8":
 		f.isTemporalSupported = true
 		f.vp8Munger = NewVP8Munger(f.logger)
-	case "video/av1":
-		// TODO : we only enable dd layer selector for av1 now, at future we can
+	case "video/av1", "video/vp9":
+		// TODO : we only enable dd layer selector for av1 and vp9 now, at future we can
 		// enable it for vp8 too
 		f.ddLayerSelector = NewDDVideoLayerSelector(f.logger)
 	}
@@ -515,7 +515,7 @@ func (f *Forwarder) AllocateOptimal(availableLayers []int32, brs Bitrates, allow
 		}
 		alloc.TargetLayers = buffer.VideoLayer{
 			Spatial:  int32(math.Min(float64(f.maxPublishedLayer), float64(maxSpatial))),
-			Temporal: buffer.DefaultMaxLayerTemporal,
+			Temporal: f.maxLayers.Temporal,
 		}
 	}
 
@@ -570,14 +570,14 @@ func (f *Forwarder) AllocateOptimal(availableLayers []int32, brs Bitrates, allow
 					alloc.TargetLayers.Spatial = l
 				}
 			}
-			alloc.TargetLayers.Temporal = buffer.DefaultMaxLayerTemporal
+			alloc.TargetLayers.Temporal = f.maxLayers.Temporal
 
 			alloc.RequestLayerSpatial = alloc.TargetLayers.Spatial
 		} else {
 			requestLayerSpatial := int32(math.Min(float64(f.maxLayers.Spatial), float64(f.maxPublishedLayer)))
 			if f.currentLayers.IsValid() && requestLayerSpatial == f.requestLayerSpatial && f.currentLayers.Spatial == f.requestLayerSpatial {
 				// current is locked to desired, stay there
-				alloc.TargetLayers = f.currentLayers
+				alloc.TargetLayers = buffer.VideoLayer{Spatial: f.requestLayerSpatial, Temporal: f.maxLayers.Temporal}
 				alloc.RequestLayerSpatial = f.requestLayerSpatial
 			} else {
 				// opportunistically latch on to anything

diff --git a/pkg/sfu/forwarder_test.go b/pkg/sfu/forwarder_test.go
@@ -387,7 +387,7 @@ func TestForwarderAllocateOptimal(t *testing.T) {
 	f.requestLayerSpatial = 0
 	expectedTargetLayers = buffer.VideoLayer{
 		Spatial:  2,
-		Temporal: 3,
+		Temporal: 1,
 	}
 	expectedResult = VideoAllocation{
 		PauseReason:         VideoPauseReasonFeedDry,
@@ -397,7 +397,7 @@ func TestForwarderAllocateOptimal(t *testing.T) {
 		TargetLayers:        expectedTargetLayers,
 		RequestLayerSpatial: 2,
 		MaxLayers:           f.maxLayers,
-		DistanceToDesired:   -1.5,
+		DistanceToDesired:   -1,
 	}
 	result = f.AllocateOptimal([]int32{0, 1}, emptyBitrates, true)
 	require.Equal(t, expectedResult, result)