Skip to content

Commit

Permalink
SampleBuilder: Port to use jitter buffer
Browse files Browse the repository at this point in the history
  • Loading branch information
@thatsnotright
thatsnotright committed Mar 25, 2024
1 parent cd73129 commit d080e21
Show file tree
Hide file tree
Showing 2 changed files with 61 additions and 33 deletions.
63 changes: 43 additions & 20 deletions pkg/media/samplebuilder/samplebuilder.go
View file
Original file line number Diff line number Diff line change
Expand Up @@ -8,6 +8,7 @@ import (
"math"
"time"

"github.com/pion/interceptor/pkg/jitterbuffer"
"github.com/pion/rtp"
"github.com/pion/webrtc/v4/pkg/media"
)
Expand All @@ -16,7 +17,7 @@ import (
type SampleBuilder struct {
maxLate uint16 // how many packets to wait until we get a valid Sample
maxLateTimestamp uint32 // max timestamp between old and new timestamps before dropping packets
buffer [math.MaxUint16 + 1]*rtp.Packet
buffer *jitterbuffer.JitterBuffer
preparedSamples [math.MaxUint16 + 1]*media.Sample

// Interface that allows us to take RTP packets to samples
Expand Down Expand Up @@ -60,7 +61,7 @@ type SampleBuilder struct {
// The depacketizer extracts media samples from RTP packets.
// Several depacketizers are available in package github.com/pion/rtp/codecs.
func New(maxLate uint16, depacketizer rtp.Depacketizer, sampleRate uint32, opts ...Option) *SampleBuilder {
s := &SampleBuilder{maxLate: maxLate, depacketizer: depacketizer, sampleRate: sampleRate}
s := &SampleBuilder{maxLate: maxLate, depacketizer: depacketizer, sampleRate: sampleRate, buffer: jitterbuffer.New()}
for _, o := range opts {
o(s)
}
Expand All @@ -76,7 +77,7 @@ func (s *SampleBuilder) tooOld(location sampleSequenceLocation) bool {
var foundTail *rtp.Packet

for i := location.head; i != location.tail; i++ {
if packet := s.buffer[i]; packet != nil {
if packet, _ := s.buffer.PeekAtSequence(i); packet != nil {
foundHead = packet
break
}
Expand All @@ -87,7 +88,7 @@ func (s *SampleBuilder) tooOld(location sampleSequenceLocation) bool {
}

for i := location.tail - 1; i != location.head; i-- {
if packet := s.buffer[i]; packet != nil {
if packet, _ := s.buffer.PeekAtSequence(i); packet != nil {
foundTail = packet
break
}
Expand All @@ -105,16 +106,16 @@ func (s *SampleBuilder) fetchTimestamp(location sampleSequenceLocation) (timesta
if location.empty() {
return 0, false
}
packet := s.buffer[location.head]
if packet == nil {
packet, err := s.buffer.PeekAtSequence(location.head)
if packet == nil || err != nil {
return 0, false
}
return packet.Timestamp, true
}

func (s *SampleBuilder) releasePacket(i uint16) {
var p *rtp.Packet
p, s.buffer[i] = s.buffer[i], nil
p, _ = s.buffer.PopAtSequence(i)
if p != nil && s.packetReleaseHandler != nil {
s.packetReleaseHandler(p)
}
Expand Down Expand Up @@ -178,7 +179,7 @@ func (s *SampleBuilder) purgeBuffers(flush bool) {
// Push does not copy the input. If you wish to reuse
// this memory make sure to copy before calling Push
func (s *SampleBuilder) Push(p *rtp.Packet) {
s.buffer[p.SequenceNumber] = p
s.buffer.Push(p)

switch s.filled.compare(p.SequenceNumber) {
case slCompareVoid:
Expand Down Expand Up @@ -220,14 +221,19 @@ func (s *SampleBuilder) buildSample(purgingBuffers bool) *media.Sample {

var consume sampleSequenceLocation

for i := s.active.head; s.buffer[i] != nil && s.active.compare(i) != slCompareAfter; i++ {
if s.depacketizer.IsPartitionTail(s.buffer[i].Marker, s.buffer[i].Payload) {
for i := s.active.head; s.active.compare(i) != slCompareAfter; i++ {
pkt, err := s.buffer.PeekAtSequence(i)
if pkt == nil || err != nil {
break
}

if s.depacketizer.IsPartitionTail(pkt.Marker, pkt.Payload) {
consume.head = s.active.head
consume.tail = i + 1
break
}
headTimestamp, hasData := s.fetchTimestamp(s.active)
if hasData && s.buffer[i].Timestamp != headTimestamp {
if hasData && pkt.Timestamp != headTimestamp {
consume.head = s.active.head
consume.tail = i
break
Expand All @@ -237,8 +243,8 @@ func (s *SampleBuilder) buildSample(purgingBuffers bool) *media.Sample {
if consume.empty() {
return nil
}

if !purgingBuffers && s.buffer[consume.tail] == nil {
pkt, _ := s.buffer.PeekAtSequence(consume.tail)
if !purgingBuffers && pkt == nil {
// wait for the next packet after this set of packets to arrive
// to ensure at least one post sample timestamp is known
// (unless we have to release right now)
Expand All @@ -250,8 +256,10 @@ func (s *SampleBuilder) buildSample(purgingBuffers bool) *media.Sample {

// scan for any packet after the current and use that time stamp as the diff point
for i := consume.tail; i < s.active.tail; i++ {
if s.buffer[i] != nil {
afterTimestamp = s.buffer[i].Timestamp
pkt, _ = s.buffer.PeekAtSequence(i)

if pkt != nil {
afterTimestamp = pkt.Timestamp
break
}
}
Expand All @@ -261,10 +269,11 @@ func (s *SampleBuilder) buildSample(purgingBuffers bool) *media.Sample {

// prior to decoding all the packets, check if this packet
// would end being disposed anyway
if !s.depacketizer.IsPartitionHead(s.buffer[consume.head].Payload) {
pkt, err := s.buffer.PeekAtSequence(consume.head)
if err == nil && !s.depacketizer.IsPartitionHead(pkt.Payload) {
isPadding := false
for i := consume.head; i != consume.tail; i++ {
if s.lastSampleTimestamp != nil && *s.lastSampleTimestamp == s.buffer[i].Timestamp && len(s.buffer[i].Payload) == 0 {
if s.lastSampleTimestamp != nil && *s.lastSampleTimestamp == pkt.Timestamp && len(pkt.Payload) == 0 {
isPadding = true
}
}
Expand All @@ -282,16 +291,22 @@ func (s *SampleBuilder) buildSample(purgingBuffers bool) *media.Sample {
var metadata interface{}
var rtpHeaders []*rtp.Header
for i := consume.head; i != consume.tail; i++ {
p, err := s.depacketizer.Unmarshal(s.buffer[i].Payload)
pkt, err := s.buffer.PeekAtSequence(i)
if err != nil {
return nil
}

Check warning on line 297 in pkg/media/samplebuilder/samplebuilder.go

View check run for this annotation

Codecov / codecov/patch

pkg/media/samplebuilder/samplebuilder.go#L296-L297 

Added lines #L296 - L297 were not covered by tests
p, err := s.depacketizer.Unmarshal(pkt.Payload)
if err != nil {
return nil
}
if i == consume.head && s.packetHeadHandler != nil {
metadata = s.packetHeadHandler(s.depacketizer)
}
if s.returnRTPHeaders {
h := s.buffer[i].Header.Clone()
rtpHeaders = append(rtpHeaders, &h)
if packet, _ := s.buffer.PeekAtSequence(i); packet != nil {
h := pkt.Header.Clone()
rtpHeaders = append(rtpHeaders, &h)
}
}

data = append(data, p...)
Expand Down Expand Up @@ -389,3 +404,11 @@ func WithRTPHeaders(enable bool) Option {
o.returnRTPHeaders = enable
}
}

// WithJitterBufferMinimumLength sets the minimum number of packets which must first
// be received before starting any playback
func WithJitterBufferMinimumLength(length uint16) Option {
return func(o *SampleBuilder) {
o.buffer = jitterbuffer.New(jitterbuffer.WithMinimumPacketCount(length))
}
}
31 changes: 18 additions & 13 deletions pkg/media/samplebuilder/samplebuilder_test.go
View file
Original file line number Diff line number Diff line change
Expand Up @@ -313,6 +313,7 @@ func TestSampleBuilder(t *testing.T) {
opts = append(opts, WithRTPHeaders(true))
}

opts = append(opts, WithJitterBufferMinimumLength(1))
d := &fakeDepacketizer{
headChecker: t.withHeadChecker,
headBytes: t.headBytes,
Expand All @@ -334,7 +335,7 @@ func TestSampleBuilder(t *testing.T) {
// SampleBuilder should respect maxLate if we popped successfully but then have a gap larger then maxLate
func TestSampleBuilderMaxLate(t *testing.T) {
assert := assert.New(t)
s := New(50, &fakeDepacketizer{}, 1)
s := New(50, &fakeDepacketizer{}, 1, WithJitterBufferMinimumLength(1))

s.Push(&rtp.Packet{Header: rtp.Header{SequenceNumber: 0, Timestamp: 1}, Payload: []byte{0x01}})
s.Push(&rtp.Packet{Header: rtp.Header{SequenceNumber: 1, Timestamp: 2}, Payload: []byte{0x01}})
Expand Down Expand Up @@ -383,7 +384,7 @@ func TestSampleBuilderCleanReference(t *testing.T) {
} {
seqStart := seqStart
t.Run(fmt.Sprintf("From%d", seqStart), func(t *testing.T) {
s := New(10, &fakeDepacketizer{}, 1)
s := New(10, &fakeDepacketizer{}, 1, WithJitterBufferMinimumLength(1))

s.Push(&rtp.Packet{Header: rtp.Header{SequenceNumber: 0 + seqStart, Timestamp: 0}, Payload: []byte{0x01}})
s.Push(&rtp.Packet{Header: rtp.Header{SequenceNumber: 1 + seqStart, Timestamp: 0}, Payload: []byte{0x02}})
Expand All @@ -394,14 +395,18 @@ func TestSampleBuilderCleanReference(t *testing.T) {
s.Push(pkt5)

for i := 0; i < 3; i++ {
if s.buffer[(i+int(seqStart))%0x10000] != nil {
pkt, err := s.buffer.PeekAtSequence(uint16((i + int(seqStart)) % 0x10000))

if pkt != nil || err == nil {
t.Errorf("Old packet (%d) is not unreferenced (maxLate: 10, pushed: 12)", i)
}
}
if s.buffer[(14+int(seqStart))%0x10000] != pkt4 {
pkt, err := s.buffer.PeekAtSequence(uint16((14 + int(seqStart)) % 0x10000))
if pkt != pkt4 || err != nil {
t.Error("New packet must be referenced after jump")
}
if s.buffer[(12+int(seqStart))%0x10000] != pkt5 {
pkt, err = s.buffer.PeekAtSequence(uint16((12 + int(seqStart)) % 0x10000))
if pkt != pkt5 || err != nil {
t.Error("New packet must be referenced after jump")
}
})
Expand All @@ -418,7 +423,7 @@ func TestSampleBuilderPushMaxZero(t *testing.T) {
headBytes: []byte{0x01},
}

s := New(0, d, 1)
s := New(0, d, 1, WithJitterBufferMinimumLength(1))
s.Push(&pkts[0])
if sample := s.Pop(); sample == nil {
t.Error("Should expect a popped sample")
Expand All @@ -439,7 +444,7 @@ func TestSampleBuilderWithPacketReleaseHandler(t *testing.T) {
{Header: rtp.Header{SequenceNumber: 13, Timestamp: 122}, Payload: []byte{0x04}},
{Header: rtp.Header{SequenceNumber: 21, Timestamp: 200}, Payload: []byte{0x05}},
}
s := New(10, &fakeDepacketizer{}, 1, WithPacketReleaseHandler(fakePacketReleaseHandler))
s := New(10, &fakeDepacketizer{}, 1, WithPacketReleaseHandler(fakePacketReleaseHandler), WithJitterBufferMinimumLength(1))
s.Push(&pkts[0])
s.Push(&pkts[1])
if len(released) == 0 {
Expand Down Expand Up @@ -476,7 +481,7 @@ func TestSampleBuilderWithPacketHeadHandler(t *testing.T) {
s := New(10, &fakeDepacketizer{}, 1, WithPacketHeadHandler(func(interface{}) interface{} {
headCount++
return true
}))
}), WithJitterBufferMinimumLength(1))

for _, pkt := range packets {
s.Push(pkt)
Expand All @@ -499,7 +504,7 @@ func TestSampleBuilderData(t *testing.T) {
s := New(10, &fakeDepacketizer{
headChecker: true,
alwaysHead: true,
}, 1)
}, 1, WithJitterBufferMinimumLength(1))
j := 0
for i := 0; i < 0x20000; i++ {
p := rtp.Packet{
Expand Down Expand Up @@ -565,7 +570,7 @@ func TestSampleBuilder_Flush(t *testing.T) {
}

func BenchmarkSampleBuilderSequential(b *testing.B) {
s := New(100, &fakeDepacketizer{}, 1)
s := New(100, &fakeDepacketizer{}, 1, WithJitterBufferMinimumLength(1))
b.ResetTimer()
j := 0
for i := 0; i < b.N; i++ {
Expand All @@ -591,7 +596,7 @@ func BenchmarkSampleBuilderSequential(b *testing.B) {
}

func BenchmarkSampleBuilderLoss(b *testing.B) {
s := New(100, &fakeDepacketizer{}, 1)
s := New(100, &fakeDepacketizer{}, 1, WithJitterBufferMinimumLength(1))
b.ResetTimer()
j := 0
for i := 0; i < b.N; i++ {
Expand Down Expand Up @@ -620,7 +625,7 @@ func BenchmarkSampleBuilderLoss(b *testing.B) {
}

func BenchmarkSampleBuilderReordered(b *testing.B) {
s := New(100, &fakeDepacketizer{}, 1)
s := New(100, &fakeDepacketizer{}, 1, WithJitterBufferMinimumLength(1))
b.ResetTimer()
j := 0
for i := 0; i < b.N; i++ {
Expand All @@ -646,7 +651,7 @@ func BenchmarkSampleBuilderReordered(b *testing.B) {
}

func BenchmarkSampleBuilderFragmented(b *testing.B) {
s := New(100, &fakeDepacketizer{}, 1)
s := New(100, &fakeDepacketizer{}, 1, WithJitterBufferMinimumLength(1))
b.ResetTimer()
j := 0
for i := 0; i < b.N; i++ {
Expand Down

0 comments on commit d080e21

Please sign in to comment.