Add support for transcoding to L16 codec on backchannel

pkg/bubble/client.go

@@ -228,7 +228,7 @@ func (c *Client) Handle() error {
 				},
 				Payload: annexb.EncodeToAVCC(b[6:], false),
 			}
-			c.videoTrack.WriteRTP(pkt)
+			c.videoTrack.Handler(pkt)
 		} else {
 			if c.audioTrack == nil {
 				continue
@@ -251,7 +251,7 @@ func (c *Client) Handle() error {
 				Payload: b[6+36:],
 			}
 			audioTS += uint32(len(pkt.Payload))
-			c.audioTrack.WriteRTP(pkt)
+			c.audioTrack.Handler(pkt)
 		}
 	}
 }

pkg/core/track.go

@@ -26,14 +26,23 @@ type Receiver struct {
 
 	ID byte // Channel for RTSP, PayloadType for MPEG-TS
 
+	Handler HandlerFunc
+
 	senders map[*Sender]chan *rtp.Packet
 	mu      sync.RWMutex
 	bytes   int
 }
 
 func NewReceiver(media *Media, codec *Codec) *Receiver {
 	Assert(codec != nil)
-	return &Receiver{Codec: codec, Media: media}
+
+	receiver := &Receiver{Codec: codec, Media: media}
+
+	receiver.Handler = func(packet *rtp.Packet) {
+		receiver.WriteRTP(packet)
+	}
+
+	return receiver
 }
 
 // WriteRTP - fast and non blocking write to all readers buffers

pkg/dvrip/producer.go

@@ -58,7 +58,7 @@ func (c *Producer) Start() error {
 
 			//log.Printf("[AVC] %v, len: %d, ts: %10d", h265.Types(payload), len(payload), packet.Timestamp)
 
-			c.video.WriteRTP(packet)
+			c.video.Handler(packet)
 
 		case 0xFA: // audio
 			if c.audio == nil {
@@ -82,7 +82,7 @@ func (c *Producer) Start() error {
 
 			//log.Printf("[DVR] len: %d, ts: %10d", len(packet.Payload), packet.Timestamp)
 
-			c.audio.WriteRTP(packet)
+			c.audio.Handler(packet)
 
 		case 0xF9: // unknown
 

pkg/flv/producer.go

@@ -67,7 +67,7 @@ func (c *Producer) Start() error {
 
 			pkt.Timestamp = TimeToRTP(pkt.Timestamp, c.audio.Codec.ClockRate)
 			pkt.Payload = pkt.Payload[2:]
-			c.audio.WriteRTP(pkt)
+			c.audio.Handler(pkt)
 
 		case TagVideo:
 			// frame type 4b, codecID 4b, avc packet type 8b, composition time 24b
@@ -77,7 +77,7 @@ func (c *Producer) Start() error {
 
 			pkt.Timestamp = TimeToRTP(pkt.Timestamp, c.video.Codec.ClockRate)
 			pkt.Payload = pkt.Payload[5:]
-			c.video.WriteRTP(pkt)
+			c.video.Handler(pkt)
 		}
 	}
 }

pkg/homekit/client.go

@@ -145,20 +145,20 @@ func (c *Client) Start() error {
 	if videoTrack != nil {
 		c.videoSession.OnReadRTP = func(packet *rtp.Packet) {
 			deadline.Reset(core.ConnDeadline)
-			videoTrack.WriteRTP(packet)
+			videoTrack.Handler(packet)
 			c.Recv += len(packet.Payload)
 		}
 
 		if audioTrack != nil {
 			c.audioSession.OnReadRTP = func(packet *rtp.Packet) {
-				audioTrack.WriteRTP(packet)
+				audioTrack.Handler(packet)
 				c.Recv += len(packet.Payload)
 			}
 		}
 	} else {
 		c.audioSession.OnReadRTP = func(packet *rtp.Packet) {
 			deadline.Reset(core.ConnDeadline)
-			audioTrack.WriteRTP(packet)
+			audioTrack.Handler(packet)
 			c.Recv += len(packet.Payload)
 		}
 	}
@@ -209,7 +209,7 @@ func (c *Client) startMJPEG() error {
 			Header:  rtp.Header{Timestamp: core.Now90000()},
 			Payload: b,
 		}
-		receiver.WriteRTP(packet)
+		receiver.Handler(packet)
 	}
 }
 

pkg/isapi/consumer.go

@@ -2,6 +2,7 @@ package isapi
 
 import (
 	"encoding/json"
+
 	"github.com/AlexxIT/go2rtc/pkg/core"
 	"github.com/pion/rtp"
 )

pkg/ivideon/client.go

@@ -272,7 +272,7 @@ func (c *Client) worker(buffer chan []byte) {
 				Header:  rtp.Header{Timestamp: ts * 90},
 				Payload: data[:size],
 			}
-			c.receiver.WriteRTP(packet)
+			c.receiver.Handler(packet)
 
 			data = data[size:]
 			ts += duration

pkg/kasa/producer.go

@@ -88,7 +88,7 @@ func (c *Producer) Start() error {
 					},
 					Payload: annexb.EncodeToAVCC(body, false),
 				}
-				video.WriteRTP(pkt)
+				video.Handler(pkt)
 			}
 
 		case MimeG711U:
@@ -102,7 +102,7 @@ func (c *Producer) Start() error {
 					},
 					Payload: body,
 				}
-				audio.WriteRTP(pkt)
+				audio.Handler(pkt)
 			}
 		}
 	}

pkg/magic/bitstream/producer.go

@@ -74,7 +74,7 @@ func (c *Producer) Start() error {
 				Header:  rtp.Header{Timestamp: core.Now90000()},
 				Payload: annexb.EncodeToAVCC(buf[:i], true),
 			}
-			c.Receivers[0].WriteRTP(pkt)
+			c.Receivers[0].Handler(pkt)
 
 			//log.Printf("[AVC] %v, len: %d", h264.Types(pkt.Payload), len(pkt.Payload))
 

pkg/magic/mjpeg/producer.go

@@ -63,7 +63,7 @@ func (c *Producer) Start() error {
 			Header:  rtp.Header{Timestamp: core.Now90000()},
 			Payload: buf[:i],
 		}
-		c.Receivers[0].WriteRTP(pkt)
+		c.Receivers[0].Handler(pkt)
 
 		//log.Printf("[mjpeg] ts=%d size=%d", pkt.Header.Timestamp, len(pkt.Payload))
 

pkg/mjpeg/client.go

@@ -39,7 +39,7 @@ func (c *Client) Handle() error {
 		Header:  rtp.Header{Timestamp: core.Now90000()},
 		Payload: body,
 	}
-	c.receiver.WriteRTP(pkt)
+	c.receiver.Handler(pkt)
 
 	c.recv += len(body)
 
@@ -67,7 +67,7 @@ func (c *Client) Handle() error {
 				Header:  rtp.Header{Timestamp: core.Now90000()},
 				Payload: body,
 			}
-			c.receiver.WriteRTP(pkt)
+			c.receiver.Handler(pkt)
 		}
 	}
 

pkg/mpegts/producer.go

@@ -45,7 +45,7 @@ func (c *Producer) Start() error {
 		for _, receiver := range c.Receivers {
 			if receiver.ID == pkt.PayloadType {
 				TimestampToRTP(pkt, receiver.Codec)
-				receiver.WriteRTP(pkt)
+				receiver.Handler(pkt)
 				break
 			}
 		}

pkg/multipart/producer.go

@@ -57,7 +57,7 @@ func (c *Producer) Start() error {
 				Header:  rtp.Header{Timestamp: core.Now90000()},
 				Payload: body,
 			}
-			mjpeg.WriteRTP(packet)
+			mjpeg.Handler(packet)
 		}
 	}
 }

pkg/rtsp/conn.go

@@ -244,7 +244,7 @@ func (c *Conn) Handle() (err error) {
 
 			for _, receiver := range c.receivers {
 				if receiver.ID == channelID {
-					receiver.WriteRTP(packet)
+					receiver.Handler(packet)
 					break
 				}
 			}

pkg/tapo/client.go

@@ -192,7 +192,7 @@ func (c *Client) Handle() error {
 			for _, receiver := range c.receivers {
 				if receiver.ID == pkt.PayloadType {
 					mpegts.TimestampToRTP(pkt, receiver.Codec)
-					receiver.WriteRTP(pkt)
+					receiver.Handler(pkt)
 					break
 				}
 			}

pkg/webrtc/conn.go

@@ -103,7 +103,7 @@ func NewConn(pc *webrtc.PeerConnection) *Conn {
 				continue
 			}
 
-			track.WriteRTP(packet)
+			track.Handler(packet)
 		}
 	})
 

pkg/webrtc/helpers.go

@@ -58,7 +58,7 @@ func UnmarshalMedias(descriptions []*sdp.MediaDescription) (medias []*core.Media
 // so it can add resampling for PCMA/PCMU and repack for PCM/PCML
 func WithResampling(medias []*core.Media) []*core.Media {
 	for _, media := range medias {
-		if media.Kind != core.KindAudio || media.Direction != core.DirectionSendonly {
+		if media.Kind != core.KindAudio {
 			continue
 		}
 
@@ -252,7 +252,7 @@ func MimeType(codec *core.Codec) string {
 	case core.CodecG722:
 		return webrtc.MimeTypeG722
 	}
-	panic("not implemented")
+	panic("codec not implemented: " + codec.Name)
 }
 
 // 4.1.2.2.  Guidelines for Choosing Type and Local Preferences

pkg/webrtc/producer.go

@@ -2,18 +2,29 @@ package webrtc
 
 import (
 	"github.com/AlexxIT/go2rtc/pkg/core"
+	"github.com/AlexxIT/go2rtc/pkg/pcm"
 	"github.com/pion/webrtc/v3"
 )
 
 func (c *Conn) GetTrack(media *core.Media, codec *core.Codec) (*core.Receiver, error) {
 	core.Assert(media.Direction == core.DirectionRecvonly)
 
 	for _, track := range c.receivers {
-		if track.Codec == codec {
+		if track.Codec.Match(codec) {
 			return track, nil
 		}
 	}
 
+	track := core.NewReceiver(media, codec)
+
+	if codec.ClockRate == 0 {
+		if codec.Name == core.CodecPCM || codec.Name == core.CodecPCML {
+			codec.Name = core.CodecPCMA
+		}
+		codec.ClockRate = 8000
+		track.Handler = pcm.ResampleToPCMA(track.Codec, 16000, track.Handler) //TODO
+	}
+
 	switch c.Mode {
 	case core.ModePassiveConsumer: // backchannel from browser
 		// set codec for consumer recv track so remote peer should send media with this codec
@@ -38,7 +49,6 @@ func (c *Conn) GetTrack(media *core.Media, codec *core.Codec) (*core.Receiver, e
 		panic(core.Caller())
 	}
 
-	track := core.NewReceiver(media, codec)
 	c.receivers = append(c.receivers, track)
 	return track, nil
 }