Why my rtp video track published to livekit by server-sdk-go is not display on flutter

[xiangxud]

I use server-sdk-go create room and publish rtp track ,some h264 video can display ,but some h264 rtp packets not display on flutter sdk, use my js code run on web is ok,how can i debug? some config i mistake ?

follow is livekit debug info,

"participant": "kvs:50:9a:4c:3c:2d:b5"

• no video play(use my device rv1126 video capture h264,
• but use kvs example h264 file transfer is ok) ,
• audio all is ok,
• on my web js code client all is ok,use same rtp forward

2022-07-06T07:34:52.905+0800	INFO	livekit	service/rtcservice.go:211	new client WS connected	{"room": "50:9a:4c:3c:2d:b5", "participant": "user", "connID": "user", "roomID": "RM_e6gBu37aP4iM"}
2022-07-06T07:34:52.908+0800	INFO	livekit	rtc/room.go:276	new participant joined	{"room": "50:9a:4c:3c:2d:b5", "roomID": "RM_e6gBu37aP4iM", "pID": "PA_5JQYKWzKnU8F", "participant": "user", "protocol": 7, "options": {"AutoSubscribe":true}}
2022-07-06T07:35:33.057+0800	INFO	livekit	service/rtcservice.go:211	new client WS connected	{"room": "50:9a:4c:3c:2d:b5", "participant": "kvs:50:9a:4c:3c:2d:b5", "connID": "kvs:50:9a:4c:3c:2d:b5", "roomID": "RM_e6gBu37aP4iM"}
2022-07-06T07:35:33.058+0800	INFO	livekit	rtc/room.go:276	new participant joined	{"room": "50:9a:4c:3c:2d:b5", "roomID": "RM_e6gBu37aP4iM", "pID": "PA_vE4HgC9pzeXV", "participant": "kvs:50:9a:4c:3c:2d:b5", "protocol": 7, "options": {"AutoSubscribe":true}}
2022-07-06T07:35:34.294+0800	INFO	livekit	rtc/participant.go:1151	mediaTrack published	{"room": "50:9a:4c:3c:2d:b5", "roomID": "RM_e6gBu37aP4iM", "participant": "kvs:50:9a:4c:3c:2d:b5", "pID": "PA_vE4HgC9pzeXV", "kind": "audio", "trackID": "TR_AMJuQZmPPHCxh7", "rid": "", "SSRC": 745830919}
2022-07-06T07:35:34.294+0800	INFO	livekit	buffer/buffer.go:187	REMB not supported, RTCP feedback will not be generated	{"room": "50:9a:4c:3c:2d:b5", "roomID": "RM_e6gBu37aP4iM", "participant": "kvs:50:9a:4c:3c:2d:b5", "pID": "PA_vE4HgC9pzeXV", "trackID": "TR_VCGKbmpLww8AWY", "mime": "video/h264", "layer": 2}
2022-07-06T07:35:34.295+0800	INFO	livekit	rtc/participant.go:1151	mediaTrack published	{"room": "50:9a:4c:3c:2d:b5", "roomID": "RM_e6gBu37aP4iM", "participant": "kvs:50:9a:4c:3c:2d:b5", "pID": "PA_vE4HgC9pzeXV", "kind": "video", "trackID": "TR_VCGKbmpLww8AWY", "rid": "", "SSRC": 560412114}

"participant": "obstest:50:9a:4c:3c:2d:b5" is obs rtmp covert to webrtc all is ok

2022-07-06T07:39:05.453+0800	INFO	livekit	service/rtcservice.go:211	new client WS connected	{"room": "50:9a:4c:3c:2d:b5", "participant": "obstest:50:9a:4c:3c:2d:b5", "connID": "obstest:50:9a:4c:3c:2d:b5", "roomID": "RM_e6gBu37aP4iM"}
2022-07-06T07:39:05.456+0800	INFO	livekit	rtc/room.go:276	new participant joined	{"room": "50:9a:4c:3c:2d:b5", "roomID": "RM_e6gBu37aP4iM", "pID": "PA_LiTqAuQrzHnr", "participant": "obstest:50:9a:4c:3c:2d:b5", "protocol": 7, "options": {"AutoSubscribe":true}}
2022-07-06T07:39:06.721+0800	INFO	livekit	buffer/buffer.go:187	REMB not supported, RTCP feedback will not be generated	{"room": "50:9a:4c:3c:2d:b5", "roomID": "RM_e6gBu37aP4iM", "participant": "obstest:50:9a:4c:3c:2d:b5", "pID": "PA_LiTqAuQrzHnr", "trackID": "TR_VCTnxmAftWjUpM", "mime": "video/h264", "layer": 2}
2022-07-06T07:39:06.722+0800	INFO	livekit	rtc/participant.go:1151	mediaTrack published	{"room": "50:9a:4c:3c:2d:b5", "roomID": "RM_e6gBu37aP4iM", "participant": "obstest:50:9a:4c:3c:2d:b5", "pID": "PA_LiTqAuQrzHnr", "kind": "audio", "trackID": "TR_AMgy7FXF2ctPap", "rid": "", "SSRC": 2974532577}
2022-07-06T07:39:06.722+0800	INFO	livekit	rtc/participant.go:1151	mediaTrack published	{"room": "50:9a:4c:3c:2d:b5", "roomID": "RM_e6gBu37aP4iM", "participant": "obstest:50:9a:4c:3c:2d:b5", "pID": "PA_LiTqAuQrzHnr", "kind": "video", "trackID": "TR_VCTnxmAftWjUpM", "rid": "", "SSRC": 882893420}

this is my published code:

	if t.VideoRTPTrack == nil {
		if strings.Contains(trackRemote.Codec().MimeType, "video") {
			videoRTPTrack, err := webrtc.NewTrackLocalStaticRTP(trackRemote.Codec().RTPCodecCapability, streamname+"-video", streamname)
			if err != nil {
				panic(err)
			}

			// r.RoomClient.MutePublishedTrack(r.Ctx,)
			// var local_video *lksdk.LocalTrackPublication
			if t.Videopub, err = t.RoomConnect.LocalParticipant.PublishTrack(videoRTPTrack, &lksdk.TrackPublicationOptions{Name: streamname + "-video"}); err != nil {
				log.Debug("Error publishing video RTP track->", err)
				return err
			}
			t.VideoRTPTrack = videoRTPTrack
			r.Localtracks[streamname] = t
			// r.Localtracks[streamname] = &LocalTrackPublication{p: local_video, Track: videoTrack, Trackname: streamname + "-video"}
			log.Debug("published video track -> ", streamname)
		}
	}
	if t.AudioRTPTrack == nil {
		if strings.Contains(trackRemote.Codec().MimeType, "audio") {
			audioRTPTrack, err := webrtc.NewTrackLocalStaticRTP(trackRemote.Codec().RTPCodecCapability, streamname+"-audio", streamname)
			//audioTrack, err := lksdk.NewLocalSampleTrack(webrtc.RTPCodecCapability{MimeType: webrtc.MimeTypeOpus})
			if err != nil {
				panic(err)
			}
			// var local_audio *lksdk.LocalTrackPublication

			if t.Audiopub, err = t.RoomConnect.LocalParticipant.PublishTrack(audioRTPTrack, &lksdk.TrackPublicationOptions{Name: streamname + "-audio"}); err != nil {
				log.Debug("Error publishing audio track", err)
				return err
			}
			t.AudioRTPTrack = audioRTPTrack
			r.Localtracks[streamname] = t
			log.Debug("published audio track -> ", streamname)
		}
	}

this is write rtp data:

func (r *Room) TrackSendRtpPackets(trackname, kind string, data []byte) (n int, err error) {
	if trackname == "" {
		log.Debug("Track name is null")
		return 0, fmt.Errorf("input trackname is null")
	}
	// var t *webrtc.TrackLocalStaticSample
	var t *webrtc.TrackLocalStaticRTP
	track := r.Localtracks[trackname]
	if track == nil {
		log.Debug("Track is nil ->", trackname, "<- no to publish")
		return 0, fmt.Errorf(" track is null,no to publish")
	}
	if kind == "video" {
		t = track.VideoRTPTrack
	} else if kind == "audio" {
		t = track.AudioRTPTrack
	}
	if t == nil {
		log.Debug("Track is nil ->", trackname, "<- no to publish")
		return 0, fmt.Errorf(" track is null,no to publish")
	}

	n, err = t.Write(data)
	return n, err

}

[davidzhao]

which platform are you testing with for Flutter? It's probably due to the profile used for H.264. WebRTC spec only supports constrained baseline; While some browsers may render other profiles fine, mobile devices can be very limiting when it comes to decoding support.

Ensure your H.264 video is encoded like this

[xiangxud]

thx,flutter for web edge ,i use h264 profile is 77 (main) not 66 baseline, let me try baseline

[xiangxud]

always only audio play ,not video play,i set config my device h264 encode profile to baseline

time="2022-07-10T10:33:15+08:00" level=info msg="ICE state checking"
time="2022-07-10T10:33:15+08:00" level=info msg="ICE state connected"
------------------
10:33:15	DEBUG	current streamname:kvs
------------------
2022/07/10 10:33:15  "level"=0 "msg"="received offer for subscriber"
2022/07/10 10:33:15  "level"=0 "msg"="ICE connected"
10:33:15	DEBUG	track->kvs<-is nil ,Connect room&{0xc0003b8400 <nil> <nil> <nil> <nil> <nil> {<nil> <nil> <nil> <nil>} {<nil> <nil> <nil> <nil>} <nil> kvs} &{{true true true true ws://192.168.0.18:7880 192.168.0.18:5551 APINrg5cyLqPK3p yhmmq0BnW2kTTgGWvwdzwD7MhyEHO5RrDUpprGeBhxe 50:9a:4c:3c:2d:b5 50:9a:4c:3c:2d:b5   } context.Background.WithCancel 0xc0003e0510 sid:"RM_EQcVJuejETwV" name:"50:9a:4c:3c:2d:b5" empty_timeout:300 creation_time:1657364287 turn_password:"tHz76liFATUZQRGWyf3T8y5yAmuv0UaFDwhxYYBl3ZY" enabled_codecs:{mime:"audio/opus"} enabled_codecs:{mime:"video/VP8"} enabled_codecs:{mime:"video/H264"} num_participants:2 {9 0} {0 0} <nil> <nil> map[]}
2022/07/10 10:33:15  "level"=0 "msg"="published track" "name"="kvs-video" "source"="CAMERA"
10:33:15	DEBUG	[RTPTrackPublished]published video track -> kvs
time="2022-07-10T10:33:15+08:00" level=info msg="Got track video/H264, pt=125 tbn=90000, fmtp=level-asymmetry-allowed=1;packetization-mode=1;profile-level-id=42e01f"
2022/07/10 10:33:15  "level"=0 "msg"="published track" "name"="kvs-audio" "source"="MICROPHONE"
time="2022-07-10T10:33:15+08:00" level=info msg="Got track audio/opus, pt=111 tbn=48000, channels=2"
10:33:15	DEBUG	[RTPTrackPublished]published audio track -> kvs
2022/07/10 10:33:15  "level"=0 "msg"="successfully set publisher answer"

[davidzhao]

what do the client logs say? can you check chrome://webrtc-internals to see if the track exists?

[xiangxud]

yes the track exists ,but not frame received ,the audio track is ok

[davidzhao]

Unfortunately, we aren't able to reproduce this. Are you able to give us a small example that could reproduce this? Ideally with our flutter example app.