From efa2b47655a3a45cfc554367141a013dc99ae58c Mon Sep 17 00:00:00 2001 From: Hiroshi Horie <548776+hiroshihorie@users.noreply.github.com> Date: Thu, 21 Oct 2021 05:30:57 +0900 Subject: [PATCH 1/7] fixed --- lib/src/rtc_engine.dart | 17 +++++++++ lib/src/track/remote_track_publication.dart | 40 ++++++++++++++------- lib/src/track/track.dart | 2 ++ lib/src/track/track_publication.dart | 2 +- 4 files changed, 47 insertions(+), 14 deletions(-) diff --git a/lib/src/rtc_engine.dart b/lib/src/rtc_engine.dart index d0ef4bc46..01f2b8f5b 100644 --- a/lib/src/rtc_engine.dart +++ b/lib/src/rtc_engine.dart @@ -337,6 +337,9 @@ class RTCEngine extends Disposable with EventsEmittable { }); subscriber?.pc.onTrack = (rtc.RTCTrackEvent event) { + // + logger.fine('[WebRTC] pc.onTrack'); + final stream = event.streams.firstOrNull; if (stream == null) { // we need the stream to get the track's id @@ -344,6 +347,14 @@ class RTCEngine extends Disposable with EventsEmittable { return; } + event.track.onEnded = () { + logger.fine('[WebRTC] track.onEnded'); + }; + + stream.onRemoveTrack = (_) { + logger.fine('[WebRTC] stream.onRemoveTrack'); + }; + events.emit(EngineTrackAddedEvent( track: event.track, stream: stream, @@ -351,6 +362,12 @@ class RTCEngine extends Disposable with EventsEmittable { )); }; + // dosn't work on mac + subscriber?.pc.onRemoveTrack = + (rtc.MediaStream stream, rtc.MediaStreamTrack track) { + logger.fine('[WebRTC] ${track.id} pc.onRemoveTrack'); + }; + // also handle messages over the pub channel, for backwards compatibility try { final lossyInit = rtc.RTCDataChannelInit() diff --git a/lib/src/track/remote_track_publication.dart b/lib/src/track/remote_track_publication.dart index fef7ba69f..b74f6c4ed 100644 --- a/lib/src/track/remote_track_publication.dart +++ b/lib/src/track/remote_track_publication.dart @@ -1,3 +1,5 @@ +import 'package:livekit_client/src/logger.dart'; + import '../events.dart'; import '../extensions.dart'; import '../participant/remote_participant.dart'; @@ -10,10 +12,8 @@ import 'track_publication.dart'; /// control if we should subscribe to the track, and its quality (for video). class RemoteTrackPublication extends TrackPublication { final RemoteParticipant _participant; - bool _unsubscribed = false; bool _disabled = false; lk_rtc.VideoQuality _videoQuality = lk_rtc.VideoQuality.HIGH; - lk_rtc.VideoQuality get videoQuality => _videoQuality; set videoQuality(lk_rtc.VideoQuality val) { @@ -29,18 +29,23 @@ class RemoteTrackPublication extends TrackPublication { _sendUpdateTrackSettings(); } - @override - bool get subscribed { - if (_unsubscribed) { - return false; - } - return super.subscribed; - } - set subscribed(bool val) { - if (_unsubscribed == !val) return; - _unsubscribed = !val; - _sendUpdateTrackSettings(); + logger.fine('setting subscribed = ${val}'); + if (val == super.subscribed) return; + _sendUpdateSubscription(subscribed: val); + if (!val && track != null) { + // Ideally, we should wait for WebRTC's onRemoveTrack event + // but it does not work reliably across platforms. + // So for now we will assume remove track succeeded. + track!.mediaStreamTrackEnded = true; + + [_participant.events, _participant.roomEvents] + .emit(TrackUnsubscribedEvent( + participant: _participant, + track: track!, + publication: this, + )); + } } /// for internal use @@ -77,6 +82,15 @@ class RemoteTrackPublication extends TrackPublication { this.track = track; } + void _sendUpdateSubscription({required bool subscribed}) { + logger.fine('Sending update subscription... ${sid} ${subscribed}'); + final subscription = lk_rtc.UpdateSubscription( + trackSids: [sid], + subscribe: subscribed, + ); + _participant.client.sendUpdateSubscription(subscription); + } + void _sendUpdateTrackSettings() { final settings = lk_rtc.UpdateTrackSettings( trackSids: [sid], diff --git a/lib/src/track/track.dart b/lib/src/track/track.dart index 0c0e3c6c9..1f4356e81 100644 --- a/lib/src/track/track.dart +++ b/lib/src/track/track.dart @@ -18,6 +18,8 @@ abstract class Track extends DisposableChangeNotifier { final String name; final lk_models.TrackType kind; rtc.MediaStreamTrack mediaStreamTrack; + @internal + bool mediaStreamTrackEnded = false; String? sid; rtc.RTCRtpTransceiver? transceiver; diff --git a/lib/src/track/track_publication.dart b/lib/src/track/track_publication.dart index a97326bfd..be2297f8a 100644 --- a/lib/src/track/track_publication.dart +++ b/lib/src/track/track_publication.dart @@ -19,7 +19,7 @@ abstract class TrackPublication extends Disposable { bool simulcasted = false; TrackDimension? dimension; - bool get subscribed => track != null; + bool get subscribed => track != null && !track!.mediaStreamTrackEnded; TrackPublication.fromInfo(lk_models.TrackInfo info) : sid = info.sid, From 12b9a4a80ce533ca9b24ac9525dcbd7ab951c58c Mon Sep 17 00:00:00 2001 From: Hiroshi Horie <548776+hiroshihorie@users.noreply.github.com> Date: Thu, 21 Oct 2021 05:37:05 +0900 Subject: [PATCH 2/7] format --- lib/src/rtc_engine.dart | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) diff --git a/lib/src/rtc_engine.dart b/lib/src/rtc_engine.dart index 01f2b8f5b..060e45db2 100644 --- a/lib/src/rtc_engine.dart +++ b/lib/src/rtc_engine.dart @@ -337,7 +337,6 @@ class RTCEngine extends Disposable with EventsEmittable { }); subscriber?.pc.onTrack = (rtc.RTCTrackEvent event) { - // logger.fine('[WebRTC] pc.onTrack'); final stream = event.streams.firstOrNull; @@ -347,10 +346,12 @@ class RTCEngine extends Disposable with EventsEmittable { return; } + // doesn't get called reliably event.track.onEnded = () { logger.fine('[WebRTC] track.onEnded'); }; + // doesn't get called reliably stream.onRemoveTrack = (_) { logger.fine('[WebRTC] stream.onRemoveTrack'); }; @@ -362,7 +363,7 @@ class RTCEngine extends Disposable with EventsEmittable { )); }; - // dosn't work on mac + // doesn't get called reliably, doesn't work on mac subscriber?.pc.onRemoveTrack = (rtc.MediaStream stream, rtc.MediaStreamTrack track) { logger.fine('[WebRTC] ${track.id} pc.onRemoveTrack'); From 19613cc2790828b2c685a8a32faeb5333b504e09 Mon Sep 17 00:00:00 2001 From: Hiroshi Horie <548776+hiroshihorie@users.noreply.github.com> Date: Thu, 21 Oct 2021 06:00:31 +0900 Subject: [PATCH 3/7] set track to null --- lib/src/track/remote_track_publication.dart | 10 +++++----- lib/src/track/track.dart | 2 -- lib/src/track/track_publication.dart | 2 +- 3 files changed, 6 insertions(+), 8 deletions(-) diff --git a/lib/src/track/remote_track_publication.dart b/lib/src/track/remote_track_publication.dart index b74f6c4ed..343553c14 100644 --- a/lib/src/track/remote_track_publication.dart +++ b/lib/src/track/remote_track_publication.dart @@ -34,17 +34,17 @@ class RemoteTrackPublication extends TrackPublication { if (val == super.subscribed) return; _sendUpdateSubscription(subscribed: val); if (!val && track != null) { - // Ideally, we should wait for WebRTC's onRemoveTrack event - // but it does not work reliably across platforms. - // So for now we will assume remove track succeeded. - track!.mediaStreamTrackEnded = true; - [_participant.events, _participant.roomEvents] .emit(TrackUnsubscribedEvent( participant: _participant, track: track!, publication: this, )); + + // Ideally, we should wait for WebRTC's onRemoveTrack event + // but it does not work reliably across platforms. + // So for now we will assume remove track succeeded. + track = null; } } diff --git a/lib/src/track/track.dart b/lib/src/track/track.dart index 1f4356e81..0c0e3c6c9 100644 --- a/lib/src/track/track.dart +++ b/lib/src/track/track.dart @@ -18,8 +18,6 @@ abstract class Track extends DisposableChangeNotifier { final String name; final lk_models.TrackType kind; rtc.MediaStreamTrack mediaStreamTrack; - @internal - bool mediaStreamTrackEnded = false; String? sid; rtc.RTCRtpTransceiver? transceiver; diff --git a/lib/src/track/track_publication.dart b/lib/src/track/track_publication.dart index be2297f8a..a97326bfd 100644 --- a/lib/src/track/track_publication.dart +++ b/lib/src/track/track_publication.dart @@ -19,7 +19,7 @@ abstract class TrackPublication extends Disposable { bool simulcasted = false; TrackDimension? dimension; - bool get subscribed => track != null && !track!.mediaStreamTrackEnded; + bool get subscribed => track != null; TrackPublication.fromInfo(lk_models.TrackInfo info) : sid = info.sid, From 7212af74b4815d7ed1f4138f37bf739b547b8430 Mon Sep 17 00:00:00 2001 From: Hiroshi Horie <548776+hiroshihorie@users.noreply.github.com> Date: Thu, 21 Oct 2021 06:02:17 +0900 Subject: [PATCH 4/7] rearrange --- lib/src/track/remote_track_publication.dart | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/lib/src/track/remote_track_publication.dart b/lib/src/track/remote_track_publication.dart index 343553c14..38729f7cb 100644 --- a/lib/src/track/remote_track_publication.dart +++ b/lib/src/track/remote_track_publication.dart @@ -34,16 +34,16 @@ class RemoteTrackPublication extends TrackPublication { if (val == super.subscribed) return; _sendUpdateSubscription(subscribed: val); if (!val && track != null) { + // Ideally, we should wait for WebRTC's onRemoveTrack event + // but it does not work reliably across platforms. + // So for now we will assume remove track succeeded. [_participant.events, _participant.roomEvents] .emit(TrackUnsubscribedEvent( participant: _participant, track: track!, publication: this, )); - - // Ideally, we should wait for WebRTC's onRemoveTrack event - // but it does not work reliably across platforms. - // So for now we will assume remove track succeeded. + // Simply set to null for now track = null; } } From e6e7ed6044df3db656ec6b2534830b8bba4b2735 Mon Sep 17 00:00:00 2001 From: Hiroshi Horie <548776+hiroshihorie@users.noreply.github.com> Date: Thu, 21 Oct 2021 06:27:14 +0900 Subject: [PATCH 5/7] toggle subscribed --- example/lib/widgets/participant.dart | 27 ++++++++++++++++++++++++--- 1 file changed, 24 insertions(+), 3 deletions(-) diff --git a/example/lib/widgets/participant.dart b/example/lib/widgets/participant.dart index cf93d5535..fefef8fb4 100644 --- a/example/lib/widgets/participant.dart +++ b/example/lib/widgets/participant.dart @@ -1,4 +1,5 @@ import 'package:collection/collection.dart'; +import 'package:eva_icons_flutter/eva_icons_flutter.dart'; import 'package:flutter/material.dart'; import 'package:flutter_webrtc/flutter_webrtc.dart'; import 'package:livekit_client/livekit_client.dart'; @@ -81,9 +82,29 @@ class _ParticipantWidgetState extends State { Align( alignment: Alignment.bottomCenter, - child: ParticipantInfoWidget( - title: widget.participant.identity, - muted: audioPub == null, + child: Column( + crossAxisAlignment: CrossAxisAlignment.end, + mainAxisSize: MainAxisSize.min, + children: [ + Material( + type: MaterialType.circle, + color: Colors.transparent, + child: IconButton( + onPressed: () { + final pub = widget.participant.audioTracks.firstOrNull; + if (pub is RemoteTrackPublication) { + print('TrackSubscribedEvent updating...'); + pub.subscribed = !pub.subscribed; + } + }, + icon: Icon(EvaIcons.volumeUpOutline), + ), + ), + ParticipantInfoWidget( + title: widget.participant.identity, + muted: audioPub == null, + ), + ], ), ), ], From 6ba75408fbe018758b112a5395e5a854e0eeb30e Mon Sep 17 00:00:00 2001 From: Hiroshi Horie <548776+hiroshihorie@users.noreply.github.com> Date: Thu, 21 Oct 2021 19:23:28 +0900 Subject: [PATCH 6/7] clean --- example/lib/pages/room.dart | 8 +------- 1 file changed, 1 insertion(+), 7 deletions(-) diff --git a/example/lib/pages/room.dart b/example/lib/pages/room.dart index 77f8d9d4c..6560d322a 100644 --- a/example/lib/pages/room.dart +++ b/example/lib/pages/room.dart @@ -71,13 +71,7 @@ class _RoomPageState extends State { // Create video track final localVideo = await LocalVideoTrack.createCameraTrack(); // Try to publish the video - await widget.room.localParticipant.publishVideoTrack( - localVideo, - // options: TrackPublishOptions( - // // simulcast: true, - // videoEncoding: VideoParameters.presetQVGA169.encoding, - // ), - ); + await widget.room.localParticipant.publishVideoTrack(localVideo); // Create mic track final localAudio = await LocalAudioTrack.create(); From 59270cb0da3f78028a8554e276cc42cbd61369c2 Mon Sep 17 00:00:00 2001 From: Hiroshi Horie <548776+hiroshihorie@users.noreply.github.com> Date: Thu, 21 Oct 2021 21:07:08 +0900 Subject: [PATCH 7/7] `RemoteTrackPublicationMenuWidget` for mute/unmute subscribe/unsubscribe --- example/lib/widgets/participant.dart | 119 ++++++++++++++++------ example/lib/widgets/participant_info.dart | 8 +- 2 files changed, 93 insertions(+), 34 deletions(-) diff --git a/example/lib/widgets/participant.dart b/example/lib/widgets/participant.dart index fefef8fb4..c0f216be1 100644 --- a/example/lib/widgets/participant.dart +++ b/example/lib/widgets/participant.dart @@ -24,8 +24,8 @@ class ParticipantWidget extends StatefulWidget { class _ParticipantWidgetState extends State { // - TrackPublication? videoPub; - TrackPublication? audioPub; + TrackPublication? firstVideoPub; + TrackPublication? firstAudioPub; @override void initState() { @@ -51,18 +51,14 @@ class _ParticipantWidgetState extends State { // register for change so Flutter will re-build the widget upon change void _onParticipantChanged() { // - final firstAudio = widget.participant.audioTracks - .firstWhereOrNull((pub) => pub.subscribed); - final firstVideo = widget.participant.videoTracks - .firstWhereOrNull((pub) => !pub.isScreenShare && pub.subscribed); - - if (firstVideo is RemoteTrackPublication) { - firstVideo.videoQuality = widget.quality; - } - setState(() { - audioPub = !(firstAudio?.muted ?? true) ? firstAudio : null; - videoPub = !(firstVideo?.muted ?? true) ? firstVideo : null; + // For simplification, We are assuming here + // there is only 1 video / audio tracks. + firstAudioPub = widget.participant.audioTracks.firstOrNull; + firstVideoPub = widget.participant.videoTracks.firstOrNull; + if (firstVideoPub is RemoteTrackPublication) { + (firstVideoPub as RemoteTrackPublication).videoQuality = widget.quality; + } }); } @@ -72,9 +68,10 @@ class _ParticipantWidgetState extends State { child: Stack( children: [ // Video - if (videoPub != null) + if (firstVideoPub?.subscribed == true && + firstVideoPub?.muted == false) VideoTrackRenderer( - videoPub!.track as VideoTrack, + firstVideoPub!.track as VideoTrack, fit: RTCVideoViewObjectFit.RTCVideoViewObjectFitCover, ) else @@ -83,26 +80,36 @@ class _ParticipantWidgetState extends State { Align( alignment: Alignment.bottomCenter, child: Column( - crossAxisAlignment: CrossAxisAlignment.end, + crossAxisAlignment: CrossAxisAlignment.stretch, mainAxisSize: MainAxisSize.min, children: [ - Material( - type: MaterialType.circle, - color: Colors.transparent, - child: IconButton( - onPressed: () { - final pub = widget.participant.audioTracks.firstOrNull; - if (pub is RemoteTrackPublication) { - print('TrackSubscribedEvent updating...'); - pub.subscribed = !pub.subscribed; - } - }, - icon: Icon(EvaIcons.volumeUpOutline), - ), + // + // Menu for Video RemoteTrackPublication + // + Row( + mainAxisSize: MainAxisSize.max, + mainAxisAlignment: MainAxisAlignment.end, + children: [ + if (firstVideoPub is RemoteTrackPublication) + RemoteTrackPublicationMenuWidget( + pub: firstVideoPub as RemoteTrackPublication, + icon: EvaIcons.videoOutline, + ), + // + // Menu for Audio RemoteTrackPublication + // + if (firstAudioPub is RemoteTrackPublication) + RemoteTrackPublicationMenuWidget( + pub: firstAudioPub as RemoteTrackPublication, + icon: EvaIcons.volumeUpOutline, + ), + ], ), + ParticipantInfoWidget( title: widget.participant.identity, - muted: audioPub == null, + audioAvailable: firstAudioPub?.muted == false && + firstAudioPub?.subscribed == true, ), ], ), @@ -111,3 +118,55 @@ class _ParticipantWidgetState extends State { ), ); } + +class RemoteTrackPublicationMenuWidget extends StatelessWidget { + final IconData icon; + final RemoteTrackPublication pub; + const RemoteTrackPublicationMenuWidget({ + required this.pub, + required this.icon, + Key? key, + }) : super(key: key); + + @override + Widget build(BuildContext context) => Material( + // type: MaterialType.card, + color: Colors.black.withOpacity(0.3), + // shape: CircleBorder(), + child: PopupMenuButton( + // shape: CircleBorder(), + icon: Icon(icon), + onSelected: (value) => value(), + itemBuilder: (BuildContext context) { + return >[ + // + // Mute/Unmute + // + if (pub.muted == false) + PopupMenuItem( + child: const Text('Mute'), + value: () => pub.muted = true, + ), + if (pub.muted == true) + PopupMenuItem( + child: const Text('Un-mute'), + value: () => pub.muted = false, + ), + // + // Subscribe/Unsubscribe + // + if (pub.subscribed == false) + PopupMenuItem( + child: const Text('Subscribe'), + value: () => pub.subscribed = true, + ), + if (pub.subscribed == true) + PopupMenuItem( + child: const Text('Un-subscribe'), + value: () => pub.subscribed = false, + ), + ]; + }, + ), + ); +} diff --git a/example/lib/widgets/participant_info.dart b/example/lib/widgets/participant_info.dart index d1354d3ec..403f11e11 100644 --- a/example/lib/widgets/participant_info.dart +++ b/example/lib/widgets/participant_info.dart @@ -4,11 +4,11 @@ import 'package:flutter/material.dart'; class ParticipantInfoWidget extends StatelessWidget { // final String? title; - final bool muted; + final bool audioAvailable; const ParticipantInfoWidget({ this.title, - this.muted = true, + this.audioAvailable = true, Key? key, }) : super(key: key); @@ -33,8 +33,8 @@ class ParticipantInfoWidget extends StatelessWidget { Padding( padding: const EdgeInsets.only(left: 5), child: Icon( - !muted ? EvaIcons.mic : EvaIcons.micOff, - color: !muted ? Colors.white : Colors.red, + audioAvailable ? EvaIcons.mic : EvaIcons.micOff, + color: audioAvailable ? Colors.white : Colors.red, size: 16, ), ),