From 4c5e3146a68392fe32965b1b52065b698ea7beed Mon Sep 17 00:00:00 2001
From: Drew Ewing <drew.ewing@satellite.im>
Date: Mon, 2 May 2022 16:51:48 -0700
Subject: [PATCH] feat(groupchat): screen mute/unmute fix

---
 components/views/media/user/User.html |   8 +-
 components/views/media/user/User.vue  |   1 +
 libraries/WebRTC/Call.ts              | 218 +++++++++++++-------------
 libraries/WebRTC/Libp2p.ts            |  12 +-
 store/webrtc/actions.ts               | 100 +++---------
 store/webrtc/mutations.ts             |   1 +
 6 files changed, 140 insertions(+), 200 deletions(-)

diff --git a/components/views/media/user/User.html b/components/views/media/user/User.html
index 6374feac44..10f1cbe80e 100644
--- a/components/views/media/user/User.html
+++ b/components/views/media/user/User.html
@@ -3,9 +3,7 @@
     <mic-off-icon class="indicator" v-if="audioMuted" size="1.2x" />
   </div>
 
-  <div
-    v-if="user && (!webrtc.streamMuted[user.peerId] || !webrtc.streamMuted[user.peerId].screen) && screenStream && screenStream.getTracks()[0].enabled"
-  >
+  <div v-if="user && screenStream">
     <video
       class="video-stream screen-stream"
       :src-object.prop="screenStream"
@@ -14,9 +12,7 @@
     ></video>
   </div>
 
-  <div
-    v-else-if="user && (!webrtc.streamMuted[user.peerId] || !webrtc.streamMuted[user.peerId].video) && videoStream && videoStream.getTracks()[0].enabled"
-  >
+  <div v-else-if="user && !screenStream && videoStream">
     <video
       class="video-stream"
       :src-object.prop="videoStream"
diff --git a/components/views/media/user/User.vue b/components/views/media/user/User.vue
index 168223ee53..26ae91e357 100644
--- a/components/views/media/user/User.vue
+++ b/components/views/media/user/User.vue
@@ -57,6 +57,7 @@ export default Vue.extend({
       return (
         this.user?.peerId &&
         !this.webrtc.streamMuted[this.user.peerId]?.video &&
+        this.webrtc.streamMuted[this.user.peerId]?.screen &&
         this.webrtc.activeCall &&
         this.webrtc.activeCall.callId &&
         $WebRTC
diff --git a/libraries/WebRTC/Call.ts b/libraries/WebRTC/Call.ts
index 7bd1d3b0af..9dfe2042c8 100644
--- a/libraries/WebRTC/Call.ts
+++ b/libraries/WebRTC/Call.ts
@@ -33,6 +33,7 @@ export class Call extends Emitter<CallEventListeners> {
   peers: { [peerId: string]: CallPeer } = {} // The Simple Peer instance for the active call
   peerConnected: { [key: string]: boolean } = {}
   peerDialingDisabled: { [key: string]: boolean } = {}
+  screenStreams: { [peerId: string]: string } = {}
   streams: { [peerId: string]: { [kind: string]: MediaStream } } = {} // Remote stream received via WebRTC
   tracks: { [peerId: string]: Set<MediaStreamTrack> } = {} // Remote stream received via WebRTC
   peerSignals: { [peerId: string]: Peer.SignalData } = {} // A variable to store the signaling data before the answer
@@ -251,24 +252,6 @@ export class Call extends Emitter<CallEventListeners> {
     await Promise.all(
       Object.values(this.peers).map(async (peer) => {
         try {
-          console.info(
-            'asking peer to unmute track',
-            peer.id,
-            audioTrack.id,
-            'audio',
-          )
-          await this.p2p.sendMessage(
-            {
-              type: 'peer:unmute',
-              payload: {
-                callId: this.callId,
-                trackId: audioTrack.id,
-                kind: 'audio',
-              },
-              sentAt: Date.now().valueOf(),
-            },
-            peer.id,
-          )
           peer.addStream(audioStream)
           peer.addTrack(audioTrack, audioStream)
         } catch (_) {}
@@ -302,24 +285,6 @@ export class Call extends Emitter<CallEventListeners> {
 
     await Promise.all(
       Object.values(this.peers).map(async (peer) => {
-        console.info(
-          'asking peer to unmute track',
-          peer.id,
-          videoTrack.id,
-          'video',
-        )
-        await this.p2p.sendMessage(
-          {
-            type: 'peer:unmute',
-            payload: {
-              callId: this.callId,
-              trackId: videoTrack.id,
-              kind: 'audio',
-            },
-            sentAt: Date.now().valueOf(),
-          },
-          peer.id,
-        )
         try {
           peer.addStream(videoStream)
           peer.addTrack(videoTrack, videoStream)
@@ -351,19 +316,12 @@ export class Call extends Emitter<CallEventListeners> {
 
     await Promise.all(
       Object.values(this.peers).map(async (peer) => {
-        console.info(
-          'asking peer to unmute track',
-          peer.id,
-          screenTrack.id,
-          'screen',
-        )
         await this.p2p.sendMessage(
           {
-            type: 'peer:unmute',
+            type: 'peer:screenshare',
             payload: {
-              callId: this.callId,
+              streamId: screenStream.id,
               trackId: screenTrack.id,
-              kind: 'screen',
             },
             sentAt: Date.now().valueOf(),
           },
@@ -596,30 +554,32 @@ export class Call extends Emitter<CallEventListeners> {
     track.enabled = false
 
     // tell all of the peers that we muted the track
-    await Promise.all(
-      Object.values(this.peers).map(async (peer) => {
-        console.info('asking peer to mute track', peer.id, track.id, kind)
-        await this.p2p.sendMessage(
-          {
-            type: 'peer:mute',
-            payload: {
-              callId: this.callId,
-              trackId: track.id,
-              kind,
+    if (peerId === this.localId) {
+      await Promise.all(
+        Object.values(this.peers).map(async (peer) => {
+          console.info('asking peer to mute track', peer.id, track.id, kind)
+          await this.p2p.sendMessage(
+            {
+              type: 'peer:mute',
+              payload: {
+                callId: this.callId,
+                trackId: track.id,
+                kind,
+              },
+              sentAt: Date.now().valueOf(),
             },
-            sentAt: Date.now().valueOf(),
-          },
-          peer.id,
-        )
-      }),
-    )
+            peer.id,
+          )
+        }),
+      )
 
-    console.info('muted local track', track, kind, stream)
-    this.emit('LOCAL_TRACK_REMOVED', {
-      track,
-      kind,
-      stream,
-    })
+      console.info('muted local track', track, kind, stream)
+      this.emit('LOCAL_TRACK_REMOVED', {
+        track,
+        kind,
+        stream,
+      })
+    }
   }
 
   /**
@@ -634,17 +594,15 @@ export class Call extends Emitter<CallEventListeners> {
     kind: string
     peerId?: string
   }) {
-    let broadcasted = false
-    if (kind === 'audio' && !this.streams[peerId]?.audio) {
-      await this.createAudioStream(true)
-      broadcasted = true
-    } else if (kind === 'video' && !this.streams[peerId]?.video) {
-      await this.createVideoStream(true)
-      broadcasted = true
-    } else if (kind === 'screen' && !this.streams[peerId]?.screen) {
-      this.screenShareEnabled = true
-      await this.createDisplayStream()
-      broadcasted = true
+    if (peerId === this.localId) {
+      if (kind === 'audio' && !this.streams[peerId]?.audio) {
+        await this.createAudioStream(true)
+      } else if (kind === 'video' && !this.streams[peerId]?.video) {
+        await this.createVideoStream(true)
+      } else if (kind === 'screen' && !this.streams[peerId]?.screen) {
+        this.screenShareEnabled = true
+        await this.createDisplayStream()
+      }
     }
 
     const stream = this.streams[peerId]?.[kind]
@@ -662,32 +620,13 @@ export class Call extends Emitter<CallEventListeners> {
     }
     track.enabled = true
 
-    if (!broadcasted) {
-      // tell all of the peers that we muted the track
-      await Promise.all(
-        Object.values(this.peers).map(async (peer) => {
-          console.info('asking peer to unmute track', peer.id, track.id, kind)
-          await this.p2p.sendMessage(
-            {
-              type: 'peer:unmute',
-              payload: {
-                callId: this.callId,
-                trackId: track.id,
-                kind,
-              },
-              sentAt: Date.now().valueOf(),
-            },
-            peer.id,
-          )
-        }),
-      )
+    if (peerId === this.localId) {
+      this.emit('LOCAL_TRACK_CREATED', {
+        track,
+        kind,
+        stream,
+      })
     }
-
-    this.emit('LOCAL_TRACK_CREATED', {
-      track,
-      kind,
-      stream,
-    })
   }
 
   /**
@@ -713,12 +652,20 @@ export class Call extends Emitter<CallEventListeners> {
     this.p2p?.on('peer:signal', this._onBusSignal.bind(this))
     this.p2p?.on('peer:refuse', this._onBusRefuse.bind(this))
     this.p2p?.on('peer:hangup', this._onBusHangup.bind(this))
+    this.p2p?.on('peer:screenshare', this._onBusScreenshare.bind(this))
+    this.p2p?.on('peer:mute', this._onBusMute.bind(this))
+    this.p2p?.on('peer:unmute', this._onBusUnmute.bind(this))
+    this.p2p?.on('peer:destroy', this._onBusDestroy.bind(this))
   }
 
   protected _unbindBusListeners() {
     this.p2p?.off('peer:signal', this._onBusSignal.bind(this))
     this.p2p?.off('peer:refuse', this._onBusRefuse.bind(this))
     this.p2p?.off('peer:hangup', this._onBusHangup.bind(this))
+    this.p2p?.off('peer:screenshare', this._onBusScreenshare.bind(this))
+    this.p2p?.off('peer:mute', this._onBusMute.bind(this))
+    this.p2p?.off('peer:unmute', this._onBusUnmute.bind(this))
+    this.p2p?.off('peer:destroy', this._onBusDestroy.bind(this))
   }
 
   /**
@@ -823,7 +770,6 @@ export class Call extends Emitter<CallEventListeners> {
     this.peerDialingDisabled[peer.id] = true
     this.emit('REMOTE_TRACK_RECEIVED', {
       peerId: peer.id,
-      callId: peer.callId,
       track,
       stream,
     })
@@ -834,13 +780,16 @@ export class Call extends Emitter<CallEventListeners> {
    * @description Callback for the Simple Peer stream event
    * @param stream MediaStream object for the audio/video stream
    */
-  protected _onStream(peer: CallPeer, stream: MediaStream, kind?: string) {
+  protected _onStream(peer: CallPeer, stream: MediaStream) {
     if (!this.streams[peer.id]) {
       this.streams[peer.id] = {}
     }
-    const k: string = kind || stream.getTracks()[0].kind
-    this.streams[peer.id][k] = stream
-    this.emit('STREAM', { peerId: peer.id, stream, kind: k })
+    const kind =
+      this.screenStreams[peer.id] === stream.id
+        ? 'screen'
+        : stream.getTracks()?.[0]?.kind
+    this.streams[peer.id][kind] = stream
+    this.emit('STREAM', { peerId: peer.id, stream, kind })
   }
 
   /**
@@ -924,6 +873,59 @@ export class Call extends Emitter<CallEventListeners> {
    * after the call started
    */
   protected _onBusHangup({ peerId }: { peerId: PeerId }) {
+    this.peerDialingDisabled[peerId.toB58String()] = true
+    this.destroyPeer(peerId.toB58String())
+  }
+
+  protected _onBusScreenshare({
+    peerId,
+    payload: { streamId },
+  }: {
+    peerId: PeerId
+    payload: { streamId: string }
+  }) {
+    const peer = this.peers[peerId.toB58String()]
+    console.info('[Call] Screenshare', peerId, streamId)
+    if (peer) {
+      this.screenStreams[peerId.toB58String()] = streamId
+    }
+  }
+
+  protected _onBusMute({
+    peerId,
+    payload: { kind, trackId },
+  }: {
+    peerId: PeerId
+    payload: { kind: string; trackId: string }
+  }) {
+    // this.mute({ peerId: peerId.toB58String(), kind })
+    Object.values(this.streams[peerId.toB58String()] || {}).forEach(
+      (stream) => {
+        const track = stream.getTrackById(trackId)
+        if (!track) return
+        track.enabled = false
+      },
+    )
+  }
+
+  protected _onBusUnmute({
+    peerId,
+    payload: { kind, trackId },
+  }: {
+    peerId: PeerId
+    payload: { kind: string; trackId: string }
+  }) {
+    // this.unmute({ peerId: peerId.toB58String(), kind })
+    Object.values(this.streams[peerId.toB58String()] || {}).forEach(
+      (stream) => {
+        const track = stream.getTrackById(trackId)
+        if (!track) return
+        track.enabled = true
+      },
+    )
+  }
+
+  protected _onBusDestroy({ peerId }: { peerId: PeerId }) {
     this.destroyPeer(peerId.toB58String())
   }
 }
diff --git a/libraries/WebRTC/Libp2p.ts b/libraries/WebRTC/Libp2p.ts
index 4cfdaa559b..e92c5de642 100644
--- a/libraries/WebRTC/Libp2p.ts
+++ b/libraries/WebRTC/Libp2p.ts
@@ -34,10 +34,15 @@ interface P2PListeners {
   'peer:destroy': (data: { peerId: PeerId; payload: unknown }) => void
   'peer:hangup': (data: { peerId: PeerId; payload: unknown }) => void
   'peer:request': (data: { peerId: PeerId }) => void
+  'peer:screenshare': (data: {
+    peerId: PeerId
+    payload: { streamId: string }
+  }) => void
   'peer:mute': (data: {
     peerId: PeerId
     payload: {
       callId: string
+      streamId: string
       trackId: string
       kind: string
     }
@@ -46,6 +51,7 @@ interface P2PListeners {
     peerId: PeerId
     payload: {
       callId: string
+      streamId: string
       trackId: string
       kind: string
     }
@@ -63,10 +69,6 @@ interface P2PListeners {
       textilePubkey: string
     }
   }) => void
-  'peer:requestDetails': (data: {
-    peerId: PeerId
-    payload: { callId: string }
-  }) => void
 }
 
 type P2PMessage = {
@@ -238,7 +240,6 @@ export class Peer2Peer extends Emitter<P2PListeners> {
             [
               'peer:call',
               'peer:details',
-              'peer:requestDetails',
               'peer:answer',
               'peer:destroy',
               'peer:announce',
@@ -246,6 +247,7 @@ export class Peer2Peer extends Emitter<P2PListeners> {
               'peer:hangup',
               'peer:mute',
               'peer:unmute',
+              'peer:screenshare',
             ].includes(message.type)
           ) {
             this.emit(message.type, { peerId, ...message })
diff --git a/store/webrtc/actions.ts b/store/webrtc/actions.ts
index 7d24e5753c..17fb8aeeb7 100644
--- a/store/webrtc/actions.ts
+++ b/store/webrtc/actions.ts
@@ -76,42 +76,6 @@ const webRTCActions = {
       }
     })
 
-    $Peer2Peer.on('peer:requestDetails', async ({ peerId, payload }) => {
-      if (
-        payload.callId &&
-        payload.callId === rootState.conversation.id &&
-        (payload.callId === peerId.toB58String() ||
-          (rootState.conversation.type === 'group' &&
-            rootState.groups.all
-              .find((group) => group.id === rootState.conversation.id)
-              ?.members?.find(
-                (member) =>
-                  rootState.friends.all.find(
-                    (friend) => friend.address === member.address,
-                  )?.peerId === peerId.toB58String(),
-              ))) &&
-        rootState.accounts.details
-      ) {
-        // tell the peer what to call us, etc...
-        const { name, profilePicture, textilePubkey } =
-          rootState.accounts.details
-        const details = {
-          callId: payload.callId,
-          name,
-          profilePicture,
-          textilePubkey,
-        }
-        await $Peer2Peer.sendMessage(
-          {
-            type: 'peer:details',
-            payload: details,
-            sentAt: Date.now().valueOf(),
-          },
-          peerId,
-        )
-      }
-    })
-
     $Peer2Peer.on('peer:disconnect', ({ peerId }) => {
       const disconnectedFriend = rootState.friends.all.find(
         (friend) => friend.peerId === peerId.toB58String(),
@@ -183,56 +147,28 @@ const webRTCActions = {
     $Peer2Peer.on(
       'peer:mute',
       ({ peerId, payload: { callId, trackId, kind } }) => {
-        console.info('muting peer track', { peerId, callId, trackId, kind })
-        const call = $WebRTC.getCall(callId)
-        if (!call || !trackId) return
-        commit('setMuted', { peerId, kind, callId, trackId, muted: true })
-        Object.values(call.streams[peerId.toB58String()] || {}).forEach(
-          (stream) => {
-            const track = stream.getTrackById(trackId)
-            if (!track) return
-            track.enabled = false
-          },
-        )
+        const peerIdStr = peerId.toB58String()
+        commit('setMuted', {
+          peerId: peerIdStr,
+          kind,
+          callId,
+          trackId,
+          muted: true,
+        })
       },
     )
 
     $Peer2Peer.on(
       'peer:unmute',
       ({ peerId, payload: { callId, trackId, kind } }) => {
-        console.info('unmuting peer track', { peerId, callId, trackId, kind })
-        const call = $WebRTC.getCall(callId)
-        if (!call || !trackId) return
-        commit('setMuted', { peerId, kind, callId, trackId, muted: false })
-        Object.values(call.streams[peerId.toB58String()] || {}).forEach(
-          (stream) => {
-            const track = stream.getTrackById(trackId)
-            if (!track) return
-            track.enabled = true
-          },
-        )
-      },
-    )
-
-    $Peer2Peer.on('peer:destroy', ({ payload }: { payload: any }) => {
-      const { callId, peerId } = payload
-      const call = $WebRTC.calls.get(callId)
-      if (!call) return
-
-      call.peerDialingDisabled[peerId.toB58String()] = true
-      call.destroyPeer(peerId)
-    })
-
-    $Peer2Peer.on(
-      'peer:hangup',
-      ({ peerId, payload }: { peerId: PeerId; payload: any }) => {
-        const { callId } = payload
-        const call = $WebRTC.calls.get(callId)
-        if (!call) return
-
-        // ignore this peer for the remainder of the call or until they call again
-        call.peerDialingDisabled[peerId.toB58String()] = true
-        call.destroyPeer(peerId.toB58String())
+        const peerIdStr = peerId.toB58String()
+        commit('setMuted', {
+          peerId: peerIdStr,
+          kind,
+          callId,
+          trackId,
+          muted: false,
+        })
       },
     )
 
@@ -362,6 +298,7 @@ const webRTCActions = {
     call.on('LOCAL_TRACK_CREATED', ({ track, kind }) => {
       $Logger.log('webrtc', `local track created: ${track.kind}#${track.id}`)
       commit('setMuted', {
+        callId: call.callId,
         peerId: $Peer2Peer.id,
         kind,
         muted: false,
@@ -371,6 +308,7 @@ const webRTCActions = {
     call.on('LOCAL_TRACK_REMOVED', ({ track, kind }) => {
       $Logger.log('webrtc', `local track removed: ${kind}#${track.id}`)
       commit('setMuted', {
+        callId: call.callId,
         peerId: $Peer2Peer.id,
         kind,
         muted: true,
@@ -378,7 +316,7 @@ const webRTCActions = {
     })
 
     call.on('STREAM', ({ peerId, kind }) => {
-      commit('setMuted', { peerId, kind, muted: false })
+      // commit('setMuted', { peerId, kind, muted: false })
     })
 
     call.on('ANSWERED', ({ peerId }) => {
diff --git a/store/webrtc/mutations.ts b/store/webrtc/mutations.ts
index e297694333..28d1d41d9f 100644
--- a/store/webrtc/mutations.ts
+++ b/store/webrtc/mutations.ts
@@ -42,6 +42,7 @@ const mutations = {
     state: WebRTCState,
     { peerId, kind, muted }: { peerId: string; kind: string; muted: boolean },
   ) {
+    console.info('streamMuted', { peerId, kind, muted })
     state.streamMuted = {
       ...state.streamMuted,
       [peerId]: { ...state.streamMuted[peerId], [kind]: muted },