From 505234ee0302be3092ec5fc342b42d94cc828f3c Mon Sep 17 00:00:00 2001 From: Drew Ewing Date: Fri, 8 Apr 2022 14:03:13 -0700 Subject: [PATCH] feat(calls): changes to calling, screen sharing, group calls WIP --- .gitignore | 5 + .vscode/launch.json | 74 ++ .vscode/settings.json | 2 +- __mocks__/it-pipe.js | 1 + components/ui/Global/Global.html | 11 +- components/ui/Global/Global.vue | 56 +- components/views/chat/chatbar/Chatbar.vue | 50 +- .../footer/is-connected/Connected.html | 15 +- .../chatbar/footer/is-connected/Connected.vue | 51 +- components/views/media/Media.html | 77 +- components/views/media/Media.vue | 100 +- components/views/media/actions/Actions.html | 17 +- components/views/media/actions/Actions.vue | 64 +- components/views/media/heading/Heading.vue | 4 +- .../media/incomingCall/IncomingCall.html | 26 +- .../views/media/incomingCall/IncomingCall.vue | 31 +- components/views/media/user/User.html | 67 +- components/views/media/user/User.vue | 120 +- .../navigation/sidebar/controls/Controls.html | 6 +- .../navigation/sidebar/controls/Controls.vue | 64 +- .../views/navigation/sidebar/live/Live.vue | 20 +- .../views/navigation/toolbar/Toolbar.html | 23 +- .../views/navigation/toolbar/Toolbar.vue | 58 +- jest.config.js | 7 +- layouts/chat.vue | 29 +- libraries/Enums/types/webrtc.ts | 1 + libraries/WebRTC/Call.ts | 1028 +++++++++++------ libraries/WebRTC/Libp2p.ts | 80 +- libraries/WebRTC/Peer.ts | 9 + libraries/WebRTC/TracksManager.test.ts | 2 +- libraries/WebRTC/WebRTC.ts | 69 +- libraries/WebRTC/types.ts | 50 +- libraries/ui/Friends.test.ts | 31 - .../ui/__snapshots__/Friends.test.ts.snap | 26 - locales/en-US.js | 3 + middleware/authenticated.test.ts | 18 +- nuxt.config.js | 5 +- pages/chat/direct/_address.vue | 5 +- pages/chat/groups/_id.vue | 2 +- plugins/local/classLoader.ts | 10 +- plugins/local/logger.ts | 5 + plugins/thirdparty/persist.ts | 1 - store/accounts/actions.test.ts | 1 - store/accounts/actions.ts | 9 +- store/accounts/mutations.test.ts | 3 + store/accounts/mutations.ts | 8 + store/audio/actions.ts | 15 +- store/conversation/actions.ts | 65 ++ store/conversation/mutations.ts | 45 + store/conversation/state.ts | 10 + store/conversation/types.ts | 32 + .../__snapshots__/mutations.test.ts.snap | 6 - .../friends/__snapshots__/state.test.ts.snap | 1 + store/friends/actions.ts | 5 +- store/friends/mutations.test.ts | 10 - store/friends/mutations.ts | 8 +- store/friends/state.ts | 1 + store/getters.test.ts | 6 +- store/groups/types.ts | 2 + .../textile/__snapshots__/state.test.ts.snap | 1 + store/textile/actions.ts | 49 +- store/textile/state.ts | 1 + store/textile/types.ts | 1 + store/ui/actions.test.ts | 1 - store/ui/actions.ts | 2 +- store/ui/mutations.test.ts | 1 - store/webrtc/__snapshots__/state.test.ts.snap | 29 +- store/webrtc/actions.ts | 424 +++++-- store/webrtc/mutations.test.ts | 289 ++--- store/webrtc/mutations.ts | 99 +- store/webrtc/state.ts | 20 +- store/webrtc/types.ts | 46 +- tsconfig.json | 12 +- types/messaging.d.ts | 1 + types/store/store.ts | 2 + types/ui/core.d.ts | 1 + types/ui/friends.d.ts | 1 - types/ui/user.d.ts | 2 + utilities/Hounddog.test.ts | 3 - utilities/Hounddog.ts | 4 +- utilities/Messaging.test.ts | 2 - utilities/__snapshots__/Hounddog.test.ts.snap | 34 +- 82 files changed, 2141 insertions(+), 1434 deletions(-) create mode 100644 .vscode/launch.json create mode 100644 __mocks__/it-pipe.js create mode 100644 libraries/WebRTC/Peer.ts create mode 100644 plugins/local/logger.ts create mode 100644 store/conversation/actions.ts create mode 100644 store/conversation/mutations.ts create mode 100644 store/conversation/state.ts create mode 100644 store/conversation/types.ts diff --git a/.gitignore b/.gitignore index 7e9711962f..122d87c780 100644 --- a/.gitignore +++ b/.gitignore @@ -95,3 +95,8 @@ sw.* # Scratchpad files for testing stuff scratchpad.ts scratchpad.js + +# Browser profiles for debugging +.vscode/firefox/ +.vscode/pwa-msedge/ +.vscode/chrome/ diff --git a/.vscode/launch.json b/.vscode/launch.json new file mode 100644 index 0000000000..1bee647f13 --- /dev/null +++ b/.vscode/launch.json @@ -0,0 +1,74 @@ +{ + "version": "0.2.0", + "configurations": [ + { + "type": "pwa-msedge", + "name": "edge: profile-1", + "request": "launch", + "url": "http://localhost:3000/?p1", + "userDataDir": "${workspaceFolder}/.vscode/pwa-msedge/profile-1" + }, + { + "type": "pwa-msedge", + "name": "edge: profile-2", + "request": "launch", + "url": "http://localhost:3000/?p2", + "userDataDir": "${workspaceFolder}/.vscode/pwa-msedge/profile-2" + }, + { + "type": "pwa-msedge", + "name": "edge: profile-3", + "request": "launch", + "url": "http://localhost:3000/?p3", + "userDataDir": "${workspaceFolder}/.vscode/pwa-msedge/profile-3" + }, + { + "type": "vscode-edge-devtools.debug", + "name": "edge: devtools", + "request": "attach", + "url": "http://localhost:3000/" + }, + { + "name": "browser: firefox", + "type": "firefox", + "request": "launch", + "url": "http://localhost:3000", + "reAttach": true, + "profile": "debug", + "keepProfileChanges": true, + "profileDir": "${workspaceRoot}/.vscode/firefox", + "webRoot": "${workspaceFolder}", + "pathMappings": [ + { + "url": "webpack:///", + "path": "${webRoot}/" + } + ] + }, + { + "type": "node", + "request": "launch", + "name": "server: nuxi", + "args": ["dev"], + "osx": { + "program": "${workspaceFolder}/node_modules/.bin/nuxi" + }, + "linux": { + "program": "${workspaceFolder}/node_modules/.bin/nuxi" + }, + "windows": { + "program": "${workspaceFolder}/node_modules/nuxt/bin/nuxi.mjs" + } + } + ], + "compounds": [ + { + "name": "fullstack: nuxi", + "configurations": ["server: nuxi", "browser: firefox"] + }, + { + "name": "edge: profile-1 w/ devtools", + "configurations": ["edge: profile-1", "edge: devtools"] + } + ] +} diff --git a/.vscode/settings.json b/.vscode/settings.json index a660af9b84..c7e50307f9 100644 --- a/.vscode/settings.json +++ b/.vscode/settings.json @@ -4,5 +4,5 @@ "[vue]": { "editor.defaultFormatter": "esbenp.prettier-vscode" }, - "cSpell.words": ["Dexie"] + "cSpell.words": ["Dexie", "minisearch"] } diff --git a/__mocks__/it-pipe.js b/__mocks__/it-pipe.js new file mode 100644 index 0000000000..3e8c144cda --- /dev/null +++ b/__mocks__/it-pipe.js @@ -0,0 +1 @@ +export function pipe() {} diff --git a/components/ui/Global/Global.html b/components/ui/Global/Global.html index 3e3e85ec23..93ad3ba13d 100644 --- a/components/ui/Global/Global.html +++ b/components/ui/Global/Global.html @@ -30,15 +30,8 @@ :set-close-timeout="5000" /> - - + + ModalWindows, }, mounted() { @@ -68,19 +72,28 @@ export default Vue.extend({ * @example */ async acceptCall(kinds: TrackKind[]) { - if (this.webrtc.activeCall) { - this.hangUp() - } - - const identifier = this.webrtc.incomingCall - const call = this.$WebRTC.getPeer(identifier) - + this.$store.commit('webrtc/setStreamMuted', { + peerId: p2p.id, + audio: true, + video: true, + screen: true, + }) + const { callId, peerId } = this.webrtc.incomingCall + const call = $WebRTC.getCall(callId) if (!call) return if (call) { try { await call.createLocalTracks(kinds) - await call.answer() + await call.answer(peerId) + this.$store.commit( + 'webrtc/setActiveCall', + { callId, peerId }, + { root: true }, + ) + this.$store.commit('webrtc/setIncomingCall', undefined, { + root: true, + }) } catch (error) { if (error instanceof Error) { this.$toast.error(this.$t(error.message) as string) @@ -88,10 +101,20 @@ export default Vue.extend({ } } - const callingPath = `/chat/direct/${identifier}` + const redirectId = + this.conversation.type === 'group' + ? `groups/${callId}` + : `direct/${ + this.$store.state.friends.all.find( + (f: Friend) => f.peerId === peerId, + )?.address || 'error' + }` + + const callingPath = `/chat/${redirectId}` if (this.$route.path !== callingPath) { this.$router.push(callingPath) } + if (this.ui.showSettings) { this.$store.commit('ui/toggleSettings', { show: false }) } @@ -102,11 +125,8 @@ export default Vue.extend({ * @example */ denyCall() { - const identifier = this.webrtc.incomingCall - - const call = this.$WebRTC.getPeer(identifier) - if (call) call.deny() - + this.$store.commit('webrtc/setIncomingCall', undefined) + this.$store.commit('ui/fullscreen', false) this.$store.dispatch('webrtc/denyCall') }, /** @@ -115,11 +135,9 @@ export default Vue.extend({ * @example */ hangUp() { - if (!this.webrtc.activeCall) return - const call = this.$WebRTC.getPeer(this.webrtc.activeCall) - if (call) call.hangUp() - this.$store.dispatch('webrtc/hangUp') + this.$store.commit('webrtc/setIncomingCall', undefined, { root: true }) this.$store.commit('ui/fullscreen', false) + this.$store.dispatch('webrtc/hangUp') }, /** * @method share diff --git a/components/views/chat/chatbar/Chatbar.vue b/components/views/chat/chatbar/Chatbar.vue index 556d90c0f9..d756cc5ccf 100644 --- a/components/views/chat/chatbar/Chatbar.vue +++ b/components/views/chat/chatbar/Chatbar.vue @@ -4,7 +4,6 @@ import Vue, { PropType } from 'vue' import { mapState, mapGetters } from 'vuex' import { throttle } from 'lodash' import { TerminalIcon } from 'satellite-lucide-icons' -import PeerId from 'peer-id' import Upload from '../../files/upload/Upload.vue' import FilePreview from '../../files/upload/filePreview/FilePreview.vue' @@ -17,8 +16,8 @@ import { PropCommonEnum, } from '~/libraries/Enums/enums' import { Config } from '~/config' -import { Peer2Peer } from '~/libraries/WebRTC/Libp2p' import { UploadDropItemType } from '~/types/files/file' +import { Group } from '~/types/messaging' declare module 'vue/types/vue' { interface Vue { @@ -43,7 +42,7 @@ export default Vue.extend({ }, props: { recipient: { - type: Object as PropType, + type: Object as PropType, default: () => {}, }, }, @@ -56,10 +55,10 @@ export default Vue.extend({ } }, computed: { - ...mapState(['ui', 'friends', 'webrtc', 'chat', 'textile']), ...mapGetters('chat', ['getFiles']), + ...mapState(['ui', 'friends', 'webrtc', 'chat', 'textile', 'conversation']), activeFriend() { - return this.$Hounddog.getActiveFriend(this.friends) + return this.conversation?.participants?.[0] }, /** * Computes the amount of characters left @@ -153,6 +152,9 @@ export default Vue.extend({ this.$data.recipientTyping = activeFriend.typingState === PropCommonEnum.TYPING }, + }, + 'conversation.participants': { + handler() {}, deep: true, }, 'recipient.address': { @@ -205,29 +207,7 @@ export default Vue.extend({ typingNotifHandler( state: PropCommonEnum.TYPING | PropCommonEnum.NOT_TYPING, ) { - const activeFriend = this.$Hounddog.getActiveFriend(this.friends) - if (activeFriend) { - try { - const p2p = Peer2Peer.getInstance() - - if (!activeFriend.peerId) return - - p2p.sendMessage( - { - type: 'TYPING_STATE', - payload: { state: 'TYPING' }, - sentAt: Date.now(), - }, - PeerId.createFromB58String(activeFriend.peerId), - ) - // const activePeer = this.$WebRTC.getPeer(activeFriend.address) - // activePeer?.send('TYPING_STATE', { state }) - } catch (error: any) { - this.$Logger.log('cannot send after peer is destroyed', 'ERROR', { - error, - }) - } - } + // TODO use conversation participants }, /** * @method throttleTyping @@ -299,25 +279,25 @@ export default Vue.extend({ } if ( this.ui.replyChatbarContent.from && - !RegExp(this.$Config.regex.uuidv4).test(this.recipient.textilePubkey) + !RegExp(this.$Config.regex.uuidv4).test((this.recipient as Group)?.id) ) { this.$store.dispatch('textile/sendReplyMessage', { - to: this.recipient.textilePubkey, + to: (this.recipient as Friend).textilePubkey, text: value, replyTo: this.ui.replyChatbarContent.messageID, replyType: MessagingTypesEnum.TEXT, }) return } - // Check if it's a group + if ( RegExp(this.$Config.regex.uuidv4).test( - this.recipient.textilePubkey.split('|')[1], + (this.recipient as Group)?.id?.split('|')[1], ) ) { if (this.ui.replyChatbarContent.from) { this.$store.dispatch('textile/sendGroupReplyMessage', { - to: this.recipient.textilePubkey, + to: (this.recipient as Group).id, text: value, replyTo: this.ui.replyChatbarContent.messageID, replyType: MessagingTypesEnum.TEXT, @@ -326,12 +306,12 @@ export default Vue.extend({ return } this.$store.dispatch('textile/sendGroupMessage', { - groupId: this.recipient.textilePubkey, + groupId: (this.recipient as Group).id, message: value, }) } else { this.$store.dispatch('textile/sendTextMessage', { - to: this.recipient.textilePubkey, + to: (this.recipient as Friend).textilePubkey, text: value, }) } diff --git a/components/views/chat/chatbar/footer/is-connected/Connected.html b/components/views/chat/chatbar/footer/is-connected/Connected.html index e555600e2c..d4f020848c 100644 --- a/components/views/chat/chatbar/footer/is-connected/Connected.html +++ b/components/views/chat/chatbar/footer/is-connected/Connected.html @@ -1,19 +1,12 @@ -
+
-
diff --git a/components/views/chat/chatbar/footer/is-connected/Connected.vue b/components/views/chat/chatbar/footer/is-connected/Connected.vue index 4d2757a487..6de29bc00e 100644 --- a/components/views/chat/chatbar/footer/is-connected/Connected.vue +++ b/components/views/chat/chatbar/footer/is-connected/Connected.vue @@ -18,7 +18,48 @@ export default Vue.extend({ }, }, computed: { - ...mapState(['ui', 'webrtc', 'friends']), + ...mapState(['ui', 'webrtc', 'friends', 'conversation']), + onlineParticipants() { + return this.conversation.participants + .filter( + (participant) => + this.friends.all.find( + (friend) => friend.address === participant.address, + )?.state === 'online', + ) + .map((participant) => participant.name) + }, + onlineParticipantsText() { + if (this.onlineParticipants.length === 1) { + return `${this.onlineParticipants[0]} ${this.$t('ui.is')} ${this.$t( + 'ui.online', + )}` + } + if (this.onlineParticipants.length > 4) { + return `${this.onlineParticipants.length} ${this.$t( + 'ui.participants', + )} ${this.$t('ui.are')} ${this.$t('ui.online')}` + } + if (this.onlineParticipants.length) { + return `${this.onlineParticipants.join(', ')} ${this.$t( + 'ui.are', + )} ${this.$t('ui.online')}` + } + if (this.conversation.participants.length === 1) { + return `${this.conversation.participants[0]} ${this.$t( + 'ui.is', + )} ${this.$t('ui.offline')}` + } + return `${this.conversation.participants.join(', ')} ${this.$t( + 'ui.are', + )} ${this.$t('ui.offline')}` + }, + }, + watch: { + 'conversation.participants': { + handler() {}, + deep: true, + }, }, methods: { /** @@ -29,10 +70,10 @@ export default Vue.extend({ * friendConnected('user1') // true */ friendConnected(friendId: string): boolean { - if (this.webrtc) { - return this.webrtc.connectedPeers.includes(friendId) - } - return false + return ( + this.friends.all.find((friend) => friend.address === friendId).state === + 'online' + ) }, }, }) diff --git a/components/views/media/Media.html b/components/views/media/Media.html index ee79e3d048..b7e32f4494 100644 --- a/components/views/media/Media.html +++ b/components/views/media/Media.html @@ -1,40 +1,37 @@ -
- -
- - - - -
- ... -
-
- - - - -
+
+ +
+ + +
+ +
+ +
+ ... +
+
+ + + + +
diff --git a/components/views/media/Media.vue b/components/views/media/Media.vue index 1a524f6270..0c4cfe3a90 100644 --- a/components/views/media/Media.vue +++ b/components/views/media/Media.vue @@ -6,6 +6,12 @@ import { mapState } from 'vuex' import { Friend } from '~/types/ui/friends' import { User } from '~/types/ui/user' import { Sounds } from '~/libraries/SoundManager/SoundManager' +import { Peer2Peer } from '~/libraries/WebRTC/Libp2p' +import { Group, GroupMember } from '~/store/groups/types' +import { $WebRTC } from '~/libraries/WebRTC/WebRTC' +import { ConversationParticipant } from '~/store/conversation/types' + +const p2p = Peer2Peer.getInstance() export default Vue.extend({ props: { @@ -35,82 +41,37 @@ export default Vue.extend({ } }, computed: { - ...mapState(['ui', 'accounts', 'friends', 'webrtc']), + ...mapState([ + 'ui', + 'accounts', + 'friends', + 'groups', + 'webrtc', + 'conversation', + ]), isActiveCall() { - return this.friends.all.find( - (friend: any) => - friend.activeChat && friend.peerId === this.webrtc.activeCall, + return ( + this.webrtc.activeCall && + this.webrtc.activeCall.callId === this.conversation.id ) }, - localAudioMuted() { - return this.webrtc.localTracks.audio.muted - }, - remoteAudioMuted() { - return this.webrtc.remoteTracks.audio.muted - }, computedUsers() { return this.fullscreen ? this.users.slice(0, this.fullscreenMaxViewableUsers) : this.users.slice(0, this.maxViewableUsers) }, - activeCall() { - return this.$store.state.friends.all.some( - (friend: any) => friend.peerId === this.webrtc.activeCall, - ) + localParticipant() { + return { ...this.accounts.details, peerId: p2p.id } }, - localVideoStream() { - const { activeCall } = this.webrtc - const { id, muted } = this.webrtc.localTracks.video - - if (muted) { - return null - } - - const call = this.$WebRTC.getPeer(activeCall) - if (!call) return null - const localVideoTrack = call.getTrackById(id) - return localVideoTrack ? new MediaStream([localVideoTrack]) : null - }, - remoteVideoStream() { - const { activeCall } = this.webrtc - const { id, muted } = this.webrtc.remoteTracks.video - - if (muted) { - return null - } - - const call = this.$WebRTC.getPeer(activeCall) - if (!call) return null - - const remoteVideoTrack = call.getTrackById(id) - return remoteVideoTrack ? new MediaStream([remoteVideoTrack]) : null + remoteParticipants() { + return this.conversation.participants.filter( + (participant: ConversationParticipant) => participant.peerId !== p2p.id, + ) }, - remoteAudioStream() { + activeCall() { const { activeCall } = this.webrtc - const { id, muted } = this.webrtc.remoteTracks.audio - - if (muted) { - return null - } - - const call = this.$WebRTC.getPeer(activeCall) - if (!call) return null - const remoteAudioTrack = call.getTrackById(id) - return remoteAudioTrack ? new MediaStream([remoteAudioTrack]) : null - }, - remoteTracks() { - const { id, muted } = this.webrtc.remoteTracks.audio - - return Boolean(id || muted) - }, - recipient() { - const isMe = this.$route.params.address === this.accounts.active - const recipient = isMe - ? null - : this.friends.all.find( - (friend: Friend) => friend.address === this.$route.params.address, - ) - return recipient + const call = $WebRTC.getCall(activeCall.callId) + return call }, ...mapState(['audio']), }, @@ -246,14 +207,7 @@ export default Vue.extend({ beforeMount() { // TODO: Create mixin/library that will handle call rejoining and closing window.onbeforeunload = (e) => { - const call = this.$WebRTC.getPeer(this.webrtc.activeCall) - - if (call) { - call.hangUp() - this.$store.dispatch('webrtc/hangUp') - this.$store.commit('ui/fullscreen', false) - } - return undefined + this.$store.dispatch('webrtc/hangUp') } }, methods: { diff --git a/components/views/media/actions/Actions.html b/components/views/media/actions/Actions.html index 3cf8337b95..5610bfce9a 100644 --- a/components/views/media/actions/Actions.html +++ b/components/views/media/actions/Actions.html @@ -7,7 +7,7 @@ class="left" :type="videoMuted ? 'danger' : 'dark'" size="small" - :action="toggleVideo" + :action="() => toggleMute('video')" :loading="isLoading" loadingText="" > @@ -22,7 +22,7 @@ @@ -32,10 +32,17 @@
- - + + +
{ - this.elapsedTime(this.webrtc.activeStream.createdAt) + this.elapsedTime(this.webrtc.createdAt) }, 1000) } } diff --git a/components/views/media/incomingCall/IncomingCall.html b/components/views/media/incomingCall/IncomingCall.html index ff65600e4c..f6f34b53b0 100644 --- a/components/views/media/incomingCall/IncomingCall.html +++ b/components/views/media/incomingCall/IncomingCall.html @@ -1,13 +1,25 @@
-