From b366967ca599ef42ddeaf1780ee9f7f76c6e9959 Mon Sep 17 00:00:00 2001 From: Evan Hahn <69474926+EvanHahn-Signal@users.noreply.github.com> Date: Tue, 17 Nov 2020 13:49:48 -0600 Subject: [PATCH] Upgrade to RingRTC v2.8.2 RC.6 --- package.json | 2 +- ts/components/CallManager.stories.tsx | 11 +- ts/components/CallManager.tsx | 5 - ts/components/CallScreen.stories.tsx | 11 +- ts/components/CallScreen.tsx | 4 - ts/components/CallingPip.stories.tsx | 4 +- ts/components/CallingPip.tsx | 5 +- ts/components/CallingPipRemoteVideo.tsx | 14 +- ts/components/GroupCallRemoteParticipant.tsx | 171 +++++++++++++----- ts/components/GroupCallRemoteParticipants.tsx | 5 +- ts/services/calling.ts | 123 +++++++------ ts/state/smart/CallManager.tsx | 4 - ts/types/Calling.ts | 10 - ts/util/lint/exceptions.json | 20 +- yarn.lock | 4 +- 15 files changed, 212 insertions(+), 181 deletions(-) diff --git a/package.json b/package.json index 137374b4bb8..2bdda56aed2 100644 --- a/package.json +++ b/package.json @@ -136,7 +136,7 @@ "redux-ts-utils": "3.2.2", "reselect": "4.0.0", "rimraf": "2.6.2", - "ringrtc": "https://github.com/signalapp/signal-ringrtc-node.git#7de95cffa71019e94c74fc43313c28eaf23c66e1", + "ringrtc": "https://github.com/signalapp/signal-ringrtc-node.git#65fbea67295005fe1a9db2cbe85fce0d5297848d", "sanitize-filename": "1.6.3", "sanitize.css": "11.0.0", "semver": "5.4.1", diff --git a/ts/components/CallManager.stories.tsx b/ts/components/CallManager.stories.tsx index 4cad4883b7c..74882fdb010 100644 --- a/ts/components/CallManager.stories.tsx +++ b/ts/components/CallManager.stories.tsx @@ -62,17 +62,10 @@ const createProps = (storyProps: Partial = {}): PropsType => ({ cancelCall: action('cancel-call'), closeNeedPermissionScreen: action('close-need-permission-screen'), declineCall: action('decline-call'), - // We allow `any` here because these are fake and actually come from RingRTC, which we + // We allow `any` here because this is fake and actually comes from RingRTC, which we // can't import. - /* eslint-disable @typescript-eslint/no-explicit-any */ - createCanvasVideoRenderer: () => - ({ - setCanvas: noop, - enable: noop, - disable: noop, - } as any), + // eslint-disable-next-line @typescript-eslint/no-explicit-any getGroupCallVideoFrameSource: noop as any, - /* eslint-enable @typescript-eslint/no-explicit-any */ hangUp: action('hang-up'), i18n, me: { diff --git a/ts/components/CallManager.tsx b/ts/components/CallManager.tsx index a4f413d7115..bc0e6f031ef 100644 --- a/ts/components/CallManager.tsx +++ b/ts/components/CallManager.tsx @@ -12,7 +12,6 @@ import { CallEndedReason, CallMode, CallState, - CanvasVideoRenderer, GroupCallJoinState, GroupCallRemoteParticipantType, VideoFrameSource, @@ -47,7 +46,6 @@ export interface PropsType { activeCall?: ActiveCallType; availableCameras: Array; cancelCall: (_: CancelCallType) => void; - createCanvasVideoRenderer: () => CanvasVideoRenderer; closeNeedPermissionScreen: () => void; getGroupCallVideoFrameSource: ( conversationId: string, @@ -89,7 +87,6 @@ const ActiveCallManager: React.FC = ({ availableCameras, cancelCall, closeNeedPermissionScreen, - createCanvasVideoRenderer, hangUp, i18n, getGroupCallVideoFrameSource, @@ -210,7 +207,6 @@ const ActiveCallManager: React.FC = ({ = ({ - ({ - setCanvas: noop, - enable: noop, - disable: noop, - } as any), + // eslint-disable-next-line @typescript-eslint/no-explicit-any getGroupCallVideoFrameSource: noop as any, - /* eslint-enable @typescript-eslint/no-explicit-any */ hangUp: action('hang-up'), hasLocalAudio: boolean('hasLocalAudio', overrideProps.hasLocalAudio || false), hasLocalVideo: boolean('hasLocalVideo', overrideProps.hasLocalVideo || false), diff --git a/ts/components/CallScreen.tsx b/ts/components/CallScreen.tsx index 7b738f728a4..b76fa4aa18a 100644 --- a/ts/components/CallScreen.tsx +++ b/ts/components/CallScreen.tsx @@ -22,7 +22,6 @@ import { CallMode, CallState, GroupCallConnectionState, - CanvasVideoRenderer, VideoFrameSource, } from '../types/Calling'; import { ColorType } from '../types/Colors'; @@ -34,7 +33,6 @@ import { GroupCallRemoteParticipants } from './GroupCallRemoteParticipants'; export type PropsType = { call: DirectCallStateType | GroupCallStateType; conversation: ConversationType; - createCanvasVideoRenderer: () => CanvasVideoRenderer; getGroupCallVideoFrameSource: (demuxId: number) => VideoFrameSource; hangUp: (_: HangUpType) => void; hasLocalAudio: boolean; @@ -62,7 +60,6 @@ export type PropsType = { export const CallScreen: React.FC = ({ call, conversation, - createCanvasVideoRenderer, getGroupCallVideoFrameSource, hangUp, hasLocalAudio, @@ -174,7 +171,6 @@ export const CallScreen: React.FC = ({ remoteParticipantsElement = ( ); diff --git a/ts/components/CallingPip.stories.tsx b/ts/components/CallingPip.stories.tsx index fda46332ec6..10a72ec3ba9 100644 --- a/ts/components/CallingPip.stories.tsx +++ b/ts/components/CallingPip.stories.tsx @@ -46,9 +46,7 @@ const defaultCall = { const createProps = (overrideProps: Partial = {}): PropsType => ({ call: overrideProps.call || defaultCall, conversation: overrideProps.conversation || conversation, - /* eslint-disable @typescript-eslint/no-explicit-any */ - createCanvasVideoRenderer: noop as any, - /* eslint-disable @typescript-eslint/no-explicit-any */ + // eslint-disable-next-line @typescript-eslint/no-explicit-any getGroupCallVideoFrameSource: noop as any, hangUp: action('hang-up'), hasLocalVideo: boolean('hasLocalVideo', overrideProps.hasLocalVideo || false), diff --git a/ts/components/CallingPip.tsx b/ts/components/CallingPip.tsx index a3c8cc1c1e9..d0316d0b52e 100644 --- a/ts/components/CallingPip.tsx +++ b/ts/components/CallingPip.tsx @@ -6,7 +6,7 @@ import Tooltip from 'react-tooltip-lite'; import { CallingPipRemoteVideo } from './CallingPipRemoteVideo'; import { LocalizerType } from '../types/Util'; import { ConversationType } from '../state/ducks/conversations'; -import { CanvasVideoRenderer, VideoFrameSource } from '../types/Calling'; +import { VideoFrameSource } from '../types/Calling'; import { DirectCallStateType, GroupCallStateType, @@ -18,7 +18,6 @@ import { export type PropsType = { call: DirectCallStateType | GroupCallStateType; conversation: ConversationType; - createCanvasVideoRenderer: () => CanvasVideoRenderer; getGroupCallVideoFrameSource: (demuxId: number) => VideoFrameSource; hangUp: (_: HangUpType) => void; hasLocalVideo: boolean; @@ -36,7 +35,6 @@ const PIP_PADDING = 8; export const CallingPip = ({ call, conversation, - createCanvasVideoRenderer, getGroupCallVideoFrameSource, hangUp, hasLocalVideo, @@ -171,7 +169,6 @@ export const CallingPip = ({ CanvasVideoRenderer; getGroupCallVideoFrameSource: (demuxId: number) => VideoFrameSource; i18n: LocalizerType; setRendererCanvas: (_: SetRendererCanvasType) => void; @@ -31,7 +26,6 @@ export interface PropsType { export const CallingPipRemoteVideo = ({ call, conversation, - createCanvasVideoRenderer, getGroupCallVideoFrameSource, i18n, setRendererCanvas, @@ -87,16 +81,12 @@ export const CallingPipRemoteVideo = ({ return (
); diff --git a/ts/components/GroupCallRemoteParticipant.tsx b/ts/components/GroupCallRemoteParticipant.tsx index 98f0a66187f..6c25e1b4c0f 100644 --- a/ts/components/GroupCallRemoteParticipant.tsx +++ b/ts/components/GroupCallRemoteParticipant.tsx @@ -1,64 +1,147 @@ // Copyright 2020 Signal Messenger, LLC // SPDX-License-Identifier: AGPL-3.0-only -import React, { useRef, useEffect, CSSProperties } from 'react'; +import React, { + useState, + useRef, + useMemo, + useCallback, + useEffect, + CSSProperties, +} from 'react'; import classNames from 'classnames'; import { noop } from 'lodash'; -import { CanvasVideoRenderer, VideoFrameSource } from '../types/Calling'; +import { VideoFrameSource } from '../types/Calling'; import { CallBackgroundBlur } from './CallBackgroundBlur'; -interface PropsType { - createCanvasVideoRenderer: () => CanvasVideoRenderer; +// The max size video frame we'll support (in RGBA) +const FRAME_BUFFER_SIZE = 1920 * 1080 * 4; + +interface BasePropsType { demuxId: number; getGroupCallVideoFrameSource: (demuxId: number) => VideoFrameSource; hasRemoteAudio: boolean; hasRemoteVideo: boolean; - height: number | string; +} + +interface InPipPropsType { + isInPip: true; +} + +interface NotInPipPropsType { + isInPip?: false; + width: number; + height: number; left: number; top: number; - width: number | string; } -export const GroupCallRemoteParticipant: React.FC = ({ - createCanvasVideoRenderer, - demuxId, - getGroupCallVideoFrameSource, - hasRemoteAudio, - hasRemoteVideo, - height, - left, - top, - width, -}) => { +type PropsType = BasePropsType & (InPipPropsType | NotInPipPropsType); + +export const GroupCallRemoteParticipant: React.FC = props => { + const { + demuxId, + getGroupCallVideoFrameSource, + hasRemoteAudio, + hasRemoteVideo, + } = props; + + const [canvasStyles, setCanvasStyles] = useState({}); + const remoteVideoRef = useRef(null); - const canvasVideoRendererRef = useRef(createCanvasVideoRenderer()); + const rafIdRef = useRef(null); + const frameBufferRef = useRef( + new ArrayBuffer(FRAME_BUFFER_SIZE) + ); - useEffect(() => { - const canvasVideoRenderer = canvasVideoRendererRef.current; - - if (hasRemoteVideo) { - canvasVideoRenderer.setCanvas(remoteVideoRef); - canvasVideoRenderer.enable(getGroupCallVideoFrameSource(demuxId)); - return () => { - canvasVideoRenderer.disable(); - }; + const videoFrameSource = useMemo( + () => getGroupCallVideoFrameSource(demuxId), + [getGroupCallVideoFrameSource, demuxId] + ); + + const renderVideoFrame = useCallback(() => { + const canvasEl = remoteVideoRef.current; + if (!canvasEl) { + return; + } + + const context = canvasEl.getContext('2d'); + if (!context) { + return; } - canvasVideoRenderer.disable(); - return noop; - }, [hasRemoteVideo, getGroupCallVideoFrameSource, demuxId]); - - // If our `width` and `height` props don't match the canvas's aspect ratio, we want to - // fill the container. This can happen when RingRTC gives us an inaccurate - // `videoAspectRatio`. - const canvasStyles: CSSProperties = {}; - const canvasEl = remoteVideoRef.current; - if (hasRemoteVideo && canvasEl) { - if (canvasEl.width > canvasEl.height) { - canvasStyles.width = '100%'; + const frameDimensions = videoFrameSource.receiveVideoFrame( + frameBufferRef.current + ); + if (!frameDimensions) { + return; + } + + const [frameWidth, frameHeight] = frameDimensions; + canvasEl.width = frameWidth; + canvasEl.height = frameHeight; + + context.putImageData( + new ImageData( + new Uint8ClampedArray( + frameBufferRef.current, + 0, + frameWidth * frameHeight * 4 + ), + frameWidth, + frameHeight + ), + 0, + 0 + ); + + // If our `width` and `height` props don't match the canvas's aspect ratio, we want to + // fill the container. This can happen when RingRTC gives us an inaccurate + // `videoAspectRatio`, or if the container is an unexpected size. + if (frameWidth > frameHeight) { + setCanvasStyles({ width: '100%' }); } else { - canvasStyles.height = '100%'; + setCanvasStyles({ height: '100%' }); + } + }, [videoFrameSource]); + + useEffect(() => { + if (!hasRemoteVideo) { + return noop; } + + const tick = () => { + renderVideoFrame(); + rafIdRef.current = requestAnimationFrame(tick); + }; + + rafIdRef.current = requestAnimationFrame(tick); + + return () => { + if (rafIdRef.current) { + cancelAnimationFrame(rafIdRef.current); + rafIdRef.current = null; + } + }; + }, [hasRemoteVideo, renderVideoFrame, videoFrameSource]); + + let containerStyles: CSSProperties; + + // TypeScript isn't smart enough to know that `isInPip` by itself disambiguates the + // types, so we have to use `props.isInPip` instead. + // eslint-disable-next-line react/destructuring-assignment + if (props.isInPip) { + containerStyles = canvasStyles; + } else { + const { top, left, width, height } = props; + + containerStyles = { + height, + left, + position: 'absolute', + top, + width, + }; } return ( @@ -69,13 +152,7 @@ export const GroupCallRemoteParticipant: React.FC = ({ 'module-ongoing-call__group-call-remote-participant--audio-muted': !hasRemoteAudio, } )} - style={{ - position: 'absolute', - width, - height, - top, - left, - }} + style={containerStyles} > {hasRemoteVideo ? ( CanvasVideoRenderer; getGroupCallVideoFrameSource: (demuxId: number) => VideoFrameSource; remoteParticipants: ReadonlyArray; } @@ -52,7 +51,6 @@ interface PropsType { // screen? The biggest scalar wins as the "best arrangement". // 4. Lay out this arrangement on the screen. export const GroupCallRemoteParticipants: React.FC = ({ - createCanvasVideoRenderer, getGroupCallVideoFrameSource, remoteParticipants, }) => { @@ -196,7 +194,6 @@ export const GroupCallRemoteParticipants: React.FC = ({ return ( = new Map([ [HttpMethod.Get, 'GET'], [HttpMethod.Put, 'PUT'], [HttpMethod.Post, 'POST'], + [HttpMethod.Delete, 'DELETE'], ]); export { @@ -323,67 +326,73 @@ export class CallingClass { let isRequestingMembershipProof = false; - const outerGroupCall = RingRTC.getGroupCall(groupIdBuffer, { - onLocalDeviceStateChanged: groupCall => { - const localDeviceState = groupCall.getLocalDeviceState(); + const outerGroupCall = RingRTC.getGroupCall( + groupIdBuffer, + RINGRTC_SFU_URL, + { + onLocalDeviceStateChanged: groupCall => { + const localDeviceState = groupCall.getLocalDeviceState(); + + if ( + localDeviceState.connectionState === ConnectionState.NotConnected + ) { + if (localDeviceState.videoMuted) { + this.disableLocalCamera(); + } + + delete this.callsByConversation[conversationId]; + } else { + this.callsByConversation[conversationId] = groupCall; - if (localDeviceState.connectionState === ConnectionState.NotConnected) { - if (localDeviceState.videoMuted) { - this.disableLocalCamera(); + if (localDeviceState.videoMuted) { + this.disableLocalCamera(); + } else { + this.videoCapturer.enableCaptureAndSend(groupCall); + } } - delete this.callsByConversation[conversationId]; - } else { - this.callsByConversation[conversationId] = groupCall; - - if (localDeviceState.videoMuted) { - this.disableLocalCamera(); - } else { - this.enableLocalCamera(); + this.syncGroupCallToRedux(conversationId, groupCall); + }, + onRemoteDeviceStatesChanged: groupCall => { + this.syncGroupCallToRedux(conversationId, groupCall); + }, + onPeekChanged: groupCall => { + this.syncGroupCallToRedux(conversationId, groupCall); + }, + async requestMembershipProof(groupCall) { + if (isRequestingMembershipProof) { + return; } - } - - this.syncGroupCallToRedux(conversationId, groupCall); - }, - onRemoteDeviceStatesChanged: groupCall => { - this.syncGroupCallToRedux(conversationId, groupCall); - }, - onJoinedMembersChanged: groupCall => { - this.syncGroupCallToRedux(conversationId, groupCall); - }, - async requestMembershipProof(groupCall) { - if (isRequestingMembershipProof) { - return; - } - isRequestingMembershipProof = true; - try { - const proof = await fetchMembershipProof({ - publicParams, - secretParams, - }); - if (proof) { - const proofArray = new TextEncoder().encode(proof); - groupCall.setMembershipProof(proofArray.buffer); + isRequestingMembershipProof = true; + try { + const proof = await fetchMembershipProof({ + publicParams, + secretParams, + }); + if (proof) { + const proofArray = new TextEncoder().encode(proof); + groupCall.setMembershipProof(proofArray.buffer); + } + } catch (err) { + window.log.error('Failed to fetch membership proof', err); + } finally { + isRequestingMembershipProof = false; } - } catch (err) { - window.log.error('Failed to fetch membership proof', err); - } finally { - isRequestingMembershipProof = false; - } - }, - requestGroupMembers(groupCall) { - groupCall.setGroupMembers( - getMembershipList(conversationId).map( - member => - new GroupMemberInfo( - uuidToArrayBuffer(member.uuid), - member.uuidCiphertext - ) - ) - ); - }, - onEnded: noop, - }); + }, + requestGroupMembers(groupCall) { + groupCall.setGroupMembers( + getMembershipList(conversationId).map( + member => + new GroupMemberInfo( + uuidToArrayBuffer(member.uuid), + member.uuidCiphertext + ) + ) + ); + }, + onEnded: noop, + } + ); if (!outerGroupCall) { // This should be very rare, likely due to RingRTC not being able to get a lock diff --git a/ts/state/smart/CallManager.tsx b/ts/state/smart/CallManager.tsx index 867b864138a..1e19f43774f 100644 --- a/ts/state/smart/CallManager.tsx +++ b/ts/state/smart/CallManager.tsx @@ -3,7 +3,6 @@ import React from 'react'; import { connect } from 'react-redux'; -import { CanvasVideoRenderer } from 'ringrtc'; import { mapDispatchToProps } from '../actions'; import { CallManager } from '../../components/CallManager'; import { calling as callingService } from '../../services/calling'; @@ -21,8 +20,6 @@ function renderDeviceSelection(): JSX.Element { return ; } -const createCanvasVideoRenderer = () => new CanvasVideoRenderer(); - const getGroupCallVideoFrameSource = callingService.getGroupCallVideoFrameSource.bind( callingService ); @@ -108,7 +105,6 @@ const mapStateToIncomingCallProp = (state: StateType) => { const mapStateToProps = (state: StateType) => ({ activeCall: mapStateToActiveCallProp(state), availableCameras: state.calling.availableCameras, - createCanvasVideoRenderer, getGroupCallVideoFrameSource, i18n: getIntl(state), incomingCall: mapStateToIncomingCallProp(state), diff --git a/ts/types/Calling.ts b/ts/types/Calling.ts index aac1fe6dad7..c2f5a2ae1b4 100644 --- a/ts/types/Calling.ts +++ b/ts/types/Calling.ts @@ -69,16 +69,6 @@ export interface GroupCallRemoteParticipantType { title: string; } -// Should match RingRTC's CanvasVideoRenderer -interface Ref { - readonly current: T | null; -} -export interface CanvasVideoRenderer { - setCanvas(canvas: Ref | undefined): void; - enable(source: VideoFrameSource): void; - disable(): void; -} - // Should match RingRTC's VideoFrameSource export interface VideoFrameSource { receiveVideoFrame(buffer: ArrayBuffer): [number, number] | undefined; diff --git a/ts/util/lint/exceptions.json b/ts/util/lint/exceptions.json index 9b5dcbf1646..2497c45f7dd 100644 --- a/ts/util/lint/exceptions.json +++ b/ts/util/lint/exceptions.json @@ -14427,7 +14427,7 @@ "rule": "React-useRef", "path": "ts/components/CallingPip.tsx", "line": " const videoContainerRef = React.useRef(null);", - "lineNumber": 48, + "lineNumber": 46, "reasonCategory": "usageTrusted", "updated": "2020-10-26T19:12:24.410Z", "reasonDetail": "Element is measured. Its HTML is not used." @@ -14436,7 +14436,7 @@ "rule": "React-useRef", "path": "ts/components/CallingPip.tsx", "line": " const localVideoRef = React.useRef(null);", - "lineNumber": 49, + "lineNumber": 47, "reasonCategory": "usageTrusted", "updated": "2020-10-26T19:12:24.410Z", "reasonDetail": "Used to get the local video element for rendering." @@ -14571,7 +14571,7 @@ "rule": "React-useRef", "path": "ts/components/GroupCallRemoteParticipant.js", "line": " const remoteVideoRef = react_1.useRef(null);", - "lineNumber": 20, + "lineNumber": 24, "reasonCategory": "usageTrusted", "updated": "2020-11-11T21:56:04.179Z", "reasonDetail": "Needed to render the remote video element." @@ -14579,19 +14579,19 @@ { "rule": "React-useRef", "path": "ts/components/GroupCallRemoteParticipant.js", - "line": " const canvasVideoRendererRef = react_1.useRef(createCanvasVideoRenderer());", - "lineNumber": 21, + "line": " const rafIdRef = react_1.useRef(null);", + "lineNumber": 25, "reasonCategory": "usageTrusted", - "updated": "2020-11-11T21:56:04.179Z", + "updated": "2020-11-17T16:24:25.480Z", "reasonDetail": "Doesn't touch the DOM." }, { "rule": "React-useRef", - "path": "ts/components/GroupCallRemoteParticipant.tsx", - "line": " const canvasVideoRendererRef = useRef(createCanvasVideoRenderer());", - "lineNumber": 34, + "path": "ts/components/GroupCallRemoteParticipant.js", + "line": " const frameBufferRef = react_1.useRef(new ArrayBuffer(FRAME_BUFFER_SIZE));", + "lineNumber": 26, "reasonCategory": "usageTrusted", - "updated": "2020-11-11T21:56:04.179Z", + "updated": "2020-11-17T16:24:25.480Z", "reasonDetail": "Doesn't touch the DOM." }, { diff --git a/yarn.lock b/yarn.lock index a6976431786..4c31ceb1885 100644 --- a/yarn.lock +++ b/yarn.lock @@ -14300,9 +14300,9 @@ rimraf@~2.4.0: dependencies: glob "^6.0.1" -"ringrtc@https://github.com/signalapp/signal-ringrtc-node.git#7de95cffa71019e94c74fc43313c28eaf23c66e1": +"ringrtc@https://github.com/signalapp/signal-ringrtc-node.git#65fbea67295005fe1a9db2cbe85fce0d5297848d": version "2.8.1" - resolved "https://github.com/signalapp/signal-ringrtc-node.git#7de95cffa71019e94c74fc43313c28eaf23c66e1" + resolved "https://github.com/signalapp/signal-ringrtc-node.git#65fbea67295005fe1a9db2cbe85fce0d5297848d" ripemd160@^2.0.0, ripemd160@^2.0.1: version "2.0.1"