Skip to content

Commit

Permalink
Group call video rendering performance improvements
Browse files Browse the repository at this point in the history
  • Loading branch information
@EvanHahn-Signal @josh-signal
EvanHahn-Signal authored and josh-signal committed Nov 23, 2020
1 parent 85de77d commit 9417871
Show file tree
Hide file tree
Showing 3 changed files with 144 additions and 122 deletions.
249 changes: 133 additions & 116 deletions ts/components/GroupCallRemoteParticipant.tsx
View file
Expand Up @@ -38,134 +38,151 @@ interface NotInPipPropsType {

type PropsType = BasePropsType & (InPipPropsType | NotInPipPropsType);

export const GroupCallRemoteParticipant: React.FC<PropsType> = props => {
const {
demuxId,
getGroupCallVideoFrameSource,
hasRemoteAudio,
hasRemoteVideo,
} = props;

const [canvasStyles, setCanvasStyles] = useState<CSSProperties>({});

const remoteVideoRef = useRef<HTMLCanvasElement | null>(null);
const rafIdRef = useRef<number | null>(null);
const frameBufferRef = useRef<ArrayBuffer>(
new ArrayBuffer(FRAME_BUFFER_SIZE)
);

const videoFrameSource = useMemo(
() => getGroupCallVideoFrameSource(demuxId),
[getGroupCallVideoFrameSource, demuxId]
);

const renderVideoFrame = useCallback(() => {
const canvasEl = remoteVideoRef.current;
if (!canvasEl) {
return;
}

const context = canvasEl.getContext('2d');
if (!context) {
return;
}
export const GroupCallRemoteParticipant: React.FC<PropsType> = React.memo(
props => {
const {
demuxId,
getGroupCallVideoFrameSource,
hasRemoteAudio,
hasRemoteVideo,
} = props;

const [isWide, setIsWide] = useState(true);

const remoteVideoRef = useRef<HTMLCanvasElement | null>(null);
const canvasContextRef = useRef<CanvasRenderingContext2D | null>(null);
const frameBufferRef = useRef<ArrayBuffer>(
new ArrayBuffer(FRAME_BUFFER_SIZE)
);

const frameDimensions = videoFrameSource.receiveVideoFrame(
frameBufferRef.current
const videoFrameSource = useMemo(
() => getGroupCallVideoFrameSource(demuxId),
[getGroupCallVideoFrameSource, demuxId]
);
if (!frameDimensions) {
return;
}

const [frameWidth, frameHeight] = frameDimensions;
canvasEl.width = frameWidth;
canvasEl.height = frameHeight;
const renderVideoFrame = useCallback(() => {
const canvasEl = remoteVideoRef.current;
if (!canvasEl) {
return;
}

const canvasContext = canvasContextRef.current;
if (!canvasContext) {
return;
}

const frameDimensions = videoFrameSource.receiveVideoFrame(
frameBufferRef.current
);
if (!frameDimensions) {
return;
}

context.putImageData(
new ImageData(
new Uint8ClampedArray(
frameBufferRef.current,
0,
frameWidth * frameHeight * 4
const [frameWidth, frameHeight] = frameDimensions;
if (frameWidth < 2 || frameHeight < 2) {
return;
}

canvasEl.width = frameWidth;
canvasEl.height = frameHeight;

canvasContext.putImageData(
new ImageData(
new Uint8ClampedArray(
frameBufferRef.current,
0,
frameWidth * frameHeight * 4
),
frameWidth,
frameHeight
),
frameWidth,
frameHeight
),
0,
0
);
0,
0
);

setIsWide(frameWidth > frameHeight);
}, [videoFrameSource]);

useEffect(() => {
if (!hasRemoteVideo) {
return noop;
}

let rafId = requestAnimationFrame(tick);

function tick() {
renderVideoFrame();
rafId = requestAnimationFrame(tick);
}

return () => {
cancelAnimationFrame(rafId);
};
}, [hasRemoteVideo, renderVideoFrame, videoFrameSource]);

let canvasStyles: CSSProperties;
let containerStyles: CSSProperties;

// If our `width` and `height` props don't match the canvas's aspect ratio, we want to
// fill the container. This can happen when RingRTC gives us an inaccurate
// `videoAspectRatio`, or if the container is an unexpected size.
if (frameWidth > frameHeight) {
setCanvasStyles({ width: '100%' });
if (isWide) {
canvasStyles = { width: '100%' };
} else {
setCanvasStyles({ height: '100%' });
canvasStyles = { height: '100%' };
}
}, [videoFrameSource]);

useEffect(() => {
if (!hasRemoteVideo) {
return noop;
// TypeScript isn't smart enough to know that `isInPip` by itself disambiguates the
// types, so we have to use `props.isInPip` instead.
// eslint-disable-next-line react/destructuring-assignment
if (props.isInPip) {
containerStyles = canvasStyles;
} else {
const { top, left, width, height } = props;

containerStyles = {
height,
left,
position: 'absolute',
top,
width,
};
}

const tick = () => {
renderVideoFrame();
rafIdRef.current = requestAnimationFrame(tick);
};

rafIdRef.current = requestAnimationFrame(tick);

return () => {
if (rafIdRef.current) {
cancelAnimationFrame(rafIdRef.current);
rafIdRef.current = null;
}
};
}, [hasRemoteVideo, renderVideoFrame, videoFrameSource]);

let containerStyles: CSSProperties;

// TypeScript isn't smart enough to know that `isInPip` by itself disambiguates the
// types, so we have to use `props.isInPip` instead.
// eslint-disable-next-line react/destructuring-assignment
if (props.isInPip) {
containerStyles = canvasStyles;
} else {
const { top, left, width, height } = props;

containerStyles = {
height,
left,
position: 'absolute',
top,
width,
};
return (
<div
className={classNames(
'module-ongoing-call__group-call-remote-participant',
{
'module-ongoing-call__group-call-remote-participant--audio-muted': !hasRemoteAudio,
}
)}
style={containerStyles}
>
{hasRemoteVideo ? (
<canvas
className="module-ongoing-call__group-call-remote-participant__remote-video"
style={canvasStyles}
ref={canvasEl => {
remoteVideoRef.current = canvasEl;
if (canvasEl) {
canvasContextRef.current = canvasEl.getContext('2d', {
alpha: false,
desynchronized: true,
storage: 'discardable',
} as CanvasRenderingContext2DSettings);
} else {
canvasContextRef.current = null;
}
}}
/>
) : (
<CallBackgroundBlur>
{/* TODO: Improve the styling here. See DESKTOP-894. */}
<span />
</CallBackgroundBlur>
)}
</div>
);
}

return (
<div
className={classNames(
'module-ongoing-call__group-call-remote-participant',
{
'module-ongoing-call__group-call-remote-participant--audio-muted': !hasRemoteAudio,
}
)}
style={containerStyles}
>
{hasRemoteVideo ? (
<canvas
className="module-ongoing-call__group-call-remote-participant__remote-video"
style={canvasStyles}
ref={remoteVideoRef}
/>
) : (
<CallBackgroundBlur>
{/* TODO: Improve the styling here. See DESKTOP-894. */}
<span />
</CallBackgroundBlur>
)}
</div>
);
};
);
13 changes: 9 additions & 4 deletions ts/components/GroupCallRemoteParticipants.tsx
View file
Expand Up @@ -162,7 +162,9 @@ export const GroupCallRemoteParticipants: React.FC<PropsType> = ({
]);

// 4. Lay out this arrangement on the screen.
const gridParticipantHeight = gridArrangement.scalar * MIN_RENDERED_HEIGHT;
const gridParticipantHeight = Math.floor(
gridArrangement.scalar * MIN_RENDERED_HEIGHT
);
const gridParticipantHeightWithMargin =
gridParticipantHeight + PARTICIPANT_MARGIN;
const gridTotalRowHeightWithMargin =
Expand All @@ -181,12 +183,15 @@ export const GroupCallRemoteParticipants: React.FC<PropsType> = ({
const totalRowWidth =
totalRowWidthWithoutMargins +
PARTICIPANT_MARGIN * (remoteParticipantsInRow.length - 1);
const leftOffset = (containerDimensions.width - totalRowWidth) / 2;
const leftOffset = Math.floor(
(containerDimensions.width - totalRowWidth) / 2
);

let rowWidthSoFar = 0;
return remoteParticipantsInRow.map(remoteParticipant => {
const renderedWidth =
remoteParticipant.videoAspectRatio * gridParticipantHeight;
const renderedWidth = Math.floor(
remoteParticipant.videoAspectRatio * gridParticipantHeight
);
const left = rowWidthSoFar + leftOffset;

rowWidthSoFar += renderedWidth + PARTICIPANT_MARGIN;
Expand Down
4 changes: 2 additions & 2 deletions ts/util/lint/exceptions.json
View file
Expand Up @@ -14579,10 +14579,10 @@
{
"rule": "React-useRef",
"path": "ts/components/GroupCallRemoteParticipant.js",
"line": " const rafIdRef = react_1.useRef(null);",
"line": " const canvasContextRef = react_1.useRef(null);",
"lineNumber": 25,
"reasonCategory": "usageTrusted",
"updated": "2020-11-17T16:24:25.480Z",
"updated": "2020-11-17T23:29:38.698Z",
"reasonDetail": "Doesn't touch the DOM."
},
{
Expand Down

0 comments on commit 9417871

Please sign in to comment.