diff --git a/index.html b/index.html index 68f616e..c8455c5 100644 --- a/index.html +++ b/index.html @@ -1688,5 +1688,104 @@
Some platforms or User Agents may provide built-in support for background segmentation of video frames, in particular for camera video streams. + Web applications may want to control whether background segmentation is computed at the source level and to get access to the computed segmentation masks. + This allows the web application for instance + to do custom framing or background blurring or replacement + while leveraging on platform computed background segmentation. + This allows the web application + to access the original unmodified frame and + to fine tune frame modifications based on its likings. + For that reason, we extend {{MediaStreamTrack}} with the following properties and {{VideoFrame}} with the following attributes. +
++partial dictionary MediaTrackSupportedConstraints { + boolean backgroundSegmentationMask = true; +}; + +partial dictionary MediaTrackConstraintSet { + ConstrainBoolean backgroundSegmentationMask; +}; + +partial dictionary MediaTrackSettings { + boolean backgroundSegmentationMask; +}; + +partial dictionary MediaTrackCapabilities { + sequence<boolean> backgroundSegmentationMask; +};+
+partial dictionary VideoFrameMetadata { + ImageBitmap backgroundSegmentationMask; +};+
backgroundSegmentationMask
+ of type
+ {{VideoFrameMetadata}}A background segmentation mask with + white denoting certainly foreground, + black denoting certainly background and + grey denoting uncertainty or ambiguity with + light shades of grey denoting likely foreground and + dark shades of grey denoting likely background.
++// Open camera. +const stream = await navigator.mediaDevices.getUserMedia({video: true}); +const [videoTrack] = stream.getVideoTracks(); + +// Try to enable background segmentation mask. +const videoCapabilities = videoTrack.getCapabilities(); +if ((videoCapabilities.backgroundSegmentationMask || []).includes(true)) { + await videoTrack.applyConstraints({backgroundSegmentationMask: {exact: true}}); +} else { + // Background segmentation mask is not supported by the platform or + // by the camera. Consider falling back to some other method. +} + +const canvasContext = document.querySelector('canvas').getContext('2d'); +const videoProcessor = new MediaStreamTrackProcessor({track: videoTrack}); +const videoProcessorReader = videoProcessor.readable.getReader(); + +for (;;) { + const {done, value: videoFrame} = await videoProcessorReader.read(); + if (done) + break; + const {backgroundSegmentationMask} = videoFrame.metadata(); + if (backgroundSegmentationMask) { + // Draw the video frame. + canvasContext.globalCompositeOperation = 'copy'; + context.drawImage(videoFrame, 0, 0); + // Draw (or multiply with) the mask. + // The result is the foreground on black. + context.globalCompositeOperation = 'multiply'; + canvasContext.drawImage(backgroundSegmentationMask, 0, 0); + } + else { + // Everything is background. Fill with black. + canvasContext.globalCompositeOperation = 'copy'; + canvasContext.fillStyle = 'black'; + canvasContext.fillRect( + 0, + 0, + canvasContext.canvas.width, + canvasContext.canvas.height); + } +} ++