From 558751d92a79234fbbd4f80491299cb42830ccea Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Eero=20H=C3=A4kkinen?= Date: Thu, 9 May 2024 00:51:05 +0300 Subject: [PATCH 1/5] Add background segmentation mask --- index.html | 52 ++++++++++++++++++++++++++++++++++++++++++++++++++++ 1 file changed, 52 insertions(+) diff --git a/index.html b/index.html index 68f616e..97d1e1b 100644 --- a/index.html +++ b/index.html @@ -1688,5 +1688,57 @@

MediaStream in workers

}; +
+

Background segmentation mask

+

Some platforms or User Agents may provide built-in support for background segmentation of video frames, in particular for camera video streams. + Web applications may want to control whether background segmentation is computed at the source level and to get access to the computed segmentation masks. + This allows the web application for instance + to do custom framing or background blurring or replacement + while leveraging on platform computed background segmentation. + This allows the web application + to access the original unmodified frame and + to fine tune frame modifications based on its likings. + For that reason, we extend {{MediaStreamTrack}} with the following properties and {{VideoFrame}} with the following attributes. +

+ 
+partial dictionary MediaTrackSupportedConstraints {
+  boolean backgroundSegmentationMask = true;
+};
+
+partial dictionary MediaTrackConstraintSet {
+  ConstrainBoolean backgroundSegmentationMask;
+};
+
+partial dictionary MediaTrackSettings {
+  boolean backgroundSegmentationMask;
+};
+
+partial dictionary MediaTrackCapabilities {
+  sequence<boolean> backgroundSegmentationMask;
+};
+
+

VideoFrame interface extensions

+ 
+partial interface VideoFrame {
+  readonly attribute VideoFrame? backgroundSegmentationMask;
+};
+
+

Attributes

+
+
backgroundSegmentationMask + of type + {{VideoFrame}}, + readonly
+
+

A background segmentation mask with + white denoting certainly foreground, + black denoting certainly background and + grey denoting uncertainty.

+
+
+
+ +
+
From 9dafe7d708312453bca81b587f1aee63df0bf4b5 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Eero=20H=C3=A4kkinen?= Date: Thu, 30 May 2024 22:22:46 +0300 Subject: [PATCH 2/5] Update index.html --- index.html | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/index.html b/index.html index 97d1e1b..c25844f 100644 --- a/index.html +++ b/index.html @@ -1733,7 +1733,9 @@

Attributes

A background segmentation mask with white denoting certainly foreground, black denoting certainly background and - grey denoting uncertainty.

+ grey denoting uncertainty or ambiguity with + light shades of grey denoting likely foreground and + dark shades of grey denoting likely background.

From 1a5a12ba9eeef7ab98857829b9b1169f021d5275 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Eero=20H=C3=A4kkinen?= Date: Thu, 6 Jun 2024 15:43:32 +0300 Subject: [PATCH 3/5] Extend VideoFrameMetadata with ImageBitmap --- index.html | 13 ++++++------- 1 file changed, 6 insertions(+), 7 deletions(-) diff --git a/index.html b/index.html index c25844f..7b46a5e 100644 --- a/index.html +++ b/index.html @@ -1717,18 +1717,17 @@

Background segmentation mask

sequence<boolean> backgroundSegmentationMask; };
-

VideoFrame interface extensions

+

VideoFrameMetadata dictionary extensions

-partial interface VideoFrame {
-  readonly attribute VideoFrame? backgroundSegmentationMask;
+partial dictionary VideoFrameMetadata {
+  ImageBitmap backgroundSegmentationMask;
 };
-

Attributes

-
+

Dictionary {{VideoFrameMetadata}} Members

+
backgroundSegmentationMask of type - {{VideoFrame}}, - readonly
+ {{VideoFrameMetadata}}

A background segmentation mask with white denoting certainly foreground, From 5aa912ddaafa48280a33a1f9e5cd966c405bc1d2 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Eero=20H=C3=A4kkinen?= Date: Fri, 14 Jun 2024 10:41:53 +0300 Subject: [PATCH 4/5] Fix formatting --- index.html | 29 ++++++++++++++--------------- 1 file changed, 14 insertions(+), 15 deletions(-) diff --git a/index.html b/index.html index 7b46a5e..728b695 100644 --- a/index.html +++ b/index.html @@ -1724,21 +1724,20 @@

VideoFrameMetadata dictionary extensions

};

Dictionary {{VideoFrameMetadata}} Members

-
-
backgroundSegmentationMask - of type - {{VideoFrameMetadata}}
-
-

A background segmentation mask with - white denoting certainly foreground, - black denoting certainly background and - grey denoting uncertainty or ambiguity with - light shades of grey denoting likely foreground and - dark shades of grey denoting likely background.

-
-
-
- +
+
backgroundSegmentationMask + of type + {{VideoFrameMetadata}}
+
+

A background segmentation mask with + white denoting certainly foreground, + black denoting certainly background and + grey denoting uncertainty or ambiguity with + light shades of grey denoting likely foreground and + dark shades of grey denoting likely background.

+
+
+
From 6593ce055875fb49eb69ec98f07e87949ea5a6df Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Eero=20H=C3=A4kkinen?= Date: Fri, 14 Jun 2024 10:42:04 +0300 Subject: [PATCH 5/5] Add example --- index.html | 47 +++++++++++++++++++++++++++++++++++++++++++++++ 1 file changed, 47 insertions(+) diff --git a/index.html b/index.html index 728b695..c8455c5 100644 --- a/index.html +++ b/index.html @@ -1739,6 +1739,53 @@

Dictionary {{VideoFrameMetadata}} Members

+
+

Example

+ 
+// Open camera.
+const stream = await navigator.mediaDevices.getUserMedia({video: true});
+const [videoTrack] = stream.getVideoTracks();
+
+// Try to enable background segmentation mask.
+const videoCapabilities = videoTrack.getCapabilities();
+if ((videoCapabilities.backgroundSegmentationMask || []).includes(true)) {
+  await videoTrack.applyConstraints({backgroundSegmentationMask: {exact: true}});
+} else {
+  // Background segmentation mask is not supported by the platform or
+  // by the camera. Consider falling back to some other method.
+}
+
+const canvasContext = document.querySelector('canvas').getContext('2d');
+const videoProcessor = new MediaStreamTrackProcessor({track: videoTrack});
+const videoProcessorReader = videoProcessor.readable.getReader();
+
+for (;;) {
+  const {done, value: videoFrame} = await videoProcessorReader.read();
+  if (done)
+    break;
+  const {backgroundSegmentationMask} = videoFrame.metadata();
+  if (backgroundSegmentationMask) {
+    // Draw the video frame.
+    canvasContext.globalCompositeOperation = 'copy';
+    context.drawImage(videoFrame, 0, 0);
+    // Draw (or multiply with) the mask.
+    // The result is the foreground on black.
+    context.globalCompositeOperation = 'multiply';
+    canvasContext.drawImage(backgroundSegmentationMask, 0, 0);
+  }
+  else {
+    // Everything is background. Fill with black.
+    canvasContext.globalCompositeOperation = 'copy';
+    canvasContext.fillStyle = 'black';
+    canvasContext.fillRect(
+      0,
+      0,
+      canvasContext.canvas.width,
+      canvasContext.canvas.height);
+  }
+}
+
+