WebGPU camera video 0-copy import

This enables the camera video 0-copy import by properly plumbing the required video frame metadata and SI usage. Bug: 1258986 Change-Id: Iafa2e122314e95dfa2acd500253b4ee8e43b9aca Reviewed-on: https://chromium-review.googlesource.com/c/chromium/src/+/4211186 Reviewed-by: Ilya Nikolaevskiy <ilnik@chromium.org> Commit-Queue: Jie A Chen <jie.a.chen@intel.com> Cr-Commit-Position: refs/heads/main@{#1100806}
chromium · Feb 3, 2023 · 45cc801 · 45cc801
1 parent 8063ecc
commit 45cc801
Show file tree

Hide file tree

Showing 5 changed files with 53 additions and 8 deletions.
diff --git a/base/trace_event/builtin_categories.h b/base/trace_event/builtin_categories.h
@@ -280,6 +280,7 @@
   X(TRACE_DISABLED_BY_DEFAULT("viz.surface_lifetime"))                   \
   X(TRACE_DISABLED_BY_DEFAULT("viz.triangles"))                          \
   X(TRACE_DISABLED_BY_DEFAULT("webaudio.audionode"))                     \
+  X(TRACE_DISABLED_BY_DEFAULT("webgpu"))                                 \
   X(TRACE_DISABLED_BY_DEFAULT("webrtc"))                                 \
   X(TRACE_DISABLED_BY_DEFAULT("worker.scheduler"))                       \
   X(TRACE_DISABLED_BY_DEFAULT("xr.debug"))

diff --git a/content/test/data/gpu/vc/webgpu_videos_mxn.html b/content/test/data/gpu/vc/webgpu_videos_mxn.html
@@ -31,6 +31,7 @@
     var useLargeSizeVideo = false;
     var enableBackPressureWorkaround = true;
     var codec = 'vp9';
+    var useLocalCamera = false;
 
     async function startMxNVideos() {
       const container = document.getElementById('container');
@@ -85,6 +86,12 @@
       else if (backPressureWorkaroundOption == '1')
         enableBackPressureWorkaround = true;
 
+      const useLocalCameraOption = parsedString['use_local_camera']
+      if (useLocalCameraOption == '0')
+        useLocalCamera = false;
+      else if (useLocalCameraOption == '1')
+        useLocalCamera = true;
+
       // Get the number of video rows and columns from the string.
       var videoRows = parsedString['rows'];
       var videoColumns = parsedString['columns'];
@@ -127,8 +134,27 @@
       // For the small video at the upper right corner.
       videos[videoRows * videoColumns].width = videoWidth / 3;
       videos[videoRows * videoColumns].height = videoHeight / 3;
-      // Use the 640x360 source to simulate the local camera.
-      videos[videoRows * videoColumns].src = GetVideoSource(1, 1, codec);
+      if (useLocalCamera) {
+        const video = videos[videoRows * videoColumns];
+        video.src = null;
+        const constraints = {
+          audio: false,
+          video: { width: 640, height: 360 }
+        };
+        await navigator.mediaDevices.getUserMedia(constraints)
+          .then((mediaStream) => {
+            video.srcObject = mediaStream;
+            video.onloadedmetadata = () => {
+              video.play();
+            };
+          })
+          .catch((err) => {
+            console.error(`${err.name}: ${err.message}`);
+          });
+      } else {
+        // Use the 640x360 source to simulate the local camera.
+        videos[videoRows * videoColumns].src = GetVideoSource(1, 1, codec);
+      }
 
       await Promise.all(videos.map(video => video.play()));
       p.remove();

diff --git a/third_party/blink/renderer/modules/webgpu/external_texture_helper.cc b/third_party/blink/renderer/modules/webgpu/external_texture_helper.cc
@@ -18,6 +18,7 @@
 #include "third_party/blink/renderer/platform/graphics/gpu/shared_gpu_context.h"
 #include "third_party/blink/renderer/platform/graphics/gpu/webgpu_mailbox_texture.h"
 #include "third_party/blink/renderer/platform/graphics/video_frame_image_util.h"
+#include "third_party/blink/renderer/platform/instrumentation/tracing/trace_event.h"
 #include "third_party/skia/include/effects/SkColorMatrix.h"
 #include "third_party/skia/modules/skcms/skcms.h"
 
@@ -200,11 +201,16 @@ ExternalTexture CreateExternalTexture(
   external_texture_desc.visibleSize = {
       static_cast<uint32_t>(visible_rect.width()),
       static_cast<uint32_t>(visible_rect.height())};
-
-  if (media_video_frame->HasTextures() &&
-      (media_video_frame->format() == media::PIXEL_FORMAT_NV12) &&
-      device_support_zero_copy &&
-      media_video_frame->metadata().is_webgpu_compatible) {
+  const bool zero_copy =
+      (media_video_frame->HasTextures() &&
+       (media_video_frame->format() == media::PIXEL_FORMAT_NV12) &&
+       device_support_zero_copy &&
+       media_video_frame->metadata().is_webgpu_compatible);
+  TRACE_EVENT_INSTANT2(TRACE_DISABLED_BY_DEFAULT("webgpu"),
+                       "CreateExternalTexture", TRACE_EVENT_SCOPE_THREAD,
+                       "zero_copy", !!zero_copy, "video_frame",
+                       media_video_frame->AsHumanReadableString());
+  if (zero_copy) {
     scoped_refptr<WebGPUMailboxTexture> mailbox_texture =
         WebGPUMailboxTexture::FromVideoFrame(
             device->GetDawnControlClient(), device->GetHandle(),

diff --git a/third_party/blink/renderer/platform/video_capture/video_capture_impl.cc b/third_party/blink/renderer/platform/video_capture/video_capture_impl.cc
@@ -433,7 +433,10 @@ bool VideoCaptureImpl::VideoFrameBufferPreparer::Initialize() {
       CHECK(buffer_context_->GetGpuMemoryBuffer());
 
       auto buffer_handle = buffer_context_->GetGpuMemoryBuffer()->CloneHandle();
-
+#if BUILDFLAG(IS_CHROMEOS)
+      is_webgpu_compatible_ =
+          buffer_handle.native_pixmap_handle.supports_zero_copy_webgpu_import;
+#endif
       // No need to propagate shared memory region further as it's already
       // exposed by |buffer_context_->data()|.
       buffer_handle.region = base::UnsafeSharedMemoryRegion();
@@ -526,6 +529,9 @@ bool VideoCaptureImpl::VideoFrameBufferPreparer::BindVideoFrameOnMediaThread(
 #if BUILDFLAG(IS_MAC)
   usage |= gpu::SHARED_IMAGE_USAGE_MACOS_VIDEO_TOOLBOX;
 #endif
+#if BUILDFLAG(IS_CHROMEOS)
+  usage |= gpu::SHARED_IMAGE_USAGE_WEBGPU;
+#endif
 
   if (base::FeatureList::IsEnabled(
           media::kMultiPlaneVideoCaptureSharedImages)) {
@@ -577,6 +583,9 @@ bool VideoCaptureImpl::VideoFrameBufferPreparer::BindVideoFrameOnMediaThread(
   }
   frame_->metadata().allow_overlay = true;
   frame_->metadata().read_lock_fences_enabled = true;
+#if BUILDFLAG(IS_CHROMEOS)
+  frame_->metadata().is_webgpu_compatible = is_webgpu_compatible_;
+#endif
   return true;
 }
 

diff --git a/third_party/blink/renderer/platform/video_capture/video_capture_impl.h b/third_party/blink/renderer/platform/video_capture/video_capture_impl.h
@@ -177,6 +177,9 @@ class PLATFORM_EXPORT VideoCaptureImpl
     scoped_refptr<BufferContext> buffer_context_;
     scoped_refptr<media::VideoFrame> frame_;
     std::unique_ptr<gfx::GpuMemoryBuffer> gpu_memory_buffer_;
+#if BUILDFLAG(IS_CHROMEOS)
+    bool is_webgpu_compatible_ = false;
+#endif
   };
 
   // Contains information about a video capture client, including capture