contentHint should be preserved when transferring a MediaStreamTrack

https://bugs.webkit.org/show_bug.cgi?id=267921
rdar://121516684

Reviewed by Eric Carlson.

We add contentHint as a slot in data holder.
We make sure to set contentHint in MediaStreamTrackPrivate constructor.
We remove the optimization in MediaStreamTrack::create for the document case since this creates two code paths that we would need to keep in sync.

* LayoutTests/http/wpt/mediastream/transfer-mediastreamtrack-to-worker-expected.txt:
* LayoutTests/http/wpt/mediastream/transfer-mediastreamtrack-to-worker.html:
* Source/WebCore/Headers.cmake:
* Source/WebCore/Modules/mediastream/MediaStreamTrack.cpp:
(WebCore::contentHintToAtomString):
(WebCore::MediaStreamTrack::contentHint const):
(WebCore::MediaStreamTrack::setContentHint):
(WebCore::MediaStreamTrack::create):
* Source/WebCore/WebCore.xcodeproj/project.pbxproj:
* Source/WebCore/platform/mediastream/MediaStreamTrackDataHolder.cpp:
(WebCore::MediaStreamTrackDataHolder::MediaStreamTrackDataHolder):
* Source/WebCore/platform/mediastream/MediaStreamTrackDataHolder.h:
* Source/WebCore/platform/mediastream/MediaStreamTrackHintValue.h: Added.
* Source/WebCore/platform/mediastream/MediaStreamTrackPrivate.cpp:
(WebCore::MediaStreamTrackPrivate::MediaStreamTrackPrivate):
(WebCore::MediaStreamTrackPrivate::setContentHint):
(WebCore::MediaStreamTrackPrivate::toDataHolder):
* Source/WebCore/platform/mediastream/MediaStreamTrackPrivate.h:

Canonical link: https://commits.webkit.org/273479@main

LayoutTests/http/wpt/mediastream/transfer-mediastreamtrack-to-worker-expected.txt

@@ -4,4 +4,5 @@ PASS Transferred audio track get muted/unmuted according page state
 PASS Transferred video track get muted/unmuted according page state
 PASS Page state gets updated when transferred tracks get stopped
 PASS Page state gets updated when clones of transferred tracks get stopped
+PASS Content hint is preserved when transferring a track
 

LayoutTests/http/wpt/mediastream/transfer-mediastreamtrack-to-worker.html

@@ -181,6 +181,44 @@
     } while (++videoCounter < 40 && internals.pageMediaState() !== "IsNotPlaying")
     assert_equals(internals.pageMediaState(), "IsNotPlaying");
 }, "Page state gets updated when clones of transferred tracks get stopped");
+
+promise_test(async test => {
+    const worker = await createWorker(`
+        self.onmessage = async (event) => {
+            if (event.data.type === "getVideoTrack") {
+                 const generator = new VideoTrackGenerator();
+                 generator.track.contentHint = event.data.hint;
+                 self.postMessage({ track: generator.track }, [generator.track]);
+                 return;
+            }
+            const track = event.data.track;
+            self.postMessage(track.contentHint);
+        }
+   `);
+    test.add_cleanup(() => worker.terminate());
+
+    const stream = await navigator.mediaDevices.getUserMedia({ audio: true, video: true });
+
+    for (let hint of ["motion", "detail", "text"]) {
+        const track = stream.getVideoTracks()[0].clone();
+        track.contentHint = hint;
+        worker.postMessage({ track }, [track]);
+        assert_equals(await new Promise(resolve => worker.onmessage = e => resolve(e.data)), hint);
+    }
+
+    for (let hint of ["speech", "music"]) {
+        const track = stream.getAudioTracks()[0].clone();
+        track.contentHint = hint;
+        worker.postMessage({ track }, [track]);
+        assert_equals(await new Promise(resolve => worker.onmessage = e => resolve(e.data)), hint);
+    }
+
+    for (let hint of ["motion", "detail", "text"]) {
+        worker.postMessage({ type: "getVideoTrack", hint });
+        const track = await new Promise(resolve => worker.onmessage = e => resolve(e.data.track));
+        assert_equals(track.contentHint, hint);
+    }
+}, "Content hint is preserved when transferring a track");
 </script>
 </body>
 </html>

Source/WebCore/Headers.cmake

@@ -2167,6 +2167,7 @@ set(WebCore_PRIVATE_FRAMEWORK_HEADERS
     platform/mediastream/MediaStreamPrivate.h
     platform/mediastream/MediaStreamRequest.h
     platform/mediastream/MediaStreamTrackDataHolder.h
+    platform/mediastream/MediaStreamTrackHintValue.h
     platform/mediastream/MediaStreamTrackPrivate.h
     platform/mediastream/MeteringMode.h
     platform/mediastream/PhotoCapabilities.h

Source/WebCore/Modules/mediastream/MediaStreamTrack.cpp

@@ -135,20 +135,20 @@ const String& MediaStreamTrack::label() const
     return m_private->label();
 }
 
-static AtomString contentHintToAtomString(MediaStreamTrackPrivate::HintValue hint)
+static AtomString contentHintToAtomString(MediaStreamTrackHintValue hint)
 {
     switch (hint) {
-    case MediaStreamTrackPrivate::HintValue::Empty:
+    case MediaStreamTrackHintValue::Empty:
         return emptyAtom();
-    case MediaStreamTrackPrivate::HintValue::Speech:
+    case MediaStreamTrackHintValue::Speech:
         return "speech"_s;
-    case MediaStreamTrackPrivate::HintValue::Music:
+    case MediaStreamTrackHintValue::Music:
         return "music"_s;
-    case MediaStreamTrackPrivate::HintValue::Motion:
+    case MediaStreamTrackHintValue::Motion:
         return "motion"_s;
-    case MediaStreamTrackPrivate::HintValue::Detail:
+    case MediaStreamTrackHintValue::Detail:
         return "detail"_s;
-    case MediaStreamTrackPrivate::HintValue::Text:
+    case MediaStreamTrackHintValue::Text:
         return "text"_s;
     default:
         return emptyAtom();
@@ -159,30 +159,31 @@ const AtomString& MediaStreamTrack::contentHint() const
 {
     if (m_contentHint.isNull())
         m_contentHint = contentHintToAtomString(m_private->contentHint());
+
     return m_contentHint;
 }
 
 void MediaStreamTrack::setContentHint(const String& hintValue)
 {
-    MediaStreamTrackPrivate::HintValue value;
+    MediaStreamTrackHintValue value;
     if (m_private->isAudio()) {
         if (hintValue.isEmpty())
-            value = MediaStreamTrackPrivate::HintValue::Empty;
+            value = MediaStreamTrackHintValue::Empty;
         else if (hintValue == "speech"_s)
-            value = MediaStreamTrackPrivate::HintValue::Speech;
+            value = MediaStreamTrackHintValue::Speech;
         else if (hintValue == "music"_s)
-            value = MediaStreamTrackPrivate::HintValue::Music;
+            value = MediaStreamTrackHintValue::Music;
         else
             return;
     } else {
         if (hintValue.isEmpty())
-            value = MediaStreamTrackPrivate::HintValue::Empty;
+            value = MediaStreamTrackHintValue::Empty;
         else if (hintValue == "detail"_s)
-            value = MediaStreamTrackPrivate::HintValue::Detail;
+            value = MediaStreamTrackHintValue::Detail;
         else if (hintValue == "motion"_s)
-            value = MediaStreamTrackPrivate::HintValue::Motion;
+            value = MediaStreamTrackHintValue::Motion;
         else if (hintValue == "text"_s)
-            value = MediaStreamTrackPrivate::HintValue::Text;
+            value = MediaStreamTrackHintValue::Text;
         else
             return;
     }
@@ -629,18 +630,13 @@ UniqueRef<MediaStreamTrackDataHolder> MediaStreamTrack::detach()
 
 Ref<MediaStreamTrack> MediaStreamTrack::create(ScriptExecutionContext& context, UniqueRef<MediaStreamTrackDataHolder>&& dataHolder)
 {
-    RefPtr<MediaStreamTrackPrivate> privateTrack;
-    if (RefPtr document = dynamicDowncast<Document>(context))
-        privateTrack = MediaStreamTrackPrivate::create(document->logger(), WTFMove(dataHolder->source), WTFMove(dataHolder->trackId));
-    else {
-        privateTrack = MediaStreamTrackPrivate::create(Logger::create(&context), WTFMove(dataHolder), [identifier = context.identifier()](Function<void()>&& task) {
-            ScriptExecutionContext::postTaskTo(identifier, [task = WTFMove(task)] (auto&) mutable {
-                task();
-            });
+    auto privateTrack = MediaStreamTrackPrivate::create(Logger::create(&context), WTFMove(dataHolder), [identifier = context.identifier()](Function<void()>&& task) {
+        ScriptExecutionContext::postTaskTo(identifier, [task = WTFMove(task)] (auto&) mutable {
+            task();
         });
-    }
+    });
 
-    return MediaStreamTrack::create(context, privateTrack.releaseNonNull(), RegisterCaptureTrackToOwner::No);
+    return MediaStreamTrack::create(context, WTFMove(privateTrack), RegisterCaptureTrackToOwner::No);
 }
 
 #if !RELEASE_LOG_DISABLED

Source/WebCore/WebCore.xcodeproj/project.pbxproj

@@ -1415,6 +1415,7 @@
 		41860F0F2847A5BE00E4A395 /* ExtendableEventInit.h in Headers */ = {isa = PBXBuildFile; fileRef = 4131F3B41F955BC30059995A /* ExtendableEventInit.h */; settings = {ATTRIBUTES = (Private, ); }; };
 		41878FAD27294721002E1EDD /* BaseAudioMediaStreamTrackRendererUnit.h in Headers */ = {isa = PBXBuildFile; fileRef = 41878FA827282055002E1EDD /* BaseAudioMediaStreamTrackRendererUnit.h */; settings = {ATTRIBUTES = (Private, ); }; };
 		41885B9311B6FDA6003383BB /* FormSubmission.h in Headers */ = {isa = PBXBuildFile; fileRef = 41885B9111B6FDA6003383BB /* FormSubmission.h */; settings = {ATTRIBUTES = (Private, ); }; };
+		418882562B613A0300F1254F /* MediaStreamTrackHintValue.h in Headers */ = {isa = PBXBuildFile; fileRef = 418882552B613A0300F1254F /* MediaStreamTrackHintValue.h */; settings = {ATTRIBUTES = (Private, ); }; };
 		4188D55C26C54D6F004858C8 /* RTCDtlsTransportBackend.h in Headers */ = {isa = PBXBuildFile; fileRef = 4188D55A26C54D6F004858C8 /* RTCDtlsTransportBackend.h */; };
 		4188D56326C551B1004858C8 /* RTCDtlsTransportState.h in Headers */ = {isa = PBXBuildFile; fileRef = 4188D56226C551B1004858C8 /* RTCDtlsTransportState.h */; };
 		418938B22429F9B4007FDC41 /* RetrieveRecordsOptions.h in Headers */ = {isa = PBXBuildFile; fileRef = 418938AF2429F9AB007FDC41 /* RetrieveRecordsOptions.h */; settings = {ATTRIBUTES = (Private, ); }; };
@@ -9965,6 +9966,7 @@
 		418807E124E458C300DDAF94 /* AbortAlgorithm.h */ = {isa = PBXFileReference; lastKnownFileType = sourcecode.c.h; path = AbortAlgorithm.h; sourceTree = "<group>"; };
 		41885B9111B6FDA6003383BB /* FormSubmission.h */ = {isa = PBXFileReference; fileEncoding = 4; lastKnownFileType = sourcecode.c.h; path = FormSubmission.h; sourceTree = "<group>"; };
 		41885B9211B6FDA6003383BB /* FormSubmission.cpp */ = {isa = PBXFileReference; fileEncoding = 4; lastKnownFileType = sourcecode.cpp.cpp; path = FormSubmission.cpp; sourceTree = "<group>"; };
+		418882552B613A0300F1254F /* MediaStreamTrackHintValue.h */ = {isa = PBXFileReference; fileEncoding = 4; lastKnownFileType = sourcecode.c.h; path = MediaStreamTrackHintValue.h; sourceTree = "<group>"; };
 		4188D55A26C54D6F004858C8 /* RTCDtlsTransportBackend.h */ = {isa = PBXFileReference; fileEncoding = 4; lastKnownFileType = sourcecode.c.h; path = RTCDtlsTransportBackend.h; sourceTree = "<group>"; };
 		4188D55D26C5511A004858C8 /* RTCDtlsTransport.idl */ = {isa = PBXFileReference; lastKnownFileType = text; path = RTCDtlsTransport.idl; sourceTree = "<group>"; };
 		4188D55E26C5511A004858C8 /* RTCDtlsTransport.h */ = {isa = PBXFileReference; lastKnownFileType = sourcecode.c.h; path = RTCDtlsTransport.h; sourceTree = "<group>"; };
@@ -20910,6 +20912,7 @@
 				078E3CBE1FE19F3000483C1D /* MediaStreamRequest.h */,
 				410398A42B593B7000C9E03D /* MediaStreamTrackDataHolder.cpp */,
 				410398A52B593B7000C9E03D /* MediaStreamTrackDataHolder.h */,
+				418882552B613A0300F1254F /* MediaStreamTrackHintValue.h */,
 				07FFDE66181AED420072D409 /* MediaStreamTrackPrivate.cpp */,
 				07FFDE67181AED420072D409 /* MediaStreamTrackPrivate.h */,
 				073930DB2AC6452700C1D1B1 /* MeteringMode.h */,
@@ -40592,6 +40595,7 @@
 				078E091817D14D1C00420AA1 /* MediaStreamTrack.h in Headers */,
 				410398A72B593B7100C9E03D /* MediaStreamTrackDataHolder.h in Headers */,
 				078E091917D14D1C00420AA1 /* MediaStreamTrackEvent.h in Headers */,
+				418882562B613A0300F1254F /* MediaStreamTrackHintValue.h in Headers */,
 				07FFDE69181AED420072D409 /* MediaStreamTrackPrivate.h in Headers */,
 				525B0CB12A9F8FD100E8A94B /* MediationRequirement.h in Headers */,
 				416F429C29C0C19700C080DC /* MediaTrackCapabilities.h in Headers */,

Source/WebCore/platform/mediastream/MediaStreamTrackDataHolder.cpp

@@ -71,16 +71,17 @@ class PreventSourceFromEndingObserverWrapper : public ThreadSafeRefCounted<Preve
     std::unique_ptr<PreventSourceFromEndingObserver> m_observer;
 };
 
-MediaStreamTrackDataHolder::MediaStreamTrackDataHolder(bool isProducingData, bool enabled, bool ended, bool muted, bool interrupted, String&& trackId, String&& label, RealtimeMediaSource::Type type, CaptureDevice::DeviceType deviceType, RealtimeMediaSourceSettings&& settings, RealtimeMediaSourceCapabilities&& capabilities, Ref<RealtimeMediaSource>&& source)
-    : isProducingData(isProducingData)
-    , enabled(enabled)
-    , ended(ended)
-    , muted(muted)
-    , interrupted(interrupted)
-    , trackId(WTFMove(trackId))
+MediaStreamTrackDataHolder::MediaStreamTrackDataHolder(String&& trackId, String&& label, RealtimeMediaSource::Type type, CaptureDevice::DeviceType deviceType, bool isEnabled, bool isEnded, MediaStreamTrackHintValue contentHint, bool isProducingData, bool isMuted, bool isInterrupted, RealtimeMediaSourceSettings settings, RealtimeMediaSourceCapabilities capabilities, Ref<RealtimeMediaSource>&& source)
+    : trackId(WTFMove(trackId))
     , label(WTFMove(label))
     , type(type)
     , deviceType(deviceType)
+    , isEnabled(isEnabled)
+    , isEnded(isEnded)
+    , contentHint(contentHint)
+    , isProducingData(isProducingData)
+    , isMuted(isMuted)
+    , isInterrupted(isInterrupted)
     , settings(WTFMove(settings))
     , capabilities(WTFMove(capabilities))
     , source(source.get())

Source/WebCore/platform/mediastream/MediaStreamTrackDataHolder.h

@@ -27,6 +27,7 @@
 #if ENABLE(MEDIA_STREAM)
 
 #include "CaptureDevice.h"
+#include "MediaStreamTrackHintValue.h"
 #include "RealtimeMediaSource.h"
 #include <wtf/FastMalloc.h>
 
@@ -37,21 +38,22 @@ class PreventSourceFromEndingObserverWrapper;
 struct MediaStreamTrackDataHolder {
     WTF_MAKE_STRUCT_FAST_ALLOCATED;
 
-    WEBCORE_EXPORT MediaStreamTrackDataHolder(bool isProducingData, bool enabled, bool ended, bool muted, bool interrupted, String&& trackId, String&& label, RealtimeMediaSource::Type, CaptureDevice::DeviceType, RealtimeMediaSourceSettings&&, RealtimeMediaSourceCapabilities&&, Ref<RealtimeMediaSource>&&);
+    WEBCORE_EXPORT MediaStreamTrackDataHolder(String&& trackId, String&& label, RealtimeMediaSource::Type, CaptureDevice::DeviceType, bool isEnabled, bool isEnded, MediaStreamTrackHintValue, bool isProducingData, bool isMuted, bool isInterrupted, RealtimeMediaSourceSettings, RealtimeMediaSourceCapabilities, Ref<RealtimeMediaSource>&&);
     WEBCORE_EXPORT ~MediaStreamTrackDataHolder();
 
     MediaStreamTrackDataHolder(const MediaStreamTrackDataHolder &) = delete;
     MediaStreamTrackDataHolder &operator=(const MediaStreamTrackDataHolder &) = delete;
 
-    bool isProducingData { false };
-    bool enabled { false };
-    bool ended { false };
-    bool muted { false };
-    bool interrupted { false };
     String trackId;
     String label;
     RealtimeMediaSource::Type type;
     CaptureDevice::DeviceType deviceType;
+    bool isEnabled { true };
+    bool isEnded { false };
+    MediaStreamTrackHintValue contentHint { MediaStreamTrackHintValue::Empty };
+    bool isProducingData { false };
+    bool isMuted { false };
+    bool isInterrupted { false };
     RealtimeMediaSourceSettings settings;
     RealtimeMediaSourceCapabilities capabilities;
     Ref<RealtimeMediaSource> source;

Source/WebCore/platform/mediastream/MediaStreamTrackHintValue.h

@@ -0,0 +1,36 @@
+/*
+ * Copyright (C) 2024 Apple Inc. All rights reserved.
+ *
+ * Redistribution and use in source and binary forms, with or without
+ * modification, are permitted provided that the following conditions
+ * are met:
+ * 1. Redistributions of source code must retain the above copyright
+ *    notice, this list of conditions and the following disclaimer.
+ * 2. Redistributions in binary form must reproduce the above copyright
+ *    notice, this list of conditions and the following disclaimer in the
+ *    documentation and/or other materials provided with the distribution.
+ *
+ * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS
+ * "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT
+ * LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR
+ * A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT
+ * OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL,
+ * SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT
+ * LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE,
+ * DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY
+ * THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
+ * (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE
+ * OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
+ */
+
+#pragma once
+
+#if ENABLE(MEDIA_STREAM)
+
+namespace WebCore {
+
+enum class MediaStreamTrackHintValue : uint8_t { Empty, Speech, Music, Motion, Detail, Text };
+
+} // namespace WebCore
+
+#endif // ENABLE(MEDIA_STREAM)

Source/WebCore/platform/mediastream/MediaStreamTrackPrivate.cpp

@@ -338,13 +338,14 @@ MediaStreamTrackPrivate::MediaStreamTrackPrivate(Ref<const Logger>&& logger, Uni
     , m_deviceType(dataHolder->deviceType)
     , m_isCaptureTrack(false)
     , m_captureDidFail(false)
+    , m_contentHint(dataHolder->contentHint)
     , m_logger(WTFMove(logger))
 #if !RELEASE_LOG_DISABLED
     , m_logIdentifier(uniqueLogIdentifier())
 #endif
     , m_isProducingData(dataHolder->isProducingData)
-    , m_isMuted(dataHolder->muted)
-    , m_isInterrupted(dataHolder->interrupted)
+    , m_isMuted(dataHolder->isMuted)
+    , m_isInterrupted(dataHolder->isInterrupted)
     , m_settings(WTFMove(dataHolder->settings))
     , m_capabilities(WTFMove(dataHolder->capabilities))
 #if ASSERT_ENABLED
@@ -394,7 +395,7 @@ void MediaStreamTrackPrivate::removeObserver(MediaStreamTrackPrivate::Observer&
     m_observers.remove(observer);
 }
 
-void MediaStreamTrackPrivate::setContentHint(HintValue hintValue)
+void MediaStreamTrackPrivate::setContentHint(MediaStreamTrackHintValue hintValue)
 {
     m_contentHint = hintValue;
 }
@@ -648,15 +649,16 @@ void MediaStreamTrackPrivate::updateReadyState()
 UniqueRef<MediaStreamTrackDataHolder> MediaStreamTrackPrivate::toDataHolder()
 {
     return makeUniqueRef<MediaStreamTrackDataHolder>(
-        m_isProducingData,
-        m_isEnabled,
-        m_isEnded,
-        m_isMuted,
-        m_isInterrupted,
         m_id.isolatedCopy(),
         m_label.isolatedCopy(),
         m_type,
         m_deviceType,
+        m_isEnabled,
+        m_isEnded,
+        m_contentHint,
+        m_isProducingData,
+        m_isMuted,
+        m_isInterrupted,
         m_settings.isolatedCopy(),
         m_capabilities.isolatedCopy(),
         Ref { m_sourceObserver->source() });

Source/WebCore/platform/mediastream/MediaStreamTrackPrivate.h

@@ -29,6 +29,7 @@
 
 #if ENABLE(MEDIA_STREAM)
 
+#include "MediaStreamTrackHintValue.h"
 #include "RealtimeMediaSource.h"
 #include <wtf/LoggerHelper.h>
 #include <wtf/RefCounted.h>
@@ -78,9 +79,8 @@ class MediaStreamTrackPrivate final
 
     bool ended() const { return m_isEnded; }
 
-    enum class HintValue { Empty, Speech, Music, Motion, Detail, Text };
-    HintValue contentHint() const { return m_contentHint; }
-    void setContentHint(HintValue);
+    MediaStreamTrackHintValue contentHint() const { return m_contentHint; }
+    void setContentHint(MediaStreamTrackHintValue);
 
     void startProducingData();
     void stopProducingData();
@@ -186,7 +186,7 @@ class MediaStreamTrackPrivate final
     bool m_isEnded { false };
     bool m_captureDidFail { false };
     bool m_hasStartedProducingData { false };
-    HintValue m_contentHint { HintValue::Empty };
+    MediaStreamTrackHintValue m_contentHint { MediaStreamTrackHintValue::Empty };
     Ref<const Logger> m_logger;
 #if !RELEASE_LOG_DISABLED
     const void* m_logIdentifier;