Ruby in WebVTT are not rendered correctly

https://bugs.webkit.org/show_bug.cgi?id=262064 rdar://116011365 Reviewed by Jer Noble and Eric Carlson. This patch creates a new class WebVTTRubyElement, a subclass of WebVTTElementImpl And RubyElement. Now, each ruby tag in a VTT is associated with a WebVTTRubyElement, Which has access to RubyElement::createElementRenderer() to render it appropriately. Added a layout test to check that ruby elements in WebVTT cues are positioned correctly. * LayoutTests/media/track/captions-webvtt/ruby.vtt: Added. * LayoutTests/media/track/webvtt-ruby.html: Added. * LayoutTests/media/track/webvtt-ruby-expected.txt: Added. * Source/WebCore/css/SelectorCheckerTestFunctions.h: (WebCore::matchesLangPseudoClass): (WebCore::matchesFutureCuePseudoClass): (WebCore::matchesPastCuePseudoClass): * Source/WebCore/dom/Node.h: (WebCore::Node::isWebVTTRubyElement const): (WebCore::Node::isWebVTTRubyTextElement const): * Source/WebCore/html/RubyElement.h: (): Deleted. * Source/WebCore/html/RubyTextElement.h: (): Deleted. * Source/WebCore/html/track/VTTCue.cpp: (WebCore::VTTCue::markFutureAndPastNodes): * Source/WebCore/html/track/WebVTTElement.cpp: (WebCore::WebVTTElement::WebVTTElement): (WebCore::WebVTTElementImpl::create): (WebCore::WebVTTElementImpl::cloneElementWithoutAttributesAndChildren): (WebCore::WebVTTElementImpl::createEquivalentHTMLElement): (WebCore::WebVTTElement::create): Deleted. (WebCore::WebVTTElement::cloneElementWithoutAttributesAndChildren): Deleted. (WebCore::WebVTTElement::createEquivalentHTMLElement): Deleted. * Source/WebCore/html/track/WebVTTElement.h: (WebCore::WebVTTElementImpl::WebVTTElementImpl): (isType): * Source/WebCore/html/track/WebVTTParser.cpp: (WebCore::WebVTTTreeBuilder::currentType const): (WebCore::WebVTTTreeBuilder::buildFromString): (WebCore::WebVTTTreeBuilder::constructTreeFromToken): * Source/WebCore/page/CaptionUserPreferencesMediaAF.cpp: (WebCore::CaptionUserPreferencesMediaAF::captionsStyleSheetOverride const): * Source/WebCore/style/ElementRuleCollector.cpp: (WebCore::Style::ElementRuleCollector::collectMatchingShadowPseudoElementRules): Canonical link: https://commits.webkit.org/268746@main
WebKit · Oct 2, 2023 · def69bf · def69bf
1 parent 81820fa
commit def69bf
Show file tree

Hide file tree

Showing 13 changed files with 218 additions and 52 deletions.
diff --git a/LayoutTests/media/track/captions-webvtt/ruby.vtt b/LayoutTests/media/track/captions-webvtt/ruby.vtt
@@ -0,0 +1,4 @@
+WEBVTT
+
+00:00:00.000 --> 00:00:05.000 align:left
+This is a ruby <ruby>base<rt>ruby</rt></ruby>
diff --git a/LayoutTests/media/track/webvtt-ruby-expected.txt b/LayoutTests/media/track/webvtt-ruby-expected.txt
@@ -0,0 +1,12 @@
+
+EVENT(canplay)
+EVENT(addtrack)
+EXPECTED (video.textTracks.length == '1') OK
+RUN(video.textTracks[0].mode = 'showing')
+RUN(video.currentTime = 1)
+EVENT(seeked)
+EXPECTED (window.internals.shadowRoot(video).querySelector('rt') != 'null') OK
+EXPECTED (rubyText.offsetTop < rubyBase.offsetTop == 'true') OK
+EXPECTED (rubyBase.offsetTop == (rubyText.offsetTop + rubyText.offsetHeight) == 'true') OK
+END OF TEST
+
diff --git a/LayoutTests/media/track/webvtt-ruby.html b/LayoutTests/media/track/webvtt-ruby.html
@@ -0,0 +1,38 @@
+<!doctype html>
+<html>
+    <head>
+        <title>WebVTTRubyText elements should appear above WebVTTRuby elements</title>
+        <script src=../../resources/js-test-pre.js></script>
+        <script src=../video-test.js></script>
+        <script src=../media-file.js></script>
+        <script>
+            async function runTest()
+            {
+                video = document.getElementById('video');
+                video.src = findMediaFile('video', '../content/test');
+                await waitFor(video, 'canplay');
+                let track = document.createElement('track');
+                track.src = 'captions-webvtt/ruby.vtt';
+                video.appendChild(track)
+
+                await waitFor(video.textTracks, 'addtrack');
+                testExpected("video.textTracks.length", 1);
+                run("video.textTracks[0].mode = 'showing'");
+
+                run("video.currentTime = 1");
+                await waitFor(video, 'seeked');
+
+                window.internals.ensureUserAgentShadowRoot(video);
+                await testExpectedEventually("window.internals.shadowRoot(video).querySelector('rt')", null, "!=", 1000);
+                rubyBase = window.internals.shadowRoot(video).querySelector('ruby');
+                rubyText = window.internals.shadowRoot(video).querySelector('rt');
+                await testExpected("rubyText.offsetTop < rubyBase.offsetTop", true);
+                await testExpected("rubyBase.offsetTop == (rubyText.offsetTop + rubyText.offsetHeight)", true);
+                endTest();
+            }
+        </script>
+    </head>
+    <body onload="runTest()">
+        <video id="video" width="320px" height="240px" paused></video>
+    </body>
+</html>
diff --git a/Source/WebCore/css/SelectorCheckerTestFunctions.h b/Source/WebCore/css/SelectorCheckerTestFunctions.h
@@ -206,6 +206,10 @@ ALWAYS_INLINE bool matchesLangPseudoClass(const Element& element, const FixedVec
 #if ENABLE(VIDEO)
     if (is<WebVTTElement>(element))
         language = downcast<WebVTTElement>(element).language();
+    else if (is<WebVTTRubyElement>(element))
+        language = downcast<WebVTTRubyElement>(element).language();
+    else if (is<WebVTTRubyTextElement>(element))
+        language = downcast<WebVTTRubyTextElement>(element).language();
     else
 #endif
         language = element.effectiveLang();
@@ -469,12 +473,24 @@ ALWAYS_INLINE bool matchesPictureInPicturePseudoClass(const Element& element)
 
 ALWAYS_INLINE bool matchesFutureCuePseudoClass(const Element& element)
 {
-    return is<WebVTTElement>(element) && !downcast<WebVTTElement>(element).isPastNode();
+    if (auto* webVTTElement = dynamicDowncast<WebVTTElement>(element))
+        return !webVTTElement->isPastNode();
+    if (auto* webVTTRubyElement = dynamicDowncast<WebVTTRubyElement>(element))
+        return !webVTTRubyElement->isPastNode();
+    if (auto* webVTTRubyTextElement = dynamicDowncast<WebVTTRubyTextElement>(element))
+        return !webVTTRubyTextElement->isPastNode();
+    return false;
 }
 
 ALWAYS_INLINE bool matchesPastCuePseudoClass(const Element& element)
 {
-    return is<WebVTTElement>(element) && downcast<WebVTTElement>(element).isPastNode();
+    if (is<WebVTTElement>(element))
+        return downcast<WebVTTElement>(element).isPastNode();
+    if (is<WebVTTRubyElement>(element))
+        return downcast<WebVTTRubyElement>(element).isPastNode();
+    if (is<WebVTTRubyTextElement>(element))
+        return downcast<WebVTTRubyTextElement>(element).isPastNode();
+    return false;
 }
 
 ALWAYS_INLINE bool matchesPlayingPseudoClass(const Element& element)

diff --git a/Source/WebCore/dom/Node.h b/Source/WebCore/dom/Node.h
@@ -220,6 +220,8 @@ class Node : public EventTarget {
 
 #if ENABLE(VIDEO)
     virtual bool isWebVTTElement() const { return false; }
+    virtual bool isWebVTTRubyElement() const { return false; }
+    virtual bool isWebVTTRubyTextElement() const { return false; }
 #endif
     bool isStyledElement() const { return hasNodeFlag(NodeFlag::IsHTMLElement) || hasNodeFlag(NodeFlag::IsSVGElement) || hasNodeFlag(NodeFlag::IsMathMLElement); }
     virtual bool isAttributeNode() const { return false; }

diff --git a/Source/WebCore/html/RubyElement.h b/Source/WebCore/html/RubyElement.h
@@ -29,14 +29,16 @@
 
 namespace WebCore {
 
-class RubyElement final : public HTMLElement {
+class RubyElement : public HTMLElement {
     WTF_MAKE_ISO_ALLOCATED(RubyElement);
 public:
     static Ref<RubyElement> create(Document&);
     static Ref<RubyElement> create(const QualifiedName&, Document&);
 
-private:
+protected:
     RubyElement(const QualifiedName&, Document&);
+
+private:
     RenderPtr<RenderElement> createElementRenderer(RenderStyle&&, const RenderTreePosition&) override;
 };
 

diff --git a/Source/WebCore/html/RubyTextElement.h b/Source/WebCore/html/RubyTextElement.h
@@ -29,14 +29,16 @@
 
 namespace WebCore {
 
-class RubyTextElement final : public HTMLElement {
+class RubyTextElement : public HTMLElement {
     WTF_MAKE_ISO_ALLOCATED(RubyTextElement);
 public:
     static Ref<RubyTextElement> create(Document&);
     static Ref<RubyTextElement> create(const QualifiedName&, Document&);
 
-private:
+protected:
     RubyTextElement(const QualifiedName&, Document&);
+
+private:
     RenderPtr<RenderElement> createElementRenderer(RenderStyle&&, const RenderTreePosition&) override;
 };
 

diff --git a/Source/WebCore/html/track/VTTCue.cpp b/Source/WebCore/html/track/VTTCue.cpp
@@ -944,12 +944,16 @@ void VTTCue::markFutureAndPastNodes(ContainerNode* root, const MediaTime& previo
                 isPastNode = false;
         }
 
-        if (is<WebVTTElement>(*child)) {
+        if (is<WebVTTElement>(*child))
             downcast<WebVTTElement>(*child).setIsPastNode(isPastNode);
-            // Make an elemenet id match a cue id for style matching purposes.
-            if (!id().isEmpty())
-                downcast<WebVTTElement>(*child).setIdAttribute(id());
-        }
+        else if (is<WebVTTRubyElement>(*child))
+            downcast<WebVTTRubyElement>(*child).setIsPastNode(isPastNode);
+        else if (is<WebVTTRubyTextElement>(*child))
+            downcast<WebVTTRubyTextElement>(*child).setIsPastNode(isPastNode);
+
+        // Make an element id match a cue id for style matching purposes.
+        if (!id().isEmpty() && is<Element>(*child))
+            downcast<Element>(*child).setIdAttribute(id());
     }
 }
 

diff --git a/Source/WebCore/html/track/WebVTTElement.cpp b/Source/WebCore/html/track/WebVTTElement.cpp
@@ -1,5 +1,5 @@
 /*
- * Copyright (C) 2013 Apple Inc.  All rights reserved.
+ * Copyright (C) 2013-2023 Apple Inc.  All rights reserved.
  *
  * Redistribution and use in source and binary forms, with or without
  * modification, are permitted provided that the following conditions
@@ -30,6 +30,9 @@
 
 #include "ElementInlines.h"
 #include "HTMLSpanElement.h"
+#include "RenderRuby.h"
+#include "RenderRubyText.h"
+#include "RenderTreePosition.h"
 #include "RubyElement.h"
 #include "RubyTextElement.h"
 #include "TextTrack.h"
@@ -38,6 +41,8 @@
 namespace WebCore {
 
 WTF_MAKE_ISO_ALLOCATED_IMPL(WebVTTElement);
+WTF_MAKE_ISO_ALLOCATED_IMPL(WebVTTRubyElement);
+WTF_MAKE_ISO_ALLOCATED_IMPL(WebVTTRubyTextElement);
 
 static const QualifiedName& nodeTypeToTagName(WebVTTNodeType nodeType)
 {
@@ -73,26 +78,37 @@ static const QualifiedName& nodeTypeToTagName(WebVTTNodeType nodeType)
     }
 }
 
-WebVTTElement::WebVTTElement(WebVTTNodeType nodeType, Document& document)
-    : Element(nodeTypeToTagName(nodeType), document, CreateElement)
-    , m_isPastNode(0)
-    , m_webVTTNodeType(nodeType)
+WebVTTElement::WebVTTElement(WebVTTNodeType nodeType, AtomString language, Document& document)
+    : WebVTTElementImpl(nodeType, language)
+    , Element(nodeTypeToTagName(nodeType), document, CreateElement)
 {
 }
 
-Ref<WebVTTElement> WebVTTElement::create(WebVTTNodeType nodeType, Document& document)
+Ref<Element> WebVTTElementImpl::create(WebVTTNodeType nodeType, AtomString language, Document& document)
 {
-    return adoptRef(*new WebVTTElement(nodeType, document));
+    switch (nodeType) {
+    default:
+    case WebVTTNodeTypeNone:
+    case WebVTTNodeTypeClass:
+    case WebVTTNodeTypeItalic:
+    case WebVTTNodeTypeLanguage:
+    case WebVTTNodeTypeBold:
+    case WebVTTNodeTypeUnderline:
+    case WebVTTNodeTypeVoice:
+        return adoptRef(*new WebVTTElement(nodeType, language, document));
+    case WebVTTNodeTypeRuby:
+        return adoptRef(*new WebVTTRubyElement(language, document));
+    case WebVTTNodeTypeRubyText:
+        return adoptRef(*new WebVTTRubyTextElement(language, document));
+    }
 }
 
-Ref<Element> WebVTTElement::cloneElementWithoutAttributesAndChildren(Document& targetDocument)
+Ref<Element> WebVTTElementImpl::cloneElementWithoutAttributesAndChildren(Document& targetDocument)
 {
-    Ref<WebVTTElement> clone = create(static_cast<WebVTTNodeType>(m_webVTTNodeType), targetDocument);
-    clone->setLanguage(m_language);
-    return clone;
+    return create(static_cast<WebVTTNodeType>(m_webVTTNodeType), m_language, targetDocument);
 }
 
-Ref<HTMLElement> WebVTTElement::createEquivalentHTMLElement(Document& document)
+Ref<HTMLElement> WebVTTElementImpl::createEquivalentHTMLElement(Document& document)
 {
     RefPtr<HTMLElement> htmlElement;
 
@@ -101,8 +117,8 @@ Ref<HTMLElement> WebVTTElement::createEquivalentHTMLElement(Document& document)
     case WebVTTNodeTypeLanguage:
     case WebVTTNodeTypeVoice:
         htmlElement = HTMLSpanElement::create(document);
-        htmlElement->setAttributeWithoutSynchronization(HTMLNames::titleAttr, attributeWithoutSynchronization(voiceAttributeName()));
-        htmlElement->setAttributeWithoutSynchronization(HTMLNames::langAttr, attributeWithoutSynchronization(langAttributeName()));
+        htmlElement->setAttributeWithoutSynchronization(HTMLNames::titleAttr, toElement().attributeWithoutSynchronization(voiceAttributeName()));
+        htmlElement->setAttributeWithoutSynchronization(HTMLNames::langAttr, toElement().attributeWithoutSynchronization(langAttributeName()));
         break;
     case WebVTTNodeTypeItalic:
         htmlElement = HTMLElement::create(HTMLNames::iTag, document);
@@ -123,7 +139,7 @@ Ref<HTMLElement> WebVTTElement::createEquivalentHTMLElement(Document& document)
 
     ASSERT(htmlElement);
     if (htmlElement)
-        htmlElement->setAttributeWithoutSynchronization(HTMLNames::classAttr, attributeWithoutSynchronization(HTMLNames::classAttr));
+        htmlElement->setAttributeWithoutSynchronization(HTMLNames::classAttr, toElement().attributeWithoutSynchronization(HTMLNames::classAttr));
     return htmlElement.releaseNonNull();
 }
 

diff --git a/Source/WebCore/html/track/WebVTTElement.h b/Source/WebCore/html/track/WebVTTElement.h
@@ -1,5 +1,5 @@
 /*
- * Copyright (C) 2013 Apple Inc.  All rights reserved.
+ * Copyright (C) 2013-2023 Apple Inc.  All rights reserved.
  *
  * Redistribution and use in source and binary forms, with or without
  * modification, are permitted provided that the following conditions
@@ -28,6 +28,8 @@
 #if ENABLE(VIDEO)
 
 #include "HTMLElement.h"
+#include "RubyElement.h"
+#include "RubyTextElement.h"
 #include <wtf/NeverDestroyed.h>
 
 namespace WebCore {
@@ -44,13 +46,14 @@ enum WebVTTNodeType {
     WebVTTNodeTypeVoice
 };
 
-class WebVTTElement final : public Element {
-    WTF_MAKE_ISO_ALLOCATED(WebVTTElement);
+class WebVTTElement;
+
+class WebVTTElementImpl {
 public:
-    static Ref<WebVTTElement> create(const WebVTTNodeType, Document&);
+    static Ref<Element> create(const WebVTTNodeType, AtomString language, Document&);
     Ref<HTMLElement> createEquivalentHTMLElement(Document&);
 
-    Ref<Element> cloneElementWithoutAttributesAndChildren(Document&) override;
+    Ref<Element> cloneElementWithoutAttributesAndChildren(Document&);
 
     void setWebVTTNodeType(WebVTTNodeType type) { m_webVTTNodeType = static_cast<unsigned>(type); }
     WebVTTNodeType webVTTNodeType() const { return static_cast<WebVTTNodeType>(m_webVTTNodeType); }
@@ -66,28 +69,89 @@ class WebVTTElement final : public Element {
         static NeverDestroyed<QualifiedName> voiceAttr(nullAtom(), "voice"_s, nullAtom());
         return voiceAttr;
     }
-    
+
     static const QualifiedName& langAttributeName()
     {
         static NeverDestroyed<QualifiedName> voiceAttr(nullAtom(), "lang"_s, nullAtom());
         return voiceAttr;
     }
 
-private:
-    WebVTTElement(WebVTTNodeType, Document&);
-
-    bool isWebVTTElement() const override { return true; }
+protected:
+    WebVTTElementImpl(WebVTTNodeType nodeType, AtomString language)
+        : m_isPastNode { false }
+        , m_webVTTNodeType { static_cast<unsigned>(nodeType) }
+        , m_language { language }
+    {
+    }
+    virtual ~WebVTTElementImpl() = default;
+    virtual Element& toElement() = 0;
 
     unsigned m_isPastNode : 1;
     unsigned m_webVTTNodeType : 4;
-    
+
     AtomString m_language;
 };
 
+class WebVTTElement final : public WebVTTElementImpl, public Element {
+    WTF_MAKE_ISO_ALLOCATED(WebVTTElement);
+public:
+    Ref<Element> cloneElementWithoutAttributesAndChildren(Document& document) final { return WebVTTElementImpl::cloneElementWithoutAttributesAndChildren(document); }
+
+private:
+    friend class WebVTTElementImpl;
+    WebVTTElement(WebVTTNodeType, AtomString language, Document&);
+
+    bool isWebVTTElement() const final { return true; }
+    Element& toElement() final { return *this; }
+};
+
+class WebVTTRubyElement final : public WebVTTElementImpl, public RubyElement {
+    WTF_MAKE_ISO_ALLOCATED(WebVTTRubyElement);
+public:
+    Ref<Element> cloneElementWithoutAttributesAndChildren(Document& document) final { return WebVTTElementImpl::cloneElementWithoutAttributesAndChildren(document); }
+
+private:
+    friend class WebVTTElementImpl;
+    WebVTTRubyElement(AtomString language, Document& document)
+        : WebVTTElementImpl(WebVTTNodeTypeRuby, language)
+        , RubyElement(HTMLNames::rubyTag, document)
+    {
+    }
+
+    bool isWebVTTRubyElement() const final { return true; }
+    Element& toElement() final { return *this; }
+};
+
+class WebVTTRubyTextElement final : public WebVTTElementImpl, public RubyTextElement {
+    WTF_MAKE_ISO_ALLOCATED(WebVTTRubyTextElement);
+public:
+    Ref<Element> cloneElementWithoutAttributesAndChildren(Document& document) final { return WebVTTElementImpl::cloneElementWithoutAttributesAndChildren(document); }
+
+private:
+    friend class WebVTTElementImpl;
+    WebVTTRubyTextElement(AtomString language, Document& document)
+        : WebVTTElementImpl(WebVTTNodeTypeRubyText, language)
+        , RubyTextElement(HTMLNames::rtTag, document)
+    {
+    }
+
+    bool isWebVTTRubyTextElement() const final { return true; }
+    Element& toElement() final { return *this; }
+};
+
+
 } // namespace WebCore
 
 SPECIALIZE_TYPE_TRAITS_BEGIN(WebCore::WebVTTElement)
     static bool isType(const WebCore::Node& node) { return node.isWebVTTElement(); }
 SPECIALIZE_TYPE_TRAITS_END()
 
+SPECIALIZE_TYPE_TRAITS_BEGIN(WebCore::WebVTTRubyElement)
+    static bool isType(const WebCore::Node& node) { return node.isWebVTTRubyElement(); }
+SPECIALIZE_TYPE_TRAITS_END()
+
+SPECIALIZE_TYPE_TRAITS_BEGIN(WebCore::WebVTTRubyTextElement)
+    static bool isType(const WebCore::Node& node) { return node.isWebVTTRubyTextElement(); }
+SPECIALIZE_TYPE_TRAITS_END()
+
 #endif // ENABLE(VIDEO)