feat(selectors): deep selector which pierces open shadow roots (#1738)

docs/selectors.md

@@ -61,7 +61,7 @@ const handle = await divHandle.$('css=span');
 
 CSS engine is equivalent to [`Document.querySelector`](https://developer.mozilla.org/en/docs/Web/API/Document/querySelector). Example: `css=.article > span:nth-child(2) li`.
 
-> **NOTE** Malformed selector not starting with `//` nor with `#` is automatically transformed to css selector. For example, Playwright converts `page.$('span > button')` to `page.$('css=span > button')`. Selectors starting with `#` are converted to [text](#text). Selectors starting with `//` are converted to [xpath](#xpath).
+> **NOTE** Malformed selector not starting with `//` nor with `"` is automatically transformed to css selector. For example, Playwright converts `page.$('span > button')` to `page.$('css=span > button')`. Selectors starting with `"` are converted to [text](#text). Selectors starting with `//` are converted to [xpath](#xpath).
 
 ### xpath
 
@@ -76,17 +76,52 @@ Text engine finds an element that contains a text node with passed text. Example
 - Text body can be escaped with double quotes for precise matching, insisting on exact match, including specified whitespace and case. This means `text="Login "` will only match `<button>Login </button>` with exactly one space after "Login".
 - Text body can also be a JavaScript-like regex wrapped in `/` symbols. This means `text=/^\\s*Login$/i` will match `<button> loGIN</button>` with any number of spaces before "Login" and no spaces after.
 
-> **NOTE** Text engine searches for elements in the light dom in the iteration order, and then recursively inside open shadow roots in the iteration order. It does not search inside closed shadow roots or iframes.
+> **NOTE** Text engine first searches for elements in the light dom in the iteration order, and then recursively inside open shadow roots in the iteration order. It does not search inside closed shadow roots or iframes.
 
 > **NOTE** Input elements of the type `button` and `submit` are rendered with their value as text, and text engine finds them. For example, `text=Login` matches `<input type=button value="Login">`.
 
 > **NOTE** Malformed selector starting with `"` is automatically transformed to text selector. For example, Playwright converts `page.click('"Login"')` to `page.click('text="Login"')`.
 
+### deep
+
+Deep engine is equivalent to CSS, but with every [Descendant combinator](https://developer.mozilla.org/en-US/docs/Web/CSS/Descendant_combinator) piercing open shadow roots, including the implicit descendant combinator at the start of the selector. [See this article](https://developer.mozilla.org/en-US/docs/Web/Web_Components/Using_shadow_DOM) for high-level overview of Shadow DOM.
+
+```html
+<article>
+  <div>In the light dom</div>
+  <div slot='myslot'>In the light dom, but goes into the shadow slot</div>
+  <open mode shadow root>
+    <div class='in-the-shadow'>
+      <span class='content'>
+        In the shadow dom
+        <open mode shadow root>
+          <li id='target'>Deep in the shadow</li>
+        </open mode shadow root>
+      </span>
+    </div>
+    <slot name='myslot'></slot>
+  </open mode shadow root>
+</article>
+```
+
+Note that `<open mode shadow root>` is not an html element, but rather a shadow root created with `element.attachShadow({mode: 'open'})`.
+
+- `"deep=article div"` matches the first `<div>In the light dom</div>`
+- `"deep=article > div"` matches two `div` elements that are direct children of the `article`
+- `"deep=article .in-the-shadow"` matches the `<div class='in-the-shadow'>`, piercing the shadow root
+- `"deep=article div > span"` matches the `<span class='content'>`, piercing the shadow root
+- `"deep=article > .in-the-shadow"` does not match anything, because `<div class='in-the-shadow'>` is not a direct child of `article`
+- `"deep=article li#target"` matches the `<li id='target'>Deep in the shadow</li>`, piercing two shadow roots
+
+> **NOTE** Only use deep engine if you need to pierce shadow roots. Otherwise, prefer the more effective CSS engine.
+
+> **NOTE** Deep engine first searches for elements in the light dom in the iteration order, and then recursively inside open shadow roots in the iteration order. It does not search inside closed shadow roots or iframes.
+
 ### id, data-testid, data-test-id, data-test
 
-Attribute engines are selecting based on the corresponding atrribute value. For example: `data-test-id=foo` is similar to `querySelector('*[data-test-id=foo]')`.
+Attribute engines are selecting based on the corresponding atrribute value. For example: `data-test-id=foo` is similar to `deep=[data-test-id="foo"]`.
 
-> **NOTE** Attribute engine searches for elements in the light dom in the iteration order, and then recursively inside open shadow roots in the iteration order. It does not search inside closed shadow roots or iframes.
+> **NOTE** Attribute engine first searches for elements in the light dom in the iteration order, and then recursively inside open shadow roots in the iteration order. It does not search inside closed shadow roots or iframes.
 
 ## Custom selector engines
 

src/injected/deepSelectorEngine.ts

@@ -0,0 +1,193 @@
+/**
+ * Copyright (c) Microsoft Corporation.
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+import { SelectorEngine, SelectorRoot } from './selectorEngine';
+
+export const DeepEngine: SelectorEngine = {
+  create(root: SelectorRoot, targetElement: Element): string | undefined {
+    return;
+  },
+
+  query(root: SelectorRoot, selector: string): Element | undefined {
+    const simple = root.querySelector(selector);
+    if (simple)
+      return simple;
+    const parts = split(selector);
+    if (!parts.length)
+      return;
+    parts.reverse();
+    return queryInternal(root, root, parts);
+  },
+
+  queryAll(root: SelectorRoot, selector: string): Element[] {
+    const result: Element[] = [];
+    const parts = split(selector);
+    if (parts.length) {
+      parts.reverse();
+      queryAllInternal(root, root, parts, result);
+    }
+    return result;
+  }
+};
+
+function queryInternal(boundary: SelectorRoot, root: SelectorRoot, parts: string[]): Element | undefined {
+  const matching = root.querySelectorAll(parts[0]);
+  for (let i = 0; i < matching.length; i++) {
+    const element = matching[i];
+    if (parts.length === 1 || matches(element, parts, boundary))
+      return element;
+  }
+  if ((root as Element).shadowRoot) {
+    const child = queryInternal(boundary, (root as Element).shadowRoot!, parts);
+    if (child)
+      return child;
+  }
+  const elements = root.querySelectorAll('*');
+  for (let i = 0; i < elements.length; i++) {
+    const element = elements[i];
+    if (element.shadowRoot) {
+      const child = queryInternal(boundary, element.shadowRoot, parts);
+      if (child)
+        return child;
+    }
+  }
+}
+
+function queryAllInternal(boundary: SelectorRoot, root: SelectorRoot, parts: string[], result: Element[]) {
+  const matching = root.querySelectorAll(parts[0]);
+  for (let i = 0; i < matching.length; i++) {
+    const element = matching[i];
+    if (parts.length === 1 || matches(element, parts, boundary))
+      result.push(element);
+  }
+  if ((root as Element).shadowRoot)
+    queryAllInternal(boundary, (root as Element).shadowRoot!, parts, result);
+  const elements = root.querySelectorAll('*');
+  for (let i = 0; i < elements.length; i++) {
+    const element = elements[i];
+    if (element.shadowRoot)
+      queryAllInternal(boundary, element.shadowRoot, parts, result);
+  }
+}
+
+function matches(element: Element | undefined, parts: string[], boundary: SelectorRoot): boolean {
+  let i = 1;
+  while (i < parts.length && (element = parentElementOrShadowHost(element!)) && element !== boundary) {
+    if (element.matches(parts[i]))
+      i++;
+  }
+  return i === parts.length;
+}
+
+function parentElementOrShadowHost(element: Element): Element | undefined {
+  if (element.parentElement)
+    return element.parentElement;
+  if (!element.parentNode)
+    return;
+  if (element.parentNode.nodeType === Node.DOCUMENT_FRAGMENT_NODE && (element.parentNode as ShadowRoot).host)
+    return (element.parentNode as ShadowRoot).host;
+}
+
+function split(selector: string): string[] {
+  let index = 0;
+  let quote: string | undefined;
+  let start = 0;
+  let space: 'none' | 'before' | 'after' = 'none';
+  const result: string[] = [];
+  const append = () => {
+    const part = selector.substring(start, index).trim();
+    if (part.length)
+      result.push(part);
+  };
+  while (index < selector.length) {
+    const c = selector[index];
+    if (!quote && c === ' ') {
+      if (space === 'none' || space === 'before')
+        space = 'before';
+      index++;
+    } else {
+      if (space === 'before') {
+        if (c === '>' || c === '+' || c === '~') {
+          space = 'after';
+        } else {
+          append();
+          start = index;
+          space = 'none';
+        }
+      } else {
+        space = 'none';
+      }
+      if (c === '\\' && index + 1 < selector.length) {
+        index += 2;
+      } else if (c === quote) {
+        quote = undefined;
+        index++;
+      } else {
+        index++;
+      }
+    }
+  }
+  append();
+  return result;
+}
+
+(DeepEngine as any)._test = () => {
+  let id = 0;
+
+  function createShadow(level: number): Element {
+    const root = document.createElement('div');
+    root.id = 'id' + id;
+    root.textContent = 'root #id' + id;
+    id++;
+    const shadow = root.attachShadow({ mode: 'open' });
+    for (let i = 0; i < 9; i++) {
+      const div = document.createElement('div');
+      div.id = 'id' + id;
+      div.textContent = '#id' + id;
+      id++;
+      shadow.appendChild(div);
+    }
+    if (level) {
+      shadow.appendChild(createShadow(level - 1));
+      shadow.appendChild(createShadow(level - 1));
+    }
+    return root;
+  }
+
+  const {query, queryAll} = DeepEngine;
+
+  document.body.textContent = '';
+  document.body.appendChild(createShadow(10));
+  console.time('found');
+  for (let i = 0; i < id; i += 17) {
+    const e = query(document, `div #id${i}`);
+    if (!e || e.id !== 'id' + i)
+      console.log(`div #id${i}`);  // eslint-disable-line no-console
+  }
+  console.timeEnd('found');
+  console.time('not found');
+  for (let i = 0; i < id; i += 17) {
+    const e = query(document, `div div div div div #d${i}`);
+    if (e)
+      console.log(`div div div div div #d${i}`);  // eslint-disable-line no-console
+  }
+  console.timeEnd('not found');
+  console.log(query(document, '#id543 + #id544'));  // eslint-disable-line no-console
+  console.log(query(document, '#id542 ~ #id545'));  // eslint-disable-line no-console
+  console.time('all');
+  queryAll(document, 'div div div + div');
+  console.timeEnd('all');
+};

src/injected/selectorEvaluator.ts

@@ -15,6 +15,7 @@
  */
 
 import { CSSEngine } from './cssSelectorEngine';
+import { DeepEngine } from './deepSelectorEngine';
 import { XPathEngine } from './xpathSelectorEngine';
 import { TextEngine } from './textSelectorEngine';
 import { SelectorEngine, SelectorRoot } from './selectorEngine';
@@ -33,6 +34,7 @@ class SelectorEvaluator {
     this.engines.set('css', CSSEngine);
     this.engines.set('xpath', XPathEngine);
     this.engines.set('text', TextEngine);
+    this.engines.set('deep', DeepEngine);
     this.engines.set('id', createAttributeEngine('id'));
     this.engines.set('data-testid', createAttributeEngine('data-testid'));
     this.engines.set('data-test-id', createAttributeEngine('data-test-id'));

src/selectors.ts

@@ -35,7 +35,7 @@ export class Selectors {
 
   constructor() {
     // Note: keep in sync with SelectorEvaluator class.
-    this._builtinEngines = new Set(['css', 'xpath', 'text', 'id', 'data-testid', 'data-test-id', 'data-test']);
+    this._builtinEngines = new Set(['css', 'xpath', 'text', 'deep', 'id', 'data-testid', 'data-test-id', 'data-test']);
     this._engines = new Map();
   }
 

test/assets/deep-shadow.html

@@ -27,5 +27,9 @@
   span3.setAttribute('data-testid', 'foo');
   span3.textContent = 'Hello from root3';
   shadowRoot3.appendChild(span3);
+  const span4 = document.createElement('span');
+  span4.textContent = 'Hello from root3 #2';
+  span4.setAttribute('attr', 'value space');
+  shadowRoot3.appendChild(span4);
 });
 </script>

test/queryselector.spec.js

@@ -155,7 +155,7 @@ describe('Page.$$eval', function() {
   it('should enter shadow roots with >> syntax', async({page, server}) => {
     await page.goto(server.PREFIX + '/deep-shadow.html');
     const spansCount = await page.$$eval('css=div >> css=div >> css=span', spans => spans.length);
-    expect(spansCount).toBe(2);
+    expect(spansCount).toBe(3);
   });
 });
 
@@ -548,6 +548,26 @@ describe('text selector', () => {
   });
 });
 
+describe('deep selector', () => {
+  it('should work for open shadow roots', async({page, server}) => {
+    await page.goto(server.PREFIX + '/deep-shadow.html');
+    expect(await page.$eval(`deep=span`, e => e.textContent)).toBe('Hello from root1');
+    expect(await page.$eval(`deep=[attr="value\\ space"]`, e => e.textContent)).toBe('Hello from root3 #2');
+    expect(await page.$eval(`deep=[attr='value\\ \\space']`, e => e.textContent)).toBe('Hello from root3 #2');
+    expect(await page.$eval(`deep=div div span`, e => e.textContent)).toBe('Hello from root2');
+    expect(await page.$eval(`deep=div span + span`, e => e.textContent)).toBe('Hello from root3 #2');
+    expect(await page.$eval(`deep=span + [attr*="value"]`, e => e.textContent)).toBe('Hello from root3 #2');
+    expect(await page.$eval(`deep=[data-testid="foo"] + [attr*="value"]`, e => e.textContent)).toBe('Hello from root3 #2');
+    expect(await page.$eval(`deep=#target`, e => e.textContent)).toBe('Hello from root2');
+    expect(await page.$eval(`deep=div #target`, e => e.textContent)).toBe('Hello from root2');
+    expect(await page.$eval(`deep=div div #target`, e => e.textContent)).toBe('Hello from root2');
+    expect(await page.$(`deep=div div div #target`)).toBe(null);
+    expect(await page.$eval(`deep=section > div div span`, e => e.textContent)).toBe('Hello from root2');
+    expect(await page.$eval(`deep=section > div div span:nth-child(2)`, e => e.textContent)).toBe('Hello from root3 #2');
+    expect(await page.$(`deep=section div div div div`)).toBe(null);
+  });
+});
+
 describe('attribute selector', () => {
   it('should work for open shadow roots', async({page, server}) => {
     await page.goto(server.PREFIX + '/deep-shadow.html');