Non interactive leaf elements are now included in the LLMs context

evals/assets/peeler.html

@@ -20,7 +20,12 @@ <h2>Knife Set</h2>
             <h2>Peeler</h2>
             <p>The ultimate tool for peeling fruits and vegetables.</p>
         </div>
-        <button onclick="location.href='cart.html?item=A'">Add to cart</button>
+        <button role="button" onclick="location.href='cart.html?item=A'">Add to cart</button>
     </div>
+    <a href="cart.html" aria-role="button">
+        <div>
+            hi world
+        </div>
+    </a>
 </body>
 </html>

evals/peeler_simple.eval.ts

@@ -8,13 +8,13 @@ const exactMatch = (args: { input; output; expected? }) => {
   };
 };
 
-Eval('Vanta', {
+Eval('Peeler Simple', {
   data: () => {
     return [
       {
         input: {
           text: 'add the peeler to cart',
-          desired: `body > div.page-wrapper > div.nav_component > div.nav_element.w-nav > div.padding-global > div > div > nav > div.nav_cta-wrapper.is-new > a.nav_cta-button-desktop.is-smaller.w-button`,
+          desired: null,
         },
       },
     ];
@@ -30,8 +30,16 @@ Eval('Vanta', {
 
     await stageHand.act({ action: input.text });
 
+    const successMessageLocator = stageHand.page.locator(
+      'text="Congratulations, you have 1 A in your cart"'
+    );
+    await successMessageLocator.waitFor({ state: 'visible', timeout: 5000 });
+    const isVisible = await successMessageLocator.isVisible();
+
     await stageHand.browser.close();
 
+    return isVisible;
+
     return false;
   },
   scores: [exactMatch],

examples/index.ts

@@ -9,7 +9,7 @@ async function example() {
   await stageHand.waitForSettledDom();
 
   await stageHand.act({
-    action: 'click the next date on the calendar that has times available',
+    action: 'click the next available date',
   });
 }
 

lib/playwright/dom.ts

@@ -0,0 +1,111 @@
+import { Locator, type Page } from '@playwright/test';
+import jsdom from 'jsdom';
+const { JSDOM } = jsdom;
+
+const interactiveElementTypes = [
+  'A',
+  'BUTTON',
+  'DETAILS',
+  'EMBED',
+  'INPUT',
+  'LABEL',
+  'MENU',
+  'MENUITEM',
+  'OBJECT',
+  'SELECT',
+  'TEXTAREA',
+  'SUMMARY',
+];
+
+const interactiveRoles = [
+  'button',
+  'menu',
+  'menuitem',
+  'link',
+  'checkbox',
+  'radio',
+  'slider',
+  'tab',
+  'tabpanel',
+  'textbox',
+  'combobox',
+  'grid',
+  'listbox',
+  'option',
+  'progressbar',
+  'scrollbar',
+  'searchbox',
+  'switch',
+  'tree',
+  'treeitem',
+  'spinbutton',
+  'tooltip',
+];
+const interactiveAriaRoles = ['menu', 'menuitem', 'button'];
+
+const isInteractiveElement = (element: HTMLElement) => {
+  const elementType = element.tagName;
+  const elementRole = element.getAttribute('role');
+  const elementAriaRole = element.getAttribute('aria-role');
+
+  if (
+    element.getAttribute('disabled') === 'true' ||
+    element.hidden ||
+    element.ariaDisabled
+  ) {
+    return false;
+  }
+
+  return (
+    (elementType && interactiveElementTypes.includes(elementType)) ||
+    (elementRole && interactiveRoles.includes(elementRole)) ||
+    (elementAriaRole && interactiveAriaRoles.includes(elementAriaRole))
+  );
+};
+
+async function cleanDOM(startingLocator: Locator) {
+  console.log('---DOM CLEANING--- starting cleaning');
+  const domString = await startingLocator.evaluate((el) => el.outerHTML);
+  if (!domString) {
+    throw new Error("error selecting DOM that doesn't exist");
+  }
+  const { document } = new JSDOM(domString).window;
+  const candidateElements: Array<HTMLElement> = [];
+  const DOMQueue: Array<HTMLElement> = [document.body];
+  while (DOMQueue.length > 0) {
+    const element = DOMQueue.pop();
+    if (element) {
+      const childrenCount = element.children.length;
+      // if you have no children you are a leaf node
+      if (childrenCount === 0) {
+        candidateElements.push(element);
+        continue;
+      } else if (isInteractiveElement(element)) {
+        candidateElements.push(element);
+        continue;
+      }
+      for (let i = childrenCount - 1; i >= 0; i--) {
+        const child = element.children[i];
+
+        DOMQueue.push(child as HTMLElement);
+      }
+    }
+  }
+
+  const cleanedHtml = candidateElements
+
+    .map((r) =>
+      r.outerHTML
+        .split('\n')
+        .map((line) => line.trim())
+        .join(' ')
+    )
+    .join(',\n');
+
+  console.log('---DOM CLEANING--- CLEANED HTML STRING');
+  console.log(cleanedHtml);
+
+  return cleanedHtml;
+}
+
+export { cleanDOM };

lib/playwright/index.ts

@@ -3,12 +3,12 @@ import {
   type Browser,
   type BrowserContext,
   chromium,
-  Locator,
 } from '@playwright/test';
 import { expect } from '@playwright/test';
 import Cache from '../cache';
 import OpenAI from 'openai';
 import crypto from 'crypto';
+import { cleanDOM } from './dom';
 
 require('dotenv').config({ path: '.env' });
 
@@ -35,46 +35,6 @@ async function getBrowser(env: 'LOCAL' | 'BROWSERBASE' = 'BROWSERBASE') {
   }
 }
 
-const interactiveElements = [
-  'a',
-  'button',
-  "[role='button']",
-  "[aria-role='button']",
-  'details',
-  'embed',
-  'input',
-  'label',
-  'menu',
-  "[role='menu']",
-  "[aria-role='menu']",
-  'menuitem',
-  "[role='menuitem']",
-  "[aria-role='menuitem']",
-  'object',
-  'select',
-  'textarea',
-  'summary',
-  "[role='link']",
-  "[role='checkbox']",
-  "[role='radio']",
-  "[role='slider']",
-  "[role='tab']",
-  "[role='tabpanel']",
-  "[role='textbox']",
-  "[role='combobox']",
-  "[role='grid']",
-  "[role='listbox']",
-  "[role='option']",
-  "[role='progressbar']",
-  "[role='scrollbar']",
-  "[role='searchbox']",
-  "[role='switch']",
-  "[role='tree']",
-  "[role='treeitem']",
-  "[role='spinbutton']",
-  "[role='tooltip']",
-];
-
 export class Stagehand {
   private openai: OpenAI;
   public observations: { [key: string]: { result: string; id: string } };
@@ -119,34 +79,6 @@ export class Stagehand {
     return this.page.evaluate(() => window.waitForDomSettle());
   }
 
-  async cleanDOM(parent: Locator) {
-    const elementsSelector = interactiveElements.join(', ');
-
-    console.log('\nCLEAN DOM SELECTOR');
-    console.log(elementsSelector);
-
-    const foundElements = await parent.locator(elementsSelector).all();
-    console.log(foundElements);
-    const results = await Promise.allSettled(
-      foundElements.map((el) => el.evaluate((el) => el.outerHTML))
-    );
-
-    console.log('\nFOUND ELEMENTS STRING');
-    console.log(results);
-
-    const cleanedHtml = results
-      .filter(
-        (r): r is PromiseFulfilledResult<string> => r.status === 'fulfilled'
-      )
-      .map((r) => r.value)
-      .join('\n');
-
-    console.log('\nCLEANED HTML STRING');
-    console.log(cleanedHtml);
-
-    return cleanedHtml;
-  }
-
   getKey(operation) {
     return crypto.createHash('sha256').update(operation).digest('hex');
   }
@@ -170,7 +102,7 @@ export class Stagehand {
       return key;
     }
 
-    const fullBody = await this.cleanDOM(this.page.locator('body'));
+    const fullBody = await cleanDOM(this.page.locator('body'));
 
     const selectorResponse = await this.openai.chat.completions.create({
       model: 'gpt-4o',
@@ -239,7 +171,6 @@ export class Stagehand {
   async act({
     observation,
     action,
-    data,
   }: {
     observation?: string;
     action: string;
@@ -274,7 +205,7 @@ export class Stagehand {
       console.log('observation', this.observations[observation].result);
     }
 
-    const area = await this.cleanDOM(
+    const area = await cleanDOM(
       observation
         ? this.page.locator(this.observations[observation].result)
         : this.page.locator('body')
@@ -286,14 +217,13 @@ export class Stagehand {
         {
           role: 'system',
           content:
-            'You are helping the user automate browser by finding one or more actions to take.\n\nyou will be given a DOM element, an overall goal, and data to use when taking actions.\n\nuse selectors that are least likely to change\n\nfor each action required to complete the goal,  follow this format in raw JSON, no markdown\n\n[{\n method: string (the required playwright function to call)\n locator: string (the locator to find the element to act on),\nargs: Array<string | number> (the required arguments)\n}]\n\n\n\n',
+            'You are helping the user automate browser by finding one or more actions to take.\n\nyou will be given a list of potential DOM elements and a goal to accompplish.\n\nuse selectors that are least likely to change and directly select the element\n\nfor each action required to complete the goal,  follow this format in raw JSON, no markdown\n\n[{\n method: string (the required playwright function to call)\n locator: string (the locator to find the element to act on),\nargs: Array<string | number> (the required arguments)\n}]\n\n\n\n',
         },
         {
           role: 'user',
           content: `
             action: ${action},
-            DOM: ${area}
-            data: ${JSON.stringify(data)}`,
+            DOM: ${area}`,
         },
       ],
 

package.json

@@ -14,6 +14,7 @@
   "license": "ISC",
   "devDependencies": {
     "@playwright/test": "^1.42.1",
+    "@types/jsdom": "^21.1.6",
     "@types/node": "^20.11.30",
     "prettier": "^3.2.5",
     "tsx": "^4.10.5"
@@ -23,6 +24,7 @@
     "braintrust": "^0.0.127",
     "dotenv": "^16.4.5",
     "e": "^0.2.33",
+    "jsdom": "^24.0.0",
     "openai": "^4.29.2"
   }
 }