fix(core): detect repeated actions across changing element refs (#430)

packages/core/src/browser/ariaBrowser.ts

@@ -99,6 +99,14 @@ export interface AriaBrowser {
    */
   performAction(ref: string, action: PageAction, value?: string): Promise<void>;
 
+  /**
+   * Look up element identity (role + accessible name) for a ref from the
+   * most recent ariaTree snapshot. Returns null if the ref is unknown.
+   * Used by the repetition detector to distinguish logical targets when
+   * the ref string itself churns between snapshots.
+   */
+  getRefIdentity(ref: string): Promise<{ role: string; name: string } | null>;
+
   /**
    * Waits for a specific load state of the page
    * @param state The load state to wait for

packages/core/src/browser/ariaTree/ariaSnapshot.ts

@@ -48,6 +48,11 @@ export function generateAndRenderAriaTree(root: Element, counter?: RefCounter):
   // that strip data-pilo-ref attributes, e.g. React reconciliation)
   (globalThis as any).__piloRefMap = new Map<string, Element>();
 
+  // Initialize identity map for ref -> {role, name} lookup. Used by the
+  // repetition detector to distinguish logical click targets when refs
+  // themselves churn between snapshots.
+  (globalThis as any).__piloIdentityMap = new Map<string, { role: string; name: string }>();
+
   const ariaTree = generateAriaTree(root);
   return renderAriaTree(ariaTree, refCounter);
 }
@@ -493,6 +498,13 @@ function renderAriaTree(root: AriaNode, counter: RefCounter): string {
       if (ariaNode.element) {
         (globalThis as any).__piloRefMap?.set(ref, ariaNode.element);
       }
+      // Capture role + accessible name for the repetition detector. Stored
+      // separately from refMap so consumers that only need identity don't
+      // have to re-walk the DOM.
+      (globalThis as any).__piloIdentityMap?.set(ref, {
+        role: String(ariaNode.role),
+        name: ariaNode.name ?? "",
+      });
     }
 
     const escapedKey = indent + "- " + yamlEscapeKeyIfNeeded(key);

packages/core/src/browser/playwrightBrowser.ts

@@ -788,6 +788,25 @@ export class PlaywrightBrowser implements AriaBrowser {
     return locator;
   }
 
+  async getRefIdentity(ref: string): Promise<{ role: string; name: string } | null> {
+    if (!this.page) return null;
+    try {
+      return await this.page.evaluate(
+        (refArg) =>
+          (
+            globalThis as unknown as {
+              __piloIdentityMap?: Map<string, { role: string; name: string }>;
+            }
+          ).__piloIdentityMap?.get(refArg) ?? null,
+        ref,
+      );
+    } catch {
+      // The page may have navigated or torn down the identity map. Identity
+      // is advisory for repetition detection — log nothing, just bail out.
+      return null;
+    }
+  }
+
   async performAction(ref: string, action: PageAction, value?: string): Promise<void> {
     if (!this.page) throw new Error("Browser not started");
     return withSpan(

packages/core/src/tools/webActionTools.ts

@@ -43,6 +43,11 @@ type ActionResult = {
   value?: string | number;
   error?: string;
   isRecoverable?: boolean;
+  // Snapshot-time role + accessible name for the targeted element. Optional
+  // (only populated when a ref is provided and the browser can resolve it).
+  // Used by the repetition detector to distinguish logical targets even
+  // when ref strings churn between snapshots.
+  targetIdentity?: { role: string; name: string };
 };
 
 /**
@@ -82,6 +87,17 @@ async function performActionWithValidation(
           value,
         });
 
+        // Capture target identity BEFORE the action runs — the snapshot's
+        // identity map is the basis for the ref the LLM emitted, and the
+        // action itself may invalidate the ref by causing navigation or DOM
+        // teardown. Identity is advisory; null means we just won't fold it
+        // into the repetition signature for this turn.
+        let targetIdentity: { role: string; name: string } | undefined;
+        if (ref) {
+          const id = await context.browser.getRefIdentity(ref);
+          if (id) targetIdentity = id;
+        }
+
         // Perform the action
         await context.browser.performAction(ref || "", action, value);
 
@@ -97,6 +113,7 @@ async function performActionWithValidation(
           action,
           ...(ref && { ref }),
           ...(value !== undefined && { value }),
+          ...(targetIdentity && { targetIdentity }),
         };
       } catch (error) {
         // For browser exceptions, emit failure with error details and return error info

packages/core/src/webAgent.ts

@@ -9,7 +9,7 @@
 
 import { streamText, ModelMessage, StreamTextResult } from "ai";
 import type { ProviderConfig } from "./provider.js";
-import { AriaBrowser } from "./browser/ariaBrowser.js";
+import { AriaBrowser, PageAction } from "./browser/ariaBrowser.js";
 import {
   BrowserReconnectedEventData,
   CdpEndpointConnectedEventData,
@@ -230,6 +230,15 @@ export class WebAgent {
   private readonly onUserDataRequired: UserDataCallback | undefined;
   private readonly taskId: string | undefined;
 
+  // Actions where same-action-same-value repetition is legitimate workflow
+  // (e.g. scrolling an infinite feed, waiting for a slow page) rather than a
+  // stuck-loop signal. The detector skips these entirely and resets state so
+  // a real loop interrupted by a scroll doesn't compound across the gap.
+  private static readonly REPETITION_EXEMPT_ACTIONS: ReadonlySet<string> = new Set<string>([
+    PageAction.Scroll,
+    PageAction.Wait,
+  ]);
+
   constructor(
     private browser: AriaBrowser,
     options: WebAgentOptions,
@@ -1154,15 +1163,24 @@ export class WebAgent {
     actionExecuted: boolean;
     error?: TaskError;
   } | null {
+    // Skip exempt actions entirely, and reset state so an in-progress repeat
+    // count doesn't carry across them (an exempt action between two clicks
+    // is treated as progress, not as a transparent gap).
+    if (WebAgent.REPETITION_EXEMPT_ACTIONS.has(actionOutput.action)) {
+      executionState.actionRepeatCount = 0;
+      executionState.lastAction = undefined;
+      return null;
+    }
+
     // Define explicit thresholds for warning and abort
     const REPETITION_WARNING_THRESHOLD = this.maxRepeatedActions + 1;
     const REPETITION_ABORT_THRESHOLD = this.maxRepeatedActions + 2;
 
     // Create signature for current action
     const currentActionSignature = this.createActionSignature(
       actionOutput.action,
-      actionOutput.ref,
       actionOutput.value,
+      actionOutput.targetIdentity,
     );
 
     // Check if this is the same action as the last one
@@ -1505,10 +1523,29 @@ export class WebAgent {
   // === Helper Methods ===
 
   /**
-   * Create a signature for an action to track repetitions
+   * Build a repetition-signature for an action. The element ref is
+   * deliberately excluded because refs are regenerated every snapshot — a
+   * logical "click Submit" gets a new ref each turn, so including it would
+   * make near-duplicate actions hash differently and bypass the detector.
+   * Element identity (role + accessible name) is folded in when present so
+   * ref-only actions like `click` don't collapse different logical targets
+   * (e.g. "click Next" vs "click Submit") onto the same signature.
+   * Value is normalized (lowercase + trim) so cosmetic input variation
+   * doesn't reset the counter on the same logical action.
    */
-  private createActionSignature(action: string, ref?: string, value?: string | number): string {
-    return `${action}:${ref || ""}:${value || ""}`;
+  private createActionSignature(
+    action: string,
+    value?: string | number,
+    identity?: { role: string; name: string },
+  ): string {
+    const normalizedValue = String(value ?? "")
+      .toLowerCase()
+      .trim();
+    if (identity) {
+      const normalizedName = identity.name.toLowerCase().trim();
+      return `${action}:${identity.role}:${normalizedName}:${normalizedValue}`;
+    }
+    return `${action}:${normalizedValue}`;
   }
 
   /**

packages/core/test/tools/webActionTools.test.ts

@@ -73,6 +73,10 @@ class MockBrowser implements AriaBrowser {
     // Mock implementation - can be configured to throw errors for testing
   }
 
+  async getRefIdentity(_ref: string): Promise<{ role: string; name: string } | null> {
+    return null;
+  }
+
   async waitForLoadState(): Promise<void> {}
 
   async runInTemporaryTab<T>(fn: (tab: any) => Promise<T>): Promise<T> {
@@ -256,6 +260,34 @@ describe("Web Action Tools", () => {
 
       await expect(tools.click.execute({ ref: "btn1" })).rejects.toThrow("Network error");
     });
+
+    it("should capture target identity from the browser and include it in the result", async () => {
+      vi.spyOn(mockBrowser, "getRefIdentity").mockResolvedValueOnce({
+        role: "button",
+        name: "Submit",
+      });
+
+      const result = await tools.click.execute({ ref: "btn1" });
+
+      expect(result).toEqual({
+        success: true,
+        action: "click",
+        ref: "btn1",
+        targetIdentity: { role: "button", name: "Submit" },
+      });
+    });
+
+    it("should omit targetIdentity when the browser returns null", async () => {
+      vi.spyOn(mockBrowser, "getRefIdentity").mockResolvedValueOnce(null);
+
+      const result = await tools.click.execute({ ref: "btn1" });
+
+      expect(result).toEqual({
+        success: true,
+        action: "click",
+        ref: "btn1",
+      });
+    });
   });
 
   describe("Fill Action", () => {