Fix key translation for goto/back batch actions

Author: cursoragent

pkg/templates/python/openai-computer-use/agent/agent.py

@@ -1,7 +1,11 @@
 import json
 import time
 from typing import Any, Callable
-from computers.kernel_computer import KernelComputer
+from computers.kernel_computer import (
+    KernelComputer,
+    _describe_action,
+    _describe_batch_actions,
+)
 from utils import (
     create_response,
     show_image,
@@ -186,56 +190,6 @@ def _extract_prompt_text(self, item: dict[str, Any]) -> str | None:
                 parts.append(text)
         return " ".join(parts) if parts else None
 
-    def _describe_action(self, action_type: str, action_args: dict[str, Any]) -> str:
-        if action_type == "click":
-            x = int(action_args.get("x", 0))
-            y = int(action_args.get("y", 0))
-            button = action_args.get("button", "left")
-            if button in ("", "left"):
-                return f"click({x}, {y})"
-            return f"click({x}, {y}, {button})"
-        if action_type == "double_click":
-            return f"double_click({int(action_args.get('x', 0))}, {int(action_args.get('y', 0))})"
-        if action_type == "type":
-            text = str(action_args.get("text", ""))
-            if len(text) > 60:
-                text = f"{text[:57]}..."
-            return f"type({text!r})"
-        if action_type == "keypress":
-            keys = action_args.get("keys", [])
-            hold_keys = action_args.get("hold_keys", [])
-            if hold_keys:
-                return f"keypress(hold={hold_keys}, keys={keys})"
-            return f"keypress({keys})"
-        if action_type == "scroll":
-            return (
-                f"scroll({int(action_args.get('x', 0))}, {int(action_args.get('y', 0))}, "
-                f"dx={int(action_args.get('scroll_x', 0))}, dy={int(action_args.get('scroll_y', 0))})"
-            )
-        if action_type == "move":
-            return f"move({int(action_args.get('x', 0))}, {int(action_args.get('y', 0))})"
-        if action_type == "drag":
-            return "drag(...)"
-        if action_type == "wait":
-            return f"wait({int(action_args.get('ms', 1000))}ms)"
-        if action_type == "goto":
-            return f"goto({action_args.get('url', '')!r})"
-        if action_type == "back":
-            return "back()"
-        if action_type == "url":
-            return "url()"
-        if action_type == "screenshot":
-            return "screenshot()"
-        return action_type
-
-    def _describe_batch_actions(self, actions: list[dict[str, Any]]) -> str:
-        pieces: list[str] = []
-        for action in actions:
-            action_type = str(action.get("type", "unknown"))
-            action_args = {k: v for k, v in action.items() if k != "type"}
-            pieces.append(self._describe_action(action_type, action_args))
-        return "batch[" + " -> ".join(pieces) + "]"
-
     def _batch_terminal_read_action(self, actions: list[dict[str, Any]]) -> str:
         if not actions:
             return ""
@@ -269,7 +223,7 @@ def handle_item(self, item):
                     typed_actions = [a for a in actions if isinstance(a, dict)]
                     payload = {
                         "action_type": "batch",
-                        "description": self._describe_batch_actions(typed_actions),
+                        "description": _describe_batch_actions(typed_actions),
                         "action": {"type": "batch", "actions": typed_actions},
                     }
                     if elapsed_ms is not None:
@@ -315,14 +269,14 @@ def handle_item(self, item):
             if len(typed_actions) == 1:
                 action_type = str(typed_actions[0].get("type", "unknown"))
                 action_payload: dict[str, Any] = typed_actions[0]
-                description = self._describe_action(
+                description = _describe_action(
                     action_type,
                     {k: v for k, v in typed_actions[0].items() if k != "type"},
                 )
             else:
                 action_type = "batch"
                 action_payload = {"type": "batch", "actions": typed_actions}
-                description = self._describe_batch_actions(typed_actions)
+                description = _describe_batch_actions(typed_actions)
 
             payload = {
                 "action_type": action_type,

pkg/templates/python/openai-computer-use/computers/kernel_computer.py

@@ -189,15 +189,15 @@ def _goto_batch_actions(url: str) -> List[Dict[str, Any]]:
     return [
         {
             "type": "press_key",
-            "press_key": {"hold_keys": ["Ctrl"], "keys": ["l"]},
+            "press_key": {"hold_keys": ["Control_L"], "keys": ["l"]},
         },
         {
             "type": "sleep",
             "sleep": {"duration_ms": GOTO_CHORD_DELAY_MS},
         },
         {
             "type": "press_key",
-            "press_key": {"hold_keys": ["Ctrl"], "keys": ["a"]},
+            "press_key": {"hold_keys": ["Control_L"], "keys": ["a"]},
         },
         {
             "type": "type_text",
@@ -214,7 +214,7 @@ def _back_batch_actions() -> List[Dict[str, Any]]:
     return [
         {
             "type": "press_key",
-            "press_key": {"hold_keys": ["Alt"], "keys": ["Left"]},
+            "press_key": {"hold_keys": ["Alt_L"], "keys": ["Left"]},
         }
     ]
 

pkg/templates/typescript/openai-computer-use/lib/kernel-computer.ts

@@ -220,17 +220,17 @@ function isBatchComputerActionType(actionType: string): boolean {
 
 function gotoBatchActions(url: string): BatchAction[] {
   return [
-    { type: 'press_key', press_key: { hold_keys: ['Ctrl'], keys: ['l'] } },
+    { type: 'press_key', press_key: { hold_keys: ['Control_L'], keys: ['l'] } },
     { type: 'sleep', sleep: { duration_ms: GOTO_CHORD_DELAY_MS } },
-    { type: 'press_key', press_key: { hold_keys: ['Ctrl'], keys: ['a'] } },
+    { type: 'press_key', press_key: { hold_keys: ['Control_L'], keys: ['a'] } },
     { type: 'type_text', type_text: { text: url } },
     { type: 'press_key', press_key: { keys: ['Return'] } },
   ];
 }
 
 function backBatchActions(): BatchAction[] {
   return [
-    { type: 'press_key', press_key: { hold_keys: ['Alt'], keys: ['Left'] } },
+    { type: 'press_key', press_key: { hold_keys: ['Alt_L'], keys: ['Left'] } },
   ];
 }