From c6d8482ef7ed5d7dc97742e47cd37f539ab8ccf2 Mon Sep 17 00:00:00 2001 From: Filip Michalsky Date: Tue, 4 Mar 2025 22:19:03 -0500 Subject: [PATCH 01/16] first attempt sync/async clients --- stagehand/__init__.py | 32 ++++++++- stagehand/async_client.py | 145 ++++++++++++++++++++++++++++++++++++++ stagehand/base.py | 75 ++++++++++++++++++++ stagehand/sync_client.py | 137 +++++++++++++++++++++++++++++++++++ 4 files changed, 386 insertions(+), 3 deletions(-) create mode 100644 stagehand/async_client.py create mode 100644 stagehand/base.py create mode 100644 stagehand/sync_client.py diff --git a/stagehand/__init__.py b/stagehand/__init__.py index 63b03284..a86ea41d 100644 --- a/stagehand/__init__.py +++ b/stagehand/__init__.py @@ -1,4 +1,30 @@ -from .client import Stagehand +from typing import Optional, Union, Dict, Any -__version__ = "0.1.0" -__all__ = ["Stagehand"] +from .async_client import AsyncStagehand +from .sync_client import SyncStagehand +from .config import StagehandConfig + +def create_client( + sync: bool = False, + config: Optional[StagehandConfig] = None, + **kwargs +) -> Union[AsyncStagehand, SyncStagehand]: + """ + Factory function to create either a synchronous or asynchronous Stagehand client. + + Args: + sync: If True, creates a synchronous client. If False, creates an asynchronous client. + config: Optional StagehandConfig object to configure the client. + **kwargs: Additional arguments to pass to the client constructor. + + Returns: + Either a SyncStagehand or AsyncStagehand instance. + """ + if sync: + return SyncStagehand(config=config, **kwargs) + return AsyncStagehand(config=config, **kwargs) + +# For backward compatibility +Stagehand = AsyncStagehand + +__all__ = ['create_client', 'AsyncStagehand', 'SyncStagehand', 'Stagehand'] diff --git a/stagehand/async_client.py b/stagehand/async_client.py new file mode 100644 index 00000000..87da3b89 --- /dev/null +++ b/stagehand/async_client.py @@ -0,0 +1,145 @@ +import asyncio +import json +import logging +from collections.abc import Awaitable +from typing import Any, Callable, Dict, Optional + +import httpx +from playwright.async_api import async_playwright + +from .base import BaseStagehand +from .page import StagehandPage +from .utils import default_log_handler, convert_dict_keys_to_camel_case + +logger = logging.getLogger(__name__) + +class AsyncStagehand(BaseStagehand): + """ + Async implementation of the Stagehand client. + """ + + # Dictionary to store one lock per session_id + _session_locks = {} + + def __init__( + self, + config: Optional[StagehandConfig] = None, + server_url: Optional[str] = None, + session_id: Optional[str] = None, + browserbase_api_key: Optional[str] = None, + browserbase_project_id: Optional[str] = None, + model_api_key: Optional[str] = None, + on_log: Optional[Callable[[Dict[str, Any]], Awaitable[None]]] = default_log_handler, + verbose: int = 1, + model_name: Optional[str] = None, + dom_settle_timeout_ms: Optional[int] = None, + debug_dom: Optional[bool] = None, + httpx_client: Optional[httpx.AsyncClient] = None, + timeout_settings: Optional[httpx.Timeout] = None, + model_client_options: Optional[Dict[str, Any]] = None, + ): + super().__init__( + config=config, + server_url=server_url, + session_id=session_id, + browserbase_api_key=browserbase_api_key, + browserbase_project_id=browserbase_project_id, + model_api_key=model_api_key, + on_log=on_log, + verbose=verbose, + model_name=model_name, + dom_settle_timeout_ms=dom_settle_timeout_ms, + debug_dom=debug_dom, + timeout_settings=timeout_settings, + model_client_options=model_client_options, + ) + self.httpx_client = httpx_client + self.playwright = None + self.browser = None + self.context = None + self.page = None + + def _get_lock_for_session(self) -> asyncio.Lock: + """Get or create a lock for the current session.""" + if self.session_id not in self._session_locks: + self._session_locks[self.session_id] = asyncio.Lock() + return self._session_locks[self.session_id] + + async def __aenter__(self): + """Async context manager entry.""" + await self.init() + return self + + async def __aexit__(self, exc_type, exc_val, exc_tb): + """Async context manager exit.""" + await self.close() + + async def init(self): + """Initialize the client and create a session if needed.""" + if not self.httpx_client: + self.httpx_client = httpx.AsyncClient(timeout=self.timeout_settings) + + await self._check_server_health() + + if not self.session_id: + await self._create_session() + + self.playwright = await async_playwright().start() + self.browser = await self.playwright.chromium.launch() + self.context = await self.browser.new_context() + self.page = await self.context.new_page() + + async def close(self): + """Close the client and cleanup resources.""" + if self.page: + await self.page.close() + if self.context: + await self.context.close() + if self.browser: + await self.browser.close() + if self.playwright: + await self.playwright.stop() + if self.httpx_client: + await self.httpx_client.aclose() + + async def _check_server_health(self, timeout: int = 10): + """Check if the server is healthy and responding.""" + try: + response = await self.httpx_client.get(f"{self.server_url}/health") + response.raise_for_status() + except Exception as e: + raise Exception(f"Server health check failed: {str(e)}") + + async def _create_session(self): + """Create a new session with the server.""" + payload = { + "browserbaseApiKey": self.browserbase_api_key, + "browserbaseProjectId": self.browserbase_project_id, + "modelApiKey": self.model_api_key, + "modelName": self.model_name, + "domSettleTimeoutMs": self.dom_settle_timeout_ms, + "debugDom": self.debug_dom, + "modelClientOptions": self.model_client_options, + } + + response = await self.httpx_client.post( + f"{self.server_url}/sessions", + json=convert_dict_keys_to_camel_case(payload) + ) + response.raise_for_status() + self.session_id = response.json()["sessionId"] + + async def _execute(self, method: str, payload: Dict[str, Any]) -> Any: + """Execute a command on the server.""" + async with self._get_lock_for_session(): + response = await self.httpx_client.post( + f"{self.server_url}/sessions/{self.session_id}/execute", + json={"method": method, "payload": convert_dict_keys_to_camel_case(payload)} + ) + response.raise_for_status() + return response.json() + + async def _handle_log(self, msg: Dict[str, Any]): + """Handle log messages from the server.""" + if self.on_log: + await self.on_log(msg) \ No newline at end of file diff --git a/stagehand/base.py b/stagehand/base.py new file mode 100644 index 00000000..82fecc3d --- /dev/null +++ b/stagehand/base.py @@ -0,0 +1,75 @@ +from abc import ABC, abstractmethod +from typing import Any, Callable, Dict, Optional, Union +from .config import StagehandConfig +from .page import StagehandPage + +class BaseStagehand(ABC): + """ + Base class for Stagehand clients that defines the common interface + and shared functionality for both sync and async implementations. + """ + + def __init__( + self, + config: Optional[StagehandConfig] = None, + server_url: Optional[str] = None, + session_id: Optional[str] = None, + browserbase_api_key: Optional[str] = None, + browserbase_project_id: Optional[str] = None, + model_api_key: Optional[str] = None, + on_log: Optional[Callable[[Dict[str, Any]], Union[None, Any]]] = None, + verbose: int = 1, + model_name: Optional[str] = None, + dom_settle_timeout_ms: Optional[int] = None, + debug_dom: Optional[bool] = None, + timeout_settings: Optional[Any] = None, + model_client_options: Optional[Dict[str, Any]] = None, + ): + self.config = config or StagehandConfig() + self.server_url = server_url or self.config.server_url + self.session_id = session_id + self.browserbase_api_key = browserbase_api_key or self.config.browserbase_api_key + self.browserbase_project_id = browserbase_project_id or self.config.browserbase_project_id + self.model_api_key = model_api_key or self.config.model_api_key + self.on_log = on_log + self.verbose = verbose + self.model_name = model_name or self.config.model_name + self.dom_settle_timeout_ms = dom_settle_timeout_ms or self.config.dom_settle_timeout_ms + self.debug_dom = debug_dom if debug_dom is not None else self.config.debug_dom + self.timeout_settings = timeout_settings or self.config.timeout_settings + self.model_client_options = model_client_options or self.config.model_client_options + + @abstractmethod + def init(self): + """Initialize the client and create a session if needed.""" + pass + + @abstractmethod + def close(self): + """Close the client and cleanup resources.""" + pass + + @abstractmethod + def _check_server_health(self, timeout: int = 10): + """Check if the server is healthy and responding.""" + pass + + @abstractmethod + def _create_session(self): + """Create a new session with the server.""" + pass + + @abstractmethod + def _execute(self, method: str, payload: Dict[str, Any]) -> Any: + """Execute a command on the server.""" + pass + + @abstractmethod + def _handle_log(self, msg: Dict[str, Any]): + """Handle log messages from the server.""" + pass + + def _log(self, message: str, level: int = 1): + """Log a message if verbose level is sufficient.""" + if self.verbose >= level: + print(f"[Stagehand] {message}") \ No newline at end of file diff --git a/stagehand/sync_client.py b/stagehand/sync_client.py new file mode 100644 index 00000000..551077a0 --- /dev/null +++ b/stagehand/sync_client.py @@ -0,0 +1,137 @@ +import json +import logging +from typing import Any, Callable, Dict, Optional + +import httpx +from playwright.sync_api import sync_playwright + +from .base import BaseStagehand +from .page import StagehandPage +from .utils import convert_dict_keys_to_camel_case + +logger = logging.getLogger(__name__) + +def default_sync_log_handler(msg: Dict[str, Any]) -> None: + """Default synchronous log handler.""" + logger.info(json.dumps(msg, indent=2)) + +class SyncStagehand(BaseStagehand): + """ + Synchronous implementation of the Stagehand client. + """ + + def __init__( + self, + config: Optional[StagehandConfig] = None, + server_url: Optional[str] = None, + session_id: Optional[str] = None, + browserbase_api_key: Optional[str] = None, + browserbase_project_id: Optional[str] = None, + model_api_key: Optional[str] = None, + on_log: Optional[Callable[[Dict[str, Any]], None]] = default_sync_log_handler, + verbose: int = 1, + model_name: Optional[str] = None, + dom_settle_timeout_ms: Optional[int] = None, + debug_dom: Optional[bool] = None, + httpx_client: Optional[httpx.Client] = None, + timeout_settings: Optional[httpx.Timeout] = None, + model_client_options: Optional[Dict[str, Any]] = None, + ): + super().__init__( + config=config, + server_url=server_url, + session_id=session_id, + browserbase_api_key=browserbase_api_key, + browserbase_project_id=browserbase_project_id, + model_api_key=model_api_key, + on_log=on_log, + verbose=verbose, + model_name=model_name, + dom_settle_timeout_ms=dom_settle_timeout_ms, + debug_dom=debug_dom, + timeout_settings=timeout_settings, + model_client_options=model_client_options, + ) + self.httpx_client = httpx_client + self.playwright = None + self.browser = None + self.context = None + self.page = None + + def __enter__(self): + """Context manager entry.""" + self.init() + return self + + def __exit__(self, exc_type, exc_val, exc_tb): + """Context manager exit.""" + self.close() + + def init(self): + """Initialize the client and create a session if needed.""" + if not self.httpx_client: + self.httpx_client = httpx.Client(timeout=self.timeout_settings) + + self._check_server_health() + + if not self.session_id: + self._create_session() + + self.playwright = sync_playwright().start() + self.browser = self.playwright.chromium.launch() + self.context = self.browser.new_context() + self.page = self.context.new_page() + + def close(self): + """Close the client and cleanup resources.""" + if self.page: + self.page.close() + if self.context: + self.context.close() + if self.browser: + self.browser.close() + if self.playwright: + self.playwright.stop() + if self.httpx_client: + self.httpx_client.close() + + def _check_server_health(self, timeout: int = 10): + """Check if the server is healthy and responding.""" + try: + response = self.httpx_client.get(f"{self.server_url}/health") + response.raise_for_status() + except Exception as e: + raise Exception(f"Server health check failed: {str(e)}") + + def _create_session(self): + """Create a new session with the server.""" + payload = { + "browserbaseApiKey": self.browserbase_api_key, + "browserbaseProjectId": self.browserbase_project_id, + "modelApiKey": self.model_api_key, + "modelName": self.model_name, + "domSettleTimeoutMs": self.dom_settle_timeout_ms, + "debugDom": self.debug_dom, + "modelClientOptions": self.model_client_options, + } + + response = self.httpx_client.post( + f"{self.server_url}/sessions", + json=convert_dict_keys_to_camel_case(payload) + ) + response.raise_for_status() + self.session_id = response.json()["sessionId"] + + def _execute(self, method: str, payload: Dict[str, Any]) -> Any: + """Execute a command on the server.""" + response = self.httpx_client.post( + f"{self.server_url}/sessions/{self.session_id}/execute", + json={"method": method, "payload": convert_dict_keys_to_camel_case(payload)} + ) + response.raise_for_status() + return response.json() + + def _handle_log(self, msg: Dict[str, Any]): + """Handle log messages from the server.""" + if self.on_log: + self.on_log(msg) \ No newline at end of file From 841c5bf1f7e5c5c8844158603f3025516295d926 Mon Sep 17 00:00:00 2001 From: Filip Michalsky Date: Tue, 4 Mar 2025 22:24:57 -0500 Subject: [PATCH 02/16] Revert latest commit --- stagehand/__init__.py | 32 +-------- stagehand/async_client.py | 145 -------------------------------------- stagehand/base.py | 75 -------------------- stagehand/sync_client.py | 137 ----------------------------------- 4 files changed, 3 insertions(+), 386 deletions(-) delete mode 100644 stagehand/async_client.py delete mode 100644 stagehand/base.py delete mode 100644 stagehand/sync_client.py diff --git a/stagehand/__init__.py b/stagehand/__init__.py index a86ea41d..63b03284 100644 --- a/stagehand/__init__.py +++ b/stagehand/__init__.py @@ -1,30 +1,4 @@ -from typing import Optional, Union, Dict, Any +from .client import Stagehand -from .async_client import AsyncStagehand -from .sync_client import SyncStagehand -from .config import StagehandConfig - -def create_client( - sync: bool = False, - config: Optional[StagehandConfig] = None, - **kwargs -) -> Union[AsyncStagehand, SyncStagehand]: - """ - Factory function to create either a synchronous or asynchronous Stagehand client. - - Args: - sync: If True, creates a synchronous client. If False, creates an asynchronous client. - config: Optional StagehandConfig object to configure the client. - **kwargs: Additional arguments to pass to the client constructor. - - Returns: - Either a SyncStagehand or AsyncStagehand instance. - """ - if sync: - return SyncStagehand(config=config, **kwargs) - return AsyncStagehand(config=config, **kwargs) - -# For backward compatibility -Stagehand = AsyncStagehand - -__all__ = ['create_client', 'AsyncStagehand', 'SyncStagehand', 'Stagehand'] +__version__ = "0.1.0" +__all__ = ["Stagehand"] diff --git a/stagehand/async_client.py b/stagehand/async_client.py deleted file mode 100644 index 87da3b89..00000000 --- a/stagehand/async_client.py +++ /dev/null @@ -1,145 +0,0 @@ -import asyncio -import json -import logging -from collections.abc import Awaitable -from typing import Any, Callable, Dict, Optional - -import httpx -from playwright.async_api import async_playwright - -from .base import BaseStagehand -from .page import StagehandPage -from .utils import default_log_handler, convert_dict_keys_to_camel_case - -logger = logging.getLogger(__name__) - -class AsyncStagehand(BaseStagehand): - """ - Async implementation of the Stagehand client. - """ - - # Dictionary to store one lock per session_id - _session_locks = {} - - def __init__( - self, - config: Optional[StagehandConfig] = None, - server_url: Optional[str] = None, - session_id: Optional[str] = None, - browserbase_api_key: Optional[str] = None, - browserbase_project_id: Optional[str] = None, - model_api_key: Optional[str] = None, - on_log: Optional[Callable[[Dict[str, Any]], Awaitable[None]]] = default_log_handler, - verbose: int = 1, - model_name: Optional[str] = None, - dom_settle_timeout_ms: Optional[int] = None, - debug_dom: Optional[bool] = None, - httpx_client: Optional[httpx.AsyncClient] = None, - timeout_settings: Optional[httpx.Timeout] = None, - model_client_options: Optional[Dict[str, Any]] = None, - ): - super().__init__( - config=config, - server_url=server_url, - session_id=session_id, - browserbase_api_key=browserbase_api_key, - browserbase_project_id=browserbase_project_id, - model_api_key=model_api_key, - on_log=on_log, - verbose=verbose, - model_name=model_name, - dom_settle_timeout_ms=dom_settle_timeout_ms, - debug_dom=debug_dom, - timeout_settings=timeout_settings, - model_client_options=model_client_options, - ) - self.httpx_client = httpx_client - self.playwright = None - self.browser = None - self.context = None - self.page = None - - def _get_lock_for_session(self) -> asyncio.Lock: - """Get or create a lock for the current session.""" - if self.session_id not in self._session_locks: - self._session_locks[self.session_id] = asyncio.Lock() - return self._session_locks[self.session_id] - - async def __aenter__(self): - """Async context manager entry.""" - await self.init() - return self - - async def __aexit__(self, exc_type, exc_val, exc_tb): - """Async context manager exit.""" - await self.close() - - async def init(self): - """Initialize the client and create a session if needed.""" - if not self.httpx_client: - self.httpx_client = httpx.AsyncClient(timeout=self.timeout_settings) - - await self._check_server_health() - - if not self.session_id: - await self._create_session() - - self.playwright = await async_playwright().start() - self.browser = await self.playwright.chromium.launch() - self.context = await self.browser.new_context() - self.page = await self.context.new_page() - - async def close(self): - """Close the client and cleanup resources.""" - if self.page: - await self.page.close() - if self.context: - await self.context.close() - if self.browser: - await self.browser.close() - if self.playwright: - await self.playwright.stop() - if self.httpx_client: - await self.httpx_client.aclose() - - async def _check_server_health(self, timeout: int = 10): - """Check if the server is healthy and responding.""" - try: - response = await self.httpx_client.get(f"{self.server_url}/health") - response.raise_for_status() - except Exception as e: - raise Exception(f"Server health check failed: {str(e)}") - - async def _create_session(self): - """Create a new session with the server.""" - payload = { - "browserbaseApiKey": self.browserbase_api_key, - "browserbaseProjectId": self.browserbase_project_id, - "modelApiKey": self.model_api_key, - "modelName": self.model_name, - "domSettleTimeoutMs": self.dom_settle_timeout_ms, - "debugDom": self.debug_dom, - "modelClientOptions": self.model_client_options, - } - - response = await self.httpx_client.post( - f"{self.server_url}/sessions", - json=convert_dict_keys_to_camel_case(payload) - ) - response.raise_for_status() - self.session_id = response.json()["sessionId"] - - async def _execute(self, method: str, payload: Dict[str, Any]) -> Any: - """Execute a command on the server.""" - async with self._get_lock_for_session(): - response = await self.httpx_client.post( - f"{self.server_url}/sessions/{self.session_id}/execute", - json={"method": method, "payload": convert_dict_keys_to_camel_case(payload)} - ) - response.raise_for_status() - return response.json() - - async def _handle_log(self, msg: Dict[str, Any]): - """Handle log messages from the server.""" - if self.on_log: - await self.on_log(msg) \ No newline at end of file diff --git a/stagehand/base.py b/stagehand/base.py deleted file mode 100644 index 82fecc3d..00000000 --- a/stagehand/base.py +++ /dev/null @@ -1,75 +0,0 @@ -from abc import ABC, abstractmethod -from typing import Any, Callable, Dict, Optional, Union -from .config import StagehandConfig -from .page import StagehandPage - -class BaseStagehand(ABC): - """ - Base class for Stagehand clients that defines the common interface - and shared functionality for both sync and async implementations. - """ - - def __init__( - self, - config: Optional[StagehandConfig] = None, - server_url: Optional[str] = None, - session_id: Optional[str] = None, - browserbase_api_key: Optional[str] = None, - browserbase_project_id: Optional[str] = None, - model_api_key: Optional[str] = None, - on_log: Optional[Callable[[Dict[str, Any]], Union[None, Any]]] = None, - verbose: int = 1, - model_name: Optional[str] = None, - dom_settle_timeout_ms: Optional[int] = None, - debug_dom: Optional[bool] = None, - timeout_settings: Optional[Any] = None, - model_client_options: Optional[Dict[str, Any]] = None, - ): - self.config = config or StagehandConfig() - self.server_url = server_url or self.config.server_url - self.session_id = session_id - self.browserbase_api_key = browserbase_api_key or self.config.browserbase_api_key - self.browserbase_project_id = browserbase_project_id or self.config.browserbase_project_id - self.model_api_key = model_api_key or self.config.model_api_key - self.on_log = on_log - self.verbose = verbose - self.model_name = model_name or self.config.model_name - self.dom_settle_timeout_ms = dom_settle_timeout_ms or self.config.dom_settle_timeout_ms - self.debug_dom = debug_dom if debug_dom is not None else self.config.debug_dom - self.timeout_settings = timeout_settings or self.config.timeout_settings - self.model_client_options = model_client_options or self.config.model_client_options - - @abstractmethod - def init(self): - """Initialize the client and create a session if needed.""" - pass - - @abstractmethod - def close(self): - """Close the client and cleanup resources.""" - pass - - @abstractmethod - def _check_server_health(self, timeout: int = 10): - """Check if the server is healthy and responding.""" - pass - - @abstractmethod - def _create_session(self): - """Create a new session with the server.""" - pass - - @abstractmethod - def _execute(self, method: str, payload: Dict[str, Any]) -> Any: - """Execute a command on the server.""" - pass - - @abstractmethod - def _handle_log(self, msg: Dict[str, Any]): - """Handle log messages from the server.""" - pass - - def _log(self, message: str, level: int = 1): - """Log a message if verbose level is sufficient.""" - if self.verbose >= level: - print(f"[Stagehand] {message}") \ No newline at end of file diff --git a/stagehand/sync_client.py b/stagehand/sync_client.py deleted file mode 100644 index 551077a0..00000000 --- a/stagehand/sync_client.py +++ /dev/null @@ -1,137 +0,0 @@ -import json -import logging -from typing import Any, Callable, Dict, Optional - -import httpx -from playwright.sync_api import sync_playwright - -from .base import BaseStagehand -from .page import StagehandPage -from .utils import convert_dict_keys_to_camel_case - -logger = logging.getLogger(__name__) - -def default_sync_log_handler(msg: Dict[str, Any]) -> None: - """Default synchronous log handler.""" - logger.info(json.dumps(msg, indent=2)) - -class SyncStagehand(BaseStagehand): - """ - Synchronous implementation of the Stagehand client. - """ - - def __init__( - self, - config: Optional[StagehandConfig] = None, - server_url: Optional[str] = None, - session_id: Optional[str] = None, - browserbase_api_key: Optional[str] = None, - browserbase_project_id: Optional[str] = None, - model_api_key: Optional[str] = None, - on_log: Optional[Callable[[Dict[str, Any]], None]] = default_sync_log_handler, - verbose: int = 1, - model_name: Optional[str] = None, - dom_settle_timeout_ms: Optional[int] = None, - debug_dom: Optional[bool] = None, - httpx_client: Optional[httpx.Client] = None, - timeout_settings: Optional[httpx.Timeout] = None, - model_client_options: Optional[Dict[str, Any]] = None, - ): - super().__init__( - config=config, - server_url=server_url, - session_id=session_id, - browserbase_api_key=browserbase_api_key, - browserbase_project_id=browserbase_project_id, - model_api_key=model_api_key, - on_log=on_log, - verbose=verbose, - model_name=model_name, - dom_settle_timeout_ms=dom_settle_timeout_ms, - debug_dom=debug_dom, - timeout_settings=timeout_settings, - model_client_options=model_client_options, - ) - self.httpx_client = httpx_client - self.playwright = None - self.browser = None - self.context = None - self.page = None - - def __enter__(self): - """Context manager entry.""" - self.init() - return self - - def __exit__(self, exc_type, exc_val, exc_tb): - """Context manager exit.""" - self.close() - - def init(self): - """Initialize the client and create a session if needed.""" - if not self.httpx_client: - self.httpx_client = httpx.Client(timeout=self.timeout_settings) - - self._check_server_health() - - if not self.session_id: - self._create_session() - - self.playwright = sync_playwright().start() - self.browser = self.playwright.chromium.launch() - self.context = self.browser.new_context() - self.page = self.context.new_page() - - def close(self): - """Close the client and cleanup resources.""" - if self.page: - self.page.close() - if self.context: - self.context.close() - if self.browser: - self.browser.close() - if self.playwright: - self.playwright.stop() - if self.httpx_client: - self.httpx_client.close() - - def _check_server_health(self, timeout: int = 10): - """Check if the server is healthy and responding.""" - try: - response = self.httpx_client.get(f"{self.server_url}/health") - response.raise_for_status() - except Exception as e: - raise Exception(f"Server health check failed: {str(e)}") - - def _create_session(self): - """Create a new session with the server.""" - payload = { - "browserbaseApiKey": self.browserbase_api_key, - "browserbaseProjectId": self.browserbase_project_id, - "modelApiKey": self.model_api_key, - "modelName": self.model_name, - "domSettleTimeoutMs": self.dom_settle_timeout_ms, - "debugDom": self.debug_dom, - "modelClientOptions": self.model_client_options, - } - - response = self.httpx_client.post( - f"{self.server_url}/sessions", - json=convert_dict_keys_to_camel_case(payload) - ) - response.raise_for_status() - self.session_id = response.json()["sessionId"] - - def _execute(self, method: str, payload: Dict[str, Any]) -> Any: - """Execute a command on the server.""" - response = self.httpx_client.post( - f"{self.server_url}/sessions/{self.session_id}/execute", - json={"method": method, "payload": convert_dict_keys_to_camel_case(payload)} - ) - response.raise_for_status() - return response.json() - - def _handle_log(self, msg: Dict[str, Any]): - """Handle log messages from the server.""" - if self.on_log: - self.on_log(msg) \ No newline at end of file From d45b04b19a9794bf17fb6fb6a2a11761e9476371 Mon Sep 17 00:00:00 2001 From: Filip Michalsky Date: Tue, 4 Mar 2025 22:43:58 -0500 Subject: [PATCH 03/16] first attempt --- stagehand/__init__.py | 3 +- stagehand/base.py | 97 +++++++++++++++ stagehand/sync_client.py | 261 +++++++++++++++++++++++++++++++++++++++ stagehand/sync_page.py | 153 +++++++++++++++++++++++ 4 files changed, 513 insertions(+), 1 deletion(-) create mode 100644 stagehand/base.py create mode 100644 stagehand/sync_client.py create mode 100644 stagehand/sync_page.py diff --git a/stagehand/__init__.py b/stagehand/__init__.py index 63b03284..b5f30d42 100644 --- a/stagehand/__init__.py +++ b/stagehand/__init__.py @@ -1,4 +1,5 @@ from .client import Stagehand +from .sync_client import SyncStagehand __version__ = "0.1.0" -__all__ = ["Stagehand"] +__all__ = ["Stagehand", "SyncStagehand"] diff --git a/stagehand/base.py b/stagehand/base.py new file mode 100644 index 00000000..5430bd12 --- /dev/null +++ b/stagehand/base.py @@ -0,0 +1,97 @@ +from abc import ABC, abstractmethod +from typing import Any, Callable, Dict, Optional, Union +from playwright.async_api import Page + +from .config import StagehandConfig +from .page import StagehandPage +from .utils import default_log_handler + +class StagehandBase(ABC): + """ + Base class for Stagehand client implementations. + Defines the common interface and functionality for both sync and async versions. + """ + def __init__( + self, + config: Optional[StagehandConfig] = None, + server_url: Optional[str] = None, + session_id: Optional[str] = None, + browserbase_api_key: Optional[str] = None, + browserbase_project_id: Optional[str] = None, + model_api_key: Optional[str] = None, + on_log: Optional[Callable[[Dict[str, Any]], Any]] = default_log_handler, + verbose: int = 1, + model_name: Optional[str] = None, + dom_settle_timeout_ms: Optional[int] = None, + debug_dom: Optional[bool] = None, + timeout_settings: Optional[float] = None, + ): + """ + Initialize the Stagehand client with common configuration. + """ + self.server_url = server_url or os.getenv("STAGEHAND_SERVER_URL") + + if config: + self.browserbase_api_key = config.api_key or browserbase_api_key or os.getenv("BROWSERBASE_API_KEY") + self.browserbase_project_id = config.project_id or browserbase_project_id or os.getenv("BROWSERBASE_PROJECT_ID") + self.model_api_key = model_api_key or ( + config.model_client_options.get("apiKey") if config.model_client_options else None + ) or os.getenv("MODEL_API_KEY") + self.session_id = config.browserbase_session_id or session_id + self.model_name = config.model_name or model_name + self.dom_settle_timeout_ms = config.dom_settle_timeout_ms or dom_settle_timeout_ms + self.debug_dom = config.debug_dom if config.debug_dom is not None else debug_dom + else: + self.browserbase_api_key = browserbase_api_key or os.getenv("BROWSERBASE_API_KEY") + self.browserbase_project_id = browserbase_project_id or os.getenv("BROWSERBASE_PROJECT_ID") + self.model_api_key = model_api_key or os.getenv("MODEL_API_KEY") + self.session_id = session_id + self.model_name = model_name + self.dom_settle_timeout_ms = dom_settle_timeout_ms + self.debug_dom = debug_dom + + self.on_log = on_log + self.verbose = verbose + self.timeout_settings = timeout_settings or 180.0 + + self.streamed_response = True + self._initialized = False + self._closed = False + self.page: Optional[StagehandPage] = None + + # Validate essential fields if session_id was provided + if self.session_id: + if not self.browserbase_api_key: + raise ValueError("browserbase_api_key is required (or set BROWSERBASE_API_KEY in env).") + if not self.browserbase_project_id: + raise ValueError("browserbase_project_id is required (or set BROWSERBASE_PROJECT_ID in env).") + + @abstractmethod + def init(self): + """ + Initialize the Stagehand client. + Must be implemented by subclasses. + """ + pass + + @abstractmethod + def close(self): + """ + Clean up resources. + Must be implemented by subclasses. + """ + pass + + def _log(self, message: str, level: int = 1): + """ + Internal logging helper that maps verbosity to logging levels. + """ + if self.verbose >= level: + timestamp = time.strftime("%Y-%m-%d %H:%M:%S", time.localtime()) + formatted_msg = f"{timestamp}::[stagehand] {message}" + if level == 1: + logger.info(formatted_msg) + elif level == 2: + logger.warning(formatted_msg) + else: + logger.debug(formatted_msg) \ No newline at end of file diff --git a/stagehand/sync_client.py b/stagehand/sync_client.py new file mode 100644 index 00000000..02569fe0 --- /dev/null +++ b/stagehand/sync_client.py @@ -0,0 +1,261 @@ +import asyncio +import os +import time +import logging +import json +from typing import Any, Dict, Optional + +import httpx +from playwright.sync_api import sync_playwright + +from .base import StagehandBase +from .config import StagehandConfig +from .sync_page import SyncStagehandPage +from .utils import default_log_handler + +logger = logging.getLogger(__name__) + +class SyncStagehand(StagehandBase): + """ + Synchronous implementation of the Stagehand client. + Wraps the async implementation using asyncio.run() + """ + def __init__( + self, + config: Optional[StagehandConfig] = None, + server_url: Optional[str] = None, + session_id: Optional[str] = None, + browserbase_api_key: Optional[str] = None, + browserbase_project_id: Optional[str] = None, + model_api_key: Optional[str] = None, + on_log: Optional[Callable[[Dict[str, Any]], Any]] = default_log_handler, + verbose: int = 1, + model_name: Optional[str] = None, + dom_settle_timeout_ms: Optional[int] = None, + debug_dom: Optional[bool] = None, + timeout_settings: Optional[float] = None, + ): + super().__init__( + config=config, + server_url=server_url, + session_id=session_id, + browserbase_api_key=browserbase_api_key, + browserbase_project_id=browserbase_project_id, + model_api_key=model_api_key, + on_log=on_log, + verbose=verbose, + model_name=model_name, + dom_settle_timeout_ms=dom_settle_timeout_ms, + debug_dom=debug_dom, + timeout_settings=timeout_settings, + ) + self._client: Optional[httpx.Client] = None + self._playwright = None + self._browser = None + self._context = None + self._playwright_page = None + + def init(self): + """ + Initialize the Stagehand client synchronously. + """ + if self._initialized: + self._log("Stagehand is already initialized; skipping init()", level=3) + return + + self._log("Initializing Stagehand...", level=3) + + if not self._client: + self._client = httpx.Client(timeout=self.timeout_settings) + + # Check server health + self._check_server_health() + + # Create session if we don't have one + if not self.session_id: + self._create_session() + self._log(f"Created new session: {self.session_id}", level=3) + + # Start Playwright and connect to remote + self._log("Starting Playwright...", level=3) + self._playwright = sync_playwright().start() + + connect_url = ( + f"wss://connect.browserbase.com?apiKey={self.browserbase_api_key}" + f"&sessionId={self.session_id}" + ) + self._log(f"Connecting to remote browser at: {connect_url}", level=3) + self._browser = self._playwright.chromium.connect_over_cdp(connect_url) + self._log(f"Connected to remote browser: {self._browser}", level=3) + + # Access or create a context + existing_contexts = self._browser.contexts + self._log(f"Existing contexts: {len(existing_contexts)}", level=3) + if existing_contexts: + self._context = existing_contexts[0] + else: + self._log("Creating a new context...", level=3) + self._context = self._browser.new_context() + + # Access or create a page + existing_pages = self._context.pages + self._log(f"Existing pages: {len(existing_pages)}", level=3) + if existing_pages: + self._log("Using existing page", level=3) + self._playwright_page = existing_pages[0] + else: + self._log("Creating a new page...", level=3) + self._playwright_page = self._context.new_page() + + # Wrap with SyncStagehandPage + self._log("Wrapping Playwright page in SyncStagehandPage", level=3) + self.page = SyncStagehandPage(self._playwright_page, self) + + self._initialized = True + + def close(self): + """ + Clean up resources synchronously. + """ + if self._closed: + return + + self._log("Closing resources...", level=3) + + # End the session on the server if we have a session ID + if self.session_id: + try: + self._log(f"Ending session {self.session_id} on the server...", level=3) + headers = { + "x-bb-api-key": self.browserbase_api_key, + "x-bb-project-id": self.browserbase_project_id, + "Content-Type": "application/json", + } + self._execute("end", {"sessionId": self.session_id}) + self._log(f"Session {self.session_id} ended successfully", level=3) + except Exception as e: + self._log(f"Error ending session: {str(e)}", level=3) + + if self._playwright: + self._log("Stopping Playwright...", level=3) + self._playwright.stop() + self._playwright = None + + if self._client: + self._log("Closing the HTTP client...", level=3) + self._client.close() + self._client = None + + self._closed = True + + def _check_server_health(self, timeout: int = 10): + """ + Check server health synchronously with exponential backoff. + """ + start = time.time() + attempt = 0 + while True: + try: + headers = { + "x-bb-api-key": self.browserbase_api_key, + } + resp = self._client.get(f"{self.server_url}/healthcheck", headers=headers) + if resp.status_code == 200: + data = resp.json() + if data.get("status") == "ok": + self._log("Healthcheck passed. Server is running.", level=3) + return + except Exception as e: + self._log(f"Healthcheck error: {str(e)}", level=3) + + if time.time() - start > timeout: + raise TimeoutError(f"Server not responding after {timeout} seconds.") + + wait_time = min(2 ** attempt * 0.5, 5.0) + time.sleep(wait_time) + attempt += 1 + + def _create_session(self): + """ + Create a new session synchronously. + """ + if not self.browserbase_api_key: + raise ValueError("browserbase_api_key is required to create a session.") + if not self.browserbase_project_id: + raise ValueError("browserbase_project_id is required to create a session.") + if not self.model_api_key: + raise ValueError("model_api_key is required to create a session.") + + payload = { + "modelName": self.model_name, + "domSettleTimeoutMs": self.dom_settle_timeout_ms, + "verbose": self.verbose, + "debugDom": self.debug_dom, + } + headers = { + "x-bb-api-key": self.browserbase_api_key, + "x-bb-project-id": self.browserbase_project_id, + "x-model-api-key": self.model_api_key, + "Content-Type": "application/json", + } + + resp = self._client.post( + f"{self.server_url}/sessions/start", + json=payload, + headers=headers, + ) + if resp.status_code != 200: + raise RuntimeError(f"Failed to create session: {resp.text}") + data = resp.json() + self._log(f"Session created: {data}", level=3) + if not data.get("success") or "sessionId" not in data.get("data", {}): + raise RuntimeError(f"Invalid response format: {resp.text}") + self.session_id = data["data"]["sessionId"] + + def _execute(self, method: str, payload: Dict[str, Any]) -> Any: + """ + Execute a command synchronously. + """ + headers = { + "x-bb-api-key": self.browserbase_api_key, + "x-bb-project-id": self.browserbase_project_id, + "Content-Type": "application/json", + "Connection": "keep-alive", + "x-stream-response": str(self.streamed_response).lower(), + } + if self.model_api_key: + headers["x-model-api-key"] = self.model_api_key + + url = f"{self.server_url}/sessions/{self.session_id}/{method}" + self._log(f"Executing {method} with payload: {payload}", level=3) + + response = self._client.post(url, json=payload, headers=headers, stream=True) + if response.status_code != 200: + raise RuntimeError(f"Error: {response.text}") + + for line in response.iter_lines(decode_unicode=True): + if not line.strip(): + continue + if line.startswith("data: "): + line = line[6:] + try: + message = json.loads(line) + msg_type = message.get("type") + if msg_type == "system": + status = message.get("data", {}).get("status") + if status == "finished": + return message.get("data", {}).get("result") + elif msg_type == "log": + log_msg = message.get("data", {}).get("message", "") + self._log(log_msg, level=3) + if self.on_log: + self.on_log(message) + else: + self._log(f"Unknown message type: {msg_type}", level=3) + if self.on_log: + self.on_log(message) + except json.JSONDecodeError: + self._log(f"Could not parse line as JSON: {line}", level=3) + continue + + raise RuntimeError("Server connection closed without sending 'finished' message") \ No newline at end of file diff --git a/stagehand/sync_page.py b/stagehand/sync_page.py new file mode 100644 index 00000000..b32f9ce9 --- /dev/null +++ b/stagehand/sync_page.py @@ -0,0 +1,153 @@ +from typing import List, Optional, Union + +from playwright.sync_api import Page + +from .schemas import ( + ActOptions, + ActResult, + ExtractOptions, + ExtractResult, + ObserveOptions, + ObserveResult, +) + + +class SyncStagehandPage: + """Synchronous wrapper around Playwright Page that integrates with Stagehand server""" + + def __init__(self, page: Page, stagehand_client): + """ + Initialize a SyncStagehandPage instance. + + Args: + page (Page): The underlying Playwright page. + stagehand_client: The sync client used to interface with the Stagehand server. + """ + self.page = page + self._stagehand = stagehand_client + + def goto( + self, + url: str, + *, + referer: Optional[str] = None, + timeout: Optional[int] = None, + wait_until: Optional[str] = None + ): + """ + Navigate to URL using the Stagehand server synchronously. + + Args: + url (str): The URL to navigate to. + referer (Optional[str]): Optional referer URL. + timeout (Optional[int]): Optional navigation timeout in milliseconds. + wait_until (Optional[str]): Optional wait condition; one of ('load', 'domcontentloaded', 'networkidle', 'commit'). + + Returns: + The result from the Stagehand server's navigation execution. + """ + options = {} + if referer is not None: + options["referer"] = referer + if timeout is not None: + options["timeout"] = timeout + if wait_until is not None: + options["wait_until"] = wait_until + options["waitUntil"] = wait_until + + payload = {"url": url} + if options: + payload["options"] = options + + result = self._stagehand._execute("navigate", payload) + return result + + def act(self, options: Union[str, ActOptions, ObserveResult]) -> ActResult: + """ + Execute an AI action via the Stagehand server synchronously. + + Args: + options (Union[str, ActOptions, ObserveResult]): + - A string with the action command to be executed by the AI + - An ActOptions object encapsulating the action command and optional parameters + - An ObserveResult with selector and method fields for direct execution without LLM + + Returns: + ActResult: The result from the Stagehand server's action execution. + """ + # Check if options is an ObserveResult with both selector and method + if isinstance(options, ObserveResult) and hasattr(options, "selector") and hasattr(options, "method"): + # For ObserveResult, we directly pass it to the server which will + # execute the method against the selector + payload = options.model_dump(exclude_none=True, by_alias=True) + # Convert string to ActOptions if needed + elif isinstance(options, str): + options = ActOptions(action=options) + payload = options.model_dump(exclude_none=True, by_alias=True) + # Otherwise, it should be an ActOptions object + else: + payload = options.model_dump(exclude_none=True, by_alias=True) + + result = self._stagehand._execute("act", payload) + if isinstance(result, dict): + return ActResult(**result) + return result + + def observe(self, options: Union[str, ObserveOptions]) -> List[ObserveResult]: + """ + Make an AI observation via the Stagehand server synchronously. + + Args: + options (Union[str, ObserveOptions]): Either a string with the observation instruction + or a Pydantic model encapsulating the observation instruction. + + Returns: + List[ObserveResult]: A list of observation results from the Stagehand server. + """ + # Convert string to ObserveOptions if needed + if isinstance(options, str): + options = ObserveOptions(instruction=options) + + payload = options.model_dump(exclude_none=True, by_alias=True) + result = self._stagehand._execute("observe", payload) + + # Convert raw result to list of ObserveResult models + if isinstance(result, list): + return [ObserveResult(**item) for item in result] + elif isinstance(result, dict): + # If single dict, wrap in list + return [ObserveResult(**result)] + return [] + + def extract(self, options: Union[str, ExtractOptions]) -> ExtractResult: + """ + Extract data using AI via the Stagehand server synchronously. + + Args: + options (Union[str, ExtractOptions]): The extraction options describing what to extract and how. + + Returns: + ExtractResult: The result from the Stagehand server's extraction execution. + """ + # Convert string to ExtractOptions if needed + if isinstance(options, str): + options = ExtractOptions(instruction=options) + + payload = options.model_dump(exclude_none=True, by_alias=True) + result = self._stagehand._execute("extract", payload) + if isinstance(result, dict): + return ExtractResult(**result) + return result + + # Forward other Page methods to underlying Playwright page + def __getattr__(self, name): + """ + Forward attribute lookups to the underlying Playwright page. + + Args: + name (str): Name of the attribute to access. + + Returns: + The attribute from the underlying Playwright page. + """ + return getattr(self.page, name) \ No newline at end of file From e8dd96b5be4515bb21bbb12fa507a37654ba145b Mon Sep 17 00:00:00 2001 From: Filip Michalsky Date: Wed, 5 Mar 2025 20:27:47 -0500 Subject: [PATCH 04/16] refactor --- stagehand/__init__.py | 4 +- stagehand/base.py | 6 ++ stagehand/client.py | 68 ++++++-------------- stagehand/sync/__init__.py | 3 + stagehand/{sync_client.py => sync/client.py} | 10 ++- stagehand/{sync_page.py => sync/page.py} | 2 +- 6 files changed, 34 insertions(+), 59 deletions(-) create mode 100644 stagehand/sync/__init__.py rename stagehand/{sync_client.py => sync/client.py} (97%) rename stagehand/{sync_page.py => sync/page.py} (99%) diff --git a/stagehand/__init__.py b/stagehand/__init__.py index b5f30d42..d0e16fbe 100644 --- a/stagehand/__init__.py +++ b/stagehand/__init__.py @@ -1,5 +1,3 @@ from .client import Stagehand -from .sync_client import SyncStagehand -__version__ = "0.1.0" -__all__ = ["Stagehand", "SyncStagehand"] +__all__ = ["Stagehand"] diff --git a/stagehand/base.py b/stagehand/base.py index 5430bd12..0500ba58 100644 --- a/stagehand/base.py +++ b/stagehand/base.py @@ -5,6 +5,12 @@ from .config import StagehandConfig from .page import StagehandPage from .utils import default_log_handler +import os +import time +import logging + +logger = logging.getLogger(__name__) + class StagehandBase(ABC): """ diff --git a/stagehand/client.py b/stagehand/client.py index 8474b869..d47f2740 100644 --- a/stagehand/client.py +++ b/stagehand/client.py @@ -10,6 +10,7 @@ from dotenv import load_dotenv from playwright.async_api import async_playwright +from .base import StagehandBase from .config import StagehandConfig from .page import StagehandPage from .utils import default_log_handler, convert_dict_keys_to_camel_case @@ -19,7 +20,7 @@ logger = logging.getLogger(__name__) -class Stagehand: +class Stagehand(StagehandBase): """ Python client for interacting with a running Stagehand server and Browserbase remote headless browser. @@ -68,49 +69,20 @@ def __init__( timeout_settings (Optional[httpx.Timeout]): Optional custom timeout settings for httpx. model_client_options (Optional[Dict[str, Any]]): Optional model client options. """ - self.server_url = server_url or os.getenv("STAGEHAND_SERVER_URL") - - if config: - self.browserbase_api_key = ( - config.api_key - or browserbase_api_key - or os.getenv("BROWSERBASE_API_KEY") - ) - self.browserbase_project_id = ( - config.project_id - or browserbase_project_id - or os.getenv("BROWSERBASE_PROJECT_ID") - ) - self.model_api_key = os.getenv("MODEL_API_KEY") - self.session_id = config.browserbase_session_id or session_id - self.model_name = config.model_name or model_name - self.dom_settle_timeout_ms = ( - config.dom_settle_timeout_ms or dom_settle_timeout_ms - ) - self.debug_dom = ( - config.debug_dom if config.debug_dom is not None else debug_dom - ) - self._custom_logger = config.logger # For future integration if needed - # Additional config parameters available for future use: - self.headless = config.headless - self.enable_caching = config.enable_caching - self.model_client_options = model_client_options - else: - self.browserbase_api_key = browserbase_api_key or os.getenv( - "BROWSERBASE_API_KEY" - ) - self.browserbase_project_id = browserbase_project_id or os.getenv( - "BROWSERBASE_PROJECT_ID" - ) - self.model_api_key = model_api_key or os.getenv("MODEL_API_KEY") - self.session_id = session_id - self.model_name = model_name - self.dom_settle_timeout_ms = dom_settle_timeout_ms - self.debug_dom = debug_dom - self.model_client_options = model_client_options - - self.on_log = on_log - self.verbose = verbose + super().__init__( + config=config, + server_url=server_url, + session_id=session_id, + browserbase_api_key=browserbase_api_key, + browserbase_project_id=browserbase_project_id, + model_api_key=model_api_key, + on_log=on_log, + verbose=verbose, + model_name=model_name, + dom_settle_timeout_ms=dom_settle_timeout_ms, + debug_dom=debug_dom, + timeout_settings=timeout_settings, + ) self.httpx_client = httpx_client self.timeout_settings = timeout_settings or httpx.Timeout( connect=180.0, @@ -118,8 +90,7 @@ def __init__( write=180.0, pool=180.0, ) - self.streamed_response = True # Default to True for streamed responses - + self.model_client_options = model_client_options self._client: Optional[httpx.AsyncClient] = None self._playwright = None self._browser = None @@ -152,7 +123,6 @@ def _get_lock_for_session(self) -> asyncio.Lock: async def __aenter__(self): self._log("Entering Stagehand context manager (__aenter__)...", level=3) - # Just call init() if not already done await self.init() return self @@ -317,7 +287,7 @@ async def _create_session(self): "debugDom": self.debug_dom, } - if hasattr(self, "model_client_options") and self.model_client_options: + if self.model_client_options: payload["modelClientOptions"] = self.model_client_options headers = { @@ -359,7 +329,7 @@ async def _execute(self, method: str, payload: Dict[str, Any]) -> Any: headers["x-model-api-key"] = self.model_api_key modified_payload = dict(payload) - if hasattr(self, "model_client_options") and self.model_client_options and "modelClientOptions" not in modified_payload: + if self.model_client_options and "modelClientOptions" not in modified_payload: modified_payload["modelClientOptions"] = self.model_client_options # Convert snake_case keys to camelCase for the API diff --git a/stagehand/sync/__init__.py b/stagehand/sync/__init__.py new file mode 100644 index 00000000..5a01f936 --- /dev/null +++ b/stagehand/sync/__init__.py @@ -0,0 +1,3 @@ +from .client import Stagehand + +__all__ = ["Stagehand"] \ No newline at end of file diff --git a/stagehand/sync_client.py b/stagehand/sync/client.py similarity index 97% rename from stagehand/sync_client.py rename to stagehand/sync/client.py index 02569fe0..5435ef4b 100644 --- a/stagehand/sync_client.py +++ b/stagehand/sync/client.py @@ -1,4 +1,3 @@ -import asyncio import os import time import logging @@ -8,17 +7,16 @@ import httpx from playwright.sync_api import sync_playwright -from .base import StagehandBase -from .config import StagehandConfig +from ..base import StagehandBase +from ..config import StagehandConfig from .sync_page import SyncStagehandPage -from .utils import default_log_handler +from ..utils import default_log_handler logger = logging.getLogger(__name__) -class SyncStagehand(StagehandBase): +class Stagehand(StagehandBase): """ Synchronous implementation of the Stagehand client. - Wraps the async implementation using asyncio.run() """ def __init__( self, diff --git a/stagehand/sync_page.py b/stagehand/sync/page.py similarity index 99% rename from stagehand/sync_page.py rename to stagehand/sync/page.py index b32f9ce9..d31261b9 100644 --- a/stagehand/sync_page.py +++ b/stagehand/sync/page.py @@ -2,7 +2,7 @@ from playwright.sync_api import Page -from .schemas import ( +from ..schemas import ( ActOptions, ActResult, ExtractOptions, From 3b174d7ab2c8b79c2b037143863b5f3a96b93121 Mon Sep 17 00:00:00 2001 From: Filip Michalsky Date: Wed, 5 Mar 2025 20:42:27 -0500 Subject: [PATCH 05/16] more refactor of the sync sdk --- stagehand/client.py | 68 ++++++++++++++++++-------- stagehand/sync/client.py | 101 +++++++++++++++++++++++++-------------- 2 files changed, 115 insertions(+), 54 deletions(-) diff --git a/stagehand/client.py b/stagehand/client.py index d47f2740..8474b869 100644 --- a/stagehand/client.py +++ b/stagehand/client.py @@ -10,7 +10,6 @@ from dotenv import load_dotenv from playwright.async_api import async_playwright -from .base import StagehandBase from .config import StagehandConfig from .page import StagehandPage from .utils import default_log_handler, convert_dict_keys_to_camel_case @@ -20,7 +19,7 @@ logger = logging.getLogger(__name__) -class Stagehand(StagehandBase): +class Stagehand: """ Python client for interacting with a running Stagehand server and Browserbase remote headless browser. @@ -69,20 +68,49 @@ def __init__( timeout_settings (Optional[httpx.Timeout]): Optional custom timeout settings for httpx. model_client_options (Optional[Dict[str, Any]]): Optional model client options. """ - super().__init__( - config=config, - server_url=server_url, - session_id=session_id, - browserbase_api_key=browserbase_api_key, - browserbase_project_id=browserbase_project_id, - model_api_key=model_api_key, - on_log=on_log, - verbose=verbose, - model_name=model_name, - dom_settle_timeout_ms=dom_settle_timeout_ms, - debug_dom=debug_dom, - timeout_settings=timeout_settings, - ) + self.server_url = server_url or os.getenv("STAGEHAND_SERVER_URL") + + if config: + self.browserbase_api_key = ( + config.api_key + or browserbase_api_key + or os.getenv("BROWSERBASE_API_KEY") + ) + self.browserbase_project_id = ( + config.project_id + or browserbase_project_id + or os.getenv("BROWSERBASE_PROJECT_ID") + ) + self.model_api_key = os.getenv("MODEL_API_KEY") + self.session_id = config.browserbase_session_id or session_id + self.model_name = config.model_name or model_name + self.dom_settle_timeout_ms = ( + config.dom_settle_timeout_ms or dom_settle_timeout_ms + ) + self.debug_dom = ( + config.debug_dom if config.debug_dom is not None else debug_dom + ) + self._custom_logger = config.logger # For future integration if needed + # Additional config parameters available for future use: + self.headless = config.headless + self.enable_caching = config.enable_caching + self.model_client_options = model_client_options + else: + self.browserbase_api_key = browserbase_api_key or os.getenv( + "BROWSERBASE_API_KEY" + ) + self.browserbase_project_id = browserbase_project_id or os.getenv( + "BROWSERBASE_PROJECT_ID" + ) + self.model_api_key = model_api_key or os.getenv("MODEL_API_KEY") + self.session_id = session_id + self.model_name = model_name + self.dom_settle_timeout_ms = dom_settle_timeout_ms + self.debug_dom = debug_dom + self.model_client_options = model_client_options + + self.on_log = on_log + self.verbose = verbose self.httpx_client = httpx_client self.timeout_settings = timeout_settings or httpx.Timeout( connect=180.0, @@ -90,7 +118,8 @@ def __init__( write=180.0, pool=180.0, ) - self.model_client_options = model_client_options + self.streamed_response = True # Default to True for streamed responses + self._client: Optional[httpx.AsyncClient] = None self._playwright = None self._browser = None @@ -123,6 +152,7 @@ def _get_lock_for_session(self) -> asyncio.Lock: async def __aenter__(self): self._log("Entering Stagehand context manager (__aenter__)...", level=3) + # Just call init() if not already done await self.init() return self @@ -287,7 +317,7 @@ async def _create_session(self): "debugDom": self.debug_dom, } - if self.model_client_options: + if hasattr(self, "model_client_options") and self.model_client_options: payload["modelClientOptions"] = self.model_client_options headers = { @@ -329,7 +359,7 @@ async def _execute(self, method: str, payload: Dict[str, Any]) -> Any: headers["x-model-api-key"] = self.model_api_key modified_payload = dict(payload) - if self.model_client_options and "modelClientOptions" not in modified_payload: + if hasattr(self, "model_client_options") and self.model_client_options and "modelClientOptions" not in modified_payload: modified_payload["modelClientOptions"] = self.model_client_options # Convert snake_case keys to camelCase for the API diff --git a/stagehand/sync/client.py b/stagehand/sync/client.py index 5435ef4b..c6ea4115 100644 --- a/stagehand/sync/client.py +++ b/stagehand/sync/client.py @@ -2,15 +2,15 @@ import time import logging import json -from typing import Any, Dict, Optional +from typing import Any, Dict, Optional, Callable -import httpx +import requests from playwright.sync_api import sync_playwright from ..base import StagehandBase from ..config import StagehandConfig -from .sync_page import SyncStagehandPage -from ..utils import default_log_handler +from .page import SyncStagehandPage +from ..utils import default_log_handler, convert_dict_keys_to_camel_case logger = logging.getLogger(__name__) @@ -32,6 +32,7 @@ def __init__( dom_settle_timeout_ms: Optional[int] = None, debug_dom: Optional[bool] = None, timeout_settings: Optional[float] = None, + model_client_options: Optional[Dict[str, Any]] = None, ): super().__init__( config=config, @@ -47,11 +48,13 @@ def __init__( debug_dom=debug_dom, timeout_settings=timeout_settings, ) - self._client: Optional[httpx.Client] = None + self._client: Optional[requests.Session] = None self._playwright = None self._browser = None self._context = None self._playwright_page = None + self.model_client_options = model_client_options + self.streamed_response = True # Default to True for streamed responses def init(self): """ @@ -64,7 +67,7 @@ def init(self): self._log("Initializing Stagehand...", level=3) if not self._client: - self._client = httpx.Client(timeout=self.timeout_settings) + self._client = requests.Session() # Check server health self._check_server_health() @@ -190,6 +193,10 @@ def _create_session(self): "verbose": self.verbose, "debugDom": self.debug_dom, } + + if self.model_client_options: + payload["modelClientOptions"] = self.model_client_options + headers = { "x-bb-api-key": self.browserbase_api_key, "x-bb-project-id": self.browserbase_project_id, @@ -224,36 +231,60 @@ def _execute(self, method: str, payload: Dict[str, Any]) -> Any: if self.model_api_key: headers["x-model-api-key"] = self.model_api_key + modified_payload = dict(payload) + if self.model_client_options and "modelClientOptions" not in modified_payload: + modified_payload["modelClientOptions"] = self.model_client_options + + # Convert snake_case keys to camelCase for the API + modified_payload = convert_dict_keys_to_camel_case(modified_payload) + url = f"{self.server_url}/sessions/{self.session_id}/{method}" - self._log(f"Executing {method} with payload: {payload}", level=3) + self._log(f"\n==== EXECUTING {method.upper()} ====", level=3) + self._log(f"URL: {url}", level=3) + self._log(f"Payload: {modified_payload}", level=3) + self._log(f"Headers: {headers}", level=3) - response = self._client.post(url, json=payload, headers=headers, stream=True) - if response.status_code != 200: - raise RuntimeError(f"Error: {response.text}") - - for line in response.iter_lines(decode_unicode=True): - if not line.strip(): - continue - if line.startswith("data: "): - line = line[6:] - try: - message = json.loads(line) - msg_type = message.get("type") - if msg_type == "system": - status = message.get("data", {}).get("status") - if status == "finished": - return message.get("data", {}).get("result") - elif msg_type == "log": - log_msg = message.get("data", {}).get("message", "") - self._log(log_msg, level=3) - if self.on_log: - self.on_log(message) - else: - self._log(f"Unknown message type: {msg_type}", level=3) - if self.on_log: - self.on_log(message) - except json.JSONDecodeError: - self._log(f"Could not parse line as JSON: {line}", level=3) - continue + try: + response = self._client.post(url, json=modified_payload, headers=headers, stream=True) + if response.status_code != 200: + error_message = response.text + self._log(f"Error: {error_message}", level=3) + return None + + self._log("Starting to process streaming response...", level=3) + for line in response.iter_lines(decode_unicode=True): + if not line.strip(): + continue + + try: + if line.startswith("data: "): + line = line[6:] + + message = json.loads(line) + msg_type = message.get("type") + + if msg_type == "system": + status = message.get("data", {}).get("status") + if status == "finished": + result = message.get("data", {}).get("result") + self._log(f"FINISHED WITH RESULT: {result}", level=3) + return result + elif msg_type == "log": + log_msg = message.get("data", {}).get("message", "") + self._log(log_msg, level=3) + if self.on_log: + self.on_log(message) + else: + self._log(f"Unknown message type: {msg_type}", level=3) + if self.on_log: + self.on_log(message) + + except json.JSONDecodeError: + self._log(f"Could not parse line as JSON: {line}", level=3) + continue + except Exception as e: + self._log(f"EXCEPTION IN _EXECUTE: {str(e)}") + raise + self._log("==== ERROR: No 'finished' message received ====", level=3) raise RuntimeError("Server connection closed without sending 'finished' message") \ No newline at end of file From f027cef5805e2695502161da8cf415935b6a9b81 Mon Sep 17 00:00:00 2001 From: Filip Michalsky Date: Wed, 5 Mar 2025 21:02:44 -0500 Subject: [PATCH 06/16] add non-streaming response --- stagehand/base.py | 4 +++- stagehand/client.py | 25 ++++++++++++++++++++++++- stagehand/config.py | 4 ++++ stagehand/sync/client.py | 19 +++++++++++++------ 4 files changed, 44 insertions(+), 8 deletions(-) diff --git a/stagehand/base.py b/stagehand/base.py index 0500ba58..4d7e7b21 100644 --- a/stagehand/base.py +++ b/stagehand/base.py @@ -31,6 +31,7 @@ def __init__( dom_settle_timeout_ms: Optional[int] = None, debug_dom: Optional[bool] = None, timeout_settings: Optional[float] = None, + stream_response: Optional[bool] = None, ): """ Initialize the Stagehand client with common configuration. @@ -47,6 +48,7 @@ def __init__( self.model_name = config.model_name or model_name self.dom_settle_timeout_ms = config.dom_settle_timeout_ms or dom_settle_timeout_ms self.debug_dom = config.debug_dom if config.debug_dom is not None else debug_dom + self.streamed_response = config.stream_response if config.stream_response is not None else stream_response else: self.browserbase_api_key = browserbase_api_key or os.getenv("BROWSERBASE_API_KEY") self.browserbase_project_id = browserbase_project_id or os.getenv("BROWSERBASE_PROJECT_ID") @@ -55,12 +57,12 @@ def __init__( self.model_name = model_name self.dom_settle_timeout_ms = dom_settle_timeout_ms self.debug_dom = debug_dom + self.streamed_response = stream_response if stream_response is not None else True self.on_log = on_log self.verbose = verbose self.timeout_settings = timeout_settings or 180.0 - self.streamed_response = True self._initialized = False self._closed = False self.page: Optional[StagehandPage] = None diff --git a/stagehand/client.py b/stagehand/client.py index 8474b869..dc8e8e52 100644 --- a/stagehand/client.py +++ b/stagehand/client.py @@ -48,6 +48,7 @@ def __init__( httpx_client: Optional[httpx.AsyncClient] = None, timeout_settings: Optional[httpx.Timeout] = None, model_client_options: Optional[Dict[str, Any]] = None, + stream_response: Optional[bool] = None, ): """ Initialize the Stagehand client. @@ -67,6 +68,7 @@ def __init__( httpx_client (Optional[httpx.AsyncClient]): Optional custom httpx.AsyncClient instance. timeout_settings (Optional[httpx.Timeout]): Optional custom timeout settings for httpx. model_client_options (Optional[Dict[str, Any]]): Optional model client options. + stream_response (Optional[bool]): Whether to stream responses from the server. """ self.server_url = server_url or os.getenv("STAGEHAND_SERVER_URL") @@ -95,6 +97,7 @@ def __init__( self.headless = config.headless self.enable_caching = config.enable_caching self.model_client_options = model_client_options + self.streamed_response = config.stream_response if config.stream_response is not None else stream_response else: self.browserbase_api_key = browserbase_api_key or os.getenv( "BROWSERBASE_API_KEY" @@ -108,6 +111,7 @@ def __init__( self.dom_settle_timeout_ms = dom_settle_timeout_ms self.debug_dom = debug_dom self.model_client_options = model_client_options + self.streamed_response = stream_response if stream_response is not None else True self.on_log = on_log self.verbose = verbose @@ -118,7 +122,6 @@ def __init__( write=180.0, pool=180.0, ) - self.streamed_response = True # Default to True for streamed responses self._client: Optional[httpx.AsyncClient] = None self._playwright = None @@ -375,6 +378,26 @@ async def _execute(self, method: str, payload: Dict[str, Any]) -> Any: async with client: try: + if not self.streamed_response: + # For non-streaming responses, just return the final result + response = await client.post( + f"{self.server_url}/sessions/{self.session_id}/{method}", + json=modified_payload, + headers=headers, + ) + if response.status_code != 200: + error_text = await response.aread() + error_message = error_text.decode("utf-8") + self._log(f"Error: {error_message}", level=3) + return None + + data = response.json() + if data.get("success"): + return data.get("data", {}).get("result") + else: + raise RuntimeError(f"Request failed: {data.get('error', 'Unknown error')}") + + # Handle streaming response async with client.stream( "POST", f"{self.server_url}/sessions/{self.session_id}/{method}", diff --git a/stagehand/config.py b/stagehand/config.py index f3a678b6..d54d2b01 100644 --- a/stagehand/config.py +++ b/stagehand/config.py @@ -21,6 +21,7 @@ class StagehandConfig(BaseModel): browserbase_session_id (Optional[str]): Session ID for resuming Browserbase sessions. model_name (Optional[str]): Name of the model to use. self_heal (Optional[bool]): Enable self-healing functionality. + stream_response (Optional[bool]): Whether to stream responses from the server. """ env: str = "BROWSERBASE" @@ -56,6 +57,9 @@ class StagehandConfig(BaseModel): self_heal: Optional[bool] = Field( True, alias="selfHeal", description="Enable self-healing functionality" ) + stream_response: Optional[bool] = Field( + True, alias="streamResponse", description="Whether to stream responses from the server" + ) class Config: populate_by_name = True diff --git a/stagehand/sync/client.py b/stagehand/sync/client.py index c6ea4115..b63d9757 100644 --- a/stagehand/sync/client.py +++ b/stagehand/sync/client.py @@ -33,6 +33,7 @@ def __init__( debug_dom: Optional[bool] = None, timeout_settings: Optional[float] = None, model_client_options: Optional[Dict[str, Any]] = None, + stream_response: Optional[bool] = None, ): super().__init__( config=config, @@ -47,6 +48,7 @@ def __init__( dom_settle_timeout_ms=dom_settle_timeout_ms, debug_dom=debug_dom, timeout_settings=timeout_settings, + stream_response=stream_response, ) self._client: Optional[requests.Session] = None self._playwright = None @@ -245,12 +247,17 @@ def _execute(self, method: str, payload: Dict[str, Any]) -> Any: self._log(f"Headers: {headers}", level=3) try: - response = self._client.post(url, json=modified_payload, headers=headers, stream=True) - if response.status_code != 200: - error_message = response.text - self._log(f"Error: {error_message}", level=3) - return None - + if not self.streamed_response: + # For non-streaming responses, just return the final result + response = self._client.post(url, json=modified_payload, headers=headers) + if response.status_code != 200: + error_message = response.text + self._log(f"Error: {error_message}", level=3) + return None + + return response.json() # Return the raw response as the result + + # Handle streaming response self._log("Starting to process streaming response...", level=3) for line in response.iter_lines(decode_unicode=True): if not line.strip(): From 7e35507a3a85f86ef6554664863e4815b2608481 Mon Sep 17 00:00:00 2001 From: Filip Michalsky Date: Wed, 5 Mar 2025 21:13:39 -0500 Subject: [PATCH 07/16] client update --- stagehand/client.py | 65 +++++++++++++-------------------------------- 1 file changed, 19 insertions(+), 46 deletions(-) diff --git a/stagehand/client.py b/stagehand/client.py index dc8e8e52..6bbfe1d3 100644 --- a/stagehand/client.py +++ b/stagehand/client.py @@ -13,13 +13,14 @@ from .config import StagehandConfig from .page import StagehandPage from .utils import default_log_handler, convert_dict_keys_to_camel_case +from .base import StagehandBase load_dotenv() logger = logging.getLogger(__name__) -class Stagehand: +class Stagehand(StagehandBase): """ Python client for interacting with a running Stagehand server and Browserbase remote headless browser. @@ -70,51 +71,22 @@ def __init__( model_client_options (Optional[Dict[str, Any]]): Optional model client options. stream_response (Optional[bool]): Whether to stream responses from the server. """ - self.server_url = server_url or os.getenv("STAGEHAND_SERVER_URL") - - if config: - self.browserbase_api_key = ( - config.api_key - or browserbase_api_key - or os.getenv("BROWSERBASE_API_KEY") - ) - self.browserbase_project_id = ( - config.project_id - or browserbase_project_id - or os.getenv("BROWSERBASE_PROJECT_ID") - ) - self.model_api_key = os.getenv("MODEL_API_KEY") - self.session_id = config.browserbase_session_id or session_id - self.model_name = config.model_name or model_name - self.dom_settle_timeout_ms = ( - config.dom_settle_timeout_ms or dom_settle_timeout_ms - ) - self.debug_dom = ( - config.debug_dom if config.debug_dom is not None else debug_dom - ) - self._custom_logger = config.logger # For future integration if needed - # Additional config parameters available for future use: - self.headless = config.headless - self.enable_caching = config.enable_caching - self.model_client_options = model_client_options - self.streamed_response = config.stream_response if config.stream_response is not None else stream_response - else: - self.browserbase_api_key = browserbase_api_key or os.getenv( - "BROWSERBASE_API_KEY" - ) - self.browserbase_project_id = browserbase_project_id or os.getenv( - "BROWSERBASE_PROJECT_ID" - ) - self.model_api_key = model_api_key or os.getenv("MODEL_API_KEY") - self.session_id = session_id - self.model_name = model_name - self.dom_settle_timeout_ms = dom_settle_timeout_ms - self.debug_dom = debug_dom - self.model_client_options = model_client_options - self.streamed_response = stream_response if stream_response is not None else True - - self.on_log = on_log - self.verbose = verbose + super().__init__( + config=config, + server_url=server_url, + session_id=session_id, + browserbase_api_key=browserbase_api_key, + browserbase_project_id=browserbase_project_id, + model_api_key=model_api_key, + on_log=on_log, + verbose=verbose, + model_name=model_name, + dom_settle_timeout_ms=dom_settle_timeout_ms, + debug_dom=debug_dom, + timeout_settings=timeout_settings, + stream_response=stream_response, + ) + self.httpx_client = httpx_client self.timeout_settings = timeout_settings or httpx.Timeout( connect=180.0, @@ -122,6 +94,7 @@ def __init__( write=180.0, pool=180.0, ) + self.model_client_options = model_client_options self._client: Optional[httpx.AsyncClient] = None self._playwright = None From 69546b7a3c9dddfd484a0b6a786cac286ea69573 Mon Sep 17 00:00:00 2001 From: Filip Michalsky Date: Wed, 5 Mar 2025 21:22:26 -0500 Subject: [PATCH 08/16] move model_client_options to base class --- stagehand/base.py | 3 +++ stagehand/client.py | 2 +- 2 files changed, 4 insertions(+), 1 deletion(-) diff --git a/stagehand/base.py b/stagehand/base.py index 4d7e7b21..99dc3c4c 100644 --- a/stagehand/base.py +++ b/stagehand/base.py @@ -32,6 +32,7 @@ def __init__( debug_dom: Optional[bool] = None, timeout_settings: Optional[float] = None, stream_response: Optional[bool] = None, + model_client_options: Optional[Dict[str, Any]] = None, ): """ Initialize the Stagehand client with common configuration. @@ -49,6 +50,7 @@ def __init__( self.dom_settle_timeout_ms = config.dom_settle_timeout_ms or dom_settle_timeout_ms self.debug_dom = config.debug_dom if config.debug_dom is not None else debug_dom self.streamed_response = config.stream_response if config.stream_response is not None else stream_response + self.model_client_options = config.model_client_options or model_client_options else: self.browserbase_api_key = browserbase_api_key or os.getenv("BROWSERBASE_API_KEY") self.browserbase_project_id = browserbase_project_id or os.getenv("BROWSERBASE_PROJECT_ID") @@ -58,6 +60,7 @@ def __init__( self.dom_settle_timeout_ms = dom_settle_timeout_ms self.debug_dom = debug_dom self.streamed_response = stream_response if stream_response is not None else True + self.model_client_options = model_client_options self.on_log = on_log self.verbose = verbose diff --git a/stagehand/client.py b/stagehand/client.py index 6bbfe1d3..22339fee 100644 --- a/stagehand/client.py +++ b/stagehand/client.py @@ -85,6 +85,7 @@ def __init__( debug_dom=debug_dom, timeout_settings=timeout_settings, stream_response=stream_response, + model_client_options=model_client_options, ) self.httpx_client = httpx_client @@ -94,7 +95,6 @@ def __init__( write=180.0, pool=180.0, ) - self.model_client_options = model_client_options self._client: Optional[httpx.AsyncClient] = None self._playwright = None From c31cb5d8fd4a518c3b3ebb894f9166e6559cbf07 Mon Sep 17 00:00:00 2001 From: Filip Michalsky Date: Wed, 5 Mar 2025 21:24:42 -0500 Subject: [PATCH 09/16] add back version and bump --- stagehand/__init__.py | 6 +++++- 1 file changed, 5 insertions(+), 1 deletion(-) diff --git a/stagehand/__init__.py b/stagehand/__init__.py index d0e16fbe..a553d9bc 100644 --- a/stagehand/__init__.py +++ b/stagehand/__init__.py @@ -1,3 +1,7 @@ from .client import Stagehand +from .config import StagehandConfig +from .page import StagehandPage -__all__ = ["Stagehand"] +__version__ = "0.2.2" + +__all__ = ["Stagehand", "StagehandConfig", "StagehandPage"] From d8431735024edbc3821c460735bc0bf6abb13c52 Mon Sep 17 00:00:00 2001 From: Filip Michalsky Date: Wed, 5 Mar 2025 21:32:09 -0500 Subject: [PATCH 10/16] update readme and reqs --- README.md | 89 +++++++++++++++++++++++++++++++++++++++--------- requirements.txt | 1 + 2 files changed, 74 insertions(+), 16 deletions(-) diff --git a/README.md b/README.md index 48504178..8b508231 100644 --- a/README.md +++ b/README.md @@ -116,7 +116,9 @@ export STAGEHAND_SERVER_URL="url-of-stagehand-server" ## Quickstart -Below is a minimal example to get started with Stagehand using the new schema-based options: +Stagehand supports both asynchronous and synchronous usage. Here are examples for both approaches: + +### Asynchronous Usage ```python import asyncio @@ -164,26 +166,68 @@ if __name__ == "__main__": asyncio.run(main()) ``` +### Synchronous Usage -## Running Evaluations +```python +import os +from stagehand.sync.client import Stagehand +from stagehand.schemas import ActOptions, ExtractOptions +from pydantic import BaseModel +from dotenv import load_dotenv -To test all evaluations, run the following command in your terminal: +load_dotenv() +class DescriptionSchema(BaseModel): + description: str -```bash -python evals/run_all_evals.py -``` +def main(): + # Create a Stagehand client - it will automatically create a new session if needed + stagehand = Stagehand( + model_name="gpt-4", # Optional: defaults are available from the server + ) + + # Initialize Stagehand and create a new session + stagehand.init() + print(f"Created new session: {stagehand.session_id}") + + # Navigate to a webpage using local Playwright controls + stagehand.page.goto("https://www.example.com") + print("Navigation complete.") -This script will dynamically discover and execute every evaluation module within the `evals` directory and print the results for each. + # Perform an action using the AI (e.g. simulate a button click) + stagehand.page.act(ActOptions(action="click on the 'Quickstart' button")) + + # Extract data from the page with schema validation + data = stagehand.page.extract( + ExtractOptions( + instruction="extract the description of the page", + schemaDefinition=DescriptionSchema.model_json_schema() + ) + ) + description = data.get("description") if isinstance(data, dict) else data.description + print("Extracted description:", description) + stagehand.close() -## More Examples +if __name__ == "__main__": + main() +``` + +### Context Manager Usage -For further examples, check out the scripts in the `examples/` directory: +Both async and sync clients support context manager usage for automatic resource cleanup: + +```python +# Async context manager +async with Stagehand() as stagehand: + await stagehand.page.goto("https://www.example.com") + await stagehand.page.act(ActOptions(action="click the 'Login' button")) -1. **examples/example.py**: Demonstrates combined server-side/page navigation with AI-based actions. -2. **examples/extract-example.py**: Shows how to use the extract functionality with a JSON schema or a Pydantic model. -3. **examples/observe-example.py**: Demonstrates the observe functionality to get natural-language readings of the page. +# Sync context manager +with Stagehand() as stagehand: + stagehand.page.goto("https://www.example.com") + stagehand.page.act(ActOptions(action="click the 'Login' button")) +``` ## Configuration @@ -197,6 +241,8 @@ Stagehand can be configured via environment variables or through a `StagehandCon - `model_name`: Optional model name for the AI. - `dom_settle_timeout_ms`: Additional time (in ms) to have the DOM settle. - `debug_dom`: Enable debug mode for DOM operations. +- `stream_response`: Whether to stream responses from the server (default: True). +- `timeout_settings`: Custom timeout settings for HTTP requests. Example using a unified configuration: @@ -211,23 +257,34 @@ config = StagehandConfig( debug_dom=True, headless=False, dom_settle_timeout_ms=3000, - model_name="gpt-4o-mini", + model_name="gpt-4", model_client_options={"apiKey": os.getenv("MODEL_API_KEY")} ) + +# Use with async client +async with Stagehand(config=config) as stagehand: + await stagehand.page.goto("https://www.example.com") + +# Use with sync client +with Stagehand(config=config) as stagehand: + stagehand.page.goto("https://www.example.com") ``` ## Features - **AI-powered Browser Control**: Execute natural language instructions over a running browser. - **Validated Data Extraction**: Use JSON schemas (or Pydantic models) to extract and validate information from pages. -- **Async/Await Support**: Built using Python's asyncio, making it easy to build scalable web automation workflows. +- **Async/Sync Support**: Choose between asynchronous and synchronous APIs based on your needs. +- **Context Manager Support**: Automatic resource cleanup with async and sync context managers. - **Extensible**: Seamlessly extend Playwright functionality with AI enrichments. +- **Streaming Support**: Sreaming responses for better performance with long-running operations. Default True. ## Requirements - Python 3.7+ -- httpx -- asyncio +- httpx (for async client) +- requests (for sync client) +- asyncio (for async client) - pydantic - python-dotenv (optional, for .env support) - playwright diff --git a/requirements.txt b/requirements.txt index 641e9eeb..561c8dd1 100644 --- a/requirements.txt +++ b/requirements.txt @@ -3,4 +3,5 @@ asyncio>=3.4.3 python-dotenv>=1.0.0 pydantic>=1.10.0 playwright>=1.42.1 +requests>=2.31.0 rich \ No newline at end of file From 8e76ff3f060abd1cd2a8e5534f8e7477fb30a45c Mon Sep 17 00:00:00 2001 From: Filip Michalsky Date: Wed, 5 Mar 2025 21:41:10 -0500 Subject: [PATCH 11/16] move model options out of stagehand config --- README.md | 29 ++++++++--------------------- stagehand/base.py | 17 +++++++++-------- stagehand/client.py | 1 - stagehand/config.py | 6 +----- 4 files changed, 18 insertions(+), 35 deletions(-) diff --git a/README.md b/README.md index 8b508231..dfb75338 100644 --- a/README.md +++ b/README.md @@ -213,21 +213,16 @@ if __name__ == "__main__": main() ``` -### Context Manager Usage +This script will dynamically discover and execute every evaluation module within the `evals` directory and print the results for each. -Both async and sync clients support context manager usage for automatic resource cleanup: -```python -# Async context manager -async with Stagehand() as stagehand: - await stagehand.page.goto("https://www.example.com") - await stagehand.page.act(ActOptions(action="click the 'Login' button")) +## More Examples -# Sync context manager -with Stagehand() as stagehand: - stagehand.page.goto("https://www.example.com") - stagehand.page.act(ActOptions(action="click the 'Login' button")) -``` +For further examples, check out the scripts in the `examples/` directory: + +1. **examples/example.py**: Demonstrates combined server-side/page navigation with AI-based actions. +2. **examples/extract-example.py**: Shows how to use the extract functionality with a JSON schema or a Pydantic model. +3. **examples/observe-example.py**: Demonstrates the observe functionality to get natural-language readings of the page. ## Configuration @@ -257,17 +252,9 @@ config = StagehandConfig( debug_dom=True, headless=False, dom_settle_timeout_ms=3000, - model_name="gpt-4", + model_name="gpt-4o-mini", model_client_options={"apiKey": os.getenv("MODEL_API_KEY")} ) - -# Use with async client -async with Stagehand(config=config) as stagehand: - await stagehand.page.goto("https://www.example.com") - -# Use with sync client -with Stagehand(config=config) as stagehand: - stagehand.page.goto("https://www.example.com") ``` ## Features diff --git a/stagehand/base.py b/stagehand/base.py index 99dc3c4c..61dff7c1 100644 --- a/stagehand/base.py +++ b/stagehand/base.py @@ -42,25 +42,26 @@ def __init__( if config: self.browserbase_api_key = config.api_key or browserbase_api_key or os.getenv("BROWSERBASE_API_KEY") self.browserbase_project_id = config.project_id or browserbase_project_id or os.getenv("BROWSERBASE_PROJECT_ID") - self.model_api_key = model_api_key or ( - config.model_client_options.get("apiKey") if config.model_client_options else None - ) or os.getenv("MODEL_API_KEY") self.session_id = config.browserbase_session_id or session_id self.model_name = config.model_name or model_name self.dom_settle_timeout_ms = config.dom_settle_timeout_ms or dom_settle_timeout_ms self.debug_dom = config.debug_dom if config.debug_dom is not None else debug_dom - self.streamed_response = config.stream_response if config.stream_response is not None else stream_response - self.model_client_options = config.model_client_options or model_client_options else: self.browserbase_api_key = browserbase_api_key or os.getenv("BROWSERBASE_API_KEY") self.browserbase_project_id = browserbase_project_id or os.getenv("BROWSERBASE_PROJECT_ID") - self.model_api_key = model_api_key or os.getenv("MODEL_API_KEY") self.session_id = session_id self.model_name = model_name self.dom_settle_timeout_ms = dom_settle_timeout_ms self.debug_dom = debug_dom - self.streamed_response = stream_response if stream_response is not None else True - self.model_client_options = model_client_options + + # Handle model-related settings directly + self.model_api_key = model_api_key or os.getenv("MODEL_API_KEY") + self.model_client_options = model_client_options or {} + if self.model_api_key and "apiKey" not in self.model_client_options: + self.model_client_options["apiKey"] = self.model_api_key + + # Handle streaming response setting directly + self.streamed_response = stream_response if stream_response is not None else True self.on_log = on_log self.verbose = verbose diff --git a/stagehand/client.py b/stagehand/client.py index 22339fee..12441151 100644 --- a/stagehand/client.py +++ b/stagehand/client.py @@ -1,7 +1,6 @@ import asyncio import json import logging -import os import time from collections.abc import Awaitable from typing import Any, Callable, Dict, Optional diff --git a/stagehand/config.py b/stagehand/config.py index d54d2b01..8f0b1340 100644 --- a/stagehand/config.py +++ b/stagehand/config.py @@ -1,4 +1,4 @@ -from typing import Any, Callable, Optional +from typing import Any, Callable, Optional, Dict from pydantic import BaseModel, Field @@ -21,7 +21,6 @@ class StagehandConfig(BaseModel): browserbase_session_id (Optional[str]): Session ID for resuming Browserbase sessions. model_name (Optional[str]): Name of the model to use. self_heal (Optional[bool]): Enable self-healing functionality. - stream_response (Optional[bool]): Whether to stream responses from the server. """ env: str = "BROWSERBASE" @@ -57,9 +56,6 @@ class StagehandConfig(BaseModel): self_heal: Optional[bool] = Field( True, alias="selfHeal", description="Enable self-healing functionality" ) - stream_response: Optional[bool] = Field( - True, alias="streamResponse", description="Whether to stream responses from the server" - ) class Config: populate_by_name = True From 0a7840c0c22381152faf531696919fd4f6e72e8c Mon Sep 17 00:00:00 2001 From: Filip Michalsky Date: Wed, 5 Mar 2025 22:07:37 -0500 Subject: [PATCH 12/16] update --- examples/example_sync.py | 115 +++++++++++++++++++++++++++++++++++++++ 1 file changed, 115 insertions(+) create mode 100644 examples/example_sync.py diff --git a/examples/example_sync.py b/examples/example_sync.py new file mode 100644 index 00000000..896ac846 --- /dev/null +++ b/examples/example_sync.py @@ -0,0 +1,115 @@ +import logging +import os + +from dotenv import load_dotenv +from rich.console import Console +from rich.panel import Panel +from rich.theme import Theme + +from stagehand.sync import Stagehand +from stagehand.config import StagehandConfig + +# Create a custom theme for consistent styling +custom_theme = Theme( + { + "info": "cyan", + "success": "green", + "warning": "yellow", + "error": "red bold", + "highlight": "magenta", + "url": "blue underline", + } +) + +# Create a Rich console instance with our theme +console = Console(theme=custom_theme) + +load_dotenv() + +# Configure logging with Rich handler +logging.basicConfig( + level=logging.WARNING, # Feel free to change this to INFO or DEBUG to see more logs + format="%(asctime)s - %(name)s - %(levelname)s - %(message)s", + datefmt="%Y-%m-%d %H:%M:%S", +) + + +def main(): + # Build a unified configuration object for Stagehand + config = StagehandConfig( + env="BROWSERBASE", + api_key=os.getenv("BROWSERBASE_API_KEY"), + project_id=os.getenv("BROWSERBASE_PROJECT_ID"), + headless=False, + dom_settle_timeout_ms=3000, + model_name="gpt-4o", + model_client_options={"apiKey": os.getenv("MODEL_API_KEY")}, + ) + + # Create a Stagehand client using the configuration object. + stagehand = Stagehand( + config=config, server_url=os.getenv("STAGEHAND_SERVER_URL"), verbose=2 + ) + + # Initialize - this creates a new session automatically. + console.print("\n🚀 [info]Initializing Stagehand...[/]") + stagehand.init() + console.print(f"\n[yellow]Created new session:[/] {stagehand.session_id}") + console.print( + f"🌐 [white]View your live browser:[/] [url]https://www.browserbase.com/sessions/{stagehand.session_id}[/]" + ) + + import time + time.sleep(2) + + console.print("\n▶️ [highlight] Navigating[/] to Google") + stagehand.page.goto("https://google.com/") + console.print("✅ [success]Navigated to Google[/]") + + console.print("\n▶️ [highlight] Clicking[/] on About link") + # Click on the "About" link using Playwright + stagehand.page.get_by_role("link", name="About", exact=True).click() + console.print("✅ [success]Clicked on About link[/]") + + time.sleep(2) + console.print("\n▶️ [highlight] Navigating[/] back to Google") + stagehand.page.goto("https://google.com/") + console.print("✅ [success]Navigated back to Google[/]") + + console.print("\n▶️ [highlight] Performing action:[/] search for openai") + stagehand.page.act("search for openai") + stagehand.page.keyboard.press("Enter") + console.print("✅ [success]Performing Action:[/] Action completed successfully") + + console.print("\n▶️ [highlight] Observing page[/] for news button") + observed = stagehand.page.observe("find the news button on the page") + if len(observed) > 0: + element = observed[0] + console.print("✅ [success]Found element:[/] News button") + stagehand.page.act(element) + else: + console.print("❌ [error]No element found[/]") + + console.print("\n▶️ [highlight] Extracting[/] first search result") + data = stagehand.page.extract("extract the first result from the search") + console.print("📊 [info]Extracted data:[/]") + console.print_json(f"{data.model_dump_json()}") + + # Close the session + console.print("\n⏹️ [warning]Closing session...[/]") + stagehand.close() + console.print("✅ [success]Session closed successfully![/]") + console.rule("[bold]End of Example[/]") + + +if __name__ == "__main__": + # Add a fancy header + console.print( + "\n", + Panel.fit( + "[light_gray]Stagehand 🤘 Python Sync Example[/]", + border_style="green", + padding=(1, 10), + ), + ) + main() \ No newline at end of file From 19d9cc33336073f42b06a65b90fe7c09bb28696f Mon Sep 17 00:00:00 2001 From: Filip Michalsky Date: Wed, 5 Mar 2025 22:09:49 -0500 Subject: [PATCH 13/16] updates --- README.md | 22 +++++++++++++++++++++- stagehand/sync/client.py | 13 ++++++++++--- 2 files changed, 31 insertions(+), 4 deletions(-) diff --git a/README.md b/README.md index dfb75338..0c5bc46b 100644 --- a/README.md +++ b/README.md @@ -215,7 +215,6 @@ if __name__ == "__main__": This script will dynamically discover and execute every evaluation module within the `evals` directory and print the results for each. - ## More Examples For further examples, check out the scripts in the `examples/` directory: @@ -276,6 +275,27 @@ config = StagehandConfig( - python-dotenv (optional, for .env support) - playwright +## Contributing + +### Running Tests + +The project uses pytest for testing. To run the tests: + +```bash +# Install development dependencies +pip install -r requirements-dev.txt + +# Run all tests +pytest + +# Run specific test categories +pytest tests/unit/ # Run unit tests only +pytest tests/functional/ # Run functional tests only + +# Run tests with verbose output +pytest -v +``` + ## License MIT License (c) 2025 Browserbase, Inc. diff --git a/stagehand/sync/client.py b/stagehand/sync/client.py index b63d9757..a44641be 100644 --- a/stagehand/sync/client.py +++ b/stagehand/sync/client.py @@ -259,6 +259,12 @@ def _execute(self, method: str, payload: Dict[str, Any]) -> Any: # Handle streaming response self._log("Starting to process streaming response...", level=3) + response = self._client.post(url, json=modified_payload, headers=headers, stream=True) + if response.status_code != 200: + error_message = response.text + self._log(f"Error: {error_message}", level=3) + return None + for line in response.iter_lines(decode_unicode=True): if not line.strip(): continue @@ -280,11 +286,12 @@ def _execute(self, method: str, payload: Dict[str, Any]) -> Any: log_msg = message.get("data", {}).get("message", "") self._log(log_msg, level=3) if self.on_log: - self.on_log(message) + # For sync implementation, we just log the message directly + self._log(f"Log message: {log_msg}", level=3) else: self._log(f"Unknown message type: {msg_type}", level=3) if self.on_log: - self.on_log(message) + self._log(f"Unknown message: {message}", level=3) except json.JSONDecodeError: self._log(f"Could not parse line as JSON: {line}", level=3) @@ -294,4 +301,4 @@ def _execute(self, method: str, payload: Dict[str, Any]) -> Any: raise self._log("==== ERROR: No 'finished' message received ====", level=3) - raise RuntimeError("Server connection closed without sending 'finished' message") \ No newline at end of file + raise RuntimeError("Server connection closed without sending 'finished' message") \ No newline at end of file From efeb65cbb8381aad3d8985756effc511f7417637 Mon Sep 17 00:00:00 2001 From: Filip Michalsky Date: Wed, 5 Mar 2025 22:21:39 -0500 Subject: [PATCH 14/16] add tests --- tests/functional/test_sync_client.py | 79 ++++++++++++++++++++++++++++ 1 file changed, 79 insertions(+) create mode 100644 tests/functional/test_sync_client.py diff --git a/tests/functional/test_sync_client.py b/tests/functional/test_sync_client.py new file mode 100644 index 00000000..7bffec39 --- /dev/null +++ b/tests/functional/test_sync_client.py @@ -0,0 +1,79 @@ +import os +import pytest +from dotenv import load_dotenv +from stagehand.sync.client import Stagehand +from stagehand.config import StagehandConfig +from stagehand.schemas import ActOptions, ObserveOptions, ExtractOptions + +# Load environment variables +load_dotenv() + + +@pytest.fixture +def stagehand_client(): + """Fixture to create and manage a Stagehand client instance.""" + config = StagehandConfig( + env=( + "BROWSERBASE" + if os.getenv("BROWSERBASE_API_KEY") and os.getenv("BROWSERBASE_PROJECT_ID") + else "LOCAL" + ), + api_key=os.getenv("BROWSERBASE_API_KEY"), + project_id=os.getenv("BROWSERBASE_PROJECT_ID"), + debug_dom=True, + headless=True, # Run tests in headless mode + dom_settle_timeout_ms=3000, + model_name="gpt-4o-mini", + model_client_options={"apiKey": os.getenv("MODEL_API_KEY")}, + ) + + client = Stagehand( + config=config, server_url=os.getenv("STAGEHAND_SERVER_URL"), verbose=2 + ) + + # Initialize the client + client.init() + + yield client + + # Cleanup + client.close() + + +def test_navigation(stagehand_client): + """Test basic navigation functionality.""" + stagehand_client.page.goto("https://www.google.com") + # Add assertions based on the page state if needed + + +def test_act_command(stagehand_client): + """Test the act command functionality.""" + stagehand_client.page.goto("https://www.google.com") + stagehand_client.page.act(ActOptions(action="search for openai")) + # Add assertions based on the action result if needed + + +def test_observe_command(stagehand_client): + """Test the observe command functionality.""" + stagehand_client.page.goto("https://www.google.com") + result = stagehand_client.page.observe(ObserveOptions(instruction="find the search input box")) + assert result is not None + assert len(result) > 0 + assert hasattr(result[0], 'selector') + assert hasattr(result[0], 'description') + + +def test_extract_command(stagehand_client): + """Test the extract command functionality.""" + stagehand_client.page.goto("https://www.google.com") + result = stagehand_client.page.extract("title") + assert result is not None + assert hasattr(result, 'extraction') + assert isinstance(result.extraction, str) + assert result.extraction is not None + + +def test_session_management(stagehand_client): + """Test session management functionality.""" + assert stagehand_client.session_id is not None + assert isinstance(stagehand_client.session_id, str) From 2372507959da351e1183aec99cb002136302b198 Mon Sep 17 00:00:00 2001 From: Filip Michalsky Date: Wed, 5 Mar 2025 22:21:54 -0500 Subject: [PATCH 15/16] update readme --- README.md | 10 +--------- 1 file changed, 1 insertion(+), 9 deletions(-) diff --git a/README.md b/README.md index 0c5bc46b..8aed1066 100644 --- a/README.md +++ b/README.md @@ -285,15 +285,7 @@ The project uses pytest for testing. To run the tests: # Install development dependencies pip install -r requirements-dev.txt -# Run all tests -pytest - -# Run specific test categories -pytest tests/unit/ # Run unit tests only -pytest tests/functional/ # Run functional tests only - -# Run tests with verbose output -pytest -v +chmod +x run_tests.sh && ./run_tests.sh ``` ## License From bb30102b0829ce04d93fc67c1984130fe0ca9690 Mon Sep 17 00:00:00 2001 From: Filip Michalsky Date: Sun, 9 Mar 2025 21:46:05 -0400 Subject: [PATCH 16/16] remove unused import config, minor readme tweaks --- README.md | 50 +++++++++++++++++++++++++-------------------- stagehand/config.py | 2 +- 2 files changed, 29 insertions(+), 23 deletions(-) diff --git a/README.md b/README.md index 8aed1066..fe46f547 100644 --- a/README.md +++ b/README.md @@ -116,14 +116,13 @@ export STAGEHAND_SERVER_URL="url-of-stagehand-server" ## Quickstart -Stagehand supports both asynchronous and synchronous usage. Here are examples for both approaches: +Stagehand supports both synchronous and asynchronous usage. Here are examples for both approaches: -### Asynchronous Usage +### Synchronous Usage ```python -import asyncio import os -from stagehand.client import Stagehand +from stagehand.sync.client import Stagehand from stagehand.schemas import ActOptions, ExtractOptions from pydantic import BaseModel from dotenv import load_dotenv @@ -133,25 +132,25 @@ load_dotenv() class DescriptionSchema(BaseModel): description: str -async def main(): +def main(): # Create a Stagehand client - it will automatically create a new session if needed stagehand = Stagehand( - model_name="gpt-4o", # Optional: defaults are available from the server + model_name="gpt-4", # Optional: defaults are available from the server ) # Initialize Stagehand and create a new session - await stagehand.init() + stagehand.init() print(f"Created new session: {stagehand.session_id}") # Navigate to a webpage using local Playwright controls - await stagehand.page.goto("https://www.example.com") + stagehand.page.goto("https://www.example.com") print("Navigation complete.") # Perform an action using the AI (e.g. simulate a button click) - await stagehand.page.act(ActOptions(action="click on the 'Quickstart' button")) + stagehand.page.act("click on the 'Quickstart' button") # Extract data from the page with schema validation - data = await stagehand.page.extract( + data = stagehand.page.extract( ExtractOptions( instruction="extract the description of the page", schemaDefinition=DescriptionSchema.model_json_schema() @@ -160,17 +159,18 @@ async def main(): description = data.get("description") if isinstance(data, dict) else data.description print("Extracted description:", description) - await stagehand.close() + stagehand.close() if __name__ == "__main__": - asyncio.run(main()) + main() ``` -### Synchronous Usage +### Asynchronous Usage ```python +import asyncio import os -from stagehand.sync.client import Stagehand +from stagehand.client import Stagehand from stagehand.schemas import ActOptions, ExtractOptions from pydantic import BaseModel from dotenv import load_dotenv @@ -180,25 +180,25 @@ load_dotenv() class DescriptionSchema(BaseModel): description: str -def main(): +async def main(): # Create a Stagehand client - it will automatically create a new session if needed stagehand = Stagehand( - model_name="gpt-4", # Optional: defaults are available from the server + model_name="gpt-4o", # Optional: defaults are available from the server ) # Initialize Stagehand and create a new session - stagehand.init() + await stagehand.init() print(f"Created new session: {stagehand.session_id}") # Navigate to a webpage using local Playwright controls - stagehand.page.goto("https://www.example.com") + await stagehand.page.goto("https://www.example.com") print("Navigation complete.") # Perform an action using the AI (e.g. simulate a button click) - stagehand.page.act(ActOptions(action="click on the 'Quickstart' button")) + await stagehand.page.act("click on the 'Quickstart' button") # Extract data from the page with schema validation - data = stagehand.page.extract( + data = await stagehand.page.extract( ExtractOptions( instruction="extract the description of the page", schemaDefinition=DescriptionSchema.model_json_schema() @@ -207,12 +207,18 @@ def main(): description = data.get("description") if isinstance(data, dict) else data.description print("Extracted description:", description) - stagehand.close() + await stagehand.close() if __name__ == "__main__": - main() + asyncio.run(main()) ``` +## Evals + +To test all evaluations, run the following command in your terminal: + +`python evals/run_all_evals.py` + This script will dynamically discover and execute every evaluation module within the `evals` directory and print the results for each. ## More Examples diff --git a/stagehand/config.py b/stagehand/config.py index 8f0b1340..f3a678b6 100644 --- a/stagehand/config.py +++ b/stagehand/config.py @@ -1,4 +1,4 @@ -from typing import Any, Callable, Optional, Dict +from typing import Any, Callable, Optional from pydantic import BaseModel, Field