cookbook/langchain-web-agent.ipynb

{
 "cells": [
  {
   "cell_type": "markdown",
   "metadata": {
    "collapsed": false
   },
   "source": [
    "# Web browsing agents with langchain\n",
    "### Setup agent functions"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 1,
   "metadata": {
    "collapsed": false
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Requirement already satisfied: langchain in /Users/asimshrestha/Library/Caches/pypoetry/virtualenvs/tarsier-KjefggzA-py3.11/lib/python3.11/site-packages (0.0.334)\n",
      "Requirement already satisfied: PyYAML>=5.3 in /Users/asimshrestha/Library/Caches/pypoetry/virtualenvs/tarsier-KjefggzA-py3.11/lib/python3.11/site-packages (from langchain) (6.0.1)\n",
      "Requirement already satisfied: SQLAlchemy<3,>=1.4 in /Users/asimshrestha/Library/Caches/pypoetry/virtualenvs/tarsier-KjefggzA-py3.11/lib/python3.11/site-packages (from langchain) (2.0.23)\n",
      "Requirement already satisfied: aiohttp<4.0.0,>=3.8.3 in /Users/asimshrestha/Library/Caches/pypoetry/virtualenvs/tarsier-KjefggzA-py3.11/lib/python3.11/site-packages (from langchain) (3.8.6)\n",
      "Requirement already satisfied: anyio<4.0 in /Users/asimshrestha/Library/Caches/pypoetry/virtualenvs/tarsier-KjefggzA-py3.11/lib/python3.11/site-packages (from langchain) (3.7.1)\n",
      "Requirement already satisfied: dataclasses-json<0.7,>=0.5.7 in /Users/asimshrestha/Library/Caches/pypoetry/virtualenvs/tarsier-KjefggzA-py3.11/lib/python3.11/site-packages (from langchain) (0.6.2)\n",
      "Requirement already satisfied: jsonpatch<2.0,>=1.33 in /Users/asimshrestha/Library/Caches/pypoetry/virtualenvs/tarsier-KjefggzA-py3.11/lib/python3.11/site-packages (from langchain) (1.33)\n",
      "Requirement already satisfied: langsmith<0.1.0,>=0.0.62 in /Users/asimshrestha/Library/Caches/pypoetry/virtualenvs/tarsier-KjefggzA-py3.11/lib/python3.11/site-packages (from langchain) (0.0.63)\n",
      "Requirement already satisfied: numpy<2,>=1 in /Users/asimshrestha/Library/Caches/pypoetry/virtualenvs/tarsier-KjefggzA-py3.11/lib/python3.11/site-packages (from langchain) (1.25.2)\n",
      "Requirement already satisfied: pydantic<3,>=1 in /Users/asimshrestha/Library/Caches/pypoetry/virtualenvs/tarsier-KjefggzA-py3.11/lib/python3.11/site-packages (from langchain) (2.4.2)\n",
      "Requirement already satisfied: requests<3,>=2 in /Users/asimshrestha/Library/Caches/pypoetry/virtualenvs/tarsier-KjefggzA-py3.11/lib/python3.11/site-packages (from langchain) (2.31.0)\n",
      "Requirement already satisfied: tenacity<9.0.0,>=8.1.0 in /Users/asimshrestha/Library/Caches/pypoetry/virtualenvs/tarsier-KjefggzA-py3.11/lib/python3.11/site-packages (from langchain) (8.2.3)\n",
      "Requirement already satisfied: attrs>=17.3.0 in /Users/asimshrestha/Library/Caches/pypoetry/virtualenvs/tarsier-KjefggzA-py3.11/lib/python3.11/site-packages (from aiohttp<4.0.0,>=3.8.3->langchain) (23.1.0)\n",
      "Requirement already satisfied: charset-normalizer<4.0,>=2.0 in /Users/asimshrestha/Library/Caches/pypoetry/virtualenvs/tarsier-KjefggzA-py3.11/lib/python3.11/site-packages (from aiohttp<4.0.0,>=3.8.3->langchain) (3.3.2)\n",
      "Requirement already satisfied: multidict<7.0,>=4.5 in /Users/asimshrestha/Library/Caches/pypoetry/virtualenvs/tarsier-KjefggzA-py3.11/lib/python3.11/site-packages (from aiohttp<4.0.0,>=3.8.3->langchain) (6.0.4)\n",
      "Requirement already satisfied: async-timeout<5.0,>=4.0.0a3 in /Users/asimshrestha/Library/Caches/pypoetry/virtualenvs/tarsier-KjefggzA-py3.11/lib/python3.11/site-packages (from aiohttp<4.0.0,>=3.8.3->langchain) (4.0.3)\n",
      "Requirement already satisfied: yarl<2.0,>=1.0 in /Users/asimshrestha/Library/Caches/pypoetry/virtualenvs/tarsier-KjefggzA-py3.11/lib/python3.11/site-packages (from aiohttp<4.0.0,>=3.8.3->langchain) (1.9.2)\n",
      "Requirement already satisfied: frozenlist>=1.1.1 in /Users/asimshrestha/Library/Caches/pypoetry/virtualenvs/tarsier-KjefggzA-py3.11/lib/python3.11/site-packages (from aiohttp<4.0.0,>=3.8.3->langchain) (1.4.0)\n",
      "Requirement already satisfied: aiosignal>=1.1.2 in /Users/asimshrestha/Library/Caches/pypoetry/virtualenvs/tarsier-KjefggzA-py3.11/lib/python3.11/site-packages (from aiohttp<4.0.0,>=3.8.3->langchain) (1.3.1)\n",
      "Requirement already satisfied: idna>=2.8 in /Users/asimshrestha/Library/Caches/pypoetry/virtualenvs/tarsier-KjefggzA-py3.11/lib/python3.11/site-packages (from anyio<4.0->langchain) (3.4)\n",
      "Requirement already satisfied: sniffio>=1.1 in /Users/asimshrestha/Library/Caches/pypoetry/virtualenvs/tarsier-KjefggzA-py3.11/lib/python3.11/site-packages (from anyio<4.0->langchain) (1.3.0)\n",
      "Requirement already satisfied: marshmallow<4.0.0,>=3.18.0 in /Users/asimshrestha/Library/Caches/pypoetry/virtualenvs/tarsier-KjefggzA-py3.11/lib/python3.11/site-packages (from dataclasses-json<0.7,>=0.5.7->langchain) (3.20.1)\n",
      "Requirement already satisfied: typing-inspect<1,>=0.4.0 in /Users/asimshrestha/Library/Caches/pypoetry/virtualenvs/tarsier-KjefggzA-py3.11/lib/python3.11/site-packages (from dataclasses-json<0.7,>=0.5.7->langchain) (0.9.0)\n",
      "Requirement already satisfied: jsonpointer>=1.9 in /Users/asimshrestha/Library/Caches/pypoetry/virtualenvs/tarsier-KjefggzA-py3.11/lib/python3.11/site-packages (from jsonpatch<2.0,>=1.33->langchain) (2.4)\n",
      "Requirement already satisfied: annotated-types>=0.4.0 in /Users/asimshrestha/Library/Caches/pypoetry/virtualenvs/tarsier-KjefggzA-py3.11/lib/python3.11/site-packages (from pydantic<3,>=1->langchain) (0.6.0)\n",
      "Requirement already satisfied: pydantic-core==2.10.1 in /Users/asimshrestha/Library/Caches/pypoetry/virtualenvs/tarsier-KjefggzA-py3.11/lib/python3.11/site-packages (from pydantic<3,>=1->langchain) (2.10.1)\n",
      "Requirement already satisfied: typing-extensions>=4.6.1 in /Users/asimshrestha/Library/Caches/pypoetry/virtualenvs/tarsier-KjefggzA-py3.11/lib/python3.11/site-packages (from pydantic<3,>=1->langchain) (4.8.0)\n",
      "Requirement already satisfied: urllib3<3,>=1.21.1 in /Users/asimshrestha/Library/Caches/pypoetry/virtualenvs/tarsier-KjefggzA-py3.11/lib/python3.11/site-packages (from requests<3,>=2->langchain) (2.0.7)\n",
      "Requirement already satisfied: certifi>=2017.4.17 in /Users/asimshrestha/Library/Caches/pypoetry/virtualenvs/tarsier-KjefggzA-py3.11/lib/python3.11/site-packages (from requests<3,>=2->langchain) (2023.7.22)\n",
      "Requirement already satisfied: packaging>=17.0 in /Users/asimshrestha/Library/Caches/pypoetry/virtualenvs/tarsier-KjefggzA-py3.11/lib/python3.11/site-packages (from marshmallow<4.0.0,>=3.18.0->dataclasses-json<0.7,>=0.5.7->langchain) (23.2)\n",
      "Requirement already satisfied: mypy-extensions>=0.3.0 in /Users/asimshrestha/Library/Caches/pypoetry/virtualenvs/tarsier-KjefggzA-py3.11/lib/python3.11/site-packages (from typing-inspect<1,>=0.4.0->dataclasses-json<0.7,>=0.5.7->langchain) (1.0.0)\n",
      "\n",
      "\u001b[1m[\u001b[0m\u001b[34;49mnotice\u001b[0m\u001b[1;39;49m]\u001b[0m\u001b[39;49m A new release of pip is available: \u001b[0m\u001b[31;49m23.1.2\u001b[0m\u001b[39;49m -> \u001b[0m\u001b[32;49m23.3.1\u001b[0m\n",
      "\u001b[1m[\u001b[0m\u001b[34;49mnotice\u001b[0m\u001b[1;39;49m]\u001b[0m\u001b[39;49m To update, run: \u001b[0m\u001b[32;49mpip install --upgrade pip\u001b[0m\n",
      "Note: you may need to restart the kernel to use updated packages.\n",
      "Requirement already satisfied: playwright in /Users/asimshrestha/Library/Caches/pypoetry/virtualenvs/tarsier-KjefggzA-py3.11/lib/python3.11/site-packages (1.39.0)\n",
      "Requirement already satisfied: greenlet==3.0.0 in /Users/asimshrestha/Library/Caches/pypoetry/virtualenvs/tarsier-KjefggzA-py3.11/lib/python3.11/site-packages (from playwright) (3.0.0)\n",
      "Requirement already satisfied: pyee==11.0.1 in /Users/asimshrestha/Library/Caches/pypoetry/virtualenvs/tarsier-KjefggzA-py3.11/lib/python3.11/site-packages (from playwright) (11.0.1)\n",
      "Requirement already satisfied: typing-extensions in /Users/asimshrestha/Library/Caches/pypoetry/virtualenvs/tarsier-KjefggzA-py3.11/lib/python3.11/site-packages (from pyee==11.0.1->playwright) (4.8.0)\n",
      "\n",
      "\u001b[1m[\u001b[0m\u001b[34;49mnotice\u001b[0m\u001b[1;39;49m]\u001b[0m\u001b[39;49m A new release of pip is available: \u001b[0m\u001b[31;49m23.1.2\u001b[0m\u001b[39;49m -> \u001b[0m\u001b[32;49m23.3.1\u001b[0m\n",
      "\u001b[1m[\u001b[0m\u001b[34;49mnotice\u001b[0m\u001b[1;39;49m]\u001b[0m\u001b[39;49m To update, run: \u001b[0m\u001b[32;49mpip install --upgrade pip\u001b[0m\n",
      "Note: you may need to restart the kernel to use updated packages.\n",
      "Requirement already satisfied: tarsier in /Users/asimshrestha/Library/Caches/pypoetry/virtualenvs/tarsier-KjefggzA-py3.11/lib/python3.11/site-packages (0.1.0)\n",
      "Requirement already satisfied: google-cloud-vision<4.0.0,>=3.4.5 in /Users/asimshrestha/Library/Caches/pypoetry/virtualenvs/tarsier-KjefggzA-py3.11/lib/python3.11/site-packages (from tarsier) (3.4.5)\n",
      "Requirement already satisfied: playwright<2.0.0,>=1.39.0 in /Users/asimshrestha/Library/Caches/pypoetry/virtualenvs/tarsier-KjefggzA-py3.11/lib/python3.11/site-packages (from tarsier) (1.39.0)\n",
      "Requirement already satisfied: pydantic<3.0.0,>=2.4.2 in /Users/asimshrestha/Library/Caches/pypoetry/virtualenvs/tarsier-KjefggzA-py3.11/lib/python3.11/site-packages (from tarsier) (2.4.2)\n",
      "Requirement already satisfied: selenium<5.0.0,>=4.15.2 in /Users/asimshrestha/Library/Caches/pypoetry/virtualenvs/tarsier-KjefggzA-py3.11/lib/python3.11/site-packages (from tarsier) (4.15.2)\n",
      "Requirement already satisfied: google-api-core[grpc]!=2.0.*,!=2.1.*,!=2.10.*,!=2.2.*,!=2.3.*,!=2.4.*,!=2.5.*,!=2.6.*,!=2.7.*,!=2.8.*,!=2.9.*,<3.0.0dev,>=1.34.0 in /Users/asimshrestha/Library/Caches/pypoetry/virtualenvs/tarsier-KjefggzA-py3.11/lib/python3.11/site-packages (from google-cloud-vision<4.0.0,>=3.4.5->tarsier) (2.14.0)\n",
      "Requirement already satisfied: proto-plus<2.0.0dev,>=1.22.0 in /Users/asimshrestha/Library/Caches/pypoetry/virtualenvs/tarsier-KjefggzA-py3.11/lib/python3.11/site-packages (from google-cloud-vision<4.0.0,>=3.4.5->tarsier) (1.22.3)\n",
      "Requirement already satisfied: protobuf!=3.20.0,!=3.20.1,!=4.21.0,!=4.21.1,!=4.21.2,!=4.21.3,!=4.21.4,!=4.21.5,<5.0.0dev,>=3.19.5 in /Users/asimshrestha/Library/Caches/pypoetry/virtualenvs/tarsier-KjefggzA-py3.11/lib/python3.11/site-packages (from google-cloud-vision<4.0.0,>=3.4.5->tarsier) (4.25.0)\n",
      "Requirement already satisfied: greenlet==3.0.0 in /Users/asimshrestha/Library/Caches/pypoetry/virtualenvs/tarsier-KjefggzA-py3.11/lib/python3.11/site-packages (from playwright<2.0.0,>=1.39.0->tarsier) (3.0.0)\n",
      "Requirement already satisfied: pyee==11.0.1 in /Users/asimshrestha/Library/Caches/pypoetry/virtualenvs/tarsier-KjefggzA-py3.11/lib/python3.11/site-packages (from playwright<2.0.0,>=1.39.0->tarsier) (11.0.1)\n",
      "Requirement already satisfied: typing-extensions in /Users/asimshrestha/Library/Caches/pypoetry/virtualenvs/tarsier-KjefggzA-py3.11/lib/python3.11/site-packages (from pyee==11.0.1->playwright<2.0.0,>=1.39.0->tarsier) (4.8.0)\n",
      "Requirement already satisfied: annotated-types>=0.4.0 in /Users/asimshrestha/Library/Caches/pypoetry/virtualenvs/tarsier-KjefggzA-py3.11/lib/python3.11/site-packages (from pydantic<3.0.0,>=2.4.2->tarsier) (0.6.0)\n",
      "Requirement already satisfied: pydantic-core==2.10.1 in /Users/asimshrestha/Library/Caches/pypoetry/virtualenvs/tarsier-KjefggzA-py3.11/lib/python3.11/site-packages (from pydantic<3.0.0,>=2.4.2->tarsier) (2.10.1)\n",
      "Requirement already satisfied: urllib3[socks]<3,>=1.26 in /Users/asimshrestha/Library/Caches/pypoetry/virtualenvs/tarsier-KjefggzA-py3.11/lib/python3.11/site-packages (from selenium<5.0.0,>=4.15.2->tarsier) (2.0.7)\n",
      "Requirement already satisfied: trio~=0.17 in /Users/asimshrestha/Library/Caches/pypoetry/virtualenvs/tarsier-KjefggzA-py3.11/lib/python3.11/site-packages (from selenium<5.0.0,>=4.15.2->tarsier) (0.23.1)\n",
      "Requirement already satisfied: trio-websocket~=0.9 in /Users/asimshrestha/Library/Caches/pypoetry/virtualenvs/tarsier-KjefggzA-py3.11/lib/python3.11/site-packages (from selenium<5.0.0,>=4.15.2->tarsier) (0.11.1)\n",
      "Requirement already satisfied: certifi>=2021.10.8 in /Users/asimshrestha/Library/Caches/pypoetry/virtualenvs/tarsier-KjefggzA-py3.11/lib/python3.11/site-packages (from selenium<5.0.0,>=4.15.2->tarsier) (2023.7.22)\n",
      "Requirement already satisfied: googleapis-common-protos<2.0.dev0,>=1.56.2 in /Users/asimshrestha/Library/Caches/pypoetry/virtualenvs/tarsier-KjefggzA-py3.11/lib/python3.11/site-packages (from google-api-core[grpc]!=2.0.*,!=2.1.*,!=2.10.*,!=2.2.*,!=2.3.*,!=2.4.*,!=2.5.*,!=2.6.*,!=2.7.*,!=2.8.*,!=2.9.*,<3.0.0dev,>=1.34.0->google-cloud-vision<4.0.0,>=3.4.5->tarsier) (1.61.0)\n",
      "Requirement already satisfied: google-auth<3.0.dev0,>=2.14.1 in /Users/asimshrestha/Library/Caches/pypoetry/virtualenvs/tarsier-KjefggzA-py3.11/lib/python3.11/site-packages (from google-api-core[grpc]!=2.0.*,!=2.1.*,!=2.10.*,!=2.2.*,!=2.3.*,!=2.4.*,!=2.5.*,!=2.6.*,!=2.7.*,!=2.8.*,!=2.9.*,<3.0.0dev,>=1.34.0->google-cloud-vision<4.0.0,>=3.4.5->tarsier) (2.23.4)\n",
      "Requirement already satisfied: requests<3.0.0.dev0,>=2.18.0 in /Users/asimshrestha/Library/Caches/pypoetry/virtualenvs/tarsier-KjefggzA-py3.11/lib/python3.11/site-packages (from google-api-core[grpc]!=2.0.*,!=2.1.*,!=2.10.*,!=2.2.*,!=2.3.*,!=2.4.*,!=2.5.*,!=2.6.*,!=2.7.*,!=2.8.*,!=2.9.*,<3.0.0dev,>=1.34.0->google-cloud-vision<4.0.0,>=3.4.5->tarsier) (2.31.0)\n",
      "Requirement already satisfied: grpcio<2.0dev,>=1.33.2 in /Users/asimshrestha/Library/Caches/pypoetry/virtualenvs/tarsier-KjefggzA-py3.11/lib/python3.11/site-packages (from google-api-core[grpc]!=2.0.*,!=2.1.*,!=2.10.*,!=2.2.*,!=2.3.*,!=2.4.*,!=2.5.*,!=2.6.*,!=2.7.*,!=2.8.*,!=2.9.*,<3.0.0dev,>=1.34.0->google-cloud-vision<4.0.0,>=3.4.5->tarsier) (1.59.2)\n",
      "Requirement already satisfied: grpcio-status<2.0.dev0,>=1.33.2 in /Users/asimshrestha/Library/Caches/pypoetry/virtualenvs/tarsier-KjefggzA-py3.11/lib/python3.11/site-packages (from google-api-core[grpc]!=2.0.*,!=2.1.*,!=2.10.*,!=2.2.*,!=2.3.*,!=2.4.*,!=2.5.*,!=2.6.*,!=2.7.*,!=2.8.*,!=2.9.*,<3.0.0dev,>=1.34.0->google-cloud-vision<4.0.0,>=3.4.5->tarsier) (1.59.2)\n",
      "Requirement already satisfied: attrs>=20.1.0 in /Users/asimshrestha/Library/Caches/pypoetry/virtualenvs/tarsier-KjefggzA-py3.11/lib/python3.11/site-packages (from trio~=0.17->selenium<5.0.0,>=4.15.2->tarsier) (23.1.0)\n",
      "Requirement already satisfied: sortedcontainers in /Users/asimshrestha/Library/Caches/pypoetry/virtualenvs/tarsier-KjefggzA-py3.11/lib/python3.11/site-packages (from trio~=0.17->selenium<5.0.0,>=4.15.2->tarsier) (2.4.0)\n",
      "Requirement already satisfied: idna in /Users/asimshrestha/Library/Caches/pypoetry/virtualenvs/tarsier-KjefggzA-py3.11/lib/python3.11/site-packages (from trio~=0.17->selenium<5.0.0,>=4.15.2->tarsier) (3.4)\n",
      "Requirement already satisfied: outcome in /Users/asimshrestha/Library/Caches/pypoetry/virtualenvs/tarsier-KjefggzA-py3.11/lib/python3.11/site-packages (from trio~=0.17->selenium<5.0.0,>=4.15.2->tarsier) (1.3.0.post0)\n",
      "Requirement already satisfied: sniffio>=1.3.0 in /Users/asimshrestha/Library/Caches/pypoetry/virtualenvs/tarsier-KjefggzA-py3.11/lib/python3.11/site-packages (from trio~=0.17->selenium<5.0.0,>=4.15.2->tarsier) (1.3.0)\n",
      "Requirement already satisfied: wsproto>=0.14 in /Users/asimshrestha/Library/Caches/pypoetry/virtualenvs/tarsier-KjefggzA-py3.11/lib/python3.11/site-packages (from trio-websocket~=0.9->selenium<5.0.0,>=4.15.2->tarsier) (1.2.0)\n",
      "Requirement already satisfied: pysocks!=1.5.7,<2.0,>=1.5.6 in /Users/asimshrestha/Library/Caches/pypoetry/virtualenvs/tarsier-KjefggzA-py3.11/lib/python3.11/site-packages (from urllib3[socks]<3,>=1.26->selenium<5.0.0,>=4.15.2->tarsier) (1.7.1)\n",
      "Requirement already satisfied: cachetools<6.0,>=2.0.0 in /Users/asimshrestha/Library/Caches/pypoetry/virtualenvs/tarsier-KjefggzA-py3.11/lib/python3.11/site-packages (from google-auth<3.0.dev0,>=2.14.1->google-api-core[grpc]!=2.0.*,!=2.1.*,!=2.10.*,!=2.2.*,!=2.3.*,!=2.4.*,!=2.5.*,!=2.6.*,!=2.7.*,!=2.8.*,!=2.9.*,<3.0.0dev,>=1.34.0->google-cloud-vision<4.0.0,>=3.4.5->tarsier) (5.3.2)\n",
      "Requirement already satisfied: pyasn1-modules>=0.2.1 in /Users/asimshrestha/Library/Caches/pypoetry/virtualenvs/tarsier-KjefggzA-py3.11/lib/python3.11/site-packages (from google-auth<3.0.dev0,>=2.14.1->google-api-core[grpc]!=2.0.*,!=2.1.*,!=2.10.*,!=2.2.*,!=2.3.*,!=2.4.*,!=2.5.*,!=2.6.*,!=2.7.*,!=2.8.*,!=2.9.*,<3.0.0dev,>=1.34.0->google-cloud-vision<4.0.0,>=3.4.5->tarsier) (0.3.0)\n",
      "Requirement already satisfied: rsa<5,>=3.1.4 in /Users/asimshrestha/Library/Caches/pypoetry/virtualenvs/tarsier-KjefggzA-py3.11/lib/python3.11/site-packages (from google-auth<3.0.dev0,>=2.14.1->google-api-core[grpc]!=2.0.*,!=2.1.*,!=2.10.*,!=2.2.*,!=2.3.*,!=2.4.*,!=2.5.*,!=2.6.*,!=2.7.*,!=2.8.*,!=2.9.*,<3.0.0dev,>=1.34.0->google-cloud-vision<4.0.0,>=3.4.5->tarsier) (4.9)\n",
      "Requirement already satisfied: charset-normalizer<4,>=2 in /Users/asimshrestha/Library/Caches/pypoetry/virtualenvs/tarsier-KjefggzA-py3.11/lib/python3.11/site-packages (from requests<3.0.0.dev0,>=2.18.0->google-api-core[grpc]!=2.0.*,!=2.1.*,!=2.10.*,!=2.2.*,!=2.3.*,!=2.4.*,!=2.5.*,!=2.6.*,!=2.7.*,!=2.8.*,!=2.9.*,<3.0.0dev,>=1.34.0->google-cloud-vision<4.0.0,>=3.4.5->tarsier) (3.3.2)\n",
      "Requirement already satisfied: h11<1,>=0.9.0 in /Users/asimshrestha/Library/Caches/pypoetry/virtualenvs/tarsier-KjefggzA-py3.11/lib/python3.11/site-packages (from wsproto>=0.14->trio-websocket~=0.9->selenium<5.0.0,>=4.15.2->tarsier) (0.14.0)\n",
      "Requirement already satisfied: pyasn1<0.6.0,>=0.4.6 in /Users/asimshrestha/Library/Caches/pypoetry/virtualenvs/tarsier-KjefggzA-py3.11/lib/python3.11/site-packages (from pyasn1-modules>=0.2.1->google-auth<3.0.dev0,>=2.14.1->google-api-core[grpc]!=2.0.*,!=2.1.*,!=2.10.*,!=2.2.*,!=2.3.*,!=2.4.*,!=2.5.*,!=2.6.*,!=2.7.*,!=2.8.*,!=2.9.*,<3.0.0dev,>=1.34.0->google-cloud-vision<4.0.0,>=3.4.5->tarsier) (0.5.0)\n",
      "\n",
      "\u001b[1m[\u001b[0m\u001b[34;49mnotice\u001b[0m\u001b[1;39;49m]\u001b[0m\u001b[39;49m A new release of pip is available: \u001b[0m\u001b[31;49m23.1.2\u001b[0m\u001b[39;49m -> \u001b[0m\u001b[32;49m23.3.1\u001b[0m\n",
      "\u001b[1m[\u001b[0m\u001b[34;49mnotice\u001b[0m\u001b[1;39;49m]\u001b[0m\u001b[39;49m To update, run: \u001b[0m\u001b[32;49mpip install --upgrade pip\u001b[0m\n",
      "Note: you may need to restart the kernel to use updated packages.\n"
     ]
    }
   ],
   "source": [
    "%pip install langchain\n",
    "%pip install playwright\n",
    "%pip install tarsier\n",
    "!playwright install"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 2,
   "metadata": {
    "ExecuteTime": {
     "end_time": "2023-11-11T08:46:31.531676Z",
     "start_time": "2023-11-11T08:46:31.501731Z"
    },
    "collapsed": false
   },
   "outputs": [],
   "source": [
    "from langchain.chat_models import ChatOpenAI\n",
    "from langchain.agents import tool\n",
    "from playwright.async_api import async_playwright\n",
    "from tarsier import Tarsier, GoogleVisionOCRService\n",
    "import os\n",
    "\n",
    "# Setup Playwright\n",
    "p = await async_playwright().__aenter__()\n",
    "browser = await p.chromium.launch(headless=False)\n",
    "page = await browser.new_page()\n",
    "\n",
    "# Setup Creds\n",
    "os.environ[\"OPENAI_API_KEY\"] = \"sk-...\"\n",
    "google_cloud_credentials = {...}\n",
    "\n",
    "# Setup Tarsier\n",
    "ocr_service = GoogleVisionOCRService(google_cloud_credentials)\n",
    "tarsier = Tarsier(ocr_service)\n",
    "tag_to_xpath = {}\n",
    "\n",
    "\n",
    "# Define tools/actions\n",
    "@tool\n",
    "async def read_page() -> str:\n",
    "    \"\"\"\n",
    "    Use to read the current state of the page\n",
    "    \"\"\"\n",
    "    return await read_page_impl()\n",
    "\n",
    "\n",
    "async def read_page_impl() -> str:\n",
    "    page_text, inner_tag_to_xpath = await tarsier.page_to_text(page)\n",
    "    tag_to_xpath.clear()\n",
    "    tag_to_xpath.update(inner_tag_to_xpath)\n",
    "    return page_text\n",
    "\n",
    "\n",
    "@tool\n",
    "async def click(element_id: int) -> str:\n",
    "    \"\"\"\n",
    "    Click on an element based on element_id and return the new page state\n",
    "    \"\"\"\n",
    "    x_path = tag_to_xpath[element_id]\n",
    "    print(x_path)\n",
    "    element = page.locator(x_path)\n",
    "    await element.scroll_into_view_if_needed()\n",
    "    await page.wait_for_timeout(1000)\n",
    "    await element.click()\n",
    "    await page.wait_for_timeout(2000)\n",
    "    return await read_page_impl()\n",
    "\n",
    "\n",
    "@tool\n",
    "async def type_text(element_id: int, text: str) -> str:\n",
    "    \"\"\"\n",
    "    Input text into a textbox based on element_id and return the new page state\n",
    "    \"\"\"\n",
    "    x_path = tag_to_xpath[element_id]\n",
    "    print(x_path)\n",
    "    await page.locator(x_path).press_sequentially(text)\n",
    "    return await read_page_impl()\n",
    "\n",
    "\n",
    "@tool\n",
    "async def press_key(key: str) -> str:\n",
    "    \"\"\"\n",
    "    Press a key on the keyboard and return the new page state\n",
    "    \"\"\"\n",
    "    await page.keyboard.press(key)\n",
    "    await page.wait_for_timeout(2000)\n",
    "    return await read_page_impl()"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {
    "collapsed": false
   },
   "source": [
    "### GPT4-V + Tarsier"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 3,
   "metadata": {
    "ExecuteTime": {
     "end_time": "2023-11-11T07:49:41.232707Z",
     "start_time": "2023-11-11T07:49:41.103809Z"
    },
    "collapsed": false
   },
   "outputs": [],
   "source": [
    "from langchain.prompts import ChatPromptTemplate\n",
    "from langchain.agents import initialize_agent, AgentType\n",
    "from langchain.chains import LLMChain\n",
    "\n",
    "template = \"\"\"\n",
    "You are a web interaction agent. Use the read page tool to understand where you currently are. \n",
    "You will be passed in OCR text of a web page where element ids are to the left of elements. \n",
    "\n",
    "You have access to the following tools:\n",
    "{tools}\n",
    "\n",
    "Use the following format:\n",
    "\n",
    "Question: the input question you must answer\n",
    "Thought: you should always think about what to do\n",
    "Action: the action to take, should be one of [{tool_names}]\n",
    "Action Input: the input to the action\n",
    "Observation: the result of the action\n",
    "... (this Thought/Action/Action Input/Observation can repeat N times)\n",
    "Thought: I now know the final answer\n",
    "Final Answer: the final answer to the original input question\n",
    "\n",
    "These were previous tasks you completed:\n",
    "\n",
    "Begin!\n",
    "\n",
    "Question: {input}\n",
    "{agent_scratchpad}\"\"\"\n",
    "prompt = ChatPromptTemplate.from_template(template)\n",
    "\n",
    "llm = ChatOpenAI(model_name=\"gpt-4\", temperature=0)\n",
    "llm_chain = LLMChain(llm=llm, prompt=prompt)\n",
    "\n",
    "# Setup chain\n",
    "tarsier_agent_chain = initialize_agent(\n",
    "    [read_page, click, type_text],\n",
    "    llm,\n",
    "    agent=AgentType.STRUCTURED_CHAT_ZERO_SHOT_REACT_DESCRIPTION,\n",
    "    verbose=True,\n",
    ")\n",
    "\n",
    "# Go to google"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 4,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "\n",
      "\n",
      "\u001b[1m> Entering new AgentExecutor chain...\u001b[0m\n",
      "\u001b[32;1m\u001b[1;3mThought: The first step is to read the current state of the page to understand where we are and what elements are available. \n",
      "\n",
      "Action:\n",
      "```\n",
      "{\n",
      "  \"action\": \"read_page\",\n",
      "  \"action_input\": {}\n",
      "}\n",
      "```\u001b[0m\n",
      "Observation: \u001b[36;1m\u001b[1;3m--------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------\n",
      "0 ] About  [ 1 ] Store\n",
      "    [\t\t\t\t\t\t\t\t\t\t\t\t\t\t\t\t\t\t    [ 2 ] Gmail\t\t\t[ 5 ] Sign\n",
      "\t\t\t\t\t\t\t\t\t\t\t\t\t\t\t\t\t\t\t\t   [ 3 ]\n",
      "\t\t\t\t\t\t\t\t\t\t\t\t\t\t\t\t\t\t\t\t\t Images\n",
      "\t\t\t\t\t\t\t\t\t\t\t\t\t\t\t\t\t\t\t\t\t\t [ 4 ]\n",
      "\t\t\t\t\t\t\t\t\t\t\t\t       GOOGLE\n",
      "\t\t\t\t\t\t\t\t\t\t\t\t\t\t\t\t\t\t\t\t Sign in to    Google\n",
      "\t\t\t\t\t\t\t\t\t\t\t\t\t\t\t\t\t\t\t\t Save your     passwords securely with\n",
      "\t\t\t\t\t\t\t\t\t\t\t\t\t\t\t\t\t\t\t\t\t\tAccount\n",
      "\t\t\t\t\t\t\t\t\t\t\t\t\t\t\t\t\t\t\t\t       Google\n",
      "\t\t\t\t\t\t\t\t\t\t\t\t\t\t\t\t\t\t\t\t your\n",
      "\t\t\t\t\t\t\t\t\t\t\t\t\t\t\t\t\t   EY\n",
      "\t\t\t\t\t\t\t\t     [ 6 ]\n",
      "\t\t\t\t\t\t\t\t\t\t\t\t\t\t\t\t\t\t\t\t\t  signed out\n",
      "\t\t\t\t\t\t\t\t\t\t\t\t\t\t\t\t\t\t\t\t    Stay\t\t    Sign in\n",
      "\t\t\t\t\t\t\t\t\t\t\t\t\t\t\t\t\t [ 9 ] [ 10\n",
      "\t\t\t\t\t\t\t   Q   { 8 }\t\t\t\t\t\t\t\t\t      ]\n",
      "\t\t\t\t\t\t\t\t\t\t\t      Search\n",
      "\t\t\t\t\t\t\t\t\t\t11 )  Google\n",
      "\t\t\t\t\t\t\t\t\t      (\t\t\t  { 12 } I'm  Feeling Lucky\n",
      "\t\t\t\t\t\t\t\t\t\t      Learn how  Google [ 13 ] supports veterans\n",
      "\t\t\t\t\t\t\t\t\t\t\t\t\t\t\t\t\t\t\t\t\t\t\t       20 ]\n",
      "\t\t\t\t\t\t\t\t\t\t\t\t\t\t\t\t\t\t\t\t\t\t\t     [\n",
      "\t\t\t\t\t\t\t       works\t\t\t\t\t\t\t\t\t\t\t\t\t      19 ] Terms\n",
      "     [ 14 ]    Advertising 15 ]   Business   16 ] How   Search\t\t\t\t\t\t\t\t\t\t\t\t   18 ]   Privacy [\n",
      "\t\t\t  [\t\t\t\t\t\t       [ 17 ]  Our  third  decade of climate action : join us\t\t\t[\n",
      "\t\t\t\t\t    [\n",
      "--------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------\u001b[0m\n",
      "Thought:\u001b[32;1m\u001b[1;3mThe current page appears to be the Google homepage. The next step is to search for \"OpenAI Dev day\". This can be done by typing the text into the search box. The element_id for the search box appears to be 8. \n",
      "\n",
      "Action:\n",
      "```\n",
      "{\n",
      "  \"action\": \"type_text\",\n",
      "  \"action_input\": {\n",
      "    \"element_id\": 8,\n",
      "    \"text\": \"OpenAI Dev day\"\n",
      "  }\n",
      "}\n",
      "```\u001b[0m//textarea[@id=\"APjFqb\"]\n",
      "\n",
      "Observation: \u001b[38;5;200m\u001b[1;3m--------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------\n",
      "[ 4 ]\n",
      "     0 ] About  [ 1 ] Store\t\t\t\t\t\t\t\t\t\t\t\t\t\t\t    [ 2 ] Gmail [ 3 ]\t\t   5 ] Sign in\n",
      "    [\t\t\t\t\t\t\t\t\t\t\t\t\t\t\t\t\t\t\t\t  Images\t     [\n",
      "\t\t\t\t\t[ 6 ] .InXdpd { max - height : 100 % ; max - width : 100 % ; object - fit : contain ; object - position : center bottom ; width : auto ; vertical - align : }\n",
      "\t\t\t\t\t\t\t\t\t\t\t\t\t\t\t\t\t\t\t       Sign in to    Google\n",
      "\t\t\t\t\t\t\t\t\t\t\t\t\t\t\t\t\t\t\t       Save your     passwords securely with\n",
      "\t\t\t\t\t\t\t\t\t\t\t\t\t\t\t\t\t\t   G\t\t\t  Account\n",
      "\t\t\t\t\t\t\t\t\t\t\t\t\t\t\t\t\t\t\t\t     Google\n",
      "\t\t\t\t\t\t\t\t\t\t\t\t\t\t\t\t\t\t\t       your\n",
      "\t\t\t\t\t\t\t\t\t\t\t\t\t\t\t\t\t\t\t\t\tsigned out\n",
      "\t\t\t\t\t\t\t\t\t\t\t\t\t\t\t\t\t\t\t\t  Stay\t\t    Sign in\n",
      "\t\t\t\t\t\t\t\t\t\t\t\t\t\t\t     [ 9 ] X\n",
      "\t\t\t\t\t\t\t\t\t\t\t\t\t\t\t\t\t10 ] [ 11 ]\n",
      "\t\t\t\t\t\t\t   Q   { 8 }   OpenAl Dev  day\t\t\t\t\t\t [\n",
      "\t\t\t\t\t\t\t\t\t       12 }  Google  Search\n",
      "\t\t\t\t\t\t\t\t\t     {\t\t\t  ( 13 ) I'm Feeling Lucky\n",
      "\t\t\t\t\t\t\t\t\t\t     Learn how  Google [ 14 ] supports veterans\n",
      "\t\t\t\t\t\t\t\t\t\t\t\t\t\t\t\t\t\t\t\t\t\t\t     21 ]\n",
      "\t\t\t\t\t\t\t\t\t\t\t\t\t\t\t\t\t\t\t\t\t\t\t    [\n",
      "\t\t\t\t\t\t\t\t\t\t\t\t\t\t\t\t\t\t\t\t\t\t      Terms\n",
      "\t\t\t\t\t\t\t       works\n",
      "\t\t\t   16 ]   Business   17 ] How   Search\t\t\t\t\t\t\t\t\t\t\t\t\t\t  20 ]\n",
      "\t\t\t\t\t\t\t\t\t\t  [ 18 ] Our third decade of climate action : join us\t\t\t  [ 19 ] Privacy\n",
      "     [ 15 ]    Advertising [\t       [\t\t\t\t\t\t\t\t\t\t\t\t\t\t\t\t   [\n",
      "--------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------\u001b[0m\n",
      "Thought:\u001b[32;1m\u001b[1;3mThe text \"OpenAI Dev day\" has been entered into the search box. The next step is to submit the search. The element_id for the Google Search button appears to be 12.\n",
      "\n",
      "Action:\n",
      "```\n",
      "{\n",
      "  \"action\": \"click\",\n",
      "  \"action_input\": {\n",
      "    \"element_id\": 12\n",
      "  }\n",
      "}\n",
      "```\n",
      "\u001b[0m//html/body/div[1][contains(concat(\" \", normalize-space(@class), \" \"), \" L3eUgb \")]/div[3][contains(concat(\" \", normalize-space(@class), \" \"), \" o3j99 \") and contains(concat(\" \", normalize-space(@class), \" \"), \" ikrT4e \") and contains(concat(\" \", normalize-space(@class), \" \"), \" om7nvf \")]/form/div[1]/div[1][contains(concat(\" \", normalize-space(@class), \" \"), \" A8SBwf \") and contains(concat(\" \", normalize-space(@class), \" \"), \" sbfc \")]/div[4][contains(concat(\" \", normalize-space(@class), \" \"), \" FPdoLc \") and contains(concat(\" \", normalize-space(@class), \" \"), \" lJ9FBc \")]/center/input[1][contains(concat(\" \", normalize-space(@class), \" \"), \" gNO89b \")]\n",
      "\n",
      "Observation: \u001b[33;1m\u001b[1;3m--------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------\n",
      "{\n",
      "\t\t\t\t\t\t\t\t\t\t\t\t\t\t     [ 9 ]\n",
      "\t\t\t\t\t\t\t\t\t\t\t\t\t\t\t\t\t\t\t\t  [ 10 ]\n",
      "\t\t\t\t\t { 5 }  OpenAl  Dev  day\t\t\t     [ 6 ] X    [ 7 ] [ 8 ]\t\t\t\t\t\t\t\t [ 12 ] Sign in\n",
      "\t\t\t\t\t\t\t\t\t\t\t\t\t\t     Q\t\t\t\t\t\t     11 ]\n",
      "\t\t\t\t\t\t\t\t\t\t\t\t\t\t\t\t\t\t\t\t\t  [\n",
      "\t\t[ 4 ]\n",
      "\t\t 13 ]   News\t\t\t    Videos   16 ]   Github   17 ]    Tutorial\t\t\t\t    20 ]     Location\n",
      "\t\t[\t      [ 14 ]   Images [ 15 ]\t  [\t\t[\t       [ 18 ]     Shopping [ 19 ]   Agenda [\t\t [ 21 ] [ 23 ] [ 24 ]\t   [ 25 ] Tools\n",
      "\n",
      "\t\t   About    29,100,000 results ( 0.35 seconds )\n",
      "\t\t\t  OpenAl  DevDay\n",
      "\t\t\t\t\t      [ 28 ]\n",
      "\t\t\t\t https://devday.openai.com\n",
      "\t\t     OpenAl     DevDay\n",
      "\t\t\t\t\t\t      on     November 6 in San  Francisco .\n",
      "\t\t     OpenAl's first developer  conference\n",
      "\t\t    Missing : ( 8 ) | Show results with : [ 29 ] { 8 }\n",
      "\t\t\t       also  ask  [ 30 ] :\n",
      "\t\t     People\n",
      "\t\t [ 31 ] Who    invested in OpenAI ?\n",
      "\t\t\t      the   found\n",
      "\t\t[ 32 ] Who is\t       of    OpenAI ?\n",
      "\t\t[ 33 ] ls  OpenAl a    nonprofit ?\n",
      "\t\t\t\t\t      free ?\n",
      "\t\t  34 ] Can I use   OpenAl for\n",
      "\t\t [\n",
      "\t\t\t\t\t\t\t\t\t\t\t\t\t    [ 35 ] Feedback\n",
      "\t\t\t  Videos  [ 36 ]\n",
      "\t\t\t\t\t    [ 38 ]\n",
      "\t\t\t\t\t\t       OpenAl   DevDay ,  Opening   Keynote\n",
      "\t\t\t       OPENAI\n",
      "\t\t\t    DE\n",
      "\t\t [ 37 ]     0\n",
      "\t\t\t\t\t\t       YouTube   OpenAl\n",
      "\t\t\t  1:15:56\n",
      "\t\t\t\t\t\t    4  days\n",
      "\t\t\t\t\t\t\t    ago\n",
      "\t\t\t\t\t\t\t      Dev  Day Al       Breakthroughs Just  Changed    Everything ...\n",
      "\t\t\t\tOPENAI      [ 40 ]     OpenAl\n",
      "\t\t  39 ]\t  DEVDAY\n",
      "\t\t [\n",
      "\t\t\t\t\t\t       YouTube . Wes Roth\n",
      "\t\t\t\t  PERCUT\n",
      "\t\t\t 37:39\n",
      "\t\t\t\t\t\t    4  days\n",
      "\t\t\t\t\t\t\t    ago\n",
      "\t\t [ 41 ]\n",
      "\t\t    key    moments in this video V\n",
      "\t\t 17\n",
      "\t\t\t    AI      PANELISTS [ 43 ]   OpenAl Dev  Day     Livestream with    Generative Al  Developer ...\n",
      "\t\t\t  PEN\n",
      "\t\t  42 ]\n",
      "\t\t [\n",
      "\t\t\t\t\t\t       YouTube    Voicebot + Synthedia\n",
      "\t\t\t  1:55:28\n",
      "\t\t\t\t\t\t    4  days\n",
      "\t\t\t\t\t\t\t    ago\n",
      "\t\t\t\t\t\t\t\t\t\t\t\t\t   [ 44 ]  Feedback\n",
      "\t\t\t\t\t\t\t\t  [ 45 ]\n",
      "\t\t\t\t\t\t\t\tView all →\n",
      "\t\t\t\t   Developer Forum\n",
      "\t\t\t  OpenAl\n",
      "\t\t\t\t   https://community.openai.com openai - dev - day - 2023 ... [ 47 ]\n",
      "\t\t\t\t\t\t\t\t      !\n",
      "\t\t\t\t    Day    2023 :\t Announcement\n",
      "\t\t     OpenAl   Dev -\n",
      "\t\t 6  days ago –    Monday , November 6 , 2023 6:00 PM - Keynote Live Stream : Tune in for Sam\n",
      "\t\t    Altman's keynote as he unveils the newest OpenAl    innovations with live ...\n",
      "\t\t\t\t   Developer Forum\n",
      "\t\t\t  OpenAl\n",
      "\t\t\t\t\t\t\t\t    ... [ 49 ] ;\n",
      "\t\t\t\t   https://community.openai.com > openai - dev - day - 2023\n",
      "\t\t     OpenAl   Dev - Day    2023 :  Dev - Day\tDiscussion !\n",
      "\t\t 6  days ago -    Welcome , everyone ! This topic is your hub for the latest from Dev - Day . ! ... We have\n",
      "\t\t a   forum  member live at Dev - Day sharing updates . Our forum team ...\n",
      "\t\t\t    Bloomberg\n",
      "\t\t   B\n",
      "\t\t\t\t\t\t     >     newsletters › op ... [ 51 ] ;\n",
      "\t\t\t\t  https://www.bloomberg.com > news\n",
      "\t\t      OpenAl's      Developer   Event    Sparks\tExcitement and    Fear in ...\n",
      "\t\t 3  days ago -    Reibman , who called OpenAl Dev Day the \" Super Bowl \" moment for Al ...\n",
      "\t\t     November 8 , 2023 at 6:52 AM PST . Save . Listen . 1:45 . Rockstar Games , a ...\n",
      "\t\t\t  gHacks    Technology News\n",
      "\t\t\t\t\t\t [ 53 ]\n",
      "\t\t\t      https://ghacks.net > Misc\n",
      "\t\t     OpenAl   Dev\n",
      "\t\t\t\t    Day : A    recap\n",
      "\t\t 4  days ago -    OpenAI , a leading artificial intelligence research laboratory , held its first OpenAl\n",
      "\t\t  Dev  Day , on   November 6 , 2023 , San  Francisco . The event ...\n",
      "\t\t\t   infoq.com\n",
      "\t\t\t\thttps://www.infoq.com > news > 2023/11 > openai - an ... [ 55 ]\n",
      "\t\t     OpenAl\tAnnounces   New      Models   and   APIs  at   First      Developer ...\n",
      "\t\t 3  days ago - The  new   model is more  capable , cheaper , and supports a 128K context window . It\n",
      "\t\t  has     knowledge of world events up to April 2023 and can the ...\n",
      "\t\t\t   HyScaler\n",
      "\t\t\t\t\t\t  devday - recap - ai - advances [ 57 ] :\n",
      "\t\t\t       https://hyscaler.com > openai -\n",
      "\t\t     OpenAl   Dev   Day     Recap :\tRevolutionary       Developments in ...\n",
      "\t\t 3  days ago - In sum ,  OpenAl Dev Day  has   emerged as a   landmark event , shaping the contours\n",
      "\t\t of an Al -   empowered future that is both exciting and within reach .\n",
      "\t\t\tThe   Indian\n",
      "\t\t\t\t     Express\n",
      "\t\t\t\t https://indianexpress.com > ... > Artificial Intelligence [ 59 ]\n",
      "\t\t   GPT - 4   Turbo ,  GPT    Store &\n",
      "\t\t\t\t\t\t\t    Assistants API : 9 big       takeaways ...\n",
      "\t\t 4  days ago -    OpenAl just  concluded its first - ever developer conference , DevDay 2023. The\n",
      "\t\t      conference was held at the SVN West facility in San Francisco .\n",
      "\t\t\t  LinkedIn\n",
      "\t\t  in\n",
      "\t\t\t\t https://www.linkedin.com > pulse › look - openai - develop ... [ 61 ]\n",
      "\t\t A    look at  the      OpenAl       Developer Day\t    announcements\n",
      "\t\t 3  days ago -   OpenAl  shows no sign of  slowing down , and at its recent developer  conference it\n",
      "\t\t      announced that the company has reached over 100 million ...\n",
      "\t\t      Related       searches [ 62 ] :\n",
      "\t\t   [ 63 ] Q\t  openai dev day   tutorial\t  [ 67 ] Q\n",
      "\t\t\t\t8\t\t\t\t\t\t       openai   devday   location\n",
      "\t\t   [ 64 ] Q\t\t\t\t\t     [ 68 ] Q\n",
      "\t\t\t\t8    openai dev day   github\t\t\t    openai dev day     summary\n",
      "\t\t\t\t\t\t    reddit\n",
      "\t\t   [ 65 ] Q\topenai dev day\t\t       [ 69 ] Q\topenai dev day    youtube\n",
      "\t\t   [ 66 ] Q\t\t   day  2023\t\t [ 70 ] Q\t\t\t   schedule\n",
      "\t\t\t\t   openai dev\t\t\t\t\t   openai   devday\n",
      "--------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------\u001b[0m\n",
      "Thought:\u001b[32;1m\u001b[1;3mThe search results for \"OpenAI Dev day\" have been displayed. The first video related to \"OpenAI Dev day\" is titled \"OpenAI DevDay, Opening Keynote\" and is hosted on YouTube. The element_id for this video appears to be 37. The next step is to click on this video.\n",
      "\n",
      "Action:\n",
      "```\n",
      "{\n",
      "  \"action\": \"click\",\n",
      "  \"action_input\": {\n",
      "    \"element_id\": 37\n",
      "  }\n",
      "}\n",
      "```\u001b[0m//div[@id=\"rso\"]/div[3][contains(concat(\" \", normalize-space(@class), \" \"), \" ULSxyf \")]/div[contains(concat(\" \", normalize-space(@class), \" \"), \" MjjYud \")]/div[contains(concat(\" \", normalize-space(@class), \" \"), \" uVMCKf \")]/div[2][contains(concat(\" \", normalize-space(@class), \" \"), \" e4xoPb \")]/div[2]/div[1][contains(concat(\" \", normalize-space(@class), \" \"), \" RzdJxc \")]/div/div[contains(concat(\" \", normalize-space(@class), \" \"), \" X4T0U \")]/div/div/div[1]/div/div[contains(concat(\" \", normalize-space(@class), \" \"), \" KYaZsb \")]/a[1][contains(concat(\" \", normalize-space(@class), \" \"), \" xMqpbd \") and contains(concat(\" \", normalize-space(@class), \" \"), \" NFjGBc \")]\n",
      "\n",
      "Observation: \u001b[33;1m\u001b[1;3m--------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------\n",
      "[ 1 ]\n",
      "\t 0 ]\n",
      "\t [\n",
      "\t\t\t\t\t\t\t\t\t\t\t\t\t\t\t\t\t\t\t\t\t\t     [ 2 ]    YouTube\n",
      "\t   ←\n",
      "\t\t\t     OpenAl\t    DevDay   ,\t  Opening\t     Keynote\n",
      "\t\t\t      YouTube .    OpenAl . Nov 6 ,  2023\n",
      "\t\t\t\t     OpenAl     DevDay ,    Opening     Keynote\n",
      "\n",
      "\t\t\t      C\n",
      "\t\t\t       ware\n",
      "\t\t\t       Devor\n",
      "\t\t\t\t\t\t\t\t\t\t\t\t\t\t OPENAI\n",
      "\t\t\t\t\t\t\t\t\t\t\t\t\t\t DEVDAY\n",
      "\t\t\t\t\t\t\t\t\t\t\t\t\t\t OPENAI\n",
      "\t\t\t\t\t\t\t\t\t\t\t\t\t\tDEVDAY\n",
      "--------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------\u001b[0m\n",
      "Thought:\u001b[32;1m\u001b[1;3mWe are now on the page of the first video titled \"OpenAI DevDay, Opening Keynote\" on YouTube. This is the information the user requested. \n",
      "\n",
      "Action:\n",
      "```\n",
      "{\n",
      "  \"action\": \"Final Answer\",\n",
      "  \"action_input\": \"We are now on the page of the first video titled 'OpenAI DevDay, Opening Keynote' on YouTube.\"\n",
      "}\n",
      "```\u001b[0m\n",
      "\n",
      "\u001b[1m> Finished chain.\u001b[0m\n"
     ]
    },
    {
     "data": {
      "text/plain": [
       "\"We are now on the page of the first video titled 'OpenAI DevDay, Opening Keynote' on YouTube.\""
      ]
     },
     "execution_count": 4,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "await page.goto(\"https://www.google.com/\")\n",
    "await tarsier_agent_chain.arun(\n",
    "    \"\"\"\n",
    "    Read the page, search for OpenAI Dev day, go to the first video\n",
    "    \"\"\"\n",
    ")"
   ]
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 3",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.11.6"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 0
}