Browser & Computer Use Agents

Browser Agents: Expanding the Action Space

Most agent tasks are LLM + API calls. But many real-world tasks happen in browsers: filling forms, navigating authenticated portals, extracting data from JavaScript-rendered pages, interacting with web applications that have no API.

Browser agents close this gap.

Playwright-based Web Agent

BrowserTool manages a persistent Chromium process across multiple page operations, opening and closing the browser for every request would be too slow in production.

# src/tools/browser_tool.py
from playwright.async_api import async_playwright, Page, Browser
from dataclasses import dataclass
import asyncio
import base64

@dataclass
class BrowserResult:
    success: bool
    data: dict | list | str | None
    screenshot: str | None = None  # base64 encoded PNG, useful for debugging failures visually
    error: str | None = None

class BrowserTool:
    """Browser automation tool with stealth and error recovery."""
    
    def __init__(self):
        self._playwright = None
        self._browser: Browser | None = None
    
    async def __aenter__(self):
        self._playwright = await async_playwright().start()
        self._browser = await self._playwright.chromium.launch(
            headless=True,    # headless=True, runs Chrome without a GUI; required in server environments
            args=[
                "--no-sandbox",                                      # required when running as root in Docker
                "--disable-dev-shm-usage",                          # prevents /dev/shm OOM crashes in containers
                "--disable-blink-features=AutomationControlled",    # removes one bot-detection signal from Chrome flags
            ],
        )
        return self
    
    async def __aexit__(self, *args):
        if self._browser:
            await self._browser.close()
        if self._playwright:
            await self._playwright.stop()
    
    async def new_page(self) -> Page:
        context = await self._browser.new_context(
            # Spoof a real Mac/Chrome user-agent so basic UA checks don't block the request
            user_agent="Mozilla/5.0 (Macintosh; Intel Mac OS X 10_15_7) AppleWebKit/537.36",
            viewport={"width": 1280, "height": 720},
        )
        page = await context.new_page()
        
        # Object.defineProperty patches navigator.webdriver to hide that this is an automated browser
        await page.add_init_script("""
            Object.defineProperty(navigator, 'webdriver', {get: () => undefined});
        """)
        
        return page
    
    async def navigate_and_extract(
        self,
        url: str,
        extract_selector: str = None,
        wait_for: str = "networkidle",
    ) -> BrowserResult:
        """Navigate to URL and extract content."""
        page = await self.new_page()
        
        try:
            # wait_until="networkidle" waits for all network requests to finish: important for JS-heavy SPAs
            await page.goto(url, wait_until=wait_for, timeout=30000)
            
            if extract_selector:
                # wait_for_selector blocks until the element appears in the DOM: avoids race conditions
                await page.wait_for_selector(extract_selector, timeout=10000)
                elements = await page.query_selector_all(extract_selector)
                texts = [await el.text_content() for el in elements]
                return BrowserResult(success=True, data=texts)
            else:
                content = await page.content()
                # Truncate to 50 KB: full HTML pages can be hundreds of KB, way beyond useful token budget
                return BrowserResult(success=True, data=content[:50000])
        
        except Exception as e:
            # Capture a screenshot on failure so you can see what the page looked like when it broke
            screenshot = await page.screenshot()
            return BrowserResult(
                success=False,
                data=None,
                screenshot=base64.b64encode(screenshot).decode(),
                error=str(e),
            )
        finally:
            await page.close()
    
    async def fill_form(self, url: str, form_data: dict) -> BrowserResult:
        """Navigate to URL and fill a form."""
        page = await self.new_page()
        
        try:
            await page.goto(url, wait_until="networkidle")
            
            for selector, value in form_data.items():
                element = await page.wait_for_selector(selector, timeout=5000)
                await element.fill(str(value))
                await asyncio.sleep(0.1)  # small delay between fields mimics human typing pace
            
            # Capture state before submission to aid debugging if the submit silently fails
            screenshot_before = await page.screenshot()
            
            # Find the submit button by attribute: more robust than a hardcoded CSS class
            submit_btn = await page.query_selector('[type="submit"], button[type="submit"]')
            if submit_btn:
                await submit_btn.click()
                await page.wait_for_load_state("networkidle")  # wait for post-submit navigation to settle
            
            screenshot_after = await page.screenshot()
            page_content = await page.content()
            
            return BrowserResult(
                success=True,
                data={"page_content": page_content[:10000]},
                screenshot=base64.b64encode(screenshot_after).decode(),
            )
        
        except Exception as e:
            screenshot = await page.screenshot()
            return BrowserResult(success=False, error=str(e), screenshot=base64.b64encode(screenshot).decode())
        finally:
            await page.close()

Browser Use: LLM-native Browser Automation

Browser Use (Python library) lets an LLM control a browser by describing what to do in natural language:

# pip install browser-use
from browser_use import Agent as BrowserAgent
from langchain_openai import ChatOpenAI

async def browser_agent_task(task: str) -> str:
    """Use an LLM to control a browser and accomplish a task."""
    
    agent = BrowserAgent(
        task=task,
        llm=ChatOpenAI(model="gpt-5.6-sol"),
        use_vision=True,  # agent sees screenshots of the page, not just the DOM text
    )
    
    # max_steps caps the action loop: without it, a confused agent can burn tokens indefinitely
    result = await agent.run(max_steps=20)
    return result.final_result()

# Usage examples
result = await browser_agent_task(
    "Go to linkedin.com/jobs and find the top 5 Python developer jobs in San Francisco. Return their titles and companies."
)

result = await browser_agent_task(
    "Go to our internal portal at http://localhost:3000, log in with [email protected] / password123, and download the Q4 sales report."
)

Anthropic Computer Use API

For applications that require genuine visual understanding of any screen:

# src/tools/computer_use.py
import anthropic
import base64
from PIL import ImageGrab  # or pyautogui for screenshots

client = anthropic.Anthropic()

async def run_computer_use_agent(task: str) -> str:
    """Anthropic Computer Use agent that sees and controls the screen."""
    
    tools = [
        {
            "type": "computer_20241022",
            "name": "computer",
            # Tell Claude the exact screen resolution so pixel coordinates it clicks are accurate
            "display_width_px": 1280,
            "display_height_px": 720,
            "display_number": 1,
        },
        {"type": "text_editor_20241022", "name": "str_replace_editor"},  # lets Claude edit files on disk
        {"type": "bash_20241022", "name": "bash"},                        # lets Claude run shell commands
    ]
    
    messages = [{"role": "user", "content": task}]
    
    # The loop continues until the model signals end_turn: each iteration is one "see → act" cycle
    while True:
        response = client.beta.messages.create(
            model="claude-opus-4-8",  # Computer Use requires a capable vision model; Opus is the recommendation
            max_tokens=4096,
            tools=tools,
            messages=messages,
            betas=["computer-use-2024-10-22"],  # opt into the beta tool definitions for computer control
        )
        
        # Append the assistant's turn to the conversation history before the next loop
        messages.append({"role": "assistant", "content": response.content})
        
        if response.stop_reason == "end_turn":
            # The model decided it's done: extract the final text response
            text_blocks = [b for b in response.content if b.type == "text"]
            return text_blocks[-1].text if text_blocks else "Task complete"
        
        # Process tool calls: the model requests actions; we execute them and return screenshots
        tool_results = []
        for block in response.content:
            if block.type == "tool_use" and block.name == "computer":
                result = await execute_computer_action(block.input)
                tool_results.append({
                    "type": "tool_result",
                    "tool_use_id": block.id,  # must echo the same ID so the model knows which call this answers
                    "content": result,
                })
        
        # Feed screenshots back as the next user turn: this is how the model "sees" what happened
        messages.append({"role": "user", "content": tool_results})

async def execute_computer_action(action: dict) -> list:
    """Execute a computer use action and return a screenshot."""
    from pyautogui import click, write, press, scroll, moveTo
    from PIL import ImageGrab
    
    action_type = action.get("action")
    
    if action_type == "screenshot":
        pass  # model just wants to see the current screen without performing any action
    elif action_type == "left_click":
        click(action["coordinate"][0], action["coordinate"][1])
    elif action_type == "type":
        write(action["text"], interval=0.05)  # small interval simulates human typing speed
    elif action_type == "key":
        press(action["key"])
    elif action_type == "scroll":
        scroll(action.get("coordinate", [0, 0])[0], action["direction"] == "up" and 3 or -3)
    
    # Always return a screenshot after every action so the model can verify the result
    screenshot = ImageGrab.grab()
    screenshot = screenshot.resize((1280, 720))  # normalize to the declared display size
    
    import io
    buffer = io.BytesIO()
    screenshot.save(buffer, format="PNG")
    screenshot_b64 = base64.b64encode(buffer.getvalue()).decode()
    
    return [{"type": "image", "source": {"type": "base64", "media_type": "image/png", "data": screenshot_b64}}]

Choosing the Right Approach

Browser agents unlock a class of tasks that API-based agents simply cannot do. The tradeoff is latency (2-10x slower than API calls), cost (screenshots are expensive tokens), and reliability (browsers break in unpredictable ways). Use them when there’s no API alternative.