diff --git a/README.md b/README.md index c920367..2210dfd 100644 --- a/README.md +++ b/README.md @@ -465,7 +465,9 @@ firecrawl agent abc123-def456-... --wait --poll-interval 10 --- -### `browser` - Browser sandbox sessions (Beta) +### `browser` - Browser sandbox sessions (Deprecated) + +> **Deprecated:** Prefer `scrape` + `interact` instead. Interact lets you scrape a page and then click, fill forms, and navigate without managing sessions manually. See the `interact` command. Launch and control cloud browser sessions. By default, commands are sent to agent-browser (pre-installed in every sandbox). Use `--python` or `--node` to run Playwright code directly instead. @@ -866,24 +868,22 @@ Add `-y` to any command to auto-approve tool permissions (maps to `--dangerously ### Live View -Use `firecrawl browser launch --json` to get a live view URL, then pass it to your agent so you can watch it work in real-time: +Use `firecrawl scrape ` + `firecrawl interact` to interact with pages. For advanced use cases requiring a raw CDP session, you can still use `firecrawl browser launch --json` to get a live view URL: ```bash -# Launch a browser session and grab the live view URL +# Preferred: scrape + interact workflow +firecrawl scrape https://myapp.com +firecrawl interact --prompt "Click on the login button and fill in the form" + +# Advanced: Launch a browser session and grab the live view URL LIVE_URL=$(firecrawl browser launch --json | jq -r '.liveViewUrl') # Pass it to Claude Code -claude --append-system-prompt "A cloud browser session is running. Live view: $LIVE_URL -- use \`firecrawl browser\` commands to interact." \ +claude --append-system-prompt "A cloud browser session is running. Live view: $LIVE_URL -- use \`firecrawl interact\` to interact with scraped pages." \ --dangerously-skip-permissions \ - "QA test https://myapp.com using the cloud browser" - -# Pass it to Codex -codex --full-auto \ - --config "instructions=A cloud browser session is running. Live view: $LIVE_URL -- use \`firecrawl browser\` commands to interact." \ - "walk through the signup flow on https://example.com" + "QA test https://myapp.com" -# Or use the built-in workflow commands (session is auto-saved for firecrawl browser) -firecrawl browser launch --json | jq -r '.liveViewUrl' +# Or use the built-in workflow commands firecrawl claude demo https://resend.com ``` diff --git a/package.json b/package.json index ce1a76e..fb8ce9e 100644 --- a/package.json +++ b/package.json @@ -1,6 +1,6 @@ { "name": "firecrawl-cli", - "version": "1.11.2", + "version": "1.12.0", "description": "Command-line interface for Firecrawl. Scrape, crawl, and extract data from any website directly from your terminal.", "main": "dist/index.js", "bin": { diff --git a/skills/firecrawl-agent/SKILL.md b/skills/firecrawl-agent/SKILL.md index 0b3577f..5fc0623 100644 --- a/skills/firecrawl-agent/SKILL.md +++ b/skills/firecrawl-agent/SKILL.md @@ -53,5 +53,5 @@ firecrawl agent "get feature list" --urls "" --wait -o .firecrawl/features. ## See also - [firecrawl-scrape](../firecrawl-scrape/SKILL.md) — simpler single-page extraction -- [firecrawl-browser](../firecrawl-browser/SKILL.md) — manual browser automation (more control) +- [firecrawl-browser](../firecrawl-browser/SKILL.md) — scrape + interact for manual page interaction (more control) - [firecrawl-crawl](../firecrawl-crawl/SKILL.md) — bulk extraction without AI diff --git a/skills/firecrawl-browser/SKILL.md b/skills/firecrawl-browser/SKILL.md index ed0f0ae..9030e9c 100644 --- a/skills/firecrawl-browser/SKILL.md +++ b/skills/firecrawl-browser/SKILL.md @@ -1,107 +1,85 @@ --- name: firecrawl-browser description: | - Cloud browser automation for pages requiring interaction — clicks, form fills, login, pagination, infinite scroll. Use this skill when the user needs to interact with a webpage, log into a site, click buttons, fill forms, navigate multi-step flows, handle pagination, or when regular scraping fails because content requires JavaScript interaction. Triggers on "click", "fill out the form", "log in to", "paginated", "infinite scroll", "interact with the page", or "scrape failed". Provides remote Chromium sessions with persistent profiles. + DEPRECATED — use scrape + interact instead. Interact lets you scrape a page and then click, fill forms, and navigate without managing sessions manually. Use this skill when the user needs to interact with a webpage, log into a site, click buttons, fill forms, navigate multi-step flows, handle pagination, or when regular scraping fails because content requires JavaScript interaction. Triggers on "click", "fill out the form", "log in to", "paginated", "infinite scroll", "interact with the page", or "scrape failed". allowed-tools: - Bash(firecrawl *) - Bash(npx firecrawl *) --- -# firecrawl browser +# firecrawl interact (formerly browser) -Cloud Chromium sessions in Firecrawl's remote sandboxed environment. Interact with pages that require clicks, form fills, pagination, or login. +> **The `browser` command is deprecated.** Use `scrape` + `interact` instead. Interact lets you scrape a page and then click, fill forms, and navigate without managing sessions manually. + +Interact with scraped pages in a live browser session. Scrape a page first, then use natural language prompts or code to click, fill forms, navigate, and extract data. ## When to use - Content requires interaction: clicks, form fills, pagination, login - `scrape` failed because content is behind JavaScript interaction - You need to navigate a multi-step flow -- Last resort in the [workflow escalation pattern](firecrawl-cli): search → scrape → map → crawl → **browser** -- **Never use browser for web searches** — use `search` instead +- Last resort in the [workflow escalation pattern](firecrawl-cli): search → scrape → map → crawl → **interact** +- **Never use interact for web searches** — use `search` instead ## Quick start ```bash -# Typical browser workflow -firecrawl browser "open " -firecrawl browser "snapshot -i" # see interactive elements with @ref IDs -firecrawl browser "click @e5" # interact with elements -firecrawl browser "fill @e3 'search query'" # fill form fields -firecrawl browser "scrape" -o .firecrawl/page.md # extract content -firecrawl browser close -``` +# 1. Scrape a page (scrape ID is saved automatically) +firecrawl scrape "" -Shorthand auto-launches a session if none exists — no setup required. +# 2. Interact with the page using natural language +firecrawl interact --prompt "Click the login button" +firecrawl interact --prompt "Fill in the email field with test@example.com" +firecrawl interact --prompt "Extract the pricing table" -## Commands +# 3. Or use code for precise control +firecrawl interact --code "agent-browser click @e5" --language bash +firecrawl interact --code "agent-browser snapshot -i" --language bash -| Command | Description | -| -------------------- | ---------------------------------------- | -| `open ` | Navigate to a URL | -| `snapshot -i` | Get interactive elements with `@ref` IDs | -| `screenshot` | Capture a PNG screenshot | -| `click <@ref>` | Click an element by ref | -| `type <@ref> ` | Type into an element | -| `fill <@ref> ` | Fill a form field (clears first) | -| `scrape` | Extract page content as markdown | -| `scroll ` | Scroll up/down/left/right | -| `wait ` | Wait for a duration | -| `eval ` | Evaluate JavaScript on the page | - -Session management: `launch-session --ttl 600`, `list`, `close` +# 4. Stop the session when done +firecrawl interact stop +``` ## Options -| Option | Description | -| ---------------------------- | -------------------------------------------------- | -| `--ttl ` | Session time-to-live | -| `--ttl-inactivity ` | Inactivity timeout | -| `--session ` | Use a specific session ID | -| `--profile ` | Use a named profile (persists state) | -| `--no-save-changes` | Read-only reconnect (don't write to session state) | -| `-o, --output ` | Output file path | +| Option | Description | +| --------------------- | ------------------------------------------------- | +| `--prompt ` | Natural language instruction (use this OR --code) | +| `--code ` | Code to execute in the browser session | +| `--language ` | Language for code: bash, python, node | +| `--timeout ` | Execution timeout (default: 30, max: 300) | +| `--scrape-id ` | Target a specific scrape (default: last scrape) | +| `-o, --output ` | Output file path | ## Profiles -Profiles survive close and can be reconnected by name. Use them for login-then-work flows: +Use `--profile` on the scrape to persist browser state (cookies, localStorage) across scrapes: ```bash # Session 1: Login and save state -firecrawl browser launch-session --profile my-app -firecrawl browser "open https://app.example.com/login" -firecrawl browser "snapshot -i" -firecrawl browser "fill @e3 'user@example.com'" -firecrawl browser "click @e7" -firecrawl browser "wait 2" -firecrawl browser close +firecrawl scrape "https://app.example.com/login" --profile my-app +firecrawl interact --prompt "Fill in email with user@example.com and click login" # Session 2: Come back authenticated -firecrawl browser launch-session --profile my-app -firecrawl browser "open https://app.example.com/dashboard" -firecrawl browser "scrape" -o .firecrawl/dashboard.md -firecrawl browser close -``` - -Read-only reconnect (no writes to session state): - -```bash -firecrawl browser launch-session --profile my-app --no-save-changes +firecrawl scrape "https://app.example.com/dashboard" --profile my-app +firecrawl interact --prompt "Extract the dashboard data" ``` -Shorthand with profile: +Read-only reconnect (no writes to profile state): ```bash -firecrawl browser --profile my-app "open https://example.com" +firecrawl scrape "https://app.example.com" --profile my-app --no-save-changes ``` ## Tips -- If you get forbidden errors, the session may have expired — create a new one. -- For parallel browser work, launch separate sessions and operate them via `--session `. -- Always `close` sessions when done to free resources. +- Always scrape first — `interact` requires a scrape ID from a previous `firecrawl scrape` call +- The scrape ID is saved automatically, so you don't need `--scrape-id` for subsequent interact calls +- Use `firecrawl interact stop` to free resources when done +- For parallel work, scrape multiple pages and interact with each using `--scrape-id` ## See also -- [firecrawl-scrape](../firecrawl-scrape/SKILL.md) — try scrape first, escalate to browser only when needed -- [firecrawl-search](../firecrawl-search/SKILL.md) — for web searches (never use browser for searching) +- [firecrawl-scrape](../firecrawl-scrape/SKILL.md) — try scrape first, escalate to interact only when needed +- [firecrawl-search](../firecrawl-search/SKILL.md) — for web searches (never use interact for searching) - [firecrawl-agent](../firecrawl-agent/SKILL.md) — AI-powered extraction (less manual control) diff --git a/skills/firecrawl-cli/SKILL.md b/skills/firecrawl-cli/SKILL.md index eecdfbf..5b1dba1 100644 --- a/skills/firecrawl-cli/SKILL.md +++ b/skills/firecrawl-cli/SKILL.md @@ -1,7 +1,7 @@ --- name: firecrawl description: | - Web scraping, search, crawling, and browser automation via the Firecrawl CLI. Use this skill whenever the user wants to search the web, find articles, research a topic, look something up online, scrape a webpage, grab content from a URL, extract data from a website, crawl documentation, download a site, or interact with pages that need clicks or logins. Also use when they say "fetch this page", "pull the content from", "get the page at https://", or reference scraping external websites. This provides real-time web search with full page content extraction and cloud browser automation — capabilities beyond what Claude can do natively with built-in tools. Do NOT trigger for local file operations, git commands, deployments, or code editing tasks. + Web scraping, search, crawling, and page interaction via the Firecrawl CLI. Use this skill whenever the user wants to search the web, find articles, research a topic, look something up online, scrape a webpage, grab content from a URL, extract data from a website, crawl documentation, download a site, or interact with pages that need clicks or logins. Also use when they say "fetch this page", "pull the content from", "get the page at https://", or reference scraping external websites. This provides real-time web search with full page content extraction and interact capabilities — beyond what Claude can do natively with built-in tools. Do NOT trigger for local file operations, git commands, deployments, or code editing tasks. allowed-tools: - Bash(firecrawl *) - Bash(npx firecrawl *) @@ -9,7 +9,7 @@ allowed-tools: # Firecrawl CLI -Web scraping, search, and browser automation CLI. Returns clean markdown optimized for LLM context windows. +Web scraping, search, and page interaction CLI. Returns clean markdown optimized for LLM context windows. Run `firecrawl --help` or `firecrawl --help` for full option details. @@ -42,25 +42,25 @@ Follow this escalation pattern: 2. **Scrape** - Have a URL. Extract its content directly. 3. **Map + Scrape** - Large site or need a specific subpage. Use `map --search` to find the right URL, then scrape it. 4. **Crawl** - Need bulk content from an entire site section (e.g., all /docs/). -5. **Browser** - Scrape failed because content is behind interaction (pagination, modals, form submissions, multi-step navigation). +5. **Interact** - Scrape first, then interact with the page (pagination, modals, form submissions, multi-step navigation). -| Need | Command | When | -| --------------------------- | ---------- | --------------------------------------------------------- | -| Find pages on a topic | `search` | No specific URL yet | -| Get a page's content | `scrape` | Have a URL, page is static or JS-rendered | -| Find URLs within a site | `map` | Need to locate a specific subpage | -| Bulk extract a site section | `crawl` | Need many pages (e.g., all /docs/) | -| AI-powered data extraction | `agent` | Need structured data from complex sites | -| Interact with a page | `browser` | Content requires clicks, form fills, pagination, or login | -| Download a site to files | `download` | Save an entire site as local files | +| Need | Command | When | +| --------------------------- | --------------------- | --------------------------------------------------------- | +| Find pages on a topic | `search` | No specific URL yet | +| Get a page's content | `scrape` | Have a URL, page is static or JS-rendered | +| Find URLs within a site | `map` | Need to locate a specific subpage | +| Bulk extract a site section | `crawl` | Need many pages (e.g., all /docs/) | +| AI-powered data extraction | `agent` | Need structured data from complex sites | +| Interact with a page | `scrape` + `interact` | Content requires clicks, form fills, pagination, or login | +| Download a site to files | `download` | Save an entire site as local files | -For detailed command reference, use the individual skill for each command (e.g., `firecrawl-search`, `firecrawl-browser`) or run `firecrawl --help`. +For detailed command reference, run `firecrawl --help`. -**Scrape vs browser:** +**Scrape vs interact:** - Use `scrape` first. It handles static pages and JS-rendered SPAs. -- Use `browser` when you need to interact with a page, such as clicking buttons, filling out forms, navigating through a complex site, infinite scroll, or when scrape fails to grab all the content you need. -- Never use browser for web searches - use `search` instead. +- Use `scrape` + `interact` when you need to interact with a page, such as clicking buttons, filling out forms, navigating through a complex site, infinite scroll, or when scrape fails to grab all the content you need. +- Never use interact for web searches - use `search` instead. **Avoid redundant fetches:** @@ -116,7 +116,7 @@ firecrawl scrape "" -o .firecrawl/3.md & wait ``` -For browser, launch separate sessions for independent tasks and operate them in parallel via `--session `. +For interact, scrape multiple pages and interact with each independently using their scrape IDs. ## Credit Usage diff --git a/skills/firecrawl-crawl/SKILL.md b/skills/firecrawl-crawl/SKILL.md index fb2f3bd..ca6e6b5 100644 --- a/skills/firecrawl-crawl/SKILL.md +++ b/skills/firecrawl-crawl/SKILL.md @@ -15,7 +15,7 @@ Bulk extract content from a website. Crawls pages following links up to a depth/ - You need content from many pages on a site (e.g., all `/docs/`) - You want to extract an entire site section -- Step 4 in the [workflow escalation pattern](firecrawl-cli): search → scrape → map → **crawl** → browser +- Step 4 in the [workflow escalation pattern](firecrawl-cli): search → scrape → map → **crawl** → interact ## Quick start diff --git a/skills/firecrawl-map/SKILL.md b/skills/firecrawl-map/SKILL.md index f8047fc..77eaacf 100644 --- a/skills/firecrawl-map/SKILL.md +++ b/skills/firecrawl-map/SKILL.md @@ -15,7 +15,7 @@ Discover URLs on a site. Use `--search` to find a specific page within a large s - You need to find a specific subpage on a large site - You want a list of all URLs on a site before scraping or crawling -- Step 3 in the [workflow escalation pattern](firecrawl-cli): search → scrape → **map** → crawl → browser +- Step 3 in the [workflow escalation pattern](firecrawl-cli): search → scrape → **map** → crawl → interact ## Quick start diff --git a/skills/firecrawl-scrape/SKILL.md b/skills/firecrawl-scrape/SKILL.md index fc313d3..150c571 100644 --- a/skills/firecrawl-scrape/SKILL.md +++ b/skills/firecrawl-scrape/SKILL.md @@ -15,7 +15,7 @@ Scrape one or more URLs. Returns clean, LLM-optimized markdown. Multiple URLs ar - You have a specific URL and want its content - The page is static or JS-rendered (SPA) -- Step 2 in the [workflow escalation pattern](firecrawl-cli): search → **scrape** → map → crawl → browser +- Step 2 in the [workflow escalation pattern](firecrawl-cli): search → **scrape** → map → crawl → interact ## Quick start @@ -55,7 +55,7 @@ firecrawl scrape "https://example.com/pricing" --query "What is the enterprise p ## Tips - **Prefer plain scrape over `--query`.** Scrape to a file, then use `grep`, `head`, or read the markdown directly — you can search and reason over the full content yourself. Use `--query` only when you want a single targeted answer without saving the page (costs 5 extra credits). -- **Try scrape before browser.** Scrape handles static pages and JS-rendered SPAs. Only escalate to browser when you need interaction (clicks, form fills, pagination). +- **Try scrape before interact.** Scrape handles static pages and JS-rendered SPAs. Only escalate to `interact` when you need interaction (clicks, form fills, pagination). - Multiple URLs are scraped concurrently — check `firecrawl --status` for your concurrency limit. - Single format outputs raw content. Multiple formats (e.g., `--format markdown,links`) output JSON. - Always quote URLs — shell interprets `?` and `&` as special characters. @@ -64,5 +64,5 @@ firecrawl scrape "https://example.com/pricing" --query "What is the enterprise p ## See also - [firecrawl-search](../firecrawl-search/SKILL.md) — find pages when you don't have a URL -- [firecrawl-browser](../firecrawl-browser/SKILL.md) — when scrape can't get the content (interaction needed) +- [firecrawl-browser](../firecrawl-browser/SKILL.md) — when scrape can't get the content, use `interact` to click, fill forms, etc. - [firecrawl-download](../firecrawl-download/SKILL.md) — bulk download an entire site to local files diff --git a/skills/firecrawl-search/SKILL.md b/skills/firecrawl-search/SKILL.md index aec4232..aeea8ed 100644 --- a/skills/firecrawl-search/SKILL.md +++ b/skills/firecrawl-search/SKILL.md @@ -15,7 +15,7 @@ Web search with optional content scraping. Returns search results as JSON, optio - You don't have a specific URL yet - You need to find pages, answer questions, or discover sources -- First step in the [workflow escalation pattern](firecrawl-cli): search → scrape → map → crawl → browser +- First step in the [workflow escalation pattern](firecrawl-cli): search → scrape → map → crawl → interact ## Quick start diff --git a/src/commands/interact.ts b/src/commands/interact.ts new file mode 100644 index 0000000..9b539f7 --- /dev/null +++ b/src/commands/interact.ts @@ -0,0 +1,198 @@ +/** + * Interact command implementation + * Execute AI prompts or code against a scraped page in a live browser session + */ + +import { getClient } from '../utils/client'; +import { getConfig, validateConfig } from '../utils/config'; +import { + getScrapeId, + loadInteractSession, + clearInteractSession, +} from '../utils/interact-session'; +import { writeOutput } from '../utils/output'; + +export interface InteractExecuteOptions { + scrapeId?: string; + prompt?: string; + code?: string; + language?: 'python' | 'node' | 'bash'; + timeout?: number; + apiKey?: string; + apiUrl?: string; + output?: string; + json?: boolean; +} + +export interface InteractStopOptions { + scrapeId?: string; + apiKey?: string; + apiUrl?: string; + output?: string; + json?: boolean; +} + +function resolveApiConfig(options: { apiKey?: string; apiUrl?: string }) { + if (options.apiKey || options.apiUrl) { + getClient({ apiKey: options.apiKey, apiUrl: options.apiUrl }); + } + const config = getConfig(); + const apiKey = options.apiKey || config.apiKey; + validateConfig(apiKey); + const apiUrl = options.apiUrl || config.apiUrl || 'https://api.firecrawl.dev'; + return { apiKey: apiKey!, apiUrl: apiUrl.replace(/\/$/, '') }; +} + +/** + * Execute a prompt or code in an interactive browser session bound to a scrape + */ +export async function handleInteractExecute( + options: InteractExecuteOptions +): Promise { + try { + const scrapeId = getScrapeId(options.scrapeId); + const { apiKey, apiUrl } = resolveApiConfig(options); + + const stored = loadInteractSession(); + if (!options.scrapeId && stored) { + process.stderr.write( + `Using scrape ${scrapeId}` + + (stored.url ? ` (${stored.url})` : '') + + '\n' + ); + } + + const body: Record = { origin: 'cli', integration: 'cli' }; + + if (options.code) { + body.code = options.code; + body.language = options.language || 'node'; + } else if (options.prompt) { + body.prompt = options.prompt; + } + + if (options.timeout !== undefined) body.timeout = options.timeout; + + const url = `${apiUrl}/v2/scrape/${scrapeId}/interact`; + const response = await fetch(url, { + method: 'POST', + headers: { + Authorization: `Bearer ${apiKey}`, + 'Content-Type': 'application/json', + }, + body: JSON.stringify(body), + }); + + if (!response.ok) { + const errorData = await response.json().catch(() => ({})); + throw new Error( + (errorData as any).error || + `HTTP ${response.status}: ${response.statusText}` + ); + } + + const data = (await response.json()) as Record; + + if (!data.success) { + console.error('Error:', data.error || 'Unknown error'); + process.exit(1); + } + + if (options.json) { + const output = JSON.stringify(data, null, 2); + writeOutput(output, options.output, !!options.output); + return; + } + + if (data.liveViewUrl) { + process.stderr.write(`Live View: ${data.liveViewUrl}\n`); + } + if (data.interactiveLiveViewUrl) { + process.stderr.write( + `Interactive Live View: ${data.interactiveLiveViewUrl}\n` + ); + } + + if (options.prompt && data.output) { + writeOutput(data.output, options.output, !!options.output); + } else { + const result = data.stdout || data.result || ''; + if (result) { + writeOutput(result.trimEnd(), options.output, !!options.output); + } + } + + if (data.stderr) { + process.stderr.write(data.stderr); + } + } catch (error) { + console.error( + 'Error:', + error instanceof Error ? error.message : 'Unknown error occurred' + ); + process.exit(1); + } +} + +/** + * Stop an interactive browser session bound to a scrape + */ +export async function handleInteractStop( + options: InteractStopOptions +): Promise { + try { + const scrapeId = getScrapeId(options.scrapeId); + const { apiKey, apiUrl } = resolveApiConfig(options); + + const url = `${apiUrl}/v2/scrape/${scrapeId}/interact`; + const response = await fetch(url, { + method: 'DELETE', + headers: { + Authorization: `Bearer ${apiKey}`, + 'Content-Type': 'application/json', + }, + }); + + if (!response.ok) { + const errorData = await response.json().catch(() => ({})); + throw new Error( + (errorData as any).error || + `HTTP ${response.status}: ${response.statusText}` + ); + } + + const data = (await response.json()) as Record; + + if (!data.success) { + console.error('Error:', data.error || 'Unknown error'); + process.exit(1); + } + + // Clear the stored session + const stored = loadInteractSession(); + if (stored && stored.scrapeId === scrapeId) { + clearInteractSession(); + } + + if (options.json) { + const output = JSON.stringify(data, null, 2); + writeOutput(output, options.output, !!options.output); + } else { + const lines: string[] = [`Session stopped (${scrapeId})`]; + if (data.sessionDurationMs !== undefined) { + const seconds = (data.sessionDurationMs / 1000).toFixed(1); + lines.push(`Duration: ${seconds}s`); + } + if (data.creditsBilled !== undefined) { + lines.push(`Credits billed: ${data.creditsBilled}`); + } + writeOutput(lines.join('\n'), options.output, !!options.output); + } + } catch (error) { + console.error( + 'Error:', + error instanceof Error ? error.message : 'Unknown error occurred' + ); + process.exit(1); + } +} diff --git a/src/commands/scrape.ts b/src/commands/scrape.ts index 7f68b5e..343f1ed 100644 --- a/src/commands/scrape.ts +++ b/src/commands/scrape.ts @@ -11,6 +11,7 @@ import type { } from '../types/scrape'; import { getClient } from '../utils/client'; import { handleScrapeOutput, writeOutput } from '../utils/output'; +import { saveInteractSession } from '../utils/interact-session'; import { getOrigin } from '../utils/url'; import { executeMap } from './map'; import { getStatus } from './status'; @@ -110,6 +111,10 @@ export async function executeScrape( scrapeParams.location = options.location; } + if (options.profile) { + scrapeParams.profile = options.profile; + } + // Execute scrape with timing - only wrap the scrape call in try-catch const requestStartTime = Date.now(); @@ -118,6 +123,19 @@ export async function executeScrape( const requestEndTime = Date.now(); outputTiming(options, requestStartTime, requestEndTime); + const scrapeId = result?.metadata?.scrapeId; + if (scrapeId) { + try { + saveInteractSession({ + scrapeId, + url: options.url, + createdAt: new Date().toISOString(), + }); + } catch { + // Non-critical — don't fail the scrape + } + } + return { success: true, data: result, diff --git a/src/index.ts b/src/index.ts index 072aa29..79fbf7a 100644 --- a/src/index.ts +++ b/src/index.ts @@ -25,6 +25,7 @@ import { handleBrowserClose, handleBrowserQuickExecute, } from './commands/browser'; +import { handleInteractExecute, handleInteractStop } from './commands/interact'; import { handleVersionCommand } from './commands/version'; import { handleLoginCommand } from './commands/login'; import { handleLogoutCommand } from './commands/logout'; @@ -62,6 +63,7 @@ const AUTH_REQUIRED_COMMANDS = [ 'search', 'agent', 'browser', + 'interact', 'credit-usage', ]; @@ -162,6 +164,14 @@ function createScrapeCommand(): Command { '-Q, --query ', 'Ask a question about the page content (query format)' ) + .option( + '--profile ', + 'Persistent browser profile name for maintaining state across scrapes' + ) + .option( + '--no-save-changes', + 'Load existing profile data without saving changes (default: saves changes)' + ) .action(async (positionalArgs, options) => { // Collect URLs from positional args and --url option @@ -735,12 +745,12 @@ function createAgentCommand(): Command { } /** - * Create and configure the browser command + * Create and configure the browser command (deprecated — prefer scrape + interact) */ function createBrowserCommand(): Command { const browserCmd = new Command('browser') .description( - 'Launch cloud browser sessions and execute Python, JavaScript, or bash code remotely via Playwright' + '[Deprecated: prefer scrape + interact] Launch cloud browser sessions and execute code remotely via Playwright' ) .argument('[code]', 'Shorthand: auto-launch session + execute command') .option( @@ -1029,12 +1039,156 @@ Examples: return browserCmd; } -// Add crawl, map, search, agent, and browser commands to main program +/** + * Create and configure the interact command + */ +function createInteractCommand(): Command { + const interactCmd = new Command('interact') + .description( + 'Interact with a scraped page in a live browser session. Run AI prompts or execute code against any previous scrape.' + ) + .argument('[args...]', 'Prompt text, or scrape-id followed by prompt text') + .option('-c, --code ', 'Code to execute in the browser sandbox') + .option( + '-p, --prompt ', + 'AI prompt (alternative to positional argument)' + ) + .option('-s, --scrape-id ', 'Scrape job ID (default: last scrape)') + .option('--node', 'Execute code as Node.js/Playwright (default)', false) + .option('--python', 'Execute code as Python/Playwright', false) + .option('--bash', 'Execute code as Bash', false) + .option( + '--timeout ', + 'Timeout in seconds (1-300, default: 30)', + parseInt + ) + .option( + '-k, --api-key ', + 'Firecrawl API key (overrides global --api-key)' + ) + .option('--api-url ', 'API URL (overrides global --api-url)') + .option('-o, --output ', 'Output file path (default: stdout)') + .option('--json', 'Output as JSON format', false) + .addHelpText( + 'after', + ` + The scrape ID is saved automatically after every scrape, so you + don't need to pass it explicitly. Just scrape and interact: + + $ firecrawl scrape https://example.com + $ firecrawl interact "Click the pricing tab" + $ firecrawl interact "What is the price of the Pro plan?" + $ firecrawl interact stop + + You can also pass a scrape ID explicitly: + + $ firecrawl interact "Click the pricing tab" + $ firecrawl interact -s "Click the pricing tab" + + Code execution: + + $ firecrawl interact -c "await page.title()" + $ firecrawl interact -c "print(await page.title())" --python + $ firecrawl interact -c "snapshot" --bash +` + ) + .action(async (positionalArgs: string[], options) => { + // Disambiguate positional args: if the first arg looks like a UUID, + // treat it as scrape-id; otherwise treat everything as prompt text. + let scrapeId: string | undefined = options.scrapeId; + let prompt: string | undefined = options.prompt; + + if (positionalArgs.length > 0) { + if (!scrapeId && isJobId(positionalArgs[0])) { + scrapeId = positionalArgs[0]; + if (positionalArgs.length > 1) { + prompt = prompt || positionalArgs.slice(1).join(' '); + } + } else { + prompt = prompt || positionalArgs.join(' '); + } + } + + if (!options.code && !prompt) { + console.error( + 'Error: Provide an AI prompt or use --code to execute code.\n' + + 'Example: firecrawl interact "Click the pricing tab"' + ); + process.exit(1); + } + + if (options.code && prompt) { + console.error('Error: Provide either a prompt or --code, not both.'); + process.exit(1); + } + + const flagCount = [options.python, options.node, options.bash].filter( + Boolean + ).length; + if (flagCount > 1) { + console.error( + 'Error: Only one of --python, --node, or --bash can be specified' + ); + process.exit(1); + } + const language = options.python + ? 'python' + : options.bash + ? 'bash' + : 'node'; + + await handleInteractExecute({ + scrapeId, + prompt: options.code ? undefined : prompt, + code: options.code, + language, + timeout: options.timeout, + apiKey: options.apiKey, + apiUrl: options.apiUrl, + output: options.output, + json: options.json, + }); + }); + + interactCmd + .command('stop') + .description('Stop the interactive browser session for a scrape') + .argument('[scrape-id]', 'Scrape job ID (default: last scrape)') + .option( + '-k, --api-key ', + 'Firecrawl API key (overrides global --api-key)' + ) + .option('--api-url ', 'API URL (overrides global --api-url)') + .option('-o, --output ', 'Output file path (default: stdout)') + .option('--json', 'Output as JSON format', false) + .addHelpText( + 'after', + ` +Examples: + $ firecrawl interact stop + $ firecrawl interact stop +` + ) + .action(async (scrapeId, options) => { + await handleInteractStop({ + scrapeId, + apiKey: options.apiKey, + apiUrl: options.apiUrl, + output: options.output, + json: options.json, + }); + }); + + return interactCmd; +} + +// Add crawl, map, search, agent, browser, and interact commands to main program program.addCommand(createCrawlCommand()); program.addCommand(createMapCommand()); program.addCommand(createSearchCommand()); program.addCommand(createAgentCommand()); program.addCommand(createBrowserCommand()); +program.addCommand(createInteractCommand()); // Experimental: AI workflow commands program.addCommand(createClaudeCommand()); diff --git a/src/types/scrape.ts b/src/types/scrape.ts index 345b797..dbdb9b1 100644 --- a/src/types/scrape.ts +++ b/src/types/scrape.ts @@ -57,6 +57,11 @@ export interface ScrapeOptions { location?: ScrapeLocation; /** Question to ask about the page content (query format) */ query?: string; + /** Persistent browser profile for maintaining state across scrapes */ + profile?: { + name: string; + saveChanges?: boolean; + }; } export interface ScrapeResult { diff --git a/src/utils/interact-session.ts b/src/utils/interact-session.ts new file mode 100644 index 0000000..188a167 --- /dev/null +++ b/src/utils/interact-session.ts @@ -0,0 +1,91 @@ +/** + * Interact session persistence utility + * Stores the last scrape ID so interact commands can reuse it automatically + */ + +import * as fs from 'fs'; +import * as path from 'path'; +import * as os from 'os'; + +export interface StoredInteractSession { + scrapeId: string; + url: string; + createdAt: string; +} + +function getConfigDir(): string { + const homeDir = os.homedir(); + const platform = os.platform(); + + switch (platform) { + case 'darwin': + return path.join( + homeDir, + 'Library', + 'Application Support', + 'firecrawl-cli' + ); + case 'win32': + return path.join(homeDir, 'AppData', 'Roaming', 'firecrawl-cli'); + default: + return path.join(homeDir, '.config', 'firecrawl-cli'); + } +} + +function getSessionPath(): string { + return path.join(getConfigDir(), 'interact-session.json'); +} + +function ensureConfigDir(): void { + const configDir = getConfigDir(); + if (!fs.existsSync(configDir)) { + fs.mkdirSync(configDir, { recursive: true, mode: 0o700 }); + } +} + +export function saveInteractSession(session: StoredInteractSession): void { + ensureConfigDir(); + fs.writeFileSync(getSessionPath(), JSON.stringify(session, null, 2), 'utf-8'); +} + +export function loadInteractSession(): StoredInteractSession | null { + try { + const sessionPath = getSessionPath(); + if (!fs.existsSync(sessionPath)) { + return null; + } + return JSON.parse( + fs.readFileSync(sessionPath, 'utf-8') + ) as StoredInteractSession; + } catch { + return null; + } +} + +export function clearInteractSession(): void { + try { + const sessionPath = getSessionPath(); + if (fs.existsSync(sessionPath)) { + fs.unlinkSync(sessionPath); + } + } catch { + // Ignore errors + } +} + +/** + * Resolve scrape ID from explicit override or stored session + */ +export function getScrapeId(overrideId?: string): string { + if (overrideId) return overrideId; + + const stored = loadInteractSession(); + if (stored) return stored.scrapeId; + + throw new Error( + 'No active scrape session. Scrape a URL first:\n' + + ' firecrawl scrape https://example.com\n' + + 'Or specify a scrape ID explicitly:\n' + + ' firecrawl interact "prompt"' + ); +} diff --git a/src/utils/options.ts b/src/utils/options.ts index 52a3a95..e8c382b 100644 --- a/src/utils/options.ts +++ b/src/utils/options.ts @@ -90,6 +90,14 @@ export function parseScrapeOptions(options: any): ScrapeOptions { } } + let profile: { name: string; saveChanges?: boolean } | undefined; + if (options.profile) { + profile = { + name: options.profile, + saveChanges: options.saveChanges, + }; + } + return { url: options.url, formats, @@ -112,5 +120,6 @@ export function parseScrapeOptions(options: any): ScrapeOptions { maxAge: options.maxAge, location, query: options.query, + profile, }; }