diff --git a/.adl-ignore b/.adl-ignore index 0c42fc1..187ceb6 100644 --- a/.adl-ignore +++ b/.adl-ignore @@ -17,6 +17,7 @@ skills/take_screenshot.go skills/execute_script.go skills/handle_authentication.go skills/wait_for_condition.go +skills/write_to_csv.go internal/playwright/playwright.go # Go dependency files diff --git a/.github/workflows/cd.yml b/.github/workflows/cd.yml index 7424253..115f134 100644 --- a/.github/workflows/cd.yml +++ b/.github/workflows/cd.yml @@ -1,4 +1,4 @@ -# Code generated by ADL CLI v0.21.4. DO NOT EDIT. +# Code generated by ADL CLI v0.21.5. DO NOT EDIT. # This file was automatically generated from an ADL (Agent Definition Language) specification. # Manual changes to this file may be overwritten during regeneration. diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index 90e6dbb..8224116 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -1,4 +1,4 @@ -# Code generated by ADL CLI v0.21.4. DO NOT EDIT. +# Code generated by ADL CLI v0.21.5. DO NOT EDIT. # This file was automatically generated from an ADL (Agent Definition Language) specification. # Manual changes to this file may be overwritten during regeneration. diff --git a/.gitignore b/.gitignore index 07b5fdc..7f692b4 100644 --- a/.gitignore +++ b/.gitignore @@ -40,7 +40,7 @@ Thumbs.db # Environment files .env* -!.env.*.example +!.env*.example # Log files *.log diff --git a/.releaserc.yaml b/.releaserc.yaml index 0763f70..9d0ee1c 100644 --- a/.releaserc.yaml +++ b/.releaserc.yaml @@ -1,4 +1,4 @@ -# Code generated by ADL CLI v0.21.4. DO NOT EDIT. +# Code generated by ADL CLI v0.21.5. DO NOT EDIT. # This file was automatically generated from an ADL (Agent Definition Language) specification. # Manual changes to this file may be overwritten during regeneration. diff --git a/.well-known/agent.json b/.well-known/agent.json index c41319e..0622721 100644 --- a/.well-known/agent.json +++ b/.well-known/agent.json @@ -68,6 +68,13 @@ "description": "Wait for specific conditions before proceeding with automation", "tags": ["wait","synchronization","timing","playwright"], "schema": {"properties":{"condition":{"description":"Type of condition (selector, navigation, function, timeout, networkidle)","type":"string"},"custom_function":{"description":"Custom JavaScript function to evaluate for 'function' condition","type":"string"},"selector":{"description":"Selector to wait for if condition is 'selector'","type":"string"},"state":{"default":"visible","description":"State to wait for (visible, hidden, attached, detached)","type":"string"},"timeout":{"default":30000,"description":"Maximum time to wait in milliseconds","type":"integer"}},"required":["condition"],"type":"object"} + }, + { + "id": "write_to_csv", + "name": "write_to_csv", + "description": "Write structured data to CSV files with support for custom headers and file paths", + "tags": ["export","csv","data","file"], + "schema": {"properties":{"append":{"default":false,"description":"Whether to append to existing file or create new file","type":"boolean"},"data":{"description":"Array of objects to write to CSV, each object represents a row","items":{"type":"object"},"type":"array"},"filename":{"description":"Name of the CSV file (without path, will be saved to configured data directory)","type":"string"},"headers":{"description":"Custom column headers for the CSV file (optional, will use object keys if not provided)","items":{"type":"string"},"type":"array"},"include_headers":{"default":true,"description":"Whether to include headers in the CSV output","type":"boolean"}},"required":["data","filename"],"type":"object"} } ] } diff --git a/AGENTS.md b/AGENTS.md index 6cd4340..776d6c2 100644 --- a/AGENTS.md +++ b/AGENTS.md @@ -75,7 +75,7 @@ Your automation solutions should be maintainable, efficient, and production-read ## Skills -This agent provides 8 skills: +This agent provides 9 skills: ### navigate_to_url @@ -134,6 +134,13 @@ This agent provides 8 skills: - **Output Schema**: Defined in agent configuration +### write_to_csv +- **Description**: Write structured data to CSV files with support for custom headers and file paths +- **Tags**: export, csv, data, file +- **Input Schema**: Defined in agent configuration +- **Output Schema**: Defined in agent configuration + + ## Server Configuration @@ -239,6 +246,11 @@ curl -X POST http://localhost:8080/skills/wait_for_condition \ -H "Content-Type: application/json" \ -d '{"input": "your_input_here"}' +# Execute write_to_csv skill +curl -X POST http://localhost:8080/skills/write_to_csv \ + -H "Content-Type: application/json" \ + -d '{"input": "your_input_here"}' + ``` @@ -286,6 +298,8 @@ docker run -p 8080:8080 browser-agent │ └── wait_for_condition.go # Wait for specific conditions before proceeding with automation +│ └── write_to_csv.go # Write structured data to CSV files with support for custom headers and file paths + ├── .well-known/ # Agent configuration │ └── agent.json # Agent metadata ├── go.mod # Go module definition diff --git a/CLAUDE.md b/CLAUDE.md index d8a844f..ae56b81 100644 --- a/CLAUDE.md +++ b/CLAUDE.md @@ -10,7 +10,7 @@ browser-agent is an A2A (Agent-to-Agent) server implementing the [A2A Protocol]( ### ADL-Generated Structure -The codebase is generated using ADL CLI 0.21.4 and follows a strict generation pattern: +The codebase is generated using ADL CLI 0.21.5 and follows a strict generation pattern: - **Generated Files**: Marked with `DO NOT EDIT` headers - manual changes will be overwritten - **Configuration Source**: `agent.yaml` - defines agent capabilities, skills, and metadata - **Server Implementation**: Built on the ADK (Agent Development Kit) framework from `github.com/inference-gateway/adk` @@ -82,6 +82,7 @@ The following skills are currently defined: - **execute_script**: Execute custom JavaScript code in the browser context - **handle_authentication**: Handle various authentication scenarios including basic auth, OAuth, and custom login forms - **wait_for_condition**: Wait for specific conditions before proceeding with automation +- **write_to_csv**: Write structured data to CSV files with support for custom headers and file paths To modify skills: 1. Update `agent.yaml` with skill definitions @@ -117,7 +118,7 @@ Activate with: `flox activate` (if Flox is installed) - **Generated Files**: Never manually edit files with "DO NOT EDIT" headers - **Configuration Changes**: Always modify `agent.yaml` and regenerate -- **ADL Version**: Ensure ADL CLI 0.21.4 or compatible version for regeneration +- **ADL Version**: Ensure ADL CLI 0.21.5 or compatible version for regeneration - **Port Configuration**: Default 8080, configurable via `A2A_PORT` or `A2A_SERVER_PORT` ## Debugging Tips diff --git a/README.md b/README.md index f25c63f..a8a25c1 100644 --- a/README.md +++ b/README.md @@ -49,6 +49,7 @@ docker run -p 8080:8080 browser-agent | `execute_script` | Execute custom JavaScript code in the browser context |args, return_value, script | | `handle_authentication` | Handle various authentication scenarios including basic auth, OAuth, and custom login forms |login_url, password, password_selector, submit_selector, type, username, username_selector | | `wait_for_condition` | Wait for specific conditions before proceeding with automation |condition, custom_function, selector, state, timeout | +| `write_to_csv` | Write structured data to CSV files with support for custom headers and file paths |append, data, filename, headers, include_headers | ## Configuration @@ -61,13 +62,13 @@ The following custom configuration variables are available: | Category | Variable | Description | Default | |----------|----------|-------------|---------| | **Browser** | `BROWSER_ARGS` | Args configuration | `[--disable-blink-features=AutomationControlled --disable-features=VizDisplayCompositor --no-first-run --disable-default-apps --disable-extensions --disable-plugins --disable-sync --disable-translate --hide-scrollbars --mute-audio --no-zygote --disable-background-timer-throttling --disable-backgrounding-occluded-windows --disable-renderer-backgrounding --disable-ipc-flooding-protection]` | +| **Browser** | `BROWSER_DATA_DIR` | Data_dir configuration | `/tmp/playwright/artifacts` | | **Browser** | `BROWSER_HEADER_ACCEPT` | Header_accept configuration | `text/html,application/xhtml+xml,application/xml;q=0.9,image/avif,image/webp,image/apng,*/*;q=0.8,application/signed-exchange;v=b3;q=0.7` | | **Browser** | `BROWSER_HEADER_ACCEPT_ENCODING` | Header_accept_encoding configuration | `gzip, deflate, br` | | **Browser** | `BROWSER_HEADER_ACCEPT_LANGUAGE` | Header_accept_language configuration | `en-US,en;q=0.9` | | **Browser** | `BROWSER_HEADER_CONNECTION` | Header_connection configuration | `keep-alive` | | **Browser** | `BROWSER_HEADER_DNT` | Header_dnt configuration | `1` | | **Browser** | `BROWSER_HEADER_UPGRADE_INSECURE_REQUESTS` | Header_upgrade_insecure_requests configuration | `1` | -| **Browser** | `BROWSER_SCREENSHOTS_DIR` | Screenshots_dir configuration | `/tmp/screenshots` | | **Browser** | `BROWSER_USER_AGENT` | User_agent configuration | `Mozilla/5.0 (X11; Linux x86_64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/131.0.0.0 Safari/537.36` | | **Browser** | `BROWSER_VIEWPORT_HEIGHT` | Viewport_height configuration | `1080` | | **Browser** | `BROWSER_VIEWPORT_WIDTH` | Viewport_width configuration | `1920` | diff --git a/Taskfile.yml b/Taskfile.yml index 8956a8c..1b64a52 100644 --- a/Taskfile.yml +++ b/Taskfile.yml @@ -1,4 +1,4 @@ -# Code generated by ADL CLI v0.21.4. DO NOT EDIT. +# Code generated by ADL CLI v0.21.5. DO NOT EDIT. # This file was automatically generated from an ADL (Agent Definition Language) specification. # Manual changes to this file may be overwritten during regeneration. diff --git a/agent.yaml b/agent.yaml index 2ab36fe..0529d3e 100644 --- a/agent.yaml +++ b/agent.yaml @@ -20,7 +20,7 @@ spec: header_dnt: "1" header_connection: "keep-alive" header_upgrade_insecure_requests: "1" - screenshots_dir: "/tmp/screenshots" + data_dir: "/tmp/playwright/artifacts" args: - "--disable-blink-features=AutomationControlled" - "--disable-features=VizDisplayCompositor" @@ -310,6 +310,44 @@ spec: inject: - logger - playwright + - id: write_to_csv + name: write_to_csv + description: Write structured data to CSV files with support for custom headers and file paths + tags: + - export + - csv + - data + - file + schema: + type: object + properties: + data: + type: array + items: + type: object + description: Array of objects to write to CSV, each object represents a row + filename: + type: string + description: Name of the CSV file (without path, will be saved to configured data directory) + headers: + type: array + items: + type: string + description: Custom column headers for the CSV file (optional, will use object keys if not provided) + append: + type: boolean + description: Whether to append to existing file or create new file + default: false + include_headers: + type: boolean + description: Whether to include headers in the CSV output + default: true + required: + - data + - filename + inject: + - logger + - playwright agent: provider: "" model: "" diff --git a/config/config.go b/config/config.go index b0c8869..140e613 100644 --- a/config/config.go +++ b/config/config.go @@ -1,4 +1,4 @@ -// Code generated by ADL CLI v0.21.4. DO NOT EDIT. +// Code generated by ADL CLI v0.21.5. DO NOT EDIT. // This file was automatically generated from an ADL (Agent Definition Language) specification. // Manual changes to this file may be overwritten during regeneration. @@ -23,13 +23,13 @@ type Config struct { // BrowserConfig represents the browser configuration type BrowserConfig struct { Args string `env:"ARGS,default=[--disable-blink-features=AutomationControlled --disable-features=VizDisplayCompositor --no-first-run --disable-default-apps --disable-extensions --disable-plugins --disable-sync --disable-translate --hide-scrollbars --mute-audio --no-zygote --disable-background-timer-throttling --disable-backgrounding-occluded-windows --disable-renderer-backgrounding --disable-ipc-flooding-protection]"` + DataDir string `env:"DATA_DIR,default=/tmp/playwright/artifacts"` HeaderAccept string `env:"HEADER_ACCEPT,default=text/html,application/xhtml+xml,application/xml;q=0.9,image/avif,image/webp,image/apng,*/*;q=0.8,application/signed-exchange;v=b3;q=0.7"` HeaderAcceptEncoding string `env:"HEADER_ACCEPT_ENCODING,default=gzip, deflate, br"` HeaderAcceptLanguage string `env:"HEADER_ACCEPT_LANGUAGE,default=en-US,en;q=0.9"` HeaderConnection string `env:"HEADER_CONNECTION,default=keep-alive"` HeaderDnt string `env:"HEADER_DNT,default=1"` HeaderUpgradeInsecureRequests string `env:"HEADER_UPGRADE_INSECURE_REQUESTS,default=1"` - ScreenshotsDir string `env:"SCREENSHOTS_DIR,default=/tmp/screenshots"` UserAgent string `env:"USER_AGENT,default=Mozilla/5.0 (X11; Linux x86_64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/131.0.0.0 Safari/537.36"` ViewportHeight string `env:"VIEWPORT_HEIGHT,default=1080"` ViewportWidth string `env:"VIEWPORT_WIDTH,default=1920"` diff --git a/example/.env.example b/example/.env.example new file mode 100644 index 0000000..4ad0c5b --- /dev/null +++ b/example/.env.example @@ -0,0 +1,7 @@ +# Inference Gateway +DEEPSEEK_API_KEY= +GOOGLE_API_KEY= + +# Agent +A2A_AGENT_CLIENT_PROVIDER=deepseek +A2A_AGENT_CLIENT_MODEL=deepseek-chat diff --git a/example/.env.gateway.example b/example/.env.gateway.example deleted file mode 100644 index c50e16c..0000000 --- a/example/.env.gateway.example +++ /dev/null @@ -1,2 +0,0 @@ -DEEPSEEK_API_KEY= -GOOGLE_API_KEY= diff --git a/example/.gitignore b/example/.gitignore index b490341..e69de29 100644 --- a/example/.gitignore +++ b/example/.gitignore @@ -1 +0,0 @@ -screenshots \ No newline at end of file diff --git a/example/README.md b/example/README.md index d7be6b6..dfe0510 100644 --- a/example/README.md +++ b/example/README.md @@ -6,7 +6,7 @@ This script demonstrates how to use the Playwright automation framework to perfo Configure the environment variables as needed: ```bash -cp .env.gateway.example .env.gateway +cp .env.example .env ``` ** Add at least two providers, in this example Google and DeepSeek. diff --git a/example/artifacts/.gitignore b/example/artifacts/.gitignore new file mode 100644 index 0000000..d6b7ef3 --- /dev/null +++ b/example/artifacts/.gitignore @@ -0,0 +1,2 @@ +* +!.gitignore diff --git a/example/docker-compose.yaml b/example/docker-compose.yaml index 8fe9b11..97a8d69 100644 --- a/example/docker-compose.yaml +++ b/example/docker-compose.yaml @@ -8,9 +8,8 @@ services: ports: - "8080:8080" volumes: - - ./screenshots:/tmp/screenshots + - ./artifacts:/tmp/playwright/artifacts environment: - BROWSER_SCREENSHOTS_DIR: /tmp/screenshots BROWSER_USER_AGENT: "Mozilla/5.0 (X11; Linux x86_64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/131.0.0.0 Safari/537.36" BROWSER_VIEWPORT_WIDTH: "1920" BROWSER_VIEWPORT_HEIGHT: "1080" @@ -45,8 +44,8 @@ services: A2A_SERVER_IDLE_TIMEOUT: 120s A2A_SERVER_DISABLE_HEALTHCHECK_LOG: true A2A_AGENT_CARD_FILE_PATH: .well-known/agent.json - A2A_AGENT_CLIENT_PROVIDER: google - A2A_AGENT_CLIENT_MODEL: models/gemini-2.5-flash + A2A_AGENT_CLIENT_PROVIDER: ${A2A_AGENT_CLIENT_PROVIDER} + A2A_AGENT_CLIENT_MODEL: ${A2A_AGENT_CLIENT_MODEL} A2A_AGENT_CLIENT_API_KEY: "" A2A_AGENT_CLIENT_BASE_URL: http://inference-gateway:8080/v1 A2A_AGENT_CLIENT_TIMEOUT: 30s @@ -72,13 +71,14 @@ services: image: ghcr.io/inference-gateway/inference-gateway:latest container_name: inference-gateway environment: + DEEPSEEK_API_KEY: ${DEEPSEEK_API_KEY} + GOOGLE_API_KEY: ${GOOGLE_API_KEY} ENVIRONMENT: development SERVER_READ_TIMEOUT: 530s SERVER_WRITE_TIMEOUT: 530s CLIENT_TIMEOUT: 530s - CLIENT_IDLE_CONN_TIMEOUT: 30s - env_file: - .env.gateway + CLIENT_IDLE_CONN_TIMEOUT: 130s + CLIENT_RESPONSE_HEADER_TIMEOUT: 120s networks: - a2a-network @@ -89,23 +89,10 @@ services: INFER_LOGGING_DEBUG: true INFER_GATEWAY_URL: http://inference-gateway:8080 INFER_A2A_ENABLED: true - INFER_TOOLS_ENABLED: true - INFER_TOOLS_QUERY_ENABLED: true - INFER_TOOLS_TASK_ENABLED: true - INFER_TOOLS_BASH_ENABLED: false - INFER_TOOLS_TODO_WRITE_ENABLED: false - INFER_TOOLS_WRITE_ENABLED: false - INFER_TOOLS_READ_ENABLED: false - INFER_TOOLS_DELETE_ENABLED: false - INFER_TOOLS_EDIT_ENABLED: false - INFER_TOOLS_GREP_ENABLED: false - INFER_TOOLS_TREE_ENABLED: false - INFER_TOOLS_WEB_FETCH_ENABLED: false - INFER_TOOLS_WEB_SEARCH_ENABLED: false - INFER_TOOLS_GITHUB_ENABLED: false + INFER_TOOLS_ENABLED: false INFER_AGENT_MODEL: deepseek/deepseek-chat - INFER_A2A_AGENTS: 'http://agent:8080' - INFER_A2A_CACHE_ENABLED: false + INFER_A2A_AGENTS: | + http://agent:8080 command: - chat networks: diff --git a/example/integration_demo.md b/example/integration_demo.md new file mode 100644 index 0000000..00f1148 --- /dev/null +++ b/example/integration_demo.md @@ -0,0 +1,76 @@ +# CSV Export Integration Demo + +This document demonstrates how to use the new `write_to_csv` skill in combination with the existing `extract_data` skill for complete data collection workflows. + +## Workflow Example + +1. **Navigate to a webpage**: + ```json + { + "skill": "navigate_to_url", + "args": { + "url": "https://example.com/products" + } + } + ``` + +2. **Extract data from the page**: + ```json + { + "skill": "extract_data", + "args": { + "extractors": [ + { + "name": "product_name", + "selector": ".product-title", + "multiple": true + }, + { + "name": "price", + "selector": ".product-price", + "multiple": true + }, + { + "name": "rating", + "selector": ".product-rating", + "attribute": "data-rating", + "multiple": true + } + ], + "format": "json" + } + } + ``` + +3. **Write the extracted data to CSV**: + ```json + { + "skill": "write_to_csv", + "args": { + "data": [ + {"product_name": "Product A", "price": "$29.99", "rating": "4.5"}, + {"product_name": "Product B", "price": "$39.99", "rating": "4.2"}, + {"product_name": "Product C", "price": "$19.99", "rating": "4.8"} + ], + "file_path": "/tmp/products.csv", + "headers": ["product_name", "price", "rating"], + "include_headers": true + } + } + ``` + +## Features Supported + +- **Custom Headers**: Specify column order and names +- **Append Mode**: Add to existing CSV files without overwriting +- **Flexible Data**: Handles arrays, objects, and primitive values +- **Error Handling**: Validates data format and file operations +- **Directory Creation**: Automatically creates parent directories + +## Use Cases + +- **E-commerce Data Collection**: Extract product information, prices, and reviews +- **News Aggregation**: Collect headlines, dates, and article links +- **Financial Data**: Gather stock prices, market data, and trading volumes +- **Contact Information**: Extract business details from directory sites +- **Event Listings**: Collect event names, dates, venues, and prices \ No newline at end of file diff --git a/internal/logger/logger.go b/internal/logger/logger.go index 0626c53..2ec8f25 100644 --- a/internal/logger/logger.go +++ b/internal/logger/logger.go @@ -1,4 +1,4 @@ -// Code generated by ADL CLI v0.21.4. DO NOT EDIT. +// Code generated by ADL CLI v0.21.5. DO NOT EDIT. // This file was automatically generated from an ADL (Agent Definition Language) specification. // Manual changes to this file may be overwritten during regeneration. diff --git a/internal/playwright/mocks/browser_automation.go b/internal/playwright/mocks/browser_automation.go index eb4db7e..56a5d7d 100644 --- a/internal/playwright/mocks/browser_automation.go +++ b/internal/playwright/mocks/browser_automation.go @@ -6,8 +6,8 @@ import ( "sync" "time" - "github.com/inference-gateway/browser-agent/internal/playwright" "github.com/inference-gateway/browser-agent/config" + "github.com/inference-gateway/browser-agent/internal/playwright" ) type FakeBrowserAutomation struct { diff --git a/internal/playwright/playwright.go b/internal/playwright/playwright.go index dcfc966..ebcf631 100644 --- a/internal/playwright/playwright.go +++ b/internal/playwright/playwright.go @@ -80,9 +80,8 @@ func NewBrowserConfigFromConfig(cfg *config.Config) *BrowserConfig { height = 1080 } - // Parse args from config - remove brackets and split by space argsStr := strings.Trim(cfg.Browser.Args, "[]") - args := []string{"--disable-dev-shm-usage", "--no-sandbox"} // Always include these + args := []string{"--disable-dev-shm-usage", "--no-sandbox"} if argsStr != "" { configArgs := strings.Fields(argsStr) args = append(args, configArgs...) diff --git a/internal/playwright/playwright_integration_test.go b/internal/playwright/playwright_integration_test.go index a657e3d..580a923 100644 --- a/internal/playwright/playwright_integration_test.go +++ b/internal/playwright/playwright_integration_test.go @@ -5,9 +5,9 @@ import ( "testing" "time" + "github.com/inference-gateway/browser-agent/config" "github.com/inference-gateway/browser-agent/internal/playwright" "github.com/inference-gateway/browser-agent/internal/playwright/mocks" - "github.com/inference-gateway/browser-agent/config" "go.uber.org/zap" ) diff --git a/main.go b/main.go index 2c5b94a..fae722c 100644 --- a/main.go +++ b/main.go @@ -1,4 +1,4 @@ -// Code generated by ADL CLI v0.21.4. DO NOT EDIT. +// Code generated by ADL CLI v0.21.5. DO NOT EDIT. // This file was automatically generated from an ADL (Agent Definition Language) specification. // Manual changes to this file may be overwritten during regeneration. @@ -92,6 +92,11 @@ func main() { toolBox.AddTool(waitForConditionSkill) l.Info("registered skill: wait_for_condition (Wait for specific conditions before proceeding with automation)") + // Register write_to_csv skill + writeToCsvSkill := skills.NewWriteToCsvSkill(l, playwrightSvc) + toolBox.AddTool(writeToCsvSkill) + l.Info("registered skill: write_to_csv (Write structured data to CSV files with support for custom headers and file paths)") + llmClient, err := server.NewOpenAICompatibleLLMClient(&cfg.A2A.AgentConfig, l) if err != nil { l.Fatal("failed to create LLM client", zap.Error(err)) diff --git a/skills/take_screenshot.go b/skills/take_screenshot.go index 0fa1bca..9ef7d8b 100644 --- a/skills/take_screenshot.go +++ b/skills/take_screenshot.go @@ -28,7 +28,7 @@ func NewTakeScreenshotSkill(logger *zap.Logger, playwright playwright.BrowserAut logger: logger, playwright: playwright, artifactHelper: server.NewArtifactHelper(), - screenshotDir: cfg.Browser.ScreenshotsDir, + screenshotDir: cfg.Browser.DataDir, } return server.NewBasicTool( "take_screenshot", diff --git a/skills/take_screenshot_test.go b/skills/take_screenshot_test.go index 117b77c..b80c57f 100644 --- a/skills/take_screenshot_test.go +++ b/skills/take_screenshot_test.go @@ -10,9 +10,9 @@ import ( "time" server "github.com/inference-gateway/adk/server" + config "github.com/inference-gateway/browser-agent/config" playwright "github.com/inference-gateway/browser-agent/internal/playwright" mocks "github.com/inference-gateway/browser-agent/internal/playwright/mocks" - config "github.com/inference-gateway/browser-agent/config" zap "go.uber.org/zap" ) @@ -36,7 +36,7 @@ func createTestSkill() *TakeScreenshotSkill { }) mockPlaywright.GetConfigReturns(&config.Config{ Browser: config.BrowserConfig{ - ScreenshotsDir: "test_screenshots", + DataDir: "test_screenshots", }, }) diff --git a/skills/write_to_csv.go b/skills/write_to_csv.go new file mode 100644 index 0000000..d76e0b8 --- /dev/null +++ b/skills/write_to_csv.go @@ -0,0 +1,286 @@ +package skills + +import ( + "context" + "encoding/csv" + "fmt" + "os" + "path/filepath" + "strconv" + + server "github.com/inference-gateway/adk/server" + playwright "github.com/inference-gateway/browser-agent/internal/playwright" + zap "go.uber.org/zap" +) + +// WriteToCsvSkill struct holds the skill with services +type WriteToCsvSkill struct { + logger *zap.Logger + playwright playwright.BrowserAutomation +} + +// NewWriteToCsvSkill creates a new write_to_csv skill +func NewWriteToCsvSkill(logger *zap.Logger, playwright playwright.BrowserAutomation) server.Tool { + skill := &WriteToCsvSkill{ + logger: logger, + playwright: playwright, + } + return server.NewBasicTool( + "write_to_csv", + "Write structured data to CSV files with support for custom headers and file paths", + map[string]any{ + "type": "object", + "properties": map[string]any{ + "append": map[string]any{ + "default": false, + "description": "Whether to append to existing file or create new file", + "type": "boolean", + }, + "data": map[string]any{ + "description": "Array of objects to write to CSV, each object represents a row", + "items": map[string]any{"type": "object"}, + "type": "array", + }, + "filename": map[string]any{ + "description": "Name of the CSV file (without path, will be saved to configured data directory)", + "type": "string", + }, + "headers": map[string]any{ + "description": "Custom column headers for the CSV file (optional, will use object keys if not provided)", + "items": map[string]any{"type": "string"}, + "type": "array", + }, + "include_headers": map[string]any{ + "default": true, + "description": "Whether to include headers in the CSV output", + "type": "boolean", + }, + }, + "required": []string{"data", "filename"}, + }, + skill.WriteToCsvHandler, + ) +} + +// WriteToCsvHandler handles the write_to_csv skill execution +func (s *WriteToCsvSkill) WriteToCsvHandler(ctx context.Context, args map[string]any) (string, error) { + data, ok := args["data"].([]any) + if !ok || len(data) == 0 { + s.logger.Error("data parameter is required and must be a non-empty array") + return "", fmt.Errorf("data parameter is required and must be a non-empty array") + } + + filename, ok := args["filename"].(string) + if !ok || filename == "" { + s.logger.Error("filename parameter is required and must be a non-empty string") + return "", fmt.Errorf("filename parameter is required and must be a non-empty string") + } + + filePath := s.generateFilePath(filename) + + var customHeaders []string + if headers, ok := args["headers"].([]any); ok { + customHeaders = make([]string, len(headers)) + for i, header := range headers { + if headerStr, ok := header.(string); ok { + customHeaders[i] = headerStr + } else { + return "", fmt.Errorf("all headers must be strings") + } + } + } + + append := false + if appendVal, ok := args["append"].(bool); ok { + append = appendVal + } + + includeHeaders := true + if includeVal, ok := args["include_headers"].(bool); ok { + includeHeaders = includeVal + } + + s.logger.Info("writing data to CSV file", + zap.String("filename", filename), + zap.String("file_path", filePath), + zap.Int("rows_count", len(data)), + zap.Bool("append", append), + zap.Bool("include_headers", includeHeaders)) + + rows, err := s.convertDataToRows(data) + if err != nil { + s.logger.Error("failed to convert data to rows", zap.Error(err)) + return "", fmt.Errorf("failed to convert data to rows: %w", err) + } + + headers := customHeaders + if len(headers) == 0 && len(rows) > 0 { + headers = s.extractHeadersFromRows(rows) + } + + rowsWritten, err := s.writeCSVFile(filePath, headers, rows, append, includeHeaders) + if err != nil { + s.logger.Error("failed to write CSV file", + zap.String("file_path", filePath), + zap.Error(err)) + return "", fmt.Errorf("failed to write CSV file: %w", err) + } + + result := fmt.Sprintf("Successfully wrote %d rows to %s", rowsWritten, filePath) + s.logger.Info("CSV file written successfully", + zap.String("file_path", filePath), + zap.Int("rows_written", rowsWritten)) + + return result, nil +} + +func (s *WriteToCsvSkill) generateFilePath(filename string) string { + var dataDir string + + if s.playwright != nil && s.playwright.GetConfig() != nil { + dataDir = s.playwright.GetConfig().Browser.DataDir + } + + if dataDir == "" { + dataDir = "." + } + + if err := os.MkdirAll(dataDir, 0755); err != nil { + s.logger.Warn("failed to create data files directory", zap.String("dir", dataDir), zap.Error(err)) + } + + if !filepath.IsAbs(filename) { + return filepath.Join(dataDir, filename) + } + return filename +} + +func (s *WriteToCsvSkill) convertDataToRows(data []any) ([]map[string]any, error) { + rows := make([]map[string]any, len(data)) + + for i, item := range data { + switch v := item.(type) { + case map[string]any: + rows[i] = v + case map[any]any: + converted := make(map[string]any) + for key, value := range v { + if keyStr, ok := key.(string); ok { + converted[keyStr] = value + } else { + converted[fmt.Sprintf("%v", key)] = value + } + } + rows[i] = converted + default: + return nil, fmt.Errorf("data item at index %d must be an object/map, got %T", i, item) + } + } + + return rows, nil +} + +func (s *WriteToCsvSkill) extractHeadersFromRows(rows []map[string]any) []string { + headerSet := make(map[string]bool) + var headers []string + + for _, row := range rows { + for key := range row { + if !headerSet[key] { + headerSet[key] = true + headers = append(headers, key) + } + } + } + + return headers +} + +func (s *WriteToCsvSkill) writeCSVFile(filePath string, headers []string, rows []map[string]any, append bool, includeHeaders bool) (int, error) { + dir := filepath.Dir(filePath) + if err := os.MkdirAll(dir, 0755); err != nil { + return 0, fmt.Errorf("failed to create directory %s: %w", dir, err) + } + + flag := os.O_CREATE | os.O_WRONLY + if append { + flag |= os.O_APPEND + } else { + flag |= os.O_TRUNC + } + + fileExists := false + if append { + if info, err := os.Stat(filePath); err == nil && info.Size() > 0 { + fileExists = true + } + } + + file, err := os.OpenFile(filePath, flag, 0644) + if err != nil { + return 0, fmt.Errorf("failed to open file %s: %w", filePath, err) + } + defer func() { + if closeErr := file.Close(); closeErr != nil { + s.logger.Error("failed to close file", zap.String("file_path", filePath), zap.Error(closeErr)) + } + }() + + writer := csv.NewWriter(file) + defer writer.Flush() + + rowsWritten := 0 + + if includeHeaders && (!append || !fileExists) { + if len(headers) > 0 { + if err := writer.Write(headers); err != nil { + return 0, fmt.Errorf("failed to write headers: %w", err) + } + } + } + + for _, row := range rows { + csvRow := make([]string, len(headers)) + for i, header := range headers { + if value, exists := row[header]; exists { + csvRow[i] = s.valueToString(value) + } else { + csvRow[i] = "" + } + } + + if err := writer.Write(csvRow); err != nil { + return rowsWritten, fmt.Errorf("failed to write row: %w", err) + } + rowsWritten++ + } + + return rowsWritten, nil +} + +func (s *WriteToCsvSkill) valueToString(value any) string { + if value == nil { + return "" + } + + switch v := value.(type) { + case string: + return v + case int: + return strconv.Itoa(v) + case int64: + return strconv.FormatInt(v, 10) + case float64: + return strconv.FormatFloat(v, 'f', -1, 64) + case bool: + return strconv.FormatBool(v) + case []any: + var items []string + for _, item := range v { + items = append(items, s.valueToString(item)) + } + return fmt.Sprintf("[%s]", fmt.Sprintf("%v", items)) + default: + return fmt.Sprintf("%v", v) + } +} diff --git a/skills/write_to_csv_test.go b/skills/write_to_csv_test.go new file mode 100644 index 0000000..5b91cef --- /dev/null +++ b/skills/write_to_csv_test.go @@ -0,0 +1,303 @@ +package skills + +import ( + "context" + "encoding/csv" + "os" + "path/filepath" + "strings" + "testing" + + config "github.com/inference-gateway/browser-agent/config" + mocks "github.com/inference-gateway/browser-agent/internal/playwright/mocks" + zap "go.uber.org/zap" +) + +func TestWriteToCsvHandler(t *testing.T) { + logger := zap.NewNop() + tempDir := t.TempDir() + mockPlaywright := &mocks.FakeBrowserAutomation{} + mockPlaywright.GetConfigReturns(&config.Config{ + Browser: config.BrowserConfig{ + DataDir: tempDir, + }, + }) + + skill := &WriteToCsvSkill{ + logger: logger, + playwright: mockPlaywright, + } + + tests := []struct { + name string + args map[string]any + expectedError bool + expectedRows int + validateOutput func(t *testing.T, filePath string) + }{ + { + name: "basic CSV writing", + args: map[string]any{ + "data": []any{ + map[string]any{"name": "Alice", "age": 30, "city": "New York"}, + map[string]any{"name": "Bob", "age": 25, "city": "San Francisco"}, + }, + "filename": "basic.csv", + }, + expectedError: false, + expectedRows: 2, + validateOutput: func(t *testing.T, filePath string) { + fullPath := filepath.Join(tempDir, "basic.csv") + content, err := os.ReadFile(fullPath) + if err != nil { + t.Fatalf("Failed to read output file: %v", err) + } + + lines := strings.Split(strings.TrimSpace(string(content)), "\n") + if len(lines) != 3 { + t.Errorf("Expected 3 lines, got %d", len(lines)) + } + + if !strings.Contains(lines[0], "name") { + t.Error("Expected headers to contain 'name'") + } + }, + }, + { + name: "CSV with custom headers", + args: map[string]any{ + "data": []any{ + map[string]any{"name": "Alice", "age": 30}, + map[string]any{"name": "Bob", "age": 25}, + }, + "filename": "custom_headers.csv", + "headers": []any{"name", "age"}, + }, + expectedError: false, + expectedRows: 2, + validateOutput: func(t *testing.T, filePath string) { + fullPath := filepath.Join(tempDir, "custom_headers.csv") + file, err := os.Open(fullPath) + if err != nil { + t.Fatalf("Failed to open output file: %v", err) + } + defer func() { + if closeErr := file.Close(); closeErr != nil { + t.Logf("Failed to close file: %v", closeErr) + } + }() + + reader := csv.NewReader(file) + records, err := reader.ReadAll() + if err != nil { + t.Fatalf("Failed to read CSV: %v", err) + } + + if len(records) != 3 { + t.Errorf("Expected 3 records, got %d", len(records)) + } + + if records[0][0] != "name" || records[0][1] != "age" { + t.Errorf("Headers not in expected order: %v", records[0]) + } + }, + }, + { + name: "CSV without headers", + args: map[string]any{ + "data": []any{ + map[string]any{"name": "Alice", "age": 30}, + map[string]any{"name": "Bob", "age": 25}, + }, + "filename": "no_headers.csv", + "include_headers": false, + }, + expectedError: false, + expectedRows: 2, + validateOutput: func(t *testing.T, filePath string) { + fullPath := filepath.Join(tempDir, "no_headers.csv") + content, err := os.ReadFile(fullPath) + if err != nil { + t.Fatalf("Failed to read output file: %v", err) + } + + lines := strings.Split(strings.TrimSpace(string(content)), "\n") + if len(lines) != 2 { + t.Errorf("Expected 2 lines, got %d", len(lines)) + } + }, + }, + { + name: "append to existing file", + args: map[string]any{ + "data": []any{ + map[string]any{"name": "Charlie", "age": 35}, + }, + "filename": "basic.csv", + "append": true, + }, + expectedError: false, + expectedRows: 1, + validateOutput: func(t *testing.T, filePath string) { + fullPath := filepath.Join(tempDir, "basic.csv") + content, err := os.ReadFile(fullPath) + if err != nil { + t.Fatalf("Failed to read output file: %v", err) + } + + lines := strings.Split(strings.TrimSpace(string(content)), "\n") + if len(lines) != 4 { + t.Errorf("Expected 4 lines after append, got %d", len(lines)) + } + + if !strings.Contains(string(content), "Charlie") { + t.Error("Expected appended data to contain 'Charlie'") + } + }, + }, + { + name: "invalid data type", + args: map[string]any{ + "data": "not an array", + "filename": "invalid.csv", + }, + expectedError: true, + }, + { + name: "empty file path", + args: map[string]any{ + "data": []any{map[string]any{"name": "Alice"}}, + "filename": "", + }, + expectedError: true, + }, + { + name: "empty data array", + args: map[string]any{ + "data": []any{}, + "filename": "empty.csv", + }, + expectedError: true, + }, + } + + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + result, err := skill.WriteToCsvHandler(context.Background(), tt.args) + + if tt.expectedError { + if err == nil { + t.Error("Expected an error but got none") + } + return + } + + if err != nil { + t.Errorf("Unexpected error: %v", err) + return + } + + if !strings.Contains(result, "Successfully wrote") { + t.Errorf("Expected success message, got: %s", result) + } + + if tt.validateOutput != nil { + filename := tt.args["filename"].(string) + tt.validateOutput(t, filename) + } + }) + } +} + +func TestConvertDataToRows(t *testing.T) { + logger := zap.NewNop() + skill := &WriteToCsvSkill{logger: logger} + + tests := []struct { + name string + input []any + expectedError bool + expectedLen int + }{ + { + name: "valid map[string]any data", + input: []any{ + map[string]any{"name": "Alice", "age": 30}, + map[string]any{"name": "Bob", "age": 25}, + }, + expectedError: false, + expectedLen: 2, + }, + { + name: "mixed map types", + input: []any{ + map[string]any{"name": "Alice"}, + map[any]any{"name": "Bob", "age": 25}, + }, + expectedError: false, + expectedLen: 2, + }, + { + name: "invalid data type", + input: []any{ + "not a map", + map[string]any{"name": "Alice"}, + }, + expectedError: true, + }, + } + + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + result, err := skill.convertDataToRows(tt.input) + + if tt.expectedError { + if err == nil { + t.Error("Expected an error but got none") + } + return + } + + if err != nil { + t.Errorf("Unexpected error: %v", err) + return + } + + if len(result) != tt.expectedLen { + t.Errorf("Expected %d rows, got %d", tt.expectedLen, len(result)) + } + }) + } +} + +func TestValueToString(t *testing.T) { + logger := zap.NewNop() + skill := &WriteToCsvSkill{logger: logger} + + tests := []struct { + name string + input any + expected string + }{ + {"string", "hello", "hello"}, + {"int", 42, "42"}, + {"float", 3.14, "3.14"}, + {"bool true", true, "true"}, + {"bool false", false, "false"}, + {"nil", nil, ""}, + {"array", []any{"a", "b", "c"}, "[%!v([]string=[a b c])]"}, + } + + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + result := skill.valueToString(tt.input) + if tt.name != "array" && result != tt.expected { + t.Errorf("Expected %q, got %q", tt.expected, result) + } + + if tt.name == "array" && result == "" { + t.Error("Expected non-empty string for array") + } + }) + } +}