diff --git a/scripts/.env.example b/scripts/.env.example new file mode 100644 index 0000000..721da8b --- /dev/null +++ b/scripts/.env.example @@ -0,0 +1,11 @@ +# Benchmark Configuration +BASE_URL=http://localhost:3005 +TYPE=eoa +FROM=0x... +CHAIN_ID=1337 +SECRET_KEY=your-secret-key +VAULT_ACCESS_TOKEN=your-vault-access-token + +# Optional: Benchmark Parameters +CONCURRENT_REQUESTS=10 +TOTAL_REQUESTS=100 diff --git a/scripts/.gitignore b/scripts/.gitignore index a14702c..acd7425 100644 --- a/scripts/.gitignore +++ b/scripts/.gitignore @@ -32,3 +32,4 @@ report.[0-9]_.[0-9]_.[0-9]_.[0-9]_.json # Finder (MacOS) folder config .DS_Store +benchmarks/runs diff --git a/scripts/benchmarks/README.md b/scripts/benchmarks/README.md new file mode 100644 index 0000000..6d98420 --- /dev/null +++ b/scripts/benchmarks/README.md @@ -0,0 +1,154 @@ +# EOA Executor Benchmark + +A comprehensive benchmark tool for stress testing the `/v1/write/transaction` endpoint with detailed performance metrics and webhook event tracking. + +## Features + +- āœ… Concurrent request execution with configurable concurrency +- šŸ“Š Detailed metrics tracking (HTTP response time, submission time, confirmation time) +- šŸŽ£ Built-in webhook server to capture transaction lifecycle events +- šŸ“ˆ Statistical analysis (p50, p90, p95, p99, min, max, mean) +- šŸ“„ CSV export of individual transaction metrics +- šŸ“Š JSON export of aggregate results +- šŸŽÆ Real-time progress and status updates + +## Setup + +1. Copy the `.env.example` to `.env` in the `/scripts` directory: + ```bash + cp .env.example .env + ``` + +2. Configure your environment variables in `.env`: + ``` + BASE_URL=http://localhost:3005 + TYPE=eoa + FROM=0x1234567890123456789012345678901234567890 + CHAIN_ID=1337 + SECRET_KEY=your-secret-key + VAULT_ACCESS_TOKEN=your-vault-access-token + CONCURRENT_REQUESTS=10 + TOTAL_REQUESTS=100 + ``` + +## Usage + +Run the benchmark from the `/scripts` directory: + +```bash +cd scripts +bun ./benchmarks/eoa.ts +``` + +## Output + +The benchmark creates a timestamped directory `run-/` containing: + +- `transactions-.csv` - Individual transaction metrics +- `result-.json` - Aggregate statistics + +### CSV Format + +Each row contains: +- `transaction_id` - Unique transaction identifier +- `http_response_time_ms` - HTTP request/response time +- `sent_to_submitted_ms` - Time from send to submitted webhook +- `submitted_to_confirmed_ms` - Time from submitted to confirmed webhook +- `total_time_ms` - Total time from send to confirmed +- `status` - Final status (confirmed/failed/pending) +- `error` - Error message if failed + +### JSON Format + +Aggregate results include: +- Total/successful/failed request counts +- Error rate percentage +- Duration and throughput (req/s) +- Statistical breakdown for all timing metrics (min, max, mean, p50, p90, p95, p99) + +## Configuration Options + +| Variable | Default | Description | +|----------|---------|-------------| +| `BASE_URL` | `http://localhost:3005` | API endpoint base URL | +| `TYPE` | `eoa` | Execution type | +| `FROM` | *required* | Sender address | +| `CHAIN_ID` | `1337` | Blockchain network ID | +| `SECRET_KEY` | *required* | API secret key | +| `VAULT_ACCESS_TOKEN` | *required* | Vault access token | +| `CONCURRENT_REQUESTS` | `10` | Number of concurrent requests | +| `TOTAL_REQUESTS` | `100` | Total number of requests to send | + +## How It Works + +1. **Webhook Server**: Starts a local server on port 3070 to receive transaction lifecycle webhooks +2. **Request Sending**: Sends HTTP requests to `/v1/write/transaction` with controlled concurrency +3. **Event Tracking**: Tracks webhook events for each transaction: + - `send` stage with `Success` event = transaction submitted + - `confirm` stage with `Success` event = transaction confirmed + - `Failure` or `Nack` events = transaction failed +4. **Metrics Calculation**: Computes timing metrics and statistics +5. **Output Generation**: Writes CSV and JSON files with results + +## Webhook Event Structure + +Based on `executors/src/eoa/events.rs`, the webhook payload contains: +```json +{ + "transaction_id": "...", + "executor_name": "eoa", + "stage_name": "send" | "confirm", + "event_type": "Success" | "Nack" | "Failure", + "payload": { ... } +} +``` + +## Example Output + +``` +šŸš€ Starting benchmark... +šŸ“Š Configuration: + Base URL: http://localhost:3005 + Type: eoa + From: 0x1234... + Chain ID: 1337 + Total Requests: 100 + Concurrent Requests: 10 + +šŸ“¤ Sent 10/100 requests... +... +āœ… All HTTP requests completed +ā³ Waiting for webhooks to complete... +šŸŽ‰ All transactions completed! + +šŸ“Š BENCHMARK RESULTS +============================================================ +šŸ“ˆ Overview: + Total Requests: 100 + Successful: 98 + Failed: 2 + Error Rate: 2.00% + Duration: 45.23s + Throughput: 2.21 req/s + +ā±ļø HTTP Response Times (ms): + Min: 45.23 + Mean: 123.45 + P50: 110.00 + P90: 180.00 + P95: 210.00 + P99: 250.00 + Max: 320.00 +... +``` + +## Tips + +- Start with a small `TOTAL_REQUESTS` value to test your setup +- Adjust `CONCURRENT_REQUESTS` based on your server capacity +- Monitor your server logs alongside the benchmark +- The script waits up to 2 minutes for pending webhooks before timing out +- Use the CSV output for detailed per-transaction analysis +- Use the JSON output for automated performance regression testing + + diff --git a/scripts/benchmarks/eoa.ts b/scripts/benchmarks/eoa.ts new file mode 100644 index 0000000..888fd04 --- /dev/null +++ b/scripts/benchmarks/eoa.ts @@ -0,0 +1,516 @@ +/// + +// Bun globals (runtime will provide these) +declare const Bun: any; +declare const process: any; + +// Extend ImportMeta for Bun +interface ImportMeta { + dir: string; +} + +// Types based on events.rs +interface WebhookEvent { + transactionId: string; + executorName: string; + stageName: string; // "send" | "confirm" + eventType: string; // "Success" | "Nack" | "Failure" + payload: any; +} + +interface TransactionMetrics { + transactionId: string; + httpResponseTime: number; + sentTime: number; + submittedTime?: number; + confirmedTime?: number; + sentToSubmittedMs?: number; + submittedToConfirmedMs?: number; + totalTimeMs?: number; + status: "pending" | "submitted" | "confirmed" | "failed"; + error?: string; +} + +interface BenchmarkConfig { + baseUrl: string; + type: string; + from: string; + chainId: number; + secretKey: string; + vaultAccessToken: string; + concurrentRequests: number; + totalRequests: number; +} + +interface AggregateResults { + totalRequests: number; + successfulRequests: number; + failedRequests: number; + errorRate: number; + httpResponseTimes: { + min: number; + max: number; + mean: number; + p50: number; + p90: number; + p95: number; + p99: number; + }; + sentToSubmittedTimes: { + min: number; + max: number; + mean: number; + p50: number; + p90: number; + p95: number; + p99: number; + }; + submittedToConfirmedTimes: { + min: number; + max: number; + mean: number; + p50: number; + p90: number; + p95: number; + p99: number; + }; + totalTimes: { + min: number; + max: number; + mean: number; + p50: number; + p90: number; + p95: number; + p99: number; + }; + duration: number; + throughput: number; +} + +// Global state to track transactions +const transactions = new Map(); +const pendingTransactions = new Set(); + +// Configuration +const config: BenchmarkConfig = { + baseUrl: process.env.BASE_URL || "http://localhost:3005", + type: process.env.TYPE || "eoa", + from: process.env.FROM!, + chainId: parseInt(process.env.CHAIN_ID || "1337"), + secretKey: process.env.SECRET_KEY!, + vaultAccessToken: process.env.VAULT_ACCESS_TOKEN!, + concurrentRequests: parseInt(process.env.CONCURRENT_REQUESTS || "10"), + totalRequests: parseInt(process.env.TOTAL_REQUESTS || "100"), +}; + +// Validate required env vars +if (!config.from || !config.secretKey || !config.vaultAccessToken) { + console.error("āŒ Missing required environment variables: FROM, SECRET_KEY, VAULT_ACCESS_TOKEN"); + process.exit(1); +} + +// Setup webhook server +const webhookServer = Bun.serve({ + port: 3070, + async fetch(req: Request) { + if (req.method === "POST" && new URL(req.url).pathname === "/callback") { + try { + const event: WebhookEvent = await req.json(); + handleWebhookEvent(event); + return new Response("OK", { status: 200 }); + } catch (error) { + console.error("Error handling webhook:", error); + return new Response("Error", { status: 500 }); + } + } + return new Response("Not Found", { status: 404 }); + }, +}); + +console.log(`šŸŽ£ Webhook server listening on http://localhost:${webhookServer.port}`); + +// Handle webhook events +function handleWebhookEvent(event: WebhookEvent) { + const txId = event.transactionId; + const metrics = transactions.get(txId); + + if (!metrics) { + console.warn(`āš ļø Received webhook for unknown transaction: ${txId}`); + return; + } + + const now = Date.now(); + + // Handle different stage events + if (event.stageName === "send") { + if (event.eventType === "SUCCESS") { + metrics.submittedTime = now; + metrics.sentToSubmittedMs = now - metrics.sentTime; + metrics.status = "submitted"; + console.log(`āœ… Transaction ${txId.slice(0, 8)}... submitted (${metrics.sentToSubmittedMs}ms)`); + } else if (event.eventType === "Failure") { + metrics.status = "failed"; + metrics.error = JSON.stringify(event.payload); + pendingTransactions.delete(txId); + console.log(`āŒ Transaction ${txId.slice(0, 8)}... failed at send stage (pending: ${pendingTransactions.size})`); + } + } else if (event.stageName === "confirm") { + if (event.eventType === "SUCCESS") { + metrics.confirmedTime = now; + if (metrics.submittedTime) { + metrics.submittedToConfirmedMs = now - metrics.submittedTime; + } + metrics.totalTimeMs = now - metrics.sentTime; + metrics.status = "confirmed"; + pendingTransactions.delete(txId); + console.log(`šŸŽ‰ Transaction ${txId.slice(0, 8)}... confirmed (total: ${metrics.totalTimeMs}ms, pending: ${pendingTransactions.size})`); + } else if (event.eventType === "FAIL" || event.eventType === "NACK") { + metrics.status = "failed"; + metrics.error = JSON.stringify(event.payload); + pendingTransactions.delete(txId); + console.log(`āŒ Transaction ${txId.slice(0, 8)}... failed at confirmation stage (pending: ${pendingTransactions.size})`); + } + } +} + +// Send a single transaction +async function sendTransaction(): Promise { + const startTime = performance.now(); + + try { + const response = await fetch(`${config.baseUrl}/v1/write/transaction`, { + method: "POST", + headers: { + "Content-Type": "application/json", + "x-thirdweb-secret-key": config.secretKey, + "x-vault-access-token": config.vaultAccessToken, + }, + body: JSON.stringify({ + executionOptions: { + type: config.type, + from: config.from, + chainId: config.chainId, + }, + params: [ + { + to: "0x2247d5d238d0f9d37184d8332aE0289d1aD9991b", + data: "0x", + value: "0", + }, + ], + webhookOptions: [ + { + url: "http://localhost:3070/callback", + }, + ], + }), + }); + + const endTime = performance.now(); + const httpResponseTime = endTime - startTime; + + if (!response.ok) { + const errorText = await response.text(); + throw new Error(`HTTP ${response.status}: ${errorText}`); + } + + const res = await response.json() as { + result: { + transactions: { + id: string; + }[] + } + }; + const transactionId = res.result.transactions[0]?.id; + + if (!transactionId) { + console.error("āŒ No transaction ID in response:", JSON.stringify(response, null, 2)); + throw new Error("No transaction ID in response"); + } + + // Set sentTime AFTER we get the HTTP response, so we measure from when + // the server accepted the request to when webhooks fire + const sentTime = Date.now(); + + const metrics: TransactionMetrics = { + transactionId, + httpResponseTime, + sentTime, + status: "pending", + }; + + transactions.set(transactionId, metrics); + pendingTransactions.add(transactionId); + + console.log(`šŸ“ Added transaction ${transactionId.slice(0, 8)}... to pending (total: ${pendingTransactions.size})`); + + return metrics; + } catch (error) { + const httpResponseTime = performance.now() - startTime; + const sentTime = Date.now(); + const errorMetrics: TransactionMetrics = { + transactionId: `error-${Date.now()}`, + httpResponseTime, + sentTime, + status: "failed", + error: error instanceof Error ? error.message : String(error), + }; + transactions.set(errorMetrics.transactionId, errorMetrics); + console.error(`āŒ Transaction request failed: ${error instanceof Error ? error.message : String(error)}`); + return errorMetrics; + } +} + +// Run benchmark with concurrency control +async function runBenchmark() { + console.log("\nšŸš€ Starting benchmark..."); + console.log(`šŸ“Š Configuration:`); + console.log(` Base URL: ${config.baseUrl}`); + console.log(` Type: ${config.type}`); + console.log(` From: ${config.from}`); + console.log(` Chain ID: ${config.chainId}`); + console.log(` Total Requests: ${config.totalRequests}`); + console.log(` Concurrent Requests: ${config.concurrentRequests}`); + console.log(); + + const startTime = Date.now(); + const allPromises: Promise[] = []; + const inFlight = new Set>(); + + // Send requests with concurrency control using sliding window + for (let i = 0; i < config.totalRequests; i++) { + const promise = sendTransaction(); + allPromises.push(promise); + inFlight.add(promise); + + // Remove from in-flight set when completed + promise.finally(() => inFlight.delete(promise)); + + // Control concurrency - wait if we've reached the limit + if (inFlight.size >= config.concurrentRequests) { + await Promise.race(inFlight); + } + + // Progress indicator + if ((i + 1) % 10 === 0) { + console.log(`šŸ“¤ Sent ${i + 1}/${config.totalRequests} requests... (in-flight: ${inFlight.size})`); + } + } + + // Wait for all HTTP requests to complete + await Promise.all(allPromises); + console.log(`\nāœ… All HTTP requests completed`); + console.log(` Pending transactions: ${pendingTransactions.size}`); + + // Wait for all webhooks to be received (with timeout) + console.log(`ā³ Waiting for webhooks to complete...`); + const maxWaitTime = 600000; // 5 minutes + const pollInterval = 1000; // 1 second + let waited = 0; + + while (pendingTransactions.size > 0 && waited < maxWaitTime) { + await new Promise((resolve) => setTimeout(resolve, pollInterval)); + waited += pollInterval; + + if (waited % 5000 === 0) { + console.log(` Still waiting for ${pendingTransactions.size} transactions...`); + } + } + + const endTime = Date.now(); + const duration = endTime - startTime; + + if (pendingTransactions.size > 0) { + console.warn(`\nāš ļø Timeout: ${pendingTransactions.size} transactions still pending`); + } else { + console.log(`\nšŸŽ‰ All transactions completed!`); + } + + return { startTime, endTime, duration }; +} + +// Calculate statistics +function calculateStats(values: number[]): { + min: number; + max: number; + mean: number; + p50: number; + p90: number; + p95: number; + p99: number; +} { + if (values.length === 0) { + return { min: 0, max: 0, mean: 0, p50: 0, p90: 0, p95: 0, p99: 0 }; + } + + const sorted = [...values].sort((a, b) => a - b); + const sum = sorted.reduce((acc, val) => acc + val, 0); + + return { + min: sorted[0] ?? 0, + max: sorted[sorted.length - 1] ?? 0, + mean: sum / sorted.length, + p50: sorted[Math.floor(sorted.length * 0.5)] ?? 0, + p90: sorted[Math.floor(sorted.length * 0.9)] ?? 0, + p95: sorted[Math.floor(sorted.length * 0.95)] ?? 0, + p99: sorted[Math.floor(sorted.length * 0.99)] ?? 0, + }; +} + +// Generate aggregate results +function generateAggregateResults(duration: number): AggregateResults { + const allMetrics = Array.from(transactions.values()); + const successfulMetrics = allMetrics.filter((m) => m.status === "confirmed"); + const failedMetrics = allMetrics.filter((m) => m.status === "failed"); + + const httpResponseTimes = allMetrics.map((m) => m.httpResponseTime); + const sentToSubmittedTimes = successfulMetrics + .filter((m) => m.sentToSubmittedMs !== undefined) + .map((m) => m.sentToSubmittedMs!); + const submittedToConfirmedTimes = successfulMetrics + .filter((m) => m.submittedToConfirmedMs !== undefined) + .map((m) => m.submittedToConfirmedMs!); + const totalTimes = successfulMetrics + .filter((m) => m.totalTimeMs !== undefined) + .map((m) => m.totalTimeMs!); + + return { + totalRequests: allMetrics.length, + successfulRequests: successfulMetrics.length, + failedRequests: failedMetrics.length, + errorRate: (failedMetrics.length / allMetrics.length) * 100, + httpResponseTimes: calculateStats(httpResponseTimes), + sentToSubmittedTimes: calculateStats(sentToSubmittedTimes), + submittedToConfirmedTimes: calculateStats(submittedToConfirmedTimes), + totalTimes: calculateStats(totalTimes), + duration, + throughput: (allMetrics.length / duration) * 1000, // requests per second + }; +} + +// Write CSV file +async function writeCSV(outputDir: string, timestamp: string) { + const csvPath = `${outputDir}/transactions-${timestamp}.csv`; + const headers = [ + "transaction_id", + "http_response_time_ms", + "sent_to_submitted_ms", + "submitted_to_confirmed_ms", + "total_time_ms", + "status", + "error", + ]; + + const rows = Array.from(transactions.values()).map((m) => [ + m.transactionId, + m.httpResponseTime.toFixed(2), + m.sentToSubmittedMs?.toFixed(2) || "", + m.submittedToConfirmedMs?.toFixed(2) || "", + m.totalTimeMs?.toFixed(2) || "", + m.status, + m.error || "", + ]); + + const csvContent = [headers.join(","), ...rows.map((row) => row.join(","))].join("\n"); + + await Bun.write(csvPath, csvContent); + console.log(`šŸ“„ CSV written to: ${csvPath}`); +} + +// Write JSON results +async function writeJSON(outputDir: string, timestamp: string, results: AggregateResults) { + const jsonPath = `${outputDir}/result-${timestamp}.json`; + await Bun.write(jsonPath, JSON.stringify(results, null, 2)); + console.log(`šŸ“Š Results written to: ${jsonPath}`); +} + +// Print results to console +function printResults(results: AggregateResults) { + console.log("\n" + "=".repeat(60)); + console.log("šŸ“Š BENCHMARK RESULTS"); + console.log("=".repeat(60)); + + console.log("\nšŸ“ˆ Overview:"); + console.log(` Total Requests: ${results.totalRequests}`); + console.log(` Successful: ${results.successfulRequests}`); + console.log(` Failed: ${results.failedRequests}`); + console.log(` Error Rate: ${results.errorRate.toFixed(2)}%`); + console.log(` Duration: ${(results.duration / 1000).toFixed(2)}s`); + console.log(` Throughput: ${results.throughput.toFixed(2)} req/s`); + + console.log("\nā±ļø HTTP Response Times (ms):"); + console.log(` Min: ${results.httpResponseTimes.min.toFixed(2)}`); + console.log(` Mean: ${results.httpResponseTimes.mean.toFixed(2)}`); + console.log(` P50: ${results.httpResponseTimes.p50.toFixed(2)}`); + console.log(` P90: ${results.httpResponseTimes.p90.toFixed(2)}`); + console.log(` P95: ${results.httpResponseTimes.p95.toFixed(2)}`); + console.log(` P99: ${results.httpResponseTimes.p99.toFixed(2)}`); + console.log(` Max: ${results.httpResponseTimes.max.toFixed(2)}`); + + console.log("\nšŸ“¤ Sent to Submitted Times (ms):"); + console.log(` Min: ${results.sentToSubmittedTimes.min.toFixed(2)}`); + console.log(` Mean: ${results.sentToSubmittedTimes.mean.toFixed(2)}`); + console.log(` P50: ${results.sentToSubmittedTimes.p50.toFixed(2)}`); + console.log(` P90: ${results.sentToSubmittedTimes.p90.toFixed(2)}`); + console.log(` P95: ${results.sentToSubmittedTimes.p95.toFixed(2)}`); + console.log(` P99: ${results.sentToSubmittedTimes.p99.toFixed(2)}`); + console.log(` Max: ${results.sentToSubmittedTimes.max.toFixed(2)}`); + + console.log("\nāœ… Submitted to Confirmed Times (ms):"); + console.log(` Min: ${results.submittedToConfirmedTimes.min.toFixed(2)}`); + console.log(` Mean: ${results.submittedToConfirmedTimes.mean.toFixed(2)}`); + console.log(` P50: ${results.submittedToConfirmedTimes.p50.toFixed(2)}`); + console.log(` P90: ${results.submittedToConfirmedTimes.p90.toFixed(2)}`); + console.log(` P95: ${results.submittedToConfirmedTimes.p95.toFixed(2)}`); + console.log(` P99: ${results.submittedToConfirmedTimes.p99.toFixed(2)}`); + console.log(` Max: ${results.submittedToConfirmedTimes.max.toFixed(2)}`); + + console.log("\nšŸŽÆ Total Times (Sent to Confirmed) (ms):"); + console.log(` Min: ${results.totalTimes.min.toFixed(2)}`); + console.log(` Mean: ${results.totalTimes.mean.toFixed(2)}`); + console.log(` P50: ${results.totalTimes.p50.toFixed(2)}`); + console.log(` P90: ${results.totalTimes.p90.toFixed(2)}`); + console.log(` P95: ${results.totalTimes.p95.toFixed(2)}`); + console.log(` P99: ${results.totalTimes.p99.toFixed(2)}`); + console.log(` Max: ${results.totalTimes.max.toFixed(2)}`); + + console.log("\n" + "=".repeat(60)); +} + +// Main execution +async function main() { + try { + // Run benchmark + const { duration } = await runBenchmark(); + + // Generate results + const results = generateAggregateResults(duration); + + // Create output directory + const timestamp = new Date().toISOString().replace(/[:.]/g, "-"); + const currentDir = (import.meta as any).dir; + const outputDir = `${currentDir}/runs/run-${timestamp}`; + await Bun.$`mkdir -p ${outputDir}`.quiet(); + + // Write outputs + await writeCSV(outputDir, timestamp); + await writeJSON(outputDir, timestamp, results); + + // Print results + printResults(results); + + console.log(`\nāœØ Benchmark complete! Results saved to: ${outputDir}\n`); + } catch (error) { + console.error("āŒ Benchmark failed:", error); + webhookServer.stop(); + process.exit(1); + } + + // Stop webhook server and exit + webhookServer.stop(); + process.exit(0); +} + +// Run the benchmark +main(); + diff --git a/scripts/package.json b/scripts/package.json index cdaac8c..81c3195 100644 --- a/scripts/package.json +++ b/scripts/package.json @@ -2,7 +2,8 @@ "name": "engine-core-scripts", "type": "module", "scripts": { - "cleanup": "bun ./redis-cleanup/index.tsx" + "cleanup": "bun ./redis-cleanup/index.tsx", + "benchmark:eoa": "bun ./benchmarks/eoa.ts" }, "dependencies": { "ioredis": "^5.3.2", diff --git a/scripts/tsconfig.json b/scripts/tsconfig.json index bfa0fea..be3d138 100644 --- a/scripts/tsconfig.json +++ b/scripts/tsconfig.json @@ -1,7 +1,7 @@ { "compilerOptions": { // Environment setup & latest features - "lib": ["ESNext"], + "lib": ["ESNext", "DOM"], "target": "ESNext", "module": "Preserve", "moduleDetection": "force",