From 7db6c185ec8b7c4b5192476ddc6ac9ff1d4abfc6 Mon Sep 17 00:00:00 2001 From: fg-nava <189638926+fg-nava@users.noreply.github.com> Date: Thu, 14 Aug 2025 16:21:30 -0700 Subject: [PATCH 1/4] feat: playwright artifacts prisma model, anthropicVertex provider --- mastra-test-app/.gitignore | 3 + mastra-test-app/package.json | 1 + mastra-test-app/pnpm-lock.yaml | 103 +++++++++ .../migration.sql | 29 +++ mastra-test-app/prisma/schema.prisma | 21 ++ .../src/mastra/agents/data-ops-agent.ts | 11 +- .../src/mastra/agents/web-automation-agent.ts | 15 +- .../src/mastra/artifact-watcher.ts | 207 ++++++++++++++++++ mastra-test-app/src/mastra/mcp.ts | 17 +- .../src/mastra/storage-artifacts.ts | 103 +++++++++ .../src/mastra/tools/artifact-tools.ts | 187 ++++++++++++++++ .../src/mastra/types/artifact-types.ts | 159 ++++++++++++++ 12 files changed, 851 insertions(+), 5 deletions(-) create mode 100644 mastra-test-app/prisma/migrations/20250814205016_add_playwright_artifacts/migration.sql create mode 100644 mastra-test-app/src/mastra/artifact-watcher.ts create mode 100644 mastra-test-app/src/mastra/storage-artifacts.ts create mode 100644 mastra-test-app/src/mastra/tools/artifact-tools.ts create mode 100644 mastra-test-app/src/mastra/types/artifact-types.ts diff --git a/mastra-test-app/.gitignore b/mastra-test-app/.gitignore index 6fd3be5..b7efda4 100644 --- a/mastra-test-app/.gitignore +++ b/mastra-test-app/.gitignore @@ -27,3 +27,6 @@ mastra.db mastra.db-* mastra-memory.db mastra-memory.db-* + +# Google Vertex AI +vertex-ai-credentials.json \ No newline at end of file diff --git a/mastra-test-app/package.json b/mastra-test-app/package.json index 348f4f5..203fe5e 100644 --- a/mastra-test-app/package.json +++ b/mastra-test-app/package.json @@ -37,6 +37,7 @@ "@mastra/pg": "^0.13.3", "@prisma/client": "^6.12.0", "@types/jsonwebtoken": "^9.0.10", + "anthropic-vertex-ai": "1.0.2", "csv-parse": "^6.1.0", "dotenv": "^17.2.1", "exa-mcp-server": "^2.0.3", diff --git a/mastra-test-app/pnpm-lock.yaml b/mastra-test-app/pnpm-lock.yaml index 4e57773..8c29e09 100644 --- a/mastra-test-app/pnpm-lock.yaml +++ b/mastra-test-app/pnpm-lock.yaml @@ -44,6 +44,9 @@ importers: '@types/jsonwebtoken': specifier: ^9.0.10 version: 9.0.10 + anthropic-vertex-ai: + specifier: 1.0.2 + version: 1.0.2(zod@3.25.76) csv-parse: specifier: ^6.1.0 version: 6.1.0 @@ -109,12 +112,25 @@ packages: peerDependencies: zod: ^3.0.0 + '@ai-sdk/provider-utils@1.0.20': + resolution: {integrity: sha512-ngg/RGpnA00eNOWEtXHenpX1MsM2QshQh4QJFjUfwcqHpM5kTfG7je7Rc3HcEDP+OkRVv2GF+X4fC1Vfcnl8Ow==} + engines: {node: '>=18'} + peerDependencies: + zod: ^3.0.0 + peerDependenciesMeta: + zod: + optional: true + '@ai-sdk/provider-utils@2.2.8': resolution: {integrity: sha512-fqhG+4sCVv8x7nFzYnFo19ryhAa3w096Kmc3hWxMQfW/TubPOmt3A6tYZhl4mUfQWWQMsuSkLrtjlWuXBVSGQA==} engines: {node: '>=18'} peerDependencies: zod: ^3.23.8 + '@ai-sdk/provider@0.0.24': + resolution: {integrity: sha512-XMsNGJdGO+L0cxhhegtqZ8+T6nn4EoShS819OvCgI2kLbYTIvk0GWFGD0AXJmxkxs3DrpsJxKAFukFR7bvTkgQ==} + engines: {node: '>=18'} + '@ai-sdk/provider@1.1.3': resolution: {integrity: sha512-qZMxYJ0qqX/RfnuIaab+zp8UAeJn/ygXXAffR5I4N0n1IrvA6qBsjc8hXLmBiMV2zoXlifkacF7sEFnYnjBcqg==} engines: {node: '>=18'} @@ -1577,6 +1593,12 @@ packages: resolution: {integrity: sha512-zbB9rCJAT1rbjiVDb2hqKFHNYLxgtk8NURxZ3IZwD3F6NtxbXZQCnnSi1Lkx+IDohdPlFp222wVALIheZJQSEg==} engines: {node: '>=8'} + anthropic-vertex-ai@1.0.2: + resolution: {integrity: sha512-4YuK04KMmBGkx6fi2UjnHkE4mhaIov7tnT5La9+DMn/gw/NSOLZoWNUx+13VY3mkcaseKBMEn1DBzdXXJFIP7A==} + engines: {node: '>=18'} + peerDependencies: + zod: ^3.0.0 + argparse@2.0.1: resolution: {integrity: sha512-8+9WqebbFzpX9OR+Wa6O29asIogeRMzcGtAINdpMHHyAg10f05aSFVBbcEqGf/PXw1EjAZ+q2/bEBg3DvurK3Q==} @@ -1976,6 +1998,10 @@ packages: resolution: {integrity: sha512-aIL5Fx7mawVa300al2BnEE4iNvo1qETxLrPI/o05L7z6go7fCw1J6EQmbK4FmJ2AS7kgVF/KEZWufBfdClMcPg==} engines: {node: '>= 0.6'} + eventsource-parser@1.1.2: + resolution: {integrity: sha512-v0eOBUbiaFojBu2s2NPBfYUoRR9GjcDNvCXVaqEf5vVfpIAh9f8RCo4vXTP8c63QRKCFwoLpMpTdPwwhEKVgzA==} + engines: {node: '>=14.18'} + eventsource-parser@3.0.3: resolution: {integrity: sha512-nVpZkTMM9rF6AQ9gPJpFsNAMt48wIzB5TQgiTLdHiuO8XEDhUgZEhqKlZWXbIzo9VmJ/HvysHqEaVeD5v9TPvA==} engines: {node: '>=20.0.0'} @@ -2174,6 +2200,10 @@ packages: resolution: {integrity: sha512-0Ia46fDOaT7k4og1PDW4YbodWWr3scS2vAr2lTbsplOt2WkKp0vQbkI9wKis/T5LV/dqPjO3bpS/z6GTJB82LA==} engines: {node: '>=18'} + google-auth-library@9.15.1: + resolution: {integrity: sha512-Jb6Z0+nvECVz+2lzSMt9u98UsoakXxA2HGHMCxh+so3n90XgYWkq5dur19JAJV7ONiJY22yBTyJB1TSkvPq9Ng==} + engines: {node: '>=14'} + google-logging-utils@0.0.2: resolution: {integrity: sha512-NEgUnEcBiP5HrPzufUkBzJOD/Sxsco3rLNo1F1TNf7ieU8ryUzBhqba8r756CjLX7rn3fHl6iLEwPYuqpoKgQQ==} engines: {node: '>=14'} @@ -2185,6 +2215,10 @@ packages: graceful-fs@4.2.11: resolution: {integrity: sha512-RbJ5/jmFcNNCcDV5o9eTnBLJ/HszWV0P73bc+Ff4nS/rJj+YaS6IGyiOL0VoBYX+l1Wrl3k63h/KrH+nhJ0XvQ==} + gtoken@7.1.0: + resolution: {integrity: sha512-pCcEwRi+TKpMlxAQObHDQ56KawURgyAf6jtIY046fJ5tIv3zDe/LEIubckAO8fj6JnAxLdmWkUfNyulQ2iKdEw==} + engines: {node: '>=14.0.0'} + has-symbols@1.1.0: resolution: {integrity: sha512-1cDNdwJ2Jaohmb3sg4OmKaMBwuC48sYni5HUw2DvsC8LjGTLK9h+eb1X6RyuOHe4hT0ULCW68iomhjUoKUqlPQ==} engines: {node: '>= 0.4'} @@ -2435,9 +2469,15 @@ packages: jwa@1.4.2: resolution: {integrity: sha512-eeH5JO+21J78qMvTIDdBXidBd6nG2kZjg5Ohz/1fpa28Z4CcsWUzJ1ZZyFq/3z3N17aZy+ZuBoHljASbL1WfOw==} + jwa@2.0.1: + resolution: {integrity: sha512-hRF04fqJIP8Abbkq5NKGN0Bbr3JxlQ+qhZufXVr0DvujKy93ZCbXZMHDL4EOtodSbCWxOqR8MS1tXA5hwqCXDg==} + jws@3.2.2: resolution: {integrity: sha512-YHlZCB6lMTllWDtSPHz/ZXTsi8S00usEV6v1tjq8tOUZzw7DpSDWVXjXDre6ed1w/pd495ODpHZYSdkRTsa0HA==} + jws@4.0.0: + resolution: {integrity: sha512-KDncfTmOZoOMTFG4mBlG0qUIOlc03fmzH+ru6RgYVZhPkyiy/92Owlt/8UEN+a4TXR1FQetfIpJE8ApdvdVxTg==} + libsql@0.5.17: resolution: {integrity: sha512-RRlj5XQI9+Wq+/5UY8EnugSWfRmHEw4hn3DKlPrkUgZONsge1PwTtHcpStP6MSNi8ohcbsRgEHJaymA33a8cBw==} cpu: [x64, arm64, wasm32, arm] @@ -2585,6 +2625,11 @@ packages: engines: {node: ^10 || ^12 || ^13.7 || ^14 || >=15.0.1} hasBin: true + nanoid@3.3.6: + resolution: {integrity: sha512-BGcqMMJuToF7i1rt+2PWSNVnWIkGCU78jBG3RxO/bZlnZPK2Cmi2QaffxGO/2RvWi9sL+FAiRiXMgsyxQ1DIDA==} + engines: {node: ^10 || ^12 || ^13.7 || ^14 || >=15.0.1} + hasBin: true + negotiator@0.6.3: resolution: {integrity: sha512-+EUsqGPLsM+j/zdChZjsnX51g4XrHFOIXwfnCVPGlQk/k5giakcKsuxCObBRu6DSm9opw/O6slWbJdghQM4bBg==} engines: {node: '>= 0.6'} @@ -3395,6 +3440,15 @@ snapshots: '@ai-sdk/provider-utils': 2.2.8(zod@3.25.76) zod: 3.25.76 + '@ai-sdk/provider-utils@1.0.20(zod@3.25.76)': + dependencies: + '@ai-sdk/provider': 0.0.24 + eventsource-parser: 1.1.2 + nanoid: 3.3.6 + secure-json-parse: 2.7.0 + optionalDependencies: + zod: 3.25.76 + '@ai-sdk/provider-utils@2.2.8(zod@3.25.76)': dependencies: '@ai-sdk/provider': 1.1.3 @@ -3402,6 +3456,10 @@ snapshots: secure-json-parse: 2.7.0 zod: 3.25.76 + '@ai-sdk/provider@0.0.24': + dependencies: + json-schema: 0.4.0 + '@ai-sdk/provider@1.1.3': dependencies: json-schema: 0.4.0 @@ -5132,6 +5190,16 @@ snapshots: dependencies: color-convert: 2.0.1 + anthropic-vertex-ai@1.0.2(zod@3.25.76): + dependencies: + '@ai-sdk/provider': 0.0.24 + '@ai-sdk/provider-utils': 1.0.20(zod@3.25.76) + google-auth-library: 9.15.1 + zod: 3.25.76 + transitivePeerDependencies: + - encoding + - supports-color + argparse@2.0.1: {} array-flatten@1.1.1: {} @@ -5492,6 +5560,8 @@ snapshots: etag@1.8.1: {} + eventsource-parser@1.1.2: {} + eventsource-parser@3.0.3: {} eventsource@3.0.7: @@ -5784,12 +5854,32 @@ snapshots: slash: 5.1.0 unicorn-magic: 0.3.0 + google-auth-library@9.15.1: + dependencies: + base64-js: 1.5.1 + ecdsa-sig-formatter: 1.0.11 + gaxios: 6.7.1 + gcp-metadata: 6.1.1 + gtoken: 7.1.0 + jws: 4.0.0 + transitivePeerDependencies: + - encoding + - supports-color + google-logging-utils@0.0.2: {} gopd@1.2.0: {} graceful-fs@4.2.11: {} + gtoken@7.1.0: + dependencies: + gaxios: 6.7.1 + jws: 4.0.0 + transitivePeerDependencies: + - encoding + - supports-color + has-symbols@1.1.0: {} has-tostringtag@1.0.2: @@ -5993,11 +6083,22 @@ snapshots: ecdsa-sig-formatter: 1.0.11 safe-buffer: 5.2.1 + jwa@2.0.1: + dependencies: + buffer-equal-constant-time: 1.0.1 + ecdsa-sig-formatter: 1.0.11 + safe-buffer: 5.2.1 + jws@3.2.2: dependencies: jwa: 1.4.2 safe-buffer: 5.2.1 + jws@4.0.0: + dependencies: + jwa: 2.0.1 + safe-buffer: 5.2.1 + libsql@0.5.17: dependencies: '@neon-rs/load': 0.0.4 @@ -6172,6 +6273,8 @@ snapshots: nanoid@3.3.11: {} + nanoid@3.3.6: {} + negotiator@0.6.3: {} negotiator@1.0.0: {} diff --git a/mastra-test-app/prisma/migrations/20250814205016_add_playwright_artifacts/migration.sql b/mastra-test-app/prisma/migrations/20250814205016_add_playwright_artifacts/migration.sql new file mode 100644 index 0000000..9fce8dc --- /dev/null +++ b/mastra-test-app/prisma/migrations/20250814205016_add_playwright_artifacts/migration.sql @@ -0,0 +1,29 @@ +-- CreateTable +CREATE TABLE "public"."mastra_artifacts" ( + "id" TEXT NOT NULL, + "sessionId" TEXT NOT NULL, + "fileName" TEXT NOT NULL, + "fileType" TEXT NOT NULL, + "mimeType" TEXT NOT NULL, + "size" INTEGER NOT NULL, + "content" BYTEA NOT NULL, + "metadata" JSONB NOT NULL DEFAULT '{}', + "traceId" TEXT, + "threadId" TEXT, + "createdAt" TIMESTAMP(3) NOT NULL DEFAULT CURRENT_TIMESTAMP, + "updatedAt" TIMESTAMP(3) NOT NULL, + + CONSTRAINT "mastra_artifacts_pkey" PRIMARY KEY ("id") +); + +-- CreateIndex +CREATE INDEX "mastra_artifacts_sessionId_idx" ON "public"."mastra_artifacts"("sessionId"); + +-- CreateIndex +CREATE INDEX "mastra_artifacts_fileType_idx" ON "public"."mastra_artifacts"("fileType"); + +-- CreateIndex +CREATE INDEX "mastra_artifacts_traceId_idx" ON "public"."mastra_artifacts"("traceId"); + +-- CreateIndex +CREATE INDEX "mastra_artifacts_createdAt_idx" ON "public"."mastra_artifacts"("createdAt"); diff --git a/mastra-test-app/prisma/schema.prisma b/mastra-test-app/prisma/schema.prisma index a6d94fc..2dd9609 100644 --- a/mastra-test-app/prisma/schema.prisma +++ b/mastra-test-app/prisma/schema.prisma @@ -66,3 +66,24 @@ model HouseholdDependent { @@map("household_dependents") } +model PlaywrightArtifact { + id String @id @default(cuid()) + sessionId String + fileName String + fileType String // 'screenshot' | 'trace' | 'session' | 'other' + mimeType String + size Int + content Bytes + metadata Json @default("{}") + traceId String? + threadId String? + createdAt DateTime @default(now()) + updatedAt DateTime @updatedAt + + @@index([sessionId]) + @@index([fileType]) + @@index([traceId]) + @@index([createdAt]) + @@map("mastra_artifacts") +} + diff --git a/mastra-test-app/src/mastra/agents/data-ops-agent.ts b/mastra-test-app/src/mastra/agents/data-ops-agent.ts index df12d61..85970eb 100644 --- a/mastra-test-app/src/mastra/agents/data-ops-agent.ts +++ b/mastra-test-app/src/mastra/agents/data-ops-agent.ts @@ -4,21 +4,30 @@ import { anthropic } from '@ai-sdk/anthropic'; import { google } from '@ai-sdk/google'; import { databaseTools } from '../tools/database-tools'; import { storageTools } from '../tools/storage-tools'; +import { artifactTools } from '../tools/artifact-tools'; export const dataOpsAgent = new Agent({ name: 'Data Ops Agent', - description: 'Agent specialized in database and Mastra storage queries (participants, threads, messages, traces).', + description: 'Agent specialized in database and Playwright artifacts and Mastra storage queries (participants, threads, messages, traces).', instructions: ` You are a concise data operations assistant. Use the provided tools to: - Query and manage participants/household records - Inspect Mastra threads, messages, and traces + - Store and retrieve Playwright artifacts (screenshots, traces, session data) - Return small, readable result sets + + The artifact tools allow you to: + - Store files from disk as artifacts in the database + - List and search stored artifacts + - Retrieve artifacts by ID or session + - Delete artifacts when no longer needed `, // model: google('gemini-2.5-pro'), model: anthropic('claude-sonnet-4-20250514'), tools: { ...Object.fromEntries(databaseTools.map(t => [t.id, t])), ...Object.fromEntries(storageTools.map(t => [t.id, t])), + ...Object.fromEntries(artifactTools.map(t => [t.id, t])), }, defaultStreamOptions: { diff --git a/mastra-test-app/src/mastra/agents/web-automation-agent.ts b/mastra-test-app/src/mastra/agents/web-automation-agent.ts index 6fc9651..ff2738a 100644 --- a/mastra-test-app/src/mastra/agents/web-automation-agent.ts +++ b/mastra-test-app/src/mastra/agents/web-automation-agent.ts @@ -3,10 +3,17 @@ import { exaMCP, playwrightMCP } from '../mcp'; import { Agent } from '@mastra/core/agent'; import { Memory } from '@mastra/memory'; -import { anthropic } from '@ai-sdk/anthropic'; import { databaseTools } from '../tools/database-tools'; +import { anthropic } from '@ai-sdk/anthropic'; import { google } from '@ai-sdk/google'; import { openai } from '@ai-sdk/openai'; +import { createAnthropicVertex } from 'anthropic-vertex-ai'; + +// Configure Anthropic Vertex AI +const anthropicVertex = createAnthropicVertex({ + region: process.env.GOOGLE_VERTEX_REGION, + projectId: process.env.GOOGLE_VERTEX_PROJECT_ID, +}); const storage = postgresStore; @@ -135,9 +142,11 @@ export const webAutomationAgent = new Agent({ Take action immediately. Don't ask for permission to proceed with your core function. `, // model: openai('gpt-5-2025-08-07'), - // // model: openai('gpt-4.1-mini'), + // model: openai('gpt-4.1-mini'), // model: anthropic('claude-sonnet-4-20250514'), - model: google('gemini-2.5-pro'), + model: anthropicVertex('claude-sonnet-4-20250514'), + // model: google('gemini-2.5-pro'), + tools: { ...Object.fromEntries(databaseTools.map(tool => [tool.id, tool])), ...(await playwrightMCP.getTools()), diff --git a/mastra-test-app/src/mastra/artifact-watcher.ts b/mastra-test-app/src/mastra/artifact-watcher.ts new file mode 100644 index 0000000..24e045e --- /dev/null +++ b/mastra-test-app/src/mastra/artifact-watcher.ts @@ -0,0 +1,207 @@ +import fs from 'fs'; +import path from 'path'; +import { artifactStorage } from './storage-artifacts'; + +export class ArtifactWatcher { + private watchers: fs.FSWatcher[] = []; + private sessionId: string; + private outputDir: string; + + constructor(outputDir: string, sessionId?: string) { + this.outputDir = outputDir; + this.sessionId = sessionId || `session_${Date.now()}`; + } + + async start(): Promise { + // Ensure output directory exists + await fs.promises.mkdir(this.outputDir, { recursive: true }); + + console.log(`Starting artifact watcher for session ${this.sessionId} in ${this.outputDir}`); + + // Watch for new files in the output directory + const watcher = fs.watch(this.outputDir, { recursive: true }, async (eventType, filename) => { + if (eventType === 'rename' && filename) { + const filePath = path.join(this.outputDir, filename); + + // Check if file exists (rename can be triggered by creation or deletion) + try { + const stats = await fs.promises.stat(filePath); + if (stats.isFile()) { + await this.handleNewFile(filePath); + } + } catch (error) { + // File might have been deleted or doesn't exist yet + console.debug(`File ${filename} not accessible:`, error); + } + } + }); + + this.watchers.push(watcher); + + // Also scan for existing files + await this.scanExistingFiles(); + } + + private async scanExistingFiles(): Promise { + try { + const files = await fs.promises.readdir(this.outputDir, { recursive: true }); + + for (const file of files) { + if (typeof file === 'string') { + const filePath = path.join(this.outputDir, file); + const stats = await fs.promises.stat(filePath); + + if (stats.isFile()) { + await this.handleNewFile(filePath); + } + } + } + } catch (error) { + console.error('Error scanning existing files:', error); + } + } + + private async handleNewFile(filePath: string): Promise { + try { + // Wait a bit to ensure file is fully written + await new Promise(resolve => setTimeout(resolve, 500)); + + const fileName = path.basename(filePath); + const content = await fs.promises.readFile(filePath); + + // Determine file type based on name and extension + const fileType = this.detectFileType(fileName); + const mimeType = this.getMimeType(fileName); + + // Extract trace and thread IDs from filename patterns + const { traceId, threadId } = this.extractIds(fileName); + + // Store in database - ensure content is properly handled as Buffer + const id = await artifactStorage.storeArtifact({ + sessionId: this.sessionId, + fileName, + fileType, + mimeType, + size: content.length, + content: Buffer.isBuffer(content) ? content : Buffer.from(content), + metadata: { + originalPath: filePath, + watchedAt: new Date().toISOString(), + directory: this.outputDir, + }, + traceId, + threadId, + }); + + console.log(`Stored artifact ${fileName} with ID ${id} (${content.length} bytes)${traceId ? `, trace: ${traceId}` : ''}${threadId ? `, thread: ${threadId}` : ''}`); + + // Optionally remove the file after storing + // await fs.promises.unlink(filePath); + + } catch (error) { + console.error(`Error handling file ${filePath}:`, error); + } + } + + private detectFileType(fileName: string): 'screenshot' | 'trace' | 'session' | 'other' { + const name = fileName.toLowerCase(); + + if (name.includes('screenshot') || name.includes('page-') && (name.endsWith('.png') || name.endsWith('.jpg') || name.endsWith('.jpeg'))) { + return 'screenshot'; + } + + if (name.includes('trace') || name.endsWith('.trace') || name.endsWith('.zip')) { + return 'trace'; + } + + if (name.includes('session') || name.endsWith('.md') || name.endsWith('.yml') || name.endsWith('.yaml')) { + return 'session'; + } + + return 'other'; + } + + private getMimeType(fileName: string): string { + const ext = path.extname(fileName).toLowerCase(); + const mimeTypes: Record = { + '.png': 'image/png', + '.jpg': 'image/jpeg', + '.jpeg': 'image/jpeg', + '.gif': 'image/gif', + '.webp': 'image/webp', + '.pdf': 'application/pdf', + '.json': 'application/json', + '.yml': 'text/yaml', + '.yaml': 'text/yaml', + '.md': 'text/markdown', + '.txt': 'text/plain', + '.zip': 'application/zip', + '.trace': 'application/octet-stream', + }; + return mimeTypes[ext] || 'application/octet-stream'; + } + + private extractIds(fileName: string): { traceId: string | null; threadId: string | null } { + let traceId: string | null = null; + let threadId: string | null = null; + + // Extract trace ID from common Playwright patterns + // Examples: trace-123abc.zip, trace_session_456def.trace + const traceMatch = fileName.match(/trace[-_]?([a-fA-F0-9]{6,})/i); + if (traceMatch) { + traceId = traceMatch[1]; + } + + // Extract thread ID from filename patterns + // Examples: thread-789ghi.json, session_thread_abc123.md + const threadMatch = fileName.match(/thread[-_]?([a-fA-F0-9]{6,})/i); + if (threadMatch) { + threadId = threadMatch[1]; + } + + // Alternative: extract from session-based patterns + // Examples: session_123_trace_456.zip + if (!traceId) { + const sessionTraceMatch = fileName.match(/session_[^_]+_trace_([a-fA-F0-9]+)/i); + if (sessionTraceMatch) { + traceId = sessionTraceMatch[1]; + } + } + + return { traceId, threadId }; + } + + stop(): void { + this.watchers.forEach(watcher => watcher.close()); + this.watchers = []; + console.log(`Stopped artifact watcher for session ${this.sessionId}`); + } + + getSessionId(): string { + return this.sessionId; + } +} + +// Global watcher instance +let globalWatcher: ArtifactWatcher | null = null; + +export function startArtifactWatcher(outputDir: string, sessionId?: string): ArtifactWatcher { + if (globalWatcher) { + globalWatcher.stop(); + } + + globalWatcher = new ArtifactWatcher(outputDir, sessionId); + globalWatcher.start(); + return globalWatcher; +} + +export function stopArtifactWatcher(): void { + if (globalWatcher) { + globalWatcher.stop(); + globalWatcher = null; + } +} + +export function getActiveWatcher(): ArtifactWatcher | null { + return globalWatcher; +} diff --git a/mastra-test-app/src/mastra/mcp.ts b/mastra-test-app/src/mastra/mcp.ts index 2b0e153..56bcd7b 100644 --- a/mastra-test-app/src/mastra/mcp.ts +++ b/mastra-test-app/src/mastra/mcp.ts @@ -1,5 +1,20 @@ import 'dotenv/config'; import { MCPClient } from "@mastra/mcp"; +import { startArtifactWatcher } from './artifact-watcher'; +import path from 'path'; + +// Create a unique session-based output directory +const createOutputDir = () => { + const sessionId = `session_${Date.now()}_${Math.random().toString(36).slice(2)}`; + const outputDir = path.join(process.cwd(), 'artifacts', sessionId); + + // Start the artifact watcher for this session + startArtifactWatcher(outputDir, sessionId); + + return { outputDir, sessionId }; +}; + +const { outputDir } = createOutputDir(); const buildPlaywrightArgs = () => { @@ -10,7 +25,7 @@ const buildPlaywrightArgs = () => { "--user-agent=Mozilla/5.0 (X11; Linux x86_64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/131.0.0.0 Safari/537.36", "--viewport-size=1920,1080", "--no-sandbox", - "--output-dir=artifacts", + `--output-dir=${outputDir}`, "--save-session", "--save-trace" ]; diff --git a/mastra-test-app/src/mastra/storage-artifacts.ts b/mastra-test-app/src/mastra/storage-artifacts.ts new file mode 100644 index 0000000..bd50ea9 --- /dev/null +++ b/mastra-test-app/src/mastra/storage-artifacts.ts @@ -0,0 +1,103 @@ +import { prisma } from '../lib/prisma'; +import type { PlaywrightArtifact } from './types/artifact-types'; + +export class ArtifactStorage { + constructor(private prismaClient = prisma) {} + + // No longer needed - Prisma handles table creation via migrations + async ensureTable(): Promise { + // This is now handled by Prisma migrations + return Promise.resolve(); + } + + async storeArtifact(artifact: Omit): Promise { + const result = await this.prismaClient.playwrightArtifact.create({ + data: { + sessionId: artifact.sessionId, + fileName: artifact.fileName, + fileType: artifact.fileType, + mimeType: artifact.mimeType, + size: artifact.size, + content: artifact.content, + metadata: artifact.metadata, + traceId: artifact.traceId, + threadId: artifact.threadId, + }, + }); + + return result.id; + } + + async getArtifact(id: string): Promise { + return await this.prismaClient.playwrightArtifact.findUnique({ + where: { id }, + }); + } + + async getSessionArtifacts(sessionId: string): Promise { + return await this.prismaClient.playwrightArtifact.findMany({ + where: { sessionId }, + orderBy: { createdAt: 'desc' }, + }); + } + + async listArtifacts(options: { + limit?: number; + offset?: number; + fileType?: string; + sessionId?: string; + } = {}): Promise<{ artifacts: Omit[]; total: number }> { + const { limit = 50, offset = 0, fileType, sessionId } = options; + + const where: any = {}; + if (fileType) where.fileType = fileType; + if (sessionId) where.sessionId = sessionId; + + // Get total count + const total = await this.prismaClient.playwrightArtifact.count({ where }); + + // Get artifacts (without content for performance) + const artifacts = await this.prismaClient.playwrightArtifact.findMany({ + where, + select: { + id: true, + sessionId: true, + fileName: true, + fileType: true, + mimeType: true, + size: true, + metadata: true, + traceId: true, + threadId: true, + createdAt: true, + updatedAt: true, + // content: false (excluded) + }, + orderBy: { createdAt: 'desc' }, + take: limit, + skip: offset, + }); + + return { artifacts, total }; + } + + async deleteArtifact(id: string): Promise { + try { + await this.prismaClient.playwrightArtifact.delete({ + where: { id }, + }); + return true; + } catch (error) { + return false; + } + } + + async deleteSessionArtifacts(sessionId: string): Promise { + const result = await this.prismaClient.playwrightArtifact.deleteMany({ + where: { sessionId }, + }); + return result.count; + } +} + +export const artifactStorage = new ArtifactStorage(); diff --git a/mastra-test-app/src/mastra/tools/artifact-tools.ts b/mastra-test-app/src/mastra/tools/artifact-tools.ts new file mode 100644 index 0000000..444f334 --- /dev/null +++ b/mastra-test-app/src/mastra/tools/artifact-tools.ts @@ -0,0 +1,187 @@ +import { createTool } from '@mastra/core/tools'; +import { artifactStorage } from '../storage-artifacts'; +import fs from 'fs/promises'; +import path from 'path'; +import { + storeFileArtifactSchema, + getArtifactSchema, + listArtifactsSchema, + getSessionArtifactsSchema, + deleteArtifactSchema, + deleteSessionArtifactsSchema, + artifactResponseSchema, + artifactDetailResponseSchema, + artifactListResponseSchema, + sessionArtifactsResponseSchema, + deleteArtifactResponseSchema, + deleteSessionArtifactsResponseSchema, + getMimeType, + detectFileType, +} from '../types/artifact-types'; + +export const storeFileArtifact = createTool({ + id: 'store-file-artifact', + description: 'Store a file as an artifact in the database', + inputSchema: storeFileArtifactSchema, + outputSchema: artifactResponseSchema, + execute: async ({ context }) => { + try { + const { filePath, sessionId, metadata, traceId, threadId } = context; + let { fileType } = context; + + // Read file + const content = await fs.readFile(filePath); + const fileName = path.basename(filePath); + const mimeType = getMimeType(fileName); + + // Auto-detect file type if not provided + if (fileType === 'other') { + fileType = detectFileType(fileName); + } + + // Store in database + const id = await artifactStorage.storeArtifact({ + sessionId, + fileName, + fileType, + mimeType, + size: content.length, + content, + metadata: { + ...metadata, + originalPath: filePath, + storedAt: new Date().toISOString(), + }, + traceId: traceId || null, + threadId: threadId || null, + }); + + return { + id, + fileName, + size: content.length, + stored: true, + }; + } catch (error) { + throw new Error(`Failed to store artifact: ${error}`); + } + }, +}); + +export const getArtifact = createTool({ + id: 'get-artifact', + description: 'Retrieve an artifact by ID', + inputSchema: getArtifactSchema, + outputSchema: artifactDetailResponseSchema, + execute: async ({ context }) => { + const artifact = await artifactStorage.getArtifact(context.id); + + if (!artifact) { + return { artifact: null }; + } + + return { + artifact: { + id: artifact.id, + sessionId: artifact.sessionId, + fileName: artifact.fileName, + fileType: artifact.fileType, + mimeType: artifact.mimeType, + size: artifact.size, + metadata: artifact.metadata, + traceId: artifact.traceId || null, + threadId: artifact.threadId || null, + createdAt: artifact.createdAt.toISOString(), + contentBase64: Buffer.from(artifact.content).toString('base64'), + }, + }; + }, +}); + +export const listArtifacts = createTool({ + id: 'list-artifacts', + description: 'List artifacts with optional filtering', + inputSchema: listArtifactsSchema, + outputSchema: artifactListResponseSchema, + execute: async ({ context }) => { + const { limit, offset, fileType, sessionId } = context; + const result = await artifactStorage.listArtifacts({ + limit, + offset, + fileType, + sessionId, + }); + + return { + artifacts: result.artifacts.map(artifact => ({ + id: artifact.id, + sessionId: artifact.sessionId, + fileName: artifact.fileName, + fileType: artifact.fileType, + mimeType: artifact.mimeType, + size: artifact.size, + metadata: artifact.metadata, + traceId: artifact.traceId || null, + threadId: artifact.threadId || null, + createdAt: artifact.createdAt.toISOString(), + })), + total: result.total, + hasMore: offset + result.artifacts.length < result.total, + }; + }, +}); + +export const getSessionArtifacts = createTool({ + id: 'get-session-artifacts', + description: 'Get all artifacts for a specific session', + inputSchema: getSessionArtifactsSchema, + outputSchema: sessionArtifactsResponseSchema, + execute: async ({ context }) => { + const artifacts = await artifactStorage.getSessionArtifacts(context.sessionId); + + return { + artifacts: artifacts.map(artifact => ({ + id: artifact.id, + fileName: artifact.fileName, + fileType: artifact.fileType, + size: artifact.size, + createdAt: artifact.createdAt.toISOString(), + })), + sessionId: context.sessionId, + }; + }, +}); + +export const deleteArtifact = createTool({ + id: 'delete-artifact', + description: 'Delete an artifact by ID', + inputSchema: deleteArtifactSchema, + outputSchema: deleteArtifactResponseSchema, + execute: async ({ context }) => { + const deleted = await artifactStorage.deleteArtifact(context.id); + return { deleted }; + }, +}); + +export const deleteSessionArtifacts = createTool({ + id: 'delete-session-artifacts', + description: 'Delete all artifacts for a session', + inputSchema: deleteSessionArtifactsSchema, + outputSchema: deleteSessionArtifactsResponseSchema, + execute: async ({ context }) => { + const deletedCount = await artifactStorage.deleteSessionArtifacts(context.sessionId); + return { + sessionId: context.sessionId, + deletedCount, + }; + }, +}); + +export const artifactTools = [ + storeFileArtifact, + getArtifact, + listArtifacts, + getSessionArtifacts, + deleteArtifact, + deleteSessionArtifacts, +]; diff --git a/mastra-test-app/src/mastra/types/artifact-types.ts b/mastra-test-app/src/mastra/types/artifact-types.ts new file mode 100644 index 0000000..866ddf5 --- /dev/null +++ b/mastra-test-app/src/mastra/types/artifact-types.ts @@ -0,0 +1,159 @@ +import { z } from 'zod'; + +// Artifact schemas for creation and manipulation +export const storeFileArtifactSchema = z.object({ + filePath: z.string().describe('Path to the file to store'), + sessionId: z.string().describe('Session ID for grouping related artifacts'), + fileType: z.enum(['screenshot', 'trace', 'session', 'other']).optional().default('other').describe('Type of artifact'), + metadata: z.record(z.any()).optional().default({}).describe('Additional metadata for the artifact'), + traceId: z.string().optional().describe('Associated trace ID'), + threadId: z.string().optional().describe('Associated thread ID'), +}); + +export const getArtifactSchema = z.object({ + id: z.string().describe('Unique artifact ID'), +}); + +export const listArtifactsSchema = z.object({ + limit: z.number().int().min(1).max(200).optional().default(50).describe('Maximum number of artifacts to return'), + offset: z.number().int().min(0).optional().default(0).describe('Number of artifacts to skip'), + fileType: z.enum(['screenshot', 'trace', 'session', 'other']).optional().describe('Filter by artifact type'), + sessionId: z.string().optional().describe('Filter by session ID'), +}); + +export const getSessionArtifactsSchema = z.object({ + sessionId: z.string().describe('Session ID to get artifacts for'), +}); + +export const deleteArtifactSchema = z.object({ + id: z.string().describe('Unique artifact ID to delete'), +}); + +export const deleteSessionArtifactsSchema = z.object({ + sessionId: z.string().describe('Session ID to delete all artifacts for'), +}); + +// Response schemas +export const artifactResponseSchema = z.object({ + id: z.string(), + fileName: z.string(), + size: z.number(), + stored: z.boolean(), +}); + +export const artifactDetailResponseSchema = z.object({ + artifact: z.object({ + id: z.string(), + sessionId: z.string(), + fileName: z.string(), + fileType: z.string(), + mimeType: z.string(), + size: z.number(), + metadata: z.record(z.any()), + traceId: z.string().nullable(), + threadId: z.string().nullable(), + createdAt: z.string(), + contentBase64: z.string(), + }).nullable(), +}); + +export const artifactListResponseSchema = z.object({ + artifacts: z.array(z.object({ + id: z.string(), + sessionId: z.string(), + fileName: z.string(), + fileType: z.string(), + mimeType: z.string(), + size: z.number(), + metadata: z.record(z.any()), + traceId: z.string().nullable(), + threadId: z.string().nullable(), + createdAt: z.string(), + })), + total: z.number(), + hasMore: z.boolean(), +}); + +export const sessionArtifactsResponseSchema = z.object({ + artifacts: z.array(z.object({ + id: z.string(), + fileName: z.string(), + fileType: z.string(), + size: z.number(), + createdAt: z.string(), + })), + sessionId: z.string(), +}); + +export const deleteArtifactResponseSchema = z.object({ + deleted: z.boolean(), +}); + +export const deleteSessionArtifactsResponseSchema = z.object({ + sessionId: z.string(), + deletedCount: z.number(), +}); + +// Type definitions from Prisma model +export type PlaywrightArtifact = { + id: string; + sessionId: string; + fileName: string; + fileType: string; + mimeType: string; + size: number; + content: Uint8Array; // Prisma uses Uint8Array for Bytes fields + metadata: any; + traceId: string | null; + threadId: string | null; + createdAt: Date; + updatedAt: Date; +}; + +// Type definitions for tool inputs/outputs +export type StoreFileArtifact = z.infer; +export type GetArtifact = z.infer; +export type ListArtifacts = z.infer; +export type GetSessionArtifacts = z.infer; +export type DeleteArtifact = z.infer; +export type DeleteSessionArtifacts = z.infer; + +// Helper function to determine MIME type +export function getMimeType(fileName: string): string { + const ext = fileName.split('.').pop()?.toLowerCase() || ''; + const mimeTypes: Record = { + png: 'image/png', + jpg: 'image/jpeg', + jpeg: 'image/jpeg', + gif: 'image/gif', + webp: 'image/webp', + pdf: 'application/pdf', + json: 'application/json', + yml: 'text/yaml', + yaml: 'text/yaml', + md: 'text/markdown', + txt: 'text/plain', + zip: 'application/zip', + trace: 'application/octet-stream', + }; + return mimeTypes[ext] || 'application/octet-stream'; +} + +// Helper function to detect file type +export function detectFileType(fileName: string): 'screenshot' | 'trace' | 'session' | 'other' { + const name = fileName.toLowerCase(); + + if (name.includes('screenshot') || name.includes('page-') && (name.endsWith('.png') || name.endsWith('.jpg') || name.endsWith('.jpeg'))) { + return 'screenshot'; + } + + if (name.includes('trace') || name.endsWith('.trace') || name.endsWith('.zip')) { + return 'trace'; + } + + if (name.includes('session') || name.endsWith('.md') || name.endsWith('.yml') || name.endsWith('.yaml')) { + return 'session'; + } + + return 'other'; +} From 72fa6f5e74c79c3e3680f535293114973f6d45c0 Mon Sep 17 00:00:00 2001 From: fg-nava <189638926+fg-nava@users.noreply.github.com> Date: Thu, 14 Aug 2025 18:29:22 -0700 Subject: [PATCH 2/4] fix: remove vertex-anthropic provider --- mastra-test-app/package.json | 1 - mastra-test-app/pnpm-lock.yaml | 103 ------------------ .../src/mastra/agents/web-automation-agent.ts | 10 +- 3 files changed, 1 insertion(+), 113 deletions(-) diff --git a/mastra-test-app/package.json b/mastra-test-app/package.json index 203fe5e..348f4f5 100644 --- a/mastra-test-app/package.json +++ b/mastra-test-app/package.json @@ -37,7 +37,6 @@ "@mastra/pg": "^0.13.3", "@prisma/client": "^6.12.0", "@types/jsonwebtoken": "^9.0.10", - "anthropic-vertex-ai": "1.0.2", "csv-parse": "^6.1.0", "dotenv": "^17.2.1", "exa-mcp-server": "^2.0.3", diff --git a/mastra-test-app/pnpm-lock.yaml b/mastra-test-app/pnpm-lock.yaml index 8c29e09..4e57773 100644 --- a/mastra-test-app/pnpm-lock.yaml +++ b/mastra-test-app/pnpm-lock.yaml @@ -44,9 +44,6 @@ importers: '@types/jsonwebtoken': specifier: ^9.0.10 version: 9.0.10 - anthropic-vertex-ai: - specifier: 1.0.2 - version: 1.0.2(zod@3.25.76) csv-parse: specifier: ^6.1.0 version: 6.1.0 @@ -112,25 +109,12 @@ packages: peerDependencies: zod: ^3.0.0 - '@ai-sdk/provider-utils@1.0.20': - resolution: {integrity: sha512-ngg/RGpnA00eNOWEtXHenpX1MsM2QshQh4QJFjUfwcqHpM5kTfG7je7Rc3HcEDP+OkRVv2GF+X4fC1Vfcnl8Ow==} - engines: {node: '>=18'} - peerDependencies: - zod: ^3.0.0 - peerDependenciesMeta: - zod: - optional: true - '@ai-sdk/provider-utils@2.2.8': resolution: {integrity: sha512-fqhG+4sCVv8x7nFzYnFo19ryhAa3w096Kmc3hWxMQfW/TubPOmt3A6tYZhl4mUfQWWQMsuSkLrtjlWuXBVSGQA==} engines: {node: '>=18'} peerDependencies: zod: ^3.23.8 - '@ai-sdk/provider@0.0.24': - resolution: {integrity: sha512-XMsNGJdGO+L0cxhhegtqZ8+T6nn4EoShS819OvCgI2kLbYTIvk0GWFGD0AXJmxkxs3DrpsJxKAFukFR7bvTkgQ==} - engines: {node: '>=18'} - '@ai-sdk/provider@1.1.3': resolution: {integrity: sha512-qZMxYJ0qqX/RfnuIaab+zp8UAeJn/ygXXAffR5I4N0n1IrvA6qBsjc8hXLmBiMV2zoXlifkacF7sEFnYnjBcqg==} engines: {node: '>=18'} @@ -1593,12 +1577,6 @@ packages: resolution: {integrity: sha512-zbB9rCJAT1rbjiVDb2hqKFHNYLxgtk8NURxZ3IZwD3F6NtxbXZQCnnSi1Lkx+IDohdPlFp222wVALIheZJQSEg==} engines: {node: '>=8'} - anthropic-vertex-ai@1.0.2: - resolution: {integrity: sha512-4YuK04KMmBGkx6fi2UjnHkE4mhaIov7tnT5La9+DMn/gw/NSOLZoWNUx+13VY3mkcaseKBMEn1DBzdXXJFIP7A==} - engines: {node: '>=18'} - peerDependencies: - zod: ^3.0.0 - argparse@2.0.1: resolution: {integrity: sha512-8+9WqebbFzpX9OR+Wa6O29asIogeRMzcGtAINdpMHHyAg10f05aSFVBbcEqGf/PXw1EjAZ+q2/bEBg3DvurK3Q==} @@ -1998,10 +1976,6 @@ packages: resolution: {integrity: sha512-aIL5Fx7mawVa300al2BnEE4iNvo1qETxLrPI/o05L7z6go7fCw1J6EQmbK4FmJ2AS7kgVF/KEZWufBfdClMcPg==} engines: {node: '>= 0.6'} - eventsource-parser@1.1.2: - resolution: {integrity: sha512-v0eOBUbiaFojBu2s2NPBfYUoRR9GjcDNvCXVaqEf5vVfpIAh9f8RCo4vXTP8c63QRKCFwoLpMpTdPwwhEKVgzA==} - engines: {node: '>=14.18'} - eventsource-parser@3.0.3: resolution: {integrity: sha512-nVpZkTMM9rF6AQ9gPJpFsNAMt48wIzB5TQgiTLdHiuO8XEDhUgZEhqKlZWXbIzo9VmJ/HvysHqEaVeD5v9TPvA==} engines: {node: '>=20.0.0'} @@ -2200,10 +2174,6 @@ packages: resolution: {integrity: sha512-0Ia46fDOaT7k4og1PDW4YbodWWr3scS2vAr2lTbsplOt2WkKp0vQbkI9wKis/T5LV/dqPjO3bpS/z6GTJB82LA==} engines: {node: '>=18'} - google-auth-library@9.15.1: - resolution: {integrity: sha512-Jb6Z0+nvECVz+2lzSMt9u98UsoakXxA2HGHMCxh+so3n90XgYWkq5dur19JAJV7ONiJY22yBTyJB1TSkvPq9Ng==} - engines: {node: '>=14'} - google-logging-utils@0.0.2: resolution: {integrity: sha512-NEgUnEcBiP5HrPzufUkBzJOD/Sxsco3rLNo1F1TNf7ieU8ryUzBhqba8r756CjLX7rn3fHl6iLEwPYuqpoKgQQ==} engines: {node: '>=14'} @@ -2215,10 +2185,6 @@ packages: graceful-fs@4.2.11: resolution: {integrity: sha512-RbJ5/jmFcNNCcDV5o9eTnBLJ/HszWV0P73bc+Ff4nS/rJj+YaS6IGyiOL0VoBYX+l1Wrl3k63h/KrH+nhJ0XvQ==} - gtoken@7.1.0: - resolution: {integrity: sha512-pCcEwRi+TKpMlxAQObHDQ56KawURgyAf6jtIY046fJ5tIv3zDe/LEIubckAO8fj6JnAxLdmWkUfNyulQ2iKdEw==} - engines: {node: '>=14.0.0'} - has-symbols@1.1.0: resolution: {integrity: sha512-1cDNdwJ2Jaohmb3sg4OmKaMBwuC48sYni5HUw2DvsC8LjGTLK9h+eb1X6RyuOHe4hT0ULCW68iomhjUoKUqlPQ==} engines: {node: '>= 0.4'} @@ -2469,15 +2435,9 @@ packages: jwa@1.4.2: resolution: {integrity: sha512-eeH5JO+21J78qMvTIDdBXidBd6nG2kZjg5Ohz/1fpa28Z4CcsWUzJ1ZZyFq/3z3N17aZy+ZuBoHljASbL1WfOw==} - jwa@2.0.1: - resolution: {integrity: sha512-hRF04fqJIP8Abbkq5NKGN0Bbr3JxlQ+qhZufXVr0DvujKy93ZCbXZMHDL4EOtodSbCWxOqR8MS1tXA5hwqCXDg==} - jws@3.2.2: resolution: {integrity: sha512-YHlZCB6lMTllWDtSPHz/ZXTsi8S00usEV6v1tjq8tOUZzw7DpSDWVXjXDre6ed1w/pd495ODpHZYSdkRTsa0HA==} - jws@4.0.0: - resolution: {integrity: sha512-KDncfTmOZoOMTFG4mBlG0qUIOlc03fmzH+ru6RgYVZhPkyiy/92Owlt/8UEN+a4TXR1FQetfIpJE8ApdvdVxTg==} - libsql@0.5.17: resolution: {integrity: sha512-RRlj5XQI9+Wq+/5UY8EnugSWfRmHEw4hn3DKlPrkUgZONsge1PwTtHcpStP6MSNi8ohcbsRgEHJaymA33a8cBw==} cpu: [x64, arm64, wasm32, arm] @@ -2625,11 +2585,6 @@ packages: engines: {node: ^10 || ^12 || ^13.7 || ^14 || >=15.0.1} hasBin: true - nanoid@3.3.6: - resolution: {integrity: sha512-BGcqMMJuToF7i1rt+2PWSNVnWIkGCU78jBG3RxO/bZlnZPK2Cmi2QaffxGO/2RvWi9sL+FAiRiXMgsyxQ1DIDA==} - engines: {node: ^10 || ^12 || ^13.7 || ^14 || >=15.0.1} - hasBin: true - negotiator@0.6.3: resolution: {integrity: sha512-+EUsqGPLsM+j/zdChZjsnX51g4XrHFOIXwfnCVPGlQk/k5giakcKsuxCObBRu6DSm9opw/O6slWbJdghQM4bBg==} engines: {node: '>= 0.6'} @@ -3440,15 +3395,6 @@ snapshots: '@ai-sdk/provider-utils': 2.2.8(zod@3.25.76) zod: 3.25.76 - '@ai-sdk/provider-utils@1.0.20(zod@3.25.76)': - dependencies: - '@ai-sdk/provider': 0.0.24 - eventsource-parser: 1.1.2 - nanoid: 3.3.6 - secure-json-parse: 2.7.0 - optionalDependencies: - zod: 3.25.76 - '@ai-sdk/provider-utils@2.2.8(zod@3.25.76)': dependencies: '@ai-sdk/provider': 1.1.3 @@ -3456,10 +3402,6 @@ snapshots: secure-json-parse: 2.7.0 zod: 3.25.76 - '@ai-sdk/provider@0.0.24': - dependencies: - json-schema: 0.4.0 - '@ai-sdk/provider@1.1.3': dependencies: json-schema: 0.4.0 @@ -5190,16 +5132,6 @@ snapshots: dependencies: color-convert: 2.0.1 - anthropic-vertex-ai@1.0.2(zod@3.25.76): - dependencies: - '@ai-sdk/provider': 0.0.24 - '@ai-sdk/provider-utils': 1.0.20(zod@3.25.76) - google-auth-library: 9.15.1 - zod: 3.25.76 - transitivePeerDependencies: - - encoding - - supports-color - argparse@2.0.1: {} array-flatten@1.1.1: {} @@ -5560,8 +5492,6 @@ snapshots: etag@1.8.1: {} - eventsource-parser@1.1.2: {} - eventsource-parser@3.0.3: {} eventsource@3.0.7: @@ -5854,32 +5784,12 @@ snapshots: slash: 5.1.0 unicorn-magic: 0.3.0 - google-auth-library@9.15.1: - dependencies: - base64-js: 1.5.1 - ecdsa-sig-formatter: 1.0.11 - gaxios: 6.7.1 - gcp-metadata: 6.1.1 - gtoken: 7.1.0 - jws: 4.0.0 - transitivePeerDependencies: - - encoding - - supports-color - google-logging-utils@0.0.2: {} gopd@1.2.0: {} graceful-fs@4.2.11: {} - gtoken@7.1.0: - dependencies: - gaxios: 6.7.1 - jws: 4.0.0 - transitivePeerDependencies: - - encoding - - supports-color - has-symbols@1.1.0: {} has-tostringtag@1.0.2: @@ -6083,22 +5993,11 @@ snapshots: ecdsa-sig-formatter: 1.0.11 safe-buffer: 5.2.1 - jwa@2.0.1: - dependencies: - buffer-equal-constant-time: 1.0.1 - ecdsa-sig-formatter: 1.0.11 - safe-buffer: 5.2.1 - jws@3.2.2: dependencies: jwa: 1.4.2 safe-buffer: 5.2.1 - jws@4.0.0: - dependencies: - jwa: 2.0.1 - safe-buffer: 5.2.1 - libsql@0.5.17: dependencies: '@neon-rs/load': 0.0.4 @@ -6273,8 +6172,6 @@ snapshots: nanoid@3.3.11: {} - nanoid@3.3.6: {} - negotiator@0.6.3: {} negotiator@1.0.0: {} diff --git a/mastra-test-app/src/mastra/agents/web-automation-agent.ts b/mastra-test-app/src/mastra/agents/web-automation-agent.ts index ff2738a..6b51a45 100644 --- a/mastra-test-app/src/mastra/agents/web-automation-agent.ts +++ b/mastra-test-app/src/mastra/agents/web-automation-agent.ts @@ -7,13 +7,6 @@ import { databaseTools } from '../tools/database-tools'; import { anthropic } from '@ai-sdk/anthropic'; import { google } from '@ai-sdk/google'; import { openai } from '@ai-sdk/openai'; -import { createAnthropicVertex } from 'anthropic-vertex-ai'; - -// Configure Anthropic Vertex AI -const anthropicVertex = createAnthropicVertex({ - region: process.env.GOOGLE_VERTEX_REGION, - projectId: process.env.GOOGLE_VERTEX_PROJECT_ID, -}); const storage = postgresStore; @@ -143,8 +136,7 @@ export const webAutomationAgent = new Agent({ `, // model: openai('gpt-5-2025-08-07'), // model: openai('gpt-4.1-mini'), - // model: anthropic('claude-sonnet-4-20250514'), - model: anthropicVertex('claude-sonnet-4-20250514'), + model: anthropic('claude-sonnet-4-20250514'), // model: google('gemini-2.5-pro'), tools: { From 833d4b8f5102b4c2812d0adac3b2d7f6bb743d50 Mon Sep 17 00:00:00 2001 From: fg-nava <189638926+fg-nava@users.noreply.github.com> Date: Mon, 18 Aug 2025 17:02:00 -0700 Subject: [PATCH 3/4] updating cors for vm --- mastra-test-app/src/mastra/index.ts | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/mastra-test-app/src/mastra/index.ts b/mastra-test-app/src/mastra/index.ts index edcbbed..17f3b2e 100644 --- a/mastra-test-app/src/mastra/index.ts +++ b/mastra-test-app/src/mastra/index.ts @@ -35,7 +35,7 @@ export const mastra = new Mastra({ host: '0.0.0.0', // Allow external connections port: 4111, cors: { - origin: ['http://localhost:4111', 'http://0.0.0.0:4111', '*'], + origin: process.env.CORS_ORIGINS?.split(',') || ['http://localhost:4111', '*'], allowMethods: ['GET', 'POST', 'PUT', 'DELETE', 'OPTIONS'], allowHeaders: ['Content-Type', 'Authorization', 'x-mastra-dev-playground'], credentials: true, From 5b59dd28f9028e694cc82177cebe42f3ce5a76f8 Mon Sep 17 00:00:00 2001 From: fg-nava <189638926+fg-nava@users.noreply.github.com> Date: Mon, 18 Aug 2025 17:14:32 -0700 Subject: [PATCH 4/4] docs: adding latest notes on GCP Postrgres DB setup and connection --- mastra-test-app/docs/DEPLOYMENT_NOTES.md | 68 ++++++- mastra-test-app/docs/GCP_POSTGRESQL_SETUP.md | 180 +++++++++++++++++++ 2 files changed, 240 insertions(+), 8 deletions(-) create mode 100644 mastra-test-app/docs/GCP_POSTGRESQL_SETUP.md diff --git a/mastra-test-app/docs/DEPLOYMENT_NOTES.md b/mastra-test-app/docs/DEPLOYMENT_NOTES.md index 6e5bc56..fb3dd03 100644 --- a/mastra-test-app/docs/DEPLOYMENT_NOTES.md +++ b/mastra-test-app/docs/DEPLOYMENT_NOTES.md @@ -6,7 +6,7 @@ This guide provides an exact, copy‑pasteable sequence to deploy `mastra-test-a - Google Cloud project with billing enabled - `gcloud` CLI installed and authenticated - API keys: OpenAI, Anthropic, Exa -- PostgreSQL connection URL (currently Neon) +- GCP Cloud SQL PostgreSQL database (see [GCP_POSTGRESQL_SETUP.md](./GCP_POSTGRESQL_SETUP.md)) --- @@ -55,14 +55,32 @@ cd labs-asp-experiments/mastra-test-app pnpm install ``` -### 5) Configure environment +### 5) Authorize VM IP for database access +```bash +# Get the VM's external IP +EXTERNAL_IP=$(gcloud compute instances describe mastra-app \ + --zone=us-west1-a \ + --format='get(networkInterfaces[0].accessConfigs[0].natIP)') + +echo "VM External IP: $EXTERNAL_IP" + +# Add VM IP to Cloud SQL authorized networks (replace existing IPs as needed) +gcloud sql instances patch app-dev --authorized-networks=$EXTERNAL_IP +``` + +### 6) Configure environment Create `.env` in the project root with required variables: ```bash cat > .env << 'EOF' OPENAI_API_KEY=your_openai_key ANTHROPIC_API_KEY=your_anthropic_key EXA_API_KEY=your_exa_key -DATABASE_URL=postgresql://user:pass@host:5432/db?sslmode=require + +# GCP Cloud SQL PostgreSQL (update with your actual values) +DATABASE_URL="postgresql://app_user:your_password@your_db_ip/app_db?sslmode=disable" + +# CORS Configuration (replace YOUR_VM_IP with actual external IP) +CORS_ORIGINS="http://localhost:4111,http://0.0.0.0:4111,http://YOUR_VM_IP:4111,*" # Required for auth middleware MASTRA_JWT_SECRET=replace_with_a_strong_random_secret @@ -72,20 +90,47 @@ NODE_ENV=production EOF ``` -### 7) Build and (optionally) apply migrations +**Note:** For production, consider using `sslmode=require` with proper SSL certificates instead of `sslmode=disable`. + +### 7) Build and apply migrations ```bash # Build the app and prepare Prisma client in the output pnpm build -# If your database is new or migrations need to be applied +# Apply database migrations npx prisma migrate deploy + +# Generate Prisma client +npx prisma generate ``` ### 8) Start the application + +#### Option A: Development/Testing (foreground) ```bash pnpm dev ``` +#### Option B: Production/Persistent (with PM2) +```bash +# Install PM2 globally +sudo npm install -g pm2 + +# Start the app with PM2 +pm2 start "pnpm dev" --name app-playground + +# Useful PM2 commands: +pm2 list # View all processes +pm2 logs app-playground # View logs +pm2 restart app-playground # Restart the app +pm2 stop app-playground # Stop the app +pm2 delete app-playground # Remove from PM2 + +# Auto-restart PM2 processes on system reboot +pm2 startup +pm2 save +``` + The server listens on `0.0.0.0:4111`. In another terminal: ```bash EXTERNAL_IP=$(gcloud compute instances describe mastra-app \ @@ -102,6 +147,13 @@ echo "http://$EXTERNAL_IP:4111/auth/login" - Auth-protected UI is served at `/auth/login` and the Web Automation Agent playground at `/agents/webAutomationAgent/chat/` after login. ### Troubleshooting -- Prisma client errors: ensure you ran `pnpm build`. If needed, run `npx prisma generate` once, then `pnpm build` again. -- Firewall: verify with `gcloud compute firewall-rules list --filter="name~allow-mastra-app"`. -- Logs: if using PM2, run `pm2 logs mastra-app`; otherwise, observe terminal output from `pnpm dev`. \ No newline at end of file +- **Prisma client errors:** ensure you ran `pnpm build`. If needed, run `npx prisma generate` once, then `pnpm build` again. +- **Firewall issues:** verify with `gcloud compute firewall-rules list --filter="name~allow-mastra-app"`. +- **Database connection:** check that VM IP is authorized in Cloud SQL and `.env` has correct DATABASE_URL. +- **Logs:** + - PM2: `pm2 logs app-playground` + - Foreground: observe terminal output from `pnpm dev` +- **PM2 process management:** + - Check status: `pm2 status` + - Restart if needed: `pm2 restart app-playground` + - Clear logs: `pm2 flush` \ No newline at end of file diff --git a/mastra-test-app/docs/GCP_POSTGRESQL_SETUP.md b/mastra-test-app/docs/GCP_POSTGRESQL_SETUP.md new file mode 100644 index 0000000..e29ef3b --- /dev/null +++ b/mastra-test-app/docs/GCP_POSTGRESQL_SETUP.md @@ -0,0 +1,180 @@ +## Overview + +We migrated to Google Cloud SQL PostgreSQL due to memory and storage limitations on our previous database. The setup uses a cost-effective shared-core instance suitable for development environments. + +## Prerequisites + +- Google Cloud SDK (`gcloud`) installed and authenticated +- Access to the `nava-labs` GCP project +- Appropriate IAM permissions for Cloud SQL + +## Database Configuration + +### Instance Details +- **Instance Name:** `app-dev` +- **Database Engine:** PostgreSQL 15 +- **Tier:** `db-g1-small` (1.7 GB RAM, shared-core) +- **Region:** `us-central1` +- **Storage:** 100GB SSD with automatic backups enabled +- **IP Address:** `your_ip_address` + +### Database & User +- **Database Name:** `app_db` +- **Username:** `app_user` +- **Password:** `your_password` + +## Setup Instructions + +### 1. Verify GCP Project Configuration + +```bash +# Check current project +gcloud config get-value project +# Should return: nava-labs +``` + +### 2. Create PostgreSQL Instance + +```bash +gcloud sql instances create app-dev \ + --database-version=POSTGRES_15 \ + --tier=db-g1-small \ + --region=us-central1 \ + --storage-type=SSD \ + --storage-size=100GB \ + --backup +``` + +**Note:** PostgreSQL on Cloud SQL only supports custom tiers or shared-core tiers (not the standard predefined tiers like `db-n1-standard-*`). + +### 3. Create Database + +```bash +gcloud sql databases create app_db --instance=app-dev +``` + +### 4. Create Database User + +```bash +gcloud sql users create app_user \ + --instance=app-dev \ + --password=your_password +``` + +### 5. Get Connection Information + +```bash +# Get the instance IP address +gcloud sql instances describe app-dev \ + --format="value(ipAddresses[0].ipAddress)" +``` + +### 6. Authorize Your IP Address + +Before you can connect to the database, you need to add your current IP address to the authorized networks: + +```bash +# Get your current public IP address +curl -s https://ipinfo.io/ip + +# Add your IP to the authorized networks (replace YOUR_IP with the actual IP) +gcloud sql instances patch app-dev --authorized-networks=YOUR_IP +``` + +**Important:** When adding a new IP address, make sure to include any previously authorized IPs, otherwise they will be overwritten. + +### 7. Update Environment Configuration + +Update your `.env` file with the new database connection string: + +```env +# New GCP Cloud SQL PostgreSQL +DATABASE_URL="postgresql://app_user:your_password@your_ip_address/app_db?sslmode=require" +``` + +## Connection String Format + +``` +postgresql://[username]:[password]@[host]:[port]/[database]?[parameters] +``` + +For our setup: +- **Username:** `app_user` +- **Password:** `your_password` +- **Host:** `your_ip_address` +- **Port:** `5432` (default, omitted) +- **Database:** `app_db` +- **SSL Mode:** `require` (mandatory for Cloud SQL) + +## Database Migration + +After setting up the new database and authorizing your IP, run Prisma migrations to set up the schema: + +```bash +# Deploy existing migrations to the new database +npx prisma migrate deploy + +# Generate the Prisma client +npx prisma generate + +# Verify the schema is synchronized (optional) +npx prisma db push --accept-data-loss +``` + +**Note:** If you get a connection error, make sure you've completed step 6 (IP authorization) above. + +## Environment Naming Convention + +Following our multi-environment strategy: +- **Development:** `app-dev` (current setup) +- **Staging:** `app-staging` (future) +- **Production:** `app-prod` (future) + +## Cost Considerations + +- **Tier:** `db-g1-small` is cost-effective for development +- **Storage:** 100GB SSD provides good performance +- **Backups:** Enabled for data safety +- **Region:** `us-central1` offers good pricing + +## Troubleshooting + +### Common Issues + +1. **"Only custom or shared-core instance Billing Tier type allowed for PostgreSQL"** + - Use `db-g1-small` or `db-custom-X-Y` tiers, not `db-n1-standard-*` + +2. **"Can't reach database server" or connection timeouts** + - Ensure your IP is authorized: `gcloud sql instances patch app-dev --authorized-networks=YOUR_IP` + - Verify SSL mode is set to `require` + - Check that the instance is running: `gcloud sql instances describe app-dev` + +3. **Prisma migration fails** + - Make sure IP authorization is complete before running migrations + - Verify the DATABASE_URL in your `.env` file is correct + +### Useful Commands + +```bash +# List all SQL instances +gcloud sql instances list + +# Get instance details +gcloud sql instances describe app-dev + +# List databases in instance +gcloud sql databases list --instance=app-dev + +# List users in instance +gcloud sql users list --instance=app-dev + +# Connect via gcloud (for testing) +gcloud sql connect app-dev --user=app_user --database=app_db +``` + +## Security Notes + +- Database password is stored in `.env` file (ensure it's in `.gitignore`) +- SSL is required for all connections +- Plan to use Cloud SQL Proxy for better security in production +- IP whitelisting should be configured for production environments