diff --git a/.agents/skills/testerarmy-cli/SKILL.md b/.agents/skills/testerarmy-cli/SKILL.md new file mode 100644 index 00000000000..d025e386e49 --- /dev/null +++ b/.agents/skills/testerarmy-cli/SKILL.md @@ -0,0 +1,228 @@ +--- +name: testerarmy-cli +description: Use TesterArmy CLI to create, organize, and run dashboard-managed QA tests. Prefer saved tests, groups, project context, credentials, and remote runs over one-off local prompts. Trigger when defining regression coverage, adding QA flows, or wiring CI checks. +license: MIT +metadata: + author: TesterArmy + tags: testerarmy, qa, cli, dashboard-tests, regression, ci +--- + +# TesterArmy CLI + +Create dashboard-managed QA coverage with `ta` / `testerarmy`. + +Default to saved dashboard tests, groups, project context, credentials, and +remote runs. Use local `ta run "..."` only for quick exploration. + +## When to Use + +- Create persistent QA coverage for a feature or product area. +- Convert acceptance criteria into dashboard tests. +- Add or update smoke, regression, auth, billing, onboarding, or mobile flows. +- Prepare CI-visible groups for PRs or releases. + +## Setup + +Check auth: + +```bash +ta status --json +``` + +If needed: + +```bash +ta auth +TESTERARMY_API_KEY= ta status --json +``` + +Discover scope: + +```bash +ta projects list --json +ta groups list --project --json +ta tests list --project --json +``` + +Use IDs exactly as returned. + +## Project Context + +Create a project: + +```bash +echo '{"name":"Example","url":"https://example.com","projectType":"web"}' | ta projects create --json +``` + +Store app knowledge: + +```bash +echo '{"category":"site_structure","title":"Auth route","content":"Login is at /login","importance":"high"}' | ta memories create --project --json +``` + +Memory categories: `site_structure`, `test_insights`, `user_preferences`. + +Create credentials for login or inbox flows: + +```bash +echo '{"kind":"login","label":"Admin","username":"admin@example.com","password":"secret"}' | ta projects credentials-create --json +echo '{"kind":"inbox","label":"Signup inbox"}' | ta projects credentials-create --json +``` + +Never print real secrets in final messages. + +## Tests + +Create: + +```bash +echo '{"title":"Login flow","description":"User can sign in and reach the dashboard","steps":[{"title":"Navigate to /login","type":"act"},{"title":"Sign in with the saved admin credentials","type":"login","credentialId":""},{"title":"Dashboard loads and shows the project list","type":"assert"}]}' | ta tests create --project --json +``` + +Create in a group: + +```bash +echo '{"title":"Pricing CTA","steps":[{"title":"Open /pricing","type":"act"},{"title":"Click the primary CTA","type":"act"},{"title":"Signup or dashboard flow starts","type":"assert"}]}' | ta tests create --project --group --json +``` + +Payload: + +```json +{ + "title": "string, required", + "description": "string, optional", + "platform": "web or mobile, optional", + "steps": [ + { "title": "User action", "type": "act" }, + { "title": "Expected result", "type": "assert" }, + { "title": "Login instruction", "type": "login", "credentialId": "uuid" }, + { "title": "Use temporary email", "type": "login", "temporaryEmail": true }, + { "title": "Screenshot label", "type": "screenshot" } + ] +} +``` + +Rules: + +- Cover one user journey. +- Prefer 3-8 meaningful steps. +- Use `act` for navigation, clicks, typing, upload, and other user actions. +- Use `assert` for visible outcomes, persisted state, email delivery, or URL changes. +- Use `login` with `credentialId` or `temporaryEmail`; do not put passwords in step titles. +- Use `screenshot` only for important visual checkpoints. +- Maximum 50 steps per test. + +Inspect before changing: + +```bash +ta tests get --json +``` + +Update title, description, or steps: + +```bash +echo '{"title":"Updated login smoke"}' | ta tests update --json +echo '{"steps":[{"title":"Open /login","type":"act"},{"title":"Sign in","type":"login","credentialId":""},{"title":"Dashboard is visible","type":"assert"}]}' | ta tests update --json +``` + +Replacing `steps` requires the complete array. + +## Groups + +Create suites: + +```bash +echo '{"projectId":"","name":"Smoke"}' | ta groups create --json +ta groups add-test --json +ta groups remove-test --json +``` + +Common groups: `Smoke`, `Auth`, `Core journeys`, `Mobile smoke`. + +## Runs + +Modes: + +- Default/local: fetches a saved test, then runs it on this machine. +- `--remote`: queues the saved test in TesterArmy cloud. + +Local debugging: + +```bash +ta tests run --url http://localhost:3000 --json +ta tests run --group --project --url http://localhost:3000 --parallel 3 --json +``` + +Remote validation: + +```bash +ta tests run --remote --wait --json +ta tests run --group --project --remote --wait --json +``` + +Defaults: + +- No `--remote`: local browser execution. +- `--remote`: cloud execution. +- `--wait`: wait for remote results. +- Local-only flags such as `--headed`, `--browser`, `--timeout`, and `--system-prompt-file` are ignored with `--remote`. +- Remote group runs can use `--environment production|preview`. +- Remote single-test runs can use `--mode fast|deep`. + +CI: + +```bash +ta ci --group --project --target-url https://staging.example.com --json +``` + +Runs: + +```bash +ta runs list --project --json +ta runs get --json +ta runs wait --timeout 600000 --json +ta runs cancel --json +``` + +## Mobile App Coverage + +Upload an iOS Simulator app before cloud runs: + +```bash +ta upload-app --app-path ios/build/Build/Products/Release-iphonesimulator/MyApp.app --project --json +ta ci --group --project --app-id --delete-app-after-run --json +``` + +Supported uploads: `.app`, `.app.zip`, `.zip`, `.app.tar.gz`, `.tar.gz`, `.tgz`. +`.ipa` and Android artifacts are not supported yet. + +## Local Prompt + +`ta run ` runs an ad hoc local browser test: + +```bash +ta run "check pricing CTA" --url https://example.com --json +``` + +Use only to explore before creating or updating dashboard tests. Use +`ta tests create` and `ta tests run --group` for durable workflows. + +## Reporting + +Report: + +- Project ID/name +- Test IDs and titles created or updated +- Group IDs/names touched +- Remote validation command and result, if run +- Run ID or artifact/output path, if available + +Do not claim durable coverage unless `ta tests create` or `ta tests update` ran. + +## References + +| File | Description | +| --- | --- | +| [reporting-template.md][reporting-template] | Dashboard coverage report template | + +[reporting-template]: references/reporting-template.md diff --git a/.agents/skills/testerarmy-cli/agents/openai.yaml b/.agents/skills/testerarmy-cli/agents/openai.yaml new file mode 100644 index 00000000000..d88cd81b8cb --- /dev/null +++ b/.agents/skills/testerarmy-cli/agents/openai.yaml @@ -0,0 +1,4 @@ +interface: + display_name: "TesterArmy CLI" + short_description: "Validate changes with ta markdown tests, artifacts, and strict pass/fail reporting" + default_prompt: "Use $testerarmy-cli to run targeted qa validation after code changes. Default to ta run with --json, add --output when needed for reuse, and report exact command + outcome." diff --git a/.agents/skills/testerarmy-cli/references/reporting-template.md b/.agents/skills/testerarmy-cli/references/reporting-template.md new file mode 100644 index 00000000000..9b825bccdfb --- /dev/null +++ b/.agents/skills/testerarmy-cli/references/reporting-template.md @@ -0,0 +1,26 @@ +# Dashboard Coverage Report Template + +Use this shape in final updates. Prefer remote `--json` validation. + +```md +TesterArmy: +- Project: Example () +- Created: Login flow () +- Group: Smoke () +- Validation: ta tests run --remote --wait --json +- Result: PASS +- Run: +``` + +If failed: + +```md +TesterArmy: +- Project: Example () +- Updated: Checkout smoke () +- Group: Smoke () +- Validation: ta tests run --group --project --remote --wait --json +- Result: FAILED +- Run: +- Failure: "Checkout success screen did not appear" +``` diff --git a/.claude/skills/testerarmy-cli b/.claude/skills/testerarmy-cli new file mode 120000 index 00000000000..e20abc8e56c --- /dev/null +++ b/.claude/skills/testerarmy-cli @@ -0,0 +1 @@ +../../.agents/skills/testerarmy-cli \ No newline at end of file diff --git a/.windsurf/skills/testerarmy-cli b/.windsurf/skills/testerarmy-cli new file mode 120000 index 00000000000..e20abc8e56c --- /dev/null +++ b/.windsurf/skills/testerarmy-cli @@ -0,0 +1 @@ +../../.agents/skills/testerarmy-cli \ No newline at end of file diff --git a/apps/api/src/app/agents/agents-webhook.controller.ts b/apps/api/src/app/agents/agents-webhook.controller.ts index 7ccc1d2e569..4db1ce95958 100644 --- a/apps/api/src/app/agents/agents-webhook.controller.ts +++ b/apps/api/src/app/agents/agents-webhook.controller.ts @@ -9,7 +9,6 @@ import { Post, Req, Res, - UseGuards, } from '@nestjs/common'; import { ApiExcludeController } from '@nestjs/swagger'; import { PinoLogger } from '@novu/application-generic'; @@ -21,7 +20,6 @@ import { ExternalApiAccessible } from '../auth/framework/external-api.decorator' import { UserSession } from '../shared/framework/user.decorator'; import { AgentReplyPayloadDto } from './dtos/agent-reply-payload.dto'; import { AgentInactiveException } from './exceptions/agent-inactive.exception'; -import { AgentConversationEnabledGuard } from './guards/agent-conversation-enabled.guard'; import type { AgentConfigResolveSource } from './services/agent-config-resolver.service'; import { ChatSdkService } from './services/chat-sdk.service'; import { ManagedAgentService } from './services/managed-agent.service'; @@ -29,7 +27,6 @@ import { HandleAgentReplyCommand } from './usecases/handle-agent-reply/handle-ag import { HandleAgentReply } from './usecases/handle-agent-reply/handle-agent-reply.usecase'; @Controller('/agents') -@UseGuards(AgentConversationEnabledGuard) @ApiExcludeController() export class AgentsWebhookController { constructor( diff --git a/apps/api/src/app/agents/agents.controller.ts b/apps/api/src/app/agents/agents.controller.ts index be33105cb6e..f0ed2aba70e 100644 --- a/apps/api/src/app/agents/agents.controller.ts +++ b/apps/api/src/app/agents/agents.controller.ts @@ -13,7 +13,6 @@ import { Query, Req, UseFilters, - UseGuards, UseInterceptors, } from '@nestjs/common'; import { ApiExcludeController, ApiExcludeEndpoint, ApiOperation } from '@nestjs/swagger'; @@ -81,7 +80,6 @@ import { SendWhatsAppTestTemplateResponseDto, } from './dtos/send-whatsapp-test-template.dto'; import { AgentRuntimeExceptionFilter } from './filters/agent-runtime-exception.filter'; -import { AgentConversationEnabledGuard } from './guards/agent-conversation-enabled.guard'; import { AddAgentIntegrationCommand } from './usecases/add-agent-integration/add-agent-integration.command'; import { AddAgentIntegration } from './usecases/add-agent-integration/add-agent-integration.usecase'; import { ConfigureTelegramAgentWebhookCommand } from './usecases/configure-telegram-agent-webhook/configure-telegram-agent-webhook.command'; @@ -148,7 +146,6 @@ import { VerifyManagedCredentials } from './usecases/verify-managed-credentials/ @ApiCommonResponses() @Controller('/agents') @UseInterceptors(ClassSerializerInterceptor) -@UseGuards(AgentConversationEnabledGuard) @ApiExcludeController() @RequireAuthentication() export class AgentsController { diff --git a/apps/api/src/app/agents/services/agent-config-resolver.service.ts b/apps/api/src/app/agents/services/agent-config-resolver.service.ts index c53b18684f9..e3c63a7f98c 100644 --- a/apps/api/src/app/agents/services/agent-config-resolver.service.ts +++ b/apps/api/src/app/agents/services/agent-config-resolver.service.ts @@ -3,7 +3,6 @@ import { AnalyticsService, decryptChannelConnectionAuth, decryptCredentials, - FeatureFlagsService, PinoLogger, } from '@novu/application-generic'; import { @@ -13,7 +12,7 @@ import { ICredentialsEntity, IntegrationRepository, } from '@novu/dal'; -import { EmailProviderIdEnum, FeatureFlagsKeysEnum } from '@novu/shared'; +import { EmailProviderIdEnum } from '@novu/shared'; import type { WellKnownEmoji } from 'chat'; import { trackAgentIntegrationFirstWebhook } from '../agent-analytics'; import { AgentPlatformEnum } from '../dtos/agent-platform.enum'; @@ -86,7 +85,6 @@ async function resolveReaction( @Injectable() export class AgentConfigResolver { constructor( - private readonly featureFlagsService: FeatureFlagsService, private readonly agentRepository: AgentRepository, private readonly agentIntegrationRepository: AgentIntegrationRepository, private readonly integrationRepository: IntegrationRepository, @@ -113,16 +111,6 @@ export class AgentConfigResolver { const { _environmentId: environmentId, _organizationId: organizationId } = agent; - const isEnabled = await this.featureFlagsService.getFlag({ - key: FeatureFlagsKeysEnum.IS_CONVERSATIONAL_AGENTS_ENABLED, - defaultValue: false, - environment: { _id: environmentId }, - organization: { _id: organizationId }, - }); - if (!isEnabled) { - throw new NotFoundException(); - } - const integration = await this.integrationRepository.findOne({ _environmentId: environmentId, _organizationId: organizationId, diff --git a/apps/api/src/app/agents/usecases/generate-managed-agent/generate-managed-agent.usecase.ts b/apps/api/src/app/agents/usecases/generate-managed-agent/generate-managed-agent.usecase.ts index b0dce51449c..3860efb706c 100644 --- a/apps/api/src/app/agents/usecases/generate-managed-agent/generate-managed-agent.usecase.ts +++ b/apps/api/src/app/agents/usecases/generate-managed-agent/generate-managed-agent.usecase.ts @@ -1,11 +1,10 @@ -import { ForbiddenException, Injectable, ServiceUnavailableException } from '@nestjs/common'; +import { Injectable, ServiceUnavailableException } from '@nestjs/common'; import { ModuleRef } from '@nestjs/core'; -import { AnalyticsService, FeatureFlagsService, InstrumentUsecase, PinoLogger } from '@novu/application-generic'; +import { AnalyticsService, InstrumentUsecase, PinoLogger } from '@novu/application-generic'; import { CLAUDE_ANTHROPIC_SKILLS, CLAUDE_BUILTIN_TOOLS, CLAUDE_DEFAULT_TOOL_TYPES, - FeatureFlagsKeysEnum, MCP_SERVERS, } from '@novu/shared'; @@ -177,7 +176,6 @@ function ensureDefaultTools(tools: string[]): string[] { export class GenerateManagedAgent { constructor( private readonly moduleRef: ModuleRef, - private readonly featureFlagsService: FeatureFlagsService, private readonly analyticsService: AnalyticsService, private readonly logger: PinoLogger ) {} @@ -188,22 +186,6 @@ export class GenerateManagedAgent { const runtime = command.runtime ?? 'managed'; const { organizationId, environmentId, _id: userId } = user; - // Self-hosted scaffolding does not provision anything on Anthropic Managed Agents — it just - // emits name/identifier/systemPrompt for the caller's own runtime. There is therefore no - // reason to gate it on IS_MANAGED_AGENT_RUNTIME_ENABLED. Managed generation still requires - // the flag because the resulting agent will be provisioned on Anthropic. - if (runtime === 'managed') { - const isEnabled = await this.featureFlagsService.getFlag({ - key: FeatureFlagsKeysEnum.IS_MANAGED_AGENT_RUNTIME_ENABLED, - defaultValue: false, - organization: { _id: organizationId }, - }); - - if (!isEnabled) { - throw new ForbiddenException('Managed agent generation is not enabled for this organization'); - } - } - const eeAi = this.loadEeAi(); const llmService = this.moduleRef.get(eeAi.LlmService, { strict: false }); const tokenUsageTracker = new eeAi.TokenUsageTracker(); diff --git a/apps/api/src/app/environments-v2/usecases/sync-strategies/agent-sync.strategy.ts b/apps/api/src/app/environments-v2/usecases/sync-strategies/agent-sync.strategy.ts index e5221adb01f..c773771c090 100644 --- a/apps/api/src/app/environments-v2/usecases/sync-strategies/agent-sync.strategy.ts +++ b/apps/api/src/app/environments-v2/usecases/sync-strategies/agent-sync.strategy.ts @@ -1,6 +1,6 @@ import { Injectable } from '@nestjs/common'; -import { FeatureFlagsService, PinoLogger } from '@novu/application-generic'; -import { FeatureFlagsKeysEnum, UserSessionData } from '@novu/shared'; +import { PinoLogger } from '@novu/application-generic'; +import { UserSessionData } from '@novu/shared'; import { IDiffResult, ISyncContext, ISyncResult, ResourceTypeEnum } from '../../types/sync.types'; import { BaseSyncStrategy } from './base/base-sync.strategy'; @@ -12,8 +12,7 @@ export class AgentSyncStrategy extends BaseSyncStrategy { constructor( logger: PinoLogger, private agentSyncOperation: AgentSyncOperation, - private agentDiffOperation: AgentDiffOperation, - private featureFlagsService: FeatureFlagsService + private agentDiffOperation: AgentDiffOperation ) { super(logger); } @@ -23,12 +22,6 @@ export class AgentSyncStrategy extends BaseSyncStrategy { } async execute(context: ISyncContext): Promise { - const isEnabled = await this.isFeatureEnabled(context.user.organizationId, context.sourceEnvironmentId); - - if (!isEnabled) { - return { resourceType: ResourceTypeEnum.AGENT, successful: [], failed: [], skipped: [], totalProcessed: 0 }; - } - return this.agentSyncOperation.execute(context); } @@ -38,31 +31,10 @@ export class AgentSyncStrategy extends BaseSyncStrategy { organizationId: string, userContext: UserSessionData ): Promise { - const isEnabled = await this.isFeatureEnabled(organizationId, sourceEnvId); - - if (!isEnabled) { - return []; - } - return this.agentDiffOperation.execute(sourceEnvId, targetEnvId, organizationId, userContext); } async getAvailableResourceIds(sourceEnvironmentId: string, organizationId: string): Promise { - const isEnabled = await this.isFeatureEnabled(organizationId, sourceEnvironmentId); - - if (!isEnabled) { - return []; - } - return this.agentSyncOperation.getAvailableResourceIds(sourceEnvironmentId, organizationId); } - - private async isFeatureEnabled(organizationId: string, environmentId: string): Promise { - return this.featureFlagsService.getFlag({ - key: FeatureFlagsKeysEnum.IS_CONVERSATIONAL_AGENTS_ENABLED, - defaultValue: false, - organization: { _id: organizationId }, - environment: { _id: environmentId }, - }); - } } diff --git a/packages/novu/package.json b/packages/novu/package.json index e7d1fb240a3..71cc7659c71 100644 --- a/packages/novu/package.json +++ b/packages/novu/package.json @@ -1,6 +1,6 @@ { "name": "novu", - "version": "2.8.1-rc.12", + "version": "2.8.1-rc.13", "description": "Novu CLI. Run Novu Studio and sync workflows with Novu Cloud", "main": "src/index.js", "publishConfig": { diff --git a/packages/novu/src/commands/connect/ui/phase-content.tsx b/packages/novu/src/commands/connect/ui/phase-content.tsx index 81b3c332871..f28444049f5 100644 --- a/packages/novu/src/commands/connect/ui/phase-content.tsx +++ b/packages/novu/src/commands/connect/ui/phase-content.tsx @@ -56,8 +56,10 @@ export function PhaseContent({ case 'pick-runtime': return ( - Where do you want the agent to run? - Choose the agent runtime. Novu connects it to Slack, email, and more. + + Where do you want the agent to run? + Choose the agent runtime. Novu connects it to Slack, email, and more. + phase.resolve(value)} /> ); @@ -362,9 +364,9 @@ function RuntimeSelect({ onChange: (value: AgentRuntimeChoice) => void; }): React.ReactElement { const options: Array<{ value: AgentRuntimeChoice; title: string; detail?: string }> = [ - { value: 'demo', title: 'Novu Demo Agent', detail: '10 conversations per month' }, - { value: 'claude', title: 'Claude Managed Agents - BYOK' }, - { value: 'claude-aws', title: 'Claude Managed Agents on AWS' }, + { value: 'demo', title: 'Demo Credentials', detail: '10 conversations per month' }, + { value: 'claude', title: 'Claude Managed Agents' }, + { value: 'claude-aws', title: 'AWS Claude Managed Agents' }, ]; const [idx, setIdx] = React.useState(0); diff --git a/packages/novu/src/commands/dev/enums.ts b/packages/novu/src/commands/dev/enums.ts index 800c0982703..43d3710d797 100644 --- a/packages/novu/src/commands/dev/enums.ts +++ b/packages/novu/src/commands/dev/enums.ts @@ -20,7 +20,7 @@ export enum ApiUrlEnum { /** Browser-auth surface for `novu connect` (distinct from the main dashboard). */ export enum ConnectDashboardUrlEnum { PROD = 'https://connect.novu.co', - STAGING = 'https://devconnect.novu.co', + STAGING = 'https://connect.novu-staging.co', } export const LOCAL_API_URL = 'https://api.novu.localhost'; diff --git a/packages/novu/src/commands/dev/resolve-region-urls.spec.ts b/packages/novu/src/commands/dev/resolve-region-urls.spec.ts index b8a39f18e7b..27305e2e783 100644 --- a/packages/novu/src/commands/dev/resolve-region-urls.spec.ts +++ b/packages/novu/src/commands/dev/resolve-region-urls.spec.ts @@ -16,7 +16,7 @@ describe('resolveRegionUrls', () => { expect(urls.apiUrl).toBe('https://api.novu-staging.co'); expect(urls.dashboardUrl).toBe('https://dashboard.novu-staging.co'); - expect(urls.connectDashboardUrl).toBe('https://devconnect.novu.co'); + expect(urls.connectDashboardUrl).toBe('https://connect.novu-staging.co'); }); it('maps local region to local dev URLs with connect dashboard matching dashboard', () => { diff --git a/skills-lock.json b/skills-lock.json index 51e073d17b9..a177a7044c5 100644 --- a/skills-lock.json +++ b/skills-lock.json @@ -18,6 +18,12 @@ "sourceType": "github", "skillPath": "skills/linear-release-setup/SKILL.md", "computedHash": "f6e1b7c9779323210fd4800a10049b30c87bc34bf254acbae606d6dea3f4478d" + }, + "testerarmy-cli": { + "source": "tester-army/cli", + "sourceType": "github", + "skillPath": "skills/testerarmy-cli/SKILL.md", + "computedHash": "25d81694661d78160561b54865cdc5a589ba3bd11bace6becaa392a0f9a9e2ff" } } }