diff --git a/.github/workflows/e2e-tests.yml b/.github/workflows/e2e-tests.yml index 05ff6bee3..6f5a46e85 100644 --- a/.github/workflows/e2e-tests.yml +++ b/.github/workflows/e2e-tests.yml @@ -86,44 +86,78 @@ jobs: app-id: ${{ vars.APP_ID }} private-key: ${{ secrets.APP_PRIVATE_KEY }} owner: aws - # Build @aws/agentcore-cdk from source for cross-package testing. - # Requires secret: CDK_REPO_NAME (org/repo). Token is generated by the App above. - - name: Build CDK package + # Clone CDK repo for bundle script (requires App token for private repo access) + - name: Clone CDK repo run: | CDK_BRANCH="${{ inputs.cdk_branch || 'main' }}" - echo "Building CDK from branch: $CDK_BRANCH" + echo "Cloning CDK from branch: $CDK_BRANCH" git clone --depth 1 --branch "$CDK_BRANCH" "https://x-access-token:${CDK_REPO_TOKEN}@github.com/${CDK_REPO}.git" /tmp/cdk-repo - cd /tmp/cdk-repo - npm ci - npm run build - TARBALL=$(npm pack --pack-destination "$RUNNER_TEMP" | tail -1) - echo "CDK_TARBALL=$RUNNER_TEMP/$TARBALL" >> "$GITHUB_ENV" env: CDK_REPO_TOKEN: ${{ steps.app-token.outputs.token }} CDK_REPO: ${{ secrets.CDK_REPO_NAME }} - run: npm ci - - run: npm run build - - name: Install CLI globally - run: npm install -g "$(npm pack | tail -1)" + + - name: Bundle GA and preview tarballs + run: | + npm run bundle + GA_TARBALL=$(ls aws-agentcore-*.tgz | grep -v preview | head -1) + PREVIEW_TARBALL=$(ls aws-agentcore-*-preview-*.tgz | head -1) + echo "GA_TARBALL=$PWD/$GA_TARBALL" >> "$GITHUB_ENV" + echo "PREVIEW_TARBALL=$PWD/$PREVIEW_TARBALL" >> "$GITHUB_ENV" + env: + AGENTCORE_CDK_PATH: /tmp/cdk-repo + + - name: Install GA CLI globally + run: npm install -g "$GA_TARBALL" - name: Detect changed e2e test files id: changed run: | BASE_SHA=${{ github.event.pull_request.base.sha || 'HEAD~1' }} - CHANGED=$(git diff --name-only "$BASE_SHA"..HEAD -- 'e2e-tests/*.test.ts' \ - | grep -v '^e2e-tests/strands-bedrock\.test\.ts$' \ - | tr '\n' ' ') - echo "extra_tests=$CHANGED" >> "$GITHUB_OUTPUT" - echo "Changed e2e tests: ${CHANGED:-none}" + # If any helper file changed, run all e2e tests + HELPERS_CHANGED=$(git diff --name-only "$BASE_SHA"..HEAD -- 'e2e-tests/*.ts' \ + | grep -v '\.test\.ts$' | head -1) + if [ -n "$HELPERS_CHANGED" ]; then + GA_EXTRA=$(find e2e-tests -name '*.test.ts' \ + | grep -v '^e2e-tests/strands-bedrock\.test\.ts$' \ + | grep -v '^e2e-tests/harness-' \ + | tr '\n' ' ') + HARNESS_EXTRA=$(find e2e-tests -name 'harness-*.test.ts' \ + | grep -v '^e2e-tests/harness-bedrock\.test\.ts$' \ + | tr '\n' ' ') + else + GA_EXTRA=$(git diff --name-only "$BASE_SHA"..HEAD -- 'e2e-tests/*.test.ts' \ + | grep -v '^e2e-tests/strands-bedrock\.test\.ts$' \ + | grep -v '^e2e-tests/harness-' \ + | tr '\n' ' ') + HARNESS_EXTRA=$(git diff --name-only "$BASE_SHA"..HEAD -- 'e2e-tests/harness-*.test.ts' \ + | grep -v '^e2e-tests/harness-bedrock\.test\.ts$' \ + | tr '\n' ' ') + fi + echo "ga_extra=$GA_EXTRA" >> "$GITHUB_OUTPUT" + echo "harness_extra=$HARNESS_EXTRA" >> "$GITHUB_OUTPUT" + echo "GA extra tests: ${GA_EXTRA:-none}" + echo "Harness extra tests: ${HARNESS_EXTRA:-none}" + + - name: Run E2E tests (GA) + env: + AWS_ACCOUNT_ID: ${{ steps.aws.outputs.account_id }} + AWS_REGION: ${{ inputs.aws_region || 'us-east-1' }} + ANTHROPIC_API_KEY: ${{ env.E2E_ANTHROPIC_API_KEY }} + OPENAI_API_KEY: ${{ env.E2E_OPENAI_API_KEY }} + GEMINI_API_KEY: ${{ env.E2E_GEMINI_API_KEY }} + run: npx vitest run --project e2e e2e-tests/strands-bedrock.test.ts ${{ steps.changed.outputs.ga_extra }} + + - name: Install preview CLI globally + run: npm install -g "$PREVIEW_TARBALL" - - name: Run E2E tests + - name: Run E2E tests (preview/harness) env: AWS_ACCOUNT_ID: ${{ steps.aws.outputs.account_id }} AWS_REGION: ${{ inputs.aws_region || 'us-east-1' }} ANTHROPIC_API_KEY: ${{ env.E2E_ANTHROPIC_API_KEY }} OPENAI_API_KEY: ${{ env.E2E_OPENAI_API_KEY }} GEMINI_API_KEY: ${{ env.E2E_GEMINI_API_KEY }} - CDK_TARBALL: ${{ env.CDK_TARBALL }} - # Always run strands-bedrock as baseline, plus any e2e test files changed in the PR - run: npx vitest run --project e2e e2e-tests/strands-bedrock.test.ts ${{ steps.changed.outputs.extra_tests }} + BUILD_PREVIEW: '1' + run: npx vitest run --project e2e e2e-tests/harness-bedrock.test.ts ${{ steps.changed.outputs.harness_extra }} diff --git a/e2e-tests/harness-e2e-helper.ts b/e2e-tests/harness-e2e-helper.ts index a0ee882c9..b10e173dc 100644 --- a/e2e-tests/harness-e2e-helper.ts +++ b/e2e-tests/harness-e2e-helper.ts @@ -1,3 +1,4 @@ +import { getHarness } from '../src/cli/aws/agentcore-harness.js'; import { hasAwsCredentials, parseJsonOutput, prereqs, retry, spawnAndCollect } from '../src/test-utils/index.js'; import { cleanupStaleCredentialProviders, @@ -7,7 +8,7 @@ import { writeAwsTargets, } from './e2e-helper.js'; import { randomUUID } from 'node:crypto'; -import { mkdir, rm } from 'node:fs/promises'; +import { mkdir, readFile, rm } from 'node:fs/promises'; import { tmpdir } from 'node:os'; import { join } from 'node:path'; import { afterAll, beforeAll, describe, expect, it } from 'vitest'; @@ -30,10 +31,11 @@ export function createHarnessE2ESuite(cfg: HarnessE2EConfig) { const providerLabel = cfg.modelProvider === 'open_ai' ? 'OpenAI' : cfg.modelProvider === 'gemini' ? 'Gemini' : 'Bedrock'; - describe.sequential(`e2e: harness/${providerLabel} — create → deploy → invoke`, () => { + describe.sequential(`e2e: harness/${providerLabel} — create → deploy → invoke → teardown`, () => { let testDir: string; let projectPath: string; let harnessName: string; + let harnessId: string; beforeAll(async () => { if (!canRun) return; @@ -76,7 +78,8 @@ export function createHarnessE2ESuite(cfg: HarnessE2EConfig) { afterAll(async () => { if (projectPath && hasAws) { - await teardownE2EProject(projectPath, harnessName, cfg.modelProvider); + // Teardown is tested as a step; this is a safety net in case earlier steps fail + await teardownE2EProject(projectPath, harnessName, cfg.modelProvider).catch((_: unknown) => undefined); } if (testDir) await rm(testDir, { recursive: true, force: true, maxRetries: 3, retryDelay: 1000 }); }, 600000); @@ -158,6 +161,62 @@ export function createHarnessE2ESuite(cfg: HarnessE2EConfig) { expect(harness, `Harness "${harnessName}" should appear in status`).toBeDefined(); expect(harness!.deploymentState).toBe('deployed'); expect(harness!.identifier, 'Deployed harness should have a harnessArn').toBeTruthy(); + + // Capture harnessId for teardown verification + const statePath = join(projectPath, 'agentcore', '.cli', 'deployed-state.json'); + const stateJson = JSON.parse(await readFile(statePath, 'utf-8')) as { + targets?: { default?: { resources?: { harnesses?: Record } } }; + }; + const harnessEntry = stateJson.targets?.default?.resources?.harnesses?.[harnessName]; + if (harnessEntry) { + harnessId = harnessEntry.harnessId; + } + }, + 120000 + ); + + it.skipIf(!canRun)( + 'remove all and deploy tears down harness', + async () => { + const removeResult = await runAgentCoreCLI(['remove', 'all', '--yes', '--json'], projectPath); + expect(removeResult.exitCode, `Remove all failed: ${removeResult.stderr}`).toBe(0); + + const removeJson = parseJsonOutput(removeResult.stdout) as { success: boolean }; + expect(removeJson.success).toBe(true); + + const deployResult = await runAgentCoreCLI(['deploy', '--yes', '--json'], projectPath); + expect(deployResult.exitCode, `Teardown deploy failed: ${deployResult.stderr}`).toBe(0); + + const deployJson = parseJsonOutput(deployResult.stdout) as { success: boolean }; + expect(deployJson.success).toBe(true); + }, + 600000 + ); + + it.skipIf(!canRun)( + 'verifies harness is deleted from AWS', + async () => { + expect(harnessId, 'harnessId should have been captured').toBeTruthy(); + + const region = process.env.AWS_REGION ?? 'us-east-1'; + await retry( + async () => { + try { + const result = await getHarness({ region, harnessId }); + expect(['DELETING', 'DELETED'], `Expected DELETING or DELETED, got ${result.harness.status}`).toContain( + result.harness.status + ); + } catch (err: unknown) { + const message = err instanceof Error ? err.message : String(err); + expect( + message.includes('not found') || message.includes('ResourceNotFoundException'), + `Expected ResourceNotFound, got: ${message}` + ).toBe(true); + } + }, + 5, + 10000 + ); }, 120000 ); diff --git a/src/cli/tui/screens/deploy/useDeployFlow.ts b/src/cli/tui/screens/deploy/useDeployFlow.ts index 8b0295744..5f895e2b5 100644 --- a/src/cli/tui/screens/deploy/useDeployFlow.ts +++ b/src/cli/tui/screens/deploy/useDeployFlow.ts @@ -1,4 +1,5 @@ import { ConfigIO } from '../../../../lib'; +import type { DeployedState, HarnessDeployedState } from '../../../../schema'; import type { CdkToolkitWrapper, DeployMessage, SwitchableIoHost } from '../../../cdk/toolkit-lib'; import { buildDeployedState, @@ -14,9 +15,11 @@ import { } from '../../../cloudformation'; import { DEFAULT_DEPLOY_ATTRS, computeDeployAttrs } from '../../../commands/deploy/utils.js'; import { getErrorMessage, isChangesetInProgressError, isExpiredTokenError } from '../../../errors'; +import { isPreviewEnabled } from '../../../feature-flags'; import { ExecLogger } from '../../../logging'; import { performStackTeardown, setupTransactionSearch } from '../../../operations/deploy'; import { getGatewayTargetStatuses } from '../../../operations/deploy/gateway-status'; +import { createDeploymentManager } from '../../../operations/deploy/imperative'; import { deleteOrphanedABTests, setupABTests } from '../../../operations/deploy/post-deploy-ab-tests'; import { resolveConfigBundleComponentKeys, @@ -319,6 +322,38 @@ export function useDeployFlow(options: DeployFlowOptions = {}): DeployFlowState setStackOutputs(outputs); const existingState = await configIO.readDeployedState().catch(() => undefined); + + // Post-CDK: deploy imperative resources (harness) — preview mode only + let deployedHarnesses: Record | undefined; + if (isPreviewEnabled()) { + const imperativeManager = createDeploymentManager(); + const imperativeDeployedState: DeployedState = existingState ?? { targets: {} }; + const imperativeContext = { + projectSpec: ctx.projectSpec, + target, + configIO, + deployedState: imperativeDeployedState, + cdkOutputs: outputs, + onProgress: (step: string, status: 'start' | 'done' | 'error') => { + logger.log(`${step}: ${status}`); + }, + }; + + if (imperativeManager.hasDeployersForPhase('post-cdk', imperativeContext)) { + logger.startStep('Deploy harnesses'); + const postCdkResult = await imperativeManager.runPhase('post-cdk', imperativeContext); + const harnessResult = postCdkResult.results.get('harness'); + if (harnessResult?.state) { + deployedHarnesses = harnessResult.state as Record; + } + if (!postCdkResult.success) { + logger.endStep('error', postCdkResult.error); + throw new Error(`Harness deployment failed: ${postCdkResult.error}`); + } + logger.endStep('success'); + } + } + let deployedState = buildDeployedState({ targetName: target.name, stackName: currentStackName, @@ -333,6 +368,7 @@ export function useDeployFlow(options: DeployFlowOptions = {}): DeployFlowState policyEngines, policies, datasets, + harnesses: deployedHarnesses, }); await configIO.writeDeployedState(deployedState); @@ -673,6 +709,37 @@ export function useDeployFlow(options: DeployFlowOptions = {}): DeployFlowState await cdkToolkitWrapper.deploy(); if (context?.isTeardownDeploy) { + // Teardown imperative resources (harnesses) before destroying the stack + if (isPreviewEnabled()) { + const teardownTarget = context.awsTargets[0]; + if (teardownTarget) { + const imperativeManager = createDeploymentManager(); + const teardownConfigIO = new ConfigIO(); + const existingTeardownState = await teardownConfigIO + .readDeployedState() + .catch(() => ({ targets: {} }) as DeployedState); + const teardownContext = { + projectSpec: context.projectSpec, + target: teardownTarget, + configIO: teardownConfigIO, + deployedState: existingTeardownState, + onProgress: (step: string, status: 'start' | 'done' | 'error') => { + logger.log(`${step}: ${status}`); + }, + }; + + if (imperativeManager.hasDeployersForPhase('post-cdk', teardownContext)) { + logger.startStep('Tear down imperative resources'); + const teardownResult = await imperativeManager.teardownAll(teardownContext); + if (!teardownResult.success) { + logger.endStep('error', teardownResult.error); + throw new Error(`Imperative teardown failed: ${teardownResult.error}`); + } + logger.endStep('success'); + } + } + } + // After deploying the empty spec, destroy the stack entirely const targetName = context.awsTargets[0]?.name; if (targetName) {