From f46d6329c8775508970337f91818adfd96f1adf9 Mon Sep 17 00:00:00 2001 From: Harrison Weinstock Date: Thu, 30 Apr 2026 20:18:21 +0000 Subject: [PATCH 1/2] fix: resolve e2e import test concurrency races Fix two independent concurrency issues causing flaky e2e import tests: 1. TOCTOU race in evaluator import (import-evaluator.ts): The beforeConfigWrite hook lists all online eval configs then fetches details for each with Promise.all. If a config is deleted between the list and get calls, the API throws 'Online evaluation configuration not found' and the entire import fails. Fixed by using Promise.allSettled and filtering out disappeared configs. 2. Resource name collisions across parallel CI shards (setup_*.py): Python setup scripts generated resource names using int(time.time()) (second-level precision). Parallel CI shards starting in the same second would collide with ConflictException. The test already passes a unique RESOURCE_SUFFIX env var but scripts ignored it for naming. Added NAME_SUFFIX to common.py that prefers RESOURCE_SUFFIX when set, and updated all setup scripts to use it. --- e2e-tests/fixtures/import/common.py | 3 +++ e2e-tests/fixtures/import/setup_evaluator.py | 4 ++-- e2e-tests/fixtures/import/setup_gateway.py | 4 ++-- e2e-tests/fixtures/import/setup_memory_full.py | 5 +++-- e2e-tests/fixtures/import/setup_runtime_basic.py | 6 +++--- src/cli/commands/import/import-evaluator.ts | 16 ++++++++++++---- 6 files changed, 25 insertions(+), 13 deletions(-) diff --git a/e2e-tests/fixtures/import/common.py b/e2e-tests/fixtures/import/common.py index c786c0c01..3573ed519 100644 --- a/e2e-tests/fixtures/import/common.py +++ b/e2e-tests/fixtures/import/common.py @@ -2,6 +2,7 @@ import json import os import time +import uuid import zipfile import tempfile @@ -9,6 +10,8 @@ REGION = os.environ.get("AWS_REGION") or os.environ.get("AWS_DEFAULT_REGION") or "us-east-1" RESOURCE_SUFFIX = os.environ.get("RESOURCE_SUFFIX", "") +# Unique suffix for resource names — avoids collisions across parallel CI shards. +NAME_SUFFIX = RESOURCE_SUFFIX or uuid.uuid4().hex[:12] SCRIPT_DIR = os.path.dirname(os.path.abspath(__file__)) APP_DIR = os.path.join(SCRIPT_DIR, "app") _resources_name = f"bugbash-resources-{RESOURCE_SUFFIX}.json" if RESOURCE_SUFFIX else "bugbash-resources.json" diff --git a/e2e-tests/fixtures/import/setup_evaluator.py b/e2e-tests/fixtures/import/setup_evaluator.py index d49787d0e..4f46a91ca 100644 --- a/e2e-tests/fixtures/import/setup_evaluator.py +++ b/e2e-tests/fixtures/import/setup_evaluator.py @@ -12,6 +12,7 @@ from common import ( get_control_client, save_resource, tag_resource, wait_for_evaluator, print_import_command, + NAME_SUFFIX, ) DEFAULT_EVALUATOR_MODEL = os.environ.get("DEFAULT_EVALUATOR_MODEL", "us.anthropic.claude-sonnet-4-5-20250929-v1:0") @@ -19,8 +20,7 @@ def main(): client = get_control_client() - ts = int(time.time()) - evaluator_name = f"bugbash_eval_{ts}" + evaluator_name = f"bugbash_eval_{NAME_SUFFIX}" print(f"Creating evaluator: {evaluator_name}") resp = client.create_evaluator( diff --git a/e2e-tests/fixtures/import/setup_gateway.py b/e2e-tests/fixtures/import/setup_gateway.py index e190d0dfc..0e94c82db 100644 --- a/e2e-tests/fixtures/import/setup_gateway.py +++ b/e2e-tests/fixtures/import/setup_gateway.py @@ -16,14 +16,14 @@ from common import ( REGION, get_control_client, ensure_role, save_resource, tag_resource, wait_for_gateway, wait_for_gateway_target, + NAME_SUFFIX, ) def main(): role_arn = ensure_role() client = get_control_client() - ts = int(time.time()) - gateway_name = f"bugbashGw{ts}" + gateway_name = f"bugbashGw{NAME_SUFFIX}" # ------------------------------------------------------------------ # 1. Create gateway diff --git a/e2e-tests/fixtures/import/setup_memory_full.py b/e2e-tests/fixtures/import/setup_memory_full.py index 5df196524..c2f29018e 100644 --- a/e2e-tests/fixtures/import/setup_memory_full.py +++ b/e2e-tests/fixtures/import/setup_memory_full.py @@ -12,18 +12,19 @@ from common import ( ensure_role, get_control_client, wait_for_memory, save_resource, print_import_command, tag_resource, + NAME_SUFFIX, ) def main(): role_arn = ensure_role() client = get_control_client() - memory_name = f"bugbash_memory_{int(time.time())}" + memory_name = f"bugbash_memory_{NAME_SUFFIX}" print(f"Creating memory: {memory_name}") resp = client.create_memory( name=memory_name, - clientToken=f"bugbash-{int(time.time())}", + clientToken=f"bugbash-{NAME_SUFFIX}", eventExpiryDuration=30, memoryExecutionRoleArn=role_arn, memoryStrategies=[ diff --git a/e2e-tests/fixtures/import/setup_runtime_basic.py b/e2e-tests/fixtures/import/setup_runtime_basic.py index 65e1585a1..80d014dd9 100644 --- a/e2e-tests/fixtures/import/setup_runtime_basic.py +++ b/e2e-tests/fixtures/import/setup_runtime_basic.py @@ -11,16 +11,16 @@ from common import ( ensure_role, get_control_client, wait_for_runtime, save_resource, print_import_command, upload_code, + NAME_SUFFIX, ) def main(): role_arn = ensure_role() client = get_control_client() - ts = int(time.time()) - runtime_name = f"bugbash_basic_{ts}" + runtime_name = f"bugbash_basic_{NAME_SUFFIX}" - bucket, s3_key = upload_code(f"bugbash-basic-{ts}") + bucket, s3_key = upload_code(f"bugbash-basic-{NAME_SUFFIX}") print(f"Creating basic runtime: {runtime_name}") resp = client.create_agent_runtime( diff --git a/src/cli/commands/import/import-evaluator.ts b/src/cli/commands/import/import-evaluator.ts index be85829f3..7c6c8b0d2 100644 --- a/src/cli/commands/import/import-evaluator.ts +++ b/src/cli/commands/import/import-evaluator.ts @@ -10,6 +10,7 @@ import { ANSI } from './constants'; import { failResult, parseAndValidateArn } from './import-utils'; import { executeResourceImport } from './resource-import'; import type { ImportResourceOptions, ImportResourceResult, ResourceImportDescriptor } from './types'; +import { ResourceNotFoundException } from '@aws-sdk/client-bedrock-agentcore-control'; import type { Command } from '@commander-js/extra-typings'; /** @@ -92,11 +93,18 @@ const evaluatorDescriptor: ResourceImportDescriptor 0) { - const oecDetails = await Promise.all( - oecSummaries.map(s => - getOnlineEvaluationConfig({ region: target.region, configId: s.onlineEvaluationConfigId }) + // Configs can be deleted between list and get (TOCTOU race). + // Skip ResourceNotFoundException — a deleted config can't be locking our evaluator. + const oecDetails = ( + await Promise.all( + oecSummaries.map(s => + getOnlineEvaluationConfig({ region: target.region, configId: s.onlineEvaluationConfigId }).catch(err => { + if (err instanceof ResourceNotFoundException) return null; + throw err; + }) + ) ) - ); + ).filter(r => r !== null); const referencingOec = oecDetails.find(oec => oec.evaluatorIds?.includes(detail.evaluatorId)); From f2f82b2090a0b5c6f57b492e871106a2da777e57 Mon Sep 17 00:00:00 2001 From: Harrison Weinstock Date: Thu, 30 Apr 2026 20:59:12 +0000 Subject: [PATCH 2/2] chore: remove unused time imports from setup scripts --- e2e-tests/fixtures/import/setup_evaluator.py | 1 - e2e-tests/fixtures/import/setup_gateway.py | 1 - e2e-tests/fixtures/import/setup_memory_full.py | 1 - e2e-tests/fixtures/import/setup_runtime_basic.py | 1 - 4 files changed, 4 deletions(-) diff --git a/e2e-tests/fixtures/import/setup_evaluator.py b/e2e-tests/fixtures/import/setup_evaluator.py index 4f46a91ca..e4573da45 100644 --- a/e2e-tests/fixtures/import/setup_evaluator.py +++ b/e2e-tests/fixtures/import/setup_evaluator.py @@ -8,7 +8,6 @@ import os sys.path.insert(0, os.path.dirname(os.path.abspath(__file__))) -import time from common import ( get_control_client, save_resource, tag_resource, wait_for_evaluator, print_import_command, diff --git a/e2e-tests/fixtures/import/setup_gateway.py b/e2e-tests/fixtures/import/setup_gateway.py index 0e94c82db..a846617aa 100644 --- a/e2e-tests/fixtures/import/setup_gateway.py +++ b/e2e-tests/fixtures/import/setup_gateway.py @@ -12,7 +12,6 @@ import os sys.path.insert(0, os.path.dirname(os.path.abspath(__file__))) -import time from common import ( REGION, get_control_client, ensure_role, save_resource, tag_resource, wait_for_gateway, wait_for_gateway_target, diff --git a/e2e-tests/fixtures/import/setup_memory_full.py b/e2e-tests/fixtures/import/setup_memory_full.py index c2f29018e..277179cfb 100644 --- a/e2e-tests/fixtures/import/setup_memory_full.py +++ b/e2e-tests/fixtures/import/setup_memory_full.py @@ -8,7 +8,6 @@ import os sys.path.insert(0, os.path.dirname(os.path.abspath(__file__))) -import time from common import ( ensure_role, get_control_client, wait_for_memory, save_resource, print_import_command, tag_resource, diff --git a/e2e-tests/fixtures/import/setup_runtime_basic.py b/e2e-tests/fixtures/import/setup_runtime_basic.py index 80d014dd9..d29ddbd1c 100644 --- a/e2e-tests/fixtures/import/setup_runtime_basic.py +++ b/e2e-tests/fixtures/import/setup_runtime_basic.py @@ -7,7 +7,6 @@ import os sys.path.insert(0, os.path.dirname(os.path.abspath(__file__))) -import time from common import ( ensure_role, get_control_client, wait_for_runtime, save_resource, print_import_command, upload_code,