From d796596c9b07f6c12418ed4fabe06b2f4334ceb3 Mon Sep 17 00:00:00 2001 From: Deepthi Rao Date: Wed, 29 Jul 2026 17:58:38 -0400 Subject: [PATCH 1/2] fix(ci): push tutorial images from buildx instead of re-pushing a stale local tag The publish path built a multi-platform image without --push, then a separate 'docker push' step shipped ':latest'. But a multi-arch buildx build can't be loaded into the local Docker image store, so no fresh local ':latest' exists; the preceding validation step's 'docker run' pulls the *existing* (stale) ':latest' from ghcr, and 'docker push' then re-pushes that same stale image. Result: ':latest' has been frozen at a Dec-2025 build, re-pushed unchanged on every run, which is why the 020_state_machine agent's mcp<2 pin never reached the published image and the scale-agentex integration test kept failing. Fix: on the publish path, pass --push to 'agentex agents build' so buildx sends the freshly-built multi-arch image straight to the registry, and drop the now-redundant separate 'docker push' step. Validation still runs and now exercises the freshly-pushed image. Also revert the CLI build cache default back to True (keep the --cache/--no-cache flag): the staleness was never a layer-cache problem, so forcing --no-cache globally only slowed builds without fixing anything. Co-Authored-By: Claude Opus 4.8 (1M context) --- .../build-and-push-tutorial-agent.yml | 18 +++++++++--------- src/agentex/lib/cli/commands/agents.py | 6 ++---- src/agentex/lib/cli/handlers/agent_handlers.py | 8 +++----- 3 files changed, 14 insertions(+), 18 deletions(-) diff --git a/.github/workflows/build-and-push-tutorial-agent.yml b/.github/workflows/build-and-push-tutorial-agent.yml index 1b9cda3eb..94786821f 100644 --- a/.github/workflows/build-and-push-tutorial-agent.yml +++ b/.github/workflows/build-and-push-tutorial-agent.yml @@ -196,8 +196,16 @@ jobs: echo "SKIP_VALIDATION=true" >> $GITHUB_ENV fi - # Always build locally first (without push) + # Build the image. On the publish path, push straight from buildx. + # A multi-platform build cannot be loaded into the local Docker image + # store, so a subsequent `docker push` has no freshly-built image to send + # and ends up re-pushing whatever :latest was pulled during validation — + # i.e. the previous, stale image. Pushing from the build itself is the + # only way the newly-built multi-arch image actually lands on the tag. BUILD_ARGS="--manifest ${{ matrix.agent_path }}/manifest.yaml --registry ${REGISTRY} --tag ${VERSION_TAG} --platforms linux/amd64,linux/arm64 --repository-name ${REPOSITORY_NAME}" + if [ "$SHOULD_PUSH" = "true" ]; then + BUILD_ARGS="$BUILD_ARGS --push" + fi agentex agents build $BUILD_ARGS echo "✅ Successfully built: ${REGISTRY}/${REPOSITORY_NAME}:${VERSION_TAG}" @@ -279,14 +287,6 @@ jobs: echo "✅ All validations passed for: $FULL_IMAGE" - - name: Push Agent Image - if: env.SHOULD_PUSH == 'true' - run: | - FULL_IMAGE="${{ env.FULL_IMAGE }}" - echo "🚀 Pushing validated image: $FULL_IMAGE" - docker push "$FULL_IMAGE" - echo "✅ Successfully pushed: $FULL_IMAGE" - deprecate-agents: name: "Deprecate Removed Agents" runs-on: ubuntu-latest diff --git a/src/agentex/lib/cli/commands/agents.py b/src/agentex/lib/cli/commands/agents.py index 801fda350..b4076d932 100644 --- a/src/agentex/lib/cli/commands/agents.py +++ b/src/agentex/lib/cli/commands/agents.py @@ -128,11 +128,9 @@ def build( help="Docker build argument in the format 'KEY=VALUE' (can be used multiple times)", ), cache: bool = typer.Option( - False, + True, "--cache/--no-cache", - help="Whether to use the build cache. Defaults to off so a stale cached layer " - "can't silently ship source that no longer matches the checkout (notably when " - "republishing a moving tag like ':latest'). Pass --cache to opt back in.", + help="Whether to use the build cache (default on). Pass --no-cache for a clean rebuild.", ), ): """ diff --git a/src/agentex/lib/cli/handlers/agent_handlers.py b/src/agentex/lib/cli/handlers/agent_handlers.py index 322154b92..3c966896e 100644 --- a/src/agentex/lib/cli/handlers/agent_handlers.py +++ b/src/agentex/lib/cli/handlers/agent_handlers.py @@ -39,7 +39,7 @@ def build_agent( secret: str | None = None, tag: str | None = None, build_args: list[str] | None = None, - cache: bool = False, + cache: bool = True, ) -> str: """Build the agent locally and optionally push to registry @@ -50,10 +50,8 @@ def build_agent( secret: Docker build secret in format 'id=secret-id,src=path-to-secret-file' tag: Image tag to use (defaults to 'latest') build_args: List of Docker build arguments in format 'KEY=VALUE' - cache: Whether to use the build cache. Defaults to False (passes --no-cache to - buildx) so a stale cached layer can't silently ship source that no longer - matches the checkout, notably when republishing a moving tag like ':latest'. - Pass True to opt back in for faster local rebuilds. + cache: Whether to use the build cache. Defaults to True. Set to False to pass + --no-cache to buildx for a clean rebuild. Returns: The image URL From 8929503463af07882943fa846295745080d73231 Mon Sep 17 00:00:00 2001 From: Deepthi Rao Date: Wed, 29 Jul 2026 18:35:38 -0400 Subject: [PATCH 2/2] validate before promoting :latest (candidate-tag then promote) Address the validate-after-publish gap: build+push to an immutable candidate tag (the commit SHA), validate that exact pushed image, then promote :latest onto it via 'docker buildx imagetools create' (a registry-side manifest copy, no rebuild). If validation fails, :latest is left on the last known-good build and only the SHA tag is dirty -- no unvalidated image ever lands on :latest. Co-Authored-By: Claude Opus 4.8 (1M context) --- .../build-and-push-tutorial-agent.yml | 37 +++++++++++++------ 1 file changed, 26 insertions(+), 11 deletions(-) diff --git a/.github/workflows/build-and-push-tutorial-agent.yml b/.github/workflows/build-and-push-tutorial-agent.yml index 94786821f..b35154389 100644 --- a/.github/workflows/build-and-push-tutorial-agent.yml +++ b/.github/workflows/build-and-push-tutorial-agent.yml @@ -181,27 +181,30 @@ jobs: AGENT_NAME="${{ steps.image-name.outputs.agent_name }}" REPOSITORY_NAME="${{ github.repository }}/tutorial-agents/${AGENT_NAME}" - # Determine if we should push based on event type + # Determine if we should publish based on event type. + # Publish path: push to an immutable candidate tag (the commit SHA) first, + # validate that exact pushed artifact, then promote :latest onto it. This + # keeps an unvalidated image off :latest — if validation fails, :latest is + # left pointing at the last known-good build, and only the SHA tag is dirty. if [ "${{ github.event_name }}" = "push" ] || [ "${{ inputs.rebuild_all }}" = "true" ]; then SHOULD_PUSH=true - VERSION_TAG="latest" - echo "🚀 Building agent (will push after validation): ${{ matrix.agent_path }}" + PROMOTE_LATEST=true + VERSION_TAG="${{ github.sha }}" + echo "🚀 Building agent (push candidate ${VERSION_TAG}, promote :latest after validation): ${{ matrix.agent_path }}" else SHOULD_PUSH=false + PROMOTE_LATEST=false VERSION_TAG="${{ github.sha }}" echo "🔍 Building agent for validation: ${{ matrix.agent_path }}" - # Set full image name for validation step (local build) - echo "FULL_IMAGE=${REGISTRY}/${REPOSITORY_NAME}:${VERSION_TAG}" >> $GITHUB_ENV # Skip image validation for PRs since Buildx doesn't load multi-platform images locally echo "SKIP_VALIDATION=true" >> $GITHUB_ENV fi - # Build the image. On the publish path, push straight from buildx. - # A multi-platform build cannot be loaded into the local Docker image - # store, so a subsequent `docker push` has no freshly-built image to send - # and ends up re-pushing whatever :latest was pulled during validation — - # i.e. the previous, stale image. Pushing from the build itself is the - # only way the newly-built multi-arch image actually lands on the tag. + # Build the image. On the publish path, push straight from buildx to the + # candidate tag: a multi-platform build cannot be loaded into the local + # Docker store, so it must be pushed by the build itself rather than by a + # later `docker push` (which would have no fresh local image and would + # re-push a stale tag instead). BUILD_ARGS="--manifest ${{ matrix.agent_path }}/manifest.yaml --registry ${REGISTRY} --tag ${VERSION_TAG} --platforms linux/amd64,linux/arm64 --repository-name ${REPOSITORY_NAME}" if [ "$SHOULD_PUSH" = "true" ]; then BUILD_ARGS="$BUILD_ARGS --push" @@ -212,7 +215,9 @@ jobs: # Set environment variables for subsequent steps echo "FULL_IMAGE=${REGISTRY}/${REPOSITORY_NAME}:${VERSION_TAG}" >> $GITHUB_ENV + echo "LATEST_IMAGE=${REGISTRY}/${REPOSITORY_NAME}:latest" >> $GITHUB_ENV echo "SHOULD_PUSH=${SHOULD_PUSH}" >> $GITHUB_ENV + echo "PROMOTE_LATEST=${PROMOTE_LATEST}" >> $GITHUB_ENV - name: Validate agent image if: env.SKIP_VALIDATION != 'true' @@ -287,6 +292,16 @@ jobs: echo "✅ All validations passed for: $FULL_IMAGE" + - name: Promote validated image to :latest + if: env.PROMOTE_LATEST == 'true' + run: | + echo "🏷️ Promoting validated ${{ env.FULL_IMAGE }} -> ${{ env.LATEST_IMAGE }}" + # Registry-side manifest copy: no rebuild, preserves the multi-arch + # manifest list, and only runs after validation passed — so :latest never + # points at an unvalidated image. + docker buildx imagetools create --tag "${{ env.LATEST_IMAGE }}" "${{ env.FULL_IMAGE }}" + echo "✅ Promoted to ${{ env.LATEST_IMAGE }}" + deprecate-agents: name: "Deprecate Removed Agents" runs-on: ubuntu-latest