From e5fa15b763889df4617ef024e671c62df0100fdd Mon Sep 17 00:00:00 2001 From: Alexander Alderman Webb Date: Tue, 25 Nov 2025 17:24:12 +0100 Subject: [PATCH 1/2] ci: Split up Test AI workflow --- .../workflows/test-integrations-agents.yml | 98 +++++++++++++++++ .../test-integrations-ai-workflow.yml | 102 ++++++++++++++++++ .github/workflows/test-integrations-ai.yml | 28 ----- .github/workflows/test-integrations-mcp.yml | 98 +++++++++++++++++ .github/workflows/test-integrations-tasks.yml | 2 +- .../split_tox_gh_actions.py | 20 ++-- .../templates/test_group.jinja | 2 +- 7 files changed, 313 insertions(+), 37 deletions(-) create mode 100644 .github/workflows/test-integrations-agents.yml create mode 100644 .github/workflows/test-integrations-ai-workflow.yml create mode 100644 .github/workflows/test-integrations-mcp.yml diff --git a/.github/workflows/test-integrations-agents.yml b/.github/workflows/test-integrations-agents.yml new file mode 100644 index 0000000000..5a7f7a057b --- /dev/null +++ b/.github/workflows/test-integrations-agents.yml @@ -0,0 +1,98 @@ +# Do not edit this YAML file. This file is generated automatically by executing +# python scripts/split_tox_gh_actions/split_tox_gh_actions.py +# The template responsible for it is in +# scripts/split_tox_gh_actions/templates/base.jinja +name: Test Agents +on: + push: + branches: + - master + - release/** + - major/** + pull_request: +# Cancel in progress workflows on pull_requests. +# https://docs.github.com/en/actions/using-jobs/using-concurrency#example-using-a-fallback-value +concurrency: + group: ${{ github.workflow }}-${{ github.head_ref || github.run_id }} + cancel-in-progress: true +permissions: + contents: read +env: + BUILD_CACHE_KEY: ${{ github.sha }} + CACHED_BUILD_PATHS: | + ${{ github.workspace }}/dist-serverless +jobs: + test-agents: + name: Agents + timeout-minutes: 30 + runs-on: ${{ matrix.os }} + strategy: + fail-fast: false + matrix: + python-version: ["3.10","3.11","3.12","3.13","3.14","3.14t"] + # python3.6 reached EOL and is no longer being supported on + # new versions of hosted runners on Github Actions + # ubuntu-20.04 is the last version that supported python3.6 + # see https://github.com/actions/setup-python/issues/544#issuecomment-1332535877 + os: [ubuntu-22.04] + # Use Docker container only for Python 3.6 + container: ${{ matrix.python-version == '3.6' && 'python:3.6' || null }} + steps: + - uses: actions/checkout@v6.0.0 + - uses: actions/setup-python@v6 + if: ${{ matrix.python-version != '3.6' }} + with: + python-version: ${{ matrix.python-version }} + allow-prereleases: true + - name: Setup Test Env + run: | + pip install "coverage[toml]" tox + - name: Erase coverage + run: | + coverage erase + - name: Test openai_agents + run: | + set -x # print commands that are executed + ./scripts/runtox.sh "py${{ matrix.python-version }}-openai_agents" + - name: Test pydantic_ai + run: | + set -x # print commands that are executed + ./scripts/runtox.sh "py${{ matrix.python-version }}-pydantic_ai" + - name: Generate coverage XML (Python 3.6) + if: ${{ !cancelled() && matrix.python-version == '3.6' }} + run: | + export COVERAGE_RCFILE=.coveragerc36 + coverage combine .coverage-sentry-* + coverage xml --ignore-errors + - name: Generate coverage XML + if: ${{ !cancelled() && matrix.python-version != '3.6' }} + run: | + coverage combine .coverage-sentry-* + coverage xml + - name: Upload coverage to Codecov + if: ${{ !cancelled() }} + uses: codecov/codecov-action@v5.5.1 + with: + token: ${{ secrets.CODECOV_TOKEN }} + files: coverage.xml + # make sure no plugins alter our coverage reports + plugins: noop + verbose: true + - name: Upload test results to Codecov + if: ${{ !cancelled() }} + uses: codecov/test-results-action@v1 + with: + token: ${{ secrets.CODECOV_TOKEN }} + files: .junitxml + verbose: true + check_required_tests: + name: All Agents tests passed + needs: test-agents + # Always run this, even if a dependent job failed + if: always() + runs-on: ubuntu-22.04 + steps: + - name: Check for failures + if: needs.test-agents.result != 'success' + run: | + echo "One of the dependent jobs has failed. You may need to re-run it." && exit 1 diff --git a/.github/workflows/test-integrations-ai-workflow.yml b/.github/workflows/test-integrations-ai-workflow.yml new file mode 100644 index 0000000000..cc7aaef9ad --- /dev/null +++ b/.github/workflows/test-integrations-ai-workflow.yml @@ -0,0 +1,102 @@ +# Do not edit this YAML file. This file is generated automatically by executing +# python scripts/split_tox_gh_actions/split_tox_gh_actions.py +# The template responsible for it is in +# scripts/split_tox_gh_actions/templates/base.jinja +name: Test AI Workflow +on: + push: + branches: + - master + - release/** + - major/** + pull_request: +# Cancel in progress workflows on pull_requests. +# https://docs.github.com/en/actions/using-jobs/using-concurrency#example-using-a-fallback-value +concurrency: + group: ${{ github.workflow }}-${{ github.head_ref || github.run_id }} + cancel-in-progress: true +permissions: + contents: read +env: + BUILD_CACHE_KEY: ${{ github.sha }} + CACHED_BUILD_PATHS: | + ${{ github.workspace }}/dist-serverless +jobs: + test-ai_workflow: + name: AI Workflow + timeout-minutes: 30 + runs-on: ${{ matrix.os }} + strategy: + fail-fast: false + matrix: + python-version: ["3.9","3.10","3.11","3.12","3.13","3.14"] + # python3.6 reached EOL and is no longer being supported on + # new versions of hosted runners on Github Actions + # ubuntu-20.04 is the last version that supported python3.6 + # see https://github.com/actions/setup-python/issues/544#issuecomment-1332535877 + os: [ubuntu-22.04] + # Use Docker container only for Python 3.6 + container: ${{ matrix.python-version == '3.6' && 'python:3.6' || null }} + steps: + - uses: actions/checkout@v6.0.0 + - uses: actions/setup-python@v6 + if: ${{ matrix.python-version != '3.6' }} + with: + python-version: ${{ matrix.python-version }} + allow-prereleases: true + - name: Setup Test Env + run: | + pip install "coverage[toml]" tox + - name: Erase coverage + run: | + coverage erase + - name: Test langchain-base + run: | + set -x # print commands that are executed + ./scripts/runtox.sh "py${{ matrix.python-version }}-langchain-base" + - name: Test langchain-notiktoken + run: | + set -x # print commands that are executed + ./scripts/runtox.sh "py${{ matrix.python-version }}-langchain-notiktoken" + - name: Test langgraph + run: | + set -x # print commands that are executed + ./scripts/runtox.sh "py${{ matrix.python-version }}-langgraph" + - name: Generate coverage XML (Python 3.6) + if: ${{ !cancelled() && matrix.python-version == '3.6' }} + run: | + export COVERAGE_RCFILE=.coveragerc36 + coverage combine .coverage-sentry-* + coverage xml --ignore-errors + - name: Generate coverage XML + if: ${{ !cancelled() && matrix.python-version != '3.6' }} + run: | + coverage combine .coverage-sentry-* + coverage xml + - name: Upload coverage to Codecov + if: ${{ !cancelled() }} + uses: codecov/codecov-action@v5.5.1 + with: + token: ${{ secrets.CODECOV_TOKEN }} + files: coverage.xml + # make sure no plugins alter our coverage reports + plugins: noop + verbose: true + - name: Upload test results to Codecov + if: ${{ !cancelled() }} + uses: codecov/test-results-action@v1 + with: + token: ${{ secrets.CODECOV_TOKEN }} + files: .junitxml + verbose: true + check_required_tests: + name: All AI Workflow tests passed + needs: test-ai_workflow + # Always run this, even if a dependent job failed + if: always() + runs-on: ubuntu-22.04 + steps: + - name: Check for failures + if: needs.test-ai_workflow.result != 'success' + run: | + echo "One of the dependent jobs has failed. You may need to re-run it." && exit 1 diff --git a/.github/workflows/test-integrations-ai.yml b/.github/workflows/test-integrations-ai.yml index de25fb3e84..84884397a0 100644 --- a/.github/workflows/test-integrations-ai.yml +++ b/.github/workflows/test-integrations-ai.yml @@ -66,30 +66,10 @@ jobs: run: | set -x # print commands that are executed ./scripts/runtox.sh "py${{ matrix.python-version }}-huggingface_hub" - - name: Test langchain-base - run: | - set -x # print commands that are executed - ./scripts/runtox.sh "py${{ matrix.python-version }}-langchain-base" - - name: Test langchain-notiktoken - run: | - set -x # print commands that are executed - ./scripts/runtox.sh "py${{ matrix.python-version }}-langchain-notiktoken" - - name: Test langgraph - run: | - set -x # print commands that are executed - ./scripts/runtox.sh "py${{ matrix.python-version }}-langgraph" - name: Test litellm run: | set -x # print commands that are executed ./scripts/runtox.sh "py${{ matrix.python-version }}-litellm" - - name: Test mcp - run: | - set -x # print commands that are executed - ./scripts/runtox.sh "py${{ matrix.python-version }}-mcp" - - name: Test fastmcp - run: | - set -x # print commands that are executed - ./scripts/runtox.sh "py${{ matrix.python-version }}-fastmcp" - name: Test openai-base run: | set -x # print commands that are executed @@ -98,14 +78,6 @@ jobs: run: | set -x # print commands that are executed ./scripts/runtox.sh "py${{ matrix.python-version }}-openai-notiktoken" - - name: Test openai_agents - run: | - set -x # print commands that are executed - ./scripts/runtox.sh "py${{ matrix.python-version }}-openai_agents" - - name: Test pydantic_ai - run: | - set -x # print commands that are executed - ./scripts/runtox.sh "py${{ matrix.python-version }}-pydantic_ai" - name: Generate coverage XML (Python 3.6) if: ${{ !cancelled() && matrix.python-version == '3.6' }} run: | diff --git a/.github/workflows/test-integrations-mcp.yml b/.github/workflows/test-integrations-mcp.yml new file mode 100644 index 0000000000..3758beda17 --- /dev/null +++ b/.github/workflows/test-integrations-mcp.yml @@ -0,0 +1,98 @@ +# Do not edit this YAML file. This file is generated automatically by executing +# python scripts/split_tox_gh_actions/split_tox_gh_actions.py +# The template responsible for it is in +# scripts/split_tox_gh_actions/templates/base.jinja +name: Test MCP +on: + push: + branches: + - master + - release/** + - major/** + pull_request: +# Cancel in progress workflows on pull_requests. +# https://docs.github.com/en/actions/using-jobs/using-concurrency#example-using-a-fallback-value +concurrency: + group: ${{ github.workflow }}-${{ github.head_ref || github.run_id }} + cancel-in-progress: true +permissions: + contents: read +env: + BUILD_CACHE_KEY: ${{ github.sha }} + CACHED_BUILD_PATHS: | + ${{ github.workspace }}/dist-serverless +jobs: + test-mcp: + name: MCP + timeout-minutes: 30 + runs-on: ${{ matrix.os }} + strategy: + fail-fast: false + matrix: + python-version: ["3.10","3.12","3.13","3.14","3.14t"] + # python3.6 reached EOL and is no longer being supported on + # new versions of hosted runners on Github Actions + # ubuntu-20.04 is the last version that supported python3.6 + # see https://github.com/actions/setup-python/issues/544#issuecomment-1332535877 + os: [ubuntu-22.04] + # Use Docker container only for Python 3.6 + container: ${{ matrix.python-version == '3.6' && 'python:3.6' || null }} + steps: + - uses: actions/checkout@v6.0.0 + - uses: actions/setup-python@v6 + if: ${{ matrix.python-version != '3.6' }} + with: + python-version: ${{ matrix.python-version }} + allow-prereleases: true + - name: Setup Test Env + run: | + pip install "coverage[toml]" tox + - name: Erase coverage + run: | + coverage erase + - name: Test mcp + run: | + set -x # print commands that are executed + ./scripts/runtox.sh "py${{ matrix.python-version }}-mcp" + - name: Test fastmcp + run: | + set -x # print commands that are executed + ./scripts/runtox.sh "py${{ matrix.python-version }}-fastmcp" + - name: Generate coverage XML (Python 3.6) + if: ${{ !cancelled() && matrix.python-version == '3.6' }} + run: | + export COVERAGE_RCFILE=.coveragerc36 + coverage combine .coverage-sentry-* + coverage xml --ignore-errors + - name: Generate coverage XML + if: ${{ !cancelled() && matrix.python-version != '3.6' }} + run: | + coverage combine .coverage-sentry-* + coverage xml + - name: Upload coverage to Codecov + if: ${{ !cancelled() }} + uses: codecov/codecov-action@v5.5.1 + with: + token: ${{ secrets.CODECOV_TOKEN }} + files: coverage.xml + # make sure no plugins alter our coverage reports + plugins: noop + verbose: true + - name: Upload test results to Codecov + if: ${{ !cancelled() }} + uses: codecov/test-results-action@v1 + with: + token: ${{ secrets.CODECOV_TOKEN }} + files: .junitxml + verbose: true + check_required_tests: + name: All MCP tests passed + needs: test-mcp + # Always run this, even if a dependent job failed + if: always() + runs-on: ubuntu-22.04 + steps: + - name: Check for failures + if: needs.test-mcp.result != 'success' + run: | + echo "One of the dependent jobs has failed. You may need to re-run it." && exit 1 diff --git a/.github/workflows/test-integrations-tasks.yml b/.github/workflows/test-integrations-tasks.yml index ff565e27e5..b9111b3a42 100644 --- a/.github/workflows/test-integrations-tasks.yml +++ b/.github/workflows/test-integrations-tasks.yml @@ -45,7 +45,7 @@ jobs: python-version: ${{ matrix.python-version }} allow-prereleases: true - name: Start Redis - uses: supercharge/redis-github-action@1.8.1 + uses: supercharge/redis-github-action@1.8.0 - name: Install Java uses: actions/setup-java@v5 with: diff --git a/scripts/split_tox_gh_actions/split_tox_gh_actions.py b/scripts/split_tox_gh_actions/split_tox_gh_actions.py index 291452dea0..a8a2ab26af 100755 --- a/scripts/split_tox_gh_actions/split_tox_gh_actions.py +++ b/scripts/split_tox_gh_actions/split_tox_gh_actions.py @@ -69,21 +69,27 @@ "Common": [ "common", ], + "MCP": [ + "mcp", + "fastmcp", + ], + "Agents": [ + "openai_agents", + "pydantic_ai", + ], + "AI Workflow": [ + "langchain-base", + "langchain-notiktoken", + "langgraph", + ], "AI": [ "anthropic", "cohere", "google_genai", "huggingface_hub", - "langchain-base", - "langchain-notiktoken", - "langgraph", "litellm", - "mcp", - "fastmcp", "openai-base", "openai-notiktoken", - "openai_agents", - "pydantic_ai", ], "Cloud": [ "aws_lambda", diff --git a/scripts/split_tox_gh_actions/templates/test_group.jinja b/scripts/split_tox_gh_actions/templates/test_group.jinja index 59369265b3..0a718e6d24 100644 --- a/scripts/split_tox_gh_actions/templates/test_group.jinja +++ b/scripts/split_tox_gh_actions/templates/test_group.jinja @@ -42,7 +42,7 @@ # Use Docker container only for Python 3.6 {% raw %}container: ${{ matrix.python-version == '3.6' && 'python:3.6' || null }}{% endraw %} steps: - - uses: actions/checkout@v5.0.0 + - uses: actions/checkout@v6.0.0 - uses: actions/setup-python@v6 {% raw %}if: ${{ matrix.python-version != '3.6' }}{% endraw %} with: From c66c3e52b69914337a1e8502d94ede012c16591b Mon Sep 17 00:00:00 2001 From: Alexander Alderman Webb Date: Tue, 25 Nov 2025 18:06:39 +0100 Subject: [PATCH 2/2] update supercharge/redis-github-action in test_group.jinja --- .github/workflows/test-integrations-tasks.yml | 2 +- scripts/split_tox_gh_actions/templates/test_group.jinja | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/.github/workflows/test-integrations-tasks.yml b/.github/workflows/test-integrations-tasks.yml index b9111b3a42..ff565e27e5 100644 --- a/.github/workflows/test-integrations-tasks.yml +++ b/.github/workflows/test-integrations-tasks.yml @@ -45,7 +45,7 @@ jobs: python-version: ${{ matrix.python-version }} allow-prereleases: true - name: Start Redis - uses: supercharge/redis-github-action@1.8.0 + uses: supercharge/redis-github-action@1.8.1 - name: Install Java uses: actions/setup-java@v5 with: diff --git a/scripts/split_tox_gh_actions/templates/test_group.jinja b/scripts/split_tox_gh_actions/templates/test_group.jinja index 0a718e6d24..10ef7cfec3 100644 --- a/scripts/split_tox_gh_actions/templates/test_group.jinja +++ b/scripts/split_tox_gh_actions/templates/test_group.jinja @@ -56,7 +56,7 @@ {% if needs_redis %} - name: Start Redis - uses: supercharge/redis-github-action@1.8.0 + uses: supercharge/redis-github-action@1.8.1 {% endif %} {% if needs_java %}