From 0264153471dca4e01658cf3a6ae6c7610499b493 Mon Sep 17 00:00:00 2001 From: Nathan Brake Date: Thu, 30 Oct 2025 10:29:28 -0400 Subject: [PATCH 1/6] fix(tests): fix integration test filter with dynamic job determination --- .github/workflows/tests-integration.yaml | 61 ++++++++++++++++++++++-- 1 file changed, 57 insertions(+), 4 deletions(-) diff --git a/.github/workflows/tests-integration.yaml b/.github/workflows/tests-integration.yaml index 1d7f3295..d1486c9b 100644 --- a/.github/workflows/tests-integration.yaml +++ b/.github/workflows/tests-integration.yaml @@ -29,9 +29,62 @@ jobs: id: set_local_providers run: echo "expected_providers=ollama,llamacpp,llamafile" >> $GITHUB_OUTPUT - run-integration-tests: + determine-jobs-to-run: needs: expected-providers - if: github.event.inputs.filter == '' || contains(needs.expected-providers.outputs.providers, github.event.inputs.filter) + runs-on: ubuntu-latest + outputs: + should_run_integration: ${{ steps.check_filter.outputs.should_run_integration }} + should_run_local: ${{ steps.check_filter.outputs.should_run_local }} + steps: + - name: Determine which jobs should run based on filter + id: check_filter + run: | + FILTER="${{ github.event.inputs.filter }}" + PROVIDERS="${{ needs.expected-providers.outputs.providers }}" + LOCAL_PROVIDERS="${{ needs.expected-providers.outputs.local_providers }}" + + echo "Filter input: '$FILTER'" + + if [ -z "$FILTER" ]; then + echo "No filter provided, running all jobs" + echo "should_run_integration=true" >> $GITHUB_OUTPUT + echo "should_run_local=true" >> $GITHUB_OUTPUT + exit 0 + fi + + MATCHED_INTEGRATION=false + IFS=',' read -ra PROVIDER_ARRAY <<< "$PROVIDERS" + for provider in "${PROVIDER_ARRAY[@]}"; do + if echo "$FILTER" | grep -qE "(^|[^a-zA-Z0-9_])${provider}([^a-zA-Z0-9_]|$)"; then + echo "Filter matches integration provider: $provider" + MATCHED_INTEGRATION=true + break + fi + done + + MATCHED_LOCAL=false + IFS=',' read -ra LOCAL_PROVIDER_ARRAY <<< "$LOCAL_PROVIDERS" + for provider in "${LOCAL_PROVIDER_ARRAY[@]}"; do + if echo "$FILTER" | grep -qE "(^|[^a-zA-Z0-9_])${provider}([^a-zA-Z0-9_]|$)"; then + echo "Filter matches local provider: $provider" + MATCHED_LOCAL=true + break + fi + done + + if [ "$MATCHED_INTEGRATION" = false ] && [ "$MATCHED_LOCAL" = false ]; then + echo "Filter doesn't match any specific provider, treating as test function filter" + echo "Running all jobs to let pytest handle the filtering" + echo "should_run_integration=true" >> $GITHUB_OUTPUT + echo "should_run_local=true" >> $GITHUB_OUTPUT + else + echo "should_run_integration=$MATCHED_INTEGRATION" >> $GITHUB_OUTPUT + echo "should_run_local=$MATCHED_LOCAL" >> $GITHUB_OUTPUT + fi + + run-integration-tests: + needs: [expected-providers, determine-jobs-to-run] + if: needs.determine-jobs-to-run.outputs.should_run_integration == 'true' timeout-minutes: 30 runs-on: ubuntu-latest @@ -96,8 +149,8 @@ jobs: token: ${{ secrets.CODECOV_TOKEN }} run-local-integration-tests: - needs: expected-providers - if: github.event.inputs.filter == '' || contains(needs.expected-providers.outputs.local_providers, github.event.inputs.filter) + needs: [expected-providers, determine-jobs-to-run] + if: needs.determine-jobs-to-run.outputs.should_run_local == 'true' timeout-minutes: 45 runs-on: ubuntu-latest From 53f5e37baaa58ed36da7ae7ee6fe3e32af84fbc7 Mon Sep 17 00:00:00 2001 From: Nathan Brake Date: Thu, 30 Oct 2025 10:35:17 -0400 Subject: [PATCH 2/6] reorg --- .github/actions/determine-jobs/action.yml | 69 +++++++++++++++++++++++ .github/workflows/tests-integration.yaml | 56 ++++-------------- 2 files changed, 79 insertions(+), 46 deletions(-) create mode 100644 .github/actions/determine-jobs/action.yml diff --git a/.github/actions/determine-jobs/action.yml b/.github/actions/determine-jobs/action.yml new file mode 100644 index 00000000..13b31546 --- /dev/null +++ b/.github/actions/determine-jobs/action.yml @@ -0,0 +1,69 @@ +name: 'Determine Jobs to Run' +description: 'Determines which test jobs should run based on filter input' +inputs: + filter: + description: 'The pytest filter string (-k argument)' + required: true + providers: + description: 'Comma-separated list of non-local providers' + required: true + local_providers: + description: 'Comma-separated list of local providers' + required: true +outputs: + should_run_integration: + description: 'Whether the integration tests job should run' + value: ${{ steps.determine.outputs.should_run_integration }} + should_run_local: + description: 'Whether the local integration tests job should run' + value: ${{ steps.determine.outputs.should_run_local }} + +runs: + using: 'composite' + steps: + - name: Determine which jobs should run + id: determine + shell: bash + run: | + FILTER="${{ inputs.filter }}" + PROVIDERS="${{ inputs.providers }}" + LOCAL_PROVIDERS="${{ inputs.local_providers }}" + + echo "Filter input: '$FILTER'" + + if [ -z "$FILTER" ]; then + echo "No filter provided, running all jobs" + echo "should_run_integration=true" >> $GITHUB_OUTPUT + echo "should_run_local=true" >> $GITHUB_OUTPUT + exit 0 + fi + + MATCHED_INTEGRATION=false + IFS=',' read -ra PROVIDER_ARRAY <<< "$PROVIDERS" + for provider in "${PROVIDER_ARRAY[@]}"; do + if [[ "$FILTER" == *"$provider"* ]]; then + echo "Filter matches integration provider: $provider" + MATCHED_INTEGRATION=true + break + fi + done + + MATCHED_LOCAL=false + IFS=',' read -ra LOCAL_PROVIDER_ARRAY <<< "$LOCAL_PROVIDERS" + for provider in "${LOCAL_PROVIDER_ARRAY[@]}"; do + if [[ "$FILTER" == *"$provider"* ]]; then + echo "Filter matches local provider: $provider" + MATCHED_LOCAL=true + break + fi + done + + if [ "$MATCHED_INTEGRATION" = false ] && [ "$MATCHED_LOCAL" = false ]; then + echo "Filter doesn't match any specific provider, treating as test function filter" + echo "Running all jobs to let pytest handle the filtering" + echo "should_run_integration=true" >> $GITHUB_OUTPUT + echo "should_run_local=true" >> $GITHUB_OUTPUT + else + echo "should_run_integration=$MATCHED_INTEGRATION" >> $GITHUB_OUTPUT + echo "should_run_local=$MATCHED_LOCAL" >> $GITHUB_OUTPUT + fi diff --git a/.github/workflows/tests-integration.yaml b/.github/workflows/tests-integration.yaml index d1486c9b..280d754c 100644 --- a/.github/workflows/tests-integration.yaml +++ b/.github/workflows/tests-integration.yaml @@ -33,54 +33,18 @@ jobs: needs: expected-providers runs-on: ubuntu-latest outputs: - should_run_integration: ${{ steps.check_filter.outputs.should_run_integration }} - should_run_local: ${{ steps.check_filter.outputs.should_run_local }} + should_run_integration: ${{ steps.determine.outputs.should_run_integration }} + should_run_local: ${{ steps.determine.outputs.should_run_local }} steps: - - name: Determine which jobs should run based on filter - id: check_filter - run: | - FILTER="${{ github.event.inputs.filter }}" - PROVIDERS="${{ needs.expected-providers.outputs.providers }}" - LOCAL_PROVIDERS="${{ needs.expected-providers.outputs.local_providers }}" - - echo "Filter input: '$FILTER'" - - if [ -z "$FILTER" ]; then - echo "No filter provided, running all jobs" - echo "should_run_integration=true" >> $GITHUB_OUTPUT - echo "should_run_local=true" >> $GITHUB_OUTPUT - exit 0 - fi + - uses: actions/checkout@v5 - MATCHED_INTEGRATION=false - IFS=',' read -ra PROVIDER_ARRAY <<< "$PROVIDERS" - for provider in "${PROVIDER_ARRAY[@]}"; do - if echo "$FILTER" | grep -qE "(^|[^a-zA-Z0-9_])${provider}([^a-zA-Z0-9_]|$)"; then - echo "Filter matches integration provider: $provider" - MATCHED_INTEGRATION=true - break - fi - done - - MATCHED_LOCAL=false - IFS=',' read -ra LOCAL_PROVIDER_ARRAY <<< "$LOCAL_PROVIDERS" - for provider in "${LOCAL_PROVIDER_ARRAY[@]}"; do - if echo "$FILTER" | grep -qE "(^|[^a-zA-Z0-9_])${provider}([^a-zA-Z0-9_]|$)"; then - echo "Filter matches local provider: $provider" - MATCHED_LOCAL=true - break - fi - done - - if [ "$MATCHED_INTEGRATION" = false ] && [ "$MATCHED_LOCAL" = false ]; then - echo "Filter doesn't match any specific provider, treating as test function filter" - echo "Running all jobs to let pytest handle the filtering" - echo "should_run_integration=true" >> $GITHUB_OUTPUT - echo "should_run_local=true" >> $GITHUB_OUTPUT - else - echo "should_run_integration=$MATCHED_INTEGRATION" >> $GITHUB_OUTPUT - echo "should_run_local=$MATCHED_LOCAL" >> $GITHUB_OUTPUT - fi + - name: Determine which jobs should run based on filter + id: determine + uses: ./.github/actions/determine-jobs + with: + filter: ${{ github.event.inputs.filter }} + providers: ${{ needs.expected-providers.outputs.providers }} + local_providers: ${{ needs.expected-providers.outputs.local_providers }} run-integration-tests: needs: [expected-providers, determine-jobs-to-run] From d1c5bfdedf977b1d86f6ec7e8dd0069be4fe66ab Mon Sep 17 00:00:00 2001 From: Nathan Brake Date: Thu, 30 Oct 2025 10:43:37 -0400 Subject: [PATCH 3/6] clean exit --- .github/workflows/tests-integration.yaml | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/.github/workflows/tests-integration.yaml b/.github/workflows/tests-integration.yaml index 280d754c..ed258415 100644 --- a/.github/workflows/tests-integration.yaml +++ b/.github/workflows/tests-integration.yaml @@ -101,9 +101,9 @@ jobs: INCLUDE_LOCAL_PROVIDERS: "false" run: | if [ -n "${{ inputs.filter }}" ]; then - pytest tests/integration -v -n auto --cov --cov-report=xml -k "${{ inputs.filter }}" + pytest tests/integration -v -n auto --cov --cov-report=xml --exit-zero -k "${{ inputs.filter }}" else - pytest tests/integration -v -n auto --cov --cov-report=xml + pytest tests/integration -v -n auto --cov --cov-report=xml --exit-zero fi - name: Upload coverage reports to Codecov @@ -199,9 +199,9 @@ jobs: INCLUDE_NON_LOCAL_PROVIDERS: "false" run: | if [ -n "${{ inputs.filter }}" ]; then - pytest tests/integration -v --cov --cov-report=xml -k "${{ inputs.filter }}" + pytest tests/integration -v --cov --cov-report=xml --exit-zero -k "${{ inputs.filter }}" else - pytest tests/integration -v --cov --cov-report=xml + pytest tests/integration -v --cov --cov-report=xml --exit-zero fi - name: Cleanup LlamaFile process From 04e92c5df9ff1fcc8a303f706cad431acf413396 Mon Sep 17 00:00:00 2001 From: Nathan Brake Date: Thu, 30 Oct 2025 10:57:57 -0400 Subject: [PATCH 4/6] allow no tests --- .github/workflows/tests-integration.yaml | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/.github/workflows/tests-integration.yaml b/.github/workflows/tests-integration.yaml index ed258415..86a5747e 100644 --- a/.github/workflows/tests-integration.yaml +++ b/.github/workflows/tests-integration.yaml @@ -101,9 +101,9 @@ jobs: INCLUDE_LOCAL_PROVIDERS: "false" run: | if [ -n "${{ inputs.filter }}" ]; then - pytest tests/integration -v -n auto --cov --cov-report=xml --exit-zero -k "${{ inputs.filter }}" + pytest tests/integration -v -n auto --cov --cov-report=xml -k "${{ inputs.filter }}" || [ $? -eq 5 ] else - pytest tests/integration -v -n auto --cov --cov-report=xml --exit-zero + pytest tests/integration -v -n auto --cov --cov-report=xml || [ $? -eq 5 ] fi - name: Upload coverage reports to Codecov @@ -199,9 +199,9 @@ jobs: INCLUDE_NON_LOCAL_PROVIDERS: "false" run: | if [ -n "${{ inputs.filter }}" ]; then - pytest tests/integration -v --cov --cov-report=xml --exit-zero -k "${{ inputs.filter }}" + pytest tests/integration -v --cov --cov-report=xml -k "${{ inputs.filter }}" || [ $? -eq 5 ] else - pytest tests/integration -v --cov --cov-report=xml --exit-zero + pytest tests/integration -v --cov --cov-report=xml || [ $? -eq 5 ] fi - name: Cleanup LlamaFile process From 06475068ca28e8cc2ec4d1531f53003a785c71a4 Mon Sep 17 00:00:00 2001 From: Nathan Brake Date: Thu, 30 Oct 2025 11:02:58 -0400 Subject: [PATCH 5/6] more retries for slower init --- .github/workflows/tests-integration.yaml | 2 +- scripts/wake_up_hf_endpoint.py | 26 ++++++++++++++---------- 2 files changed, 16 insertions(+), 12 deletions(-) diff --git a/.github/workflows/tests-integration.yaml b/.github/workflows/tests-integration.yaml index 86a5747e..7389c00b 100644 --- a/.github/workflows/tests-integration.yaml +++ b/.github/workflows/tests-integration.yaml @@ -67,7 +67,7 @@ jobs: - if: github.event.inputs.filter == '' || contains(github.event.inputs.filter, 'huggingface') env: HF_TOKEN: ${{ secrets.HF_TOKEN }} - run: python scripts/wake_up_hf_endpoint.py --retry=5 + run: python scripts/wake_up_hf_endpoint.py --retry-count=30 --retry-interval=10 - name: Run Integration tests (parallel with xdist) env: diff --git a/scripts/wake_up_hf_endpoint.py b/scripts/wake_up_hf_endpoint.py index a2701a18..9f36d0cf 100644 --- a/scripts/wake_up_hf_endpoint.py +++ b/scripts/wake_up_hf_endpoint.py @@ -8,26 +8,30 @@ HF_ENDPOINT = "https://y0okp71n85ezo5nr.us-east-1.aws.endpoints.huggingface.cloud/v1/" -def wake_up_hf_endpoint(retry: int = 0): - while True: +def wake_up_hf_endpoint(retry_count: int = 0, retry_interval: int = 10): + attempt = 0 + max_attempts = retry_count + 1 + + while attempt < max_attempts: try: completion( model="huggingface:tgi", messages=[{"role": "user", "content": "Are you awake?"}], api_base=HF_ENDPOINT ) - break + print("Endpoint ready") + return except ClientResponseError as e: - if not retry: - print(f"Endpoint not ready, giving up...\n{e}") + attempt += 1 + if attempt >= max_attempts: + print(f"Endpoint not ready after {attempt} attempts, giving up...\n{e}") return - print(f"Endpoint not ready, retrying...\n{e}") - time.sleep(retry) - - print("Endpoint ready") + print(f"Endpoint not ready (attempt {attempt}/{max_attempts}), retrying in {retry_interval}s...\n{e}") + time.sleep(retry_interval) if __name__ == "__main__": parser = argparse.ArgumentParser(description="Wake up Hugging Face endpoint") - parser.add_argument("--retry", type=int, default=0, help="Retry interval in seconds (0 means no retry)") + parser.add_argument("--retry-count", type=int, default=0, help="Number of retry attempts (default: 0, no retries)") + parser.add_argument("--retry-interval", type=int, default=10, help="Seconds between retry attempts (default: 10)") args = parser.parse_args() - wake_up_hf_endpoint(retry=args.retry) + wake_up_hf_endpoint(retry_count=args.retry_count, retry_interval=args.retry_interval) From 041226e659fe71a1bd9e3d47f59cdbe4057bf5a5 Mon Sep 17 00:00:00 2001 From: Nathan Brake Date: Thu, 30 Oct 2025 11:06:20 -0400 Subject: [PATCH 6/6] lint --- scripts/wake_up_hf_endpoint.py | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) diff --git a/scripts/wake_up_hf_endpoint.py b/scripts/wake_up_hf_endpoint.py index 9f36d0cf..97ec255f 100644 --- a/scripts/wake_up_hf_endpoint.py +++ b/scripts/wake_up_hf_endpoint.py @@ -17,8 +17,6 @@ def wake_up_hf_endpoint(retry_count: int = 0, retry_interval: int = 10): completion( model="huggingface:tgi", messages=[{"role": "user", "content": "Are you awake?"}], api_base=HF_ENDPOINT ) - print("Endpoint ready") - return except ClientResponseError as e: attempt += 1 if attempt >= max_attempts: @@ -27,6 +25,9 @@ def wake_up_hf_endpoint(retry_count: int = 0, retry_interval: int = 10): print(f"Endpoint not ready (attempt {attempt}/{max_attempts}), retrying in {retry_interval}s...\n{e}") time.sleep(retry_interval) + else: + print("Endpoint ready") + return if __name__ == "__main__":