fix(tests): fix integration test filter with dynamic job determination

.github/actions/determine-jobs/action.yml

@@ -0,0 +1,69 @@
+name: 'Determine Jobs to Run'
+description: 'Determines which test jobs should run based on filter input'
+inputs:
+  filter:
+    description: 'The pytest filter string (-k argument)'
+    required: true
+  providers:
+    description: 'Comma-separated list of non-local providers'
+    required: true
+  local_providers:
+    description: 'Comma-separated list of local providers'
+    required: true
+outputs:
+  should_run_integration:
+    description: 'Whether the integration tests job should run'
+    value: ${{ steps.determine.outputs.should_run_integration }}
+  should_run_local:
+    description: 'Whether the local integration tests job should run'
+    value: ${{ steps.determine.outputs.should_run_local }}
+
+runs:
+  using: 'composite'
+  steps:
+    - name: Determine which jobs should run
+      id: determine
+      shell: bash
+      run: |
+        FILTER="${{ inputs.filter }}"
+        PROVIDERS="${{ inputs.providers }}"
+        LOCAL_PROVIDERS="${{ inputs.local_providers }}"
+
+        echo "Filter input: '$FILTER'"
+
+        if [ -z "$FILTER" ]; then
+          echo "No filter provided, running all jobs"
+          echo "should_run_integration=true" >> $GITHUB_OUTPUT
+          echo "should_run_local=true" >> $GITHUB_OUTPUT
+          exit 0
+        fi
+
+        MATCHED_INTEGRATION=false
+        IFS=',' read -ra PROVIDER_ARRAY <<< "$PROVIDERS"
+        for provider in "${PROVIDER_ARRAY[@]}"; do
+          if [[ "$FILTER" == *"$provider"* ]]; then
+            echo "Filter matches integration provider: $provider"
+            MATCHED_INTEGRATION=true
+            break
+          fi
+        done
+
+        MATCHED_LOCAL=false
+        IFS=',' read -ra LOCAL_PROVIDER_ARRAY <<< "$LOCAL_PROVIDERS"
+        for provider in "${LOCAL_PROVIDER_ARRAY[@]}"; do
+          if [[ "$FILTER" == *"$provider"* ]]; then
+            echo "Filter matches local provider: $provider"
+            MATCHED_LOCAL=true
+            break
+          fi
+        done
+
+        if [ "$MATCHED_INTEGRATION" = false ] && [ "$MATCHED_LOCAL" = false ]; then
+          echo "Filter doesn't match any specific provider, treating as test function filter"
+          echo "Running all jobs to let pytest handle the filtering"
+          echo "should_run_integration=true" >> $GITHUB_OUTPUT
+          echo "should_run_local=true" >> $GITHUB_OUTPUT
+        else
+          echo "should_run_integration=$MATCHED_INTEGRATION" >> $GITHUB_OUTPUT
+          echo "should_run_local=$MATCHED_LOCAL" >> $GITHUB_OUTPUT
+        fi

.github/workflows/tests-integration.yaml

@@ -29,9 +29,26 @@ jobs:
         id: set_local_providers
         run: echo "expected_providers=ollama,llamacpp,llamafile" >> $GITHUB_OUTPUT
 
-  run-integration-tests:
+  determine-jobs-to-run:
     needs: expected-providers
-    if: github.event.inputs.filter == '' || contains(needs.expected-providers.outputs.providers, github.event.inputs.filter)
+    runs-on: ubuntu-latest
+    outputs:
+      should_run_integration: ${{ steps.determine.outputs.should_run_integration }}
+      should_run_local: ${{ steps.determine.outputs.should_run_local }}
+    steps:
+      - uses: actions/checkout@v5
+
+      - name: Determine which jobs should run based on filter
+        id: determine
+        uses: ./.github/actions/determine-jobs
+        with:
+          filter: ${{ github.event.inputs.filter }}
+          providers: ${{ needs.expected-providers.outputs.providers }}
+          local_providers: ${{ needs.expected-providers.outputs.local_providers }}
+
+  run-integration-tests:
+    needs: [expected-providers, determine-jobs-to-run]
+    if: needs.determine-jobs-to-run.outputs.should_run_integration == 'true'
     timeout-minutes: 30
     runs-on: ubuntu-latest
 
@@ -50,7 +67,7 @@ jobs:
       - if: github.event.inputs.filter == '' || contains(github.event.inputs.filter, 'huggingface')
         env:
           HF_TOKEN: ${{ secrets.HF_TOKEN }}
-        run: python scripts/wake_up_hf_endpoint.py --retry=5
+        run: python scripts/wake_up_hf_endpoint.py --retry-count=30 --retry-interval=10
 
       - name: Run Integration tests (parallel with xdist)
         env:
@@ -83,9 +100,9 @@ jobs:
           INCLUDE_LOCAL_PROVIDERS: "false"
         run: |
           if [ -n "${{ inputs.filter }}" ]; then
-            pytest tests/integration -v -n auto --cov --cov-report=xml -k "${{ inputs.filter }}"
+            pytest tests/integration -v -n auto --cov --cov-report=xml -k "${{ inputs.filter }}" || [ $? -eq 5 ]
           else
-            pytest tests/integration -v -n auto --cov --cov-report=xml
+            pytest tests/integration -v -n auto --cov --cov-report=xml || [ $? -eq 5 ]
           fi
 
       - name: Upload coverage reports to Codecov
@@ -95,8 +112,8 @@ jobs:
           token: ${{ secrets.CODECOV_TOKEN }}
 
   run-local-integration-tests:
-    needs: expected-providers
-    if: github.event.inputs.filter == '' || contains(needs.expected-providers.outputs.local_providers, github.event.inputs.filter)
+    needs: [expected-providers, determine-jobs-to-run]
+    if: needs.determine-jobs-to-run.outputs.should_run_local == 'true'
     timeout-minutes: 45
     runs-on: ubuntu-latest
 
@@ -181,9 +198,9 @@ jobs:
           INCLUDE_NON_LOCAL_PROVIDERS: "false"
         run: |
           if [ -n "${{ inputs.filter }}" ]; then
-            pytest tests/integration -v --cov --cov-report=xml -k "${{ inputs.filter }}"
+            pytest tests/integration -v --cov --cov-report=xml -k "${{ inputs.filter }}" || [ $? -eq 5 ]
           else
-            pytest tests/integration -v --cov --cov-report=xml
+            pytest tests/integration -v --cov --cov-report=xml || [ $? -eq 5 ]
           fi
 
       - name: Cleanup LlamaFile process

scripts/wake_up_hf_endpoint.py

@@ -8,26 +8,31 @@
 HF_ENDPOINT = "https://y0okp71n85ezo5nr.us-east-1.aws.endpoints.huggingface.cloud/v1/"
 
 
-def wake_up_hf_endpoint(retry: int = 0):
-    while True:
+def wake_up_hf_endpoint(retry_count: int = 0, retry_interval: int = 10):
+    attempt = 0
+    max_attempts = retry_count + 1
+
+    while attempt < max_attempts:
         try:
             completion(
                 model="huggingface:tgi", messages=[{"role": "user", "content": "Are you awake?"}], api_base=HF_ENDPOINT
             )
-            break
         except ClientResponseError as e:
-            if not retry:
-                print(f"Endpoint not ready, giving up...\n{e}")
+            attempt += 1
+            if attempt >= max_attempts:
+                print(f"Endpoint not ready after {attempt} attempts, giving up...\n{e}")
                 return
 
-            print(f"Endpoint not ready, retrying...\n{e}")
-            time.sleep(retry)
-
-    print("Endpoint ready")
+            print(f"Endpoint not ready (attempt {attempt}/{max_attempts}), retrying in {retry_interval}s...\n{e}")
+            time.sleep(retry_interval)
+        else:
+            print("Endpoint ready")
+            return
 
 
 if __name__ == "__main__":
     parser = argparse.ArgumentParser(description="Wake up Hugging Face endpoint")
-    parser.add_argument("--retry", type=int, default=0, help="Retry interval in seconds (0 means no retry)")
+    parser.add_argument("--retry-count", type=int, default=0, help="Number of retry attempts (default: 0, no retries)")
+    parser.add_argument("--retry-interval", type=int, default=10, help="Seconds between retry attempts (default: 10)")
     args = parser.parse_args()
-    wake_up_hf_endpoint(retry=args.retry)
+    wake_up_hf_endpoint(retry_count=args.retry_count, retry_interval=args.retry_interval)