diff --git a/.github/actions/determine-jobs/action.yml b/.github/actions/determine-jobs/action.yml new file mode 100644 index 00000000..13b31546 --- /dev/null +++ b/.github/actions/determine-jobs/action.yml @@ -0,0 +1,69 @@ +name: 'Determine Jobs to Run' +description: 'Determines which test jobs should run based on filter input' +inputs: + filter: + description: 'The pytest filter string (-k argument)' + required: true + providers: + description: 'Comma-separated list of non-local providers' + required: true + local_providers: + description: 'Comma-separated list of local providers' + required: true +outputs: + should_run_integration: + description: 'Whether the integration tests job should run' + value: ${{ steps.determine.outputs.should_run_integration }} + should_run_local: + description: 'Whether the local integration tests job should run' + value: ${{ steps.determine.outputs.should_run_local }} + +runs: + using: 'composite' + steps: + - name: Determine which jobs should run + id: determine + shell: bash + run: | + FILTER="${{ inputs.filter }}" + PROVIDERS="${{ inputs.providers }}" + LOCAL_PROVIDERS="${{ inputs.local_providers }}" + + echo "Filter input: '$FILTER'" + + if [ -z "$FILTER" ]; then + echo "No filter provided, running all jobs" + echo "should_run_integration=true" >> $GITHUB_OUTPUT + echo "should_run_local=true" >> $GITHUB_OUTPUT + exit 0 + fi + + MATCHED_INTEGRATION=false + IFS=',' read -ra PROVIDER_ARRAY <<< "$PROVIDERS" + for provider in "${PROVIDER_ARRAY[@]}"; do + if [[ "$FILTER" == *"$provider"* ]]; then + echo "Filter matches integration provider: $provider" + MATCHED_INTEGRATION=true + break + fi + done + + MATCHED_LOCAL=false + IFS=',' read -ra LOCAL_PROVIDER_ARRAY <<< "$LOCAL_PROVIDERS" + for provider in "${LOCAL_PROVIDER_ARRAY[@]}"; do + if [[ "$FILTER" == *"$provider"* ]]; then + echo "Filter matches local provider: $provider" + MATCHED_LOCAL=true + break + fi + done + + if [ "$MATCHED_INTEGRATION" = false ] && [ "$MATCHED_LOCAL" = false ]; then + echo "Filter doesn't match any specific provider, treating as test function filter" + echo "Running all jobs to let pytest handle the filtering" + echo "should_run_integration=true" >> $GITHUB_OUTPUT + echo "should_run_local=true" >> $GITHUB_OUTPUT + else + echo "should_run_integration=$MATCHED_INTEGRATION" >> $GITHUB_OUTPUT + echo "should_run_local=$MATCHED_LOCAL" >> $GITHUB_OUTPUT + fi diff --git a/.github/workflows/tests-integration.yaml b/.github/workflows/tests-integration.yaml index 5cd7ec37..cbaf9c83 100644 --- a/.github/workflows/tests-integration.yaml +++ b/.github/workflows/tests-integration.yaml @@ -29,9 +29,26 @@ jobs: id: set_local_providers run: echo "expected_providers=ollama,llamacpp,llamafile" >> $GITHUB_OUTPUT - run-integration-tests: + determine-jobs-to-run: needs: expected-providers - if: github.event.inputs.filter == '' || contains(needs.expected-providers.outputs.providers, github.event.inputs.filter) + runs-on: ubuntu-latest + outputs: + should_run_integration: ${{ steps.determine.outputs.should_run_integration }} + should_run_local: ${{ steps.determine.outputs.should_run_local }} + steps: + - uses: actions/checkout@v5 + + - name: Determine which jobs should run based on filter + id: determine + uses: ./.github/actions/determine-jobs + with: + filter: ${{ github.event.inputs.filter }} + providers: ${{ needs.expected-providers.outputs.providers }} + local_providers: ${{ needs.expected-providers.outputs.local_providers }} + + run-integration-tests: + needs: [expected-providers, determine-jobs-to-run] + if: needs.determine-jobs-to-run.outputs.should_run_integration == 'true' timeout-minutes: 30 runs-on: ubuntu-latest @@ -50,7 +67,7 @@ jobs: - if: github.event.inputs.filter == '' || contains(github.event.inputs.filter, 'huggingface') env: HF_TOKEN: ${{ secrets.HF_TOKEN }} - run: python scripts/wake_up_hf_endpoint.py --retry=5 + run: python scripts/wake_up_hf_endpoint.py --retry-count=30 --retry-interval=10 - name: Run Integration tests (parallel with xdist) env: @@ -83,9 +100,9 @@ jobs: INCLUDE_LOCAL_PROVIDERS: "false" run: | if [ -n "${{ inputs.filter }}" ]; then - pytest tests/integration -v -n auto --cov --cov-report=xml -k "${{ inputs.filter }}" + pytest tests/integration -v -n auto --cov --cov-report=xml -k "${{ inputs.filter }}" || [ $? -eq 5 ] else - pytest tests/integration -v -n auto --cov --cov-report=xml + pytest tests/integration -v -n auto --cov --cov-report=xml || [ $? -eq 5 ] fi - name: Upload coverage reports to Codecov @@ -95,8 +112,8 @@ jobs: token: ${{ secrets.CODECOV_TOKEN }} run-local-integration-tests: - needs: expected-providers - if: github.event.inputs.filter == '' || contains(needs.expected-providers.outputs.local_providers, github.event.inputs.filter) + needs: [expected-providers, determine-jobs-to-run] + if: needs.determine-jobs-to-run.outputs.should_run_local == 'true' timeout-minutes: 45 runs-on: ubuntu-latest @@ -181,9 +198,9 @@ jobs: INCLUDE_NON_LOCAL_PROVIDERS: "false" run: | if [ -n "${{ inputs.filter }}" ]; then - pytest tests/integration -v --cov --cov-report=xml -k "${{ inputs.filter }}" + pytest tests/integration -v --cov --cov-report=xml -k "${{ inputs.filter }}" || [ $? -eq 5 ] else - pytest tests/integration -v --cov --cov-report=xml + pytest tests/integration -v --cov --cov-report=xml || [ $? -eq 5 ] fi - name: Cleanup LlamaFile process diff --git a/scripts/wake_up_hf_endpoint.py b/scripts/wake_up_hf_endpoint.py index a2701a18..97ec255f 100644 --- a/scripts/wake_up_hf_endpoint.py +++ b/scripts/wake_up_hf_endpoint.py @@ -8,26 +8,31 @@ HF_ENDPOINT = "https://y0okp71n85ezo5nr.us-east-1.aws.endpoints.huggingface.cloud/v1/" -def wake_up_hf_endpoint(retry: int = 0): - while True: +def wake_up_hf_endpoint(retry_count: int = 0, retry_interval: int = 10): + attempt = 0 + max_attempts = retry_count + 1 + + while attempt < max_attempts: try: completion( model="huggingface:tgi", messages=[{"role": "user", "content": "Are you awake?"}], api_base=HF_ENDPOINT ) - break except ClientResponseError as e: - if not retry: - print(f"Endpoint not ready, giving up...\n{e}") + attempt += 1 + if attempt >= max_attempts: + print(f"Endpoint not ready after {attempt} attempts, giving up...\n{e}") return - print(f"Endpoint not ready, retrying...\n{e}") - time.sleep(retry) - - print("Endpoint ready") + print(f"Endpoint not ready (attempt {attempt}/{max_attempts}), retrying in {retry_interval}s...\n{e}") + time.sleep(retry_interval) + else: + print("Endpoint ready") + return if __name__ == "__main__": parser = argparse.ArgumentParser(description="Wake up Hugging Face endpoint") - parser.add_argument("--retry", type=int, default=0, help="Retry interval in seconds (0 means no retry)") + parser.add_argument("--retry-count", type=int, default=0, help="Number of retry attempts (default: 0, no retries)") + parser.add_argument("--retry-interval", type=int, default=10, help="Seconds between retry attempts (default: 10)") args = parser.parse_args() - wake_up_hf_endpoint(retry=args.retry) + wake_up_hf_endpoint(retry_count=args.retry_count, retry_interval=args.retry_interval)