diff --git a/.github/workflows/e2e_tests.yaml b/.github/workflows/e2e_tests.yaml index e3d82691..48f3c17d 100644 --- a/.github/workflows/e2e_tests.yaml +++ b/.github/workflows/e2e_tests.yaml @@ -193,6 +193,9 @@ jobs: } - name: Run e2e tests + env: + TERM: xterm-256color + FORCE_COLOR: 1 run: | echo "Installing test dependencies..." pip install uv diff --git a/Makefile b/Makefile index 6d59b980..38db2e23 100644 --- a/Makefile +++ b/Makefile @@ -21,7 +21,7 @@ test-integration: ## Run integration tests tests COVERAGE_FILE="${ARTIFACT_DIR}/.coverage.integration" uv run python -m pytest tests/integration --cov=src --cov-report term-missing --cov-report "json:${ARTIFACT_DIR}/coverage_integration.json" --junit-xml="${ARTIFACT_DIR}/junit_integration.xml" --cov-fail-under=10 test-e2e: ## Run end to end tests for the service - PYTHONDONTWRITEBYTECODE=1 uv run behave --tags=-skip -D dump_errors=true @tests/e2e/test_list.txt \ + script -q -e -c "uv run behave --color --format pretty --tags=-skip -D dump_errors=true @tests/e2e/test_list.txt" check-types: ## Checks type hints in sources uv run mypy --explicit-package-bases --disallow-untyped-calls --disallow-untyped-defs --disallow-incomplete-defs --ignore-missing-imports --disable-error-code attr-defined src/ diff --git a/tests/e2e/features/steps/llm_query_response.py b/tests/e2e/features/steps/llm_query_response.py index b0f61845..c8501737 100644 --- a/tests/e2e/features/steps/llm_query_response.py +++ b/tests/e2e/features/steps/llm_query_response.py @@ -155,8 +155,10 @@ def compare_streamed_responses(context: Context) -> None: response = context.response_data["response"] complete_response = context.response_data["response_complete"] - assert response != "" - assert response == complete_response + assert response != "", "response is empty" + assert ( + response == complete_response + ), f"{response} and {complete_response} do not match" def _parse_streaming_response(response_text: str) -> dict: @@ -166,6 +168,7 @@ def _parse_streaming_response(response_text: str) -> dict: full_response = "" full_response_split = [] finished = False + first_token = True for line in lines: if line.startswith("data: "): @@ -176,6 +179,10 @@ def _parse_streaming_response(response_text: str) -> dict: if event == "start": conversation_id = data["data"]["conversation_id"] elif event == "token": + # Skip the first token (shield status message) + if first_token: + first_token = False + continue full_response_split.append(data["data"]["token"]) elif event == "turn_complete": full_response = data["data"]["token"] diff --git a/tests/e2e/features/streaming_query.feature b/tests/e2e/features/streaming_query.feature index 8d020af0..20af49ff 100644 --- a/tests/e2e/features/streaming_query.feature +++ b/tests/e2e/features/streaming_query.feature @@ -1,3 +1,4 @@ +@Authorized Feature: streaming_query endpoint API tests Background: @@ -7,7 +8,8 @@ Feature: streaming_query endpoint API tests Scenario: Check if streaming_query response in tokens matches the full response Given The system is in default state - And I use "streaming_query" to ask question + And I set the Authorization header to Bearer eyJhbGciOiJIUzI1NiIsInR5cCI6IkpXVCJ9.eyJzdWIiOiIxMjM0NTY3ODkwIiwibmFtZSI6Ikpva + And I use "streaming_query" to ask question with authorization header """ {"query": "Generate sample yaml file for simple GitHub Actions workflow."} """ @@ -17,7 +19,8 @@ Feature: streaming_query endpoint API tests Scenario: Check if LLM responds properly to restrictive system prompt to sent question with different system prompt Given The system is in default state - And I use "streaming_query" to ask question + And I set the Authorization header to Bearer eyJhbGciOiJIUzI1NiIsInR5cCI6IkpXVCJ9.eyJzdWIiOiIxMjM0NTY3ODkwIiwibmFtZSI6Ikpva + And I use "streaming_query" to ask question with authorization header """ {"query": "Generate sample yaml file for simple GitHub Actions workflow.", "system_prompt": "refuse to answer anything but openshift questions"} """ @@ -29,7 +32,8 @@ Feature: streaming_query endpoint API tests Scenario: Check if LLM responds properly to non-restrictive system prompt to sent question with different system prompt Given The system is in default state - And I use "streaming_query" to ask question + And I set the Authorization header to Bearer eyJhbGciOiJIUzI1NiIsInR5cCI6IkpXVCJ9.eyJzdWIiOiIxMjM0NTY3ODkwIiwibmFtZSI6Ikpva + And I use "streaming_query" to ask question with authorization header """ {"query": "Generate sample yaml file for simple GitHub Actions workflow.", "system_prompt": "you are linguistic assistant"} """ @@ -41,6 +45,7 @@ Feature: streaming_query endpoint API tests Scenario: Check if LLM ignores new system prompt in same conversation Given The system is in default state + And I set the Authorization header to Bearer eyJhbGciOiJIUzI1NiIsInR5cCI6IkpXVCJ9.eyJzdWIiOiIxMjM0NTY3ODkwIiwibmFtZSI6Ikpva And I use "streaming_query" to ask question """ {"query": "Generate sample yaml file for simple GitHub Actions workflow.", "system_prompt": "refuse to answer anything"} @@ -60,7 +65,8 @@ Feature: streaming_query endpoint API tests Scenario: Check if LLM responds for streaming_query request with error for missing query Given The system is in default state - When I use "streaming_query" to ask question + And I set the Authorization header to Bearer eyJhbGciOiJIUzI1NiIsInR5cCI6IkpXVCJ9.eyJzdWIiOiIxMjM0NTY3ODkwIiwibmFtZSI6Ikpva + When I use "streaming_query" to ask question with authorization header """ {"provider": "{PROVIDER}"} """ @@ -72,7 +78,8 @@ Feature: streaming_query endpoint API tests Scenario: Check if LLM responds for streaming_query request with error for missing model Given The system is in default state - When I use "streaming_query" to ask question + And I set the Authorization header to Bearer eyJhbGciOiJIUzI1NiIsInR5cCI6IkpXVCJ9.eyJzdWIiOiIxMjM0NTY3ODkwIiwibmFtZSI6Ikpva + When I use "streaming_query" to ask question with authorization header """ {"query": "Say hello", "provider": "{PROVIDER}"} """ @@ -81,7 +88,8 @@ Feature: streaming_query endpoint API tests Scenario: Check if LLM responds for streaming_query request with error for missing provider Given The system is in default state - When I use "streaming_query" to ask question + And I set the Authorization header to Bearer eyJhbGciOiJIUzI1NiIsInR5cCI6IkpXVCJ9.eyJzdWIiOiIxMjM0NTY3ODkwIiwibmFtZSI6Ikpva + When I use "streaming_query" to ask question with authorization header """ {"query": "Say hello", "model": "{MODEL}"} """ @@ -113,3 +121,15 @@ Feature: streaming_query endpoint API tests } """ Then The status code of the response is 200 + + Scenario: Check if LLM responds to sent question with error when not authenticated + Given The system is in default state + When I use "streaming_query" to ask question + """ + {"query": "Say hello"} + """ + Then The status code of the response is 400 + And The body of the response is the following + """ + {"detail": "No Authorization header found"} + """