Skip to content
Merged
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
10 changes: 9 additions & 1 deletion tests/e2e/features/steps/common_http.py
Original file line number Diff line number Diff line change
Expand Up @@ -3,7 +3,7 @@
import json

import requests
from behave import given, then, when
from behave import given, then, when # pyright: ignore[reportAttributeAccessIssue]
from behave.runner import Context
from tests.e2e.utils.utils import normalize_endpoint, validate_json

Expand Down Expand Up @@ -37,6 +37,8 @@ def request_endpoint_with_json(
# initial value
context.response = None

assert context.text is not None, "Payload needs to be specified"

# perform REST API call
context.response = requests.get(
f"http://{hostname}:{port}/{endpoint}",
Expand All @@ -54,6 +56,8 @@ def request_endpoint_with_url_params(
"""Perform a request to the server defined by URL to a given endpoint."""
params = {}

assert context.table is not None, "Request parameters needs to be specified"

for row in context.table:
name = row["param"]
value = row["value"]
Expand Down Expand Up @@ -120,6 +124,7 @@ def check_content_type(context: Context, content_type: str) -> None:
def check_response_body_schema(context: Context) -> None:
"""Check that response body is compliant with a given schema."""
assert context.response is not None, "Request needs to be performed first"
assert context.text is not None, "Response does not contain any payload"
schema = json.loads(context.text)
body = context.response.json()

Expand All @@ -139,6 +144,7 @@ def check_response_body_contains(context: Context, substring: str) -> None:
def check_prediction_result(context: Context) -> None:
"""Check the content of the response to be exactly the same."""
assert context.response is not None, "Request needs to be performed first"
assert context.text is not None, "Response does not contain any payload"
expected_body = json.loads(context.text)
result = context.response.json()

Expand All @@ -150,6 +156,7 @@ def check_prediction_result(context: Context) -> None:
def check_prediction_result_ignoring_field(context: Context, field: str) -> None:
"""Check the content of the response to be exactly the same."""
assert context.response is not None, "Request needs to be performed first"
assert context.text is not None, "Response does not contain any payload"
expected_body = json.loads(context.text).copy()
result = context.response.json().copy()

Expand Down Expand Up @@ -217,6 +224,7 @@ def access_rest_api_endpoint_post(context: Context, endpoint: str) -> None:
path = f"{context.api_prefix}/{endpoint}".replace("//", "/")
url = base + path

assert context.text is not None, "Payload needs to be specified"
data = json.loads(context.text)
# initial value
context.response = None
Expand Down
4 changes: 3 additions & 1 deletion tests/e2e/features/steps/llm_query_response.py
Original file line number Diff line number Diff line change
@@ -1,7 +1,7 @@
"""LLM query and response steps."""

import requests
from behave import then, when
from behave import then, when # pyright: ignore[reportAttributeAccessIssue]
from behave.runner import Context

DEFAULT_LLM_TIMEOUT = 60
Expand Down Expand Up @@ -41,6 +41,8 @@ def check_fragments_in_response(context: Context) -> None:
response_json = context.response.json()
response = response_json["response"]

assert context.table is not None, "Fragments are not specified in table"

for fragment in context.table:
expected = fragment["Fragments in LLM response"]
assert (
Expand Down
15 changes: 10 additions & 5 deletions tests/e2e/gen_scenario_list.py
Original file line number Diff line number Diff line change
Expand Up @@ -21,8 +21,11 @@

import os

# repository URL
REPO_URL = "https://github.com/lightspeed-core/lightspeed-stack/"

# URL prefix to create links to feature files
FEATURES_URL_PREFIX = "https://github.com/lightspeed-core/lightspeed-stack/blob/main/tests/e2e/features" # noqa E501
FEATURES_URL_PREFIX = f"{REPO_URL}blob/main/tests/e2e/features"

# list of prefixes for scenarios or scenario outlines
PREFIXES = ("Scenario: ", "Scenario Outline: ")
Expand All @@ -40,16 +43,18 @@
print()

# generage list of scenarios
directory = FEATURE_DIRECTORY

# files within one subdirectory needs to be sorted so the
# resulting scenario list will have stable structure across versions
files = sorted(os.listdir(directory))
files = sorted(os.listdir(FEATURE_DIRECTORY))
for filename in files:
# grep all .feature files
if filename.endswith(".feature"):
# feature file header
print("## [`{}`]({}/{})\n".format(filename, FEATURES_URL_PREFIX, filename))
with open(os.path.join(directory, filename), "r") as fin:
print(f"## [`{filename}`]({FEATURES_URL_PREFIX}/{filename})\n")
with open(
os.path.join(FEATURE_DIRECTORY, filename), "r", encoding="utf-8"
) as fin:
for line in fin.readlines():
line = line.strip()
# process all scenarios and scenario outlines
Expand Down