diff --git a/Makefile b/Makefile index 363cd1f71..5fbd89e99 100644 --- a/Makefile +++ b/Makefile @@ -36,7 +36,7 @@ CONTAINER_RUNTIME ?= $(shell command -v podman 2>/dev/null || command -v docker docs/models/database.svg run-stack: ## Run lightspeed-stack directly, without building dependent service/s - uv run src/lightspeed_stack.py -c $(CONFIG) + uv run opentelemetry-instrument python3.12 src/lightspeed_stack.py -c $(CONFIG) run: start-llama-stack-container ## Run the service locally with dependent services @echo "Starting Lightspeed Core Stack..." diff --git a/README.md b/README.md index e3a59d743..4e1cf0a02 100644 --- a/README.md +++ b/README.md @@ -1130,7 +1130,7 @@ ENV PATH="/app-root/.venv/bin:$PATH" # Run the application EXPOSE 8080 -ENTRYPOINT ["python3.12", "src/lightspeed_stack.py"] +ENTRYPOINT ["opentelemetry-instrument", "python3.12", "src/lightspeed_stack.py"] USER 1001 ``` diff --git a/deploy/lightspeed-stack/Containerfile b/deploy/lightspeed-stack/Containerfile index 772c91a7a..61bd07281 100644 --- a/deploy/lightspeed-stack/Containerfile +++ b/deploy/lightspeed-stack/Containerfile @@ -135,7 +135,7 @@ ENV PYTHONPATH="/app-root" # Run the application EXPOSE 8080 -ENTRYPOINT ["python3.12", "src/lightspeed_stack.py"] +ENTRYPOINT ["opentelemetry-instrument", "python3.12", "src/lightspeed_stack.py"] LABEL vendor="Red Hat, Inc." \ name="lightspeed-core/lightspeed-stack-rhel9" \ diff --git a/docs/user_doc/deployment_guide.md b/docs/user_doc/deployment_guide.md index bd8b1849a..dc2a9a3d8 100644 --- a/docs/user_doc/deployment_guide.md +++ b/docs/user_doc/deployment_guide.md @@ -468,7 +468,7 @@ make run ``` ``` -uv run src/lightspeed_stack.py +uv run opentelemetry-instrument python3.12 src/lightspeed_stack.py [07/29/25 15:43:35] INFO Initializing app main.py:19 INFO Including routers main.py:68 INFO: Started server process [1922983] @@ -561,7 +561,7 @@ cp examples/lightspeed-stack-lls-library.yaml lightspeed-stack.yaml ``` 1. Check the output ```text - uv run src/lightspeed_stack.py + uv run opentelemetry-instrument python3.12 src/lightspeed_stack.py Using config run.yaml: apis: - agents diff --git a/tests/e2e-prow/rhoai/manifests/lightspeed/lightspeed-stack.yaml b/tests/e2e-prow/rhoai/manifests/lightspeed/lightspeed-stack.yaml index 83e2f173a..f0ce49df9 100644 --- a/tests/e2e-prow/rhoai/manifests/lightspeed/lightspeed-stack.yaml +++ b/tests/e2e-prow/rhoai/manifests/lightspeed/lightspeed-stack.yaml @@ -37,7 +37,7 @@ spec: key: key optional: true image: ${LIGHTSPEED_STACK_IMAGE} - command: ["/bin/sh", "-c", "mkdir -p /tmp/data && exec python3.12 src/lightspeed_stack.py"] + command: ["/bin/sh", "-c", "mkdir -p /tmp/data && exec opentelemetry-instrument python3.12 src/lightspeed_stack.py"] ports: - containerPort: 8080 # TCP probes avoid HTTP/auth. LCS + Llama handshake and large images can take 60–120s before :8080 listens; diff --git a/tests/e2e/utils/prow_utils.py b/tests/e2e/utils/prow_utils.py index 6abd4a431..dddc04558 100644 --- a/tests/e2e/utils/prow_utils.py +++ b/tests/e2e/utils/prow_utils.py @@ -62,8 +62,12 @@ def run_e2e_ops( ) -def wait_for_pod_health(pod_name: str, max_attempts: int = 12) -> None: - """Wait for pod to be ready in OpenShift/Prow environment.""" +def wait_for_pod_health(pod_name: str, max_attempts: int = 20) -> None: + """Wait for pod to be ready in OpenShift/Prow environment. + + Generous number of attempts to account for OpenTelemetry instrumentation + initialization overhead during service startup. + """ actual_pod_name = get_pod_name(pod_name) try: result = run_e2e_ops("wait-for-pod", [actual_pod_name, str(max_attempts)]) diff --git a/tests/e2e/utils/utils.py b/tests/e2e/utils/utils.py index a93f05246..178adb02d 100644 --- a/tests/e2e/utils/utils.py +++ b/tests/e2e/utils/utils.py @@ -181,7 +181,7 @@ def validate_json(message: Any, schema: Any) -> None: assert False, "The provided schema is faulty:" + str(e) -def wait_for_container_health(container_name: str, max_attempts: int = 6) -> None: +def wait_for_container_health(container_name: str, max_attempts: int = 20) -> None: """Wait for container to be healthy. Polls a Docker container until its health status becomes `healthy` or the @@ -192,6 +192,9 @@ def wait_for_container_health(container_name: str, max_attempts: int = 6) -> Non inspect errors or timeouts are ignored and retried; the function returns after the container is observed healthy or after all attempts complete. + OpenTelemetry instrumentation adds initialization overhead, so the default + has been set to 20 attempts (40 seconds) to prevent timeouts. + Returns: ------- None @@ -199,7 +202,7 @@ def wait_for_container_health(container_name: str, max_attempts: int = 6) -> Non Parameters: ---------- container_name (str): Docker container name or ID to check. - max_attempts (int): Maximum number of health check attempts (default 6). + max_attempts (int): Maximum number of health check attempts (default 20). """ if is_prow_environment(): wait_for_pod_health(container_name, max_attempts) @@ -468,9 +471,10 @@ def restart_container(container_name: str) -> None: # Wait for container to be healthy. # Library mode embeds llama-stack, so the container takes longer to start - # (~45-60s vs ~10s in server mode). Use a generous attempt count so - # MCP-auth scenarios that restart the container don't time out. - wait_for_container_health(container_name, max_attempts=12) + # (~45-60s vs ~10s in server mode). OpenTelemetry instrumentation adds + # initialization overhead. Use a generous attempt count so MCP-auth scenarios + # that restart the container don't time out. + wait_for_container_health(container_name, max_attempts=20) if container_name == "llama-stack": from tests.e2e.features.steps.health import (