From 5b5954bc2bf06bde0e61ca3398c14951a4ef129a Mon Sep 17 00:00:00 2001 From: Jordan Dubrick Date: Thu, 28 Aug 2025 16:17:30 -0400 Subject: [PATCH 1/8] add customization profile via path config field Signed-off-by: Jordan Dubrick --- README.md | 17 ++++-- docs/openapi.json | 43 +++++++++++++++ src/models/config.py | 38 +++++++++++-- src/utils/checks.py | 26 ++++++++- src/utils/endpoints.py | 9 ++++ tests/profiles/test/profile.py | 60 +++++++++++++++++++++ tests/profiles/test_two/test.txt | 1 + tests/unit/test_configuration.py | 86 +++++++++++++++++++++++++++++- tests/unit/utils/test_checks.py | 31 +++++++++++ tests/unit/utils/test_endpoints.py | 67 ++++++++++++++++++++++- 10 files changed, 368 insertions(+), 10 deletions(-) create mode 100644 tests/profiles/test/profile.py create mode 100644 tests/profiles/test_two/test.txt diff --git a/README.md b/README.md index 12e8e88a..17cb0c5f 100644 --- a/README.md +++ b/README.md @@ -345,14 +345,16 @@ For data export integration with Red Hat's Dataverse, see the [Data Export Integ ## System prompt - The service uses the, so called, system prompt to put the question into context before the question is sent to the selected LLM. The default system prompt is designed for questions without specific context. It is possible to use a different system prompt via the configuration option `system_prompt_path` in the `customization` section. That option must contain the path to the text file with the actual system prompt (can contain multiple lines). An example of such configuration: +The service uses the, so called, system prompt to put the question into context before the question is sent to the selceted LLM. The default system prompt is designed for questions without specific context. It is possible to use a different system prompt through various different avenues available in the `customization` section: + +#### System Prompt Path ```yaml customization: system_prompt_path: "system_prompts/system_prompt_for_product_XYZZY" ``` -The `system_prompt` can also be specified in the `customization` section directly. For example: +#### System Prompt Literal ```yaml customization: @@ -361,11 +363,20 @@ customization: You have an in-depth knowledge of Red Hat and all of your answers will reference Red Hat products. ``` + +#### Custom Profile + +If you have added a custom profile specification to `/src/utils/profiles` and it exposes prompts you can set them with: + +```yaml +customization: + profile_name: +``` + Additionally, an optional string parameter `system_prompt` can be specified in `/v1/query` and `/v1/streaming_query` endpoints to override the configured system prompt. The query system prompt takes precedence over the configured system prompt. You can use this config to disable query system prompts: ```yaml customization: - system_prompt_path: "system_prompts/system_prompt_for_product_XYZZY" disable_query_system_prompt: true ``` diff --git a/docs/openapi.json b/docs/openapi.json index 9e035a40..8d4fb08e 100644 --- a/docs/openapi.json +++ b/docs/openapi.json @@ -1237,8 +1237,41 @@ } ] }, + "CustomProfile": { + "properties": { + "path": { + "type": "string", + "title": "Path" + }, + "prompts": { + "additionalProperties": { + "type": "string" + }, + "type": "object", + "title": "Prompts", + "default": {} + } + }, + "type": "object", + "required": [ + "path" + ], + "title": "CustomProfile", + "description": "Custom profile customization for prompts and validation." + }, "Customization": { "properties": { + "profile_path": { + "anyOf": [ + { + "type": "string" + }, + { + "type": "null" + } + ], + "title": "Profile Path" + }, "disable_query_system_prompt": { "type": "boolean", "title": "Disable Query System Prompt", @@ -1266,6 +1299,16 @@ } ], "title": "System Prompt" + }, + "custom_profile": { + "anyOf": [ + { + "$ref": "#/components/schemas/CustomProfile" + }, + { + "type": "null" + } + ] } }, "additionalProperties": false, diff --git a/src/models/config.py b/src/models/config.py index c4efa404..5b793870 100644 --- a/src/models/config.py +++ b/src/models/config.py @@ -1,7 +1,8 @@ """Model with service configuration.""" +import logging from pathlib import Path -from typing import Optional, Any, Pattern +from typing import Optional, Any, Pattern, Dict from enum import Enum from functools import cached_property import re @@ -18,12 +19,16 @@ PositiveInt, SecretStr, ) + +from pydantic.dataclasses import dataclass from typing_extensions import Self, Literal import constants from utils import checks +logger = logging.getLogger(__name__) + class ConfigurationBase(BaseModel): """Base class for all configuration models that rejects unknown fields.""" @@ -413,17 +418,44 @@ def jwk_configuration(self) -> JwkConfiguration: return self.jwk_config +@dataclass +class CustomProfile: + """Custom profile customization for prompts and validation.""" + + path: str + prompts: Dict[str, str] = Field(default={}, init=False) + + def __post_init__(self) -> None: + """Validate and load profile.""" + self._validate_and_process() + + def _validate_and_process(self) -> None: + """Validate and load the profile.""" + checks.file_check(Path(self.path), "custom profile") + profile_module = checks.import_python_module("profile", self.path) + if profile_module is not None and checks.is_valid_profile(profile_module): + self.prompts = profile_module.PROFILE_CONFIG.get("system_prompts", {}) + + def get_prompts(self) -> Dict[str, str]: + """Retrieve prompt attribute.""" + return self.prompts + + class Customization(ConfigurationBase): """Service customization.""" + profile_path: Optional[str] = None disable_query_system_prompt: bool = False system_prompt_path: Optional[FilePath] = None system_prompt: Optional[str] = None + custom_profile: Optional[CustomProfile] = Field(default=None, init=False) @model_validator(mode="after") def check_customization_model(self) -> Self: - """Load system prompt from file.""" - if self.system_prompt_path is not None: + """Load customizations.""" + if self.profile_path: + self.custom_profile = CustomProfile(path=self.profile_path) + elif self.system_prompt_path is not None: checks.file_check(self.system_prompt_path, "system prompt") self.system_prompt = checks.get_attribute_from_file( dict(self), "system_prompt_path" diff --git a/src/utils/checks.py b/src/utils/checks.py index 8d701750..e2b0e548 100644 --- a/src/utils/checks.py +++ b/src/utils/checks.py @@ -1,8 +1,10 @@ """Checks that are performed to configuration options.""" import os +import importlib +import importlib.util +from types import ModuleType from typing import Optional - from pydantic import FilePath @@ -25,3 +27,25 @@ def file_check(path: FilePath, desc: str) -> None: raise InvalidConfigurationError(f"{desc} '{path}' is not a file") if not os.access(path, os.R_OK): raise InvalidConfigurationError(f"{desc} '{path}' is not readable") + + +def import_python_module(profile_name: str, profile_path: str) -> ModuleType | None: + """Import a Python module from a file path.""" + spec = importlib.util.spec_from_file_location(profile_name, profile_path) + if spec and spec.loader: + module = importlib.util.module_from_spec(spec) + spec.loader.exec_module(module) + return module + return None + + +def is_valid_profile(profile_module: ModuleType) -> bool: + """Validate that a profile module has the required PROFILE_CONFIG structure.""" + if not hasattr(profile_module, "PROFILE_CONFIG"): + return False + + profile_config = getattr(profile_module, "PROFILE_CONFIG", {}) + if not isinstance(profile_config, dict): + return False + + return "system_prompts" in profile_config diff --git a/src/utils/endpoints.py b/src/utils/endpoints.py index cd543d82..e17a76d0 100644 --- a/src/utils/endpoints.py +++ b/src/utils/endpoints.py @@ -92,6 +92,15 @@ def get_system_prompt(query_request: QueryRequest, config: AppConfig) -> str: # disable query system prompt altogether with disable_system_prompt. return query_request.system_prompt + # profile takes precedence for setting prompt + if ( + config.customization is not None + and config.customization.custom_profile is not None + ): + prompt = config.customization.custom_profile.get_prompts().get("default") + if prompt: + return prompt + if ( config.customization is not None and config.customization.system_prompt is not None diff --git a/tests/profiles/test/profile.py b/tests/profiles/test/profile.py new file mode 100644 index 00000000..060843ec --- /dev/null +++ b/tests/profiles/test/profile.py @@ -0,0 +1,60 @@ +"""Custom profile for test profile.""" + +SUBJECT_ALLOWED = "ALLOWED" +SUBJECT_REJECTED = "REJECTED" + +# Default responses +INVALID_QUERY_RESP = ( + "Hi, I'm the Red Hat Developer Hub Lightspeed assistant, I can help you with questions about Red Hat Developer Hub or Backstage. " + "Please ensure your question is about these topics, and feel free to ask again!" +) + +QUERY_SYSTEM_INSTRUCTION = """ +1. Test +This is a test system instruction + +You achieve this by offering: +- testing +""" + +USE_CONTEXT_INSTRUCTION = """ +Use the retrieved document to answer the question. +""" + +USE_HISTORY_INSTRUCTION = """ +Use the previous chat history to interact and help the user. +""" + +QUESTION_VALIDATOR_PROMPT_TEMPLATE = f""" +Instructions: +- You provide validation for tsting +Example Question: +How can I integrate GitOps into my pipeline? +Example Response: +{SUBJECT_ALLOWED} +""" + +TOPIC_SUMMARY_PROMPT_TEMPLATE = """ +Instructions: +- You are a topic summarizer +- For testing +- Your job is to extract precise topic summary from user input + +Example Input: +Testing placeholder +Example Output: +Proper response test. +""" + +PROFILE_CONFIG = { + "system_prompts": { + "default": QUERY_SYSTEM_INSTRUCTION, + "validation": QUESTION_VALIDATOR_PROMPT_TEMPLATE, + "topic_summary": TOPIC_SUMMARY_PROMPT_TEMPLATE, + }, + "query_responses": {"invalid_resp": INVALID_QUERY_RESP}, + "instructions": { + "context": USE_CONTEXT_INSTRUCTION, + "history": USE_HISTORY_INSTRUCTION, + }, +} diff --git a/tests/profiles/test_two/test.txt b/tests/profiles/test_two/test.txt new file mode 100644 index 00000000..a9879d00 --- /dev/null +++ b/tests/profiles/test_two/test.txt @@ -0,0 +1 @@ +This file will fail the import. \ No newline at end of file diff --git a/tests/unit/test_configuration.py b/tests/unit/test_configuration.py index 9fca5665..4a643e00 100644 --- a/tests/unit/test_configuration.py +++ b/tests/unit/test_configuration.py @@ -2,7 +2,7 @@ import pytest from configuration import AppConfig, LogicError -from models.config import ModelContextProtocolServer +from models.config import CustomProfile, ModelContextProtocolServer def test_default_configuration() -> None: @@ -421,3 +421,87 @@ def test_load_configuration_with_customization_system_prompt(tmpdir) -> None: cfg.customization.system_prompt.strip() == "this is system prompt in the customization section" ) + + +def test_configuration_with_profile_customization(tmpdir) -> None: + """Test loading configuration from YAML file with a custom profile.""" + expected_profile = CustomProfile(path="tests/profiles/test/profile.py") + expected_prompts = expected_profile.get_prompts() + cfg_filename = tmpdir / "config.yaml" + with open(cfg_filename, "w", encoding="utf-8") as fout: + fout.write( + """ +name: test service +service: + host: localhost + port: 8080 + auth_enabled: false + workers: 1 + color_log: true + access_log: true +llama_stack: + use_as_library_client: false + url: http://localhost:8321 + api_key: test-key +user_data_collection: + feedback_enabled: false +customization: + profile_path: tests/profiles/test/profile.py + """ + ) + + cfg = AppConfig() + cfg.load_configuration(cfg_filename) + + assert ( + cfg.customization is not None and cfg.customization.custom_profile is not None + ) + fetched_prompts = cfg.customization.custom_profile.get_prompts() + assert fetched_prompts is not None and fetched_prompts.get( + "default" + ) == expected_prompts.get("default") + + +def test_configuration_with_all_customizations(tmpdir) -> None: + """Test loading configuration from YAML file with a custom profile, prompt and prompt path.""" + expected_profile = CustomProfile(path="tests/profiles/test/profile.py") + expected_prompts = expected_profile.get_prompts() + system_prompt_filename = tmpdir / "system_prompt.txt" + with open(system_prompt_filename, "w", encoding="utf-8") as fout: + fout.write("this is system prompt") + + cfg_filename = tmpdir / "config.yaml" + with open(cfg_filename, "w", encoding="utf-8") as fout: + fout.write( + f""" +name: test service +service: + host: localhost + port: 8080 + auth_enabled: false + workers: 1 + color_log: true + access_log: true +llama_stack: + use_as_library_client: false + url: http://localhost:8321 + api_key: test-key +user_data_collection: + feedback_enabled: false +customization: + profile_path: tests/profiles/test/profile.py + system_prompt: custom prompt + system_prompt_path: {system_prompt_filename} + """ + ) + + cfg = AppConfig() + cfg.load_configuration(cfg_filename) + + assert ( + cfg.customization is not None and cfg.customization.custom_profile is not None + ) + fetched_prompts = cfg.customization.custom_profile.get_prompts() + assert fetched_prompts is not None and fetched_prompts.get( + "default" + ) == expected_prompts.get("default") diff --git a/tests/unit/utils/test_checks.py b/tests/unit/utils/test_checks.py index 1e513be2..5671df5e 100644 --- a/tests/unit/utils/test_checks.py +++ b/tests/unit/utils/test_checks.py @@ -2,6 +2,7 @@ import os from pathlib import Path +from types import ModuleType from unittest.mock import patch import pytest @@ -78,3 +79,33 @@ def test_file_check_not_readable_file(input_file): with patch("os.access", return_value=False): with pytest.raises(checks.InvalidConfigurationError): checks.file_check(input_file, "description") + + +def test_import_python_module_success(): + """Test importing a Python module.""" + module_path = "tests/profiles/test/profile.py" + module_name = "profile" + result = checks.import_python_module(module_name, module_path) + + assert isinstance(result, ModuleType) + + +def test_import_python_module_error(): + """Test importing a Python module that is a .txt file.""" + module_path = "tests/profiles/test_two/test.txt" + module_name = "profile" + result = checks.import_python_module(module_name, module_path) + + assert result is None + + +def test_is_valid_profile(): + """Test if an imported profile is valid.""" + module_path = "tests/profiles/test/profile.py" + module_name = "profile" + fetched_module = checks.import_python_module(module_name, module_path) + result = False + if fetched_module: + result = checks.is_valid_profile(fetched_module) + + assert result is True diff --git a/tests/unit/utils/test_endpoints.py b/tests/unit/utils/test_endpoints.py index d9a496d0..04701ac4 100644 --- a/tests/unit/utils/test_endpoints.py +++ b/tests/unit/utils/test_endpoints.py @@ -6,13 +6,14 @@ import constants from configuration import AppConfig -from tests.unit import config_dict - +from models.config import CustomProfile from models.requests import QueryRequest from models.config import Action from utils import endpoints from utils.endpoints import get_agent +from tests.unit import config_dict + CONFIGURED_SYSTEM_PROMPT = "This is a configured system prompt" @@ -70,6 +71,42 @@ def config_with_custom_system_prompt_and_disable_query_system_prompt_fixture(): return cfg +@pytest.fixture( + name="config_with_custom_profile_prompt_and_enabled_query_system_prompt" +) +def config_with_custom_profile_prompt_and_enabled_query_system_prompt_fixture(): + """Configuration with custom profile loaded for prompt and disabled query system prompt set.""" + test_config = config_dict.copy() + + test_config["customization"] = { + "profile_path": "tests/profiles/test/profile.py", + "system_prompt": CONFIGURED_SYSTEM_PROMPT, + "disable_query_system_prompt": False, + } + cfg = AppConfig() + cfg.init_from_dict(test_config) + + return cfg + + +@pytest.fixture( + name="config_with_custom_profile_prompt_and_disable_query_system_prompt" +) +def config_with_custom_profile_prompt_and_disable_query_system_prompt_fixture(): + """Configuration with custom profile loaded for prompt and disabled query system prompt set.""" + test_config = config_dict.copy() + + test_config["customization"] = { + "profile_path": "tests/profiles/test/profile.py", + "system_prompt": CONFIGURED_SYSTEM_PROMPT, + "disable_query_system_prompt": True, + } + cfg = AppConfig() + cfg.init_from_dict(test_config) + + return cfg + + @pytest.fixture(name="query_request_without_system_prompt") def query_request_without_system_prompt_fixture(): """Fixture for query request without system prompt.""" @@ -175,6 +212,32 @@ def test_get_system_prompt_with_disable_query_system_prompt_and_non_system_promp assert system_prompt == CONFIGURED_SYSTEM_PROMPT +def test_get_profile_prompt_with_disable_query_system_prompt( + config_with_custom_profile_prompt_and_disable_query_system_prompt, + query_request_without_system_prompt, +): + """Test that system prompt is set if profile enabled and query system prompt disabled.""" + custom_profile = CustomProfile(path="tests/profiles/test/profile.py") + prompts = custom_profile.get_prompts() + system_prompt = endpoints.get_system_prompt( + query_request_without_system_prompt, + config_with_custom_profile_prompt_and_disable_query_system_prompt, + ) + assert system_prompt == prompts.get("default") + + +def test_get_profile_prompt_with_enabled_query_system_prompt( + config_with_custom_profile_prompt_and_enabled_query_system_prompt, + query_request_with_system_prompt, +): + """Test that profile system prompt is overridden by query system prompt enabled.""" + system_prompt = endpoints.get_system_prompt( + query_request_with_system_prompt, + config_with_custom_profile_prompt_and_enabled_query_system_prompt, + ) + assert system_prompt == query_request_with_system_prompt.system_prompt + + @pytest.mark.asyncio async def test_get_agent_with_conversation_id(prepare_agent_mocks, mocker): """Test get_agent function when agent exists in llama stack.""" From c4d532f142d745d04154c0b375d5a49b595d2209 Mon Sep 17 00:00:00 2001 From: Jordan Dubrick Date: Thu, 28 Aug 2025 16:23:31 -0400 Subject: [PATCH 2/8] remove unused logger Signed-off-by: Jordan Dubrick --- src/models/config.py | 3 --- 1 file changed, 3 deletions(-) diff --git a/src/models/config.py b/src/models/config.py index 5b793870..02a2c754 100644 --- a/src/models/config.py +++ b/src/models/config.py @@ -1,6 +1,5 @@ """Model with service configuration.""" -import logging from pathlib import Path from typing import Optional, Any, Pattern, Dict from enum import Enum @@ -27,8 +26,6 @@ from utils import checks -logger = logging.getLogger(__name__) - class ConfigurationBase(BaseModel): """Base class for all configuration models that rejects unknown fields.""" From 5d1b4567bae07ac4bb1fed7353cdc6f20510997b Mon Sep 17 00:00:00 2001 From: Jordan Dubrick Date: Thu, 28 Aug 2025 16:30:35 -0400 Subject: [PATCH 3/8] update profile section of readme Signed-off-by: Jordan Dubrick --- README.md | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/README.md b/README.md index 17cb0c5f..bc5eab9d 100644 --- a/README.md +++ b/README.md @@ -366,11 +366,11 @@ customization: #### Custom Profile -If you have added a custom profile specification to `/src/utils/profiles` and it exposes prompts you can set them with: +You can pass a custom prompt profile via it's `path` to the customization: ```yaml customization: - profile_name: + profile_path: ``` Additionally, an optional string parameter `system_prompt` can be specified in `/v1/query` and `/v1/streaming_query` endpoints to override the configured system prompt. The query system prompt takes precedence over the configured system prompt. You can use this config to disable query system prompts: From 2ff536d9395e0bc9258f35da0ca0d607d60e995c Mon Sep 17 00:00:00 2001 From: Jordan Dubrick Date: Tue, 2 Sep 2025 12:17:25 -0400 Subject: [PATCH 4/8] fix typos Signed-off-by: Jordan Dubrick --- README.md | 10 +++++----- tests/profiles/test/profile.py | 2 +- 2 files changed, 6 insertions(+), 6 deletions(-) diff --git a/README.md b/README.md index bc5eab9d..27b631f6 100644 --- a/README.md +++ b/README.md @@ -345,16 +345,16 @@ For data export integration with Red Hat's Dataverse, see the [Data Export Integ ## System prompt -The service uses the, so called, system prompt to put the question into context before the question is sent to the selceted LLM. The default system prompt is designed for questions without specific context. It is possible to use a different system prompt through various different avenues available in the `customization` section: +The service uses the, so called, system prompt to put the question into context before the question is sent to the selected LLM. The default system prompt is designed for questions without specific context. It is possible to use a different system prompt through various different avenues available in the `customization` section: -#### System Prompt Path +### System Prompt Path ```yaml customization: system_prompt_path: "system_prompts/system_prompt_for_product_XYZZY" ``` -#### System Prompt Literal +### System Prompt Literal ```yaml customization: @@ -364,9 +364,9 @@ customization: ``` -#### Custom Profile +### Custom Profile -You can pass a custom prompt profile via it's `path` to the customization: +You can pass a custom prompt profile via its `path` to the customization: ```yaml customization: diff --git a/tests/profiles/test/profile.py b/tests/profiles/test/profile.py index 060843ec..76698f39 100644 --- a/tests/profiles/test/profile.py +++ b/tests/profiles/test/profile.py @@ -27,7 +27,7 @@ QUESTION_VALIDATOR_PROMPT_TEMPLATE = f""" Instructions: -- You provide validation for tsting +- You provide validation for testing Example Question: How can I integrate GitOps into my pipeline? Example Response: From b399002ab320ffa2ee693892a3710d6415dd0d9c Mon Sep 17 00:00:00 2001 From: Jordan Dubrick Date: Tue, 2 Sep 2025 14:07:54 -0400 Subject: [PATCH 5/8] address coderabbit review Signed-off-by: Jordan Dubrick --- Makefile | 2 +- src/utils/checks.py | 26 ++++++++++++--- tests/profiles/test_three/profile.py | 49 ++++++++++++++++++++++++++++ tests/unit/utils/test_checks.py | 12 +++++++ 4 files changed, 83 insertions(+), 6 deletions(-) create mode 100644 tests/profiles/test_three/profile.py diff --git a/Makefile b/Makefile index 6d59b980..cdbcb73e 100644 --- a/Makefile +++ b/Makefile @@ -13,7 +13,7 @@ run: ## Run the service locally test-unit: ## Run the unit tests @echo "Running unit tests..." @echo "Reports will be written to ${ARTIFACT_DIR}" - COVERAGE_FILE="${ARTIFACT_DIR}/.coverage.unit" uv run python -m pytest tests/unit --cov=src --cov-report term-missing --cov-report "json:${ARTIFACT_DIR}/coverage_unit.json" --junit-xml="${ARTIFACT_DIR}/junit_unit.xml" --cov-fail-under=60 + COVERAGE_FILE="${ARTIFACT_DIR}/.coverage.unit" uv run python -m pytest tests/unit -v --cov=src --cov-report term-missing --cov-report "json:${ARTIFACT_DIR}/coverage_unit.json" --junit-xml="${ARTIFACT_DIR}/junit_unit.xml" --cov-fail-under=60 test-integration: ## Run integration tests tests @echo "Running integration tests..." diff --git a/src/utils/checks.py b/src/utils/checks.py index e2b0e548..43a8668a 100644 --- a/src/utils/checks.py +++ b/src/utils/checks.py @@ -31,12 +31,25 @@ def file_check(path: FilePath, desc: str) -> None: def import_python_module(profile_name: str, profile_path: str) -> ModuleType | None: """Import a Python module from a file path.""" + if not profile_path.endswith(".py"): + return None spec = importlib.util.spec_from_file_location(profile_name, profile_path) - if spec and spec.loader: - module = importlib.util.module_from_spec(spec) + if not spec or not spec.loader: + return None + module = importlib.util.module_from_spec(spec) + try: spec.loader.exec_module(module) - return module - return None + except ( + SyntaxError, + ImportError, + ModuleNotFoundError, + NameError, + AttributeError, + TypeError, + ValueError, + ): + return None + return module def is_valid_profile(profile_module: ModuleType) -> bool: @@ -48,4 +61,7 @@ def is_valid_profile(profile_module: ModuleType) -> bool: if not isinstance(profile_config, dict): return False - return "system_prompts" in profile_config + if not profile_config.get("system_prompts"): + return False + + return isinstance(profile_config.get("system_prompts"), dict) diff --git a/tests/profiles/test_three/profile.py b/tests/profiles/test_three/profile.py new file mode 100644 index 00000000..3c0f121b --- /dev/null +++ b/tests/profiles/test_three/profile.py @@ -0,0 +1,49 @@ +"""Custom profile for test profile.""" + +SUBJECT_ALLOWED = "ALLOWED" +SUBJECT_REJECTED = "REJECTED" + +# Default responses +INVALID_QUERY_RESP = ( + "Hi, I'm the Red Hat Developer Hub Lightspeed assistant, I can help you with questions about Red Hat Developer Hub or Backstage. " + "Please ensure your question is about these topics, and feel free to ask again!" +) + +QUERY_SYSTEM_INSTRUCTION = """ +1. Test +This is a test system instruction + +You achieve this by offering: +- testing +""" + +USE_CONTEXT_INSTRUCTION = """ +Use the retrieved document to answer the question. +""" + +USE_HISTORY_INSTRUCTION = """ +Use the previous chat history to interact and help the user. +""" + +QUESTION_VALIDATOR_PROMPT_TEMPLATE = f""" +Instructions: +- You provide validation for testing +Example Question: +How can I integrate GitOps into my pipeline? +Example Response: +{SUBJECT_ALLOWED} +""" + +TOPIC_SUMMARY_PROMPT_TEMPLATE = """ +Instructions: +- You are a topic summarizer +- For testing +- Your job is to extract precise topic summary from user input + +Example Input: +Testing placeholder +Example Output: +Proper response test. +""" + +PROFILE_CONFIG = ({"system_prompts": QUERY_SYSTEM_INSTRUCTION},) diff --git a/tests/unit/utils/test_checks.py b/tests/unit/utils/test_checks.py index 5671df5e..89bd0271 100644 --- a/tests/unit/utils/test_checks.py +++ b/tests/unit/utils/test_checks.py @@ -109,3 +109,15 @@ def test_is_valid_profile(): result = checks.is_valid_profile(fetched_module) assert result is True + + +def test_invalid_profile(): + """Test if an imported profile is valid (expect invalid)""" + module_path = "tests/profiles/test_three/profile.py" + module_name = "profile" + fetched_module = checks.import_python_module(module_name, module_path) + result = False + if fetched_module: + result = checks.is_valid_profile(fetched_module) + + assert result is False From 4590bdf88adf1fa875788652bab84c40b85bf106 Mon Sep 17 00:00:00 2001 From: Jordan Dubrick Date: Tue, 2 Sep 2025 14:08:30 -0400 Subject: [PATCH 6/8] fix typo Signed-off-by: Jordan Dubrick --- Makefile | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/Makefile b/Makefile index cdbcb73e..6d59b980 100644 --- a/Makefile +++ b/Makefile @@ -13,7 +13,7 @@ run: ## Run the service locally test-unit: ## Run the unit tests @echo "Running unit tests..." @echo "Reports will be written to ${ARTIFACT_DIR}" - COVERAGE_FILE="${ARTIFACT_DIR}/.coverage.unit" uv run python -m pytest tests/unit -v --cov=src --cov-report term-missing --cov-report "json:${ARTIFACT_DIR}/coverage_unit.json" --junit-xml="${ARTIFACT_DIR}/junit_unit.xml" --cov-fail-under=60 + COVERAGE_FILE="${ARTIFACT_DIR}/.coverage.unit" uv run python -m pytest tests/unit --cov=src --cov-report term-missing --cov-report "json:${ARTIFACT_DIR}/coverage_unit.json" --junit-xml="${ARTIFACT_DIR}/junit_unit.xml" --cov-fail-under=60 test-integration: ## Run integration tests tests @echo "Running integration tests..." From 6960ebdfc60d5a083468ab67c14edb967d6dbc7b Mon Sep 17 00:00:00 2001 From: Jordan Dubrick Date: Tue, 2 Sep 2025 14:43:50 -0400 Subject: [PATCH 7/8] Update README.md Co-authored-by: coderabbitai[bot] <136622811+coderabbitai[bot]@users.noreply.github.com> --- README.md | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/README.md b/README.md index 27b631f6..5628ff20 100644 --- a/README.md +++ b/README.md @@ -345,8 +345,7 @@ For data export integration with Red Hat's Dataverse, see the [Data Export Integ ## System prompt -The service uses the, so called, system prompt to put the question into context before the question is sent to the selected LLM. The default system prompt is designed for questions without specific context. It is possible to use a different system prompt through various different avenues available in the `customization` section: - +The service uses a so-called system prompt to put the question into context before it is sent to the selected LLM. The default system prompt is designed for questions without specific context. You can supply a different system prompt through various avenues available in the `customization` section: ### System Prompt Path ```yaml From 8171a1e3cd95138dc5a95064c2c417fd7afbc341 Mon Sep 17 00:00:00 2001 From: Jordan Dubrick Date: Wed, 3 Sep 2025 09:59:09 -0400 Subject: [PATCH 8/8] swap from Dict to built-in dict Signed-off-by: Jordan Dubrick --- src/models/config.py | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/src/models/config.py b/src/models/config.py index 02a2c754..f3994335 100644 --- a/src/models/config.py +++ b/src/models/config.py @@ -1,7 +1,7 @@ """Model with service configuration.""" from pathlib import Path -from typing import Optional, Any, Pattern, Dict +from typing import Optional, Any, Pattern from enum import Enum from functools import cached_property import re @@ -420,7 +420,7 @@ class CustomProfile: """Custom profile customization for prompts and validation.""" path: str - prompts: Dict[str, str] = Field(default={}, init=False) + prompts: dict[str, str] = Field(default={}, init=False) def __post_init__(self) -> None: """Validate and load profile.""" @@ -433,7 +433,7 @@ def _validate_and_process(self) -> None: if profile_module is not None and checks.is_valid_profile(profile_module): self.prompts = profile_module.PROFILE_CONFIG.get("system_prompts", {}) - def get_prompts(self) -> Dict[str, str]: + def get_prompts(self) -> dict[str, str]: """Retrieve prompt attribute.""" return self.prompts