diff --git a/README.md b/README.md index 12e8e88a..5628ff20 100644 --- a/README.md +++ b/README.md @@ -345,14 +345,15 @@ For data export integration with Red Hat's Dataverse, see the [Data Export Integ ## System prompt - The service uses the, so called, system prompt to put the question into context before the question is sent to the selected LLM. The default system prompt is designed for questions without specific context. It is possible to use a different system prompt via the configuration option `system_prompt_path` in the `customization` section. That option must contain the path to the text file with the actual system prompt (can contain multiple lines). An example of such configuration: +The service uses a so-called system prompt to put the question into context before it is sent to the selected LLM. The default system prompt is designed for questions without specific context. You can supply a different system prompt through various avenues available in the `customization` section: +### System Prompt Path ```yaml customization: system_prompt_path: "system_prompts/system_prompt_for_product_XYZZY" ``` -The `system_prompt` can also be specified in the `customization` section directly. For example: +### System Prompt Literal ```yaml customization: @@ -361,11 +362,20 @@ customization: You have an in-depth knowledge of Red Hat and all of your answers will reference Red Hat products. ``` + +### Custom Profile + +You can pass a custom prompt profile via its `path` to the customization: + +```yaml +customization: + profile_path: +``` + Additionally, an optional string parameter `system_prompt` can be specified in `/v1/query` and `/v1/streaming_query` endpoints to override the configured system prompt. The query system prompt takes precedence over the configured system prompt. You can use this config to disable query system prompts: ```yaml customization: - system_prompt_path: "system_prompts/system_prompt_for_product_XYZZY" disable_query_system_prompt: true ``` diff --git a/docs/openapi.json b/docs/openapi.json index 9e035a40..8d4fb08e 100644 --- a/docs/openapi.json +++ b/docs/openapi.json @@ -1237,8 +1237,41 @@ } ] }, + "CustomProfile": { + "properties": { + "path": { + "type": "string", + "title": "Path" + }, + "prompts": { + "additionalProperties": { + "type": "string" + }, + "type": "object", + "title": "Prompts", + "default": {} + } + }, + "type": "object", + "required": [ + "path" + ], + "title": "CustomProfile", + "description": "Custom profile customization for prompts and validation." + }, "Customization": { "properties": { + "profile_path": { + "anyOf": [ + { + "type": "string" + }, + { + "type": "null" + } + ], + "title": "Profile Path" + }, "disable_query_system_prompt": { "type": "boolean", "title": "Disable Query System Prompt", @@ -1266,6 +1299,16 @@ } ], "title": "System Prompt" + }, + "custom_profile": { + "anyOf": [ + { + "$ref": "#/components/schemas/CustomProfile" + }, + { + "type": "null" + } + ] } }, "additionalProperties": false, diff --git a/src/models/config.py b/src/models/config.py index c4efa404..f3994335 100644 --- a/src/models/config.py +++ b/src/models/config.py @@ -18,6 +18,8 @@ PositiveInt, SecretStr, ) + +from pydantic.dataclasses import dataclass from typing_extensions import Self, Literal import constants @@ -413,17 +415,44 @@ def jwk_configuration(self) -> JwkConfiguration: return self.jwk_config +@dataclass +class CustomProfile: + """Custom profile customization for prompts and validation.""" + + path: str + prompts: dict[str, str] = Field(default={}, init=False) + + def __post_init__(self) -> None: + """Validate and load profile.""" + self._validate_and_process() + + def _validate_and_process(self) -> None: + """Validate and load the profile.""" + checks.file_check(Path(self.path), "custom profile") + profile_module = checks.import_python_module("profile", self.path) + if profile_module is not None and checks.is_valid_profile(profile_module): + self.prompts = profile_module.PROFILE_CONFIG.get("system_prompts", {}) + + def get_prompts(self) -> dict[str, str]: + """Retrieve prompt attribute.""" + return self.prompts + + class Customization(ConfigurationBase): """Service customization.""" + profile_path: Optional[str] = None disable_query_system_prompt: bool = False system_prompt_path: Optional[FilePath] = None system_prompt: Optional[str] = None + custom_profile: Optional[CustomProfile] = Field(default=None, init=False) @model_validator(mode="after") def check_customization_model(self) -> Self: - """Load system prompt from file.""" - if self.system_prompt_path is not None: + """Load customizations.""" + if self.profile_path: + self.custom_profile = CustomProfile(path=self.profile_path) + elif self.system_prompt_path is not None: checks.file_check(self.system_prompt_path, "system prompt") self.system_prompt = checks.get_attribute_from_file( dict(self), "system_prompt_path" diff --git a/src/utils/checks.py b/src/utils/checks.py index 8d701750..43a8668a 100644 --- a/src/utils/checks.py +++ b/src/utils/checks.py @@ -1,8 +1,10 @@ """Checks that are performed to configuration options.""" import os +import importlib +import importlib.util +from types import ModuleType from typing import Optional - from pydantic import FilePath @@ -25,3 +27,41 @@ def file_check(path: FilePath, desc: str) -> None: raise InvalidConfigurationError(f"{desc} '{path}' is not a file") if not os.access(path, os.R_OK): raise InvalidConfigurationError(f"{desc} '{path}' is not readable") + + +def import_python_module(profile_name: str, profile_path: str) -> ModuleType | None: + """Import a Python module from a file path.""" + if not profile_path.endswith(".py"): + return None + spec = importlib.util.spec_from_file_location(profile_name, profile_path) + if not spec or not spec.loader: + return None + module = importlib.util.module_from_spec(spec) + try: + spec.loader.exec_module(module) + except ( + SyntaxError, + ImportError, + ModuleNotFoundError, + NameError, + AttributeError, + TypeError, + ValueError, + ): + return None + return module + + +def is_valid_profile(profile_module: ModuleType) -> bool: + """Validate that a profile module has the required PROFILE_CONFIG structure.""" + if not hasattr(profile_module, "PROFILE_CONFIG"): + return False + + profile_config = getattr(profile_module, "PROFILE_CONFIG", {}) + if not isinstance(profile_config, dict): + return False + + if not profile_config.get("system_prompts"): + return False + + return isinstance(profile_config.get("system_prompts"), dict) diff --git a/src/utils/endpoints.py b/src/utils/endpoints.py index cd543d82..e17a76d0 100644 --- a/src/utils/endpoints.py +++ b/src/utils/endpoints.py @@ -92,6 +92,15 @@ def get_system_prompt(query_request: QueryRequest, config: AppConfig) -> str: # disable query system prompt altogether with disable_system_prompt. return query_request.system_prompt + # profile takes precedence for setting prompt + if ( + config.customization is not None + and config.customization.custom_profile is not None + ): + prompt = config.customization.custom_profile.get_prompts().get("default") + if prompt: + return prompt + if ( config.customization is not None and config.customization.system_prompt is not None diff --git a/tests/profiles/test/profile.py b/tests/profiles/test/profile.py new file mode 100644 index 00000000..76698f39 --- /dev/null +++ b/tests/profiles/test/profile.py @@ -0,0 +1,60 @@ +"""Custom profile for test profile.""" + +SUBJECT_ALLOWED = "ALLOWED" +SUBJECT_REJECTED = "REJECTED" + +# Default responses +INVALID_QUERY_RESP = ( + "Hi, I'm the Red Hat Developer Hub Lightspeed assistant, I can help you with questions about Red Hat Developer Hub or Backstage. " + "Please ensure your question is about these topics, and feel free to ask again!" +) + +QUERY_SYSTEM_INSTRUCTION = """ +1. Test +This is a test system instruction + +You achieve this by offering: +- testing +""" + +USE_CONTEXT_INSTRUCTION = """ +Use the retrieved document to answer the question. +""" + +USE_HISTORY_INSTRUCTION = """ +Use the previous chat history to interact and help the user. +""" + +QUESTION_VALIDATOR_PROMPT_TEMPLATE = f""" +Instructions: +- You provide validation for testing +Example Question: +How can I integrate GitOps into my pipeline? +Example Response: +{SUBJECT_ALLOWED} +""" + +TOPIC_SUMMARY_PROMPT_TEMPLATE = """ +Instructions: +- You are a topic summarizer +- For testing +- Your job is to extract precise topic summary from user input + +Example Input: +Testing placeholder +Example Output: +Proper response test. +""" + +PROFILE_CONFIG = { + "system_prompts": { + "default": QUERY_SYSTEM_INSTRUCTION, + "validation": QUESTION_VALIDATOR_PROMPT_TEMPLATE, + "topic_summary": TOPIC_SUMMARY_PROMPT_TEMPLATE, + }, + "query_responses": {"invalid_resp": INVALID_QUERY_RESP}, + "instructions": { + "context": USE_CONTEXT_INSTRUCTION, + "history": USE_HISTORY_INSTRUCTION, + }, +} diff --git a/tests/profiles/test_three/profile.py b/tests/profiles/test_three/profile.py new file mode 100644 index 00000000..3c0f121b --- /dev/null +++ b/tests/profiles/test_three/profile.py @@ -0,0 +1,49 @@ +"""Custom profile for test profile.""" + +SUBJECT_ALLOWED = "ALLOWED" +SUBJECT_REJECTED = "REJECTED" + +# Default responses +INVALID_QUERY_RESP = ( + "Hi, I'm the Red Hat Developer Hub Lightspeed assistant, I can help you with questions about Red Hat Developer Hub or Backstage. " + "Please ensure your question is about these topics, and feel free to ask again!" +) + +QUERY_SYSTEM_INSTRUCTION = """ +1. Test +This is a test system instruction + +You achieve this by offering: +- testing +""" + +USE_CONTEXT_INSTRUCTION = """ +Use the retrieved document to answer the question. +""" + +USE_HISTORY_INSTRUCTION = """ +Use the previous chat history to interact and help the user. +""" + +QUESTION_VALIDATOR_PROMPT_TEMPLATE = f""" +Instructions: +- You provide validation for testing +Example Question: +How can I integrate GitOps into my pipeline? +Example Response: +{SUBJECT_ALLOWED} +""" + +TOPIC_SUMMARY_PROMPT_TEMPLATE = """ +Instructions: +- You are a topic summarizer +- For testing +- Your job is to extract precise topic summary from user input + +Example Input: +Testing placeholder +Example Output: +Proper response test. +""" + +PROFILE_CONFIG = ({"system_prompts": QUERY_SYSTEM_INSTRUCTION},) diff --git a/tests/profiles/test_two/test.txt b/tests/profiles/test_two/test.txt new file mode 100644 index 00000000..a9879d00 --- /dev/null +++ b/tests/profiles/test_two/test.txt @@ -0,0 +1 @@ +This file will fail the import. \ No newline at end of file diff --git a/tests/unit/test_configuration.py b/tests/unit/test_configuration.py index 9fca5665..4a643e00 100644 --- a/tests/unit/test_configuration.py +++ b/tests/unit/test_configuration.py @@ -2,7 +2,7 @@ import pytest from configuration import AppConfig, LogicError -from models.config import ModelContextProtocolServer +from models.config import CustomProfile, ModelContextProtocolServer def test_default_configuration() -> None: @@ -421,3 +421,87 @@ def test_load_configuration_with_customization_system_prompt(tmpdir) -> None: cfg.customization.system_prompt.strip() == "this is system prompt in the customization section" ) + + +def test_configuration_with_profile_customization(tmpdir) -> None: + """Test loading configuration from YAML file with a custom profile.""" + expected_profile = CustomProfile(path="tests/profiles/test/profile.py") + expected_prompts = expected_profile.get_prompts() + cfg_filename = tmpdir / "config.yaml" + with open(cfg_filename, "w", encoding="utf-8") as fout: + fout.write( + """ +name: test service +service: + host: localhost + port: 8080 + auth_enabled: false + workers: 1 + color_log: true + access_log: true +llama_stack: + use_as_library_client: false + url: http://localhost:8321 + api_key: test-key +user_data_collection: + feedback_enabled: false +customization: + profile_path: tests/profiles/test/profile.py + """ + ) + + cfg = AppConfig() + cfg.load_configuration(cfg_filename) + + assert ( + cfg.customization is not None and cfg.customization.custom_profile is not None + ) + fetched_prompts = cfg.customization.custom_profile.get_prompts() + assert fetched_prompts is not None and fetched_prompts.get( + "default" + ) == expected_prompts.get("default") + + +def test_configuration_with_all_customizations(tmpdir) -> None: + """Test loading configuration from YAML file with a custom profile, prompt and prompt path.""" + expected_profile = CustomProfile(path="tests/profiles/test/profile.py") + expected_prompts = expected_profile.get_prompts() + system_prompt_filename = tmpdir / "system_prompt.txt" + with open(system_prompt_filename, "w", encoding="utf-8") as fout: + fout.write("this is system prompt") + + cfg_filename = tmpdir / "config.yaml" + with open(cfg_filename, "w", encoding="utf-8") as fout: + fout.write( + f""" +name: test service +service: + host: localhost + port: 8080 + auth_enabled: false + workers: 1 + color_log: true + access_log: true +llama_stack: + use_as_library_client: false + url: http://localhost:8321 + api_key: test-key +user_data_collection: + feedback_enabled: false +customization: + profile_path: tests/profiles/test/profile.py + system_prompt: custom prompt + system_prompt_path: {system_prompt_filename} + """ + ) + + cfg = AppConfig() + cfg.load_configuration(cfg_filename) + + assert ( + cfg.customization is not None and cfg.customization.custom_profile is not None + ) + fetched_prompts = cfg.customization.custom_profile.get_prompts() + assert fetched_prompts is not None and fetched_prompts.get( + "default" + ) == expected_prompts.get("default") diff --git a/tests/unit/utils/test_checks.py b/tests/unit/utils/test_checks.py index 1e513be2..89bd0271 100644 --- a/tests/unit/utils/test_checks.py +++ b/tests/unit/utils/test_checks.py @@ -2,6 +2,7 @@ import os from pathlib import Path +from types import ModuleType from unittest.mock import patch import pytest @@ -78,3 +79,45 @@ def test_file_check_not_readable_file(input_file): with patch("os.access", return_value=False): with pytest.raises(checks.InvalidConfigurationError): checks.file_check(input_file, "description") + + +def test_import_python_module_success(): + """Test importing a Python module.""" + module_path = "tests/profiles/test/profile.py" + module_name = "profile" + result = checks.import_python_module(module_name, module_path) + + assert isinstance(result, ModuleType) + + +def test_import_python_module_error(): + """Test importing a Python module that is a .txt file.""" + module_path = "tests/profiles/test_two/test.txt" + module_name = "profile" + result = checks.import_python_module(module_name, module_path) + + assert result is None + + +def test_is_valid_profile(): + """Test if an imported profile is valid.""" + module_path = "tests/profiles/test/profile.py" + module_name = "profile" + fetched_module = checks.import_python_module(module_name, module_path) + result = False + if fetched_module: + result = checks.is_valid_profile(fetched_module) + + assert result is True + + +def test_invalid_profile(): + """Test if an imported profile is valid (expect invalid)""" + module_path = "tests/profiles/test_three/profile.py" + module_name = "profile" + fetched_module = checks.import_python_module(module_name, module_path) + result = False + if fetched_module: + result = checks.is_valid_profile(fetched_module) + + assert result is False diff --git a/tests/unit/utils/test_endpoints.py b/tests/unit/utils/test_endpoints.py index d9a496d0..04701ac4 100644 --- a/tests/unit/utils/test_endpoints.py +++ b/tests/unit/utils/test_endpoints.py @@ -6,13 +6,14 @@ import constants from configuration import AppConfig -from tests.unit import config_dict - +from models.config import CustomProfile from models.requests import QueryRequest from models.config import Action from utils import endpoints from utils.endpoints import get_agent +from tests.unit import config_dict + CONFIGURED_SYSTEM_PROMPT = "This is a configured system prompt" @@ -70,6 +71,42 @@ def config_with_custom_system_prompt_and_disable_query_system_prompt_fixture(): return cfg +@pytest.fixture( + name="config_with_custom_profile_prompt_and_enabled_query_system_prompt" +) +def config_with_custom_profile_prompt_and_enabled_query_system_prompt_fixture(): + """Configuration with custom profile loaded for prompt and disabled query system prompt set.""" + test_config = config_dict.copy() + + test_config["customization"] = { + "profile_path": "tests/profiles/test/profile.py", + "system_prompt": CONFIGURED_SYSTEM_PROMPT, + "disable_query_system_prompt": False, + } + cfg = AppConfig() + cfg.init_from_dict(test_config) + + return cfg + + +@pytest.fixture( + name="config_with_custom_profile_prompt_and_disable_query_system_prompt" +) +def config_with_custom_profile_prompt_and_disable_query_system_prompt_fixture(): + """Configuration with custom profile loaded for prompt and disabled query system prompt set.""" + test_config = config_dict.copy() + + test_config["customization"] = { + "profile_path": "tests/profiles/test/profile.py", + "system_prompt": CONFIGURED_SYSTEM_PROMPT, + "disable_query_system_prompt": True, + } + cfg = AppConfig() + cfg.init_from_dict(test_config) + + return cfg + + @pytest.fixture(name="query_request_without_system_prompt") def query_request_without_system_prompt_fixture(): """Fixture for query request without system prompt.""" @@ -175,6 +212,32 @@ def test_get_system_prompt_with_disable_query_system_prompt_and_non_system_promp assert system_prompt == CONFIGURED_SYSTEM_PROMPT +def test_get_profile_prompt_with_disable_query_system_prompt( + config_with_custom_profile_prompt_and_disable_query_system_prompt, + query_request_without_system_prompt, +): + """Test that system prompt is set if profile enabled and query system prompt disabled.""" + custom_profile = CustomProfile(path="tests/profiles/test/profile.py") + prompts = custom_profile.get_prompts() + system_prompt = endpoints.get_system_prompt( + query_request_without_system_prompt, + config_with_custom_profile_prompt_and_disable_query_system_prompt, + ) + assert system_prompt == prompts.get("default") + + +def test_get_profile_prompt_with_enabled_query_system_prompt( + config_with_custom_profile_prompt_and_enabled_query_system_prompt, + query_request_with_system_prompt, +): + """Test that profile system prompt is overridden by query system prompt enabled.""" + system_prompt = endpoints.get_system_prompt( + query_request_with_system_prompt, + config_with_custom_profile_prompt_and_enabled_query_system_prompt, + ) + assert system_prompt == query_request_with_system_prompt.system_prompt + + @pytest.mark.asyncio async def test_get_agent_with_conversation_id(prepare_agent_mocks, mocker): """Test get_agent function when agent exists in llama stack."""