From b8873b84ee15fc2e009d19776e94cebb37c46c4c Mon Sep 17 00:00:00 2001 From: Fahad Ali Shaikh Date: Wed, 3 Jul 2024 15:27:48 +0530 Subject: [PATCH] Litellm (#1265) * litellm base version * 1. added missing test cases 2. removed stream from hyperparameters from prompt action * test cased fixed * 1. added missing test case 2. updated litellm * 1. added missing test case * 1. added missing test case * 1. added missing test case 2. removed deprecated api * test cases fixed * removed unused variable * fixed unused variable * fixed unused variable * added test cased for fetching logs * litellm base version * 1. added missing test cases 2. removed stream from hyperparameters from prompt action * test cased fixed * 1. added missing test case 2. updated litellm * 1. added missing test case * 1. added missing test case * 1. added missing test case 2. removed deprecated api * test cases fixed * removed unused variable * fixed unused variable * fixed unused variable * added test cased for fetching logs * added test cased for fetching logs * removed unused import * added invocation in metadata for litellm * 1. changed rasa rule policy to allow max history 2. changed rasa domain.yml schemas to allow unicode Alphabets for slots and form name * litellm base version * 1. added missing test cases 2. removed stream from hyperparameters from prompt action * test cased fixed * 1. added missing test case 2. updated litellm * 1. added missing test case * 1. added missing test case * 1. added missing test case 2. removed deprecated api * test cases fixed * removed unused variable * fixed unused variable * fixed unused variable * added test cased for fetching logs * added test cased for fetching logs * litellm base version * 1. added missing test cases 2. removed stream from hyperparameters from prompt action * test cased fixed * 1. added missing test case 2. updated litellm * 1. added missing test case * 1. added missing test case * 1. added missing test case 2. removed deprecated api * test cases fixed * removed unused variable * fixed unused variable * fixed unused variable * added test cased for fetching logs * removed unused import * added invocation in metadata for litellm * 1. changed rasa rule policy to allow max history 2. changed rasa domain.yml schemas to allow unicode Alphabets for slots and form name * test cases fixed after merging --- augmentation/paraphrase/gpt3/gpt.py | 7 +- custom/__init__.py | 0 custom/fallback.py | 58 - custom/ner.py | 169 --- docker/Dockerfile | 2 + kairon/actions/definitions/database.py | 2 +- kairon/actions/definitions/prompt.py | 31 +- kairon/api/app/routers/bot/bot.py | 23 +- kairon/api/models.py | 28 +- kairon/chat/agent/message_processor.py | 9 - kairon/importer/validator/file_validator.py | 31 +- kairon/shared/actions/data_objects.py | 8 +- kairon/shared/actions/utils.py | 3 +- .../concurrency/actors/pyscript_runner.py | 10 +- kairon/shared/data/constant.py | 1 + kairon/shared/data/processor.py | 4 +- kairon/shared/llm/base.py | 4 +- kairon/shared/llm/clients/__init__.py | 0 kairon/shared/llm/clients/azure.py | 25 - kairon/shared/llm/clients/base.py | 8 - kairon/shared/llm/clients/factory.py | 18 - kairon/shared/llm/clients/gpt3.py | 92 -- kairon/shared/llm/data_objects.py | 13 + kairon/shared/llm/factory.py | 17 - kairon/shared/llm/logger.py | 41 + kairon/shared/llm/{gpt3.py => processor.py} | 135 +- kairon/shared/rest_client.py | 31 +- kairon/shared/schemas/domain.yml | 142 +++ kairon/shared/utils.py | 85 +- kairon/shared/vector_embeddings/db/base.py | 8 +- kairon/shared/vector_embeddings/db/qdrant.py | 30 +- kairon/train.py | 9 +- metadata/integrations.yml | 123 +- requirements/dev.txt | 13 +- requirements/prod.txt | 68 +- tests/conftest.py | 1 - tests/integration_test/action_service_test.py | 1101 ++++++++++------- tests/integration_test/chat_service_test.py | 12 +- tests/integration_test/event_service_test.py | 2 +- .../integration_test/history_services_test.py | 2 +- tests/integration_test/services_test.py | 318 ++++- tests/unit_test/action/action_test.py | 11 +- tests/unit_test/api/api_processor_test.py | 9 +- .../augmentation/gpt_augmentation_test.py | 10 +- tests/unit_test/chat/chat_test.py | 27 +- tests/unit_test/cli_test.py | 17 +- .../data_processor/agent_processor_test.py | 2 +- .../data_processor/data_processor_test.py | 211 ++-- .../unit_test/data_processor/history_test.py | 2 +- tests/unit_test/events/definitions_test.py | 4 +- tests/unit_test/events/events_test.py | 13 +- tests/unit_test/events/scheduler_test.py | 2 +- tests/unit_test/idp/test_idp_helper.py | 1 - tests/unit_test/llm_test.py | 1096 +++++++++------- tests/unit_test/plugins_test.py | 3 +- tests/unit_test/rest_client_test.py | 17 + tests/unit_test/utility_test.py | 746 +---------- .../validator/training_data_validator_test.py | 73 +- .../vector_embeddings/qdrant_test.py | 31 +- tests/unit_test/verification_test.py | 2 +- training_data/ReadMe.md | 1 - 61 files changed, 2320 insertions(+), 2642 deletions(-) delete mode 100644 custom/__init__.py delete mode 100644 custom/fallback.py delete mode 100644 custom/ner.py delete mode 100644 kairon/shared/llm/clients/__init__.py delete mode 100644 kairon/shared/llm/clients/azure.py delete mode 100644 kairon/shared/llm/clients/base.py delete mode 100644 kairon/shared/llm/clients/factory.py delete mode 100644 kairon/shared/llm/clients/gpt3.py create mode 100644 kairon/shared/llm/data_objects.py delete mode 100644 kairon/shared/llm/factory.py create mode 100644 kairon/shared/llm/logger.py rename kairon/shared/llm/{gpt3.py => processor.py} (67%) create mode 100644 kairon/shared/schemas/domain.yml delete mode 100644 training_data/ReadMe.md diff --git a/augmentation/paraphrase/gpt3/gpt.py b/augmentation/paraphrase/gpt3/gpt.py index 340c220c0..b11a8eac5 100644 --- a/augmentation/paraphrase/gpt3/gpt.py +++ b/augmentation/paraphrase/gpt3/gpt.py @@ -1,7 +1,7 @@ """Creates the Example and GPT classes for a user to interface with the OpenAI API.""" -import openai +from openai import OpenAI import uuid @@ -95,8 +95,9 @@ def submit_request(self, prompt, num_responses, api_key): """Calls the OpenAI API with the specified parameters.""" if num_responses < 1: num_responses = 1 - response = openai.Completion.create(api_key=api_key, - engine=self.get_engine(), + client = OpenAI(api_key=api_key) + response = client.completions.create( + model=self.get_engine(), prompt=self.craft_query(prompt), max_tokens=self.get_max_tokens(), temperature=self.get_temperature(), diff --git a/custom/__init__.py b/custom/__init__.py deleted file mode 100644 index e69de29bb..000000000 diff --git a/custom/fallback.py b/custom/fallback.py deleted file mode 100644 index 4c396c039..000000000 --- a/custom/fallback.py +++ /dev/null @@ -1,58 +0,0 @@ -''' -Custom component to get fallback action intent -Reference: https://forum.rasa.com/t/fallback-intents-for-context-sensitive-fallbacks/963 -''' - -from rasa.nlu.classifiers.classifier import IntentClassifier - -class FallbackIntentFilter(IntentClassifier): - - # Name of the component to be used when integrating it in a - # pipeline. E.g. ``[ComponentA, ComponentB]`` - # will be a proper pipeline definition where ``ComponentA`` - # is the name of the first component of the pipeline. - name = "FallbackIntentFilter" - - # Defines what attributes the pipeline component will - # provide when called. The listed attributes - # should be set by the component on the message object - # during test and train, e.g. - # ```message.set("entities", [...])``` - provides = [] - - # Which attributes on a message are required by this - # component. e.g. if requires contains "tokens", than a - # previous component in the pipeline needs to have "tokens" - # within the above described `provides` property. - requires = [] - - # Defines the default configuration parameters of a component - # these values can be overwritten in the pipeline configuration - # of the model. The component should choose sensible defaults - # and should be able to create reasonable results with the defaults. - defaults = {} - - # Defines what language(s) this component can handle. - # This attribute is designed for instance method: `can_handle_language`. - # Default value is None which means it can handle all languages. - # This is an important feature for backwards compatibility of components. - language_list = None - - def __init__(self, component_config=None, low_threshold=0.3, high_threshold=0.4, fallback_intent="fallback", - out_of_scope_intent="out_of_scope"): - super().__init__(component_config) - self.fb_low_threshold = low_threshold - self.fb_high_threshold = high_threshold - self.fallback_intent = fallback_intent - self.out_of_scope_intent = out_of_scope_intent - - def process(self, message, **kwargs): - message_confidence = message.data['intent']['confidence'] - new_intent = None - if message_confidence <= self.fb_low_threshold: - new_intent = {'name': self.out_of_scope_intent, 'confidence': message_confidence} - elif message_confidence <= self.fb_high_threshold: - new_intent = {'name': self.fallback_intent, 'confidence': message_confidence} - if new_intent is not None: - message.data['intent'] = new_intent - message.data['intent_ranking'].insert(0, new_intent) diff --git a/custom/ner.py b/custom/ner.py deleted file mode 100644 index 1a38897fe..000000000 --- a/custom/ner.py +++ /dev/null @@ -1,169 +0,0 @@ -from rasa.nlu.components import Component -from typing import Any, Optional, Text, Dict, TYPE_CHECKING -import os -import spacy -import pickle -from spacy.matcher import Matcher -from rasa.nlu.extractors.extractor import EntityExtractor - - -if TYPE_CHECKING: - from rasa.nlu.model import Metadata - -PATTERN_NER_FILE = 'pattern_ner.pkl' -class SpacyPatternNER(EntityExtractor): - """A new component""" - name = "pattern_ner_spacy" - # Defines what attributes the pipeline component will - # provide when called. The listed attributes - # should be set by the component on the message object - # during test and train, e.g. - # ```message.set("entities", [...])``` - provides = ["entities"] - - # Which attributes on a message are required by this - # component. e.g. if requires contains "tokens", than a - # previous component in the pipeline needs to have "tokens" - # within the above described `provides` property. - requires = ["tokens"] - - # Defines the default configuration parameters of a component - # these values can be overwritten in the pipeline configuration - # of the model. The component should choose sensible defaults - # and should be able to create reasonable results with the defaults. - defaults = {} - - # Defines what language(s) this component can handle. - # This attribute is designed for instance method: `can_handle_language`. - # Default value is None which means it can handle all languages. - # This is an important feature for backwards compatibility of components. - language_list = None - - def __init__(self, component_config=None, matcher=None): - super(SpacyPatternNER, self).__init__(component_config) - if matcher: - self.matcher = matcher - self.spacy_nlp = spacy.blank('en') - self.spacy_nlp.vocab = self.matcher.vocab - else: - self.spacy_nlp = spacy.blank('en') - self.matcher = Matcher(self.spacy_nlp.vocab) - - def train(self, training_data, cfg, **kwargs): - """Train this component. - - This is the components chance to train itself provided - with the training data. The component can rely on - any context attribute to be present, that gets created - by a call to :meth:`components.Component.pipeline_init` - of ANY component and - on any context attributes created by a call to - :meth:`components.Component.train` - of components previous to this one.""" - for lookup_table in training_data.lookup_tables: - key = lookup_table['name'] - pattern = [] - for element in lookup_table['elements']: - tokens = [{'LOWER': token.lower()} for token in str(element).split()] - pattern.append(tokens) - self.matcher.add(key, pattern) - - def process(self, message, **kwargs): - """Process an incoming message. - - This is the components chance to process an incoming - message. The component can rely on - any context attribute to be present, that gets created - by a call to :meth:`components.Component.pipeline_init` - of ANY component and - on any context attributes created by a call to - :meth:`components.Component.process` - of components previous to this one.""" - entities = [] - - # with plural forms - doc = self.spacy_nlp(message.data['text'].lower()) - matches = self.matcher(doc) - entities = self.getNewEntityObj(doc, matches, entities) - - # Without plural forms - doc = self.spacy_nlp(' '.join([token.lemma_ for token in doc])) - matches = self.matcher(doc) - entities = self.getNewEntityObj(doc, matches, entities) - - # Remove duplicates - seen = set() - new_entities = [] - - for entityObj in entities: - record = tuple(entityObj.items()) - if record not in seen: - seen.add(record) - new_entities.append(entityObj) - - message.set("entities", message.get("entities", []) + new_entities, add_to_output=True) - - - def getNewEntityObj(self, doc, matches, entities): - - for ent_id, start, end in matches: - new_entity_value = doc[start:end].text - new_entity_value_len = len(new_entity_value.split()) - is_add = True - - for old_entity in entities: - old_entity_value = old_entity["value"] - old_entity_value_len = len(old_entity_value.split()) - - if old_entity_value_len > new_entity_value_len and new_entity_value in old_entity_value: - is_add = False - elif old_entity_value_len < new_entity_value_len and old_entity_value in new_entity_value: - entities.remove(old_entity) - - if is_add: - entities.append({ - 'start': start, - 'end': end, - 'value': doc[start:end].text, - 'entity': self.matcher.vocab.strings[ent_id], - 'confidence': None, - 'extractor': self.name - }) - - return entities - - - def persist(self, file_name: Text, model_dir: Text) -> Optional[Dict[Text, Any]]: - """Persist this component to disk for future loading.""" - if self.matcher: - modelFile = os.path.join(model_dir, PATTERN_NER_FILE) - self.saveModel(modelFile) - return {"pattern_ner_file": PATTERN_NER_FILE} - - - @classmethod - def load( - cls, - meta: Dict[Text, Any], - model_dir: Optional[Text] = None, - model_metadata: Optional["Metadata"] = None, - cached_component: Optional["Component"] = None, - **kwargs: Any - ) -> "Component": - """Load this component from file.""" - - file_name = meta.get("pattern_ner_file", PATTERN_NER_FILE) - modelFile = os.path.join(model_dir, file_name) - if os.path.exists(modelFile): - modelLoad = open(modelFile, "rb") - matcher = pickle.load(modelLoad) - modelLoad.close() - return cls(meta, matcher) - else: - return cls(meta) - - - def saveModel(self, modelFile): - modelSave = open(modelFile, "wb") - pickle.dump(self.matcher, modelSave) - modelSave.close() \ No newline at end of file diff --git a/docker/Dockerfile b/docker/Dockerfile index 58f48e6ba..4b9cadd30 100644 --- a/docker/Dockerfile +++ b/docker/Dockerfile @@ -38,6 +38,8 @@ COPY . . RUN rm -rf ${TEMPLATE_DIR_DEFAULT}/models/* && \ rasa train --data ${TEMPLATE_DIR_DEFAULT}/data --config ${TEMPLATE_DIR_DEFAULT}/config.yml --domain ${TEMPLATE_DIR_DEFAULT}/domain.yml --out ${TEMPLATE_DIR_DEFAULT}/models +RUN cp kairon/shared/rule_policy.py /usr/local/lib/python3.10/site-packages/rasa/core/policies/rule_policy.py +RUN cp kairon/shared/schemas/domain.yml /usr/local/lib/python3.10/site-packages/rasa/shared/utils/schemas/domain.yml ENV HF_HOME="/home/cache" SENTENCE_TRANSFORMERS_HOME="/home/cache" diff --git a/kairon/actions/definitions/database.py b/kairon/actions/definitions/database.py index 6f0e48271..0d54abd49 100644 --- a/kairon/actions/definitions/database.py +++ b/kairon/actions/definitions/database.py @@ -83,7 +83,7 @@ async def execute(self, dispatcher: CollectingDispatcher, tracker: Tracker, doma request_body = ActionUtility.get_payload(payload, tracker) msg_logger.append(request_body) tracker_data = ActionUtility.build_context(tracker, True) - response = await vector_db.perform_operation(operation_type, request_body) + response = await vector_db.perform_operation(operation_type, request_body, user=tracker.sender_id) logger.info("response: " + str(response)) response_context = self.__add_user_context_to_http_response(response, tracker_data) bot_response, bot_resp_log, _ = ActionUtility.compose_response(vector_action_config['response'], response_context) diff --git a/kairon/actions/definitions/prompt.py b/kairon/actions/definitions/prompt.py index 3de92f413..2d3b5257b 100644 --- a/kairon/actions/definitions/prompt.py +++ b/kairon/actions/definitions/prompt.py @@ -4,7 +4,6 @@ from rasa_sdk import Tracker from rasa_sdk.executor import CollectingDispatcher -from kairon import Utility from kairon.actions.definitions.base import ActionsBase from kairon.shared.actions.data_objects import ActionServerLogs from kairon.shared.actions.exception import ActionFailure @@ -12,8 +11,8 @@ from kairon.shared.actions.utils import ActionUtility from kairon.shared.constants import FAQ_DISABLED_ERR, KaironSystemSlots, KAIRON_USER_MSG_ENTITY from kairon.shared.data.constant import DEFAULT_NLU_FALLBACK_RESPONSE -from kairon.shared.llm.factory import LLMFactory from kairon.shared.models import LlmPromptType, LlmPromptSource +from kairon.shared.llm.processor import LLMProcessor class ActionPrompt(ActionsBase): @@ -62,14 +61,18 @@ async def execute(self, dispatcher: CollectingDispatcher, tracker: Tracker, doma time_taken_slots = 0 final_slots = {"type": "slots_to_fill"} llm_response_log = {"type": "llm_response"} - + llm_processor = None try: k_faq_action_config, bot_settings = self.retrieve_config() user_question = k_faq_action_config.get('user_question') user_msg = self.__get_user_msg(tracker, user_question) + llm_type = k_faq_action_config['llm_type'] llm_params = await self.__get_llm_params(k_faq_action_config, dispatcher, tracker, domain) - llm = LLMFactory.get_instance("faq")(self.bot, bot_settings["llm_settings"]) - llm_response, time_taken_llm_response = await llm.predict(user_msg, **llm_params) + llm_processor = LLMProcessor(self.bot, llm_type) + llm_response, time_taken_llm_response = await llm_processor.predict(user_msg, + user=tracker.sender_id, + invocation='prompt_action', + **llm_params) status = "FAILURE" if llm_response.get("is_failure", False) is True else status exception = llm_response.get("exception") bot_response = llm_response['content'] @@ -93,8 +96,8 @@ async def execute(self, dispatcher: CollectingDispatcher, tracker: Tracker, doma total_time_elapsed = time_taken_llm_response + time_taken_slots events_to_extend = [llm_response_log, final_slots] events.extend(events_to_extend) - if llm: - llm_logs = llm.logs + if llm_processor: + llm_logs = llm_processor.logs ActionServerLogs( type=ActionType.prompt_action.value, intent=tracker.get_intent_of_latest_message(skip_fallback_intent=False), @@ -119,16 +122,6 @@ async def execute(self, dispatcher: CollectingDispatcher, tracker: Tracker, doma return slots_to_fill async def __get_llm_params(self, k_faq_action_config: dict, dispatcher: CollectingDispatcher, tracker: Tracker, domain: Dict[Text, Any]): - implementations = { - "GPT3_FAQ_EMBED": self.__get_gpt_params, - } - - llm_type = Utility.environment['llm']["faq"] - if not implementations.get(llm_type): - raise ActionFailure(f'{llm_type} type LLM is not supported') - return await implementations[Utility.environment['llm']["faq"]](k_faq_action_config, dispatcher, tracker, domain) - - async def __get_gpt_params(self, k_faq_action_config: dict, dispatcher: CollectingDispatcher, tracker: Tracker, domain: Dict[Text, Any]): from kairon.actions.definitions.factory import ActionFactory system_prompt = None @@ -147,7 +140,7 @@ async def __get_gpt_params(self, k_faq_action_config: dict, dispatcher: Collecti history_prompt = ActionUtility.prepare_bot_responses(tracker, num_bot_responses) elif prompt['source'] == LlmPromptSource.bot_content.value and prompt['is_enabled']: use_similarity_prompt = True - hyperparameters = prompt.get('hyperparameters', {}) + hyperparameters = prompt.get("hyperparameters", {}) similarity_prompt.append({'similarity_prompt_name': prompt['name'], 'similarity_prompt_instructions': prompt['instructions'], 'collection': prompt['data'], @@ -179,7 +172,7 @@ async def __get_gpt_params(self, k_faq_action_config: dict, dispatcher: Collecti is_query_prompt_enabled = True query_prompt_dict.update({'query_prompt': query_prompt, 'use_query_prompt': is_query_prompt_enabled}) - params["hyperparameters"] = k_faq_action_config.get('hyperparameters', Utility.get_llm_hyperparameters()) + params["hyperparameters"] = k_faq_action_config['hyperparameters'] params["system_prompt"] = system_prompt params["context_prompt"] = context_prompt params["query_prompt"] = query_prompt_dict diff --git a/kairon/api/app/routers/bot/bot.py b/kairon/api/app/routers/bot/bot.py index 6dfeda5c5..839ae0fe8 100644 --- a/kairon/api/app/routers/bot/bot.py +++ b/kairon/api/app/routers/bot/bot.py @@ -26,7 +26,7 @@ from kairon.shared.actions.data_objects import ActionServerLogs from kairon.shared.auth import Authentication from kairon.shared.constants import TESTER_ACCESS, DESIGNER_ACCESS, CHAT_ACCESS, UserActivityType, ADMIN_ACCESS, \ - VIEW_ACCESS, EventClass, AGENT_ACCESS + EventClass, AGENT_ACCESS from kairon.shared.data.assets_processor import AssetsProcessor from kairon.shared.data.audit.processor import AuditDataProcessor from kairon.shared.data.constant import EVENT_STATUS, ENDPOINT_TYPE, TOKEN_TYPE, ModelTestType, \ @@ -38,11 +38,11 @@ from kairon.shared.data.utils import DataUtility from kairon.shared.importer.data_objects import ValidationLogs from kairon.shared.importer.processor import DataImporterLogProcessor +from kairon.shared.live_agent.live_agent import LiveAgentHandler +from kairon.shared.llm.processor import LLMProcessor from kairon.shared.models import User, TemplateType from kairon.shared.test.processor import ModelTestingLogProcessor from kairon.shared.utils import Utility -from kairon.shared.live_agent.live_agent import LiveAgentHandler - router = APIRouter() v2 = APIRouter() @@ -1668,3 +1668,20 @@ async def get_live_agent_token(current_user: User = Security(Authentication.get_ data = await LiveAgentHandler.authenticate_agent(current_user.get_user(), current_user.get_bot()) return Response(data=data) + +@router.get("/llm/logs", response_model=Response) +async def get_llm_logs( + start_idx: int = 0, page_size: int = 10, + current_user: User = Security(Authentication.get_current_user_and_bot, scopes=TESTER_ACCESS) +): + """ + Get data llm event logs. + """ + logs = list(LLMProcessor.get_logs(current_user.get_bot(), start_idx, page_size)) + row_cnt = LLMProcessor.get_row_count(current_user.get_bot()) + data = { + "logs": logs, + "total": row_cnt + } + return Response(data=data) + diff --git a/kairon/api/models.py b/kairon/api/models.py index 1eaa034f2..d61a98126 100644 --- a/kairon/api/models.py +++ b/kairon/api/models.py @@ -15,7 +15,7 @@ ACTIVITY_STATUS, INTEGRATION_STATUS, FALLBACK_MESSAGE, - DEFAULT_NLU_FALLBACK_RESPONSE, + DEFAULT_NLU_FALLBACK_RESPONSE ) from ..shared.actions.models import ( ActionParameterType, @@ -37,6 +37,7 @@ CognitionDataType, CognitionMetadataType, ) +from kairon.shared.utils import Utility class RecaptchaVerifiedRequest(BaseModel): @@ -1057,7 +1058,8 @@ class PromptActionConfigRequest(BaseModel): num_bot_responses: int = 5 failure_message: str = DEFAULT_NLU_FALLBACK_RESPONSE user_question: UserQuestionModel = UserQuestionModel() - hyperparameters: dict = None + llm_type: str + hyperparameters: dict llm_prompts: List[LlmPromptRequest] instructions: List[str] = [] set_slots: List[SetSlotsUsingActionResponse] = [] @@ -1078,16 +1080,28 @@ def validate_num_bot_responses(cls, v, values, **kwargs): raise ValueError("num_bot_responses should not be greater than 5") return v + @validator("llm_type") + def validate_llm_type(cls, v, values, **kwargs): + if v not in Utility.get_llms(): + raise ValueError("Invalid llm type") + return v + + @validator("hyperparameters") + def validate_llm_hyperparameters(cls, v, values, **kwargs): + if values.get('llm_type'): + Utility.validate_llm_hyperparameters(v, values['llm_type'], ValueError) + @root_validator def check(cls, values): from kairon.shared.utils import Utility - if not values.get("hyperparameters"): - values["hyperparameters"] = {} + if values.get("llm_type"): + if not values.get("hyperparameters"): + values["hyperparameters"] = {} - for key, value in Utility.get_llm_hyperparameters().items(): - if key not in values["hyperparameters"]: - values["hyperparameters"][key] = value + for key, value in Utility.get_llm_hyperparameters(values.get("llm_type")).items(): + if key not in values["hyperparameters"]: + values["hyperparameters"][key] = value return values diff --git a/kairon/chat/agent/message_processor.py b/kairon/chat/agent/message_processor.py index 827404063..e4ae506d2 100644 --- a/kairon/chat/agent/message_processor.py +++ b/kairon/chat/agent/message_processor.py @@ -294,15 +294,6 @@ def predict_next_with_tracker_if_should( Raises: ActionLimitReached if the limit of actions to predict has been reached. """ - should_predict_another_action = self.should_predict_another_action( - tracker.latest_action_name - ) - - if self.is_action_limit_reached(tracker, should_predict_another_action): - raise ActionLimitReached( - "The limit of actions to predict has been reached." - ) - prediction = self._predict_next_with_tracker(tracker) action = self.action_for_index( diff --git a/kairon/importer/validator/file_validator.py b/kairon/importer/validator/file_validator.py index b55b3f0e6..ad2062c01 100644 --- a/kairon/importer/validator/file_validator.py +++ b/kairon/importer/validator/file_validator.py @@ -695,9 +695,9 @@ def __validate_prompt_actions(prompt_actions: list): data_error.append( f'num_bot_responses should not be greater than 5 and of type int: {action.get("name")}') llm_prompts_errors = TrainingDataValidator.__validate_llm_prompts(action['llm_prompts']) - if action.get('hyperparameters') is not None: - llm_hyperparameters_errors = TrainingDataValidator.__validate_llm_prompts_hyperparamters( - action.get('hyperparameters')) + if action.get('hyperparameters'): + llm_hyperparameters_errors = TrainingDataValidator.__validate_llm_prompts_hyperparameters( + action.get('hyperparameters'), action.get("llm_type", "openai")) data_error.extend(llm_hyperparameters_errors) data_error.extend(llm_prompts_errors) if action['name'] in actions_present: @@ -785,27 +785,12 @@ def __validate_llm_prompts(llm_prompts: dict): return error_list @staticmethod - def __validate_llm_prompts_hyperparamters(hyperparameters: dict): + def __validate_llm_prompts_hyperparameters(hyperparameters: dict, llm_type: str): error_list = [] - for key, value in hyperparameters.items(): - if key == 'temperature' and not 0.0 <= value <= 2.0: - error_list.append("Temperature must be between 0.0 and 2.0!") - elif key == 'presence_penalty' and not -2.0 <= value <= 2.0: - error_list.append("presence_penality must be between -2.0 and 2.0!") - elif key == 'frequency_penalty' and not -2.0 <= value <= 2.0: - error_list.append("frequency_penalty must be between -2.0 and 2.0!") - elif key == 'top_p' and not 0.0 <= value <= 1.0: - error_list.append("top_p must be between 0.0 and 1.0!") - elif key == 'n' and not 1 <= value <= 5: - error_list.append("n must be between 1 and 5!") - elif key == 'max_tokens' and not 5 <= value <= 4096: - error_list.append("max_tokens must be between 5 and 4096!") - elif key == 'logit_bias' and not isinstance(value, dict): - error_list.append("logit_bias must be a dictionary!") - elif key == 'stop': - if value and (not isinstance(value, (str, int, list)) or (isinstance(value, list) and len(value) > 4)): - error_list.append( - "Stop must be None, a string, an integer, or an array of 4 or fewer strings or integers.") + try: + Utility.validate_llm_hyperparameters(hyperparameters, llm_type, AppException) + except AppException as e: + error_list.append(e.__str__()) return error_list @staticmethod diff --git a/kairon/shared/actions/data_objects.py b/kairon/shared/actions/data_objects.py index b43a16da2..d02e88b02 100644 --- a/kairon/shared/actions/data_objects.py +++ b/kairon/shared/actions/data_objects.py @@ -34,6 +34,7 @@ KAIRON_TWO_STAGE_FALLBACK, FALLBACK_MESSAGE, DEFAULT_NLU_FALLBACK_RESPONSE, + DEFAULT_LLM ) from kairon.shared.data.signals import push_notification, auditlogger from kairon.shared.models import LlmPromptType, LlmPromptSource @@ -784,7 +785,8 @@ class PromptAction(Auditlog): bot = StringField(required=True) user = StringField(required=True) timestamp = DateTimeField(default=datetime.utcnow) - hyperparameters = DictField(default=Utility.get_llm_hyperparameters) + llm_type = StringField(default=DEFAULT_LLM, choices=Utility.get_llms()) + hyperparameters = DictField(default=Utility.get_default_llm_hyperparameters) llm_prompts = EmbeddedDocumentListField(LlmPrompt, required=True) instructions = ListField(StringField()) set_slots = EmbeddedDocumentListField(SetSlotsFromResponse) @@ -794,7 +796,7 @@ class PromptAction(Auditlog): meta = {"indexes": [{"fields": ["bot", ("bot", "name", "status")]}]} def clean(self): - for key, value in Utility.get_llm_hyperparameters().items(): + for key, value in Utility.get_llm_hyperparameters(self.llm_type).items(): if key not in self.hyperparameters: self.hyperparameters.update({key: value}) @@ -812,7 +814,7 @@ def validate(self, clean=True): dict_data["llm_prompts"], ValidationError ) Utility.validate_llm_hyperparameters( - dict_data["hyperparameters"], ValidationError + dict_data["hyperparameters"], self.llm_type, ValidationError ) diff --git a/kairon/shared/actions/utils.py b/kairon/shared/actions/utils.py index 00911f55c..d0c97d72f 100644 --- a/kairon/shared/actions/utils.py +++ b/kairon/shared/actions/utils.py @@ -5,6 +5,8 @@ import re from datetime import datetime from typing import Any, List, Text, Dict +from ..utils import Utility +Utility.load_system_metadata() import requests from aiohttp import ContentTypeError @@ -26,7 +28,6 @@ from ..data.data_objects import Slots, KeyVault from ..plugins.factory import PluginFactory from ..rest_client import AioRestClient -from ..utils import Utility from ...exceptions import AppException diff --git a/kairon/shared/concurrency/actors/pyscript_runner.py b/kairon/shared/concurrency/actors/pyscript_runner.py index a68e352c9..bd5286739 100644 --- a/kairon/shared/concurrency/actors/pyscript_runner.py +++ b/kairon/shared/concurrency/actors/pyscript_runner.py @@ -1,20 +1,26 @@ from types import ModuleType from typing import Text, Dict, Optional, Callable +import orjson as json from AccessControl.ZopeGuards import _safe_globals from RestrictedPython import compile_restricted from RestrictedPython.Guards import safer_getattr from loguru import logger from timeout_decorator import timeout_decorator -import orjson as json -from ..actors.base import BaseActor from kairon.exceptions import AppException +from ..actors.base import BaseActor +from AccessControl.SecurityInfo import allow_module + +allow_module("datetime") +allow_module("time") + global_safe = _safe_globals global_safe['_getattr_'] = safer_getattr global_safe['json'] = json + class PyScriptRunner(BaseActor): def execute(self, source_code: Text, predefined_objects: Optional[Dict] = None, **kwargs): diff --git a/kairon/shared/data/constant.py b/kairon/shared/data/constant.py index ad88ee625..afa40a619 100644 --- a/kairon/shared/data/constant.py +++ b/kairon/shared/data/constant.py @@ -215,6 +215,7 @@ class ModelTestType(str, Enum): DEFAULT_SYSTEM_PROMPT = ( "You are a personal assistant. Answer question based on the context below" ) +DEFAULT_LLM = "openai" class AuditlogActions(str, Enum): diff --git a/kairon/shared/data/processor.py b/kairon/shared/data/processor.py index 41db4c7f2..d63f6befc 100644 --- a/kairon/shared/data/processor.py +++ b/kairon/shared/data/processor.py @@ -7326,9 +7326,7 @@ def edit_prompt_action( action.failure_message = request_data.get("failure_message") action.user_question = UserQuestion(**request_data.get("user_question")) action.num_bot_responses = request_data.get("num_bot_responses", 5) - action.hyperparameters = request_data.get( - "hyperparameters", Utility.get_llm_hyperparameters() - ) + action.hyperparameters = request_data.get("hyperparameters") action.llm_prompts = [ LlmPrompt(**prompt) for prompt in request_data.get("llm_prompts", []) ] diff --git a/kairon/shared/llm/base.py b/kairon/shared/llm/base.py index 4babc6a23..006e38a3d 100644 --- a/kairon/shared/llm/base.py +++ b/kairon/shared/llm/base.py @@ -8,9 +8,9 @@ def __init__(self, bot: Text): self.bot = bot @abstractmethod - async def train(self, *args, **kwargs) -> Dict: + async def train(self, user, *args, **kwargs) -> Dict: pass @abstractmethod - async def predict(self, query, *args, **kwargs) -> Dict: + async def predict(self, query, user, *args, **kwargs) -> Dict: pass diff --git a/kairon/shared/llm/clients/__init__.py b/kairon/shared/llm/clients/__init__.py deleted file mode 100644 index e69de29bb..000000000 diff --git a/kairon/shared/llm/clients/azure.py b/kairon/shared/llm/clients/azure.py deleted file mode 100644 index 9b980d0d6..000000000 --- a/kairon/shared/llm/clients/azure.py +++ /dev/null @@ -1,25 +0,0 @@ -from typing import Text - -from kairon.shared.constants import GPT3ResourceTypes -from kairon.shared.llm.clients.gpt3 import GPT3Resources - - -class AzureGPT3Resources(GPT3Resources): - resource_url = "https://kairon.openai.azure.com/openai/deployments" - - def __init__(self, api_key: Text, **kwargs): - super().__init__(api_key) - self.api_key = api_key - self.api_version = kwargs.get("api_version") - self.model_id = { - GPT3ResourceTypes.embeddings.value: kwargs.get("embeddings_model_id"), - GPT3ResourceTypes.chat_completion.value: kwargs.get("chat_completion_model_id") - } - - def get_headers(self): - return {"api-key": self.api_key} - - def get_resource_url(self, resource: Text): - model_id = self.model_id[resource] - resource_url = f"{self.resource_url}/{model_id}/{resource}?api-version={self.api_version}" - return resource_url diff --git a/kairon/shared/llm/clients/base.py b/kairon/shared/llm/clients/base.py deleted file mode 100644 index 71ef7037e..000000000 --- a/kairon/shared/llm/clients/base.py +++ /dev/null @@ -1,8 +0,0 @@ -from abc import ABC -from typing import Text - - -class LLMResources(ABC): - - async def invoke(self, resource: Text, engine: Text, **kwargs): - raise NotImplementedError("Provider not implemented") diff --git a/kairon/shared/llm/clients/factory.py b/kairon/shared/llm/clients/factory.py deleted file mode 100644 index def8d09c3..000000000 --- a/kairon/shared/llm/clients/factory.py +++ /dev/null @@ -1,18 +0,0 @@ -from typing import Text -from kairon.exceptions import AppException -from kairon.shared.constants import LLMResourceProvider -from kairon.shared.llm.clients.azure import AzureGPT3Resources -from kairon.shared.llm.clients.gpt3 import GPT3Resources - - -class LLMClientFactory: - __implementations = { - LLMResourceProvider.openai.value: GPT3Resources, - LLMResourceProvider.azure.value: AzureGPT3Resources - } - - @staticmethod - def get_resource_provider(_type: Text): - if not LLMClientFactory.__implementations.get(_type): - raise AppException(f'{_type} client not supported') - return LLMClientFactory.__implementations[_type] diff --git a/kairon/shared/llm/clients/gpt3.py b/kairon/shared/llm/clients/gpt3.py deleted file mode 100644 index d6f2c5679..000000000 --- a/kairon/shared/llm/clients/gpt3.py +++ /dev/null @@ -1,92 +0,0 @@ -import ujson as json -import random -from json import JSONDecodeError -from ujson import JSONDecodeError as UJSONDecodeError -from typing import Text -from loguru import logger -from openai.api_requestor import parse_stream_helper -from kairon.exceptions import AppException -from kairon.shared.constants import GPT3ResourceTypes -from kairon.shared.llm.clients.base import LLMResources -from kairon.shared.rest_client import AioRestClient - - -class GPT3Resources(LLMResources): - resource_url = "https://api.openai.com/v1" - - def __init__(self, api_key: Text, **kwargs): - self.api_key = api_key - - def get_headers(self): - return {"Authorization": f"Bearer {self.api_key}"} - - def get_resource_url(self, resource: Text): - return f"{self.resource_url}/{resource}" - - async def invoke(self, resource: Text, model: Text, **kwargs): - client = None - http_url = self.get_resource_url(resource) - request_body = kwargs.copy() - request_body.update({"model": model}) - is_streaming_resp = kwargs.get("stream", False) - try: - client = AioRestClient(False) - resp = await client.request("POST", http_url, request_body, self.get_headers(), - return_json=False, is_streaming_resp=is_streaming_resp, max_retries=3) - if resp.status != 200: - try: - resp = await resp.json() - logger.debug(f"GPT response error: {resp}") - raise AppException(f"{resp['error'].get('message')}. Request id: {resp['error'].get('id')}") - except JSONDecodeError: - raise AppException(f"Received non 200 status code ({resp.status}): {resp.text}") - - if is_streaming_resp: - resp = client.streaming_response - - data = await self.__parse_response(resource, resp, **kwargs) - finally: - if client: - await client.cleanup() - return data - - async def __parse_response(self, resource: Text, response, **kwargs): - parsers = { - GPT3ResourceTypes.embeddings.value: self._parse_embeddings_response, - GPT3ResourceTypes.chat_completion.value: self.__parse_completion_response - } - return await parsers[resource](response, **kwargs) - - async def _parse_embeddings_response(self, response, **hyperparameters): - raw_response = await response.json() - formatted_response = raw_response["data"][0]["embedding"] - return formatted_response, raw_response - - async def __parse_completion_response(self, response, **kwargs): - if kwargs.get("stream"): - formatted_response = await self._parse_streaming_response(response, kwargs.get("n", 1)) - raw_response = response - else: - formatted_response, raw_response = await self._parse_api_response(response) - return formatted_response, raw_response - - async def _parse_api_response(self, response): - raw_response = await response.json() - msg_choice = random.choice(raw_response['choices']) - formatted_response = msg_choice['message']['content'] - return formatted_response, raw_response - - async def _parse_streaming_response(self, response, num_choices): - formatted_response = '' - msg_choice = random.randint(0, num_choices - 1) - try: - for chunk in response or []: - line = parse_stream_helper(chunk) - if line: - line = json.loads(line) - if line["choices"][0].get("index") == msg_choice and line["choices"][0]['delta'].get('content'): - formatted_response = f"{formatted_response}{line['choices'][0]['delta']['content']}" - except Exception as e: - logger.exception(e) - raise AppException(f"Failed to parse streaming response: {chunk}") - return formatted_response diff --git a/kairon/shared/llm/data_objects.py b/kairon/shared/llm/data_objects.py new file mode 100644 index 000000000..7713444ba --- /dev/null +++ b/kairon/shared/llm/data_objects.py @@ -0,0 +1,13 @@ +from mongoengine import Document, DynamicField, StringField, FloatField, DateTimeField, DictField + + +class LLMLogs(Document): + response = DynamicField() + start_time = DateTimeField() + end_time = DateTimeField() + cost = FloatField() + llm_call_id = StringField() + llm_provider = StringField() + model = StringField() + model_params = DictField() + metadata = DictField() \ No newline at end of file diff --git a/kairon/shared/llm/factory.py b/kairon/shared/llm/factory.py deleted file mode 100644 index 5424d1eea..000000000 --- a/kairon/shared/llm/factory.py +++ /dev/null @@ -1,17 +0,0 @@ -from typing import Text -from kairon.exceptions import AppException -from kairon.shared.utils import Utility -from kairon.shared.llm.gpt3 import GPT3FAQEmbedding - - -class LLMFactory: - __implementations = { - "GPT3_FAQ_EMBED": GPT3FAQEmbedding - } - - @staticmethod - def get_instance(_type: Text): - llm_type = Utility.environment['llm'][_type] - if not LLMFactory.__implementations.get(llm_type): - raise AppException(f'{llm_type} type LLM is not supported') - return LLMFactory.__implementations[llm_type] diff --git a/kairon/shared/llm/logger.py b/kairon/shared/llm/logger.py new file mode 100644 index 000000000..3af7a7870 --- /dev/null +++ b/kairon/shared/llm/logger.py @@ -0,0 +1,41 @@ +from litellm.integrations.custom_logger import CustomLogger +from .data_objects import LLMLogs +import ujson as json +from loguru import logger + + +class LiteLLMLogger(CustomLogger): + + def log_stream_event(self, kwargs, response_obj, start_time, end_time): + self.__logs_litellm(**kwargs) + + def log_success_event(self, kwargs, response_obj, start_time, end_time): + self.__logs_litellm(**kwargs) + + def log_failure_event(self, kwargs, response_obj, start_time, end_time): + self.__logs_litellm(**kwargs) + + async def async_log_stream_event(self, kwargs, response_obj, start_time, end_time): + self.__logs_litellm(**kwargs) + + async def async_log_success_event(self, kwargs, response_obj, start_time, end_time): + self.__logs_litellm(**kwargs) + + async def async_log_failure_event(self, kwargs, response_obj, start_time, end_time): + self.__logs_litellm(**kwargs) + + def __logs_litellm(self, **kwargs): + logger.info("logging llms call") + litellm_params = kwargs.get('litellm_params') + self.__save_logs(**{'response': json.loads(kwargs.get('original_response')) if kwargs.get('original_response') else None, + 'start_time': kwargs.get('start_time'), + 'end_time': kwargs.get('end_time'), + 'cost': kwargs.get("response_cost"), + 'llm_call_id': litellm_params.get('litellm_call_id'), + 'llm_provider': litellm_params.get('custom_llm_provider'), + 'model_params': kwargs.get("additional_args", {}).get("complete_input_dict"), + 'metadata': litellm_params.get('metadata')}) + + def __save_logs(self, **kwargs): + logs = LLMLogs(**kwargs).save().to_mongo().to_dict() + logger.info(f"llm logs: {logs}") diff --git a/kairon/shared/llm/gpt3.py b/kairon/shared/llm/processor.py similarity index 67% rename from kairon/shared/llm/gpt3.py rename to kairon/shared/llm/processor.py index 4e991ca7c..5b6831fe3 100644 --- a/kairon/shared/llm/gpt3.py +++ b/kairon/shared/llm/processor.py @@ -1,9 +1,9 @@ +from secrets import randbelow, choice import time - from typing import Text, Dict, List, Tuple from urllib.parse import urljoin -import openai +import litellm from loguru import logger as logging from tiktoken import get_encoding from tqdm import tqdm @@ -13,35 +13,36 @@ from kairon.shared.admin.processor import Sysadmin from kairon.shared.cognition.data_objects import CognitionData from kairon.shared.cognition.processor import CognitionDataProcessor -from kairon.shared.constants import GPT3ResourceTypes from kairon.shared.data.constant import DEFAULT_SYSTEM_PROMPT, DEFAULT_CONTEXT_PROMPT from kairon.shared.llm.base import LLMBase -from kairon.shared.llm.clients.factory import LLMClientFactory +from kairon.shared.llm.logger import LiteLLMLogger +from kairon.shared.llm.data_objects import LLMLogs from kairon.shared.models import CognitionDataType from kairon.shared.rest_client import AioRestClient from kairon.shared.utils import Utility +litellm.callbacks = [LiteLLMLogger()] + -class GPT3FAQEmbedding(LLMBase): +class LLMProcessor(LLMBase): __embedding__ = 1536 - def __init__(self, bot: Text, llm_settings: dict): + def __init__(self, bot: Text, llm_type: str): super().__init__(bot) self.db_url = Utility.environment['vector']['db'] self.headers = {} if Utility.environment['vector']['key']: self.headers = {"api-key": Utility.environment['vector']['key']} self.suffix = "_faq_embd" - self.vector_config = {'size': 1536, 'distance': 'Cosine'} - self.llm_settings = llm_settings + self.llm_type = llm_type + self.vector_config = {'size': self.__embedding__, 'distance': 'Cosine'} self.api_key = Sysadmin.get_bot_secret(bot, BotSecretType.gpt_key.value, raise_err=True) - self.client = LLMClientFactory.get_resource_provider(llm_settings["provider"])(self.api_key, - **self.llm_settings) self.tokenizer = get_encoding("cl100k_base") self.EMBEDDING_CTX_LENGTH = 8191 self.__logs = [] - async def train(self, *args, **kwargs) -> Dict: + async def train(self, user, *args, **kwargs) -> Dict: + invocation = kwargs.pop('invocation', None) await self.__delete_collections() count = 0 processor = CognitionDataProcessor() @@ -51,35 +52,38 @@ async def train(self, *args, **kwargs) -> Dict: {'$project': {'collection': "$_id", 'content': 1, '_id': 0}} ])) for collections in collection_groups: - collection = f"{self.bot}_{collections['collection']}{self.suffix}" if collections['collection'] else f"{self.bot}{self.suffix}" + collection = f"{self.bot}_{collections['collection']}{self.suffix}" if collections[ + 'collection'] else f"{self.bot}{self.suffix}" await self.__create_collection__(collection) for content in tqdm(collections['content'], desc="Training FAQ"): if content['content_type'] == CognitionDataType.json.value: metadata = processor.find_matching_metadata(self.bot, content['data'], content.get('collection')) - search_payload, embedding_payload = Utility.retrieve_search_payload_and_embedding_payload(content['data'], metadata) + search_payload, embedding_payload = Utility.retrieve_search_payload_and_embedding_payload( + content['data'], metadata) else: search_payload, embedding_payload = {'content': content["data"]}, content["data"] - search_payload['collection_name'] = collection - embeddings = await self.__get_embedding(embedding_payload) + embeddings = await self.get_embedding(embedding_payload, user, invocation=invocation) points = [{'id': content['vector_id'], 'vector': embeddings, 'payload': search_payload}] - await self.__collection_upsert__(collection, {'points': points}, err_msg="Unable to train FAQ! Contact support") + await self.__collection_upsert__(collection, {'points': points}, + err_msg="Unable to train FAQ! Contact support") count += 1 return {"faq": count} - async def predict(self, query: Text, *args, **kwargs) -> Tuple: + async def predict(self, query: Text, user, *args, **kwargs) -> Tuple: start_time = time.time() embeddings_created = False + invocation = kwargs.pop('invocation', None) try: - query_embedding = await self.__get_embedding(query) + query_embedding = await self.get_embedding(query, user, invocation=invocation) embeddings_created = True system_prompt = kwargs.pop('system_prompt', DEFAULT_SYSTEM_PROMPT) context_prompt = kwargs.pop('context_prompt', DEFAULT_CONTEXT_PROMPT) context = await self.__attach_similarity_prompt_if_enabled(query_embedding, context_prompt, **kwargs) - answer = await self.__get_answer(query, system_prompt, context, **kwargs) + answer = await self.__get_answer(query, system_prompt, context, user, invocation=invocation,**kwargs) response = {"content": answer} - except openai.error.APIConnectionError as e: + except Exception as e: logging.exception(e) if embeddings_created: failure_stage = "Retrieving chat completion for the provided query." @@ -87,9 +91,6 @@ async def predict(self, query: Text, *args, **kwargs) -> Tuple: failure_stage = "Creating a new embedding for the provided query." self.__logs.append({'error': f"{failure_stage} {str(e)}"}) response = {"is_failure": True, "exception": str(e), "content": None} - except Exception as e: - logging.exception(e) - response = {"is_failure": True, "exception": str(e), "content": None} end_time = time.time() elapsed_time = end_time - start_time @@ -102,26 +103,54 @@ def truncate_text(self, text: Text) -> Text: tokens = self.tokenizer.encode(text)[:self.EMBEDDING_CTX_LENGTH] return self.tokenizer.decode(tokens) - async def __get_embedding(self, text: Text) -> List[float]: + async def get_embedding(self, text: Text, user, **kwargs) -> List[float]: truncated_text = self.truncate_text(text) - result, _ = await self.client.invoke(GPT3ResourceTypes.embeddings.value, model="text-embedding-3-small", - input=truncated_text) - return result + result = await litellm.aembedding(model="text-embedding-3-small", + input=[truncated_text], + metadata={'user': user, 'bot': self.bot, 'invocation': kwargs.get("invocation")}, + api_key=self.api_key, + num_retries=3) + return result["data"][0]["embedding"] + + async def __parse_completion_response(self, response, **kwargs): + if kwargs.get("stream"): + formatted_response = '' + msg_choice = randbelow(kwargs.get("n", 1)) + if response["choices"][0].get("index") == msg_choice and response["choices"][0]['delta'].get('content'): + formatted_response = f"{response['choices'][0]['delta']['content']}" + else: + msg_choice = choice(response['choices']) + formatted_response = msg_choice['message']['content'] + return formatted_response + + async def __get_completion(self, messages, hyperparameters, user, **kwargs): + response = await litellm.acompletion(messages=messages, + metadata={'user': user, 'bot': self.bot, 'invocation': kwargs.get("invocation")}, + api_key=self.api_key, + num_retries=3, + **hyperparameters) + formatted_response = await self.__parse_completion_response(response, + **hyperparameters) + return formatted_response, response - async def __get_answer(self, query, system_prompt: Text, context: Text, **kwargs): + async def __get_answer(self, query, system_prompt: Text, context: Text, user, **kwargs): use_query_prompt = False query_prompt = '' + invocation = kwargs.pop('invocation') if kwargs.get('query_prompt', {}): query_prompt_dict = kwargs.pop('query_prompt') query_prompt = query_prompt_dict.get('query_prompt', '') use_query_prompt = query_prompt_dict.get('use_query_prompt') previous_bot_responses = kwargs.get('previous_bot_responses') - hyperparameters = kwargs.get('hyperparameters', Utility.get_llm_hyperparameters()) + hyperparameters = kwargs['hyperparameters'] instructions = kwargs.get('instructions', []) instructions = '\n'.join(instructions) if use_query_prompt and query_prompt: - query = await self.__rephrase_query(query, system_prompt, query_prompt, hyperparameters=hyperparameters) + query = await self.__rephrase_query(query, system_prompt, query_prompt, + hyperparameters=hyperparameters, + user=user, + invocation=f"{invocation}_rephrase") messages = [ {"role": "system", "content": system_prompt}, ] @@ -130,21 +159,26 @@ async def __get_answer(self, query, system_prompt: Text, context: Text, **kwargs messages.append({"role": "user", "content": f"{context} \n{instructions} \nQ: {query} \nA:"}) if instructions \ else messages.append({"role": "user", "content": f"{context} \nQ: {query} \nA:"}) - completion, raw_response = await self.client.invoke(GPT3ResourceTypes.chat_completion.value, messages=messages, - **hyperparameters) + completion, raw_response = await self.__get_completion(messages=messages, + hyperparameters=hyperparameters, + user=user, + invocation=invocation) self.__logs.append({'messages': messages, 'raw_completion_response': raw_response, 'type': 'answer_query', 'hyperparameters': hyperparameters}) return completion - async def __rephrase_query(self, query, system_prompt: Text, query_prompt: Text, **kwargs): + async def __rephrase_query(self, query, system_prompt: Text, query_prompt: Text, user, **kwargs): + invocation = kwargs.pop('invocation') messages = [ {"role": "system", "content": system_prompt}, {"role": "user", "content": f"{query_prompt}\n\n Q: {query}\n A:"} ] - hyperparameters = kwargs.get('hyperparameters', Utility.get_llm_hyperparameters()) + hyperparameters = kwargs['hyperparameters'] - completion, raw_response = await self.client.invoke(GPT3ResourceTypes.chat_completion.value, messages=messages, - **hyperparameters) + completion, raw_response = await self.__get_completion(messages=messages, + hyperparameters=hyperparameters, + user=user, + invocation=invocation) self.__logs.append({'messages': messages, 'raw_completion_response': raw_response, 'type': 'rephrase_query', 'hyperparameters': hyperparameters}) return completion @@ -153,9 +187,9 @@ async def __delete_collections(self): client = AioRestClient(False) try: response = await client.request(http_url=urljoin(self.db_url, "/collections"), - request_method="GET", - headers=self.headers, - timeout=5) + request_method="GET", + headers=self.headers, + timeout=5) if response.get('result'): for collection in response['result'].get('collections') or []: if collection['name'].startswith(self.bot): @@ -234,3 +268,26 @@ async def __attach_similarity_prompt_if_enabled(self, query_embedding, context_p similarity_context = f"Instructions on how to use {similarity_prompt_name}:\n{extracted_values}\n{similarity_prompt_instructions}\n" context_prompt = f"{context_prompt}\n{similarity_context}" return context_prompt + + @staticmethod + def get_logs(bot: str, start_idx: int = 0, page_size: int = 10): + """ + Get all logs for data importer event. + @param bot: bot id. + @param start_idx: start index + @param page_size: page size + @return: list of logs. + """ + for log in LLMLogs.objects(metadata__bot=bot).order_by("-start_time").skip(start_idx).limit(page_size): + llm_log = log.to_mongo().to_dict() + llm_log.pop('_id') + yield llm_log + + @staticmethod + def get_row_count(bot: str): + """ + Gets the count of rows in a LLMLogs for a particular bot. + :param bot: bot id + :return: Count of rows + """ + return LLMLogs.objects(metadata__bot=bot).count() diff --git a/kairon/shared/rest_client.py b/kairon/shared/rest_client.py index a76faad82..5c144a0df 100644 --- a/kairon/shared/rest_client.py +++ b/kairon/shared/rest_client.py @@ -32,14 +32,6 @@ def __init__(self, close_session_with_rqst_completion=True): self._time_elapsed = None self._status_code = None - @property - def streaming_response(self): - return self._streaming_response - - @streaming_response.setter - def streaming_response(self, resp): - self._streaming_response = resp - @property def time_elapsed(self): return self._time_elapsed @@ -60,7 +52,7 @@ async def request(self, request_method: str, http_url: str, request_body: Union[ headers: dict = None, return_json: bool = True, **kwargs): max_retries = kwargs.get("max_retries", 1) - status_forcelist = kwargs.get("status_forcelist", [104, 502, 503, 504]) + status_forcelist = set(kwargs.get("status_forcelist", [104, 502, 503, 504])) timeout = ClientTimeout(total=kwargs['timeout']) if kwargs.get('timeout') else None is_streaming_resp = kwargs.pop("is_streaming_resp", False) content_type = kwargs.pop("content_type", HttpRequestContentType.json.value) @@ -124,12 +116,9 @@ async def __trigger(self, client, *args, **kwargs) -> ClientResponse: logger.debug(f"Content-type: {response.headers['content-type']}") logger.debug(f"Status code: {str(response.status)}") self.status_code = response.status - if is_streaming_resp: - streaming_resp = await AioRestClient.parse_streaming_response(response) - self.streaming_response = streaming_resp - logger.debug(f"Raw streaming response: {streaming_resp}") - text = await response.text() - logger.debug(f"Raw response: {text}") + if not is_streaming_resp: + text = await response.text() + logger.debug(f"Raw response: {text}") return response def __validate_response(self, response: ClientResponse, **kwargs): @@ -149,14 +138,4 @@ async def cleanup(self): Close underlying connector to release all acquired resources. """ if not self.session.closed: - await self.session.close() - - @staticmethod - async def parse_streaming_response(response): - chunks = [] - async for chunk in response.content: - if not chunk: - break - chunks.append(chunk) - - return chunks + await self.session.close() \ No newline at end of file diff --git a/kairon/shared/schemas/domain.yml b/kairon/shared/schemas/domain.yml new file mode 100644 index 000000000..b5ceed4c3 --- /dev/null +++ b/kairon/shared/schemas/domain.yml @@ -0,0 +1,142 @@ +allowempty: True +mapping: + version: + type: "str" + required: False + allowempty: False + intents: + type: "seq" + sequence: + - type: "map" + mapping: + use_entities: + type: "any" + ignore_entities: + type: "any" + allowempty: True + - type: "str" + entities: + type: "seq" + matching: "any" + sequence: + - type: "map" + mapping: + roles: + type: "seq" + sequence: + - type: "str" + groups: + type: "seq" + sequence: + - type: "str" + allowempty: True + - type: "str" + actions: + type: seq + matching: "any" + seq: + - type: str + - type: map + mapping: + regex;([A-Za-z]+): + type: map + mapping: + send_domain: + type: "bool" + responses: + # see shared/nlu/training_data/schemas/responses.yml + include: responses + + slots: + type: "map" + allowempty: True + mapping: + regex;([A-Za-z\u00C0-\u017F\u0400-\u04FF\u0370-\u03FF\u0530-\u058F\u0590-\u05FF\u0600-\u06FF\u0980-\u09FF\u0A80-\u0AFF\u0B80-\u0BFF\u4E00-\u9FFF\u3040-\u309F\u30A0-\u30FF\uAC00-\uD7AF]+): + type: "map" + allowempty: True + mapping: + influence_conversation: + type: "bool" + required: False + type: + type: "any" + required: True + values: + type: "seq" + sequence: + - type: "any" + required: False + min_value: + type: "number" + required: False + max_value: + type: "number" + required: False + initial_value: + type: "any" + required: False + mappings: + type: "seq" + required: True + allowempty: False + sequence: + - type: "map" + allowempty: True + mapping: + type: + type: "str" + intent: + type: "any" + not_intent: + type: "any" + entity: + type: "str" + role: + type: "str" + group: + type: "str" + value: + type: "any" + action: + type: "str" + conditions: + type: "seq" + sequence: + - type: "map" + mapping: + active_loop: + type: "str" + nullable: True + requested_slot: + type: "str" + forms: + type: "map" + required: False + mapping: + regex;([A-Za-z\u00C0-\u017F\u0400-\u04FF\u0370-\u03FF\u0530-\u058F\u0590-\u05FF\u0600-\u06FF\u0980-\u09FF\u0A80-\u0AFF\u0B80-\u0BFF\u4E00-\u9FFF\u3040-\u309F\u30A0-\u30FF\uAC00-\uD7AF]+): + type: "map" + mapping: + required_slots: + type: "seq" + sequence: + - type: str + required: False + allowempty: True + ignored_intents: + type: any + config: + type: "map" + allowempty: True + mapping: + store_entities_as_slots: + type: "bool" + session_config: + type: "map" + allowempty: True + mapping: + session_expiration_time: + type: "number" + range: + min: 0 + carry_over_slots_to_new_session: + type: "bool" diff --git a/kairon/shared/utils.py b/kairon/shared/utils.py index 0d9fd41a9..59cb9a5e2 100644 --- a/kairon/shared/utils.py +++ b/kairon/shared/utils.py @@ -78,6 +78,7 @@ TOKEN_TYPE, KAIRON_TWO_STAGE_FALLBACK, SLOT_TYPE, + DEFAULT_LLM ) from kairon.shared.kairon_yaml_story_writer import kaironYAMLStoryWriter from .data.dto import KaironStoryStep @@ -2052,73 +2053,33 @@ def verify_email(email: Text): raise AppException("Invalid or disposable Email!") @staticmethod - def get_llm_hyperparameters(): + def get_llms(): + return Utility.system_metadata.get("llm", {}).keys() + + @staticmethod + def get_default_llm_hyperparameters(): + return Utility.get_llm_hyperparameters(DEFAULT_LLM) + + @staticmethod + def get_llm_hyperparameters(llm_type): hyperparameters = {} - if Utility.environment["llm"]["faq"] in {"GPT3_FAQ_EMBED"}: - for key, value in Utility.system_metadata["llm"]["gpt"].items(): + if llm_type in Utility.system_metadata["llm"].keys(): + for key, value in Utility.system_metadata["llm"][llm_type]['properties'].items(): hyperparameters[key] = value["default"] return hyperparameters - raise AppException("Could not find any hyperparameters for configured LLM.") + raise AppException(f"Could not find any hyperparameters for {llm_type} LLM.") @staticmethod - def validate_llm_hyperparameters(hyperparameters: dict, exception_class): - params = Utility.system_metadata["llm"]["gpt"] - for key, value in hyperparameters.items(): - if ( - key == "temperature" - and not params["temperature"]["min"] - <= value - <= params["temperature"]["max"] - ): - raise exception_class( - f"Temperature must be between {params['temperature']['min']} and {params['temperature']['max']}!" - ) - elif ( - key == "presence_penalty" - and not params["presence_penalty"]["min"] - <= value - <= params["presence_penalty"]["max"] - ): - raise exception_class( - f"Presence penalty must be between {params['presence_penalty']['min']} and {params['presence_penalty']['max']}!" - ) - elif ( - key == "frequency_penalty" - and not params["presence_penalty"]["min"] - <= value - <= params["presence_penalty"]["max"] - ): - raise exception_class( - f"Frequency penalty must be between {params['presence_penalty']['min']} and {params['presence_penalty']['max']}!" - ) - elif ( - key == "top_p" - and not params["top_p"]["min"] <= value <= params["top_p"]["max"] - ): - raise exception_class( - f"top_p must be between {params['top_p']['min']} and {params['top_p']['max']}!" - ) - elif key == "n" and not params["n"]["min"] <= value <= params["n"]["max"]: - raise exception_class( - f"n must be between {params['n']['min']} and {params['n']['max']} and should not be 0!" - ) - elif ( - key == "max_tokens" - and not params["max_tokens"]["min"] - <= value - <= params["max_tokens"]["max"] - ): - raise exception_class( - f"max_tokens must be between {params['max_tokens']['min']} and {params['max_tokens']['max']} and should not be 0!" - ) - elif key == "logit_bias" and not isinstance(value, dict): - raise exception_class("logit_bias must be a dictionary!") - elif key == "stop": - exc_msg = "Stop must be None, a string, an integer, or an array of 4 or fewer strings or integers." - if value and not isinstance(value, (str, int, list)): - raise exception_class(exc_msg) - elif value and (isinstance(value, list) and len(value) > 4): - raise exception_class(exc_msg) + def validate_llm_hyperparameters(hyperparameters: dict, llm_type: str, exception_class): + from jsonschema_rs import JSONSchema, ValidationError as JValidationError + schema = Utility.system_metadata["llm"][llm_type] + try: + validator = JSONSchema(schema) + validator.validate(hyperparameters) + except JValidationError as e: + message = f"{e.instance_path}: {e.message}" + raise exception_class(message) + @staticmethod def create_uuid_from_string(val: str): diff --git a/kairon/shared/vector_embeddings/db/base.py b/kairon/shared/vector_embeddings/db/base.py index 178ee25de..887be41bb 100644 --- a/kairon/shared/vector_embeddings/db/base.py +++ b/kairon/shared/vector_embeddings/db/base.py @@ -8,16 +8,16 @@ class VectorEmbeddingsDbBase(ABC): @abstractmethod - async def embedding_search(self, request_body: Dict): + async def embedding_search(self, request_body: Dict, user: str, **kwargs): raise NotImplementedError("Provider not implemented") @abstractmethod - async def payload_search(self, request_body: Dict): + async def payload_search(self, request_body: Dict, user: str, **kwargs): raise NotImplementedError("Provider not implemented") - async def perform_operation(self, op_type: Text, request_body: Dict): + async def perform_operation(self, op_type: Text, request_body: Dict, user: str, **kwargs): supported_ops = {DbActionOperationType.payload_search.value: self.payload_search, DbActionOperationType.embedding_search.value: self.embedding_search} if op_type not in supported_ops.keys(): raise AppException("Operation type not supported") - return await supported_ops[op_type](request_body) + return await supported_ops[op_type](request_body, user, **kwargs) diff --git a/kairon/shared/vector_embeddings/db/qdrant.py b/kairon/shared/vector_embeddings/db/qdrant.py index d2ff7e69c..400e3103c 100644 --- a/kairon/shared/vector_embeddings/db/qdrant.py +++ b/kairon/shared/vector_embeddings/db/qdrant.py @@ -5,10 +5,8 @@ from kairon import Utility from kairon.shared.actions.utils import ActionUtility -from kairon.shared.admin.constants import BotSecretType -from kairon.shared.admin.processor import Sysadmin -from kairon.shared.constants import GPT3ResourceTypes -from kairon.shared.llm.clients.factory import LLMClientFactory +from kairon.shared.llm.processor import LLMProcessor +from kairon.shared.data.constant import DEFAULT_LLM from kairon.shared.vector_embeddings.db.base import VectorEmbeddingsDbBase @@ -25,31 +23,19 @@ def __init__(self, bot: Text, collection_name: Text, llm_settings: dict, db_url: if Utility.environment['vector']['key']: self.headers = {"api-key": Utility.environment['vector']['key']} self.llm_settings = llm_settings - self.api_key = Sysadmin.get_bot_secret(self.bot, BotSecretType.gpt_key.value, raise_err=True) - self.client = LLMClientFactory.get_resource_provider(llm_settings["provider"])(self.api_key, - **self.llm_settings) + self.llm = LLMProcessor(self.bot, DEFAULT_LLM) self.tokenizer = get_encoding("cl100k_base") self.EMBEDDING_CTX_LENGTH = 8191 - def truncate_text(self, text: Text) -> Text: - """ - Truncate text to 8191 tokens for openai - """ - tokens = self.tokenizer.encode(text)[:self.EMBEDDING_CTX_LENGTH] - return self.tokenizer.decode(tokens) + async def __get_embedding(self, text: Text, user: str, **kwargs) -> List[float]: + return await self.llm.get_embedding(text, user=user, invocation='db_action_qdrant') - async def __get_embedding(self, text: Text) -> List[float]: - truncated_text = self.truncate_text(text) - result, _ = await self.client.invoke(GPT3ResourceTypes.embeddings.value, model="text-embedding-3-small", - input=truncated_text) - return result - - async def embedding_search(self, request_body: Dict): + async def embedding_search(self, request_body: Dict, user: str, **kwargs): url = urljoin(self.db_url, f"/collections/{self.collection_name}/points") if request_body.get("text"): url = urljoin(self.db_url, f"/collections/{self.collection_name}/points/search") user_msg = request_body.get("text") - vector = await self.__get_embedding(user_msg) + vector = await self.__get_embedding(user_msg, user, **kwargs) request_body = {'vector': vector, 'limit': 10, 'with_payload': True, 'score_threshold': 0.70} embedding_search_result = ActionUtility.execute_http_request(http_url=url, request_method='POST', @@ -57,7 +43,7 @@ async def embedding_search(self, request_body: Dict): request_body=request_body) return embedding_search_result - async def payload_search(self, request_body: Dict): + async def payload_search(self, request_body: Dict, user, **kwargs): url = urljoin(self.db_url, f"/collections/{self.collection_name}/points/scroll") payload_filter_result = ActionUtility.execute_http_request(http_url=url, request_method='POST', diff --git a/kairon/train.py b/kairon/train.py index 05e761dc5..aa19943cb 100644 --- a/kairon/train.py +++ b/kairon/train.py @@ -6,17 +6,17 @@ from rasa.api import train from rasa.model import DEFAULT_MODELS_PATH from rasa.shared.constants import DEFAULT_CONFIG_PATH, DEFAULT_DATA_PATH, DEFAULT_DOMAIN_PATH - +from kairon.shared.data.constant import DEFAULT_LLM from kairon.chat.agent.agent import KaironAgent from kairon.exceptions import AppException from kairon.shared.account.processor import AccountProcessor from kairon.shared.data.constant import EVENT_STATUS from kairon.shared.data.model_processor import ModelProcessor from kairon.shared.data.processor import MongoProcessor -from kairon.shared.llm.factory import LLMFactory from kairon.shared.metering.constants import MetricType from kairon.shared.metering.metering_processor import MeteringProcessor from kairon.shared.utils import Utility +from kairon.shared.llm.processor import LLMProcessor def train_model_for_bot(bot: str): @@ -81,6 +81,7 @@ def train_model_for_bot(bot: str): raise AppException(e) return model + def start_training(bot: str, user: str, token: str = None): """ prevents training of the bot, @@ -100,8 +101,8 @@ def start_training(bot: str, user: str, token: str = None): settings = processor.get_bot_settings(bot, user) settings = settings.to_mongo().to_dict() if settings["llm_settings"]['enable_faq']: - llm = LLMFactory.get_instance("faq")(bot, settings["llm_settings"]) - faqs = asyncio.run(llm.train()) + llm_processor = LLMProcessor(bot, DEFAULT_LLM) + faqs = asyncio.run(llm_processor.train(user=user, invocation='model_training')) account = AccountProcessor.get_bot(bot)['account'] MeteringProcessor.add_metrics(bot=bot, metric_type=MetricType.faq_training.value, account=account, **faqs) agent_url = Utility.environment['model']['agent'].get('url') diff --git a/metadata/integrations.yml b/metadata/integrations.yml index f65e67b57..6ba78b15f 100644 --- a/metadata/integrations.yml +++ b/metadata/integrations.yml @@ -95,59 +95,70 @@ live_agents: websocket_url: wss://app.chatwoot.com/cable llm: - gpt: - temperature: - type: float - default: 0.0 - min: 0.0 - max: 2.0 - description: "The temperature hyperparameter controls the creativity or randomness of the generated responses." - max_tokens: - type: int - default: 300 - min: 5 - max: 4096 - description: "The max_tokens hyperparameter limits the length of generated responses in chat completion using ChatGPT." - model: - type: str - default: "gpt-3.5-turbo" - description: "The model hyperparameter is the ID of the model to use such as gpt-2, gpt-3, or a custom model that you have trained or fine-tuned." - top_p: - type: float - default: 0.0 - min: 0.0 - max: 1.0 - description: "The top_p hyperparameter is a value that controls the diversity of the generated responses." - n: - type: int - default: 1 - min: 1 - max: 5 - description: "The n hyperparameter controls the number of different response options that are generated by the model." - stream: - type: bool - default: false - description: "the stream hyperparameter controls whether the model generates a continuous stream of text or a single response." - stop: - type: - - str - - array - - int - default: null - description: "The stop hyperparameter is used to specify a list of tokens that should be used to indicate the end of a generated response." - presence_penalty: - type: float - default: 0.0 - min: -2.0 - max: 2.0 - description: "The presence_penalty hyperparameter penalizes the model for generating words that are not present in the context or input prompt. " - frequency_penalty: - type: float - default: 0.0 - min: -2.0 - max: 2.0 - description: "The frequency_penalty hyperparameter penalizes the model for generating words that have already been generated in the current response." - logit_bias: - type: dict - default: {} - description: "The logit_bias hyperparameter helps prevent GPT-3 from generating unwanted tokens or even to encourage generation of tokens that you do want. " + openai: + $schema: "https://json-schema.org/draft/2020-12/schema" + type: object + description: "Open AI Models for Prompt" + properties: + temperature: + type: number + default: 0.0 + minimum: 0.0 + maximum: 2.0 + description: "The temperature hyperparameter controls the creativity or randomness of the generated responses." + max_tokens: + type: integer + default: 300 + minimum: 5 + maximum: 4096 + description: "The max_tokens hyperparameter limits the length of generated responses in chat completion using ChatGPT." + model: + type: string + default: "gpt-3.5-turbo" + enum: ["gpt-3.5-turbo", "gpt-3.5-turbo-instruct"] + description: "The model hyperparameter is the ID of the model to use such as gpt-2, gpt-3, or a custom model that you have trained or fine-tuned." + top_p: + type: number + default: 0.0 + minimum: 0.0 + maximum: 1.0 + description: "The top_p hyperparameter is a value that controls the diversity of the generated responses." + n: + type: integer + default: 1 + minimum: 1 + maximum: 5 + description: "The n hyperparameter controls the number of different response options that are generated by the model." + stop: + anyOf: + - type: "string" + - type: "array" + maxItems: 4 + items: + type: "string" + - type: "integer" + - type: "null" + + type: + - "string" + - "array" + - "integer" + - "null" + default: null + description: "The stop hyperparameter is used to specify a list of tokens that should be used to indicate the end of a generated response." + presence_penalty: + type: number + default: 0.0 + minimum: -2.0 + maximum: 2.0 + description: "The presence_penalty hyperparameter penalizes the model for generating words that are not present in the context or input prompt. " + frequency_penalty: + type: number + default: 0.0 + minimum: -2.0 + maximum: 2.0 + description: "The frequency_penalty hyperparameter penalizes the model for generating words that have already been generated in the current response." + logit_bias: + type: object + default: {} + description: "The logit_bias hyperparameter helps prevent GPT-3 from generating unwanted tokens or even to encourage generation of tokens that you do want. " diff --git a/requirements/dev.txt b/requirements/dev.txt index d411ab022..19268e061 100644 --- a/requirements/dev.txt +++ b/requirements/dev.txt @@ -1,16 +1,15 @@ -r prod.txt -pytest==8.1.1 +pytest==8.2.2 pytest_httpx==0.30.0 -pytest-asyncio==0.23.6 -responses==0.25.0 +pytest-asyncio==0.23.7 +responses==0.25.2 mock==5.1.0 -moto[all]==5.0.5 +moto[all]==5.0.9 mongomock==4.1.2 black==22.12.0 -locust==2.25.0 +locust==2.29.0 deepdiff==7.0.1 pytest-cov==5.0.0 pytest-html==4.1.1 pytest-aioresponses==0.2.0 -aioresponses==0.7.6 -pykwalify==1.8.0 \ No newline at end of file +aioresponses==0.7.6 \ No newline at end of file diff --git a/requirements/prod.txt b/requirements/prod.txt index 19c6817ff..13fcc40e7 100644 --- a/requirements/prod.txt +++ b/requirements/prod.txt @@ -1,65 +1,69 @@ rasa[full]==3.6.20 mongoengine==0.28.2 fastapi==0.110.2 -uvicorn[standard]==0.29.0 +uvicorn[standard]==0.30.1 smart-config==0.1.3 fastapi_sso==0.9.1 fastapi-keycloak==1.0.10 pykka==3.1.1 zenpy==2.0.42 -validators==0.28.0 +validators==0.28.3 secure==0.3.0 password-strength==0.0.3.post2 beautifulsoup4==4.12.3 uuid6==2024.01.12 passlib[bcrypt]==1.7.4 -openai==0.28.1 json2html==1.3.0 -google-api-python-client==2.110.0 -jira==3.5.2 +google-api-python-client==2.133.0 +jira==3.8.0 pipedrive-python-lib==1.2.3 -google-cloud-translate==3.13.0 -blinker==1.7.0 -pymupdf==1.23.7 -python-docx==1.1.0 +google-cloud-translate==3.15.3 +blinker==1.8.2 +pymupdf==1.24.5 +python-docx==1.1.2 python-multipart==0.0.9 pandas==2.2.2 -openpyxl==3.1.2 +openpyxl==3.1.4 sentencepiece==0.1.99 -dramatiq==1.15.0 +dramatiq==1.17.0 dramatiq-mongodb==0.8.3 nlpaug==1.1.11 keybert==0.8.4 -pyTelegramBotAPI==4.17.0 +pyTelegramBotAPI==4.19.1 APScheduler==3.9.1.post1 -croniter==2.0.3 +croniter==2.0.5 faiss-cpu==1.8.0 -tiktoken==0.6.0 +tiktoken==0.7.0 RestrictedPython==7.1 -AccessControl==6.3 +AccessControl==7.0 timeout-decorator==0.5.0 -googlesearch-python==1.2.3 +googlesearch-python==1.2.4 aiohttp-retry==2.8.3 pqdict==1.4.0 google-businessmessages==1.0.5 google-apitools==0.5.32 -orjson==3.10.1 -opentelemetry-distro[otlp]==0.45b0 +orjson==3.10.5 +opentelemetry-distro[otlp]==0.46b0 opentelemetry-sdk-extension-aws==2.0.1 opentelemetry-propagator-aws-xray==1.0.1 -opentelemetry-instrumentation-fastapi==0.45b0 -opentelemetry-instrumentation-aiohttp-client==0.45b0 -opentelemetry-instrumentation-asyncio==0.45b0 -opentelemetry-instrumentation-aws-lambda==0.45b0 -opentelemetry-instrumentation-boto==0.45b0 -opentelemetry-instrumentation-botocore==0.45b0 -opentelemetry-instrumentation-httpx==0.45b0 -opentelemetry-instrumentation-logging==0.45b0 -opentelemetry-instrumentation-pymongo==0.45b0 -opentelemetry-instrumentation-requests==0.45b0 -opentelemetry-instrumentation-system-metrics==0.45b0 -opentelemetry-instrumentation-grpc==0.45b0 -opentelemetry-instrumentation-sklearn==0.45b0 -opentelemetry-instrumentation-asgi==0.45b0 +opentelemetry-instrumentation-fastapi==0.46b0 +opentelemetry-instrumentation-aiohttp-client==0.46b0 +opentelemetry-instrumentation-asyncio==0.46b0 +opentelemetry-instrumentation-aws-lambda==0.46b0 +opentelemetry-instrumentation-boto==0.46b0 +opentelemetry-instrumentation-botocore==0.46b0 +opentelemetry-instrumentation-httpx==0.46b0 +opentelemetry-instrumentation-logging==0.46b0 +opentelemetry-instrumentation-pymongo==0.46b0 +opentelemetry-instrumentation-requests==0.46b0 +opentelemetry-instrumentation-system-metrics==0.46b0 +opentelemetry-instrumentation-grpc==0.46b0 +opentelemetry-instrumentation-sklearn==0.46b0 +opentelemetry-instrumentation-asgi==0.46b0 +opentelemetry-instrumentation-requests==0.46b0 +opentelemetry-instrumentation-sklearn==0.46b0 pykwalify==1.8.0 gunicorn==22.0.0 +litellm==1.39.5 +jsonschema_rs==0.18.0 +mongoengine-jsonschema==0.1.3 \ No newline at end of file diff --git a/tests/conftest.py b/tests/conftest.py index c613d74a5..10bd6434c 100644 --- a/tests/conftest.py +++ b/tests/conftest.py @@ -1,6 +1,5 @@ from kairon.shared.concurrency.actors.factory import ActorFactory import pytest -from mock import patch import os diff --git a/tests/integration_test/action_service_test.py b/tests/integration_test/action_service_test.py index ac56df754..6933be2f0 100644 --- a/tests/integration_test/action_service_test.py +++ b/tests/integration_test/action_service_test.py @@ -2,7 +2,8 @@ import os from urllib.parse import urlencode, urljoin -import mock +import litellm +from unittest import mock import numpy as np import pytest import responses @@ -10,8 +11,11 @@ from deepdiff import DeepDiff from fastapi.testclient import TestClient from jira import JIRAError -from mock import patch -from mongoengine import connect, DoesNotExist +from mongoengine import connect + +from kairon.shared.utils import Utility + +Utility.load_system_metadata() from kairon.actions.definitions.live_agent import ActionLiveAgent from kairon.actions.definitions.set_slot import ActionSetSlot @@ -33,15 +37,13 @@ DEFAULT_NLU_FALLBACK_RESPONSE from kairon.shared.data.data_objects import Slots, KeyVault, BotSettings, LLMSettings from kairon.shared.data.processor import MongoProcessor -from kairon.shared.llm.clients.gpt3 import GPT3Resources -from kairon.shared.llm.gpt3 import GPT3FAQEmbedding -from kairon.shared.utils import Utility from kairon.shared.vector_embeddings.db.qdrant import Qdrant os.environ['ASYNC_TEST_TIMEOUT'] = "360" os.environ["system_file"] = "./tests/testing_data/system.yaml" client = TestClient(action) +OPENAI_EMBEDDING_OUTPUT = 1536 @pytest.fixture(autouse=True, scope='class') @@ -83,7 +85,8 @@ def test_live_agent_action_execution(aioresponses): aioresponses.add( method="POST", url=f"{Utility.environment['live_agent']['url']}/conversation/request", - payload={"success": True, "data": {"identifier": "asjlbceuwvbalncouabvlvnlavni", "msg": None }, "message": None, "error_code": 0}, + payload={"success": True, "data": {"identifier": "asjlbceuwvbalncouabvlvnlavni", "msg": None}, "message": None, + "error_code": 0}, body={'bot_id': '5f50fd0a56b698ca10d35d2z', 'sender_id': 'default', 'channel': 'messenger'}, status=200 ) @@ -186,7 +189,9 @@ def test_live_agent_action_execution_no_agent_available(aioresponses): aioresponses.add( method="POST", url=f"{Utility.environment['live_agent']['url']}/conversation/request", - payload={"success": True, "data": {"identifier": "asjlbceuwvbalncouabvlvnlavni", "msg": "live agent is not available" }, "message": None, "error_code": 0}, + payload={"success": True, + "data": {"identifier": "asjlbceuwvbalncouabvlvnlavni", "msg": "live agent is not available"}, + "message": None, "error_code": 0}, body={'bot_id': '5f50fd0a56b698ca10d35d2z', 'sender_id': 'default', 'channel': 'messenger'}, status=200 ) @@ -276,7 +281,6 @@ def test_live_agent_action_execution_no_agent_available(aioresponses): assert response_json['responses'][0]['text'] == 'live agent is not available' - def test_live_agent_action_execution_with_exception(aioresponses): bot_settings = BotSettings(bot='5f50fd0a56b698ca10d35d21', user='user') bot_settings.live_agent_enabled = True @@ -385,7 +389,9 @@ def test_live_agent_action_execution_with_exception(aioresponses): assert response.status_code == 200 assert len(response_json['responses']) == 1 assert response_json['responses'][0]['text'] == 'Connecting to live agent' - assert response_json == {'events': [], 'responses': [{'text': 'Connecting to live agent', 'buttons': [], 'elements': [], 'custom': {}, 'template': None, 'response': None, 'image': None, 'attachment': None}]} + assert response_json == {'events': [], 'responses': [ + {'text': 'Connecting to live agent', 'buttons': [], 'elements': [], 'custom': {}, 'template': None, + 'response': None, 'image': None, 'attachment': None}]} def test_live_agent_action_execution_with_exception(aioresponses): @@ -496,11 +502,12 @@ def test_live_agent_action_execution_with_exception(aioresponses): assert response.status_code == 200 assert len(response_json['responses']) == 1 assert response_json['responses'][0]['text'] == 'Connecting to live agent' - assert response_json == {'events': [], 'responses': [{'text': 'Connecting to live agent', 'buttons': [], 'elements': [], 'custom': {}, 'template': None, 'response': None, 'image': None, 'attachment': None}]} + assert response_json == {'events': [], 'responses': [ + {'text': 'Connecting to live agent', 'buttons': [], 'elements': [], 'custom': {}, 'template': None, + 'response': None, 'image': None, 'attachment': None}]} def test_retrieve_config_failure(): - patch('kairon.actions.definitions.live_agent.LiveAgentActionConfig.objects().get', side_effect=DoesNotExist) action_live_agent = ActionLiveAgent(bot='test_bot', name='test_action') with pytest.raises(ActionFailure, match="No Live Agent action found for given action and bot"): action_live_agent.retrieve_config() @@ -533,14 +540,22 @@ def test_pyscript_action_execution(): json={"success": True, "data": {"bot_response": {'numbers': [1, 2, 3, 4, 5], 'total': 15, 'i': 5}, "slots": {"location": "Bangalore", "langauge": "Kannada"}, "type": "json"}, "message": None, "error_code": 0}, - match=[responses.matchers.json_params_matcher( {'source_code': script, - 'predefined_objects': {'chat_log': [], 'intent': 'pyscript_action', - 'kairon_user_msg': None, 'key_vault': {}, 'latest_message': {'intent_ranking': [{'name': 'pyscript_action'}], 'text': 'get intents'}, - 'sender_id': 'default', 'session_started': None, - 'slot': {'bot': '5f50fd0a56b698ca10d35d2z', 'langauge': 'Kannada', 'location': 'Bangalore'}, - 'user_message': 'get intents'} - - })] + match=[responses.matchers.json_params_matcher({'source_code': script, + 'predefined_objects': {'chat_log': [], + 'intent': 'pyscript_action', + 'kairon_user_msg': None, 'key_vault': {}, + 'latest_message': {'intent_ranking': [ + {'name': 'pyscript_action'}], + 'text': 'get intents'}, + 'sender_id': 'default', + 'session_started': None, + 'slot': { + 'bot': '5f50fd0a56b698ca10d35d2z', + 'langauge': 'Kannada', + 'location': 'Bangalore'}, + 'user_message': 'get intents'} + + })] ) request_object = { @@ -666,6 +681,7 @@ def test_pyscript_action_execution_with_multiple_utterances(): assert response_json['responses'][0]['custom'] == {'text': 'Hello!'} assert response_json['responses'][1]['text'] == 'How can I help you?' + @responses.activate def test_pyscript_action_execution_with_multiple_integer_utterances(): import textwrap @@ -779,7 +795,9 @@ def test_pyscript_action_execution_with_bot_response_none(): "message": None, "error_code": 0}, match=[responses.matchers.json_params_matcher( {'source_code': script, - 'predefined_objects': {'sender_id': 'default', 'user_message': 'get intents', 'latest_message': {'intent_ranking': [{'name': 'pyscript_action'}], 'text': 'get intents'}, + 'predefined_objects': {'sender_id': 'default', 'user_message': 'get intents', + 'latest_message': {'intent_ranking': [{'name': 'pyscript_action'}], + 'text': 'get intents'}, 'slot': {'bot': '5f50fd0a56b698ca10d35d2z', 'location': 'Bangalore', 'langauge': 'Kannada'}, 'intent': 'pyscript_action', 'chat_log': [], 'key_vault': {}, 'kairon_user_msg': None, 'session_started': None}})] @@ -854,7 +872,9 @@ def test_pyscript_action_execution_with_type_json_bot_response_none(): "message": None, "error_code": 0}, match=[responses.matchers.json_params_matcher( {'source_code': script, - 'predefined_objects': {'sender_id': 'default', 'user_message': 'get intents', 'latest_message': {'intent_ranking': [{'name': 'pyscript_action'}], 'text': 'get intents'}, + 'predefined_objects': {'sender_id': 'default', 'user_message': 'get intents', + 'latest_message': {'intent_ranking': [{'name': 'pyscript_action'}], + 'text': 'get intents'}, 'slot': {'bot': '5f50fd0a56b698ca10d35d2z', 'location': 'Bangalore', 'langauge': 'Kannada'}, 'intent': 'pyscript_action', 'chat_log': [], 'key_vault': {}, 'kairon_user_msg': None, 'session_started': None}})] @@ -929,7 +949,9 @@ def test_pyscript_action_execution_with_type_json_bot_response_str(): "message": None, "error_code": 0}, match=[responses.matchers.json_params_matcher( {'source_code': script, - 'predefined_objects': {'sender_id': 'default', 'user_message': 'get intents', 'latest_message': {'intent_ranking': [{'name': 'pyscript_action'}], 'text': 'get intents'}, + 'predefined_objects': {'sender_id': 'default', 'user_message': 'get intents', + 'latest_message': {'intent_ranking': [{'name': 'pyscript_action'}], + 'text': 'get intents'}, 'slot': {'bot': '5f50fd0a56b698ca10d35d2z', 'location': 'Bangalore', 'langauge': 'Kannada'}, 'intent': 'pyscript_action', 'chat_log': [], 'key_vault': {}, 'kairon_user_msg': None, 'session_started': None}})] @@ -1006,7 +1028,9 @@ def test_pyscript_action_execution_with_other_type(): "message": None, "error_code": 0}, match=[responses.matchers.json_params_matcher( {'source_code': script, - 'predefined_objects': {'sender_id': 'default', 'user_message': 'get intents', 'latest_message': {'intent_ranking': [{'name': 'pyscript_action'}], 'text': 'get intents'}, + 'predefined_objects': {'sender_id': 'default', 'user_message': 'get intents', + 'latest_message': {'intent_ranking': [{'name': 'pyscript_action'}], + 'text': 'get intents'}, 'slot': {'bot': '5f50fd0a56b698ca10d35d2z', 'location': 'Bangalore', 'langauge': 'Kannada'}, 'intent': 'pyscript_action', 'chat_log': [], 'key_vault': {}, 'kairon_user_msg': None, 'session_started': None}})] @@ -1081,7 +1105,9 @@ def test_pyscript_action_execution_with_slots_not_dict_type(): "slots": "invalid slots values"}, "message": None, "error_code": 0}, match=[responses.matchers.json_params_matcher( {'source_code': script, - 'predefined_objects': {'sender_id': 'default', 'user_message': 'get intents', 'latest_message': {'intent_ranking': [{'name': 'pyscript_action'}], 'text': 'get intents'}, + 'predefined_objects': {'sender_id': 'default', 'user_message': 'get intents', + 'latest_message': {'intent_ranking': [{'name': 'pyscript_action'}], + 'text': 'get intents'}, 'slot': {'bot': '5f50fd0a56b698ca10d35d2z', 'location': 'Bangalore', 'langauge': 'Kannada'}, 'intent': 'pyscript_action', 'chat_log': [], 'key_vault': {}, 'kairon_user_msg': None, 'session_started': None}})] @@ -1176,7 +1202,7 @@ def test_pyscript_action_execution_without_pyscript_evaluator_url(mock_trigger_l }, "version": "version" } - with patch("kairon.shared.utils.Utility.environment", new=mock_environment): + with mock.patch("kairon.shared.utils.Utility.environment", new=mock_environment): mock_trigger_lambda.return_value = \ {"Payload": {"body": {"bot_response": "Successfully Evaluated the pyscript", "slots": {"location": "Bangalore", "langauge": "Kannada"}}}, "StatusCode": 200} @@ -1186,15 +1212,17 @@ def test_pyscript_action_execution_without_pyscript_evaluator_url(mock_trigger_l assert len(response_json['events']) == 3 assert len(response_json['responses']) == 1 assert response_json['events'] == [ - {'event': 'slot', 'timestamp': None, 'name': 'location', 'value': 'Bangalore'}, - {'event': 'slot', 'timestamp': None, 'name': 'langauge', 'value': 'Kannada'}, - {'event': 'slot', 'timestamp': None, 'name': 'kairon_action_response', - 'value': "Successfully Evaluated the pyscript"}] + {'event': 'slot', 'timestamp': None, 'name': 'location', 'value': 'Bangalore'}, + {'event': 'slot', 'timestamp': None, 'name': 'langauge', 'value': 'Kannada'}, + {'event': 'slot', 'timestamp': None, 'name': 'kairon_action_response', + 'value': "Successfully Evaluated the pyscript"}] assert response_json['responses'][0]['text'] == "Successfully Evaluated the pyscript" called_args = mock_trigger_lambda.call_args assert called_args.args[1] == \ {'source_code': script, - 'predefined_objects': {'sender_id': 'default', 'user_message': 'get intents', 'latest_message': {'intent_ranking': [{'name': 'pyscript_action'}], 'text': 'get intents'}, + 'predefined_objects': {'sender_id': 'default', 'user_message': 'get intents', + 'latest_message': {'intent_ranking': [{'name': 'pyscript_action'}], + 'text': 'get intents'}, 'slot': {"bot": "5f50fd0a56b698ca10d35d2z", "location": "Bangalore", "langauge": "Kannada"}, 'intent': 'pyscript_action', 'chat_log': [], 'key_vault': {}, @@ -1253,7 +1281,7 @@ def test_pyscript_action_execution_without_pyscript_evaluator_url_raise_exceptio }, "version": "version" } - with patch("kairon.shared.utils.Utility.environment", new=mock_environment): + with mock.patch("kairon.shared.utils.Utility.environment", new=mock_environment): mock_trigger_lambda.return_value = {"Payload": {"body": "Failed to evaluated the pyscript"}, "StatusCode": 422} response = client.post("/webhook", json=request_object) response_json = response.json() @@ -1261,8 +1289,8 @@ def test_pyscript_action_execution_without_pyscript_evaluator_url_raise_exceptio assert len(response_json['events']) == 1 assert len(response_json['responses']) == 1 assert response_json['events'] == [ - {'event': 'slot', 'timestamp': None, 'name': 'kairon_action_response', - 'value': "I have failed to process your request"}] + {'event': 'slot', 'timestamp': None, 'name': 'kairon_action_response', + 'value': "I have failed to process your request"}] log = ActionServerLogs.objects(action=action_name).get().to_mongo().to_dict() assert log['exception'] == "Failed to evaluated the pyscript" @@ -1297,7 +1325,9 @@ def raise_custom_exception(request): "POST", Utility.environment['evaluator']['pyscript']['url'], callback=raise_custom_exception, match=[responses.matchers.json_params_matcher( {'source_code': script, - 'predefined_objects': {'sender_id': 'default', 'user_message': 'get intents', 'latest_message': {'intent_ranking': [{'name': 'pyscript_action'}], 'text': 'get intents'}, + 'predefined_objects': {'sender_id': 'default', 'user_message': 'get intents', + 'latest_message': {'intent_ranking': [{'name': 'pyscript_action'}], + 'text': 'get intents'}, 'slot': {'bot': '5f50fd0a56b698ca10d35d2z', 'location': 'Bangalore', 'langauge': 'Kannada'}, 'intent': 'pyscript_action', 'chat_log': [], 'key_vault': {}, 'kairon_user_msg': None, 'session_started': None}})] @@ -1308,7 +1338,7 @@ def raise_custom_exception(request): "tracker": { "sender_id": "default", "conversation_id": "default", - "slots": {"bot": "5f50fd0a56b698ca10d35d2z", "location": "Bangalore", "langauge": "Kannada"}, + "slots": {"bot": "5f50fd0a56b698ca10d35d2z", "location": "Bangalore", "langauge": "Kannada"}, "latest_message": {'text': 'get intents', 'intent_ranking': [{'name': 'pyscript_action'}]}, "latest_event_time": 1537645578.314389, "followup_action": "action_listen", @@ -1368,7 +1398,9 @@ def test_pyscript_action_execution_with_invalid_response(): "error_code": 422}, match=[responses.matchers.json_params_matcher( {'source_code': script, - 'predefined_objects': {'sender_id': 'default', 'user_message': 'get intents', 'latest_message': {'intent_ranking': [{'name': 'pyscript_action'}], 'text': 'get intents'}, + 'predefined_objects': {'sender_id': 'default', 'user_message': 'get intents', + 'latest_message': {'intent_ranking': [{'name': 'pyscript_action'}], + 'text': 'get intents'}, 'slot': {'bot': '5f50fd0a56b698ca10d35d2z', 'location': 'Bangalore', 'langauge': 'Kannada'}, 'intent': 'pyscript_action', 'chat_log': [], 'key_vault': {}, 'kairon_user_msg': None, 'session_started': None}})] @@ -1410,7 +1442,8 @@ def test_pyscript_action_execution_with_invalid_response(): {'event': 'slot', 'timestamp': None, 'name': 'kairon_action_response', 'value': 'I have failed to process your request'}] log = ActionServerLogs.objects(action=action_name).get().to_mongo().to_dict() - assert log['exception'] == 'Pyscript evaluation failed: {\'success\': False, \'data\': None, \'message\': \'Script execution error: ("Line 2: SyntaxError: invalid syntax at statement: for i in 10",)\', \'error_code\': 422}' + assert log[ + 'exception'] == 'Pyscript evaluation failed: {\'success\': False, \'data\': None, \'message\': \'Script execution error: ("Line 2: SyntaxError: invalid syntax at statement: for i in 10",)\', \'error_code\': 422}' def test_http_action_execution(aioresponses): @@ -1510,8 +1543,42 @@ def test_http_action_execution(aioresponses): if event.get('time_elapsed') is not None: del event['time_elapsed'] print(events) - assert events == [{'type': 'slots', 'data': [{'name': 'val_d', 'value': '${data.a.b.d}', 'evaluation_type': 'expression', 'slot_value': None}, {'name': 'val_d_0', 'value': '${data.a.b.d.0}', 'evaluation_type': 'expression', 'slot_value': None}]}, {'type': 'response', 'dispatch_bot_response': True, 'dispatch_type': 'text', 'data': 'The value of ${data.a.b.3} in ${data.a.b.d.0} is ${data.a.b.d}', 'evaluation_type': 'expression', 'response': "The value of 2 in red is ['red', 'buggy', 'bumpers']", 'bot_response_log': ['evaluation_type: expression', 'expression: The value of ${data.a.b.3} in ${data.a.b.d.0} is ${data.a.b.d}', "data: {'data': {'a': {'b': {'3': 2, '43': 30, 'c': [], 'd': ['red', 'buggy', 'bumpers']}}}, 'context': {'sender_id': 'default', 'user_message': 'get intents', 'slot': {'bot': '5f50fd0a56b698ca10d35d2e'}, 'intent': 'test_run', 'chat_log': [], 'key_vault': {'EMAIL': 'uditpandey@digite.com', 'FIRSTNAME': 'udit'}, 'latest_message': {'text': 'get intents', 'intent_ranking': [{'name': 'test_run'}]}, 'kairon_user_msg': None, 'session_started': None, 'bot': '5f50fd0a56b698ca10d35d2e'}, 'http_status_code': 200}", "response: The value of 2 in red is ['red', 'buggy', 'bumpers']"]}, {'type': 'api_call', 'headers': {'botid': '**********************2e', 'userid': '****', 'tag': '******ot', 'email': '*******************om'}, 'method': 'GET', 'url': 'http://localhost:8081/mock', 'payload': {'bot': '5f50fd0a56b698ca10d35d2e', 'user': '1011', 'tag': 'from_bot', 'name': 'udit', 'contact': ''}, 'response': {'a': {'b': {'3': 2, '43': 30, 'c': [], 'd': ['red', 'buggy', 'bumpers']}}}, 'status_code': 200}, {'type': 'params_list', 'request_body': {'bot': '5f50fd0a56b698ca10d35d2e', 'user': '1011', 'tag': 'from_bot', 'name': 'udit', 'contact': ''}, 'request_params': {'bot': '**********************2e', 'user': '1011', 'tag': '******ot', 'name': '****', 'contact': None}}, {'type': 'filled_slots', 'data': {'val_d': "['red', 'buggy', 'bumpers']", 'val_d_0': 'red'}, 'slot_eval_log': ['initiating slot evaluation', 'Slot: val_d', 'evaluation_type: expression', 'expression: ${data.a.b.d}', "data: {'data': {'a': {'b': {'3': 2, '43': 30, 'c': [], 'd': ['red', 'buggy', 'bumpers']}}}, 'context': {'sender_id': 'default', 'user_message': 'get intents', 'slot': {'bot': '5f50fd0a56b698ca10d35d2e'}, 'intent': 'test_run', 'chat_log': [], 'key_vault': {'EMAIL': 'uditpandey@digite.com', 'FIRSTNAME': 'udit'}, 'latest_message': {'text': 'get intents', 'intent_ranking': [{'name': 'test_run'}]}, 'kairon_user_msg': None, 'session_started': None, 'bot': '5f50fd0a56b698ca10d35d2e'}, 'http_status_code': 200}", "response: ['red', 'buggy', 'bumpers']", 'Slot: val_d_0', 'evaluation_type: expression', 'expression: ${data.a.b.d.0}', "data: {'data': {'a': {'b': {'3': 2, '43': 30, 'c': [], 'd': ['red', 'buggy', 'bumpers']}}}, 'context': {'sender_id': 'default', 'user_message': 'get intents', 'slot': {'bot': '5f50fd0a56b698ca10d35d2e'}, 'intent': 'test_run', 'chat_log': [], 'key_vault': {'EMAIL': 'uditpandey@digite.com', 'FIRSTNAME': 'udit'}, 'latest_message': {'text': 'get intents', 'intent_ranking': [{'name': 'test_run'}]}, 'kairon_user_msg': None, 'session_started': None, 'bot': '5f50fd0a56b698ca10d35d2e'}, 'http_status_code': 200}", 'response: red']}] - assert log == {'type': 'http_action', 'intent': 'test_run', 'action': 'test_http_action_execution', 'sender': 'default', 'headers': {}, 'url': 'http://localhost:8081/mock', 'request_method': 'GET', 'bot_response': "The value of 2 in red is ['red', 'buggy', 'bumpers']", 'bot': '5f50fd0a56b698ca10d35d2e', 'status': 'SUCCESS', 'fail_reason': None, 'user_msg': 'get intents', 'http_status_code': 200} + assert events == [{'type': 'slots', 'data': [ + {'name': 'val_d', 'value': '${data.a.b.d}', 'evaluation_type': 'expression', 'slot_value': None}, + {'name': 'val_d_0', 'value': '${data.a.b.d.0}', 'evaluation_type': 'expression', 'slot_value': None}]}, + {'type': 'response', 'dispatch_bot_response': True, 'dispatch_type': 'text', + 'data': 'The value of ${data.a.b.3} in ${data.a.b.d.0} is ${data.a.b.d}', + 'evaluation_type': 'expression', + 'response': "The value of 2 in red is ['red', 'buggy', 'bumpers']", + 'bot_response_log': ['evaluation_type: expression', + 'expression: The value of ${data.a.b.3} in ${data.a.b.d.0} is ${data.a.b.d}', + "data: {'data': {'a': {'b': {'3': 2, '43': 30, 'c': [], 'd': ['red', 'buggy', 'bumpers']}}}, 'context': {'sender_id': 'default', 'user_message': 'get intents', 'slot': {'bot': '5f50fd0a56b698ca10d35d2e'}, 'intent': 'test_run', 'chat_log': [], 'key_vault': {'EMAIL': 'uditpandey@digite.com', 'FIRSTNAME': 'udit'}, 'latest_message': {'text': 'get intents', 'intent_ranking': [{'name': 'test_run'}]}, 'kairon_user_msg': None, 'session_started': None, 'bot': '5f50fd0a56b698ca10d35d2e'}, 'http_status_code': 200}", + "response: The value of 2 in red is ['red', 'buggy', 'bumpers']"]}, + {'type': 'api_call', + 'headers': {'botid': '**********************2e', 'userid': '****', 'tag': '******ot', + 'email': '*******************om'}, 'method': 'GET', + 'url': 'http://localhost:8081/mock', + 'payload': {'bot': '5f50fd0a56b698ca10d35d2e', 'user': '1011', 'tag': 'from_bot', 'name': 'udit', + 'contact': ''}, + 'response': {'a': {'b': {'3': 2, '43': 30, 'c': [], 'd': ['red', 'buggy', 'bumpers']}}}, + 'status_code': 200}, {'type': 'params_list', + 'request_body': {'bot': '5f50fd0a56b698ca10d35d2e', 'user': '1011', + 'tag': 'from_bot', 'name': 'udit', 'contact': ''}, + 'request_params': {'bot': '**********************2e', 'user': '1011', + 'tag': '******ot', 'name': '****', 'contact': None}}, + {'type': 'filled_slots', 'data': {'val_d': "['red', 'buggy', 'bumpers']", 'val_d_0': 'red'}, + 'slot_eval_log': ['initiating slot evaluation', 'Slot: val_d', 'evaluation_type: expression', + 'expression: ${data.a.b.d}', + "data: {'data': {'a': {'b': {'3': 2, '43': 30, 'c': [], 'd': ['red', 'buggy', 'bumpers']}}}, 'context': {'sender_id': 'default', 'user_message': 'get intents', 'slot': {'bot': '5f50fd0a56b698ca10d35d2e'}, 'intent': 'test_run', 'chat_log': [], 'key_vault': {'EMAIL': 'uditpandey@digite.com', 'FIRSTNAME': 'udit'}, 'latest_message': {'text': 'get intents', 'intent_ranking': [{'name': 'test_run'}]}, 'kairon_user_msg': None, 'session_started': None, 'bot': '5f50fd0a56b698ca10d35d2e'}, 'http_status_code': 200}", + "response: ['red', 'buggy', 'bumpers']", 'Slot: val_d_0', + 'evaluation_type: expression', 'expression: ${data.a.b.d.0}', + "data: {'data': {'a': {'b': {'3': 2, '43': 30, 'c': [], 'd': ['red', 'buggy', 'bumpers']}}}, 'context': {'sender_id': 'default', 'user_message': 'get intents', 'slot': {'bot': '5f50fd0a56b698ca10d35d2e'}, 'intent': 'test_run', 'chat_log': [], 'key_vault': {'EMAIL': 'uditpandey@digite.com', 'FIRSTNAME': 'udit'}, 'latest_message': {'text': 'get intents', 'intent_ranking': [{'name': 'test_run'}]}, 'kairon_user_msg': None, 'session_started': None, 'bot': '5f50fd0a56b698ca10d35d2e'}, 'http_status_code': 200}", + 'response: red']}] + assert log == {'type': 'http_action', 'intent': 'test_run', 'action': 'test_http_action_execution', + 'sender': 'default', 'headers': {}, 'url': 'http://localhost:8081/mock', 'request_method': 'GET', + 'bot_response': "The value of 2 in red is ['red', 'buggy', 'bumpers']", + 'bot': '5f50fd0a56b698ca10d35d2e', 'status': 'SUCCESS', 'fail_reason': None, + 'user_msg': 'get intents', 'http_status_code': 200} def test_http_action_execution_returns_custom_json(aioresponses): @@ -1974,8 +2041,41 @@ def test_http_action_execution_no_response_dispatch(aioresponses): if event.get('time_elapsed') is not None: del event['time_elapsed'] print(events) - assert events == [{'type': 'slots', 'data': [{'name': 'val_d', 'value': '${data.a.b.d}', 'evaluation_type': 'expression', 'slot_value': None}, {'name': 'val_d_0', 'value': '${data.a.b.d.0}', 'evaluation_type': 'expression', 'slot_value': None}]}, {'type': 'response', 'dispatch_bot_response': False, 'dispatch_type': 'text', 'data': 'The value of ${data.a.b.3} in ${data.a.b.d.0} is ${data.a.b.d}', 'evaluation_type': 'expression', 'response': "The value of 2 in red is ['red', 'buggy', 'bumpers']", 'bot_response_log': ['evaluation_type: expression', 'expression: The value of ${data.a.b.3} in ${data.a.b.d.0} is ${data.a.b.d}', "data: {'data': {'a': {'b': {'3': 2, '43': 30, 'c': [], 'd': ['red', 'buggy', 'bumpers']}}}, 'context': {'sender_id': 'default', 'user_message': 'get intents', 'slot': {'bot': '5f50fd0a56b698ca10d35d2e'}, 'intent': 'test_run', 'chat_log': [], 'key_vault': {'EMAIL': 'uditpandey@digite.com', 'FIRSTNAME': 'udit'}, 'latest_message': {'text': 'get intents', 'intent_ranking': [{'name': 'test_run'}]}, 'kairon_user_msg': None, 'session_started': None, 'bot': '5f50fd0a56b698ca10d35d2e'}, 'http_status_code': 200}", "response: The value of 2 in red is ['red', 'buggy', 'bumpers']"]}, {'type': 'api_call', 'headers': {'botid': '5f50fd0a56b698ca10d35d2e', 'userid': '****', 'tag': '******ot'}, 'method': 'GET', 'url': 'http://localhost:8081/mock', 'payload': {'bot': '5f50fd0a56b698ca10d35d2e', 'user': '1011', 'tag': 'from_bot'}, 'response': {'a': {'b': {'3': 2, '43': 30, 'c': [], 'd': ['red', 'buggy', 'bumpers']}}}, 'status_code': 200}, {'type': 'params_list', 'request_body': {'bot': '5f50fd0a56b698ca10d35d2e', 'user': '1011', 'tag': 'from_bot'}, 'request_params': {'bot': '5f50fd0a56b698ca10d35d2e', 'user': '1011', 'tag': '******ot'}}, {'type': 'filled_slots', 'data': {'val_d': "['red', 'buggy', 'bumpers']", 'val_d_0': 'red'}, 'slot_eval_log': ['initiating slot evaluation', 'Slot: val_d', 'evaluation_type: expression', 'expression: ${data.a.b.d}', "data: {'data': {'a': {'b': {'3': 2, '43': 30, 'c': [], 'd': ['red', 'buggy', 'bumpers']}}}, 'context': {'sender_id': 'default', 'user_message': 'get intents', 'slot': {'bot': '5f50fd0a56b698ca10d35d2e'}, 'intent': 'test_run', 'chat_log': [], 'key_vault': {'EMAIL': 'uditpandey@digite.com', 'FIRSTNAME': 'udit'}, 'latest_message': {'text': 'get intents', 'intent_ranking': [{'name': 'test_run'}]}, 'kairon_user_msg': None, 'session_started': None, 'bot': '5f50fd0a56b698ca10d35d2e'}, 'http_status_code': 200}", "response: ['red', 'buggy', 'bumpers']", 'Slot: val_d_0', 'evaluation_type: expression', 'expression: ${data.a.b.d.0}', "data: {'data': {'a': {'b': {'3': 2, '43': 30, 'c': [], 'd': ['red', 'buggy', 'bumpers']}}}, 'context': {'sender_id': 'default', 'user_message': 'get intents', 'slot': {'bot': '5f50fd0a56b698ca10d35d2e'}, 'intent': 'test_run', 'chat_log': [], 'key_vault': {'EMAIL': 'uditpandey@digite.com', 'FIRSTNAME': 'udit'}, 'latest_message': {'text': 'get intents', 'intent_ranking': [{'name': 'test_run'}]}, 'kairon_user_msg': None, 'session_started': None, 'bot': '5f50fd0a56b698ca10d35d2e'}, 'http_status_code': 200}", 'response: red']}] - assert log == {'type': 'http_action', 'intent': 'test_run', 'action': 'test_http_action_execution_no_response_dispatch', 'sender': 'default', 'headers': {}, 'url': 'http://localhost:8081/mock', 'request_method': 'GET', 'bot_response': "The value of 2 in red is ['red', 'buggy', 'bumpers']", 'bot': '5f50fd0a56b698ca10d35d2e', 'status': 'SUCCESS', 'fail_reason': None, 'user_msg': 'get intents', 'http_status_code': 200} + assert events == [{'type': 'slots', 'data': [ + {'name': 'val_d', 'value': '${data.a.b.d}', 'evaluation_type': 'expression', 'slot_value': None}, + {'name': 'val_d_0', 'value': '${data.a.b.d.0}', 'evaluation_type': 'expression', 'slot_value': None}]}, + {'type': 'response', 'dispatch_bot_response': False, 'dispatch_type': 'text', + 'data': 'The value of ${data.a.b.3} in ${data.a.b.d.0} is ${data.a.b.d}', + 'evaluation_type': 'expression', + 'response': "The value of 2 in red is ['red', 'buggy', 'bumpers']", + 'bot_response_log': ['evaluation_type: expression', + 'expression: The value of ${data.a.b.3} in ${data.a.b.d.0} is ${data.a.b.d}', + "data: {'data': {'a': {'b': {'3': 2, '43': 30, 'c': [], 'd': ['red', 'buggy', 'bumpers']}}}, 'context': {'sender_id': 'default', 'user_message': 'get intents', 'slot': {'bot': '5f50fd0a56b698ca10d35d2e'}, 'intent': 'test_run', 'chat_log': [], 'key_vault': {'EMAIL': 'uditpandey@digite.com', 'FIRSTNAME': 'udit'}, 'latest_message': {'text': 'get intents', 'intent_ranking': [{'name': 'test_run'}]}, 'kairon_user_msg': None, 'session_started': None, 'bot': '5f50fd0a56b698ca10d35d2e'}, 'http_status_code': 200}", + "response: The value of 2 in red is ['red', 'buggy', 'bumpers']"]}, + {'type': 'api_call', + 'headers': {'botid': '5f50fd0a56b698ca10d35d2e', 'userid': '****', 'tag': '******ot'}, + 'method': 'GET', 'url': 'http://localhost:8081/mock', + 'payload': {'bot': '5f50fd0a56b698ca10d35d2e', 'user': '1011', 'tag': 'from_bot'}, + 'response': {'a': {'b': {'3': 2, '43': 30, 'c': [], 'd': ['red', 'buggy', 'bumpers']}}}, + 'status_code': 200}, {'type': 'params_list', + 'request_body': {'bot': '5f50fd0a56b698ca10d35d2e', 'user': '1011', + 'tag': 'from_bot'}, + 'request_params': {'bot': '5f50fd0a56b698ca10d35d2e', 'user': '1011', + 'tag': '******ot'}}, + {'type': 'filled_slots', 'data': {'val_d': "['red', 'buggy', 'bumpers']", 'val_d_0': 'red'}, + 'slot_eval_log': ['initiating slot evaluation', 'Slot: val_d', 'evaluation_type: expression', + 'expression: ${data.a.b.d}', + "data: {'data': {'a': {'b': {'3': 2, '43': 30, 'c': [], 'd': ['red', 'buggy', 'bumpers']}}}, 'context': {'sender_id': 'default', 'user_message': 'get intents', 'slot': {'bot': '5f50fd0a56b698ca10d35d2e'}, 'intent': 'test_run', 'chat_log': [], 'key_vault': {'EMAIL': 'uditpandey@digite.com', 'FIRSTNAME': 'udit'}, 'latest_message': {'text': 'get intents', 'intent_ranking': [{'name': 'test_run'}]}, 'kairon_user_msg': None, 'session_started': None, 'bot': '5f50fd0a56b698ca10d35d2e'}, 'http_status_code': 200}", + "response: ['red', 'buggy', 'bumpers']", 'Slot: val_d_0', + 'evaluation_type: expression', 'expression: ${data.a.b.d.0}', + "data: {'data': {'a': {'b': {'3': 2, '43': 30, 'c': [], 'd': ['red', 'buggy', 'bumpers']}}}, 'context': {'sender_id': 'default', 'user_message': 'get intents', 'slot': {'bot': '5f50fd0a56b698ca10d35d2e'}, 'intent': 'test_run', 'chat_log': [], 'key_vault': {'EMAIL': 'uditpandey@digite.com', 'FIRSTNAME': 'udit'}, 'latest_message': {'text': 'get intents', 'intent_ranking': [{'name': 'test_run'}]}, 'kairon_user_msg': None, 'session_started': None, 'bot': '5f50fd0a56b698ca10d35d2e'}, 'http_status_code': 200}", + 'response: red']}] + assert log == {'type': 'http_action', 'intent': 'test_run', + 'action': 'test_http_action_execution_no_response_dispatch', 'sender': 'default', 'headers': {}, + 'url': 'http://localhost:8081/mock', 'request_method': 'GET', + 'bot_response': "The value of 2 in red is ['red', 'buggy', 'bumpers']", + 'bot': '5f50fd0a56b698ca10d35d2e', 'status': 'SUCCESS', 'fail_reason': None, + 'user_msg': 'get intents', 'http_status_code': 200} @responses.activate @@ -2074,8 +2174,22 @@ def test_http_action_execution_script_evaluation(aioresponses): if event.get('time_elapsed') is not None: del event['time_elapsed'] print(events) - assert events == [{'type': 'response', 'dispatch_bot_response': True, 'dispatch_type': 'text', 'data': "bot_response = data['b']['name']", 'evaluation_type': 'script', 'response': 'Mayank', 'bot_response_log': ['evaluation_type: script', "script: bot_response = data['b']['name']", "data: {'data': {'a': 10, 'b': {'name': 'Mayank', 'arr': ['red', 'green', 'hotpink']}}, 'context': {'sender_id': 'default', 'user_message': 'get intents', 'slot': {'bot': '5f50fd0a56b698ca10d35d2e'}, 'intent': 'test_run', 'chat_log': [], 'key_vault': {'EMAIL': 'uditpandey@digite.com', 'FIRSTNAME': 'udit'}, 'latest_message': {'text': 'get intents', 'intent_ranking': [{'name': 'test_run'}]}, 'kairon_user_msg': None, 'session_started': None, 'bot': '5f50fd0a56b698ca10d35d2e'}, 'http_status_code': 200}", 'raise_err_on_failure: True']}, {'type': 'api_call', 'headers': {'botid': '5f50fd0a56b698ca10d35d2e', 'userid': '****', 'tag': '******ot'}, 'method': 'GET', 'url': 'http://localhost:8081/mock', 'payload': {}, 'response': {'a': 10, 'b': {'name': 'Mayank', 'arr': ['red', 'green', 'hotpink']}}, 'status_code': 200}, {'type': 'params_list', 'request_body': {}, 'request_params': {}}, {'type': 'filled_slots', 'data': {}, 'slot_eval_log': ['initiating slot evaluation']}] - assert log == {'type': 'http_action', 'intent': 'test_run', 'action': 'test_http_action_execution_script_evaluation', 'sender': 'default', 'headers': {}, 'url': 'http://localhost:8081/mock', 'request_method': 'GET', 'bot_response': 'Mayank', 'bot': '5f50fd0a56b698ca10d35d2e', 'status': 'SUCCESS', 'fail_reason': None, 'user_msg': 'get intents', 'http_status_code': 200} + assert events == [{'type': 'response', 'dispatch_bot_response': True, 'dispatch_type': 'text', + 'data': "bot_response = data['b']['name']", 'evaluation_type': 'script', 'response': 'Mayank', + 'bot_response_log': ['evaluation_type: script', "script: bot_response = data['b']['name']", + "data: {'data': {'a': 10, 'b': {'name': 'Mayank', 'arr': ['red', 'green', 'hotpink']}}, 'context': {'sender_id': 'default', 'user_message': 'get intents', 'slot': {'bot': '5f50fd0a56b698ca10d35d2e'}, 'intent': 'test_run', 'chat_log': [], 'key_vault': {'EMAIL': 'uditpandey@digite.com', 'FIRSTNAME': 'udit'}, 'latest_message': {'text': 'get intents', 'intent_ranking': [{'name': 'test_run'}]}, 'kairon_user_msg': None, 'session_started': None, 'bot': '5f50fd0a56b698ca10d35d2e'}, 'http_status_code': 200}", + 'raise_err_on_failure: True']}, {'type': 'api_call', 'headers': { + 'botid': '5f50fd0a56b698ca10d35d2e', 'userid': '****', 'tag': '******ot'}, 'method': 'GET', + 'url': 'http://localhost:8081/mock', + 'payload': {}, 'response': {'a': 10, 'b': { + 'name': 'Mayank', 'arr': ['red', 'green', 'hotpink']}}, 'status_code': 200}, + {'type': 'params_list', 'request_body': {}, 'request_params': {}}, + {'type': 'filled_slots', 'data': {}, 'slot_eval_log': ['initiating slot evaluation']}] + assert log == {'type': 'http_action', 'intent': 'test_run', + 'action': 'test_http_action_execution_script_evaluation', 'sender': 'default', 'headers': {}, + 'url': 'http://localhost:8081/mock', 'request_method': 'GET', 'bot_response': 'Mayank', + 'bot': '5f50fd0a56b698ca10d35d2e', 'status': 'SUCCESS', 'fail_reason': None, + 'user_msg': 'get intents', 'http_status_code': 200} @responses.activate @@ -2200,8 +2314,34 @@ def test_http_action_execution_script_evaluation_with_dynamic_params_post(aiores if event.get('time_elapsed') is not None: del event['time_elapsed'] print(events) - assert events == [{'type': 'response', 'dispatch_bot_response': True, 'dispatch_type': 'text', 'data': "bot_response = data['b']['name']", 'evaluation_type': 'script', 'response': 'Mayank', 'bot_response_log': ['evaluation_type: script', "script: bot_response = data['b']['name']", "data: {'data': {'a': 10, 'b': {'name': 'Mayank', 'arr': ['red', 'green', 'hotpink']}}, 'context': {'sender_id': 'default', 'user_message': 'get intents', 'slot': {'bot': '5f50fd0a56b698ca10d35d2e'}, 'intent': 'test_run', 'chat_log': [], 'key_vault': {'EMAIL': 'uditpandey@digite.com', 'FIRSTNAME': 'udit'}, 'latest_message': {'text': 'get intents', 'intent_ranking': [{'name': 'test_run'}]}, 'kairon_user_msg': None, 'session_started': None, 'bot': '5f50fd0a56b698ca10d35d2e'}, 'http_status_code': 200}", 'raise_err_on_failure: True']}, {'type': 'api_call', 'headers': {'botid': '5f50fd0a56b698ca10d35d2e', 'userid': '****', 'tag': '******ot'}, 'method': 'POST', 'url': 'http://localhost:8081/mock', 'payload': {'sender_id': 'default', 'user_message': 'get intents', 'intent': 'test_run'}, 'response': {'a': 10, 'b': {'name': 'Mayank', 'arr': ['red', 'green', 'hotpink']}}, 'status_code': 200}, {'type': 'dynamic_params', 'data': "body = {'sender_id': sender_id, 'user_message': user_message, 'intent': intent}", 'response': {'sender_id': 'default', 'user_message': 'get intents', 'intent': 'test_run'}, 'slots': {}, 'request_params': ['evaluation_type: script', "script: body = {'sender_id': sender_id, 'user_message': user_message, 'intent': intent}", "data: {'sender_id': 'default', 'user_message': 'get intents', 'slot': {'bot': '5f50fd0a56b698ca10d35d2e'}, 'intent': 'test_run', 'chat_log': [], 'key_vault': {'EMAIL': 'uditpandey@digite.com', 'FIRSTNAME': 'udit'}, 'latest_message': {'text': 'get intents', 'intent_ranking': [{'name': 'test_run'}]}, 'kairon_user_msg': None, 'session_started': None, 'bot': '5f50fd0a56b698ca10d35d2e'}", 'raise_err_on_failure: True']}, {'type': 'filled_slots', 'data': {}, 'slot_eval_log': ['initiating slot evaluation']}] - assert log == {'type': 'http_action', 'intent': 'test_run', 'action': 'test_http_action_execution_script_evaluation_with_dynamic_params_post', 'sender': 'default', 'headers': {}, 'url': 'http://localhost:8081/mock', 'request_method': 'POST', 'bot_response': 'Mayank', 'bot': '5f50fd0a56b698ca10d35d2e', 'status': 'SUCCESS', 'fail_reason': None, 'user_msg': 'get intents', 'http_status_code': 200} + assert events == [{'type': 'response', 'dispatch_bot_response': True, 'dispatch_type': 'text', + 'data': "bot_response = data['b']['name']", 'evaluation_type': 'script', 'response': 'Mayank', + 'bot_response_log': ['evaluation_type: script', "script: bot_response = data['b']['name']", + "data: {'data': {'a': 10, 'b': {'name': 'Mayank', 'arr': ['red', 'green', 'hotpink']}}, 'context': {'sender_id': 'default', 'user_message': 'get intents', 'slot': {'bot': '5f50fd0a56b698ca10d35d2e'}, 'intent': 'test_run', 'chat_log': [], 'key_vault': {'EMAIL': 'uditpandey@digite.com', 'FIRSTNAME': 'udit'}, 'latest_message': {'text': 'get intents', 'intent_ranking': [{'name': 'test_run'}]}, 'kairon_user_msg': None, 'session_started': None, 'bot': '5f50fd0a56b698ca10d35d2e'}, 'http_status_code': 200}", + 'raise_err_on_failure: True']}, {'type': 'api_call', 'headers': { + 'botid': '5f50fd0a56b698ca10d35d2e', 'userid': '****', 'tag': '******ot'}, 'method': 'POST', + 'url': 'http://localhost:8081/mock', + 'payload': {'sender_id': 'default', + 'user_message': 'get intents', + 'intent': 'test_run'}, + 'response': {'a': 10, + 'b': {'name': 'Mayank', + 'arr': ['red', 'green', + 'hotpink']}}, + 'status_code': 200}, + {'type': 'dynamic_params', + 'data': "body = {'sender_id': sender_id, 'user_message': user_message, 'intent': intent}", + 'response': {'sender_id': 'default', 'user_message': 'get intents', 'intent': 'test_run'}, + 'slots': {}, 'request_params': ['evaluation_type: script', + "script: body = {'sender_id': sender_id, 'user_message': user_message, 'intent': intent}", + "data: {'sender_id': 'default', 'user_message': 'get intents', 'slot': {'bot': '5f50fd0a56b698ca10d35d2e'}, 'intent': 'test_run', 'chat_log': [], 'key_vault': {'EMAIL': 'uditpandey@digite.com', 'FIRSTNAME': 'udit'}, 'latest_message': {'text': 'get intents', 'intent_ranking': [{'name': 'test_run'}]}, 'kairon_user_msg': None, 'session_started': None, 'bot': '5f50fd0a56b698ca10d35d2e'}", + 'raise_err_on_failure: True']}, + {'type': 'filled_slots', 'data': {}, 'slot_eval_log': ['initiating slot evaluation']}] + assert log == {'type': 'http_action', 'intent': 'test_run', + 'action': 'test_http_action_execution_script_evaluation_with_dynamic_params_post', + 'sender': 'default', 'headers': {}, 'url': 'http://localhost:8081/mock', 'request_method': 'POST', + 'bot_response': 'Mayank', 'bot': '5f50fd0a56b698ca10d35d2e', 'status': 'SUCCESS', + 'fail_reason': None, 'user_msg': 'get intents', 'http_status_code': 200} @responses.activate @@ -2325,8 +2465,34 @@ def test_http_action_execution_script_evaluation_with_dynamic_params(aioresponse if event.get('time_elapsed') is not None: del event['time_elapsed'] print(events) - assert events == [{'type': 'response', 'dispatch_bot_response': True, 'dispatch_type': 'text', 'data': "bot_response = data['b']['name']", 'evaluation_type': 'script', 'response': 'Mayank', 'bot_response_log': ['evaluation_type: script', "script: bot_response = data['b']['name']", "data: {'data': {'a': 10, 'b': {'name': 'Mayank', 'arr': ['red', 'green', 'hotpink']}}, 'context': {'sender_id': 'default', 'user_message': 'get intents', 'slot': {'bot': '5f50fd0a56b698ca10d35d2e'}, 'intent': 'test_run', 'chat_log': [], 'key_vault': {'EMAIL': 'uditpandey@digite.com', 'FIRSTNAME': 'udit'}, 'latest_message': {'text': 'get intents', 'intent_ranking': [{'name': 'test_run'}]}, 'kairon_user_msg': None, 'session_started': None, 'bot': '5f50fd0a56b698ca10d35d2e'}, 'http_status_code': 200}", 'raise_err_on_failure: True']}, {'type': 'api_call', 'headers': {'botid': '5f50fd0a56b698ca10d35d2e', 'userid': '****', 'tag': '******ot'}, 'method': 'GET', 'url': 'http://localhost:8081/mock', 'payload': {'sender_id': 'default', 'user_message': 'get intents', 'intent': 'test_run'}, 'response': {'a': 10, 'b': {'name': 'Mayank', 'arr': ['red', 'green', 'hotpink']}}, 'status_code': 200}, {'type': 'dynamic_params', 'data': "body = {'sender_id': sender_id, 'user_message': user_message, 'intent': intent}", 'response': {'sender_id': 'default', 'user_message': 'get intents', 'intent': 'test_run'}, 'slots': {}, 'request_params': ['evaluation_type: script', "script: body = {'sender_id': sender_id, 'user_message': user_message, 'intent': intent}", "data: {'sender_id': 'default', 'user_message': 'get intents', 'slot': {'bot': '5f50fd0a56b698ca10d35d2e'}, 'intent': 'test_run', 'chat_log': [], 'key_vault': {'EMAIL': 'uditpandey@digite.com', 'FIRSTNAME': 'udit'}, 'latest_message': {'text': 'get intents', 'intent_ranking': [{'name': 'test_run'}]}, 'kairon_user_msg': None, 'session_started': None, 'bot': '5f50fd0a56b698ca10d35d2e'}", 'raise_err_on_failure: True']}, {'type': 'filled_slots', 'data': {}, 'slot_eval_log': ['initiating slot evaluation']}] - assert log == {'type': 'http_action', 'intent': 'test_run', 'action': 'test_http_action_execution_script_evaluation_with_dynamic_params', 'sender': 'default', 'headers': {}, 'url': 'http://localhost:8081/mock', 'request_method': 'GET', 'bot_response': 'Mayank', 'bot': '5f50fd0a56b698ca10d35d2e', 'status': 'SUCCESS', 'fail_reason': None, 'user_msg': 'get intents', 'http_status_code': 200} + assert events == [{'type': 'response', 'dispatch_bot_response': True, 'dispatch_type': 'text', + 'data': "bot_response = data['b']['name']", 'evaluation_type': 'script', 'response': 'Mayank', + 'bot_response_log': ['evaluation_type: script', "script: bot_response = data['b']['name']", + "data: {'data': {'a': 10, 'b': {'name': 'Mayank', 'arr': ['red', 'green', 'hotpink']}}, 'context': {'sender_id': 'default', 'user_message': 'get intents', 'slot': {'bot': '5f50fd0a56b698ca10d35d2e'}, 'intent': 'test_run', 'chat_log': [], 'key_vault': {'EMAIL': 'uditpandey@digite.com', 'FIRSTNAME': 'udit'}, 'latest_message': {'text': 'get intents', 'intent_ranking': [{'name': 'test_run'}]}, 'kairon_user_msg': None, 'session_started': None, 'bot': '5f50fd0a56b698ca10d35d2e'}, 'http_status_code': 200}", + 'raise_err_on_failure: True']}, {'type': 'api_call', 'headers': { + 'botid': '5f50fd0a56b698ca10d35d2e', 'userid': '****', 'tag': '******ot'}, 'method': 'GET', + 'url': 'http://localhost:8081/mock', + 'payload': {'sender_id': 'default', + 'user_message': 'get intents', + 'intent': 'test_run'}, + 'response': {'a': 10, + 'b': {'name': 'Mayank', + 'arr': ['red', 'green', + 'hotpink']}}, + 'status_code': 200}, + {'type': 'dynamic_params', + 'data': "body = {'sender_id': sender_id, 'user_message': user_message, 'intent': intent}", + 'response': {'sender_id': 'default', 'user_message': 'get intents', 'intent': 'test_run'}, + 'slots': {}, 'request_params': ['evaluation_type: script', + "script: body = {'sender_id': sender_id, 'user_message': user_message, 'intent': intent}", + "data: {'sender_id': 'default', 'user_message': 'get intents', 'slot': {'bot': '5f50fd0a56b698ca10d35d2e'}, 'intent': 'test_run', 'chat_log': [], 'key_vault': {'EMAIL': 'uditpandey@digite.com', 'FIRSTNAME': 'udit'}, 'latest_message': {'text': 'get intents', 'intent_ranking': [{'name': 'test_run'}]}, 'kairon_user_msg': None, 'session_started': None, 'bot': '5f50fd0a56b698ca10d35d2e'}", + 'raise_err_on_failure: True']}, + {'type': 'filled_slots', 'data': {}, 'slot_eval_log': ['initiating slot evaluation']}] + assert log == {'type': 'http_action', 'intent': 'test_run', + 'action': 'test_http_action_execution_script_evaluation_with_dynamic_params', 'sender': 'default', + 'headers': {}, 'url': 'http://localhost:8081/mock', 'request_method': 'GET', + 'bot_response': 'Mayank', 'bot': '5f50fd0a56b698ca10d35d2e', 'status': 'SUCCESS', + 'fail_reason': None, 'user_msg': 'get intents', 'http_status_code': 200} @responses.activate @@ -2451,8 +2617,31 @@ def test_http_action_execution_script_evaluation_with_dynamic_params_returns_cus if event.get('time_elapsed') is not None: del event['time_elapsed'] print(events) - assert events == [{'type': 'response', 'dispatch_bot_response': True, 'dispatch_type': 'json', 'data': 'bot_response = data', 'evaluation_type': 'script', 'response': {'a': 10, 'b': {'name': 'Mayank', 'arr': ['red', 'green', 'hotpink']}}, 'bot_response_log': ['evaluation_type: script', 'script: bot_response = data', "data: {'data': {'a': 10, 'b': {'name': 'Mayank', 'arr': ['red', 'green', 'hotpink']}}, 'context': {'sender_id': 'default', 'user_message': 'get intents', 'slot': {'bot': '5f50fd0a56b698ca10d35d2e'}, 'intent': 'test_run', 'chat_log': [], 'key_vault': {'EMAIL': 'uditpandey@digite.com', 'FIRSTNAME': 'udit'}, 'latest_message': {'text': 'get intents', 'intent_ranking': [{'name': 'test_run'}]}, 'kairon_user_msg': None, 'session_started': None, 'bot': '5f50fd0a56b698ca10d35d2e'}, 'http_status_code': 200}", 'raise_err_on_failure: True']}, {'type': 'api_call', 'headers': {'botid': '5f50fd0a56b698ca10d35d2e', 'userid': '****', 'tag': '******ot'}, 'method': 'GET', 'url': 'http://localhost:8081/mock', 'payload': {'sender_id': 'default', 'user_message': 'get intents', 'intent': 'test_run'}, 'response': {'a': 10, 'b': {'name': 'Mayank', 'arr': ['red', 'green', 'hotpink']}}, 'status_code': 200}, {'type': 'dynamic_params', 'data': "body = {'sender_id': sender_id, 'user_message': user_message, 'intent': intent}", 'response': {'sender_id': 'default', 'user_message': 'get intents', 'intent': 'test_run'}, 'slots': {}, 'request_params': ['evaluation_type: script', "script: body = {'sender_id': sender_id, 'user_message': user_message, 'intent': intent}", "data: {'sender_id': 'default', 'user_message': 'get intents', 'slot': {'bot': '5f50fd0a56b698ca10d35d2e'}, 'intent': 'test_run', 'chat_log': [], 'key_vault': {'EMAIL': 'uditpandey@digite.com', 'FIRSTNAME': 'udit'}, 'latest_message': {'text': 'get intents', 'intent_ranking': [{'name': 'test_run'}]}, 'kairon_user_msg': None, 'session_started': None, 'bot': '5f50fd0a56b698ca10d35d2e'}", 'raise_err_on_failure: True']}, {'type': 'filled_slots', 'data': {}, 'slot_eval_log': ['initiating slot evaluation']}] - assert log == {'type': 'http_action', 'intent': 'test_run', 'action': 'test_http_action_execution_script_evaluation_with_dynamic_params_returns_custom_json', 'sender': 'default', 'headers': {}, 'url': 'http://localhost:8081/mock', 'request_method': 'GET', 'bot_response': "{'a': 10, 'b': {'name': 'Mayank', 'arr': ['red', 'green', 'hotpink']}}", 'bot': '5f50fd0a56b698ca10d35d2e', 'status': 'SUCCESS', 'fail_reason': None, 'user_msg': 'get intents', 'http_status_code': 200} + assert events == [ + {'type': 'response', 'dispatch_bot_response': True, 'dispatch_type': 'json', 'data': 'bot_response = data', + 'evaluation_type': 'script', + 'response': {'a': 10, 'b': {'name': 'Mayank', 'arr': ['red', 'green', 'hotpink']}}, + 'bot_response_log': ['evaluation_type: script', 'script: bot_response = data', + "data: {'data': {'a': 10, 'b': {'name': 'Mayank', 'arr': ['red', 'green', 'hotpink']}}, 'context': {'sender_id': 'default', 'user_message': 'get intents', 'slot': {'bot': '5f50fd0a56b698ca10d35d2e'}, 'intent': 'test_run', 'chat_log': [], 'key_vault': {'EMAIL': 'uditpandey@digite.com', 'FIRSTNAME': 'udit'}, 'latest_message': {'text': 'get intents', 'intent_ranking': [{'name': 'test_run'}]}, 'kairon_user_msg': None, 'session_started': None, 'bot': '5f50fd0a56b698ca10d35d2e'}, 'http_status_code': 200}", + 'raise_err_on_failure: True']}, + {'type': 'api_call', 'headers': {'botid': '5f50fd0a56b698ca10d35d2e', 'userid': '****', 'tag': '******ot'}, + 'method': 'GET', 'url': 'http://localhost:8081/mock', + 'payload': {'sender_id': 'default', 'user_message': 'get intents', 'intent': 'test_run'}, + 'response': {'a': 10, 'b': {'name': 'Mayank', 'arr': ['red', 'green', 'hotpink']}}, 'status_code': 200}, + {'type': 'dynamic_params', + 'data': "body = {'sender_id': sender_id, 'user_message': user_message, 'intent': intent}", + 'response': {'sender_id': 'default', 'user_message': 'get intents', 'intent': 'test_run'}, 'slots': {}, + 'request_params': ['evaluation_type: script', + "script: body = {'sender_id': sender_id, 'user_message': user_message, 'intent': intent}", + "data: {'sender_id': 'default', 'user_message': 'get intents', 'slot': {'bot': '5f50fd0a56b698ca10d35d2e'}, 'intent': 'test_run', 'chat_log': [], 'key_vault': {'EMAIL': 'uditpandey@digite.com', 'FIRSTNAME': 'udit'}, 'latest_message': {'text': 'get intents', 'intent_ranking': [{'name': 'test_run'}]}, 'kairon_user_msg': None, 'session_started': None, 'bot': '5f50fd0a56b698ca10d35d2e'}", + 'raise_err_on_failure: True']}, + {'type': 'filled_slots', 'data': {}, 'slot_eval_log': ['initiating slot evaluation']}] + assert log == {'type': 'http_action', 'intent': 'test_run', + 'action': 'test_http_action_execution_script_evaluation_with_dynamic_params_returns_custom_json', + 'sender': 'default', 'headers': {}, 'url': 'http://localhost:8081/mock', 'request_method': 'GET', + 'bot_response': "{'a': 10, 'b': {'name': 'Mayank', 'arr': ['red', 'green', 'hotpink']}}", + 'bot': '5f50fd0a56b698ca10d35d2e', 'status': 'SUCCESS', 'fail_reason': None, + 'user_msg': 'get intents', 'http_status_code': 200} @responses.activate @@ -2577,8 +2766,34 @@ def test_http_action_execution_script_evaluation_with_dynamic_params_no_response if event.get('time_elapsed') is not None: del event['time_elapsed'] print(events) - assert events == [{'type': 'response', 'dispatch_bot_response': False, 'dispatch_type': 'text', 'data': "bot_response = data['b']['name']", 'evaluation_type': 'script', 'response': 'Mayank', 'bot_response_log': ['evaluation_type: script', "script: bot_response = data['b']['name']", "data: {'data': {'a': 10, 'b': {'name': 'Mayank', 'arr': ['red', 'green', 'hotpink']}}, 'context': {'sender_id': 'default', 'user_message': 'get intents', 'slot': {'bot': '5f50fd0a56b698ca10d35d2e'}, 'intent': 'test_run', 'chat_log': [], 'key_vault': {'EMAIL': 'uditpandey@digite.com', 'FIRSTNAME': 'udit'}, 'latest_message': {'text': 'get intents', 'intent_ranking': [{'name': 'test_run'}]}, 'kairon_user_msg': None, 'session_started': None, 'bot': '5f50fd0a56b698ca10d35d2e'}, 'http_status_code': 200}", 'raise_err_on_failure: True']}, {'type': 'api_call', 'headers': {'botid': '5f50fd0a56b698ca10d35d2e', 'userid': '****', 'tag': '******ot'}, 'method': 'GET', 'url': 'http://localhost:8081/mock', 'payload': {'sender_id': 'default', 'user_message': 'get intents', 'intent': 'test_run'}, 'response': {'a': 10, 'b': {'name': 'Mayank', 'arr': ['red', 'green', 'hotpink']}}, 'status_code': 200}, {'type': 'dynamic_params', 'data': "body = {'sender_id': sender_id, 'user_message': user_message, 'intent': intent}", 'response': {'sender_id': 'default', 'user_message': 'get intents', 'intent': 'test_run'}, 'slots': {}, 'request_params': ['evaluation_type: script', "script: body = {'sender_id': sender_id, 'user_message': user_message, 'intent': intent}", "data: {'sender_id': 'default', 'user_message': 'get intents', 'slot': {'bot': '5f50fd0a56b698ca10d35d2e'}, 'intent': 'test_run', 'chat_log': [], 'key_vault': {'EMAIL': 'uditpandey@digite.com', 'FIRSTNAME': 'udit'}, 'latest_message': {'text': 'get intents', 'intent_ranking': [{'name': 'test_run'}]}, 'kairon_user_msg': None, 'session_started': None, 'bot': '5f50fd0a56b698ca10d35d2e'}", 'raise_err_on_failure: True']}, {'type': 'filled_slots', 'data': {}, 'slot_eval_log': ['initiating slot evaluation']}] - assert log == {'type': 'http_action', 'intent': 'test_run', 'action': 'test_http_action_execution_script_evaluation_with_dynamic_params_no_response_dispatch', 'sender': 'default', 'headers': {}, 'url': 'http://localhost:8081/mock', 'request_method': 'GET', 'bot_response': 'Mayank', 'bot': '5f50fd0a56b698ca10d35d2e', 'status': 'SUCCESS', 'fail_reason': None, 'user_msg': 'get intents', 'http_status_code': 200} + assert events == [{'type': 'response', 'dispatch_bot_response': False, 'dispatch_type': 'text', + 'data': "bot_response = data['b']['name']", 'evaluation_type': 'script', 'response': 'Mayank', + 'bot_response_log': ['evaluation_type: script', "script: bot_response = data['b']['name']", + "data: {'data': {'a': 10, 'b': {'name': 'Mayank', 'arr': ['red', 'green', 'hotpink']}}, 'context': {'sender_id': 'default', 'user_message': 'get intents', 'slot': {'bot': '5f50fd0a56b698ca10d35d2e'}, 'intent': 'test_run', 'chat_log': [], 'key_vault': {'EMAIL': 'uditpandey@digite.com', 'FIRSTNAME': 'udit'}, 'latest_message': {'text': 'get intents', 'intent_ranking': [{'name': 'test_run'}]}, 'kairon_user_msg': None, 'session_started': None, 'bot': '5f50fd0a56b698ca10d35d2e'}, 'http_status_code': 200}", + 'raise_err_on_failure: True']}, {'type': 'api_call', 'headers': { + 'botid': '5f50fd0a56b698ca10d35d2e', 'userid': '****', 'tag': '******ot'}, 'method': 'GET', + 'url': 'http://localhost:8081/mock', + 'payload': {'sender_id': 'default', + 'user_message': 'get intents', + 'intent': 'test_run'}, + 'response': {'a': 10, + 'b': {'name': 'Mayank', + 'arr': ['red', 'green', + 'hotpink']}}, + 'status_code': 200}, + {'type': 'dynamic_params', + 'data': "body = {'sender_id': sender_id, 'user_message': user_message, 'intent': intent}", + 'response': {'sender_id': 'default', 'user_message': 'get intents', 'intent': 'test_run'}, + 'slots': {}, 'request_params': ['evaluation_type: script', + "script: body = {'sender_id': sender_id, 'user_message': user_message, 'intent': intent}", + "data: {'sender_id': 'default', 'user_message': 'get intents', 'slot': {'bot': '5f50fd0a56b698ca10d35d2e'}, 'intent': 'test_run', 'chat_log': [], 'key_vault': {'EMAIL': 'uditpandey@digite.com', 'FIRSTNAME': 'udit'}, 'latest_message': {'text': 'get intents', 'intent_ranking': [{'name': 'test_run'}]}, 'kairon_user_msg': None, 'session_started': None, 'bot': '5f50fd0a56b698ca10d35d2e'}", + 'raise_err_on_failure: True']}, + {'type': 'filled_slots', 'data': {}, 'slot_eval_log': ['initiating slot evaluation']}] + assert log == {'type': 'http_action', 'intent': 'test_run', + 'action': 'test_http_action_execution_script_evaluation_with_dynamic_params_no_response_dispatch', + 'sender': 'default', 'headers': {}, 'url': 'http://localhost:8081/mock', 'request_method': 'GET', + 'bot_response': 'Mayank', 'bot': '5f50fd0a56b698ca10d35d2e', 'status': 'SUCCESS', + 'fail_reason': None, 'user_msg': 'get intents', 'http_status_code': 200} @responses.activate @@ -2836,7 +3051,7 @@ def test_http_action_execution_script_evaluation_with_dynamic_params_and_params_ resp_msg = json.dumps(data_obj) aioresponses.add( method=responses.GET, - url=http_url+"?intent=test_run&sender_id=default&user_message=get+intents", + url=http_url + "?intent=test_run&sender_id=default&user_message=get+intents", body=resp_msg, status=200 ) @@ -2901,8 +3116,35 @@ def test_http_action_execution_script_evaluation_with_dynamic_params_and_params_ if event.get('time_elapsed') is not None: del event['time_elapsed'] print(events) - assert events == [{'type': 'response', 'dispatch_bot_response': True, 'dispatch_type': 'text', 'data': "bot_response = data['b']['name']", 'evaluation_type': 'script', 'response': 'Mayank', 'bot_response_log': ['evaluation_type: script', "script: bot_response = data['b']['name']", "data: {'data': {'a': 10, 'b': {'name': 'Mayank', 'arr': ['red', 'green', 'hotpink']}}, 'context': {'sender_id': 'default', 'user_message': 'get intents', 'slot': {'bot': '5f50fd0a56b698ca10d35d2e'}, 'intent': 'test_run', 'chat_log': [], 'key_vault': {'EMAIL': 'uditpandey@digite.com', 'FIRSTNAME': 'udit'}, 'latest_message': {'text': 'get intents', 'intent_ranking': [{'name': 'test_run'}]}, 'kairon_user_msg': None, 'session_started': None, 'bot': '5f50fd0a56b698ca10d35d2e'}, 'http_status_code': 200}", 'raise_err_on_failure: True']}, {'type': 'api_call', 'headers': {'botid': '5f50fd0a56b698ca10d35d2e', 'userid': '****', 'tag': '******ot'}, 'method': 'GET', 'url': 'http://localhost:8081/mock', 'payload': {'sender_id': 'default', 'user_message': 'get intents', 'intent': 'test_run'}, 'response': {'a': 10, 'b': {'name': 'Mayank', 'arr': ['red', 'green', 'hotpink']}}, 'status_code': 200}, {'type': 'dynamic_params', 'data': "body = {'sender_id': sender_id, 'user_message': user_message, 'intent': intent}", 'response': {'sender_id': 'default', 'user_message': 'get intents', 'intent': 'test_run'}, 'slots': {}, 'request_params': ['evaluation_type: script', "script: body = {'sender_id': sender_id, 'user_message': user_message, 'intent': intent}", "data: {'sender_id': 'default', 'user_message': 'get intents', 'slot': {'bot': '5f50fd0a56b698ca10d35d2e'}, 'intent': 'test_run', 'chat_log': [], 'key_vault': {'EMAIL': 'uditpandey@digite.com', 'FIRSTNAME': 'udit'}, 'latest_message': {'text': 'get intents', 'intent_ranking': [{'name': 'test_run'}]}, 'kairon_user_msg': None, 'session_started': None, 'bot': '5f50fd0a56b698ca10d35d2e'}", 'raise_err_on_failure: True']}, {'type': 'filled_slots', 'data': {}, 'slot_eval_log': ['initiating slot evaluation']}] - assert not DeepDiff(log, {'type': 'http_action', 'intent': 'test_run', 'action': 'test_http_action_execution_script_evaluation_with_dynamic_params_and_params_list', 'sender': 'default', 'headers': {}, 'url': 'http://localhost:8081/mock', 'request_method': 'GET', 'bot_response': 'Mayank', 'bot': '5f50fd0a56b698ca10d35d2e', 'status': 'SUCCESS', 'fail_reason': None, 'user_msg': 'get intents', 'http_status_code': 200}, ignore_order=True) + assert events == [{'type': 'response', 'dispatch_bot_response': True, 'dispatch_type': 'text', + 'data': "bot_response = data['b']['name']", 'evaluation_type': 'script', 'response': 'Mayank', + 'bot_response_log': ['evaluation_type: script', "script: bot_response = data['b']['name']", + "data: {'data': {'a': 10, 'b': {'name': 'Mayank', 'arr': ['red', 'green', 'hotpink']}}, 'context': {'sender_id': 'default', 'user_message': 'get intents', 'slot': {'bot': '5f50fd0a56b698ca10d35d2e'}, 'intent': 'test_run', 'chat_log': [], 'key_vault': {'EMAIL': 'uditpandey@digite.com', 'FIRSTNAME': 'udit'}, 'latest_message': {'text': 'get intents', 'intent_ranking': [{'name': 'test_run'}]}, 'kairon_user_msg': None, 'session_started': None, 'bot': '5f50fd0a56b698ca10d35d2e'}, 'http_status_code': 200}", + 'raise_err_on_failure: True']}, {'type': 'api_call', 'headers': { + 'botid': '5f50fd0a56b698ca10d35d2e', 'userid': '****', 'tag': '******ot'}, 'method': 'GET', + 'url': 'http://localhost:8081/mock', + 'payload': {'sender_id': 'default', + 'user_message': 'get intents', + 'intent': 'test_run'}, + 'response': {'a': 10, + 'b': {'name': 'Mayank', + 'arr': ['red', 'green', + 'hotpink']}}, + 'status_code': 200}, + {'type': 'dynamic_params', + 'data': "body = {'sender_id': sender_id, 'user_message': user_message, 'intent': intent}", + 'response': {'sender_id': 'default', 'user_message': 'get intents', 'intent': 'test_run'}, + 'slots': {}, 'request_params': ['evaluation_type: script', + "script: body = {'sender_id': sender_id, 'user_message': user_message, 'intent': intent}", + "data: {'sender_id': 'default', 'user_message': 'get intents', 'slot': {'bot': '5f50fd0a56b698ca10d35d2e'}, 'intent': 'test_run', 'chat_log': [], 'key_vault': {'EMAIL': 'uditpandey@digite.com', 'FIRSTNAME': 'udit'}, 'latest_message': {'text': 'get intents', 'intent_ranking': [{'name': 'test_run'}]}, 'kairon_user_msg': None, 'session_started': None, 'bot': '5f50fd0a56b698ca10d35d2e'}", + 'raise_err_on_failure: True']}, + {'type': 'filled_slots', 'data': {}, 'slot_eval_log': ['initiating slot evaluation']}] + assert not DeepDiff(log, {'type': 'http_action', 'intent': 'test_run', + 'action': 'test_http_action_execution_script_evaluation_with_dynamic_params_and_params_list', + 'sender': 'default', 'headers': {}, 'url': 'http://localhost:8081/mock', + 'request_method': 'GET', 'bot_response': 'Mayank', 'bot': '5f50fd0a56b698ca10d35d2e', + 'status': 'SUCCESS', 'fail_reason': None, 'user_msg': 'get intents', + 'http_status_code': 200}, ignore_order=True) @responses.activate @@ -2942,7 +3184,7 @@ def test_http_action_execution_script_evaluation_failure_no_dispatch(aioresponse aioresponses.add( method=responses.GET, - url=http_url+"?bot=5f50fd0a56b698ca10d35d2d&tag=from_bot&user=1011", + url=http_url + "?bot=5f50fd0a56b698ca10d35d2d&tag=from_bot&user=1011", body=resp_msg, status=200 ) @@ -3040,7 +3282,7 @@ def test_http_action_execution_script_evaluation_failure_and_dispatch(aiorespons aioresponses.add( method=responses.GET, - url=http_url+"?bot=5f50fd0a56b698ca10d35d2d&tag=from_bot&user=1011", + url=http_url + "?bot=5f50fd0a56b698ca10d35d2d&tag=from_bot&user=1011", body=resp_msg, status=200, ) @@ -3199,12 +3441,12 @@ def test_http_action_execution_script_evaluation_failure_and_dispatch_2(aiorespo assert response_json['events'] == [ {"event": "slot", "timestamp": None, "name": "kairon_action_response", "value": "I have failed to process your request"}, - {"event": "slot", "timestamp": None, "name": "http_status_code", "value": 200},] + {"event": "slot", "timestamp": None, "name": "http_status_code", "value": 200}, ] assert response_json['responses'][0]['text'] == "I have failed to process your request" -@patch("kairon.shared.actions.utils.ActionUtility.get_action") -@patch("kairon.actions.definitions.http.ActionHTTP.retrieve_config") +@mock.patch("kairon.shared.actions.utils.ActionUtility.get_action") +@mock.patch("kairon.actions.definitions.http.ActionHTTP.retrieve_config") @mock.patch("kairon.shared.rest_client.AioRestClient._AioRestClient__trigger", autospec=True) def test_http_action_failed_execution(mock_trigger_request, mock_action_config, mock_action): action_name = "test_run_with_get" @@ -3272,8 +3514,18 @@ def _get_action(*arge, **kwargs): if event.get('time_elapsed') is not None: del event['time_elapsed'] print(events) - assert events == [{'type': 'response', 'dispatch_bot_response': True, 'dispatch_type': 'text', 'data': 'The value of ${a.b.3} in ${a.b.d.0} is ${a.b.d}', 'evaluation_type': 'expression', 'exception': 'I have failed to process your request'}, {'type': 'api_call', 'headers': {}, 'method': 'GET', 'url': 'http://localhost:8800/mock', 'payload': {}, 'response': None, 'status_code': 408, 'exception': "Got non-200 status code:408 http_response:{'data': None, 'context': {'sender_id': 'default', 'user_message': 'get intents', 'slot': {'bot': '5f50fd0a56b698ca10d35d2e'}, 'intent': 'test_run', 'chat_log': [], 'key_vault': {'EMAIL': 'uditpandey@digite.com', 'FIRSTNAME': 'udit'}, 'latest_message': {'text': 'get intents', 'intent_ranking': [{'name': 'test_run'}]}, 'kairon_user_msg': None, 'session_started': None, 'bot': '5f50fd0a56b698ca10d35d2e'}, 'http_status_code': 408}"}, {'type': 'params_list', 'request_body': {}, 'request_params': {}}, {'type': 'filled_slots'}] - assert log == {'type': 'http_action', 'intent': 'test_run', 'action': 'test_run_with_get', 'sender': 'default', 'headers': {}, 'url': 'http://localhost:8800/mock', 'request_method': 'GET', 'bot_response': 'I have failed to process your request', 'bot': '5f50fd0a56b698ca10d35d2e', 'status': 'FAILURE', 'fail_reason': 'Got non-200 status code:408 http_response:None', 'user_msg': 'get intents', 'time_elapsed': 0, 'http_status_code': 408} + assert events == [{'type': 'response', 'dispatch_bot_response': True, 'dispatch_type': 'text', + 'data': 'The value of ${a.b.3} in ${a.b.d.0} is ${a.b.d}', 'evaluation_type': 'expression', + 'exception': 'I have failed to process your request'}, + {'type': 'api_call', 'headers': {}, 'method': 'GET', 'url': 'http://localhost:8800/mock', + 'payload': {}, 'response': None, 'status_code': 408, + 'exception': "Got non-200 status code:408 http_response:{'data': None, 'context': {'sender_id': 'default', 'user_message': 'get intents', 'slot': {'bot': '5f50fd0a56b698ca10d35d2e'}, 'intent': 'test_run', 'chat_log': [], 'key_vault': {'EMAIL': 'uditpandey@digite.com', 'FIRSTNAME': 'udit'}, 'latest_message': {'text': 'get intents', 'intent_ranking': [{'name': 'test_run'}]}, 'kairon_user_msg': None, 'session_started': None, 'bot': '5f50fd0a56b698ca10d35d2e'}, 'http_status_code': 408}"}, + {'type': 'params_list', 'request_body': {}, 'request_params': {}}, {'type': 'filled_slots'}] + assert log == {'type': 'http_action', 'intent': 'test_run', 'action': 'test_run_with_get', 'sender': 'default', + 'headers': {}, 'url': 'http://localhost:8800/mock', 'request_method': 'GET', + 'bot_response': 'I have failed to process your request', 'bot': '5f50fd0a56b698ca10d35d2e', + 'status': 'FAILURE', 'fail_reason': 'Got non-200 status code:408 http_response:None', + 'user_msg': 'get intents', 'time_elapsed': 0, 'http_status_code': 408} def test_http_action_missing_action_name(): @@ -3533,7 +3785,7 @@ def test_vectordb_action_execution_embedding_search_from_value(mock_embedding): BotSecrets(secret_type=BotSecretType.gpt_key.value, value="key_value", bot="5f50fd0a56b698ca10d75d2e", user="user").save() embedding = list(np.random.random(Qdrant.__embedding__)) - mock_embedding.return_value = embedding + mock_embedding.return_value = {'data': [{'embedding': embedding}]} http_url = 'http://localhost:6333/collections/5f50fd0a56b698ca10d75d2e_test_vectordb_action_execution_faq_embd/points' resp_msg = json.dumps( @@ -3976,8 +4228,8 @@ def test_vectordb_action_execution_invalid_operation_type(): log.pop('timestamp') -@patch("kairon.shared.actions.utils.ActionUtility.get_action") -@patch("kairon.actions.definitions.database.ActionDatabase.retrieve_config") +@mock.patch("kairon.shared.actions.utils.ActionUtility.get_action") +@mock.patch("kairon.actions.definitions.database.ActionDatabase.retrieve_config") def test_vectordb_action_failed_execution(mock_action_config, mock_action): action_name = "test_run_with_get_action" payload_body = {"ids": [0], "with_payload": True, "with_vector": True} @@ -3997,7 +4249,6 @@ def test_vectordb_action_failed_execution(mock_action_config, mock_action): BotSecrets(secret_type=BotSecretType.gpt_key.value, value="key_value", bot="5f50fd0a56b697ca10d35d2e", user="user").save() - def _get_action_config(*arge, **kwargs): return action_config.to_mongo().to_dict(), bot_settings.to_mongo().to_dict() @@ -4167,9 +4418,9 @@ def _get_action(*arge, **kwargs): }, "version": "version" } - with patch.object(ActionUtility, "get_action") as mock_action: + with mock.patch.object(ActionUtility, "get_action") as mock_action: mock_action.side_effect = _get_action - with patch.object(ActionSetSlot, "retrieve_config") as mocked: + with mock.patch.object(ActionSetSlot, "retrieve_config") as mocked: mocked.side_effect = _get_action_config response = client.post("/webhook", json=request_object) response_json = response.json() @@ -4230,9 +4481,9 @@ def _get_action(*arge, **kwargs): }, "version": "version" } - with patch.object(ActionUtility, "get_action") as mock_action: + with mock.patch.object(ActionUtility, "get_action") as mock_action: mock_action.side_effect = _get_action - with patch.object(ActionSetSlot, "retrieve_config") as mocked: + with mock.patch.object(ActionSetSlot, "retrieve_config") as mocked: mocked.side_effect = _get_action_config response = client.post("/webhook", json=request_object) response_json = response.json() @@ -4292,9 +4543,9 @@ def _get_action(*arge, **kwargs): }, "version": "version" } - with patch.object(ActionUtility, "get_action") as mock_action: + with mock.patch.object(ActionUtility, "get_action") as mock_action: mock_action.side_effect = _get_action - with patch.object(ActionSetSlot, "retrieve_config") as mocked: + with mock.patch.object(ActionSetSlot, "retrieve_config") as mocked: mocked.side_effect = _get_action_config response = client.post("/webhook", json=request_object) response_json = response.json() @@ -5295,9 +5546,9 @@ def test_form_validation_action_with_is_required_true_and_semantics(): @responses.activate -@patch("kairon.shared.actions.utils.ActionUtility.get_action") -@patch("kairon.actions.definitions.email.ActionEmail.retrieve_config") -@patch("kairon.shared.utils.SMTP", autospec=True) +@mock.patch("kairon.shared.actions.utils.ActionUtility.get_action") +@mock.patch("kairon.actions.definitions.email.ActionEmail.retrieve_config") +@mock.patch("kairon.shared.utils.SMTP", autospec=True) def test_email_action_execution_script_evaluation(mock_smtp, mock_action_config, mock_action): Utility.email_conf['email']['templates']['custom_text_mail'] = open('template/emails/custom_text_mail.html', 'rb').read().decode() @@ -5380,9 +5631,9 @@ def _get_action_config(*arge, **kwargs): assert str(args[2]).__contains__("Content-Type: text/html") -@patch("kairon.shared.actions.utils.ActionUtility.get_action") -@patch("kairon.actions.definitions.email.ActionEmail.retrieve_config") -@patch("kairon.shared.utils.SMTP", autospec=True) +@mock.patch("kairon.shared.actions.utils.ActionUtility.get_action") +@mock.patch("kairon.actions.definitions.email.ActionEmail.retrieve_config") +@mock.patch("kairon.shared.utils.SMTP", autospec=True) def test_email_action_execution(mock_smtp, mock_action_config, mock_action): Utility.email_conf['email']['templates']['conversation'] = open('template/emails/conversation.html', 'rb').read().decode() @@ -5532,9 +5783,9 @@ def _get_action_config(*arge, **kwargs): assert str(args[2]).__contains__("Subject: default test") -@patch("kairon.shared.actions.utils.ActionUtility.get_action") -@patch("kairon.actions.definitions.email.ActionEmail.retrieve_config") -@patch("kairon.shared.utils.SMTP", autospec=True) +@mock.patch("kairon.shared.actions.utils.ActionUtility.get_action") +@mock.patch("kairon.actions.definitions.email.ActionEmail.retrieve_config") +@mock.patch("kairon.shared.utils.SMTP", autospec=True) def test_email_action_execution_with_sender_email_from_slot(mock_smtp, mock_action_config, mock_action): Utility.email_conf['email']['templates']['conversation'] = open('template/emails/conversation.html', 'rb').read().decode() @@ -5684,9 +5935,9 @@ def _get_action_config(*arge, **kwargs): assert str(args[2]).__contains__("Subject: mahesh.sattala test") -@patch("kairon.shared.actions.utils.ActionUtility.get_action") -@patch("kairon.actions.definitions.email.ActionEmail.retrieve_config") -@patch("kairon.shared.utils.SMTP", autospec=True) +@mock.patch("kairon.shared.actions.utils.ActionUtility.get_action") +@mock.patch("kairon.actions.definitions.email.ActionEmail.retrieve_config") +@mock.patch("kairon.shared.utils.SMTP", autospec=True) def test_email_action_execution_with_receiver_email_list_from_slot(mock_smtp, mock_action_config, mock_action): Utility.email_conf['email']['templates']['conversation'] = open('template/emails/conversation.html', 'rb').read().decode() @@ -5837,9 +6088,9 @@ def _get_action_config(*arge, **kwargs): assert str(args[2]).__contains__("Subject: mahesh.sattala test") -@patch("kairon.shared.actions.utils.ActionUtility.get_action") -@patch("kairon.actions.definitions.email.ActionEmail.retrieve_config") -@patch("kairon.shared.utils.SMTP", autospec=True) +@mock.patch("kairon.shared.actions.utils.ActionUtility.get_action") +@mock.patch("kairon.actions.definitions.email.ActionEmail.retrieve_config") +@mock.patch("kairon.shared.utils.SMTP", autospec=True) def test_email_action_execution_with_single_receiver_email_from_slot(mock_smtp, mock_action_config, mock_action): Utility.email_conf['email']['templates']['conversation'] = open('template/emails/conversation.html', 'rb').read().decode() @@ -5990,9 +6241,9 @@ def _get_action_config(*arge, **kwargs): assert str(args[2]).__contains__("Subject: mahesh.sattala test") -@patch("kairon.shared.actions.utils.ActionUtility.get_action") -@patch("kairon.actions.definitions.email.ActionEmail.retrieve_config") -@patch("kairon.shared.utils.SMTP", autospec=True) +@mock.patch("kairon.shared.actions.utils.ActionUtility.get_action") +@mock.patch("kairon.actions.definitions.email.ActionEmail.retrieve_config") +@mock.patch("kairon.shared.utils.SMTP", autospec=True) def test_email_action_execution_with_invalid_from_email(mock_smtp, mock_action_config, mock_action): Utility.email_conf['email']['templates']['conversation'] = open('template/emails/conversation.html', 'rb').read().decode() @@ -6119,9 +6370,9 @@ def _get_action_config(*arge, **kwargs): assert logs.exception == "Invalid 'from_email' type. It must be of type str." -@patch("kairon.shared.actions.utils.ActionUtility.get_action") -@patch("kairon.actions.definitions.email.ActionEmail.retrieve_config") -@patch("kairon.shared.utils.SMTP", autospec=True) +@mock.patch("kairon.shared.actions.utils.ActionUtility.get_action") +@mock.patch("kairon.actions.definitions.email.ActionEmail.retrieve_config") +@mock.patch("kairon.shared.utils.SMTP", autospec=True) def test_email_action_execution_with_invalid_to_email(mock_smtp, mock_action_config, mock_action): Utility.email_conf['email']['templates']['conversation'] = open('template/emails/conversation.html', 'rb').read().decode() @@ -6248,9 +6499,9 @@ def _get_action_config(*arge, **kwargs): assert logs.exception == "Invalid 'from_email' type. It must be of type str." -@patch("kairon.shared.actions.utils.ActionUtility.get_action") -@patch("kairon.actions.definitions.email.ActionEmail.retrieve_config") -@patch("kairon.shared.utils.SMTP", autospec=True) +@mock.patch("kairon.shared.actions.utils.ActionUtility.get_action") +@mock.patch("kairon.actions.definitions.email.ActionEmail.retrieve_config") +@mock.patch("kairon.shared.utils.SMTP", autospec=True) def test_email_action_execution_with_invalid_to_email(mock_smtp, mock_action_config, mock_action): Utility.email_conf['email']['templates']['conversation'] = open('template/emails/conversation.html', 'rb').read().decode() @@ -6378,9 +6629,9 @@ def _get_action_config(*arge, **kwargs): assert logs.exception == "Invalid 'to_email' type. It must be of type str or list." -@patch("kairon.shared.actions.utils.ActionUtility.get_action") -@patch("kairon.actions.definitions.email.ActionEmail.retrieve_config") -@patch("kairon.shared.utils.SMTP", autospec=True) +@mock.patch("kairon.shared.actions.utils.ActionUtility.get_action") +@mock.patch("kairon.actions.definitions.email.ActionEmail.retrieve_config") +@mock.patch("kairon.shared.utils.SMTP", autospec=True) def test_email_action_execution_varied_utterances(mock_smtp, mock_action_config, mock_action): Utility.email_conf['email']['templates']['conversation'] = open('template/emails/conversation.html', 'rb').read().decode() @@ -6834,8 +7085,8 @@ def _get_action_config(*arge, **kwargs): assert logs.status == "SUCCESS" -@patch("kairon.shared.actions.utils.ActionUtility.get_action") -@patch("kairon.actions.definitions.email.ActionEmail.retrieve_config") +@mock.patch("kairon.shared.actions.utils.ActionUtility.get_action") +@mock.patch("kairon.actions.definitions.email.ActionEmail.retrieve_config") def test_email_action_failed_execution(mock_action_config, mock_action): action_name = "test_run_email_action" action = Actions(name=action_name, type=ActionType.email_action.value, bot="bot", user="user") @@ -7221,7 +7472,7 @@ def _run_action(*args, **kwargs): }, "version": "version" } - with patch.object(ActionUtility, "perform_google_search") as mocked: + with mock.patch.object(ActionUtility, "perform_google_search") as mocked: mocked.side_effect = _run_action response = client.post("/webhook", json=request_object) response_json = response.json() @@ -7250,7 +7501,7 @@ def _run_action(*args, **kwargs): 'intent_ranking': [{'name': 'test_run'}], "entities": [{"value": "my custom text", "entity": KAIRON_USER_MSG_ENTITY}] } - with patch.object(ActionUtility, "perform_google_search") as mocked: + with mock.patch.object(ActionUtility, "perform_google_search") as mocked: mocked.side_effect = _run_action response = client.post("/webhook", json=request_object) response_json = response.json() @@ -7277,7 +7528,7 @@ def _run_action(*args, **kwargs): request_object["tracker"]["latest_message"] = { 'text': '/action_google_search', 'intent_ranking': [{'name': 'test_run'}] } - with patch.object(ActionUtility, "perform_google_search") as mocked: + with mock.patch.object(ActionUtility, "perform_google_search") as mocked: mocked.side_effect = _run_action response = client.post("/webhook", json=request_object) response_json = response.json() @@ -7316,7 +7567,7 @@ def _run_action(*args, **kwargs): request_object["tracker"]["sender_id"] = user request_object["tracker"]["latest_message"]['text'] = "what is Kanban?" - with patch.object(ActionUtility, "perform_google_search") as mocked: + with mock.patch.object(ActionUtility, "perform_google_search") as mocked: mocked.side_effect = _run_action response = client.post("/webhook", json=request_object) response_json = response.json() @@ -7351,7 +7602,7 @@ def _run_action(*args, **kwargs): request_object["tracker"]["sender_id"] = user request_object["tracker"]["latest_message"]['text'] = "what is Kanban?" - with patch.object(ActionUtility, "perform_google_search") as mocked: + with mock.patch.object(ActionUtility, "perform_google_search") as mocked: mocked.side_effect = _run_action response = client.post("/webhook", json=request_object) response_json = response.json() @@ -7388,7 +7639,7 @@ def _run_action(*args, **kwargs): request_object["tracker"]["sender_id"] = user request_object["tracker"]["latest_message"]['text'] = "what is Kanban?" - with patch.object(ActionUtility, "perform_google_search") as mocked: + with mock.patch.object(ActionUtility, "perform_google_search") as mocked: mocked.side_effect = _run_action response = client.post("/webhook", json=request_object) response_json = response.json() @@ -7488,7 +7739,7 @@ def _run_action(*args, **kwargs): }, "version": "version" } - with patch.object(ActionUtility, "perform_google_search") as mocked: + with mock.patch.object(ActionUtility, "perform_google_search") as mocked: mocked.side_effect = _run_action response = client.post("/webhook", json=request_object) response_json = response.json() @@ -7591,7 +7842,7 @@ def _run_action(*args, **kwargs): }, "version": "version" } - with patch.object(ActionUtility, "perform_google_search") as mocked: + with mock.patch.object(ActionUtility, "perform_google_search") as mocked: mocked.side_effect = _run_action response = client.post("/webhook", json=request_object) response_json = response.json() @@ -7789,7 +8040,7 @@ def _perform_web_search(*args, **kwargs): }, "version": "version" } - with patch.object(ActionUtility, "perform_web_search") as mocked: + with mock.patch.object(ActionUtility, "perform_web_search") as mocked: mocked.side_effect = _perform_web_search response = client.post("/webhook", json=request_object) response_json = response.json() @@ -8030,7 +8281,7 @@ def _perform_web_search(*args, **kwargs): }, "version": "version" } - with patch.object(ActionUtility, "perform_web_search") as mocked: + with mock.patch.object(ActionUtility, "perform_web_search") as mocked: mocked.side_effect = _perform_web_search response = client.post("/webhook", json=request_object) response_json = response.json() @@ -8149,7 +8400,7 @@ def _perform_web_search(*args, **kwargs): }, "version": "version" } - with patch.object(ActionUtility, "perform_web_search") as mocked: + with mock.patch.object(ActionUtility, "perform_web_search") as mocked: mocked.side_effect = _perform_web_search response = client.post("/webhook", json=request_object) response_json = response.json() @@ -8160,9 +8411,9 @@ def _perform_web_search(*args, **kwargs): {'event': 'slot', 'timestamp': None, 'name': 'kairon_action_response', 'value': 'Data science combines math, statistics, programming, analytics, AI, and machine learning to uncover insights from data. Learn how data science works, what it entails, and how it differs from data science and BI.\nTo know more, please visit: What is Data Science? | IBM\n\nData science is an interdisciplinary field that uses algorithms, procedures, and processes to examine large amounts of data in order to uncover hidden patterns, generate insights, and direct decision-making.\nTo know more, please visit: What Is Data Science? Definition, Examples, Jobs, and More'}], 'responses': [{ - 'text': 'Data science combines math, statistics, programming, analytics, AI, and machine learning to uncover insights from data. Learn how data science works, what it entails, and how it differs from data science and BI.\nTo know more, please visit: What is Data Science? | IBM\n\nData science is an interdisciplinary field that uses algorithms, procedures, and processes to examine large amounts of data in order to uncover hidden patterns, generate insights, and direct decision-making.\nTo know more, please visit: What Is Data Science? Definition, Examples, Jobs, and More', - 'buttons': [], 'elements': [], 'custom': {}, 'template': None, 'response': None, - 'image': None, 'attachment': None}]} + 'text': 'Data science combines math, statistics, programming, analytics, AI, and machine learning to uncover insights from data. Learn how data science works, what it entails, and how it differs from data science and BI.\nTo know more, please visit: What is Data Science? | IBM\n\nData science is an interdisciplinary field that uses algorithms, procedures, and processes to examine large amounts of data in order to uncover hidden patterns, generate insights, and direct decision-making.\nTo know more, please visit: What Is Data Science? Definition, Examples, Jobs, and More', + 'buttons': [], 'elements': [], 'custom': {}, 'template': None, 'response': None, + 'image': None, 'attachment': None}]} log = ActionServerLogs.objects(bot=bot, type=ActionType.web_search_action.value, status="SUCCESS").get() assert log['user_msg'] == '/action_public_search' @@ -8190,7 +8441,7 @@ def _perform_web_search(*args, **kwargs): request_object["tracker"]["sender_id"] = user request_object["tracker"]["latest_message"]['text'] = "What is Python?" - with patch.object(ActionUtility, "perform_web_search") as mocked: + with mock.patch.object(ActionUtility, "perform_web_search") as mocked: mocked.side_effect = _perform_web_search response = client.post("/webhook", json=request_object) response_json = response.json() @@ -8293,7 +8544,7 @@ def _perform_web_search(*args, **kwargs): }, "version": "version" } - with patch.object(ActionUtility, "perform_web_search") as mocked: + with mock.patch.object(ActionUtility, "perform_web_search") as mocked: mocked.side_effect = _perform_web_search response = client.post("/webhook", json=request_object) response_json = response.json() @@ -8397,7 +8648,7 @@ def _perform_web_search(*args, **kwargs): }, "version": "version" } - with patch.object(ActionUtility, "perform_web_search") as mocked: + with mock.patch.object(ActionUtility, "perform_web_search") as mocked: mocked.side_effect = _perform_web_search response = client.post("/webhook", json=request_object) response_json = response.json() @@ -8420,7 +8671,7 @@ def test_process_jira_action(): def _mock_response(*args, **kwargs): return None - with patch('kairon.shared.actions.data_objects.JiraAction.validate', new=_mock_response): + with mock.patch('kairon.shared.actions.data_objects.JiraAction.validate', new=_mock_response): Actions(name=action_name, type=ActionType.jira_action.value, bot=bot, user=user).save() JiraAction( name=action_name, bot=bot, user=user, url='https://test-digite.atlassian.net', @@ -8507,7 +8758,7 @@ def _mock_response(*args, **kwargs): }, "version": "version" } - with patch.object(ActionUtility, "create_jira_issue") as mocked: + with mock.patch.object(ActionUtility, "create_jira_issue") as mocked: mocked.side_effect = _mock_response response = client.post("/webhook", json=request_object) response_json = response.json() @@ -8529,7 +8780,7 @@ def _mock_validation(*args, **kwargs): def _mock_response(*args, **kwargs): raise JIRAError(status_code=404, url='https://test-digite.atlassian.net') - with patch('kairon.shared.actions.data_objects.JiraAction.validate', new=_mock_validation): + with mock.patch('kairon.shared.actions.data_objects.JiraAction.validate', new=_mock_validation): Actions(name=action_name, type=ActionType.jira_action.value, bot=bot, user='test_user').save() JiraAction( name=action_name, bot=bot, user=user, url='https://test-digite.atlassian.net', @@ -8616,7 +8867,7 @@ def _mock_response(*args, **kwargs): }, "version": "version" } - with patch.object(ActionUtility, "create_jira_issue") as mocked: + with mock.patch.object(ActionUtility, "create_jira_issue") as mocked: mocked.side_effect = _mock_response response = client.post("/webhook", json=request_object) response_json = response.json() @@ -8812,7 +9063,7 @@ def test_process_zendesk_action(): user = 'test_user' Actions(name=action_name, type=ActionType.zendesk_action.value, bot=bot, user='test_user').save() - with patch('zenpy.Zenpy'): + with mock.patch('zenpy.Zenpy'): ZendeskAction(name=action_name, subdomain='digite751', user_name='udit.pandey@digite.com', api_token=CustomActionRequestParameters(value='1234567890'), subject='new ticket', response='ticket created', @@ -8897,7 +9148,7 @@ def test_process_zendesk_action(): "version": "version" } - with patch('zenpy.Zenpy'): + with mock.patch('zenpy.Zenpy'): response = client.post("/webhook", json=request_object) response_json = response.json() assert response_json == {'events': [ @@ -8914,7 +9165,7 @@ def test_process_zendesk_action_failure(): user = 'test_user' Actions(name=action_name, type=ActionType.zendesk_action.value, bot=bot, user='test_user').save() - with patch('zenpy.Zenpy'): + with mock.patch('zenpy.Zenpy') as zen: ZendeskAction(name=action_name, subdomain='digite751', user_name='udit.pandey@digite.com', api_token=CustomActionRequestParameters(value='1234567890'), subject='new ticket', response='ticket created', @@ -9003,8 +9254,8 @@ def __mock_zendesk_error(*args, **kwargs): from zenpy.lib.exception import APIException raise APIException({"error": {"title": "No help desk at digite751.zendesk.com"}}) - with patch('zenpy.Zenpy') as mock: - mock.side_effect = __mock_zendesk_error + with mock.patch('zenpy.Zenpy') as zen: + zen.side_effect = __mock_zendesk_error response = client.post("/webhook", json=request_object) response_json = response.json() assert response_json == {'events': [ @@ -9110,7 +9361,7 @@ def test_process_pipedrive_leads_action(): user = 'test_user' Actions(name=action_name, type=ActionType.pipedrive_leads_action.value, bot=bot, user='test_user').save() - with patch('pipedrive.client.Client'): + with mock.patch('pipedrive.client.Client'): metadata = {'name': 'name', 'org_name': 'organization', 'email': 'email', 'phone': 'phone'} PipedriveLeadsAction(name=action_name, domain='https://digite751.pipedrive.com/', api_token=CustomActionRequestParameters(value='1234567890'), @@ -9209,10 +9460,10 @@ def __mock_create_leads(*args, **kwargs): def __mock_create_note(*args, **kwargs): return {"success": True, "data": {"id": 2}} - with patch('pipedrive.organizations.Organizations.create_organization', __mock_create_organization): - with patch('pipedrive.persons.Persons.create_person', __mock_create_person): - with patch('pipedrive.leads.Leads.create_lead', __mock_create_leads): - with patch('pipedrive.notes.Notes.create_note', __mock_create_note): + with mock.patch('pipedrive.organizations.Organizations.create_organization', __mock_create_organization): + with mock.patch('pipedrive.persons.Persons.create_person', __mock_create_person): + with mock.patch('pipedrive.leads.Leads.create_lead', __mock_create_leads): + with mock.patch('pipedrive.notes.Notes.create_note', __mock_create_note): response = client.post("/webhook", json=request_object) response_json = response.json() assert response_json == {'events': [ @@ -9386,7 +9637,7 @@ def test_process_pipedrive_leads_action_failure(): user = 'test_user' Actions(name=action_name, type=ActionType.pipedrive_leads_action.value, bot=bot, user='test_user').save() - with patch('pipedrive.client.Client'): + with mock.patch('pipedrive.client.Client'): metadata = {'name': 'name', 'org_name': 'organization', 'email': 'email', 'phone': 'phone'} PipedriveLeadsAction(name=action_name, domain='https://digite751.pipedrive.com/', api_token=CustomActionRequestParameters(value='1234567890'), @@ -9476,7 +9727,7 @@ def __mock_pipedrive_error(*args, **kwargs): from pipedrive.exceptions import BadRequestError raise BadRequestError('Invalid request raised', {'error_code': 402}) - with patch('pipedrive.organizations.Organizations.create_organization', __mock_pipedrive_error): + with mock.patch('pipedrive.organizations.Organizations.create_organization', __mock_pipedrive_error): response = client.post("/webhook", json=request_object) response_json = response.json() assert response_json == {'events': [ @@ -9921,7 +10172,7 @@ def _mock_search(*args, **kwargs): {"text": "yes", "payload": "yes"}]: yield result - with patch.object(MongoProcessor, "search_training_examples") as mock_action: + with mock.patch.object(MongoProcessor, "search_training_examples") as mock_action: mock_action.side_effect = _mock_search response = client.post("/webhook", json=request_object) response_json = response.json() @@ -9933,7 +10184,7 @@ def _mock_search(*args, **kwargs): for _ in []: yield - with patch.object(MongoProcessor, "search_training_examples") as mock_action: + with mock.patch.object(MongoProcessor, "search_training_examples") as mock_action: mock_action.side_effect = _mock_search response = client.post("/webhook", json=request_object) response_json = response.json() @@ -10824,7 +11075,7 @@ def __mock_error(*args, **kwargs): "e2e_actions": []}, "version": "2.8.15" } - with patch.object(ActionUtility, "trigger_rephrase") as mock_utils: + with mock.patch.object(ActionUtility, "trigger_rephrase") as mock_utils: mock_utils.side_effect = __mock_error response = client.post("/webhook", json=request_object) @@ -11013,7 +11264,7 @@ async def mock_process_actions(*args, **kwargs): from rasa_sdk import ActionExecutionRejection raise ActionExecutionRejection("Action Execution Rejection") - with patch('kairon.actions.handlers.action.ActionHandler.process_actions', mock_process_actions): + with mock.patch('kairon.actions.handlers.action.ActionHandler.process_actions', mock_process_actions): response = client.post("/webhook", json=request_object) response_json = response.json() assert response_json == {'error': "Custom action 'Action Execution Rejection' rejected execution.", @@ -11023,18 +11274,17 @@ async def mock_process_actions(*args, **kwargs): from rasa_sdk.interfaces import ActionNotFoundException raise ActionNotFoundException("Action Not Found Exception") - with patch('kairon.actions.handlers.action.ActionHandler.process_actions', mock_process_actions): + with mock.patch('kairon.actions.handlers.action.ActionHandler.process_actions', mock_process_actions): response = client.post("/webhook", json=request_object) response_json = response.json() assert response_json == {'error': "No registered action found for name 'Action Not Found Exception'.", 'action_name': 'Action Not Found Exception'} -@mock.patch.object(GPT3FAQEmbedding, "_GPT3FAQEmbedding__get_answer", autospec=True) -@mock.patch.object(GPT3FAQEmbedding, "_GPT3FAQEmbedding__get_embedding", autospec=True) -@patch("kairon.shared.rest_client.AioRestClient.request", autospec=True) +@mock.patch.object(litellm, "acompletion", autospec=True) +@mock.patch.object(litellm, "aembedding", autospec=True) +@mock.patch("kairon.shared.rest_client.AioRestClient.request", autospec=True) def test_prompt_action_response_action_with_prompt_question_from_slot(mock_search, mock_embedding, mock_completion): - from kairon.shared.llm.gpt3 import GPT3FAQEmbedding from uuid6 import uuid7 action_name = "test_prompt_action_response_action_with_prompt_question_from_slot" @@ -11058,9 +11308,9 @@ def test_prompt_action_response_action_with_prompt_question_from_slot(mock_searc 'is_enabled': True} ] - embedding = list(np.random.random(GPT3FAQEmbedding.__embedding__)) - mock_embedding.return_value = embedding - mock_completion.return_value = generated_text + embedding = list(np.random.random(OPENAI_EMBEDDING_OUTPUT)) + mock_embedding.return_value = {'data': [{'embedding': embedding}]} + mock_completion.return_value = {'choices': [{'message': {'content': generated_text, 'role': 'assistant'}}]} mock_search.return_value = { 'result': [{'id': uuid7().__str__(), 'score': 0.80, 'payload': {'content': bot_content}}]} Actions(name=action_name, type=ActionType.prompt_action.value, bot=bot, user=user).save() @@ -11088,31 +11338,20 @@ def test_prompt_action_response_action_with_prompt_question_from_slot(mock_searc {'text': generated_text, 'buttons': [], 'elements': [], 'custom': {}, 'template': None, 'response': None, 'image': None, 'attachment': None} ] + expected = {'messages': [ + {'role': 'system', 'content': 'You are a personal assistant. Answer question based on the context below.\n'}, + {'role': 'user', 'content': 'hello'}, {'role': 'assistant', 'content': 'how are you'}, {'role': 'user', + 'content': "\nInstructions on how to use Similarity Prompt:\n['Python is a high-level, general-purpose programming language. Its design philosophy emphasizes code readability with the use of significant indentation. Python is dynamically typed and garbage-collected.']\nAnswer question based on the context above, if answer is not in the context go check previous logs.\n \nQ: What kind of language is python? \nA:"}], + 'metadata': {'user': 'udit.pandey', 'bot': '5f50fd0a56b698ca10d35d2l', 'invocation': 'prompt_action'}, 'api_key': 'keyvalue', + 'num_retries': 3, 'temperature': 0.0, 'max_tokens': 300, 'model': 'gpt-3.5-turbo', 'top_p': 0.0, 'n': 1, + 'stop': None, 'presence_penalty': 0.0, 'frequency_penalty': 0.0, 'logit_bias': {}} + assert not DeepDiff(mock_completion.call_args[1], expected, ignore_order=True) - assert mock_completion.call_args.args[1] == 'What kind of language is python?' - assert mock_completion.call_args.args[ - 2] == """You are a personal assistant. Answer question based on the context below.\n""" - print(mock_completion.call_args.args[3]) - assert mock_completion.call_args.args[ - 3] == "\nInstructions on how to use Similarity Prompt:\n['Python is a high-level, general-purpose programming language. Its design philosophy emphasizes code readability with the use of significant indentation. Python is dynamically typed and garbage-collected.']\nAnswer question based on the context above, if answer is not in the context go check previous logs.\n" - print(mock_completion.call_args.kwargs) - assert mock_completion.call_args.kwargs == { - 'hyperparameters': {'temperature': 0.0, 'max_tokens': 300, 'model': 'gpt-3.5-turbo', 'top_p': 0.0, 'n': 1, - 'stream': False, 'stop': None, 'presence_penalty': 0.0, 'frequency_penalty': 0.0, - 'logit_bias': {}}, 'query_prompt': {}, - 'previous_bot_responses': [{'role': 'user', 'content': 'hello'}, - {'role': 'assistant', 'content': 'how are you'}], 'similarity_prompt': [ - {'similarity_prompt_name': 'Similarity Prompt', - 'similarity_prompt_instructions': 'Answer question based on the context above, if answer is not in the context go check previous logs.', - 'collection': 'python', 'use_similarity_prompt': True, 'top_results': 10, 'similarity_threshold': 0.7}], - 'instructions': []} - - -@mock.patch.object(GPT3FAQEmbedding, "_GPT3FAQEmbedding__get_answer", autospec=True) -@mock.patch.object(GPT3FAQEmbedding, "_GPT3FAQEmbedding__get_embedding", autospec=True) -@patch("kairon.shared.rest_client.AioRestClient.request", autospec=True) + +@mock.patch.object(litellm, "acompletion", autospec=True) +@mock.patch.object(litellm, "aembedding", autospec=True) +@mock.patch("kairon.shared.rest_client.AioRestClient.request", autospec=True) def test_prompt_action_response_action_with_bot_responses(mock_search, mock_embedding, mock_completion): - from kairon.shared.llm.gpt3 import GPT3FAQEmbedding from uuid6 import uuid7 action_name = "test_prompt_action" @@ -11136,9 +11375,9 @@ def test_prompt_action_response_action_with_bot_responses(mock_search, mock_embe 'is_enabled': True} ] - embedding = list(np.random.random(GPT3FAQEmbedding.__embedding__)) - mock_embedding.return_value = embedding - mock_completion.return_value = generated_text + embedding = list(np.random.random(OPENAI_EMBEDDING_OUTPUT)) + mock_embedding.return_value = {'data': [{'embedding': embedding}]} + mock_completion.return_value = {'choices': [{'message': {'content': generated_text, 'role': 'assistant'}}]} mock_search.return_value = { 'result': [{'id': uuid7().__str__(), 'score': 0.80, 'payload': {'content': bot_content}}]} Actions(name=action_name, type=ActionType.prompt_action.value, bot=bot, user=user).save() @@ -11166,32 +11405,21 @@ def test_prompt_action_response_action_with_bot_responses(mock_search, mock_embe {'text': generated_text, 'buttons': [], 'elements': [], 'custom': {}, 'template': None, 'response': None, 'image': None, 'attachment': None} ] + expected = {'messages': [ + {'role': 'system', 'content': 'You are a personal assistant. Answer question based on the context below.\n'}, + {'role': 'user', 'content': 'hello'}, {'role': 'assistant', 'content': 'how are you'}, {'role': 'user', + 'content': "\nInstructions on how to use Similarity Prompt:\n['Python is a high-level, general-purpose programming language. Its design philosophy emphasizes code readability with the use of significant indentation. Python is dynamically typed and garbage-collected.']\nAnswer question based on the context above, if answer is not in the context go check previous logs.\n \nQ: What kind of language is python? \nA:"}], + 'metadata': {'user': 'udit.pandey', 'bot': '5f50fd0a56b698ca10d35d2k', 'invocation': 'prompt_action'}, 'api_key': 'keyvalue', + 'num_retries': 3, 'temperature': 0.0, 'max_tokens': 300, 'model': 'gpt-3.5-turbo', 'top_p': 0.0, 'n': 1, + 'stop': None, 'presence_penalty': 0.0, 'frequency_penalty': 0.0, 'logit_bias': {}} + assert not DeepDiff(mock_completion.call_args[1], expected, ignore_order=True) + - assert mock_completion.call_args.args[1] == 'What kind of language is python?' - assert mock_completion.call_args.args[ - 2] == """You are a personal assistant. Answer question based on the context below.\n""" - print(mock_completion.call_args.args[3]) - assert mock_completion.call_args.args[ - 3] == "\nInstructions on how to use Similarity Prompt:\n['Python is a high-level, general-purpose programming language. Its design philosophy emphasizes code readability with the use of significant indentation. Python is dynamically typed and garbage-collected.']\nAnswer question based on the context above, if answer is not in the context go check previous logs.\n" - print(mock_completion.call_args.kwargs) - assert mock_completion.call_args.kwargs == { - 'hyperparameters': {'temperature': 0.0, 'max_tokens': 300, 'model': 'gpt-3.5-turbo', 'top_p': 0.0, 'n': 1, - 'stream': False, 'stop': None, 'presence_penalty': 0.0, 'frequency_penalty': 0.0, - 'logit_bias': {}}, 'query_prompt': {}, - 'previous_bot_responses': [{'role': 'user', 'content': 'hello'}, - {'role': 'assistant', 'content': 'how are you'}], 'similarity_prompt': [ - {'similarity_prompt_name': 'Similarity Prompt', - 'similarity_prompt_instructions': 'Answer question based on the context above, if answer is not in the context go check previous logs.', - 'collection': 'python', 'use_similarity_prompt': True, 'top_results': 10, 'similarity_threshold': 0.7}], - 'instructions': []} - - -@mock.patch.object(GPT3FAQEmbedding, "_GPT3FAQEmbedding__get_answer", autospec=True) -@mock.patch.object(GPT3FAQEmbedding, "_GPT3FAQEmbedding__get_embedding", autospec=True) -@patch("kairon.shared.rest_client.AioRestClient.request", autospec=True) +@mock.patch.object(litellm, "acompletion", autospec=True) +@mock.patch.object(litellm, "aembedding", autospec=True) +@mock.patch("kairon.shared.rest_client.AioRestClient.request", autospec=True) def test_prompt_action_response_action_with_bot_responses_with_instructions(mock_search, mock_embedding, mock_completion): - from kairon.shared.llm.gpt3 import GPT3FAQEmbedding from uuid6 import uuid7 action_name = "test_prompt_action_with_bot_responses_with_instructions" @@ -11216,9 +11444,9 @@ def test_prompt_action_response_action_with_bot_responses_with_instructions(mock 'is_enabled': True} ] - embedding = list(np.random.random(GPT3FAQEmbedding.__embedding__)) - mock_embedding.return_value = embedding - mock_completion.return_value = generated_text + embedding = list(np.random.random(OPENAI_EMBEDDING_OUTPUT)) + mock_embedding.return_value = {'data': [{'embedding': embedding}]} + mock_completion.return_value = {'choices': [{'message': {'content': generated_text, 'role': 'assistant'}}]} mock_search.return_value = { 'result': [{'id': uuid7().__str__(), 'score': 0.80, 'payload': {'content': bot_content}}]} Actions(name=action_name, type=ActionType.prompt_action.value, bot=bot, user=user).save() @@ -11247,31 +11475,20 @@ def test_prompt_action_response_action_with_bot_responses_with_instructions(mock {'text': generated_text, 'buttons': [], 'elements': [], 'custom': {}, 'template': None, 'response': None, 'image': None, 'attachment': None} ] + expected = {'messages': [ + {'role': 'system', 'content': 'You are a personal assistant. Answer question based on the context below.\n'}, + {'role': 'user', 'content': 'hello'}, {'role': 'assistant', 'content': 'how are you'}, {'role': 'user', + 'content': "\nInstructions on how to use Similarity Prompt:\n['Python is a high-level, general-purpose programming language. Its design philosophy emphasizes code readability with the use of significant indentation. Python is dynamically typed and garbage-collected.']\nAnswer question based on the context above, if answer is not in the context go check previous logs.\n \nAnswer in a short way.\nKeep it simple. \nQ: What kind of language is python? \nA:"}], + 'metadata': {'user': 'udit.pandey', 'bot': '5f50fd0a56b678ca10d35d2k', 'invocation': 'prompt_action'}, 'api_key': 'keyvalue', + 'num_retries': 3, 'temperature': 0.0, 'max_tokens': 300, 'model': 'gpt-3.5-turbo', 'top_p': 0.0, 'n': 1, + 'stop': None, 'presence_penalty': 0.0, 'frequency_penalty': 0.0, 'logit_bias': {}} + assert not DeepDiff(mock_completion.call_args[1], expected, ignore_order=True) + - assert mock_completion.call_args.args[1] == 'What kind of language is python?' - assert mock_completion.call_args.args[ - 2] == """You are a personal assistant. Answer question based on the context below.\n""" - print(mock_completion.call_args.args[3]) - assert mock_completion.call_args.args[ - 3] == "\nInstructions on how to use Similarity Prompt:\n['Python is a high-level, general-purpose programming language. Its design philosophy emphasizes code readability with the use of significant indentation. Python is dynamically typed and garbage-collected.']\nAnswer question based on the context above, if answer is not in the context go check previous logs.\n" - print(mock_completion.call_args.kwargs) - assert mock_completion.call_args.kwargs == { - 'hyperparameters': {'temperature': 0.0, 'max_tokens': 300, 'model': 'gpt-3.5-turbo', 'top_p': 0.0, 'n': 1, - 'stream': False, 'stop': None, 'presence_penalty': 0.0, 'frequency_penalty': 0.0, - 'logit_bias': {}}, 'query_prompt': {}, - 'previous_bot_responses': [{'role': 'user', 'content': 'hello'}, - {'role': 'assistant', 'content': 'how are you'}], 'similarity_prompt': [ - {'similarity_prompt_name': 'Similarity Prompt', - 'similarity_prompt_instructions': 'Answer question based on the context above, if answer is not in the context go check previous logs.', - 'collection': 'python', 'use_similarity_prompt': True, 'top_results': 10, 'similarity_threshold': 0.7}], - 'instructions': ['Answer in a short way.', 'Keep it simple.']} - - -@mock.patch.object(GPT3Resources, "invoke", autospec=True) -@mock.patch.object(GPT3FAQEmbedding, "_GPT3FAQEmbedding__get_embedding", autospec=True) -@patch("kairon.shared.rest_client.AioRestClient.request", autospec=True) +@mock.patch.object(litellm, "acompletion", autospec=True) +@mock.patch.object(litellm, "aembedding", autospec=True) +@mock.patch("kairon.shared.rest_client.AioRestClient.request", autospec=True) def test_prompt_action_response_action_with_query_prompt(mock_search, mock_embedding, mock_completion): - from kairon.shared.llm.gpt3 import GPT3FAQEmbedding from uuid6 import uuid7 action_name = "test_prompt_action_response_action_with_query_prompt" @@ -11295,20 +11512,18 @@ def test_prompt_action_response_action_with_query_prompt(mock_search, mock_embed 'instructions': 'Answer according to the context', 'type': 'query', 'source': 'static', 'is_enabled': True}, {'name': 'Query Prompt', - 'data': 'If there is no specific query, assume that user is aking about java programming.', + 'data': 'If there is no specific query, assume that user is asking about java programming.', 'instructions': 'Answer according to the context', 'type': 'query', 'source': 'static', 'is_enabled': True} ] - mock_completion_for_query_prompt = rephrased_query, { - 'choices': [{'message': {'content': rephrased_query, 'role': 'assistant'}}]} + mock_completion_for_query_prompt = {'choices': [{'message': {'content': rephrased_query, 'role': 'assistant'}}]} - mock_completion_for_answer = generated_text, { - 'choices': [{'message': {'content': generated_text, 'role': 'assistant'}}]} + mock_completion_for_answer = {'choices': [{'message': {'content': generated_text, 'role': 'assistant'}}]} mock_completion.side_effect = [mock_completion_for_query_prompt, mock_completion_for_answer] - embedding = list(np.random.random(GPT3FAQEmbedding.__embedding__)) - mock_embedding.return_value = embedding + embedding = list(np.random.random(OPENAI_EMBEDDING_OUTPUT)) + mock_embedding.return_value = {'data': [{'embedding': embedding}]} mock_search.return_value = { 'result': [{'id': uuid7().__str__(), 'score': 0.80, 'payload': {'content': bot_content}}]} Actions(name=action_name, type=ActionType.prompt_action.value, bot=bot, user=user).save() @@ -11343,10 +11558,9 @@ def test_prompt_action_response_action_with_query_prompt(mock_search, mock_embed 'content': "\nInstructions on how to use Similarity Prompt:\n['Python is a high-level, general-purpose programming language. Its design philosophy emphasizes code readability with the use of significant indentation. Python is dynamically typed and garbage-collected.']\nAnswer question based on the context above, if answer is not in the context go check previous logs.\n \nQ: Explain python is called high level programming language in laymen terms? \nA:"}] -@mock.patch.object(GPT3FAQEmbedding, "_GPT3FAQEmbedding__get_answer", autospec=True) -@mock.patch.object(GPT3FAQEmbedding, "_GPT3FAQEmbedding__get_embedding", autospec=True) +@mock.patch.object(litellm, "acompletion", autospec=True) +@mock.patch.object(litellm, "aembedding", autospec=True) def test_prompt_response_action(mock_embedding, mock_completion, aioresponses): - from kairon.shared.llm.gpt3 import GPT3FAQEmbedding from uuid6 import uuid7 action_name = GPT_LLM_FAQ @@ -11368,7 +11582,7 @@ def test_prompt_response_action(mock_embedding, mock_completion, aioresponses): }, {'name': 'Data science prompt', 'instructions': 'Answer question based on the context above.', 'type': 'user', 'source': 'bot_content', - 'data': 'data_science'} + 'data': 'data_science'}, ] aioresponses.add( url=urljoin(Utility.environment['vector']['db'], @@ -11384,11 +11598,14 @@ def test_prompt_response_action(mock_embedding, mock_completion, aioresponses): status=200, payload={ 'result': [{'id': uuid7().__str__(), 'score': 0.80, 'payload': {'content': bot_content_two}}]}) - embedding = list(np.random.random(GPT3FAQEmbedding.__embedding__)) - mock_embedding.return_value = embedding - mock_completion.return_value = generated_text + embedding = list(np.random.random(OPENAI_EMBEDDING_OUTPUT)) + mock_embedding.return_value = {'data': [{'embedding': embedding}]} + mock_completion.return_value = {'choices': [{'message': {'content': generated_text, 'role': 'assistant'}}]} Actions(name=action_name, type=ActionType.prompt_action.value, bot=bot, user=user).save() - PromptAction(name=action_name, bot=bot, user=user, llm_prompts=llm_prompts).save() + PromptAction(name=action_name, + bot=bot, + user=user, + llm_prompts=llm_prompts).save() BotSettings(llm_settings=LLMSettings(enable_faq=True), bot=bot, user=user).save() BotSecrets(secret_type=BotSecretType.gpt_key.value, value=value, bot=bot, user=user).save() @@ -11408,11 +11625,10 @@ def test_prompt_response_action(mock_embedding, mock_completion, aioresponses): ] -@mock.patch.object(GPT3FAQEmbedding, "_GPT3FAQEmbedding__get_answer", autospec=True) -@mock.patch.object(GPT3FAQEmbedding, "_GPT3FAQEmbedding__get_embedding", autospec=True) -@patch("kairon.shared.rest_client.AioRestClient.request", autospec=True) +@mock.patch.object(litellm, "acompletion", autospec=True) +@mock.patch.object(litellm, "aembedding", autospec=True) +@mock.patch("kairon.shared.rest_client.AioRestClient.request", autospec=True) def test_prompt_response_action_with_instructions(mock_search, mock_embedding, mock_completion): - from kairon.shared.llm.gpt3 import GPT3FAQEmbedding from uuid6 import uuid7 action_name = 'test_prompt_response_action_with_instructions' @@ -11433,9 +11649,9 @@ def test_prompt_response_action_with_instructions(mock_search, mock_embedding, m 'is_enabled': True } ] - embedding = list(np.random.random(GPT3FAQEmbedding.__embedding__)) - mock_embedding.return_value = embedding - mock_completion.return_value = generated_text + embedding = list(np.random.random(OPENAI_EMBEDDING_OUTPUT)) + mock_embedding.return_value = {'data': [{'embedding': embedding}]} + mock_completion.return_value = {'choices': [{'message': {'content': generated_text, 'role': 'assistant'}}]} mock_search.return_value = { 'result': [{'id': uuid7().__str__(), 'score': 0.80, 'payload': {'content': bot_content}}]} Actions(name=action_name, type=ActionType.prompt_action.value, bot=bot, user=user).save() @@ -11459,11 +11675,10 @@ def test_prompt_response_action_with_instructions(mock_search, mock_embedding, m ] -@mock.patch.object(GPT3Resources, "invoke", autospec=True) -@mock.patch.object(GPT3FAQEmbedding, "_GPT3FAQEmbedding__get_embedding", autospec=True) -@patch("kairon.shared.rest_client.AioRestClient.request", autospec=True) +@mock.patch.object(litellm, "acompletion", autospec=True) +@mock.patch.object(litellm, "aembedding", autospec=True) +@mock.patch("kairon.shared.rest_client.AioRestClient.request", autospec=True) def test_prompt_response_action_streaming_enabled(mock_search, mock_embedding, mock_completion): - from kairon.shared.llm.gpt3 import GPT3FAQEmbedding from uuid6 import uuid7 action_name = GPT_LLM_FAQ @@ -11489,9 +11704,9 @@ def test_prompt_response_action_streaming_enabled(mock_search, mock_embedding, m 'presence_penalty': 0.0, 'frequency_penalty': 0.0, 'logit_bias': {}} - embedding = list(np.random.random(GPT3FAQEmbedding.__embedding__)) - mock_embedding.return_value = embedding - mock_completion.return_value = generated_text, generated_text + embedding = list(np.random.random(OPENAI_EMBEDDING_OUTPUT)) + mock_embedding.return_value = {'data': [{'embedding': embedding}]} + mock_completion.return_value = {'choices': [{'delta': {'role': 'assistant', 'content': generated_text}, 'finish_reason': None, 'index': 0}]} mock_search.return_value = { 'result': [{'id': uuid7().__str__(), 'score': 0.80, 'payload': {'content': bot_content}}]} Actions(name=action_name, type=ActionType.prompt_action.value, bot=bot, user=user).save() @@ -11507,28 +11722,23 @@ def test_prompt_response_action_streaming_enabled(mock_search, mock_embedding, m response = client.post("/webhook", json=request_object) response_json = response.json() + print(response_json['events']) assert response_json['events'] == [ {'event': 'slot', 'timestamp': None, 'name': 'kairon_action_response', 'value': generated_text}] assert response_json['responses'] == [ {'text': generated_text, 'buttons': [], 'elements': [], 'custom': {}, 'template': None, 'response': None, 'image': None, 'attachment': None} ] - print(mock_completion.call_args.kwargs) - assert mock_completion.call_args.kwargs == { - 'messages': [{'role': 'system', 'content': 'You are a personal assistant.\n'}, {'role': 'user', - 'content': "\nInstructions on how to use Similarity Prompt:\n['Python is a high-level, general-purpose programming language. Its design philosophy emphasizes code readability with the use of significant indentation. Python is dynamically typed and garbage-collected.']\nAnswer question based on the context above.\n \nQ: What kind of language is python? \nA:"}], - 'temperature': 0.0, 'max_tokens': 300, 'model': 'gpt-3.5-turbo', 'top_p': 0.0, 'n': 1, 'stream': True, - 'stop': None, 'presence_penalty': 0.0, 'frequency_penalty': 0.0, 'logit_bias': {}} - assert mock_completion.call_args.args[1] == 'chat/completions' + expected = {'messages': [{'role': 'system', 'content': 'You are a personal assistant.\n'}, {'role': 'user', + 'content': "\nInstructions on how to use Similarity Prompt:\n['Python is a high-level, general-purpose programming language. Its design philosophy emphasizes code readability with the use of significant indentation. Python is dynamically typed and garbage-collected.']\nAnswer question based on the context above.\n \nQ: What kind of language is python? \nA:"}], + 'metadata': {'user': 'udit.pandeyy', 'bot': '5f50k90a56b698ca10d35d2e', 'invocation': 'prompt_action'}, 'api_key': 'keyvalue', + 'num_retries': 3, 'temperature': 0.0, 'max_tokens': 300, 'model': 'gpt-3.5-turbo', 'top_p': 0.0, 'n': 1, + 'stream': True, 'stop': None, 'presence_penalty': 0.0, 'frequency_penalty': 0.0, 'logit_bias': {}} + assert not DeepDiff(mock_completion.call_args.kwargs, expected, ignore_order=True) -@patch("kairon.shared.llm.gpt3.openai.ChatCompletion.create", autospec=True) -@patch("kairon.shared.llm.gpt3.openai.Embedding.create", autospec=True) -@patch("kairon.shared.rest_client.AioRestClient.request", autospec=True) -def test_prompt_response_action_failure(mock_search, mock_embedding, mock_completion): - from kairon.shared.llm.gpt3 import GPT3FAQEmbedding - from openai.util import convert_to_openai_object - from openai.openai_response import OpenAIResponse +@mock.patch("kairon.shared.rest_client.AioRestClient.request", autospec=True) +def test_prompt_response_action_failure(mock_search): from uuid6 import uuid7 action_name = GPT_LLM_FAQ @@ -11537,10 +11747,7 @@ def test_prompt_response_action_failure(mock_search, mock_embedding, mock_comple user_msg = "What kind of language is python?" bot_content = "Python is a high-level, general-purpose programming language. Its design philosophy emphasizes code readability with the use of significant indentation. Python is dynamically typed and garbage-collected." generated_text = "I don't know." - embedding = list(np.random.random(GPT3FAQEmbedding.__embedding__)) - mock_embedding.return_value = convert_to_openai_object(OpenAIResponse({'data': [{'embedding': embedding}]}, {})) - mock_completion.return_value = convert_to_openai_object( - OpenAIResponse({'choices': [{'message': {'content': generated_text, 'role': 'assistant'}}]}, {})) + embedding = list(np.random.random(OPENAI_EMBEDDING_OUTPUT)) mock_search.return_value = { 'result': [{'id': uuid7().__str__(), 'score': 0.80, 'payload': {'content': bot_content}}]} Actions(name=action_name, type=ActionType.prompt_action.value, bot=bot, user=user).save() @@ -11611,13 +11818,10 @@ def test_prompt_action_response_action_does_not_exists(): assert len(response_json['responses']) == 0 -@mock.patch.object(GPT3FAQEmbedding, "_GPT3FAQEmbedding__get_answer", autospec=True) -@mock.patch.object(GPT3FAQEmbedding, "_GPT3FAQEmbedding__get_embedding", autospec=True) -@patch("kairon.shared.rest_client.AioRestClient.request", autospec=True) +@mock.patch.object(litellm, "acompletion", autospec=True) +@mock.patch.object(litellm, "aembedding", autospec=True) +@mock.patch("kairon.shared.rest_client.AioRestClient.request", autospec=True) def test_prompt_action_response_action_with_static_user_prompt(mock_search, mock_embedding, mock_completion): - from kairon.shared.llm.gpt3 import GPT3FAQEmbedding - from openai.util import convert_to_openai_object - from openai.openai_response import OpenAIResponse from uuid6 import uuid7 action_name = "kairon_faq_action" @@ -11646,8 +11850,7 @@ def test_prompt_action_response_action_with_static_user_prompt(mock_search, mock ] def mock_completion_for_answer(*args, **kwargs): - return convert_to_openai_object( - OpenAIResponse({'choices': [{'message': {'content': generated_text, 'role': 'assistant'}}]}, {})) + return {'choices': [{'message': {'content': generated_text, 'role': 'assistant'}}]} def __mock_search_cache(*args, **kwargs): return {'result': []} @@ -11660,9 +11863,9 @@ def __mock_cache_result(*args, **kwargs): mock_completion.return_value = mock_completion_for_answer() - embedding = list(np.random.random(GPT3FAQEmbedding.__embedding__)) - mock_embedding.return_value = embedding - mock_completion.return_value = generated_text + embedding = list(np.random.random(OPENAI_EMBEDDING_OUTPUT)) + mock_embedding.return_value = {'data': [{'embedding': embedding}]} + mock_completion.return_value = {'choices': [{'message': {'content': generated_text, 'role': 'assistant'}}]} mock_search.side_effect = [__mock_search_cache(), __mock_fetch_similar(), __mock_cache_result()] Actions(name=action_name, type=ActionType.prompt_action.value, bot=bot, user=user).save() BotSettings(llm_settings=LLMSettings(enable_faq=True), bot=bot, user=user).save() @@ -11692,13 +11895,10 @@ def __mock_cache_result(*args, **kwargs): @responses.activate -@mock.patch.object(GPT3FAQEmbedding, "_GPT3FAQEmbedding__get_answer", autospec=True) -@mock.patch.object(GPT3FAQEmbedding, "_GPT3FAQEmbedding__get_embedding", autospec=True) -@patch("kairon.shared.llm.gpt3.GPT3FAQEmbedding.__collection_search__", autospec=True) +@mock.patch.object(litellm, "acompletion", autospec=True) +@mock.patch.object(litellm, "aembedding", autospec=True) +@mock.patch("kairon.shared.llm.processor.LLMProcessor.__collection_search__", autospec=True) def test_prompt_action_response_action_with_action_prompt(mock_search, mock_embedding, mock_completion, aioresponses): - from kairon.shared.llm.gpt3 import GPT3FAQEmbedding - from openai.util import convert_to_openai_object - from openai.openai_response import OpenAIResponse from uuid6 import uuid7 action_name = "kairon_faq_action" @@ -11765,16 +11965,15 @@ def test_prompt_action_response_action_with_action_prompt(mock_search, mock_embe ] def mock_completion_for_answer(*args, **kwargs): - return convert_to_openai_object( - OpenAIResponse({'choices': [{'message': {'content': generated_text, 'role': 'assistant'}}]}, {})) + return {'choices': [{'message': {'content': generated_text, 'role': 'assistant'}}]} def __mock_fetch_similar(*args, **kwargs): return {'result': [{'id': uuid7().__str__(), 'score': 0.80, 'payload': {'content': bot_content}}]} mock_completion.return_value = mock_completion_for_answer() - embedding = list(np.random.random(GPT3FAQEmbedding.__embedding__)) - mock_embedding.return_value = embedding - mock_completion.return_value = generated_text + embedding = list(np.random.random(OPENAI_EMBEDDING_OUTPUT)) + mock_embedding.return_value = {'data': [{'embedding': embedding}]} + mock_completion.return_value = {'choices': [{'message': {'content': generated_text, 'role': 'assistant'}}]} mock_search.return_value = __mock_fetch_similar() Actions(name=action_name, type=ActionType.prompt_action.value, bot=bot, user=user).save() BotSettings(llm_settings=LLMSettings(enable_faq=True), bot=bot, user=user).save() @@ -11802,24 +12001,29 @@ def __mock_fetch_similar(*args, **kwargs): assert response_json['responses'] == [ {'text': generated_text, 'buttons': [], 'elements': [], 'custom': {}, 'template': None, 'response': None, 'image': None, 'attachment': None}] - log = ActionServerLogs.objects(bot=bot, type=ActionType.prompt_action.value, status="SUCCESS").get() - assert log['llm_logs'] == [] - assert mock_completion.call_args.args[1] == 'What kind of language is python?' - assert mock_completion.call_args.args[2] == 'You are a personal assistant.\n' - with open('tests/testing_data/actions/action_prompt.txt', 'r') as file: - prompt_data = file.read() - print(mock_completion.call_args.args[3]) - assert mock_completion.call_args.args[3] == prompt_data - - -@mock.patch.object(GPT3FAQEmbedding, "_GPT3FAQEmbedding__get_answer", autospec=True) -@mock.patch.object(GPT3FAQEmbedding, "_GPT3FAQEmbedding__get_embedding", autospec=True) + log = ActionServerLogs.objects(bot=bot, type=ActionType.prompt_action.value, + status="SUCCESS").get().to_mongo().to_dict() + expected = [{'messages': [{'role': 'system', 'content': 'You are a personal assistant.\n'}, {'role': 'user', + 'content': "Python Prompt:\nA programming language is a system of notation for writing computer programs.[1] Most programming languages are text-based formal languages, but they may also be graphical. They are a kind of computer language.\nInstructions on how to use Python Prompt:\nAnswer according to the context\n\nJava Prompt:\nJava is a programming language and computing platform first released by Sun Microsystems in 1995.\nInstructions on how to use Java Prompt:\nAnswer according to the context\n\nAction Prompt:\nPython is a scripting language because it uses an interpreter to translate and run its code.\nInstructions on how to use Action Prompt:\nAnswer according to the context\n\n\nInstructions on how to use Similarity Prompt:\n['Python is a high-level, general-purpose programming language. Its design philosophy emphasizes code readability with the use of significant indentation. Python is dynamically typed and garbage-collected.']\nAnswer question based on the context above, if answer is not in the context go check previous logs.\n \nQ: What kind of language is python? \nA:"}], + 'raw_completion_response': {'choices': [{'message': { + 'content': 'Python is dynamically typed, garbage-collected, high level, general purpose programming.', + 'role': 'assistant'}}]}, 'type': 'answer_query', + 'hyperparameters': {'temperature': 0.0, 'max_tokens': 300, 'model': 'gpt-3.5-turbo', 'top_p': 0.0, + 'n': 1, 'stop': None, 'presence_penalty': 0.0, + 'frequency_penalty': 0.0, 'logit_bias': {}}}] + assert not DeepDiff(log['llm_logs'], expected, ignore_order=True) + expected = {'messages': [{'role': 'system', 'content': 'You are a personal assistant.\n'}, {'role': 'user', + 'content': "Python Prompt:\nA programming language is a system of notation for writing computer programs.[1] Most programming languages are text-based formal languages, but they may also be graphical. They are a kind of computer language.\nInstructions on how to use Python Prompt:\nAnswer according to the context\n\nJava Prompt:\nJava is a programming language and computing platform first released by Sun Microsystems in 1995.\nInstructions on how to use Java Prompt:\nAnswer according to the context\n\nAction Prompt:\nPython is a scripting language because it uses an interpreter to translate and run its code.\nInstructions on how to use Action Prompt:\nAnswer according to the context\n\n\nInstructions on how to use Similarity Prompt:\n['Python is a high-level, general-purpose programming language. Its design philosophy emphasizes code readability with the use of significant indentation. Python is dynamically typed and garbage-collected.']\nAnswer question based on the context above, if answer is not in the context go check previous logs.\n \nQ: What kind of language is python? \nA:"}], + 'metadata': {'user': 'nupur.khare', 'bot': '5u08kd0a56b698ca10d98e6s', 'invocation': 'prompt_action'}, 'api_key': 'keyvalue', + 'num_retries': 3, 'temperature': 0.0, 'max_tokens': 300, 'model': 'gpt-3.5-turbo', 'top_p': 0.0, 'n': 1, + 'stop': None, 'presence_penalty': 0.0, 'frequency_penalty': 0.0, 'logit_bias': {}} + assert not DeepDiff(mock_completion.call_args[1], expected, ignore_order=True) + + +@mock.patch.object(litellm, "acompletion", autospec=True) +@mock.patch.object(litellm, "aembedding", autospec=True) @mock.patch.object(ActionUtility, "perform_google_search", autospec=True) def test_kairon_faq_response_with_google_search_prompt(mock_google_search, mock_embedding, mock_completion): - from kairon.shared.llm.gpt3 import GPT3FAQEmbedding - from openai.util import convert_to_openai_object - from openai.openai_response import OpenAIResponse - action_name = "kairon_faq_action" google_action_name = "custom_search_action" bot = "5u08kd0a56b698ca10hgjgjkhgjks" @@ -11860,12 +12064,11 @@ def _run_action(*args, **kwargs): PromptAction(name=action_name, bot=bot, user=user, llm_prompts=llm_prompts).save() def mock_completion_for_answer(*args, **kwargs): - return convert_to_openai_object( - OpenAIResponse({'choices': [{'message': {'content': generated_text, 'role': 'assistant'}}]}, {})) + return {'choices': [{'message': {'content': generated_text, 'role': 'assistant'}}]} - mock_completion.return_value = generated_text - embedding = list(np.random.random(GPT3FAQEmbedding.__embedding__)) - mock_embedding.return_value = embedding + mock_completion.return_value = {'choices': [{'message': {'content': generated_text, 'role': 'assistant'}}]} + embedding = list(np.random.random(OPENAI_EMBEDDING_OUTPUT)) + mock_embedding.return_value = {'data': [{'embedding': embedding}]} mock_google_search.side_effect = _run_action request_object = json.load(open("tests/testing_data/actions/action-request.json")) @@ -11882,12 +12085,24 @@ def mock_completion_for_answer(*args, **kwargs): 'buttons': [], 'elements': [], 'custom': {}, 'template': None, 'response': None, 'image': None, 'attachment': None}] - log = ActionServerLogs.objects(bot=bot, type=ActionType.prompt_action.value, status="SUCCESS").get() - assert log['llm_logs'] == [] - assert mock_completion.call_args.args[1] == 'What is kanban' - assert mock_completion.call_args.args[2] == 'You are a personal assistant.\n' - assert mock_completion.call_args.args[ - 3] == 'Google search Prompt:\nKanban visualizes both the process (the workflow) and the actual work passing through that process.\nTo know more, please visit: Kanban\n\nKanban project management is one of the emerging PM methodologies, and the Kanban approach is suitable for every team and goal.\nTo know more, please visit: Kanban Project management\n\nKanban is a popular framework used to implement agile and DevOps software development.\nTo know more, please visit: Kanban agile\nInstructions on how to use Google search Prompt:\nAnswer according to the context\n\n' + log = ActionServerLogs.objects(bot=bot, type=ActionType.prompt_action.value, + status="SUCCESS").get().to_mongo().to_dict() + expected = [{'messages': [{'role': 'system', 'content': 'You are a personal assistant.\n'}, {'role': 'user', + 'content': 'Google search Prompt:\nKanban visualizes both the process (the workflow) and the actual work passing through that process.\nTo know more, please visit: Kanban\n\nKanban project management is one of the emerging PM methodologies, and the Kanban approach is suitable for every team and goal.\nTo know more, please visit: Kanban Project management\n\nKanban is a popular framework used to implement agile and DevOps software development.\nTo know more, please visit: Kanban agile\nInstructions on how to use Google search Prompt:\nAnswer according to the context\n\n \nQ: What is kanban \nA:'}], + 'raw_completion_response': {'choices': [{'message': { + 'content': 'Kanban is a workflow management tool which visualizes both the process (the workflow) and the actual work passing through that process.', + 'role': 'assistant'}}]}, 'type': 'answer_query', + 'hyperparameters': {'temperature': 0.0, 'max_tokens': 300, 'model': 'gpt-3.5-turbo', 'top_p': 0.0, + 'n': 1, 'stop': None, 'presence_penalty': 0.0, + 'frequency_penalty': 0.0, 'logit_bias': {}}}] + + assert not DeepDiff(log['llm_logs'], expected, ignore_order=True) + expected = {'messages': [{'role': 'system', 'content': 'You are a personal assistant.\n'}, {'role': 'user', + 'content': 'Google search Prompt:\nKanban visualizes both the process (the workflow) and the actual work passing through that process.\nTo know more, please visit: Kanban\n\nKanban project management is one of the emerging PM methodologies, and the Kanban approach is suitable for every team and goal.\nTo know more, please visit: Kanban Project management\n\nKanban is a popular framework used to implement agile and DevOps software development.\nTo know more, please visit: Kanban agile\nInstructions on how to use Google search Prompt:\nAnswer according to the context\n\n \nQ: What is kanban \nA:'}], + 'metadata': {'user': 'test_user', 'bot': '5u08kd0a56b698ca10hgjgjkhgjks', 'invocation': 'prompt_action'}, 'api_key': 'keyvalue', + 'num_retries': 3, 'temperature': 0.0, 'max_tokens': 300, 'model': 'gpt-3.5-turbo', 'top_p': 0.0, 'n': 1, + 'stop': None, 'presence_penalty': 0.0, 'frequency_penalty': 0.0, 'logit_bias': {}} + assert not DeepDiff(mock_completion.call_args[1], expected, ignore_order=True) def test_prompt_response_action_with_action_not_found(): @@ -11919,13 +12134,10 @@ def test_prompt_response_action_with_action_not_found(): log['exception'] = 'No action found for given bot and name' -@mock.patch.object(GPT3FAQEmbedding, "_GPT3FAQEmbedding__get_answer", autospec=True) -@mock.patch.object(GPT3FAQEmbedding, "_GPT3FAQEmbedding__get_embedding", autospec=True) -@patch("kairon.shared.rest_client.AioRestClient.request", autospec=True) +@mock.patch.object(litellm, "acompletion", autospec=True) +@mock.patch.object(litellm, "aembedding", autospec=True) +@mock.patch("kairon.shared.rest_client.AioRestClient.request", autospec=True) def test_prompt_action_dispatch_response_disabled(mock_search, mock_embedding, mock_completion): - from kairon.shared.llm.gpt3 import GPT3FAQEmbedding - from openai.util import convert_to_openai_object - from openai.openai_response import OpenAIResponse from uuid6 import uuid7 action_name = "kairon_faq_action" @@ -11949,17 +12161,16 @@ def test_prompt_action_dispatch_response_disabled(mock_search, mock_embedding, m ] def mock_completion_for_answer(*args, **kwargs): - return convert_to_openai_object( - OpenAIResponse({'choices': [{'message': {'content': generated_text, 'role': 'assistant'}}]}, {})) + return {'choices': [{'message': {'content': generated_text, 'role': 'assistant'}}]} def __mock_fetch_similar(*args, **kwargs): return {'result': [{'id': uuid7().__str__(), 'score': 0.80, 'payload': {'content': bot_content}}]} mock_completion.return_value = mock_completion_for_answer() - embedding = list(np.random.random(GPT3FAQEmbedding.__embedding__)) - mock_embedding.return_value = embedding - mock_completion.return_value = generated_text + embedding = list(np.random.random(OPENAI_EMBEDDING_OUTPUT)) + mock_embedding.return_value = {'data': [{'embedding': embedding}]} + mock_completion.return_value = {'choices': [{'message': {'content': generated_text, 'role': 'assistant'}}]} mock_search.return_value = __mock_fetch_similar() Actions(name=action_name, type=ActionType.prompt_action.value, bot=bot, user=user).save() BotSettings(llm_settings=LLMSettings(enable_faq=True), bot=bot, user=user).save() @@ -12001,23 +12212,28 @@ def __mock_fetch_similar(*args, **kwargs): {'content': 'Python is dynamically typed, garbage-collected, high level, general purpose programming.'} }, {'type': 'slots_to_fill', 'data': {}, 'slot_eval_log': ['initiating slot evaluation']} ] - assert log['llm_logs'] == [] - assert mock_completion.call_args.args[1] == 'What is the name of prompt?' - assert mock_completion.call_args.args[2] == 'You are a personal assistant.\n' - with open('tests/testing_data/actions/slot_prompt.txt', 'r') as file: - prompt_data = file.read() - print(mock_completion.call_args.args[3]) - assert mock_completion.call_args.args[3] == prompt_data - - -@mock.patch.object(GPT3FAQEmbedding, "_GPT3FAQEmbedding__get_answer", autospec=True) -@mock.patch.object(GPT3FAQEmbedding, "_GPT3FAQEmbedding__get_embedding", autospec=True) -@patch("kairon.shared.actions.utils.ActionUtility.compose_response", autospec=True) -@patch("kairon.shared.rest_client.AioRestClient.request", autospec=True) + expected = [{'messages': [{'role': 'system', 'content': 'You are a personal assistant.\n'}, {'role': 'user', + 'content': "Language Prompt:\nPython is an interpreted, object-oriented, high-level programming language with dynamic semantics.\nInstructions on how to use Language Prompt:\nAnswer according to the context\n\n\nInstructions on how to use Similarity Prompt:\n['Python is a high-level, general-purpose programming language. Its design philosophy emphasizes code readability with the use of significant indentation. Python is dynamically typed and garbage-collected.']\nAnswer question based on the context above, if answer is not in the context go check previous logs.\n \nQ: What is the name of prompt? \nA:"}], + 'raw_completion_response': {'choices': [{'message': { + 'content': 'Python is dynamically typed, garbage-collected, high level, general purpose programming.', + 'role': 'assistant'}}]}, 'type': 'answer_query', + 'hyperparameters': {'temperature': 0.0, 'max_tokens': 300, 'model': 'gpt-3.5-turbo', 'top_p': 0.0, + 'n': 1, 'stop': None, 'presence_penalty': 0.0, + 'frequency_penalty': 0.0, 'logit_bias': {}}}] + assert not DeepDiff(log['llm_logs'], expected, ignore_order=True) + expected = {'messages': [{'role': 'system', 'content': 'You are a personal assistant.\n'}, {'role': 'user', + 'content': "Language Prompt:\nPython is an interpreted, object-oriented, high-level programming language with dynamic semantics.\nInstructions on how to use Language Prompt:\nAnswer according to the context\n\n\nInstructions on how to use Similarity Prompt:\n['Python is a high-level, general-purpose programming language. Its design philosophy emphasizes code readability with the use of significant indentation. Python is dynamically typed and garbage-collected.']\nAnswer question based on the context above, if answer is not in the context go check previous logs.\n \nQ: What is the name of prompt? \nA:"}], + 'metadata': {'user': 'udit.pandey', 'bot': '5u80fd0a56c908ca10d35d2sjhj', 'invocation': 'prompt_action'}, 'api_key': 'keyvalue', + 'num_retries': 3, 'temperature': 0.0, 'max_tokens': 300, 'model': 'gpt-3.5-turbo', 'top_p': 0.0, 'n': 1, + 'stop': None, 'presence_penalty': 0.0, 'frequency_penalty': 0.0, 'logit_bias': {}} + assert not DeepDiff(mock_completion.call_args[1], expected, ignore_order=True) + + +@mock.patch.object(litellm, "acompletion", autospec=True) +@mock.patch.object(litellm, "aembedding", autospec=True) +@mock.patch("kairon.shared.actions.utils.ActionUtility.compose_response", autospec=True) +@mock.patch("kairon.shared.rest_client.AioRestClient.request", autospec=True) def test_prompt_action_set_slots(mock_search, mock_slot_set, mock_mock_embedding, mock_completion): - from openai.util import convert_to_openai_object - from openai.openai_response import OpenAIResponse - action_name = "kairon_faq_action" bot = "5u80fd0a56c908ca10d35d2sjhjhjhj" user = "udit.pandey" @@ -12040,11 +12256,10 @@ def test_prompt_action_set_slots(mock_search, mock_slot_set, mock_mock_embedding ] def mock_completion_for_answer(*args, **kwargs): - return convert_to_openai_object( - OpenAIResponse({'choices': [{'message': {'content': generated_text, 'role': 'assistant'}}]}, {})) + return {'choices': [{'message': {'content': generated_text, 'role': 'assistant'}}]} mock_completion.return_value = mock_completion_for_answer() - mock_completion.return_value = generated_text + mock_completion.return_value = {'choices': [{'message': {'content': generated_text, 'role': 'assistant'}}]} log1 = ['Slot: api_type', 'evaluation_type: expression', f"data: {generated_text}", 'response: filter'] log2 = ['Slot: query', 'evaluation_type: expression', f"data: {generated_text}", 'response: {\"must\": [{\"key\": \"Date Added\", \"match\": {\"value\": 1673721000.0}}]}'] @@ -12093,26 +12308,38 @@ def mock_completion_for_answer(*args, **kwargs): assert events == [ {'type': 'llm_response', 'response': '{"api_type": "filter", {"filter": {"must": [{"key": "Date Added", "match": {"value": 1673721000.0}}]}}}', - 'llm_response_log': {'content': '{"api_type": "filter", {"filter": {"must": [{"key": "Date Added", "match": {"value": 1673721000.0}}]}}}'}}, - {'type': 'slots_to_fill', 'data': {'api_type': 'filter', 'query': '{"must": [{"key": "Date Added", "match": {"value": 1673721000.0}}]}'}, - 'slot_eval_log': ['initiating slot evaluation', 'Slot: api_type', 'Slot: api_type', 'evaluation_type: expression', + 'llm_response_log': { + 'content': '{"api_type": "filter", {"filter": {"must": [{"key": "Date Added", "match": {"value": 1673721000.0}}]}}}'}}, + {'type': 'slots_to_fill', + 'data': {'api_type': 'filter', 'query': '{"must": [{"key": "Date Added", "match": {"value": 1673721000.0}}]}'}, + 'slot_eval_log': ['initiating slot evaluation', 'Slot: api_type', 'Slot: api_type', + 'evaluation_type: expression', 'data: {"api_type": "filter", {"filter": {"must": [{"key": "Date Added", "match": {"value": 1673721000.0}}]}}}', 'response: filter', 'Slot: query', 'Slot: query', 'evaluation_type: expression', 'data: {"api_type": "filter", {"filter": {"must": [{"key": "Date Added", "match": {"value": 1673721000.0}}]}}}', 'response: {"must": [{"key": "Date Added", "match": {"value": 1673721000.0}}]}']} ] - assert log['llm_logs'] == [] - assert mock_completion.call_args.args[1] == user_msg - assert mock_completion.call_args.args[2] == 'You are a personal assistant.\n' - - -@mock.patch.object(GPT3FAQEmbedding, "_GPT3FAQEmbedding__get_answer", autospec=True) -@mock.patch.object(GPT3FAQEmbedding, "_GPT3FAQEmbedding__get_embedding", autospec=True) -@patch("kairon.shared.rest_client.AioRestClient.request", autospec=True) + expected = [{'messages': [{'role': 'system', 'content': 'You are a personal assistant.\n'}, {'role': 'user', + 'content': 'Qdrant Prompt:\nConvert user questions into json requests in qdrant such that they will either filter, apply range queries and search the payload in qdrant. Sample payload present in qdrant looks like below with each of the points starting with 1 to 5 is a record in qdrant.1. {"Category (Risk, Issue, Action Item)": "Risk", "Date Added": 1673721000.0,2. {"Category (Risk, Issue, Action Item)": "Action Item", "Date Added": 1673721000.0,For eg: to find category of record created on 15/01/2023, the filter query is:{"filter": {"must": [{"key": "Date Added", "match": {"value": 1673721000.0}}]}}\nInstructions on how to use Qdrant Prompt:\nCreate qdrant filter query out of user message based on above instructions.\n\n \nQ: category of record created on 15/01/2023? \nA:'}], + 'raw_completion_response': {'choices': [{'message': { + 'content': '{"api_type": "filter", {"filter": {"must": [{"key": "Date Added", "match": {"value": 1673721000.0}}]}}}', + 'role': 'assistant'}}]}, 'type': 'answer_query', + 'hyperparameters': {'temperature': 0.0, 'max_tokens': 300, 'model': 'gpt-3.5-turbo', 'top_p': 0.0, + 'n': 1, 'stop': None, 'presence_penalty': 0.0, + 'frequency_penalty': 0.0, 'logit_bias': {}}}] + assert not DeepDiff(log['llm_logs'], expected, ignore_order=True) + expected = {'messages': [{'role': 'system', 'content': 'You are a personal assistant.\n'}, {'role': 'user', + 'content': 'Qdrant Prompt:\nConvert user questions into json requests in qdrant such that they will either filter, apply range queries and search the payload in qdrant. Sample payload present in qdrant looks like below with each of the points starting with 1 to 5 is a record in qdrant.1. {"Category (Risk, Issue, Action Item)": "Risk", "Date Added": 1673721000.0,2. {"Category (Risk, Issue, Action Item)": "Action Item", "Date Added": 1673721000.0,For eg: to find category of record created on 15/01/2023, the filter query is:{"filter": {"must": [{"key": "Date Added", "match": {"value": 1673721000.0}}]}}\nInstructions on how to use Qdrant Prompt:\nCreate qdrant filter query out of user message based on above instructions.\n\n \nQ: category of record created on 15/01/2023? \nA:'}], + 'metadata': {'user': 'udit.pandey', 'bot': '5u80fd0a56c908ca10d35d2sjhjhjhj', 'invocation': 'prompt_action'}, 'api_key': 'keyvalue', + 'num_retries': 3, 'temperature': 0.0, 'max_tokens': 300, 'model': 'gpt-3.5-turbo', 'top_p': 0.0, 'n': 1, + 'stop': None, 'presence_penalty': 0.0, 'frequency_penalty': 0.0, 'logit_bias': {}} + assert not DeepDiff(mock_completion.call_args[1], expected, ignore_order=True) + + +@mock.patch.object(litellm, "acompletion", autospec=True) +@mock.patch.object(litellm, "aembedding", autospec=True) +@mock.patch("kairon.shared.rest_client.AioRestClient.request", autospec=True) def test_prompt_action_response_action_slot_prompt(mock_search, mock_embedding, mock_completion): - from kairon.shared.llm.gpt3 import GPT3FAQEmbedding - from openai.util import convert_to_openai_object - from openai.openai_response import OpenAIResponse from uuid6 import uuid7 action_name = "kairon_faq_action" @@ -12136,17 +12363,16 @@ def test_prompt_action_response_action_slot_prompt(mock_search, mock_embedding, ] def mock_completion_for_answer(*args, **kwargs): - return convert_to_openai_object( - OpenAIResponse({'choices': [{'message': {'content': generated_text, 'role': 'assistant'}}]}, {})) + return {'choices': [{'message': {'content': generated_text, 'role': 'assistant'}}]} def __mock_fetch_similar(*args, **kwargs): return {'result': [{'id': uuid7().__str__(), 'score': 0.80, 'payload': {'content': bot_content}}]} mock_completion.return_value = mock_completion_for_answer() - embedding = list(np.random.random(GPT3FAQEmbedding.__embedding__)) - mock_embedding.return_value = embedding - mock_completion.return_value = generated_text + embedding = list(np.random.random(OPENAI_EMBEDDING_OUTPUT)) + mock_embedding.return_value = {'data': [{'embedding': embedding}]} + mock_completion.return_value = {'choices': [{'message': {'content': generated_text, 'role': 'assistant'}}]} mock_search.return_value = __mock_fetch_similar() Actions(name=action_name, type=ActionType.prompt_action.value, bot=bot, user=user).save() BotSettings(llm_settings=LLMSettings(enable_faq=True), bot=bot, user=user).save() @@ -12191,22 +12417,27 @@ def __mock_fetch_similar(*args, **kwargs): {'content': 'Python is dynamically typed, garbage-collected, high level, general purpose programming.'} }, {'type': 'slots_to_fill', 'data': {}, 'slot_eval_log': ['initiating slot evaluation']} ] - assert log['llm_logs'] == [] - assert mock_completion.call_args.args[1] == 'What is the name of prompt?' - assert mock_completion.call_args.args[2] == 'You are a personal assistant.\n' - with open('tests/testing_data/actions/slot_prompt.txt', 'r') as file: - prompt_data = file.read() - print(mock_completion.call_args.args[3]) - assert mock_completion.call_args.args[3] == prompt_data - - -@mock.patch.object(GPT3FAQEmbedding, "_GPT3FAQEmbedding__get_answer", autospec=True) -@mock.patch.object(GPT3FAQEmbedding, "_GPT3FAQEmbedding__get_embedding", autospec=True) -@patch("kairon.shared.rest_client.AioRestClient.request", autospec=True) + expected = [{'messages': [{'role': 'system', 'content': 'You are a personal assistant.\n'}, {'role': 'user', + 'content': "Language Prompt:\nPython is an interpreted, object-oriented, high-level programming language with dynamic semantics.\nInstructions on how to use Language Prompt:\nAnswer according to the context\n\n\nInstructions on how to use Similarity Prompt:\n['Python is a high-level, general-purpose programming language. Its design philosophy emphasizes code readability with the use of significant indentation. Python is dynamically typed and garbage-collected.']\nAnswer question based on the context above, if answer is not in the context go check previous logs.\n \nQ: What is the name of prompt? \nA:"}], + 'raw_completion_response': {'choices': [{'message': { + 'content': 'Python is dynamically typed, garbage-collected, high level, general purpose programming.', + 'role': 'assistant'}}]}, 'type': 'answer_query', + 'hyperparameters': {'temperature': 0.0, 'max_tokens': 300, 'model': 'gpt-3.5-turbo', 'top_p': 0.0, + 'n': 1, 'stop': None, 'presence_penalty': 0.0, + 'frequency_penalty': 0.0, 'logit_bias': {}}}] + assert not DeepDiff(log['llm_logs'], expected, ignore_order=True) + expected = {'messages': [{'role': 'system', 'content': 'You are a personal assistant.\n'}, {'role': 'user', + 'content': "Language Prompt:\nPython is an interpreted, object-oriented, high-level programming language with dynamic semantics.\nInstructions on how to use Language Prompt:\nAnswer according to the context\n\n\nInstructions on how to use Similarity Prompt:\n['Python is a high-level, general-purpose programming language. Its design philosophy emphasizes code readability with the use of significant indentation. Python is dynamically typed and garbage-collected.']\nAnswer question based on the context above, if answer is not in the context go check previous logs.\n \nQ: What is the name of prompt? \nA:"}], + 'metadata': {'user': 'udit.pandey', 'bot': '5u80fd0a56c908ca10d35d2s', 'invocation': 'prompt_action'}, 'api_key': 'keyvalue', + 'num_retries': 3, 'temperature': 0.0, 'max_tokens': 300, 'model': 'gpt-3.5-turbo', 'top_p': 0.0, 'n': 1, + 'stop': None, 'presence_penalty': 0.0, 'frequency_penalty': 0.0, 'logit_bias': {}} + assert not DeepDiff(mock_completion.call_args[1], expected, ignore_order=True) + + +@mock.patch.object(litellm, "acompletion", autospec=True) +@mock.patch.object(litellm, "aembedding", autospec=True) +@mock.patch("kairon.shared.rest_client.AioRestClient.request", autospec=True) def test_prompt_action_user_message_in_slot(mock_search, mock_embedding, mock_completion): - from kairon.shared.llm.gpt3 import GPT3FAQEmbedding - from openai.util import convert_to_openai_object - from openai.openai_response import OpenAIResponse from uuid6 import uuid7 action_name = "kairon_faq_action" @@ -12226,17 +12457,16 @@ def test_prompt_action_user_message_in_slot(mock_search, mock_embedding, mock_co ] def mock_completion_for_answer(*args, **kwargs): - return convert_to_openai_object( - OpenAIResponse({'choices': [{'message': {'content': generated_text, 'role': 'assistant'}}]}, {})) + return {'choices': [{'message': {'content': generated_text, 'role': 'assistant'}}]} def __mock_fetch_similar(*args, **kwargs): return {'result': [{'id': uuid7().__str__(), 'score': 0.80, 'payload': {'content': bot_content}}]} mock_completion.return_value = mock_completion_for_answer() - embedding = list(np.random.random(GPT3FAQEmbedding.__embedding__)) - mock_embedding.return_value = embedding - mock_completion.return_value = generated_text + embedding = list(np.random.random(OPENAI_EMBEDDING_OUTPUT)) + mock_embedding.return_value = {'data': [{'embedding': embedding}]} + mock_completion.return_value = {'choices': [{'message': {'content': generated_text, 'role': 'assistant'}}]} mock_search.return_value = __mock_fetch_similar() Actions(name=action_name, type=ActionType.prompt_action.value, bot=bot, user=user).save() BotSettings(llm_settings=LLMSettings(enable_faq=True), bot=bot, user=user).save() @@ -12260,28 +12490,23 @@ def __mock_fetch_similar(*args, **kwargs): {'text': generated_text, 'buttons': [], 'elements': [], 'custom': {}, 'template': None, 'response': None, 'image': None, 'attachment': None} ] - assert mock_completion.call_args[0][1] == 'Kanban And Scrum Together?' - assert mock_completion.call_args[0][2] == 'You are a personal assistant.\n' - print(mock_completion.call_args[0][3]) - assert mock_completion.call_args[0][3] == """ -Instructions on how to use Similarity Prompt: -['Scrum teams using Kanban as a visual management tool can get work delivered faster and more often. Prioritized tasks are completed first as the team collectively decides what is best using visual cues from the Kanban board. The best part is that Scrum teams can use Kanban and Scrum at the same time.'] -Answer question based on the context above, if answer is not in the context go check previous logs. -""" - -@mock.patch.object(GPT3FAQEmbedding, "_GPT3FAQEmbedding__get_answer", autospec=True) -@mock.patch.object(GPT3FAQEmbedding, "_GPT3FAQEmbedding__get_embedding", autospec=True) -@patch("kairon.shared.rest_client.AioRestClient.request", autospec=True) -def test_prompt_action_response_action_when_similarity_is_empty(mock_search, mock_embedding, mock_completion): - from kairon.shared.llm.gpt3 import GPT3FAQEmbedding - from uuid6 import uuid7 + expected = {'messages': [{'role': 'system', 'content': 'You are a personal assistant.\n'}, {'role': 'user', + 'content': "\nInstructions on how to use Similarity Prompt:\n['Scrum teams using Kanban as a visual management tool can get work delivered faster and more often. Prioritized tasks are completed first as the team collectively decides what is best using visual cues from the Kanban board. The best part is that Scrum teams can use Kanban and Scrum at the same time.']\nAnswer question based on the context above, if answer is not in the context go check previous logs.\n \nQ: Kanban And Scrum Together? \nA:"}], + 'metadata': {'user': user, 'bot': bot, 'invocation': 'prompt_action'}, 'api_key': value, + 'num_retries': 3, 'temperature': 0.0, 'max_tokens': 300, 'model': 'gpt-3.5-turbo', 'top_p': 0.0, 'n': 1, + 'stop': None, 'presence_penalty': 0.0, 'frequency_penalty': 0.0, 'logit_bias': {}} + assert not DeepDiff(mock_completion.call_args[1], expected, ignore_order=True) + +@mock.patch.object(litellm, "acompletion", autospec=True) +@mock.patch.object(litellm, "aembedding", autospec=True) +@mock.patch("kairon.shared.rest_client.AioRestClient.request", autospec=True) +def test_prompt_action_response_action_when_similarity_is_empty(mock_search, mock_embedding, mock_completion): action_name = "test_prompt_action_response_action_when_similarity_is_empty" bot = "5f50fd0a56b698ca10d35d2C" user = "udit.pandey" value = "keyvalue" user_msg = "What kind of language is python?" - bot_content = "Python is a high-level, general-purpose programming language. Its design philosophy emphasizes code readability with the use of significant indentation. Python is dynamically typed and garbage-collected." generated_text = "Python is dynamically typed, garbage-collected, high level, general purpose programming." llm_prompts = [ {'name': 'System Prompt', @@ -12297,9 +12522,9 @@ def test_prompt_action_response_action_when_similarity_is_empty(mock_search, moc 'is_enabled': True} ] - embedding = list(np.random.random(GPT3FAQEmbedding.__embedding__)) - mock_embedding.return_value = embedding - mock_completion.return_value = generated_text + embedding = list(np.random.random(OPENAI_EMBEDDING_OUTPUT)) + mock_embedding.return_value = {'data': [{'embedding': embedding}]} + mock_completion.return_value = {'choices': [{'message': {'content': generated_text, 'role': 'assistant'}}]} mock_search.return_value = {'result': []} Actions(name=action_name, type=ActionType.prompt_action.value, bot=bot, user=user).save() BotSettings(llm_settings=LLMSettings(enable_faq=True), bot=bot, user=user).save() @@ -12327,29 +12552,20 @@ def test_prompt_action_response_action_when_similarity_is_empty(mock_search, moc 'response': None, 'image': None, 'attachment': None} ] - assert mock_completion.call_args.args[1] == 'What kind of language is python?' - assert mock_completion.call_args.args[ - 2] == """You are a personal assistant. Answer question based on the context below.\n""" - print(mock_completion.call_args.args[3]) - assert not mock_completion.call_args.args[3] - print(mock_completion.call_args.kwargs) - assert mock_completion.call_args.kwargs == { - 'hyperparameters': {'temperature': 0.0, 'max_tokens': 300, 'model': 'gpt-3.5-turbo', 'top_p': 0.0, 'n': 1, - 'stream': False, 'stop': None, 'presence_penalty': 0.0, 'frequency_penalty': 0.0, - 'logit_bias': {}}, 'query_prompt': {}, - 'previous_bot_responses': [{'role': 'user', 'content': 'hello'}, - {'role': 'assistant', 'content': 'how are you'}], 'similarity_prompt': [ - {'similarity_prompt_name': 'Similarity Prompt', - 'similarity_prompt_instructions': 'Answer question based on the context above, if answer is not in the context go check previous logs.', - 'collection': 'python', 'use_similarity_prompt': True, 'top_results': 10, 'similarity_threshold': 0.7}], - 'instructions': []} - - -@mock.patch.object(GPT3FAQEmbedding, "_GPT3FAQEmbedding__get_answer", autospec=True) -@mock.patch.object(GPT3FAQEmbedding, "_GPT3FAQEmbedding__get_embedding", autospec=True) -@patch("kairon.shared.rest_client.AioRestClient.request", autospec=True) + expected = {'messages': [ + {'role': 'system', 'content': 'You are a personal assistant. Answer question based on the context below.\n'}, + {'role': 'user', 'content': 'hello'}, {'role': 'assistant', 'content': 'how are you'}, + {'role': 'user', 'content': ' \nQ: What kind of language is python? \nA:'}], + 'metadata': {'user': 'udit.pandey', 'bot': bot, 'invocation': 'prompt_action'}, 'api_key': value, + 'num_retries': 3, 'temperature': 0.0, 'max_tokens': 300, 'model': 'gpt-3.5-turbo', 'top_p': 0.0, 'n': 1, + 'stop': None, 'presence_penalty': 0.0, 'frequency_penalty': 0.0, 'logit_bias': {}} + assert not DeepDiff(mock_completion.call_args[1], expected, ignore_order=True) + + +@mock.patch.object(litellm, "acompletion", autospec=True) +@mock.patch.object(litellm, "aembedding", autospec=True) +@mock.patch("kairon.shared.rest_client.AioRestClient.request", autospec=True) def test_prompt_action_response_action_when_similarity_disabled(mock_search, mock_embedding, mock_completion): - from kairon.shared.llm.gpt3 import GPT3FAQEmbedding from uuid6 import uuid7 action_name = "test_prompt_action_response_action_when_similarity_disabled" @@ -12373,10 +12589,11 @@ def test_prompt_action_response_action_when_similarity_disabled(mock_search, moc 'is_enabled': False} ] - embedding = list(np.random.random(GPT3FAQEmbedding.__embedding__)) - mock_embedding.return_value = embedding - mock_completion.return_value = generated_text - mock_search.return_value = {'result': [{'id': uuid7().__str__(), 'score': 0.80, 'payload': {'content': bot_content}}]} + embedding = list(np.random.random(OPENAI_EMBEDDING_OUTPUT)) + mock_embedding.return_value = {'data': [{'embedding': embedding}]} + mock_completion.return_value = {'choices': [{'message': {'content': generated_text, 'role': 'assistant'}}]} + mock_search.return_value = { + 'result': [{'id': uuid7().__str__(), 'score': 0.80, 'payload': {'content': bot_content}}]} Actions(name=action_name, type=ActionType.prompt_action.value, bot=bot, user=user).save() BotSettings(llm_settings=LLMSettings(enable_faq=True), bot=bot, user=user).save() PromptAction(name=action_name, bot=bot, user=user, num_bot_responses=2, llm_prompts=llm_prompts).save() @@ -12402,17 +12619,11 @@ def test_prompt_action_response_action_when_similarity_disabled(mock_search, moc {'text': generated_text, 'buttons': [], 'elements': [], 'custom': {}, 'template': None, 'response': None, 'image': None, 'attachment': None} ] - - assert mock_completion.call_args.args[1] == 'What kind of language is python?' - assert mock_completion.call_args.args[ - 2] == """You are a personal assistant. Answer question based on the context below.\n""" - print(mock_completion.call_args.args[3]) - assert not mock_completion.call_args.args[3] - print(mock_completion.call_args.kwargs) - assert mock_completion.call_args.kwargs == { - 'hyperparameters': {'temperature': 0.0, 'max_tokens': 300, 'model': 'gpt-3.5-turbo', 'top_p': 0.0, 'n': 1, - 'stream': False, 'stop': None, 'presence_penalty': 0.0, 'frequency_penalty': 0.0, - 'logit_bias': {}}, 'query_prompt': {}, - 'previous_bot_responses': [{'role': 'user', 'content': 'hello'}, - {'role': 'assistant', 'content': 'how are you'}], 'similarity_prompt': [], - 'instructions': []} \ No newline at end of file + expected = {'messages': [ + {'role': 'system', 'content': 'You are a personal assistant. Answer question based on the context below.\n'}, + {'role': 'user', 'content': 'hello'}, {'role': 'assistant', 'content': 'how are you'}, + {'role': 'user', 'content': ' \nQ: What kind of language is python? \nA:'}], + 'metadata': {'user': user, 'bot': bot, 'invocation': 'prompt_action'}, 'api_key': value, + 'num_retries': 3, 'temperature': 0.0, 'max_tokens': 300, 'model': 'gpt-3.5-turbo', 'top_p': 0.0, 'n': 1, + 'stop': None, 'presence_penalty': 0.0, 'frequency_penalty': 0.0, 'logit_bias': {}} + assert not DeepDiff(mock_completion.call_args[1], expected, ignore_order=True) diff --git a/tests/integration_test/chat_service_test.py b/tests/integration_test/chat_service_test.py index 45597ff76..04d65276a 100644 --- a/tests/integration_test/chat_service_test.py +++ b/tests/integration_test/chat_service_test.py @@ -5,17 +5,21 @@ from datetime import datetime, timedelta from unittest import mock from urllib.parse import urlencode, quote_plus - -from kairon.shared.live_agent.live_agent import LiveAgentHandler from kairon.shared.utils import Utility - os.environ["system_file"] = "./tests/testing_data/system.yaml" os.environ["ASYNC_TEST_TIMEOUT"] = "3600" Utility.load_environment() +Utility.load_system_metadata() + +from kairon.shared.live_agent.live_agent import LiveAgentHandler + + + + import pytest import responses -from mock import patch +from unittest.mock import patch from mongoengine import connect from slack_sdk.web.slack_response import SlackResponse from starlette.exceptions import HTTPException diff --git a/tests/integration_test/event_service_test.py b/tests/integration_test/event_service_test.py index 445a37949..f03f65817 100644 --- a/tests/integration_test/event_service_test.py +++ b/tests/integration_test/event_service_test.py @@ -3,7 +3,7 @@ from dramatiq.brokers.stub import StubBroker from loguru import logger -from mock import patch +from unittest.mock import patch from starlette.testclient import TestClient from kairon.shared.constants import EventClass, EventExecutor diff --git a/tests/integration_test/history_services_test.py b/tests/integration_test/history_services_test.py index 2d8cc807a..46e683b43 100644 --- a/tests/integration_test/history_services_test.py +++ b/tests/integration_test/history_services_test.py @@ -8,7 +8,7 @@ from mongomock import MongoClient from kairon.history.processor import HistoryProcessor from pymongo.collection import Collection -import mock +from unittest import mock from urllib.parse import urlencode diff --git a/tests/integration_test/services_test.py b/tests/integration_test/services_test.py index 64f553c7f..363f02ec6 100644 --- a/tests/integration_test/services_test.py +++ b/tests/integration_test/services_test.py @@ -1,3 +1,5 @@ +import time + import ujson as json import os import re @@ -6,12 +8,14 @@ import tempfile from datetime import datetime, timedelta from io import BytesIO + +from unittest.mock import patch import yaml -from mock import patch + from urllib.parse import urljoin from zipfile import ZipFile -import mock +from unittest import mock import pytest import responses from botocore.exceptions import ClientError @@ -24,6 +28,9 @@ from pydantic import SecretStr from rasa.shared.utils.io import read_config_file from slack_sdk.web.slack_response import SlackResponse +from kairon.shared.utils import Utility, MailUtility + +Utility.load_system_metadata() from kairon.api.app.main import app from kairon.events.definitions.multilingual import MultilingualEvent @@ -45,6 +52,7 @@ KAIRON_TWO_STAGE_FALLBACK, FeatureMappings, DEFAULT_NLU_FALLBACK_RESPONSE, + DEFAULT_LLM ) from kairon.shared.data.data_objects import ( Stories, @@ -71,7 +79,6 @@ from kairon.shared.multilingual.utils.translator import Translator from kairon.shared.organization.processor import OrgProcessor from kairon.shared.sso.clients.google import GoogleSSO -from kairon.shared.utils import Utility, MailUtility from urllib.parse import urlencode from deepdiff import DeepDiff @@ -1629,7 +1636,6 @@ def test_get_live_agent_with_no_live_agent(): def test_enable_live_agent(): - bot_settings = BotSettings.objects(bot=pytest.bot).get() bot_settings.live_agent_enabled = True bot_settings.save() @@ -1652,7 +1658,6 @@ def test_enable_live_agent(): assert actual["success"] - def test_get_live_agent_after_enabled_no_bot_settings_enabled(): bot_settings = BotSettings.objects(bot=pytest.bot).get() bot_settings.live_agent_enabled = False @@ -1669,6 +1674,7 @@ def test_get_live_agent_after_enabled_no_bot_settings_enabled(): assert not actual["message"] assert actual["success"] + def test_get_live_agent_after_enabled(): bot_settings = BotSettings.objects(bot=pytest.bot).get() bot_settings.live_agent_enabled = True @@ -2438,7 +2444,9 @@ def _mock_get_bot_settings(*args, **kwargs): 'source': 'static', 'is_enabled': True}], 'instructions': ['Answer in a short manner.', 'Keep it simple.'], 'num_bot_responses': 5, - "failure_message": DEFAULT_NLU_FALLBACK_RESPONSE} + "failure_message": DEFAULT_NLU_FALLBACK_RESPONSE, + "llm_type": DEFAULT_LLM, + "hyperparameters": Utility.get_default_llm_hyperparameters()} response = client.post( f"/api/bot/{pytest.bot}/action/prompt", json=action, @@ -3175,6 +3183,8 @@ def _mock_get_bot_settings(*args, **kwargs): ], "num_bot_responses": 5, "failure_message": DEFAULT_NLU_FALLBACK_RESPONSE, + "llm_type": DEFAULT_LLM, + "hyperparameters": Utility.get_default_llm_hyperparameters() } response = client.post( f"/api/bot/{pytest.bot}/action/prompt", @@ -3232,6 +3242,8 @@ def _mock_get_bot_settings(*args, **kwargs): ], "num_bot_responses": 5, "failure_message": DEFAULT_NLU_FALLBACK_RESPONSE, + "llm_type": DEFAULT_LLM, + "hyperparameters": Utility.get_default_llm_hyperparameters() } response = client.post( f"/api/bot/{pytest.bot}/action/prompt", @@ -3276,6 +3288,8 @@ def test_add_prompt_action_with_invalid_query_prompt(): ], "num_bot_responses": 5, "failure_message": DEFAULT_NLU_FALLBACK_RESPONSE, + "llm_type": DEFAULT_LLM, + "hyperparameters": Utility.get_default_llm_hyperparameters() } response = client.post( f"/api/bot/{pytest.bot}/action/prompt", @@ -3333,6 +3347,8 @@ def test_add_prompt_action_with_invalid_num_bot_responses(): ], "failure_message": DEFAULT_NLU_FALLBACK_RESPONSE, "num_bot_responses": 10, + "llm_type": DEFAULT_LLM, + "hyperparameters": Utility.get_default_llm_hyperparameters() } response = client.post( f"/api/bot/{pytest.bot}/action/prompt", @@ -3389,7 +3405,9 @@ def test_add_prompt_action_with_invalid_system_prompt_source(): }, ], "num_bot_responses": 5, - "failure_message": DEFAULT_NLU_FALLBACK_RESPONSE + "failure_message": DEFAULT_NLU_FALLBACK_RESPONSE, + "llm_type": DEFAULT_LLM, + "hyperparameters": Utility.get_default_llm_hyperparameters() } response = client.post( f"/api/bot/{pytest.bot}/action/prompt", @@ -3454,7 +3472,9 @@ def test_add_prompt_action_with_multiple_system_prompt(): }, ], "num_bot_responses": 5, - "failure_message": DEFAULT_NLU_FALLBACK_RESPONSE + "failure_message": DEFAULT_NLU_FALLBACK_RESPONSE, + "llm_type": DEFAULT_LLM, + "hyperparameters": Utility.get_default_llm_hyperparameters() } response = client.post( f"/api/bot/{pytest.bot}/action/prompt", @@ -3511,7 +3531,9 @@ def test_add_prompt_action_with_empty_llm_prompt_name(): }, ], "num_bot_responses": 5, - "failure_message": DEFAULT_NLU_FALLBACK_RESPONSE + "failure_message": DEFAULT_NLU_FALLBACK_RESPONSE, + "llm_type": DEFAULT_LLM, + "hyperparameters": Utility.get_default_llm_hyperparameters() } response = client.post( f"/api/bot/{pytest.bot}/action/prompt", @@ -3568,7 +3590,9 @@ def test_add_prompt_action_with_empty_data_for_static_prompt(): }, ], "num_bot_responses": 5, - "failure_message": DEFAULT_NLU_FALLBACK_RESPONSE + "failure_message": DEFAULT_NLU_FALLBACK_RESPONSE, + "llm_type": DEFAULT_LLM, + "hyperparameters": Utility.get_default_llm_hyperparameters() } response = client.post( f"/api/bot/{pytest.bot}/action/prompt", @@ -3629,7 +3653,9 @@ def test_add_prompt_action_with_multiple_history_source_prompts(): }, ], "num_bot_responses": 5, - "failure_message": DEFAULT_NLU_FALLBACK_RESPONSE + "failure_message": DEFAULT_NLU_FALLBACK_RESPONSE, + "llm_type": DEFAULT_LLM, + "hyperparameters": Utility.get_default_llm_hyperparameters() } response = client.post( f"/api/bot/{pytest.bot}/action/prompt", @@ -3689,6 +3715,8 @@ def test_add_prompt_action_with_gpt_feature_disabled(): "failure_message": DEFAULT_NLU_FALLBACK_RESPONSE, "top_results": 10, "similarity_threshold": 0.70, + "llm_type": DEFAULT_LLM, + "hyperparameters": Utility.get_default_llm_hyperparameters() } response = client.post( f"/api/bot/{pytest.bot}/action/prompt", @@ -3705,6 +3733,138 @@ def test_add_prompt_action_with_gpt_feature_disabled(): assert actual["error_code"] == 422 +def test_add_prompt_action_with_invalid_llm_type(monkeypatch): + def _mock_get_bot_settings(*args, **kwargs): + return BotSettings( + bot=pytest.bot, + user="integration@demo.ai", + llm_settings=LLMSettings(enable_faq=True), + ) + + monkeypatch.setattr(MongoProcessor, "get_bot_settings", _mock_get_bot_settings) + action = { + "name": "test_add_prompt_action_with_invalid_llm_type", 'user_question': {'type': 'from_user_message'}, + "llm_prompts": [ + { + "name": "System Prompt", + "data": "You are a personal assistant.", + "type": "system", + "source": "static", + "is_enabled": True, + }, + { + "name": "Similarity Prompt", + "data": "Bot_collection", + "instructions": "Answer question based on the context above, if answer is not in the context go check previous logs.", + "type": "user", + "source": "bot_content", + "is_enabled": True, + }, + { + "name": "Query Prompt", + "data": "A programming language is a system of notation for writing computer programs.[1] Most programming languages are text-based formal languages, but they may also be graphical. They are a kind of computer language.", + "instructions": "Answer according to the context", + "type": "query", + "source": "static", + "is_enabled": True, + }, + { + "name": "Query Prompt", + "data": "If there is no specific query, assume that user is aking about java programming.", + "instructions": "Answer according to the context", + "type": "query", + "source": "static", + "is_enabled": True, + }, + ], + "instructions": ["Answer in a short manner.", "Keep it simple."], + "num_bot_responses": 5, + "failure_message": DEFAULT_NLU_FALLBACK_RESPONSE, + "llm_type": "test", + "hyperparameters": Utility.get_default_llm_hyperparameters() + } + response = client.post( + f"/api/bot/{pytest.bot}/action/prompt", + json=action, + headers={"Authorization": pytest.token_type + " " + pytest.access_token}, + ) + actual = response.json() + assert not DeepDiff(actual["message"], + [{'loc': ['body', 'llm_type'], 'msg': 'Invalid llm type', 'type': 'value_error'}], + ignore_order=True) + assert not actual["success"] + assert not actual["data"] + assert actual["error_code"] == 422 + + +def test_add_prompt_action_with_invalid_hyperameters(monkeypatch): + temp = Utility.get_default_llm_hyperparameters() + temp['temperature'] = 3.0 + + def _mock_get_bot_settings(*args, **kwargs): + return BotSettings( + bot=pytest.bot, + user="integration@demo.ai", + llm_settings=LLMSettings(enable_faq=True), + ) + + monkeypatch.setattr(MongoProcessor, "get_bot_settings", _mock_get_bot_settings) + action = { + "name": "test_add_prompt_action_with_invalid_hyperameters", 'user_question': {'type': 'from_user_message'}, + "llm_prompts": [ + { + "name": "System Prompt", + "data": "You are a personal assistant.", + "type": "system", + "source": "static", + "is_enabled": True, + }, + { + "name": "Similarity Prompt", + "data": "Bot_collection", + "instructions": "Answer question based on the context above, if answer is not in the context go check previous logs.", + "type": "user", + "source": "bot_content", + "is_enabled": True, + }, + { + "name": "Query Prompt", + "data": "A programming language is a system of notation for writing computer programs.[1] Most programming languages are text-based formal languages, but they may also be graphical. They are a kind of computer language.", + "instructions": "Answer according to the context", + "type": "query", + "source": "static", + "is_enabled": True, + }, + { + "name": "Query Prompt", + "data": "If there is no specific query, assume that user is aking about java programming.", + "instructions": "Answer according to the context", + "type": "query", + "source": "static", + "is_enabled": True, + }, + ], + "instructions": ["Answer in a short manner.", "Keep it simple."], + "num_bot_responses": 5, + "failure_message": DEFAULT_NLU_FALLBACK_RESPONSE, + "llm_type": DEFAULT_LLM, + "hyperparameters": temp + } + response = client.post( + f"/api/bot/{pytest.bot}/action/prompt", + json=action, + headers={"Authorization": pytest.token_type + " " + pytest.access_token}, + ) + actual = response.json() + assert not DeepDiff(actual["message"], + [{'loc': ['body', 'hyperparameters'], + 'msg': "['temperature']: 3.0 is greater than the maximum of 2.0", 'type': 'value_error'}], + ignore_order=True) + assert not actual["success"] + assert not actual["data"] + assert actual["error_code"] == 422 + + def test_add_prompt_action(monkeypatch): def _mock_get_bot_settings(*args, **kwargs): return BotSettings( @@ -3751,7 +3911,9 @@ def _mock_get_bot_settings(*args, **kwargs): ], "instructions": ["Answer in a short manner.", "Keep it simple."], "num_bot_responses": 5, - "failure_message": DEFAULT_NLU_FALLBACK_RESPONSE + "failure_message": DEFAULT_NLU_FALLBACK_RESPONSE, + "llm_type": DEFAULT_LLM, + "hyperparameters": Utility.get_default_llm_hyperparameters() } response = client.post( f"/api/bot/{pytest.bot}/action/prompt", @@ -3812,7 +3974,9 @@ def _mock_get_bot_settings(*args, **kwargs): ], "instructions": ["Answer in a short manner.", "Keep it simple."], "num_bot_responses": 5, - "failure_message": DEFAULT_NLU_FALLBACK_RESPONSE + "failure_message": DEFAULT_NLU_FALLBACK_RESPONSE, + "llm_type": DEFAULT_LLM, + "hyperparameters": Utility.get_default_llm_hyperparameters() } response = client.post( f"/api/bot/{pytest.bot}/action/prompt", @@ -3863,7 +4027,9 @@ def test_update_prompt_action_does_not_exist(): }, ], "num_bot_responses": 5, - "failure_message": DEFAULT_NLU_FALLBACK_RESPONSE + "failure_message": DEFAULT_NLU_FALLBACK_RESPONSE, + "llm_type": DEFAULT_LLM, + "hyperparameters": Utility.get_default_llm_hyperparameters() } response = client.put( f"/api/bot/{pytest.bot}/action/prompt/61512cc2c6219f0aae7bba3d", @@ -3915,7 +4081,9 @@ def test_update_prompt_action_with_invalid_similarity_threshold(): }, ], "num_bot_responses": 5, - "failure_message": "updated_failure_message" + "failure_message": "updated_failure_message", + "llm_type": DEFAULT_LLM, + "hyperparameters": Utility.get_default_llm_hyperparameters() } response = client.put( f"/api/bot/{pytest.bot}/action/prompt/{pytest.action_id}", @@ -3960,7 +4128,9 @@ def test_update_prompt_action_with_invalid_top_results(): }, ], "num_bot_responses": 5, - "failure_message": "updated_failure_message" + "failure_message": "updated_failure_message", + "llm_type": DEFAULT_LLM, + "hyperparameters": Utility.get_default_llm_hyperparameters() } response = client.put( f"/api/bot/{pytest.bot}/action/prompt/{pytest.action_id}", @@ -4004,7 +4174,9 @@ def test_update_prompt_action_with_invalid_num_bot_responses(): }, ], "num_bot_responses": 50, - "failure_message": "updated_failure_message" + "failure_message": "updated_failure_message", + "llm_type": DEFAULT_LLM, + "hyperparameters": Utility.get_default_llm_hyperparameters() } response = client.put( f"/api/bot/{pytest.bot}/action/prompt/{pytest.action_id}", @@ -4053,6 +4225,8 @@ def test_update_prompt_action_with_invalid_query_prompt(): "num_bot_responses": 5, "use_query_prompt": True, "query_prompt": "", + "llm_type": DEFAULT_LLM, + "hyperparameters": Utility.get_default_llm_hyperparameters() } response = client.put( f"/api/bot/{pytest.bot}/action/prompt/{pytest.action_id}", @@ -4110,6 +4284,8 @@ def test_update_prompt_action_with_query_prompt_with_false(): }, ], "dispatch_response": False, + "llm_type": DEFAULT_LLM, + "hyperparameters": Utility.get_default_llm_hyperparameters() } response = client.put( f"/api/bot/{pytest.bot}/action/prompt/{pytest.action_id}", @@ -4166,7 +4342,9 @@ def test_update_prompt_action(): ], "instructions": ["Answer in a short manner.", "Keep it simple."], "num_bot_responses": 5, - "failure_message": "updated_failure_message" + "failure_message": "updated_failure_message", + "llm_type": DEFAULT_LLM, + "hyperparameters": Utility.get_default_llm_hyperparameters() } response = client.put( f"/api/bot/{pytest.bot}/action/prompt/{pytest.action_id}", @@ -4191,11 +4369,12 @@ def test_get_prompt_action(): assert actual["error_code"] == 0 assert not actual["message"] actual["data"][0].pop("_id") - assert actual["data"] == [ + assert not DeepDiff(actual["data"], [ {'name': 'test_update_prompt_action', 'num_bot_responses': 5, 'failure_message': 'updated_failure_message', 'user_question': {'type': 'from_slot', 'value': 'prompt_question'}, + 'llm_type': 'openai', 'hyperparameters': {'temperature': 0.0, 'max_tokens': 300, 'model': 'gpt-3.5-turbo', 'top_p': 0.0, 'n': 1, - 'stream': False, 'stop': None, 'presence_penalty': 0.0, 'frequency_penalty': 0.0, + 'stop': None, 'presence_penalty': 0.0, 'frequency_penalty': 0.0, 'logit_bias': {}}, 'llm_prompts': [ {'name': 'System Prompt', 'data': 'You are a personal assistant.', 'type': 'system', 'source': 'static', 'is_enabled': True}, {'name': 'Similarity_analytical Prompt', 'data': 'Bot_collection', @@ -4209,7 +4388,7 @@ def test_get_prompt_action(): 'instructions': 'Answer according to the context', 'type': 'query', 'source': 'static', 'is_enabled': True}], 'instructions': ['Answer in a short manner.', 'Keep it simple.'], 'set_slots': [], 'dispatch_response': True, - 'status': True}] + 'status': True}], ignore_order=True) def test_add_prompt_action_with_empty_collection_for_bot_content_prompt(monkeypatch): @@ -4259,7 +4438,9 @@ def _mock_get_bot_settings(*args, **kwargs): ], "instructions": ["Answer in a short manner.", "Keep it simple."], "num_bot_responses": 5, - "failure_message": DEFAULT_NLU_FALLBACK_RESPONSE + "failure_message": DEFAULT_NLU_FALLBACK_RESPONSE, + "llm_type": DEFAULT_LLM, + "hyperparameters": Utility.get_default_llm_hyperparameters() } response = client.post( f"/api/bot/{pytest.bot}/action/prompt", @@ -4282,12 +4463,13 @@ def _mock_get_bot_settings(*args, **kwargs): assert actual["error_code"] == 0 assert not actual["message"] actual["data"][1].pop("_id") - assert actual["data"][1] == { + assert not DeepDiff(actual["data"][1], { 'name': 'test_add_prompt_action_with_empty_collection_for_bot_content_prompt', 'num_bot_responses': 5, 'failure_message': "I'm sorry, I didn't quite understand that. Could you rephrase?", 'user_question': {'type': 'from_user_message'}, + 'llm_type': 'openai', 'hyperparameters': {'temperature': 0.0, 'max_tokens': 300, 'model': 'gpt-3.5-turbo', 'top_p': 0.0, 'n': 1, - 'stream': False, 'stop': None, 'presence_penalty': 0.0, 'frequency_penalty': 0.0, + 'stop': None, 'presence_penalty': 0.0, 'frequency_penalty': 0.0, 'logit_bias': {}}, 'llm_prompts': [{'name': 'System Prompt', 'data': 'You are a personal assistant.', 'type': 'system', 'source': 'static', 'is_enabled': True}, @@ -4303,7 +4485,7 @@ def _mock_get_bot_settings(*args, **kwargs): 'instructions': 'Answer according to the context', 'type': 'query', 'source': 'static', 'is_enabled': True}], 'instructions': ['Answer in a short manner.', 'Keep it simple.'], 'set_slots': [], - 'dispatch_response': True, 'status': True} + 'dispatch_response': True, 'status': True}, ignore_order=True) def test_add_prompt_action_with_bot_content_prompt_with_payload(monkeypatch): @@ -4356,7 +4538,10 @@ def _mock_get_bot_settings(*args, **kwargs): 'source': 'static', 'is_enabled': True}], 'instructions': ['Answer in a short manner.', 'Keep it simple.'], 'num_bot_responses': 5, - "failure_message": DEFAULT_NLU_FALLBACK_RESPONSE} + "failure_message": DEFAULT_NLU_FALLBACK_RESPONSE, + "llm_type": DEFAULT_LLM, + "hyperparameters": Utility.get_default_llm_hyperparameters() + } response = client.post( f"/api/bot/{pytest.bot}/action/prompt", json=action, @@ -4370,12 +4555,13 @@ def _mock_get_bot_settings(*args, **kwargs): ) actual = response.json() actual["data"][2].pop("_id") - assert actual["data"][2] == { + assert not DeepDiff(actual["data"][2], { 'name': 'test_add_prompt_action_with_bot_content_prompt_with_payload', 'num_bot_responses': 5, 'failure_message': "I'm sorry, I didn't quite understand that. Could you rephrase?", 'user_question': {'type': 'from_user_message'}, + 'llm_type': 'openai', 'hyperparameters': {'temperature': 0.0, 'max_tokens': 300, 'model': 'gpt-3.5-turbo', 'top_p': 0.0, 'n': 1, - 'stream': False, 'stop': None, 'presence_penalty': 0.0, + 'stop': None, 'presence_penalty': 0.0, 'frequency_penalty': 0.0, 'logit_bias': {}}, 'llm_prompts': [{'name': 'System Prompt', 'data': 'You are a personal assistant.', 'type': 'system', 'source': 'static', 'is_enabled': True}, @@ -4390,7 +4576,7 @@ def _mock_get_bot_settings(*args, **kwargs): 'data': 'If there is no specific query, assume that user is aking about java programming.', 'instructions': 'Answer according to the context', 'type': 'query', 'source': 'static', 'is_enabled': True}], 'instructions': ['Answer in a short manner.', 'Keep it simple.'], - 'set_slots': [], 'dispatch_response': True, 'status': True} + 'set_slots': [], 'dispatch_response': True, 'status': True}, ignore_order=True) assert actual["success"] assert actual["error_code"] == 0 assert not actual["message"] @@ -4447,7 +4633,10 @@ def _mock_get_bot_settings(*args, **kwargs): 'source': 'static', 'is_enabled': True}], 'instructions': ['Answer in a short manner.', 'Keep it simple.'], 'num_bot_responses': 5, - "failure_message": DEFAULT_NLU_FALLBACK_RESPONSE} + "failure_message": DEFAULT_NLU_FALLBACK_RESPONSE, + "llm_type": DEFAULT_LLM, + "hyperparameters": Utility.get_default_llm_hyperparameters() + } response = client.post( f"/api/bot/{pytest.bot}/action/prompt", json=action, @@ -4461,12 +4650,13 @@ def _mock_get_bot_settings(*args, **kwargs): ) actual = response.json() actual["data"][3].pop("_id") - assert actual["data"][3] == { + assert not DeepDiff(actual["data"][3], { 'name': 'test_add_prompt_action_with_bot_content_prompt_with_content', 'num_bot_responses': 5, 'failure_message': "I'm sorry, I didn't quite understand that. Could you rephrase?", 'user_question': {'type': 'from_user_message'}, + 'llm_type': 'openai', 'hyperparameters': {'temperature': 0.0, 'max_tokens': 300, 'model': 'gpt-3.5-turbo', 'top_p': 0.0, 'n': 1, - 'stream': False, 'stop': None, 'presence_penalty': 0.0, 'frequency_penalty': 0.0, + 'stop': None, 'presence_penalty': 0.0, 'frequency_penalty': 0.0, 'logit_bias': {}}, 'llm_prompts': [{'name': 'System Prompt', 'data': 'You are a personal assistant.', 'type': 'system', 'source': 'static', 'is_enabled': True}, @@ -4482,7 +4672,7 @@ def _mock_get_bot_settings(*args, **kwargs): 'instructions': 'Answer according to the context', 'type': 'query', 'source': 'static', 'is_enabled': True}], 'instructions': ['Answer in a short manner.', 'Keep it simple.'], 'set_slots': [], - 'dispatch_response': True, 'status': True} + 'dispatch_response': True, 'status': True}, ignore_order=True) assert actual["success"] assert actual["error_code"] == 0 assert not actual["message"] @@ -5121,7 +5311,8 @@ def test_get_data_importer_logs(): assert actual['data']["logs"][3]['event_status'] == EVENT_STATUS.COMPLETED.value assert actual['data']["logs"][3]['status'] == 'Failure' assert set(actual['data']["logs"][3]['files_received']) == {'rules', 'stories', 'nlu', 'domain', 'config', - 'actions', 'chat_client_config', 'multiflow_stories', 'bot_content'} + 'actions', 'chat_client_config', 'multiflow_stories', + 'bot_content'} assert actual['data']["logs"][3]['is_data_uploaded'] assert actual['data']["logs"][3]['start_timestamp'] assert actual['data']["logs"][3]['end_timestamp'] @@ -5153,7 +5344,8 @@ def test_get_data_importer_logs(): ] assert actual['data']["logs"][3]['is_data_uploaded'] assert set(actual['data']["logs"][3]['files_received']) == {'rules', 'stories', 'nlu', 'config', 'domain', - 'actions', 'chat_client_config', 'multiflow_stories','bot_content'} + 'actions', 'chat_client_config', 'multiflow_stories', + 'bot_content'} @responses.activate @@ -6229,6 +6421,7 @@ def test_add_story_lone_intent(): } ] + def test_add_story_consecutive_intents(): response = client.post( f"/api/bot/{pytest.bot}/stories", @@ -10523,6 +10716,7 @@ def test_login_for_verified(): pytest.access_token = actual["data"]["access_token"] pytest.token_type = actual["data"]["token_type"] + def test_list_bots_for_different_user(): response = client.get( "/api/account/bot", @@ -19641,7 +19835,7 @@ def test_set_templates_with_sysadmin_as_user(): intents = Intents.objects(bot=pytest.bot) intents = [{k: v for k, v in intent.to_mongo().to_dict().items() if k not in ['_id', 'bot', 'timestamp']} for - intent in intents] + intent in intents] assert intents == [ {'name': 'greet', 'user': 'sysadmin', 'status': True, 'is_integration': False, 'use_entities': False}, @@ -19717,7 +19911,6 @@ def test_add_channel_config(monkeypatch): def test_add_bot_with_template_with_sysadmin_as_user(monkeypatch): - def mock_reload_model(*args, **kwargs): mock_reload_model.called_with = (args, kwargs) return None @@ -19756,7 +19949,7 @@ def mock_reload_model(*args, **kwargs): rules = Rules.objects(bot=bot_id) rules = [{k: v for k, v in rule.to_mongo().to_dict().items() if k not in ['_id', 'bot', 'timestamp']} for - rule in rules] + rule in rules] assert rules == [ {'block_name': 'ask the user to rephrase whenever they send a message with low nlu confidence', @@ -19770,7 +19963,7 @@ def mock_reload_model(*args, **kwargs): utterances = Utterances.objects(bot=bot_id) utterances = [{k: v for k, v in utterance.to_mongo().to_dict().items() if k not in ['_id', 'bot', 'timestamp']} for - utterance in utterances] + utterance in utterances] assert utterances == [ {'name': 'utter_please_rephrase', 'user': 'sysadmin', 'status': True}, @@ -20885,7 +21078,7 @@ def test_get_bot_settings(): 'whatsapp': 'meta', 'cognition_collections_limit': 3, 'cognition_columns_per_collection_limit': 5, - 'integrations_per_user_limit':3 } + 'integrations_per_user_limit': 3} def test_update_analytics_settings_with_empty_value(): @@ -20963,7 +21156,7 @@ def test_update_analytics_settings(): 'live_agent_enabled': False, 'cognition_collections_limit': 3, 'cognition_columns_per_collection_limit': 5, - 'integrations_per_user_limit':3 } + 'integrations_per_user_limit': 3} def test_delete_channels_config(): @@ -24114,6 +24307,47 @@ def test_trigger_widget(): assert actual["error_code"] == 0 assert len(actual["data"]) == 2 assert not actual["message"] + + +def test_get_llm_logs(): + from kairon.shared.llm.logger import LiteLLMLogger + import litellm + import asyncio + + loop = asyncio.new_event_loop() + user = "test" + litellm.callbacks = [LiteLLMLogger()] + + messages = [{"role": "user", "content": "Hi"}] + expected = "Hi, How may i help you?" + + result = loop.run_until_complete(litellm.acompletion(messages=messages, + model="gpt-3.5-turbo", + mock_response=expected, + metadata={'user': user, 'bot': pytest.bot})) + assert result['choices'][0]['message']['content'] == expected + + time.sleep(2) + + response = client.get( + f"/api/bot/{pytest.bot}/llm/logs?start_idx=0&page_size=10", + headers={"Authorization": pytest.token_type + " " + pytest.access_token}, + ) + actual = response.json() + print(actual) + assert actual["success"] + assert actual["error_code"] == 0 + assert len(actual["data"]["logs"]) == 1 + assert actual["data"]["total"] == 1 + assert actual["data"]["logs"][0]['start_time'] + assert actual["data"]["logs"][0]['end_time'] + assert actual["data"]["logs"][0]['cost'] + assert actual["data"]["logs"][0]['llm_call_id'] + assert actual["data"]["logs"][0]["llm_provider"] == "openai" + assert not actual["data"]["logs"][0].get("model") + assert actual["data"]["logs"][0]["model_params"] == {} + assert actual["data"]["logs"][0]["metadata"]['bot'] == pytest.bot + assert actual["data"]["logs"][0]["metadata"]['user'] == "test" def test_add_custom_widget_invalid_config(): @@ -25013,4 +25247,4 @@ def test_list_system_metadata(): actual = response.json() assert actual["error_code"] == 0 assert actual["success"] - assert len(actual["data"]) == 17 + assert len(actual["data"]) == 17 \ No newline at end of file diff --git a/tests/unit_test/action/action_test.py b/tests/unit_test/action/action_test.py index d40eb9eb4..221d6d1aa 100644 --- a/tests/unit_test/action/action_test.py +++ b/tests/unit_test/action/action_test.py @@ -3,9 +3,10 @@ import re from unittest import mock -import mock from googleapiclient.http import HttpRequest from pipedrive.exceptions import UnauthorizedError, BadRequestError +from kairon.shared.utils import Utility +Utility.load_system_metadata() from kairon.actions.definitions.email import ActionEmail from kairon.actions.definitions.factory import ActionFactory @@ -42,10 +43,10 @@ from kairon.actions.handlers.processor import ActionProcessor from kairon.shared.actions.utils import ActionUtility from kairon.shared.actions.exception import ActionFailure -from kairon.shared.utils import Utility from unittest.mock import patch from urllib.parse import urlencode + class TestActions: @pytest.fixture(autouse=True, scope='class') @@ -2660,9 +2661,10 @@ def test_get_prompt_action_config(self): 'failure_message': "I'm sorry, I didn't quite understand that. Could you rephrase?", 'user_question': {'type': 'from_user_message'}, 'bot': 'test_action_server', 'hyperparameters': {'temperature': 0.0, 'max_tokens': 300, 'model': 'gpt-3.5-turbo', - 'top_p': 0.0, 'n': 1, 'stream': False, 'stop': None, + 'top_p': 0.0, 'n': 1, 'stop': None, 'presence_penalty': 0.0, 'frequency_penalty': 0.0, 'logit_bias': {}}, + 'llm_type': 'openai', 'llm_prompts': [{'name': 'System Prompt', 'data': 'You are a personal assistant.', 'type': 'system', 'source': 'static', 'is_enabled': True}, {'name': 'History Prompt', 'type': 'user', @@ -3951,9 +3953,10 @@ def test_get_prompt_action_config_2(self): 'failure_message': "I'm sorry, I didn't quite understand that. Could you rephrase?", 'bot': 'test_bot_action_test', 'user': 'test_user_action_test', 'hyperparameters': {'temperature': 0.0, 'max_tokens': 300, 'model': 'gpt-3.5-turbo', - 'top_p': 0.0, 'n': 1, 'stream': False, 'stop': None, + 'top_p': 0.0, 'n': 1, 'stop': None, 'presence_penalty': 0.0, 'frequency_penalty': 0.0, 'logit_bias': {}}, 'dispatch_response': True, 'set_slots': [], + 'llm_type': 'openai', 'llm_prompts': [{'name': 'System Prompt', 'data': 'You are a personal assistant.', 'type': 'system', 'source': 'static', 'is_enabled': True}, {'name': 'History Prompt', 'type': 'user', diff --git a/tests/unit_test/api/api_processor_test.py b/tests/unit_test/api/api_processor_test.py index 34a46ad81..363272358 100644 --- a/tests/unit_test/api/api_processor_test.py +++ b/tests/unit_test/api/api_processor_test.py @@ -7,6 +7,8 @@ from unittest import mock from unittest.mock import patch from urllib.parse import urljoin +from kairon.shared.utils import Utility, MailUtility +Utility.load_system_metadata() import jwt import pytest @@ -22,7 +24,6 @@ from starlette.requests import Request from starlette.responses import RedirectResponse -from kairon.api.app.routers.idp import get_idp_config from kairon.api.models import RegisterAccount, EventConfig, IDPConfig, StoryRequest, HttpActionParameters, Password from kairon.exceptions import AppException from kairon.idp.data_objects import IdpConfig @@ -42,12 +43,6 @@ from kairon.shared.organization.processor import OrgProcessor from kairon.shared.sso.clients.facebook import FacebookSSO from kairon.shared.sso.clients.google import GoogleSSO -from kairon.shared.utils import Utility, MailUtility -from kairon.exceptions import AppException -import time -from kairon.idp.data_objects import IdpConfig -from kairon.api.models import RegisterAccount, EventConfig, IDPConfig, StoryRequest, HttpActionParameters, Password -from mongomock import MongoClient os.environ["system_file"] = "./tests/testing_data/system.yaml" diff --git a/tests/unit_test/augmentation/gpt_augmentation_test.py b/tests/unit_test/augmentation/gpt_augmentation_test.py index dfdb7d42e..e743fb49c 100644 --- a/tests/unit_test/augmentation/gpt_augmentation_test.py +++ b/tests/unit_test/augmentation/gpt_augmentation_test.py @@ -1,7 +1,7 @@ from augmentation.paraphrase.gpt3.generator import GPT3ParaphraseGenerator from augmentation.paraphrase.gpt3.models import GPTRequest from augmentation.paraphrase.gpt3.gpt import GPT -import openai +from openai.resources.completions import Completions import pytest import responses @@ -61,7 +61,7 @@ def test_questions_set_generation(monkeypatch): def test_generate_questions(monkeypatch): - monkeypatch.setattr(openai.Completion, 'create', mock_create) + monkeypatch.setattr(Completions, 'create', mock_create) request_data = GPTRequest(api_key="MockKey", data=["Are there any more test questions?"], num_responses=2) @@ -73,7 +73,7 @@ def test_generate_questions(monkeypatch): def test_generate_questions_empty_api_key(monkeypatch): - monkeypatch.setattr(openai.Completion, 'create', mock_create) + monkeypatch.setattr(Completions, 'create', mock_create) request_data = GPTRequest(api_key="", data=["Are there any more test questions?"], num_responses=2) @@ -84,7 +84,7 @@ def test_generate_questions_empty_api_key(monkeypatch): def test_generate_questions_empty_data(monkeypatch): - monkeypatch.setattr(openai.Completion, 'create', mock_create) + monkeypatch.setattr(Completions, 'create', mock_create) request_data = GPTRequest(api_key="MockKey", data=[], num_responses=2) @@ -125,6 +125,6 @@ def test_generate_questions_invalid_api_key(): data=["Are there any more test questions?"], num_responses=2) gpt3_generator = GPT3ParaphraseGenerator(request_data=request_data) - with pytest.raises(APIError, match=r'.*Incorrect API key provided: InvalidKey. You can find your API key at https://beta.openai.com..*'): + with pytest.raises(APIError, match=r'.*Incorrect API key provided: InvalidKey. You can find your API key at https://platform.openai.com/account/..*'): gpt3_generator.paraphrases() diff --git a/tests/unit_test/chat/chat_test.py b/tests/unit_test/chat/chat_test.py index 9abbcab6b..6336152bb 100644 --- a/tests/unit_test/chat/chat_test.py +++ b/tests/unit_test/chat/chat_test.py @@ -3,7 +3,7 @@ import ujson as json import os from re import escape -from unittest.mock import patch +from unittest import mock from urllib.parse import urlencode, quote_plus import mongomock @@ -21,7 +21,6 @@ from kairon.shared.data.constant import ACCESS_ROLES, TOKEN_TYPE from kairon.shared.data.utils import DataUtility from kairon.shared.utils import Utility -import mock from pymongo.errors import ServerSelectionTimeoutError @@ -49,7 +48,7 @@ def test_save_channel_config_invalid(self): "test", "test" ) - with patch("slack_sdk.web.client.WebClient.team_info") as mock_slack_resp: + with mock.patch("slack_sdk.web.client.WebClient.team_info") as mock_slack_resp: mock_slack_resp.return_value = SlackResponse( client=self, http_verb="POST", @@ -93,7 +92,7 @@ def test_save_channel_config_invalid(self): "test" ) - @patch("kairon.shared.utils.Utility.get_slack_team_info", autospec=True) + @mock.patch("kairon.shared.utils.Utility.get_slack_team_info", autospec=True) def test_save_channel_config_slack_team_id_error(self, mock_slack_info): mock_slack_info.side_effect = AppException("The request to the Slack API failed. ") with pytest.raises(AppException, match="The request to the Slack API failed.*"): @@ -108,7 +107,7 @@ def __mock_get_bot(*args, **kwargs): return {"account": 1000} monkeypatch.setattr(AccountProcessor, "get_bot", __mock_get_bot) - with patch("slack_sdk.web.client.WebClient.team_info") as mock_slack_resp: + with mock.patch("slack_sdk.web.client.WebClient.team_info") as mock_slack_resp: mock_slack_resp.return_value = SlackResponse( client=self, http_verb="POST", @@ -133,7 +132,7 @@ def __mock_get_bot(*args, **kwargs): "client_secret": "a23456789sfdghhtyutryuivcbn", "is_primary": True}}, "test", "test" ) - @patch("kairon.shared.utils.Utility.get_slack_team_info", autospec=True) + @mock.patch("kairon.shared.utils.Utility.get_slack_team_info", autospec=True) def test_save_channel_config_slack_secondary_app_team_id_error(self, mock_slack_info ): mock_slack_info.side_effect = AppException("The request to the Slack API failed. ") with pytest.raises(AppException, match="The request to the Slack API failed.*"): @@ -149,7 +148,7 @@ def __mock_get_bot(*args, **kwargs): return {"account": 1000} monkeypatch.setattr(AccountProcessor, "get_bot", __mock_get_bot) - with patch("slack_sdk.web.client.WebClient.team_info") as mock_slack_resp: + with mock.patch("slack_sdk.web.client.WebClient.team_info") as mock_slack_resp: mock_slack_resp.return_value = SlackResponse( client=self, http_verb="POST", @@ -252,7 +251,7 @@ def __mock_get_bot(*args, **kwargs): return {"account": 1000} monkeypatch.setattr(AccountProcessor, "get_bot", __mock_get_bot) - with patch("slack_sdk.web.client.WebClient.team_info") as mock_slack_resp: + with mock.patch("slack_sdk.web.client.WebClient.team_info") as mock_slack_resp: mock_slack_resp.return_value = SlackResponse( client=self, http_verb="POST", @@ -317,7 +316,7 @@ def __mock_get_bot(*args, **kwargs): return {"account": 1000} monkeypatch.setattr(AccountProcessor, "get_bot", __mock_get_bot) - with patch("slack_sdk.web.client.WebClient.team_info") as mock_slack_resp: + with mock.patch("slack_sdk.web.client.WebClient.team_info") as mock_slack_resp: mock_slack_resp.return_value = SlackResponse( client=self, http_verb="POST", @@ -384,7 +383,7 @@ def test_save_channel_config_telegram(self): def __mock_endpoint(*args): return f"https://test@test.com/api/bot/telegram/tests/test" - with patch('kairon.shared.data.utils.DataUtility.get_channel_endpoint', __mock_endpoint): + with mock.patch('kairon.shared.data.utils.DataUtility.get_channel_endpoint', __mock_endpoint): ChatDataProcessor.save_channel_config({"connector_type": "telegram", "config": { "access_token": access_token, @@ -405,7 +404,7 @@ def test_save_channel_config_telegram_invalid(self): def __mock_endpoint(*args): return f"https://test@test.com/api/bot/telegram/tests/test" - with patch('kairon.shared.data.utils.DataUtility.get_channel_endpoint', __mock_endpoint): + with mock.patch('kairon.shared.data.utils.DataUtility.get_channel_endpoint', __mock_endpoint): ChatDataProcessor.save_channel_config({"connector_type": "telegram", "config": { "access_token": access_token, @@ -487,7 +486,7 @@ def test_save_channel_config_business_messages_with_invalid_private_key(self): def __mock_endpoint(*args): return f"https://test@test.com/api/bot/business_messages/tests/test" - with patch('kairon.shared.data.utils.DataUtility.get_channel_endpoint', __mock_endpoint): + with mock.patch('kairon.shared.data.utils.DataUtility.get_channel_endpoint', __mock_endpoint): channel_endpoint = ChatDataProcessor.save_channel_config( { "connector_type": "business_messages", @@ -515,7 +514,7 @@ def test_save_channel_config_business_messages(self): def __mock_endpoint(*args): return f"https://test@test.com/api/bot/business_messages/tests/test" - with patch('kairon.shared.data.utils.DataUtility.get_channel_endpoint', __mock_endpoint): + with mock.patch('kairon.shared.data.utils.DataUtility.get_channel_endpoint', __mock_endpoint): channel_endpoint = ChatDataProcessor.save_channel_config( { "connector_type": "business_messages", @@ -603,7 +602,7 @@ def test_get_channel_end_point_whatsapp(self, monkeypatch): def _mock_generate_integration_token(*arge, **kwargs): return "testtoken", "ignore" - with patch.object(Authentication, "generate_integration_token", _mock_generate_integration_token): + with mock.patch.object(Authentication, "generate_integration_token", _mock_generate_integration_token): channel_url = ChatDataProcessor.save_channel_config({ "connector_type": "whatsapp", "config": { "app_secret": "app123", diff --git a/tests/unit_test/cli_test.py b/tests/unit_test/cli_test.py index 80f1e6bf2..1f28f1fd2 100644 --- a/tests/unit_test/cli_test.py +++ b/tests/unit_test/cli_test.py @@ -1,8 +1,10 @@ +import argparse +import os from datetime import datetime -from unittest.mock import patch +from unittest import mock import pytest -import os +from mongoengine import connect from kairon import cli from kairon.cli.conversations_deletion import initiate_history_deletion_archival @@ -10,22 +12,17 @@ from kairon.cli.delete_logs import delete_logs from kairon.cli.importer import validate_and_import from kairon.cli.message_broadcast import send_notifications -from kairon.cli.training import train from kairon.cli.testing import run_tests_on_model +from kairon.cli.training import train from kairon.cli.translator import translate_multilingual_bot from kairon.events.definitions.data_generator import DataGenerationEvent from kairon.events.definitions.data_importer import TrainingDataImporterEvent from kairon.events.definitions.history_delete import DeleteHistoryEvent -from kairon.events.definitions.message_broadcast import MessageBroadcastEvent from kairon.events.definitions.model_testing import ModelTestingEvent from kairon.events.definitions.multilingual import MultilingualEvent from kairon.shared.concurrency.actors.factory import ActorFactory -from kairon.shared.utils import Utility -from mongoengine import connect -import mock -import argparse - from kairon.shared.constants import EventClass +from kairon.shared.utils import Utility class TestTrainingCli: @@ -395,7 +392,7 @@ def test_message_broadcast_no_event_id(self, monkeypatch): return_value=argparse.Namespace(func=send_notifications, bot="test_cli", user="testUser", event_id="65432123456789876543")) def test_message_broadcast_all_arguments(self, mock_namespace): - with patch('kairon.events.definitions.message_broadcast.MessageBroadcastEvent.execute', autospec=True): + with mock.patch('kairon.events.definitions.message_broadcast.MessageBroadcastEvent.execute', autospec=True): cli() for proxy in ActorFactory._ActorFactory__actors.values(): diff --git a/tests/unit_test/data_processor/agent_processor_test.py b/tests/unit_test/data_processor/agent_processor_test.py index ec62c2dec..8d37b1541 100644 --- a/tests/unit_test/data_processor/agent_processor_test.py +++ b/tests/unit_test/data_processor/agent_processor_test.py @@ -16,7 +16,7 @@ from kairon.shared.data.constant import EVENT_STATUS from kairon.shared.data.model_processor import ModelProcessor -from mock import patch +from unittest.mock import patch from mongoengine import connect diff --git a/tests/unit_test/data_processor/data_processor_test.py b/tests/unit_test/data_processor/data_processor_test.py index 0e1557969..abf591ae0 100644 --- a/tests/unit_test/data_processor/data_processor_test.py +++ b/tests/unit_test/data_processor/data_processor_test.py @@ -8,8 +8,13 @@ from datetime import datetime, timedelta, timezone from io import BytesIO from typing import List +from kairon.shared.utils import Utility +os.environ["system_file"] = "./tests/testing_data/system.yaml" +Utility.load_environment() +Utility.load_system_metadata() -from mock import patch + +from unittest.mock import patch import numpy as np import pandas as pd import pytest @@ -55,7 +60,7 @@ from kairon.shared.data.constant import UTTERANCE_TYPE, EVENT_STATUS, STORY_EVENT, ALLOWED_DOMAIN_FORMATS, \ ALLOWED_CONFIG_FORMATS, ALLOWED_NLU_FORMATS, ALLOWED_STORIES_FORMATS, ALLOWED_RULES_FORMATS, REQUIREMENTS, \ DEFAULT_NLU_FALLBACK_RULE, SLOT_TYPE, KAIRON_TWO_STAGE_FALLBACK, AuditlogActions, TOKEN_TYPE, GPT_LLM_FAQ, \ - DEFAULT_CONTEXT_PROMPT, DEFAULT_NLU_FALLBACK_RESPONSE, DEFAULT_SYSTEM_PROMPT + DEFAULT_CONTEXT_PROMPT, DEFAULT_NLU_FALLBACK_RESPONSE, DEFAULT_SYSTEM_PROMPT, DEFAULT_LLM from kairon.shared.data.data_objects import (TrainingExamples, Slots, Entities, EntitySynonyms, RegexFeatures, @@ -67,8 +72,7 @@ Utterances, BotSettings, ChatClientConfig, LookupTables, Forms, SlotMapping, KeyVault, MultiflowStories, LLMSettings, MultiflowStoryEvents, Synonyms, - Lookup, - DemoRequestLogs + Lookup ) from kairon.shared.data.history_log_processor import HistoryDeletionLogProcessor from kairon.shared.data.model_processor import ModelProcessor @@ -77,19 +81,15 @@ from kairon.shared.data.utils import DataUtility from kairon.shared.importer.processor import DataImporterLogProcessor from kairon.shared.live_agent.live_agent import LiveAgentHandler -from kairon.shared.llm.gpt3 import GPT3FAQEmbedding from kairon.shared.metering.constants import MetricType from kairon.shared.metering.data_object import Metering from kairon.shared.models import StoryEventType, HttpContentType, CognitionDataType from kairon.shared.multilingual.processor import MultilingualLogProcessor from kairon.shared.test.data_objects import ModelTestingLogs from kairon.shared.test.processor import ModelTestingLogProcessor -from kairon.shared.utils import Utility from kairon.train import train_model_for_bot, start_training - -os.environ["system_file"] = "./tests/testing_data/system.yaml" -Utility.load_environment() from deepdiff import DeepDiff +import litellm class TestMongoProcessor: @@ -152,50 +152,6 @@ def test_add_complex_story_with_slot(self): {'name': 'persona', 'type': 'SLOT', 'value': 'positive'}, {'name': 'utter_welcome_user', 'type': 'BOT'}] - def test_add_demo_request_with_empty_first_name(self): - processor = MongoProcessor() - processor.add_demo_request( - first_name="", last_name="Sattala", email="mahesh.sattala@digite.com", phone="+919876543210", - message="This is test message", recaptcha_response="Svw2mPVxM0SkO4_2yxTcDQQ7iKNUDeDhGf4l6C2i" - ) - - def test_add_demo_request_with_empty_last_name(self): - processor = MongoProcessor() - processor.add_demo_request( - first_name="Mahesh", last_name="", email="mahesh.sattala@digite.com", phone="+919876543210", - message="This is test message", recaptcha_response="Svw2mPVxM0SkO4_2yxTcDQQ7iKNUDeDhGf4l6C2i" - ) - - def test_add_demo_request_with_invalid_email(self): - processor = MongoProcessor() - processor.add_demo_request( - first_name="Mahesh", last_name="Sattala", email="mahesh.sattala", phone="+919876543210", - message="This is test message", recaptcha_response="Svw2mPVxM0SkO4_2yxTcDQQ7iKNUDeDhGf4l6C2i" - ) - - def test_add_demo_request_with_invalid_status(self): - processor = MongoProcessor() - processor.add_demo_request( - first_name="Mahesh", last_name="Sattala", email="mahesh.sattala@digite.com", - phone="+919876543210", message="This is test message", status="Invalid_status", - recaptcha_response="Svw2mPVxM0SkO4_2yxTcDQQ7iKNUDeDhGf4l6C2i" - ) - - def test_add_demo_request(self): - processor = MongoProcessor() - processor.add_demo_request(first_name="Mahesh", last_name="Sattala", email="mahesh.sattala@nimblework.com", - phone="+919876543210", message="This is test message", status="demo_given", - recaptcha_response="Svw2mPVxM0SkO4_2yxTcDQQ7iKNUDeDhGf4l6C2i") - demo_request_logs = DemoRequestLogs.objects(first_name="Mahesh", last_name="Sattala", - email="mahesh.sattala@nimblework.com").get().to_mongo().to_dict() - assert demo_request_logs['first_name'] == "Mahesh" - assert demo_request_logs['last_name'] == "Sattala" - assert demo_request_logs['email'] == "mahesh.sattala@nimblework.com" - assert demo_request_logs['phone'] == "+919876543210" - assert demo_request_logs['status'] == "demo_given" - assert demo_request_logs['message'] == "This is test message" - assert demo_request_logs['recaptcha_response'] == "Svw2mPVxM0SkO4_2yxTcDQQ7iKNUDeDhGf4l6C2i" - def test_add_prompt_action_with_gpt_feature_disabled(self): processor = MongoProcessor() bot = 'test' @@ -213,9 +169,9 @@ def test_add_prompt_action_with_invalid_slots(self): BotSettings(bot=bot, user=user, llm_settings=LLMSettings(enable_faq=True)).save() request = {'name': 'test_add_prompt_action_with_invalid_slots', 'num_bot_responses': 5, 'failure_message': DEFAULT_NLU_FALLBACK_RESPONSE, - 'hyperparameters': {'temperature': 0.0, 'max_tokens': 300, 'model': 'gpt - 3.5 - turbo', + 'hyperparameters': {'temperature': 0.0, 'max_tokens': 300, 'model': 'gpt-3.5-turbo', 'top_p': 0.0, - 'n': 1, 'stream': False, 'stop': None, 'presence_penalty': 0.0, + 'n': 1, 'stop': None, 'presence_penalty': 0.0, 'frequency_penalty': 0.0, 'logit_bias': {}}, 'llm_prompts': [{'name': 'System Prompt', 'data': 'You are a personal assistant.', 'instructions': 'Answer question based on the context below.', 'type': 'system', @@ -241,9 +197,9 @@ def test_add_prompt_action_with_invalid_http_action(self): BotSettings(bot=bot, user=user, llm_settings=LLMSettings(enable_faq=True)).save() request = {'name': 'test_add_prompt_action_with_invalid_http_action', 'num_bot_responses': 5, 'failure_message': DEFAULT_NLU_FALLBACK_RESPONSE, - 'hyperparameters': {'temperature': 0.0, 'max_tokens': 300, 'model': 'gpt - 3.5 - turbo', + 'hyperparameters': {'temperature': 0.0, 'max_tokens': 300, 'model': 'gpt-3.5-turbo', 'top_p': 0.0, - 'n': 1, 'stream': False, 'stop': None, 'presence_penalty': 0.0, + 'n': 1, 'stop': None, 'presence_penalty': 0.0, 'frequency_penalty': 0.0, 'logit_bias': {}}, 'llm_prompts': [{'name': 'System Prompt', 'data': 'You are a personal assistant.', 'instructions': 'Answer question based on the context below.', 'type': 'system', @@ -270,9 +226,9 @@ def test_add_prompt_action_with_invalid_similarity_threshold(self): BotSettings(bot=bot, user=user, llm_settings=LLMSettings(enable_faq=True)).save() request = {'name': 'test_prompt_action_similarity', 'num_bot_responses': 5, 'failure_message': DEFAULT_NLU_FALLBACK_RESPONSE, - 'hyperparameters': {'temperature': 0.0, 'max_tokens': 300, 'model': 'gpt - 3.5 - turbo', + 'hyperparameters': {'temperature': 0.0, 'max_tokens': 300, 'model': 'gpt-3.5-turbo', 'top_p': 0.0, - 'n': 1, 'stream': False, 'stop': None, 'presence_penalty': 0.0, + 'n': 1, 'stop': None, 'presence_penalty': 0.0, 'frequency_penalty': 0.0, 'logit_bias': {}}, 'llm_prompts': [{'name': 'System Prompt', 'data': 'You are a personal assistant.', 'type': 'system', 'source': 'static', 'is_enabled': True}, @@ -300,9 +256,9 @@ def test_add_prompt_action_with_invalid_top_results(self): user = 'test_user' request = {'name': 'test_prompt_action_invalid_top_results', 'num_bot_responses': 5, 'failure_message': DEFAULT_NLU_FALLBACK_RESPONSE, - 'hyperparameters': {'temperature': 0.0, 'max_tokens': 300, 'model': 'gpt - 3.5 - turbo', + 'hyperparameters': {'temperature': 0.0, 'max_tokens': 300, 'model': 'gpt-3.5-turbo', 'top_p': 0.0, - 'n': 1, 'stream': False, 'stop': None, 'presence_penalty': 0.0, + 'n': 1, 'stop': None, 'presence_penalty': 0.0, 'frequency_penalty': 0.0, 'logit_bias': {}}, 'llm_prompts': [{'name': 'System Prompt', 'data': 'You are a personal assistant.', 'type': 'system', 'source': 'static', 'is_enabled': True}, @@ -348,14 +304,15 @@ def test_add_prompt_action_with_empty_collection_for_bot_content_prompt(self): processor.add_prompt_action(request, bot, user) prompt_action = processor.get_prompt_action(bot) prompt_action[0].pop("_id") - assert prompt_action == [ + assert not DeepDiff(prompt_action, [ {'name': 'test_add_prompt_action_with_empty_collection_for_bot_content_prompt', 'num_bot_responses': 5, 'failure_message': "I'm sorry, I didn't quite understand that. Could you rephrase?", 'user_question': {'type': 'from_user_message'}, 'hyperparameters': {'temperature': 0.0, 'max_tokens': 300, 'model': 'gpt-3.5-turbo', 'top_p': 0.0, - 'n': 1, 'stream': False, 'stop': None, 'presence_penalty': 0.0, + 'n': 1, 'stop': None, 'presence_penalty': 0.0, 'frequency_penalty': 0.0, 'logit_bias': {}}, + 'llm_type': 'openai', 'llm_prompts': [{'name': 'System Prompt', 'data': 'You are a personal assistant.', 'type': 'system', 'source': 'static', 'is_enabled': True}, {'name': 'Similarity Prompt', 'data': 'default', @@ -366,7 +323,7 @@ def test_add_prompt_action_with_empty_collection_for_bot_content_prompt(self): 'is_enabled': True}, {'name': 'Query Prompt', 'data': 'If there is no specific query, assume that user is aking about java programming.', 'instructions': 'Answer according to the context', 'type': 'query', 'source': 'static', 'is_enabled': True}], - 'instructions': [], 'set_slots': [], 'dispatch_response': True, 'status': True}] + 'instructions': [], 'set_slots': [], 'dispatch_response': True, 'status': True}], ignore_order=True) def test_add_prompt_action_with_bot_content_prompt(self): processor = MongoProcessor() @@ -392,15 +349,15 @@ def test_add_prompt_action_with_bot_content_prompt(self): processor.add_prompt_action(request, bot, user) prompt_action = processor.get_prompt_action(bot) prompt_action[1].pop("_id") - print(prompt_action) - assert prompt_action[1] == { + assert not DeepDiff(prompt_action[1], { 'name': 'test_add_prompt_action_with_bot_content_prompt', 'num_bot_responses': 5, 'failure_message': "I'm sorry, I didn't quite understand that. Could you rephrase?", 'user_question': {'type': 'from_user_message'}, 'hyperparameters': {'temperature': 0.0, 'max_tokens': 300, 'model': 'gpt-3.5-turbo', 'top_p': 0.0, - 'n': 1, 'stream': False, 'stop': None, 'presence_penalty': 0.0, + 'n': 1, 'stop': None, 'presence_penalty': 0.0, 'frequency_penalty': 0.0, 'logit_bias': {}}, + 'llm_type': 'openai', 'llm_prompts': [{'name': 'System Prompt', 'data': 'You are a personal assistant.', 'type': 'system', 'source': 'static', 'is_enabled': True}, {'name': 'Similarity Prompt', 'data': 'Bot_collection', @@ -411,7 +368,7 @@ def test_add_prompt_action_with_bot_content_prompt(self): 'is_enabled': True}, {'name': 'Query Prompt', 'data': 'If there is no specific query, assume that user is aking about java programming.', 'instructions': 'Answer according to the context', 'type': 'query', 'source': 'static', 'is_enabled': True}], - 'instructions': [], 'set_slots': [], 'dispatch_response': True, 'status': True} + 'instructions': [], 'set_slots': [], 'dispatch_response': True, 'status': True}, ignore_order=True) def test_add_prompt_action_with_invalid_query_prompt(self): processor = MongoProcessor() @@ -596,9 +553,9 @@ def test_add_prompt_action_with_empty_llm_prompts(self): user = 'test_user' request = {'name': 'test_add_prompt_action_with_empty_llm_prompts', 'num_bot_responses': 5, 'failure_message': DEFAULT_NLU_FALLBACK_RESPONSE, - 'hyperparameters': {'temperature': 0.0, 'max_tokens': 300, 'model': 'gpt - 3.5 - turbo', + 'hyperparameters': {'temperature': 0.0, 'max_tokens': 300, 'model': 'gpt-3.5-turbo', 'top_p': 0.0, - 'n': 1, 'stream': False, 'stop': None, 'presence_penalty': 0.0, + 'n': 1, 'stop': None, 'presence_penalty': 0.0, 'frequency_penalty': 0.0, 'logit_bias': {}}, 'llm_prompts': []} with pytest.raises(ValidationError, match="llm_prompts are required!"): @@ -621,13 +578,13 @@ def test_add_prompt_action_faq_action_with_default_values_and_instructions(self) pytest.action_id = processor.add_prompt_action(request, bot, user) action = list(processor.get_prompt_action(bot)) action[0].pop("_id") - print(action) - assert action == [{'name': 'test_add_prompt_action_faq_action_with_default_values', 'num_bot_responses': 5, + assert not DeepDiff(action, [{'name': 'test_add_prompt_action_faq_action_with_default_values', 'num_bot_responses': 5, 'failure_message': "I'm sorry, I didn't quite understand that. Could you rephrase?", 'user_question': {'type': 'from_slot', 'value': 'prompt_question'}, 'hyperparameters': {'temperature': 0.0, 'max_tokens': 300, 'model': 'gpt-3.5-turbo', - 'top_p': 0.0, 'n': 1, 'stream': False, 'stop': None, + 'top_p': 0.0, 'n': 1, 'stop': None, 'presence_penalty': 0.0, 'frequency_penalty': 0.0, 'logit_bias': {}}, + 'llm_type': 'openai', 'llm_prompts': [ {'name': 'System Prompt', 'data': 'You are a personal assistant.', 'type': 'system', 'source': 'static', 'is_enabled': True}, @@ -635,7 +592,7 @@ def test_add_prompt_action_faq_action_with_default_values_and_instructions(self) 'instructions': ['Answer in a short manner.', 'Keep it simple.'], 'set_slots': [{'name': 'gpt_result', 'value': '${data}', 'evaluation_type': 'expression'}, {'name': 'gpt_result_type', 'value': '${data.type}', - 'evaluation_type': 'script'}], 'dispatch_response': False, 'status': True}] + 'evaluation_type': 'script'}], 'dispatch_response': False, 'status': True}], ignore_order=True) def test_add_prompt_action_with_invalid_temperature_hyperparameter(self): processor = MongoProcessor() @@ -644,14 +601,14 @@ def test_add_prompt_action_with_invalid_temperature_hyperparameter(self): BotSettings(bot=bot, user=user, llm_settings=LLMSettings(enable_faq=True)).save() request = {'name': 'test_add_prompt_action_with_invalid_temperature_hyperparameter', 'num_bot_responses': 5, 'failure_message': DEFAULT_NLU_FALLBACK_RESPONSE, - 'hyperparameters': {'temperature': 3.0, 'max_tokens': 300, 'model': 'gpt - 3.5 - turbo', + 'hyperparameters': {'temperature': 3.0, 'max_tokens': 300, 'model': 'gpt-3.5-turbo', 'top_p': 0.0, - 'n': 1, 'stream': False, 'stop': None, 'presence_penalty': 0.0, + 'n': 1, 'stop': None, 'presence_penalty': 0.0, 'frequency_penalty': 0.0, 'logit_bias': {}}, 'llm_prompts': [{'name': 'System Prompt', 'data': 'You are a personal assistant.', 'type': 'system', 'source': 'static', 'is_enabled': True}, {'name': 'History Prompt', 'type': 'user', 'source': 'history', 'is_enabled': True}]} - with pytest.raises(ValidationError, match="Temperature must be between 0.0 and 2.0!"): + with pytest.raises(ValidationError, match=re.escape("['temperature']: 3.0 is greater than the maximum of 2.0")): processor.add_prompt_action(request, bot, user) def test_add_prompt_action_with_invalid_stop_hyperparameter(self): @@ -661,16 +618,16 @@ def test_add_prompt_action_with_invalid_stop_hyperparameter(self): BotSettings(bot=bot, user=user, llm_settings=LLMSettings(enable_faq=True)).save() request = {'name': 'test_add_prompt_action_with_invalid_stop_hyperparameter', 'num_bot_responses': 5, 'failure_message': DEFAULT_NLU_FALLBACK_RESPONSE, - 'hyperparameters': {'temperature': 0.0, 'max_tokens': 300, 'model': 'gpt - 3.5 - turbo', + 'hyperparameters': {'temperature': 0.0, 'max_tokens': 300, 'model': 'gpt-3.5-turbo', 'top_p': 0.0, - 'n': 1, 'stream': False, 'stop': ["\n", ".", "?", "!", ";"], + 'n': 1, 'stop': ["\n", ".", "?", "!", ";"], 'presence_penalty': 0.0, 'frequency_penalty': 0.0, 'logit_bias': {}}, 'llm_prompts': [{'name': 'System Prompt', 'data': 'You are a personal assistant.', 'type': 'system', 'source': 'static', 'is_enabled': True}, {'name': 'History Prompt', 'type': 'user', 'source': 'history', 'is_enabled': True}]} with pytest.raises(ValidationError, - match="Stop must be None, a string, an integer, or an array of 4 or fewer strings or integers."): + match=re.escape('[\'stop\']: ["\\n",".","?","!",";"] is not valid under any of the schemas listed in the \'anyOf\' keyword')): processor.add_prompt_action(request, bot, user) def test_add_prompt_action_with_invalid_presence_penalty_hyperparameter(self): @@ -681,14 +638,14 @@ def test_add_prompt_action_with_invalid_presence_penalty_hyperparameter(self): request = {'name': 'test_add_prompt_action_with_invalid_presence_penalty_hyperparameter', 'num_bot_responses': 5, 'failure_message': DEFAULT_NLU_FALLBACK_RESPONSE, - 'hyperparameters': {'temperature': 0.0, 'max_tokens': 300, 'model': 'gpt - 3.5 - turbo', + 'hyperparameters': {'temperature': 0.0, 'max_tokens': 300, 'model': 'gpt-3.5-turbo', 'top_p': 0.0, - 'n': 1, 'stream': False, 'stop': '?', 'presence_penalty': -3.0, + 'n': 1, 'stop': '?', 'presence_penalty': -3.0, 'frequency_penalty': 0.0, 'logit_bias': {}}, 'llm_prompts': [{'name': 'System Prompt', 'data': 'You are a personal assistant.', 'type': 'system', 'source': 'static', 'is_enabled': True}, {'name': 'History Prompt', 'type': 'user', 'source': 'history', 'is_enabled': True}]} - with pytest.raises(ValidationError, match="Presence penalty must be between -2.0 and 2.0!"): + with pytest.raises(ValidationError, match=re.escape("['presence_penalty']: -3.0 is less than the minimum of -2.0")): processor.add_prompt_action(request, bot, user) def test_add_prompt_action_with_invalid_frequency_penalty_hyperparameter(self): @@ -699,14 +656,14 @@ def test_add_prompt_action_with_invalid_frequency_penalty_hyperparameter(self): request = {'name': 'test_add_prompt_action_with_invalid_frequency_penalty_hyperparameter', 'num_bot_responses': 5, 'failure_message': DEFAULT_NLU_FALLBACK_RESPONSE, - 'hyperparameters': {'temperature': 0.0, 'max_tokens': 300, 'model': 'gpt - 3.5 - turbo', + 'hyperparameters': {'temperature': 0.0, 'max_tokens': 300, 'model': 'gpt-3.5-turbo', 'top_p': 0.0, - 'n': 1, 'stream': False, 'stop': '?', 'presence_penalty': 0.0, + 'n': 1, 'stop': '?', 'presence_penalty': 0.0, 'frequency_penalty': 3.0, 'logit_bias': {}}, 'llm_prompts': [{'name': 'System Prompt', 'data': 'You are a personal assistant.', 'type': 'system', 'source': 'static', 'is_enabled': True}, {'name': 'History Prompt', 'type': 'user', 'source': 'history', 'is_enabled': True}]} - with pytest.raises(ValidationError, match="Frequency penalty must be between -2.0 and 2.0!"): + with pytest.raises(ValidationError, match=re.escape("['frequency_penalty']: 3.0 is greater than the maximum of 2.0")): processor.add_prompt_action(request, bot, user) def test_add_prompt_action_with_invalid_max_tokens_hyperparameter(self): @@ -716,14 +673,14 @@ def test_add_prompt_action_with_invalid_max_tokens_hyperparameter(self): BotSettings(bot=bot, user=user, llm_settings=LLMSettings(enable_faq=True)).save() request = {'name': 'test_add_prompt_action_with_invalid_max_tokens_hyperparameter', 'num_bot_responses': 5, 'failure_message': DEFAULT_NLU_FALLBACK_RESPONSE, - 'hyperparameters': {'temperature': 0.0, 'max_tokens': 2, 'model': 'gpt - 3.5 - turbo', + 'hyperparameters': {'temperature': 0.0, 'max_tokens': 2, 'model': 'gpt-3.5-turbo', 'top_p': 0.0, - 'n': 1, 'stream': False, 'stop': '?', 'presence_penalty': 0.0, + 'n': 1, 'stop': '?', 'presence_penalty': 0.0, 'frequency_penalty': 0.0, 'logit_bias': {}}, 'llm_prompts': [{'name': 'System Prompt', 'data': 'You are a personal assistant.', 'type': 'system', 'source': 'static', 'is_enabled': True}, {'name': 'History Prompt', 'type': 'user', 'source': 'history', 'is_enabled': True}]} - with pytest.raises(ValidationError, match="max_tokens must be between 5 and 4096 and should not be 0!"): + with pytest.raises(ValidationError, match=re.escape("['max_tokens']: 2 is less than the minimum of 5")): processor.add_prompt_action(request, bot, user) def test_add_prompt_action_with_zero_max_tokens_hyperparameter(self): @@ -733,14 +690,14 @@ def test_add_prompt_action_with_zero_max_tokens_hyperparameter(self): BotSettings(bot=bot, user=user, llm_settings=LLMSettings(enable_faq=True)).save() request = {'name': 'test_add_prompt_action_with_zero_max_tokens_hyperparameter', 'num_bot_responses': 5, 'failure_message': DEFAULT_NLU_FALLBACK_RESPONSE, - 'hyperparameters': {'temperature': 0.0, 'max_tokens': 0, 'model': 'gpt - 3.5 - turbo', + 'hyperparameters': {'temperature': 0.0, 'max_tokens': 0, 'model': 'gpt-3.5-turbo', 'top_p': 0.0, - 'n': 1, 'stream': False, 'stop': '?', 'presence_penalty': 0.0, + 'n': 1, 'stop': '?', 'presence_penalty': 0.0, 'frequency_penalty': 0.0, 'logit_bias': {}}, 'llm_prompts': [{'name': 'System Prompt', 'data': 'You are a personal assistant.', 'type': 'system', 'source': 'static', 'is_enabled': True}, {'name': 'History Prompt', 'type': 'user', 'source': 'history', 'is_enabled': True}]} - with pytest.raises(ValidationError, match="max_tokens must be between 5 and 4096 and should not be 0!"): + with pytest.raises(ValidationError, match=re.escape("['max_tokens']: 0 is less than the minimum of 5")): processor.add_prompt_action(request, bot, user) def test_add_prompt_action_with_invalid_top_p_hyperparameter(self): @@ -750,14 +707,14 @@ def test_add_prompt_action_with_invalid_top_p_hyperparameter(self): BotSettings(bot=bot, user=user, llm_settings=LLMSettings(enable_faq=True)).save() request = {'name': 'test_add_prompt_action_with_invalid_top_p_hyperparameter', 'num_bot_responses': 5, 'failure_message': DEFAULT_NLU_FALLBACK_RESPONSE, - 'hyperparameters': {'temperature': 0.0, 'max_tokens': 256, 'model': 'gpt - 3.5 - turbo', + 'hyperparameters': {'temperature': 0.0, 'max_tokens': 256, 'model': 'gpt-3.5-turbo', 'top_p': 3.0, - 'n': 1, 'stream': False, 'stop': '?', 'presence_penalty': 0.0, + 'n': 1, 'stop': '?', 'presence_penalty': 0.0, 'frequency_penalty': 0.0, 'logit_bias': {}}, 'llm_prompts': [{'name': 'System Prompt', 'data': 'You are a personal assistant.', 'type': 'system', 'source': 'static', 'is_enabled': True}, {'name': 'History Prompt', 'type': 'user', 'source': 'history', 'is_enabled': True}]} - with pytest.raises(ValidationError, match="top_p must be between 0.0 and 1.0!"): + with pytest.raises(ValidationError, match=re.escape("['top_p']: 3.0 is greater than the maximum of 1.0")): processor.add_prompt_action(request, bot, user) def test_add_prompt_action_with_invalid_n_hyperparameter(self): @@ -767,14 +724,14 @@ def test_add_prompt_action_with_invalid_n_hyperparameter(self): BotSettings(bot=bot, user=user, llm_settings=LLMSettings(enable_faq=True)).save() request = {'name': 'test_add_prompt_action_with_invalid_n_hyperparameter', 'num_bot_responses': 5, 'failure_message': DEFAULT_NLU_FALLBACK_RESPONSE, - 'hyperparameters': {'temperature': 0.0, 'max_tokens': 200, 'model': 'gpt - 3.5 - turbo', + 'hyperparameters': {'temperature': 0.0, 'max_tokens': 200, 'model': 'gpt-3.5-turbo', 'top_p': 0.0, - 'n': 7, 'stream': False, 'stop': '?', 'presence_penalty': 0.0, + 'n': 7, 'stop': '?', 'presence_penalty': 0.0, 'frequency_penalty': 0.0, 'logit_bias': {}}, 'llm_prompts': [{'name': 'System Prompt', 'data': 'You are a personal assistant.', 'type': 'system', 'source': 'static', 'is_enabled': True}, {'name': 'History Prompt', 'type': 'user', 'source': 'history', 'is_enabled': True}]} - with pytest.raises(ValidationError, match="n must be between 1 and 5 and should not be 0!"): + with pytest.raises(ValidationError, match=re.escape("['n']: 7 is greater than the maximum of 5")): processor.add_prompt_action(request, bot, user) def test_add_prompt_action_with_zero_n_hyperparameter(self): @@ -784,14 +741,14 @@ def test_add_prompt_action_with_zero_n_hyperparameter(self): BotSettings(bot=bot, user=user, llm_settings=LLMSettings(enable_faq=True)).save() request = {'name': 'test_add_prompt_action_with_zero_n_hyperparameter', 'num_bot_responses': 5, 'failure_message': DEFAULT_NLU_FALLBACK_RESPONSE, - 'hyperparameters': {'temperature': 0.0, 'max_tokens': 200, 'model': 'gpt - 3.5 - turbo', + 'hyperparameters': {'temperature': 0.0, 'max_tokens': 200, 'model': 'gpt-3.5-turbo', 'top_p': 0.0, - 'n': 0, 'stream': False, 'stop': '?', 'presence_penalty': 0.0, + 'n': 0, 'stop': '?', 'presence_penalty': 0.0, 'frequency_penalty': 0.0, 'logit_bias': {}}, 'llm_prompts': [{'name': 'System Prompt', 'data': 'You are a personal assistant.', 'type': 'system', 'source': 'static', 'is_enabled': True}, {'name': 'History Prompt', 'type': 'user', 'source': 'history', 'is_enabled': True}]} - with pytest.raises(ValidationError, match="n must be between 1 and 5 and should not be 0!"): + with pytest.raises(ValidationError, match=re.escape("['n']: 0 is less than the minimum of 1")): processor.add_prompt_action(request, bot, user) def test_add_prompt_action_with_invalid_logit_bias_hyperparameter(self): @@ -801,14 +758,14 @@ def test_add_prompt_action_with_invalid_logit_bias_hyperparameter(self): BotSettings(bot=bot, user=user, llm_settings=LLMSettings(enable_faq=True)).save() request = {'name': 'test_add_prompt_action_with_invalid_logit_bias_hyperparameter', 'num_bot_responses': 5, 'failure_message': DEFAULT_NLU_FALLBACK_RESPONSE, - 'hyperparameters': {'temperature': 0.0, 'max_tokens': 200, 'model': 'gpt - 3.5 - turbo', + 'hyperparameters': {'temperature': 0.0, 'max_tokens': 200, 'model': 'gpt-3.5-turbo', 'top_p': 0.0, - 'n': 2, 'stream': False, 'stop': '?', 'presence_penalty': 0.0, + 'n': 2, 'stop': '?', 'presence_penalty': 0.0, 'frequency_penalty': 0.0, 'logit_bias': 'a'}, 'llm_prompts': [{'name': 'System Prompt', 'data': 'You are a personal assistant.', 'type': 'system', 'source': 'static', 'is_enabled': True}, {'name': 'History Prompt', 'type': 'user', 'source': 'history', 'is_enabled': True}]} - with pytest.raises(ValidationError, match="logit_bias must be a dictionary!"): + with pytest.raises(ValidationError, match=re.escape('[\'logit_bias\']: "a" is not of type "object"')): processor.add_prompt_action(request, bot, user) def test_add_prompt_action_faq_action_already_exist(self): @@ -867,7 +824,7 @@ def test_edit_prompt_action_faq_action(self): 'source': 'static', 'is_enabled': True}], "failure_message": "updated_failure_message", "use_query_prompt": True, "use_bot_responses": True, "query_prompt": "updated_query_prompt", - "num_bot_responses": 5, "hyperparameters": Utility.get_llm_hyperparameters(), + "num_bot_responses": 5, "hyperparameters": Utility.get_llm_hyperparameters('openai'), "set_slots": [{"name": "gpt_result", "value": "${data}", "evaluation_type": "expression"}, {"name": "gpt_result_type", "value": "${data.type}", "evaluation_type": "script"}], "dispatch_response": False @@ -875,12 +832,12 @@ def test_edit_prompt_action_faq_action(self): processor.edit_prompt_action(pytest.action_id, request, bot, user) action = list(processor.get_prompt_action(bot)) action[0].pop("_id") - print(action) - assert action == [{'name': 'test_edit_prompt_action_faq_action', 'num_bot_responses': 5, + assert not DeepDiff(action, [{'name': 'test_edit_prompt_action_faq_action', 'num_bot_responses': 5, 'failure_message': 'updated_failure_message', 'user_question': {'type': 'from_user_message'}, 'hyperparameters': {'temperature': 0.0, 'max_tokens': 300, 'model': 'gpt-3.5-turbo', - 'top_p': 0.0, 'n': 1, 'stream': False, 'stop': None, + 'top_p': 0.0, 'n': 1, 'stop': None, 'presence_penalty': 0.0, 'frequency_penalty': 0.0, 'logit_bias': {}}, + 'llm_type': 'openai', 'llm_prompts': [ {'name': 'System Prompt', 'data': 'You are a personal assistant.', 'type': 'system', 'source': 'static', 'is_enabled': True}, @@ -898,7 +855,8 @@ def test_edit_prompt_action_faq_action(self): 'is_enabled': True}], 'instructions': [], 'set_slots': [{'name': 'gpt_result', 'value': '${data}', 'evaluation_type': 'expression'}, {'name': 'gpt_result_type', 'value': '${data.type}', - 'evaluation_type': 'script'}], 'dispatch_response': False, 'status': True}] + 'evaluation_type': 'script'}], 'dispatch_response': False, 'status': True}], + ignore_order=True) request = {'name': 'test_edit_prompt_action_faq_action_again', 'user_question': {'type': 'from_slot', 'value': 'prompt_question'}, 'llm_prompts': [{'name': 'System Prompt', 'data': 'You are a personal assistant.', 'type': 'system', @@ -907,18 +865,18 @@ def test_edit_prompt_action_faq_action(self): processor.edit_prompt_action(pytest.action_id, request, bot, user) action = list(processor.get_prompt_action(bot)) action[0].pop("_id") - print(action) - assert action == [{'name': 'test_edit_prompt_action_faq_action_again', 'num_bot_responses': 5, + assert not DeepDiff(action, [{'name': 'test_edit_prompt_action_faq_action_again', 'num_bot_responses': 5, 'failure_message': "I'm sorry, I didn't quite understand that. Could you rephrase?", 'user_question': {'type': 'from_slot', 'value': 'prompt_question'}, + 'llm_type': 'openai', 'hyperparameters': {'temperature': 0.0, 'max_tokens': 300, 'model': 'gpt-3.5-turbo', - 'top_p': 0.0, 'n': 1, 'stream': False, 'stop': None, + 'top_p': 0.0, 'n': 1, 'stop': None, 'presence_penalty': 0.0, 'frequency_penalty': 0.0, 'logit_bias': {}}, 'llm_prompts': [ {'name': 'System Prompt', 'data': 'You are a personal assistant.', 'type': 'system', 'source': 'static', 'is_enabled': True}], 'instructions': ['Answer in a short manner.', 'Keep it simple.'], 'set_slots': [], - 'dispatch_response': True, 'status': True}] + 'dispatch_response': True, 'status': True}], ignore_order=True) def test_edit_prompt_action_with_less_hyperparameters(self): processor = MongoProcessor() @@ -951,13 +909,13 @@ def test_edit_prompt_action_with_less_hyperparameters(self): processor.edit_prompt_action(pytest.action_id, request, bot, user) action = list(processor.get_prompt_action(bot)) action[0].pop("_id") - print(action) - assert action == [{'name': 'test_edit_prompt_action_with_less_hyperparameters', 'num_bot_responses': 5, + assert not DeepDiff(action, [{'name': 'test_edit_prompt_action_with_less_hyperparameters', 'num_bot_responses': 5, 'failure_message': 'updated_failure_message', 'user_question': {'type': 'from_slot', 'value': 'prompt_question'}, 'hyperparameters': {'temperature': 0.0, 'max_tokens': 300, 'model': 'gpt-3.5-turbo', - 'top_p': 0.0, 'n': 1, 'stream': False, 'stop': None, + 'top_p': 0.0, 'n': 1, 'stop': None, 'presence_penalty': 0.0, 'frequency_penalty': 0.0, 'logit_bias': {}}, + 'llm_type': 'openai', 'llm_prompts': [ {'name': 'System Prompt', 'data': 'You are a personal assistant.', 'type': 'system', 'source': 'static', 'is_enabled': True}, @@ -973,7 +931,7 @@ def test_edit_prompt_action_with_less_hyperparameters(self): 'data': 'If there is no specific query, assume that user is aking about java programming.', 'instructions': 'Answer according to the context', 'type': 'query', 'source': 'static', 'is_enabled': True}], 'instructions': [], 'set_slots': [], 'dispatch_response': True, - 'status': True}] + 'status': True}], ignore_order=True) def test_get_prompt_action_does_not_exist(self): processor = MongoProcessor() @@ -986,13 +944,13 @@ def test_get_prompt_faq_action(self): bot = 'test_bot' action = list(processor.get_prompt_action(bot)) action[0].pop("_id") - print(action) - assert action == [{'name': 'test_edit_prompt_action_with_less_hyperparameters', 'num_bot_responses': 5, + assert not DeepDiff(action, [{'name': 'test_edit_prompt_action_with_less_hyperparameters', 'num_bot_responses': 5, 'failure_message': 'updated_failure_message', 'user_question': {'type': 'from_slot', 'value': 'prompt_question'}, 'hyperparameters': {'temperature': 0.0, 'max_tokens': 300, 'model': 'gpt-3.5-turbo', - 'top_p': 0.0, 'n': 1, 'stream': False, 'stop': None, + 'top_p': 0.0, 'n': 1, 'stop': None, 'presence_penalty': 0.0, 'frequency_penalty': 0.0, 'logit_bias': {}}, + 'llm_type': 'openai', 'llm_prompts': [ {'name': 'System Prompt', 'data': 'You are a personal assistant.', 'type': 'system', 'source': 'static', 'is_enabled': True}, @@ -1008,8 +966,7 @@ def test_get_prompt_faq_action(self): 'data': 'If there is no specific query, assume that user is aking about java programming.', 'instructions': 'Answer according to the context', 'type': 'query', 'source': 'static', 'is_enabled': True}], 'instructions': [], 'set_slots': [], 'dispatch_response': True, - 'status': True}] - + 'status': True}], ignore_order=True) def test_delete_prompt_action(self): processor = MongoProcessor() bot = 'test_bot' @@ -2674,7 +2631,7 @@ def test_start_training_fail(self): assert model_training.__len__() == 1 assert model_training.first().exception in str("Training data does not exists!") - @patch.object(GPT3FAQEmbedding, "_GPT3FAQEmbedding__get_embedding", autospec=True) + @patch.object(litellm, "aembedding", autospec=True) @patch("kairon.shared.rest_client.AioRestClient.request", autospec=True) @patch("kairon.shared.account.processor.AccountProcessor.get_bot", autospec=True) @patch("kairon.train.train_model_for_bot", autospec=True) @@ -2695,8 +2652,8 @@ def test_start_training_with_llm_faq( settings = BotSettings.objects(bot=bot).get() settings.llm_settings = LLMSettings(enable_faq=True) settings.save() - embedding = list(np.random.random(GPT3FAQEmbedding.__embedding__)) - mock_openai.return_value = embedding + embedding = list(np.random.random(1532)) + mock_openai.return_value = {'data': [{'embedding': embedding}]} mock_bot.return_value = {"account": 1} mock_train.return_value = f"/models/{bot}" start_training(bot, user) @@ -8549,7 +8506,9 @@ def test_delete_action_with_attached_http_action(self): 'data': 'tester_action', 'instructions': 'Answer according to the context', 'type': 'user', 'source': 'action', - 'is_enabled': True}] + 'is_enabled': True}], + llm_type=DEFAULT_LLM, + hyperparameters=Utility.get_default_llm_hyperparameters() ) processor.add_http_action_config(http_action_config.dict(), user, bot) processor.add_prompt_action(prompt_action_config.dict(), bot, user) diff --git a/tests/unit_test/data_processor/history_test.py b/tests/unit_test/data_processor/history_test.py index 3428736b9..8d7532bc2 100644 --- a/tests/unit_test/data_processor/history_test.py +++ b/tests/unit_test/data_processor/history_test.py @@ -2,7 +2,7 @@ import os from datetime import datetime -import mock +from unittest import mock import mongomock import pytest diff --git a/tests/unit_test/events/definitions_test.py b/tests/unit_test/events/definitions_test.py index aff9455f7..988b14072 100644 --- a/tests/unit_test/events/definitions_test.py +++ b/tests/unit_test/events/definitions_test.py @@ -4,11 +4,11 @@ from io import BytesIO from urllib.parse import urljoin -import mock +from unittest import mock import pytest import responses from fastapi import UploadFile -from mock.mock import patch +from unittest.mock import patch from mongoengine import connect from augmentation.utils import WebsiteParser diff --git a/tests/unit_test/events/events_test.py b/tests/unit_test/events/events_test.py index 98ae5ff10..7f1448a9d 100644 --- a/tests/unit_test/events/events_test.py +++ b/tests/unit_test/events/events_test.py @@ -8,7 +8,7 @@ from unittest.mock import patch from urllib.parse import urljoin -import mock +from unittest import mock import mongomock import pytest import responses @@ -17,15 +17,15 @@ from rasa.shared.constants import DEFAULT_DOMAIN_PATH, DEFAULT_DATA_PATH, DEFAULT_CONFIG_PATH from rasa.shared.importers.rasa import RasaFileImporter from responses import matchers +from kairon.shared.utils import Utility + +Utility.load_system_metadata() from kairon.shared.channels.broadcast.whatsapp import WhatsappBroadcast from kairon.shared.chat.data_objects import ChannelLogs os.environ["system_file"] = "./tests/testing_data/system.yaml" -from kairon.events.definitions.message_broadcast import MessageBroadcastEvent - -from kairon.shared.chat.broadcast.processor import MessageBroadcastProcessor from kairon.events.definitions.data_importer import TrainingDataImporterEvent from kairon.events.definitions.faq_importer import FaqDataImporterEvent from kairon.events.definitions.history_delete import DeleteHistoryEvent @@ -42,7 +42,6 @@ from kairon.shared.data.processor import MongoProcessor from kairon.shared.importer.processor import DataImporterLogProcessor from kairon.shared.test.processor import ModelTestingLogProcessor -from kairon.shared.utils import Utility from kairon.test.test_models import ModelTester os.environ["system_file"] = "./tests/testing_data/system.yaml" @@ -2020,7 +2019,7 @@ def test_execute_message_broadcast_with_pyscript_failure(self, mock_is_exist, mo bot = 'test_execute_message_broadcast_with_pyscript_failure' user = 'test_user' script = """ - import time + import os """ script = textwrap.dedent(script) config = { @@ -2057,7 +2056,7 @@ def test_execute_message_broadcast_with_pyscript_failure(self, mock_is_exist, mo logs[0][0].pop("timestamp", None) assert logs[0][0] == {"event_id": event_id, 'reference_id': reference_id, 'log_type': 'common', 'bot': bot, 'status': 'Fail', - 'user': user, "exception": "Script execution error: import of 'time' is unauthorized"} + 'user': user, "exception": "Script execution error: import of 'os' is unauthorized"} with pytest.raises(AppException, match="Notification settings not found!"): MessageBroadcastProcessor.get_settings(event_id, bot) diff --git a/tests/unit_test/events/scheduler_test.py b/tests/unit_test/events/scheduler_test.py index ec6426415..678c11d2e 100644 --- a/tests/unit_test/events/scheduler_test.py +++ b/tests/unit_test/events/scheduler_test.py @@ -1,7 +1,7 @@ import os import re -from mock import patch +from unittest.mock import patch import pytest from apscheduler.jobstores.mongodb import MongoDBJobStore diff --git a/tests/unit_test/idp/test_idp_helper.py b/tests/unit_test/idp/test_idp_helper.py index db6ff9609..0d74cc842 100644 --- a/tests/unit_test/idp/test_idp_helper.py +++ b/tests/unit_test/idp/test_idp_helper.py @@ -17,7 +17,6 @@ from kairon.shared.organization.processor import OrgProcessor from kairon.shared.utils import Utility from stress_test.data_objects import User -from mock import patch def get_user(): diff --git a/tests/unit_test/llm_test.py b/tests/unit_test/llm_test.py index 7494e1c0a..bf6b54d50 100644 --- a/tests/unit_test/llm_test.py +++ b/tests/unit_test/llm_test.py @@ -1,22 +1,25 @@ import os +from unittest import mock from urllib.parse import urljoin -import mock import numpy as np import pytest import ujson as json from aiohttp import ClientConnectionError from mongoengine import connect +from kairon.shared.utils import Utility + +Utility.load_system_metadata() + from kairon.exceptions import AppException from kairon.shared.admin.constants import BotSecretType from kairon.shared.admin.data_objects import BotSecrets from kairon.shared.cognition.data_objects import CognitionData, CognitionSchema -from kairon.shared.data.constant import DEFAULT_SYSTEM_PROMPT -from kairon.shared.data.data_objects import LLMSettings -from kairon.shared.llm.factory import LLMFactory -from kairon.shared.llm.gpt3 import GPT3FAQEmbedding, LLMBase -from kairon.shared.utils import Utility +from kairon.shared.data.constant import DEFAULT_SYSTEM_PROMPT, DEFAULT_LLM +from kairon.shared.llm.processor import LLMProcessor +import litellm +from deepdiff import DeepDiff class TestLLM: @@ -26,36 +29,9 @@ def init_connection(self): Utility.load_environment() connect(**Utility.mongoengine_connection(Utility.environment['database']["url"])) - def test_llm_base_train(self): - with pytest.raises(Exception): - base = LLMBase("Test") - base.train() - - def test_llm_base_predict(self): - with pytest.raises(Exception): - base = LLMBase('Test') - base.predict("Sample") - - def test_llm_factory_invalid_type(self): - with pytest.raises(Exception): - LLMFactory.get_instance("sample")("test", LLMSettings(provider="openai").to_mongo().to_dict()) - - def test_llm_factory_faq_type(self): - BotSecrets(secret_type=BotSecretType.gpt_key.value, value='value', bot='test', user='test').save() - inst = LLMFactory.get_instance("faq")("test", LLMSettings(provider="openai").to_mongo().to_dict()) - assert isinstance(inst, GPT3FAQEmbedding) - assert inst.db_url == Utility.environment['vector']['db'] - assert inst.headers == {} - - def test_llm_factory_faq_type_set_vector_key(self): - with mock.patch.dict(Utility.environment, {'vector': {"db": "http://test:6333", 'key': 'test'}}): - inst = LLMFactory.get_instance("faq")("test", LLMSettings(provider="openai").to_mongo().to_dict()) - assert isinstance(inst, GPT3FAQEmbedding) - assert inst.db_url == Utility.environment['vector']['db'] - assert inst.headers == {'api-key': Utility.environment['vector']['key']} - @pytest.mark.asyncio - async def test_gpt3_faq_embedding_train(self, aioresponses): + @mock.patch.object(litellm, "aembedding", autospec=True) + async def test_gpt3_faq_embedding_train(self, mock_embedding, aioresponses): bot = "test_embed_faq" user = "test" value = "nupurkhare" @@ -64,19 +40,11 @@ async def test_gpt3_faq_embedding_train(self, aioresponses): bot=bot, user=user).save() secret = BotSecrets(secret_type=BotSecretType.gpt_key.value, value=value, bot=bot, user=user).save() - embedding = list(np.random.random(GPT3FAQEmbedding.__embedding__)) - request_header = {"Authorization": "Bearer nupurkhare"} - + embedding = list(np.random.random(LLMProcessor.__embedding__)) with mock.patch.dict(Utility.environment, {'llm': {"faq": "GPT3_FAQ_EMBED", 'api_key': secret}, 'vector': {'db': "http://kairon:6333", "key": None}}): - aioresponses.add( - url="https://api.openai.com/v1/embeddings", - method="POST", - status=200, - payload={'data': [{'embedding': embedding}]} - ) - - gpt3 = GPT3FAQEmbedding(test_content.bot, LLMSettings(provider="openai").to_mongo().to_dict()) + mock_embedding.return_value = {'data': [{'embedding': embedding}]} + gpt3 = LLMProcessor(test_content.bot, DEFAULT_LLM) aioresponses.add( url=urljoin(Utility.environment['vector']['db'], f"/collections"), @@ -100,23 +68,26 @@ async def test_gpt3_faq_embedding_train(self, aioresponses): payload={"result": {"operation_id": 0, "status": "acknowledged"}, "status": "ok", "time": 0.003612634} ) - response = await gpt3.train() + response = await gpt3.train(user=user) assert response['faq'] == 1 assert list(aioresponses.requests.values())[1][0].kwargs['json'] == {'name': gpt3.bot + gpt3.suffix, 'vectors': gpt3.vector_config} - assert list(aioresponses.requests.values())[2][0].kwargs['json'] == {"model": "text-embedding-3-small", - "input": test_content.data} - assert list(aioresponses.requests.values())[2][0].kwargs['headers'] == request_header - - assert list(aioresponses.requests.values())[3][0].kwargs['json'] == { + assert list(aioresponses.requests.values())[2][0].kwargs['json'] == { 'points': [{'id': test_content.vector_id, 'vector': embedding, - 'payload': {"collection_name": f"{gpt3.bot}{gpt3.suffix}", 'content': test_content.data} + 'payload': {'content': test_content.data} }]} + expected = {"model": "text-embedding-3-small", + "input": [test_content.data], 'metadata': {'user': user, 'bot': bot, 'invocation': None}, + "api_key": value, + "num_retries": 3} + assert not DeepDiff(mock_embedding.call_args[1], expected, ignore_order=True) + @pytest.mark.asyncio - async def test_gpt3_faq_embedding_train_payload_text(self, aioresponses): + @mock.patch.object(litellm, "aembedding", autospec=True) + async def test_gpt3_faq_embedding_train_payload_text(self, mock_embedding, aioresponses): bot = "test_embed_faq_text" user = "test" value = "nupurkhare" @@ -149,24 +120,17 @@ async def test_gpt3_faq_embedding_train_payload_text(self, aioresponses): bot=bot, user=user).save() secret = BotSecrets(secret_type=BotSecretType.gpt_key.value, value=value, bot=bot, user=user).save() - embedding = list(np.random.random(GPT3FAQEmbedding.__embedding__)) - request_header = {"Authorization": "Bearer nupurkhare"} - - aioresponses.add( - url="https://api.openai.com/v1/embeddings", - method="POST", - status=200, - payload={'data': [{'embedding': embedding}]}, - repeat=True - ) - - gpt3 = GPT3FAQEmbedding(test_content.bot, LLMSettings(provider="openai").to_mongo().to_dict()) + embedding = list(np.random.random(LLMProcessor.__embedding__)) + mock_embedding.side_effect = {'data': [{'embedding': embedding}]}, {'data': [{'embedding': embedding}]}, { + 'data': [{'embedding': embedding}]} + gpt3 = LLMProcessor(bot, DEFAULT_LLM) with mock.patch.dict(Utility.environment, {'llm': {"faq": "GPT3_FAQ_EMBED", 'api_key': secret}}): aioresponses.add( url=urljoin(Utility.environment['vector']['db'], f"/collections"), method="GET", - payload={"time": 0, "status": "ok", "result": {"collections": [{"name": "test_embed_faq_text_swift_faq_embd"}, - {"name": "example_bot_swift_faq_embd"}]}} + payload={"time": 0, "status": "ok", + "result": {"collections": [{"name": "test_embed_faq_text_swift_faq_embd"}, + {"name": "example_bot_swift_faq_embd"}]}} ) aioresponses.add( @@ -181,19 +145,22 @@ async def test_gpt3_faq_embedding_train_payload_text(self, aioresponses): ) aioresponses.add( - url=urljoin(Utility.environment['vector']['db'], f"/collections/{gpt3.bot}_user_details{gpt3.suffix}/points"), + url=urljoin(Utility.environment['vector']['db'], + f"/collections/{gpt3.bot}_user_details{gpt3.suffix}/points"), method="PUT", payload={"result": {"operation_id": 0, "status": "acknowledged"}, "status": "ok", "time": 0.003612634} ) aioresponses.add( - url=urljoin(Utility.environment['vector']['db'], f"/collections/{gpt3.bot}_country_details{gpt3.suffix}"), + url=urljoin(Utility.environment['vector']['db'], + f"/collections/{gpt3.bot}_country_details{gpt3.suffix}"), method="PUT", status=200 ) aioresponses.add( - url=urljoin(Utility.environment['vector']['db'], f"/collections/{gpt3.bot}_country_details{gpt3.suffix}/points"), + url=urljoin(Utility.environment['vector']['db'], + f"/collections/{gpt3.bot}_country_details{gpt3.suffix}/points"), method="PUT", payload={"result": {"operation_id": 0, "status": "acknowledged"}, "status": "ok", "time": 0.003612634} ) @@ -205,39 +172,41 @@ async def test_gpt3_faq_embedding_train_payload_text(self, aioresponses): payload={"result": {"operation_id": 0, "status": "acknowledged"}, "status": "ok", "time": 0.003612634} ) - response = await gpt3.train() + response = await gpt3.train(user=user) assert response['faq'] == 3 - assert list(aioresponses.requests.values())[2][0].kwargs['json'] == {'name': f"{gpt3.bot}_country_details{gpt3.suffix}", - 'vectors': gpt3.vector_config} + assert list(aioresponses.requests.values())[2][0].kwargs['json'] == { + 'name': f"{gpt3.bot}_country_details{gpt3.suffix}", + 'vectors': gpt3.vector_config} - assert list(aioresponses.requests.values())[3][0].kwargs['json'] == {"model": "text-embedding-3-small", - "input": '{"country":"Spain","lang":"spanish"}'} - assert list(aioresponses.requests.values())[3][0].kwargs['headers'] == request_header - assert list(aioresponses.requests.values())[3][1].kwargs['json'] == {"model": "text-embedding-3-small", - 'input': '{"lang":"spanish","role":"ds"}'} - assert list(aioresponses.requests.values())[3][1].kwargs['headers'] == request_header - assert list(aioresponses.requests.values())[3][2].kwargs['json'] == {"model": "text-embedding-3-small", - "input": '{"name":"Nupur","city":"Pune"}'} - assert list(aioresponses.requests.values())[3][2].kwargs['headers'] == request_header - assert list(aioresponses.requests.values())[4][0].kwargs['json'] == {'points': [{'id': test_content_two.vector_id, - 'vector': embedding, - 'payload': {'collection_name': f"{gpt3.bot}_country_details{gpt3.suffix}", - 'country': 'Spain'}}]} - assert list(aioresponses.requests.values())[4][1].kwargs['json'] == {'points': [{'id': test_content_three.vector_id, - 'vector': embedding, - 'payload': {'collection_name': f"{gpt3.bot}_country_details{gpt3.suffix}", 'role': 'ds'}}]} - - assert list(aioresponses.requests.values())[5][0].kwargs['json'] == {'name': f"{gpt3.bot}_user_details{gpt3.suffix}", - 'vectors': gpt3.vector_config} - assert list(aioresponses.requests.values())[6][0].kwargs['json'] == {'points': [{'id': test_content.vector_id, - 'vector': embedding, - 'payload': {'collection_name': f"{gpt3.bot}_user_details{gpt3.suffix}", - 'name': 'Nupur'}}]} + assert list(aioresponses.requests.values())[3][0].kwargs['json'] == { + 'points': [{'id': test_content_two.vector_id, + 'vector': embedding, + 'payload': {'country': 'Spain'}}]} + assert list(aioresponses.requests.values())[3][1].kwargs['json'] == { + 'points': [{'id': test_content_three.vector_id, + 'vector': embedding, + 'payload': {'role': 'ds'}}]} + + assert list(aioresponses.requests.values())[4][0].kwargs['json'] == { + 'name': f"{gpt3.bot}_user_details{gpt3.suffix}", + 'vectors': gpt3.vector_config} + assert list(aioresponses.requests.values())[5][0].kwargs['json'] == { + 'points': [{'id': test_content.vector_id, + 'vector': embedding, + 'payload': {'name': 'Nupur'}}]} assert response['faq'] == 3 + expected = {"model": "text-embedding-3-small", + "input": [json.dumps(test_content.data)], 'metadata': {'user': user, 'bot': bot, 'invocation': None}, + "api_key": value, + "num_retries": 3} + print(mock_embedding.call_args) + assert not DeepDiff(mock_embedding.call_args[1], expected, ignore_order=True) + @pytest.mark.asyncio - async def test_gpt3_faq_embedding_train_payload_with_int(self, aioresponses): + @mock.patch.object(litellm, "aembedding", autospec=True) + async def test_gpt3_faq_embedding_train_payload_with_int(self, mock_embedding, aioresponses): bot = "test_embed_faq_json" user = "test" value = "nupurkhare" @@ -254,19 +223,14 @@ async def test_gpt3_faq_embedding_train_payload_with_int(self, aioresponses): bot=bot, user=user).save() secret = BotSecrets(secret_type=BotSecretType.gpt_key.value, value=value, bot=bot, user=user).save() - embedding = list(np.random.random(GPT3FAQEmbedding.__embedding__)) - request_header = {"Authorization": "Bearer nupurkhare"} + embedding = list(np.random.random(LLMProcessor.__embedding__)) input = {"name": "Ram", "color": "red"} - aioresponses.add( - url="https://api.openai.com/v1/embeddings", - method="POST", - status=200, - payload={'data': [{'embedding': embedding}]} - ) + mock_embedding.return_value = {'data': [{'embedding': embedding}]} - gpt3 = GPT3FAQEmbedding(test_content.bot, LLMSettings(provider="openai").to_mongo().to_dict()) + gpt3 = LLMProcessor(bot, DEFAULT_LLM) aioresponses.add( - url=urljoin(Utility.environment['vector']['db'], f"/collections/test_embed_faq_json_payload_with_int_faq_embd"), + url=urljoin(Utility.environment['vector']['db'], + f"/collections/test_embed_faq_json_payload_with_int_faq_embd"), method="PUT", status=200 ) @@ -276,27 +240,34 @@ async def test_gpt3_faq_embedding_train_payload_with_int(self, aioresponses): payload={"time": 0, "status": "ok", "result": {"collections": []}}) aioresponses.add( - url=urljoin(Utility.environment['vector']['db'], f"/collections/test_embed_faq_json_payload_with_int_faq_embd/points"), + url=urljoin(Utility.environment['vector']['db'], + f"/collections/test_embed_faq_json_payload_with_int_faq_embd/points"), method="PUT", payload={"result": {"operation_id": 0, "status": "acknowledged"}, "status": "ok", "time": 0.003612634} ) with mock.patch.dict(Utility.environment, {'llm': {"faq": "GPT3_FAQ_EMBED", 'api_key': secret}}): - response = await gpt3.train() + response = await gpt3.train(user=user) assert response['faq'] == 1 - assert list(aioresponses.requests.values())[1][0].kwargs['json'] == {'name': 'test_embed_faq_json_payload_with_int_faq_embd', - 'vectors': gpt3.vector_config} - assert list(aioresponses.requests.values())[2][0].kwargs['json'] == {"model": "text-embedding-3-small", - "input": json.dumps(input)} - assert list(aioresponses.requests.values())[2][0].kwargs['headers'] == request_header - assert list(aioresponses.requests.values())[3][0].kwargs['json'] == {'points': [{'id': test_content.vector_id, + assert list(aioresponses.requests.values())[1][0].kwargs['json'] == { + 'name': 'test_embed_faq_json_payload_with_int_faq_embd', + 'vectors': gpt3.vector_config} + assert list(aioresponses.requests.values())[2][0].kwargs['json'] == { + 'points': [{'id': test_content.vector_id, 'vector': embedding, - 'payload': {'name': 'Ram', 'age': 23, 'color': 'red', "collection_name": "test_embed_faq_json_payload_with_int_faq_embd"} + 'payload': {'name': 'Ram', 'age': 23, 'color': 'red'} }]} + expected = {"model": "text-embedding-3-small", + "input": [json.dumps(input)], 'metadata': {'user': user, 'bot': bot, 'invocation': None}, + "api_key": value, + "num_retries": 3} + assert not DeepDiff(mock_embedding.call_args[1], expected, ignore_order=True) + @pytest.mark.asyncio - async def test_gpt3_faq_embedding_train_int(self, aioresponses): + @mock.patch.object(litellm, "aembedding", autospec=True) + async def test_gpt3_faq_embedding_train_int(self, mock_embedding, aioresponses): bot = "test_int" user = "test" value = "nupurkhare" @@ -313,18 +284,11 @@ async def test_gpt3_faq_embedding_train_int(self, aioresponses): bot=bot, user=user).save() secret = BotSecrets(secret_type=BotSecretType.gpt_key.value, value=value, bot=bot, user=user).save() - embedding = list(np.random.random(GPT3FAQEmbedding.__embedding__)) - request_header = {"Authorization": "Bearer nupurkhare"} + embedding = list(np.random.random(LLMProcessor.__embedding__)) input = {"name": "Ram", "color": "red"} - aioresponses.add( - url="https://api.openai.com/v1/embeddings", - method="POST", - status=200, - payload={'data': [{'embedding': embedding}]} - ) - + mock_embedding.return_value = {'data': [{'embedding': embedding}]} with mock.patch.dict(Utility.environment, {'llm': {"faq": "GPT3_FAQ_EMBED", 'api_key': secret}}): - gpt3 = GPT3FAQEmbedding(test_content.bot, LLMSettings(provider="openai").to_mongo().to_dict()) + gpt3 = LLMProcessor(bot, DEFAULT_LLM) aioresponses.add( url=urljoin(Utility.environment['vector']['db'], f"/collections"), @@ -354,28 +318,32 @@ async def test_gpt3_faq_embedding_train_int(self, aioresponses): payload={"result": {"operation_id": 0, "status": "acknowledged"}, "status": "ok", "time": 0.003612634} ) - response = await gpt3.train() + response = await gpt3.train(user=user) assert response['faq'] == 1 assert list(aioresponses.requests.values())[1][0].kwargs['json'] == {'name': 'test_int_embd_int_faq_embd', 'vectors': gpt3.vector_config} - assert list(aioresponses.requests.values())[2][0].kwargs['json'] == {"model": "text-embedding-3-small", - "input": json.dumps(input)} - assert list(aioresponses.requests.values())[2][0].kwargs['headers'] == request_header expected_payload = test_content.data - expected_payload['collection_name'] = 'test_int_embd_int_faq_embd' - assert list(aioresponses.requests.values())[3][0].kwargs['json'] == { + #expected_payload['collection_name'] = 'test_int_embd_int_faq_embd' + assert list(aioresponses.requests.values())[2][0].kwargs['json'] == { 'points': [{'id': test_content.vector_id, 'vector': embedding, 'payload': expected_payload }]} + expected = {"model": "text-embedding-3-small", + "input": [json.dumps(input)], 'metadata': {'user': user, 'bot': bot, 'invocation': None}, + "api_key": value, + "num_retries": 3} + assert not DeepDiff(mock_embedding.call_args[1], expected, ignore_order=True) + def test_gpt3_faq_embedding_train_failure(self): with pytest.raises(AppException, match=f"Bot secret '{BotSecretType.gpt_key.value}' not configured!"): - GPT3FAQEmbedding('test_failure', LLMSettings(provider="openai").to_mongo().to_dict()) + LLMProcessor('test_failure', DEFAULT_LLM) @pytest.mark.asyncio - async def test_gpt3_faq_embedding_train_upsert_error(self, aioresponses): + @mock.patch.object(litellm, "aembedding", autospec=True) + async def test_gpt3_faq_embedding_train_upsert_error(self, mock_embedding, aioresponses): bot = "test_embed_faq_not_exists" user = "test" value = "nupurk" @@ -384,19 +352,12 @@ async def test_gpt3_faq_embedding_train_upsert_error(self, aioresponses): bot=bot, user=user).save() secret = BotSecrets(secret_type=BotSecretType.gpt_key.value, value=value, bot=bot, user=user).save() - embedding = list(np.random.random(GPT3FAQEmbedding.__embedding__)) + embedding = list(np.random.random(LLMProcessor.__embedding__)) - request_header = {"Authorization": "Bearer nupurk"} - - aioresponses.add( - url="https://api.openai.com/v1/embeddings", - method="POST", - status=200, - payload={'data': [{'embedding': embedding}]} - ) + mock_embedding.return_value = {'data': [{'embedding': embedding}]} with mock.patch.dict(Utility.environment, {'llm': {"faq": "GPT3_FAQ_EMBED", 'api_key': secret}}): - gpt3 = GPT3FAQEmbedding(test_content.bot, LLMSettings(provider="openai").to_mongo().to_dict()) + gpt3 = LLMProcessor(test_content.bot, DEFAULT_LLM) aioresponses.add( url=urljoin(Utility.environment['vector']['db'], f"/collections"), @@ -418,22 +379,28 @@ async def test_gpt3_faq_embedding_train_upsert_error(self, aioresponses): url=urljoin(Utility.environment['vector']['db'], f"/collections/{gpt3.bot}{gpt3.suffix}/points"), method="PUT", payload={"result": None, - 'status': {'error': 'Json deserialize error: missing field `vectors` at line 1 column 34779'}, - "time": 0.003612634} + 'status': {'error': 'Json deserialize error: missing field `vectors` at line 1 column 34779'}, + "time": 0.003612634} ) with pytest.raises(AppException, match="Unable to train FAQ! Contact support"): - await gpt3.train() + await gpt3.train(user=user) - assert list(aioresponses.requests.values())[1][0].kwargs['json'] == {'name': gpt3.bot + gpt3.suffix, 'vectors': gpt3.vector_config} - assert list(aioresponses.requests.values())[2][0].kwargs['json'] == {"model": "text-embedding-3-small", "input": test_content.data} - assert list(aioresponses.requests.values())[2][0].kwargs['headers'] == request_header - assert list(aioresponses.requests.values())[3][0].kwargs['json'] == {'points': [{'id': test_content.vector_id, - 'vector': embedding, 'payload': {'collection_name': f"{bot}{gpt3.suffix}",'content': test_content.data}}]} + assert list(aioresponses.requests.values())[1][0].kwargs['json'] == {'name': gpt3.bot + gpt3.suffix, + 'vectors': gpt3.vector_config} + assert list(aioresponses.requests.values())[2][0].kwargs['json'] == { + 'points': [{'id': test_content.vector_id, + 'vector': embedding, 'payload': {'content': test_content.data}}]} + expected = {"model": "text-embedding-3-small", + "input": [test_content.data], 'metadata': {'user': user, 'bot': bot, 'invocation': None}, + "api_key": value, + "num_retries": 3} + assert not DeepDiff(mock_embedding.call_args[1], expected, ignore_order=True) @pytest.mark.asyncio - async def test_gpt3_faq_embedding_train_payload_upsert_error_json(self, aioresponses): + @mock.patch.object(litellm, "aembedding", autospec=True) + async def test_gpt3_faq_embedding_train_payload_upsert_error_json(self, mock_embedding, aioresponses): bot = "payload_upsert_error" user = "test" value = "nupurk" @@ -450,19 +417,11 @@ async def test_gpt3_faq_embedding_train_payload_upsert_error_json(self, aiorespo bot=bot, user=user).save() secret = BotSecrets(secret_type=BotSecretType.gpt_key.value, value=value, bot=bot, user=user).save() - embedding = list(np.random.random(GPT3FAQEmbedding.__embedding__)) - - request_header = {"Authorization": "Bearer nupurk"} - - aioresponses.add( - url="https://api.openai.com/v1/embeddings", - method="POST", - status=200, - payload={'data': [{'embedding': embedding}]} - ) + embedding = list(np.random.random(LLMProcessor.__embedding__)) + mock_embedding.return_value = {'data': [{'embedding': embedding}]} with mock.patch.dict(Utility.environment, {'llm': {"faq": "GPT3_FAQ_EMBED", 'api_key': secret}}): - gpt3 = GPT3FAQEmbedding(test_content.bot, LLMSettings(provider="openai").to_mongo().to_dict()) + gpt3 = LLMProcessor(test_content.bot, DEFAULT_LLM) aioresponses.add( url=urljoin(Utility.environment['vector']['db'], f"/collections"), @@ -471,39 +430,50 @@ async def test_gpt3_faq_embedding_train_payload_upsert_error_json(self, aiorespo aioresponses.add( method="DELETE", - url=urljoin(Utility.environment['vector']['db'], f"/collections/payload_upsert_error_error_json_faq_embd"), + url=urljoin(Utility.environment['vector']['db'], + f"/collections/payload_upsert_error_error_json_faq_embd"), ) aioresponses.add( - url=urljoin(Utility.environment['vector']['db'], f"/collections/payload_upsert_error_error_json_faq_embd"), + url=urljoin(Utility.environment['vector']['db'], + f"/collections/payload_upsert_error_error_json_faq_embd"), method="PUT", status=200 ) aioresponses.add( - url=urljoin(Utility.environment['vector']['db'], f"/collections/payload_upsert_error_error_json_faq_embd/points"), + url=urljoin(Utility.environment['vector']['db'], + f"/collections/payload_upsert_error_error_json_faq_embd/points"), method="PUT", payload={"result": None, - 'status': {'error': 'Json deserialize error: missing field `vectors` at line 1 column 34779'}, - "time": 0.003612634} + 'status': {'error': 'Json deserialize error: missing field `vectors` at line 1 column 34779'}, + "time": 0.003612634} ) with pytest.raises(AppException, match="Unable to train FAQ! Contact support"): - await gpt3.train() + await gpt3.train(user=user) - assert list(aioresponses.requests.values())[1][0].kwargs['json'] == {'name': 'payload_upsert_error_error_json_faq_embd', 'vectors': gpt3.vector_config} - assert list(aioresponses.requests.values())[2][0].kwargs['json'] == {"model": "text-embedding-3-small", "input": json.dumps(test_content.data)} - assert list(aioresponses.requests.values())[2][0].kwargs['headers'] == request_header + assert list(aioresponses.requests.values())[1][0].kwargs['json'] == { + 'name': 'payload_upsert_error_error_json_faq_embd', 'vectors': gpt3.vector_config} expected_payload = test_content.data - expected_payload['collection_name'] = 'payload_upsert_error_error_json_faq_embd' - assert list(aioresponses.requests.values())[3][0].kwargs['json'] == {'points': [{'id': test_content.vector_id, - 'vector': embedding, - 'payload': expected_payload - }]} + #expected_payload['collection_name'] = 'payload_upsert_error_error_json_faq_embd' + assert list(aioresponses.requests.values())[2][0].kwargs['json'] == { + 'points': [{'id': test_content.vector_id, + 'vector': embedding, + 'payload': expected_payload + }]} + + expected = {"model": "text-embedding-3-small", + "input": [json.dumps(test_content.data)], 'metadata': {'user': user, 'bot': bot, 'invocation': None}, + "api_key": value, + "num_retries": 3} + assert not DeepDiff(mock_embedding.call_args[1], expected, ignore_order=True) @pytest.mark.asyncio - async def test_gpt3_faq_embedding_predict(self, aioresponses): - embedding = list(np.random.random(GPT3FAQEmbedding.__embedding__)) + @mock.patch.object(litellm, "acompletion", autospec=True) + @mock.patch.object(litellm, "aembedding", autospec=True) + async def test_gpt3_faq_embedding_predict(self, mock_embedding, mock_completion, aioresponses): + embedding = list(np.random.random(LLMProcessor.__embedding__)) bot = "test_embed_faq_predict" user = "test" @@ -516,15 +486,17 @@ async def test_gpt3_faq_embedding_predict(self, aioresponses): generated_text = "Python is dynamically typed, garbage-collected, high level, general purpose programming." query = "What kind of language is python?" + hyperparameters = Utility.get_default_llm_hyperparameters() k_faq_action_config = { "system_prompt": "You are a personal assistant. Answer the question according to the below context", "context_prompt": "Based on below context answer question, if answer not in context check previous logs.", "similarity_prompt": [{"top_results": 10, "similarity_threshold": 0.70, 'use_similarity_prompt': True, - 'similarity_prompt_name': 'Similarity Prompt', - 'similarity_prompt_instructions': 'Answer according to this context.', - 'collection': 'python'}]} - hyperparameters = Utility.get_llm_hyperparameters() + 'similarity_prompt_name': 'Similarity Prompt', + 'similarity_prompt_instructions': 'Answer according to this context.', + 'collection': 'python'}], + "hyperparameters": hyperparameters + } mock_completion_request = {"messages": [ {'role': 'system', 'content': 'You are a personal assistant. Answer the question according to the below context'}, @@ -532,50 +504,43 @@ async def test_gpt3_faq_embedding_predict(self, aioresponses): 'content': "Based on below context answer question, if answer not in context check previous logs.\nInstructions on how to use Similarity Prompt:\n['Python is a high-level, general-purpose programming language. Its design philosophy emphasizes code readability with the use of significant indentation. Python is dynamically typed and garbage-collected.']\nAnswer according to this context.\n \nQ: What kind of language is python? \nA:"} ]} mock_completion_request.update(hyperparameters) - request_header = {"Authorization": "Bearer knupur"} - - aioresponses.add( - url="https://api.openai.com/v1/embeddings", - method="POST", - status=200, - payload={'data': [{'embedding': embedding}]} - ) - - aioresponses.add( - url="https://api.openai.com/v1/chat/completions", - method="POST", - status=200, - payload={'choices': [{'message': {'content': generated_text, 'role': 'assistant'}}]} - ) - + mock_embedding.return_value = {'data': [{'embedding': embedding}]} + mock_completion.return_value = {'choices': [{'message': {'content': generated_text, 'role': 'assistant'}}]} with mock.patch.dict(Utility.environment, {'llm': {"faq": "GPT3_FAQ_EMBED", 'api_key': secret}}): - gpt3 = GPT3FAQEmbedding(test_content.bot, LLMSettings(provider="openai").to_mongo().to_dict()) + gpt3 = LLMProcessor(test_content.bot, DEFAULT_LLM) aioresponses.add( - url=urljoin(Utility.environment['vector']['db'], f"/collections/{gpt3.bot}_{test_content.collection}{gpt3.suffix}/points/search"), + url=urljoin(Utility.environment['vector']['db'], + f"/collections/{gpt3.bot}_{test_content.collection}{gpt3.suffix}/points/search"), method="POST", payload={'result': [ {'id': test_content.vector_id, 'score': 0.80, "payload": {'content': test_content.data}}]} ) - response, time_elapsed = await gpt3.predict(query, **k_faq_action_config) - assert response['content'] == generated_text - - assert list(aioresponses.requests.values())[0][0].kwargs['json'] == {"model": "text-embedding-3-small", - "input": query} - assert list(aioresponses.requests.values())[0][0].kwargs['headers'] == request_header + response, time_elapsed = await gpt3.predict(query, user=user, **k_faq_action_config) - assert list(aioresponses.requests.values())[1][0].kwargs['json'] == {'vector': embedding, 'limit': 10, + assert list(aioresponses.requests.values())[0][0].kwargs['json'] == {'vector': embedding, 'limit': 10, 'with_payload': True, 'score_threshold': 0.70} - - assert list(aioresponses.requests.values())[2][0].kwargs['json'] == mock_completion_request - assert list(aioresponses.requests.values())[2][0].kwargs['headers'] == request_header assert isinstance(time_elapsed, float) and time_elapsed > 0.0 + expected = {"model": "text-embedding-3-small", + "input": [query], 'metadata': {'user': user, 'bot': bot, 'invocation': None}, + "api_key": value, + "num_retries": 3} + assert not DeepDiff(mock_embedding.call_args[1], expected, ignore_order=True) + expected = mock_completion_request.copy() + expected['metadata'] = {'user': user, 'bot': bot, 'invocation': None} + expected['api_key'] = value + expected['num_retries'] = 3 + assert not DeepDiff(mock_completion.call_args[1], expected, ignore_order=True) + @pytest.mark.asyncio - async def test_gpt3_faq_embedding_predict_with_default_collection(self, aioresponses): - embedding = list(np.random.random(GPT3FAQEmbedding.__embedding__)) + @mock.patch.object(litellm, "acompletion", autospec=True) + @mock.patch.object(litellm, "aembedding", autospec=True) + async def test_gpt3_faq_embedding_predict_with_default_collection(self, mock_embedding, mock_completion, + aioresponses): + embedding = list(np.random.random(LLMProcessor.__embedding__)) bot = "test_embed_faq_predict_with_default_collection" user = "test" @@ -588,15 +553,17 @@ async def test_gpt3_faq_embedding_predict_with_default_collection(self, aiorespo generated_text = "Python is dynamically typed, garbage-collected, high level, general purpose programming." query = "What kind of language is python?" - + hyperparameters = Utility.get_default_llm_hyperparameters() k_faq_action_config = { "system_prompt": "You are a personal assistant. Answer the question according to the below context", "context_prompt": "Based on below context answer question, if answer not in context check previous logs.", "similarity_prompt": [{"top_results": 10, "similarity_threshold": 0.70, 'use_similarity_prompt': True, 'similarity_prompt_name': 'Similarity Prompt', 'similarity_prompt_instructions': 'Answer according to this context.', - 'collection': 'default'}]} - hyperparameters = Utility.get_llm_hyperparameters() + 'collection': 'default'}], + 'hyperparameters': hyperparameters + } + mock_completion_request = {"messages": [ {'role': 'system', 'content': 'You are a personal assistant. Answer the question according to the below context'}, @@ -604,24 +571,10 @@ async def test_gpt3_faq_embedding_predict_with_default_collection(self, aiorespo 'content': "Based on below context answer question, if answer not in context check previous logs.\nInstructions on how to use Similarity Prompt:\n['Python is a high-level, general-purpose programming language. Its design philosophy emphasizes code readability with the use of significant indentation. Python is dynamically typed and garbage-collected.']\nAnswer according to this context.\n \nQ: What kind of language is python? \nA:"} ]} mock_completion_request.update(hyperparameters) - request_header = {"Authorization": "Bearer knupur"} - - aioresponses.add( - url="https://api.openai.com/v1/embeddings", - method="POST", - status=200, - payload={'data': [{'embedding': embedding}]} - ) - - aioresponses.add( - url="https://api.openai.com/v1/chat/completions", - method="POST", - status=200, - payload={'choices': [{'message': {'content': generated_text, 'role': 'assistant'}}]} - ) - + mock_embedding.return_value = {'data': [{'embedding': embedding}]} + mock_completion.return_value = {'choices': [{'message': {'content': generated_text, 'role': 'assistant'}}]} with mock.patch.dict(Utility.environment, {'llm': {"faq": "GPT3_FAQ_EMBED", 'api_key': secret}}): - gpt3 = GPT3FAQEmbedding(test_content.bot, LLMSettings(provider="openai").to_mongo().to_dict()) + gpt3 = LLMProcessor(test_content.bot, DEFAULT_LLM) aioresponses.add( url=urljoin(Utility.environment['vector']['db'], @@ -631,40 +584,52 @@ async def test_gpt3_faq_embedding_predict_with_default_collection(self, aiorespo {'id': test_content.vector_id, 'score': 0.80, "payload": {'content': test_content.data}}]} ) - response, time_elapsed = await gpt3.predict(query, **k_faq_action_config) + response, time_elapsed = await gpt3.predict(query, user=user, **k_faq_action_config) assert response['content'] == generated_text - assert list(aioresponses.requests.values())[0][0].kwargs['json'] == {"model": "text-embedding-3-small", - "input": query} - assert list(aioresponses.requests.values())[0][0].kwargs['headers'] == request_header - - assert list(aioresponses.requests.values())[1][0].kwargs['json'] == {'vector': embedding, 'limit': 10, + assert list(aioresponses.requests.values())[0][0].kwargs['json'] == {'vector': embedding, 'limit': 10, 'with_payload': True, 'score_threshold': 0.70} - assert list(aioresponses.requests.values())[2][0].kwargs['json'] == mock_completion_request - assert list(aioresponses.requests.values())[2][0].kwargs['headers'] == request_header assert isinstance(time_elapsed, float) and time_elapsed > 0.0 + expected = {"model": "text-embedding-3-small", + "input": [query], 'metadata': {'user': user, 'bot': bot, 'invocation': None}, + "api_key": value, + "num_retries": 3} + assert not DeepDiff(mock_embedding.call_args[1], expected, ignore_order=True) + expected = mock_completion_request.copy() + expected['metadata'] = {'user': user, 'bot': bot, 'invocation': None} + expected['api_key'] = value + expected['num_retries'] = 3 + assert not DeepDiff(mock_completion.call_args[1], expected, ignore_order=True) + @pytest.mark.asyncio - async def test_gpt3_faq_embedding_predict_with_values(self, aioresponses): - embedding = list(np.random.random(GPT3FAQEmbedding.__embedding__)) + @mock.patch.object(litellm, "acompletion", autospec=True) + @mock.patch.object(litellm, "aembedding", autospec=True) + async def test_gpt3_faq_embedding_predict_with_values(self, mock_embedding, mock_completion, aioresponses): + embedding = list(np.random.random(LLMProcessor.__embedding__)) test_content = CognitionData( data="Python is a high-level, general-purpose programming language. Its design philosophy emphasizes code readability with the use of significant indentation. Python is dynamically typed and garbage-collected.", - collection='python', bot="test_embed_faq_predict", user="test").save() + collection='python', bot="test_gpt3_faq_embedding_predict_with_values", user="test").save() generated_text = "Python is dynamically typed, garbage-collected, high level, general purpose programming." query = "What kind of language is python?" + hyperparameters = Utility.get_default_llm_hyperparameters() + key = 'test' + user = "tests" + BotSecrets(secret_type=BotSecretType.gpt_key.value, value=key, bot=test_content.bot, user=user).save() k_faq_action_config = { "system_prompt": "You are a personal assistant. Answer the question according to the below context", "context_prompt": "Based on below context answer question, if answer not in context check previous logs.", "similarity_prompt": [{"top_results": 10, "similarity_threshold": 0.70, 'use_similarity_prompt': True, - 'similarity_prompt_name': 'Similarity Prompt', - 'similarity_prompt_instructions': 'Answer according to this context.', - 'collection': 'python'}]} + 'similarity_prompt_name': 'Similarity Prompt', + 'similarity_prompt_instructions': 'Answer according to this context.', + 'collection': 'python'}], + "hyperparameters": hyperparameters + } - hyperparameters = Utility.get_llm_hyperparameters() mock_completion_request = {"messages": [ {"role": "system", "content": "You are a personal assistant. Answer the question according to the below context"}, @@ -672,33 +637,21 @@ async def test_gpt3_faq_embedding_predict_with_values(self, aioresponses): 'content': "Based on below context answer question, if answer not in context check previous logs.\nInstructions on how to use Similarity Prompt:\n['Python is a high-level, general-purpose programming language. Its design philosophy emphasizes code readability with the use of significant indentation. Python is dynamically typed and garbage-collected.']\nAnswer according to this context.\n \nQ: What kind of language is python? \nA:"} ]} mock_completion_request.update(hyperparameters) - request_header = {"Authorization": "Bearer knupur"} - - aioresponses.add( - url="https://api.openai.com/v1/embeddings", - method="POST", - status=200, - payload={'data': [{'embedding': embedding}]} - ) - - aioresponses.add( - url="https://api.openai.com/v1/chat/completions", - method="POST", - status=200, - payload={'choices': [{'message': {'content': generated_text, 'role': 'assistant'}}]} - ) + mock_embedding.return_value = {'data': [{'embedding': embedding}]} + mock_completion.return_value = {'choices': [{'message': {'content': generated_text, 'role': 'assistant'}}]} - with mock.patch.dict(Utility.environment, {'llm': {"faq": "GPT3_FAQ_EMBED", 'api_key': 'test'}}): - gpt3 = GPT3FAQEmbedding(test_content.bot, LLMSettings(provider="openai").to_mongo().to_dict()) + with mock.patch.dict(Utility.environment, {'vector': {"key": "test", 'db': "http://localhost:6333"}}): + gpt3 = LLMProcessor(test_content.bot, DEFAULT_LLM) aioresponses.add( - url=urljoin(Utility.environment['vector']['db'], f"/collections/{gpt3.bot}_{test_content.collection}{gpt3.suffix}/points/search"), + url=urljoin(Utility.environment['vector']['db'], + f"/collections/{gpt3.bot}_{test_content.collection}{gpt3.suffix}/points/search"), method="POST", payload={'result': [ {'id': test_content.vector_id, 'score': 0.80, "payload": {'content': test_content.data}}]} ) - response, time_elapsed = await gpt3.predict(query, **k_faq_action_config) + response, time_elapsed = await gpt3.predict(query, user=user, **k_faq_action_config) assert response['content'] == generated_text assert gpt3.logs == [ {'messages': [{'role': 'system', @@ -711,225 +664,381 @@ async def test_gpt3_faq_embedding_predict_with_values(self, aioresponses): 'role': 'assistant'}}]}, 'type': 'answer_query', 'hyperparameters': {'temperature': 0.0, 'max_tokens': 300, 'model': 'gpt-3.5-turbo', 'top_p': 0.0, - 'n': 1, 'stream': False, 'stop': None, 'presence_penalty': 0.0, + 'n': 1, 'stop': None, 'presence_penalty': 0.0, 'frequency_penalty': 0.0, 'logit_bias': {}}}] - assert list(aioresponses.requests.values())[0][0].kwargs['json'] == {"model": "text-embedding-3-small", "input": query} - assert list(aioresponses.requests.values())[0][0].kwargs['headers'] == request_header - - assert list(aioresponses.requests.values())[1][0].kwargs['json'] == {'vector': embedding, 'limit': 10, 'with_payload': True, 'score_threshold': 0.70} + assert list(aioresponses.requests.values())[0][0].kwargs['json'] == {'vector': embedding, 'limit': 10, + 'with_payload': True, + 'score_threshold': 0.70} - assert list(aioresponses.requests.values())[2][0].kwargs['json'] == mock_completion_request - assert list(aioresponses.requests.values())[2][0].kwargs['headers'] == request_header assert isinstance(time_elapsed, float) and time_elapsed > 0.0 + expected = {"model": "text-embedding-3-small", + "input": [query], 'metadata': {'user': user, 'bot': gpt3.bot, 'invocation': None}, + "api_key": key, + "num_retries": 3} + assert not DeepDiff(mock_embedding.call_args[1], expected, ignore_order=True) + expected = mock_completion_request.copy() + expected['metadata'] = {'user': user, 'bot': gpt3.bot, 'invocation': None} + expected['api_key'] = key + expected['num_retries'] = 3 + assert not DeepDiff(mock_completion.call_args[1], expected, ignore_order=True) + @pytest.mark.asyncio - async def test_gpt3_faq_embedding_predict_with_values_with_instructions(self, aioresponses): - embedding = list(np.random.random(GPT3FAQEmbedding.__embedding__)) + @mock.patch.object(litellm, "acompletion", autospec=True) + @mock.patch.object(litellm, "aembedding", autospec=True) + async def test_gpt3_faq_embedding_predict_with_values_and_stream(self, mock_embedding, mock_completion, + aioresponses): + embedding = list(np.random.random(LLMProcessor.__embedding__)) test_content = CognitionData( - data="Java is a high-level, general-purpose programming language. Java is known for its write once, run anywhere capability. ", - collection='java', bot="test_embed_faq_predict", user="test").save() + data="Python is a high-level, general-purpose programming language. Its design philosophy emphasizes code readability with the use of significant indentation. Python is dynamically typed and garbage-collected.", + collection='python', bot="test_gpt3_faq_embedding_predict_with_values_and_stream", user="test").save() generated_text = "Python is dynamically typed, garbage-collected, high level, general purpose programming." query = "What kind of language is python?" + hyperparameters = Utility.get_default_llm_hyperparameters() + hyperparameters['stream'] = True + key = 'test' + user = "tests" + BotSecrets(secret_type=BotSecretType.gpt_key.value, value=key, bot=test_content.bot, user=user).save() k_faq_action_config = { "system_prompt": "You are a personal assistant. Answer the question according to the below context", "context_prompt": "Based on below context answer question, if answer not in context check previous logs.", "similarity_prompt": [{"top_results": 10, "similarity_threshold": 0.70, 'use_similarity_prompt': True, - 'similarity_prompt_name': 'Similarity Prompt', - 'similarity_prompt_instructions': 'Answer according to this context.', - "collection": "java"}], - 'instructions': ['Answer in a short way.', 'Keep it simple.']} + 'similarity_prompt_name': 'Similarity Prompt', + 'similarity_prompt_instructions': 'Answer according to this context.', + 'collection': 'python'}], + "hyperparameters": hyperparameters + } - hyperparameters = Utility.get_llm_hyperparameters() mock_completion_request = {"messages": [ - {'role': 'system', - 'content': 'You are a personal assistant. Answer the question according to the below context'}, + {"role": "system", + "content": "You are a personal assistant. Answer the question according to the below context"}, {'role': 'user', - 'content': "Based on below context answer question, if answer not in context check previous logs.\nInstructions on how to use Similarity Prompt:\n['Java is a high-level, general-purpose programming language. Java is known for its write once, run anywhere capability. ']\nAnswer according to this context.\n \nAnswer in a short way.\nKeep it simple. \nQ: What kind of language is python? \nA:"} + 'content': "Based on below context answer question, if answer not in context check previous logs.\nInstructions on how to use Similarity Prompt:\n['Python is a high-level, general-purpose programming language. Its design philosophy emphasizes code readability with the use of significant indentation. Python is dynamically typed and garbage-collected.']\nAnswer according to this context.\n \nQ: What kind of language is python? \nA:"} ]} mock_completion_request.update(hyperparameters) - request_header = {"Authorization": "Bearer knupur"} - - aioresponses.add( - url="https://api.openai.com/v1/embeddings", - method="POST", - status=200, - payload={'data': [{'embedding': embedding}]} - ) - - aioresponses.add( - url="https://api.openai.com/v1/chat/completions", - method="POST", - status=200, - payload={'choices': [{'message': {'content': generated_text, 'role': 'assistant'}}]} - ) - - with mock.patch.dict(Utility.environment, {'llm': {"faq": "GPT3_FAQ_EMBED", 'api_key': 'test'}}): - gpt3 = GPT3FAQEmbedding(test_content.bot, LLMSettings(provider="openai").to_mongo().to_dict()) + mock_embedding.return_value = {'data': [{'embedding': embedding}]} + mock_completion.side_effect = [{'choices': [ + {'delta': {'role': 'assistant', 'content': generated_text[0:29]}, 'finish_reason': None, 'index': 0}]}, + {'choices': [{'delta': {'role': 'assistant', 'content': generated_text[29:60]}, + 'finish_reason': None, 'index': 0}]}, + {'choices': [{'delta': {'role': 'assistant', 'content': generated_text[60:]}, + 'finish_reason': 'stop', 'index': 0}]} + ] + + with mock.patch.dict(Utility.environment, {'llm': {"faq": "GPT3_FAQ_EMBED", 'api_key': key}}): + gpt3 = LLMProcessor(test_content.bot, DEFAULT_LLM) aioresponses.add( - url=urljoin(Utility.environment['vector']['db'], f"/collections/{gpt3.bot}_{test_content.collection}{gpt3.suffix}/points/search"), + url=urljoin(Utility.environment['vector']['db'], + f"/collections/{gpt3.bot}_{test_content.collection}{gpt3.suffix}/points/search"), method="POST", payload={'result': [ {'id': test_content.vector_id, 'score': 0.80, "payload": {'content': test_content.data}}]} ) - response, time_elapsed = await gpt3.predict(query, **k_faq_action_config) - assert response['content'] == generated_text + response, time_elapsed = await gpt3.predict(query, user=user, **k_faq_action_config) + assert response['content'] == "Python is dynamically typed, " assert gpt3.logs == [ {'messages': [{'role': 'system', 'content': 'You are a personal assistant. Answer the question according to the below context'}, {'role': 'user', - 'content': "Based on below context answer question, if answer not in context check previous logs.\nInstructions on how to use Similarity Prompt:\n['Java is a high-level, general-purpose programming language. Java is known for its write once, run anywhere capability. ']\nAnswer according to this context.\n \nAnswer in a short way.\nKeep it simple. \nQ: What kind of language is python? \nA:"}], - 'raw_completion_response': { - 'choices': [{'message': {'content': 'Python is dynamically typed, garbage-collected, ' - 'high level, general purpose programming.', - 'role': 'assistant'}}]}, + 'content': "Based on below context answer question, if answer not in context check previous logs.\nInstructions on how to use Similarity Prompt:\n['Python is a high-level, general-purpose programming language. Its design philosophy emphasizes code readability with the use of significant indentation. Python is dynamically typed and garbage-collected.']\nAnswer according to this context.\n \nQ: What kind of language is python? \nA:"}], + 'raw_completion_response': {'choices': [ + {'delta': {'role': 'assistant', 'content': generated_text[0:29]}, 'finish_reason': None, + 'index': 0}]}, 'type': 'answer_query', 'hyperparameters': {'temperature': 0.0, 'max_tokens': 300, 'model': 'gpt-3.5-turbo', 'top_p': 0.0, - 'n': 1, - 'stream': False, 'stop': None, 'presence_penalty': 0.0, 'frequency_penalty': 0.0, - 'logit_bias': {}}}] - - assert list(aioresponses.requests.values())[0][0].kwargs['json'] == {"model": "text-embedding-3-small", "input": query} - assert list(aioresponses.requests.values())[0][0].kwargs['headers'] == request_header + 'n': 1, 'stream': True, 'stop': None, 'presence_penalty': 0.0, + 'frequency_penalty': 0.0, 'logit_bias': {}}}] - assert list(aioresponses.requests.values())[1][0].kwargs['json'] == {'vector': embedding, 'limit': 10, 'with_payload': True, 'score_threshold': 0.70} + assert list(aioresponses.requests.values())[0][0].kwargs['json'] == {'vector': embedding, 'limit': 10, + 'with_payload': True, + 'score_threshold': 0.70} - assert list(aioresponses.requests.values())[2][0].kwargs['json'] == mock_completion_request - assert list(aioresponses.requests.values())[2][0].kwargs['headers'] == request_header assert isinstance(time_elapsed, float) and time_elapsed > 0.0 + expected = {"model": "text-embedding-3-small", + "input": [query], 'metadata': {'user': user, 'bot': gpt3.bot, 'invocation': None}, + "api_key": key, + "num_retries": 3} + assert not DeepDiff(mock_embedding.call_args[1], expected, ignore_order=True) + expected = mock_completion_request.copy() + expected['metadata'] = {'user': user, 'bot': gpt3.bot, 'invocation': None} + expected['api_key'] = key + expected['num_retries'] = 3 + assert not DeepDiff(mock_completion.call_args[1], expected, ignore_order=True) + @pytest.mark.asyncio - @mock.patch.object(GPT3FAQEmbedding, "_GPT3FAQEmbedding__get_answer", autospec=True) - @mock.patch.object(GPT3FAQEmbedding, "_GPT3FAQEmbedding__get_embedding", autospec=True) - async def test_gpt3_faq_embedding_predict_completion_connection_error(self, mock_embedding, mock_completion, aioresponses): - embedding = list(np.random.random(GPT3FAQEmbedding.__embedding__)) + @mock.patch.object(litellm, "acompletion", autospec=True) + @mock.patch.object(litellm, "aembedding", autospec=True) + async def test_gpt3_faq_embedding_predict_with_values_with_instructions(self, + mock_embedding, + mock_completion, + aioresponses): + embedding = list(np.random.random(LLMProcessor.__embedding__)) + user = "test" + bot = "payload_with_instruction" + key = 'test' + CognitionSchema( + metadata=[{"column_name": "name", "data_type": "str", "enable_search": True, "create_embeddings": True}, + {"column_name": "city", "data_type": "str", "enable_search": True, "create_embeddings": True}], + collection_name="User_details", + bot=bot, user=user + ).save() + test_content1 = CognitionData( + data={"name": "Nupur", "city": "Pune"}, + content_type="json", + collection="User_details", + bot=bot, user=user).save() + test_content2 = CognitionData( + data={"name": "Fahad", "city": "Mumbai"}, + content_type="json", + collection="User_details", + bot=bot, user=user).save() + test_content3 = CognitionData( + data={"name": "Hitesh", "city": "Mumbai"}, + content_type="json", + collection="User_details", + bot=bot, user=user).save() + + BotSecrets(secret_type=BotSecretType.gpt_key.value, value=key, bot=bot, user=user).save() + + generated_text = "Hitesh and Fahad lives in mumbai city." + query = "List all the user lives in mumbai city" + hyperparameters = Utility.get_default_llm_hyperparameters() + k_faq_action_config = { + "system_prompt": "You are a personal assistant. Answer the question according to the below context", + "context_prompt": "Based on below context answer question, if answer not in context check previous logs.", + "similarity_prompt": [{"top_results": 10, + "similarity_threshold": 0.70, + 'use_similarity_prompt': True, + 'similarity_prompt_name': 'Similarity Prompt', + 'similarity_prompt_instructions': 'Answer according to this context.', + "collection": "user_details"}], + 'instructions': ['Answer in a short way.', 'Keep it simple.'], + "hyperparameters": hyperparameters + } + + mock_completion_request = {"messages": [ + {'role': 'system', + 'content': 'You are a personal assistant. Answer the question according to the below context'}, + {'role': 'user', + 'content': "Based on below context answer question, if answer not in context check previous logs.\nInstructions on how to use Similarity Prompt:\n[{'name': 'Fahad', 'city': 'Mumbai'}, {'name': 'Hitesh', 'city': 'Mumbai'}]\nAnswer according to this context.\n \nAnswer in a short way.\nKeep it simple. \nQ: List all the user lives in mumbai city \nA:"} + ]} + mock_completion_request.update(hyperparameters) + mock_embedding.return_value = {'data': [{'embedding': embedding}]} + mock_completion.return_value = {'choices': [{'message': {'content': generated_text, 'role': 'assistant'}}]} + + gpt3 = LLMProcessor(bot, DEFAULT_LLM) + aioresponses.add( + url=urljoin(Utility.environment['vector']['db'], + f"/collections/{gpt3.bot}_{test_content1.collection}{gpt3.suffix}/points/search"), + method="POST", + payload={'result': [ + {'id': test_content2.vector_id, 'score': 0.80, "payload": test_content2.data}, + {'id': test_content3.vector_id, 'score': 0.80, "payload": test_content3.data} + ]} + ) + + response, time_elapsed = await gpt3.predict(query, user=user, **k_faq_action_config) + assert response['content'] == generated_text + assert gpt3.logs == [{'messages': [{'role': 'system', + 'content': 'You are a personal assistant. Answer the question according to the below context'}, + {'role': 'user', + 'content': "Based on below context answer question, if answer not in context check previous logs.\nInstructions on how to use Similarity Prompt:\n[{'name': 'Fahad', 'city': 'Mumbai'}, {'name': 'Hitesh', 'city': 'Mumbai'}]\nAnswer according to this context.\n \nAnswer in a short way.\nKeep it simple. \nQ: List all the user lives in mumbai city \nA:"}], + 'raw_completion_response': {'choices': [{'message': { + 'content': 'Hitesh and Fahad lives in mumbai city.', 'role': 'assistant'}}]}, + 'type': 'answer_query', + 'hyperparameters': {'temperature': 0.0, 'max_tokens': 300, 'model': 'gpt-3.5-turbo', + 'top_p': 0.0, 'n': 1, 'stop': None, + 'presence_penalty': 0.0, 'frequency_penalty': 0.0, 'logit_bias': {}}}] + + assert list(aioresponses.requests.values())[0][0].kwargs['json'] == {'vector': embedding, 'limit': 10, + 'with_payload': True, + 'score_threshold': 0.70} + + assert isinstance(time_elapsed, float) and time_elapsed > 0.0 + + expected = {"model": "text-embedding-3-small", + "input": [query], 'metadata': {'user': user, 'bot': bot, 'invocation': None}, + "api_key": key, + "num_retries": 3} + assert not DeepDiff(mock_embedding.call_args[1], expected, ignore_order=True) + expected = mock_completion_request.copy() + expected['metadata'] = {'user': user, 'bot': bot, 'invocation': None} + expected['api_key'] = key + expected['num_retries'] = 3 + assert not DeepDiff(mock_completion.call_args[1], expected, ignore_order=True) + + @pytest.mark.asyncio + @mock.patch.object(litellm, "acompletion", autospec=True) + @mock.patch.object(litellm, "aembedding", autospec=True) + async def test_gpt3_faq_embedding_predict_completion_connection_error(self, mock_embedding, mock_completion, + aioresponses): + embedding = list(np.random.random(LLMProcessor.__embedding__)) + bot = "test_gpt3_faq_embedding_predict_completion_connection_error" + user = 'test' + key = "test" test_content = CognitionData( data="Python is a high-level, general-purpose programming language. Its design philosophy emphasizes code readability with the use of significant indentation. Python is dynamically typed and garbage-collected.", - collection='python', bot="test_embed_faq_predict", user="test").save() + collection='python', bot=bot, user=user).save() + BotSecrets(secret_type=BotSecretType.gpt_key.value, value=key, bot=test_content.bot, user=user).save() + hyperparameters = Utility.get_default_llm_hyperparameters() generated_text = "Python is dynamically typed, garbage-collected, high level, general purpose programming." query = "What kind of language is python?" + k_faq_action_config = { "system_prompt": "You are a personal assistant. Answer the question according to the below context", "context_prompt": "Based on below context answer question, if answer not in context check previous logs.", "similarity_prompt": [{"top_results": 10, "similarity_threshold": 0.70, 'use_similarity_prompt': True, - 'similarity_prompt_name': 'Similarity Prompt', - 'similarity_prompt_instructions': 'Answer according to this context.', - "collection": 'python'}]} + 'similarity_prompt_name': 'Similarity Prompt', + 'similarity_prompt_instructions': 'Answer according to this context.', + "collection": 'python'}], + "hyperparameters": hyperparameters + } def __mock_connection_error(*args, **kwargs): - import openai + raise Exception("Connection reset by peer!") - raise openai.error.APIConnectionError("Connection reset by peer!") - - mock_embedding.return_value = embedding + mock_embedding.return_value = {'data': [{'embedding': embedding}]} mock_completion.side_effect = __mock_connection_error - with mock.patch.dict(Utility.environment, {'llm': {"faq": "GPT3_FAQ_EMBED", 'api_key': 'test'}}): - gpt3 = GPT3FAQEmbedding(test_content.bot, LLMSettings(provider="openai").to_mongo().to_dict()) + gpt3 = LLMProcessor(test_content.bot, DEFAULT_LLM) - aioresponses.add( - url=urljoin(Utility.environment['vector']['db'], f"/collections/{gpt3.bot}_{test_content.collection}{gpt3.suffix}/points/search"), - method="POST", - payload={'result': [ - {'id': test_content.vector_id, 'score': 0.80, "payload": {'content': test_content.data}}]} - ) - - response, time_elapsed = await gpt3.predict(query, **k_faq_action_config) - print(mock_completion.call_args.args[3]) + aioresponses.add( + url=urljoin(Utility.environment['vector']['db'], + f"/collections/{gpt3.bot}_{test_content.collection}{gpt3.suffix}/points/search"), + method="POST", + payload={'result': [ + {'id': test_content.vector_id, 'score': 0.80, "payload": {'content': test_content.data}}]} + ) - assert response == {'exception': "Connection reset by peer!", 'is_failure': True, "content": None} + response, time_elapsed = await gpt3.predict(query, user=user, **k_faq_action_config) + assert response == {'exception': "Connection reset by peer!", 'is_failure': True, "content": None} - assert mock_embedding.call_args.args[1] == query + assert gpt3.logs == [{'error': 'Retrieving chat completion for the provided query. Connection reset by peer!'}] - assert mock_completion.call_args.args[1] == 'What kind of language is python?' - assert mock_completion.call_args.args[ - 2] == 'You are a personal assistant. Answer the question according to the below context' - assert mock_completion.call_args.args[3] == """Based on below context answer question, if answer not in context check previous logs.\nInstructions on how to use Similarity Prompt:\n['Python is a high-level, general-purpose programming language. Its design philosophy emphasizes code readability with the use of significant indentation. Python is dynamically typed and garbage-collected.']\nAnswer according to this context.\n""" - assert mock_completion.call_args.kwargs == {'similarity_prompt': [ - {'top_results': 10, 'similarity_threshold': 0.7, 'use_similarity_prompt': True, - 'similarity_prompt_name': 'Similarity Prompt', - 'similarity_prompt_instructions': 'Answer according to this context.', 'collection': 'python'}]} - assert gpt3.logs == [{'error': 'Retrieving chat completion for the provided query. Connection reset by peer!'}] + assert list(aioresponses.requests.values())[0][0].kwargs['json'] == {'vector': embedding, 'limit': 10, + 'with_payload': True, + 'score_threshold': 0.70} + assert isinstance(time_elapsed, float) and time_elapsed > 0.0 - assert list(aioresponses.requests.values())[0][0].kwargs['json'] == {'vector': embedding, 'limit': 10, 'with_payload': True, 'score_threshold': 0.70} - assert isinstance(time_elapsed, float) and time_elapsed > 0.0 + expected = {"model": "text-embedding-3-small", + "input": [query], 'metadata': {'user': user, 'bot': bot, 'invocation': None}, + "api_key": key, + "num_retries": 3} + assert not DeepDiff(mock_embedding.call_args[1], expected, ignore_order=True) + expected = {'messages': [{'role': 'system', + 'content': 'You are a personal assistant. Answer the question according to the below context'}, + {'role': 'user', + 'content': "Based on below context answer question, if answer not in context check previous logs.\nInstructions on how to use Similarity Prompt:\n['Python is a high-level, general-purpose programming language. Its design philosophy emphasizes code readability with the use of significant indentation. Python is dynamically typed and garbage-collected.']\nAnswer according to this context.\n \nQ: What kind of language is python? \nA:"}], + 'metadata': {'user': 'test', 'bot': 'test_gpt3_faq_embedding_predict_completion_connection_error', 'invocation': None}, + 'api_key': 'test', 'num_retries': 3, 'temperature': 0.0, 'max_tokens': 300, + 'model': 'gpt-3.5-turbo', 'top_p': 0.0, 'n': 1, 'stop': None, + 'presence_penalty': 0.0, 'frequency_penalty': 0.0, 'logit_bias': {}} + assert not DeepDiff(mock_completion.call_args[1], expected, ignore_order=True) @pytest.mark.asyncio @mock.patch("kairon.shared.rest_client.AioRestClient._AioRestClient__trigger", autospec=True) - @mock.patch.object(GPT3FAQEmbedding, "_GPT3FAQEmbedding__get_embedding", autospec=True) + @mock.patch.object(litellm, "aembedding", autospec=True) async def test_gpt3_faq_embedding_predict_exact_match(self, mock_embedding, mock_llm_request): - embedding = list(np.random.random(GPT3FAQEmbedding.__embedding__)) - + embedding = list(np.random.random(LLMProcessor.__embedding__)) + user = "test" + bot = "test_gpt3_faq_embedding_predict_exact_match" + key = 'test' test_content = CognitionData( data="Python is a high-level, general-purpose programming language. Its design philosophy emphasizes code readability with the use of significant indentation. Python is dynamically typed and garbage-collected.", - collection='python', bot="test_embed_faq_predict", user="test").save() + collection='python', bot=bot, user=user).save() + BotSecrets(secret_type=BotSecretType.gpt_key.value, value=key, bot=test_content.bot, user=user).save() query = "What kind of language is python?" + hyperparameters = Utility.get_default_llm_hyperparameters() k_faq_action_config = { "system_prompt": "You are a personal assistant. Answer the question according to the below context", "context_prompt": "Based on below context answer question, if answer not in context check previous logs.", "similarity_prompt": [{"top_results": 10, "similarity_threshold": 0.70, 'use_similarity_prompt': True, - 'similarity_prompt_name': 'Similarity Prompt', - 'similarity_prompt_instructions': 'Answer according to this context.', - "collection": 'python'}]} + 'similarity_prompt_name': 'Similarity Prompt', + 'similarity_prompt_instructions': 'Answer according to this context.', + "collection": 'python'}], + "hyperparameters": hyperparameters + } - mock_embedding.return_value = embedding + mock_embedding.return_value = {'data': [{'embedding': embedding}]} mock_llm_request.side_effect = ClientConnectionError() - with mock.patch.dict(Utility.environment, {'llm': {"faq": "GPT3_FAQ_EMBED", 'api_key': 'test'}}): - gpt3 = GPT3FAQEmbedding(test_content.bot, LLMSettings(provider="openai").to_mongo().to_dict()) + gpt3 = LLMProcessor(test_content.bot, DEFAULT_LLM) - response, time_elapsed = await gpt3.predict(query, **k_faq_action_config) - assert response == {'exception': 'Failed to connect to service: localhost', 'is_failure': True, "content": None} + response, time_elapsed = await gpt3.predict(query, user="test", **k_faq_action_config) + assert response == {'exception': 'Failed to connect to service: localhost', 'is_failure': True, "content": None} - assert mock_embedding.call_args.args[1] == query - assert gpt3.logs == [] - assert isinstance(time_elapsed, float) and time_elapsed > 0.0 + assert gpt3.logs == [ + {'error': 'Retrieving chat completion for the provided query. Failed to connect to service: localhost'}] + assert isinstance(time_elapsed, float) and time_elapsed > 0.0 + + expected = {"model": "text-embedding-3-small", + "input": [query], 'metadata': {'user': user, 'bot': bot, 'invocation': None}, + "api_key": key, + "num_retries": 3} + assert not DeepDiff(mock_embedding.call_args[1], expected, ignore_order=True) @pytest.mark.asyncio - @mock.patch.object(GPT3FAQEmbedding, "_GPT3FAQEmbedding__get_embedding", autospec=True) + @mock.patch.object(litellm, "aembedding", autospec=True) async def test_gpt3_faq_embedding_predict_embedding_connection_error(self, mock_embedding): - import openai - - embedding = list(np.random.random(GPT3FAQEmbedding.__embedding__)) - + embedding = list(np.random.random(LLMProcessor.__embedding__)) + user = "test" + bot = "test_gpt3_faq_embedding_predict_embedding_connection_error" + key = "test" test_content = CognitionData( data="Python is a high-level, general-purpose programming language. Its design philosophy emphasizes code readability with the use of significant indentation. Python is dynamically typed and garbage-collected.", - bot="test_embed_faq_predict", user="test").save() + bot=bot, user=user).save() + BotSecrets(secret_type=BotSecretType.gpt_key.value, value=key, bot=test_content.bot, user=user).save() + hyperparameters = Utility.get_default_llm_hyperparameters() generated_text = "Python is dynamically typed, garbage-collected, high level, general purpose programming." query = "What kind of language is python?" + k_faq_action_config = { "system_prompt": "You are a personal assistant. Answer the question according to the below context", - "context_prompt": "Based on below context answer question, if answer not in context check previous logs."} + "context_prompt": "Based on below context answer question, if answer not in context check previous logs.", + "hyperparameters": hyperparameters + } + mock_embedding.side_effect = [Exception("Connection reset by peer!"), {'data': [{'embedding': embedding}]}] - mock_embedding.side_effect = [openai.error.APIConnectionError("Connection reset by peer!"), embedding] + gpt3 = LLMProcessor(test_content.bot, DEFAULT_LLM) + mock_embedding.side_effect = [Exception("Connection reset by peer!"), embedding] - with mock.patch.dict(Utility.environment, {'llm': {"faq": "GPT3_FAQ_EMBED", 'api_key': 'test'}}): - gpt3 = GPT3FAQEmbedding(test_content.bot, LLMSettings(provider="openai").to_mongo().to_dict()) + response, time_elapsed = await gpt3.predict(query, user="test", **k_faq_action_config) + assert response == {'exception': 'Connection reset by peer!', 'is_failure': True, "content": None} - response, time_elapsed = await gpt3.predict(query, **k_faq_action_config) - assert response == {'exception': 'Connection reset by peer!', 'is_failure': True, "content": None} + assert gpt3.logs == [{'error': 'Creating a new embedding for the provided query. Connection reset by peer!'}] + assert isinstance(time_elapsed, float) and time_elapsed > 0.0 - assert mock_embedding.call_args.args[1] == query - assert gpt3.logs == [{'error': 'Creating a new embedding for the provided query. Connection reset by peer!'}] - assert isinstance(time_elapsed, float) and time_elapsed > 0.0 + expected = {"model": "text-embedding-3-small", + "input": [query], 'metadata': {'user': user, 'bot': bot, 'invocation': None}, + "api_key": key, + "num_retries": 3} + assert not DeepDiff(mock_embedding.call_args[1], expected, ignore_order=True) @pytest.mark.asyncio - async def test_gpt3_faq_embedding_predict_with_previous_bot_responses(self, aioresponses): - embedding = list(np.random.random(GPT3FAQEmbedding.__embedding__)) + @mock.patch.object(litellm, "acompletion", autospec=True) + @mock.patch.object(litellm, "aembedding", autospec=True) + async def test_gpt3_faq_embedding_predict_with_previous_bot_responses(self, mock_embedding, mock_completion, + aioresponses): + embedding = list(np.random.random(LLMProcessor.__embedding__)) - bot = "test_embed_faq_predict" + bot = "test_gpt3_faq_embedding_predict_with_previous_bot_responses" user = "test" + key = "test" + hyperparameters = Utility.get_default_llm_hyperparameters() test_content = CognitionData( data="Python is a high-level, general-purpose programming language. Its design philosophy emphasizes code readability with the use of significant indentation. Python is dynamically typed and garbage-collected.", collection='python', bot=bot, user=user).save() + BotSecrets(secret_type=BotSecretType.gpt_key.value, value=key, bot=test_content.bot, user=user).save() generated_text = "Python is dynamically typed, garbage-collected, high level, general purpose programming." query = "What kind of language is python?" @@ -940,9 +1049,10 @@ async def test_gpt3_faq_embedding_predict_with_previous_bot_responses(self, aior ], "similarity_prompt": [{'use_similarity_prompt': True, 'similarity_prompt_name': 'Similarity Prompt', 'similarity_prompt_instructions': 'Answer according to this context.', - "collection": 'python'}] + "collection": 'python'}], + "hyperparameters": hyperparameters } - hyperparameters = Utility.get_llm_hyperparameters() + mock_completion_request = {"messages": [ {'role': 'system', 'content': 'You are a personal assistant. Answer question based on the context below'}, {'role': 'user', 'content': 'hello'}, @@ -951,69 +1061,68 @@ async def test_gpt3_faq_embedding_predict_with_previous_bot_responses(self, aior 'content': "Answer question based on the context below, if answer is not in the context go check previous logs.\nInstructions on how to use Similarity Prompt:\n['Python is a high-level, general-purpose programming language. Its design philosophy emphasizes code readability with the use of significant indentation. Python is dynamically typed and garbage-collected.']\nAnswer according to this context.\n \nQ: What kind of language is python? \nA:"} ]} mock_completion_request.update(hyperparameters) - request_header = {"Authorization": "Bearer knupur"} - - aioresponses.add( - url="https://api.openai.com/v1/embeddings", - method="POST", - status=200, - payload={'data': [{'embedding': embedding}]} - ) - - aioresponses.add( - url="https://api.openai.com/v1/chat/completions", - method="POST", - status=200, - payload={'choices': [{'message': {'content': generated_text, 'role': 'assistant'}}]} - ) + mock_embedding.return_value = {'data': [{'embedding': embedding}]} + mock_completion.return_value = {'choices': [{'message': {'content': generated_text, 'role': 'assistant'}}]} - gpt3 = GPT3FAQEmbedding(test_content.bot, LLMSettings(provider="openai").to_mongo().to_dict()) + gpt3 = LLMProcessor(test_content.bot, DEFAULT_LLM) aioresponses.add( - url=urljoin(Utility.environment['vector']['db'], f"/collections/{gpt3.bot}_{test_content.collection}{gpt3.suffix}/points/search"), + url=urljoin(Utility.environment['vector']['db'], + f"/collections/{gpt3.bot}_{test_content.collection}{gpt3.suffix}/points/search"), method="POST", payload={'result': [ {'id': test_content.vector_id, 'score': 0.80, "payload": {'content': test_content.data}}]} ) - response, time_elapsed = await gpt3.predict(query, **k_faq_action_config) - print(list(aioresponses.requests.values())[2][0].kwargs['json']) + response, time_elapsed = await gpt3.predict(query, user=user, **k_faq_action_config) assert response['content'] == generated_text - assert list(aioresponses.requests.values())[0][0].kwargs['json'] == {"model": "text-embedding-3-small", - "input": query} - assert list(aioresponses.requests.values())[0][0].kwargs['headers'] == request_header - - assert list(aioresponses.requests.values())[1][0].kwargs['json'] == {'vector': embedding, 'limit': 10, + assert list(aioresponses.requests.values())[0][0].kwargs['json'] == {'vector': embedding, 'limit': 10, 'with_payload': True, 'score_threshold': 0.70} - assert list(aioresponses.requests.values())[2][0].kwargs['json'] == mock_completion_request - assert list(aioresponses.requests.values())[2][0].kwargs['headers'] == request_header assert isinstance(time_elapsed, float) and time_elapsed > 0.0 + expected = {"model": "text-embedding-3-small", + "input": [query], 'metadata': {'user': user, 'bot': bot, 'invocation': None}, + "api_key": key, + "num_retries": 3} + assert not DeepDiff(mock_embedding.call_args[1], expected, ignore_order=True) + expected = mock_completion_request.copy() + expected['metadata'] = {'user': user, 'bot': bot, 'invocation': None} + expected['api_key'] = key + expected['num_retries'] = 3 + assert not DeepDiff(mock_completion.call_args[1], expected, ignore_order=True) + @pytest.mark.asyncio - async def test_gpt3_faq_embedding_predict_with_query_prompt(self, aioresponses): - embedding = list(np.random.random(GPT3FAQEmbedding.__embedding__)) + @mock.patch.object(litellm, "acompletion", autospec=True) + @mock.patch.object(litellm, "aembedding", autospec=True) + async def test_gpt3_faq_embedding_predict_with_query_prompt(self, mock_embedding, mock_completion, aioresponses): + embedding = list(np.random.random(LLMProcessor.__embedding__)) - bot = "test_embed_faq_predict" + bot = "test_gpt3_faq_embedding_predict_with_query_prompt" user = "test" + key = "test" test_content = CognitionData( data="Python is a high-level, general-purpose programming language. Its design philosophy emphasizes code readability with the use of significant indentation. Python is dynamically typed and garbage-collected.", collection='python', bot=bot, user=user).save() + BotSecrets(secret_type=BotSecretType.gpt_key.value, value=key, bot=test_content.bot, user=user).save() generated_text = "Python is dynamically typed, garbage-collected, high level, general purpose programming." query = "What kind of language is python?" rephrased_query = "Explain python is called high level programming language in laymen terms?" + hyperparameters = Utility.get_default_llm_hyperparameters() + k_faq_action_config = {"query_prompt": { "query_prompt": "A programming language is a system of notation for writing computer programs.[1] Most programming languages are text-based formal languages, but they may also be graphical. They are a kind of computer language.", "use_query_prompt": True}, - "similarity_prompt": [ - {'use_similarity_prompt': True, 'similarity_prompt_name': 'Similarity Prompt', - 'similarity_prompt_instructions': 'Answer according to this context.', - "collection": 'python'}] - } - hyperparameters = Utility.get_llm_hyperparameters() + "similarity_prompt": [ + {'use_similarity_prompt': True, 'similarity_prompt_name': 'Similarity Prompt', + 'similarity_prompt_instructions': 'Answer according to this context.', + "collection": 'python'}], + "hyperparameters": hyperparameters + } + mock_rephrase_request = {"messages": [ {"role": "system", "content": DEFAULT_SYSTEM_PROMPT}, @@ -1029,51 +1138,110 @@ async def test_gpt3_faq_embedding_predict_with_query_prompt(self, aioresponses): ]} mock_rephrase_request.update(hyperparameters) mock_completion_request.update(hyperparameters) - request_header = {"Authorization": "Bearer knupur"} - aioresponses.add( - url="https://api.openai.com/v1/embeddings", - method="POST", - status=200, - payload={'data': [{'embedding': embedding}]} - ) + mock_embedding.return_value = {'data': [{'embedding': embedding}]} + mock_completion.side_effect = {'choices': [{'message': {'content': rephrased_query, 'role': 'assistant'}}]}, { + 'choices': [{'message': {'content': generated_text, 'role': 'assistant'}}]} - aioresponses.add( - url="https://api.openai.com/v1/chat/completions", - method="POST", - status=200, - payload={'choices': [{'message': {'content': rephrased_query, 'role': 'assistant'}}]} - ) + gpt3 = LLMProcessor(test_content.bot, DEFAULT_LLM) aioresponses.add( - url="https://api.openai.com/v1/chat/completions", - method="POST", - status=200, - payload={'choices': [{'message': {'content': generated_text, 'role': 'assistant'}}]}, - repeat=True - ) - - gpt3 = GPT3FAQEmbedding(test_content.bot, LLMSettings(provider="openai").to_mongo().to_dict()) - - aioresponses.add( - url=urljoin(Utility.environment['vector']['db'], f"/collections/{gpt3.bot}_{test_content.collection}{gpt3.suffix}/points/search"), + url=urljoin(Utility.environment['vector']['db'], + f"/collections/{gpt3.bot}_{test_content.collection}{gpt3.suffix}/points/search"), method="POST", payload={'result': [ {'id': test_content.vector_id, 'score': 0.80, "payload": {'content': test_content.data}}]} ) - response, time_elapsed = await gpt3.predict(query, **k_faq_action_config) - print(list(aioresponses.requests.values())[2][1].kwargs['json']) + response, time_elapsed = await gpt3.predict(query, user=user, **k_faq_action_config) assert response['content'] == generated_text - assert list(aioresponses.requests.values())[0][0].kwargs['json'] == {"model": "text-embedding-3-small", - "input": query} - assert list(aioresponses.requests.values())[0][0].kwargs['headers'] == request_header - assert list(aioresponses.requests.values())[1][0].kwargs['json'] == {'vector': embedding, 'limit': 10, + assert list(aioresponses.requests.values())[0][0].kwargs['json'] == {'vector': embedding, 'limit': 10, 'with_payload': True, 'score_threshold': 0.70} - assert list(aioresponses.requests.values())[2][0].kwargs['json'] == mock_rephrase_request - assert list(aioresponses.requests.values())[2][0].kwargs['headers'] == request_header - assert list(aioresponses.requests.values())[2][1].kwargs['json'] == mock_completion_request - assert list(aioresponses.requests.values())[2][1].kwargs['headers'] == request_header assert isinstance(time_elapsed, float) and time_elapsed > 0.0 + + expected = {"model": "text-embedding-3-small", + "input": [query], 'metadata': {'user': user, 'bot': bot, 'invocation': None}, + "api_key": key, + "num_retries": 3} + assert not DeepDiff(mock_embedding.call_args[1], expected, ignore_order=True) + expected = mock_completion_request.copy() + expected['metadata'] = {'user': user, 'bot': bot, 'invocation': None} + expected['api_key'] = key + expected['num_retries'] = 3 + assert not DeepDiff(mock_completion.call_args[1], expected, ignore_order=True) + + @pytest.mark.asyncio + async def test_llm_logging(self): + from kairon.shared.llm.logger import LiteLLMLogger + bot = "test_llm_logging" + user = "test" + litellm.callbacks = [LiteLLMLogger()] + + messages = [{"role":"user", "content":"Hi"}] + expected = "Hi, How may i help you?" + + result = await litellm.acompletion(messages=messages, + model="gpt-3.5-turbo", + mock_response=expected, + metadata={'user': user, 'bot': bot, 'invocation': None}) + assert result['choices'][0]['message']['content'] == expected + + result = litellm.completion(messages=messages, + model="gpt-3.5-turbo", + mock_response=expected, + metadata={'user': user, 'bot': bot, 'invocation': None}) + assert result['choices'][0]['message']['content'] == expected + + result = litellm.completion(messages=messages, + model="gpt-3.5-turbo", + mock_response=expected, + stream=True, + metadata={'user': user, 'bot': bot, 'invocation': None}) + response = '' + for chunk in result: + content = chunk["choices"][0]["delta"]["content"] + if content: + response = response + content + + assert response == expected + + result = await litellm.acompletion(messages=messages, + model="gpt-3.5-turbo", + mock_response=expected, + stream=True, + metadata={'user': user, 'bot': bot, 'invocation': None}) + response = '' + async for chunk in result: + content = chunk["choices"][0]["delta"]["content"] + print(chunk) + if content: + response += content + + assert response.__contains__(expected) + + with pytest.raises(Exception) as e: + await litellm.acompletion(messages=messages, + model="gpt-3.5-turbo", + mock_response=Exception("Authentication error"), + metadata={'user': user, 'bot': bot, 'invocation': None}) + + assert str(e) == "Authentication error" + + with pytest.raises(Exception) as e: + litellm.completion(messages=messages, + model="gpt-3.5-turbo", + mock_response=Exception("Authentication error"), + metadata={'user': user, 'bot': bot, 'invocation': None}) + + assert str(e) == "Authentication error" + + with pytest.raises(Exception) as e: + await litellm.acompletion(messages=messages, + model="gpt-3.5-turbo", + mock_response=Exception("Authentication error"), + stream=True, + metadata={'user': user, 'bot': bot, 'invocation': None}) + + assert str(e) == "Authentication error" \ No newline at end of file diff --git a/tests/unit_test/plugins_test.py b/tests/unit_test/plugins_test.py index cc24f8f04..bb9de7443 100644 --- a/tests/unit_test/plugins_test.py +++ b/tests/unit_test/plugins_test.py @@ -1,7 +1,7 @@ import os import re -import mock +from unittest import mock import pytest import requests import responses @@ -11,7 +11,6 @@ from kairon.shared.constants import PluginTypes from kairon.shared.plugins.factory import PluginFactory from kairon.shared.utils import Utility -from mongomock import MongoClient class TestUtility: diff --git a/tests/unit_test/rest_client_test.py b/tests/unit_test/rest_client_test.py index 0d6afa160..6daa1e5e4 100644 --- a/tests/unit_test/rest_client_test.py +++ b/tests/unit_test/rest_client_test.py @@ -1,4 +1,5 @@ import asyncio +import ujson as json from unittest import mock import pytest @@ -101,3 +102,19 @@ async def test_aio_rest_client_timeout_error(self, aioresponses): with pytest.raises(AppException, match="Request timed out: Request timed out"): await AioRestClient().request("get", url, request_body={"name": "udit.pandey", "loc": "blr"}, headers={"Authorization": "Bearer sasdfghjkytrtyui"}, max_retries=3) + + @pytest.mark.asyncio + async def test_aio_rest_client_post_request_stream(self, aioresponses): + url = 'http://kairon.com' + aioresponses.post("http://kairon.com", status=200, body=json.dumps({'data': 'hi!'})) + resp = await AioRestClient().request("post", url, request_body={"name": "udit.pandey", "loc": "blr"}, + headers={"Authorization": "Bearer sasdfghjkytrtyui"}, is_streaming_resp=True) + response = '' + async for content in resp.content: + response += content.decode() + + assert json.loads(response) == {"data": "hi!"} + assert list(aioresponses.requests.values())[0][0].kwargs == {'allow_redirects': True, 'headers': { + 'Authorization': 'Bearer sasdfghjkytrtyui'}, 'json': {'loc': 'blr', 'name': 'udit.pandey'}, 'timeout': None, + 'data': None, + 'trace_request_ctx': {'current_attempt': 1}} \ No newline at end of file diff --git a/tests/unit_test/utility_test.py b/tests/unit_test/utility_test.py index b32c4a3b8..5fa44f128 100644 --- a/tests/unit_test/utility_test.py +++ b/tests/unit_test/utility_test.py @@ -8,6 +8,9 @@ from io import BytesIO from unittest.mock import patch, MagicMock from urllib.parse import urlencode +from kairon.shared.utils import Utility, MailUtility + +Utility.load_system_metadata() import numpy as np import pandas as pd @@ -36,12 +39,7 @@ from kairon.shared.data.data_objects import EventConfig, Slots, LLMSettings, DemoRequestLogs from kairon.shared.data.processor import MongoProcessor from kairon.shared.data.utils import DataUtility -from kairon.shared.llm.clients.azure import AzureGPT3Resources -from kairon.shared.llm.clients.factory import LLMClientFactory -from kairon.shared.llm.clients.gpt3 import GPT3Resources -from kairon.shared.llm.gpt3 import GPT3FAQEmbedding from kairon.shared.models import TemplateType -from kairon.shared.utils import Utility, MailUtility from kairon.shared.verification.email import QuickEmailVerification @@ -2122,7 +2120,7 @@ def publish_auditlog(*args, **kwargs): return None monkeypatch.setattr(AuditDataProcessor, "publish_auditlog", publish_auditlog) - bot = "tests" + bot = "publish_auditlog" user = "testuser" event_config = EventConfig( bot=bot, user=user, ws_url="http://localhost:5000/event_url" @@ -2134,14 +2132,14 @@ def publish_auditlog(*args, **kwargs): count = AuditLogData.objects( attributes=[{"key": "bot", "value": bot}], user=user, action="save" ).count() - assert count == 2 + assert count == 1 def test_save_and_publish_auditlog_action_save_another(self, monkeypatch): def publish_auditlog(*args, **kwargs): return None monkeypatch.setattr(AuditDataProcessor, "publish_auditlog", publish_auditlog) - bot = "tests" + bot = "publish_auditlog" user = "testuser" event_config = EventConfig( bot=bot, @@ -2157,14 +2155,14 @@ def publish_auditlog(*args, **kwargs): count = AuditLogData.objects( attributes=[{"key": "bot", "value": bot}], user=user, action="save" ).count() - assert count == 3 + assert count == 2 def test_save_and_publish_auditlog_action_update(self, monkeypatch): def publish_auditlog(*args, **kwargs): return None monkeypatch.setattr(AuditDataProcessor, "publish_auditlog", publish_auditlog) - bot = "tests" + bot = "publish_auditlog" user = "testuser" event_config = EventConfig( bot=bot, @@ -2179,14 +2177,14 @@ def publish_auditlog(*args, **kwargs): count = AuditLogData.objects( attributes=[{"key": "bot", "value": bot}], user=user, action="update" ).count() - assert count == 2 + assert count == 1 def test_save_and_publish_auditlog_total_count(self, monkeypatch): def publish_auditlog(*args, **kwargs): return None monkeypatch.setattr(AuditDataProcessor, "publish_auditlog", publish_auditlog) - bot = "tests" + bot = "publish_auditlog" user = "testuser" event_config = EventConfig( bot=bot, @@ -2208,7 +2206,7 @@ def execute_http_request(*args, **kwargs): return None monkeypatch.setattr(Utility, "execute_http_request", execute_http_request) - bot = "tests" + bot = "publish_auditlog" user = "testuser" event_config = EventConfig( bot=bot, @@ -2223,11 +2221,11 @@ def execute_http_request(*args, **kwargs): count = AuditLogData.objects( attributes=[{"key": "bot", "value": bot}], user=user ).count() - assert count >= 3 + assert count >= 2 @responses.activate def test_publish_auditlog(self): - bot = "secret" + bot = "publish_auditlog" user = "secret_user" config = { "bot_user_oAuth_token": "xoxb-801939352912-801478018484-v3zq6MYNu62oSs8vammWOY8K", @@ -2263,7 +2261,7 @@ def test_publish_auditlog(self): count = AuditLogData.objects( attributes=[{"key": "bot", "value": bot}], user=user ).count() - assert count == 4 + assert count == 1 @pytest.mark.asyncio async def test_messageConverter_messenger_button_one(self): @@ -2956,14 +2954,13 @@ def test_verify_email_enable_valid_email(self): Utility.verify_email(email) def test_get_llm_hyperparameters(self): - hyperparameters = Utility.get_llm_hyperparameters() + hyperparameters = Utility.get_llm_hyperparameters("openai") assert hyperparameters == { "temperature": 0.0, "max_tokens": 300, "model": "gpt-3.5-turbo", "top_p": 0.0, "n": 1, - "stream": False, "stop": None, "presence_penalty": 0.0, "frequency_penalty": 0.0, @@ -2971,508 +2968,10 @@ def test_get_llm_hyperparameters(self): } def test_get_llm_hyperparameters_not_found(self, monkeypatch): - monkeypatch.setitem(Utility.environment["llm"], "faq", None) - with pytest.raises( - AppException, match="Could not find any hyperparameters for configured LLM." - ): - Utility.get_llm_hyperparameters() - - @pytest.mark.asyncio - async def test_trigger_gpt3_client_completion_with_generated_text( - self, aioresponses - ): - api_key = "test" - generated_text = "Python is dynamically typed, garbage-collected, high level, general purpose programming." - messages = { - "messages": [ - {"role": "system", "content": DEFAULT_SYSTEM_PROMPT}, - { - "role": "user", - "content": "Answer question based on the context below, if answer is not in the context go check previous logs.\nSimilarity Prompt:\nPython is a high-level, general-purpose programming language. Its design philosophy emphasizes code readability with the use of significant indentation. Python is dynamically typed and garbage-collected.\nInstructions on how to use Similarity Prompt: Answer according to this context.\n \n Q: Explain python is called high level programming language in laymen terms?\n A:", - }, - ] - } - hyperparameters = Utility.get_llm_hyperparameters() - request_header = {"Authorization": f"Bearer {api_key}"} - mock_completion_request = messages - mock_completion_request.update(hyperparameters) - - aioresponses.add( - url="https://api.openai.com/v1/chat/completions", - method="POST", - status=200, - payload={ - "choices": [ - {"message": {"content": generated_text, "role": "assistant"}} - ] - }, - ) - - resp = await GPT3Resources("test").invoke( - GPT3ResourceTypes.chat_completion.value, **mock_completion_request - ) - assert resp[0] == generated_text - - assert ( - list(aioresponses.requests.values())[0][0].kwargs["json"] - == mock_completion_request - ) - assert ( - list(aioresponses.requests.values())[0][0].kwargs["headers"] - == request_header - ) - - @pytest.mark.asyncio - async def test_trigger_gpt3_client_completion_with_response(self, aioresponses): - api_key = "test" - generated_text = "Python is dynamically typed, garbage-collected, high level, general purpose programming." - hyperparameters = Utility.get_llm_hyperparameters() - request_header = {"Authorization": f"Bearer {api_key}"} - mock_completion_request = { - "messages": [ - {"role": "system", "content": DEFAULT_SYSTEM_PROMPT}, - { - "role": "user", - "content": "Answer question based on the context below, if answer is not in the context go check previous logs.\nSimilarity Prompt:\nPython is a high-level, general-purpose programming language. Its design philosophy emphasizes code readability with the use of significant indentation. Python is dynamically typed and garbage-collected.\nInstructions on how to use Similarity Prompt: Answer according to this context.\n \n Q: Explain python is called high level programming language in laymen terms?\n A:", - }, - ] - } - mock_completion_request.update(hyperparameters) - - aioresponses.add( - url="https://api.openai.com/v1/chat/completions", - method="POST", - status=200, - payload={ - "choices": [ - {"message": {"content": generated_text, "role": "assistant"}} - ] - }, - ) - - formatted_response, raw_response = await GPT3Resources("test").invoke( - GPT3ResourceTypes.chat_completion.value, **mock_completion_request - ) - assert formatted_response == generated_text - assert raw_response == { - "choices": [{"message": {"content": generated_text, "role": "assistant"}}] - } - - assert ( - list(aioresponses.requests.values())[0][0].kwargs["json"] - == mock_completion_request - ) - assert ( - list(aioresponses.requests.values())[0][0].kwargs["headers"] - == request_header - ) - - @pytest.mark.asyncio - async def test_trigger_gp3_client_completion(self, aioresponses): - api_key = "test" - generated_text = "Python is dynamically typed, garbage-collected, high level, general purpose programming." - hyperparameters = Utility.get_llm_hyperparameters() - request_header = {"Authorization": f"Bearer {api_key}"} - mock_completion_request = { - "messages": [ - {"role": "system", "content": DEFAULT_SYSTEM_PROMPT}, - { - "role": "user", - "content": "Answer question based on the context below, if answer is not in the context go check previous logs.\nSimilarity Prompt:\nPython is a high-level, general-purpose programming language. Its design philosophy emphasizes code readability with the use of significant indentation. Python is dynamically typed and garbage-collected.\nInstructions on how to use Similarity Prompt: Answer according to this context.\n \n Q: Explain python is called high level programming language in laymen terms?\n A:", - }, - ] - } - mock_completion_request.update(hyperparameters) - - aioresponses.add( - url="https://api.openai.com/v1/chat/completions", - method="POST", - status=200, - payload={ - "choices": [ - {"message": {"content": generated_text, "role": "assistant"}} - ] - }, - ) - formatted_response, raw_response = await GPT3Resources(api_key).invoke( - GPT3ResourceTypes.chat_completion.value, **mock_completion_request - ) - assert formatted_response == generated_text - assert raw_response == { - "choices": [{"message": {"content": generated_text, "role": "assistant"}}] - } - - assert ( - list(aioresponses.requests.values())[0][0].kwargs["json"] - == mock_completion_request - ) - assert ( - list(aioresponses.requests.values())[0][0].kwargs["headers"] - == request_header - ) - - @pytest.mark.asyncio - async def test_trigger_gp3_client_completion_failure(self, aioresponses): - api_key = "test" - hyperparameters = Utility.get_llm_hyperparameters() - request_header = {"Authorization": f"Bearer {api_key}"} - mock_completion_request = { - "messages": [ - {"role": "system", "content": DEFAULT_SYSTEM_PROMPT}, - { - "role": "user", - "content": "Answer question based on the context below, if answer is not in the context go check previous logs.\nSimilarity Prompt:\nPython is a high-level, general-purpose programming language. Its design philosophy emphasizes code readability with the use of significant indentation. Python is dynamically typed and garbage-collected.\nInstructions on how to use Similarity Prompt: Answer according to this context.\n \n Q: Explain python is called high level programming language in laymen terms?\n A:", - }, - ] - } - mock_completion_request.update(hyperparameters) - - aioresponses.add( - url="https://api.openai.com/v1/chat/completions", - method="POST", - status=504, - payload={"error": {"message": "Server unavailable!", "id": 876543456789}}, - ) - with pytest.raises( - AppException, match="Failed to connect to service: api.openai.com" - ): - await GPT3Resources(api_key).invoke( - GPT3ResourceTypes.chat_completion.value, **mock_completion_request - ) - - assert ( - list(aioresponses.requests.values())[0][0].kwargs["json"] - == mock_completion_request - ) - assert ( - list(aioresponses.requests.values())[0][0].kwargs["headers"] - == request_header - ) - - aioresponses.add( - url="https://api.openai.com/v1/chat/completions", - method="POST", - status=201, - body="openai".encode(), - repeat=True, - ) - with pytest.raises(AppException): - await GPT3Resources(api_key).invoke( - GPT3ResourceTypes.chat_completion.value, **mock_completion_request - ) - - @pytest.mark.asyncio - async def test_trigger_gp3_client_embedding(self, aioresponses): - api_key = "test" - query = "What kind of language is python?" - embedding = list(np.random.random(GPT3FAQEmbedding.__embedding__)) - request_header = {"Authorization": f"Bearer {api_key}"} - - aioresponses.add( - url="https://api.openai.com/v1/embeddings", - method="POST", - status=200, - payload={"data": [{"embedding": embedding}]}, - ) - formatted_response, raw_response = await GPT3Resources(api_key).invoke( - GPT3ResourceTypes.embeddings.value, - model="text-embedding-3-small", - input=query, - ) - assert formatted_response == embedding - assert raw_response == {"data": [{"embedding": embedding}]} - - assert list(aioresponses.requests.values())[0][0].kwargs["json"] == { - "model": "text-embedding-3-small", - "input": query, - } - assert ( - list(aioresponses.requests.values())[0][0].kwargs["headers"] - == request_header - ) - - @pytest.mark.asyncio - async def test_trigger_gp3_client_embedding_failure(self, aioresponses): - api_key = "test" - query = "What kind of language is python?" - request_header = {"Authorization": f"Bearer {api_key}"} - - aioresponses.add( - url="https://api.openai.com/v1/embeddings", method="POST", status=504 - ) - - with pytest.raises( - AppException, match="Failed to connect to service: api.openai.com" - ): - await GPT3Resources(api_key).invoke( - GPT3ResourceTypes.embeddings.value, - model="text-embedding-3-small", - input=query, - ) - - aioresponses.add( - url="https://api.openai.com/v1/embeddings", - method="POST", - status=204, - payload={"error": {"message": "Server unavailable!", "id": 876543456789}}, - repeat=True, - ) - - with pytest.raises( - AppException, match="Server unavailable!. Request id: 876543456789" - ): - await GPT3Resources(api_key).invoke( - GPT3ResourceTypes.embeddings.value, - model="text-embedding-3-small", - input=query, - ) - - assert list(aioresponses.requests.values())[0][0].kwargs["json"] == { - "model": "text-embedding-3-small", - "input": query, - } - assert ( - list(aioresponses.requests.values())[0][0].kwargs["headers"] - == request_header - ) - - assert list(aioresponses.requests.values())[0][1].kwargs["json"] == { - "model": "text-embedding-3-small", - "input": query, - } - assert ( - list(aioresponses.requests.values())[0][1].kwargs["headers"] - == request_header - ) - - @pytest.mark.asyncio - async def test_trigger_gp3_client_streaming_completion(self, aioresponses): - api_key = "test" - generated_text = "Python is dynamically typed, garbage-collected, high level, general purpose programming." - hyperparameters = Utility.get_llm_hyperparameters() - request_header = {"Authorization": f"Bearer {api_key}"} - mock_completion_request = { - "messages": [ - {"role": "system", "content": DEFAULT_SYSTEM_PROMPT}, - { - "role": "user", - "content": "Answer question based on the context below, if answer is not in the context go check previous logs.\nSimilarity Prompt:\nPython is a high-level, general-purpose programming language. Its design philosophy emphasizes code readability with the use of significant indentation. Python is dynamically typed and garbage-collected.\nInstructions on how to use Similarity Prompt: Answer according to this context.\n \n Q: Explain python is called high level programming language in laymen terms?\n A:", - }, - ] - } - mock_completion_request.update(hyperparameters) - mock_completion_request["stream"] = True - - content = """data: {"choices": [{"delta": {"role": "assistant"}, "index": 0, "finish_reason": null}]}\n\n -data: {"choices": [{"delta": {"content": "Python"}, "index": 0, "finish_reason": null}]}\n\n -data: {"choices": [{"delta": {"content": " is"}, "index": 0, "finish_reason": null}]}\n\n -data: {"choices": [{"delta": {"content": " dynamically"}, "index": 0, "finish_reason": null}]}\n\n -data: {"choices": [{"delta": {"content": " typed"}, "index": 0, "finish_reason": null}]}\n\n -data: {"choices": [{"delta": {"content": ","}, "index": 0, "finish_reason": null}]}\n\n -data: {"choices": [{"delta": {"content": " garbage-collected"}, "index": 0, "finish_reason": null}]}\n\n -data: {"choices": [{"delta": {"content": ","}, "index": 0, "finish_reason": null}]}\n\n -data: {"choices": [{"delta": {"content": " high"}, "index": 0, "finish_reason": null}]}\n\n -data: {"choices": [{"delta": {"content": " level"}, "index": 0, "finish_reason": null}]}\n\n -data: {"choices": [{"delta": {"content": ","}, "index": 0, "finish_reason": null}]}\n\n -data: {"choices": [{"delta": {"content": " general"}, "index": 0, "finish_reason": null}]}\n\n -data: {"choices": [{"delta": {"content": " purpose"}, "index": 0, "finish_reason": null}]}\n\n -data: {"choices": [{"delta": {"content": " programming"}, "index": 0, "finish_reason": null}]}\n\n -data: {"choices": [{"delta": {"content": "."}, "index": 0, "finish_reason": null}]}\n\n -data: {"choices": [{"delta": {}, "index": 0, "finish_reason": "stop"}]}\n\n -data: [DONE]\n\n""" - - aioresponses.add( - url="https://api.openai.com/v1/chat/completions", - method="POST", - status=200, - body=content.encode(), - content_type="text/event-stream", - ) - - formatted_response, raw_response = await GPT3Resources(api_key).invoke( - GPT3ResourceTypes.chat_completion.value, **mock_completion_request - ) - assert formatted_response == generated_text - assert raw_response == [ - b'data: {"choices": [{"delta": {"role": "assistant"}, "index": 0, "finish_reason": null}]}\n', - b"\n", - b"\n", - b'data: {"choices": [{"delta": {"content": "Python"}, "index": 0, "finish_reason": null}]}\n', - b"\n", - b"\n", - b'data: {"choices": [{"delta": {"content": " is"}, "index": 0, "finish_reason": null}]}\n', - b"\n", - b"\n", - b'data: {"choices": [{"delta": {"content": " dynamically"}, "index": 0, "finish_reason": null}]}\n', - b"\n", - b"\n", - b'data: {"choices": [{"delta": {"content": " typed"}, "index": 0, "finish_reason": null}]}\n', - b"\n", - b"\n", - b'data: {"choices": [{"delta": {"content": ","}, "index": 0, "finish_reason": null}]}\n', - b"\n", - b"\n", - b'data: {"choices": [{"delta": {"content": " garbage-collected"}, "index": 0, "finish_reason": null}]}\n', - b"\n", - b"\n", - b'data: {"choices": [{"delta": {"content": ","}, "index": 0, "finish_reason": null}]}\n', - b"\n", - b"\n", - b'data: {"choices": [{"delta": {"content": " high"}, "index": 0, "finish_reason": null}]}\n', - b"\n", - b"\n", - b'data: {"choices": [{"delta": {"content": " level"}, "index": 0, "finish_reason": null}]}\n', - b"\n", - b"\n", - b'data: {"choices": [{"delta": {"content": ","}, "index": 0, "finish_reason": null}]}\n', - b"\n", - b"\n", - b'data: {"choices": [{"delta": {"content": " general"}, "index": 0, "finish_reason": null}]}\n', - b"\n", - b"\n", - b'data: {"choices": [{"delta": {"content": " purpose"}, "index": 0, "finish_reason": null}]}\n', - b"\n", - b"\n", - b'data: {"choices": [{"delta": {"content": " programming"}, "index": 0, "finish_reason": null}]}\n', - b"\n", - b"\n", - b'data: {"choices": [{"delta": {"content": "."}, "index": 0, "finish_reason": null}]}\n', - b"\n", - b"\n", - b'data: {"choices": [{"delta": {}, "index": 0, "finish_reason": "stop"}]}\n', - b"\n", - b"\n", - b"data: [DONE]\n", - b"\n", - ] - assert ( - list(aioresponses.requests.values())[0][0].kwargs["json"] - == mock_completion_request - ) - assert ( - list(aioresponses.requests.values())[0][0].kwargs["headers"] - == request_header - ) - - @pytest.mark.asyncio - async def test_trigger_gp3_client_streaming_connection_error(self, aioresponses): - api_key = "test" - generated_text = "Python is dynamically typed, garbage-collected, high level, general purpose programming." - hyperparameters = Utility.get_llm_hyperparameters() - request_header = {"Authorization": f"Bearer {api_key}"} - mock_completion_request = { - "messages": [ - {"role": "system", "content": DEFAULT_SYSTEM_PROMPT}, - { - "role": "user", - "content": "Answer question based on the context below, if answer is not in the context go check previous logs.\nSimilarity Prompt:\nPython is a high-level, general-purpose programming language. Its design philosophy emphasizes code readability with the use of significant indentation. Python is dynamically typed and garbage-collected.\nInstructions on how to use Similarity Prompt: Answer according to this context.\n \n Q: Explain python is called high level programming language in laymen terms?\n A:", - }, - ] - } - mock_completion_request.update(hyperparameters) - mock_completion_request["stream"] = True - - aioresponses.add( - url="https://api.openai.com/v1/chat/completions", - method="POST", - status=401, - ) - - with pytest.raises( - AppException, - match=re.escape( - "Failed to execute the url: 401, message='Unauthorized', url=URL('https://api.openai.com/v1/chat/completions')" - ), - ): - await GPT3Resources(api_key).invoke( - GPT3ResourceTypes.chat_completion.value, **mock_completion_request - ) - - @pytest.mark.asyncio - async def test_trigger_gp3_client_streaming_completion_failure(self, aioresponses): - api_key = "test" - hyperparameters = Utility.get_llm_hyperparameters() - request_header = {"Authorization": f"Bearer {api_key}"} - mock_completion_request = { - "messages": [ - {"role": "system", "content": DEFAULT_SYSTEM_PROMPT}, - { - "role": "user", - "content": "Answer question based on the context below, if answer is not in the context go check previous logs.\nSimilarity Prompt:\nPython is a high-level, general-purpose programming language. Its design philosophy emphasizes code readability with the use of significant indentation. Python is dynamically typed and garbage-collected.\nInstructions on how to use Similarity Prompt: Answer according to this context.\n \n Q: Explain python is called high level programming language in laymen terms?\n A:", - }, - ] - } - mock_completion_request.update(hyperparameters) - mock_completion_request["stream"] = True - - content = "data: {'choices': [{'delta': {'role': 'assistant'}}]}\n\n" - aioresponses.add( - url="https://api.openai.com/v1/chat/completions", - method="POST", - status=200, - body=content.encode(), - content_type="text/event-stream", - ) - with pytest.raises( - AppException, - match=re.escape( - "Failed to parse streaming response: b\"data: {'choices': [{'delta': {'role': 'assistant'}}]}\\n\"" - ), - ): - await GPT3Resources(api_key).invoke( - GPT3ResourceTypes.chat_completion.value, **mock_completion_request - ) - - assert ( - list(aioresponses.requests.values())[0][0].kwargs["json"] - == mock_completion_request - ) - assert ( - list(aioresponses.requests.values())[0][0].kwargs["headers"] - == request_header - ) - - @pytest.mark.asyncio - async def test_trigger_gp3_client_completion_failure_invalid_json( - self, aioresponses - ): - api_key = "test" - hyperparameters = Utility.get_llm_hyperparameters() - request_header = {"Authorization": f"Bearer {api_key}"} - mock_completion_request = { - "messages": [ - {"role": "system", "content": DEFAULT_SYSTEM_PROMPT}, - { - "role": "user", - "content": "Answer question based on the context below, if answer is not in the context go check previous logs.\nSimilarity Prompt:\nPython is a high-level, general-purpose programming language. Its design philosophy emphasizes code readability with the use of significant indentation. Python is dynamically typed and garbage-collected.\nInstructions on how to use Similarity Prompt: Answer according to this context.\n \n Q: Explain python is called high level programming language in laymen terms?\n A:", - }, - ] - } - mock_completion_request.update(hyperparameters) - mock_completion_request["stream"] = True - - content = "data: {'choices': [{'delta': {'role': 'assistant'}}]}\n\n" - aioresponses.add( - url="https://api.openai.com/v1/chat/completions", - method="POST", - status=504, - body=content.encode(), - ) with pytest.raises( - AppException, match="Failed to connect to service: api.openai.com" + AppException, match="Could not find any hyperparameters for claude LLM." ): - await GPT3Resources(api_key).invoke( - GPT3ResourceTypes.chat_completion.value, **mock_completion_request - ) - - assert ( - list(aioresponses.requests.values())[0][0].kwargs["json"] - == mock_completion_request - ) - assert ( - list(aioresponses.requests.values())[0][0].kwargs["headers"] - == request_header - ) + Utility.get_llm_hyperparameters("claude") def test_get_client_ip_with_request_client(self): request = MagicMock() @@ -3481,217 +2980,6 @@ def test_get_client_ip_with_request_client(self): ip = Utility.get_client_ip(request) assert "58.0.127.89" == ip - def test_llm_resource_provider_factory(self): - client = LLMClientFactory.get_resource_provider(LLMResourceProvider.azure.value) - assert isinstance(client("test"), AzureGPT3Resources) - - client = LLMClientFactory.get_resource_provider( - LLMResourceProvider.openai.value - ) - assert isinstance(client("test"), GPT3Resources) - - def test_llm_resource_provider_not_implemented(self): - with pytest.raises(AppException, match="aws client not supported"): - LLMClientFactory.get_resource_provider("aws") - - @pytest.mark.asyncio - async def test_trigger_azure_client_completion(self, aioresponses): - api_key = "test" - generated_text = "Python is dynamically typed, garbage-collected, high level, general purpose programming." - hyperparameters = Utility.get_llm_hyperparameters() - request_header = {"api-key": api_key} - mock_completion_request = { - "messages": [ - {"role": "system", "content": DEFAULT_SYSTEM_PROMPT}, - { - "role": "user", - "content": "Answer question based on the context below, if answer is not in the context go check previous logs.\nSimilarity Prompt:\nPython is a high-level, general-purpose programming language. Its design philosophy emphasizes code readability with the use of significant indentation. Python is dynamically typed and garbage-collected.\nInstructions on how to use Similarity Prompt: Answer according to this context.\n \n Q: Explain python is called high level programming language in laymen terms?\n A:", - }, - ] - } - mock_completion_request.update(hyperparameters) - llm_settings = ( - LLMSettings( - enable_faq=True, - provider="azure", - embeddings_model_id="openaimodel_embd", - chat_completion_model_id="openaimodel_completion", - api_version="2023-03-16", - ) - .to_mongo() - .to_dict() - ) - aioresponses.add( - url=f"https://kairon.openai.azure.com/openai/deployments/{llm_settings['chat_completion_model_id']}/{GPT3ResourceTypes.chat_completion.value}?api-version={llm_settings['api_version']}", - method="POST", - status=200, - payload={ - "choices": [ - {"message": {"content": generated_text, "role": "assistant"}} - ] - }, - ) - - client = LLMClientFactory.get_resource_provider( - LLMResourceProvider.azure.value - )(api_key, **llm_settings) - formatted_response, raw_response = await client.invoke( - GPT3ResourceTypes.chat_completion.value, **mock_completion_request - ) - assert formatted_response == generated_text - assert raw_response == { - "choices": [{"message": {"content": generated_text, "role": "assistant"}}] - } - - assert ( - list(aioresponses.requests.values())[0][0].kwargs["json"] - == mock_completion_request - ) - assert ( - list(aioresponses.requests.values())[0][0].kwargs["headers"] - == request_header - ) - - @pytest.mark.asyncio - async def test_trigger_azure_client_embedding(self, aioresponses): - api_key = "test" - query = "What kind of language is python?" - embedding = list(np.random.random(GPT3FAQEmbedding.__embedding__)) - request_header = {"api-key": api_key} - llm_settings = ( - LLMSettings( - enable_faq=True, - provider="azure", - embeddings_model_id="openaimodel_embd", - chat_completion_model_id="openaimodel_completion", - api_version="2023-03-16", - ) - .to_mongo() - .to_dict() - ) - - aioresponses.add( - url=f"https://kairon.openai.azure.com/openai/deployments/{llm_settings['embeddings_model_id']}/{GPT3ResourceTypes.embeddings.value}?api-version={llm_settings['api_version']}", - method="POST", - status=200, - payload={"data": [{"embedding": embedding}]}, - ) - client = LLMClientFactory.get_resource_provider( - LLMResourceProvider.azure.value - )(api_key, **llm_settings) - formatted_response, raw_response = await client.invoke( - GPT3ResourceTypes.embeddings.value, - model="text-embedding-3-small", - input=query, - ) - assert formatted_response == embedding - assert raw_response == {"data": [{"embedding": embedding}]} - - assert list(aioresponses.requests.values())[0][0].kwargs["json"] == { - "model": "text-embedding-3-small", - "input": query, - } - assert ( - list(aioresponses.requests.values())[0][0].kwargs["headers"] - == request_header - ) - - @pytest.mark.asyncio - async def test_trigger_azure_client_embedding_failure(self, aioresponses): - api_key = "test" - query = "What kind of language is python?" - request_header = {"api-key": api_key} - llm_settings = ( - LLMSettings( - enable_faq=True, - provider="azure", - embeddings_model_id="openaimodel_embd", - chat_completion_model_id="openaimodel_completion", - api_version="2023-03-16", - ) - .to_mongo() - .to_dict() - ) - - aioresponses.add( - url=f"https://kairon.openai.azure.com/openai/deployments/{llm_settings['embeddings_model_id']}/{GPT3ResourceTypes.embeddings.value}?api-version={llm_settings['api_version']}", - method="POST", - status=504, - ) - client = LLMClientFactory.get_resource_provider( - LLMResourceProvider.azure.value - )(api_key, **llm_settings) - - with pytest.raises( - AppException, match="Failed to connect to service: kairon.openai.azure.com" - ): - await client.invoke( - GPT3ResourceTypes.embeddings.value, - model="text-embedding-3-small", - input=query, - ) - - assert list(aioresponses.requests.values())[0][0].kwargs["json"] == { - "model": "text-embedding-3-small", - "input": query, - } - assert ( - list(aioresponses.requests.values())[0][0].kwargs["headers"] - == request_header - ) - - @pytest.mark.asyncio - async def test_trigger_azure_client_completion_failure(self, aioresponses): - api_key = "test" - hyperparameters = Utility.get_llm_hyperparameters() - request_header = {"api-key": api_key} - llm_settings = ( - LLMSettings( - enable_faq=True, - provider="azure", - embeddings_model_id="openaimodel_embd", - chat_completion_model_id="openaimodel_completion", - api_version="2023-03-16", - ) - .to_mongo() - .to_dict() - ) - mock_completion_request = { - "messages": [ - {"role": "system", "content": DEFAULT_SYSTEM_PROMPT}, - { - "role": "user", - "content": "Answer question based on the context below, if answer is not in the context go check previous logs.\nSimilarity Prompt:\nPython is a high-level, general-purpose programming language. Its design philosophy emphasizes code readability with the use of significant indentation. Python is dynamically typed and garbage-collected.\nInstructions on how to use Similarity Prompt: Answer according to this context.\n \n Q: Explain python is called high level programming language in laymen terms?\n A:", - }, - ] - } - mock_completion_request.update(hyperparameters) - - aioresponses.add( - url=f"https://kairon.openai.azure.com/openai/deployments/{llm_settings['chat_completion_model_id']}/{GPT3ResourceTypes.chat_completion.value}?api-version={llm_settings['api_version']}", - method="POST", - status=504, - payload={"error": {"message": "Server unavailable!", "id": 876543456789}}, - ) - - client = LLMClientFactory.get_resource_provider( - LLMResourceProvider.azure.value - )(api_key, **llm_settings) - with pytest.raises( - AppException, match="Failed to connect to service: kairon.openai.azure.com" - ): - await client.invoke( - GPT3ResourceTypes.chat_completion.value, **mock_completion_request - ) - - assert ( - list(aioresponses.requests.values())[0][0].kwargs["json"] - == mock_completion_request - ) - assert ( - list(aioresponses.requests.values())[0][0].kwargs["headers"] - == request_header - ) @pytest.mark.asyncio async def test_messageConverter_whatsapp_dropdown(self): diff --git a/tests/unit_test/validator/training_data_validator_test.py b/tests/unit_test/validator/training_data_validator_test.py index 2633c01f1..f3aa8dc66 100644 --- a/tests/unit_test/validator/training_data_validator_test.py +++ b/tests/unit_test/validator/training_data_validator_test.py @@ -3,6 +3,8 @@ import pytest import yaml from mongoengine import connect +from kairon.shared.utils import Utility +Utility.load_system_metadata() from kairon.exceptions import AppException from kairon.importer.validator.file_validator import TrainingDataValidator @@ -797,55 +799,34 @@ def test_validate_custom_actions_with_errors(self): assert len(error_summary['google_search_actions']) == 2 assert len(error_summary['zendesk_actions']) == 2 assert len(error_summary['pipedrive_leads_actions']) == 3 - assert len(error_summary['prompt_actions']) == 49 + assert len(error_summary['prompt_actions']) == 36 assert len(error_summary['razorpay_actions']) == 3 assert len(error_summary['pyscript_actions']) == 3 assert len(error_summary['database_actions']) == 6 - required_fields_error = error_summary["prompt_actions"][21] - assert re.match(r"Required fields .* not found in action: prompt_action_with_no_llm_prompts", required_fields_error) - del error_summary["prompt_actions"][21] - print(error_summary['prompt_actions']) - assert error_summary['prompt_actions'] == ['top_results should not be greater than 30 and of type int!', - 'similarity_threshold should be within 0.3 and 1.0 and of type int or float!', - 'Collection is required for bot content prompts!', - 'System prompt is required', 'Query prompt must have static source', - 'Name cannot be empty', 'System prompt is required', - 'num_bot_responses should not be greater than 5 and of type int: prompt_action_invalid_num_bot_responses', - 'Collection is required for bot content prompts!', - 'data field in prompts should of type string.', - 'data is required for static prompts', - 'Temperature must be between 0.0 and 2.0!', - 'max_tokens must be between 5 and 4096!', - 'top_p must be between 0.0 and 1.0!', 'n must be between 1 and 5!', - 'presence_penality must be between -2.0 and 2.0!', - 'frequency_penalty must be between -2.0 and 2.0!', - 'logit_bias must be a dictionary!', - 'System prompt must have static source', - 'Collection is required for bot content prompts!', - 'Collection is required for bot content prompts!', - 'Duplicate action found: test_add_prompt_action_one', - 'Invalid action configuration format. Dictionary expected.', - 'Temperature must be between 0.0 and 2.0!', - 'max_tokens must be between 5 and 4096!', - 'top_p must be between 0.0 and 1.0!', 'n must be between 1 and 5!', - 'Stop must be None, a string, an integer, or an array of 4 or fewer strings or integers.', - 'presence_penality must be between -2.0 and 2.0!', - 'frequency_penalty must be between -2.0 and 2.0!', - 'logit_bias must be a dictionary!', - 'Only one system prompt can be present', 'Invalid prompt type', - 'Invalid prompt source', 'Only one system prompt can be present', - 'Invalid prompt type', 'Invalid prompt source', - 'type in LLM Prompts should be of type string.', - 'source in LLM Prompts should be of type string.', - 'Instructions in LLM Prompts should be of type string.', - 'Only one system prompt can be present', - 'Data must contain action name', - 'Only one system prompt can be present', - 'Data must contain slot name', - 'Only one system prompt can be present', - 'Only one system prompt can be present', - 'Only one system prompt can be present', - 'Only one history source can be present'] + expected_errors = ['top_results should not be greater than 30 and of type int!', + 'similarity_threshold should be within 0.3 and 1.0 and of type int or float!', + 'Collection is required for bot content prompts!', 'System prompt is required', + 'Query prompt must have static source', 'Name cannot be empty', 'System prompt is required', + 'num_bot_responses should not be greater than 5 and of type int: prompt_action_invalid_num_bot_responses', + 'Collection is required for bot content prompts!', + 'data field in prompts should of type string.', 'data is required for static prompts', + "['frequency_penalty']: 5 is greater than the maximum of 2.0", + 'System prompt must have static source', 'Collection is required for bot content prompts!', + 'Collection is required for bot content prompts!', + "Required fields ['llm_prompts', 'name'] not found in action: prompt_action_with_no_llm_prompts", + 'Duplicate action found: test_add_prompt_action_one', + 'Invalid action configuration format. Dictionary expected.', + "['frequency_penalty']: 5 is greater than the maximum of 2.0", + 'Only one system prompt can be present', 'Invalid prompt type', + 'Only one system prompt can be present', 'Invalid prompt type', 'Invalid prompt source', + 'type in LLM Prompts should be of type string.', + 'source in LLM Prompts should be of type string.', + 'Instructions in LLM Prompts should be of type string.', + 'Only one system prompt can be present', 'Data must contain action name', + 'Only one system prompt can be present', 'Data must contain slot name', + 'Only one system prompt can be present', 'Only one system prompt can be present', + 'Only one system prompt can be present', 'Only one history source can be present'] + assert not DeepDiff(error_summary['prompt_actions'], expected_errors, ignore_order=True) assert component_count == {'http_actions': 7, 'slot_set_actions': 10, 'form_validation_actions': 9, 'email_actions': 5, 'google_search_actions': 5, 'jira_actions': 6, 'zendesk_actions': 4, 'pipedrive_leads_actions': 5, 'prompt_actions': 8, diff --git a/tests/unit_test/vector_embeddings/qdrant_test.py b/tests/unit_test/vector_embeddings/qdrant_test.py index 7bf166116..667285715 100644 --- a/tests/unit_test/vector_embeddings/qdrant_test.py +++ b/tests/unit_test/vector_embeddings/qdrant_test.py @@ -14,7 +14,9 @@ from kairon.shared.data.data_objects import LLMSettings from kairon.shared.vector_embeddings.db.factory import VectorEmbeddingsDbFactory from kairon.shared.vector_embeddings.db.qdrant import Qdrant - +import litellm +from kairon.shared.llm.processor import LLMProcessor +import numpy as np class TestQdrant: @@ -25,16 +27,21 @@ def init_connection(self): connect(**Utility.mongoengine_connection(Utility.environment['database']["url"])) @pytest.mark.asyncio + @mock.patch.dict(Utility.environment, {'vector': {"key": "TEST", 'db': 'http://localhost:6333'}}) + @mock.patch.object(litellm, "aembedding", autospec=True) @mock.patch.object(ActionUtility, "execute_http_request", autospec=True) - async def test_embedding_search_valid_request_body(self, mock_http_request): + async def test_embedding_search_valid_request_body(self, mock_http_request, mock_embedding): + embedding = list(np.random.random(LLMProcessor.__embedding__)) + user = "test" Utility.load_environment() secret = BotSecrets(secret_type=BotSecretType.gpt_key.value, value="key_value", bot="5f50fd0a56v098ca10d75d2g", user="user").save() qdrant = Qdrant('5f50fd0a56v098ca10d75d2g', '5f50fd0a56v098ca10d75d2g', LLMSettings(provider="openai").to_mongo().to_dict()) - request_body = {"ids": [0], "with_payload": True, "with_vector": True} + request_body = {"ids": [0], "with_payload": True, "with_vector": True, 'text': "Hi"} mock_http_request.return_value = 'expected_result' - result = await qdrant.embedding_search(request_body) + mock_embedding.return_value = {'data': [{'embedding': embedding}]} + result = await qdrant.embedding_search(request_body, user=user) assert result == 'expected_result' @pytest.mark.asyncio @@ -46,29 +53,31 @@ async def test_payload_search_valid_request_body(self, mock_http_request): request_body = {"filter": {"should": [{"key": "city", "match": {"value": "London"}}, {"key": "color", "match": {"value": "red"}}]}} mock_http_request.return_value = 'expected_result' - result = await qdrant.payload_search(request_body) + result = await qdrant.payload_search(request_body, user="test") assert result == 'expected_result' @pytest.mark.asyncio @mock.patch.object(ActionUtility, "execute_http_request", autospec=True) async def test_perform_operation_valid_op_type_and_request_body(self, mock_http_request): Utility.load_environment() + user = "test" qdrant = Qdrant('5f50fd0a56v098ca10d75d2g', '5f50fd0a56v098ca10d75d2g', LLMSettings(provider="openai").to_mongo().to_dict()) request_body = {} mock_http_request.return_value = 'expected_result' - result_embedding = await qdrant.perform_operation('embedding_search', request_body) + result_embedding = await qdrant.perform_operation('embedding_search', request_body, user=user) assert result_embedding == 'expected_result' - result_payload = await qdrant.perform_operation('payload_search', request_body) + result_payload = await qdrant.perform_operation('payload_search', request_body, user=user) assert result_payload == 'expected_result' @pytest.mark.asyncio async def test_embedding_search_empty_request_body(self): Utility.load_environment() + user = "test" qdrant = Qdrant('5f50fd0a56v098ca10d75d2g', '5f50fd0a56v098ca10d75d2g', LLMSettings(provider="openai").to_mongo().to_dict()) with pytest.raises(ActionFailure): - await qdrant.embedding_search({}) + await qdrant.embedding_search({}, user=user) @pytest.mark.asyncio async def test_payload_search_empty_request_body(self): @@ -76,7 +85,7 @@ async def test_payload_search_empty_request_body(self): qdrant = Qdrant('5f50fd0a56v098ca10d75d2g', '5f50fd0a56v098ca10d75d2g', LLMSettings(provider="openai").to_mongo().to_dict()) with pytest.raises(ActionFailure): - await qdrant.payload_search({}) + await qdrant.payload_search({}, user="test") @pytest.mark.asyncio async def test_perform_operation_invalid_op_type(self): @@ -85,7 +94,7 @@ async def test_perform_operation_invalid_op_type(self): LLMSettings(provider="openai").to_mongo().to_dict()) request_body = {} with pytest.raises(AppException, match="Operation type not supported"): - await qdrant.perform_operation("vector_search", request_body) + await qdrant.perform_operation("vector_search", request_body, user="test") def test_get_instance_raises_exception_when_db_not_implemented(self): with pytest.raises(AppException, match="Database not yet implemented!"): @@ -99,7 +108,7 @@ async def test_embedding_search_valid_request_body_payload(self, mock_http_reque LLMSettings(provider="openai").to_mongo().to_dict()) request_body = {'ids': [0], 'with_payload': True, 'with_vector': True} mock_http_request.return_value = 'expected_result' - result = await qdrant.embedding_search(request_body) + result = await qdrant.embedding_search(request_body, user="test") assert result == 'expected_result' mock_http_request.assert_called_once() diff --git a/tests/unit_test/verification_test.py b/tests/unit_test/verification_test.py index 4e6c2b360..6462ae579 100644 --- a/tests/unit_test/verification_test.py +++ b/tests/unit_test/verification_test.py @@ -2,7 +2,7 @@ import responses from kairon.shared.verification.email import QuickEmailVerification from urllib.parse import urlencode -import mock +from unittest import mock from kairon.shared.utils import Utility import os diff --git a/training_data/ReadMe.md b/training_data/ReadMe.md deleted file mode 100644 index f827d7c61..000000000 --- a/training_data/ReadMe.md +++ /dev/null @@ -1 +0,0 @@ -Trained Data Directory \ No newline at end of file