DB Action Changes

kairon/shared/actions/models.py

@@ -91,6 +91,7 @@ class DispatchType(str, Enum):
 class DbActionOperationType(str, Enum):
     payload_search = "payload_search"
     embedding_search = "embedding_search"
+    payload_and_keyword_search = "payload_and_keyword_search"
 
 
 class DbQueryValueType(str, Enum):

kairon/shared/actions/utils.py

@@ -622,6 +622,8 @@ def get_payload(payload: Dict, tracker: Tracker):
                     rqst_payload = {"text": msg}
             else:
                 rqst_payload = {"text": rqst_payload}
+            if payload.get('value') and isinstance(payload.get('value'), str):
+                rqst_payload.update(json.loads(payload.get('value')))
         else:
             rqst_payload = payload.get('value')
 

kairon/shared/vector_embeddings/db/base.py

@@ -15,9 +15,14 @@ async def embedding_search(self, request_body: Dict):
     async def payload_search(self, request_body: Dict):
         raise NotImplementedError("Provider not implemented")
 
+    @abstractmethod
+    async def payload_and_keyword_search(self, request_body: Dict):
+        raise NotImplementedError("Provider not implemented")
+
     async def perform_operation(self, op_type: Text, request_body: Dict):
         supported_ops = {DbActionOperationType.payload_search.value: self.payload_search,
-                         DbActionOperationType.embedding_search.value: self.embedding_search}
+                         DbActionOperationType.embedding_search.value: self.embedding_search,
+                         DbActionOperationType.payload_and_keyword_search.value: self.payload_and_keyword_search}
         if op_type not in supported_ops.keys():
             raise AppException("Operation type not supported")
         return await supported_ops[op_type](request_body)

kairon/shared/vector_embeddings/db/qdrant.py

@@ -63,3 +63,15 @@ async def payload_search(self, request_body: Dict):
                                                                    request_method='POST',
                                                                    request_body=request_body)
         return payload_filter_result
+
+    async def payload_and_keyword_search(self, request_body: Dict):
+        url = urljoin(self.db_url, f"/collections/{self.collection_name}/points/search")
+        user_msg = request_body.pop("text")
+        vector = await self.__get_embedding(user_msg)
+        request_body.update({'vector': vector, 'limit': 10, 'with_payload': True, 'score_threshold': 0.70})
+
+        search_result = ActionUtility.execute_http_request(http_url=url,
+                                                           request_method='POST',
+                                                           headers=self.headers,
+                                                           request_body=request_body)
+        return search_result

tests/integration_test/action_service_test.py

@@ -3004,6 +3004,90 @@ def test_vectordb_action_execution_payload_search_from_user_message():
     log.pop('timestamp')
 
 
+@responses.activate
+@mock.patch.object(Qdrant, "_Qdrant__get_embedding", autospec=True)
+def test_vectordb_action_execution_payload_search_and_keyword_search_from_user_message(mock_embedding):
+    action_name = "test_vectordb_action_execution_payload_search_and_keyword_search_from_user_message"
+    Actions(name=action_name, type=ActionType.database_action.value, bot="5f50md0a56b698ca10d35d2a",
+            user="user").save()
+    payload_body = json.dumps({"filter": {
+        "should": [{"key": "city", "match": {"value": "London"}}, {"key": "color", "match": {"value": "red"}}]}})
+    DatabaseAction(
+        name=action_name,
+        collection='test_vectordb_action_execution_payload_search_and_keyword_search_from_user_message',
+        query_type=DbActionOperationType.payload_and_keyword_search.value,
+        payload=DbQuery(type=DbQueryValueType.from_user_message.value, value=payload_body),
+        response=HttpActionResponse(value="The value of ${data.0.city} with color ${data.0.color} is ${data.0.id}"),
+        set_slots=[SetSlotsFromResponse(name="city_value", value="${data.0.id}")],
+        bot="5f50md0a56b698ca10d35d2a",
+        user="user"
+    ).save()
+    BotSettings(llm_settings=LLMSettings(enable_faq=True), bot="5f50md0a56b698ca10d35d2a", user="user").save()
+    BotSecrets(secret_type=BotSecretType.gpt_key.value, value="key_value",
+               bot="5f50md0a56b698ca10d35d2a", user="user").save()
+
+    http_url = 'http://localhost:6333/collections/5f50md0a56b698ca10d35d2a_test_vectordb_action_execution_payload_search_and_keyword_search_from_user_message_faq_embd/points/search'
+    resp_msg = json.dumps(
+        [{"id": 2, "city": "London", "color": "red"}]
+    )
+    json_params_matcher = {'text': json.dumps({"filter": {
+        "should": [{"key": "city", "match": {"value": "London"}}, {"key": "color", "match": {"value": "red"}}]}})}
+    embedding = list(np.random.random(Qdrant.__embedding__))
+    mock_embedding.return_value = embedding
+    output = [{'id': 2, 'payload': {'city': 'London', 'color': 'Red'}, 'score': 1.0, 'vector': None, 'version': 0}]
+
+    responses.add(
+        method=responses.POST,
+        url=http_url,
+        body=resp_msg,
+        status=200,
+        # match=[responses.matchers.json_params_matcher(json_params_matcher)],
+    )
+
+    request_object = {
+        "next_action": action_name,
+        "tracker": {
+            "sender_id": "default",
+            "conversation_id": "default",
+            "slots": {"bot": "5f50md0a56b698ca10d35d2a"},
+            "latest_message": {'text': payload_body, 'intent_ranking': [{'name': 'user_story'}],
+                               "entities": [{"value": payload_body, "entity": KAIRON_USER_MSG_ENTITY}]},
+            "latest_event_time": 1537645578.314389,
+            "followup_action": "action_listen",
+            "paused": False,
+            "events": [{"event1": "hello"}, {"event2": "how are you"}],
+            "latest_input_channel": "rest",
+            "active_loop": {},
+            "latest_action": {},
+        },
+        "domain": {
+            "config": {},
+            "session_config": {},
+            "intents": [],
+            "entities": [],
+            "slots": {"bot": "5f50md0a56b698ca10d35d2a"},
+            "responses": {},
+            "actions": [],
+            "forms": {},
+            "e2e_actions": []
+        },
+        "version": "version"
+    }
+    response = client.post("/webhook", json=request_object)
+    response_json = response.json()
+    assert response.status_code == 200
+    assert len(response_json['events']) == 2
+    assert len(response_json['responses']) == 1
+    assert response_json['events'] == [
+        {'event': 'slot', 'timestamp': None, 'name': 'city_value', 'value': '2'},
+        {'event': 'slot', 'timestamp': None, 'name': 'kairon_action_response',
+         'value': 'The value of London with color red is 2'}]
+    assert response_json['responses'][0]['text'] == "The value of London with color red is 2"
+    log = ActionServerLogs.objects(action=action_name, bot='5f50md0a56b698ca10d35d2a').get().to_mongo().to_dict()
+    log.pop('_id')
+    log.pop('timestamp')
+
+
 @responses.activate
 def test_vectordb_action_execution_payload_search_from_user_message_in_slot():
     action_name = "test_vectordb_action_execution_payload_search_from_user_message_in_slot"

tests/integration_test/services_test.py

@@ -9689,9 +9689,9 @@ def test_add_vectordb_action_empty_operation_value():
 
     assert actual["error_code"] == 422
     assert actual["message"] == [{'loc': ['body', 'query_type'],
-                                  'msg': "value is not a valid enumeration member; permitted: 'payload_search', 'embedding_search'",
+                                  'msg': "value is not a valid enumeration member; permitted: 'payload_search', 'embedding_search', 'payload_and_keyword_search'",
                                   'type': 'type_error.enum',
-                                  'ctx': {'enum_values': ['payload_search', 'embedding_search']}}]
+                                  'ctx': {'enum_values': ['payload_search', 'embedding_search', 'payload_and_keyword_search']}}]
     assert not actual["success"]