feat: SDK, adapter version bumps

backend/adapter_processor/adapter_processor.py

@@ -9,11 +9,9 @@
     InternalServiceError,
     InValidAdapterId,
     TestAdapterException,
-    TestAdapterInputException,
 )
 from django.conf import settings
 from django.core.exceptions import ObjectDoesNotExist
-from platform_settings.exceptions import ActiveKeyNotFound, InvalidRequest
 from platform_settings.platform_auth_service import (
     PlatformAuthenticationService,
 )
@@ -119,18 +117,8 @@ def test_adapter(adapter_id: str, adapter_metadata: dict[str, Any]) -> bool:
             test_result: bool = adapter_instance.test_connection()
             logger.info(f"{adapter_id} test result: {test_result}")
             return test_result
-        except ActiveKeyNotFound:
-            raise ActiveKeyNotFound()
-        except InvalidRequest as e:
-            raise InvalidRequest(str(e.detail))
-        except Exception as e:
-            logger.error(f"Error while testing : {adapter_id}: {e}")
-            if isinstance(e, AdapterError):
-                raise TestAdapterInputException(e.message)
-            elif isinstance(e, ActiveKeyNotFound):
-                raise e
-            else:
-                raise TestAdapterException(str(e))
+        except AdapterError as e:
+            raise TestAdapterException(str(e))
 
     @staticmethod
     def __fetch_adapters_by_key_value(key: str, value: Any) -> Adapter:

backend/prompt_studio/prompt_studio_core/exceptions.py

@@ -19,7 +19,7 @@ class PromptNotValid(APIException):
 
 
 class IndexingError(APIException):
-    status_code = 400
+    status_code = 500
     default_detail = "Error while indexing file"
 
 

backend/prompt_studio/prompt_studio_core/prompt_studio_helper.py

@@ -23,10 +23,11 @@
 from prompt_studio.prompt_studio_core.prompt_ide_base_tool import (
     PromptIdeBaseTool,
 )
-from prompt_studio.prompt_studio_index_manager.prompt_studio_index_helper import (
+from prompt_studio.prompt_studio_index_manager.prompt_studio_index_helper import (  # noqa: E501
     PromptStudioIndexHelper,
 )
 from unstract.sdk.constants import LogLevel
+from unstract.sdk.exceptions import SdkError
 from unstract.sdk.index import ToolIndex
 from unstract.sdk.prompt import PromptTool
 from unstract.sdk.utils.tool_utils import ToolUtils
@@ -151,7 +152,7 @@ def index_document(
         doc_id = PromptStudioHelper.dynamic_indexer(
             profile_manager=default_profile,
             tool_id=tool_id,
-            file_name=file_path,
+            file_path=file_path,
             org_id=org_id,
             document_id=document_id,
             is_summary=is_summary,
@@ -174,7 +175,7 @@ def prompt_responder(
         file_name: str,
         org_id: str,
         user_id: str,
-        document_id: str
+        document_id: str,
     ) -> Any:
         """Execute chain/single run of the prompts. Makes a call to prompt
         service and returns the dict of response.
@@ -227,8 +228,7 @@ def prompt_responder(
                     ),
                 )
                 if not prompt_instance:
-                    logger.error(
-                        f"Prompt id {id} does not have any data in db")
+                    logger.error(f"Prompt id {id} does not have any data in db")
                     raise PromptNotValid()
             except Exception as exc:
                 logger.error(f"Error while fetching prompt {exc}")
@@ -257,7 +257,7 @@ def prompt_responder(
                 tool=tool,
                 prompts=prompts,
                 org_id=org_id,
-                document_id=document_id
+                document_id=document_id,
             )
             stream_log.publish(
                 tool.tool_id,
@@ -277,7 +277,7 @@ def _fetch_response(
         path: str,
         prompts: list[ToolStudioPrompt],
         org_id: str,
-        document_id: str
+        document_id: str,
     ) -> Any:
         """Utility function to invoke prompt service. Used internally.
 
@@ -315,7 +315,7 @@ def _fetch_response(
                 raise DefaultProfileError()
             PromptStudioHelper.dynamic_indexer(
                 profile_manager=prompt_profile_manager,
-                file_name=path,
+                file_path=path,
                 tool_id=str(tool.tool_id),
                 org_id=org_id,
                 document_id=document_id,
@@ -397,11 +397,28 @@ def _fetch_response(
     def dynamic_indexer(
         profile_manager: ProfileManager,
         tool_id: str,
-        file_name: str,
+        file_path: str,
         org_id: str,
         document_id: str,
         is_summary: bool = False,
     ) -> str:
+        """Used to index a file based on the passed arguments.
+
+        This is useful when a file needs to be indexed dynamically as the
+        parameters meant for indexing changes. The file
+
+        Args:
+            profile_manager (ProfileManager): Profile manager instance that hold
+                values such as chunk size, chunk overlap and adapter IDs
+            tool_id (str): UUID of the prompt studio tool
+            file_path (str): Path to the file that needs to be indexed
+            org_id (str): ID of the organization
+            is_summary (bool, optional): Flag to ensure if extracted contents
+                need to be persisted.  Defaults to False.
+
+        Returns:
+            str: Index key for the combination of arguments
+        """
         try:
             util = PromptIdeBaseTool(log_level=LogLevel.INFO, org_id=org_id)
             tool_index = ToolIndex(tool=util)
@@ -411,33 +428,34 @@ def dynamic_indexer(
         embedding_model = str(profile_manager.embedding_model.id)
         vector_db = str(profile_manager.vector_store.id)
         x2text_adapter = str(profile_manager.x2text.id)
-        file_hash = ToolUtils.get_hash_from_file(file_path=file_name)
+        file_hash = ToolUtils.get_hash_from_file(file_path=file_path)
         extract_file_path: Optional[str] = None
         if not is_summary:
-            directory, filename = os.path.split(file_name)
+            directory, filename = os.path.split(file_path)
             extract_file_path = os.path.join(
                 directory, "extract", os.path.splitext(filename)[0] + ".txt"
             )
-        doc_id = str(
-            tool_index.index_file(
+
+        try:
+            doc_id: str = tool_index.index_file(
                 tool_id=tool_id,
                 embedding_type=embedding_model,
                 vector_db=vector_db,
                 x2text_adapter=x2text_adapter,
-                file_path=file_name,
+                file_path=file_path,
                 file_hash=file_hash,
                 chunk_size=profile_manager.chunk_size,
                 chunk_overlap=profile_manager.chunk_overlap,
                 reindex=profile_manager.reindex,
                 output_file_path=extract_file_path,
             )
-        )
-
-        PromptStudioIndexHelper.handle_index_manager(
-            document_id=document_id,
-            is_summary=is_summary,
-            profile_manager=profile_manager,
-            doc_id=doc_id,
-        )
 
-        return doc_id
+            PromptStudioIndexHelper.handle_index_manager(
+                document_id=document_id,
+                is_summary=is_summary,
+                profile_manager=profile_manager,
+                doc_id=doc_id,
+            )
+            return doc_id
+        except SdkError as e:
+            raise IndexingError(str(e))

backend/prompt_studio/prompt_studio_core/views.py

@@ -198,42 +198,39 @@ def index_document(self, request: HttpRequest) -> Response:
             ToolStudioPromptKeys.TOOL_ID
         )
         document_id: str = serializer.validated_data.get(
-            ToolStudioPromptKeys.DOCUMENT_ID)
+            ToolStudioPromptKeys.DOCUMENT_ID
+        )
         document: DocumentManager = DocumentManager.objects.get(pk=document_id)
         file_name: str = document.document_name
-        try:
-            unique_id = PromptStudioHelper.index_document(
+        unique_id = PromptStudioHelper.index_document(
+            tool_id=tool_id,
+            file_name=file_name,
+            org_id=request.org_id,
+            user_id=request.user.user_id,
+            document_id=document_id,
+        )
+
+        for processor_plugin in self.processor_plugins:
+            cls = processor_plugin[ProcessorConfig.METADATA][
+                ProcessorConfig.METADATA_SERVICE_CLASS
+            ]
+            cls.process(
                 tool_id=tool_id,
                 file_name=file_name,
                 org_id=request.org_id,
                 user_id=request.user.user_id,
                 document_id=document_id,
             )
 
-            for processor_plugin in self.processor_plugins:
-                cls = processor_plugin[ProcessorConfig.METADATA][
-                    ProcessorConfig.METADATA_SERVICE_CLASS
-                ]
-                cls.process(
-                    tool_id=tool_id,
-                    file_name=file_name,
-                    org_id=request.org_id,
-                    user_id=request.user.user_id,
-                    document_id=document_id,
-                )
-
-            if unique_id:
-                return Response(
-                    {"message": "Document indexed successfully."},
-                    status=status.HTTP_200_OK,
-                )
-            else:
-                logger.error(
-                    "Error occured while indexing. Unique ID is not valid."
-                )
-                raise IndexingError()
-        except Exception as exc:
-            logger.error(f"Error occured while indexing {exc}")
+        if unique_id:
+            return Response(
+                {"message": "Document indexed successfully."},
+                status=status.HTTP_200_OK,
+            )
+        else:
+            logger.error(
+                "Error occured while indexing. Unique ID is not valid."
+            )
             raise IndexingError()
 
     @action(detail=True, methods=["post"])

backend/prompt_studio/prompt_studio_document_manager/prompt_studio_document_helper.py

@@ -8,17 +8,17 @@
 
 
 class PromptStudioDocumentHelper:
-
     @staticmethod
-    def create(tool_id: str, document_name: str):
+    def create(tool_id: str, document_name: str) -> DocumentManager:
         tool: CustomTool = CustomTool.objects.get(pk=tool_id)
         document: DocumentManager = DocumentManager.objects.create(
-            tool=tool, document_name=document_name)
+            tool=tool, document_name=document_name
+        )
         logger.info("Successfully created the record")
         return document
 
     @staticmethod
-    def delete(document_id: str):
+    def delete(document_id: str) -> None:
         document: DocumentManager = DocumentManager.objects.get(pk=document_id)
         document.delete()
         logger.info("Successfully deleted the record")

backend/prompt_studio/prompt_studio_document_manager/serializers.py

@@ -1,3 +1,5 @@
+from typing import Any
+
 from backend.serializers import AuditSerializer
 
 from .constants import PSDMKeys
@@ -9,8 +11,11 @@ class Meta:
         model = DocumentManager
         fields = "__all__"
 
-    def to_representation(self, instance):
+    def to_representation(self, instance: DocumentManager) -> dict[str, Any]:
         rep: dict[str, str] = super().to_representation(instance)
-        required_fields = [PSDMKeys.DOCUMENT_NAME,
-                           PSDMKeys.TOOL, PSDMKeys.DOCUMENT_ID]
+        required_fields = [
+            PSDMKeys.DOCUMENT_NAME,
+            PSDMKeys.TOOL,
+            PSDMKeys.DOCUMENT_ID,
+        ]
         return {key: rep[key] for key in required_fields if key in rep}

backend/prompt_studio/prompt_studio_index_manager/prompt_studio_index_helper.py

@@ -1,5 +1,6 @@
 import json
 import logging
+from typing import Optional
 
 from prompt_studio.prompt_profile_manager.models import ProfileManager
 from prompt_studio.prompt_studio_document_manager.models import DocumentManager
@@ -10,31 +11,27 @@
 
 
 class PromptStudioIndexHelper:
-
     @staticmethod
     def handle_index_manager(
         document_id: str,
         is_summary: bool,
         profile_manager: ProfileManager,
-        doc_id: str
-    ):
+        doc_id: str,
+    ) -> IndexManager:
         document: DocumentManager = DocumentManager.objects.get(pk=document_id)
 
         index_id = "raw_index_id"
         if is_summary:
             index_id = "summarize_index_id"
 
         try:
-            index_manager: IndexManager = IndexManager.objects.get(**{
-                "document_manager": document,
-                "profile_manager": profile_manager,
-            })
-        except Exception:
+            index_manager: Optional[IndexManager] = IndexManager.objects.get(
+                document_manager=document, profile_manager=profile_manager
+            )
+        except IndexManager.DoesNotExist:
             index_manager = None
 
-        args: dict = {
-            f'{index_id}': doc_id,
-        }
+        args: dict[str, str] = {f"{index_id}": doc_id}
 
         index_ids_list = []
         if index_manager:
@@ -49,9 +46,10 @@ def handle_index_manager(
 
         if index_manager:
             result: IndexManager = IndexManager.objects.filter(
-                index_manager_id=index_manager.index_manager_id).update(**args)
+                index_manager_id=index_manager.index_manager_id
+            ).update(**args)
         else:
             args["document_manager"] = document
             args["profile_manager"] = profile_manager
-            result: IndexManager = IndexManager.objects.create(**args)
+            result = IndexManager.objects.create(**args)
         return result

backend/pyproject.toml

@@ -30,8 +30,8 @@ dependencies = [
     "python-socketio==5.9.0", # For log_events
     "social-auth-app-django==5.3.0", # For OAuth
     "social-auth-core==4.4.2", # For OAuth
-    "unstract-sdk~=0.14.0",
-    "unstract-adapters~=0.4.0",
+    "unstract-sdk~=0.15.0",
+    "unstract-adapters~=0.4.1",
     # ! IMPORTANT!
     # Indirect local dependencies usually need to be added in their own projects
     # as: https://pdm-project.org/latest/usage/dependency/#local-dependencies.

backend/sample.env

@@ -86,9 +86,9 @@ PROMPT_PORT=3003
 PROMPT_STUDIO_FILE_PATH=/app/prompt-studio-data
 
 # Structure Tool
-STRUCTURE_TOOL_IMAGE_URL="docker:unstract/tool-structure:0.0.2"
+STRUCTURE_TOOL_IMAGE_URL="docker:unstract/tool-structure:0.0.3"
 STRUCTURE_TOOL_IMAGE_NAME="unstract/tool-structure"
-STRUCTURE_TOOL_IMAGE_TAG="0.0.2"
+STRUCTURE_TOOL_IMAGE_TAG="0.0.3"
 
 # Feature Flags
 EVALUATION_SERVER_IP=localhost