Geearl/7045 show processing log

app/backend/app.py

@@ -22,7 +22,7 @@
     ResourceTypes,
     generate_account_sas,
 )
-from shared_code.status_log import State, StatusClassification, StatusLog
+from shared_code.status_log import State, StatusClassification, StatusLog, StatusQueryLevel
 from shared_code.tags_helper import TagsHelper
 from azure.cosmos import CosmosClient
 
@@ -279,9 +279,11 @@ async def get_all_upload_status(request: Request):
     state = json_body.get("state")
     folder = json_body.get("folder")
     try:
-        results = statusLog.read_files_status_by_timeframe(timeframe, State[state], 
-            folder, os.environ["AZURE_BLOB_STORAGE_UPLOAD_CONTAINER"])
-
+        results = statusLog.read_files_status_by_timeframe(timeframe, 
+            State[state], 
+            folder, 
+            os.environ["AZURE_BLOB_STORAGE_UPLOAD_CONTAINER"])
+
         # retrieve tags for each file
          # Initialize an empty list to hold the tags
         items = []              
@@ -365,6 +367,7 @@ async def delete_Items(request: Request):
     return True
 
 
+
 @app.post("/resubmitItems")
 async def resubmit_Items(request: Request):
     """
@@ -576,7 +579,6 @@ async def retryFile(request: Request):
             raw_file = blob.download_blob().readall()
             # Overwrite the existing blob with new data
             blob.upload_blob(raw_file, overwrite=True) 
-
             statusLog.upsert_document(document_path=filePath,
                         status='Resubmitted to the processing pipeline',
                         status_classification=StatusClassification.INFO,

app/frontend/src/api/models.ts

@@ -69,12 +69,20 @@ export type FileUploadBasicStatus = {
     start_timestamp: string;
     state_description: string;
     state_timestamp: string;
+    status_updates: StatusUpdates[];
+}
+
+export type StatusUpdates = {
+    status: string;
+    status_timestamp: string;
+    status_classification: string;
 }
 
 export type AllFilesUploadStatus = {
     statuses: FileUploadBasicStatus[];
 }
 
+
 export type AllFolders = {
     folders: string;
 }

app/frontend/src/components/FileStatus/DocumentsDetailList.module.css

@@ -64,3 +64,9 @@
   text-overflow: clip; /* Clip text that overflows */
   white-space: normal; /* Allow text to wrap */
 }
+
+.scrollableDialogContent {
+  max-height: 400px; /* Adjust the height as needed */
+  max-width: 400px;  /* Adjust the width as needed */
+  overflow-y: auto;  /* This enables vertical scrolling */
+}

app/frontend/src/components/FileStatus/DocumentsDetailList.tsx

@@ -1,7 +1,7 @@
 // Copyright (c) Microsoft Corporation.
 // Licensed under the MIT license.
 
-import React, { useState, useEffect, useRef } from "react";
+import React, { useState, useEffect, useRef, useLayoutEffect  } from "react";
 import { DetailsList, 
     DetailsListLayoutMode, 
     SelectionMode, 
@@ -30,9 +30,15 @@ export interface IDocument {
     state_description: string;
     upload_timestamp: string;
     modified_timestamp: string;
+    status_updates: Array<{
+        status: string;
+        status_timestamp: string;
+        status_classification: string;
+    }>;
     isSelected?: boolean; // Optional property to track selection state
 }
 
+
 interface Props {
     items: IDocument[];
     onFilesSorted?: (items: IDocument[]) => void;
@@ -139,19 +145,19 @@ export const DocumentsDetailList = ({ items, onFilesSorted}: Props) => {
     // *************************************************************
     // Delete processing
     // New state for managing dialog visibility and selected items
-    const [isDialogVisible, setIsDialogVisible] = useState(false);
+    const [isDeleteDialogVisible, setIsDeleteDialogVisible] = useState(false);
     const [selectedItemsForDeletion, setSelectedItemsForDeletion] = useState<IDocument[]>([]);
 
     // Function to open the dialog with selected items
     const showDeleteConfirmation = () => {
         const selectedItems = selectionRef.current.getSelection() as IDocument[];
         setSelectedItemsForDeletion(selectedItems);
-        setIsDialogVisible(true);
+        setIsDeleteDialogVisible(true);
     };
 
     // Function to handle actual deletion
     const handleDelete = () => {
-        setIsDialogVisible(false);
+        setIsDeleteDialogVisible(false);
         console.log("Items to delete:", selectedItemsForDeletion);
         selectedItemsForDeletion.forEach(item => {
             console.log(`Deleting item: ${item.name}`);
@@ -171,7 +177,6 @@ export const DocumentsDetailList = ({ items, onFilesSorted}: Props) => {
         showDeleteConfirmation();
     };
 
-
     // *************************************************************
     // Resubmit processing
     // New state for managing resubmit dialog visibility and selected items
@@ -201,12 +206,65 @@ export const DocumentsDetailList = ({ items, onFilesSorted}: Props) => {
         setNotification({ show: true, message: 'Processing resubmit. Hit \'Refresh\' to track progress' });
     };
 
-
     // Function to handle the resubmit button click
     const handleResubmitClick = () => {
         showResubmitConfirmation();
     };
 
+    // ********************************************************************
+    // State detail dialog
+    const [stateDialogVisible, setStateDialogVisible] = useState(false);
+    const [stateDialogContent, setStateDialogContent] = useState<React.ReactNode>(null);
+    const scrollableContentRef = useRef<HTMLDivElement>(null);
+
+    // const onStateColumnClick = async (item: IDocument) => {
+    //     try {
+    //         //const text = await getTextForState(item);
+    //         // const text = item.status_updates[0].status;
+    //         const text = item.status_updates.map(update => update.status).join("\n");       
+    //         setStateDialogContent(text);
+    //         setStateDialogVisible(true);
+    //     } catch (error) {
+    //         console.error("Error on state column click:", error);
+    //         // Handle error here, perhaps show an error message to the user
+    //     }
+    // };
+
+
+    const onStateColumnClick = (item: IDocument) => {
+        try {
+            const statusElements = item.status_updates.map((update, index) => (
+                <div key={index}>
+                    <b>{update.status_timestamp}</b> - {update.status}
+                </div>
+            ));
+            setStateDialogContent(statusElements);
+            setStateDialogVisible(true);
+        } catch (error) {
+            console.error("Error on state column click:", error);
+            // Handle error here, perhaps show an error message to the user
+        }
+    };
+
+
+    const dialogStyles = {
+        main: {
+            width: '400px',  // Set the width to 400 pixels
+            maxWidth: '400px', // Set the maximum width to 400 pixels
+            maxHeight: '400px', // Set the maximum height to 400 pixels
+            overflowY: 'auto', // Enable vertical scrolling for the entire dialog if needed
+        },
+    };
+
+
+    useEffect(() => {
+        // Scroll to the top when the dialog opens
+        window.scrollTo({ top: 0, left: 0, behavior: 'smooth' });
+    }, []);
+
+    // ********************************************************************
+
+
 
     const [columns, setColumns] = useState<IColumn[]> ([
         {
@@ -251,11 +309,14 @@ export const DocumentsDetailList = ({ items, onFilesSorted}: Props) => {
             ariaLabel: 'Column operations for state, Press to sort by states',
             onColumnClick: onColumnClick,
             data: 'string',
-            onRender: (item: IDocument) => (  
-                <TooltipHost content={`${item.state} `}>  
-                    <span>{item.state}</span>  
-                    {item.state === 'Error' && <a href="javascript:void(0);" onClick={() => retryErroredFile(item)}> - Retry File</a>}  
-                </TooltipHost>  
+            onRender: (item: IDocument) => (
+                <TooltipHost content={`${item.state} `}>
+                    <span onClick={() => onStateColumnClick(item)} style={{ cursor: 'pointer' }}>
+                        {item.state}
+                    </span>
+                    {item.state === 'Error' && <a href="javascript:void(0);" onClick={() => retryErroredFile(item)}> - Retry File</a>}
+                </TooltipHost>
+
             ), 
             isPadded: true,
         },
@@ -269,9 +330,12 @@ export const DocumentsDetailList = ({ items, onFilesSorted}: Props) => {
             ariaLabel: 'Column operations for folder, Press to sort by folder',
             onColumnClick: onColumnClick,
             data: 'string',
-            onRender: (item: IDocument) => {
-                return <span>{item.filePath}</span>;
-            },
+            onRender: (item: IDocument) => (  
+                <TooltipHost content={`${item.state} `}>  
+                    <span>{item.filePath.split('/').slice(1, -1).join('/')}</span>  
+                    {item.filePath === 'Error' && <a href="javascript:void(0);" onClick={() => retryErroredFile(item)}> Retry File</a>}  
+                </TooltipHost>  
+            ), 
             isPadded: true,
         },
         {
@@ -322,7 +386,9 @@ export const DocumentsDetailList = ({ items, onFilesSorted}: Props) => {
             onColumnClick: onColumnClick,
             onRender: (item: IDocument) => (
                 <TooltipHost content={`${item.state_description} `}>
-                    <span>{item.state}</span>
+                    <span onClick={() => onStateColumnClick(item)} style={{ cursor: 'pointer' }}>
+                        {item.state_description}
+                    </span>
                 </TooltipHost>
             )
         }
@@ -348,8 +414,8 @@ export const DocumentsDetailList = ({ items, onFilesSorted}: Props) => {
             <Button text="Resubmit" onClick={handleResubmitClick} />
             {/* Dialog for delete confirmation */}
             <Dialog
-                hidden={!isDialogVisible}
-                onDismiss={() => setIsDialogVisible(false)}
+                hidden={!isDeleteDialogVisible}
+                onDismiss={() => setIsDeleteDialogVisible(false)}
                 dialogContentProps={{
                     type: DialogType.normal,
                     title: 'Delete Confirmation',
@@ -362,7 +428,7 @@ export const DocumentsDetailList = ({ items, onFilesSorted}: Props) => {
             >
                 <DialogFooter>
                     <PrimaryButton onClick={handleDelete} text="Delete" />
-                    <DefaultButton onClick={() => setIsDialogVisible(false)} text="Cancel" />
+                    <DefaultButton onClick={() => setIsDeleteDialogVisible(false)} text="Cancel" />
                 </DialogFooter>
             </Dialog>
             {/* Dialog for resubmit confirmation */}
@@ -387,6 +453,25 @@ export const DocumentsDetailList = ({ items, onFilesSorted}: Props) => {
             <div>
                 <Notification message={notification.message} />
             </div>
+            <Dialog
+                hidden={!stateDialogVisible}
+                onDismiss={() => setStateDialogVisible(false)}
+                dialogContentProps={{
+                    type: DialogType.normal,
+                    title: 'State Details',
+                    closeButtonAriaLabel: 'Close',
+                }}
+                modalProps={{
+                    styles: dialogStyles,
+                }}
+            >
+                <div className="scrollableDialogContent" ref={scrollableContentRef}>
+                    {stateDialogContent}
+                </div>
+                <DialogFooter>
+                    <PrimaryButton onClick={() => setStateDialogVisible(false)} text="OK" />
+                </DialogFooter>
+            </Dialog>
         </div>
     );
 }

app/frontend/src/components/FileStatus/FileStatus.tsx

@@ -165,6 +165,11 @@ export const FileStatus = ({ className }: Props) => {
                     state_description: fileList[i].state_description,
                     upload_timestamp: fileList[i].start_timestamp,
                     modified_timestamp: fileList[i].state_timestamp,
+                    status_updates: fileList[i].status_updates.map(su => ({
+                        status: su.status,
+                        status_timestamp: su.status_timestamp,
+                        status_classification: su.status_classification,
+                    })),
                     value: fileList[i].id,
                 });
             }

functions/FileUploadedFunc/__init__.py

@@ -14,6 +14,7 @@
 from azure.core.credentials import AzureKeyCredential
 
 
+
 azure_blob_connection_string = os.environ["BLOB_CONNECTION_STRING"]
 cosmosdb_url = os.environ["COSMOSDB_URL"]
 cosmosdb_key = os.environ["COSMOSDB_KEY"]
@@ -40,11 +41,10 @@ def main(myblob: func.InputStream):
     try:
         time.sleep(random.randint(1, 2))  # add a random delay
         statusLog = StatusLog(cosmosdb_url, cosmosdb_key, cosmosdb_log_database_name, cosmosdb_log_container_name)
-        statusLog.upsert_document(myblob.name, 'Pipeline triggered by Blob Upload', StatusClassification.INFO, State.PROCESSING, False)            
+        statusLog.upsert_document(myblob.name, 'Pipeline triggered by Blob Upload', StatusClassification.INFO, State.PROCESSING, True)            
         statusLog.upsert_document(myblob.name, f'{function_name} - FileUploadedFunc function started', StatusClassification.DEBUG)    
 
-        # Create message structure to send to queue
-
+        # Create message structure to send to queue      
         file_extension = os.path.splitext(myblob.name)[1][1:].lower()
         if file_extension == 'pdf':
              # If the file is a PDF a message is sent to the PDF processing queue.

functions/shared_code/status_log.py

@@ -105,7 +105,7 @@ def read_files_status_by_timeframe(self,
         """
 
         query_string = "SELECT c.id,  c.file_path, c.file_name, c.state, \
-            c.start_timestamp, c.state_description, c.state_timestamp \
+            c.start_timestamp, c.state_description, c.state_timestamp, c.status_updates \
             FROM c"
 
         conditions = []    

functions/shared_code/utilities.py

@@ -446,6 +446,7 @@ def build_chunks(self, document_map, myblob_name, myblob_uri, chunk_target_size)
         previous_subtitle_name = document_map['structure'][0]["subtitle"]
         page_list = []
         chunk_count = 0
+        previous_paragraph_element_is_a_table = False
 
         # iterate over the paragraphs and build a chuck based on a section
         # and/or title of the document