Slack bot messages, fix indexing gui

Roey7 · Roey7 · commit 6a3a63313f84 · 2023-04-09T14:50:31.000+03:00
diff --git a/app/data_source/sources/slack/slack.py b/app/data_source/sources/slack/slack.py
@@ -49,7 +49,7 @@ def validate_config(config: Dict) -> None:
 
     @staticmethod
     def _is_valid_message(message: Dict) -> bool:
-        return 'client_msg_id' in message
+        return 'client_msg_id' in message or 'bot_id' in message
 
     def __init__(self, *args, **kwargs):
         super().__init__(*args, **kwargs)
@@ -62,15 +62,15 @@ def _list_conversations(self) -> List[SlackConversation]:
         return [SlackConversation(id=conv['id'], name=conv['name'])
                 for conv in conversations['channels']]
 
-    def _join_conversations(self, conversations: List[SlackConversation]) -> List[SlackConversation]:
+    def _feed_conversations(self, conversations: List[SlackConversation]) -> List[SlackConversation]:
         joined_conversations = []
 
         for conv in conversations:
             try:
                 result = self._slack.conversations_join(channel=conv.id)
                 if result['ok']:
-                    logger.info(f'Joined channel {conv.name}')
-                    joined_conversations.append(conv)
+                    logger.info(f'Joined channel {conv.name}, adding a fetching task...')
+                    self.add_task_to_queue(self._feed_conversation, conv=conv)
             except Exception as e:
                 logger.warning(f'Could not join channel {conv.name}: {e}')
 
@@ -92,11 +92,7 @@ def _feed_new_documents(self) -> None:
         conversations = self._list_conversations()
         logger.info(f'Found {len(conversations)} conversations')
 
-        joined_conversations = self._join_conversations(conversations)
-        logger.info(f'Joined {len(joined_conversations)} conversations')
-
-        for conv in joined_conversations:
-            self.add_task_to_queue(self._feed_conversation, conv=conv)
+        self._feed_conversations(conversations)
 
     def _feed_conversation(self, conv: SlackConversation):
         logger.info(f'Feeding conversation {conv.name}')
@@ -112,8 +108,14 @@ def _feed_conversation(self, conv: SlackConversation):
                 continue
 
             text = message['text']
-            author_id = message['user']
-            author = self._get_author_details(author_id)
+            if author_id := message.get('user'):
+                author = self._get_author_details(author_id)
+            elif message.get('bot_id'):
+                author = SlackAuthor(name=message.get('username'), image_url=message.get('icons', {}).get('image_48'))
+            else:
+                logger.warning(f'Unknown message author: {message}')
+                continue
+
             if last_msg is not None:
                 if last_msg.author == author.name:
                     last_msg.content += f"\n{text}"
@@ -123,7 +125,7 @@ def _feed_conversation(self, conv: SlackConversation):
                     last_msg = None
 
             timestamp = message['ts']
-            message_id = message['client_msg_id']
+            message_id = message.get('client_msg_id') or timestamp
             readable_timestamp = datetime.datetime.fromtimestamp(float(timestamp))
             message_url = f"https://slack.com/app_redirect?channel={conv.id}&message_ts={timestamp}"
             last_msg = BasicDocument(title=author.name, content=text, author=author.name,
diff --git a/app/indexing/background_indexer.py b/app/indexing/background_indexer.py
@@ -13,11 +13,16 @@ class BackgroundIndexer:
     _thread = None
     _stop_event = threading.Event()
     _currently_indexing_count = 0
+    _total_indexed_count = 0
 
     @classmethod
-    def get_currently_indexing(cls):
+    def get_currently_indexing(cls) -> int:
         return cls._currently_indexing_count
 
+    @classmethod
+    def get_indexed_count(cls) -> int:
+        return cls._total_indexed_count
+
     @classmethod
     def start(cls):
         cls._thread = threading.Thread(target=cls.run)
@@ -39,16 +44,20 @@ def run():
         logger.info(f'Background indexer started...')
 
         while not BackgroundIndexer._stop_event.is_set():
-            queue_items = docs_queue_instance.consume_all()
-            if not queue_items:
-                continue
+            try:
+                queue_items = docs_queue_instance.consume_all()
+                if not queue_items:
+                    continue
 
-            BackgroundIndexer._currently_indexing_count = len(queue_items)
-            logger.info(f'Got chunk of {len(queue_items)} documents')
+                BackgroundIndexer._currently_indexing_count = len(queue_items)
+                logger.info(f'Got chunk of {len(queue_items)} documents')
 
-            docs = [doc.doc for doc in queue_items]
-            Indexer.index_documents(docs)
-            BackgroundIndexer._ack_chunk(docs_queue_instance, [doc.queue_item_id for doc in queue_items])
+                docs = [doc.doc for doc in queue_items]
+                Indexer.index_documents(docs)
+                BackgroundIndexer._ack_chunk(docs_queue_instance, [doc.queue_item_id for doc in queue_items])
+            except Exception as e:
+                logger.exception(e)
+                logger.error('Error while indexing documents...')
 
     @staticmethod
     def _ack_chunk(queue: IndexQueue, ids: List[int]):
@@ -57,4 +66,5 @@ def _ack_chunk(queue: IndexQueue, ids: List[int]):
             queue.ack(id=item_id)
 
         logger.info(f'Acked {len(ids)} documents.')
+        BackgroundIndexer._total_indexed_count += len(ids)
         BackgroundIndexer._currently_indexing_count = 0
diff --git a/app/main.py b/app/main.py
@@ -114,14 +114,16 @@ async def shutdown_event():
 
 
 @app.get("/api/v1/status")
-async def status():
+def status():
     @dataclass
     class Status:
         docs_in_indexing: int
         docs_left_to_index: int
+        docs_indexed: int
 
     return Status(docs_in_indexing=BackgroundIndexer.get_currently_indexing(),
-                  docs_left_to_index=IndexQueue.get_instance().qsize() + TaskQueue.get_instance().qsize())
+                  docs_left_to_index=IndexQueue.get_instance().qsize() + TaskQueue.get_instance().qsize(),
+                  docs_indexed=BackgroundIndexer.get_indexed_count())
 
 
 @app.post("/clear-index")
diff --git a/ui/src/App.tsx b/ui/src/App.tsx
@@ -40,17 +40,21 @@ export interface AppState {
   isServerDown: boolean
   isStartedFetching: boolean
   isPreparingIndexing: boolean
+  isIndexing: boolean
   didPassDiscord: boolean
   discordCodeInput: string
   docsLeftToIndex: number
   docsInIndexing: number
-  lastServerDownTimestamp: number
+  docsIndexed: number
+  timeSinceLastIndexing: number
+  serverDownCount: number
   showResultsPage: boolean
 }
 
 export interface ServerStatus {
   docs_in_indexing: number
   docs_left_to_index: number
+  docs_indexed: number
 }
 
 Modal.setAppElement('#root');
@@ -103,8 +107,11 @@ export default class App extends React.Component <{}, AppState>{
       didPassDiscord: false,
       docsLeftToIndex: 0,
       docsInIndexing: 0,
+      docsIndexed: 0,
+      isIndexing: false,
+      serverDownCount: 0,
+      timeSinceLastIndexing: 0,
       searchDuration: 0,
-      lastServerDownTimestamp: 0,
       showResultsPage: false
     }
 
@@ -164,11 +171,9 @@ export default class App extends React.Component <{}, AppState>{
   }
 
   async fetchStatsusForever() {
-    let successSleepSeconds = 1;
     let timeBetweenFailToast = 5;
     let failSleepSeconds = 1;
-
-    api.get<ServerStatus>('/status').then((res) => {
+    api.get<ServerStatus>('/status', {timeout: 3000}).then((res) => {
       if (this.state.isServerDown) {
         toast.dismiss();
         if (!document.hidden) {
@@ -178,33 +183,37 @@ export default class App extends React.Component <{}, AppState>{
       }
 
       let isPreparingIndexing = this.state.isPreparingIndexing;
-      if (this.state.isPreparingIndexing && (res.data.docs_in_indexing > 0 || res.data.docs_left_to_index > 0)) {
+      let isIndexing = this.state.isIndexing;
+      let lastIndexingTime = this.state.timeSinceLastIndexing;
+      if (res.data.docs_in_indexing > 0 || res.data.docs_left_to_index > 0 || (res.data.docs_indexed > this.state.docsIndexed && this.state.docsIndexed > 0)) {
+        isIndexing = true;
+        lastIndexingTime = Date.now();
         isPreparingIndexing = false;
-      }
-
-      if(this.state.docsInIndexing > 0 && (res.data.docs_in_indexing === 0 && res.data.docs_left_to_index === 0)) {
+      } else if (isIndexing && Date.now() - lastIndexingTime > (1000 * 60 * 1)) {
+        isIndexing = false;
         toast.success("Indexing finished.", {autoClose: 2000});
       }
 
       this.setState({isServerDown: false, docsLeftToIndex: res.data.docs_left_to_index,
-                    docsInIndexing: res.data.docs_in_indexing, isPreparingIndexing: isPreparingIndexing});
+                    docsInIndexing: res.data.docs_in_indexing, isPreparingIndexing: isPreparingIndexing,
+                    docsIndexed: res.data.docs_indexed, isIndexing: isIndexing, timeSinceLastIndexing: lastIndexingTime});
 
-      let timeToSleep = isPreparingIndexing ? 1000 : successSleepSeconds * 1000;
+      let timeToSleep = 1000;
       setTimeout(() => this.fetchStatsusForever(), timeToSleep);
     }).catch((err) => {
-      this.setState({isServerDown: true});
+      this.setState({serverDownCount: this.state.serverDownCount + 1});
 
-      if (Date.now() - this.state.lastServerDownTimestamp > 6000 && !document.hidden) {  // if it's 6 seconds since last server down, show a toast
+      if (this.state.serverDownCount > 5 && !document.hidden) {  // if it's 6 seconds since last server down, show a toast
         toast.dismiss();
-        toast.error(`Server is down, retrying in ${timeBetweenFailToast} seconds...`, {autoClose: (timeBetweenFailToast-1) * 1000});
-        this.setState({lastServerDownTimestamp: Date.now()});
+        toast.error(`Server is not responding (retrying...)`, {autoClose: (timeBetweenFailToast-1) * 1000});
+        this.setState({isServerDown: true, serverDownCount: 0});
       }
       setTimeout(() => this.fetchStatsusForever(), failSleepSeconds * 1000);
     })    
   }
 
   inIndexing() {
-    return this.state.isPreparingIndexing || this.state.docsInIndexing > 0 || this.state.docsLeftToIndex > 0;
+    return this.state.isPreparingIndexing || this.state.isIndexing;
   }
 
   getIndexingStatusText() {
@@ -213,17 +222,36 @@ export default class App extends React.Component <{}, AppState>{
     }
 
     if (this.state.docsInIndexing > 0) {
-      let text = "Indexing " + this.state.docsInIndexing + " documents... it might take a while.";
+      let text = "Indexing " + this.state.docsInIndexing + " documents...";
       if (this.state.docsLeftToIndex > 0) {
-        text += " (" + this.state.docsLeftToIndex + "~ left in queue)";
+        text += " (" + this.state.docsLeftToIndex + " left in queue";
+        if (this.state.docsIndexed > 0) {
+          text += ", " + this.state.docsIndexed + " documents are indexed & searchable)";
+        } else {
+          text += ")";
+        }
+
+      } else {
+        if (this.state.docsIndexed > 0) {
+          text += " (" + this.state.docsIndexed + " documents are indexed & searchable)";
+        }
       }
 
       return text;
     }
 
     if (this.state.docsLeftToIndex > 0) {
-      return "Preparing to index...";
+      let text = "Preparing to index";
+      if (this.state.docsIndexed > 0) {
+        text += " (" + this.state.docsIndexed + " documents are indexed & searchable)";
+      } else {
+        text += "...";
+      }
+      return text;
     }
+
+    return `Indexing... (${this.state.docsIndexed} documents are indexed & searchable)`;
+    
   }
   
   openModal() {
diff --git a/ui/src/components/search-result.tsx b/ui/src/components/search-result.tsx
@@ -87,8 +87,14 @@ export const SearchResult = (props: SearchResultProps) => {
                                 <span className='flex flex-row items-center justify-center ml-2 mt-[5px]'>
                                     Commented {getDaysAgo(props.resultDetails.time)}
                                 </span>
-                            )
-                                
+                            )     
+                        }
+                        {
+                            props.resultDetails.type === ResultType.Message && (
+                                <span className='flex flex-row items-center justify-center ml-2 mt-[5px]'>
+                                    Sent {getDaysAgo(props.resultDetails.time)}
+                                </span>
+                            )     
                         }
                         {props.resultDetails.type === ResultType.Issue &&
                             <span className='flex flex-row items-center ml-2 px-[7px] py-[1px] font-poppins font-medium text-[15px] bg-[#392E58] text-[#8F76C6] rounded-lg'>
@@ -120,12 +126,15 @@ export const SearchResult = (props: SearchResultProps) => {
                                 {props.resultDetails.type === ResultType.Message && '#'}
                                 {props.resultDetails.location} </span>
                         </span>
-                        <span className="ml-1 flex flex-row items-center">
-                            <Img alt="author" className="inline-block ml-[6px] mr-2 h-4 rounded-xl"
-                                src={[props.resultDetails.author_image_url, props.resultDetails.author_image_data, DefaultUserImage]}></Img>
-                            <span className='capitalize'>{props.resultDetails.author} </span>
-                        </span>
-                        {props.resultDetails.child === null && DateSpan(props)}
+                        {
+                        props.resultDetails.type !== ResultType.Message &&
+                            <span className="ml-1 flex flex-row items-center">
+                                <Img alt="author" className="inline-block ml-[6px] mr-2 h-4 rounded-xl"
+                                    src={[props.resultDetails.author_image_url, props.resultDetails.author_image_data, DefaultUserImage]}></Img>
+                                <span className='capitalize'>{props.resultDetails.author} </span>
+                            </span>
+                        }
+                        {props.resultDetails.child === null && props.resultDetails.type !== ResultType.Message && DateSpan(props)}
                         <span className="flex flex-row items-center">
                             &thinsp; |&thinsp;
                             <img alt="file-type" className="inline ml-2 mx-1  h-[12px] w-[12px] grayscale-[0.55]"
