Delete data source

Author: Roey7

app/api/data_source.py

@@ -70,11 +70,11 @@ async def delete_data_source(data_source_id: int):
 
 
 @router.post("")
-async def add_integration(dto: AddDataSource, background_tasks: BackgroundTasks):
+async def add_integration(dto: AddDataSource, background_tasks: BackgroundTasks) -> int:
     data_source = DataSourceContext.create_data_source(name=dto.name, config=dto.config)
 
     # in main.py we have a background task that runs every 5 minutes and indexes the data source
     # but here we want to index the data source immediately
     background_tasks.add_task(data_source.index)
 
-    return {"success": "Data source added successfully"}
+    return data_source.get_id()

app/data_source/api/base_data_source.py

@@ -83,6 +83,9 @@ def __init__(self, config: Dict, data_source_id: int, last_index_time: datetime
         self._last_index_time = last_index_time
         self._last_task_time = None
 
+    def get_id(self):
+        return self._data_source_id
+
     def _save_index_time_in_db(self) -> None:
         """
         Sets the index time in the database, to be now

app/indexing/bm25_index.py

@@ -1,16 +1,14 @@
 import os
 import pickle
+from typing import List
+
 import nltk
 import numpy as np
 from rank_bm25 import BM25Okapi
-from sqlalchemy import Connection
-from typing import List
 
 from db_engine import Session
-from schemas import Paragraph
 from paths import BM25_INDEX_PATH
-
-
+from schemas import Paragraph
 
 
 def _add_metadata_for_indexing(paragraph: Paragraph) -> str:
@@ -60,13 +58,13 @@ def _update(self, session):
         self.index = BM25Okapi(corpus)
         self.id_map = id_map
 
-    def update(self, session = None):
+    def update(self, session=None):
         if session is None:
             with Session() as session:
                 self._update(session)
         else:
             self._update(session)
-        
+
         self._save()
 
     def search(self, query: str, top_k: int) -> List[int]:
@@ -87,4 +85,4 @@ def clear(self):
 
     def _save(self):
         with open(BM25_INDEX_PATH, 'wb') as f:
-            pickle.dump(self, f)
+            pickle.dump(self, f)

ui/src/App.tsx

@@ -22,15 +22,15 @@ import 'react-toastify/dist/ReactToastify.css';
 import { ClipLoader } from "react-spinners";
 import { FiSettings } from "react-icons/fi";
 import {AiFillWarning} from "react-icons/ai";
-import { ConnectedDataSourceType, DataSourceType } from "./data-source";
+import { ConnectedDataSource, DataSourceType } from "./data-source";
 
 export interface AppState {
   query: string
   results: SearchResultDetails[]
   searchDuration: number
   dataSourceTypes: DataSourceType[]
   dataSourceTypesDict: { [key: string]: DataSourceType }
-  connectedDataSources: string[]
+  connectedDataSources: ConnectedDataSource[]
   isLoading: boolean
   isNoResults: boolean
   isModalOpen: boolean
@@ -137,9 +137,8 @@ export default class App extends React.Component <{}, AppState>{
 
   async listConnectedDataSources() {
     try {
-      const response = await api.get<ConnectedDataSourceType[]>('/data-sources/connected');
-      let nameList = response.data.map((dataSource) => dataSource.name);
-      this.setState({ connectedDataSources: nameList })
+      const response = await api.get<ConnectedDataSource[]>('/data-sources/connected');
+      this.setState({ connectedDataSources: response.data });
     } catch (error) {
     }
   }
@@ -168,7 +167,7 @@ export default class App extends React.Component <{}, AppState>{
       }
 
       this.setState({isServerDown: false, docsLeftToIndex: res.data.docs_left_to_index,
-                     docsInIndexing: res.data.docs_in_indexing, isPreparingIndexing: isPreparingIndexing});
+                    docsInIndexing: res.data.docs_in_indexing, isPreparingIndexing: isPreparingIndexing});
 
       let timeToSleep = isPreparingIndexing ? 1000 : successSleepSeconds * 1000;
       setTimeout(() => this.fetchStatsusForever(), timeToSleep);
@@ -184,7 +183,7 @@ export default class App extends React.Component <{}, AppState>{
     })    
   }
 
-  shouldShowIndexingStatus() {
+  inIndexing() {
     return this.state.isPreparingIndexing || this.state.docsInIndexing > 0 || this.state.docsLeftToIndex > 0;
   }
 
@@ -261,8 +260,8 @@ export default class App extends React.Component <{}, AppState>{
     toast.success("Code accepted. Welcome!", {autoClose: 3000});
   }
 
-  dataSourcesAdded = (dataSourceType: string) => {
-    this.setState({isPreparingIndexing: true, connectedDataSources: [...this.state.connectedDataSources, dataSourceType]});
+  dataSourcesAdded = (newlyConnected: ConnectedDataSource) => {
+    this.setState({isPreparingIndexing: true, connectedDataSources: [...this.state.connectedDataSources, newlyConnected]});
     // if had no data from server, show toast after 30 seconds
     setTimeout(() => {
       if (this.state.isPreparingIndexing) {
@@ -272,6 +271,10 @@ export default class App extends React.Component <{}, AppState>{
     }, 1000 * 120);
   }
 
+  dataSourceRemoved = (removed: ConnectedDataSource) => {
+    this.setState({connectedDataSources: this.state.connectedDataSources.filter((ds) => ds.id !== removed.id)});
+  }
+
   render() {
     return (
     <div>
@@ -280,7 +283,7 @@ export default class App extends React.Component <{}, AppState>{
         className="absolute right-0 z-30 float-right mr-6 mt-6 text-[42px] hover:cursor-pointer hover:rotate-90 transition-all duration-300 hover:drop-shadow-2xl">
       </FiSettings>
         {
-          this.shouldShowIndexingStatus() &&
+          this.inIndexing() &&
           <div className="absolute mx-auto left-0 right-0 w-fit z-20 top-6">
             <div className="text-xs bg-[#191919] border-[#4F4F4F] border-[.8px] rounded-full inline-block px-3 py-1">
               <div className="text-[#E4E4E4] font-medium font-inter text-sm flex flex-row justify-center items-center">
@@ -348,7 +351,8 @@ export default class App extends React.Component <{}, AppState>{
           contentLabel="Example Modal"
           style={customStyles}>
           <DataSourcePanel onClose={this.closeModal} connectedDataSources={this.state.connectedDataSources}
-            onAdded={this.dataSourcesAdded} dataSourceTypesDict={this.state.dataSourceTypesDict}/>
+            inIndexing={this.inIndexing()}
+            onAdded={this.dataSourcesAdded} dataSourceTypesDict={this.state.dataSourceTypesDict} onRemoved={this.dataSourceRemoved} />
         </Modal>
 
         {/* front search page*/}

ui/src/components/data-source-panel.tsx

@@ -5,7 +5,7 @@ import copy from 'copy-to-clipboard';
 import CopyThis from '../assets/images/copy-this.png';
 import LeftPane from '../assets/images/left-pane-instructions.png';
 
-import { BsFillPencilFill } from 'react-icons/bs';
+import { FaRegEdit } from "react-icons/fa";
 import { AiFillCheckCircle } from "react-icons/ai";
 import { BiLinkExternal } from 'react-icons/bi';
 import { IoMdClose, IoMdCloseCircle } from "react-icons/io";
@@ -14,7 +14,7 @@ import { RxCopy } from 'react-icons/rx';
 import { ClipLoader } from "react-spinners";
 import { toast } from 'react-toastify';
 import { api } from "../api";
-import { ConfigField, DataSourceType } from "../data-source";
+import { ConfigField, ConnectedDataSource, DataSourceType } from "../data-source";
 
 
 export interface SelectOption {
@@ -34,8 +34,10 @@ export interface DataSourcePanelState {
 
 export interface DataSourcePanelProps {
    dataSourceTypesDict: { [key: string]: DataSourceType }
-   connectedDataSources: string[]
-   onAdded: (dataSourceType: string) => void
+   connectedDataSources: ConnectedDataSource[]
+   inIndexing: boolean
+   onAdded: (newlyConnected: ConnectedDataSource) => void
+   onRemoved: (removed: ConnectedDataSource) => void
    onClose: () => void
 }
 
@@ -136,19 +138,19 @@ export default class DataSourcePanel extends React.Component<DataSourcePanelProp
                !this.state.isAdding && (
                   <div className="w-full">
                      <h1 className="text-2xl block text-white mb-4">
-                        {this.props.connectedDataSources.length > 0 ? 'Active data sources:' : 'No Active Data Sources. Add Now!'}
-                        <BsFillPencilFill key="pencil" onClick={this.swithcMode} className='text-white float-right inline hover:text-[#9875d4] hover:cursor-pointer' />
+                        {this.props.connectedDataSources.length > 0 ? (this.state.editMode ? 'Edit mode:' : 'Active data sources:') : 'No Active Data Sources. Add Now!'}
+                        {this.props.connectedDataSources.length > 0 && <FaRegEdit key="pencil" onClick={this.swithcMode} className='text-white mt-1 float-right inline hover:text-[#9875d4] hover:cursor-pointer' />}
                      </h1>
                      <div className="flex flex-row w-[100%] flex-wrap">
-                        {this.props.connectedDataSources.map((data_source) => {
+                        {this.props.connectedDataSources.map((dataSource) => {
                            return (
                               // connected data source
                               <div className="flex py-2 pl-5 pr-3 m-2 flex-row items-center justify-center bg-[#352C45] hover:shadow-inner shadow-blue-500/50 rounded-lg font-poppins leading-[28px] border-b-[#916CCD] border-b-2">
-                                 <img alt="data-source" className={"mr-2 h-[20px]"} src={this.props.dataSourceTypesDict[data_source].image_base64}></img>
-                                 <h1 className="text-white width-full">{this.props.dataSourceTypesDict[data_source].display_name}</h1>
+                                 <img alt="data-source" className={"mr-2 h-[20px]"} src={this.props.dataSourceTypesDict[dataSource.name].image_base64}></img>
+                                 <h1 className="text-white width-full">{this.props.dataSourceTypesDict[dataSource.name].display_name}</h1>
 
                                  {this.state.editMode ? (
-                                    <IoMdCloseCircle onClick={this.removeDataSource} className="ml-6 fill-[#df335e] text-2xl hover:fill-[#ff85c2]" />
+                                    <IoMdCloseCircle onClick={() => this.removeDataSource(dataSource)} className="transition duration-150 ease-in-out  ml-6 fill-[#7d4ac3] hover:cursor-pointer text-2xl hover:fill-[#d80b0b]" />
                                  ) : (
                                     <AiFillCheckCircle className="ml-6 text-[#9875d4] text-2xl" />
                                  )
@@ -161,7 +163,7 @@ export default class DataSourcePanel extends React.Component<DataSourcePanelProp
                         {
                            Object.keys(this.props.dataSourceTypesDict).map((key) => {
                               let dataSource = this.props.dataSourceTypesDict[key];
-                              if (!this.props.connectedDataSources.includes(dataSource.name) && !this.state.editMode) {
+                              if (!this.state.editMode && !this.props.connectedDataSources.find((connectedDataSource) => connectedDataSource.name === dataSource.name)) {
                                  return (
                                     // unconnected data source
                                     <div onClick={() => this.dataSourceToAddSelected(dataSource)} className="flex hover:text-[#9875d4] py-2 pl-5 pr-3 m-2 flex-row items-center justify-center bg-[#36323b] hover:border-[#9875d4] rounded-lg font-poppins leading-[28px] border-[#777777] border-b-[.5px] transition duration-300 ease-in-out">
@@ -407,7 +409,7 @@ export default class DataSourcePanel extends React.Component<DataSourcePanelProp
             field.value = '';
          });
          this.setState({ selectedDataSource: selectedDataSource });
-         this.props.onAdded(this.state.selectedDataSource.value);
+         this.props.onAdded({name: this.state.selectedDataSource.value, id: response.data});
          this.setState({ isAddingLoading: false, isAdding: false, selectedDataSource: this.state.selectOptions[0] });
       }).catch(error => {
          toast.error("Error adding data source: " + error.response.data, { autoClose: 10000 });
@@ -419,10 +421,19 @@ export default class DataSourcePanel extends React.Component<DataSourcePanelProp
       this.setState({ selectedDataSource: event })
    }
 
-   removeDataSource = () => {
-      let connected = this.props.connectedDataSources;
-      let index = connected.indexOf(this.state.selectedDataSource.value);
-      connected.splice(index, 1);
+   removeDataSource = (connectedDataSource: ConnectedDataSource) => {
+      if (this.props.inIndexing) {
+         toast.error("Cannot remove data source while indexing is in progress");
+         return;
+      }
+
+      api.delete(`/data-sources/${connectedDataSource.id}`).then(response => {
+         toast.success(`${this.capitilize(connectedDataSource.name)} removed.`);
+         this.props.onRemoved(connectedDataSource);
+      }).catch(error => {
+         toast.error("Error adding data source: " + error.response.data, { autoClose: 10000 });
+         this.setState({ isAddingLoading: false });
+      });
    }
 
 

ui/src/data-source.ts

@@ -20,7 +20,7 @@ export interface DataSourceType {
     image_base64: string
 }
 
-export interface ConnectedDataSourceType {
+export interface ConnectedDataSource {
     id: number
     name: string
 }