fix: integrations

backend/modules/sync/dto/inputs.py

@@ -93,6 +93,7 @@ class SyncFileInput(BaseModel):
     syncs_active_id: int
     last_modified: str
     brain_id: str
+    supported: Optional[bool] = True
 
 
 class SyncFileUpdateInput(BaseModel):
@@ -103,4 +104,5 @@ class SyncFileUpdateInput(BaseModel):
         last_modified (datetime.datetime): The updated last modified date and time.
     """
 
-    last_modified: str
+    last_modified: Optional[str] = None
+    supported: Optional[bool] = None

backend/modules/sync/entity/sync.py

@@ -1,8 +1,16 @@
 from datetime import datetime
+from typing import Optional
 
 from pydantic import BaseModel
 
 
+class SyncsUser(BaseModel):
+    id: int
+    user_id: str
+    name: str
+    provider: str
+
+
 class SyncsActive(BaseModel):
     id: int
     name: str
@@ -12,6 +20,7 @@ class SyncsActive(BaseModel):
     last_synced: datetime
     sync_interval_minutes: int
     brain_id: str
+    syncs_user: Optional[SyncsUser]
 
 
 class SyncsFiles(BaseModel):
@@ -20,3 +29,4 @@ class SyncsFiles(BaseModel):
     syncs_active_id: int
     last_modified: str
     brain_id: str
+    supported: bool

backend/modules/sync/repository/sync.py

@@ -64,7 +64,10 @@ def get_syncs_active(self, user_id: str) -> List[SyncsActive]:
         """
         logger.info("Retrieving active syncs for user_id: %s", user_id)
         response = (
-            self.db.from_("syncs_active").select("*").eq("user_id", user_id).execute()
+            self.db.from_("syncs_active")
+            .select("*, syncs_user(*)")
+            .eq("user_id", user_id)
+            .execute()
         )
         if response.data:
             logger.info("Active syncs retrieved successfully: %s", response.data)

backend/modules/sync/repository/sync_files.py

@@ -81,9 +81,9 @@ def update_sync_file(self, sync_file_id: int, sync_file_input: SyncFileUpdateInp
             sync_file_id,
             sync_file_input,
         )
-        self.db.from_("syncs_files").update(sync_file_input.model_dump()).eq(
-            "id", sync_file_id
-        ).execute()
+        self.db.from_("syncs_files").update(
+            sync_file_input.model_dump(exclude_unset=True)
+        ).eq("id", sync_file_id).execute()
         logger.info("Sync file updated successfully")
 
     def delete_sync_file(self, sync_file_id: int):

backend/modules/sync/repository/sync_interfaces.py

@@ -45,7 +45,11 @@ def update_sync_user(
 
     @abstractmethod
     def get_files_folder_user_sync(
-        self, sync_active_id: int, user_id: str, folder_id: int = None
+        self,
+        sync_active_id: int,
+        user_id: str,
+        folder_id: int = None,
+        recursive: bool = False,
     ):
         pass
 

backend/modules/sync/repository/sync_user.py

@@ -159,7 +159,7 @@ def update_sync_user(
         logger.info("Sync user updated successfully")
 
     def get_files_folder_user_sync(
-        self, sync_active_id: int, user_id: str, folder_id: str = None
+        self, sync_active_id: int, user_id: str, folder_id: str = None, recursive: bool = False
     ):
         """
         Retrieve files from a user's sync folder, either from Google Drive or Azure.
@@ -195,10 +195,12 @@ def get_files_folder_user_sync(
         provider = sync_user["provider"].lower()
         if provider == "google":
             logger.info("Getting files for Google sync")
-            return get_google_drive_files(sync_user["credentials"], folder_id)
+            return {
+                "files": get_google_drive_files(sync_user["credentials"], folder_id)
+            }
         elif provider == "azure":
             logger.info("Getting files for Azure sync")
-            return list_azure_files(sync_user["credentials"], folder_id)
+            return {"files": list_azure_files(sync_user["credentials"], folder_id, recursive)}
         else:
-            logger.warning("No sync found for provider: %s", sync_user["provider"])
+            logger.warning("No sync found for provider: %s", sync_user["provider"], recursive)
             return "No sync found"

backend/modules/sync/service/sync_service.py

@@ -36,7 +36,7 @@ def delete_sync_user(self, sync_id: str, user_id: str):
 
     def get_sync_user_by_state(self, state: dict):
         return self.repository.get_sync_user_by_state(state)
-    
+
     def get_sync_user_by_id(self, sync_id: int):
         return self.repository.get_sync_user_by_id(sync_id)
 
@@ -46,10 +46,14 @@ def update_sync_user(
         return self.repository.update_sync_user(sync_user_id, state, sync_user_input)
 
     def get_files_folder_user_sync(
-        self, sync_active_id: int, user_id: str, folder_id: str = None
+        self,
+        sync_active_id: int,
+        user_id: str,
+        folder_id: str = None,
+        recursive: bool = False,
     ):
         return self.repository.get_files_folder_user_sync(
-            sync_active_id, user_id, folder_id
+            sync_active_id, user_id, folder_id, recursive
         )
 
 

backend/modules/sync/utils/googleutils.py

@@ -16,7 +16,10 @@
 )
 from modules.sync.repository.sync_files import SyncFiles
 from modules.sync.service.sync_service import SyncService, SyncUserService
-from modules.sync.utils.list_files import get_google_drive_files
+from modules.sync.utils.list_files import (
+    get_google_drive_files,
+    get_google_drive_files_by_id,
+)
 from modules.sync.utils.upload import upload_file
 from modules.upload.service.upload_file import check_file_exists
 from pydantic import BaseModel, ConfigDict
@@ -131,20 +134,23 @@ async def _upload_files(
                     filename=file_name,
                 )
 
-                await upload_file(to_upload_file, brain_id, current_user)  # type: ignore
-
                 # Check if the file already exists in the database
                 existing_files = self.sync_files_repo.get_sync_files(sync_active_id)
                 existing_file = next(
                     (f for f in existing_files if f.path == file_name), None
                 )
+                supported = False
+                if (existing_file and existing_file.supported) or not existing_file:
+                    supported = True
+                    await upload_file(to_upload_file, brain_id, current_user)  # type: ignore
 
                 if existing_file:
                     # Update the existing file record
                     self.sync_files_repo.update_sync_file(
                         existing_file.id,
                         SyncFileUpdateInput(
                             last_modified=modified_time,
+                            supported=supported,
                         ),
                     )
                 else:
@@ -155,6 +161,7 @@ async def _upload_files(
                             syncs_active_id=sync_active_id,
                             last_modified=modified_time,
                             brain_id=brain_id,
+                            supported=supported,
                         )
                     )
 
@@ -164,6 +171,30 @@ async def _upload_files(
                     "An error occurred while downloading Google Drive files: %s",
                     error,
                 )
+                # Check if the file already exists in the database
+                existing_files = self.sync_files_repo.get_sync_files(sync_active_id)
+                existing_file = next(
+                    (f for f in existing_files if f.path == file["name"]), None
+                )
+                # Update the existing file record
+                if existing_file:
+                    self.sync_files_repo.update_sync_file(
+                        existing_file.id,
+                        SyncFileUpdateInput(
+                            supported=False,
+                        ),
+                    )
+                else:
+                    # Create a new file record
+                    self.sync_files_repo.create_sync_file(
+                        SyncFileInput(
+                            path=file["name"],
+                            syncs_active_id=sync_active_id,
+                            last_modified=file["last_modified"],
+                            brain_id=brain_id,
+                            supported=False,
+                        )
+                    )
         return {"downloaded_files": downloaded_files}
 
     async def sync(self, sync_active_id: int, user_id: str):
@@ -231,12 +262,25 @@ async def sync(self, sync_active_id: int, user_id: str):
             sync_active_id,
         )
 
-        # Get the folder id from the settings from sync_active
         settings = sync_active.get("settings", {})
         folders = settings.get("folders", [])
-        files = get_google_drive_files(
-            sync_user["credentials"], folder_id=folders[0] if folders else None
-        )
+        files_to_download = settings.get("files", [])
+        files = []
+        if len(folders) > 0:
+            files = []
+            for folder in folders:
+                files.extend(
+                    get_google_drive_files(
+                        sync_user["credentials"],
+                        folder_id=folder,
+                        recursive=True,
+                    )
+                )
+        if len(files_to_download) > 0:
+            files_metadata = get_google_drive_files_by_id(
+                sync_user["credentials"], files_to_download
+            )
+        files = files + files_metadata  # type: ignore
         if "error" in files:
             logger.error(
                 "Failed to download files from Google Drive for sync_active_id: %s",
@@ -249,7 +293,7 @@ async def sync(self, sync_active_id: int, user_id: str):
 
         files_to_download = [
             file
-            for file in files.get("files", [])
+            for file in files
             if not file["is_folder"]
             and (
                 (