feat: add Confluence integration with configurable parameters for doc…

.vscode/launch.json

@@ -1,6 +1,22 @@
 {
     "version": "0.2.0",
     "configurations": [
+        {
+            "name": "Debug Pytest (current file)",
+            "type": "python",
+            "request": "launch",
+            "python": "${command:python.interpreterPath}",
+            // run pytest as a module
+            "module": "pytest",
+            "args": [
+              "--maxfail=1",
+              "--disable-warnings",
+              "-q",
+              "${file}"
+            ],
+            "console": "integratedTerminal",
+            "justMyCode": false,
+        },
         {
             "name": "rag_backend",
             "type": "python",

.vscode/settings.json

@@ -6,11 +6,22 @@
         "./rag-core-library/rag-core-api/src",
         "./rag-core-library/rag-core-lib/src",
         "./rag-core-library/extractor-api-lib/src",
+        "./admin-backend",
+        "./rag-backend",
+        "./document-extractor"
     ],
     "[yaml]": {
         "editor.tabSize": 2,
         "editor.insertSpaces": true,
         "editor.formatOnType": true,
         "editor.autoIndent": "advanced"
     },
+    "python.testing.pytestEnabled": true,
+    "python.testing.unittestEnabled": false,
+    "python.testing.pytestArgs": ["--import-mode","importlib"],
+    "python.testing.autoTestDiscoverOnSaveEnabled": true,
+    "python.envFile": "${workspaceFolder}/.env",
+    "python-envs.defaultEnvManager": "ms-python.python:conda",
+    "python-envs.defaultPackageManager": "ms-python.python:conda",
+    "python-envs.pythonProjects": [],
 }

admin-backend/Dockerfile

@@ -19,7 +19,7 @@ COPY admin-backend/pyproject.toml admin-backend/poetry.lock ./
 RUN mkdir log && chmod 700 log
 RUN touch /app/admin-backend/log/logfile.log && chmod 600 /app/admin-backend/log/logfile.log
 
-RUN poetry config virtualenvs.create false &&\
+RUN poetry config virtualenvs.create false && \
     if [ "$dev" = "1" ]; then \
         poetry install --no-interaction --no-ansi --no-root --with dev; \
     else \

conftest.py

@@ -0,0 +1,15 @@
+import sys
+import os
+from pathlib import Path
+
+# Add project root and specific directories to Python path
+project_root = Path(__file__).parent
+sys.path.insert(0, str(project_root))
+sys.path.insert(0, str(project_root / "admin-backend"))
+sys.path.insert(0, str(project_root / "rag-backend"))
+sys.path.insert(0, str(project_root / "document-extractor"))
+
+# point at each rag-core library's src folder so their packages (admin_api_lib, rag_core_api, etc.) are importable
+lib_root = project_root / "rag-core-library"
+for lib in ["admin-api-lib", "rag-core-api", "rag-core-lib", "extractor-api-lib"]:
+    sys.path.insert(0, str(lib_root / lib / "src"))

frontend/libs/admin-app/data-access/+state/documents.store.ts

@@ -3,7 +3,7 @@ import { ref } from 'vue';
 import { DocumentModel } from "../../models/document.model.ts";
 import { ErrorType } from "../../models/error-type";
 import { UploadedDocument, mapToUploadDocument } from "../../models/uploaded-document.model";
-import { DocumentAPI } from "../document.api";
+import { DocumentAPI, ConfluenceConfig } from "../document.api";
 
 export const useDocumentsStore = defineStore('chat', () => {
     const uploadedDocuments = ref<UploadedDocument[]>([]);
@@ -52,11 +52,12 @@ export const useDocumentsStore = defineStore('chat', () => {
         }
     };
 
-    const loadConfluence = async () => {
+    const loadConfluence = async (config: ConfluenceConfig) => {
       isLoadingConfluence.value = true;
       error.value = null;
       try {
-        await DocumentAPI.loadConfluence();
+        // provide confluence configuration from frontend
+        await DocumentAPI.loadConfluence(config);
         await loadDocuments(); // Refresh the document list after uploading
       } catch(err) {
         if (err.response && err.response.status === 501) {

frontend/libs/admin-app/data-access/document.api.ts

@@ -7,6 +7,15 @@ axios.defaults.auth = {
     password: import.meta.env.VITE_AUTH_PASSWORD
 };
 
+// confluence configuration interface
+export interface ConfluenceConfig {
+  spaceKey: string;
+  token: string;
+  url: string;
+  maxPages: number;
+  name: string;
+}
+
 export class DocumentAPI {
     static async loadDocuments(): Promise<DocumentModel[]> {
         try {
@@ -20,9 +29,9 @@ export class DocumentAPI {
     static async uploadDocument(file: File, onUploadProgress: (progressEvent: AxiosProgressEvent) => void): Promise<null> {
         try {
             const formData = new FormData();
-            formData.append('body', file);
+            formData.append('file', file);
 
-            const response = await axios.post<null>('/upload_documents', formData, {
+            const response = await axios.post<null>('/upload_file', formData, {
                 headers: {
                     'Content-Type': 'multipart/form-data'
                 },
@@ -35,9 +44,19 @@ export class DocumentAPI {
         }
     }
 
-    static async loadConfluence(): Promise<void> {
+    static async loadConfluence(config: ConfluenceConfig): Promise<void> {
         try {
-            await axios.post<void>('/load_confluence');
+            // convert config to list of key/value items for backend
+            const payload = [
+                { key: 'url', value: config.url },
+                { key: 'token', value: config.token },
+                { key: 'space_key', value: config.spaceKey },
+                { key: 'max_pages', value: String(config.maxPages) }
+            ];
+            // include required query parameters
+            await axios.post<void>('/upload_source', payload, {
+                params: { source_type: 'confluence', name: config.name }
+            });
         } catch(error) {
             this.handleError(error);
         }

frontend/libs/admin-app/feature-document/DocumentUploadContainer.vue

@@ -15,6 +15,14 @@ const isInvalidFileType = ref(false);
 const allowedFileTypes = ['application/pdf', 'application/vnd.openxmlformats-officedocument.wordprocessingml.document', 'application/vnd.openxmlformats-officedocument.presentationml.presentation', 'text/xml'];
 const uploadMethod = ref<'file' | 'confluence'>('file');
 
+
+// confluence configuration refs
+const confluenceName = ref('');
+const spaceKey = ref('');
+const confluenceToken = ref('');
+const confluenceUrl = ref('');
+const maxPages = ref<number>();
+
 const error = computed(() => store.error);
 
 const uploadDocuments = (files: File[]) => {
@@ -55,7 +63,14 @@ const onRemoveDocument = (documentId: string) => {
 }
 
 const handleConfluenceUpload = () => {
-    store.loadConfluence();
+    // send configured parameters to backend
+    store.loadConfluence({
+        name: confluenceName.value,
+        spaceKey: spaceKey.value,
+        token: confluenceToken.value,
+        url: confluenceUrl.value,
+        maxPages: maxPages.value
+    });
 }
 
 const clearError = () => {
@@ -113,7 +128,7 @@ const getErrorMessage = (errorType: string) => {
 
             <!-- File upload area -->
             <div v-if="uploadMethod === 'file'"
-                class="flex flex-col m-auto justify-center items-center w-full h-64 bg-base-100 rounded-box border border-base-300 border-dashed"
+                class="flex flex-col m-auto justify-center items-center w-full h-96 bg-base-100 rounded-box border border-base-300 border-dashed"
                 :class="{'bg-base-200': isDragOver}" @dragover.prevent="onDragOver" @dragleave.prevent="onDragLeave"
                 @drop.prevent="onDrop">
                 <div class="flex flex-col justify-center items-center pt-5 pb-6">
@@ -130,10 +145,23 @@ const getErrorMessage = (errorType: string) => {
 
             <!-- Confluence load area -->
             <div v-else
-                class="flex flex-col m-auto justify-center items-center w-full h-64 bg-base-100 rounded-box border border-base-300">
+                class="flex flex-col m-auto justify-center items-center w-full h-112 bg-base-100 rounded-box border border-base-300">
                 <div class="flex flex-col justify-center items-center pt-5 pb-6">
                     <GlobeAltIcon class="w-10 h-10 mb-4 text-accent-content" />
                     <p class="mb-1 font-bold">{{ t('documents.confluenceLoadTitle') }}</p>
+                    <!-- configuration inputs -->
+                    <div class="space-y-2 mb-4 w-full max-w-sm">
+                      <label for="confluenceUrl" class="sr-only">Confluence URL</label>
+                      <input v-model="confluenceUrl" type="url" placeholder="URL" class="input input-bordered w-full" />
+                      <label for="confluenceName" class="sr-only"> Confluence Name</label>
+                      <input v-model="confluenceName" type="text" placeholder="Name" class="input input-bordered w-full" />
+                      <label for="spaceKey" class="sr-only">Space key</label>
+                      <input v-model="spaceKey" type="text" placeholder="Space key" class="input input-bordered w-full" />
+                      <label for="confluenceToken" class="sr-only">Token</label>
+                      <input v-model="confluenceToken" type="password" placeholder="Token" class="input input-bordered w-full" />
+                      <label for="maxPages" class="sr-only">Max pages</label>
+                      <input v-model.number="maxPages" type="number" placeholder="Max number of pages" class="input input-bordered w-full" />
+                    </div>
                     <p class="text-xs opacity-50 mb-4">{{ t('documents.confluenceLoadDescription') }}</p>
                     <button class="btn btn-sm btn-accent" @click="handleConfluenceUpload">
                         {{ t('documents.loadConfluence') }}

frontend/libs/i18n/admin/en.json

@@ -12,7 +12,7 @@
     "fileTypeNotAllowedDescription": "Only PDF, DOCX, PPTX, and XML files are allowed",
     "fileUpload": "File Upload",
     "confluenceUpload": "Confluence",
-    "confluenceLoadTitle": "Load Confluence Pages",
+    "confluenceLoadTitle": "Load all Confluence pages from a space",
     "confluenceLoadDescription": "Click the button below to load pages from Confluence",
     "loadConfluence": "Load Confluence",
     "select": "Select",