Update connection should not replace existing attributes

nesis/api/core/controllers/datasources.py

@@ -37,8 +37,6 @@ def operate_datasources():
         return jsonify(error_message("Unauthorized access")), 401
     except util.PermissionException:
         return jsonify(error_message("Forbidden action on resource")), 403
-    except util.ValidationException:
-        return jsonify(error_message("Unable to validate datasource connection")), 403
     except:
         _LOG.exception("Error getting user")
         return jsonify(error_message("Server error")), 500

nesis/api/core/document_loaders/minio.py

@@ -269,4 +269,8 @@ def validate_connection_info(connection: Dict[str, Any]) -> Dict[str, Any]:
     assert not isblank(
         connection.get("dataobjects")
     ), "One or more buckets must be supplied"
-    return {key: val for key, val in connection.items() if key in _valid_keys}
+    return {
+        key: val
+        for key, val in connection.items()
+        if key in _valid_keys and not isblank(connection[key])
+    }

nesis/api/core/document_loaders/s3.py

@@ -294,4 +294,8 @@ def validate_connection_info(connection: Dict[str, Any]) -> Dict[str, Any]:
     assert not isblank(
         connection.get("dataobjects")
     ), "One or more buckets must be supplied"
-    return {key: val for key, val in connection.items() if key in _valid_keys}
+    return {
+        key: val
+        for key, val in connection.items()
+        if key in _valid_keys and not isblank(connection[key])
+    }

nesis/api/core/document_loaders/samba.py

@@ -53,40 +53,36 @@ def fetch_documents(
 
 
 def validate_connection_info(connection: Dict[str, Any]) -> Dict[str, Any]:
-    port = connection.get("port")
+    port = connection.get("port") or DEFAULT_SAMBA_PORT
     _valid_keys = ["port", "endpoint", "user", "password", "dataobjects"]
-    if port is None or not port:
-        connection["port"] = DEFAULT_SAMBA_PORT
-    elif not port.isnumeric():
-        raise ValidationException("Port value cannot be non numeric")
+    if not str(port).isnumeric():
+        raise ValueError("Port value cannot be non numeric")
 
     assert not isblank(
         connection.get("endpoint")
     ), "A valid share address must be supplied"
-    assert not isblank(connection.get("user")), "A valid user"
-    assert not isblank(connection.get("password")), "A valid password"
 
     try:
         _connect_samba_server(connection)
-    except ValidationException as sb:
+    except Exception as ex:
         _LOG.exception(
             f"Failed to connect to samba server at {connection['endpoint']}",
-            stack_info=True,
         )
-        raise
-    return {key: val for key, val in connection.items() if key in _valid_keys}
+        raise ValueError(ex)
+    connection["port"] = port
+    return {
+        key: val
+        for key, val in connection.items()
+        if key in _valid_keys and not isblank(connection[key])
+    }
 
 
 def _connect_samba_server(connection):
-    username = connection["user"]
-    password = connection["password"]
-    endpoint = connection["endpoint"]
-    port = connection["port"]
-    try:
-        scandir(endpoint, username=username, password=password, port=port)
-    except Exception as ex:
-        _LOG.exception(f"Error while connecting to samba server {endpoint} - {ex}")
-        raise
+    username = connection.get("user")
+    password = connection.get("password")
+    endpoint = connection.get("endpoint")
+    port = connection.get("port")
+    next(scandir(endpoint, username=username, password=password, port=port))
 
 
 def _sync_samba_documents(

nesis/api/core/document_loaders/sharepoint.py

@@ -216,4 +216,8 @@ def validate_connection_info(connection: Dict[str, Any]) -> Dict[str, Any]:
     assert not isblank(
         connection.get("certificate")
     ), "A valid certificate must be supplied"
-    return {key: val for key, val in connection.items() if key in _valid_keys}
+    return {
+        key: val
+        for key, val in connection.items()
+        if key in _valid_keys and not isblank(connection[key])
+    }

nesis/api/core/services/datasources.py

@@ -291,7 +291,10 @@ def update(self, **kwargs):
             if datasource.get("connection"):
                 try:
                     connection = validators.validate_datasource_connection(datasource)
-                    datasource_record.connection = connection
+                    datasource_record.connection = {
+                        **datasource_record.connection,
+                        **connection,
+                    }
                 except (ValueError, AssertionError) as ve:
                     raise ServiceException(ve)
 

nesis/api/core/util/__init__.py

@@ -42,4 +42,4 @@ def run_sql(engine, path):
 
 
 def isblank(item: str) -> bool:
-    return item is None or item == "" or item.isspace()
+    return item is None or item == "" or str(item).isspace()

nesis/api/tests/tasks/test_document_management.py

@@ -94,11 +94,13 @@ def test_ingest_datasource_minio(
     )
 
 
+@mock.patch("nesis.api.core.document_loaders.samba.scandir")
 @mock.patch("nesis.api.core.tasks.document_management.samba._unsync_samba_documents")
 @mock.patch("nesis.api.core.tasks.document_management.samba._sync_samba_documents")
 def test_ingest_datasource_samba(
     _sync_samba_documents: mock.MagicMock,
     _unsync_samba_documents: mock.MagicMock,
+    scandir,
     tc,
     cache_client,
     http_client,

nesis/frontend/client/src/pages/Settings/Datasources/DatasourcesDetailPage.js

@@ -466,6 +466,18 @@ function sambaConnection() {
             />
           </td>
         </tr>
+        <tr>
+          <td>Port</td>
+          <td>
+            <TextField
+              type="text"
+              id="port"
+              placeholder="Port (defaults to 445)"
+              name="connection.port"
+              validate={required}
+            />
+          </td>
+        </tr>
         <tr>
           <td>Username</td>
           <td>

nesis/frontend/client/src/pages/Settings/Datasources/DatasourcesPage.js

@@ -35,7 +35,7 @@ import { ReactComponent as BinIcon } from '../../../images/BinIcon.svg';
 import DatasourcesDetailPage from './DatasourcesDetailPage';
 
 const TypeOptions = {
-  minio: 'S3 Compatible',
+  minio: 'MinIO',
   windows_share: 'Windows Share',
   sharepoint: 'Sharepoint',
   google_drive: 'Google Drive',

nesis/rag/settings.yaml

@@ -17,7 +17,7 @@ llm:
 embedding:
   # Should be matching the value above in most cases
   mode: ${NESIS_RAG_EMBEDDING_MODE:local}
-  ingest_mode: ${NESIS_RAG_EMBEDDING_INGEST_MODE:simple }
+  ingest_mode: ${NESIS_RAG_EMBEDDING_INGEST_MODE:simple}
 
 vectorstore:
   database: pgvector

version.txt

@@ -1 +1 @@
-0.0.2
+0.1.0