Merge pull request #3170 from centerofci/db_config_api

API for database config

config/settings/common_settings.py

@@ -250,3 +250,5 @@ def pipe_delim(pipe_string):
 }
 # List of Template names that contains additional script tags to be added to the base template
 BASE_TEMPLATE_ADDITIONAL_SCRIPT_TEMPLATES = []
+
+SALT_KEY = SECRET_KEY

db/types/install.py

@@ -1,8 +1,8 @@
 from db.types.custom import email, money, multicurrency, uri, json_array, json_object
 from db.types.base import SCHEMA
 from db.schemas.operations.create import create_schema
-from db.schemas.operations.drop import drop_schema
 from db.types.operations.cast import install_all_casts
+import psycopg
 
 
 def create_type_schema(engine):
@@ -22,8 +22,6 @@ def install_mathesar_on_database(engine):
 
 
 def uninstall_mathesar_from_database(engine):
-    _cascade_type_schema(engine)
-
-
-def _cascade_type_schema(engine):
-    drop_schema(SCHEMA, engine, cascade=True)
+    conn_str = str(engine.url)
+    with psycopg.connect(conn_str) as conn:
+        conn.execute(f"DROP SCHEMA IF EXISTS __msar, msar, {SCHEMA} CASCADE")

demo/management/commands/load_arxiv_data.py

@@ -8,6 +8,7 @@
 
 from demo.install.arxiv_skeleton import get_arxiv_db_and_schema_log_path
 from mathesar.database.base import create_mathesar_engine
+from mathesar.models.base import Database
 
 
 class Command(BaseCommand):
@@ -28,7 +29,8 @@ def update_our_arxiv_dbs():
         logging.error(e, exc_info=True)
         return
     for db_name, schema_name in db_schema_pairs:
-        engine = create_mathesar_engine(db_name)
+        db = Database.current_objects.get(name=db_name)
+        engine = create_mathesar_engine(db)
         update_arxiv_schema(engine, schema_name, papers)
         engine.dispose()
 

demo/management/commands/setup_demo_template_db.py

@@ -6,6 +6,7 @@
 from db.install import install_mathesar
 from demo.install.datasets import load_datasets
 from mathesar.database.base import create_mathesar_engine
+from mathesar.models.base import Database
 
 
 class Command(BaseCommand):
@@ -19,19 +20,30 @@ def _setup_demo_template_db():
     print("Initializing demo template database...")
 
     template_db_name = settings.MATHESAR_DEMO_TEMPLATE
-    root_engine = create_mathesar_engine(settings.DATABASES["default"]["NAME"])
+    django_model = Database.current_objects.get(name=settings.DATABASES["default"]["NAME"])
+    root_engine = create_mathesar_engine(django_model)
     with root_engine.connect() as conn:
         conn.execution_options(isolation_level="AUTOCOMMIT")
         conn.execute(text(f"DROP DATABASE IF EXISTS {template_db_name} WITH (FORCE)"))
     root_engine.dispose()
+    db_model, _ = Database.current_objects.get_or_create(
+        name=template_db_name,
+        defaults={
+            'db_name': template_db_name,
+            'username': django_model.username,
+            'password': django_model.password,
+            'host': django_model.host,
+            'port': django_model.port
+        }
+    )
     install_mathesar(
         database_name=template_db_name,
-        username=settings.DATABASES["default"]["USER"],
-        password=settings.DATABASES["default"]["PASSWORD"],
-        hostname=settings.DATABASES["default"]["HOST"],
-        port=settings.DATABASES["default"]["PORT"],
+        hostname=db_model.host,
+        username=db_model.username,
+        password=db_model.password,
+        port=db_model.port,
         skip_confirm=True
     )
-    user_engine = create_mathesar_engine(template_db_name)
+    user_engine = create_mathesar_engine(db_model)
     load_datasets(user_engine)
     user_engine.dispose()

demo/middleware.py

@@ -24,20 +24,29 @@ def __init__(self, get_response):
     def __call__(self, request):
         sessionid = request.COOKIES.get('sessionid', None)
         db_name = get_name(str(sessionid))
-        database, created = Database.current_objects.get_or_create(name=db_name)
+        database, created = Database.current_objects.get_or_create(
+            name=db_name,
+            defaults={
+                'db_name': db_name,
+                'username': settings.DATABASES['default']['USER'],
+                'password': settings.DATABASES['default']['PASSWORD'],
+                'host': settings.DATABASES['default']['HOST'],
+                'port': settings.DATABASES['default']['PORT']
+            }
+        )
         if created:
             create_demo_database(
                 db_name,
-                settings.DATABASES["default"]["USER"],
-                settings.DATABASES["default"]["PASSWORD"],
-                settings.DATABASES["default"]["HOST"],
-                settings.DATABASES["default"]["NAME"],
-                settings.DATABASES["default"]["PORT"],
+                database.username,
+                database.password,
+                database.host,
+                settings.DATABASES['default']['NAME'],
+                database.port,
                 settings.MATHESAR_DEMO_TEMPLATE
             )
             append_db_and_arxiv_schema_to_log(db_name, ARXIV)
             reset_reflection(db_name=db_name)
-            engine = create_mathesar_engine(db_name)
+            engine = create_mathesar_engine(database)
             customize_settings(engine)
             load_custom_explorations(engine)
             engine.dispose()

mathesar/api/db/permissions/database.py

@@ -14,6 +14,12 @@ class DatabaseAccessPolicy(AccessPolicy):
             'action': ['list', 'retrieve', 'types', 'functions'],
             'principal': 'authenticated',
             'effect': 'allow',
+        },
+        {
+            'action': ['create', 'partial_update', 'destroy'],
+            'principal': 'authenticated',
+            'effect': 'allow',
+            'condition': 'is_superuser'
         }
     ]
 

mathesar/api/db/viewsets/databases.py

@@ -1,8 +1,7 @@
 from django_filters import rest_framework as filters
 from rest_access_policy import AccessViewSetMixin
-from rest_framework import viewsets
+from rest_framework import viewsets, status
 from rest_framework.decorators import action
-from rest_framework.mixins import ListModelMixin, RetrieveModelMixin
 from rest_framework.response import Response
 
 from mathesar.api.db.permissions.database import DatabaseAccessPolicy
@@ -14,12 +13,13 @@
 
 from db.functions.operations.check_support import get_supported_db_functions
 from mathesar.api.serializers.functions import DBFunctionSerializer
-
 from db.types.base import get_available_known_db_types
+from db.types.install import uninstall_mathesar_from_database
 from mathesar.api.serializers.db_types import DBTypeSerializer
+from mathesar.api.exceptions.validation_exceptions.exceptions import EditingDBCredentialsNotAllowed
 
 
-class DatabaseViewSet(AccessViewSetMixin, viewsets.GenericViewSet, ListModelMixin, RetrieveModelMixin):
+class DatabaseViewSet(AccessViewSetMixin, viewsets.ModelViewSet):
     serializer_class = DatabaseSerializer
     pagination_class = DefaultLimitOffsetPagination
     filter_backends = (filters.DjangoFilterBackend,)
@@ -32,6 +32,37 @@ def get_queryset(self):
             Database.objects.all().order_by('-created_at')
         )
 
+    def create(self, request):
+        serializer = DatabaseSerializer(data=request.data)
+        serializer.is_valid(raise_exception=True)
+        credentials = serializer.validated_data
+        Database.objects.create(
+            name=credentials['name'],
+            db_name=credentials['db_name'],
+            username=credentials['username'],
+            password=credentials['password'],
+            host=credentials['host'],
+            port=credentials['port']
+        ).save()
+        return Response(serializer.data, status=status.HTTP_201_CREATED)
+
+    def partial_update(self, request, pk=None):
+        db_object = self.get_object()
+        if db_object.editable:
+            serializer = DatabaseSerializer(db_object, data=request.data, partial=True)
+            serializer.is_valid(raise_exception=True)
+            serializer.save()
+            return Response(serializer.data)
+        raise EditingDBCredentialsNotAllowed()
+
+    def destroy(self, request, pk=None):
+        db_object = self.get_object()
+        if request.query_params.get('del_msar_schemas'):
+            engine = db_object._sa_engine
+            uninstall_mathesar_from_database(engine)
+        db_object.delete()
+        return Response(status=status.HTTP_204_NO_CONTENT)
+
     @action(methods=['get'], detail=True)
     def functions(self, request, pk=None):
         database = self.get_object()

mathesar/api/exceptions/error_codes.py

@@ -39,8 +39,10 @@ class ErrorCodes(Enum):
     UniqueImportViolation = 4303
 
     # Validation Error
+    BadDBCredentials = 4428
     ColumnSizeMismatch = 4401
     DistinctColumnNameRequired = 4402
+    EditingNotAllowed = 4429
     MappingsNotFound = 4417
     MultipleDataFiles = 4400
     MoneyDisplayOptionConflict = 4407

mathesar/api/exceptions/generic_exceptions/base_exceptions.py

@@ -116,3 +116,17 @@ def __init__(
             status_code=status.HTTP_503_SERVICE_UNAVAILABLE
     ):
         super().__init__(exception, error_code, message, field, details, status_code)
+
+
+class BadDBCredentials(MathesarAPIException):
+    error_code = ErrorCodes.BadDBCredentials.value
+
+    def __init__(
+            self,
+            exception,
+            field=None,
+            detail=None,
+            status_code=status.HTTP_500_INTERNAL_SERVER_ERROR
+    ):
+        message = f"Bad credentials for connecting to the requested database. The reported error is {exception.args[0]}"
+        super().__init__(exception, self.error_code, message, field, detail, status_code)

mathesar/api/exceptions/validation_exceptions/exceptions.py

@@ -181,3 +181,14 @@ def __init__(
             field=None,
     ):
         super().__init__(None, self.error_code, message, field)
+
+
+class EditingDBCredentialsNotAllowed(MathesarValidationException):
+    error_code = ErrorCodes.EditingNotAllowed.value
+
+    def __init__(
+            self,
+            message="Cannot edit the DB credentials created from .env",
+            field=None
+    ):
+        super().__init__(None, self.error_code, message, field)

mathesar/api/serializers/databases.py

@@ -4,24 +4,52 @@
 from mathesar.api.display_options import DISPLAY_OPTIONS_BY_UI_TYPE
 from mathesar.api.exceptions.mixins import MathesarErrorMessageMixin
 from mathesar.models.base import Database
+from mathesar.api.utils import is_valid_pg_creds
+from db.install import install_mathesar
 
 
 class DatabaseSerializer(MathesarErrorMessageMixin, serializers.ModelSerializer):
     supported_types_url = serializers.SerializerMethodField()
 
     class Meta:
         model = Database
-        fields = ['id', 'name', 'deleted', 'supported_types_url']
-        read_only_fields = ['id', 'name', 'deleted', 'supported_types_url']
+        fields = ['id', 'name', 'db_name', 'deleted', 'editable', 'supported_types_url', 'username', 'password', 'host', 'port']
+        read_only_fields = ['id', 'deleted', 'supported_types_url', 'editable']
+        extra_kwargs = {
+            'password': {'write_only': True}
+        }
 
     def get_supported_types_url(self, obj):
-        if isinstance(obj, Database):
+        if isinstance(obj, Database) and not self.partial:
             # Only get records if we are serializing an existing table
             request = self.context['request']
             return request.build_absolute_uri(reverse('database-types', kwargs={'pk': obj.pk}))
         else:
             return None
 
+    def validate(self, credentials):
+        if self.partial:
+            db_model = self.instance
+            for attr, value in credentials.items():
+                setattr(db_model, attr, value)
+            credentials = {
+                'db_name': db_model.db_name,
+                'host': db_model.host,
+                'username': db_model.username,
+                'password': db_model.password,
+                'port': db_model.port
+            }
+        if is_valid_pg_creds(credentials):
+            install_mathesar(
+                database_name=credentials["db_name"],
+                hostname=credentials["host"],
+                username=credentials["username"],
+                password=credentials["password"],
+                port=credentials["port"],
+                skip_confirm=True
+            )
+        return super().validate(credentials)
+
 
 class TypeSerializer(MathesarErrorMessageMixin, serializers.Serializer):
     identifier = serializers.CharField()

mathesar/api/utils.py

@@ -1,5 +1,6 @@
 from uuid import UUID
 from rest_framework.exceptions import NotFound
+from rest_framework import status
 import mathesar.api.exceptions.generic_exceptions.base_exceptions as generic_api_exceptions
 import re
 
@@ -8,6 +9,8 @@
 from mathesar.models.base import Table
 from mathesar.models.query import UIQuery
 from mathesar.utils.preview import column_alias_from_preview_template
+from mathesar.api.exceptions.generic_exceptions.base_exceptions import BadDBCredentials
+import psycopg
 
 DATA_KEY = 'data'
 METADATA_KEY = 'metadata'
@@ -158,3 +161,20 @@ def is_valid_uuid_v4(value):
         return True
     except ValueError:
         return False
+
+
+def is_valid_pg_creds(credentials):
+    dbname = credentials["db_name"]
+    user = credentials["username"]
+    password = credentials["password"]
+    host = credentials["host"]
+    port = credentials["port"]
+    conn_str = f'dbname={dbname} user={user} password={password} host={host} port={port}'
+    try:
+        with psycopg.connect(conn_str):
+            return True
+    except psycopg.errors.OperationalError as e:
+        raise BadDBCredentials(
+            exception=e,
+            status_code=status.HTTP_400_BAD_REQUEST
+        )

mathesar/apps.py

@@ -4,13 +4,7 @@
 
 
 def _prepare_database_model(**kwargs):
-    from mathesar.models.base import Database  # noqa
     from mathesar.state import make_sure_initial_reflection_happened  # noqa
-    dbs_in_settings = set(settings.DATABASES)
-    # We only want to track non-django dbs
-    dbs_in_settings.remove('default')
-    for db_name in dbs_in_settings:
-        Database.current_objects.get_or_create(name=db_name)
     # TODO fix test DB loading to make this unnecessary
     if not settings.TEST:
         make_sure_initial_reflection_happened()