[ Feature: document module ] document endpoints

.env

@@ -43,3 +43,8 @@ SENTRY_DSN=
 # Configure these with your own Docker registry images
 DOCKER_IMAGE_BACKEND=backend
 DOCKER_IMAGE_FRONTEND=frontend
+
+# AWS
+AWS_ACCESS_KEY_ID=
+AWS_SECRET_ACCESS_KEY=
+AWS_DEFAULT_REGION=

.gitignore

@@ -14,4 +14,7 @@ ENV/
 
 
 # .DS_Store: macOS Finder metadata file that stores folder view settings and icon positions.
-**/.DS_Store
+**/.DS_Store
+
+# Emacs
+*~

backend/app/api/routes/documents.py

@@ -1,33 +1,96 @@
-from fastapi import APIRouter
-from sqlmodel import select, and_
+import warnings
+from uuid import UUID, uuid4
 
-from app.api.deps import CurrentUser, SessionDep
+from fastapi import APIRouter, File, UploadFile, HTTPException
+
+from sqlalchemy.exc import NoResultFound, MultipleResultsFound, SQLAlchemyError
+
+from app.crud import DocumentCrud
 from app.models import Document, DocumentList
+from app.api.deps import CurrentUser, SessionDep
+from app.core.cloud import AmazonCloudStorage, CloudStorageError
 
 router = APIRouter(prefix="/documents", tags=["documents"])
 
+def raise_from_unknown(error: Exception):
+    warnings.warn('Unexpected exception "{}": {}'.format(
+        type(error).__name__,
+        error,
+    ))
+    raise HTTPException(status_code=500, detail=str(error))
 
-@router.get(
-    "/ls",
-    response_model=DocumentList,
-)
+@router.get("/ls", response_model=DocumentList)
 def list_docs(
         session: SessionDep,
         current_user: CurrentUser,
         skip: int = 0,
         limit: int = 100,
 ):
-    statement = (select(Document)
-                 .where(
-                     and_(
-                         Document.owner_id == current_user.id,
-                         Document.deleted_at.is_(None),
-                     ),
-                 )
-                 .offset(skip)
-                 .limit(limit))
-    docs = (session
-            .exec(statement)
-            .all())
-
-    return DocumentList(docs=docs)
+    crud = DocumentCrud(session, current_user.id)
+    try:
+        return crud.read_many(skip, limit)
+    except (ValueError, SQLAlchemyError) as err:
+        raise HTTPException(status_code=500, detail=str(err))
+    except Exception as err:
+        raise_from_unknown(err)
+
+@router.post("/cp")
+def upload_doc(
+        session: SessionDep,
+        current_user: CurrentUser,
+        src: UploadFile = File(...),
+):
+    storage = AmazonCloudStorage(current_user)
+    basename = uuid4()
+    try:
+        object_store_url = storage.put(src, str(basename))
+    except CloudStorageError as err:
+        raise HTTPException(status_code=503, detail=str(err))
+    except Exception as err:
+        raise_from_unknown(err)
+
+    crud = DocumentCrud(session, current_user.id)
+    document = Document(
+        id=basename,
+        fname=src.filename,
+        object_store_url=str(object_store_url)
+    )
+
+    try:
+        return crud.update(document)
+    except SQLAlchemyError as err:
+        raise HTTPException(status_code=503, detail=str(err))
+    except Exception as err:
+        raise_from_unknown(err)
+
+@router.get("/rm/{doc_id}")
+def delete_doc(
+        session: SessionDep,
+        current_user: CurrentUser,
+        doc_id: UUID,
+):
+    crud = DocumentCrud(session, current_user.id)
+    try:
+        return crud.delete(doc_id)
+    except NoResultFound as err:
+        raise HTTPException(status_code=404, detail=str(err))
+    except Exception as err:
+        raise_from_unknown(err)
+
+    # TODO: perform delete on the collection
+
+@router.get("/stat/{doc_id}", response_model=Document)
+def doc_info(
+        session: SessionDep,
+        current_user: CurrentUser,
+        doc_id: UUID,
+):
+    crud = DocumentCrud(session, current_user.id)
+    try:
+        return crud.read_one(doc_id)
+    except NoResultFound as err:
+        raise HTTPException(status_code=404, detail=str(err))
+    except MultipleResultsFound as err:
+        raise HTTPException(status_code=503, detail=str(err))
+    except Exception as err:
+        raise_from_unknown(err)

backend/app/core/cloud/__init__.py

@@ -0,0 +1,5 @@
+from .storage import (
+    AmazonCloudStorage,
+    AmazonCloudStorageClient,
+    CloudStorageError,
+)

backend/app/core/cloud/storage.py

@@ -0,0 +1,96 @@
+import os
+import functools as ft
+from pathlib import Path
+from dataclasses import dataclass, asdict
+from urllib.parse import ParseResult, urlunparse
+
+import boto3
+from fastapi import UploadFile
+from botocore.exceptions import ClientError
+
+from app.api.deps import CurrentUser
+from app.core.config import settings
+
+class CloudStorageError(Exception):
+    pass
+
+class AmazonCloudStorageClient:
+    @ft.cached_property
+    def client(self):
+        kwargs = {}
+        cred_params = (
+            ('aws_access_key_id', 'AWS_ACCESS_KEY_ID'),
+            ('aws_secret_access_key', 'AWS_SECRET_ACCESS_KEY'),
+            ('region_name', 'AWS_DEFAULT_REGION'),
+        )
+
+        for (i, j) in cred_params:
+            kwargs[i] = os.environ.get(j, getattr(settings, j))
+
+        return boto3.client('s3', **kwargs)
+
+    def create(self):
+        try:
+            # does the bucket exist...
+            self.client.head_bucket(Bucket=settings.AWS_S3_BUCKET)
+        except ClientError as err:
+            response = int(err.response['Error']['Code'])
+            if response != 404:
+                raise CloudStorageError(err) from err
+            # ... if not create it
+            self.client.create_bucket(
+                Bucket=settings.AWS_S3_BUCKET,
+                CreateBucketConfiguration={
+                    'LocationConstraint': settings.AWS_DEFAULT_REGION,
+                },
+            )
+
+@dataclass(frozen=True)
+class SimpleStorageName:
+    Key: str
+    Bucket: str = settings.AWS_S3_BUCKET
+
+    def __str__(self):
+        return urlunparse(self.to_url())
+
+    def to_url(self):
+        kwargs = {
+            'scheme': 's3',
+            'netloc': self.Bucket,
+            'path': self.Key,
+        }
+        for k in ParseResult._fields:
+            kwargs.setdefault(k)
+
+        return ParseResult(**kwargs)
+
+class CloudStorage:
+    def __init__(self, user: CurrentUser):
+        self.user = user
+
+    def put(self, source: UploadFile, basename: str):
+        raise NotImplementedError()
+
+class AmazonCloudStorage(CloudStorage):
+    def __init__(self, user: CurrentUser):
+        super().__init__(user)
+        self.aws = AmazonCloudStorageClient()
+
+    def put(self, source: UploadFile, basename: str):
+        key = Path(str(self.user.id), basename)
+        destination = SimpleStorageName(str(key))
+
+        kwargs = asdict(destination)
+        try:
+            self.aws.client.upload_fileobj(
+                source.file,
+                ExtraArgs={
+                    # 'Metadata': self.user.model_dump(),
+                    'ContentType': source.content_type,
+                },
+                **kwargs,
+            )
+        except ClientError as err:
+            raise CloudStorageError(f'AWS Error: "{err}"') from err
+
+        return destination

backend/app/core/config.py

@@ -95,6 +95,15 @@ def emails_enabled(self) -> bool:
     FIRST_SUPERUSER: EmailStr
     FIRST_SUPERUSER_PASSWORD: str
 
+    AWS_ACCESS_KEY_ID: str
+    AWS_SECRET_ACCESS_KEY: str
+    AWS_DEFAULT_REGION: str
+
+    @computed_field  # type: ignore[prop-decorator]
+    @property
+    def AWS_S3_BUCKET(self) -> str:
+        return f'ai-platform-documents-{self.ENVIRONMENT}'
+
     def _check_default_secret(self, var_name: str, value: str | None) -> None:
         if value == "changethis":
             message = (

backend/app/core/util.py

@@ -0,0 +1,4 @@
+from datetime import datetime, timezone
+
+def now():
+    return datetime.now(timezone.utc).replace(tzinfo=None)

backend/app/crud/__init__.py

@@ -4,4 +4,6 @@
     create_user,
     get_user_by_email,
     update_user,
-)
+)
+
+from .document import DocumentCrud

backend/app/crud/document.py

@@ -0,0 +1,69 @@
+from uuid import UUID
+from typing import Optional
+
+from sqlmodel import Session, select, and_
+
+from app.models import Document, DocumentList
+from app.core.util import now
+
+class DocumentCrud:
+    def __init__(self, session: Session, owner_id: UUID):
+        self.session = session
+        self.owner_id = owner_id
+
+    def read_one(self, doc_id: UUID):
+        statement = (
+            select(Document)
+            .where(and_(
+                Document.owner_id == self.owner_id,
+                Document.id == doc_id,
+            ))
+        )
+
+        return self.session.exec(statement).one()
+
+    def read_many(
+            self,
+            skip: Optional[int] = None,
+            limit: Optional[int] = None,
+    ):
+        statement = (
+            select(Document)
+            .where(and_(
+                Document.owner_id == self.owner_id,
+                Document.deleted_at.is_(None),
+            ))
+        )
+        if skip is not None:
+            if skip < 0:
+                raise ValueError(f'Negative skip: {skip}')
+            statement = statement.offset(skip)
+        if limit is not None:
+            if limit < 0:
+                raise ValueError(f'Negative limit: {limit}')
+            statement = statement.limit(limit)
+        docs = self.session.exec(statement).all()
+
+        return DocumentList(docs=docs)
+
+    def update(self, document: Document):
+        if not document.owner_id:
+            document.owner_id = self.owner_id
+        elif document.owner_id != self.owner_id:
+            error = 'Invalid document ownership: owner={} attempter={}'.format(
+                self.owner_id,
+                document.owner_id,
+            )
+            raise PermissionError(error)
+
+        self.session.add(document)
+        self.session.commit()
+        self.session.refresh(document)
+
+        return document
+
+    def delete(self, doc_id: UUID):
+        document = self.read_one(doc_id)
+        document.deleted_at = now()
+
+        return self.update(document)

backend/app/initial_storage.py

@@ -0,0 +1,25 @@
+import logging
+
+from botocore.exceptions import ClientError
+
+from app.core.cloud import AmazonCloudStorageClient, CloudStorageError
+from app.core.config import settings
+
+logging.basicConfig(level=logging.INFO)
+logger = logging.getLogger(__name__)
+
+def init() -> None:
+    aws = AmazonCloudStorageClient()
+    try:
+        aws.create()
+    except CloudStorageError as err:
+        logging.error(err)
+
+def main() -> None:
+    logger.info("START: setup cloud storage")
+    init()
+    logger.info("END: setup cloud storage")
+
+
+if __name__ == "__main__":
+    main()