Merge branch 'main' into eslint

.github/workflows/build-frontend.yaml

@@ -4,7 +4,11 @@ on:
   push:
     branches:
       - main
+    paths:
+      - website/**
   pull_request:
+    paths:
+      - website/**
   workflow_call:
 
 jobs:

.pre-commit-config.yaml

@@ -14,7 +14,7 @@ repos:
         # and which break the standard YAML check.  The alternative would be to
         # skip any unsafe errors (and thus break YAML compatibility) or use
         # some other checker that may not work in general.
-        exclude: copilot/web/addons/*
+        exclude: "^copilot/web/addons/.*$"
       - id: check-json
       - id: check-case-conflict
       - id: detect-private-key

README.md

@@ -107,9 +107,7 @@ To start the demo, run this, in root directory:
 docker compose up --build
 ```
 
-Then, navigate to `http://localhost:3000` and interact with the website. When
-logging in, navigate to `http://localhost:1080` to get the magic email login
-link.
+Then, navigate to `http://localhost:3000` and interact with the website.
 
 ### Website
 

ansible/dev.yaml

@@ -51,6 +51,7 @@
         env:
           POSTGRES_HOST: oasst-postgres
           DEBUG_ALLOW_ANY_API_KEY: "true"
+          DEBUG_USE_SEED_DATA: "true"
           MAX_WORKERS: "1"
         ports:
           - 8080:8080

backend/main.py

@@ -1,14 +1,21 @@
 # -*- coding: utf-8 -*-
 from http import HTTPStatus
 from pathlib import Path
+from typing import Optional
 
 import alembic.command
 import alembic.config
 import fastapi
+import pydantic
 from loguru import logger
+from oasst_backend.api.deps import get_dummy_api_client
 from oasst_backend.api.v1.api import api_router
 from oasst_backend.config import settings
+from oasst_backend.database import engine
 from oasst_backend.exceptions import OasstError, OasstErrorCode
+from oasst_backend.prompt_repository import PromptRepository
+from oasst_shared.schemas import protocol as protocol_schema
+from sqlmodel import Session
 from starlette.middleware.cors import CORSMiddleware
 
 app = fastapi.FastAPI(title=settings.PROJECT_NAME, openapi_url=f"{settings.API_V1_STR}/openapi.json")
@@ -56,4 +63,118 @@ def alembic_upgrade():
             logger.exception("Alembic upgrade failed on startup")
 
 
+if settings.DEBUG_USE_SEED_DATA:
+
+    @app.on_event("startup")
+    def seed_data():
+        class DummyPost(pydantic.BaseModel):
+            task_post_id: str
+            user_post_id: str
+            parent_post_id: Optional[str]
+            text: str
+            role: str
+
+        try:
+            logger.info("Seed data check began")
+            with Session(engine) as db:
+                api_client = get_dummy_api_client(db)
+                dummy_user = protocol_schema.User(id="__dummy_user__", display_name="Dummy User", auth_method="local")
+                pr = PromptRepository(db=db, api_client=api_client, user=dummy_user)
+
+                dummy_posts = [
+                    DummyPost(
+                        task_post_id="de111fa8",
+                        user_post_id="6f1d0711",
+                        parent_post_id=None,
+                        text="Hi!",
+                        role="user",
+                    ),
+                    DummyPost(
+                        task_post_id="74c381d4",
+                        user_post_id="4a24530b",
+                        parent_post_id="6f1d0711",
+                        text="Hello! How can I help you?",
+                        role="assistant",
+                    ),
+                    DummyPost(
+                        task_post_id="3d5dc440",
+                        user_post_id="a8c01c04",
+                        parent_post_id="4a24530b",
+                        text="Do you have a recipe for potato soup?",
+                        role="user",
+                    ),
+                    DummyPost(
+                        task_post_id="643716c1",
+                        user_post_id="f43a93b7",
+                        parent_post_id="4a24530b",
+                        text="Who were the 8 presidents before George Washington?",
+                        role="user",
+                    ),
+                    DummyPost(
+                        task_post_id="2e4e1e6",
+                        user_post_id="c886920",
+                        parent_post_id="6f1d0711",
+                        text="Hey buddy! How can I serve you?",
+                        role="assistant",
+                    ),
+                    DummyPost(
+                        task_post_id="970c437d",
+                        user_post_id="cec432cf",
+                        parent_post_id=None,
+                        text="euirdteunvglfe23908230892309832098 AAAAAAAA",
+                        role="user",
+                    ),
+                    DummyPost(
+                        task_post_id="6066118e",
+                        user_post_id="4f85f637",
+                        parent_post_id="cec432cf",
+                        text="Sorry, I did not understand your request and it is unclear to me what you want me to do. Could you describe it in a different way?",
+                        role="assistant",
+                    ),
+                    DummyPost(
+                        task_post_id="ba87780d",
+                        user_post_id="0e276b98",
+                        parent_post_id="cec432cf",
+                        text="I'm unsure how to interpret this. Is it a riddle?",
+                        role="assistant",
+                    ),
+                ]
+
+                for p in dummy_posts:
+                    wp = pr.fetch_workpackage_by_postid(p.task_post_id)
+                    if wp and not wp.ack:
+                        logger.warning("Deleting unacknowledged seed data work package")
+                        db.delete(wp)
+                        wp = None
+                    if not wp:
+                        if p.parent_post_id is None:
+                            wp = pr.store_task(
+                                protocol_schema.InitialPromptTask(hint=""), thread_id=None, parent_post_id=None
+                            )
+                        else:
+                            print("p.parent_post_id", p.parent_post_id)
+                            parent_post = pr.fetch_post_by_frontend_post_id(p.parent_post_id, fail_if_missing=True)
+                            wp = pr.store_task(
+                                protocol_schema.AssistantReplyTask(
+                                    conversation=protocol_schema.Conversation(
+                                        messages=[protocol_schema.ConversationMessage(text="dummy", is_assistant=False)]
+                                    )
+                                ),
+                                thread_id=parent_post.thread_id,
+                                parent_post_id=parent_post.id,
+                            )
+                        pr.bind_frontend_post_id(wp.id, p.task_post_id)
+                        post = pr.store_text_reply(p.text, p.task_post_id, p.user_post_id)
+
+                        logger.info(
+                            f"Inserted: post_id: {post.id}, payload: {post.payload.payload}, parent_post_id: {post.parent_id}"
+                        )
+                    else:
+                        logger.debug(f"seed data work_package found: {wp.id}")
+                logger.info("Seed data check completed")
+
+        except Exception:
+            logger.exception("Seed data insertion failed")
+
+
 app.include_router(api_router, prefix=settings.API_V1_STR)

backend/oasst_backend/api/deps.py

@@ -33,23 +33,27 @@ async def get_api_key(
         return api_key_header
 
 
+def get_dummy_api_client(db: Session) -> ApiClient:
+    # make sure that a dummy api key exits in db (foreign key references)
+    ANY_API_KEY_ID = UUID("00000000-1111-2222-3333-444444444444")
+    api_client: ApiClient = db.query(ApiClient).filter(ApiClient.id == ANY_API_KEY_ID).first()
+    if api_client is None:
+        token = token_hex(32)
+        logger.info(f"ANY_API_KEY missing, inserting api_key: {token}")
+        api_client = ApiClient(id=ANY_API_KEY_ID, api_key=token, description="ANY_API_KEY, random token")
+        db.add(api_client)
+        db.commit()
+    return api_client
+
+
 def api_auth(
     api_key: APIKey,
     db: Session,
 ) -> ApiClient:
     if api_key or settings.DEBUG_SKIP_API_KEY_CHECK:
 
         if settings.DEBUG_SKIP_API_KEY_CHECK or settings.DEBUG_ALLOW_ANY_API_KEY:
-            # make sure that a dummy api key exits in db (foreign key references)
-            ANY_API_KEY_ID = UUID("00000000-1111-2222-3333-444444444444")
-            api_client: ApiClient = db.query(ApiClient).filter(ApiClient.id == ANY_API_KEY_ID).first()
-            if api_client is None:
-                token = token_hex(32)
-                logger.info(f"ANY_API_KEY missing, inserting api_key: {token}")
-                api_client = ApiClient(id=ANY_API_KEY_ID, api_key=token, description="ANY_API_KEY, random token")
-                db.add(api_client)
-                db.commit()
-            return api_client
+            return get_dummy_api_client(db)
 
         api_client = db.query(ApiClient).filter(ApiClient.api_key == api_key).first()
         if api_client is not None and api_client.enabled:

backend/oasst_backend/config.py

@@ -17,6 +17,7 @@ class Settings(BaseSettings):
 
     DEBUG_ALLOW_ANY_API_KEY: bool = False
     DEBUG_SKIP_API_KEY_CHECK: bool = False
+    DEBUG_USE_SEED_DATA: bool = False
 
     @validator("DATABASE_URI", pre=True)
     def assemble_db_connection(cls, v: Optional[str], values: Dict[str, Any]) -> Any:

backend/oasst_backend/exceptions.py

@@ -34,6 +34,7 @@ class OasstErrorCode(IntEnum):
     INVALID_TASK_TYPE = 2004
     USER_NOT_SPECIFIED = 2005
     NO_THREADS_FOUND = 2006
+    NO_REPLIES_FOUND = 2007
     WORK_PACKAGE_NOT_FOUND = 2100
     WORK_PACKAGE_EXPIRED = 2101
     WORK_PACKAGE_PAYLOAD_TYPE_MISMATCH = 2102

backend/oasst_backend/prompt_repository.py

@@ -397,7 +397,7 @@ def fetch_random_thread(self, require_role: str = None) -> list[Post]:
             distinct_threads = distinct_threads.filter(Post.role == require_role)
         distinct_threads = distinct_threads.subquery()
 
-        random_thread = self.db.query(distinct_threads).order_by(func.random()).limit(1).subquery()
+        random_thread = self.db.query(distinct_threads).order_by(func.random()).limit(1)
         thread_posts = self.db.query(Post).filter(Post.thread_id.in_(random_thread)).all()
         return thread_posts
 
@@ -443,8 +443,10 @@ def fetch_multiple_random_replies(self, max_size: int = 5, post_role: str = None
         if post_role:
             parent = parent.filter(Post.role == post_role)
 
-        parent = parent.order_by(func.random()).limit(1).subquery()
+        parent = parent.order_by(func.random()).limit(1)
         replies = self.db.query(Post).filter(Post.parent_id.in_(parent)).order_by(func.random()).limit(max_size).all()
+        if not replies:
+            raise OasstError("No replies found", OasstErrorCode.NO_REPLIES_FOUND)
 
         thread = self.fetch_thread(replies[0].thread_id)
         thread = {p.id: p for p in thread}

docker-compose.yaml

@@ -71,6 +71,7 @@ services:
     environment:
       - POSTGRES_HOST=db
       - DEBUG_SKIP_API_KEY_CHECK=True
+      - DEBUG_USE_SEED_DATA=True
       - MAX_WORKERS=1
     depends_on:
       db:
@@ -92,6 +93,7 @@ services:
       - EMAIL_SERVER_PORT=1025
       - EMAIL_FROM=info@example.com
       - NEXTAUTH_URL=http://localhost:3000
+      - DEBUG_LOGIN=true
     depends_on:
       webdb:
         condition: service_healthy

scripts/backend-development/run-local.sh

@@ -5,6 +5,7 @@ parent_path=$( cd "$(dirname "${BASH_SOURCE[0]}")" ; pwd -P )
 pushd "$parent_path/../../backend"
 
 export DEBUG_SKIP_API_KEY_CHECK=True
+export DEBUG_USE_SEED_DATA=True
 
 uvicorn main:app --reload --port 8080 --host 0.0.0.0
 

website/README.md

@@ -64,9 +64,9 @@ If you're doing active development we suggest the following workflow:
 
 ### Using debug user credentials
 
-Whenever the website runs in development mode, you can use the debug credentials provider to log in without fancy emails or OAuth.
+You can use the debug credentials provider to log in without fancy emails or OAuth.
 
-1. Development mode is automatically active when you start the website with `npm run dev`.
+1. This feature is automatically on in development mode, i.e. when you run `npm run dev`. In case you want to do the same with a production build (for example, the docker image), then run the website with environment variable `DEBUG_LOGIN=true`.
 1. Use the `Login` button in the top right to go to the login page.
 1. You should see a section for debug credentials. Enter any username you wish, you will be logged in as that user.