From ac35e2286d0c3865aa2bb31f20b4baae031e6071 Mon Sep 17 00:00:00 2001 From: Victor Porton Date: Thu, 12 Oct 2023 23:51:04 +0300 Subject: [PATCH 1/8] DB schema --- script.sql | 23 +++++++++++++++-------- server/src/api/apps.py | 4 ++-- server/src/core/agent/agent_service.py | 6 +++--- server/src/core/history/chat_history.py | 14 +++++++------- server/src/core/history/history_dao.py | 12 ++++++------ 5 files changed, 33 insertions(+), 26 deletions(-) diff --git a/script.sql b/script.sql index 27185d3..bcb35b3 100644 --- a/script.sql +++ b/script.sql @@ -1,14 +1,22 @@ - +create table users +( + id serial primary key, + email null varchar(250) + password null varchar(50), + name varchar(50) null +); create table chat_messages ( - id serial - primary key, - user_id varchar(250), + id serial primary key, + user_ref not null int, chatbot_id varchar(50), message text, is_bot_reply boolean, - createdat timestamp default (now() AT TIME ZONE 'UTC'::text) + createdat timestamp default (now() AT TIME ZONE 'UTC'::text), + CONSTRAINT fk_user + FOREIGN KEY(user_ref) + REFERENCES users(id) ); alter table chat_messages @@ -20,6 +28,5 @@ create index idx_timestamp create index idx_chatbot_id on chat_messages (chatbot_id); -create index idx_user_id - on chat_messages (user_id); - +create index idx_user_email + on users (email); diff --git a/server/src/api/apps.py b/server/src/api/apps.py index e88ac03..e2ab7d5 100644 --- a/server/src/api/apps.py +++ b/server/src/api/apps.py @@ -92,11 +92,11 @@ def delete_app(app_key: str, current_user: User = Depends(get_current_user)): @r.get("/applications/{app_key}/history") def get_app_conversation(app_key: str, user: User = Depends(get_current_user)): if app_key == "chat": - return chat_history.get_latest_messages(user.email, app_key) + return chat_history.get_latest_messages(user.user_ref, app_key) for a in apps.get_by_user_email(user.email): if app_key == a.app_key: - return chat_history.get_latest_messages(user.email, app_key) + return chat_history.get_latest_messages(user.user_ref, app_key) raise HTTPException(status_code=404, detail="App not found") diff --git a/server/src/core/agent/agent_service.py b/server/src/core/agent/agent_service.py index 610a857..667de37 100644 --- a/server/src/core/agent/agent_service.py +++ b/server/src/core/agent/agent_service.py @@ -48,7 +48,7 @@ def is_action(self, req: UserInputDto) -> bool: def handle_user_input(self, req: UserInputDto) -> dict: current_user = req.user - self.history.add_message(req.user.email, req.app.app_key, MessageCompletion( + self.history.add_message(str(req.user.user_ref), req.app.app_key, MessageCompletion( role=MessageRole.USER, context=req.context, query=req.question @@ -63,7 +63,7 @@ def handle_user_input(self, req: UserInputDto) -> dict: self.cache.put(req.question, llm_resp.message) message = llm_resp.message - self.history.add_message(req.user.email, req.app.app_key, MessageCompletion( + self.history.add_message(str(req.user.user_ref), req.app.app_key, MessageCompletion( role=MessageRole.ASSISTANT, response=message )) @@ -85,7 +85,7 @@ def create_user_resp_obj(self, message: str): } def _get_llm_response(self, req: UserInputDto, is_action) -> LLMResponse: - history_key = req.user.email + "_" + req.app.app_key + history_key = str(req.user.user_ref) + "_" + req.app.app_key user_history = self.history.get_history(history_key) if user_history is None: user_history = [] diff --git a/server/src/core/history/chat_history.py b/server/src/core/history/chat_history.py index f873c8b..9be2c0b 100644 --- a/server/src/core/history/chat_history.py +++ b/server/src/core/history/chat_history.py @@ -13,8 +13,8 @@ def __init__(self, dao: HistoryDao): self.dao = dao self.history = CacheMemory(30) - def add_message(self, user_email, app_key, message: MessageCompletion): - key = user_email + "_" + app_key + def add_message(self, user_ref, app_key, message: MessageCompletion): + key = str(user_ref) + "_" + app_key if self.history.get(key) is None: self.history.put(key, [message]) else: @@ -22,18 +22,18 @@ def add_message(self, user_email, app_key, message: MessageCompletion): history.append(message) self.history.put(key, history) - self.persist_message(user_email, app_key, message) + self.persist_message(user_ref, app_key, message) def get_history(self, key): return self.history.get(key) - def persist_message(self, user_email, app_key, message): + def persist_message(self, user_ref, app_key, message): is_bot_replay = message.role == MessageRole.ASSISTANT msg = message.response if is_bot_replay else message.query - self.dao.persist_message(user_email, app_key, msg, is_bot_replay) + self.dao.persist_message(user_ref, app_key, msg, is_bot_replay) - def get_latest_messages(self, user_email: str, app_key: str) -> List[Dict]: - return self.dao.get_latest_messages(user_email, app_key) + def get_latest_messages(self, user_ref: str, app_key: str) -> List[Dict]: + return self.dao.get_latest_messages(user_ref, app_key) def factory_chat_history(pg_conn: DBConnection): diff --git a/server/src/core/history/history_dao.py b/server/src/core/history/history_dao.py index b526987..8fcd6ab 100644 --- a/server/src/core/history/history_dao.py +++ b/server/src/core/history/history_dao.py @@ -9,21 +9,21 @@ class HistoryDao: def __init__(self, db: DBConnection): self.db = db - def persist_message(self, user_email, app_key, msg, is_bot_replay): + def persist_message(self, user_ref, app_key, msg, is_bot_replay): insert_query = sql.SQL( """ - INSERT INTO chat_messages(user_id, chatbot_id, message, is_bot_reply) + INSERT INTO chat_messages(user_ref, chatbot_id, message, is_bot_reply) VALUES(%s, %s, %s, %s) """ ) - self.db.execute(insert_query, (user_email, app_key, msg, is_bot_replay)) + self.db.execute(insert_query, (user_ref, app_key, msg, is_bot_replay)) - def get_latest_messages(self, user_email: str, app_key: str) -> List[Dict]: + def get_latest_messages(self, user_ref: str, app_key: str) -> List[Dict]: return self.db.fetch_all( """ - SELECT * FROM chat_messages where chatbot_id = %s and user_id = %s + SELECT * FROM chat_messages where chatbot_id = %s and user_ref = %s ORDER BY createdat DESC LIMIT 50 """, - (app_key, user_email), + (app_key, user_ref), ) From 50a69fd6ff570db8fd76508f386e2e574be06836 Mon Sep 17 00:00:00 2001 From: Victor Porton Date: Thu, 12 Oct 2023 23:59:04 +0300 Subject: [PATCH 2/8] bug fix --- server/src/api/apps.py | 4 ++-- server/src/core/agent/agent_service.py | 6 +++--- 2 files changed, 5 insertions(+), 5 deletions(-) diff --git a/server/src/api/apps.py b/server/src/api/apps.py index e2ab7d5..85c3deb 100644 --- a/server/src/api/apps.py +++ b/server/src/api/apps.py @@ -92,11 +92,11 @@ def delete_app(app_key: str, current_user: User = Depends(get_current_user)): @r.get("/applications/{app_key}/history") def get_app_conversation(app_key: str, user: User = Depends(get_current_user)): if app_key == "chat": - return chat_history.get_latest_messages(user.user_ref, app_key) + return chat_history.get_latest_messages(user.pk, app_key) for a in apps.get_by_user_email(user.email): if app_key == a.app_key: - return chat_history.get_latest_messages(user.user_ref, app_key) + return chat_history.get_latest_messages(user.pk, app_key) raise HTTPException(status_code=404, detail="App not found") diff --git a/server/src/core/agent/agent_service.py b/server/src/core/agent/agent_service.py index 667de37..4e759a3 100644 --- a/server/src/core/agent/agent_service.py +++ b/server/src/core/agent/agent_service.py @@ -48,7 +48,7 @@ def is_action(self, req: UserInputDto) -> bool: def handle_user_input(self, req: UserInputDto) -> dict: current_user = req.user - self.history.add_message(str(req.user.user_ref), req.app.app_key, MessageCompletion( + self.history.add_message(str(req.user.pk), req.app.app_key, MessageCompletion( role=MessageRole.USER, context=req.context, query=req.question @@ -63,7 +63,7 @@ def handle_user_input(self, req: UserInputDto) -> dict: self.cache.put(req.question, llm_resp.message) message = llm_resp.message - self.history.add_message(str(req.user.user_ref), req.app.app_key, MessageCompletion( + self.history.add_message(str(req.user.pk), req.app.app_key, MessageCompletion( role=MessageRole.ASSISTANT, response=message )) @@ -85,7 +85,7 @@ def create_user_resp_obj(self, message: str): } def _get_llm_response(self, req: UserInputDto, is_action) -> LLMResponse: - history_key = str(req.user.user_ref) + "_" + req.app.app_key + history_key = str(req.user.pk) + "_" + req.app.app_key user_history = self.history.get_history(history_key) if user_history is None: user_history = [] From bed9c0ae014be379ca4e95a1178ec53113232e81 Mon Sep 17 00:00:00 2001 From: Victor Porton Date: Fri, 13 Oct 2023 02:36:17 +0300 Subject: [PATCH 3/8] remaining conflict resolutions --- server/src/core/agent/agent_service.py | 26 ++------------------------ 1 file changed, 2 insertions(+), 24 deletions(-) diff --git a/server/src/core/agent/agent_service.py b/server/src/core/agent/agent_service.py index b6d6b79..d861af3 100644 --- a/server/src/core/agent/agent_service.py +++ b/server/src/core/agent/agent_service.py @@ -52,15 +52,8 @@ def is_action(self, req: UserInputDto) -> bool: async def handle_user_input(self, req: UserInputDto) -> dict: current_user = req.user -<<<<<<< HEAD - self.history.add_message(str(req.user.pk), req.app.app_key, MessageCompletion( - role=MessageRole.USER, - context=req.context, - query=req.question - )) -======= add_message_dto = AddMessageDto( - user_email=current_user.email, + user_ref=int(req.user.pk), app_key=req.app.app_key, session_id=req.session_id, message=MessageCompletion( @@ -70,7 +63,6 @@ async def handle_user_input(self, req: UserInputDto) -> dict: ) ) self.history.add_message(add_message_dto) ->>>>>>> main is_action = self.is_action(req) # TODO: Eliminate `cache.exists` call: @@ -83,14 +75,8 @@ async def handle_user_input(self, req: UserInputDto) -> dict: await self.cache.put(req.question, llm_resp.message) message = llm_resp.message -<<<<<<< HEAD - self.history.add_message(str(req.user.pk), req.app.app_key, MessageCompletion( - role=MessageRole.ASSISTANT, - response=message - )) -======= add_message_dto = AddMessageDto( - user_email=current_user.email, + user_ref=req.user.pk, app_key=req.app.app_key, session_id=req.session_id, message=MessageCompletion( @@ -99,7 +85,6 @@ async def handle_user_input(self, req: UserInputDto) -> dict: ) ) self.history.add_message(add_message_dto) ->>>>>>> main if is_action: commands = self.resp_handler.extract_json_schema(message) @@ -133,14 +118,7 @@ def create_user_resp_obj(self, message: str): } def _get_llm_response(self, req: UserInputDto, is_action) -> LLMResponse: -<<<<<<< HEAD - history_key = str(req.user.pk) + "_" + req.app.app_key - user_history = self.history.get_history(history_key) - if user_history is None: - user_history = [] -======= user_history = self._get_user_history(req) ->>>>>>> main if is_action: return self.llm.get_task_command(user_history, app=req.app) return self.llm.get_question_answer(req.question, req.app, user_history) From 1a4edba3160c706151489c5bd44aaf4933305149 Mon Sep 17 00:00:00 2001 From: Victor Porton Date: Fri, 13 Oct 2023 02:38:19 +0300 Subject: [PATCH 4/8] more user_id -> user_ref change --- server/src/core/agent/agent_service.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/server/src/core/agent/agent_service.py b/server/src/core/agent/agent_service.py index d861af3..af368ca 100644 --- a/server/src/core/agent/agent_service.py +++ b/server/src/core/agent/agent_service.py @@ -96,7 +96,7 @@ async def handle_user_input(self, req: UserInputDto) -> dict: def user_history_process(self, req: UserInputDto) -> dict: add_message_dto = AddMessageDto( - user_email=req.user.email, + user_ref=req.user.pk, app_key=req.app.app_key, session_id=req.session_id, message=MessageCompletion( From b2a13aa3fdbce8305d617bc27e864f966b79d00a Mon Sep 17 00:00:00 2001 From: Victor Porton Date: Fri, 13 Oct 2023 02:39:00 +0300 Subject: [PATCH 5/8] removed superfluous int(...) --- server/src/core/agent/agent_service.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/server/src/core/agent/agent_service.py b/server/src/core/agent/agent_service.py index af368ca..b755ab1 100644 --- a/server/src/core/agent/agent_service.py +++ b/server/src/core/agent/agent_service.py @@ -53,7 +53,7 @@ def is_action(self, req: UserInputDto) -> bool: async def handle_user_input(self, req: UserInputDto) -> dict: current_user = req.user add_message_dto = AddMessageDto( - user_ref=int(req.user.pk), + user_ref=req.user.pk, app_key=req.app.app_key, session_id=req.session_id, message=MessageCompletion( From 8dbeb4f861c4a6c3137f058a5e618326460eff8c Mon Sep 17 00:00:00 2001 From: Victor Porton Date: Fri, 13 Oct 2023 03:15:26 +0300 Subject: [PATCH 6/8] bug fixes --- script.sql | 6 +++--- server/src/core/user/user_dao.py | 3 +++ server/src/core/user/user_service.py | 2 +- 3 files changed, 7 insertions(+), 4 deletions(-) diff --git a/script.sql b/script.sql index bcb35b3..7dcfef3 100644 --- a/script.sql +++ b/script.sql @@ -1,15 +1,15 @@ create table users ( id serial primary key, - email null varchar(250) - password null varchar(50), + email varchar(250) null, + password varchar(50) null, name varchar(50) null ); create table chat_messages ( id serial primary key, - user_ref not null int, + user_ref int not null, chatbot_id varchar(50), message text, is_bot_reply boolean, diff --git a/server/src/core/user/user_dao.py b/server/src/core/user/user_dao.py index 949b999..0ef28ba 100644 --- a/server/src/core/user/user_dao.py +++ b/server/src/core/user/user_dao.py @@ -6,17 +6,20 @@ class User(BaseModel): + pk: int email: str password: str name: str = None +# FIXME: rewrite with DB. class UserDao: def __init__(self): self.db = FileDB('./file_db/users') def get_all(self) -> Dict[str, User]: users = self.db.get("all") + print("USERS: ", users) if users is None: return {} return users diff --git a/server/src/core/user/user_service.py b/server/src/core/user/user_service.py index 5fa37a4..c974e3b 100644 --- a/server/src/core/user/user_service.py +++ b/server/src/core/user/user_service.py @@ -9,7 +9,7 @@ def __init__(self, user_storage: UserDao, app_dao: AppDao): self.app_dao = app_dao self.dao = user_storage if self.dao.get_all() == {}: - self.add_user(User(name="Alex", password="123", email="admin@gmail.com")) + self.add_user(User(pk=1, name="Alex", password="123", email="admin@gmail.com")) def authenticate_user(self, email, password): user = self.dao.get(email) From af154b62007c9f064870dae58177f4ec5c2392a1 Mon Sep 17 00:00:00 2001 From: Victor Porton Date: Fri, 13 Oct 2023 04:12:40 +0300 Subject: [PATCH 7/8] finished handling users through a DB --- script.sql | 2 +- server/src/api/factory.py | 2 +- server/src/core/common/pg.py | 10 +++++ server/src/core/user/user_dao.py | 60 ++++++++++++++-------------- server/src/core/user/user_service.py | 16 ++++++-- 5 files changed, 55 insertions(+), 35 deletions(-) diff --git a/script.sql b/script.sql index 7dcfef3..6fa8cf5 100644 --- a/script.sql +++ b/script.sql @@ -28,5 +28,5 @@ create index idx_timestamp create index idx_chatbot_id on chat_messages (chatbot_id); -create index idx_user_email +create unique index idx_user_email on users (email); diff --git a/server/src/api/factory.py b/server/src/api/factory.py index 98886bd..b5d49a6 100644 --- a/server/src/api/factory.py +++ b/server/src/api/factory.py @@ -16,5 +16,5 @@ chat_history = factory_chat_history(pg_conn) agent = agent_factory(chat_history, cost_service, llm_service) apps = AppDao() -user_service = UserService(UserDao(), apps) +user_service = UserService(UserDao(pg_conn), apps) current_session = {} diff --git a/server/src/core/common/pg.py b/server/src/core/common/pg.py index bb2cc71..e1b4108 100644 --- a/server/src/core/common/pg.py +++ b/server/src/core/common/pg.py @@ -33,3 +33,13 @@ def fetch_all(self, query: str, params=None) -> List[Dict]: for row in rows: result.append(dict(zip(cols, row))) return result + + def fetch_one(self, query: str, params=None) -> Dict: + self.cursor.execute(query, params) + row = self.cursor.fetchone() + if row is None: + return None + # Convert rows to list of dictionaries so they're easier to work with + cols = [desc[0] for desc in self.cursor.description] + result = [] + return dict(zip(cols, row)) diff --git a/server/src/core/user/user_dao.py b/server/src/core/user/user_dao.py index 0ef28ba..a8accf5 100644 --- a/server/src/core/user/user_dao.py +++ b/server/src/core/user/user_dao.py @@ -1,4 +1,6 @@ -from typing import Dict +from typing import Dict, Optional + +from psycopg2 import IntegrityError from pydantic import BaseModel @@ -6,46 +8,46 @@ class User(BaseModel): - pk: int + pk: Optional[int] = None email: str - password: str + password: str # TODO: encrypted password name: str = None -# FIXME: rewrite with DB. class UserDao: - def __init__(self): - self.db = FileDB('./file_db/users') + def __init__(self, pg_conn): + self.db = pg_conn def get_all(self) -> Dict[str, User]: - users = self.db.get("all") - print("USERS: ", users) - if users is None: - return {} - return users + users = self.db.fetch_all("SELECT * FROM users") + return [User(**u) for u in users] - def get(self, user) -> User: - return self.get_all().get(user) + def get_by_email(self, email: str) -> User: + user = self.db.fetch_one("SELECT * FROM users WHERE email=%s", (email,)) + return User(**user) def add(self, user: User): - if self.get(user.email) is not None: + try: + self.db.execute( + "INSERT INTO users (email, password, name) VALUES (%s, %s, %s)", + (user.email, user.password, user.name) + ) + except IntegrityError as e: + print("Inserting user:", e) return False - users = self.get_all() - users[user.email] = user - - self.db.put("all", users) - def edit(self, user: User): - users = self.get_all() - if users.get(user.email) is None: + try: + self.db.execute( + "UPDATE users SET email=%s, password=%s, name=%s", + (user.email, user.password, user.name) + ) + except IntegrityError as e: + print("Updating user:", e) return False - users[user.email] = user - self.db.put("all", users) - def remove(self, user: str): - if self.get(user) is None: - return False - users = self.get_all() - del users[user] - self.db.put("all", users) + def remove(self, pk: int): + self.db.execute( + "DELETE FROM users WHERE id=%s", + (pk,) + ) diff --git a/server/src/core/user/user_service.py b/server/src/core/user/user_service.py index c974e3b..d5e7da3 100644 --- a/server/src/core/user/user_service.py +++ b/server/src/core/user/user_service.py @@ -1,5 +1,7 @@ from typing import List +from psycopg2 import IntegrityError + from core.app.app_dao import AppDao from core.user.user_dao import UserDao, User @@ -8,11 +10,15 @@ class UserService: def __init__(self, user_storage: UserDao, app_dao: AppDao): self.app_dao = app_dao self.dao = user_storage - if self.dao.get_all() == {}: - self.add_user(User(pk=1, name="Alex", password="123", email="admin@gmail.com")) + try: + self.add_user(User(name="Alex", password="123", email="admin@gmail.com")) + self.dao.db.conn.commit() + except IntegrityError as e: + print("IGNORED EXCEPTION", e) # TODO: Remove this line. + self.dao.db.conn.rollback() def authenticate_user(self, email, password): - user = self.dao.get(email) + user = self.dao.get_by_email(email) if user is None: return None if user.password == password: @@ -20,6 +26,7 @@ def authenticate_user(self, email, password): return None + # TODO: wrong responsibility def exists_app(self, email: str, app_key: str) -> bool: app = self.app_dao.get_by_id(email, app_key) if app is None: @@ -27,11 +34,12 @@ def exists_app(self, email: str, app_key: str) -> bool: return True def get_user_by_email(self, email): - return self.dao.get(email) + return self.dao.get_by_email(email) def add_user(self, user: User): self.dao.add(user) + # TODO: Should be removed. def get_all_users(self) -> List[User]: users = self.dao.get_all().values() for user in users: From 8ec1b8d31758ff7209fb2ff301ef0b1f24ab6463 Mon Sep 17 00:00:00 2001 From: Victor Porton Date: Sat, 14 Oct 2023 20:57:34 +0300 Subject: [PATCH 8/8] bug fix --- server/src/core/user/user_dao.py | 1 + 1 file changed, 1 insertion(+) diff --git a/server/src/core/user/user_dao.py b/server/src/core/user/user_dao.py index a8accf5..fd24966 100644 --- a/server/src/core/user/user_dao.py +++ b/server/src/core/user/user_dao.py @@ -24,6 +24,7 @@ def get_all(self) -> Dict[str, User]: def get_by_email(self, email: str) -> User: user = self.db.fetch_one("SELECT * FROM users WHERE email=%s", (email,)) + user["pk"] = user["id"] # slightly a hack return User(**user) def add(self, user: User):