From 23665f34ac63414da182495c425ca61e79705d5f Mon Sep 17 00:00:00 2001 From: madgrizzle Date: Mon, 6 May 2024 22:14:42 -0400 Subject: [PATCH 1/8] Update db.py --- memgpt/agent_store/db.py | 8 ++++++-- 1 file changed, 6 insertions(+), 2 deletions(-) diff --git a/memgpt/agent_store/db.py b/memgpt/agent_store/db.py index 8584f46d06..ee38c4cdef 100644 --- a/memgpt/agent_store/db.py +++ b/memgpt/agent_store/db.py @@ -379,7 +379,7 @@ def list_data_sources(self): unique_data_sources = session.query(self.db_model.data_source).filter(*self.filters).distinct().all() return unique_data_sources - def query_date(self, start_date, end_date, offset=0, limit=None): + def query_date(self, start_date, end_date, limit=None, offset=0): filters = self.get_filters({}) with self.session_maker() as session: query = ( @@ -387,6 +387,8 @@ def query_date(self, start_date, end_date, offset=0, limit=None): .filter(*filters) .filter(self.db_model.created_at >= start_date) .filter(self.db_model.created_at <= end_date) + .filter(self.db_model.role != 'system') + .filter(self.db_model.role != 'tool') .offset(offset) ) if limit: @@ -394,7 +396,7 @@ def query_date(self, start_date, end_date, offset=0, limit=None): results = query.all() return [result.to_record() for result in results] - def query_text(self, query, offset=0, limit=None): + def query_text(self, query, limit=None, offset=0): # todo: make fuzz https://stackoverflow.com/questions/42388956/create-a-full-text-search-index-with-sqlalchemy-on-postgresql/42390204#42390204 filters = self.get_filters({}) with self.session_maker() as session: @@ -402,6 +404,8 @@ def query_text(self, query, offset=0, limit=None): session.query(self.db_model) .filter(*filters) .filter(func.lower(self.db_model.text).contains(func.lower(query))) + .filter(self.db_model.role != 'system') + .filter(self.db_model.role != 'tool') .offset(offset) ) if limit: From bacaa0adeb034042fb6c3900db24de036675ab8c Mon Sep 17 00:00:00 2001 From: madgrizzle Date: Tue, 7 May 2024 20:35:07 -0400 Subject: [PATCH 2/8] updates --- db/run_postgres.sh | 2 +- memgpt/agent_store/db.py | 25 +++++++++++++++++++ memgpt/data_types.py | 5 ++++ .../simple_summary_wrapper.py | 2 +- memgpt/memory.py | 4 +-- 5 files changed, 34 insertions(+), 4 deletions(-) mode change 100644 => 100755 db/run_postgres.sh diff --git a/db/run_postgres.sh b/db/run_postgres.sh old mode 100644 new mode 100755 index 650219381f..7747774189 --- a/db/run_postgres.sh +++ b/db/run_postgres.sh @@ -4,7 +4,7 @@ docker build -f db/Dockerfile.simple -t pg-test . # run container docker run -d --rm \ --name memgpt-db-test \ - -p 8888:5432 \ + -p 5432:5432 \ -e POSTGRES_PASSWORD=password \ -v memgpt_db_test:/var/lib/postgresql/data \ pg-test:latest diff --git a/memgpt/agent_store/db.py b/memgpt/agent_store/db.py index ee38c4cdef..4373badc5c 100644 --- a/memgpt/agent_store/db.py +++ b/memgpt/agent_store/db.py @@ -531,6 +531,31 @@ def update(self, record: RecordType): # Commit the changes to the database session.commit() + def str_to_datetime(self, str_date): + val = str_date.split('-') + _datetime = datetime(int(val[0]), int(val[1]), int(val[2])) + return _datetime + + def query_date(self, start_date, end_date, limit=None, offset=0): + filters = self.get_filters({}) + _start_date=self.str_to_datetime(start_date) + print(f"{start_date}->{_start_date}") + _end_date=self.str_to_datetime(end_date) + print(f"{end_date}->{_end_date}") + with self.session_maker() as session: + query = ( + session.query(self.db_model) + .filter(*filters) + .filter(self.db_model.created_at >= _start_date) + .filter(self.db_model.created_at <= _end_date) + .filter(self.db_model.role != 'system') + .filter(self.db_model.role != 'tool') + .offset(offset) + ) + if limit: + query = query.limit(limit) + results = query.all() + return [result.to_record() for result in results] class SQLLiteStorageConnector(SQLStorageConnector): def __init__(self, table_type: str, config: MemGPTConfig, user_id, agent_id=None): diff --git a/memgpt/data_types.py b/memgpt/data_types.py index 9e40be6388..24d2b276b4 100644 --- a/memgpt/data_types.py +++ b/memgpt/data_types.py @@ -244,6 +244,11 @@ def dict_to_message( tool_call_id=openai_message_dict["tool_call_id"] if "tool_call_id" in openai_message_dict else None, ) + def to_openai_dict_search_results(self, max_tool_id_length=TOOL_CALL_ID_MAX_LEN) -> dict: + result_json = self.to_openai_dict() + search_result_json = {"timestamp": self.created_at, "message":{"content": result_json["content"], "role": result_json["role"]}} + return search_result_json + def to_openai_dict(self, max_tool_id_length=TOOL_CALL_ID_MAX_LEN) -> dict: """Go from Message class to ChatCompletion message object""" diff --git a/memgpt/local_llm/llm_chat_completion_wrappers/simple_summary_wrapper.py b/memgpt/local_llm/llm_chat_completion_wrappers/simple_summary_wrapper.py index 1f0c72d7e9..4e9f21ada0 100644 --- a/memgpt/local_llm/llm_chat_completion_wrappers/simple_summary_wrapper.py +++ b/memgpt/local_llm/llm_chat_completion_wrappers/simple_summary_wrapper.py @@ -108,7 +108,7 @@ def create_function_call(function_call): elif message["role"] == "assistant": prompt += f"\nASSISTANT: {message['content']}" # need to add the function call if there was one - if message["function_call"]: + if "function_call" in message and message["function_call"]: prompt += f"\n{create_function_call(message['function_call'])}" elif message["role"] in ["function", "tool"]: # TODO find a good way to add this diff --git a/memgpt/memory.py b/memgpt/memory.py index b07b0263c0..a3f6b3f946 100644 --- a/memgpt/memory.py +++ b/memgpt/memory.py @@ -327,12 +327,12 @@ def get_all(self, start=0, count=None): def text_search(self, query_string, count=None, start=None): results = self.storage.query_text(query_string, count, start) - results_json = [message.to_openai_dict() for message in results] + results_json = [message.to_openai_dict_search_results() for message in results] return results_json, len(results) def date_search(self, start_date, end_date, count=None, start=None): results = self.storage.query_date(start_date, end_date, count, start) - results_json = [message.to_openai_dict() for message in results] + results_json = [message.to_openai_dict_search_results() for message in results] return results_json, len(results) def __repr__(self) -> str: From df5eb35dd1d69feff83eef5c1f829739b31f7ee9 Mon Sep 17 00:00:00 2001 From: madgrizzle Date: Tue, 7 May 2024 21:05:02 -0400 Subject: [PATCH 3/8] update --- memgpt/agent_store/db.py | 1 + 1 file changed, 1 insertion(+) diff --git a/memgpt/agent_store/db.py b/memgpt/agent_store/db.py index 4373badc5c..c76c8f546f 100644 --- a/memgpt/agent_store/db.py +++ b/memgpt/agent_store/db.py @@ -1,6 +1,7 @@ import base64 import os import uuid +from datetime import datetime from typing import Dict, Iterator, List, Optional import numpy as np From d15e1dbcfb92e01b94433f98c9e49a9008c4509e Mon Sep 17 00:00:00 2001 From: madgrizzle Date: Tue, 7 May 2024 21:25:37 -0400 Subject: [PATCH 4/8] update black --- memgpt/agent_store/db.py | 25 +++++++++++++------------ memgpt/data_types.py | 4 ++-- 2 files changed, 15 insertions(+), 14 deletions(-) diff --git a/memgpt/agent_store/db.py b/memgpt/agent_store/db.py index c76c8f546f..46e7b81c72 100644 --- a/memgpt/agent_store/db.py +++ b/memgpt/agent_store/db.py @@ -388,8 +388,8 @@ def query_date(self, start_date, end_date, limit=None, offset=0): .filter(*filters) .filter(self.db_model.created_at >= start_date) .filter(self.db_model.created_at <= end_date) - .filter(self.db_model.role != 'system') - .filter(self.db_model.role != 'tool') + .filter(self.db_model.role != "system") + .filter(self.db_model.role != "tool") .offset(offset) ) if limit: @@ -405,8 +405,8 @@ def query_text(self, query, limit=None, offset=0): session.query(self.db_model) .filter(*filters) .filter(func.lower(self.db_model.text).contains(func.lower(query))) - .filter(self.db_model.role != 'system') - .filter(self.db_model.role != 'tool') + .filter(self.db_model.role != "system") + .filter(self.db_model.role != "tool") .offset(offset) ) if limit: @@ -533,30 +533,31 @@ def update(self, record: RecordType): session.commit() def str_to_datetime(self, str_date): - val = str_date.split('-') + val = str_date.split("-") _datetime = datetime(int(val[0]), int(val[1]), int(val[2])) return _datetime def query_date(self, start_date, end_date, limit=None, offset=0): filters = self.get_filters({}) - _start_date=self.str_to_datetime(start_date) - print(f"{start_date}->{_start_date}") - _end_date=self.str_to_datetime(end_date) - print(f"{end_date}->{_end_date}") + _start_date = self.str_to_datetime(start_date) + print(f"{start_date}->{_start_date}") + _end_date = self.str_to_datetime(end_date) + print(f"{end_date}->{_end_date}") with self.session_maker() as session: query = ( session.query(self.db_model) .filter(*filters) .filter(self.db_model.created_at >= _start_date) .filter(self.db_model.created_at <= _end_date) - .filter(self.db_model.role != 'system') - .filter(self.db_model.role != 'tool') + .filter(self.db_model.role != "system") + .filter(self.db_model.role != "tool") .offset(offset) ) if limit: query = query.limit(limit) results = query.all() - return [result.to_record() for result in results] + return [result.to_record() for result in results] + class SQLLiteStorageConnector(SQLStorageConnector): def __init__(self, table_type: str, config: MemGPTConfig, user_id, agent_id=None): diff --git a/memgpt/data_types.py b/memgpt/data_types.py index 24d2b276b4..682270e363 100644 --- a/memgpt/data_types.py +++ b/memgpt/data_types.py @@ -246,9 +246,9 @@ def dict_to_message( def to_openai_dict_search_results(self, max_tool_id_length=TOOL_CALL_ID_MAX_LEN) -> dict: result_json = self.to_openai_dict() - search_result_json = {"timestamp": self.created_at, "message":{"content": result_json["content"], "role": result_json["role"]}} + search_result_json = {"timestamp": self.created_at, "message": {"content": result_json["content"], "role": result_json["role"]}} return search_result_json - + def to_openai_dict(self, max_tool_id_length=TOOL_CALL_ID_MAX_LEN) -> dict: """Go from Message class to ChatCompletion message object""" From 33c4d6911b9ea2ccccaf0870d2876135838c2aa6 Mon Sep 17 00:00:00 2001 From: madgrizzle Date: Tue, 7 May 2024 21:28:25 -0400 Subject: [PATCH 5/8] remove debug print statements --- memgpt/agent_store/db.py | 2 -- 1 file changed, 2 deletions(-) diff --git a/memgpt/agent_store/db.py b/memgpt/agent_store/db.py index 46e7b81c72..52146db4f8 100644 --- a/memgpt/agent_store/db.py +++ b/memgpt/agent_store/db.py @@ -540,9 +540,7 @@ def str_to_datetime(self, str_date): def query_date(self, start_date, end_date, limit=None, offset=0): filters = self.get_filters({}) _start_date = self.str_to_datetime(start_date) - print(f"{start_date}->{_start_date}") _end_date = self.str_to_datetime(end_date) - print(f"{end_date}->{_end_date}") with self.session_maker() as session: query = ( session.query(self.db_model) From d3399996eebf2e6f863eb69cdf678258a6238e87 Mon Sep 17 00:00:00 2001 From: madgrizzle Date: Wed, 8 May 2024 19:29:31 -0400 Subject: [PATCH 6/8] revert db/run_postgres.sh --- db/run_postgres.sh | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/db/run_postgres.sh b/db/run_postgres.sh index 7747774189..650219381f 100755 --- a/db/run_postgres.sh +++ b/db/run_postgres.sh @@ -4,7 +4,7 @@ docker build -f db/Dockerfile.simple -t pg-test . # run container docker run -d --rm \ --name memgpt-db-test \ - -p 5432:5432 \ + -p 8888:5432 \ -e POSTGRES_PASSWORD=password \ -v memgpt_db_test:/var/lib/postgresql/data \ pg-test:latest From dcb2e32e6cf378665d90c554a785fc6f1a3e7bc3 Mon Sep 17 00:00:00 2001 From: madgrizzle Date: Fri, 10 May 2024 19:46:55 -0400 Subject: [PATCH 7/8] update to avoid overriding config uri's in server.py --- memgpt/config.py | 1 - memgpt/server/server.py | 2 +- memgpt/settings.py | 24 +++++++++++++++++------- 3 files changed, 18 insertions(+), 9 deletions(-) diff --git a/memgpt/config.py b/memgpt/config.py index 0db59ab2dd..30e4e8dfea 100644 --- a/memgpt/config.py +++ b/memgpt/config.py @@ -171,7 +171,6 @@ def load(cls) -> "MemGPTConfig": "config_path": config_path, "memgpt_version": get_field(config, "version", "memgpt_version"), } - # Don't include null values config_dict = {k: v for k, v in config_dict.items() if v is not None} diff --git a/memgpt/server/server.py b/memgpt/server/server.py index cda83e0379..00c776d4c5 100644 --- a/memgpt/server/server.py +++ b/memgpt/server/server.py @@ -216,7 +216,7 @@ def __init__( # Update storage URI to match passed in settings # TODO: very hack, fix in the future for memory_type in ("archival", "recall", "metadata"): - if settings.memgpt_pg_uri: + if settings.memgpt_pg_uri_no_default: # override with env setattr(self.config, f"{memory_type}_storage_uri", settings.memgpt_pg_uri) self.config.save() diff --git a/memgpt/settings.py b/memgpt/settings.py index 8b3da82aa2..1154de1269 100644 --- a/memgpt/settings.py +++ b/memgpt/settings.py @@ -7,11 +7,11 @@ class Settings(BaseSettings): model_config = SettingsConfigDict(env_prefix="memgpt_") server_pass: Optional[str] = None - pg_db: Optional[str] = "memgpt" - pg_user: Optional[str] = "memgpt" - pg_password: Optional[str] = "memgpt" - pg_host: Optional[str] = "localhost" - pg_port: Optional[int] = 5432 + pg_db: Optional[str] = None + pg_user: Optional[str] = None + pg_password: Optional[str] = None + pg_host: Optional[str] = None + pg_port: Optional[int] = None pg_uri: Optional[str] = None # option to specifiy full uri cors_origins: Optional[list] = ["http://memgpt.localhost", "http://localhost:8283", "http://localhost:8083"] @@ -22,8 +22,18 @@ def memgpt_pg_uri(self) -> str: elif self.pg_db and self.pg_user and self.pg_password and self.pg_host and self.pg_port: return f"postgresql+pg8000://{self.pg_user}:{self.pg_password}@{self.pg_host}:{self.pg_port}/{self.pg_db}" else: - return None - + return f"postgresql+pg8000://memgpt:memgpt@localhost:5432/memgpt" + # add this property to avoid being returned the default + # reference: https://github.com/cpacker/MemGPT/issues/1362 + @property + def memgpt_pt_uri_no_default(self) -> str: + if self.pg_uri: + return self.pg_uri + elif self.pg_db and self.pg_user and self.pg_password and self.pg_host and self.pg_port: + return f"postgresql+pg8000://{self.pg_user}:{self.pg_password}@{self.pg_host}:{self.pg_port}/{self.pg_db}" + else: + return None + # singleton settings = Settings() From c7400a43343aa28dbf2ba8eaa0b50d5c2cfe85e5 Mon Sep 17 00:00:00 2001 From: madgrizzle Date: Fri, 10 May 2024 20:35:22 -0400 Subject: [PATCH 8/8] black'ed commit --- memgpt/settings.py | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) diff --git a/memgpt/settings.py b/memgpt/settings.py index 1154de1269..6fceff379f 100644 --- a/memgpt/settings.py +++ b/memgpt/settings.py @@ -26,7 +26,7 @@ def memgpt_pg_uri(self) -> str: # add this property to avoid being returned the default # reference: https://github.com/cpacker/MemGPT/issues/1362 - @property + @property def memgpt_pt_uri_no_default(self) -> str: if self.pg_uri: return self.pg_uri @@ -34,6 +34,7 @@ def memgpt_pt_uri_no_default(self) -> str: return f"postgresql+pg8000://{self.pg_user}:{self.pg_password}@{self.pg_host}:{self.pg_port}/{self.pg_db}" else: return None - + + # singleton settings = Settings()