From 9b7aa600a1a4e1dfe25bf766d5ae369d154b0e8a Mon Sep 17 00:00:00 2001 From: chunyu li <78344051+fridayL@users.noreply.github.com> Date: Tue, 17 Mar 2026 10:53:06 +0800 Subject: [PATCH 01/11] feat(memreader/LLM): add backup config for openai memreader (#1246) feat: add backend for openai Co-authored-by: harvey_xiang --- src/memos/api/config.py | 47 +++++++++++++++------- src/memos/configs/llm.py | 24 ++++++++--- src/memos/llms/openai.py | 83 ++++++++++++++++++++++++++++----------- tests/configs/test_llm.py | 5 +++ 4 files changed, 114 insertions(+), 45 deletions(-) diff --git a/src/memos/api/config.py b/src/memos/api/config.py index 06aa50c65..f24e28559 100644 --- a/src/memos/api/config.py +++ b/src/memos/api/config.py @@ -321,23 +321,40 @@ def get_activation_config() -> dict[str, Any]: @staticmethod def get_memreader_config() -> dict[str, Any]: - """Get MemReader configuration for chat/doc extraction (fine-tuned 0.6B model).""" - return { - "backend": "openai", - "config": { - "model_name_or_path": os.getenv("MEMRADER_MODEL", "gpt-4o-mini"), - "temperature": 0.6, - "max_tokens": int(os.getenv("MEMRADER_MAX_TOKENS", "8000")), - "top_p": 0.95, - "top_k": 20, - "api_key": os.getenv("MEMRADER_API_KEY", "EMPTY"), - # Default to OpenAI base URL when env var is not provided to satisfy pydantic - # validation requirements during tests/import. - "api_base": os.getenv("MEMRADER_API_BASE", "https://api.openai.com/v1"), - "remove_think_prefix": True, - }, + """Get MemReader configuration for chat/doc extraction (fine-tuned 0.6B model). + + When MEMREADER_GENERAL_MODEL is configured (i.e. a separate stable LLM exists), + the backup client is automatically enabled so that primary failures (self-deployed + model) fall back to the general LLM. + """ + config = { + "model_name_or_path": os.getenv("MEMRADER_MODEL", "gpt-4o-mini"), + "temperature": 0.6, + "max_tokens": int(os.getenv("MEMRADER_MAX_TOKENS", "8000")), + "top_p": 0.95, + "top_k": 20, + "api_key": os.getenv("MEMRADER_API_KEY", "EMPTY"), + # Default to OpenAI base URL when env var is not provided to satisfy pydantic + # validation requirements during tests/import. + "api_base": os.getenv("MEMRADER_API_BASE", "https://api.openai.com/v1"), + "remove_think_prefix": True, } + general_model = os.getenv("MEMREADER_GENERAL_MODEL") + enable_backup = os.getenv("MEMREADER_ENABLE_BACKUP", "false").lower() == "true" + if general_model and enable_backup: + config["backup_client"] = True + config["backup_model_name_or_path"] = general_model + config["backup_api_key"] = os.getenv( + "MEMREADER_GENERAL_API_KEY", os.getenv("OPENAI_API_KEY", "EMPTY") + ) + config["backup_api_base"] = os.getenv( + "MEMREADER_GENERAL_API_BASE", + os.getenv("OPENAI_API_BASE", "https://api.openai.com/v1"), + ) + + return {"backend": "openai", "config": config} + @staticmethod def get_memreader_general_llm_config() -> dict[str, Any]: """Get general LLM configuration for non-chat/doc tasks. diff --git a/src/memos/configs/llm.py b/src/memos/configs/llm.py index 5487d117c..11c39b33c 100644 --- a/src/memos/configs/llm.py +++ b/src/memos/configs/llm.py @@ -28,6 +28,22 @@ class OpenAILLMConfig(BaseLLMConfig): default="https://api.openai.com/v1", description="Base URL for OpenAI API" ) extra_body: Any = Field(default=None, description="extra body") + backup_client: bool = Field( + default=False, + description="Whether to enable backup client for fallback on primary failure", + ) + backup_api_key: str | None = Field( + default=None, description="API key for backup OpenAI-compatible endpoint" + ) + backup_api_base: str | None = Field( + default=None, description="Base URL for backup OpenAI-compatible endpoint" + ) + backup_model_name_or_path: str | None = Field( + default=None, description="Model name for backup endpoint" + ) + backup_headers: dict[str, Any] | None = Field( + default=None, description="Default headers for backup client requests" + ) class OpenAIResponsesLLMConfig(BaseLLMConfig): @@ -42,22 +58,18 @@ class OpenAIResponsesLLMConfig(BaseLLMConfig): ) -class QwenLLMConfig(BaseLLMConfig): - api_key: str = Field(..., description="API key for DashScope (Qwen)") +class QwenLLMConfig(OpenAILLMConfig): api_base: str = Field( default="https://dashscope-intl.aliyuncs.com/compatible-mode/v1", description="Base URL for Qwen OpenAI-compatible API", ) - extra_body: Any = Field(default=None, description="extra body") -class DeepSeekLLMConfig(BaseLLMConfig): - api_key: str = Field(..., description="API key for DeepSeek") +class DeepSeekLLMConfig(OpenAILLMConfig): api_base: str = Field( default="https://api.deepseek.com", description="Base URL for DeepSeek OpenAI-compatible API", ) - extra_body: Any = Field(default=None, description="Extra options for API") class AzureLLMConfig(BaseLLMConfig): diff --git a/src/memos/llms/openai.py b/src/memos/llms/openai.py index 93dac42fb..f6bb4efc1 100644 --- a/src/memos/llms/openai.py +++ b/src/memos/llms/openai.py @@ -27,7 +27,39 @@ def __init__(self, config: OpenAILLMConfig): self.client = openai.Client( api_key=config.api_key, base_url=config.api_base, default_headers=config.default_headers ) - logger.info("OpenAI LLM instance initialized") + self.use_backup_client = config.backup_client + if self.use_backup_client: + self.backup_client = openai.Client( + api_key=config.backup_api_key, + base_url=config.backup_api_base, + default_headers=config.backup_headers, + ) + logger.info( + f"OpenAI LLM instance initialized with backup " + f"(model={config.backup_model_name_or_path})" + ) + else: + self.backup_client = None + logger.info("OpenAI LLM instance initialized") + + def _parse_response(self, response) -> str: + """Extract text content from a chat completion response.""" + if not response.choices: + logger.warning("OpenAI response has no choices") + return "" + + tool_calls = getattr(response.choices[0].message, "tool_calls", None) + if isinstance(tool_calls, list) and len(tool_calls) > 0: + return self.tool_call_parser(tool_calls) + response_content = response.choices[0].message.content + reasoning_content = getattr(response.choices[0].message, "reasoning_content", None) + if isinstance(reasoning_content, str) and reasoning_content: + reasoning_content = f"{reasoning_content}" + if self.config.remove_think_prefix: + return remove_thinking_tags(response_content) + if reasoning_content: + return reasoning_content + (response_content or "") + return response_content or "" @timed_with_status( log_prefix="OpenAI LLM", @@ -50,29 +82,32 @@ def generate(self, messages: MessageList, **kwargs) -> str: start_time = time.perf_counter() logger.info(f"OpenAI LLM Request body: {request_body}") - response = self.client.chat.completions.create(**request_body) - - cost_time = time.perf_counter() - start_time - logger.info( - f"Request body: {request_body}, Response from OpenAI: {response.model_dump_json()}, Cost time: {cost_time}" - ) - - if not response.choices: - logger.warning("OpenAI response has no choices") - return "" - - tool_calls = getattr(response.choices[0].message, "tool_calls", None) - if isinstance(tool_calls, list) and len(tool_calls) > 0: - return self.tool_call_parser(tool_calls) - response_content = response.choices[0].message.content - reasoning_content = getattr(response.choices[0].message, "reasoning_content", None) - if isinstance(reasoning_content, str) and reasoning_content: - reasoning_content = f"{reasoning_content}" - if self.config.remove_think_prefix: - return remove_thinking_tags(response_content) - if reasoning_content: - return reasoning_content + (response_content or "") - return response_content or "" + try: + response = self.client.chat.completions.create(**request_body) + cost_time = time.perf_counter() - start_time + logger.info( + f"Request body: {request_body}, Response from OpenAI: " + f"{response.model_dump_json()}, Cost time: {cost_time}" + ) + return self._parse_response(response) + except Exception as e: + if not self.use_backup_client: + raise + logger.warning( + f"Primary LLM request failed with {type(e).__name__}: {e}, " + f"falling back to backup client" + ) + backup_body = { + **request_body, + "model": self.config.backup_model_name_or_path or request_body["model"], + } + backup_response = self.backup_client.chat.completions.create(**backup_body) + cost_time = time.perf_counter() - start_time + logger.info( + f"Backup LLM request succeeded, Response: " + f"{backup_response.model_dump_json()}, Cost time: {cost_time}" + ) + return self._parse_response(backup_response) @timed_with_status( log_prefix="OpenAI LLM Stream", diff --git a/tests/configs/test_llm.py b/tests/configs/test_llm.py index 6562c9a95..f3d4549b5 100644 --- a/tests/configs/test_llm.py +++ b/tests/configs/test_llm.py @@ -56,6 +56,11 @@ def test_openai_llm_config(): "remove_think_prefix", "extra_body", "default_headers", + "backup_client", + "backup_api_key", + "backup_api_base", + "backup_model_name_or_path", + "backup_headers", ], ) From 776a5f45d33b692ac5e68e05c02b10dcabe24e28 Mon Sep 17 00:00:00 2001 From: chunyu li <78344051+fridayL@users.noreply.github.com> Date: Tue, 17 Mar 2026 11:51:40 +0800 Subject: [PATCH 02/11] feat(Scheduler): change Scheduler default llm to general_llm (#1252) * feat: add backend for openai * feat: change memreader for scheaduer --------- Co-authored-by: harvey_xiang --- src/memos/api/handlers/component_init.py | 4 ++-- src/memos/mem_os/core.py | 2 +- .../general_modules/init_components_for_scheduler.py | 2 +- 3 files changed, 4 insertions(+), 4 deletions(-) diff --git a/src/memos/api/handlers/component_init.py b/src/memos/api/handlers/component_init.py index aa2525878..fc8ce311f 100644 --- a/src/memos/api/handlers/component_init.py +++ b/src/memos/api/handlers/component_init.py @@ -233,7 +233,7 @@ def init_server() -> dict[str, Any]: searcher: Searcher = tree_mem.get_searcher( manual_close_internet=os.getenv("ENABLE_INTERNET", "true").lower() == "false", moscube=False, - process_llm=mem_reader.llm, + process_llm=mem_reader.general_llm, ) logger.debug("Searcher created") @@ -260,7 +260,7 @@ def init_server() -> dict[str, Any]: mem_scheduler: OptimizedScheduler = SchedulerFactory.from_config(scheduler_config) mem_scheduler.initialize_modules( chat_llm=llm, - process_llm=mem_reader.llm, + process_llm=mem_reader.general_llm, db_engine=BaseDBManager.create_default_sqlite_engine(), mem_reader=mem_reader, redis_client=redis_client, diff --git a/src/memos/mem_os/core.py b/src/memos/mem_os/core.py index 22cd0e9cb..54f8f01e0 100644 --- a/src/memos/mem_os/core.py +++ b/src/memos/mem_os/core.py @@ -132,7 +132,7 @@ def _initialize_mem_scheduler(self) -> GeneralScheduler: # Configure scheduler general_modules self._mem_scheduler.initialize_modules( chat_llm=self.chat_llm, - process_llm=self.mem_reader.llm, + process_llm=self.mem_reader.general_llm, db_engine=self.user_manager.engine, ) self._mem_scheduler.start() diff --git a/src/memos/mem_scheduler/general_modules/init_components_for_scheduler.py b/src/memos/mem_scheduler/general_modules/init_components_for_scheduler.py index 8777b9f2e..ec431c253 100644 --- a/src/memos/mem_scheduler/general_modules/init_components_for_scheduler.py +++ b/src/memos/mem_scheduler/general_modules/init_components_for_scheduler.py @@ -305,7 +305,7 @@ def init_components() -> dict[str, Any]: searcher: Searcher = tree_mem.get_searcher( manual_close_internet=os.getenv("ENABLE_INTERNET", "true").lower() == "false", moscube=False, - process_llm=mem_reader.llm, + process_llm=mem_reader.general_llm, ) # Initialize feedback server feedback_server = SimpleMemFeedback( From 6d2d965ef34d6d9fa92608f72d39545a9f62e4f1 Mon Sep 17 00:00:00 2001 From: Dubberman <48425266+whipser030@users.noreply.github.com> Date: Tue, 17 Mar 2026 14:09:45 +0800 Subject: [PATCH 03/11] fix: The issue of the file name being stored as part of the memory as well (#1222) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit * finemode will not save filename * expand content list --------- Co-authored-by: 黑布林 <11641432+heiheiyouyou@user.noreply.gitee.com> Co-authored-by: CaralHsi --- src/memos/mem_feedback/feedback.py | 1 - src/memos/mem_reader/multi_modal_struct.py | 114 +++++++++++++++++++-- 2 files changed, 103 insertions(+), 12 deletions(-) diff --git a/src/memos/mem_feedback/feedback.py b/src/memos/mem_feedback/feedback.py index b8019004d..135058a7d 100644 --- a/src/memos/mem_feedback/feedback.py +++ b/src/memos/mem_feedback/feedback.py @@ -243,7 +243,6 @@ def _single_add_operation( datetime.now().isoformat() ) to_add_memory.metadata.background = new_memory_item.metadata.background - to_add_memory.metadata.sources = [] added_ids = self._retry_db_operation( lambda: self.memory_manager.add([to_add_memory], user_name=user_name, use_batch=False) diff --git a/src/memos/mem_reader/multi_modal_struct.py b/src/memos/mem_reader/multi_modal_struct.py index 4c0d4dcd0..2745a1bee 100644 --- a/src/memos/mem_reader/multi_modal_struct.py +++ b/src/memos/mem_reader/multi_modal_struct.py @@ -982,6 +982,9 @@ def _process_multi_modal_data( # Use MultiModalParser to parse the scene data # If it's a list, parse each item; otherwise parse as single message if isinstance(scene_data_info, list): + # Pre-expand multimodal messages + expanded_messages = self._expand_multimodal_messages(scene_data_info) + # Parse each message in the list all_memory_items = [] # Use thread pool to parse each message in parallel, but keep the original order @@ -996,7 +999,7 @@ def _process_multi_modal_data( need_emb=False, **kwargs, ) - for msg in scene_data_info + for msg in expanded_messages ] # collect results in original order for future in futures: @@ -1014,20 +1017,23 @@ def _process_multi_modal_data( if mode == "fast": return fast_memory_items else: + non_file_url_fast_items = [ + item for item in fast_memory_items if not self._is_file_url_only_item(item) + ] + # Part A: call llm in parallel using thread pool fine_memory_items = [] with ContextThreadPoolExecutor(max_workers=4) as executor: future_string = executor.submit( - self._process_string_fine, fast_memory_items, info, custom_tags, **kwargs + self._process_string_fine, non_file_url_fast_items, info, custom_tags, **kwargs ) future_tool = executor.submit( - self._process_tool_trajectory_fine, fast_memory_items, info, **kwargs + self._process_tool_trajectory_fine, non_file_url_fast_items, info, **kwargs ) - # Use general_llm for skill memory extraction (not fine-tuned for this task) future_skill = executor.submit( process_skill_memory_fine, - fast_memory_items=fast_memory_items, + fast_memory_items=non_file_url_fast_items, info=info, searcher=self.searcher, graph_db=self.graph_db, @@ -1039,7 +1045,7 @@ def _process_multi_modal_data( ) future_pref = executor.submit( process_preference_fine, - fast_memory_items, + non_file_url_fast_items, info, self.llm, self.embedder, @@ -1094,19 +1100,21 @@ def _process_transfer_multi_modal_data( **(raw_nodes[0].metadata.info or {}), } + # Filter out file-URL-only items for Part A fine processing (same as _process_multi_modal_data) + non_file_url_nodes = [node for node in raw_nodes if not self._is_file_url_only_item(node)] + fine_memory_items = [] # Part A: call llm in parallel using thread pool with ContextThreadPoolExecutor(max_workers=4) as executor: future_string = executor.submit( - self._process_string_fine, raw_nodes, info, custom_tags, **kwargs + self._process_string_fine, non_file_url_nodes, info, custom_tags, **kwargs ) future_tool = executor.submit( - self._process_tool_trajectory_fine, raw_nodes, info, **kwargs + self._process_tool_trajectory_fine, non_file_url_nodes, info, **kwargs ) - # Use general_llm for skill memory extraction (not fine-tuned for this task) future_skill = executor.submit( process_skill_memory_fine, - raw_nodes, + non_file_url_nodes, info, searcher=self.searcher, llm=self.general_llm, @@ -1118,7 +1126,7 @@ def _process_transfer_multi_modal_data( ) # Add preference memory extraction future_pref = executor.submit( - process_preference_fine, raw_nodes, info, self.general_llm, self.embedder, **kwargs + process_preference_fine, non_file_url_nodes, info, self.llm, self.embedder, **kwargs ) # Collect results @@ -1148,6 +1156,90 @@ def _process_transfer_multi_modal_data( fine_memory_items.extend(items) return fine_memory_items + @staticmethod + def _expand_multimodal_messages(messages: list) -> list: + """ + Expand messages whose ``content`` is a list into individual + sub-messages so that each modality is routed to its specialised + parser during fast-mode parsing. + + For a message like:: + + { + "content": [ + {"type": "text", "text": "Analyze this file"}, + {"type": "file", "file": {"file_data": "https://...", ...}}, + {"type": "image_url", "image_url": {"url": "https://..."}}, + ], + "role": "user", + "chat_time": "03:14 PM on 13 March, 2026", + } + + The result will be:: + + [ + {"content": "Analyze this file", "role": "user", "chat_time": "..."}, + {"type": "file", "file": {"file_data": "https://...", ...}}, + {"type": "image_url", "image_url": {"url": "https://..."}}, + ] + + Messages whose ``content`` is already a plain string (or that are + not dicts) are passed through unchanged. + """ + expanded: list = [] + for msg in messages: + if not isinstance(msg, dict): + expanded.append(msg) + continue + + content = msg.get("content") + if not isinstance(content, list): + expanded.append(msg) + continue + + # ---- content is a list: split by modality ---- + text_parts: list[str] = [] + for part in content: + if not isinstance(part, dict): + text_parts.append(str(part)) + continue + + part_type = part.get("type", "") + if part_type == "text": + text_parts.append(part.get("text", "")) + elif part_type in ("file", "image", "image_url"): + # Extract as a standalone message for its specialised parser + expanded.append(part) + else: + text_parts.append(f"[{part_type}]") + + # Reconstruct a text-only version of the original message + # (preserving role, chat_time, message_id, etc.) + text_content = "\n".join(t for t in text_parts if t.strip()) + if text_content.strip(): + text_msg = {k: v for k, v in msg.items() if k != "content"} + text_msg["content"] = text_content + expanded.append(text_msg) + + return expanded + + @staticmethod + def _is_file_url_only_item(item: TextualMemoryItem) -> bool: + """ + Check if a fast memory item contains only file-URL sources. + Args: + item: TextualMemoryItem to check + + Returns: + True if all sources are file-type with URL info (metadata only) + """ + sources = item.metadata.sources or [] + if not sources: + return False + return all( + getattr(s, "type", None) == "file" and getattr(s, "file_info", None) for s in sources + ) + def get_scene_data_info(self, scene_data: list, type: str) -> list[list[Any]]: """ Convert normalized MessagesType scenes into scene data info. From b82af21733356c826201a7f85a86fd6932aca139 Mon Sep 17 00:00:00 2001 From: Benny Chan <39545407+baranchen@users.noreply.github.com> Date: Tue, 17 Mar 2026 14:10:52 +0800 Subject: [PATCH 04/11] =?UTF-8?q?fix(graph=5Fdbs):=20add=20missing=20statu?= =?UTF-8?q?s=20parameter=20to=20Neo4jCommunityGraphDB=E2=80=A6=20(#1179)?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit fix(graph_dbs): add missing status parameter to Neo4jCommunityGraphDB.get_by_metadata The get_by_metadata method in neo4j_community.py was missing the `status` parameter that exists in the base class and other implementations (neo4j.py, postgres.py). This caused "unexpected keyword argument 'status'" errors when the search API tried to filter by status. Changes: - Add `user_name_flag: bool = True` parameter for consistency - Add `status: str | None = None` parameter - Add status filter logic in WHERE clause when status is provided - Update log messages to include status parameter Fixes search functionality when using Neo4j Community Edition backend. Co-authored-by: ariesshen Co-authored-by: CaralHsi --- src/memos/graph_dbs/neo4j_community.py | 11 +++++++++-- 1 file changed, 9 insertions(+), 2 deletions(-) diff --git a/src/memos/graph_dbs/neo4j_community.py b/src/memos/graph_dbs/neo4j_community.py index 470d8cd8e..283e15115 100644 --- a/src/memos/graph_dbs/neo4j_community.py +++ b/src/memos/graph_dbs/neo4j_community.py @@ -721,6 +721,8 @@ def get_by_metadata( user_name: str | None = None, filter: dict | None = None, knowledgebase_ids: list[str] | None = None, + user_name_flag: bool = True, + status: str | None = None, ) -> list[str]: """ Retrieve node IDs that match given metadata filters. @@ -745,15 +747,20 @@ def get_by_metadata( - Can be used for faceted recall or prefiltering before embedding rerank. """ logger.info( - f"[get_by_metadata] filters: {filters},user_name: {user_name},filter: {filter},knowledgebase_ids: {knowledgebase_ids}" + f"[get_by_metadata] filters: {filters},user_name: {user_name},filter: {filter},knowledgebase_ids: {knowledgebase_ids},status: {status}" ) print( - f"[get_by_metadata] filters: {filters},user_name: {user_name},filter: {filter},knowledgebase_ids: {knowledgebase_ids}" + f"[get_by_metadata] filters: {filters},user_name: {user_name},filter: {filter},knowledgebase_ids: {knowledgebase_ids},status: {status}" ) user_name = user_name if user_name else self.config.user_name where_clauses = [] params = {} + # Add status filter if provided + if status: + where_clauses.append("n.status = $status") + params["status"] = status + for i, f in enumerate(filters): field = f["field"] op = f.get("op", "=") From 87170e3946fc0c235faf06f05848324958eb77de Mon Sep 17 00:00:00 2001 From: chunyu li <78344051+fridayL@users.noreply.github.com> Date: Tue, 17 Mar 2026 14:50:54 +0800 Subject: [PATCH 05/11] feat(memcube): single cube use general_llm (#1253) * feat: add backend for openai * feat: change memreader for scheaduer * feat: change singlecube llm --------- Co-authored-by: harvey_xiang --- src/memos/multi_mem_cube/single_cube.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/memos/multi_mem_cube/single_cube.py b/src/memos/multi_mem_cube/single_cube.py index 6df410c19..6a91f436f 100644 --- a/src/memos/multi_mem_cube/single_cube.py +++ b/src/memos/multi_mem_cube/single_cube.py @@ -617,7 +617,7 @@ def add_before_search( # 3. Call LLM try: - raw = self.mem_reader.llm.generate([{"role": "user", "content": prompt}]) + raw = self.mem_reader.general_llm.generate([{"role": "user", "content": prompt}]) success, parsed_result = parse_keep_filter_response(raw) if not success: From 2a7a5ce47dc0f166517f201bfbfa15226cd3f1fe Mon Sep 17 00:00:00 2001 From: Hustzdy <67457465+wustzdy@users.noreply.github.com> Date: Tue, 17 Mar 2026 21:09:45 +0800 Subject: [PATCH 06/11] fix: Fix the issue where scheduled tasks are not using Redis problem (#1267) * fix: add filter of nolike && MOS_SCHEDULER_THREAD_POOL_MAX_WORKERS * fix: create_config null * fix: add use_redis_queue config * fix: add related_id --- src/memos/api/config.py | 4 ++- src/memos/api/handlers/component_init.py | 3 ++- src/memos/configs/mem_scheduler.py | 5 +++- src/memos/graph_dbs/polardb.py | 31 ++++++++++++++++++++++++ 4 files changed, 40 insertions(+), 3 deletions(-) diff --git a/src/memos/api/config.py b/src/memos/api/config.py index f24e28559..87f1efd8e 100644 --- a/src/memos/api/config.py +++ b/src/memos/api/config.py @@ -854,7 +854,7 @@ def get_scheduler_config() -> dict[str, Any]: ), "context_window_size": int(os.getenv("MOS_SCHEDULER_CONTEXT_WINDOW_SIZE", "5")), "thread_pool_max_workers": int( - os.getenv("MOS_SCHEDULER_THREAD_POOL_MAX_WORKERS", "10000") + os.getenv("MOS_SCHEDULER_THREAD_POOL_MAX_WORKERS", "200") ), "consume_interval_seconds": float( os.getenv("MOS_SCHEDULER_CONSUME_INTERVAL_SECONDS", "0.01") @@ -867,6 +867,8 @@ def get_scheduler_config() -> dict[str, Any]: "MOS_SCHEDULER_ENABLE_ACTIVATION_MEMORY", "false" ).lower() == "true", + "use_redis_queue": os.getenv("MEMSCHEDULER_USE_REDIS_QUEUE", "False").lower() + == "true", }, } diff --git a/src/memos/api/handlers/component_init.py b/src/memos/api/handlers/component_init.py index fc8ce311f..7894ff7dc 100644 --- a/src/memos/api/handlers/component_init.py +++ b/src/memos/api/handlers/component_init.py @@ -255,7 +255,8 @@ def init_server() -> dict[str, Any]: # Initialize Scheduler scheduler_config_dict = APIConfig.get_scheduler_config() scheduler_config = SchedulerConfigFactory( - backend="optimized_scheduler", config=scheduler_config_dict + backend=scheduler_config_dict["backend"], + config=scheduler_config_dict["config"], ) mem_scheduler: OptimizedScheduler = SchedulerFactory.from_config(scheduler_config) mem_scheduler.initialize_modules( diff --git a/src/memos/configs/mem_scheduler.py b/src/memos/configs/mem_scheduler.py index 9807f42c3..f76ddecc4 100644 --- a/src/memos/configs/mem_scheduler.py +++ b/src/memos/configs/mem_scheduler.py @@ -155,7 +155,10 @@ def validate_backend(cls, backend: str) -> str: @model_validator(mode="after") def create_config(self) -> "SchedulerConfigFactory": config_class = self.backend_to_class[self.backend] - self.config = config_class(**self.config) + raw = self.config + if isinstance(raw, dict) and "config" in raw and "use_redis_queue" not in raw: + raw = raw["config"] + self.config = config_class(**raw) return self diff --git a/src/memos/graph_dbs/polardb.py b/src/memos/graph_dbs/polardb.py index d740ad1d2..6db31990d 100644 --- a/src/memos/graph_dbs/polardb.py +++ b/src/memos/graph_dbs/polardb.py @@ -4667,6 +4667,36 @@ def build_filter_condition(condition_dict: dict) -> str: condition_parts.append( f"ag_catalog.agtype_access_operator(properties, '\"{key}\"'::agtype)::text LIKE '%{op_value}%'" ) + elif op == "nolike": + if key.startswith("info."): + info_field = key[5:] + if isinstance(op_value, str): + escaped_value = ( + escape_sql_string(op_value) + .replace("%", "\\%") + .replace("_", "\\_") + ) + condition_parts.append( + f"ag_catalog.agtype_access_operator(VARIADIC ARRAY[properties, '\"info\"'::ag_catalog.agtype, '\"{info_field}\"'::ag_catalog.agtype])::text NOT LIKE '%{escaped_value}%'" + ) + else: + condition_parts.append( + f"ag_catalog.agtype_access_operator(VARIADIC ARRAY[properties, '\"info\"'::ag_catalog.agtype, '\"{info_field}\"'::ag_catalog.agtype])::text NOT LIKE '%{op_value}%'" + ) + else: + if isinstance(op_value, str): + escaped_value = ( + escape_sql_string(op_value) + .replace("%", "\\%") + .replace("_", "\\_") + ) + condition_parts.append( + f"ag_catalog.agtype_access_operator(properties, '\"{key}\"'::agtype)::text NOT LIKE '%{escaped_value}%'" + ) + else: + condition_parts.append( + f"ag_catalog.agtype_access_operator(properties, '\"{key}\"'::agtype)::text NOT LIKE '%{op_value}%'" + ) # Check if key starts with "info." prefix (for simple equality) elif key.startswith("info."): # Extract the field name after "info." @@ -4756,6 +4786,7 @@ def parse_filter( "project_id", "manager_user_id", "delete_time", + "related_id", } def process_condition(condition): From 48f7320d4ac784e2e424ef013568cadc5e8ac0a6 Mon Sep 17 00:00:00 2001 From: HarveyXiang Date: Wed, 18 Mar 2026 16:33:34 +0800 Subject: [PATCH 07/11] feat: add test cov ci (#1272) * feat: add test cov ci * ci: update pr tpl * feat: update poetry * ci: update pr tpl --------- Co-authored-by: harvey_xiang --- .github/PULL_REQUEST_TEMPLATE.md | 2 +- .github/workflows/python-tests.yml | 8 +- .gitignore | 2 + Makefile | 17 ++- poetry.lock | 192 ++++++++++++++++++++++++++++- pyproject.toml | 19 +++ 6 files changed, 231 insertions(+), 9 deletions(-) diff --git a/.github/PULL_REQUEST_TEMPLATE.md b/.github/PULL_REQUEST_TEMPLATE.md index 1d21d77c2..4d0aada42 100644 --- a/.github/PULL_REQUEST_TEMPLATE.md +++ b/.github/PULL_REQUEST_TEMPLATE.md @@ -2,7 +2,7 @@ Please include a summary of the change, the problem it solves, the implementation approach, and relevant context. List any dependencies required for this change. -Related Issue (Required): Fixes @issue_number +Related Issue (Required): Fixes #issue_number ## Type of change diff --git a/.github/workflows/python-tests.yml b/.github/workflows/python-tests.yml index de300c193..1a1338408 100644 --- a/.github/workflows/python-tests.yml +++ b/.github/workflows/python-tests.yml @@ -102,7 +102,11 @@ jobs: if: ${{ !startsWith(matrix.os, 'macos-13') }} run: | poetry install --no-interaction --extras all - - name: PyTest unit tests + - name: PyTest unit tests with coverage if: ${{ !startsWith(matrix.os, 'macos-13') }} + shell: bash run: | - poetry run pytest tests -vv --durations=10 + poetry run pytest tests -vv --durations=10 \ + --cov=src/memos \ + --cov-report=term-missing \ + --cov-fail-under=28 diff --git a/.gitignore b/.gitignore index ece7e45ba..7d1be5a25 100644 --- a/.gitignore +++ b/.gitignore @@ -63,6 +63,8 @@ pip-delete-this-directory.txt # Unit test / coverage reports htmlcov/ +report/ +cov-report/ .tox/ .nox/ .coverage diff --git a/Makefile b/Makefile index 57ede5838..788504a73 100644 --- a/Makefile +++ b/Makefile @@ -1,4 +1,4 @@ -.PHONY: test +.PHONY: test test-report test-cov install: poetry install --extras all --with dev --with test @@ -9,10 +9,25 @@ clean: rm -rf .pytest_cache rm -rf .ruff_cache rm -rf tmp + rm -rf report cov-report + rm -f .coverage .coverage.* test: poetry run pytest tests +test-report: + poetry run pytest tests -vv --durations=10 \ + --html=report/index.html \ + --cov=src/memos \ + --cov-report=term-missing \ + --cov-report=html:cov-report/src + +test-cov: + poetry run pytest tests \ + --cov=src/memos \ + --cov-report=term-missing \ + --cov-report=html:cov-report/src + format: poetry run ruff check --fix poetry run ruff format diff --git a/poetry.lock b/poetry.lock index ba31d1a31..72049f025 100644 --- a/poetry.lock +++ b/poetry.lock @@ -1,4 +1,4 @@ -# This file is automatically @generated by Poetry 2.1.3 and should not be changed by hand. +# This file is automatically @generated by Poetry 2.3.2 and should not be changed by hand. [[package]] name = "absl-py" @@ -599,6 +599,128 @@ mypy = ["bokeh", "contourpy[bokeh,docs]", "docutils-stubs", "mypy (==1.15.0)", " test = ["Pillow", "contourpy[test-no-images]", "matplotlib"] test-no-images = ["pytest", "pytest-cov", "pytest-rerunfailures", "pytest-xdist", "wurlitzer"] +[[package]] +name = "coverage" +version = "7.13.5" +description = "Code coverage measurement for Python" +optional = false +python-versions = ">=3.10" +groups = ["test"] +files = [ + {file = "coverage-7.13.5-cp310-cp310-macosx_10_9_x86_64.whl", hash = "sha256:e0723d2c96324561b9aa76fb982406e11d93cdb388a7a7da2b16e04719cf7ca5"}, + {file = "coverage-7.13.5-cp310-cp310-macosx_11_0_arm64.whl", hash = "sha256:52f444e86475992506b32d4e5ca55c24fc88d73bcbda0e9745095b28ef4dc0cf"}, + {file = "coverage-7.13.5-cp310-cp310-manylinux1_i686.manylinux_2_28_i686.manylinux_2_5_i686.whl", hash = "sha256:704de6328e3d612a8f6c07000a878ff38181ec3263d5a11da1db294fa6a9bdf8"}, + {file = "coverage-7.13.5-cp310-cp310-manylinux1_x86_64.manylinux_2_28_x86_64.manylinux_2_5_x86_64.whl", hash = "sha256:a1a6d79a14e1ec1832cabc833898636ad5f3754a678ef8bb4908515208bf84f4"}, + {file = "coverage-7.13.5-cp310-cp310-manylinux2014_aarch64.manylinux_2_17_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:79060214983769c7ba3f0cee10b54c97609dca4d478fa1aa32b914480fd5738d"}, + {file = "coverage-7.13.5-cp310-cp310-manylinux2014_ppc64le.manylinux_2_17_ppc64le.manylinux_2_28_ppc64le.whl", hash = "sha256:356e76b46783a98c2a2fe81ec79df4883a1e62895ea952968fb253c114e7f930"}, + {file = "coverage-7.13.5-cp310-cp310-manylinux_2_31_riscv64.manylinux_2_39_riscv64.whl", hash = "sha256:0cef0cdec915d11254a7f549c1170afecce708d30610c6abdded1f74e581666d"}, + {file = "coverage-7.13.5-cp310-cp310-musllinux_1_2_aarch64.whl", hash = "sha256:dc022073d063b25a402454e5712ef9e007113e3a676b96c5f29b2bda29352f40"}, + {file = "coverage-7.13.5-cp310-cp310-musllinux_1_2_i686.whl", hash = "sha256:9b74db26dfea4f4e50d48a4602207cd1e78be33182bc9cbf22da94f332f99878"}, + {file = "coverage-7.13.5-cp310-cp310-musllinux_1_2_ppc64le.whl", hash = "sha256:ad146744ca4fd09b50c482650e3c1b1f4dfa1d4792e0a04a369c7f23336f0400"}, + {file = "coverage-7.13.5-cp310-cp310-musllinux_1_2_riscv64.whl", hash = "sha256:c555b48be1853fe3997c11c4bd521cdd9a9612352de01fa4508f16ec341e6fe0"}, + {file = "coverage-7.13.5-cp310-cp310-musllinux_1_2_x86_64.whl", hash = "sha256:7034b5c56a58ae5e85f23949d52c14aca2cfc6848a31764995b7de88f13a1ea0"}, + {file = "coverage-7.13.5-cp310-cp310-win32.whl", hash = "sha256:eb7fdf1ef130660e7415e0253a01a7d5a88c9c4d158bcf75cbbd922fd65a5b58"}, + {file = "coverage-7.13.5-cp310-cp310-win_amd64.whl", hash = "sha256:3e1bb5f6c78feeb1be3475789b14a0f0a5b47d505bfc7267126ccbd50289999e"}, + {file = "coverage-7.13.5-cp311-cp311-macosx_10_9_x86_64.whl", hash = "sha256:66a80c616f80181f4d643b0f9e709d97bcea413ecd9631e1dedc7401c8e6695d"}, + {file = "coverage-7.13.5-cp311-cp311-macosx_11_0_arm64.whl", hash = "sha256:145ede53ccbafb297c1c9287f788d1bc3efd6c900da23bf6931b09eafc931587"}, + {file = "coverage-7.13.5-cp311-cp311-manylinux1_i686.manylinux_2_28_i686.manylinux_2_5_i686.whl", hash = "sha256:0672854dc733c342fa3e957e0605256d2bf5934feeac328da9e0b5449634a642"}, + {file = "coverage-7.13.5-cp311-cp311-manylinux1_x86_64.manylinux_2_28_x86_64.manylinux_2_5_x86_64.whl", hash = "sha256:ec10e2a42b41c923c2209b846126c6582db5e43a33157e9870ba9fb70dc7854b"}, + {file = "coverage-7.13.5-cp311-cp311-manylinux2014_aarch64.manylinux_2_17_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:be3d4bbad9d4b037791794ddeedd7d64a56f5933a2c1373e18e9e568b9141686"}, + {file = "coverage-7.13.5-cp311-cp311-manylinux2014_ppc64le.manylinux_2_17_ppc64le.manylinux_2_28_ppc64le.whl", hash = "sha256:4d2afbc5cc54d286bfb54541aa50b64cdb07a718227168c87b9e2fb8f25e1743"}, + {file = "coverage-7.13.5-cp311-cp311-manylinux_2_31_riscv64.manylinux_2_39_riscv64.whl", hash = "sha256:3ad050321264c49c2fa67bb599100456fc51d004b82534f379d16445da40fb75"}, + {file = "coverage-7.13.5-cp311-cp311-musllinux_1_2_aarch64.whl", hash = "sha256:7300c8a6d13335b29bb76d7651c66af6bd8658517c43499f110ddc6717bfc209"}, + {file = "coverage-7.13.5-cp311-cp311-musllinux_1_2_i686.whl", hash = "sha256:eb07647a5738b89baab047f14edd18ded523de60f3b30e75c2acc826f79c839a"}, + {file = "coverage-7.13.5-cp311-cp311-musllinux_1_2_ppc64le.whl", hash = "sha256:9adb6688e3b53adffefd4a52d72cbd8b02602bfb8f74dcd862337182fd4d1a4e"}, + {file = "coverage-7.13.5-cp311-cp311-musllinux_1_2_riscv64.whl", hash = "sha256:7c8d4bc913dd70b93488d6c496c77f3aff5ea99a07e36a18f865bca55adef8bd"}, + {file = "coverage-7.13.5-cp311-cp311-musllinux_1_2_x86_64.whl", hash = "sha256:0e3c426ffc4cd952f54ee9ffbdd10345709ecc78a3ecfd796a57236bfad0b9b8"}, + {file = "coverage-7.13.5-cp311-cp311-win32.whl", hash = "sha256:259b69bb83ad9894c4b25be2528139eecba9a82646ebdda2d9db1ba28424a6bf"}, + {file = "coverage-7.13.5-cp311-cp311-win_amd64.whl", hash = "sha256:258354455f4e86e3e9d0d17571d522e13b4e1e19bf0f8596bcf9476d61e7d8a9"}, + {file = "coverage-7.13.5-cp311-cp311-win_arm64.whl", hash = "sha256:bff95879c33ec8da99fc9b6fe345ddb5be6414b41d6d1ad1c8f188d26f36e028"}, + {file = "coverage-7.13.5-cp312-cp312-macosx_10_13_x86_64.whl", hash = "sha256:460cf0114c5016fa841214ff5564aa4864f11948da9440bc97e21ad1f4ba1e01"}, + {file = "coverage-7.13.5-cp312-cp312-macosx_11_0_arm64.whl", hash = "sha256:0e223ce4b4ed47f065bfb123687686512e37629be25cc63728557ae7db261422"}, + {file = "coverage-7.13.5-cp312-cp312-manylinux1_i686.manylinux_2_28_i686.manylinux_2_5_i686.whl", hash = "sha256:6e3370441f4513c6252bf042b9c36d22491142385049243253c7e48398a15a9f"}, + {file = "coverage-7.13.5-cp312-cp312-manylinux1_x86_64.manylinux_2_28_x86_64.manylinux_2_5_x86_64.whl", hash = "sha256:03ccc709a17a1de074fb1d11f217342fb0d2b1582ed544f554fc9fc3f07e95f5"}, + {file = "coverage-7.13.5-cp312-cp312-manylinux2014_aarch64.manylinux_2_17_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:3f4818d065964db3c1c66dc0fbdac5ac692ecbc875555e13374fdbe7eedb4376"}, + {file = "coverage-7.13.5-cp312-cp312-manylinux2014_ppc64le.manylinux_2_17_ppc64le.manylinux_2_28_ppc64le.whl", hash = "sha256:012d5319e66e9d5a218834642d6c35d265515a62f01157a45bcc036ecf947256"}, + {file = "coverage-7.13.5-cp312-cp312-manylinux_2_31_riscv64.manylinux_2_39_riscv64.whl", hash = "sha256:8dd02af98971bdb956363e4827d34425cb3df19ee550ef92855b0acb9c7ce51c"}, + {file = "coverage-7.13.5-cp312-cp312-musllinux_1_2_aarch64.whl", hash = "sha256:f08fd75c50a760c7eb068ae823777268daaf16a80b918fa58eea888f8e3919f5"}, + {file = "coverage-7.13.5-cp312-cp312-musllinux_1_2_i686.whl", hash = "sha256:843ea8643cf967d1ac7e8ecd4bb00c99135adf4816c0c0593fdcc47b597fcf09"}, + {file = "coverage-7.13.5-cp312-cp312-musllinux_1_2_ppc64le.whl", hash = "sha256:9d44d7aa963820b1b971dbecd90bfe5fe8f81cff79787eb6cca15750bd2f79b9"}, + {file = "coverage-7.13.5-cp312-cp312-musllinux_1_2_riscv64.whl", hash = "sha256:7132bed4bd7b836200c591410ae7d97bf7ae8be6fc87d160b2bd881df929e7bf"}, + {file = "coverage-7.13.5-cp312-cp312-musllinux_1_2_x86_64.whl", hash = "sha256:a698e363641b98843c517817db75373c83254781426e94ada3197cabbc2c919c"}, + {file = "coverage-7.13.5-cp312-cp312-win32.whl", hash = "sha256:bdba0a6b8812e8c7df002d908a9a2ea3c36e92611b5708633c50869e6d922fdf"}, + {file = "coverage-7.13.5-cp312-cp312-win_amd64.whl", hash = "sha256:d2c87e0c473a10bffe991502eac389220533024c8082ec1ce849f4218dded810"}, + {file = "coverage-7.13.5-cp312-cp312-win_arm64.whl", hash = "sha256:bf69236a9a81bdca3bff53796237aab096cdbf8d78a66ad61e992d9dac7eb2de"}, + {file = "coverage-7.13.5-cp313-cp313-macosx_10_13_x86_64.whl", hash = "sha256:5ec4af212df513e399cf11610cc27063f1586419e814755ab362e50a85ea69c1"}, + {file = "coverage-7.13.5-cp313-cp313-macosx_11_0_arm64.whl", hash = "sha256:941617e518602e2d64942c88ec8499f7fbd49d3f6c4327d3a71d43a1973032f3"}, + {file = "coverage-7.13.5-cp313-cp313-manylinux1_i686.manylinux_2_28_i686.manylinux_2_5_i686.whl", hash = "sha256:da305e9937617ee95c2e39d8ff9f040e0487cbf1ac174f777ed5eddd7a7c1f26"}, + {file = "coverage-7.13.5-cp313-cp313-manylinux1_x86_64.manylinux_2_28_x86_64.manylinux_2_5_x86_64.whl", hash = "sha256:78e696e1cc714e57e8b25760b33a8b1026b7048d270140d25dafe1b0a1ee05a3"}, + {file = "coverage-7.13.5-cp313-cp313-manylinux2014_aarch64.manylinux_2_17_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:02ca0eed225b2ff301c474aeeeae27d26e2537942aa0f87491d3e147e784a82b"}, + {file = "coverage-7.13.5-cp313-cp313-manylinux2014_ppc64le.manylinux_2_17_ppc64le.manylinux_2_28_ppc64le.whl", hash = "sha256:04690832cbea4e4663d9149e05dba142546ca05cb1848816760e7f58285c970a"}, + {file = "coverage-7.13.5-cp313-cp313-manylinux_2_31_riscv64.manylinux_2_39_riscv64.whl", hash = "sha256:0590e44dd2745c696a778f7bab6aa95256de2cbc8b8cff4f7db8ff09813d6969"}, + {file = "coverage-7.13.5-cp313-cp313-musllinux_1_2_aarch64.whl", hash = "sha256:d7cfad2d6d81dd298ab6b89fe72c3b7b05ec7544bdda3b707ddaecff8d25c161"}, + {file = "coverage-7.13.5-cp313-cp313-musllinux_1_2_i686.whl", hash = "sha256:e092b9499de38ae0fbfbc603a74660eb6ff3e869e507b50d85a13b6db9863e15"}, + {file = "coverage-7.13.5-cp313-cp313-musllinux_1_2_ppc64le.whl", hash = "sha256:48c39bc4a04d983a54a705a6389512883d4a3b9862991b3617d547940e9f52b1"}, + {file = "coverage-7.13.5-cp313-cp313-musllinux_1_2_riscv64.whl", hash = "sha256:2d3807015f138ffea1ed9afeeb8624fd781703f2858b62a8dd8da5a0994c57b6"}, + {file = "coverage-7.13.5-cp313-cp313-musllinux_1_2_x86_64.whl", hash = "sha256:ee2aa19e03161671ec964004fb74b2257805d9710bf14a5c704558b9d8dbaf17"}, + {file = "coverage-7.13.5-cp313-cp313-win32.whl", hash = "sha256:ce1998c0483007608c8382f4ff50164bfc5bd07a2246dd272aa4043b75e61e85"}, + {file = "coverage-7.13.5-cp313-cp313-win_amd64.whl", hash = "sha256:631efb83f01569670a5e866ceb80fe483e7c159fac6f167e6571522636104a0b"}, + {file = "coverage-7.13.5-cp313-cp313-win_arm64.whl", hash = "sha256:f4cd16206ad171cbc2470dbea9103cf9a7607d5fe8c242fdf1edf36174020664"}, + {file = "coverage-7.13.5-cp313-cp313t-macosx_10_13_x86_64.whl", hash = "sha256:0428cbef5783ad91fe240f673cc1f76b25e74bbfe1a13115e4aa30d3f538162d"}, + {file = "coverage-7.13.5-cp313-cp313t-macosx_11_0_arm64.whl", hash = "sha256:e0b216a19534b2427cc201a26c25da4a48633f29a487c61258643e89d28200c0"}, + {file = "coverage-7.13.5-cp313-cp313t-manylinux1_i686.manylinux_2_28_i686.manylinux_2_5_i686.whl", hash = "sha256:972a9cd27894afe4bc2b1480107054e062df08e671df7c2f18c205e805ccd806"}, + {file = "coverage-7.13.5-cp313-cp313t-manylinux1_x86_64.manylinux_2_28_x86_64.manylinux_2_5_x86_64.whl", hash = "sha256:4b59148601efcd2bac8c4dbf1f0ad6391693ccf7a74b8205781751637076aee3"}, + {file = "coverage-7.13.5-cp313-cp313t-manylinux2014_aarch64.manylinux_2_17_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:505d7083c8b0c87a8fa8c07370c285847c1f77739b22e299ad75a6af6c32c5c9"}, + {file = "coverage-7.13.5-cp313-cp313t-manylinux2014_ppc64le.manylinux_2_17_ppc64le.manylinux_2_28_ppc64le.whl", hash = "sha256:60365289c3741e4db327e7baff2a4aaacf22f788e80fa4683393891b70a89fbd"}, + {file = "coverage-7.13.5-cp313-cp313t-manylinux_2_31_riscv64.manylinux_2_39_riscv64.whl", hash = "sha256:1b88c69c8ef5d4b6fe7dea66d6636056a0f6a7527c440e890cf9259011f5e606"}, + {file = "coverage-7.13.5-cp313-cp313t-musllinux_1_2_aarch64.whl", hash = "sha256:5b13955d31d1633cf9376908089b7cebe7d15ddad7aeaabcbe969a595a97e95e"}, + {file = "coverage-7.13.5-cp313-cp313t-musllinux_1_2_i686.whl", hash = "sha256:f70c9ab2595c56f81a89620e22899eea8b212a4041bd728ac6f4a28bf5d3ddd0"}, + {file = "coverage-7.13.5-cp313-cp313t-musllinux_1_2_ppc64le.whl", hash = "sha256:084b84a8c63e8d6fc7e3931b316a9bcafca1458d753c539db82d31ed20091a87"}, + {file = "coverage-7.13.5-cp313-cp313t-musllinux_1_2_riscv64.whl", hash = "sha256:ad14385487393e386e2ea988b09d62dd42c397662ac2dabc3832d71253eee479"}, + {file = "coverage-7.13.5-cp313-cp313t-musllinux_1_2_x86_64.whl", hash = "sha256:7f2c47b36fe7709a6e83bfadf4eefb90bd25fbe4014d715224c4316f808e59a2"}, + {file = "coverage-7.13.5-cp313-cp313t-win32.whl", hash = "sha256:67e9bc5449801fad0e5dff329499fb090ba4c5800b86805c80617b4e29809b2a"}, + {file = "coverage-7.13.5-cp313-cp313t-win_amd64.whl", hash = "sha256:da86cdcf10d2519e10cabb8ac2de03da1bcb6e4853790b7fbd48523332e3a819"}, + {file = "coverage-7.13.5-cp313-cp313t-win_arm64.whl", hash = "sha256:0ecf12ecb326fe2c339d93fc131816f3a7367d223db37817208905c89bded911"}, + {file = "coverage-7.13.5-cp314-cp314-macosx_10_15_x86_64.whl", hash = "sha256:fbabfaceaeb587e16f7008f7795cd80d20ec548dc7f94fbb0d4ec2e038ce563f"}, + {file = "coverage-7.13.5-cp314-cp314-macosx_11_0_arm64.whl", hash = "sha256:9bb2a28101a443669a423b665939381084412b81c3f8c0fcfbac57f4e30b5b8e"}, + {file = "coverage-7.13.5-cp314-cp314-manylinux1_i686.manylinux_2_28_i686.manylinux_2_5_i686.whl", hash = "sha256:bd3a2fbc1c6cccb3c5106140d87cc6a8715110373ef42b63cf5aea29df8c217a"}, + {file = "coverage-7.13.5-cp314-cp314-manylinux1_x86_64.manylinux_2_28_x86_64.manylinux_2_5_x86_64.whl", hash = "sha256:6c36ddb64ed9d7e496028d1d00dfec3e428e0aabf4006583bb1839958d280510"}, + {file = "coverage-7.13.5-cp314-cp314-manylinux2014_aarch64.manylinux_2_17_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:380e8e9084d8eb38db3a9176a1a4f3c0082c3806fa0dc882d1d87abc3c789247"}, + {file = "coverage-7.13.5-cp314-cp314-manylinux2014_ppc64le.manylinux_2_17_ppc64le.manylinux_2_28_ppc64le.whl", hash = "sha256:e808af52a0513762df4d945ea164a24b37f2f518cbe97e03deaa0ee66139b4d6"}, + {file = "coverage-7.13.5-cp314-cp314-manylinux_2_31_riscv64.manylinux_2_39_riscv64.whl", hash = "sha256:e301d30dd7e95ae068671d746ba8c34e945a82682e62918e41b2679acd2051a0"}, + {file = "coverage-7.13.5-cp314-cp314-musllinux_1_2_aarch64.whl", hash = "sha256:800bc829053c80d240a687ceeb927a94fd108bbdc68dfbe505d0d75ab578a882"}, + {file = "coverage-7.13.5-cp314-cp314-musllinux_1_2_i686.whl", hash = "sha256:0b67af5492adb31940ee418a5a655c28e48165da5afab8c7fa6fd72a142f8740"}, + {file = "coverage-7.13.5-cp314-cp314-musllinux_1_2_ppc64le.whl", hash = "sha256:c9136ff29c3a91e25b1d1552b5308e53a1e0653a23e53b6366d7c2dcbbaf8a16"}, + {file = "coverage-7.13.5-cp314-cp314-musllinux_1_2_riscv64.whl", hash = "sha256:cff784eef7f0b8f6cb28804fbddcfa99f89efe4cc35fb5627e3ac58f91ed3ac0"}, + {file = "coverage-7.13.5-cp314-cp314-musllinux_1_2_x86_64.whl", hash = "sha256:68a4953be99b17ac3c23b6efbc8a38330d99680c9458927491d18700ef23ded0"}, + {file = "coverage-7.13.5-cp314-cp314-win32.whl", hash = "sha256:35a31f2b1578185fbe6aa2e74cea1b1d0bbf4c552774247d9160d29b80ed56cc"}, + {file = "coverage-7.13.5-cp314-cp314-win_amd64.whl", hash = "sha256:2aa055ae1857258f9e0045be26a6d62bdb47a72448b62d7b55f4820f361a2633"}, + {file = "coverage-7.13.5-cp314-cp314-win_arm64.whl", hash = "sha256:1b11eef33edeae9d142f9b4358edb76273b3bfd30bc3df9a4f95d0e49caf94e8"}, + {file = "coverage-7.13.5-cp314-cp314t-macosx_10_15_x86_64.whl", hash = "sha256:10a0c37f0b646eaff7cce1874c31d1f1ccb297688d4c747291f4f4c70741cc8b"}, + {file = "coverage-7.13.5-cp314-cp314t-macosx_11_0_arm64.whl", hash = "sha256:b5db73ba3c41c7008037fa731ad5459fc3944cb7452fc0aa9f822ad3533c583c"}, + {file = "coverage-7.13.5-cp314-cp314t-manylinux1_i686.manylinux_2_28_i686.manylinux_2_5_i686.whl", hash = "sha256:750db93a81e3e5a9831b534be7b1229df848b2e125a604fe6651e48aa070e5f9"}, + {file = "coverage-7.13.5-cp314-cp314t-manylinux1_x86_64.manylinux_2_28_x86_64.manylinux_2_5_x86_64.whl", hash = "sha256:9ddb4f4a5479f2539644be484da179b653273bca1a323947d48ab107b3ed1f29"}, + {file = "coverage-7.13.5-cp314-cp314t-manylinux2014_aarch64.manylinux_2_17_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:d8a7a2049c14f413163e2bdabd37e41179b1d1ccb10ffc6ccc4b7a718429c607"}, + {file = "coverage-7.13.5-cp314-cp314t-manylinux2014_ppc64le.manylinux_2_17_ppc64le.manylinux_2_28_ppc64le.whl", hash = "sha256:e1c85e0b6c05c592ea6d8768a66a254bfb3874b53774b12d4c89c481eb78cb90"}, + {file = "coverage-7.13.5-cp314-cp314t-manylinux_2_31_riscv64.manylinux_2_39_riscv64.whl", hash = "sha256:777c4d1eff1b67876139d24288aaf1817f6c03d6bae9c5cc8d27b83bcfe38fe3"}, + {file = "coverage-7.13.5-cp314-cp314t-musllinux_1_2_aarch64.whl", hash = "sha256:6697e29b93707167687543480a40f0db8f356e86d9f67ddf2e37e2dfd91a9dab"}, + {file = "coverage-7.13.5-cp314-cp314t-musllinux_1_2_i686.whl", hash = "sha256:8fdf453a942c3e4d99bd80088141c4c6960bb232c409d9c3558e2dbaa3998562"}, + {file = "coverage-7.13.5-cp314-cp314t-musllinux_1_2_ppc64le.whl", hash = "sha256:32ca0c0114c9834a43f045a87dcebd69d108d8ffb666957ea65aa132f50332e2"}, + {file = "coverage-7.13.5-cp314-cp314t-musllinux_1_2_riscv64.whl", hash = "sha256:8769751c10f339021e2638cd354e13adeac54004d1941119b2c96fe5276d45ea"}, + {file = "coverage-7.13.5-cp314-cp314t-musllinux_1_2_x86_64.whl", hash = "sha256:cec2d83125531bd153175354055cdb7a09987af08a9430bd173c937c6d0fba2a"}, + {file = "coverage-7.13.5-cp314-cp314t-win32.whl", hash = "sha256:0cd9ed7a8b181775459296e402ca4fb27db1279740a24e93b3b41942ebe4b215"}, + {file = "coverage-7.13.5-cp314-cp314t-win_amd64.whl", hash = "sha256:301e3b7dfefecaca37c9f1aa6f0049b7d4ab8dd933742b607765d757aca77d43"}, + {file = "coverage-7.13.5-cp314-cp314t-win_arm64.whl", hash = "sha256:9dacc2ad679b292709e0f5fc1ac74a6d4d5562e424058962c7bb0c658ad25e45"}, + {file = "coverage-7.13.5-py3-none-any.whl", hash = "sha256:34b02417cf070e173989b3db962f7ed56d2f644307b2cf9d5a0f258e13084a61"}, + {file = "coverage-7.13.5.tar.gz", hash = "sha256:c81f6515c4c40141f83f502b07bbfa5c240ba25bbe73da7b33f1e5b6120ff179"}, +] + +[package.dependencies] +tomli = {version = "*", optional = true, markers = "python_full_version <= \"3.11.0a6\" and extra == \"toml\""} + +[package.extras] +toml = ["tomli ; python_full_version <= \"3.11.0a6\""] + [[package]] name = "crcmod-plus" version = "2.3.1" @@ -1621,7 +1743,7 @@ version = "3.1.6" description = "A very fast and expressive template engine." optional = false python-versions = ">=3.7" -groups = ["main", "eval"] +groups = ["main", "eval", "test"] files = [ {file = "jinja2-3.1.6-py3-none-any.whl", hash = "sha256:85ece4451f492d0c13c5dd7c13a64681a86afae63a5f347908daf103ce6d2f67"}, {file = "jinja2-3.1.6.tar.gz", hash = "sha256:0137fb05990d35f1275a587e9aee6d56da821fc83491a0fb838183be43f66d6d"}, @@ -1775,7 +1897,7 @@ files = [ [package.dependencies] attrs = ">=22.2.0" -jsonschema-specifications = ">=2023.03.6" +jsonschema-specifications = ">=2023.3.6" referencing = ">=0.28.4" rpds-py = ">=0.7.1" @@ -2263,7 +2385,7 @@ version = "3.0.2" description = "Safely add untrusted strings to HTML/XML markup." optional = false python-versions = ">=3.9" -groups = ["main", "eval"] +groups = ["main", "eval", "test"] files = [ {file = "MarkupSafe-3.0.2-cp310-cp310-macosx_10_9_universal2.whl", hash = "sha256:7e94c425039cde14257288fd61dcfb01963e658efbc0ff54f5306b06054700f8"}, {file = "MarkupSafe-3.0.2-cp310-cp310-macosx_11_0_arm64.whl", hash = "sha256:9e2d922824181480953426608b81967de705c3cef4d1af983af849d7bd619158"}, @@ -2550,6 +2672,7 @@ files = [ {file = "nltk-3.9.1-py3-none-any.whl", hash = "sha256:4fa26829c5b00715afe3061398a8989dc643b92ce7dd93fb4585a70930d168a1"}, {file = "nltk-3.9.1.tar.gz", hash = "sha256:87d127bd3de4bd89a4f81265e5fa59cb1b199b27440175370f7417d2bc7ae868"}, ] +markers = {main = "extra == \"all\""} [package.dependencies] click = "*" @@ -3961,6 +4084,65 @@ pytest = ">=7.0.0,<9" docs = ["sphinx (>=5.3)", "sphinx-rtd-theme (>=1.0)"] testing = ["coverage (>=6.2)", "hypothesis (>=5.7.1)"] +[[package]] +name = "pytest-cov" +version = "6.3.0" +description = "Pytest plugin for measuring coverage." +optional = false +python-versions = ">=3.9" +groups = ["test"] +files = [ + {file = "pytest_cov-6.3.0-py3-none-any.whl", hash = "sha256:440db28156d2468cafc0415b4f8e50856a0d11faefa38f30906048fe490f1749"}, + {file = "pytest_cov-6.3.0.tar.gz", hash = "sha256:35c580e7800f87ce892e687461166e1ac2bcb8fb9e13aea79032518d6e503ff2"}, +] + +[package.dependencies] +coverage = {version = ">=7.5", extras = ["toml"]} +pluggy = ">=1.2" +pytest = ">=6.2.5" + +[package.extras] +testing = ["fields", "hunter", "process-tests", "pytest-xdist", "virtualenv"] + +[[package]] +name = "pytest-html" +version = "4.2.0" +description = "pytest plugin for generating HTML reports" +optional = false +python-versions = ">=3.9" +groups = ["test"] +files = [ + {file = "pytest_html-4.2.0-py3-none-any.whl", hash = "sha256:ff5caf3e17a974008e5816edda61168e6c3da442b078a44f8744865862a85636"}, + {file = "pytest_html-4.2.0.tar.gz", hash = "sha256:b6a88cba507500d8709959201e2e757d3941e859fd17cfd4ed87b16fc0c67912"}, +] + +[package.dependencies] +jinja2 = ">=3" +pytest = ">=7" +pytest-metadata = ">=2" + +[package.extras] +docs = ["pip-tools (>=6.13)"] +test = ["assertpy (>=1.1)", "beautifulsoup4 (>=4.11.1)", "black (>=22.1)", "flake8 (>=4.0.1)", "pre-commit (>=2.17)", "pytest-mock (>=3.7)", "pytest-rerunfailures (>=11.1.2)", "pytest-xdist (>=2.4)", "selenium (>=4.3)", "tox (>=3.24.5)"] + +[[package]] +name = "pytest-metadata" +version = "3.1.1" +description = "pytest plugin for test session metadata" +optional = false +python-versions = ">=3.8" +groups = ["test"] +files = [ + {file = "pytest_metadata-3.1.1-py3-none-any.whl", hash = "sha256:c8e0844db684ee1c798cfa38908d20d67d0463ecb6137c72e91f418558dd5f4b"}, + {file = "pytest_metadata-3.1.1.tar.gz", hash = "sha256:d2a29b0355fbc03f168aa96d41ff88b1a3b44a3b02acbe491801c98a048017c8"}, +] + +[package.dependencies] +pytest = ">=7.0.0" + +[package.extras] +test = ["black (>=22.1.0)", "flake8 (>=4.0.1)", "pre-commit (>=2.17.0)", "tox (>=3.24.5)"] + [[package]] name = "python-dateutil" version = "2.9.0.post0" @@ -6373,4 +6555,4 @@ tree-mem = ["neo4j", "schedule"] [metadata] lock-version = "2.1" python-versions = ">=3.10,<4.0" -content-hash = "faff240c05a74263a404e8d9324ffd2f342cb4f0a4c1f5455b87349f6ccc61a5" +content-hash = "e0427aa672e57215033fe964847474521abf61b0a63f443744a6ec0b8c5ff2e2" diff --git a/pyproject.toml b/pyproject.toml index 9f17c0000..ff7c9699a 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -170,6 +170,8 @@ optional = true [tool.poetry.group.test.dependencies] pytest = "^8.3.5" pytest-asyncio = "^0.23.5" +pytest-cov = "^6.1" +pytest-html = "^4.2" ruff = "^0.11.8" [tool.poetry.group.eval] @@ -208,6 +210,23 @@ filterwarnings = [ ] +[tool.coverage.run] +source = ["src/memos"] +branch = true + +[tool.coverage.report] +show_missing = true +skip_empty = true +exclude_lines = [ + "pragma: no cover", + "if TYPE_CHECKING:", + "if __name__ == .__main__.", +] + +[tool.coverage.html] +directory = "cov-report" + + [tool.ruff] ############################################################################## # Ruff is a fast Python linter and formatter. From cca474552c3440684718b0e1055b3ee7697f0a36 Mon Sep 17 00:00:00 2001 From: Hustzdy <67457465+wustzdy@users.noreply.github.com> Date: Wed, 18 Mar 2026 19:09:51 +0800 Subject: [PATCH 08/11] feat:optimzie search_by_embedding && logs (#1284) * fix: optimize search_by_embedding * fix: optimize search_by_embedding * feat:optimize get_grouped_counts * feat:optimize get_grouped_counts * feat:optimize get_by_metadata * fix: remove self._refresh_memory_size --- src/memos/graph_dbs/polardb.py | 132 +++++++----------- .../mem_scheduler/schemas/general_schemas.py | 2 +- .../tree_text_memory/organize/manager.py | 1 - 3 files changed, 54 insertions(+), 81 deletions(-) diff --git a/src/memos/graph_dbs/polardb.py b/src/memos/graph_dbs/polardb.py index 6db31990d..4d88844df 100644 --- a/src/memos/graph_dbs/polardb.py +++ b/src/memos/graph_dbs/polardb.py @@ -432,14 +432,13 @@ def node_not_exist(self, scope: str, user_name: str | None = None) -> int: def remove_oldest_memory( self, memory_type: str, keep_latest: int, user_name: str | None = None ) -> None: - """ - Remove all WorkingMemory nodes except the latest `keep_latest` entries. - - Args: - memory_type (str): Memory type (e.g., 'WorkingMemory', 'LongTermMemory'). - keep_latest (int): Number of latest WorkingMemory entries to keep. - user_name (str, optional): User name for filtering in non-multi-db mode - """ + start_time = time.perf_counter() + logger.info( + "remove_oldest_memory by memory_type:%s,keep_latest: %s,user_name:%s", + memory_type, + keep_latest, + user_name, + ) user_name = user_name if user_name else self._get_config_value("user_name") # Use actual OFFSET logic, consistent with nebular.py @@ -456,6 +455,9 @@ def remove_oldest_memory( self.format_param_value(user_name), keep_latest, ] + logger.info( + f"remove_oldest_memory by select_query:{select_query},select_params:{select_params}" + ) try: with self._get_connection() as conn, conn.cursor() as cursor: # Execute query to get IDs to delete @@ -482,6 +484,8 @@ def remove_oldest_memory( f"keeping {keep_latest} latest for user {user_name}, " f"removed ids: {ids_to_delete}" ) + elapsed = (time.perf_counter() - start_time) * 1000.0 + logger.info("remove_oldest_memory internal took %.1f ms", elapsed) except Exception as e: logger.error(f"[remove_oldest_memory] Failed: {e}", exc_info=True) raise @@ -1840,9 +1844,8 @@ def search_by_embedding( **kwargs, ) -> list[dict]: logger.info( - "search_by_embedding user_name:%s,filter: %s, knowledgebase_ids: %s,scope:%s,status:%s,search_filter:%s,filter:%s,knowledgebase_ids:%s,return_fields:%s", + "search_by_embedding by user_name:%s,knowledgebase_ids: %s,scope:%s,status:%s,search_filter:%s,filter:%s,knowledgebase_ids:%s,return_fields:%s", user_name, - filter, knowledgebase_ids, scope, status, @@ -1895,20 +1898,21 @@ def search_by_embedding( where_clause = f"WHERE {' AND '.join(where_clauses)}" if where_clauses else "" query = f""" + set hnsw.ef_search = 100;set hnsw.iterative_scan = relaxed_order; WITH t AS ( SELECT id, properties, timeline, ag_catalog.agtype_access_operator(properties, '"id"'::agtype) AS old_id, - (1 - (embedding <=> %s::vector(1024))) AS scope + (embedding <=> %s::vector(1024)) AS scope_distance FROM "{self.db_name}_graph"."Memory" {where_clause} - ORDER BY scope DESC + ORDER BY scope_distance ASC LIMIT {top_k} ) - SELECT * + SELECT *,(1 - scope_distance) AS scope FROM t - WHERE scope > 0.1; + WHERE scope_distance < 0.9; """ vector_str = convert_to_vector(vector) query = query.replace("%s::vector(1024)", f"'{vector_str}'::vector(1024)") @@ -1953,7 +1957,7 @@ def search_by_embedding( output.append(item) elapsed_time = (time.perf_counter() - start_time) * 1000.0 logger.info( - "search_by_embedding query embedding completed time took %.1f ms", elapsed_time + "search_by_embedding query by embedding completed time took %.1f ms", elapsed_time ) return output[:top_k] @@ -1966,30 +1970,13 @@ def get_by_metadata( knowledgebase_ids: list | None = None, user_name_flag: bool = True, ) -> list[str]: - """ - Retrieve node IDs that match given metadata filters. - Supports exact match. - - Args: - filters: List of filter dicts like: - [ - {"field": "key", "op": "in", "value": ["A", "B"]}, - {"field": "confidence", "op": ">=", "value": 80}, - {"field": "tags", "op": "contains", "value": "AI"}, - ... - ] - user_name (str, optional): User name for filtering in non-multi-db mode - - Returns: - list[str]: Node IDs whose metadata match the filter conditions. (AND logic). - """ + start_time = time.perf_counter() logger.info( f" get_by_metadata user_name:{user_name},filter: {filter}, knowledgebase_ids: {knowledgebase_ids},filters:{filters}" ) user_name = user_name if user_name else self._get_config_value("user_name") - # Build WHERE conditions for cypher query where_conditions = [] for f in filters: @@ -1997,18 +1984,13 @@ def get_by_metadata( op = f.get("op", "=") value = f["value"] - # Format value if isinstance(value, str): - # Escape single quotes using backslash when inside $$ dollar-quoted strings - # In $$ delimiters, Cypher string literals can use \' to escape single quotes escaped_str = value.replace("'", "\\'") escaped_value = f"'{escaped_str}'" elif isinstance(value, list): - # Handle list values - use double quotes for Cypher arrays list_items = [] for v in value: if isinstance(v, str): - # Escape double quotes in string values for Cypher escaped_str = v.replace('"', '\\"') list_items.append(f'"{escaped_str}"') else: @@ -2016,7 +1998,6 @@ def get_by_metadata( escaped_value = f"[{', '.join(list_items)}]" else: escaped_value = f"'{value}'" if isinstance(value, str) else str(value) - # Build WHERE conditions if op == "=": where_conditions.append(f"n.{field} = {escaped_value}") elif op == "in": @@ -2045,22 +2026,19 @@ def get_by_metadata( knowledgebase_ids=knowledgebase_ids, default_user_name=self._get_config_value("user_name"), ) - logger.info(f"[get_by_metadata] user_name_conditions: {user_name_conditions}") + logger.info(f"get_by_metadata user_name_conditions: {user_name_conditions}") - # Add user_name WHERE clause if user_name_conditions: if len(user_name_conditions) == 1: where_conditions.append(user_name_conditions[0]) else: where_conditions.append(f"({' OR '.join(user_name_conditions)})") - # Build filter conditions using common method filter_where_clause = self._build_filter_conditions_cypher(filter) - logger.info(f"[get_by_metadata] filter_where_clause: {filter_where_clause}") + logger.info(f"get_by_metadata filter_where_clause: {filter_where_clause}") where_str = " AND ".join(where_conditions) + filter_where_clause - # Use cypher query cypher_query = f""" SELECT * FROM cypher('{self.db_name}_graph', $$ MATCH (n:Memory) @@ -2070,7 +2048,7 @@ def get_by_metadata( """ ids = [] - logger.info(f"[get_by_metadata] cypher_query: {cypher_query}") + logger.info(f"get_by_metadata cypher_query: {cypher_query}") try: with self._get_connection() as conn, conn.cursor() as cursor: cursor.execute(cypher_query) @@ -2078,7 +2056,8 @@ def get_by_metadata( ids = [str(item[0]).strip('"') for item in results] except Exception as e: logger.warning(f"Failed to get metadata: {e}, query is {cypher_query}") - + elapsed = (time.perf_counter() - start_time) * 1000.0 + logger.info("get_by_metadata internal took %.1f ms", elapsed) return ids @timed @@ -2165,25 +2144,19 @@ def get_grouped_counts( params: dict[str, Any] | None = None, user_name: str | None = None, ) -> list[dict[str, Any]]: - """ - Count nodes grouped by any fields. - - Args: - group_fields (list[str]): Fields to group by, e.g., ["memory_type", "status"] - where_clause (str, optional): Extra WHERE condition. E.g., - "WHERE n.status = 'activated'" - params (dict, optional): Parameters for WHERE clause. - user_name (str, optional): User name for filtering in non-multi-db mode - - Returns: - list[dict]: e.g., [{ 'memory_type': 'WorkingMemory', 'status': 'active', 'count': 10 }, ...] - """ + start_time = time.perf_counter() + logger.info( + "get_grouped_counts by group_fields:%s,where_clause: %s,params:%s,user_name:%s", + group_fields, + where_clause, + params, + user_name, + ) if not group_fields: raise ValueError("group_fields cannot be empty") user_name = user_name if user_name else self._get_config_value("user_name") - # Build user clause user_clause = f"ag_catalog.agtype_access_operator(properties, '\"user_name\"'::agtype) = '\"{user_name}\"'::agtype" if where_clause: where_clause = where_clause.strip() @@ -2194,44 +2167,43 @@ def get_grouped_counts( else: where_clause = f"WHERE {user_clause}" - # Inline parameters if provided if params and isinstance(params, dict): for key, value in params.items(): - # Handle different value types appropriately if isinstance(value, str): value = f"'{value}'" where_clause = where_clause.replace(f"${key}", str(value)) - # Handle user_name parameter in where_clause if "user_name = %s" in where_clause: where_clause = where_clause.replace( "user_name = %s", f"ag_catalog.agtype_access_operator(properties, '\"user_name\"'::agtype) = '\"{user_name}\"'::agtype", ) - # Build return fields and group by fields - return_fields = [] - group_by_fields = [] - + cte_select_list = [] + aliases = [] for field in group_fields: alias = field.replace(".", "_") - return_fields.append( - f"ag_catalog.agtype_access_operator(properties, '\"{field}\"'::agtype)::text AS {alias}" - ) - group_by_fields.append( - f"ag_catalog.agtype_access_operator(properties, '\"{field}\"'::agtype)::text" + aliases.append(alias) + cte_select_list.append( + f"ag_catalog.agtype_access_operator(properties, '\"{field}\"'::agtype) AS {alias}" ) - - # Full SQL query construction + outer_select = ", ".join(f"{a}::text" for a in aliases) + outer_group_by = ", ".join(aliases) query = f""" - SELECT {", ".join(return_fields)}, COUNT(*) AS count - FROM "{self.db_name}_graph"."Memory" - {where_clause} - GROUP BY {", ".join(group_by_fields)} + WITH t AS ( + SELECT {", ".join(cte_select_list)} + FROM "{self.db_name}_graph"."Memory" + {where_clause} + LIMIT 1000 + ) + SELECT {outer_select}, count(*) AS count + FROM t + GROUP BY {outer_group_by} """ + logger.info(f"get_grouped_counts query:{query},params:{params}") + try: with self._get_connection() as conn, conn.cursor() as cursor: - # Handle parameterized query if params and isinstance(params, list): cursor.execute(query, params) else: @@ -2250,6 +2222,8 @@ def get_grouped_counts( count_value = row[-1] # Last column is count output.append({**group_values, "count": int(count_value)}) + elapsed = (time.perf_counter() - start_time) * 1000.0 + logger.info("get_grouped_counts internal took %.1f ms", elapsed) return output except Exception as e: diff --git a/src/memos/mem_scheduler/schemas/general_schemas.py b/src/memos/mem_scheduler/schemas/general_schemas.py index 06910ba17..cd44cd171 100644 --- a/src/memos/mem_scheduler/schemas/general_schemas.py +++ b/src/memos/mem_scheduler/schemas/general_schemas.py @@ -20,7 +20,7 @@ DEFAULT_DISPATCHER_MONITOR_CHECK_INTERVAL = 300 DEFAULT_DISPATCHER_MONITOR_MAX_FAILURES = 2 DEFAULT_STUCK_THREAD_TOLERANCE = 10 -DEFAULT_MAX_INTERNAL_MESSAGE_QUEUE_SIZE = -1 +DEFAULT_MAX_INTERNAL_MESSAGE_QUEUE_SIZE = 200 DEFAULT_TOP_K = 5 DEFAULT_CONTEXT_WINDOW_SIZE = 5 DEFAULT_USE_REDIS_QUEUE = os.getenv("MEMSCHEDULER_USE_REDIS_QUEUE", "False").lower() == "true" diff --git a/src/memos/memories/textual/tree_text_memory/organize/manager.py b/src/memos/memories/textual/tree_text_memory/organize/manager.py index 96453f5a0..ecd58f309 100644 --- a/src/memos/memories/textual/tree_text_memory/organize/manager.py +++ b/src/memos/memories/textual/tree_text_memory/organize/manager.py @@ -114,7 +114,6 @@ def add( if mode == "sync": self._cleanup_working_memory(user_name) - self._refresh_memory_size(user_name=user_name) return added_ids From 93b10143807ee29a3e92232c4069d499102d8ade Mon Sep 17 00:00:00 2001 From: Wang Daoji <75928131+Wang-Daoji@users.noreply.github.com> Date: Thu, 19 Mar 2026 00:07:34 +0800 Subject: [PATCH 09/11] feat: repair bug 2.0.10 (#1282) * chore: change version number to v2.0.9 * fix: llm rsp content error fix * fix: add sources in tool and skill memory --------- Co-authored-by: jiang Co-authored-by: Jiang <33757498+hijzy@users.noreply.github.com> Co-authored-by: CaralHsi Co-authored-by: yuan.wang --- pyproject.toml | 2 +- src/memos/__init__.py | 2 +- src/memos/llms/ollama.py | 2 +- src/memos/llms/openai.py | 4 +-- src/memos/mem_reader/multi_modal_struct.py | 5 +++ .../read_skill_memory/process_skill_memory.py | 34 +++++++++++++++++-- 6 files changed, 41 insertions(+), 8 deletions(-) diff --git a/pyproject.toml b/pyproject.toml index ff7c9699a..ba2f03de9 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -4,7 +4,7 @@ ############################################################################## name = "MemoryOS" -version = "2.0.8" +version = "2.0.9" description = "Intelligence Begins with Memory" license = {text = "Apache-2.0"} readme = "README.md" diff --git a/src/memos/__init__.py b/src/memos/__init__.py index 36cc0b5b5..783c1d7dc 100644 --- a/src/memos/__init__.py +++ b/src/memos/__init__.py @@ -1,4 +1,4 @@ -__version__ = "2.0.8" +__version__ = "2.0.9" from memos.configs.mem_cube import GeneralMemCubeConfig from memos.configs.mem_os import MOSConfig diff --git a/src/memos/llms/ollama.py b/src/memos/llms/ollama.py index bd92f9625..e87734bd5 100644 --- a/src/memos/llms/ollama.py +++ b/src/memos/llms/ollama.py @@ -88,7 +88,7 @@ def generate(self, messages: MessageList, **kwargs) -> Any: ) str_response = response.message.content if self.config.remove_think_prefix: - return remove_thinking_tags(str_response) + return remove_thinking_tags(str_response or "") else: return str_thinking + str_response diff --git a/src/memos/llms/openai.py b/src/memos/llms/openai.py index f6bb4efc1..9a29ea68a 100644 --- a/src/memos/llms/openai.py +++ b/src/memos/llms/openai.py @@ -56,7 +56,7 @@ def _parse_response(self, response) -> str: if isinstance(reasoning_content, str) and reasoning_content: reasoning_content = f"{reasoning_content}" if self.config.remove_think_prefix: - return remove_thinking_tags(response_content) + return remove_thinking_tags(response_content or "") if reasoning_content: return reasoning_content + (response_content or "") return response_content or "" @@ -202,7 +202,7 @@ def generate(self, messages: MessageList, **kwargs) -> str: return self.tool_call_parser(response.choices[0].message.tool_calls) response_content = response.choices[0].message.content if self.config.remove_think_prefix: - return remove_thinking_tags(response_content) + return remove_thinking_tags(response_content or "") else: return response_content or "" diff --git a/src/memos/mem_reader/multi_modal_struct.py b/src/memos/mem_reader/multi_modal_struct.py index 2745a1bee..7aa8b1c5e 100644 --- a/src/memos/mem_reader/multi_modal_struct.py +++ b/src/memos/mem_reader/multi_modal_struct.py @@ -927,6 +927,10 @@ def _process_tool_trajectory_fine( project_id = user_context.project_id if user_context else None for fast_item in fast_memory_items: + sources = fast_item.metadata.sources or [] + if not isinstance(sources, list): + sources = [sources] + # Extract memory text (string content) mem_str = fast_item.memory or "" if not mem_str.strip() or ( @@ -954,6 +958,7 @@ def _process_tool_trajectory_fine( tool_used_status=m.get("tool_used_status", []), manager_user_id=manager_user_id, project_id=project_id, + sources=sources, ) fine_memory_items.append(node) except Exception as e: diff --git a/src/memos/mem_reader/read_skill_memory/process_skill_memory.py b/src/memos/mem_reader/read_skill_memory/process_skill_memory.py index a9a727b08..0b0c04252 100644 --- a/src/memos/mem_reader/read_skill_memory/process_skill_memory.py +++ b/src/memos/mem_reader/read_skill_memory/process_skill_memory.py @@ -19,7 +19,11 @@ from memos.llms.base import BaseLLM from memos.log import get_logger from memos.mem_reader.read_multi_modal import detect_lang -from memos.memories.textual.item import TextualMemoryItem, TreeNodeTextualMemoryMetadata +from memos.memories.textual.item import ( + SourceMessage, + TextualMemoryItem, + TreeNodeTextualMemoryMetadata, +) from memos.memories.textual.tree_text_memory.retrieve.searcher import Searcher from memos.templates.skill_mem_prompt import ( OTHERS_GENERATION_PROMPT, @@ -91,6 +95,7 @@ def _batch_extract_skills( try: skill_memory = future.result() if skill_memory: + skill_memory["_task_type"] = task_type results.append((skill_memory, task_type, task_chunks.get(task_type, []))) except Exception as e: logger.warning( @@ -901,6 +906,7 @@ def create_skill_memory_item( skill_memory: dict[str, Any], info: dict[str, Any], embedder: BaseEmbedder | None = None, + sources: list[SourceMessage] | None = None, **kwargs: Any, ) -> TextualMemoryItem: info_ = info.copy() @@ -923,7 +929,7 @@ def create_skill_memory_item( status="activated", tags=skill_memory.get("tags") or skill_memory.get("trigger", []), key=skill_memory.get("name", ""), - sources=[], + sources=sources or [], usage=[], background="", confidence=0.99, @@ -1097,6 +1103,7 @@ def _simple_extract(): try: skill_memory = future.result() if skill_memory: + skill_memory["_task_type"] = task_type memories.append(skill_memory) except Exception as e: logger.warning( @@ -1223,11 +1230,32 @@ def _full_extract(): except Exception as cleanup_error: logger.warning(f"[PROCESS_SKILLS] Error cleaning up local files: {cleanup_error}") + # Build source lookup: (role, content) → SourceMessage from fast_memory_items + source_lookup: dict[tuple[str, str], SourceMessage] = {} + for fast_item in fast_memory_items: + for source in getattr(fast_item.metadata, "sources", []) or []: + source_lookup.setdefault((source.role, source.content), source) + # Create TextualMemoryItem objects skill_memory_items = [] for skill_memory in skill_memories: try: - memory_item = create_skill_memory_item(skill_memory, info, embedder, **kwargs) + # Match sources precisely via the task chunk messages that produced this skill + task_type = skill_memory.pop("_task_type", None) + chunk_messages = task_chunks.get(task_type, []) if task_type else [] + skill_sources = [] + seen = set() + for msg in chunk_messages: + key = (msg.get("role"), msg.get("content")) + if key not in seen: + seen.add(key) + source = source_lookup.get(key) + if source: + skill_sources.append(source) + + memory_item = create_skill_memory_item( + skill_memory, info, embedder, sources=skill_sources, **kwargs + ) skill_memory_items.append(memory_item) except Exception as e: logger.warning(f"[PROCESS_SKILLS] Error creating skill memory item: {e}") From c2262897754a6b89950e5cd34a4adf45ed51d0a6 Mon Sep 17 00:00:00 2001 From: chunyu li <78344051+fridayL@users.noreply.github.com> Date: Thu, 19 Mar 2026 11:50:41 +0800 Subject: [PATCH 10/11] feat(memreader): update prefermem config (#1294) * feat: add backend for openai * feat: change memreader for scheaduer * feat: change singlecube llm --------- Co-authored-by: harvey_xiang --- src/memos/mem_reader/multi_modal_struct.py | 9 +++++++-- 1 file changed, 7 insertions(+), 2 deletions(-) diff --git a/src/memos/mem_reader/multi_modal_struct.py b/src/memos/mem_reader/multi_modal_struct.py index 7aa8b1c5e..092f29ac6 100644 --- a/src/memos/mem_reader/multi_modal_struct.py +++ b/src/memos/mem_reader/multi_modal_struct.py @@ -1052,7 +1052,7 @@ def _process_multi_modal_data( process_preference_fine, non_file_url_fast_items, info, - self.llm, + self.general_llm, self.embedder, **kwargs, ) @@ -1131,7 +1131,12 @@ def _process_transfer_multi_modal_data( ) # Add preference memory extraction future_pref = executor.submit( - process_preference_fine, non_file_url_nodes, info, self.llm, self.embedder, **kwargs + process_preference_fine, + non_file_url_nodes, + info, + self.general_llm, + self.embedder, + **kwargs, ) # Collect results From f2407190eae3ae23c0b7d3b27926cf0fb670d6da Mon Sep 17 00:00:00 2001 From: harvey_xiang Date: Fri, 20 Mar 2026 17:40:24 +0800 Subject: [PATCH 11/11] chore: update version to 2.0.10 --- pyproject.toml | 2 +- src/memos/__init__.py | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/pyproject.toml b/pyproject.toml index ba2f03de9..de8e66ad1 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -4,7 +4,7 @@ ############################################################################## name = "MemoryOS" -version = "2.0.9" +version = "2.0.10" description = "Intelligence Begins with Memory" license = {text = "Apache-2.0"} readme = "README.md" diff --git a/src/memos/__init__.py b/src/memos/__init__.py index 783c1d7dc..0a45d89d5 100644 --- a/src/memos/__init__.py +++ b/src/memos/__init__.py @@ -1,4 +1,4 @@ -__version__ = "2.0.9" +__version__ = "2.0.10" from memos.configs.mem_cube import GeneralMemCubeConfig from memos.configs.mem_os import MOSConfig