Merge pull request #328 from algorithmicsuperintelligence/fix-embedding-async-error

Fix embedding async error

Author: codelion

openevolve/_version.py

@@ -1,3 +1,3 @@
 """Version information for openevolve package."""
 
-__version__ = "0.2.20"
+__version__ = "0.2.21"

openevolve/database.py

@@ -975,20 +975,36 @@ def _llm_judge_novelty(self, program: Program, similar_program: Program) -> bool
         """
         import asyncio
         from openevolve.novelty_judge import NOVELTY_SYSTEM_MSG, NOVELTY_USER_MSG
-        
+
         user_msg = NOVELTY_USER_MSG.format(
             language=program.language,
             existing_code=similar_program.code,
             proposed_code=program.code,
         )
-        
+
         try:
-            content: str = asyncio.run(
-                self.novelty_llm.generate_with_context(
-                    system_message=NOVELTY_SYSTEM_MSG,
-                    messages=[{"role": "user", "content": user_msg}],
+            # Check if we're already in an event loop
+            try:
+                loop = asyncio.get_running_loop()
+                # We're in an async context, need to run in a new thread
+                import concurrent.futures
+                with concurrent.futures.ThreadPoolExecutor() as executor:
+                    future = executor.submit(
+                        asyncio.run,
+                        self.novelty_llm.generate_with_context(
+                            system_message=NOVELTY_SYSTEM_MSG,
+                            messages=[{"role": "user", "content": user_msg}],
+                        )
+                    )
+                    content: str = future.result()
+            except RuntimeError:
+                # No event loop running, safe to use asyncio.run()
+                content: str = asyncio.run(
+                    self.novelty_llm.generate_with_context(
+                        system_message=NOVELTY_SYSTEM_MSG,
+                        messages=[{"role": "user", "content": user_msg}],
+                    )
                 )
-            )
 
             if content is None or content is None:
                 logger.warning("Novelty LLM returned empty response")
@@ -999,24 +1015,24 @@ def _llm_judge_novelty(self, program: Program, similar_program: Program) -> bool
             # Parse the response
             NOVEL_i = content.upper().find("NOVEL")
             NOT_NOVEL_i = content.upper().find("NOT NOVEL")
-            
+
             if NOVEL_i == -1 and NOT_NOVEL_i == -1:
                 logger.warning(f"Unexpected novelty LLM response: {content}")
                 return True  # Assume novel if we can't parse
-            
+
             if NOVEL_i != -1 and NOT_NOVEL_i != -1:
                 # Both found, take the one that appears first
                 is_novel = NOVEL_i < NOT_NOVEL_i
             elif NOVEL_i != -1:
                 is_novel = True
             else:
                 is_novel = False
-                
+
             return is_novel
 
         except Exception as e:
             logger.error(f"Error in novelty LLM check: {e}")
-    
+
         return True
 
     def _is_novel(self, program_id: int, island_idx: int) -> bool:

tests/test_novelty_asyncio_issue.py

@@ -0,0 +1,169 @@
+"""
+Test for issue #313: asyncio.run() error in novelty checking
+https://github.com/algorithmicsuperintelligence/openevolve/issues/313
+
+This test reproduces the bug where calling database.add() from within an async context
+triggers a novelty check that uses asyncio.run(), which fails because it's already
+running in an event loop.
+"""
+
+import unittest
+import asyncio
+from unittest.mock import AsyncMock, MagicMock, patch, Mock
+from openevolve.config import Config
+from openevolve.database import Program, ProgramDatabase
+
+
+class MockLLM:
+    """Mock LLM that implements the async interface"""
+
+    async def generate_with_context(self, system_message: str, messages: list):
+        """Mock async generate method that returns NOVEL"""
+        return "NOVEL"
+
+
+class TestNoveltyAsyncioIssue(unittest.TestCase):
+    """Test for asyncio.run() error in novelty checking (issue #313)"""
+
+    @patch('openevolve.embedding.EmbeddingClient')
+    def setUp(self, mock_embedding_client_class):
+        """Set up test database with novelty checking enabled"""
+        # Mock the embedding client
+        mock_instance = MagicMock()
+        mock_instance.get_embedding.return_value = [0.1] * 1536  # Mock embedding vector
+        mock_embedding_client_class.return_value = mock_instance
+
+        config = Config()
+        config.database.in_memory = True
+        config.database.embedding_model = "text-embedding-3-small"
+        config.database.similarity_threshold = 0.99
+        config.database.novelty_llm = MockLLM()
+
+        self.db = ProgramDatabase(config.database)
+        self.mock_embedding_client_class = mock_embedding_client_class
+
+    def test_novelty_check_from_async_context_works(self):
+        """
+        Test that novelty checking works correctly when called from within
+        an async context (this was the bug in issue #313).
+
+        Expected behavior: Should successfully run the novelty check without
+        any asyncio.run() errors, properly using ThreadPoolExecutor to handle
+        the async LLM call from within a running event loop.
+        """
+        import logging
+
+        # Create two programs with similar embeddings to trigger LLM novelty check
+        program1 = Program(
+            id="prog1",
+            code="def test(): return 1",
+            language="python",
+            metrics={"score": 0.5},
+        )
+
+        program2 = Program(
+            id="prog2",
+            code="def test(): return 2",
+            language="python",
+            metrics={"score": 0.6},
+            parent_id="prog1",
+        )
+
+        async def async_add_programs():
+            """Add programs from async context - this simulates controller.run()"""
+            # Add first program (no novelty check, no similar programs yet)
+            prog1_id = self.db.add(program1)
+            self.assertIsNotNone(prog1_id)
+
+            # Add second program - this triggers novelty check
+            # Since embeddings are similar (both [0.1] * 1536), it will call
+            # _llm_judge_novelty which should now work correctly
+            prog2_id = self.db.add(program2)
+
+            # The novelty check should succeed without errors
+            # The program should be added (MockLLM returns "NOVEL")
+            self.assertIsNotNone(prog2_id)
+
+            return True
+
+        # This should work without any errors now
+        result = asyncio.run(async_add_programs())
+        self.assertTrue(result)
+
+        # Verify both programs were added
+        self.assertIn("prog1", self.db.programs)
+        self.assertIn("prog2", self.db.programs)
+
+    def test_novelty_check_from_sync_context_works(self):
+        """
+        Test that novelty checking also works correctly when called from
+        a synchronous (non-async) context.
+
+        Expected behavior: Should successfully run the novelty check using
+        asyncio.run() since there's no running event loop.
+        """
+        # Create two programs with similar embeddings to trigger LLM novelty check
+        program1 = Program(
+            id="prog3",
+            code="def test(): return 3",
+            language="python",
+            metrics={"score": 0.5},
+        )
+
+        program2 = Program(
+            id="prog4",
+            code="def test(): return 4",
+            language="python",
+            metrics={"score": 0.6},
+            parent_id="prog3",
+        )
+
+        # Add programs from synchronous context (no event loop running)
+        prog1_id = self.db.add(program1)
+        self.assertIsNotNone(prog1_id)
+
+        prog2_id = self.db.add(program2)
+        self.assertIsNotNone(prog2_id)
+
+        # Verify both programs were added
+        self.assertIn("prog3", self.db.programs)
+        self.assertIn("prog4", self.db.programs)
+
+    def test_novelty_check_disabled_works_fine(self):
+        """
+        Test that when novelty checking is disabled, adding programs
+        from async context works fine (this is the workaround from issue #313).
+        """
+        # Create a new database with novelty checking disabled
+        config = Config()
+        config.database.in_memory = True
+        config.database.similarity_threshold = 0.0  # Disable novelty checking
+        db_no_novelty = ProgramDatabase(config.database)
+
+        program1 = Program(
+            id="prog1",
+            code="def test(): return 1",
+            language="python",
+            metrics={"score": 0.5},
+        )
+
+        program2 = Program(
+            id="prog2",
+            code="def test(): return 2",
+            language="python",
+            metrics={"score": 0.6},
+        )
+
+        async def async_add_programs():
+            """Add programs from async context"""
+            db_no_novelty.add(program1)
+            db_no_novelty.add(program2)
+            return True
+
+        # This should work fine without novelty checking
+        result = asyncio.run(async_add_programs())
+        self.assertTrue(result)
+
+
+if __name__ == "__main__":
+    unittest.main()