Fix extract content with cache

prompting/rewards/web_retrieval.py

@@ -15,6 +15,7 @@
 from pydantic import BaseModel
 from scipy import spatial
 from thefuzz import fuzz
+from functools import lru_cache
 
 from prompting.datasets.random_website import DDGDataset, DDGDatasetEntry
 from prompting.rewards.relevance import RelevanceRewardModel
@@ -32,6 +33,11 @@ class WebsiteResult(BaseModel):
     relevant: str
 
 
+@lru_cache(maxsize=1000)
+def extract_content_with_cache(url: str):
+    return DDGDataset.extract_website_content(url)
+
+
 class WebRetrievalRewardModel(RelevanceRewardModel):
     def _cosine_similarity(self, content1: str, content2: str) -> float:
         """Calculate the cosine similarity between sentence embeddings of the reference and completions."""
@@ -46,7 +52,7 @@ def score_website_result(
             return 0
 
         # Content scraped from the URL provided in the completion.
-        reference_website_content = DDGDataset.extract_website_content(response_url)
+        reference_website_content = extract_content_with_cache(response_url)
         if not reference_website_content or len(reference_website_content) == 0:
             logger.debug(f"Failed to extract miner's content from website: {response_url}")
             return 0