implemented graph_config, fixed smart_scraper and speech graph

Author: PeriniM

examples/graph_examples/smart_scraper_example.py

@@ -7,20 +7,26 @@
 from scrapegraphai.graphs import SmartScraperGraph
 
 load_dotenv()
-
-# Define the configuration for the language model
 openai_key = os.getenv("OPENAI_APIKEY")
-llm_config = {
-    "api_key": openai_key,
-    "model_name": "gpt-3.5-turbo",
-}
 
-# Define URL and PROMPT
-URL = "https://www.ansa.it/veneto/"
-PROMPT = "List me all the news with their description."
+# Define the configuration for the graph
+graph_config = {
+    "llm": {
+        "api_key": openai_key,
+        "model": "gpt-3.5-turbo",
+    },
+    # "embedding_model": {
+    #     "api_key": openai_key,
+    #     "model": "gpt-3.5-turbo",
+    # },
+}
 
 # Create the SmartScraperGraph instance
-smart_scraper_graph = SmartScraperGraph(PROMPT, URL, llm_config)
+smart_scraper_graph = SmartScraperGraph(
+    prompt = "List me all the news with their description.",
+    url = "https://www.ansa.it/veneto/",
+    config = graph_config
+)
 
 answer = smart_scraper_graph.run()
 print(answer)

examples/graph_examples/speech_graph_example.py

@@ -0,0 +1,38 @@
+""" 
+Basic example of scraping pipeline using SpeechSummaryGraph
+"""
+
+import os
+from dotenv import load_dotenv
+from scrapegraphai.graphs import SpeechGraph
+
+load_dotenv()
+openai_key = os.getenv("OPENAI_APIKEY")
+
+# Save the audio to a file
+file_name = "website_summary.mp3"
+curr_dir = os.path.dirname(os.path.realpath(__file__))
+output_path = os.path.join(curr_dir, file_name)
+
+# Define the configuration for the graph
+graph_config = {
+    "llm": {
+        "api_key": openai_key,
+        "model": "gpt-3.5-turbo",
+    },
+    "tts_model": {
+        "api_key": openai_key,
+        "model": "tts-1",
+        "voice": "alloy"
+    },
+    "output_path": output_path,
+}
+
+speech_graph = SpeechGraph(
+    prompt = "List me all the projects and generate and audio for me to listen to.",
+    url = "https://perinim.github.io/projects/",
+    config = graph_config,
+)
+
+final_state = speech_graph.run()
+print(final_state.get("answer", "No answer found."))

examples/graph_examples/speech_summary_graph_example.py

@@ -3,4 +3,4 @@
 """
 from .base_graph import BaseGraph
 from .smart_scraper_graph import SmartScraperGraph
-from .speech_summary_graph import SpeechSummaryGraph
+from .speech_graph import SpeechGraph

scrapegraphai/graphs/__init__.py

@@ -4,7 +4,7 @@
 from ..models import OpenAI
 from .base_graph import BaseGraph
 from ..nodes import (
-    FetchHTMLNode,
+    FetchNode,
     ParseNode,
     RAGNode,
     GenerateAnswerNode
@@ -34,17 +34,17 @@ class SmartScraperGraph:
                            'temperature', and 'streaming'.
     """
 
-    def __init__(self, prompt: str, url: str, llm_config: dict):
+    def __init__(self, prompt: str, url: str, config: dict):
         """
         Initializes the SmartScraper with a prompt, URL, and language model configuration.
         """
         self.prompt = prompt
         self.url = url
-        self.llm_config = llm_config
-        self.llm = self._create_llm()
+        self.config = config
+        self.llm_model = self._create_llm(config["llm"])
         self.graph = self._create_graph()
 
-    def _create_llm(self):
+    def _create_llm(self, llm_config: dict):
         """
         Creates an instance of the ChatOpenAI class with the provided language model configuration.
 
@@ -55,12 +55,11 @@ def _create_llm(self):
             ValueError: If 'api_key' is not provided in llm_config.
         """
         llm_defaults = {
-            "model_name": "gpt-3.5-turbo",
             "temperature": 0,
             "streaming": True
         }
         # Update defaults with any LLM parameters that were provided
-        llm_params = {**llm_defaults, **self.llm_config}
+        llm_params = {**llm_defaults, **llm_config}
         # Ensure the api_key is set, raise an error if it's not
         if "api_key" not in llm_params:
             raise ValueError("LLM configuration must include an 'api_key'.")
@@ -75,24 +74,38 @@ def _create_graph(self):
             BaseGraph: An instance of the BaseGraph class.
         """
         # define the nodes for the graph
-        fetch_html_node = FetchHTMLNode("fetch_html")
-        parse_document_node = ParseNode(doc_type="html", chunks_size=4000, node_name="parse_document")
-        rag_node = RAGNode(self.llm, "rag")
-        generate_answer_node = GenerateAnswerNode(self.llm, "generate_answer")
+        fetch_node = FetchNode(
+            input="url | local_dir",
+            output=["doc"],
+            )
+        parse_node = ParseNode(
+            input="doc",
+            output=["parsed_doc"],
+            )
+        rag_node = RAGNode(
+            input="user_prompt & (parsed_doc | doc)",
+            output=["relevant_chunks"],
+            model_config={"llm_model": self.llm_model},
+            )
+        generate_answer_node = GenerateAnswerNode(
+            input="user_prompt & (relevant_chunks | parsed_doc | doc)",
+            output=["answer"],
+            model_config={"llm_model": self.llm_model},
+            )
 
         return BaseGraph(
             nodes={
-                fetch_html_node,
-                parse_document_node,
+                fetch_node,
+                parse_node,
                 rag_node,
                 generate_answer_node,
             },
             edges={
-                (fetch_html_node, parse_document_node),
-                (parse_document_node, rag_node),
+                (fetch_node, parse_node),
+                (parse_node, rag_node),
                 (rag_node, generate_answer_node)
             },
-            entry_point=fetch_html_node
+            entry_point=fetch_node
         )
 
     def run(self) -> str:
@@ -102,7 +115,7 @@ def run(self) -> str:
         Returns:
             str: The answer extracted from the web page, corresponding to the given prompt.
         """
-        inputs = {"user_input": self.prompt, "url": self.url}
+        inputs = {"user_prompt": self.prompt, "url": self.url}
         final_state = self.graph.execute(inputs)
 
         return final_state.get("answer", "No answer found.")

scrapegraphai/graphs/smart_scraper_graph.py

@@ -5,15 +5,15 @@
 from ..models import OpenAI, OpenAITextToSpeech
 from .base_graph import BaseGraph
 from ..nodes import (
-    FetchHTMLNode,
+    FetchNode,
     ParseNode,
     RAGNode,
     GenerateAnswerNode,
     TextToSpeechNode,
 )
 
 
-class SpeechSummaryGraph:
+class SpeechGraph:
     """
     SpeechSummaryGraph is a tool that automates the process of extracting and summarizing
     information from web pages, then converting that summary into spoken word via an MP3 file.
@@ -35,21 +35,18 @@ class SpeechSummaryGraph:
         output_path (str): The file path where the generated MP3 should be saved.
     """
 
-    def __init__(self, prompt: str, url: str, llm_config: dict,
-                 output_path: str = "website_summary.mp3"):
+    def __init__(self, prompt: str, url: str, config: dict):
         """
         Initializes the SmartScraper with a prompt, URL, and language model configuration.
         """
-        self.prompt = f"{prompt} - Save the summary in a key called 'summary'."
+        self.prompt = prompt
         self.url = url
-        self.llm_config = llm_config
-        self.llm = self._create_llm()
-        self.output_path = output_path
-        self.text_to_speech_model = OpenAITextToSpeech(
-            llm_config, model="tts-1", voice="alloy")
+        self.llm_model = self._create_llm(config["llm"])
+        self.output_path = config.get("output_path", "output.mp3")
+        self.text_to_speech_model = OpenAITextToSpeech(config["tts_model"])
         self.graph = self._create_graph()
 
-    def _create_llm(self):
+    def _create_llm(self, llm_config: dict):
         """
         Creates an instance of the ChatOpenAI class with the provided language model configuration.
 
@@ -60,12 +57,11 @@ def _create_llm(self):
             ValueError: If 'api_key' is not provided in llm_config.
         """
         llm_defaults = {
-            "model_name": "gpt-3.5-turbo",
             "temperature": 0,
             "streaming": True
         }
         # Update defaults with any LLM parameters that were provided
-        llm_params = {**llm_defaults, **self.llm_config}
+        llm_params = {**llm_defaults, **llm_config}
         # Ensure the api_key is set, raise an error if it's not
         if "api_key" not in llm_params:
             raise ValueError("LLM configuration must include an 'api_key'.")
@@ -79,28 +75,46 @@ def _create_graph(self):
         Returns:
             BaseGraph: An instance of the BaseGraph class.
         """
-        fetch_html_node = FetchHTMLNode("fetch_html")
-        parse_document_node = ParseNode(doc_type="html", chunks_size=4000, node_name="parse_document")
-        rag_node = RAGNode(self.llm, "rag")
-        generate_answer_node = GenerateAnswerNode(self.llm, "generate_answer")
+        # define the nodes for the graph
+        fetch_node = FetchNode(
+            input="url | local_dir",
+            output=["doc"],
+            )
+        parse_node = ParseNode(
+            input="doc",
+            output=["parsed_doc"],
+            )
+        rag_node = RAGNode(
+            input="user_prompt & (parsed_doc | doc)",
+            output=["relevant_chunks"],
+            model_config={"llm_model": self.llm_model},
+            )
+        generate_answer_node = GenerateAnswerNode(
+            input="user_prompt & (relevant_chunks | parsed_doc | doc)",
+            output=["answer"],
+            model_config={"llm_model": self.llm_model},
+            )
         text_to_speech_node = TextToSpeechNode(
-            self.text_to_speech_model, "text_to_speech")
+            input="answer",
+            output=["audio"],
+            model_config={"tts_model": self.text_to_speech_model},
+        )
 
         return BaseGraph(
             nodes={
-                fetch_html_node,
-                parse_document_node,
+                fetch_node,
+                parse_node,
                 rag_node,
                 generate_answer_node,
                 text_to_speech_node
             },
             edges={
-                (fetch_html_node, parse_document_node),
-                (parse_document_node, rag_node),
+                (fetch_node, parse_node),
+                (parse_node, rag_node),
                 (rag_node, generate_answer_node),
                 (generate_answer_node, text_to_speech_node)
             },
-            entry_point=fetch_html_node
+            entry_point=fetch_node
         )
 
     def run(self) -> str:
@@ -110,7 +124,7 @@ def run(self) -> str:
         Returns:
             str: The answer extracted from the web page, corresponding to the given prompt.
         """
-        inputs = {"user_input": self.prompt, "url": self.url}
+        inputs = {"user_prompt": self.prompt, "url": self.url}
         final_state = self.graph.execute(inputs)
 
         audio = final_state.get("audio", None)

…pegraphai/graphs/speech_summary_graph.py scrapegraphai/graphs/speech_graph.pyscrapegraphai/graphs/speech_summary_graph.py renamed to scrapegraphai/graphs/speech_graph.py scrapegraphai/graphs/speech_summary_graph.py renamed to scrapegraphai/graphs/speech_graph.py

@@ -22,7 +22,7 @@ class OpenAITextToSpeech:
         bytes of the generated speech.
     """
 
-    def __init__(self, llm_config: dict, model: str = "tts-1", voice: str = "alloy"):
+    def __init__(self, tts_config: dict):
         """
         Initializes an instance of the OpenAITextToSpeech class.
 
@@ -35,9 +35,9 @@ def __init__(self, llm_config: dict, model: str = "tts-1", voice: str = "alloy")
         """
 
         # convert model_name to model
-        self.client = OpenAI(api_key=llm_config.get("api_key"))
-        self.model = model
-        self.voice = voice
+        self.client = OpenAI(api_key=tts_config.get("api_key"))
+        self.model = tts_config.get("model", "tts-1")
+        self.voice = tts_config.get("voice", "alloy")
 
     def run(self, text):
         """

scrapegraphai/models/openai_tts.py

@@ -35,7 +35,7 @@ class FetchNode(BaseNode):
                         to succeed.
     """
 
-    def __init__(self, input: str, output: List[str], node_name: str = "FetchNode"):
+    def __init__(self, input: str, output: List[str], node_name: str = "Fetch"):
         """
         Initializes the FetchHTMLNode with a node name and node type.
         Arguments:

scrapegraphai/nodes/fetch_node.py

@@ -38,7 +38,7 @@ class GenerateAnswerNode(BaseNode):
                         updating the state with the generated answer under the 'answer' key.
     """
 
-    def __init__(self, input: str, output: List[str], model_config: dict, node_name: str = "GenerateAnswerNode"):
+    def __init__(self, input: str, output: List[str], model_config: dict, node_name: str = "GenerateAnswer"):
         """
         Initializes the GenerateAnswerNode with a language model client and a node name.
         Args: