Improve doc pipeline to add persistence (#119)

emrgnt-cmplxty · Jul 5, 2023 · ee7fb58 · ee7fb58
1 parent 1d5dd46
commit ee7fb58
Show file tree

Hide file tree

Showing 42 changed files with 989 additions and 498 deletions.
diff --git a/automata/core/code_handling/py/writer.py b/automata/core/code_handling/py/writer.py
@@ -398,22 +398,45 @@ def generate_index_files(self, docs_dir: str) -> None:
                 doc_directory_manager.root, root_relative_to_base
             )
 
+            index_path = os.path.join(root, "index.rst")
             if rst_files or dirs:
-                with open(os.path.join(root, "index.rst"), "w") as index_file:
-                    index_file.write(PyDocWriter.get_payload(root))
-
-                    index_file.write(".. toctree::\n")
-                    index_file.write(
-                        "   :maxdepth: 2\n\n"
-                        if not root_dir_node or root_dir_node.is_root_dir()  # type: ignore
-                        else "   :maxdepth: 1\n\n"
-                    )
-
-                    for sub_dir_ in sorted(dirs):
-                        index_file.write(f"   {sub_dir_}/index\n")
-                    for file in sorted(rst_files):
-                        if file != "index.rst":
-                            index_file.write(f"   {file[:-4]}\n")  # Remove .rst extension
+                if os.path.exists(index_path):
+                    with open(index_path, "r") as index_file:
+                        existing_content = index_file.read()
+                else:
+                    existing_content = ""
+
+                # Identify start and end of the auto-generated content
+                auto_start_marker = "\n..  AUTO-GENERATED CONTENT START\n..\n\n"
+                auto_end_marker = "\n..  AUTO-GENERATED CONTENT END\n..\n\n"
+
+                # Remove the auto-generated part if it already exists
+                if auto_start_marker in existing_content and auto_end_marker in existing_content:
+                    start_idx = existing_content.index(auto_start_marker)
+                    end_idx = existing_content.index(auto_end_marker) + len(auto_end_marker)
+                    existing_content = existing_content[:start_idx] + existing_content[end_idx:]
+
+                # Add new auto-generated content
+                auto_content = auto_start_marker
+                auto_content += "    .. toctree::\n"
+                auto_content += (
+                    "       :maxdepth: 2\n\n"
+                    if not root_dir_node or root_dir_node.is_root_dir()  # type: ignore
+                    else "       :maxdepth: 1\n\n"
+                )
+                for file in sorted(rst_files):
+                    if file != "index.rst":
+                        auto_content += f"       {file[:-4]}\n"  # Remove .rst extension
+                for sub_dir_ in sorted(dirs):
+                    auto_content += f"       {sub_dir_}/index\n"
+                auto_content += auto_end_marker
+
+                # Write everything back to the file
+                with open(index_path, "w") as index_file:
+                    if existing_content.strip() == "":
+                        index_file.write(PyDocWriter.get_payload(root) + auto_content)
+                    else:
+                        index_file.write(existing_content + auto_content)
 
                 self.generate_module_summary(root)
 
@@ -442,15 +465,14 @@ def get_payload(directory: str) -> str:
 Check out the :doc:`usage` section for further information, including
 how to :ref:`installation` the project.
 
-.. note::
 
 """
 
     @staticmethod
     def generate_summary(content: str) -> str:
         """This method should implement the logic to generate summary from the content."""
         # TODO: Implement summary generation function.
-        return "Summary of content"
+        return ""
 
     @staticmethod
     def camel_to_snake(name: str) -> str:

diff --git a/automata/docs/config/base/index.rst b/automata/docs/config/base/index.rst
@@ -8,10 +8,25 @@ how to :ref:`installation` the project.
 
 .. note::
 
-.. toctree::
-   :maxdepth: 1
 
-   config
 
 
-Summary of content
+
+
+
+
+
+
+..  AUTO-GENERATED CONTENT START
+..
+
+    .. toctree::
+       :maxdepth: 1
+
+       config
+
+..  AUTO-GENERATED CONTENT END
+..
+
+
+
diff --git a/automata/docs/config/index.rst b/automata/docs/config/index.rst
@@ -1,26 +1,30 @@
-config
-======
+Config Modules
+==============
 
 **Automata** is a Python library for autonomous providers.
 
 Check out the :doc:`usage` section for further information, including
 how to :ref:`installation` the project.
 
-.. note::
+..  AUTO-GENERATED CONTENT START
+..
 
-.. toctree::
-   :maxdepth: 1
+    .. toctree::
+       :maxdepth: 1
+
+       agent_config
+       agent_config_builder
+       agent_config_name
+       config_category
+       instruction_config_version
+       llm_provider
+       open_ai_automata_agent_config
+       open_ai_automata_agent_config_builder
+       base/index
+       openai_agent/index
+
+..  AUTO-GENERATED CONTENT END
+..
 
-   base/index
-   openai_agent/index
-   agent_config
-   agent_config_builder
-   agent_config_name
-   config_category
-   instruction_config_version
-   llm_provider
-   open_ai_automata_agent_config
-   open_ai_automata_agent_config_builder
 
 
-Summary of content
diff --git a/automata/docs/config/openai_agent/index.rst b/automata/docs/config/openai_agent/index.rst
@@ -8,11 +8,26 @@ how to :ref:`installation` the project.
 
 .. note::
 
-.. toctree::
-   :maxdepth: 1
 
-   config
-   template_formatter
 
 
-Summary of content
+
+
+
+
+
+
+..  AUTO-GENERATED CONTENT START
+..
+
+    .. toctree::
+       :maxdepth: 1
+
+       config
+       template_formatter
+
+..  AUTO-GENERATED CONTENT END
+..
+
+
+
diff --git a/automata/docs/core/agent/agent/index.rst b/automata/docs/core/agent/agent/index.rst
@@ -8,10 +8,25 @@ how to :ref:`installation` the project.
 
 .. note::
 
-.. toctree::
-   :maxdepth: 1
 
-   config
 
 
-Summary of content
+
+
+
+
+
+
+..  AUTO-GENERATED CONTENT START
+..
+
+    .. toctree::
+       :maxdepth: 1
+
+       config
+
+..  AUTO-GENERATED CONTENT END
+..
+
+
+
diff --git a/automata/docs/core/agent/index.rst b/automata/docs/core/agent/index.rst
@@ -8,28 +8,43 @@ how to :ref:`installation` the project.
 
 .. note::
 
-.. toctree::
-   :maxdepth: 1
-
-   agent/index
-   instances/index
-   agent
-   agent_database_error
-   agent_general_error
-   agent_instance
-   agent_max_iter_error
-   agent_result_error
-   agent_stop_iteration
-   agent_task_general_error
-   agent_task_git_error
-   agent_task_instructions
-   agent_task_state_error
-   agent_toolkit_builder
-   agent_toolkit_names
-   open_ai_agent_toolkit_builder
-   open_ai_automata_agent
-   open_ai_automata_agent_instance
-   unknown_tool_error
-
-
-Summary of content
+
+
+
+
+
+
+
+
+
+..  AUTO-GENERATED CONTENT START
+..
+
+    .. toctree::
+       :maxdepth: 1
+
+       agent
+       agent_database_error
+       agent_general_error
+       agent_instance
+       agent_max_iter_error
+       agent_result_error
+       agent_stop_iteration
+       agent_task_general_error
+       agent_task_git_error
+       agent_task_instructions
+       agent_task_state_error
+       agent_toolkit_builder
+       agent_toolkit_names
+       open_ai_agent_toolkit_builder
+       open_ai_automata_agent
+       open_ai_automata_agent_instance
+       unknown_tool_error
+       agent/index
+       instances/index
+
+..  AUTO-GENERATED CONTENT END
+..
+
+
+
diff --git a/automata/docs/core/agent/instances/index.rst b/automata/docs/core/agent/instances/index.rst
@@ -8,10 +8,25 @@ how to :ref:`installation` the project.
 
 .. note::
 
-.. toctree::
-   :maxdepth: 1
 
-   config
 
 
-Summary of content
+
+
+
+
+
+
+..  AUTO-GENERATED CONTENT START
+..
+
+    .. toctree::
+       :maxdepth: 1
+
+       config
+
+..  AUTO-GENERATED CONTENT END
+..
+
+
+
diff --git a/automata/docs/core/base/database/index.rst b/automata/docs/core/base/database/index.rst
@@ -8,13 +8,28 @@ how to :ref:`installation` the project.
 
 .. note::
 
-.. toctree::
-   :maxdepth: 1
 
-   json_vector_database
-   relational_database
-   sql_database
-   vector_database_provider
 
 
-Summary of content
+
+
+
+
+
+
+..  AUTO-GENERATED CONTENT START
+..
+
+    .. toctree::
+       :maxdepth: 1
+
+       json_vector_database
+       relational_database
+       sql_database
+       vector_database_provider
+
+..  AUTO-GENERATED CONTENT END
+..
+
+
+
diff --git a/automata/docs/core/base/index.rst b/automata/docs/core/base/index.rst
@@ -8,11 +8,26 @@ how to :ref:`installation` the project.
 
 .. note::
 
-.. toctree::
-   :maxdepth: 1
 
-   database/index
-   patterns/index
 
 
-Summary of content
+
+
+
+
+
+
+..  AUTO-GENERATED CONTENT START
+..
+
+    .. toctree::
+       :maxdepth: 1
+
+       database/index
+       patterns/index
+
+..  AUTO-GENERATED CONTENT END
+..
+
+
+