Implement directory-based plugin system

.github/PULL_REQUEST_TEMPLATE.md

@@ -41,7 +41,7 @@ By following these guidelines, your PRs are more likely to be merged quickly aft
     black .
     isort .
     mypy
-    autoflake --remove-all-unused-imports --recursive --ignore-init-module-imports autogpt tests --in-place
+    autoflake --remove-all-unused-imports --recursive --ignore-init-module-imports --ignore-pass-after-docstring autogpt tests --in-place
     ```
 
 <!-- If you haven't added tests, please explain why. If you have, check the appropriate box. If you've ensured your PR is atomic and well-documented, check the corresponding boxes. -->

.github/workflows/ci.yml

@@ -69,7 +69,7 @@ jobs:
 
       - name: Check for unused imports and pass statements
         run: |
-          cmd="autoflake --remove-all-unused-imports --recursive --ignore-init-module-imports autogpt tests"
+          cmd="autoflake --remove-all-unused-imports --recursive --ignore-init-module-imports --ignore-pass-after-docstring autogpt tests"
           $cmd --check || (echo "You have unused imports or pass statements, please run '${cmd} --in-place'" && exit 1)
 
   test:

.pre-commit-config.yaml

@@ -31,7 +31,7 @@ repos:
     hooks:
       - id: autoflake
         name: autoflake
-        entry: autoflake --in-place --remove-all-unused-imports --recursive --ignore-init-module-imports autogpt tests
+        entry: autoflake --in-place --remove-all-unused-imports --recursive --ignore-init-module-imports --ignore-pass-after-docstring autogpt tests
         language: python
         types: [ python ]
       - id: pytest-check

autogpt/plugins.py

@@ -1,8 +1,10 @@
 """Handles loading of plugins."""
 
 import importlib.util
+import inspect
 import json
 import os
+import sys
 import zipfile
 from pathlib import Path
 from typing import List
@@ -217,6 +219,28 @@ def scan_plugins(cfg: Config, debug: bool = False) -> List[AutoGPTPluginTemplate
     logger.debug(f"Allowlisted Plugins: {cfg.plugins_allowlist}")
     logger.debug(f"Denylisted Plugins: {cfg.plugins_denylist}")
 
+    # Directory-based plugins
+    for plugin_path in [f.path for f in os.scandir(cfg.plugins_dir) if f.is_dir()]:
+        # Avoid going into __pycache__ or other hidden directories
+        if plugin_path.startswith("__"):
+            continue
+
+        plugin_module_path = plugin_path.split(os.path.sep)
+        plugin_module_name = plugin_module_path[-1]
+        qualified_module_name = ".".join(plugin_module_path)
+
+        __import__(qualified_module_name)
+        plugin = sys.modules[qualified_module_name]
+
+        for _, class_obj in inspect.getmembers(plugin):
+            if (
+                hasattr(class_obj, "_abc_impl")
+                and AutoGPTPluginTemplate in class_obj.__bases__
+                and denylist_allowlist_check(plugin_module_name, cfg)
+            ):
+                loaded_plugins.append(class_obj())
+
+    # Zip-based plugins
     for plugin in plugins_path_path.glob("*.zip"):
         if moduleList := inspect_zip_for_modules(str(plugin), debug):
             for module in moduleList:
@@ -236,6 +260,7 @@ def scan_plugins(cfg: Config, debug: bool = False) -> List[AutoGPTPluginTemplate
                         and denylist_allowlist_check(a_module.__name__, cfg)
                     ):
                         loaded_plugins.append(a_module())
+
     # OpenAI plugins
     if cfg.plugins_openai:
         manifests_specs = fetch_openai_plugins_manifest_and_spec(cfg)

scripts/install_plugin_deps.py

@@ -2,6 +2,7 @@
 import subprocess
 import sys
 import zipfile
+from glob import glob
 from pathlib import Path
 
 
@@ -16,6 +17,8 @@ def install_plugin_dependencies():
         None
     """
     plugins_dir = Path(os.getenv("PLUGINS_DIR", "plugins"))
+
+    # Install zip-based plugins
     for plugin in plugins_dir.glob("*.zip"):
         with zipfile.ZipFile(str(plugin), "r") as zfile:
             try:
@@ -30,6 +33,13 @@ def install_plugin_dependencies():
             except KeyError:
                 continue
 
+    # Install directory-based plugins
+    for requirements_file in glob(f"{plugins_dir}/*/requirements.txt"):
+        subprocess.check_call(
+            [sys.executable, "-m", "pip", "install", "-r", requirements_file],
+            stdout=subprocess.DEVNULL,
+        )
+
 
 if __name__ == "__main__":
     install_plugin_dependencies()

tests/integration/test_plugins.py

@@ -39,7 +39,7 @@ class MockConfig:
 
         plugins_dir = PLUGINS_TEST_DIR
         plugins_openai = [PLUGIN_TEST_OPENAI]
-        plugins_denylist = ["AutoGPTPVicuna"]
+        plugins_denylist = ["AutoGPTPVicuna", "auto_gpt_guanaco"]
         plugins_allowlist = [PLUGIN_TEST_OPENAI]
 
     return MockConfig()
@@ -60,12 +60,12 @@ class MockConfig:
         plugins_dir = PLUGINS_TEST_DIR
         plugins_openai = []
         plugins_denylist = []
-        plugins_allowlist = ["AutoGPTPVicuna"]
+        plugins_allowlist = ["AutoGPTPVicuna", "auto_gpt_guanaco"]
 
     return MockConfig()
 
 
 def test_scan_plugins_generic(mock_config_generic_plugin):
     # Test that the function returns the correct number of plugins
     result = scan_plugins(mock_config_generic_plugin, debug=True)
-    assert len(result) == 1
+    assert len(result) == 2

tests/integration/test_web_selenium.py

@@ -1,9 +1,13 @@
+import pytest
 from pytest_mock import MockerFixture
 
 from autogpt.commands.web_selenium import browse_website
 from autogpt.config import Config
+from tests.utils import requires_api_key
 
 
+@pytest.mark.vcr
+@requires_api_key("OPENAI_API_KEY")
 def test_browse_website(config: Config, patched_api_requestor: MockerFixture):
     url = "https://barrel-roll.com"
     question = "How to execute a barrel roll"