Skip to content
New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Implement directory-based plugin system #4548

Merged
merged 5 commits into from
Jun 10, 2023
Merged
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension


Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
2 changes: 1 addition & 1 deletion .github/PULL_REQUEST_TEMPLATE.md
Original file line number Diff line number Diff line change
Expand Up @@ -41,7 +41,7 @@ By following these guidelines, your PRs are more likely to be merged quickly aft
black .
isort .
mypy
autoflake --remove-all-unused-imports --recursive --ignore-init-module-imports autogpt tests --in-place
autoflake --remove-all-unused-imports --recursive --ignore-init-module-imports --ignore-pass-after-docstring autogpt tests --in-place
ntindle marked this conversation as resolved.
Show resolved Hide resolved
```

<!-- If you haven't added tests, please explain why. If you have, check the appropriate box. If you've ensured your PR is atomic and well-documented, check the corresponding boxes. -->
Expand Down
2 changes: 1 addition & 1 deletion .github/workflows/ci.yml
Original file line number Diff line number Diff line change
Expand Up @@ -69,7 +69,7 @@ jobs:

- name: Check for unused imports and pass statements
run: |
cmd="autoflake --remove-all-unused-imports --recursive --ignore-init-module-imports autogpt tests"
cmd="autoflake --remove-all-unused-imports --recursive --ignore-init-module-imports --ignore-pass-after-docstring autogpt tests"
$cmd --check || (echo "You have unused imports or pass statements, please run '${cmd} --in-place'" && exit 1)

test:
Expand Down
2 changes: 1 addition & 1 deletion .pre-commit-config.yaml
Original file line number Diff line number Diff line change
Expand Up @@ -31,7 +31,7 @@ repos:
hooks:
- id: autoflake
name: autoflake
entry: autoflake --in-place --remove-all-unused-imports --recursive --ignore-init-module-imports autogpt tests
entry: autoflake --in-place --remove-all-unused-imports --recursive --ignore-init-module-imports --ignore-pass-after-docstring autogpt tests
language: python
types: [ python ]
- id: pytest-check
Expand Down
25 changes: 25 additions & 0 deletions autogpt/plugins.py
Original file line number Diff line number Diff line change
@@ -1,8 +1,10 @@
"""Handles loading of plugins."""

import importlib.util
import inspect
import json
import os
import sys
import zipfile
from pathlib import Path
from typing import List
Expand Down Expand Up @@ -217,6 +219,28 @@ def scan_plugins(cfg: Config, debug: bool = False) -> List[AutoGPTPluginTemplate
logger.debug(f"Allowlisted Plugins: {cfg.plugins_allowlist}")
logger.debug(f"Denylisted Plugins: {cfg.plugins_denylist}")

# Directory-based plugins
for plugin_path in [f.path for f in os.scandir(cfg.plugins_dir) if f.is_dir()]:
# Avoid going into __pycache__ or other hidden directories
if plugin_path.startswith("__"):
continue

plugin_module_path = plugin_path.split(os.path.sep)
plugin_module_name = plugin_module_path[-1]
qualified_module_name = ".".join(plugin_module_path)

__import__(qualified_module_name)
plugin = sys.modules[qualified_module_name]

for _, class_obj in inspect.getmembers(plugin):
if (
hasattr(class_obj, "_abc_impl")
and AutoGPTPluginTemplate in class_obj.__bases__
and denylist_allowlist_check(plugin_module_name, cfg)
):
loaded_plugins.append(class_obj())

# Zip-based plugins
for plugin in plugins_path_path.glob("*.zip"):
if moduleList := inspect_zip_for_modules(str(plugin), debug):
for module in moduleList:
Expand All @@ -236,6 +260,7 @@ def scan_plugins(cfg: Config, debug: bool = False) -> List[AutoGPTPluginTemplate
and denylist_allowlist_check(a_module.__name__, cfg)
):
loaded_plugins.append(a_module())

# OpenAI plugins
if cfg.plugins_openai:
manifests_specs = fetch_openai_plugins_manifest_and_spec(cfg)
Expand Down
10 changes: 10 additions & 0 deletions scripts/install_plugin_deps.py
Original file line number Diff line number Diff line change
Expand Up @@ -2,6 +2,7 @@
import subprocess
import sys
import zipfile
from glob import glob
from pathlib import Path


Expand All @@ -16,6 +17,8 @@ def install_plugin_dependencies():
None
"""
plugins_dir = Path(os.getenv("PLUGINS_DIR", "plugins"))

# Install zip-based plugins
for plugin in plugins_dir.glob("*.zip"):
with zipfile.ZipFile(str(plugin), "r") as zfile:
try:
Expand All @@ -30,6 +33,13 @@ def install_plugin_dependencies():
except KeyError:
continue

# Install directory-based plugins
for requirements_file in glob(f"{plugins_dir}/*/requirements.txt"):
subprocess.check_call(
[sys.executable, "-m", "pip", "install", "-r", requirements_file],
stdout=subprocess.DEVNULL,
)


if __name__ == "__main__":
install_plugin_dependencies()
6 changes: 3 additions & 3 deletions tests/integration/test_plugins.py
Original file line number Diff line number Diff line change
Expand Up @@ -39,7 +39,7 @@ class MockConfig:

plugins_dir = PLUGINS_TEST_DIR
plugins_openai = [PLUGIN_TEST_OPENAI]
plugins_denylist = ["AutoGPTPVicuna"]
plugins_denylist = ["AutoGPTPVicuna", "auto_gpt_guanaco"]
waynehamadi marked this conversation as resolved.
Show resolved Hide resolved
plugins_allowlist = [PLUGIN_TEST_OPENAI]

return MockConfig()
Expand All @@ -60,12 +60,12 @@ class MockConfig:
plugins_dir = PLUGINS_TEST_DIR
plugins_openai = []
plugins_denylist = []
plugins_allowlist = ["AutoGPTPVicuna"]
plugins_allowlist = ["AutoGPTPVicuna", "auto_gpt_guanaco"]

return MockConfig()


def test_scan_plugins_generic(mock_config_generic_plugin):
# Test that the function returns the correct number of plugins
result = scan_plugins(mock_config_generic_plugin, debug=True)
assert len(result) == 1
assert len(result) == 2
4 changes: 4 additions & 0 deletions tests/integration/test_web_selenium.py
Original file line number Diff line number Diff line change
@@ -1,9 +1,13 @@
import pytest
from pytest_mock import MockerFixture

from autogpt.commands.web_selenium import browse_website
from autogpt.config import Config
from tests.utils import requires_api_key


@pytest.mark.vcr
@requires_api_key("OPENAI_API_KEY")
def test_browse_website(config: Config, patched_api_requestor: MockerFixture):
url = "https://barrel-roll.com"
question = "How to execute a barrel roll"
Expand Down
Loading