diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index 8d44f2a4b2a..7f96529932b 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -5,7 +5,7 @@ on: branches: [ master, ci-test* ] paths-ignore: - 'tests/Auto-GPT-test-cassettes' - - 'tests/integration/challenges/current_score.json' + - 'tests/challenges/current_score.json' pull_request: branches: [ stable, master ] pull_request_target: @@ -148,8 +148,9 @@ jobs: - name: Run pytest with coverage run: | - pytest -n auto --cov=autogpt --cov-report term-missing --cov-branch --cov-report xml --cov-report term - python tests/integration/challenges/utils/build_current_score.py + pytest -n auto --cov=autogpt --cov-branch --cov-report term-missing --cov-report xml \ + tests/unit tests/integration tests/challenges + python tests/challenges/utils/build_current_score.py env: CI: true PROXY: ${{ secrets.PROXY }} @@ -179,7 +180,7 @@ jobs: - name: Push updated challenge scores if: github.event_name == 'push' run: | - score_file="tests/integration/challenges/current_score.json" + score_file="tests/challenges/current_score.json" if ! git diff --quiet $score_file; then git add $score_file diff --git a/.github/workflows/docker-ci.yml b/.github/workflows/docker-ci.yml index a61b707d8e7..ff43666c446 100644 --- a/.github/workflows/docker-ci.yml +++ b/.github/workflows/docker-ci.yml @@ -5,7 +5,7 @@ on: branches: [ master ] paths-ignore: - 'tests/Auto-GPT-test-cassettes' - - 'tests/integration/challenges/current_score.json' + - 'tests/challenges/current_score.json' pull_request: branches: [ master, stable ] @@ -108,15 +108,18 @@ jobs: set +e test_output=$( docker run --env CI --env OPENAI_API_KEY --entrypoint python ${{ env.IMAGE_NAME }} -m \ - pytest -n auto --cov=autogpt --cov-report term-missing --cov-branch --cov-report xml --cov-report term 2>&1 + pytest -n auto --cov=autogpt --cov-branch --cov-report term-missing \ + tests/unit tests/integration 2>&1 ) test_failure=$? - + echo "$test_output" - + cat << $EOF >> $GITHUB_STEP_SUMMARY # Tests $([ $test_failure = 0 ] && echo '✅' || echo '❌') \`\`\` $test_output \`\`\` $EOF + + exit $test_failure diff --git a/.github/workflows/pr-label.yml b/.github/workflows/pr-label.yml index 0bab56385b3..62a197de53b 100644 --- a/.github/workflows/pr-label.yml +++ b/.github/workflows/pr-label.yml @@ -6,7 +6,7 @@ on: branches: [ master ] paths-ignore: - 'tests/Auto-GPT-test-cassettes' - - 'tests/integration/challenges/current_score.json' + - 'tests/challenges/current_score.json' # So that the `dirtyLabel` is removed if conflicts are resolve # We recommend `pull_request_target` so that github secrets are available. # In `pull_request` we wouldn't be able to change labels of fork PRs diff --git a/autogpt/app.py b/autogpt/app.py index 0804b482715..525deddcb5e 100644 --- a/autogpt/app.py +++ b/autogpt/app.py @@ -142,7 +142,7 @@ def get_text_summary(url: str, question: str, config: Config) -> str: Returns: str: The summary of the text """ - text = scrape_text(url) + text = scrape_text(url, config) summary, _ = summarize_text(text, question=question) return f""" "Result" : {summary}""" diff --git a/docs/challenges/building_challenges.md b/docs/challenges/building_challenges.md index 09ab3bf513b..0bd416cc260 100644 --- a/docs/challenges/building_challenges.md +++ b/docs/challenges/building_challenges.md @@ -70,7 +70,7 @@ def kubernetes_agent( ``` ## Creating your challenge -Go to `tests/integration/challenges`and create a file that is called `test_your_test_description.py` and add it to the appropriate folder. If no category exists you can create a new one. +Go to `tests/challenges`and create a file that is called `test_your_test_description.py` and add it to the appropriate folder. If no category exists you can create a new one. Your test could look something like this @@ -84,7 +84,7 @@ import yaml from autogpt.commands.file_operations import read_file, write_to_file from tests.integration.agent_utils import run_interaction_loop -from tests.integration.challenges.utils import run_multiple_times +from tests.challenges.utils import run_multiple_times from tests.utils import requires_api_key diff --git a/docs/challenges/information_retrieval/challenge_a.md b/docs/challenges/information_retrieval/challenge_a.md index de21066ea55..bf1b7b104bd 100644 --- a/docs/challenges/information_retrieval/challenge_a.md +++ b/docs/challenges/information_retrieval/challenge_a.md @@ -5,7 +5,7 @@ **Command to try**: ``` -pytest -s tests/integration/challenges/information_retrieval/test_information_retrieval_challenge_a.py --level=2 +pytest -s tests/challenges/information_retrieval/test_information_retrieval_challenge_a.py --level=2 ``` ## Description diff --git a/docs/challenges/information_retrieval/challenge_b.md b/docs/challenges/information_retrieval/challenge_b.md index bf77a984f64..f4e68a151ed 100644 --- a/docs/challenges/information_retrieval/challenge_b.md +++ b/docs/challenges/information_retrieval/challenge_b.md @@ -5,7 +5,7 @@ **Command to try**: ``` -pytest -s tests/integration/challenges/information_retrieval/test_information_retrieval_challenge_b.py +pytest -s tests/challenges/information_retrieval/test_information_retrieval_challenge_b.py ``` ## Description diff --git a/docs/challenges/memory/challenge_b.md b/docs/challenges/memory/challenge_b.md index 49c7c40f0a9..abc6da6bef8 100644 --- a/docs/challenges/memory/challenge_b.md +++ b/docs/challenges/memory/challenge_b.md @@ -4,7 +4,7 @@ **Command to try**: ``` -pytest -s tests/integration/challenges/memory/test_memory_challenge_b.py --level=3 +pytest -s tests/challenges/memory/test_memory_challenge_b.py --level=3 `` ## Description @@ -41,4 +41,3 @@ Write all the task_ids into the file output.txt. The file has not been created y ## Objective The objective of this challenge is to test the agent's ability to follow instructions and maintain memory of the task IDs throughout the process. The agent successfully completed this challenge if it wrote the task ids in a file. - diff --git a/docs/challenges/memory/challenge_c.md b/docs/challenges/memory/challenge_c.md index fd02a4a56c6..e197ddbd2c6 100644 --- a/docs/challenges/memory/challenge_c.md +++ b/docs/challenges/memory/challenge_c.md @@ -4,7 +4,7 @@ **Command to try**: ``` -pytest -s tests/integration/challenges/memory/test_memory_challenge_c.py --level=2 +pytest -s tests/challenges/memory/test_memory_challenge_c.py --level=2 `` ## Description diff --git a/mypy.ini b/mypy.ini index b977deb04f4..275cd2602b4 100644 --- a/mypy.ini +++ b/mypy.ini @@ -2,7 +2,7 @@ follow_imports = skip check_untyped_defs = True disallow_untyped_defs = True -files = tests/integration/challenges/**/*.py +files = tests/challenges/**/*.py [mypy-requests.*] ignore_missing_imports = True diff --git a/tests/integration/challenges/__init__.py b/tests/challenges/__init__.py similarity index 100% rename from tests/integration/challenges/__init__.py rename to tests/challenges/__init__.py diff --git a/tests/integration/challenges/basic_abilities/__init__.py b/tests/challenges/basic_abilities/__init__.py similarity index 100% rename from tests/integration/challenges/basic_abilities/__init__.py rename to tests/challenges/basic_abilities/__init__.py diff --git a/tests/integration/challenges/basic_abilities/goal_oriented_tasks.md b/tests/challenges/basic_abilities/goal_oriented_tasks.md similarity index 100% rename from tests/integration/challenges/basic_abilities/goal_oriented_tasks.md rename to tests/challenges/basic_abilities/goal_oriented_tasks.md diff --git a/tests/integration/challenges/basic_abilities/test_browse_website.py b/tests/challenges/basic_abilities/test_browse_website.py similarity index 76% rename from tests/integration/challenges/basic_abilities/test_browse_website.py rename to tests/challenges/basic_abilities/test_browse_website.py index 09e5ab2200a..b918434c0ec 100644 --- a/tests/integration/challenges/basic_abilities/test_browse_website.py +++ b/tests/challenges/basic_abilities/test_browse_website.py @@ -1,10 +1,8 @@ import pytest from autogpt.agent import Agent -from tests.integration.challenges.challenge_decorator.challenge_decorator import ( - challenge, -) -from tests.integration.challenges.utils import run_interaction_loop +from tests.challenges.challenge_decorator.challenge_decorator import challenge +from tests.challenges.utils import run_interaction_loop from tests.utils import requires_api_key CYCLE_COUNT = 2 diff --git a/tests/integration/challenges/basic_abilities/test_write_file.py b/tests/challenges/basic_abilities/test_write_file.py similarity index 81% rename from tests/integration/challenges/basic_abilities/test_write_file.py rename to tests/challenges/basic_abilities/test_write_file.py index 393dbfd05d8..033f76e6bf3 100644 --- a/tests/integration/challenges/basic_abilities/test_write_file.py +++ b/tests/challenges/basic_abilities/test_write_file.py @@ -4,10 +4,8 @@ from autogpt.agent import Agent from autogpt.commands.file_operations import read_file from autogpt.config import Config -from tests.integration.challenges.challenge_decorator.challenge_decorator import ( - challenge, -) -from tests.integration.challenges.utils import run_interaction_loop +from tests.challenges.challenge_decorator.challenge_decorator import challenge +from tests.challenges.utils import run_interaction_loop from tests.utils import requires_api_key CYCLE_COUNT = 3 diff --git a/tests/integration/challenges/challenge_decorator/__init__.py b/tests/challenges/challenge_decorator/__init__.py similarity index 100% rename from tests/integration/challenges/challenge_decorator/__init__.py rename to tests/challenges/challenge_decorator/__init__.py diff --git a/tests/integration/challenges/challenge_decorator/challenge.py b/tests/challenges/challenge_decorator/challenge.py similarity index 100% rename from tests/integration/challenges/challenge_decorator/challenge.py rename to tests/challenges/challenge_decorator/challenge.py diff --git a/tests/integration/challenges/challenge_decorator/challenge_decorator.py b/tests/challenges/challenge_decorator/challenge_decorator.py similarity index 90% rename from tests/integration/challenges/challenge_decorator/challenge_decorator.py rename to tests/challenges/challenge_decorator/challenge_decorator.py index fe12317eed8..1d1fbf9146f 100644 --- a/tests/integration/challenges/challenge_decorator/challenge_decorator.py +++ b/tests/challenges/challenge_decorator/challenge_decorator.py @@ -4,11 +4,9 @@ import pytest -from tests.integration.challenges.challenge_decorator.challenge import Challenge -from tests.integration.challenges.challenge_decorator.challenge_utils import ( - create_challenge, -) -from tests.integration.challenges.challenge_decorator.score_utils import ( +from tests.challenges.challenge_decorator.challenge import Challenge +from tests.challenges.challenge_decorator.challenge_utils import create_challenge +from tests.challenges.challenge_decorator.score_utils import ( get_scores, update_new_score, ) diff --git a/tests/integration/challenges/challenge_decorator/challenge_utils.py b/tests/challenges/challenge_decorator/challenge_utils.py similarity index 96% rename from tests/integration/challenges/challenge_decorator/challenge_utils.py rename to tests/challenges/challenge_decorator/challenge_utils.py index 7db7648fa4b..74f4cf5654f 100644 --- a/tests/integration/challenges/challenge_decorator/challenge_utils.py +++ b/tests/challenges/challenge_decorator/challenge_utils.py @@ -1,7 +1,7 @@ import os from typing import Any, Callable, Dict, Optional, Tuple -from tests.integration.challenges.challenge_decorator.challenge import Challenge +from tests.challenges.challenge_decorator.challenge import Challenge CHALLENGE_PREFIX = "test_" diff --git a/tests/integration/challenges/challenge_decorator/score_utils.py b/tests/challenges/challenge_decorator/score_utils.py similarity index 95% rename from tests/integration/challenges/challenge_decorator/score_utils.py rename to tests/challenges/challenge_decorator/score_utils.py index 0a3b71a8cb6..1a8be74439d 100644 --- a/tests/integration/challenges/challenge_decorator/score_utils.py +++ b/tests/challenges/challenge_decorator/score_utils.py @@ -2,7 +2,7 @@ import os from typing import Any, Dict, Optional, Tuple -from tests.integration.challenges.challenge_decorator.challenge import Challenge +from tests.challenges.challenge_decorator.challenge import Challenge CURRENT_SCORE_LOCATION = "../current_score" NEW_SCORE_LOCATION = "../new_score" diff --git a/tests/integration/challenges/conftest.py b/tests/challenges/conftest.py similarity index 89% rename from tests/integration/challenges/conftest.py rename to tests/challenges/conftest.py index 5514a1293fb..dff45f11323 100644 --- a/tests/integration/challenges/conftest.py +++ b/tests/challenges/conftest.py @@ -5,9 +5,8 @@ from _pytest.config.argparsing import Parser from _pytest.fixtures import FixtureRequest -from tests.integration.challenges.challenge_decorator.challenge import Challenge -from tests.integration.conftest import BASE_VCR_CONFIG -from tests.vcr.vcr_filter import before_record_response +from tests.challenges.challenge_decorator.challenge import Challenge +from tests.vcr import BASE_VCR_CONFIG, before_record_response def before_record_response_filter_errors( diff --git a/tests/integration/challenges/current_score.json b/tests/challenges/current_score.json similarity index 100% rename from tests/integration/challenges/current_score.json rename to tests/challenges/current_score.json diff --git a/tests/integration/challenges/debug_code/data/two_sum.py b/tests/challenges/debug_code/data/two_sum.py similarity index 100% rename from tests/integration/challenges/debug_code/data/two_sum.py rename to tests/challenges/debug_code/data/two_sum.py diff --git a/tests/integration/challenges/debug_code/data/two_sum_tests.py b/tests/challenges/debug_code/data/two_sum_tests.py similarity index 100% rename from tests/integration/challenges/debug_code/data/two_sum_tests.py rename to tests/challenges/debug_code/data/two_sum_tests.py diff --git a/tests/integration/challenges/debug_code/test_debug_code_challenge_a.py b/tests/challenges/debug_code/test_debug_code_challenge_a.py similarity index 89% rename from tests/integration/challenges/debug_code/test_debug_code_challenge_a.py rename to tests/challenges/debug_code/test_debug_code_challenge_a.py index 008e562ce30..93df754e744 100644 --- a/tests/integration/challenges/debug_code/test_debug_code_challenge_a.py +++ b/tests/challenges/debug_code/test_debug_code_challenge_a.py @@ -7,10 +7,8 @@ from autogpt.commands.execute_code import execute_python_file from autogpt.commands.file_operations import append_to_file, write_to_file from autogpt.config import Config -from tests.integration.challenges.challenge_decorator.challenge_decorator import ( - challenge, -) -from tests.integration.challenges.utils import run_interaction_loop +from tests.challenges.challenge_decorator.challenge_decorator import challenge +from tests.challenges.utils import run_interaction_loop from tests.utils import requires_api_key CYCLE_COUNT = 5 diff --git a/tests/integration/challenges/information_retrieval/test_information_retrieval_challenge_a.py b/tests/challenges/information_retrieval/test_information_retrieval_challenge_a.py similarity index 89% rename from tests/integration/challenges/information_retrieval/test_information_retrieval_challenge_a.py rename to tests/challenges/information_retrieval/test_information_retrieval_challenge_a.py index 2f61fef3902..eb3d0c949b7 100644 --- a/tests/integration/challenges/information_retrieval/test_information_retrieval_challenge_a.py +++ b/tests/challenges/information_retrieval/test_information_retrieval_challenge_a.py @@ -3,10 +3,8 @@ from autogpt.commands.file_operations import read_file from autogpt.config import Config -from tests.integration.challenges.challenge_decorator.challenge_decorator import ( - challenge, -) -from tests.integration.challenges.utils import run_interaction_loop +from tests.challenges.challenge_decorator.challenge_decorator import challenge +from tests.challenges.utils import run_interaction_loop from tests.utils import requires_api_key CYCLE_COUNT = 3 diff --git a/tests/integration/challenges/information_retrieval/test_information_retrieval_challenge_b.py b/tests/challenges/information_retrieval/test_information_retrieval_challenge_b.py similarity index 90% rename from tests/integration/challenges/information_retrieval/test_information_retrieval_challenge_b.py rename to tests/challenges/information_retrieval/test_information_retrieval_challenge_b.py index 6461e13fbf4..51195f77ff4 100644 --- a/tests/integration/challenges/information_retrieval/test_information_retrieval_challenge_b.py +++ b/tests/challenges/information_retrieval/test_information_retrieval_challenge_b.py @@ -6,10 +6,8 @@ from autogpt.agent import Agent from autogpt.commands.file_operations import read_file from autogpt.config import Config -from tests.integration.challenges.challenge_decorator.challenge_decorator import ( - challenge, -) -from tests.integration.challenges.utils import run_interaction_loop +from tests.challenges.challenge_decorator.challenge_decorator import challenge +from tests.challenges.utils import run_interaction_loop from tests.utils import requires_api_key CYCLE_COUNT = 3 diff --git a/tests/integration/challenges/kubernetes/test_kubernetes_template_challenge_a.py b/tests/challenges/kubernetes/test_kubernetes_template_challenge_a.py similarity index 88% rename from tests/integration/challenges/kubernetes/test_kubernetes_template_challenge_a.py rename to tests/challenges/kubernetes/test_kubernetes_template_challenge_a.py index aa46ac4d4f3..8ea20b94fad 100644 --- a/tests/integration/challenges/kubernetes/test_kubernetes_template_challenge_a.py +++ b/tests/challenges/kubernetes/test_kubernetes_template_challenge_a.py @@ -5,10 +5,8 @@ from autogpt.agent import Agent from autogpt.commands.file_operations import read_file from autogpt.config import Config -from tests.integration.challenges.challenge_decorator.challenge_decorator import ( - challenge, -) -from tests.integration.challenges.utils import run_interaction_loop +from tests.challenges.challenge_decorator.challenge_decorator import challenge +from tests.challenges.utils import run_interaction_loop from tests.utils import requires_api_key CYCLE_COUNT = 3 diff --git a/tests/integration/challenges/memory/__init__.py b/tests/challenges/memory/__init__.py similarity index 100% rename from tests/integration/challenges/memory/__init__.py rename to tests/challenges/memory/__init__.py diff --git a/tests/integration/challenges/memory/test_memory_challenge_a.py b/tests/challenges/memory/test_memory_challenge_a.py similarity index 90% rename from tests/integration/challenges/memory/test_memory_challenge_a.py rename to tests/challenges/memory/test_memory_challenge_a.py index 08f461bdd86..3e3251dddea 100644 --- a/tests/integration/challenges/memory/test_memory_challenge_a.py +++ b/tests/challenges/memory/test_memory_challenge_a.py @@ -4,10 +4,8 @@ from autogpt.agent import Agent from autogpt.commands.file_operations import read_file, write_to_file from autogpt.config import Config -from tests.integration.challenges.challenge_decorator.challenge_decorator import ( - challenge, -) -from tests.integration.challenges.utils import run_interaction_loop +from tests.challenges.challenge_decorator.challenge_decorator import challenge +from tests.challenges.utils import run_interaction_loop from tests.utils import requires_api_key diff --git a/tests/integration/challenges/memory/test_memory_challenge_b.py b/tests/challenges/memory/test_memory_challenge_b.py similarity index 91% rename from tests/integration/challenges/memory/test_memory_challenge_b.py rename to tests/challenges/memory/test_memory_challenge_b.py index c82763129ac..011fa17a4da 100644 --- a/tests/integration/challenges/memory/test_memory_challenge_b.py +++ b/tests/challenges/memory/test_memory_challenge_b.py @@ -4,10 +4,8 @@ from autogpt.agent import Agent from autogpt.commands.file_operations import read_file, write_to_file from autogpt.config import Config -from tests.integration.challenges.challenge_decorator.challenge_decorator import ( - challenge, -) -from tests.integration.challenges.utils import generate_noise, run_interaction_loop +from tests.challenges.challenge_decorator.challenge_decorator import challenge +from tests.challenges.utils import generate_noise, run_interaction_loop from tests.utils import requires_api_key NOISE = 1000 diff --git a/tests/integration/challenges/memory/test_memory_challenge_c.py b/tests/challenges/memory/test_memory_challenge_c.py similarity index 95% rename from tests/integration/challenges/memory/test_memory_challenge_c.py rename to tests/challenges/memory/test_memory_challenge_c.py index ab8ece10526..d7cc6994057 100644 --- a/tests/integration/challenges/memory/test_memory_challenge_c.py +++ b/tests/challenges/memory/test_memory_challenge_c.py @@ -4,10 +4,8 @@ from autogpt.agent import Agent from autogpt.commands.file_operations import read_file, write_to_file from autogpt.config import Config -from tests.integration.challenges.challenge_decorator.challenge_decorator import ( - challenge, -) -from tests.integration.challenges.utils import generate_noise, run_interaction_loop +from tests.challenges.challenge_decorator.challenge_decorator import challenge +from tests.challenges.utils import generate_noise, run_interaction_loop from tests.utils import requires_api_key NOISE = 1000 diff --git a/tests/integration/challenges/test_challenge_should_be_formatted_properly.py b/tests/challenges/test_challenge_should_be_formatted_properly.py similarity index 100% rename from tests/integration/challenges/test_challenge_should_be_formatted_properly.py rename to tests/challenges/test_challenge_should_be_formatted_properly.py diff --git a/tests/integration/challenges/utils.py b/tests/challenges/utils.py similarity index 100% rename from tests/integration/challenges/utils.py rename to tests/challenges/utils.py diff --git a/tests/integration/challenges/utils/build_current_score.py b/tests/challenges/utils/build_current_score.py similarity index 84% rename from tests/integration/challenges/utils/build_current_score.py rename to tests/challenges/utils/build_current_score.py index 743b1328424..aec125b4004 100644 --- a/tests/integration/challenges/utils/build_current_score.py +++ b/tests/challenges/utils/build_current_score.py @@ -26,12 +26,8 @@ def recursive_sort_dict(data: dict) -> dict: cwd = os.getcwd() # get current working directory -new_score_filename_pattern = os.path.join( - cwd, "tests/integration/challenges/new_score_*.json" -) -current_score_filename = os.path.join( - cwd, "tests/integration/challenges/current_score.json" -) +new_score_filename_pattern = os.path.join(cwd, "tests/challenges/new_score_*.json") +current_score_filename = os.path.join(cwd, "tests/challenges/current_score.json") merged_data: Dict[str, Any] = {} for filename in glob.glob(new_score_filename_pattern): diff --git a/tests/conftest.py b/tests/conftest.py index 98bebc9e34c..0ee023b58d8 100644 --- a/tests/conftest.py +++ b/tests/conftest.py @@ -1,4 +1,3 @@ -import os from pathlib import Path import pytest @@ -8,15 +7,11 @@ from autogpt.llm.api_manager import ApiManager from autogpt.workspace import Workspace -pytest_plugins = ["tests.integration.agent_factory", "tests.integration.memory.utils"] - -PROXY = os.environ.get("PROXY") - - -@pytest.fixture() -def vcr_cassette_dir(request): - test_name = os.path.splitext(request.node.name)[0] - return os.path.join("tests/Auto-GPT-test-cassettes", test_name) +pytest_plugins = [ + "tests.integration.agent_factory", + "tests.integration.memory.utils", + "tests.vcr", +] @pytest.fixture() diff --git a/tests/integration/conftest.py b/tests/integration/conftest.py deleted file mode 100644 index 686f50be405..00000000000 --- a/tests/integration/conftest.py +++ /dev/null @@ -1,56 +0,0 @@ -import os - -import openai.api_requestor -import pytest -from pytest_mock import MockerFixture - -from tests.conftest import PROXY -from tests.vcr.vcr_filter import before_record_request, before_record_response - -BASE_VCR_CONFIG = { - "record_mode": "new_episodes", - "before_record_request": before_record_request, - "before_record_response": before_record_response, - "filter_headers": [ - "Authorization", - "X-OpenAI-Client-User-Agent", - "User-Agent", - ], - "match_on": ["method", "body"], -} - - -@pytest.fixture(scope="session") -def vcr_config(): - # this fixture is called by the pytest-recording vcr decorator. - return BASE_VCR_CONFIG - - -def patch_api_base(requestor): - new_api_base = f"{PROXY}/v1" - requestor.api_base = new_api_base - return requestor - - -@pytest.fixture -def patched_api_requestor(mocker: MockerFixture): - original_init = openai.api_requestor.APIRequestor.__init__ - original_validate_headers = openai.api_requestor.APIRequestor._validate_headers - - def patched_init(requestor, *args, **kwargs): - original_init(requestor, *args, **kwargs) - patch_api_base(requestor) - - def patched_validate_headers(self, supplied_headers): - headers = original_validate_headers(self, supplied_headers) - headers["AGENT-MODE"] = os.environ.get("AGENT_MODE") - headers["AGENT-TYPE"] = os.environ.get("AGENT_TYPE") - return headers - - if PROXY: - mocker.patch("openai.api_requestor.APIRequestor.__init__", new=patched_init) - mocker.patch.object( - openai.api_requestor.APIRequestor, - "_validate_headers", - new=patched_validate_headers, - ) diff --git a/tests/integration/test_commands.py b/tests/integration/test_commands.py deleted file mode 100644 index 1cbb3929667..00000000000 --- a/tests/integration/test_commands.py +++ /dev/null @@ -1,32 +0,0 @@ -"""Unit tests for the commands module""" -from unittest.mock import MagicMock, patch - -import pytest - -from autogpt.app import list_agents, start_agent -from tests.utils import requires_api_key - - -@pytest.mark.vcr -@pytest.mark.integration_test -@requires_api_key("OPENAI_API_KEY") -def test_make_agent(patched_api_requestor, config) -> None: - """Test that an agent can be created""" - # Use the mock agent manager to avoid creating a real agent - with patch("openai.ChatCompletion.create") as mock: - response = MagicMock() - # del response.error - response.choices[0].messages[0].content = "Test message" - response.usage.prompt_tokens = 1 - response.usage.completion_tokens = 1 - mock.return_value = response - start_agent( - "Test Agent", "chat", "Hello, how are you?", config, "gpt-3.5-turbo" - ) - agents = list_agents(config) - assert "List of agents:\n0: chat" == agents - start_agent( - "Test Agent 2", "write", "Hello, how are you?", config, "gpt-3.5-turbo" - ) - agents = list_agents(config) - assert "List of agents:\n0: chat\n1: write" == agents diff --git a/tests/test_analyze_code.py b/tests/test_analyze_code.py deleted file mode 100644 index 98ab8b724dd..00000000000 --- a/tests/test_analyze_code.py +++ /dev/null @@ -1,74 +0,0 @@ -# Date: 2023-5-13 -# Author: Generated by GoCodeo. -import pytest - -from autogpt.commands.analyze_code import analyze_code -from autogpt.config import Config - - -@pytest.fixture -def mock_call_ai_function(mocker): - return mocker.patch("autogpt.commands.analyze_code.call_ai_function") - - -class TestAnalyzeCode: - def test_positive_analyze_code(self, mock_call_ai_function): - # Positive Test - mock_call_ai_function.return_value = ["Suggestion 1", "Suggestion 2"] - code = "def example_function():\n pass" - config = Config() - result = analyze_code(code, config) - assert result == ["Suggestion 1", "Suggestion 2"] - mock_call_ai_function.assert_called_once_with( - "def analyze_code(code: str) -> list[str]:", - [code], - "Analyzes the given code and returns a list of suggestions for improvements.", - config=config, - ) - - def test_negative_analyze_code( - self, - mock_call_ai_function, - config: Config, - ): - # Negative Test - mock_call_ai_function.return_value = [] - code = "def example_function():\n pass" - result = analyze_code(code, config) - assert result == [] - mock_call_ai_function.assert_called_once_with( - "def analyze_code(code: str) -> list[str]:", - [code], - "Analyzes the given code and returns a list of suggestions for improvements.", - config=config, - ) - - def test_error_analyze_code(self, mock_call_ai_function, config: Config): - # Error Test - mock_call_ai_function.side_effect = Exception("Error occurred") - code = "def example_function():\n pass" - with pytest.raises(Exception): - result = analyze_code(code, config) - mock_call_ai_function.assert_called_once_with( - "def analyze_code(code: str) -> list[str]:", - [code], - "Analyzes the given code and returns a list of suggestions for improvements.", - config=config, - ) - - def test_edge_analyze_code_empty_code( - self, - mock_call_ai_function, - config: Config, - ): - # Edge Test - mock_call_ai_function.return_value = ["Suggestion 1", "Suggestion 2"] - code = "" - result = analyze_code(code, config) - assert result == ["Suggestion 1", "Suggestion 2"] - mock_call_ai_function.assert_called_once_with( - "def analyze_code(code: str) -> list[str]:", - [code], - "Analyzes the given code and returns a list of suggestions for improvements.", - config=config, - ) diff --git a/tests/test_audio_text_read_audio.py b/tests/test_audio_text_read_audio.py deleted file mode 100644 index 4385da321cc..00000000000 --- a/tests/test_audio_text_read_audio.py +++ /dev/null @@ -1,56 +0,0 @@ -# Date: 2023-5-13 -# Author: Generated by GoCodeo. -import json -from unittest.mock import MagicMock, patch - -import pytest - -from autogpt.commands.audio_text import read_audio - - -class TestReadAudio: - @patch("requests.post") - def test_positive_read_audio(self, mock_post, config): - # Positive Test - audio_data = b"test_audio_data" - mock_response = MagicMock() - mock_response.content.decode.return_value = json.dumps( - {"text": "Hello, world!"} - ) - mock_post.return_value = mock_response - - config.huggingface_api_token = "testing-token" - result = read_audio(audio_data, config) - assert result == "The audio says: Hello, world!" - mock_post.assert_called_once_with( - f"https://api-inference.huggingface.co/models/{config.huggingface_audio_to_text_model}", - headers={"Authorization": f"Bearer {config.huggingface_api_token}"}, - data=audio_data, - ) - - @patch("requests.post") - def test_negative_read_audio(self, mock_post, config): - # Negative Test - audio_data = b"test_audio_data" - mock_response = MagicMock() - mock_response.content.decode.return_value = json.dumps({"text": ""}) - mock_post.return_value = mock_response - config.huggingface_api_token = "testing-token" - result = read_audio(audio_data, config) - assert result == "The audio says: " - mock_post.assert_called_once_with( - f"https://api-inference.huggingface.co/models/{config.huggingface_audio_to_text_model}", - headers={"Authorization": f"Bearer {config.huggingface_api_token}"}, - data=audio_data, - ) - - def test_error_read_audio(self, config): - # Error Test - config.huggingface_api_token = None - with pytest.raises(ValueError): - read_audio(b"test_audio_data", config) - - def test_edge_read_audio_empty_audio(self, config): - # Edge Test - with pytest.raises(ValueError): - read_audio(b"", config) diff --git a/tests/test_audio_text_read_audio_from_file.py b/tests/test_audio_text_read_audio_from_file.py deleted file mode 100644 index c8d66a06085..00000000000 --- a/tests/test_audio_text_read_audio_from_file.py +++ /dev/null @@ -1,55 +0,0 @@ -# Date: 2023-5-13 -# Author: Generated by GoCodeo. - - -from unittest.mock import mock_open, patch - -import pytest - -from autogpt.commands.audio_text import read_audio_from_file -from autogpt.config import Config - - -@pytest.fixture -def mock_read_audio(mocker): - return mocker.patch("autogpt.commands.audio_text.read_audio") - - -class TestReadAudioFromFile: - def test_positive_read_audio_from_file(self, mock_read_audio): - # Positive test - mock_read_audio.return_value = "This is a sample text." - mock_file_data = b"Audio data" - m = mock_open(read_data=mock_file_data) - - with patch("builtins.open", m): - result = read_audio_from_file("test_audio.wav", Config()) - assert result == "This is a sample text." - m.assert_called_once_with("test_audio.wav", "rb") - - def test_negative_read_audio_from_file(self, mock_read_audio): - # Negative test - mock_read_audio.return_value = "This is a sample text." - mock_file_data = b"Audio data" - m = mock_open(read_data=mock_file_data) - - with patch("builtins.open", m): - result = read_audio_from_file("test_audio.wav", Config()) - assert result != "Incorrect text." - m.assert_called_once_with("test_audio.wav", "rb") - - def test_error_read_audio_from_file(self): - # Error test - with pytest.raises(FileNotFoundError): - read_audio_from_file("non_existent_file.wav", Config()) - - def test_edge_empty_audio_file(self, mock_read_audio): - # Edge test - mock_read_audio.return_value = "" - mock_file_data = b"" - m = mock_open(read_data=mock_file_data) - - with patch("builtins.open", m): - result = read_audio_from_file("empty_audio.wav", Config()) - assert result == "" - m.assert_called_once_with("empty_audio.wav", "rb") diff --git a/tests/test_agent.py b/tests/unit/test_agent.py similarity index 100% rename from tests/test_agent.py rename to tests/unit/test_agent.py diff --git a/tests/test_agent_manager.py b/tests/unit/test_agent_manager.py similarity index 100% rename from tests/test_agent_manager.py rename to tests/unit/test_agent_manager.py diff --git a/tests/test_ai_config.py b/tests/unit/test_ai_config.py similarity index 100% rename from tests/test_ai_config.py rename to tests/unit/test_ai_config.py diff --git a/tests/test_api_manager.py b/tests/unit/test_api_manager.py similarity index 100% rename from tests/test_api_manager.py rename to tests/unit/test_api_manager.py diff --git a/tests/test_commands.py b/tests/unit/test_commands.py similarity index 100% rename from tests/test_commands.py rename to tests/unit/test_commands.py diff --git a/tests/test_config.py b/tests/unit/test_config.py similarity index 100% rename from tests/test_config.py rename to tests/unit/test_config.py diff --git a/tests/integration/test_git_commands.py b/tests/unit/test_git_commands.py similarity index 100% rename from tests/integration/test_git_commands.py rename to tests/unit/test_git_commands.py diff --git a/tests/integration/test_google_search.py b/tests/unit/test_google_search.py similarity index 100% rename from tests/integration/test_google_search.py rename to tests/unit/test_google_search.py diff --git a/tests/test_logs.py b/tests/unit/test_logs.py similarity index 100% rename from tests/test_logs.py rename to tests/unit/test_logs.py diff --git a/tests/unit/test_make_agent.py b/tests/unit/test_make_agent.py new file mode 100644 index 00000000000..cff20ee399e --- /dev/null +++ b/tests/unit/test_make_agent.py @@ -0,0 +1,24 @@ +from unittest.mock import MagicMock + +from pytest_mock import MockerFixture + +from autogpt.app import list_agents, start_agent +from autogpt.config import Config + + +def test_make_agent(config: Config, mocker: MockerFixture) -> None: + """Test that an agent can be created""" + mock = mocker.patch("openai.ChatCompletion.create") + + response = MagicMock() + # del response.error + response.choices[0].messages[0].content = "Test message" + response.usage.prompt_tokens = 1 + response.usage.completion_tokens = 1 + mock.return_value = response + start_agent("Test Agent", "chat", "Hello, how are you?", config, "gpt-3.5-turbo") + agents = list_agents(config) + assert "List of agents:\n0: chat" == agents + start_agent("Test Agent 2", "write", "Hello, how are you?", config, "gpt-3.5-turbo") + agents = list_agents(config) + assert "List of agents:\n0: chat\n1: write" == agents diff --git a/tests/test_prompt_config.py b/tests/unit/test_prompt_config.py similarity index 100% rename from tests/test_prompt_config.py rename to tests/unit/test_prompt_config.py diff --git a/tests/test_prompt_generator.py b/tests/unit/test_prompt_generator.py similarity index 100% rename from tests/test_prompt_generator.py rename to tests/unit/test_prompt_generator.py diff --git a/tests/test_text_file_parsers.py b/tests/unit/test_text_file_parsers.py similarity index 100% rename from tests/test_text_file_parsers.py rename to tests/unit/test_text_file_parsers.py diff --git a/tests/unit/test_url_validation.py b/tests/unit/test_url_validation.py index 16eb8cd5008..5d6e8124e91 100644 --- a/tests/unit/test_url_validation.py +++ b/tests/unit/test_url_validation.py @@ -49,25 +49,17 @@ def test_url_validation_succeeds(url): assert dummy_method(url) == url -bad_protocol_data = ( - ("htt://example.com"), - ("httppp://example.com"), - (" https://example.com"), +@pytest.mark.parametrize( + "url,expected_error", + [ + ("htt://example.com", "Invalid URL format"), + ("httppp://example.com", "Invalid URL format"), + (" https://example.com", "Invalid URL format"), + ("http://?query=q", "Missing Scheme or Network location"), + ], ) - - -@pytest.mark.parametrize("url", bad_protocol_data) -def test_url_validation_fails_bad_protocol(url): - with raises(ValueError, match="Invalid URL format"): - dummy_method(url) - - -missing_loc = (("http://?query=q"),) - - -@pytest.mark.parametrize("url", missing_loc) -def test_url_validation_fails_bad_protocol(url): - with raises(ValueError, match="Missing Scheme or Network location"): +def test_url_validation_fails_invalid_url(url, expected_error): + with raises(ValueError, match=expected_error): dummy_method(url) diff --git a/tests/test_utils.py b/tests/unit/test_utils.py similarity index 98% rename from tests/test_utils.py rename to tests/unit/test_utils.py index c0ce28cc1fe..099176baee7 100644 --- a/tests/test_utils.py +++ b/tests/unit/test_utils.py @@ -1,7 +1,6 @@ import os from unittest.mock import patch -import pytest import requests from autogpt.utils import ( @@ -151,7 +150,3 @@ def test_get_current_git_branch_failure(mock_repo): branch_name = get_current_git_branch() assert branch_name == "" - - -if __name__ == "__main__": - pytest.main() diff --git a/tests/test_workspace.py b/tests/unit/test_workspace.py similarity index 100% rename from tests/test_workspace.py rename to tests/unit/test_workspace.py diff --git a/tests/vcr/__init__.py b/tests/vcr/__init__.py index e69de29bb2d..e1a2620ca70 100644 --- a/tests/vcr/__init__.py +++ b/tests/vcr/__init__.py @@ -0,0 +1,61 @@ +import os + +import openai.api_requestor +import pytest +from pytest_mock import MockerFixture + +from .vcr_filter import PROXY, before_record_request, before_record_response + +BASE_VCR_CONFIG = { + "record_mode": "new_episodes", + "before_record_request": before_record_request, + "before_record_response": before_record_response, + "filter_headers": [ + "Authorization", + "X-OpenAI-Client-User-Agent", + "User-Agent", + ], + "match_on": ["method", "body"], +} + + +@pytest.fixture(scope="session") +def vcr_config(): + # this fixture is called by the pytest-recording vcr decorator. + return BASE_VCR_CONFIG + + +@pytest.fixture() +def vcr_cassette_dir(request): + test_name = os.path.splitext(request.node.name)[0] + return os.path.join("tests/Auto-GPT-test-cassettes", test_name) + + +def patch_api_base(requestor): + new_api_base = f"{PROXY}/v1" + requestor.api_base = new_api_base + return requestor + + +@pytest.fixture +def patched_api_requestor(mocker: MockerFixture): + original_init = openai.api_requestor.APIRequestor.__init__ + original_validate_headers = openai.api_requestor.APIRequestor._validate_headers + + def patched_init(requestor, *args, **kwargs): + original_init(requestor, *args, **kwargs) + patch_api_base(requestor) + + def patched_validate_headers(self, supplied_headers): + headers = original_validate_headers(self, supplied_headers) + headers["AGENT-MODE"] = os.environ.get("AGENT_MODE") + headers["AGENT-TYPE"] = os.environ.get("AGENT_TYPE") + return headers + + if PROXY: + mocker.patch("openai.api_requestor.APIRequestor.__init__", new=patched_init) + mocker.patch.object( + openai.api_requestor.APIRequestor, + "_validate_headers", + new=patched_validate_headers, + ) diff --git a/tests/vcr/vcr_filter.py b/tests/vcr/vcr_filter.py index 4cc49fd3019..1ba433a76ec 100644 --- a/tests/vcr/vcr_filter.py +++ b/tests/vcr/vcr_filter.py @@ -1,8 +1,9 @@ import json +import os import re from typing import Any, Dict, List -from tests.conftest import PROXY +PROXY = os.environ.get("PROXY") REPLACEMENTS: List[Dict[str, str]] = [ {