diff --git a/.vscode/settings.json b/.vscode/settings.json index 612f4083a3db..d818d30862a2 100644 --- a/.vscode/settings.json +++ b/.vscode/settings.json @@ -23,7 +23,10 @@ ".vscode test": true }, "[python]": { - "editor.formatOnSave": true + "editor.formatOnSave": true, + "editor.codeActionsOnSave": { + "source.organizeImports": true + } }, "[typescript]": { "editor.defaultFormatter": "esbenp.prettier-vscode", diff --git a/news/1 Enhancements/17242.md b/news/1 Enhancements/17242.md new file mode 100644 index 000000000000..204de1ba5158 --- /dev/null +++ b/news/1 Enhancements/17242.md @@ -0,0 +1 @@ +Rewrite support for unittest test discovery. diff --git a/package.nls.json b/package.nls.json index 0feacbd3df62..c1c90bcbb5bd 100644 --- a/package.nls.json +++ b/package.nls.json @@ -71,6 +71,7 @@ "Common.moreInfo": "More Info", "Common.and": "and", "Common.ok": "Ok", + "Common.error": "Error", "Common.install": "Install", "Common.learnMore": "Learn more", "Common.reportThisIssue": "Report this issue", @@ -157,6 +158,9 @@ "debug.pyramidEnterDevelopmentIniPathInvalidFilePathError": "Enter a valid file path", "Testing.configureTests": "Configure Test Framework", "Testing.testNotConfigured": "No test framework configured.", + "Testing.cancelUnittestDiscovery": "Canceled unittest test discovery", + "Testing.errorUnittestDiscovery": "Unittest test discovery error", + "Testing.seePythonOutput": "(see Output > Python)", "Common.openOutputPanel": "Show output", "LanguageService.statusItem.name": "Python IntelliSense Status", "LanguageService.statusItem.text": "Partial Mode", diff --git a/pythonFiles/testing_tools/socket_manager.py b/pythonFiles/testing_tools/socket_manager.py new file mode 100644 index 000000000000..372a50b5e012 --- /dev/null +++ b/pythonFiles/testing_tools/socket_manager.py @@ -0,0 +1,44 @@ +# Copyright (c) Microsoft Corporation. All rights reserved. +# Licensed under the MIT License. + +import socket +import sys + + +class SocketManager(object): + """Create a socket and connect to the given address. + + The address is a (host: str, port: int) tuple. + Example usage: + + ``` + with SocketManager(("localhost", 6767)) as sock: + request = json.dumps(payload) + result = s.socket.sendall(request.encode("utf-8")) + ``` + """ + + def __init__(self, addr): + self.addr = addr + self.socket = None + + def __enter__(self): + self.socket = socket.socket( + socket.AF_INET, socket.SOCK_STREAM, socket.IPPROTO_TCP + ) + if sys.platform == "win32": + addr_use = socket.SO_EXCLUSIVEADDRUSE + else: + addr_use = socket.SO_REUSEADDR + self.socket.setsockopt(socket.SOL_SOCKET, addr_use, 1) + self.socket.connect(self.addr) + + return self + + def __exit__(self, *_): + if self.socket: + try: + self.socket.shutdown(socket.SHUT_RDWR) + except Exception: + pass + self.socket.close() diff --git a/pythonFiles/tests/unittestadapter/.data/discovery_empty.py b/pythonFiles/tests/unittestadapter/.data/discovery_empty.py new file mode 100644 index 000000000000..9af5071303ce --- /dev/null +++ b/pythonFiles/tests/unittestadapter/.data/discovery_empty.py @@ -0,0 +1,15 @@ +# Copyright (c) Microsoft Corporation. All rights reserved. +# Licensed under the MIT License. + +import unittest + + +class DiscoveryEmpty(unittest.TestCase): + """Test class for the test_empty_discovery test. + + The discover_tests function should return a dictionary with a "success" status, no errors, and no test tree + if unittest discovery was performed successfully but no tests were found. + """ + + def something(self) -> bool: + return True diff --git a/pythonFiles/tests/unittestadapter/.data/discovery_error/file_one.py b/pythonFiles/tests/unittestadapter/.data/discovery_error/file_one.py new file mode 100644 index 000000000000..42f84f046760 --- /dev/null +++ b/pythonFiles/tests/unittestadapter/.data/discovery_error/file_one.py @@ -0,0 +1,20 @@ +# Copyright (c) Microsoft Corporation. All rights reserved. +# Licensed under the MIT License. + +import unittest + +import something_else # type: ignore + + +class DiscoveryErrorOne(unittest.TestCase): + """Test class for the test_error_discovery test. + + The discover_tests function should return a dictionary with an "error" status, the discovered tests, and a list of errors + if unittest discovery failed at some point. + """ + + def test_one(self) -> None: + self.assertGreater(2, 1) + + def test_two(self) -> None: + self.assertNotEqual(2, 1) diff --git a/pythonFiles/tests/unittestadapter/.data/discovery_error/file_two.py b/pythonFiles/tests/unittestadapter/.data/discovery_error/file_two.py new file mode 100644 index 000000000000..5d6d54f886a1 --- /dev/null +++ b/pythonFiles/tests/unittestadapter/.data/discovery_error/file_two.py @@ -0,0 +1,18 @@ +# Copyright (c) Microsoft Corporation. All rights reserved. +# Licensed under the MIT License. + +import unittest + + +class DiscoveryErrorTwo(unittest.TestCase): + """Test class for the test_error_discovery test. + + The discover_tests function should return a dictionary with an "error" status, the discovered tests, and a list of errors + if unittest discovery failed at some point. + """ + + def test_one(self) -> None: + self.assertGreater(2, 1) + + def test_two(self) -> None: + self.assertNotEqual(2, 1) diff --git a/pythonFiles/tests/unittestadapter/.data/discovery_simple.py b/pythonFiles/tests/unittestadapter/.data/discovery_simple.py new file mode 100644 index 000000000000..1859436d5b5b --- /dev/null +++ b/pythonFiles/tests/unittestadapter/.data/discovery_simple.py @@ -0,0 +1,18 @@ +# Copyright (c) Microsoft Corporation. All rights reserved. +# Licensed under the MIT License. + +import unittest + + +class DiscoverySimple(unittest.TestCase): + """Test class for the test_simple_discovery test. + + The discover_tests function should return a dictionary with a "success" status, no errors, and a test tree + if unittest discovery was performed successfully. + """ + + def test_one(self) -> None: + self.assertGreater(2, 1) + + def test_two(self) -> None: + self.assertNotEqual(2, 1) diff --git a/pythonFiles/tests/unittestadapter/.data/utils_decorated_tree.py b/pythonFiles/tests/unittestadapter/.data/utils_decorated_tree.py new file mode 100644 index 000000000000..90fdfc89a27b --- /dev/null +++ b/pythonFiles/tests/unittestadapter/.data/utils_decorated_tree.py @@ -0,0 +1,29 @@ +# Copyright (c) Microsoft Corporation. All rights reserved. +# Licensed under the MIT License. + +import unittest +from functools import wraps + + +def my_decorator(f): + @wraps(f) + def wrapper(*args, **kwds): + print("Calling decorated function") + return f(*args, **kwds) + + return wrapper + + +class TreeOne(unittest.TestCase): + """Test class for the test_build_decorated_tree test. + + build_test_tree should build a test tree with these test cases. + """ + + @my_decorator + def test_one(self) -> None: + self.assertGreater(2, 1) + + @my_decorator + def test_two(self) -> None: + self.assertNotEqual(2, 1) diff --git a/pythonFiles/tests/unittestadapter/.data/utils_nested_cases/file_one.py b/pythonFiles/tests/unittestadapter/.data/utils_nested_cases/file_one.py new file mode 100644 index 000000000000..84f7fefc4ebd --- /dev/null +++ b/pythonFiles/tests/unittestadapter/.data/utils_nested_cases/file_one.py @@ -0,0 +1,17 @@ +# Copyright (c) Microsoft Corporation. All rights reserved. +# Licensed under the MIT License. + +import unittest + + +class CaseTwoFileOne(unittest.TestCase): + """Test class for the test_nested_test_cases test. + + get_test_case should return tests from the test suites in this folder. + """ + + def test_one(self) -> None: + self.assertGreater(2, 1) + + def test_two(self) -> None: + self.assertNotEqual(2, 1) diff --git a/pythonFiles/tests/unittestadapter/.data/utils_nested_cases/folder/__init__.py b/pythonFiles/tests/unittestadapter/.data/utils_nested_cases/folder/__init__.py new file mode 100644 index 000000000000..5b7f7a925cc0 --- /dev/null +++ b/pythonFiles/tests/unittestadapter/.data/utils_nested_cases/folder/__init__.py @@ -0,0 +1,2 @@ +# Copyright (c) Microsoft Corporation. All rights reserved. +# Licensed under the MIT License. diff --git a/pythonFiles/tests/unittestadapter/.data/utils_nested_cases/folder/file_two.py b/pythonFiles/tests/unittestadapter/.data/utils_nested_cases/folder/file_two.py new file mode 100644 index 000000000000..235a104016a3 --- /dev/null +++ b/pythonFiles/tests/unittestadapter/.data/utils_nested_cases/folder/file_two.py @@ -0,0 +1,17 @@ +# Copyright (c) Microsoft Corporation. All rights reserved. +# Licensed under the MIT License. + +import unittest + + +class CaseTwoFileTwo(unittest.TestCase): + """Test class for the test_nested_test_cases test. + + get_test_case should return tests from the test suites in this folder. + """ + + def test_one(self) -> None: + self.assertGreater(2, 1) + + def test_two(self) -> None: + self.assertNotEqual(2, 1) diff --git a/pythonFiles/tests/unittestadapter/.data/utils_simple_cases.py b/pythonFiles/tests/unittestadapter/.data/utils_simple_cases.py new file mode 100644 index 000000000000..fb3ae7eb7909 --- /dev/null +++ b/pythonFiles/tests/unittestadapter/.data/utils_simple_cases.py @@ -0,0 +1,17 @@ +# Copyright (c) Microsoft Corporation. All rights reserved. +# Licensed under the MIT License. + +import unittest + + +class CaseOne(unittest.TestCase): + """Test class for the test_simple_test_cases test. + + get_test_case should return tests from the test suite. + """ + + def test_one(self) -> None: + self.assertGreater(2, 1) + + def test_two(self) -> None: + self.assertNotEqual(2, 1) diff --git a/pythonFiles/tests/unittestadapter/.data/utils_simple_tree.py b/pythonFiles/tests/unittestadapter/.data/utils_simple_tree.py new file mode 100644 index 000000000000..6db51a4fd80b --- /dev/null +++ b/pythonFiles/tests/unittestadapter/.data/utils_simple_tree.py @@ -0,0 +1,17 @@ +# Copyright (c) Microsoft Corporation. All rights reserved. +# Licensed under the MIT License. + +import unittest + + +class TreeOne(unittest.TestCase): + """Test class for the test_build_simple_tree test. + + build_test_tree should build a test tree with these test cases. + """ + + def test_one(self) -> None: + self.assertGreater(2, 1) + + def test_two(self) -> None: + self.assertNotEqual(2, 1) diff --git a/pythonFiles/tests/unittestadapter/__init__.py b/pythonFiles/tests/unittestadapter/__init__.py new file mode 100644 index 000000000000..5b7f7a925cc0 --- /dev/null +++ b/pythonFiles/tests/unittestadapter/__init__.py @@ -0,0 +1,2 @@ +# Copyright (c) Microsoft Corporation. All rights reserved. +# Licensed under the MIT License. diff --git a/pythonFiles/tests/unittestadapter/conftest.py b/pythonFiles/tests/unittestadapter/conftest.py new file mode 100644 index 000000000000..19af85d1e095 --- /dev/null +++ b/pythonFiles/tests/unittestadapter/conftest.py @@ -0,0 +1,8 @@ +# Copyright (c) Microsoft Corporation. All rights reserved. +# Licensed under the MIT License. + +import sys + +# Ignore the contents of this folder for Python 2 tests. +if sys.version_info[0] < 3: + collect_ignore_glob = ["*.py"] diff --git a/pythonFiles/tests/unittestadapter/helpers.py b/pythonFiles/tests/unittestadapter/helpers.py new file mode 100644 index 000000000000..303d021368f7 --- /dev/null +++ b/pythonFiles/tests/unittestadapter/helpers.py @@ -0,0 +1,32 @@ +# Copyright (c) Microsoft Corporation. All rights reserved. +# Licensed under the MIT License. + +import pathlib + +TEST_DATA_PATH = pathlib.Path(__file__).parent / ".data" + + +def is_same_tree(tree1, tree2) -> bool: + """Helper function to test if two test trees are the same. + + `is_same_tree` starts by comparing the root attributes, and then checks if all children are the same. + """ + # Compare the root. + if any(tree1[key] != tree2[key] for key in ["path", "name", "type_"]): + return False + + # Compare child test nodes if they exist, otherwise compare test items. + if "children" in tree1 and "children" in tree2: + children1 = tree1["children"] + children2 = tree2["children"] + + # Compare test nodes. + if len(children1) != len(children2): + return False + else: + return all(is_same_tree(*pair) for pair in zip(children1, children2)) + elif "id_" in tree1 and "id_" in tree2: + # Compare test items. + return all(tree1[key] == tree2[key] for key in ["id_", "lineno"]) + + return False diff --git a/pythonFiles/tests/unittestadapter/test_discovery.py b/pythonFiles/tests/unittestadapter/test_discovery.py new file mode 100644 index 000000000000..9ffe8cd7462d --- /dev/null +++ b/pythonFiles/tests/unittestadapter/test_discovery.py @@ -0,0 +1,197 @@ +# Copyright (c) Microsoft Corporation. All rights reserved. +# Licensed under the MIT License. + +import os +import pathlib + +import pytest +from unittestadapter.discovery import ( + DEFAULT_PORT, + discover_tests, + parse_cli_args, + parse_unittest_args, +) +from unittestadapter.utils import TestNodeTypeEnum + +from .helpers import TEST_DATA_PATH, is_same_tree + + +@pytest.mark.parametrize( + "args, expected", + [ + (["--port", "6767", "--uuid", "some-uuid"], (6767, "some-uuid")), + (["--foo", "something", "--bar", "another"], (int(DEFAULT_PORT), None)), + (["--port", "4444", "--foo", "something", "--port", "9999"], (9999, None)), + ( + ["--uuid", "first-uuid", "--bar", "other", "--uuid", "second-uuid"], + (int(DEFAULT_PORT), "second-uuid"), + ), + ], +) +def test_parse_cli_args(args, expected) -> None: + """The parse_cli_args function should parse and return the port and uuid passed as command-line options. + + If there were no --port or --uuid command-line option, it should return default values). + If there are multiple options, the last one wins. + """ + actual = parse_cli_args(args) + + assert expected == actual + + +@pytest.mark.parametrize( + "args, expected", + [ + ( + ["-s", "something", "-p", "other*", "-t", "else"], + ("something", "other*", "else"), + ), + ( + [ + "--start-directory", + "foo", + "--pattern", + "bar*", + "--top-level-directory", + "baz", + ], + ("foo", "bar*", "baz"), + ), + ( + ["--foo", "something"], + (".", "test*.py", None), + ), + ], +) +def test_parse_unittest_args(args, expected) -> None: + """The parse_unittest_args function should return values for the start_dir, pattern, and top_level_dir arguments + when passed as command-line options, and ignore unrecognized arguments. + """ + actual = parse_unittest_args(args) + + assert actual == expected + + +def test_simple_discovery() -> None: + """The discover_tests function should return a dictionary with a "success" status, a uuid, no errors, and a test tree + if unittest discovery was performed successfully. + """ + start_dir = os.fsdecode(TEST_DATA_PATH) + pattern = "discovery_simple*" + file_path = os.fsdecode(pathlib.PurePath(TEST_DATA_PATH / "discovery_simple.py")) + + expected = { + "path": start_dir, + "type_": TestNodeTypeEnum.folder, + "name": ".data", + "children": [ + { + "name": "discovery_simple.py", + "type_": TestNodeTypeEnum.file, + "path": file_path, + "children": [ + { + "name": "DiscoverySimple", + "path": file_path, + "type_": TestNodeTypeEnum.class_, + "children": [ + { + "id_": "discovery_simple.DiscoverySimple.test_one", + "name": "test_one", + "path": file_path, + "type_": TestNodeTypeEnum.test, + "lineno": "14", + }, + { + "id_": "discovery_simple.DiscoverySimple.test_two", + "name": "test_two", + "path": file_path, + "type_": TestNodeTypeEnum.test, + "lineno": "17", + }, + ], + } + ], + } + ], + } + + uuid = "some-uuid" + actual = discover_tests(start_dir, pattern, None, uuid) + + assert actual["status"] == "success" + assert actual["uuid"] == uuid + assert is_same_tree(actual.get("tests"), expected) + assert "errors" not in actual + + +def test_empty_discovery() -> None: + """The discover_tests function should return a dictionary with a "success" status, a uuid, no errors, and no test tree + if unittest discovery was performed successfully but no tests were found. + """ + start_dir = os.fsdecode(TEST_DATA_PATH) + pattern = "discovery_empty*" + + uuid = "some-uuid" + actual = discover_tests(start_dir, pattern, None, uuid) + + assert actual["status"] == "success" + assert actual["uuid"] == uuid + assert "tests" not in actual + assert "errors" not in actual + + +def test_error_discovery() -> None: + """The discover_tests function should return a dictionary with an "error" status, a uuid, the discovered tests, and a list of errors + if unittest discovery failed at some point. + """ + # Discover tests in .data/discovery_error/. + start_path = pathlib.PurePath(TEST_DATA_PATH / "discovery_error") + start_dir = os.fsdecode(start_path) + pattern = "file*" + + file_path = os.fsdecode(start_path / "file_two.py") + + expected = { + "path": start_dir, + "type_": TestNodeTypeEnum.folder, + "name": "discovery_error", + "children": [ + { + "name": "file_two.py", + "type_": TestNodeTypeEnum.file, + "path": file_path, + "children": [ + { + "name": "DiscoveryErrorTwo", + "path": file_path, + "type_": TestNodeTypeEnum.class_, + "children": [ + { + "id_": "file_two.DiscoveryErrorTwo.test_one", + "name": "test_one", + "path": file_path, + "type_": TestNodeTypeEnum.test, + "lineno": "14", + }, + { + "id_": "file_two.DiscoveryErrorTwo.test_two", + "name": "test_two", + "path": file_path, + "type_": TestNodeTypeEnum.test, + "lineno": "17", + }, + ], + } + ], + } + ], + } + + uuid = "some-uuid" + actual = discover_tests(start_dir, pattern, None, uuid) + + assert actual["status"] == "error" + assert actual["uuid"] == uuid + assert is_same_tree(expected, actual.get("tests")) + assert len(actual.get("errors", [])) == 1 diff --git a/pythonFiles/tests/unittestadapter/test_utils.py b/pythonFiles/tests/unittestadapter/test_utils.py new file mode 100644 index 000000000000..cb9a9b3344a7 --- /dev/null +++ b/pythonFiles/tests/unittestadapter/test_utils.py @@ -0,0 +1,279 @@ +# Copyright (c) Microsoft Corporation. All rights reserved. +# Licensed under the MIT License. + +import os +import pathlib +import unittest + +import pytest +from unittestadapter.utils import ( + TestNode, + TestNodeTypeEnum, + build_test_tree, + get_child_node, + get_test_case, +) + +from .helpers import TEST_DATA_PATH, is_same_tree + + +@pytest.mark.parametrize( + "directory, pattern, expected", + [ + ( + ".", + "utils_simple_cases*", + [ + "utils_simple_cases.CaseOne.test_one", + "utils_simple_cases.CaseOne.test_two", + ], + ), + ( + "utils_nested_cases", + "file*", + [ + "file_one.CaseTwoFileOne.test_one", + "file_one.CaseTwoFileOne.test_two", + "folder.file_two.CaseTwoFileTwo.test_one", + "folder.file_two.CaseTwoFileTwo.test_two", + ], + ), + ], +) +def test_simple_test_cases(directory, pattern, expected) -> None: + """The get_test_case fuction should return tests from all test suites.""" + + actual = [] + + # Discover tests in .data/. + start_dir = os.fsdecode(TEST_DATA_PATH / directory) + + loader = unittest.TestLoader() + suite = loader.discover(start_dir, pattern) + + # Iterate on get_test_case and save the test id. + for test in get_test_case(suite): + actual.append(test.id()) + + assert expected == actual + + +def test_get_existing_child_node() -> None: + """The get_child_node fuction should return the child node of a test tree if it exists.""" + + tree: TestNode = { + "name": "root", + "path": "foo", + "type_": TestNodeTypeEnum.folder, + "children": [ + { + "name": "childOne", + "path": "child/one", + "type_": TestNodeTypeEnum.folder, + "children": [ + { + "name": "nestedOne", + "path": "nested/one", + "type_": TestNodeTypeEnum.folder, + "children": [], + }, + { + "name": "nestedTwo", + "path": "nested/two", + "type_": TestNodeTypeEnum.folder, + "children": [], + }, + ], + }, + { + "name": "childTwo", + "path": "child/two", + "type_": TestNodeTypeEnum.folder, + "children": [], + }, + ], + } + + get_child_node("childTwo", "child/two", TestNodeTypeEnum.folder, tree) + tree_copy = tree.copy() + + # Check that the tree didn't get mutated by get_child_node. + assert is_same_tree(tree, tree_copy) + + +def test_no_existing_child_node() -> None: + """The get_child_node fuction should add a child node to a test tree and return it if it does not exist.""" + + tree: TestNode = { + "name": "root", + "path": "foo", + "type_": TestNodeTypeEnum.folder, + "children": [ + { + "name": "childOne", + "path": "child/one", + "type_": TestNodeTypeEnum.folder, + "children": [ + { + "name": "nestedOne", + "path": "nested/one", + "type_": TestNodeTypeEnum.folder, + "children": [], + }, + { + "name": "nestedTwo", + "path": "nested/two", + "type_": TestNodeTypeEnum.folder, + "children": [], + }, + ], + }, + { + "name": "childTwo", + "path": "child/two", + "type_": TestNodeTypeEnum.folder, + "children": [], + }, + ], + } + + # Make a separate copy of tree["children"]. + tree_before = tree.copy() + tree_before["children"] = tree["children"][:] + + get_child_node("childThree", "child/three", TestNodeTypeEnum.folder, tree) + + tree_after = tree.copy() + tree_after["children"] = tree_after["children"][:-1] + + # Check that all pre-existing items in the tree didn't get mutated by get_child_node. + assert is_same_tree(tree_before, tree_after) + + # Check for the added node. + last_child = tree["children"][-1] + assert last_child["name"] == "childThree" + + +def test_build_simple_tree() -> None: + """The build_test_tree function should build and return a test tree from discovered test suites, + and an empty list of errors if there are none in the discovered data. + """ + + # Discovery tests in utils_simple_tree.py. + start_dir = os.fsdecode(TEST_DATA_PATH) + pattern = "utils_simple_tree*" + file_path = os.fsdecode(pathlib.PurePath(TEST_DATA_PATH, "utils_simple_tree.py")) + + expected: TestNode = { + "path": start_dir, + "type_": TestNodeTypeEnum.folder, + "name": ".data", + "children": [ + { + "name": "utils_simple_tree.py", + "type_": TestNodeTypeEnum.file, + "path": file_path, + "children": [ + { + "name": "TreeOne", + "path": file_path, + "type_": TestNodeTypeEnum.class_, + "children": [ + { + "id_": "utils_simple_tree.TreeOne.test_one", + "name": "test_one", + "path": file_path, + "type_": TestNodeTypeEnum.test, + "lineno": "13", + }, + { + "id_": "utils_simple_tree.TreeOne.test_two", + "name": "test_two", + "path": file_path, + "type_": TestNodeTypeEnum.test, + "lineno": "16", + }, + ], + } + ], + } + ], + } + + loader = unittest.TestLoader() + suite = loader.discover(start_dir, pattern) + tests, errors = build_test_tree(suite, start_dir) + + assert is_same_tree(expected, tests) + assert not errors + + +def test_build_decorated_tree() -> None: + """The build_test_tree function should build and return a test tree from discovered test suites, + with correct line numbers for decorated test, + and an empty list of errors if there are none in the discovered data. + """ + + # Discovery tests in utils_decorated_tree.py. + start_dir = os.fsdecode(TEST_DATA_PATH) + pattern = "utils_decorated_tree*" + file_path = os.fsdecode(pathlib.PurePath(TEST_DATA_PATH, "utils_decorated_tree.py")) + + expected: TestNode = { + "path": start_dir, + "type_": TestNodeTypeEnum.folder, + "name": ".data", + "children": [ + { + "name": "utils_decorated_tree.py", + "type_": TestNodeTypeEnum.file, + "path": file_path, + "children": [ + { + "name": "TreeOne", + "path": file_path, + "type_": TestNodeTypeEnum.class_, + "children": [ + { + "id_": "utils_decorated_tree.TreeOne.test_one", + "name": "test_one", + "path": file_path, + "type_": TestNodeTypeEnum.test, + "lineno": "24", + }, + { + "id_": "utils_decorated_tree.TreeOne.test_two", + "name": "test_two", + "path": file_path, + "type_": TestNodeTypeEnum.test, + "lineno": "28", + }, + ], + } + ], + } + ], + } + + loader = unittest.TestLoader() + suite = loader.discover(start_dir, pattern) + tests, errors = build_test_tree(suite, start_dir) + + assert is_same_tree(expected, tests) + assert not errors + + +def test_build_empty_tree() -> None: + """The build_test_tree function should return None if there are no discovered test suites, and an empty list of errors if there are none in the discovered data.""" + + start_dir = os.fsdecode(TEST_DATA_PATH) + pattern = "does_not_exist*" + + expected = None + + loader = unittest.TestLoader() + suite = loader.discover(start_dir, pattern) + tests, errors = build_test_tree(suite, start_dir) + + assert expected == tests + assert not errors diff --git a/pythonFiles/unittestadapter/__init__.py b/pythonFiles/unittestadapter/__init__.py new file mode 100644 index 000000000000..5b7f7a925cc0 --- /dev/null +++ b/pythonFiles/unittestadapter/__init__.py @@ -0,0 +1,2 @@ +# Copyright (c) Microsoft Corporation. All rights reserved. +# Licensed under the MIT License. diff --git a/pythonFiles/unittestadapter/discovery.py b/pythonFiles/unittestadapter/discovery.py new file mode 100644 index 000000000000..7efd6cabb242 --- /dev/null +++ b/pythonFiles/unittestadapter/discovery.py @@ -0,0 +1,157 @@ +# Copyright (c) Microsoft Corporation. All rights reserved. +# Licensed under the MIT License. + +import argparse +import json +import os +import sys +import traceback +import unittest +from typing import List, Literal, Optional, Tuple, TypedDict, Union + +# Add the path to pythonFiles to sys.path to find testing_tools.socket_manager. +PYTHON_FILES = os.path.dirname(os.path.dirname(os.path.abspath(__file__))) +sys.path.insert(0, PYTHON_FILES) + +from testing_tools import socket_manager + +# If I use from utils then there will be an import error in test_discovery.py. +from unittestadapter.utils import TestNode, build_test_tree + +# Add the lib path to sys.path to find the typing_extensions module. +sys.path.insert(0, os.path.join(PYTHON_FILES, "lib", "python")) + +from typing_extensions import NotRequired + +DEFAULT_PORT = "45454" + + +def parse_cli_args(args: List[str]) -> Tuple[int, Union[str, None]]: + """Parse command-line arguments that should be processed by the script. + + So far this includes the port number that it needs to connect to, and the uuid passed by the TS side. + The port is passed to the discovery.py script when it is executed, and + defaults to DEFAULT_PORT if it can't be parsed. + The uuid should be passed to the discovery.py script when it is executed, and defaults to None if it can't be parsed. + If the arguments appear several times, the value returned by parse_cli_args will be the value of the last argument. + """ + arg_parser = argparse.ArgumentParser() + arg_parser.add_argument("--port", default=DEFAULT_PORT) + arg_parser.add_argument("--uuid") + parsed_args, _ = arg_parser.parse_known_args(args) + + return int(parsed_args.port), parsed_args.uuid + + +def parse_unittest_args(args: List[str]) -> Tuple[str, str, Union[str, None]]: + """Parse command-line arguments that should be forwarded to unittest. + + Valid unittest arguments are: -v, -s, -p, -t and their long-form counterparts, + however we only care about the last three. + + The returned tuple contains the following items + - start_directory: The directory where to start discovery, defaults to . + - pattern: The pattern to match test files, defaults to test*.py + - top_level_directory: The top-level directory of the project, defaults to None, and unittest will use start_directory behind the scenes. + """ + + arg_parser = argparse.ArgumentParser() + arg_parser.add_argument("--start-directory", "-s", default=".") + arg_parser.add_argument("--pattern", "-p", default="test*.py") + arg_parser.add_argument("--top-level-directory", "-t", default=None) + + parsed_args, _ = arg_parser.parse_known_args(args) + + return ( + parsed_args.start_directory, + parsed_args.pattern, + parsed_args.top_level_directory, + ) + + +class PayloadDict(TypedDict): + cwd: str + uuid: Union[str, None] + status: Literal["success", "error"] + tests: NotRequired[TestNode] + errors: NotRequired[List[str]] + + +def discover_tests( + start_dir: str, pattern: str, top_level_dir: Optional[str], uuid: Optional[str] +) -> PayloadDict: + """Returns a dictionary containing details of the discovered tests. + + The returned dict has the following keys: + + - cwd: Absolute path to the test start directory; + - uuid: UUID sent by the caller of the Python script, that needs to be sent back as an integrity check; + - status: Test discovery status, can be "success" or "error"; + - tests: Discoverered tests if any, not present otherwise. Note that the status can be "error" but the payload can still contain tests; + - errors: Discovery errors if any, not present otherwise. + + Payload format for a successful discovery: + { + "status": "success", + "cwd": , + "tests": + } + + Payload format for a successful discovery with no tests: + { + "status": "success", + "cwd": , + } + + Payload format when there are errors: + { + "cwd": + "errors": [list of errors] + "status": "error", + } + """ + cwd = os.path.abspath(start_dir) + payload: PayloadDict = {"cwd": cwd, "status": "success", "uuid": uuid} + tests = None + errors = [] + + try: + loader = unittest.TestLoader() + suite = loader.discover(start_dir, pattern, top_level_dir) + + tests, errors = build_test_tree(suite, cwd) + except Exception: + errors.append(traceback.format_exc()) + + if tests is not None: + payload["tests"] = tests + + if len(errors): + payload["status"] = "error" + payload["errors"] = errors + + return payload + + +if __name__ == "__main__": + # Get unittest discovery arguments. + argv = sys.argv[1:] + index = argv.index("--udiscovery") + + start_dir, pattern, top_level_dir = parse_unittest_args(argv[index + 1 :]) + + # Perform test discovery. + port, uuid = parse_cli_args(argv[:index]) + payload = discover_tests(start_dir, pattern, top_level_dir, uuid) + + # Build the request data (it has to be a POST request or the Node side will not process it), and send it. + addr = ("localhost", port) + with socket_manager.SocketManager(addr) as s: + data = json.dumps(payload) + request = f"""POST / HTTP/1.1 +Host: localhost:{port} +Content-Length: {len(data)} +Content-Type: application/json + +{data}""" + result = s.socket.sendall(request.encode("utf-8")) # type: ignore diff --git a/pythonFiles/unittestadapter/utils.py b/pythonFiles/unittestadapter/utils.py new file mode 100644 index 000000000000..40aa802df3a1 --- /dev/null +++ b/pythonFiles/unittestadapter/utils.py @@ -0,0 +1,193 @@ +# Copyright (c) Microsoft Corporation. All rights reserved. +# Licensed under the MIT License. + +import enum +import inspect +import os +import pathlib +import unittest +from typing import List, Tuple, TypedDict, Union + +# Types + + +# Inherit from str so it's JSON serializable. +class TestNodeTypeEnum(str, enum.Enum): + class_ = "class" + file = "file" + folder = "folder" + test = "test" + + +class TestData(TypedDict): + name: str + path: str + type_: TestNodeTypeEnum + + +class TestItem(TestData): + id_: str + lineno: str + + +class TestNode(TestData): + children: "List[TestNode | TestItem]" + + +# Helper functions for data retrieval. + + +def get_test_case(suite): + """Iterate through a unittest test suite and return all test cases.""" + for test in suite: + if isinstance(test, unittest.TestCase): + yield test + else: + for test_case in get_test_case(test): + yield test_case + + +def get_source_line(obj) -> str: + """Get the line number of a test case start line.""" + try: + sourcelines, lineno = inspect.getsourcelines(obj) + except: + try: + # tornado-specific, see https://github.com/microsoft/vscode-python/issues/17285. + sourcelines, lineno = inspect.getsourcelines(obj.orig_method) + except: + return "*" + + # Return the line number of the first line of the test case definition. + for i, v in enumerate(sourcelines): + if v.strip().startswith(("def", "async def")): + return str(lineno + i) + + return "*" + + +# Helper functions for test tree building. + + +def build_test_node(path: str, name: str, type_: TestNodeTypeEnum) -> TestNode: + """Build a test node with no children. A test node can be a folder, a file or a class.""" + return { + "path": path, + "name": name, + "type_": type_, + "children": [], + } + + +def get_child_node( + name: str, path: str, type_: TestNodeTypeEnum, root: TestNode +) -> TestNode: + """Find a child node in a test tree given its name and type. If the node doesn't exist, create it.""" + try: + result = next( + node + for node in root["children"] + if node["name"] == name and node["type_"] == type_ + ) + except StopIteration: + result = build_test_node(path, name, type_) + root["children"].append(result) + + return result # type:ignore + + +def build_test_tree( + suite: unittest.TestSuite, test_directory: str +) -> Tuple[Union[TestNode, None], List[str]]: + """Build a test tree from a unittest test suite. + + This function returns the test tree, and any errors found by unittest. + If no tests were discovered, return `None` and a list of errors (if any). + + Test tree structure: + { + "path": , + "type": "folder", + "name": , + "children": [ + { files and folders } + ... + { + "path": , + "name": filename.py, + "type_": "file", + "children": [ + { + "path": , + "name": , + "type_": "class", + "children": [ + { + "path": , + "name": , + "type_": "test", + "lineno": + "id": , + } + ] + } + ] + } + ] + } + """ + errors = [] + directory_path = pathlib.PurePath(test_directory) + root = build_test_node(test_directory, directory_path.name, TestNodeTypeEnum.folder) + + for test_case in get_test_case(suite): + test_id = test_case.id() + if test_id.startswith("unittest.loader._FailedTest"): + errors.append(str(test_case._exception)) # type: ignore + else: + # Get the static test path components: filename, class name and function name. + components = test_id.split(".") + *folders, filename, class_name, function_name = components + py_filename = f"{filename}.py" + + current_node = root + + # Find/build nodes for the intermediate folders in the test path. + for folder in folders: + current_node = get_child_node( + folder, + os.fsdecode(pathlib.PurePath(current_node["path"], folder)), + TestNodeTypeEnum.folder, + current_node, + ) + + # Find/build file node. + path_components = [test_directory] + folders + [py_filename] + file_path = os.fsdecode(pathlib.PurePath("/".join(path_components))) + current_node = get_child_node( + py_filename, file_path, TestNodeTypeEnum.file, current_node + ) + + # Find/build class node. + current_node = get_child_node( + class_name, file_path, TestNodeTypeEnum.class_, current_node + ) + + # Get test line number. + test_method = getattr(test_case, test_case._testMethodName) + lineno = get_source_line(test_method) + + # Add test node. + test_node: TestItem = { + "id_": test_id, + "name": function_name, + "path": file_path, + "lineno": lineno, + "type_": TestNodeTypeEnum.test, + } + current_node["children"].append(test_node) + + if not root["children"]: + root = None + + return root, errors diff --git a/requirements.in b/requirements.in index 59019732fd9e..62c30103d6fd 100644 --- a/requirements.in +++ b/requirements.in @@ -3,5 +3,8 @@ # 1) pip install pip-tools # 2) pip-compile --generate-hashes requirements.in +# Unittest test adapter +typing-extensions==4.0.1 + # Sort Imports isort==5.10.1 diff --git a/requirements.txt b/requirements.txt index 691346939e7a..2e34b3d763a0 100644 --- a/requirements.txt +++ b/requirements.txt @@ -8,3 +8,7 @@ isort==5.10.1 \ --hash=sha256:6f62d78e2f89b4500b080fe3a81690850cd254227f27f75c3a0c491a1f351ba7 \ --hash=sha256:e8443a5e7a020e9d7f97f1d7d9cd17c88bcb3bc7e218bf9cf5095fe550be2951 # via -r requirements.in +typing-extensions==4.0.1 \ + --hash=sha256:4ca091dea149f945ec56afb48dae714f21e8692ef22a395223bcd328961b6a0e \ + --hash=sha256:7f001e5ac290a0c0401508864c7ec868be4e701886d5b573a9528ed3973d9d3b + # via -r requirements.in diff --git a/src/client/common/utils/localize.ts b/src/client/common/utils/localize.ts index e3b4f760ff9f..86c79139cf3a 100644 --- a/src/client/common/utils/localize.ts +++ b/src/client/common/utils/localize.ts @@ -74,6 +74,7 @@ export namespace Common { export const canceled = localize('Common.canceled', 'Canceled'); export const cancel = localize('Common.cancel', 'Cancel'); export const ok = localize('Common.ok', 'Ok'); + export const error = localize('Common.error', 'Error'); export const gotIt = localize('Common.gotIt', 'Got it!'); export const install = localize('Common.install', 'Install'); export const loadingExtension = localize('Common.loadingPythonExtension', 'Python extension loading...'); @@ -512,6 +513,12 @@ export namespace DebugConfigStrings { export namespace Testing { export const configureTests = localize('Testing.configureTests', 'Configure Test Framework'); export const testNotConfigured = localize('Testing.testNotConfigured', 'No test framework configured.'); + export const cancelUnittestDiscovery = localize( + 'Testing.cancelUnittestDiscovery', + 'Canceled unittest test discovery', + ); + export const errorUnittestDiscovery = localize('Testing.errorUnittestDiscovery', 'Unittest test discovery error'); + export const seePythonOutput = localize('Testing.seePythonOutput', '(see Output > Python)'); } export namespace OutdatedDebugger { diff --git a/src/client/testing/testController/common/server.ts b/src/client/testing/testController/common/server.ts new file mode 100644 index 000000000000..9193275a1a21 --- /dev/null +++ b/src/client/testing/testController/common/server.ts @@ -0,0 +1,112 @@ +// Copyright (c) Microsoft Corporation. All rights reserved. +// Licensed under the MIT License. + +import * as http from 'http'; +import * as net from 'net'; +import * as crypto from 'crypto'; +import { Disposable, Event, EventEmitter } from 'vscode'; +import { + ExecutionFactoryCreateWithEnvironmentOptions, + IPythonExecutionFactory, + SpawnOptions, +} from '../../../common/process/types'; +import { traceLog } from '../../../logging'; +import { DataReceivedEvent, ITestServer, TestCommandOptions } from './types'; +import { DEFAULT_TEST_PORT } from './utils'; + +export class PythonTestServer implements ITestServer, Disposable { + private _onDataReceived: EventEmitter = new EventEmitter(); + + private uuids: Map; + + private server: http.Server; + + public port: number; + + constructor(private executionFactory: IPythonExecutionFactory) { + this.uuids = new Map(); + + this.port = DEFAULT_TEST_PORT; + + const requestListener: http.RequestListener = async (request, response) => { + const buffers = []; + + try { + for await (const chunk of request) { + buffers.push(chunk); + } + + const data = Buffer.concat(buffers).toString(); + + response.end(); + + const { uuid } = JSON.parse(data); + + // Check if the uuid we received exists in the list of active ones. + // If yes, process the response, if not, ignore it. + const cwd = this.uuids.get(uuid); + if (cwd) { + this._onDataReceived.fire({ cwd, data }); + this.uuids.delete(uuid); + } + } catch (ex) { + traceLog(`Error processing test server request: ${ex}`); + this._onDataReceived.fire({ cwd: '', data: '' }); + } + }; + + this.server = http.createServer(requestListener); + this.server.listen(() => { + this.port = (this.server.address() as net.AddressInfo).port; + }); + } + + public dispose(): void { + this.server.close(); + this._onDataReceived.dispose(); + } + + public get onDataReceived(): Event { + return this._onDataReceived.event; + } + + async sendCommand(options: TestCommandOptions): Promise { + const uuid = crypto.randomUUID(); + const spawnOptions: SpawnOptions = { + token: options.token, + cwd: options.cwd, + throwOnStdErr: true, + }; + + this.uuids.set(uuid, options.cwd); + + // Create the Python environment in which to execute the command. + const creationOptions: ExecutionFactoryCreateWithEnvironmentOptions = { + allowEnvironmentFetchExceptions: false, + resource: options.workspaceFolder, + }; + const execService = await this.executionFactory.createActivatedEnvironment(creationOptions); + + // Add the generated UUID to the data to be sent (expecting to receive it back). + const args = [options.command.script, '--port', this.port.toString(), '--uuid', uuid].concat( + options.command.args, + ); + + if (options.outChannel) { + options.outChannel.appendLine(`python ${args.join(' ')}`); + } + + try { + await execService.exec(args, spawnOptions); + } catch (ex) { + this.uuids.delete(uuid); + this._onDataReceived.fire({ + cwd: options.cwd, + data: JSON.stringify({ + status: 'error', + errors: [(ex as Error).message], + }), + }); + } + } +} diff --git a/src/client/testing/testController/common/testItemUtilities.ts b/src/client/testing/testController/common/testItemUtilities.ts index 525e36e4235d..8b8b59051ec4 100644 --- a/src/client/testing/testController/common/testItemUtilities.ts +++ b/src/client/testing/testController/common/testItemUtilities.ts @@ -50,10 +50,9 @@ export function removeItemByIdFromChildren( }); } -export function createErrorTestItem( - testController: TestController, - options: { id: string; label: string; error: string }, -): TestItem { +export type ErrorTestItemOptions = { id: string; label: string; error: string }; + +export function createErrorTestItem(testController: TestController, options: ErrorTestItemOptions): TestItem { const testItem = testController.createTestItem(options.id, options.label); testItem.canResolveChildren = false; testItem.error = options.error; diff --git a/src/client/testing/testController/common/types.ts b/src/client/testing/testController/common/types.ts index 50c324471168..ab0365f2698d 100644 --- a/src/client/testing/testController/common/types.ts +++ b/src/client/testing/testController/common/types.ts @@ -4,6 +4,7 @@ import { CancellationToken, Event, + OutputChannel, TestController, TestItem, TestRun, @@ -122,3 +123,65 @@ export type RawDiscoveredTests = { parents: RawTestParent[]; tests: RawTest[]; }; + +// New test discovery adapter types + +export type DataReceivedEvent = { + cwd: string; + data: string; +}; + +export type TestDiscoveryCommand = { + script: string; + args: string[]; +}; + +export type TestCommandOptions = { + workspaceFolder: Uri; + cwd: string; + command: TestDiscoveryCommand; + token?: CancellationToken; + outChannel?: OutputChannel; +}; + +/** + * Interface describing the server that will send test commands to the Python side, and process responses. + * + * Consumers will call sendCommand in order to execute Python-related code, + * and will subscribe to the onDataReceived event to wait for the results. + */ +export interface ITestServer { + readonly onDataReceived: Event; + sendCommand(options: TestCommandOptions): Promise; +} + +export interface ITestDiscoveryAdapter { + discoverTests(uri: Uri): Promise; +} + +// Same types as in pythonFiles/unittestadapter/utils.py +export type DiscoveredTestType = 'folder' | 'file' | 'class' | 'test'; + +export type DiscoveredTestCommon = { + path: string; + name: string; + // Trailing underscore to avoid collision with the 'type' Python keyword. + type_: DiscoveredTestType; +}; + +export type DiscoveredTestItem = DiscoveredTestCommon & { + lineno: number; + // Trailing underscore to avoid collision with the 'id' Python keyword. + id_: string; +}; + +export type DiscoveredTestNode = DiscoveredTestCommon & { + children: (DiscoveredTestNode | DiscoveredTestItem)[]; +}; + +export type DiscoveredTestPayload = { + cwd: string; + tests?: DiscoveredTestNode; + status: 'success' | 'error'; + errors?: string[]; +}; diff --git a/src/client/testing/testController/common/utils.ts b/src/client/testing/testController/common/utils.ts index 13fc76a37199..f2c201dc9e0f 100644 --- a/src/client/testing/testController/common/utils.ts +++ b/src/client/testing/testController/common/utils.ts @@ -1,6 +1,8 @@ // Copyright (c) Microsoft Corporation. All rights reserved. // Licensed under the MIT License. +export const DEFAULT_TEST_PORT = 45454; + export function fixLogLines(content: string): string { const lines = content.split(/\r?\n/g); return `${lines.join('\r\n')}\r\n`; diff --git a/src/client/testing/testController/controller.ts b/src/client/testing/testController/controller.ts index 66774f8842e8..6e38722ee30a 100644 --- a/src/client/testing/testController/controller.ts +++ b/src/client/testing/testController/controller.ts @@ -18,14 +18,19 @@ import { } from 'vscode'; import { IExtensionSingleActivationService } from '../../activation/types'; import { IWorkspaceService } from '../../common/application/types'; +import { IPythonExecutionFactory } from '../../common/process/types'; import { IConfigurationService, IDisposableRegistry, Resource } from '../../common/types'; import { DelayedTrigger, IDelayedTrigger } from '../../common/utils/delayTrigger'; import { traceVerbose } from '../../logging'; import { IEventNamePropertyMapping, sendTelemetryEvent } from '../../telemetry'; import { EventName } from '../../telemetry/constants'; import { PYTEST_PROVIDER, UNITTEST_PROVIDER } from '../common/constants'; +import { TestProvider } from '../types'; +import { PythonTestServer } from './common/server'; import { DebugTestTag, getNodeByUri, RunTestTag } from './common/testItemUtilities'; -import { ITestController, ITestFrameworkController, TestRefreshOptions } from './common/types'; +import { ITestController, ITestDiscoveryAdapter, ITestFrameworkController, TestRefreshOptions } from './common/types'; +import { UnittestTestDiscoveryAdapter } from './unittest/testDiscoveryAdapter'; +import { WorkspaceTestAdapter } from './workspaceTestAdapter'; // Types gymnastics to make sure that sendTriggerTelemetry only accepts the correct types. type EventPropertyType = IEventNamePropertyMapping[EventName.UNITTEST_DISCOVERY_TRIGGER]; @@ -36,6 +41,8 @@ type TriggerType = EventPropertyType[TriggerKeyType]; export class PythonTestController implements ITestController, IExtensionSingleActivationService { public readonly supportedWorkspaceTypes = { untrustedWorkspace: false, virtualWorkspace: false }; + private readonly testAdapters: Map = new Map(); + private readonly triggerTypes: TriggerType[] = []; private readonly testController: TestController; @@ -52,6 +59,8 @@ export class PythonTestController implements ITestController, IExtensionSingleAc WorkspaceFolder[] >(); + private pythonTestServer: PythonTestServer; + public readonly onRefreshingCompleted = this.refreshingCompletedEvent.event; public readonly onRefreshingStarted = this.refreshingStartedEvent.event; @@ -66,6 +75,7 @@ export class PythonTestController implements ITestController, IExtensionSingleAc @inject(ITestFrameworkController) @named(PYTEST_PROVIDER) private readonly pytest: ITestFrameworkController, @inject(ITestFrameworkController) @named(UNITTEST_PROVIDER) private readonly unittest: ITestFrameworkController, @inject(IDisposableRegistry) private readonly disposables: IDisposableRegistry, + @inject(IPythonExecutionFactory) private readonly pythonExecFactory: IPythonExecutionFactory, ) { this.refreshCancellation = new CancellationTokenSource(); @@ -102,10 +112,37 @@ export class PythonTestController implements ITestController, IExtensionSingleAc ), ); this.testController.resolveHandler = this.resolveChildren.bind(this); + + this.pythonTestServer = new PythonTestServer(this.pythonExecFactory); } public async activate(): Promise { - this.watchForTestChanges(); + const workspaces: readonly WorkspaceFolder[] = this.workspaceService.workspaceFolders || []; + workspaces.forEach((workspace) => { + const settings = this.configSettings.getSettings(workspace.uri); + + let discoveryAdapter: ITestDiscoveryAdapter; + let testProvider: TestProvider; + if (settings.testing.unittestEnabled) { + discoveryAdapter = new UnittestTestDiscoveryAdapter(this.pythonTestServer, this.configSettings); + testProvider = UNITTEST_PROVIDER; + } else { + // TODO: PYTEST DISCOVERY ADAPTER + // this is a placeholder for now + discoveryAdapter = new UnittestTestDiscoveryAdapter(this.pythonTestServer, { ...this.configSettings }); + testProvider = PYTEST_PROVIDER; + } + + const workspaceTestAdapter = new WorkspaceTestAdapter(testProvider, discoveryAdapter, workspace.uri); + + this.testAdapters.set(workspace.uri, workspaceTestAdapter); + + if (settings.testing.autoTestDiscoverOnSaveEnabled) { + traceVerbose(`Testing: Setting up watcher for ${workspace.uri.fsPath}`); + this.watchForSettingsChanges(workspace); + this.watchForTestContentChanges(workspace); + } + }); } public refreshTestData(uri?: Resource, options?: TestRefreshOptions): Promise { @@ -155,6 +192,7 @@ export class PythonTestController implements ITestController, IExtensionSingleAc await this.pytest.refreshTestData(this.testController, uri, this.refreshCancellation.token); } else if (settings.testing.unittestEnabled) { + // TODO: Use new test discovery mechanism traceVerbose(`Testing: Refreshing test data for ${uri.fsPath}`); // Ensure we send test telemetry if it gets disabled again @@ -309,18 +347,6 @@ export class PythonTestController implements ITestController, IExtensionSingleAc }); } - private watchForTestChanges(): void { - const workspaces: readonly WorkspaceFolder[] = this.workspaceService.workspaceFolders || []; - for (const workspace of workspaces) { - const settings = this.configSettings.getSettings(workspace.uri); - if (settings.testing.autoTestDiscoverOnSaveEnabled) { - traceVerbose(`Testing: Setting up watcher for ${workspace.uri.fsPath}`); - this.watchForSettingsChanges(workspace); - this.watchForTestContentChanges(workspace); - } - } - } - private watchForSettingsChanges(workspace: WorkspaceFolder): void { const pattern = new RelativePattern(workspace, '**/{settings.json,pytest.ini,pyproject.toml,setup.cfg}'); const watcher = this.workspaceService.createFileSystemWatcher(pattern); diff --git a/src/client/testing/testController/unittest/testDiscoveryAdapter.ts b/src/client/testing/testController/unittest/testDiscoveryAdapter.ts new file mode 100644 index 000000000000..f0a5a957807c --- /dev/null +++ b/src/client/testing/testController/unittest/testDiscoveryAdapter.ts @@ -0,0 +1,72 @@ +// Copyright (c) Microsoft Corporation. All rights reserved. +// Licensed under the MIT License. + +import * as path from 'path'; +import { Uri } from 'vscode'; +import { IConfigurationService } from '../../../common/types'; +import { createDeferred, Deferred } from '../../../common/utils/async'; +import { EXTENSION_ROOT_DIR } from '../../../constants'; +import { + DataReceivedEvent, + DiscoveredTestPayload, + ITestDiscoveryAdapter, + ITestServer, + TestCommandOptions, + TestDiscoveryCommand, +} from '../common/types'; + +/** + * Wrapper class for unittest test discovery. This is where we call `runTestCommand`. + */ +export class UnittestTestDiscoveryAdapter implements ITestDiscoveryAdapter { + private deferred: Deferred | undefined; + + private cwd: string | undefined; + + constructor(public testServer: ITestServer, public configSettings: IConfigurationService) { + testServer.onDataReceived(this.onDataReceivedHandler, this); + } + + public onDataReceivedHandler({ cwd, data }: DataReceivedEvent): void { + if (this.deferred && cwd === this.cwd) { + const testData: DiscoveredTestPayload = JSON.parse(data); + + this.deferred.resolve(testData); + this.deferred = undefined; + } + } + + public async discoverTests(uri: Uri): Promise { + if (!this.deferred) { + const settings = this.configSettings.getSettings(uri); + const { unittestArgs } = settings.testing; + + const command = buildDiscoveryCommand(unittestArgs); + + this.cwd = uri.fsPath; + + const options: TestCommandOptions = { + workspaceFolder: uri, + command, + cwd: this.cwd, + }; + + this.deferred = createDeferred(); + + // Send the test command to the server. + // The server will fire an onDataReceived event once it gets a response. + this.testServer.sendCommand(options); + } + + return this.deferred.promise; + } +} + +function buildDiscoveryCommand(args: string[]): TestDiscoveryCommand { + const discoveryScript = path.join(EXTENSION_ROOT_DIR, 'pythonFiles', 'unittestadapter', 'discovery.py'); + + return { + script: discoveryScript, + args: ['--udiscovery', ...args], + }; +} diff --git a/src/client/testing/testController/workspaceTestAdapter.ts b/src/client/testing/testController/workspaceTestAdapter.ts new file mode 100644 index 000000000000..1aa5e707d88a --- /dev/null +++ b/src/client/testing/testController/workspaceTestAdapter.ts @@ -0,0 +1,311 @@ +// Copyright (c) Microsoft Corporation. All rights reserved. +// Licensed under the MIT License. + +import * as path from 'path'; +import * as util from 'util'; +import { CancellationToken, Position, Range, TestController, TestItem, Uri } from 'vscode'; +import { createDeferred, Deferred } from '../../common/utils/async'; +import { Testing } from '../../common/utils/localize'; +import { traceError } from '../../logging'; +import { sendTelemetryEvent } from '../../telemetry'; +import { EventName } from '../../telemetry/constants'; +import { TestProvider } from '../types'; +import { createErrorTestItem, DebugTestTag, ErrorTestItemOptions, RunTestTag } from './common/testItemUtilities'; +import { DiscoveredTestItem, DiscoveredTestNode, DiscoveredTestType, ITestDiscoveryAdapter } from './common/types'; + +/** + * This class exposes a test-provider-agnostic way of discovering tests. + * + * It gets instantiated by the `PythonTestController` class in charge of reflecting test data in the UI, + * and then instantiates provider-specific adapters under the hood depending on settings. + * + * This class formats the JSON test data returned by the `[Unittest|Pytest]TestDiscoveryAdapter` into test UI elements, + * and uses them to insert/update/remove items in the `TestController` instance behind the testing UI whenever the `PythonTestController` requests a refresh. + */ +export class WorkspaceTestAdapter { + private discovering: Deferred | undefined = undefined; + + private testData: DiscoveredTestNode | undefined; + + constructor( + private testProvider: TestProvider, + private discoveryAdapter: ITestDiscoveryAdapter, + // TODO: Implement test running + // private runningAdapter: ITestRunningAdapter, + private workspaceUri: Uri, + ) {} + + public async discoverTests( + testController: TestController, + token?: CancellationToken, + isMultiroot?: boolean, + workspaceFilePath?: string, + ): Promise { + sendTelemetryEvent(EventName.UNITTEST_DISCOVERING, undefined, { tool: this.testProvider }); + + const workspacePath = this.workspaceUri.fsPath; + + // Discovery is expensive. If it is already running, use the existing promise. + if (this.discovering) { + return this.discovering.promise; + } + + const deferred = createDeferred(); + this.discovering = deferred; + + let rawTestData; + try { + rawTestData = await this.discoveryAdapter.discoverTests(this.workspaceUri); + + deferred.resolve(); + } catch (ex) { + sendTelemetryEvent(EventName.UNITTEST_DISCOVERY_DONE, undefined, { tool: this.testProvider, failed: true }); + + const cancel = token?.isCancellationRequested + ? Testing.cancelUnittestDiscovery() + : Testing.errorUnittestDiscovery(); + + traceError(`${cancel}\r\n`, ex); + + // Report also on the test view. + const message = util.format(`${cancel} ${Testing.seePythonOutput()}\r\n`, ex); + const options = buildErrorNodeOptions(this.workspaceUri, message); + const errorNode = createErrorTestItem(testController, options); + testController.items.add(errorNode); + + deferred.reject(ex as Error); + } finally { + // Discovery has finished running, we have the data, + // we don't need the deferred promise anymore. + this.discovering = undefined; + } + + if (!rawTestData) { + // No test data is available + return Promise.resolve(); + } + + // Check if there were any errors in the discovery process. + if (rawTestData.status === 'error') { + const { errors } = rawTestData; + traceError(Testing.errorUnittestDiscovery(), '\r\n', errors!.join('\r\n\r\n')); + + let errorNode = testController.items.get(`DiscoveryError:${workspacePath}`); + const message = util.format( + `${Testing.errorUnittestDiscovery()} ${Testing.seePythonOutput()}\r\n`, + errors!.join('\r\n\r\n'), + ); + + if (errorNode === undefined) { + const options = buildErrorNodeOptions(this.workspaceUri, message); + errorNode = createErrorTestItem(testController, options); + testController.items.add(errorNode); + } + errorNode.error = message; + } else { + // Remove the error node if necessary, + // then parse and insert test data. + testController.items.delete(`DiscoveryError:${workspacePath}`); + + // Wrap the data under a root node named after the test provider. + const wrappedTests = rawTestData.tests; + + // If we are in a multiroot workspace scenario, wrap the current folder's test result in a tree under the overall root + the current folder name. + let rootPath = workspacePath; + let childrenRootPath = rootPath; + let childrenRootName = path.basename(rootPath); + + if (isMultiroot) { + rootPath = workspaceFilePath!; + childrenRootPath = workspacePath; + childrenRootName = path.basename(workspacePath); + } + + const children = [ + { + path: childrenRootPath, + name: childrenRootName, + type_: 'folder' as DiscoveredTestType, + children: wrappedTests ? [wrappedTests] : [], + }, + ]; + + // Update the raw test data with the wrapped data. + rawTestData.tests = { + path: rootPath, + name: this.testProvider, + type_: 'folder', + children, + }; + + const workspaceNode = testController.items.get(rootPath); + + if (rawTestData.tests) { + // If the test root for this folder exists: Workspace refresh, update its children. + // Otherwise, it is a freshly discovered workspace, and we need to create a new test root and populate the test tree. + if (workspaceNode) { + updateTestTree(testController, rawTestData.tests, this.testData, workspaceNode, token); + } else { + populateTestTree(testController, rawTestData.tests, undefined, token); + } + } else { + // Delete everything from the test controller. + testController.items.replace([]); + } + + // Save new test data state. + this.testData = rawTestData.tests; + } + + sendTelemetryEvent(EventName.UNITTEST_DISCOVERY_DONE, undefined, { tool: this.testProvider, failed: false }); + return Promise.resolve(); + } +} + +function isTestItem(test: DiscoveredTestNode | DiscoveredTestItem): test is DiscoveredTestItem { + return test.type_ === 'test'; +} + +function deleteTestTree(testController: TestController, root?: TestItem) { + if (root) { + const { children } = root; + + children.forEach((child) => { + deleteTestTree(testController, child); + + const { id } = child; + testController.items.delete(id); + }); + + testController.items.delete(root.id); + } +} + +function updateTestTree( + testController: TestController, + updatedData: DiscoveredTestNode, + localData: DiscoveredTestNode | undefined, + testRoot: TestItem | undefined, + token?: CancellationToken, +): void { + // If testRoot is undefined, use the info of the root item of testTreeData to create a test item, and append it to the test controller. + if (!testRoot) { + testRoot = testController.createTestItem(updatedData.path, updatedData.name, Uri.file(updatedData.path)); + testRoot.canResolveChildren = true; + testRoot.tags = [RunTestTag, DebugTestTag]; + + testController.items.add(testRoot); + } + + // Delete existing items if they don't exist in the updated tree. + if (localData) { + localData.children.forEach((local) => { + if (!token?.isCancellationRequested) { + const exists = updatedData.children.find( + (node) => local.name === node.name && local.path === node.path && local.type_ === node.type_, + ); + + if (!exists) { + // Delete this node and all its children. + const testItem = testController.items.get(local.path); + deleteTestTree(testController, testItem); + } + } + }); + } + + // Go through the updated tree, update the existing nodes, and create new ones if necessary. + updatedData.children.forEach((child) => { + if (!token?.isCancellationRequested) { + const root = testController.items.get(child.path); + if (root) { + root.busy = true; + // Update existing test node or item. + if (isTestItem(child)) { + // Update the only property that can be updated. + root.label = child.name; + } else { + const localNode = localData?.children.find( + (node) => child.name === node.name && child.path === node.path && child.type_ === node.type_, + ); + updateTestTree(testController, child, localNode as DiscoveredTestNode, root, token); + } + root.busy = false; + } else { + // Create new test node or item. + let testItem; + if (isTestItem(child)) { + testItem = testController.createTestItem(child.id_, child.name, Uri.file(child.path)); + const range = new Range(new Position(child.lineno - 1, 0), new Position(child.lineno, 0)); + + testItem.canResolveChildren = false; + testItem.tags = [RunTestTag, DebugTestTag]; + testItem.range = range; + + testRoot!.children.add(testItem); + } else { + testItem = testController.createTestItem(child.path, child.name, Uri.file(child.path)); + testItem.canResolveChildren = true; + testItem.tags = [RunTestTag, DebugTestTag]; + + testRoot!.children.add(testItem); + + // Populate the test tree under the newly created node. + populateTestTree(testController, child, testItem, token); + } + } + } + }); +} + +function populateTestTree( + testController: TestController, + testTreeData: DiscoveredTestNode, + testRoot: TestItem | undefined, + token?: CancellationToken, +): void { + // If testRoot is undefined, use the info of the root item of testTreeData to create a test item, and append it to the test controller. + if (!testRoot) { + testRoot = testController.createTestItem(testTreeData.path, testTreeData.name, Uri.file(testTreeData.path)); + testRoot.canResolveChildren = true; + testRoot.tags = [RunTestTag, DebugTestTag]; + + testController.items.add(testRoot); + } + + // Recursively populate the tree with test data. + testTreeData.children.forEach((child) => { + if (!token?.isCancellationRequested) { + if (isTestItem(child)) { + const testItem = testController.createTestItem(child.id_, child.name, Uri.file(child.path)); + const range = new Range(new Position(child.lineno - 1, 0), new Position(child.lineno, 0)); + + testItem.canResolveChildren = false; + testItem.range = range; + testItem.tags = [RunTestTag, DebugTestTag]; + + testRoot!.children.add(testItem); + } else { + let node = testController.items.get(child.path); + + if (!node) { + node = testController.createTestItem(child.path, child.name, Uri.file(child.path)); + + node.canResolveChildren = true; + node.tags = [RunTestTag, DebugTestTag]; + + testRoot!.children.add(node); + } + populateTestTree(testController, child, node, token); + } + } + }); +} + +function buildErrorNodeOptions(uri: Uri, message: string): ErrorTestItemOptions { + return { + id: `DiscoveryError:${uri.fsPath}`, + label: `Unittest Discovery Error [${path.basename(uri.fsPath)}]`, + error: message, + }; +} diff --git a/src/test/testing/testController/server.unit.test.ts b/src/test/testing/testController/server.unit.test.ts new file mode 100644 index 000000000000..56209fbbf554 --- /dev/null +++ b/src/test/testing/testController/server.unit.test.ts @@ -0,0 +1,271 @@ +// Copyright (c) Microsoft Corporation. All rights reserved. +// Licensed under the MIT License. + +import * as assert from 'assert'; +import * as http from 'http'; +import * as sinon from 'sinon'; +import * as crypto from 'crypto'; +import { OutputChannel, Uri } from 'vscode'; +import { IPythonExecutionFactory, IPythonExecutionService } from '../../../client/common/process/types'; +import { createDeferred } from '../../../client/common/utils/async'; +import { PythonTestServer } from '../../../client/testing/testController/common/server'; +import * as logging from '../../../client/logging'; + +suite('Python Test Server', () => { + const fakeUuid = 'fake-uuid'; + + let stubExecutionFactory: IPythonExecutionFactory; + let stubExecutionService: IPythonExecutionService; + let server: PythonTestServer; + let sandbox: sinon.SinonSandbox; + let execArgs: string[]; + let v4Stub: sinon.SinonStub; + let traceLogStub: sinon.SinonStub; + + setup(() => { + sandbox = sinon.createSandbox(); + v4Stub = sandbox.stub(crypto, 'randomUUID'); + traceLogStub = sandbox.stub(logging, 'traceLog'); + + v4Stub.returns(fakeUuid); + stubExecutionService = ({ + exec: (args: string[]) => { + execArgs = args; + return Promise.resolve({ stdout: '', stderr: '' }); + }, + } as unknown) as IPythonExecutionService; + + stubExecutionFactory = ({ + createActivatedEnvironment: () => Promise.resolve(stubExecutionService), + } as unknown) as IPythonExecutionFactory; + }); + + teardown(() => { + sandbox.restore(); + execArgs = []; + server.dispose(); + }); + + test('sendCommand should add the port and uuid to the command being sent', async () => { + const options = { + command: { script: 'myscript', args: ['-foo', 'foo'] }, + workspaceFolder: Uri.file('/foo/bar'), + cwd: '/foo/bar', + }; + + server = new PythonTestServer(stubExecutionFactory); + + await server.sendCommand(options); + const { port } = server; + + assert.deepStrictEqual(execArgs, ['myscript', '--port', `${port}`, '--uuid', fakeUuid, '-foo', 'foo']); + }); + + test('sendCommand should write to an output channel if it is provided as an option', async () => { + const output: string[] = []; + const outChannel = { + appendLine: (str: string) => { + output.push(str); + }, + } as OutputChannel; + const options = { + command: { script: 'myscript', args: ['-foo', 'foo'] }, + workspaceFolder: Uri.file('/foo/bar'), + cwd: '/foo/bar', + outChannel, + }; + + server = new PythonTestServer(stubExecutionFactory); + + await server.sendCommand(options); + + const { port } = server; + const expected = ['python', 'myscript', '--port', `${port}`, '--uuid', fakeUuid, '-foo', 'foo'].join(' '); + + assert.deepStrictEqual(output, [expected]); + }); + + test('If script execution fails during sendCommand, an onDataReceived event should be fired with the "error" status', async () => { + let eventData: { status: string; errors: string[] }; + stubExecutionService = ({ + exec: () => { + throw new Error('Failed to execute'); + }, + } as unknown) as IPythonExecutionService; + + const options = { + command: { script: 'myscript', args: ['-foo', 'foo'] }, + workspaceFolder: Uri.file('/foo/bar'), + cwd: '/foo/bar', + }; + + server = new PythonTestServer(stubExecutionFactory); + server.onDataReceived(({ data }) => { + eventData = JSON.parse(data); + }); + + await server.sendCommand(options); + + assert.deepStrictEqual(eventData!.status, 'error'); + assert.deepStrictEqual(eventData!.errors, ['Failed to execute']); + }); + + test('If the server receives data, it should fire an event if it is a known uuid', async () => { + const deferred = createDeferred(); + const options = { + command: { script: 'myscript', args: ['-foo', 'foo'] }, + workspaceFolder: Uri.file('/foo/bar'), + cwd: '/foo/bar', + }; + + let response; + + server = new PythonTestServer(stubExecutionFactory); + server.onDataReceived(({ data }) => { + response = data; + deferred.resolve(); + }); + + await server.sendCommand(options); + + // Send data back. + const { port } = server; + const requestOptions = { + hostname: 'localhost', + method: 'POST', + port, + }; + + const request = http.request(requestOptions, (res) => { + res.setEncoding('utf8'); + }); + const postData = JSON.stringify({ status: 'success', uuid: fakeUuid }); + request.write(postData); + request.end(); + + await deferred.promise; + + assert.deepStrictEqual(response, postData); + }); + test('If the server receives malformed data, it should display a log message, and not fire an event', async () => { + const deferred = createDeferred(); + const options = { + command: { script: 'myscript', args: ['-foo', 'foo'] }, + workspaceFolder: Uri.file('/foo/bar'), + cwd: '/foo/bar', + }; + + let response; + + server = new PythonTestServer(stubExecutionFactory); + server.onDataReceived(({ data }) => { + response = data; + deferred.resolve(); + }); + + await server.sendCommand(options); + + // Send data back. + const { port } = server; + const requestOptions = { + hostname: 'localhost', + method: 'POST', + port, + }; + + const request = http.request(requestOptions, (res) => { + res.setEncoding('utf8'); + }); + const postData = '[test'; + request.write(postData); + request.end(); + + await deferred.promise; + + sinon.assert.calledOnce(traceLogStub); + assert.deepStrictEqual(response, ''); + }); + + test('If the server receives data, it should not fire an event if it is an unknown uuid', async () => { + const deferred = createDeferred(); + const options = { + command: { script: 'myscript', args: ['-foo', 'foo'] }, + workspaceFolder: Uri.file('/foo/bar'), + cwd: '/foo/bar', + }; + + let response; + + server = new PythonTestServer(stubExecutionFactory); + server.onDataReceived(({ data }) => { + response = data; + deferred.resolve(); + }); + + await server.sendCommand(options); + + // Send data back. + const { port } = server; + const requestOptions = { + hostname: 'localhost', + method: 'POST', + port, + }; + + const request = http.request(requestOptions, (res) => { + res.setEncoding('utf8'); + }); + const postData = JSON.stringify({ status: 'success', uuid: fakeUuid, payload: 'foo' }); + request.write(postData); + request.end(); + + await deferred.promise; + + assert.deepStrictEqual(response, postData); + }); + + test('If the server receives data, it should not fire an event if there is no uuid', async () => { + const deferred = createDeferred(); + const options = { + command: { script: 'myscript', args: ['-foo', 'foo'] }, + workspaceFolder: Uri.file('/foo/bar'), + cwd: '/foo/bar', + }; + + let response; + + server = new PythonTestServer(stubExecutionFactory); + server.onDataReceived(({ data }) => { + response = data; + deferred.resolve(); + }); + + await server.sendCommand(options); + + // Send data back. + const { port } = server; + const requestOptions = { + hostname: 'localhost', + method: 'POST', + port, + }; + + const requestOne = http.request(requestOptions, (res) => { + res.setEncoding('utf8'); + }); + const postDataOne = JSON.stringify({ status: 'success', uuid: 'some-other-uuid', payload: 'foo' }); + requestOne.write(postDataOne); + requestOne.end(); + + const requestTwo = http.request(requestOptions, (res) => { + res.setEncoding('utf8'); + }); + const postDataTwo = JSON.stringify({ status: 'success', uuid: fakeUuid, payload: 'foo' }); + requestTwo.write(postDataTwo); + requestTwo.end(); + + await deferred.promise; + + assert.deepStrictEqual(response, postDataTwo); + }); +}); diff --git a/src/test/testing/testController/unittest/testDiscoveryAdapter.unit.test.ts b/src/test/testing/testController/unittest/testDiscoveryAdapter.unit.test.ts new file mode 100644 index 000000000000..cee4353db09a --- /dev/null +++ b/src/test/testing/testController/unittest/testDiscoveryAdapter.unit.test.ts @@ -0,0 +1,97 @@ +// Copyright (c) Microsoft Corporation. All rights reserved. +// Licensed under the MIT License. + +import * as assert from 'assert'; +import * as path from 'path'; +import { Uri } from 'vscode'; +import { IConfigurationService } from '../../../../client/common/types'; +import { EXTENSION_ROOT_DIR } from '../../../../client/constants'; +import { ITestServer, TestCommandOptions } from '../../../../client/testing/testController/common/types'; +import { UnittestTestDiscoveryAdapter } from '../../../../client/testing/testController/unittest/testDiscoveryAdapter'; + +suite('Unittest test discovery adapter', () => { + let stubConfigSettings: IConfigurationService; + + setup(() => { + stubConfigSettings = ({ + getSettings: () => ({ + testing: { unittestArgs: ['-v', '-s', '.', '-p', 'test*'] }, + }), + } as unknown) as IConfigurationService; + }); + + test('discoverTests should send the discovery command to the test server', async () => { + let options: TestCommandOptions | undefined; + + const stubTestServer = ({ + sendCommand(opt: TestCommandOptions): Promise { + options = opt; + return Promise.resolve(); + }, + onDataReceived: () => { + // no body + }, + } as unknown) as ITestServer; + + const uri = Uri.file('/foo/bar'); + const script = path.join(EXTENSION_ROOT_DIR, 'pythonFiles', 'unittestadapter', 'discovery.py'); + + const adapter = new UnittestTestDiscoveryAdapter(stubTestServer, stubConfigSettings); + adapter.discoverTests(uri); + + assert.deepStrictEqual(options, { + workspaceFolder: uri, + cwd: uri.fsPath, + command: { script, args: ['--udiscovery', '-v', '-s', '.', '-p', 'test*'] }, + }); + }); + + test("onDataReceivedHandler should parse the data if the cwd from the payload matches the test adapter's cwd", async () => { + const stubTestServer = ({ + sendCommand(): Promise { + return Promise.resolve(); + }, + onDataReceived: () => { + // no body + }, + } as unknown) as ITestServer; + + const uri = Uri.file('/foo/bar'); + const data = { status: 'success' }; + + const adapter = new UnittestTestDiscoveryAdapter(stubTestServer, stubConfigSettings); + const promise = adapter.discoverTests(uri); + + adapter.onDataReceivedHandler({ cwd: uri.fsPath, data: JSON.stringify(data) }); + + const result = await promise; + + assert.deepStrictEqual(result, data); + }); + + test("onDataReceivedHandler should ignore the data if the cwd from the payload does not match the test adapter's cwd", async () => { + const stubTestServer = ({ + sendCommand(): Promise { + return Promise.resolve(); + }, + onDataReceived: () => { + // no body + }, + } as unknown) as ITestServer; + + const uri = Uri.file('/foo/bar'); + + const adapter = new UnittestTestDiscoveryAdapter(stubTestServer, stubConfigSettings); + const promise = adapter.discoverTests(uri); + + const data = { status: 'success' }; + adapter.onDataReceivedHandler({ cwd: 'some/other/path', data: JSON.stringify(data) }); + + const nextData = { status: 'error' }; + adapter.onDataReceivedHandler({ cwd: uri.fsPath, data: JSON.stringify(nextData) }); + + const result = await promise; + + assert.deepStrictEqual(result, nextData); + }); +}); diff --git a/src/test/testing/testController/workspaceTestAdapter.unit.test.ts b/src/test/testing/testController/workspaceTestAdapter.unit.test.ts new file mode 100644 index 000000000000..7b94d73aea6d --- /dev/null +++ b/src/test/testing/testController/workspaceTestAdapter.unit.test.ts @@ -0,0 +1,188 @@ +// Copyright (c) Microsoft Corporation. All rights reserved. +// Licensed under the MIT License. + +import * as assert from 'assert'; +import * as sinon from 'sinon'; + +import { TestController, TestItem, Uri } from 'vscode'; +import { IConfigurationService } from '../../../client/common/types'; +import { UnittestTestDiscoveryAdapter } from '../../../client/testing/testController/unittest/testDiscoveryAdapter'; +import { WorkspaceTestAdapter } from '../../../client/testing/testController/workspaceTestAdapter'; +import * as Telemetry from '../../../client/telemetry'; +import { EventName } from '../../../client/telemetry/constants'; +import { ITestServer } from '../../../client/testing/testController/common/types'; + +suite('Workspace test adapter', () => { + suite('Test discovery', () => { + let stubTestServer: ITestServer; + let stubConfigSettings: IConfigurationService; + + let discoverTestsStub: sinon.SinonStub; + let sendTelemetryStub: sinon.SinonStub; + + let telemetryEvent: { eventName: EventName; properties: Record }[] = []; + + // Stubbed test controller (see comment around L.40) + let testController: TestController; + let log: string[] = []; + + const sandbox = sinon.createSandbox(); + + setup(() => { + stubConfigSettings = ({ + getSettings: () => ({ + testing: { unittestArgs: ['--foo'] }, + }), + } as unknown) as IConfigurationService; + + stubTestServer = ({ + sendCommand(): Promise { + return Promise.resolve(); + }, + onDataReceived: () => { + // no body + }, + } as unknown) as ITestServer; + + // For some reason the 'tests' namespace in vscode returns undefined. + // While I figure out how to expose to the tests, they will run + // against a stub test controller and stub test items. + const testItem = ({ + canResolveChildren: false, + tags: [], + children: { + add: () => { + // empty + }, + }, + } as unknown) as TestItem; + + testController = ({ + items: { + get: () => { + log.push('get'); + }, + add: () => { + log.push('add'); + }, + replace: () => { + log.push('replace'); + }, + delete: () => { + log.push('delete'); + }, + }, + createTestItem: () => { + log.push('createTestItem'); + return testItem; + }, + dispose: () => { + // empty + }, + } as unknown) as TestController; + + // testController = tests.createTestController('mock-python-tests', 'Mock Python Tests'); + + const mockSendTelemetryEvent = ( + eventName: EventName, + _: number | Record | undefined, + properties: unknown, + ) => { + telemetryEvent.push({ + eventName, + properties: properties as Record, + }); + }; + + discoverTestsStub = sandbox.stub(UnittestTestDiscoveryAdapter.prototype, 'discoverTests'); + sendTelemetryStub = sandbox.stub(Telemetry, 'sendTelemetryEvent').callsFake(mockSendTelemetryEvent); + }); + + teardown(() => { + telemetryEvent = []; + log = []; + testController.dispose(); + sandbox.restore(); + }); + + test("When discovering tests, the workspace test adapter should call the test discovery adapter's discoverTest method", async () => { + discoverTestsStub.resolves(); + + const testDiscoveryAdapter = new UnittestTestDiscoveryAdapter(stubTestServer, stubConfigSettings); + + const workspaceTestAdapter = new WorkspaceTestAdapter('unittest', testDiscoveryAdapter, Uri.parse('foo')); + + await workspaceTestAdapter.discoverTests(testController); + + sinon.assert.calledOnce(discoverTestsStub); + }); + + test('If discovery is already running, do not call discoveryAdapter.discoverTests again', async () => { + discoverTestsStub.callsFake( + async () => + new Promise((resolve) => { + setTimeout(() => { + // Simulate time taken by discovery. + resolve(); + }, 2000); + }), + ); + + const testDiscoveryAdapter = new UnittestTestDiscoveryAdapter(stubTestServer, stubConfigSettings); + + const workspaceTestAdapter = new WorkspaceTestAdapter('unittest', testDiscoveryAdapter, Uri.parse('foo')); + + // Try running discovery twice + const one = workspaceTestAdapter.discoverTests(testController); + const two = workspaceTestAdapter.discoverTests(testController); + + Promise.all([one, two]); + + sinon.assert.calledOnce(discoverTestsStub); + }); + + test('If discovery succeeds, send a telemetry event with the "failed" key set to false', async () => { + discoverTestsStub.resolves({ status: 'success' }); + + const testDiscoveryAdapter = new UnittestTestDiscoveryAdapter(stubTestServer, stubConfigSettings); + + const workspaceTestAdapter = new WorkspaceTestAdapter('unittest', testDiscoveryAdapter, Uri.parse('foo')); + + await workspaceTestAdapter.discoverTests(testController); + + sinon.assert.calledWith(sendTelemetryStub, EventName.UNITTEST_DISCOVERY_DONE); + assert.strictEqual(telemetryEvent.length, 2); + + const lastEvent = telemetryEvent[1]; + assert.strictEqual(lastEvent.properties.failed, false); + }); + + test('If discovery failed, send a telemetry event with the "failed" key set to true, and add an error node to the test controller', async () => { + discoverTestsStub.rejects(new Error('foo')); + + const testDiscoveryAdapter = new UnittestTestDiscoveryAdapter(stubTestServer, stubConfigSettings); + + const workspaceTestAdapter = new WorkspaceTestAdapter('unittest', testDiscoveryAdapter, Uri.parse('foo')); + + await workspaceTestAdapter.discoverTests(testController); + + sinon.assert.calledWith(sendTelemetryStub, EventName.UNITTEST_DISCOVERY_DONE); + assert.strictEqual(telemetryEvent.length, 2); + + const lastEvent = telemetryEvent[1]; + assert.ok(lastEvent.properties.failed); + + assert.deepStrictEqual(log, ['createTestItem', 'add']); + }); + + /** + * TODO To test: + * - successful discovery but no data: delete everything from the test controller + * - successful discovery with error status: add error node to tree + * - single root: populate tree if there's no root node + * - single root: update tree if there's a root node + * - single root: delete tree if there are no tests in the test data + * - multiroot: update the correct folders + */ + }); +});