Test discovery using Python pytest (#4795)

For #4035.

This is a refactor of #4695.

news/1 Enhancements/4795.md

@@ -0,0 +1 @@
+Use `Python` code for discovery of tests when using `pytest`.

src/client/telemetry/constants.ts

@@ -43,6 +43,7 @@ export enum EventName {
     UNITTEST_DISABLE = 'UNITTEST.DISABLE',
     UNITTEST_RUN = 'UNITTEST.RUN',
     UNITTEST_DISCOVER = 'UNITTEST.DISCOVER',
+    UNITTEST_DISCOVER_WITH_PYCODE = 'UNITTEST.DISCOVER.WITH.PYTHONCODE',
     UNITTEST_CONFIGURE = 'UNITTEST.CONFIGURE',
     UNITTEST_CONFIGURING = 'UNITTEST.CONFIGURING',
     UNITTEST_VIEW_OUTPUT = 'UNITTEST.VIEW_OUTPUT',

src/client/telemetry/index.ts

@@ -310,6 +310,7 @@ export interface IEventNamePropertyMapping {
     [EventName.UNITTEST_CONFIGURING]: TestConfiguringTelemetry;
     [EventName.TERMINAL_CREATE]: TerminalTelemetry;
     [EventName.UNITTEST_DISCOVER]: TestDiscoverytTelemetry;
+    [EventName.UNITTEST_DISCOVER_WITH_PYCODE]: never | undefined;
     [EventName.UNITTEST_RUN]: TestRunTelemetry;
     [EventName.UNITTEST_STOP]: never | undefined;
     [EventName.UNITTEST_DISABLE]: never | undefined;

src/client/unittests/common/services/discoveredTestParser.ts

@@ -0,0 +1,296 @@
+// Copyright (c) Microsoft Corporation. All rights reserved.
+// Licensed under the MIT License.
+
+'use strict';
+
+import { inject, injectable } from 'inversify';
+import * as path from 'path';
+import { Uri } from 'vscode';
+import { IWorkspaceService } from '../../../common/application/types';
+import { traceError } from '../../../common/logger';
+import { IFileSystem } from '../../../common/platform/types';
+import { TestDataItem } from '../../types';
+import { getParentFile, getParentSuite, getTestType } from '../testUtils';
+import { FlattenedTestFunction, FlattenedTestSuite, SubtestParent, TestFile, TestFolder, TestFunction, Tests, TestSuite, TestType } from '../types';
+import { DiscoveredTests, ITestDiscoveredTestParser, TestContainer, TestItem } from './types';
+
+@injectable()
+export class TestDiscoveredTestParser implements ITestDiscoveredTestParser {
+    constructor(@inject(IFileSystem) private readonly fs: IFileSystem,
+        @inject(IWorkspaceService) private readonly workspaceService: IWorkspaceService) { }
+    public parse(resource: Uri, discoveredTests: DiscoveredTests[]): Tests {
+        const tests: Tests = {
+            rootTestFolders: [],
+            summary: { errors: 0, failures: 0, passed: 0, skipped: 0 },
+            testFiles: [],
+            testFolders: [],
+            testFunctions: [],
+            testSuites: []
+        };
+
+        const workspace = this.workspaceService.getWorkspaceFolder(resource);
+        if (!workspace) {
+            traceError('Resource does not belong to any workspace folder');
+            return tests;
+        }
+
+        // If the root is the workspace folder, then ignore that.
+        for (const data of discoveredTests) {
+            const rootFolder = {
+                name: data.root, folders: [], time: 0,
+                testFiles: [], resource: resource, nameToRun: data.rootid
+            };
+            tests.rootTestFolders.push(rootFolder);
+            tests.testFolders.push(rootFolder);
+            this.buildChildren(rootFolder, rootFolder, data, tests);
+            this.fixRootFolders(workspace.uri, data, tests);
+        }
+
+        return tests;
+    }
+    /**
+     * Users workspace folder is not to be treated as the root.
+     * All root folders are relative to the worspace folder.
+     * @protected
+     * @param {Uri} workspaceFolder
+     * @param {DiscoveredTests} discoveredTests
+     * @param {Tests} tests
+     * @returns {void}
+     * @memberof TestDiscoveredTestParser
+     */
+    protected fixRootFolders(workspaceFolder: Uri, discoveredTests: DiscoveredTests, tests: Tests): void {
+        const isWorkspaceFolderTheRoot = this.fs.arePathsSame(discoveredTests.root, workspaceFolder.fsPath);
+        if (!isWorkspaceFolderTheRoot) {
+            return;
+        }
+        const indexToRemove = tests.rootTestFolders.findIndex(folder => folder.name === discoveredTests.root);
+        const rootFolder = tests.rootTestFolders.splice(indexToRemove, 1)[0];
+        tests.rootTestFolders.push(...rootFolder.folders);
+    }
+    /**
+     * Not the best solution to use `case statements`, but it keeps the code simple and easy to read in one place.
+     * Could go with separate classes for each type and use stratergies, but that just ends up a class for
+     * 10 lines of code. Hopefully this is more readable and maintainable than having multiple classes for
+     * the simple processing of the children.
+     *
+     * @protected
+     * @param {TestFolder} rootFolder
+     * @param {TestDataItem} parent
+     * @param {DiscoveredTests} discoveredTests
+     * @param {Tests} tests
+     * @memberof TestsDiscovery
+     */
+    protected buildChildren(rootFolder: TestFolder, parent: TestDataItem, discoveredTests: DiscoveredTests, tests: Tests) {
+        const parentType = getTestType(parent);
+        switch (parentType) {
+            case TestType.testFolder: {
+                this.processFolder(rootFolder, parent as TestFolder, discoveredTests, tests);
+                break;
+            }
+            case TestType.testFile: {
+                this.processFile(rootFolder, parent as TestFile, discoveredTests, tests);
+                break;
+            }
+            case TestType.testSuite: {
+                this.processSuite(rootFolder, parent as TestSuite, discoveredTests, tests);
+                break;
+            }
+            default:
+                break;
+        }
+    }
+    /**
+     * Process the children of a folder.
+     * A folder can only contain other folders and files.
+     * Hence limit processing to those items.
+     *
+     * @protected
+     * @param {TestFolder} rootFolder
+     * @param {TestFolder} parentFolder
+     * @param {DiscoveredTests} discoveredTests
+     * @param {Tests} tests
+     * @memberof TestDiscoveredTestParser
+     */
+    protected processFolder(rootFolder: TestFolder, parentFolder: TestFolder, discoveredTests: DiscoveredTests, tests: Tests) {
+        const folders = discoveredTests.parents
+            .filter(child => child.kind === 'folder' && child.parentid === parentFolder.nameToRun)
+            .map(folder => createTestFolder(rootFolder, folder));
+
+        const files = discoveredTests.parents
+            .filter(child => child.kind === 'file' && child.parentid === parentFolder.nameToRun)
+            .map(file => createTestFile(rootFolder, file));
+
+        parentFolder.folders.push(...folders);
+        parentFolder.testFiles.push(...files);
+        tests.testFolders.push(...folders);
+        tests.testFiles.push(...files);
+        [...folders, ...files].forEach(item => this.buildChildren(rootFolder, item, discoveredTests, tests));
+    }
+    /**
+     * Process the children of a file.
+     * A file can only contain suites, functions and paramerterized functions.
+     * Hence limit processing just to those items.
+     *
+     * @protected
+     * @param {TestFolder} rootFolder
+     * @param {TestFile} parentFile
+     * @param {DiscoveredTests} discoveredTests
+     * @param {Tests} tests
+     * @memberof TestDiscoveredTestParser
+     */
+    protected processFile(rootFolder: TestFolder, parentFile: TestFile, discoveredTests: DiscoveredTests, tests: Tests) {
+        const suites = discoveredTests.parents
+            .filter(child => child.kind === 'suite' && child.parentid === parentFile.nameToRun)
+            .map(suite => createTestSuite(parentFile, rootFolder.resource, suite));
+
+        const functions = discoveredTests.tests
+            .filter(func => func.parentid === parentFile.nameToRun)
+            .map(func => createTestFunction(rootFolder, func));
+
+        parentFile.suites.push(...suites);
+        parentFile.functions.push(...functions);
+        tests.testSuites.push(...suites.map(suite => createFlattenedSuite(tests, suite)));
+        tests.testFunctions.push(...functions.map(func => createFlattenedFunction(tests, func)));
+        suites.forEach(item => this.buildChildren(rootFolder, item, discoveredTests, tests));
+
+        const parameterizedFunctions = discoveredTests.parents
+            .filter(child => child.kind === 'function' && child.parentid === parentFile.nameToRun)
+            .map(func => createParameterizedTestFunction(rootFolder, func));
+        parameterizedFunctions.forEach(func => this.processParameterizedFunction(rootFolder, parentFile, func, discoveredTests, tests));
+    }
+    /**
+     * Process the children of a suite.
+     * A suite can only contain suites, functions and paramerterized functions.
+     * Hence limit processing just to those items.
+     *
+     * @protected
+     * @param {TestFolder} rootFolder
+     * @param {TestSuite} parentSuite
+     * @param {DiscoveredTests} discoveredTests
+     * @param {Tests} tests
+     * @memberof TestDiscoveredTestParser
+     */
+    protected processSuite(rootFolder: TestFolder, parentSuite: TestSuite, discoveredTests: DiscoveredTests, tests: Tests) {
+        const suites = discoveredTests.parents
+            .filter(child => child.kind === 'suite' && child.parentid === parentSuite.nameToRun)
+            .map(suite => createTestSuite(parentSuite, rootFolder.resource, suite));
+
+        const functions = discoveredTests.tests
+            .filter(func => func.parentid === parentSuite.nameToRun)
+            .map(func => createTestFunction(rootFolder, func));
+
+        parentSuite.suites.push(...suites);
+        parentSuite.functions.push(...functions);
+        tests.testSuites.push(...suites.map(suite => createFlattenedSuite(tests, suite)));
+        tests.testFunctions.push(...functions.map(func => createFlattenedFunction(tests, func)));
+        suites.forEach(item => this.buildChildren(rootFolder, item, discoveredTests, tests));
+
+        const parameterizedFunctions = discoveredTests.parents
+            .filter(child => child.kind === 'function' && child.parentid === parentSuite.nameToRun)
+            .map(func => createParameterizedTestFunction(rootFolder, func));
+        parameterizedFunctions.forEach(func => this.processParameterizedFunction(rootFolder, parentSuite, func, discoveredTests, tests));
+    }
+    /**
+     * Process the children of a parameterized function.
+     * A parameterized function can only contain functions (in tests).
+     * Hence limit processing just to those items.
+     *
+     * @protected
+     * @param {TestFolder} rootFolder
+     * @param {TestFunction} parentFunction
+     * @param {DiscoveredTests} discoveredTests
+     * @param {Tests} tests
+     * @returns
+     * @memberof TestDiscoveredTestParser
+     */
+    protected processParameterizedFunction(rootFolder: TestFolder, parent: TestFile | TestSuite, parentFunction: SubtestParent, discoveredTests: DiscoveredTests, tests: Tests) {
+        if (!parentFunction.asSuite) {
+            return;
+        }
+        const functions = discoveredTests.tests
+            .filter(func => func.parentid === parentFunction.nameToRun)
+            .map(func => createTestFunction(rootFolder, func));
+        functions.map(func => func.subtestParent = parentFunction);
+        parentFunction.asSuite.functions.push(...functions);
+        parent.functions.push(...functions);
+        tests.testFunctions.push(...functions.map(func => createFlattenedParameterizedFunction(tests, func, parent)));
+    }
+}
+
+function createTestFolder(root: TestFolder, item: TestContainer): TestFolder {
+    return {
+        name: item.name, nameToRun: item.id, resource: root.resource, time: 0, folders: [], testFiles: []
+    };
+}
+function createTestFile(root: TestFolder, item: TestContainer): TestFile {
+    const fullyQualifiedName = path.isAbsolute(item.id) ? item.id : path.resolve(root.name, item.id);
+    return {
+        fullPath: fullyQualifiedName, functions: [], name: item.name,
+        nameToRun: item.id, resource: root.resource, suites: [], time: 0, xmlName: createXmlName(item.id)
+    };
+}
+function createTestSuite(parentSuiteFile: TestFile | TestSuite, resource: Uri, item: TestContainer): TestSuite {
+    const suite = {
+        functions: [], name: item.name, nameToRun: item.id, resource: resource,
+        suites: [], time: 0, xmlName: '', isInstance: false, isUnitTest: false
+    };
+    suite.xmlName = `${parentSuiteFile.xmlName}.${item.name}`;
+    return suite;
+}
+function createFlattenedSuite(tests: Tests, suite: TestSuite): FlattenedTestSuite {
+    const parentFile = getParentFile(tests, suite);
+    return {
+        parentTestFile: parentFile, testSuite: suite, xmlClassName: parentFile.xmlName
+    };
+}
+function createFlattenedParameterizedFunction(tests: Tests, func: TestFunction, parent: TestFile | TestSuite): FlattenedTestFunction {
+    const type = getTestType(parent);
+    const parentFile = (type && type === TestType.testSuite) ? getParentFile(tests, func) : parent as TestFile;
+    const parentSuite = (type && type === TestType.testSuite) ? parent as TestSuite : undefined;
+    return {
+        parentTestFile: parentFile, parentTestSuite: parentSuite,
+        xmlClassName: parentSuite ? parentSuite.xmlName : parentFile.xmlName, testFunction: func
+    };
+}
+function createFlattenedFunction(tests: Tests, func: TestFunction): FlattenedTestFunction {
+    const parent = getParentFile(tests, func);
+    const type = parent ? getTestType(parent) : undefined;
+    const parentFile = (type && type === TestType.testSuite) ? getParentFile(tests, func) : parent as TestFile;
+    const parentSuite = getParentSuite(tests, func);
+    return {
+        parentTestFile: parentFile, parentTestSuite: parentSuite,
+        xmlClassName: parentSuite ? parentSuite.xmlName : parentFile.xmlName, testFunction: func
+    };
+}
+function createParameterizedTestFunction(root: TestFolder, item: TestContainer): SubtestParent {
+    const suite: TestSuite = {
+        functions: [], isInstance: false, isUnitTest: false,
+        name: item.name, nameToRun: item.id, resource: root.resource,
+        time: 0, suites: [], xmlName: ''
+    };
+    return {
+        asSuite: suite, name: item.name, nameToRun: item.id, time: 0
+    };
+}
+function createTestFunction(root: TestFolder, item: TestItem): TestFunction {
+    return {
+        name: item.name, nameToRun: item.id, resource: root.resource,
+        time: 0, file: item.source.substr(0, item.source.lastIndexOf(':'))
+    };
+}
+/**
+ * Creates something known as an Xml Name, used to identify items
+ * from an xunit test result.
+ * Once we have the test runner done in Python, this can be discarded.
+ * @param {string} fileId
+ * @returns
+ */
+function createXmlName(fileId: string) {
+    let name = path.join(path.dirname(fileId), path.basename(fileId, path.extname(fileId)));
+    name = name.replace(/\\/g, '.').replace(/\//g, '.');
+    // Remove leading . & / & \
+    while (name.startsWith('.') || name.startsWith('/') || name.startsWith('\\')) {
+        name = name.substring(1);
+    }
+    return name;
+}

src/client/unittests/common/services/discovery.ts

@@ -0,0 +1,50 @@
+// Copyright (c) Microsoft Corporation. All rights reserved.
+// Licensed under the MIT License.
+
+'use strict';
+
+import { inject, injectable } from 'inversify';
+import * as path from 'path';
+import { traceError } from '../../../common/logger';
+import { ExecutionFactoryCreateWithEnvironmentOptions, ExecutionResult, IPythonExecutionFactory, SpawnOptions } from '../../../common/process/types';
+import { EXTENSION_ROOT_DIR } from '../../../constants';
+import { captureTelemetry } from '../../../telemetry';
+import { EventName } from '../../../telemetry/constants';
+import { ITestDiscoveryService, TestDiscoveryOptions, Tests } from '../types';
+import { DiscoveredTests, ITestDiscoveredTestParser } from './types';
+
+const DISCOVERY_FILE = path.join(EXTENSION_ROOT_DIR, 'pythonFiles', 'testing_tools', 'run_adapter.py');
+
+@injectable()
+export class TestsDiscoveryService implements ITestDiscoveryService {
+    constructor(@inject(IPythonExecutionFactory) private readonly execFactory: IPythonExecutionFactory,
+        @inject(ITestDiscoveredTestParser) private readonly parser: ITestDiscoveredTestParser) { }
+    @captureTelemetry(EventName.UNITTEST_DISCOVER_WITH_PYCODE, undefined, true)
+    public async discoverTests(options: TestDiscoveryOptions): Promise<Tests> {
+        let output: ExecutionResult<string> | undefined;
+        try {
+            output = await this.exec(options);
+            const discoveredTests = JSON.parse(output.stdout) as DiscoveredTests[];
+            return this.parser.parse(options.workspaceFolder, discoveredTests);
+        } catch (ex) {
+            if (output) {
+                traceError('Failed to parse discovered Test', new Error(output.stdout));
+            }
+            traceError('Failed to parse discovered Test', ex);
+            throw ex;
+        }
+    }
+    public async exec(options: TestDiscoveryOptions): Promise<ExecutionResult<string>> {
+        const creationOptions: ExecutionFactoryCreateWithEnvironmentOptions = {
+            allowEnvironmentFetchExceptions: false,
+            resource: options.workspaceFolder
+        };
+        const execService = await this.execFactory.createActivatedEnvironment(creationOptions);
+        const spawnOptions: SpawnOptions = {
+            token: options.token,
+            cwd: options.cwd,
+            throwOnStdErr: true
+        };
+        return execService.exec([DISCOVERY_FILE, ...options.args], spawnOptions);
+    }
+}

src/client/unittests/common/services/types.ts

@@ -0,0 +1,31 @@
+// Copyright (c) Microsoft Corporation. All rights reserved.
+// Licensed under the MIT License.
+
+'use strict';
+
+import { Uri } from 'vscode';
+import { Tests } from '../types';
+
+export type TestContainer = {
+    id: string;
+    kind: 'file' | 'folder' | 'suite' | 'function';
+    name: string;
+    parentid: string;
+};
+export type TestItem = {
+    id: string;
+    name: string;
+    source: string;
+    parentid: string;
+};
+export type DiscoveredTests = {
+    rootid: string;
+    root: string;
+    parents: TestContainer[];
+    tests: TestItem[];
+};
+
+export const ITestDiscoveredTestParser = Symbol('ITestDiscoveredTestParser');
+export interface ITestDiscoveredTestParser {
+    parse(resource: Uri, discoveredTests: DiscoveredTests[]): Tests;
+}