added a hook to ImportResolver for custom importing logic. (microsoft…

…#515)

server/src/analyzer/importResolver.ts

@@ -95,7 +95,7 @@ export class ImportResolver {
             // Look for it in the root directory of the execution environment.
             importFailureInfo.push(`Looking in root directory of execution environment ` +
                 `'${ execEnv.root }'`);
-            let localImport = this._resolveAbsoluteImport(
+            let localImport = this.resolveAbsoluteImport(
                 execEnv.root, moduleDescriptor, importName, importFailureInfo);
             if (localImport && localImport.isImportFound) {
                 return this._addResultsToCache(execEnv, importName, localImport,
@@ -105,7 +105,7 @@ export class ImportResolver {
 
             for (const extraPath of execEnv.extraPaths) {
                 importFailureInfo.push(`Looking in extraPath '${ extraPath }'`);
-                localImport = this._resolveAbsoluteImport(extraPath, moduleDescriptor,
+                localImport = this.resolveAbsoluteImport(extraPath, moduleDescriptor,
                     importName, importFailureInfo);
                 if (localImport && localImport.isImportFound) {
                     return this._addResultsToCache(execEnv, importName, localImport,
@@ -121,7 +121,7 @@ export class ImportResolver {
             // Check for a typings file.
             if (this._configOptions.typingsPath) {
                 importFailureInfo.push(`Looking in typingsPath '${ this._configOptions.typingsPath }'`);
-                const typingsImport = this._resolveAbsoluteImport(
+                const typingsImport = this.resolveAbsoluteImport(
                     this._configOptions.typingsPath, moduleDescriptor, importName, importFailureInfo);
                 if (typingsImport && typingsImport.isImportFound) {
                     // We will treat typings files as "local" rather than "third party".
@@ -149,17 +149,20 @@ export class ImportResolver {
                     // Allow partial resolution because some third-party packages
                     // use tricks to populate their package namespaces.
                     importFailureInfo.push(`Looking in python search path '${ searchPath }'`);
-                    const thirdPartyImport = this._resolveAbsoluteImport(
+                    const thirdPartyImport = this.resolveAbsoluteImport(
                         searchPath, moduleDescriptor, importName, importFailureInfo,
                         true, true, true);
                     if (thirdPartyImport) {
                         thirdPartyImport.importType = ImportType.ThirdParty;
 
-                        if (thirdPartyImport.isImportFound) {
+                        if (thirdPartyImport.isImportFound && thirdPartyImport.isStubFile) {
                             return this._addResultsToCache(execEnv, importName,
                                 thirdPartyImport, moduleDescriptor.importedSymbols);
                         }
 
+                        // We did not find it, or we did and it's not from a
+                        // stub, so give chance for resolveImportEx to find
+                        // one from a stub.
                         if (bestResultSoFar === undefined ||
                             thirdPartyImport.resolvedPaths.length > bestResultSoFar.resolvedPaths.length) {
                             bestResultSoFar = thirdPartyImport;
@@ -170,6 +173,12 @@ export class ImportResolver {
                 importFailureInfo.push('No python interpreter search path');
             }
 
+            const extraResults = this.resolveImportEx(sourceFilePath, execEnv, moduleDescriptor, importName, importFailureInfo);
+            if (extraResults !== undefined) {
+                return this._addResultsToCache(execEnv, importName, extraResults,
+                    moduleDescriptor.importedSymbols);
+            }
+
             // We weren't able to find an exact match, so return the best
             // partial match.
             if (bestResultSoFar) {
@@ -193,6 +202,15 @@ export class ImportResolver {
         return this._addResultsToCache(execEnv, importName, notFoundResult, undefined);
     }
 
+    // Intended to be overridden by subclasses to provide additional stub
+    // resolving capabilities. Return undefined if no stubs were found for
+    // this import.
+    protected resolveImportEx(sourceFilePath: string, execEnv: ExecutionEnvironment,
+        moduleDescriptor: ImportedModuleDescriptor, importName: string,
+        importFailureInfo: string[] = []): ImportResult | undefined {
+        return undefined;
+    }
+
     getCompletionSuggestions(sourceFilePath: string, execEnv: ExecutionEnvironment,
         moduleDescriptor: ImportedModuleDescriptor, similarityLimit: number): string[] {
 
@@ -414,7 +432,7 @@ export class ImportResolver {
                 minorVersion === 0 ? '3' : '2and3';
             const testPath = combinePaths(typeshedPath, pythonVersionString);
             if (this.fileSystem.existsSync(testPath)) {
-                const importInfo = this._resolveAbsoluteImport(testPath, moduleDescriptor,
+                const importInfo = this.resolveAbsoluteImport(testPath, moduleDescriptor,
                     importName, importFailureInfo);
                 if (importInfo && importInfo.isImportFound) {
                     importInfo.importType = isStdLib ? ImportType.BuiltIn : ImportType.ThirdParty;
@@ -536,7 +554,7 @@ export class ImportResolver {
         }
 
         // Now try to match the module parts from the current directory location.
-        const absImport = this._resolveAbsoluteImport(curDir, moduleDescriptor,
+        const absImport = this.resolveAbsoluteImport(curDir, moduleDescriptor,
             importName, importFailureInfo);
         if (!absImport) {
             return undefined;
@@ -565,7 +583,7 @@ export class ImportResolver {
 
     // Follows import resolution algorithm defined in PEP-420:
     // https://www.python.org/dev/peps/pep-0420/
-    private _resolveAbsoluteImport(rootPath: string, moduleDescriptor: ImportedModuleDescriptor,
+    protected resolveAbsoluteImport(rootPath: string, moduleDescriptor: ImportedModuleDescriptor,
         importName: string, importFailureInfo: string[], allowPartial = false,
         allowPydFile = false, allowStubsFolder = false): ImportResult | undefined {
 
@@ -893,3 +911,5 @@ export class ImportResolver {
         return name + moduleDescriptor.nameParts.map(part => part).join('.');
     }
 }
+
+export type ImportResolverFactory = (fs: VirtualFileSystem, options: ConfigOptions) => ImportResolver;

server/src/analyzer/service.ts

@@ -26,7 +26,7 @@ import { Duration, timingStats } from '../common/timing';
 import { FileWatcher, VirtualFileSystem } from '../common/vfs';
 import { HoverResults } from '../languageService/hoverProvider';
 import { SignatureHelpResults } from '../languageService/signatureHelpProvider';
-import { ImportedModuleDescriptor, ImportResolver } from './importResolver';
+import { ImportedModuleDescriptor, ImportResolver, ImportResolverFactory } from './importResolver';
 import { MaxAnalysisTime, Program } from './program';
 import * as PythonPathUtils from './pythonPathUtils';
 
@@ -50,6 +50,7 @@ export class AnalyzerService {
     private _instanceName: string;
     private _program: Program;
     private _configOptions: ConfigOptions;
+    private _importResolverFactory: ImportResolverFactory;
     private _importResolver: ImportResolver;
     private _executionRootPath: string;
     private _typeStubTargetImportName: string | undefined;
@@ -68,11 +69,12 @@ export class AnalyzerService {
     private _requireTrackedFileUpdate = true;
     private _lastUserInteractionTime = Date.now();
 
-    constructor(instanceName: string, fs: VirtualFileSystem, console?: ConsoleInterface, configOptions?: ConfigOptions) {
+    constructor(instanceName: string, fs: VirtualFileSystem, console?: ConsoleInterface, importResolverFactory?: ImportResolverFactory, configOptions?: ConfigOptions) {
         this._instanceName = instanceName;
         this._console = console || new StandardConsole();
         this._configOptions = configOptions ?? new ConfigOptions(process.cwd());
-        this._importResolver = new ImportResolver(fs, this._configOptions);
+        this._importResolverFactory = importResolverFactory || AnalyzerService.createImportResolver;
+        this._importResolver = this._importResolverFactory(fs, this._configOptions);
         this._program = new Program(this._importResolver, this._configOptions, this._console);
         this._executionRootPath = '';
         this._typeStubTargetImportName = undefined;
@@ -85,6 +87,10 @@ export class AnalyzerService {
         this._clearReanalysisTimer();
     }
 
+    static createImportResolver(fs: VirtualFileSystem, options: ConfigOptions): ImportResolver {
+        return new ImportResolver(fs, options);
+    }
+
     setCompletionCallback(callback: AnalysisCompleteCallback | undefined): void {
         this._onCompletionCallback = callback;
     }
@@ -815,7 +821,7 @@ export class AnalyzerService {
     private _applyConfigOptions() {
         // Allocate a new import resolver because the old one has information
         // cached based on the previous config options.
-        this._importResolver = new ImportResolver(this._fs, this._configOptions);
+        this._importResolver = this._importResolverFactory(this._fs, this._configOptions);
         this._program.setImportResolver(this._importResolver);
 
         this._updateSourceFileWatchers();

server/src/languageServerBase.ts

@@ -17,6 +17,8 @@ import { CommandLineOptions } from './common/commandLineOptions';
 import { Diagnostic as AnalyzerDiagnostic, DiagnosticCategory } from './common/diagnostic';
 import './common/extensions';
 import { combinePaths, convertPathToUri, convertUriToPath, normalizePath } from './common/pathUtils';
+import { ConfigOptions } from './common/configOptions';
+import { ImportResolver } from './analyzer/importResolver';
 import { Position } from './common/textRange';
 import { createFromRealFileSystem, VirtualFileSystem } from './common/vfs';
 import { CompletionItemData } from './languageService/completionProvider';
@@ -92,11 +94,15 @@ export abstract class LanguageServerBase {
         return this.connection.workspace.getConfiguration(item);
     }
 
+    protected createImportResolver(fs: VirtualFileSystem, options: ConfigOptions): ImportResolver {
+        return new ImportResolver(fs, options);
+    }
+
     // Creates a service instance that's used for analyzing a
     // program within a workspace.
     createAnalyzerService(name: string): AnalyzerService {
         this.connection.console.log(`Starting service instance "${ name }"`);
-        const service = new AnalyzerService(name, this.fs, this.connection.console);
+        const service = new AnalyzerService(name, this.fs, this.connection.console, this.createImportResolver);
 
         // Don't allow the analysis engine to go too long without
         // reporting results. This will keep it responsive.

server/src/tests/filesystem.test.ts

@@ -144,7 +144,8 @@ test('createFromFileSystem1', () => {
     const content = '# test';
 
     // file system will map physical file system to virtual one
-    const fs = factory.createFromFileSystem(host.HOST, false,
+    const resolver = factory.createResolver(host.HOST);
+    const fs = factory.createFromFileSystem(host.HOST, resolver, false,
         { documents: [new factory.TextDocument(filepath, content)], cwd: factory.srcFolder });
 
     // check existing typeshed folder on virtual path inherited from base snapshot from physical file system
@@ -156,14 +157,16 @@ test('createFromFileSystem1', () => {
 });
 
 test('createFromFileSystem2', () => {
-    const fs = factory.createFromFileSystem(host.HOST, /* ignoreCase */ true, { cwd: factory.srcFolder });
+    const resolver = factory.createResolver(host.HOST);
+    const fs = factory.createFromFileSystem(host.HOST, resolver, /* ignoreCase */ true, { cwd: factory.srcFolder });
     const entries = fs.readdirSync(factory.typeshedFolder.toUpperCase());
     assert(entries.length > 0);
 });
 
 test('createFromFileSystemWithCustomTypeshedPath', () => {
     const invalidpath = normalizeSlashes(combinePaths(host.HOST.getWorkspaceRoot(), '../docs'));
-    const fs = factory.createFromFileSystem(host.HOST, /* ignoreCase */ false, {
+    const resolver = factory.createResolver(host.HOST);
+    const fs = factory.createFromFileSystem(host.HOST, resolver, /* ignoreCase */ false, {
         cwd: factory.srcFolder, meta: { [factory.typeshedFolder]: invalidpath }
     });
 
@@ -172,7 +175,8 @@ test('createFromFileSystemWithCustomTypeshedPath', () => {
 });
 
 test('createFromFileSystemWithMetadata', () => {
-    const fs = factory.createFromFileSystem(host.HOST, /* ignoreCase */ false, {
+    const resolver = factory.createResolver(host.HOST);
+    const fs = factory.createFromFileSystem(host.HOST, resolver, /* ignoreCase */ false, {
         cwd: factory.srcFolder, meta: { 'unused': 'unused' }
     });
 

server/src/tests/fourSlashParser.test.ts

@@ -273,7 +273,8 @@ test('fourSlashWithFileSystem', () => {
     const documents = data.files.map(f => new factory.TextDocument(f.fileName, f.content,
         new Map<string, string>(Object.entries(f.fileOptions))));
 
-    const fs = factory.createFromFileSystem(host.HOST, /* ignoreCase */ false, { documents, cwd: normalizeSlashes('/') });
+    const resolver = factory.createResolver(host.HOST);
+    const fs = factory.createFromFileSystem(host.HOST, resolver, /* ignoreCase */ false, { documents, cwd: normalizeSlashes('/') });
 
     for (const file of data.files) {
         assert.equal(fs.readFileSync(file.fileName, 'utf8'), getContent(getBaseFileName(file.fileName, '.py', false)));

server/src/tests/fourslash/importnotresolved.fourslash.ts

@@ -0,0 +1,14 @@
+/// <reference path="fourslash.ts" />
+
+// @filename: importnotresolved.py
+//// # these will not be resolve, no typestubs for django in typeshed
+//// 
+//// import [|/*marker1*/notexistant|]
+//// import [|/*marker2*/django|]
+//// 
+
+
+helper.verifyDiagnostics({
+    "marker1": { category: "error", message: "Import 'notexistant' could not be resolved" },
+    "marker2": { category: "error", message: "Import 'django' could not be resolved" },
+});

server/src/tests/harness/fourslash/runner.ts

@@ -11,16 +11,17 @@ import { combinePaths } from '../../../common/pathUtils';
 import * as host from '../host';
 import { parseTestData } from './fourSlashParser';
 import { TestState } from './testState';
+import { ImportResolverFactory } from '../../../analyzer/importResolver';
 
 /**
  * run given fourslash test file
  *
  * @param basePath this is used as a base path of the virtual file system the test will run upon
  * @param fileName this is the file path where fourslash test file will be read from
  */
-export function runFourSlashTest(basePath: string, fileName: string) {
+export function runFourSlashTest(basePath: string, fileName: string, extraMountedPaths?: Map<string, string>, importResolverFactory?: ImportResolverFactory) {
     const content = (host.HOST.readFile(fileName)!);
-    runFourSlashTestContent(basePath, fileName, content);
+    runFourSlashTestContent(basePath, fileName, content, extraMountedPaths, importResolverFactory);
 }
 
 /**
@@ -31,14 +32,14 @@ export function runFourSlashTest(basePath: string, fileName: string) {
  *                 if fourslash markup `content` doesn't have explicit `@filename` option
  * @param content  this is fourslash markup string
  */
-export function runFourSlashTestContent(basePath: string, fileName: string, content: string) {
+export function runFourSlashTestContent(basePath: string, fileName: string, content: string, extraMountedPaths?: Map<string, string>, importResolverFactory?: ImportResolverFactory) {
     // give file paths an absolute path for the virtual file system
     const absoluteBasePath = combinePaths('/', basePath);
     const absoluteFileName = combinePaths('/', fileName);
 
     // parse out the files and their metadata
     const testData = parseTestData(absoluteBasePath, content, absoluteFileName);
-    const state = new TestState(absoluteBasePath, testData);
+    const state = new TestState(absoluteBasePath, testData, extraMountedPaths, importResolverFactory);
     const output = ts.transpileModule(content, { reportDiagnostics: true, compilerOptions: { target: ts.ScriptTarget.ES2015 } });
     if (output.diagnostics!.length > 0) {
         throw new Error(`Syntax error in ${ absoluteBasePath }: ${ output.diagnostics![0].messageText }`);

server/src/tests/harness/fourslash/testState.ts

@@ -10,7 +10,7 @@
 import * as assert from 'assert';
 import * as path from 'path';
 import Char from 'typescript-char';
-import { ImportResolver } from '../../../analyzer/importResolver';
+import { ImportResolver, ImportResolverFactory } from '../../../analyzer/importResolver';
 import { Program } from '../../../analyzer/program';
 import { ConfigOptions } from '../../../common/configOptions';
 import { NullConsole } from '../../../common/console';
@@ -23,10 +23,11 @@ import { getStringComparer } from '../../../common/stringUtils';
 import { Position, TextRange } from '../../../common/textRange';
 import * as host from '../host';
 import { stringify } from '../utils';
-import { createFromFileSystem } from '../vfs/factory';
+import { createFromFileSystem, createResolver } from '../vfs/factory';
 import * as vfs from '../vfs/filesystem';
 import { CompilerSettings, FourSlashData, FourSlashFile, GlobalMetadataOptionNames, Marker,
     MultiMap, pythonSettingFilename, Range, TestCancellationToken } from './fourSlashTypes';
+import { AnalyzerService } from '../../../analyzer/service';
 
 export interface TextChange {
     span: TextRange;
@@ -52,7 +53,7 @@ export class TestState {
     // The file that's currently 'opened'
     activeFile!: FourSlashFile;
 
-    constructor(private _basePath: string, public testData: FourSlashData) {
+    constructor(private _basePath: string, public testData: FourSlashData, extraMountedPaths?: Map<string, string>, importResolverFactory?: ImportResolverFactory) {
         const strIgnoreCase = GlobalMetadataOptionNames.ignoreCase;
         const ignoreCase = testData.globalOptions[strIgnoreCase]?.toUpperCase() === 'TRUE';
 
@@ -76,10 +77,19 @@ export class TestState {
             }
         }
 
-        const fs = createFromFileSystem(host.HOST, ignoreCase, { cwd: _basePath, files, meta: testData.globalOptions });
+        const resolver = createResolver(host.HOST);
+        const fs = createFromFileSystem(host.HOST, resolver, ignoreCase, { cwd: _basePath, files, meta: testData.globalOptions });
+
+        if (extraMountedPaths) {
+            extraMountedPaths.forEach((physicalPath, virtualPath) => {
+                fs.mountSync(physicalPath, virtualPath, resolver);
+            });
+        }
+
+        importResolverFactory = importResolverFactory || AnalyzerService.createImportResolver;
 
         // this should be change to AnalyzerService rather than Program
-        const importResolver = new ImportResolver(fs, configOptions);
+        const importResolver = importResolverFactory(fs, configOptions);
         const program = new Program(importResolver, configOptions);
         program.setTrackedFiles(Object.keys(files));