Cherry picked fixes from main

.eslintignore

@@ -186,7 +186,6 @@ src/test/common/terminals/service.unit.test.ts
 src/test/common/terminals/helper.unit.test.ts
 src/test/common/terminals/activation.unit.test.ts
 src/test/common/terminals/shellDetectors/shellDetectors.unit.test.ts
-src/test/common/terminals/environmentActivationProviders/pipEnvActivationProvider.unit.test.ts
 src/test/common/terminals/environmentActivationProviders/terminalActivation.testvirtualenvs.ts
 src/test/common/socketStream.test.ts
 src/test/common/configSettings.test.ts
@@ -210,8 +209,6 @@ src/test/common/featureDeprecationManager.unit.test.ts
 src/test/common/serviceRegistry.unit.test.ts
 src/test/common/extensions.unit.test.ts
 src/test/common/variables/envVarsService.unit.test.ts
-src/test/common/variables/environmentVariablesProvider.unit.test.ts
-src/test/common/variables/envVarsProvider.multiroot.test.ts
 src/test/common/nuget/azureBobStoreRepository.unit.test.ts
 src/test/common/helpers.test.ts
 src/test/common/application/commands/reloadCommand.unit.test.ts
@@ -383,7 +380,6 @@ src/client/providers/docStringFoldingProvider.ts
 src/client/providers/linterProvider.ts
 src/client/providers/simpleRefactorProvider.ts
 src/client/providers/completionProvider.ts
-src/client/providers/jediProxy.ts
 src/client/providers/definitionProvider.ts
 src/client/providers/referenceProvider.ts
 src/client/providers/terminalProvider.ts
@@ -549,7 +545,6 @@ src/client/common/terminal/shellDetectors/vscEnvironmentShellDetector.ts
 src/client/common/terminal/shellDetectors/terminalNameShellDetector.ts
 src/client/common/terminal/shellDetectors/settingsShellDetector.ts
 src/client/common/terminal/shellDetectors/baseShellDetector.ts
-src/client/common/terminal/environmentActivationProviders/pipEnvActivationProvider.ts
 src/client/common/terminal/environmentActivationProviders/baseActivationProvider.ts
 src/client/common/terminal/environmentActivationProviders/commandPrompt.ts
 src/client/common/terminal/environmentActivationProviders/bash.ts
@@ -578,7 +573,6 @@ src/client/common/constants.ts
 src/client/common/variables/serviceRegistry.ts
 src/client/common/variables/environment.ts
 src/client/common/variables/types.ts
-src/client/common/variables/environmentVariablesProvider.ts
 src/client/common/variables/sysTypes.ts
 src/client/common/variables/systemVariables.ts
 src/client/common/nuget/azureBlobStoreNugetRepository.ts

CHANGELOG.md

@@ -653,6 +653,8 @@ part of!
    ([#14182](https://github.com/Microsoft/vscode-python/issues/14182))
 1. Fix exporting from the interactive window.
    ([#14210](https://github.com/Microsoft/vscode-python/issues/14210))
+1. Fix for CVE-2020-16977
+   ([CVE-2020-16977](https://msrc.microsoft.com/update-guide/vulnerability/CVE-2020-16977))
 
 ### Thanks
 

news/2 Fixes/15364.md

@@ -0,0 +1 @@
+Allow support for using notebook APIs in the VS code stable build.

package.json

@@ -1044,7 +1044,6 @@
                             "DeprecatePythonPath - experiment",
                             "RunByLine - experiment",
                             "tryPylance",
-                            "jediLSP",
                             "debuggerDataViewer",
                             "pythonSendEntireLineToREPL",
                             "pythonTensorboardExperiment",
@@ -1074,7 +1073,6 @@
                             "DeprecatePythonPath - experiment",
                             "RunByLine - experiment",
                             "tryPylance",
-                            "jediLSP",
                             "debuggerDataViewer",
                             "pythonSendEntireLineToREPL",
                             "pythonTensorboardExperiment",

src/client/activation/jedi/analysisOptions.ts

@@ -1,24 +1,61 @@
 // Copyright (c) Microsoft Corporation. All rights reserved.
 // Licensed under the MIT License.
 import { inject, injectable } from 'inversify';
+import * as path from 'path';
+import { WorkspaceFolder } from 'vscode';
+import { IWorkspaceService } from '../../common/application/types';
+import { IConfigurationService, Resource } from '../../common/types';
 
 import { IEnvironmentVariablesProvider } from '../../common/variables/types';
+import { PythonEnvironment } from '../../pythonEnvironments/info';
 import { LanguageServerAnalysisOptionsWithEnv } from '../common/analysisOptions';
 import { ILanguageServerOutputChannel } from '../types';
 
 /* eslint-disable @typescript-eslint/explicit-module-boundary-types, class-methods-use-this */
 
 @injectable()
 export class JediLanguageServerAnalysisOptions extends LanguageServerAnalysisOptionsWithEnv {
-    // eslint-disable-next-line @typescript-eslint/no-useless-constructor
+    private resource: Resource | undefined;
+
     constructor(
         @inject(IEnvironmentVariablesProvider) envVarsProvider: IEnvironmentVariablesProvider,
         @inject(ILanguageServerOutputChannel) lsOutputChannel: ILanguageServerOutputChannel,
+        @inject(IConfigurationService) private readonly configurationService: IConfigurationService,
+        @inject(IWorkspaceService) private readonly workspace: IWorkspaceService,
     ) {
         super(envVarsProvider, lsOutputChannel);
+        this.resource = undefined;
+    }
+
+    public async initialize(resource: Resource, interpreter: PythonEnvironment | undefined) {
+        this.resource = resource;
+        return super.initialize(resource, interpreter);
+    }
+
+    protected getWorkspaceFolder(): WorkspaceFolder | undefined {
+        return this.workspace.getWorkspaceFolder(this.resource);
     }
 
     protected async getInitializationOptions() {
+        const pythonSettings = this.configurationService.getSettings(this.resource);
+        const workspacePath = this.getWorkspaceFolder()?.uri.fsPath;
+        const extraPaths = pythonSettings.autoComplete
+            ? pythonSettings.autoComplete.extraPaths.map((extraPath) => {
+                  if (path.isAbsolute(extraPath)) {
+                      return extraPath;
+                  }
+                  return workspacePath ? path.join(workspacePath, extraPath) : '';
+              })
+            : [];
+
+        if (workspacePath) {
+            extraPaths.unshift(workspacePath);
+        }
+
+        const distinctExtraPaths = extraPaths
+            .filter((value) => value.length > 0)
+            .filter((value, index, self) => self.indexOf(value) === index);
+
         return {
             markupKindPreferred: 'markdown',
             completion: {
@@ -31,6 +68,9 @@ export class JediLanguageServerAnalysisOptions extends LanguageServerAnalysisOpt
                 didSave: true,
                 didChange: true,
             },
+            workspace: {
+                extraPaths: distinctExtraPaths,
+            },
         };
     }
 }

src/client/common/process/proc.ts

@@ -1,23 +1,17 @@
 // Copyright (c) Microsoft Corporation. All rights reserved.
 // Licensed under the MIT License.
-import { exec, execSync, spawn } from 'child_process';
 import { EventEmitter } from 'events';
-import { Observable } from 'rxjs/Observable';
-import { Readable } from 'stream';
 
 import { IDisposable } from '../types';
-import { createDeferred } from '../utils/async';
 import { EnvironmentVariables } from '../variables/types';
-import { DEFAULT_ENCODING } from './constants';
+import { execObservable, killPid, plainExec, shellExec } from './rawProcessApis';
 import {
     ExecutionResult,
     IBufferDecoder,
     IProcessService,
     ObservableExecutionResult,
-    Output,
     ShellOptions,
     SpawnOptions,
-    StdErrError,
 } from './types';
 
 export class ProcessService extends EventEmitter implements IProcessService {
@@ -37,16 +31,7 @@ export class ProcessService extends EventEmitter implements IProcessService {
     }
 
     public static kill(pid: number): void {
-        try {
-            if (process.platform === 'win32') {
-                // Windows doesn't support SIGTERM, so execute taskkill to kill the process
-                execSync(`taskkill /pid ${pid} /T /F`); // NOSONAR
-            } else {
-                process.kill(pid);
-            }
-        } catch {
-            // Ignore.
-        }
+        killPid(pid);
     }
 
     public dispose(): void {
@@ -61,199 +46,18 @@ export class ProcessService extends EventEmitter implements IProcessService {
     }
 
     public execObservable(file: string, args: string[], options: SpawnOptions = {}): ObservableExecutionResult<string> {
-        const spawnOptions = this.getDefaultOptions(options);
-        const encoding = spawnOptions.encoding ? spawnOptions.encoding : 'utf8';
-        const proc = spawn(file, args, spawnOptions);
-        let procExited = false;
-        const disposable: IDisposable = {
-            dispose() {
-                if (proc && !proc.killed && !procExited) {
-                    ProcessService.kill(proc.pid);
-                }
-                if (proc) {
-                    proc.unref();
-                }
-            },
-        };
-        this.processesToKill.add(disposable);
-
-        const output = new Observable<Output<string>>((subscriber) => {
-            const disposables: IDisposable[] = [];
-
-            // eslint-disable-next-line @typescript-eslint/ban-types
-            const on = (ee: Readable | null, name: string, fn: Function) => {
-                // eslint-disable-next-line @typescript-eslint/no-explicit-any
-                ee?.on(name, fn as any);
-                // eslint-disable-next-line @typescript-eslint/no-explicit-any
-                disposables.push({ dispose: () => ee?.removeListener(name, fn as any) as any });
-            };
-
-            if (options.token) {
-                disposables.push(
-                    options.token.onCancellationRequested(() => {
-                        if (!procExited && !proc.killed) {
-                            proc.kill();
-                            procExited = true;
-                        }
-                    }),
-                );
-            }
-
-            const sendOutput = (source: 'stdout' | 'stderr', data: Buffer) => {
-                const out = this.decoder.decode([data], encoding);
-                if (source === 'stderr' && options.throwOnStdErr) {
-                    subscriber.error(new StdErrError(out));
-                } else {
-                    subscriber.next({ source, out });
-                }
-            };
-
-            on(proc.stdout, 'data', (data: Buffer) => sendOutput('stdout', data));
-            on(proc.stderr, 'data', (data: Buffer) => sendOutput('stderr', data));
-
-            proc.once('close', () => {
-                procExited = true;
-                subscriber.complete();
-                disposables.forEach((d) => d.dispose());
-            });
-            proc.once('exit', () => {
-                procExited = true;
-                subscriber.complete();
-                disposables.forEach((d) => d.dispose());
-            });
-            proc.once('error', (ex) => {
-                procExited = true;
-                subscriber.error(ex);
-                disposables.forEach((d) => d.dispose());
-            });
-        });
-
+        const result = execObservable(file, args, options, this.decoder, this.env, this.processesToKill);
         this.emit('exec', file, args, options);
-
-        return {
-            proc,
-            out: output,
-            dispose: disposable.dispose,
-        };
+        return result;
     }
 
     public exec(file: string, args: string[], options: SpawnOptions = {}): Promise<ExecutionResult<string>> {
-        const spawnOptions = this.getDefaultOptions(options);
-        const encoding = spawnOptions.encoding ? spawnOptions.encoding : 'utf8';
-        const proc = spawn(file, args, spawnOptions);
-        const deferred = createDeferred<ExecutionResult<string>>();
-        const disposable: IDisposable = {
-            dispose: () => {
-                if (!proc.killed && !deferred.completed) {
-                    proc.kill();
-                }
-            },
-        };
-        this.processesToKill.add(disposable);
-        const disposables: IDisposable[] = [];
-
-        // eslint-disable-next-line @typescript-eslint/ban-types
-        const on = (ee: Readable | null, name: string, fn: Function) => {
-            // eslint-disable-next-line @typescript-eslint/no-explicit-any
-            ee?.on(name, fn as any);
-            // eslint-disable-next-line @typescript-eslint/no-explicit-any
-            disposables.push({ dispose: () => ee?.removeListener(name, fn as any) as any });
-        };
-
-        if (options.token) {
-            disposables.push(options.token.onCancellationRequested(disposable.dispose));
-        }
-
-        const stdoutBuffers: Buffer[] = [];
-        on(proc.stdout, 'data', (data: Buffer) => stdoutBuffers.push(data));
-        const stderrBuffers: Buffer[] = [];
-        on(proc.stderr, 'data', (data: Buffer) => {
-            if (options.mergeStdOutErr) {
-                stdoutBuffers.push(data);
-                stderrBuffers.push(data);
-            } else {
-                stderrBuffers.push(data);
-            }
-        });
-
-        proc.once('close', () => {
-            if (deferred.completed) {
-                return;
-            }
-            const stderr: string | undefined =
-                stderrBuffers.length === 0 ? undefined : this.decoder.decode(stderrBuffers, encoding);
-            if (stderr && stderr.length > 0 && options.throwOnStdErr) {
-                deferred.reject(new StdErrError(stderr));
-            } else {
-                const stdout = this.decoder.decode(stdoutBuffers, encoding);
-                deferred.resolve({ stdout, stderr });
-            }
-            disposables.forEach((d) => d.dispose());
-        });
-        proc.once('error', (ex) => {
-            deferred.reject(ex);
-            disposables.forEach((d) => d.dispose());
-        });
-
+        const promise = plainExec(file, args, options, this.decoder, this.env, this.processesToKill);
         this.emit('exec', file, args, options);
-
-        return deferred.promise;
+        return promise;
     }
 
     public shellExec(command: string, options: ShellOptions = {}): Promise<ExecutionResult<string>> {
-        const shellOptions = this.getDefaultOptions(options);
-        return new Promise((resolve, reject) => {
-            const proc = exec(command, shellOptions, (e, stdout, stderr) => {
-                if (e && e !== null) {
-                    reject(e);
-                } else if (shellOptions.throwOnStdErr && stderr && stderr.length) {
-                    reject(new Error(stderr));
-                } else {
-                    // Make sure stderr is undefined if we actually had none. This is checked
-                    // elsewhere because that's how exec behaves.
-                    resolve({ stderr: stderr && stderr.length > 0 ? stderr : undefined, stdout });
-                }
-            }); // NOSONAR
-            const disposable: IDisposable = {
-                dispose: () => {
-                    if (!proc.killed) {
-                        proc.kill();
-                    }
-                },
-            };
-            this.processesToKill.add(disposable);
-        });
-    }
-
-    private getDefaultOptions<T extends ShellOptions | SpawnOptions>(options: T): T {
-        const defaultOptions = { ...options };
-        const execOptions = defaultOptions as SpawnOptions;
-        if (execOptions) {
-            execOptions.encoding =
-                typeof execOptions.encoding === 'string' && execOptions.encoding.length > 0
-                    ? execOptions.encoding
-                    : DEFAULT_ENCODING;
-            const { encoding } = execOptions;
-            delete execOptions.encoding;
-            execOptions.encoding = encoding;
-        }
-        if (!defaultOptions.env || Object.keys(defaultOptions.env).length === 0) {
-            const env = this.env ? this.env : process.env;
-            defaultOptions.env = { ...env };
-        } else {
-            defaultOptions.env = { ...defaultOptions.env };
-        }
-
-        if (execOptions && execOptions.extraVariables) {
-            defaultOptions.env = { ...defaultOptions.env, ...execOptions.extraVariables };
-        }
-
-        // Always ensure we have unbuffered output.
-        defaultOptions.env.PYTHONUNBUFFERED = '1';
-        if (!defaultOptions.env.PYTHONIOENCODING) {
-            defaultOptions.env.PYTHONIOENCODING = 'utf-8';
-        }
-
-        return defaultOptions;
+        return shellExec(command, options, this.env, this.processesToKill);
     }
 }