Skip to content

Commit

Permalink
Add PythonEnvInfo-related helpers. (microsoft#14051)
Browse files Browse the repository at this point in the history 
This PR adds some basic helpers that we use in a subsequent PR. The following small drive-by changes are also included:

* drop PythonEnvInfo.id property
* make some internal helpers public
  • Loading branch information
@ericsnowcurrently
ericsnowcurrently authored and Kartik Raj committed Sep 29, 2020
1 parent 218dd8d commit 98c0387
Show file tree
Hide file tree
Showing 4 changed files with 219 additions and 185 deletions.
100 changes: 26 additions & 74 deletions src/client/pythonEnvironments/base/info/env.ts
View file
Original file line number Diff line number Diff line change
Expand Up @@ -37,6 +37,8 @@ export function buildEnvInfo(init?: {
},
name: '',
location: '',
searchLocation: undefined,
defaultDisplayName: undefined,
version: {
major: -1,
minor: -1,
Expand Down Expand Up @@ -65,7 +67,7 @@ export function buildEnvInfo(init?: {
export function copyEnvInfo(
env: PythonEnvInfo,
updates?: {
kind?: PythonEnvKind,
kind?: PythonEnvKind;
},
): PythonEnvInfo {
// We don't care whether or not extra/hidden properties
Expand All @@ -77,12 +79,15 @@ export function copyEnvInfo(
return copied;
}

function updateEnv(env: PythonEnvInfo, updates: {
kind?: PythonEnvKind;
executable?: string;
location?: string;
version?: PythonVersion;
}): void {
function updateEnv(
env: PythonEnvInfo,
updates: {
kind?: PythonEnvKind;
executable?: string;
location?: string;
version?: PythonVersion;
},
): void {
if (updates.kind !== undefined) {
env.kind = updates.kind;
}
Expand All @@ -109,7 +114,9 @@ export function getMinimalPartialInfo(env: string | Partial<PythonEnvInfo>): Par
return undefined;
}
return {
executable: { filename: env, sysPrefix: '', ctime: -1, mtime: -1 },
executable: {
filename: env, sysPrefix: '', ctime: -1, mtime: -1,
},
};
}
if (env.executable === undefined) {
Expand All @@ -136,7 +143,7 @@ export function getMinimalPartialInfo(env: string | Partial<PythonEnvInfo>): Par
export function areSameEnv(
left: string | Partial<PythonEnvInfo>,
right: string | Partial<PythonEnvInfo>,
allowPartialMatch?: boolean,
allowPartialMatch = true,
): boolean | undefined {
const leftInfo = getMinimalPartialInfo(left);
const rightInfo = getMinimalPartialInfo(right);
Expand Down Expand Up @@ -173,7 +180,7 @@ export function areSameEnv(
* weighted by most important to least important fields.
* Wn > Wn-1 + Wn-2 + ... W0
*/
function getPythonVersionInfoHeuristic(version:PythonVersion): number {
function getPythonVersionInfoHeuristic(version: PythonVersion): number {
let infoLevel = 0;
if (version.major > 0) {
infoLevel += 20; // W4
Expand Down Expand Up @@ -205,7 +212,7 @@ function getPythonVersionInfoHeuristic(version:PythonVersion): number {
* weighted by most important to least important fields.
* Wn > Wn-1 + Wn-2 + ... W0
*/
function getFileInfoHeuristic(file:FileInfo): number {
function getFileInfoHeuristic(file: FileInfo): number {
let infoLevel = 0;
if (file.filename.length > 0) {
infoLevel += 5; // W2
Expand All @@ -229,7 +236,7 @@ function getFileInfoHeuristic(file:FileInfo): number {
* weighted by most important to least important fields.
* Wn > Wn-1 + Wn-2 + ... W0
*/
function getDistroInfoHeuristic(distro:PythonDistroInfo):number {
function getDistroInfoHeuristic(distro: PythonDistroInfo): number {
let infoLevel = 0;
if (distro.org.length > 0) {
infoLevel += 20; // W3
Expand All @@ -250,62 +257,6 @@ function getDistroInfoHeuristic(distro:PythonDistroInfo):number {
return infoLevel;
}

/**
* Gets a prioritized list of environment types for identification.
* @returns {PythonEnvKind[]} : List of environments ordered by identification priority
*
* Remarks: This is the order of detection based on how the various distributions and tools
* configure the environment, and the fall back for identification.
* Top level we have the following environment types, since they leave a unique signature
* in the environment or * use a unique path for the environments they create.
* 1. Conda
* 2. Windows Store
* 3. PipEnv
* 4. Pyenv
* 5. Poetry
*
* Next level we have the following virtual environment tools. The are here because they
* are consumed by the tools above, and can also be used independently.
* 1. venv
* 2. virtualenvwrapper
* 3. virtualenv
*
* Last category is globally installed python, or system python.
*/
export function getPrioritizedEnvironmentKind(): PythonEnvKind[] {
return [
PythonEnvKind.CondaBase,
PythonEnvKind.Conda,
PythonEnvKind.WindowsStore,
PythonEnvKind.Pipenv,
PythonEnvKind.Pyenv,
PythonEnvKind.Poetry,
PythonEnvKind.Venv,
PythonEnvKind.VirtualEnvWrapper,
PythonEnvKind.VirtualEnv,
PythonEnvKind.OtherVirtual,
PythonEnvKind.OtherGlobal,
PythonEnvKind.MacDefault,
PythonEnvKind.System,
PythonEnvKind.Custom,
PythonEnvKind.Unknown,
];
}

/**
* Selects an environment based on the environment selection priority. This should
* match the priority in the environment identifier.
*/
export function sortEnvInfoByPriority(...envs: PythonEnvInfo[]): PythonEnvInfo[] {
// tslint:disable-next-line: no-suspicious-comment
// TODO: When we consolidate the PythonEnvKind and EnvironmentType we should have
// one location where we define priority and
const envKindByPriority:PythonEnvKind[] = getPrioritizedEnvironmentKind();
return envs.sort(
(a:PythonEnvInfo, b:PythonEnvInfo) => envKindByPriority.indexOf(a.kind) - envKindByPriority.indexOf(b.kind),
);
}

/**
* Merges properties of the `target` environment and `other` environment and returns the merged environment.
* if the value in the `target` environment is not defined or has less information. This does not mutate
Expand All @@ -318,18 +269,19 @@ export function mergeEnvironments(target: PythonEnvInfo, other: PythonEnvInfo):

const version = cloneDeep(
getPythonVersionInfoHeuristic(target.version) > getPythonVersionInfoHeuristic(other.version)
? target.version : other.version,
? target.version
: other.version,
);

const executable = cloneDeep(
getFileInfoHeuristic(target.executable) > getFileInfoHeuristic(other.executable)
? target.executable : other.executable,
? target.executable
: other.executable,
);
executable.sysPrefix = target.executable.sysPrefix ?? other.executable.sysPrefix;

const distro = cloneDeep(
getDistroInfoHeuristic(target.distro) > getDistroInfoHeuristic(other.distro)
? target.distro : other.distro,
getDistroInfoHeuristic(target.distro) > getDistroInfoHeuristic(other.distro) ? target.distro : other.distro,
);

merged.arch = merged.arch === Architecture.Unknown ? other.arch : target.arch;
Expand All @@ -341,8 +293,8 @@ export function mergeEnvironments(target: PythonEnvInfo, other: PythonEnvInfo):
// preferred env based on kind.
merged.kind = target.kind;

merged.location = merged.location ?? other.location;
merged.name = merged.name ?? other.name;
merged.location = merged.location.length ? merged.location : other.location;
merged.name = merged.name.length ? merged.name : other.name;
merged.searchLocation = merged.searchLocation ?? other.searchLocation;
merged.version = version;

Expand Down
124 changes: 69 additions & 55 deletions src/client/pythonEnvironments/base/locators/composite/environmentsReducer.ts
View file
Original file line number Diff line number Diff line change
@@ -1,15 +1,15 @@
// Copyright (c) Microsoft Corporation. All rights reserved.
// Licensed under the MIT License.

import { cloneDeep, isEqual } from 'lodash';
import { isEqual } from 'lodash';
import { Event, EventEmitter } from 'vscode';
import { traceVerbose } from '../../../../common/logger';
import { createDeferred } from '../../../../common/utils/async';
import { PythonEnvInfo, PythonEnvKind } from '../../info';
import { areSameEnv } from '../../info/env';
import { areSameEnv, mergeEnvironments } from '../../info/env';
import {
ILocator, IPythonEnvsIterator, PythonEnvUpdatedEvent, PythonLocatorQuery,
} from '../../locator';
import { getEnvs } from '../../locatorUtils';
import { PythonEnvsChangedEvent } from '../../watcher';

/**
Expand All @@ -23,28 +23,11 @@ export class PythonEnvsReducer implements ILocator {
constructor(private readonly parentLocator: ILocator) {}

public async resolveEnv(env: string | PythonEnvInfo): Promise<PythonEnvInfo | undefined> {
let environment: PythonEnvInfo | undefined;
const waitForUpdatesDeferred = createDeferred<void>();
const iterator = this.iterEnvs();
iterator.onUpdated!((event) => {
if (event === null) {
waitForUpdatesDeferred.resolve();
} else if (environment && areSameEnv(environment, event.update)) {
environment = event.update;
}
});
let result = await iterator.next();
while (!result.done) {
if (areSameEnv(result.value, env)) {
environment = result.value;
}
// eslint-disable-next-line no-await-in-loop
result = await iterator.next();
}
const environments = await getEnvs(this.iterEnvs());
const environment = environments.find((e) => areSameEnv(e, env));
if (!environment) {
return undefined;
}
await waitForUpdatesDeferred.promise;
return this.parentLocator.resolveEnv(environment);
}

Expand Down Expand Up @@ -72,15 +55,12 @@ async function* iterEnvsIterator(
if (event === null) {
state.done = true;
checkIfFinishedAndNotify(state, didUpdate);
} else if (seen[event.index] !== undefined) {
state.pending += 1;
resolveDifferencesInBackground(event.index, event.update, state, didUpdate, seen).ignoreErrors();
} else {
if (seen[event.index] !== undefined) {
state.pending += 1;
resolveDifferencesInBackground(event.index, event.update, state, didUpdate, seen)
.ignoreErrors();
} else {
// This implies a problem in a downstream locator
traceVerbose(`Expected already iterated env, got ${event.old} (#${event.index})`);
}
// This implies a problem in a downstream locator
traceVerbose(`Expected already iterated env, got ${event.old} (#${event.index})`);
}
});
}
Expand Down Expand Up @@ -114,7 +94,7 @@ async function resolveDifferencesInBackground(
seen: PythonEnvInfo[],
) {
const oldEnv = seen[oldIndex];
const merged = mergeEnvironments(oldEnv, newEnv);
const merged = resolveEnvCollision(oldEnv, newEnv);
if (!isEqual(oldEnv, merged)) {
seen[oldIndex] = merged;
didUpdate.fire({ index: oldIndex, old: oldEnv, update: merged });
Expand All @@ -138,29 +118,63 @@ function checkIfFinishedAndNotify(
}
}

export function mergeEnvironments(environment: PythonEnvInfo, other: PythonEnvInfo): PythonEnvInfo {
const result = cloneDeep(environment);
// Preserve type information.
// Possible we identified environment as unknown, but a later provider has identified env type.
if (environment.kind === PythonEnvKind.Unknown && other.kind && other.kind !== PythonEnvKind.Unknown) {
result.kind = other.kind;
}
const props: (keyof PythonEnvInfo)[] = [
'version',
'kind',
'executable',
'name',
'arch',
'distro',
'defaultDisplayName',
'searchLocation',
function resolveEnvCollision(oldEnv: PythonEnvInfo, newEnv: PythonEnvInfo): PythonEnvInfo {
const [env, other] = sortEnvInfoByPriority(oldEnv, newEnv);
return mergeEnvironments(env, other);
}

/**
* Selects an environment based on the environment selection priority. This should
* match the priority in the environment identifier.
*/
function sortEnvInfoByPriority(...envs: PythonEnvInfo[]): PythonEnvInfo[] {
// tslint:disable-next-line: no-suspicious-comment
// TODO: When we consolidate the PythonEnvKind and EnvironmentType we should have
// one location where we define priority and
const envKindByPriority: PythonEnvKind[] = getPrioritizedEnvironmentKind();
return envs.sort(
(a: PythonEnvInfo, b: PythonEnvInfo) => envKindByPriority.indexOf(a.kind) - envKindByPriority.indexOf(b.kind),
);
}

/**
* Gets a prioritized list of environment types for identification.
* @returns {PythonEnvKind[]} : List of environments ordered by identification priority
*
* Remarks: This is the order of detection based on how the various distributions and tools
* configure the environment, and the fall back for identification.
* Top level we have the following environment types, since they leave a unique signature
* in the environment or * use a unique path for the environments they create.
* 1. Conda
* 2. Windows Store
* 3. PipEnv
* 4. Pyenv
* 5. Poetry
*
* Next level we have the following virtual environment tools. The are here because they
* are consumed by the tools above, and can also be used independently.
* 1. venv
* 2. virtualenvwrapper
* 3. virtualenv
*
* Last category is globally installed python, or system python.
*/
function getPrioritizedEnvironmentKind(): PythonEnvKind[] {
return [
PythonEnvKind.CondaBase,
PythonEnvKind.Conda,
PythonEnvKind.WindowsStore,
PythonEnvKind.Pipenv,
PythonEnvKind.Pyenv,
PythonEnvKind.Poetry,
PythonEnvKind.Venv,
PythonEnvKind.VirtualEnvWrapper,
PythonEnvKind.VirtualEnv,
PythonEnvKind.OtherVirtual,
PythonEnvKind.OtherGlobal,
PythonEnvKind.MacDefault,
PythonEnvKind.System,
PythonEnvKind.Custom,
PythonEnvKind.Unknown,
];
props.forEach((prop) => {
if (!result[prop] && other[prop]) {
// tslint:disable: no-any
// eslint-disable-next-line @typescript-eslint/no-explicit-any
(result as any)[prop] = other[prop];
}
});
return result;
}
Loading

0 comments on commit 98c0387

Please sign in to comment.