Skip to content
New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Jump to bottom

cache builtn extensions locations from gallery #183467

Merged
merged 1 commit into from May 25, 2023
Merged

cache builtn extensions locations from gallery #183467

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
1 commit
Select commit
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Jump to
Jump to file
Failed to load files.
Diff view
Diff view
9 changes: 7 additions & 2 deletions src/vs/platform/extensionResourceLoader/common/extensionResourceLoader.ts
View file
Open in desktop
Expand Up @@ -40,6 +40,11 @@ export interface IExtensionResourceLoaderService {
*/
readonly supportsExtensionGalleryResources: boolean;

/**
* Return true if the given URI is a extension gallery resource.
*/
isExtensionGalleryResource(uri: URI): boolean;

/**
* Computes the URL of a extension gallery resource. Returns `undefined` if gallery does not provide extension resources.
*/
Expand Down Expand Up @@ -104,8 +109,8 @@ export abstract class AbstractExtensionResourceLoaderService implements IExtensi

public abstract readExtensionResource(uri: URI): Promise<string>;

protected isExtensionGalleryResource(uri: URI) {
return this._extensionGalleryAuthority && this._extensionGalleryAuthority === this._getExtensionGalleryAuthority(uri);
isExtensionGalleryResource(uri: URI): boolean {
return !!this._extensionGalleryAuthority && this._extensionGalleryAuthority === this._getExtensionGalleryAuthority(uri);
}

protected async getExtensionGalleryRequestHeaders(): Promise<IHeaders> {
Expand Down
13 changes: 1 addition & 12 deletions src/vs/workbench/browser/web.api.ts
View file
Open in desktop
Expand Up @@ -18,8 +18,6 @@ import type { IProgress, IProgressCompositeOptions, IProgressDialogOptions, IPro
import type { ITextEditorOptions } from 'vs/platform/editor/common/editor';
import type { EditorGroupLayout } from 'vs/workbench/services/editor/common/editorGroupsService';
import type { IEmbedderTerminalOptions } from 'vs/workbench/services/terminal/common/embedderTerminalService';
import { IExtensionManifest } from 'vs/platform/extensions/common/extensions';
import { ITranslations } from 'vs/platform/extensionManagement/common/extensionNls';

/**
* The `IWorkbench` interface is the API facade for web embedders
Expand Down Expand Up @@ -222,7 +220,7 @@ export interface IWorkbenchConstructionOptions {
* - an extension in the Marketplace
* - location of the extension where it is hosted.
*/
readonly additionalBuiltinExtensions?: readonly (MarketplaceExtension | UriComponents | HostedExtension)[];
readonly additionalBuiltinExtensions?: readonly (MarketplaceExtension | UriComponents)[];

/**
* List of extensions to be enabled if they are installed.
Expand Down Expand Up @@ -375,15 +373,6 @@ export interface IResourceUriProvider {
export type ExtensionId = string;

export type MarketplaceExtension = ExtensionId | { readonly id: ExtensionId; preRelease?: boolean; migrateStorageFrom?: ExtensionId };
export interface HostedExtension {
readonly location: UriComponents;
readonly preRelease?: boolean;
readonly packageJSON?: IExtensionManifest;
readonly defaultPackageTranslations?: ITranslations | null;
readonly packageNLSUris?: Map<string, UriComponents>;
readonly readmeUri?: UriComponents;
readonly changelogUri?: UriComponents;
}

export interface ICommonTelemetryPropertiesResolver {
(): { [key: string]: any };
Expand Down
177 changes: 117 additions & 60 deletions src/vs/workbench/services/extensionManagement/browser/webExtensionsScannerService.ts
View file
Open in desktop
Expand Up @@ -45,15 +45,6 @@ import { IUserDataProfilesService } from 'vs/platform/userDataProfile/common/use
import { IUriIdentityService } from 'vs/platform/uriIdentity/common/uriIdentity';

type GalleryExtensionInfo = { readonly id: string; preRelease?: boolean; migrateStorageFrom?: string };
interface HostedExtensionInfo {
readonly location: UriComponents;
readonly preRelease?: boolean;
readonly packageJSON?: IExtensionManifest;
readonly defaultPackageTranslations?: ITranslations | null;
readonly packageNLSUris?: Map<string, UriComponents>;
readonly readmeUri?: UriComponents;
readonly changelogUri?: UriComponents;
}
type ExtensionInfo = { readonly id: string; preRelease: boolean };

function isGalleryExtensionInfo(obj: unknown): obj is GalleryExtensionInfo {
Expand All @@ -63,16 +54,6 @@ function isGalleryExtensionInfo(obj: unknown): obj is GalleryExtensionInfo {
&& (galleryExtensionInfo.migrateStorageFrom === undefined || typeof galleryExtensionInfo.migrateStorageFrom === 'string');
}

function isHostedExtensionInfo(obj: unknown): obj is HostedExtensionInfo {
const hostedExtensionInfo = obj as HostedExtensionInfo | undefined;
return isUriComponents(hostedExtensionInfo?.location)
&& (hostedExtensionInfo?.preRelease === undefined || typeof hostedExtensionInfo.preRelease === 'boolean')
&& (hostedExtensionInfo?.packageJSON === undefined || typeof hostedExtensionInfo.packageJSON === 'object')
&& (hostedExtensionInfo?.defaultPackageTranslations === undefined || hostedExtensionInfo?.defaultPackageTranslations === null || typeof hostedExtensionInfo.defaultPackageTranslations === 'object')
&& (hostedExtensionInfo?.changelogUri === undefined || isUriComponents(hostedExtensionInfo?.changelogUri))
&& (hostedExtensionInfo?.readmeUri === undefined || isUriComponents(hostedExtensionInfo?.readmeUri));
}

function isUriComponents(thing: unknown): thing is UriComponents {
if (!thing) {
return false;
Expand Down Expand Up @@ -144,12 +125,13 @@ export class WebExtensionsScannerService extends Disposable implements IWebExten
}
}

private _customBuiltinExtensionsInfoPromise: Promise<{ extensions: ExtensionInfo[]; extensionsToMigrate: [string, string][]; extensionLocations: HostedExtensionInfo[] }> | undefined;
private readCustomBuiltinExtensionsInfoFromEnv(): Promise<{ extensions: ExtensionInfo[]; extensionsToMigrate: [string, string][]; extensionLocations: HostedExtensionInfo[] }> {
private _customBuiltinExtensionsInfoPromise: Promise<{ extensions: ExtensionInfo[]; extensionsToMigrate: [string, string][]; extensionLocations: URI[]; extensionGalleryResources: URI[] }> | undefined;
private readCustomBuiltinExtensionsInfoFromEnv(): Promise<{ extensions: ExtensionInfo[]; extensionsToMigrate: [string, string][]; extensionLocations: URI[]; extensionGalleryResources: URI[] }> {
if (!this._customBuiltinExtensionsInfoPromise) {
this._customBuiltinExtensionsInfoPromise = (async () => {
let extensions: ExtensionInfo[] = [];
const extensionLocations: HostedExtensionInfo[] = [];
const extensionLocations: URI[] = [];
const extensionGalleryResources: URI[] = [];
const extensionsToMigrate: [string, string][] = [];
const customBuiltinExtensionsInfo = this.environmentService.options && Array.isArray(this.environmentService.options.additionalBuiltinExtensions)
? this.environmentService.options.additionalBuiltinExtensions.map(additionalBuiltinExtension => isString(additionalBuiltinExtension) ? { id: additionalBuiltinExtension } : additionalBuiltinExtension)
Expand All @@ -160,11 +142,12 @@ export class WebExtensionsScannerService extends Disposable implements IWebExten
if (e.migrateStorageFrom) {
extensionsToMigrate.push([e.migrateStorageFrom, e.id]);
}
} else {
if (isHostedExtensionInfo(e)) {
extensionLocations.push(e);
} else if (isUriComponents(e)) {
const extensionLocation = URI.revive(e);
if (this.extensionResourceLoaderService.isExtensionGalleryResource(extensionLocation)) {
extensionGalleryResources.push(extensionLocation);
} else {
extensionLocations.push({ location: e });
extensionLocations.push(extensionLocation);
}
}
}
Expand All @@ -177,7 +160,10 @@ export class WebExtensionsScannerService extends Disposable implements IWebExten
if (extensionLocations.length) {
this.logService.info('Found additional builtin location extensions in env', extensionLocations.map(e => e.toString()));
}
return { extensions, extensionsToMigrate, extensionLocations };
if (extensionGalleryResources.length) {
this.logService.info('Found additional builtin extension gallery resources in env', extensionGalleryResources.map(e => e.toString()));
}
return { extensions, extensionsToMigrate, extensionLocations, extensionGalleryResources };
})();
}
return this._customBuiltinExtensionsInfoPromise;
Expand Down Expand Up @@ -243,15 +229,9 @@ export class WebExtensionsScannerService extends Disposable implements IWebExten
return [];
}
const result: IScannedExtension[] = [];
await Promise.allSettled(extensionLocations.map(async ({ location, preRelease, packageNLSUris, packageJSON, defaultPackageTranslations, readmeUri, changelogUri }) => {
await Promise.allSettled(extensionLocations.map(async extensionLocation => {
try {
const webExtension = await this.toWebExtension(URI.revive(location), undefined,
packageJSON,
packageNLSUris ? [...packageNLSUris.entries()].reduce((result, [key, value]) => { result.set(key, URI.revive(value)); return result; }, new Map<string, URI>()) : undefined,
defaultPackageTranslations,
URI.revive(readmeUri),
URI.revive(changelogUri),
{ isPreReleaseVersion: preRelease });
const webExtension = await this.toWebExtension(extensionLocation);
const extension = await this.toScannedExtension(webExtension, true);
if (extension.isValid || !scanOptions?.skipInvalidExtensions) {
result.push(extension);
Expand All @@ -271,9 +251,13 @@ export class WebExtensionsScannerService extends Disposable implements IWebExten
return [];
}
const result: IScannedExtension[] = [];
const { extensions } = await this.readCustomBuiltinExtensionsInfoFromEnv();
const { extensions, extensionGalleryResources } = await this.readCustomBuiltinExtensionsInfoFromEnv();
try {
const useCache = this.storageService.get('additionalBuiltinExtensions', StorageScope.APPLICATION, '[]') === JSON.stringify(extensions);
const cacheValue = JSON.stringify({
extensions: extensions.sort((a, b) => a.id.localeCompare(b.id)),
extensionGalleryResources: extensionGalleryResources.map(e => e.toString()).sort()
});
const useCache = this.storageService.get('additionalBuiltinExtensions', StorageScope.APPLICATION, '{}') === cacheValue;
const webExtensions = await (useCache ? this.getCustomBuiltinExtensionsFromCache() : this.updateCustomBuiltinExtensionsCache());
if (webExtensions.length) {
await Promise.all(webExtensions.map(async webExtension => {
Expand All @@ -289,7 +273,7 @@ export class WebExtensionsScannerService extends Disposable implements IWebExten
}
}));
}
this.storageService.store('additionalBuiltinExtensions', JSON.stringify(extensions), StorageScope.APPLICATION, StorageTarget.MACHINE);
this.storageService.store('additionalBuiltinExtensions', cacheValue, StorageScope.APPLICATION, StorageTarget.MACHINE);
} catch (error) {
this.logService.info('Ignoring following additional builtin extensions as there is an error while fetching them from gallery', extensions.map(({ id }) => id), getErrorMessage(error));
}
Expand Down Expand Up @@ -366,31 +350,95 @@ export class WebExtensionsScannerService extends Disposable implements IWebExten
if (!this._updateCustomBuiltinExtensionsCachePromise) {
this._updateCustomBuiltinExtensionsCachePromise = (async () => {
this.logService.info('Updating additional builtin extensions cache');
const webExtensions: IWebExtension[] = [];
const { extensions } = await this.readCustomBuiltinExtensionsInfoFromEnv();
if (extensions.length) {
const galleryExtensionsMap = await this.getExtensionsWithDependenciesAndPackedExtensions(extensions);
const missingExtensions = extensions.filter(({ id }) => !galleryExtensionsMap.has(id.toLowerCase()));
if (missingExtensions.length) {
this.logService.info('Skipping the additional builtin extensions because their compatible versions are not found.', missingExtensions);
}
await Promise.all([...galleryExtensionsMap.values()].map(async gallery => {
try {
const webExtension = await this.toWebExtensionFromGallery(gallery, { isPreReleaseVersion: gallery.properties.isPreReleaseVersion, preRelease: gallery.properties.isPreReleaseVersion, isBuiltin: true });
webExtensions.push(webExtension);
} catch (error) {
this.logService.info(`Ignoring additional builtin extension ${gallery.identifier.id} because there is an error while converting it into web extension`, getErrorMessage(error));
}
}));
const { extensions, extensionGalleryResources } = await this.readCustomBuiltinExtensionsInfoFromEnv();
const [galleryWebExtensions, extensionGalleryResourceWebExtensions] = await Promise.all([
this.resolveBuiltinGalleryExtensions(extensions),
this.resolveBuiltinExtensionGalleryResources(extensionGalleryResources)
]);
const webExtensionsMap = new Map<string, IWebExtension>();
for (const webExtension of [...galleryWebExtensions, ...extensionGalleryResourceWebExtensions]) {
webExtensionsMap.set(webExtension.identifier.id.toLowerCase(), webExtension);
}
await this.resolveDependenciesAndPackedExtensions(extensionGalleryResourceWebExtensions, webExtensionsMap);
const webExtensions = [...webExtensionsMap.values()];
await this.writeCustomBuiltinExtensionsCache(() => webExtensions);
return webExtensions;
})();
}
return this._updateCustomBuiltinExtensionsCachePromise;
}

private async getExtensionsWithDependenciesAndPackedExtensions(toGet: IExtensionInfo[], result: Map<string, IGalleryExtension> = new Map<string, IGalleryExtension>()): Promise<Map<string, IGalleryExtension>> {
private async resolveBuiltinExtensionGalleryResources(extensionGalleryResources: URI[]): Promise<IWebExtension[]> {
if (extensionGalleryResources.length === 0) {
return [];
}
const result = new Map<string, IWebExtension>();
const extensionInfos: IExtensionInfo[] = [];
await Promise.all(extensionGalleryResources.map(async extensionGalleryResource => {
const webExtension = await this.toWebExtensionFromExtensionGalleryResource(extensionGalleryResource);
result.set(webExtension.identifier.id.toLowerCase(), webExtension);
extensionInfos.push({ id: webExtension.identifier.id, version: webExtension.version });
}));
const galleryExtensions = await this.galleryService.getExtensions(extensionInfos, CancellationToken.None);
for (const galleryExtension of galleryExtensions) {
const webExtension = result.get(galleryExtension.identifier.id.toLowerCase());
if (webExtension) {
result.set(galleryExtension.identifier.id.toLowerCase(), {
...webExtension,
readmeUri: galleryExtension.assets.readme ? URI.parse(galleryExtension.assets.readme.uri) : undefined,
changelogUri: galleryExtension.assets.changelog ? URI.parse(galleryExtension.assets.changelog.uri) : undefined,
metadata: { isPreReleaseVersion: galleryExtension.properties.isPreReleaseVersion, preRelease: galleryExtension.properties.isPreReleaseVersion, isBuiltin: true }
});
}
}
return [...result.values()];
}

private async resolveBuiltinGalleryExtensions(extensions: IExtensionInfo[]): Promise<IWebExtension[]> {
if (extensions.length === 0) {
return [];
}
const webExtensions: IWebExtension[] = [];
const galleryExtensionsMap = await this.getExtensionsWithDependenciesAndPackedExtensions(extensions);
const missingExtensions = extensions.filter(({ id }) => !galleryExtensionsMap.has(id.toLowerCase()));
if (missingExtensions.length) {
this.logService.info('Skipping the additional builtin extensions because their compatible versions are not found.', missingExtensions);
}
await Promise.all([...galleryExtensionsMap.values()].map(async gallery => {
try {
const webExtension = await this.toWebExtensionFromGallery(gallery, { isPreReleaseVersion: gallery.properties.isPreReleaseVersion, preRelease: gallery.properties.isPreReleaseVersion, isBuiltin: true });
webExtensions.push(webExtension);
} catch (error) {
this.logService.info(`Ignoring additional builtin extension ${gallery.identifier.id} because there is an error while converting it into web extension`, getErrorMessage(error));
}
}));
return webExtensions;
}

private async resolveDependenciesAndPackedExtensions(webExtensions: IWebExtension[], result: Map<string, IWebExtension>): Promise<void> {
const extensionInfos: IExtensionInfo[] = [];
for (const webExtension of webExtensions) {
for (const e of [...(webExtension.manifest?.extensionDependencies ?? []), ...(webExtension.manifest?.extensionPack ?? [])]) {
if (!result.has(e.toLowerCase())) {
extensionInfos.push({ id: e, version: webExtension.version });
}
}
}
if (extensionInfos.length === 0) {
return;
}
const galleryExtensions = await this.getExtensionsWithDependenciesAndPackedExtensions(extensionInfos, new Set<string>([...result.keys()]));
await Promise.all([...galleryExtensions.values()].map(async gallery => {
try {
const webExtension = await this.toWebExtensionFromGallery(gallery, { isPreReleaseVersion: gallery.properties.isPreReleaseVersion, preRelease: gallery.properties.isPreReleaseVersion, isBuiltin: true });
result.set(webExtension.identifier.id.toLowerCase(), webExtension);
} catch (error) {
this.logService.info(`Ignoring additional builtin extension ${gallery.identifier.id} because there is an error while converting it into web extension`, getErrorMessage(error));
}
}));
}

private async getExtensionsWithDependenciesAndPackedExtensions(toGet: IExtensionInfo[], seen: Set<string> = new Set<string>(), result: Map<string, IGalleryExtension> = new Map<string, IGalleryExtension>()): Promise<Map<string, IGalleryExtension>> {
if (toGet.length === 0) {
return result;
}
Expand All @@ -399,13 +447,13 @@ export class WebExtensionsScannerService extends Disposable implements IWebExten
for (const extension of extensions) {
result.set(extension.identifier.id.toLowerCase(), extension);
for (const id of [...(isNonEmptyArray(extension.properties.dependencies) ? extension.properties.dependencies : []), ...(isNonEmptyArray(extension.properties.extensionPack) ? extension.properties.extensionPack : [])]) {
if (!result.has(id.toLowerCase()) && !packsAndDependencies.has(id.toLowerCase())) {
if (!result.has(id.toLowerCase()) && !packsAndDependencies.has(id.toLowerCase()) && !seen.has(id.toLowerCase())) {
const extensionInfo = toGet.find(e => areSameExtensions(e, extension.identifier));
packsAndDependencies.set(id.toLowerCase(), { id, preRelease: extensionInfo?.preRelease });
}
}
}
return this.getExtensionsWithDependenciesAndPackedExtensions([...packsAndDependencies.values()].filter(({ id }) => !result.has(id.toLowerCase())), result);
return this.getExtensionsWithDependenciesAndPackedExtensions([...packsAndDependencies.values()].filter(({ id }) => !result.has(id.toLowerCase())), seen, result);
}

async scanSystemExtensions(): Promise<IExtension[]> {
Expand Down Expand Up @@ -606,19 +654,28 @@ export class WebExtensionsScannerService extends Disposable implements IWebExten
if (!extensionLocation) {
throw new Error('No extension gallery service configured.');
}

return this.toWebExtensionFromExtensionGalleryResource(extensionLocation,
galleryExtension.identifier,
galleryExtension.assets.readme ? URI.parse(galleryExtension.assets.readme.uri) : undefined,
galleryExtension.assets.changelog ? URI.parse(galleryExtension.assets.changelog.uri) : undefined,
metadata);
}

private async toWebExtensionFromExtensionGalleryResource(extensionLocation: URI, identifier?: IExtensionIdentifier, readmeUri?: URI, changelogUri?: URI, metadata?: Metadata): Promise<IWebExtension> {
const extensionResources = await this.listExtensionResources(extensionLocation);
const packageNLSResources = this.getPackageNLSResourceMapFromResources(extensionResources);

// The fallback, in English, will fill in any gaps missing in the localized file.
const fallbackPackageNLSResource = extensionResources.find(e => basename(e) === 'package.nls.json');
return this.toWebExtension(
extensionLocation,
galleryExtension.identifier,
identifier,
undefined,
packageNLSResources,
fallbackPackageNLSResource ? URI.parse(fallbackPackageNLSResource) : null,
galleryExtension.assets.readme ? URI.parse(galleryExtension.assets.readme.uri) : undefined,
galleryExtension.assets.changelog ? URI.parse(galleryExtension.assets.changelog.uri) : undefined,
readmeUri,
changelogUri,
metadata);
}

Expand Down