diff --git a/package.json b/package.json index 9d7b78457b62..27f962d26e18 100644 --- a/package.json +++ b/package.json @@ -1636,10 +1636,22 @@ "description": "Allows a user to import a jupyter notebook into a python file anytime one is opened.", "scope": "resource" }, - "python.dataScience.loadWidgetScriptsFromThirdPartySource": { - "type": "boolean", - "default": false, - "description": "Enables loading of scripts files for Widgets (ipywidgest, bqplot, beakerx, ipyleaflet, etc) from https://unpkg.com.", + "python.dataScience.widgetScriptSources": { + "type": "array", + "default": [], + "items": { + "type": "string", + "enum": [ + "jsdelivr.com", + "unpkg.com" + ], + "enumDescriptions": [ + "Loads widget (javascript) scripts from https://www.jsdelivr.com/", + "Loads widget (javascript) scripts from https://unpkg.com/" + ] + }, + "uniqueItems": true, + "markdownDescription": "Defines the location and order of the sources where scripts files for Widgets are downloaded from (e.g. ipywidgest, bqplot, beakerx, ipyleaflet, etc). Not selecting any of these could result in widgets not rendering or function correctly. See [here](https://aka.ms/PVSCIPyWidgets) for more information. Once updated you will need to restart the Kernel.", "scope": "machine" }, "python.dataScience.gatherRules": { diff --git a/package.nls.json b/package.nls.json index c929edcf5089..8262076060c8 100644 --- a/package.nls.json +++ b/package.nls.json @@ -145,6 +145,7 @@ "Common.reload": "Reload", "Common.moreInfo": "More Info", "Common.and": "and", + "Common.ok": "Ok", "Common.install": "Install", "Common.learnMore": "Learn more", "OutputChannelNames.languageServer": "Python Language Server", @@ -464,6 +465,7 @@ "DataScience.jupyterSelectURIRemoteDetail": "Specify the URI of an existing server", "DataScience.gatherQuality": "Did gather work as desired?", "DataScience.loadClassFailedWithNoInternet": "Error loading {0}:{1}. Internet connection required for loading 3rd party widgets.", - "DataScience.loadThirdPartyWidgetScriptsDisabled": "Loading of Widgets is disabled by default. Click here to enable the setting 'python.dataScience.loadWidgetScriptsFromThirdPartySource'. Once enabled you will need to restart the Kernel", - "DataScience.loadThirdPartyWidgetScriptsPostEnabled": "Please restart the Kernel when changing the setting 'loadWidgetScriptsFromThirdPartySource'." + "DataScience.useCDNForWidgets": "Widgets require us to download supporting files from a 3rd party website. Click [here](https://aka.ms/PVSCIPyWidgets) for more information.", + "DataScience.loadThirdPartyWidgetScriptsPostEnabled": "Please restart the Kernel when changing the setting 'python.dataScience.widgetScriptSources'.", + "DataScience.enableCDNForWidgetsSetting": "Widgets require us to download supporting files from a 3rd party website. Click here to enable this or click here for more information. (Error loading {0}:{1})." } diff --git a/src/client/common/application/types.ts b/src/client/common/application/types.ts index 9e1eba45a6db..c942eba09151 100644 --- a/src/client/common/application/types.ts +++ b/src/client/common/application/types.ts @@ -1045,6 +1045,18 @@ export type WebPanelMessage = { // Wraps the VS Code webview panel export const IWebPanel = Symbol('IWebPanel'); export interface IWebPanel { + /** + * Convert a uri for the local file system to one that can be used inside webviews. + * + * Webviews cannot directly load resources from the workspace or local file system using `file:` uris. The + * `asWebviewUri` function takes a local `file:` uri and converts it into a uri that can be used inside of + * a webview to load the same resource: + * + * ```ts + * webview.html = `` + * ``` + */ + asWebviewUri(localResource: Uri): Uri; setTitle(val: string): void; /** * Makes the webpanel show up. diff --git a/src/client/common/application/webPanels/webPanel.ts b/src/client/common/application/webPanels/webPanel.ts index 9b36befee4a1..40719293572f 100644 --- a/src/client/common/application/webPanels/webPanel.ts +++ b/src/client/common/application/webPanels/webPanel.ts @@ -71,6 +71,12 @@ export class WebPanel implements IWebPanel { this.panel.dispose(); } } + public asWebviewUri(localResource: Uri) { + if (!this.panel) { + throw new Error('WebView not initialized, too early to get a Uri'); + } + return this.panel?.webview.asWebviewUri(localResource); + } public isVisible(): boolean { return this.panel ? this.panel.visible : false; @@ -161,7 +167,7 @@ export class WebPanel implements IWebPanel { VS Code Python React UI - + diff --git a/src/client/common/net/httpClient.ts b/src/client/common/net/httpClient.ts index 14cfb7fa20a9..adceb9b1e8a3 100644 --- a/src/client/common/net/httpClient.ts +++ b/src/client/common/net/httpClient.ts @@ -5,7 +5,7 @@ import { inject, injectable } from 'inversify'; import { parse, ParseError } from 'jsonc-parser'; -import * as requestTypes from 'request'; +import type * as requestTypes from 'request'; import { IHttpClient } from '../../common/types'; import { IServiceContainer } from '../../ioc/types'; import { IWorkspaceService } from '../application/types'; @@ -26,7 +26,7 @@ export class HttpClient implements IHttpClient { } public async getJSON(uri: string, strict: boolean = true): Promise { - const body = await this.getBody(uri); + const body = await this.getContents(uri); return this.parseBodyToJSON(body, strict); } @@ -44,7 +44,21 @@ export class HttpClient implements IHttpClient { } } - public async getBody(uri: string): Promise { + public async exists(uri: string): Promise { + // tslint:disable-next-line:no-require-imports + const request = require('request') as typeof requestTypes; + return new Promise((resolve) => { + try { + request + .get(uri, this.requestOptions) + .on('response', (response) => resolve(response.statusCode === 200)) + .on('error', () => resolve(false)); + } catch { + resolve(false); + } + }); + } + private async getContents(uri: string): Promise { // tslint:disable-next-line:no-require-imports const request = require('request') as typeof requestTypes; return new Promise((resolve, reject) => { diff --git a/src/client/common/types.ts b/src/client/common/types.ts index 3baf18cf5d23..0d634bcbad98 100644 --- a/src/client/common/types.ts +++ b/src/client/common/types.ts @@ -393,9 +393,11 @@ export interface IDataScienceSettings { variableQueries: IVariableQuery[]; disableJupyterAutoStart?: boolean; jupyterCommandLineArguments: string[]; - loadWidgetScriptsFromThirdPartySource?: boolean; + widgetScriptSources: WidgetCDNs[]; } +export type WidgetCDNs = 'unpkg.com' | 'jsdelivr.com'; + export const IConfigurationService = Symbol('IConfigurationService'); export interface IConfigurationService { getSettings(resource?: Uri): IPythonSettings; @@ -466,6 +468,10 @@ export interface IHttpClient { * @param strict Set `false` to allow trailing comma and comments in the JSON, defaults to `true` */ getJSON(uri: string, strict?: boolean): Promise; + /** + * Returns the url is valid (i.e. return status code of 200). + */ + exists(uri: string): Promise; } export const IExtensionContext = Symbol('ExtensionContext'); diff --git a/src/client/common/utils/localize.ts b/src/client/common/utils/localize.ts index 86e972070afc..8d5d91df58d6 100644 --- a/src/client/common/utils/localize.ts +++ b/src/client/common/utils/localize.ts @@ -63,6 +63,7 @@ export namespace Common { export const bannerLabelNo = localize('Common.bannerLabelNo', 'No'); export const canceled = localize('Common.canceled', 'Canceled'); export const cancel = localize('Common.cancel', 'Cancel'); + export const ok = localize('Common.ok', 'Ok'); export const gotIt = localize('Common.gotIt', 'Got it!'); export const install = localize('Common.install', 'Install'); export const loadingExtension = localize('Common.loadingPythonExtension', 'Python extension loading...'); @@ -838,7 +839,15 @@ export namespace DataScience { ); export const loadThirdPartyWidgetScriptsPostEnabled = localize( 'DataScience.loadThirdPartyWidgetScriptsPostEnabled', - "Once you have updated the setting 'loadWidgetScriptsFromThirdPartySource' you will need to restart the Kernel." + "Please restart the Kernel when changing the setting 'python.dataScience.widgetScriptSources'." + ); + export const useCDNForWidgets = localize( + 'DataScience.useCDNForWidgets', + 'Widgets require us to download supporting files from a 3rd party website. Click [here](https://aka.ms/PVSCIPyWidgets) for more information.' + ); + export const enableCDNForWidgetsSetting = localize( + 'DataScience.enableCDNForWidgetsSetting', + "Widgets require us to download supporting files from a 3rd party website. Click here to enable this or click here for more information. (Error loading {0}:{1})." ); } diff --git a/src/client/common/utils/serializers.ts b/src/client/common/utils/serializers.ts index c55ac74f5b2f..7626c0780264 100644 --- a/src/client/common/utils/serializers.ts +++ b/src/client/common/utils/serializers.ts @@ -31,8 +31,12 @@ export function serializeDataViews(buffers: undefined | (ArrayBuffer | ArrayBuff // tslint:disable-next-line: no-any } as any); } else { + // Do not use `Array.apply`, it will not work for large arrays. + // Nodejs will throw `stackoverflow` exceptions. + // Else following ipynb fails https://github.com/K3D-tools/K3D-jupyter/blob/821a59ed88579afaafababd6291e8692d70eb088/examples/camera_manipulation.ipynb + // Yet another case where 99% can work, but 1% can fail when testing. // tslint:disable-next-line: no-any - newBufferView.push(Array.apply(null, new Uint8Array(item as any) as any) as any); + newBufferView.push([...new Uint8Array(item as any)]); } } diff --git a/src/client/datascience/commands/commandRegistry.ts b/src/client/datascience/commands/commandRegistry.ts index b4a0f225d9e4..aaa234262daa 100644 --- a/src/client/datascience/commands/commandRegistry.ts +++ b/src/client/datascience/commands/commandRegistry.ts @@ -106,14 +106,14 @@ export class CommandRegistry implements IDisposable { } private enableLoadingWidgetScriptsFromThirdParty(): void { - if (this.configService.getSettings(undefined).datascience.loadWidgetScriptsFromThirdPartySource) { + if (this.configService.getSettings(undefined).datascience.widgetScriptSources.length > 0) { return; } // Update the setting and once updated, notify user to restart kernel. this.configService .updateSetting( - 'dataScience.loadWidgetScriptsFromThirdPartySource', - true, + 'dataScience.widgetScriptSources', + ['jsdelivr.com', 'unpkg.com'], undefined, ConfigurationTarget.Global ) diff --git a/src/client/datascience/constants.ts b/src/client/datascience/constants.ts index 0a735f6aa8bf..8090d3137e1c 100644 --- a/src/client/datascience/constants.ts +++ b/src/client/datascience/constants.ts @@ -82,7 +82,8 @@ export namespace Commands { export const SaveAsNotebookNonCustomEditor = 'python.datascience.notebookeditor.saveAs'; export const OpenNotebookNonCustomEditor = 'python.datascience.notebookeditor.open'; export const GatherQuality = 'python.datascience.gatherquality'; - export const EnableLoadingWidgetsFrom3rdPartySource = 'python.datascience.loadWidgetScriptsFromThirdPartySource'; + export const EnableLoadingWidgetsFrom3rdPartySource = + 'python.datascience.enableLoadingWidgetScriptsFromThirdPartySource'; } export namespace CodeLensCommands { @@ -292,6 +293,13 @@ export enum Telemetry { IPyWidgetLoadSuccess = 'DS_INTERNAL.IPYWIDGET_LOAD_SUCCESS', IPyWidgetLoadFailure = 'DS_INTERNAL.IPYWIDGET_LOAD_FAILURE', IPyWidgetLoadDisabled = 'DS_INTERNAL.IPYWIDGET_LOAD_DISABLED', + HashedIPyWidgetNameUsed = 'DS_INTERNAL.IPYWIDGET_USED_BY_USER', + HashedIPyWidgetNameDiscovered = 'DS_INTERNAL.IPYWIDGET_DISCOVERED', + HashedIPyWidgetScriptDiscoveryError = 'DS_INTERNAL.IPYWIDGET_DISCOVERY_ERRORED', + DiscoverIPyWidgetNamesLocalPerf = 'DS_INTERNAL.IPYWIDGET_TEST_AVAILABILITY_ON_LOCAL', + DiscoverIPyWidgetNamesCDNPerf = 'DS_INTERNAL.IPYWIDGET_TEST_AVAILABILITY_ON_CDN', + IPyWidgetPromptToUseCDN = 'DS_INTERNAL.IPYWIDGET_PROMPT_TO_USE_CDN', + IPyWidgetPromptToUseCDNSelection = 'DS_INTERNAL.IPYWIDGET_PROMPT_TO_USE_CDN_SELECTION', IPyWidgetOverhead = 'DS_INTERNAL.IPYWIDGET_OVERHEAD', IPyWidgetRenderFailure = 'DS_INTERNAL.IPYWIDGET_RENDER_FAILURE' } diff --git a/src/client/datascience/interactive-common/interactiveBase.ts b/src/client/datascience/interactive-common/interactiveBase.ts index f7aeddebc7e8..86d4f8babf1a 100644 --- a/src/client/datascience/interactive-common/interactiveBase.ts +++ b/src/client/datascience/interactive-common/interactiveBase.ts @@ -177,6 +177,12 @@ export abstract class InteractiveBase extends WebViewHost l.postMessage((e) => this.postMessageInternal(e.message, e.payload))); + // Channel for listeners to send messages to the interactive base. + this.listeners.forEach((l) => { + if (l.postInternalMessage) { + l.postInternalMessage((e) => this.onMessage(e.message, e.payload)); + } + }); // Tell each listener our identity. Can't do it here though as were in the constructor for the base class setTimeout(() => { @@ -204,6 +210,12 @@ export abstract class InteractiveBase extends WebViewHost & Messa [CommonActionType.FOCUS_INPUT]: MessageType.other, [CommonActionType.LOAD_IPYWIDGET_CLASS_SUCCESS]: MessageType.other, [CommonActionType.LOAD_IPYWIDGET_CLASS_FAILURE]: MessageType.other, - [CommonActionType.LOAD_IPYWIDGET_CLASS_DISABLED_FAILURE]: MessageType.other, [CommonActionType.IPYWIDGET_RENDER_FAILURE]: MessageType.other, // Types from InteractiveWindowMessages @@ -116,7 +115,6 @@ const messageWithMessageTypes: MessageMapping & Messa [InteractiveWindowMessages.Interrupt]: MessageType.other, [InteractiveWindowMessages.IPyWidgetLoadSuccess]: MessageType.other, [InteractiveWindowMessages.IPyWidgetLoadFailure]: MessageType.other, - [InteractiveWindowMessages.IPyWidgetLoadDisabled]: MessageType.other, [InteractiveWindowMessages.IPyWidgetRenderFailure]: MessageType.other, [InteractiveWindowMessages.LoadAllCells]: MessageType.other, [InteractiveWindowMessages.LoadAllCellsComplete]: MessageType.other, @@ -177,6 +175,8 @@ const messageWithMessageTypes: MessageMapping & Messa [InteractiveWindowMessages.UpdateDisplayData]: MessageType.syncWithLiveShare, [InteractiveWindowMessages.VariableExplorerToggle]: MessageType.other, [InteractiveWindowMessages.VariablesComplete]: MessageType.other, + [InteractiveWindowMessages.ConvertUriForUseInWebViewRequest]: MessageType.other, + [InteractiveWindowMessages.ConvertUriForUseInWebViewResponse]: MessageType.other, // Types from CssMessages [CssMessages.GetCssRequest]: MessageType.other, [CssMessages.GetCssResponse]: MessageType.other, @@ -187,9 +187,12 @@ const messageWithMessageTypes: MessageMapping & Messa [SharedMessages.Started]: MessageType.other, [SharedMessages.UpdateSettings]: MessageType.other, // IpyWidgets - [IPyWidgetMessages.IPyWidgets_kernelOptions]: MessageType.noIdea, + [IPyWidgetMessages.IPyWidgets_kernelOptions]: MessageType.syncAcrossSameNotebooks, [IPyWidgetMessages.IPyWidgets_Ready]: MessageType.noIdea, - [IPyWidgetMessages.IPyWidgets_onRestartKernel]: MessageType.noIdea, + [IPyWidgetMessages.IPyWidgets_WidgetScriptSourceRequest]: MessageType.noIdea, + [IPyWidgetMessages.IPyWidgets_WidgetScriptSourceResponse]: MessageType.syncAcrossSameNotebooks, + [IPyWidgetMessages.IPyWidgets_onKernelChanged]: MessageType.syncAcrossSameNotebooks, + [IPyWidgetMessages.IPyWidgets_onRestartKernel]: MessageType.syncAcrossSameNotebooks, [IPyWidgetMessages.IPyWidgets_msg]: MessageType.noIdea, [IPyWidgetMessages.IPyWidgets_binary_msg]: MessageType.noIdea, [IPyWidgetMessages.IPyWidgets_msg_handled]: MessageType.noIdea, diff --git a/src/client/datascience/ipywidgets/cdnWidgetScriptSourceProvider.ts b/src/client/datascience/ipywidgets/cdnWidgetScriptSourceProvider.ts new file mode 100644 index 000000000000..b7e69cfc317c --- /dev/null +++ b/src/client/datascience/ipywidgets/cdnWidgetScriptSourceProvider.ts @@ -0,0 +1,91 @@ +// Copyright (c) Microsoft Corporation. All rights reserved. +// Licensed under the MIT License. + +'use strict'; + +import { traceWarning } from '../../common/logger'; +import { IConfigurationService, IHttpClient, WidgetCDNs } from '../../common/types'; +import { StopWatch } from '../../common/utils/stopWatch'; +import { sendTelemetryEvent } from '../../telemetry'; +import { Telemetry } from '../constants'; +import { IWidgetScriptSourceProvider, WidgetScriptSource } from './types'; + +// Source borrowed from https://github.com/jupyter-widgets/ipywidgets/blob/54941b7a4b54036d089652d91b39f937bde6b6cd/packages/html-manager/src/libembed-amd.ts#L33 +const unpgkUrl = 'https://unpkg.com/'; +const jsdelivrUrl = 'https://cdn.jsdelivr.net/npm/'; +function moduleNameToCDNUrl(cdn: string, moduleName: string, moduleVersion: string) { + let packageName = moduleName; + let fileName = 'index'; // default filename + // if a '/' is present, like 'foo/bar', packageName is changed to 'foo', and path to 'bar' + // We first find the first '/' + let index = moduleName.indexOf('/'); + if (index !== -1 && moduleName[0] === '@') { + // if we have a namespace, it's a different story + // @foo/bar/baz should translate to @foo/bar and baz + // so we find the 2nd '/' + index = moduleName.indexOf('/', index + 1); + } + if (index !== -1) { + fileName = moduleName.substr(index + 1); + packageName = moduleName.substr(0, index); + } + return `${cdn}${packageName}@${moduleVersion}/dist/${fileName}`; +} + +function getCDNPrefix(cdn?: WidgetCDNs): string | undefined { + switch (cdn) { + case 'unpkg.com': + return unpgkUrl; + case 'jsdelivr.com': + return jsdelivrUrl; + default: + break; + } +} +/** + * Widget scripts are found in CDN. + * Given an widget module name & version, this will attempt to find the Url on a CDN. + * We'll need to stick to the order of preference prescribed by the user. + */ +export class CDNWidgetScriptSourceProvider implements IWidgetScriptSourceProvider { + private get cdnProviders(): readonly WidgetCDNs[] { + const settings = this.configurationSettings.getSettings(undefined); + return settings.datascience.widgetScriptSources; + } + public static validUrls = new Map(); + constructor( + private readonly configurationSettings: IConfigurationService, + private readonly httpClient: IHttpClient + ) {} + public dispose() { + // Noop. + } + public async getWidgetScriptSource(moduleName: string, moduleVersion: string): Promise { + const cdns = [...this.cdnProviders]; + while (cdns.length) { + const cdn = cdns.shift(); + const cdnBaseUrl = getCDNPrefix(cdn); + if (!cdnBaseUrl || !cdn) { + continue; + } + const scriptUri = moduleNameToCDNUrl(cdnBaseUrl, moduleName, moduleVersion); + const exists = await this.getUrlForWidget(cdn, scriptUri); + if (exists) { + return { moduleName, scriptUri, source: 'cdn' }; + } + } + traceWarning(`Widget Script not found for ${moduleName}@${moduleVersion}`); + return { moduleName }; + } + private async getUrlForWidget(cdn: string, url: string): Promise { + if (CDNWidgetScriptSourceProvider.validUrls.has(url)) { + return CDNWidgetScriptSourceProvider.validUrls.get(url)!; + } + + const stopWatch = new StopWatch(); + const exists = await this.httpClient.exists(url); + sendTelemetryEvent(Telemetry.DiscoverIPyWidgetNamesCDNPerf, stopWatch.elapsedTime, { cdn, exists }); + CDNWidgetScriptSourceProvider.validUrls.set(url, exists); + return exists; + } +} diff --git a/src/client/datascience/ipywidgets/ipyWidgetMessageDispatcherFactory.ts b/src/client/datascience/ipywidgets/ipyWidgetMessageDispatcherFactory.ts index 4193b4362b5c..551bf1d18dca 100644 --- a/src/client/datascience/ipywidgets/ipyWidgetMessageDispatcherFactory.ts +++ b/src/client/datascience/ipywidgets/ipyWidgetMessageDispatcherFactory.ts @@ -75,7 +75,7 @@ class IPyWidgetMessageDispatcherWithOldMessages implements IIPyWidgetMessageDisp */ @injectable() export class IPyWidgetMessageDispatcherFactory implements IDisposable { - private readonly ipywidgetMulticasters = new Map(); + private readonly messageDispatchers = new Map(); private readonly messages: IPyWidgetMessage[] = []; private disposed = false; private disposables: IDisposable[] = []; @@ -98,17 +98,17 @@ export class IPyWidgetMessageDispatcherFactory implements IDisposable { } } public create(identity: Uri): IIPyWidgetMessageDispatcher { - let baseDispatcher = this.ipywidgetMulticasters.get(identity.fsPath); + let baseDispatcher = this.messageDispatchers.get(identity.fsPath); if (!baseDispatcher) { baseDispatcher = new IPyWidgetMessageDispatcher(this.notebookProvider, identity); - this.ipywidgetMulticasters.set(identity.fsPath, baseDispatcher); + this.messageDispatchers.set(identity.fsPath, baseDispatcher); // Capture all messages so we can re-play messages that others missed. this.disposables.push(baseDispatcher.postMessage(this.onMessage, this)); } // If we have messages upto this point, then capture those messages, - // & pass to the multicaster so it can re-broadcast those old messages. + // & pass to the dispatcher so it can re-broadcast those old messages. // If there are no old messages, even then return a new instance of the class. // This way, the reference to that will be controlled by calling code. const dispatcher = new IPyWidgetMessageDispatcherWithOldMessages( @@ -124,9 +124,9 @@ export class IPyWidgetMessageDispatcherFactory implements IDisposable { } notebook.onDisposed( () => { - const item = this.ipywidgetMulticasters.get(notebook.identity.fsPath); + const item = this.messageDispatchers.get(notebook.identity.fsPath); item?.dispose(); // NOSONAR - this.ipywidgetMulticasters.delete(notebook.identity.fsPath); + this.messageDispatchers.delete(notebook.identity.fsPath); }, this, this.disposables diff --git a/src/client/datascience/ipywidgets/ipyWidgetScriptSource.ts b/src/client/datascience/ipywidgets/ipyWidgetScriptSource.ts new file mode 100644 index 000000000000..78866c2291bc --- /dev/null +++ b/src/client/datascience/ipywidgets/ipyWidgetScriptSource.ts @@ -0,0 +1,302 @@ +// Copyright (c) Microsoft Corporation. All rights reserved. +// Licensed under the MIT License. + +'use strict'; +import type * as jupyterlabService from '@jupyterlab/services'; +import type * as serialize from '@jupyterlab/services/lib/kernel/serialize'; +import { inject, injectable } from 'inversify'; +import { IDisposable } from 'monaco-editor'; +import { Event, EventEmitter, Uri } from 'vscode'; +import type { Data as WebSocketData } from 'ws'; +import { IApplicationShell, IWorkspaceService } from '../../common/application/types'; +import { traceError } from '../../common/logger'; +import { IFileSystem } from '../../common/platform/types'; +import { IConfigurationService, IDisposableRegistry, IHttpClient, IPersistentStateFactory } from '../../common/types'; +import { createDeferred, Deferred } from '../../common/utils/async'; +import { IInterpreterService, PythonInterpreter } from '../../interpreter/contracts'; +import { sendTelemetryEvent } from '../../telemetry'; +import { Telemetry } from '../constants'; +import { + INotebookIdentity, + InteractiveWindowMessages, + IPyWidgetMessages +} from '../interactive-common/interactiveWindowTypes'; +import { + IInteractiveWindowListener, + ILocalResourceUriConverter, + INotebook, + INotebookProvider, + KernelSocketInformation +} from '../types'; +import { IPyWidgetScriptSourceProvider } from './ipyWidgetScriptSourceProvider'; +import { WidgetScriptSource } from './types'; + +@injectable() +export class IPyWidgetScriptSource implements IInteractiveWindowListener, ILocalResourceUriConverter { + // tslint:disable-next-line: no-any + public get postMessage(): Event<{ message: string; payload: any }> { + return this.postEmitter.event; + } + // tslint:disable-next-line: no-any + public get postInternalMessage(): Event<{ message: string; payload: any }> { + return this.postInternalMessageEmitter.event; + } + private notebookIdentity?: Uri; + private postEmitter = new EventEmitter<{ + message: string; + // tslint:disable-next-line: no-any + payload: any; + }>(); + private postInternalMessageEmitter = new EventEmitter<{ + message: string; + // tslint:disable-next-line: no-any + payload: any; + }>(); + private notebook?: INotebook; + private jupyterLab?: typeof jupyterlabService; + private scriptProvider?: IPyWidgetScriptSourceProvider; + private disposables: IDisposable[] = []; + private interpreterForWhichWidgetSourcesWereFetched?: PythonInterpreter; + private kernelSocketInfo?: KernelSocketInformation; + private subscribedToKernelSocket: boolean = false; + /** + * Key value pair of widget modules along with the version that needs to be loaded. + */ + private pendingModuleRequests = new Map(); + private jupyterSerialize?: typeof serialize; + private get deserialize(): typeof serialize.deserialize { + if (!this.jupyterSerialize) { + // tslint:disable-next-line: no-require-imports + this.jupyterSerialize = require('@jupyterlab/services/lib/kernel/serialize') as typeof serialize; + } + return this.jupyterSerialize.deserialize; + } + private readonly uriConversionPromises = new Map>(); + constructor( + @inject(IDisposableRegistry) disposables: IDisposableRegistry, + @inject(INotebookProvider) private readonly notebookProvider: INotebookProvider, + @inject(IFileSystem) private readonly fs: IFileSystem, + @inject(IInterpreterService) private readonly interpreterService: IInterpreterService, + @inject(IConfigurationService) private readonly configurationSettings: IConfigurationService, + @inject(IHttpClient) private readonly httpClient: IHttpClient, + @inject(IApplicationShell) private readonly appShell: IApplicationShell, + @inject(IWorkspaceService) private readonly workspaceService: IWorkspaceService, + @inject(IPersistentStateFactory) private readonly stateFactory: IPersistentStateFactory + ) { + disposables.push(this); + this.notebookProvider.onNotebookCreated( + (e) => { + if (e.identity.toString() === this.notebookIdentity?.toString()) { + this.initialize().catch(traceError.bind('Failed to initialize')); + } + }, + this, + this.disposables + ); + } + public asWebviewUri(localResource: Uri): Promise { + const key = localResource.toString(); + if (!this.uriConversionPromises.has(key)) { + this.uriConversionPromises.set(key, createDeferred()); + // Send a request for the translation. + this.postInternalMessageEmitter.fire({ + message: InteractiveWindowMessages.ConvertUriForUseInWebViewRequest, + payload: localResource + }); + } + return this.uriConversionPromises.get(key)!.promise; + } + + public dispose() { + while (this.disposables.length) { + this.disposables.shift()?.dispose(); // NOSONAR + } + } + + // tslint:disable-next-line: no-any + public onMessage(message: string, payload?: any): void { + if (message === InteractiveWindowMessages.NotebookIdentity) { + this.saveIdentity(payload).catch((ex) => + traceError(`Failed to initialize ${(this as Object).constructor.name}`, ex) + ); + } else if (message === InteractiveWindowMessages.ConvertUriForUseInWebViewResponse) { + const response: undefined | { request: Uri; response: Uri } = payload; + if (response && this.uriConversionPromises.get(response.request.toString())) { + this.uriConversionPromises.get(response.request.toString())!.resolve(response.response); + } + } else if (message === IPyWidgetMessages.IPyWidgets_WidgetScriptSourceRequest) { + if (payload) { + const { moduleName, moduleVersion } = payload as { moduleName: string; moduleVersion: string }; + this.sendWidgetSource(moduleName, moduleVersion).catch( + traceError.bind('Failed to send widget sources upon ready') + ); + } + } + } + + /** + * Send the widget script source for a specific widget module & version. + * This is a request made when a widget is certainly used in a notebook. + */ + private async sendWidgetSource(moduleName: string, moduleVersion: string) { + // Standard widgets area already available, hence no need to look for them. + if (moduleName.startsWith('@jupyter') || moduleName === 'azureml_widgets') { + return; + } + if (!this.notebook || !this.scriptProvider) { + this.pendingModuleRequests.set(moduleName, moduleVersion); + return; + } + + let widgetSource: WidgetScriptSource = { moduleName }; + try { + widgetSource = await this.scriptProvider.getWidgetScriptSource(moduleName, moduleVersion); + } catch (ex) { + traceError('Failed to get widget source due to an error', ex); + sendTelemetryEvent(Telemetry.HashedIPyWidgetScriptDiscoveryError); + } finally { + // Send to UI (even if there's an error) continues instead of hanging while waiting for a response. + this.postEmitter.fire({ + message: IPyWidgetMessages.IPyWidgets_WidgetScriptSourceResponse, + payload: widgetSource + }); + } + } + private async saveIdentity(args: INotebookIdentity) { + this.notebookIdentity = Uri.parse(args.resource); + await this.initialize(); + } + + private async initialize() { + if (!this.jupyterLab) { + // Lazy load jupyter lab for faster extension loading. + // tslint:disable-next-line:no-require-imports + this.jupyterLab = require('@jupyterlab/services') as typeof jupyterlabService; // NOSONAR + } + + if (!this.notebookIdentity) { + return; + } + if (!this.notebook) { + this.notebook = await this.notebookProvider.getOrCreateNotebook({ + identity: this.notebookIdentity, + disableUI: true, + getOnly: true + }); + } + if (!this.notebook) { + return; + } + if (this.scriptProvider) { + return; + } + this.scriptProvider = new IPyWidgetScriptSourceProvider( + this.notebook, + this, + this.fs, + this.interpreterService, + this.appShell, + this.configurationSettings, + this.workspaceService, + this.stateFactory, + this.httpClient + ); + await this.initializeNotebook(); + } + private async initializeNotebook() { + if (!this.notebook) { + return; + } + this.subscribeToKernelSocket(); + this.notebook.onDisposed(() => this.dispose()); + // When changing a kernel, we might have a new interpreter. + this.notebook.onKernelChanged( + () => { + // If underlying interpreter has changed, then refresh list of widget sources. + // After all, different kernels have different widgets. + if ( + this.notebook?.getMatchingInterpreter() && + this.notebook?.getMatchingInterpreter() === this.interpreterForWhichWidgetSourcesWereFetched + ) { + return; + } + // Let UI know that kernel has changed. + this.postEmitter.fire({ message: IPyWidgetMessages.IPyWidgets_onKernelChanged, payload: undefined }); + }, + this, + this.disposables + ); + this.handlePendingRequests(); + } + private subscribeToKernelSocket() { + if (this.subscribedToKernelSocket || !this.notebook) { + return; + } + this.subscribedToKernelSocket = true; + // Listen to changes to kernel socket (e.g. restarts or changes to kernel). + this.notebook.kernelSocket.subscribe((info) => { + // Remove old handlers. + this.kernelSocketInfo?.socket?.removeReceiveHook(this.onKernelSocketMessage.bind(this)); // NOSONAR + + if (!info || !info.socket) { + // No kernel socket information, hence nothing much we can do. + this.kernelSocketInfo = undefined; + return; + } + + this.kernelSocketInfo = info; + this.kernelSocketInfo.socket?.addReceiveHook(this.onKernelSocketMessage.bind(this)); // NOSONAR + }); + } + /** + * If we get a comm open message, then we know a widget will be displayed. + * In this case get hold of the name and send it up (pre-fetch it before UI makes a request for it). + */ + private async onKernelSocketMessage(message: WebSocketData): Promise { + // tslint:disable-next-line: no-any + const msg = this.deserialize(message as any); + if (this.jupyterLab?.KernelMessage.isCommOpenMsg(msg) && msg.content.target_module) { + this.sendWidgetSource(msg.content.target_module, '').catch( + traceError.bind('Failed to pre-load Widget Script') + ); + } else if ( + this.jupyterLab?.KernelMessage.isCommOpenMsg(msg) && + msg.content.data && + msg.content.data.state && + // tslint:disable-next-line: no-any + ((msg.content.data.state as any)._view_module || (msg.content.data.state as any)._model_module) + ) { + // tslint:disable-next-line: no-any + const viewModule: string = (msg.content.data.state as any)._view_module; + // tslint:disable-next-line: no-any + const viewModuleVersion: string = (msg.content.data.state as any)._view_module_version; + // tslint:disable-next-line: no-any + const modelModule = (msg.content.data.state as any)._model_module; + // tslint:disable-next-line: no-any + const modelModuleVersion = (msg.content.data.state as any)._model_module_version; + if (viewModule) { + this.sendWidgetSource(viewModule, modelModuleVersion || '').catch( + traceError.bind('Failed to pre-load Widget Script') + ); + } + if (modelModule) { + this.sendWidgetSource(viewModule, viewModuleVersion || '').catch( + traceError.bind('Failed to pre-load Widget Script') + ); + } + } + } + private handlePendingRequests() { + const pendingModuleNames = Array.from(this.pendingModuleRequests.keys()); + while (pendingModuleNames.length) { + const moduleName = pendingModuleNames.shift(); + if (moduleName) { + const moduleVersion = this.pendingModuleRequests.get(moduleName)!; + this.pendingModuleRequests.delete(moduleName); + this.sendWidgetSource(moduleName, moduleVersion).catch( + traceError.bind(`Failed to send WidgetScript for ${moduleName}`) + ); + } + } + } +} diff --git a/src/client/datascience/ipywidgets/ipyWidgetScriptSourceProvider.ts b/src/client/datascience/ipywidgets/ipyWidgetScriptSourceProvider.ts new file mode 100644 index 000000000000..b9ecc802ba94 --- /dev/null +++ b/src/client/datascience/ipywidgets/ipyWidgetScriptSourceProvider.ts @@ -0,0 +1,198 @@ +// Copyright (c) Microsoft Corporation. All rights reserved. +// Licensed under the MIT License. + +'use strict'; + +import { sha256 } from 'hash.js'; +import { ConfigurationChangeEvent, ConfigurationTarget } from 'vscode'; +import { IApplicationShell, IWorkspaceService } from '../../common/application/types'; +import { traceError } from '../../common/logger'; +import { IFileSystem } from '../../common/platform/types'; +import { + IConfigurationService, + IHttpClient, + IPersistentState, + IPersistentStateFactory, + WidgetCDNs +} from '../../common/types'; +import { createDeferred, Deferred } from '../../common/utils/async'; +import { Common, DataScience } from '../../common/utils/localize'; +import { IInterpreterService } from '../../interpreter/contracts'; +import { sendTelemetryEvent } from '../../telemetry'; +import { Telemetry } from '../constants'; +import { ILocalResourceUriConverter, INotebook } from '../types'; +import { CDNWidgetScriptSourceProvider } from './cdnWidgetScriptSourceProvider'; +import { LocalWidgetScriptSourceProvider } from './localWidgetScriptSourceProvider'; +import { RemoteWidgetScriptSourceProvider } from './remoteWidgetScriptSourceProvider'; +import { IWidgetScriptSourceProvider, WidgetScriptSource } from './types'; + +const GlobalStateKeyToTrackIfUserConfiguredCDNAtLeastOnce = 'IPYWidgetCDNConfigured'; + +/** + * This class decides where to get widget scripts from. + * Whether its cdn or local or other, and also controls the order/priority. + * If user changes the order, this will react to those configuration setting changes. + * If user has not configured antying, user will be presented with a prompt. + */ +export class IPyWidgetScriptSourceProvider implements IWidgetScriptSourceProvider { + private scriptProviders?: IWidgetScriptSourceProvider[]; + private configurationPromise?: Deferred; + private get configuredScriptSources(): readonly WidgetCDNs[] { + const settings = this.configurationSettings.getSettings(undefined); + return settings.datascience.widgetScriptSources; + } + private readonly userConfiguredCDNAtLeastOnce: IPersistentState; + constructor( + private readonly notebook: INotebook, + private readonly localResourceUriConverter: ILocalResourceUriConverter, + private readonly fs: IFileSystem, + private readonly interpreterService: IInterpreterService, + private readonly appShell: IApplicationShell, + private readonly configurationSettings: IConfigurationService, + private readonly workspaceService: IWorkspaceService, + private readonly stateFactory: IPersistentStateFactory, + private readonly httpClient: IHttpClient + ) { + this.userConfiguredCDNAtLeastOnce = this.stateFactory.createGlobalPersistentState( + GlobalStateKeyToTrackIfUserConfiguredCDNAtLeastOnce, + false + ); + } + public initialize() { + this.workspaceService.onDidChangeConfiguration(this.onSettingsChagned.bind(this)); + } + public dispose() { + this.disposeScriptProviders(); + } + /** + * We know widgets are being used, at this point prompt user if required. + */ + public async getWidgetScriptSource( + moduleName: string, + moduleVersion: string + ): Promise> { + await this.configureWidgets(); + if (!this.scriptProviders) { + this.rebuildProviders(); + } + + // Get script sources in order, if one works, then get out. + const scriptSourceProviders = (this.scriptProviders || []).slice(); + let found: WidgetScriptSource = { moduleName }; + while (scriptSourceProviders.length) { + const scriptProvider = scriptSourceProviders.shift(); + if (!scriptProvider) { + continue; + } + const source = await scriptProvider.getWidgetScriptSource(moduleName, moduleVersion); + // If we found the script source, then use that. + if (source.scriptUri) { + found = source; + break; + } + } + + sendTelemetryEvent(Telemetry.HashedIPyWidgetNameUsed, undefined, { + hashedName: sha256().update(found.moduleName).digest('hex'), + source: found.source + }); + + if (!found.scriptUri) { + traceError(`Script source for Widget ${moduleName}@${moduleVersion} not found`); + } + return found; + } + private onSettingsChagned(e: ConfigurationChangeEvent) { + if (e.affectsConfiguration('dataScience.widgetScriptSources')) { + this.rebuildProviders(); + } + } + private disposeScriptProviders() { + while (this.scriptProviders && this.scriptProviders.length) { + const item = this.scriptProviders.shift(); + if (item) { + item.dispose(); + } + } + } + private rebuildProviders() { + this.disposeScriptProviders(); + + const scriptProviders: IWidgetScriptSourceProvider[] = []; + + // If we're allowed to use CDN providers, then use them, and use in order of preference. + if (this.configuredScriptSources.length > 0) { + scriptProviders.push(new CDNWidgetScriptSourceProvider(this.configurationSettings, this.httpClient)); + } + if (this.notebook.connection.localLaunch) { + scriptProviders.push( + new LocalWidgetScriptSourceProvider( + this.notebook, + this.localResourceUriConverter, + this.fs, + this.interpreterService + ) + ); + } else { + scriptProviders.push(new RemoteWidgetScriptSourceProvider(this.notebook.connection, this.httpClient)); + } + + this.scriptProviders = scriptProviders; + } + + private async configureWidgets(): Promise { + if (this.configuredScriptSources.length !== 0) { + return; + } + + if (this.userConfiguredCDNAtLeastOnce.value) { + return; + } + + if (this.configurationPromise) { + return this.configurationPromise.promise; + } + this.configurationPromise = createDeferred(); + sendTelemetryEvent(Telemetry.IPyWidgetPromptToUseCDN); + const selection = await this.appShell.showInformationMessage( + DataScience.useCDNForWidgets(), + Common.ok(), + Common.cancel(), + Common.doNotShowAgain() + ); + + let selectionForTelemetry: 'ok' | 'cancel' | 'dismissed' | 'doNotShowAgain' = 'dismissed'; + switch (selection) { + case Common.ok(): { + selectionForTelemetry = 'ok'; + // always search local interpreter or attempt to fetch scripts from remote jupyter server as backups. + await Promise.all([ + this.updateScriptSources(['jsdelivr.com', 'unpkg.com']), + this.userConfiguredCDNAtLeastOnce.updateValue(true) + ]); + break; + } + case Common.doNotShowAgain(): { + selectionForTelemetry = 'doNotShowAgain'; + // At a minimum search local interpreter or attempt to fetch scripts from remote jupyter server. + await Promise.all([this.updateScriptSources([]), this.userConfiguredCDNAtLeastOnce.updateValue(true)]); + break; + } + default: + selectionForTelemetry = selection === Common.cancel() ? 'cancel' : 'dismissed'; + break; + } + + sendTelemetryEvent(Telemetry.IPyWidgetPromptToUseCDNSelection, undefined, { selection: selectionForTelemetry }); + this.configurationPromise.resolve(); + } + private async updateScriptSources(scriptSources: WidgetCDNs[]) { + const targetSetting = 'dataScience.widgetScriptSources'; + await this.configurationSettings.updateSetting( + targetSetting, + scriptSources, + undefined, + ConfigurationTarget.Global + ); + } +} diff --git a/src/client/datascience/ipywidgets/ipywidgetHandler.ts b/src/client/datascience/ipywidgets/ipywidgetHandler.ts index a61c42e72689..f3d259bb3b47 100644 --- a/src/client/datascience/ipywidgets/ipywidgetHandler.ts +++ b/src/client/datascience/ipywidgets/ipywidgetHandler.ts @@ -7,7 +7,6 @@ import { inject, injectable } from 'inversify'; import { Event, EventEmitter, Uri } from 'vscode'; import { ILoadIPyWidgetClassFailureAction, - LoadIPyWidgetClassDisabledAction, LoadIPyWidgetClassLoadAction } from '../../../datascience-ui/interactive-common/redux/reducers/types'; import { EnableIPyWidgets } from '../../common/experimentGroups'; @@ -45,7 +44,8 @@ export class IPyWidgetHandler implements IInteractiveWindowListener { constructor( @inject(INotebookProvider) notebookProvider: INotebookProvider, @inject(IDisposableRegistry) private readonly disposables: IDisposableRegistry, - @inject(IPyWidgetMessageDispatcherFactory) private readonly factory: IPyWidgetMessageDispatcherFactory, + @inject(IPyWidgetMessageDispatcherFactory) + private readonly widgetMessageDispatcherFactory: IPyWidgetMessageDispatcherFactory, @inject(IExperimentsManager) readonly experimentsManager: IExperimentsManager ) { disposables.push( @@ -71,8 +71,6 @@ export class IPyWidgetHandler implements IInteractiveWindowListener { this.sendLoadSucceededTelemetry(payload); } else if (message === InteractiveWindowMessages.IPyWidgetLoadFailure) { this.sendLoadFailureTelemetry(payload); - } else if (message === InteractiveWindowMessages.IPyWidgetLoadDisabled) { - this.sendLoadDisabledTelemetry(payload); } else if (message === InteractiveWindowMessages.IPyWidgetRenderFailure) { this.sendRenderFailureTelemetry(payload); } @@ -106,17 +104,6 @@ export class IPyWidgetHandler implements IInteractiveWindowListener { // do nothing on failure } } - private sendLoadDisabledTelemetry(payload: LoadIPyWidgetClassDisabledAction) { - try { - sendTelemetryEvent(Telemetry.IPyWidgetLoadDisabled, 0, { - moduleHash: this.hash(payload.moduleName), - moduleVersion: payload.moduleVersion - }); - } catch { - // do nothing on failure - } - } - private sendRenderFailureTelemetry(payload: Error) { try { traceError('Error rendering a widget: ', payload); @@ -129,7 +116,7 @@ export class IPyWidgetHandler implements IInteractiveWindowListener { if (!this.notebookIdentity || !this.enabled) { return; } - this.ipyWidgetMessageDispatcher = this.factory.create(this.notebookIdentity); + this.ipyWidgetMessageDispatcher = this.widgetMessageDispatcherFactory.create(this.notebookIdentity); return this.ipyWidgetMessageDispatcher; } diff --git a/src/client/datascience/ipywidgets/localWidgetScriptSourceProvider.ts b/src/client/datascience/ipywidgets/localWidgetScriptSourceProvider.ts new file mode 100644 index 000000000000..dc2cd6861500 --- /dev/null +++ b/src/client/datascience/ipywidgets/localWidgetScriptSourceProvider.ts @@ -0,0 +1,113 @@ +// Copyright (c) Microsoft Corporation. All rights reserved. +// Licensed under the MIT License. + +'use strict'; + +import * as path from 'path'; +import { Uri } from 'vscode'; +import { traceError } from '../../common/logger'; +import { IFileSystem } from '../../common/platform/types'; +import { IInterpreterService, PythonInterpreter } from '../../interpreter/contracts'; +import { captureTelemetry } from '../../telemetry'; +import { Telemetry } from '../constants'; +import { ILocalResourceUriConverter, INotebook } from '../types'; +import { IWidgetScriptSourceProvider, WidgetScriptSource } from './types'; + +/** + * Widget scripts are found in /share/jupyter/nbextensions. + * Here's an example: + * /share/jupyter/nbextensions/k3d/index.js + * /share/jupyter/nbextensions/nglview/index.js + * /share/jupyter/nbextensions/bqplot/index.js + */ +export class LocalWidgetScriptSourceProvider implements IWidgetScriptSourceProvider { + private cachedWidgetScripts?: Promise; + constructor( + private readonly notebook: INotebook, + private readonly localResourceUriConverter: ILocalResourceUriConverter, + private readonly fs: IFileSystem, + private readonly interpreterService: IInterpreterService + ) {} + public async getWidgetScriptSource(moduleName: string): Promise> { + const sources = await this.getWidgetScriptSources(); + const found = sources.find((item) => item.moduleName.toLowerCase() === moduleName.toLowerCase()); + return found || { moduleName }; + } + public dispose() { + // Noop. + } + public async getWidgetScriptSources(ignoreCache?: boolean): Promise> { + if (!ignoreCache && this.cachedWidgetScripts) { + return this.cachedWidgetScripts; + } + return (this.cachedWidgetScripts = this.getWidgetScriptSourcesWithoutCache()); + } + @captureTelemetry(Telemetry.DiscoverIPyWidgetNamesLocalPerf) + private async getWidgetScriptSourcesWithoutCache(): Promise { + const sysPrefix = await this.getSysPrefixOfKernel(); + if (!sysPrefix) { + return []; + } + + const nbextensionsPath = path.join(sysPrefix, 'share', 'jupyter', 'nbextensions'); + // Search only one level deep, hence `*/index.js`. + const files = await this.fs.search(`*${path.sep}index.js`, nbextensionsPath); + + const validFiles = files.filter((file) => { + // Should be of the form `/index.js` + const parts = file.split(path.sep); + if (parts.length !== 2) { + traceError('Incorrect file found when searching for nnbextension entrypoints'); + return false; + } + return true; + }); + + const mappedFiles = validFiles.map(async (file) => { + // Should be of the form `/index.js` + const parts = file.split(path.sep); + const moduleName = parts[0]; + + // Drop the `.js`. + const fileUri = Uri.file(path.join(nbextensionsPath, moduleName, 'index')); + const scriptUri = (await this.localResourceUriConverter.asWebviewUri(fileUri)).toString(); + // tslint:disable-next-line: no-unnecessary-local-variable + const widgetScriptSource: WidgetScriptSource = { moduleName, scriptUri, source: 'local' }; + return widgetScriptSource; + }); + // tslint:disable-next-line: no-any + return Promise.all(mappedFiles as any); + } + private async getSysPrefixOfKernel() { + const interpreter = this.getInterpreter(); + if (interpreter?.sysPrefix) { + return interpreter?.sysPrefix; + } + if (!interpreter?.path) { + return; + } + const interpreterInfo = await this.interpreterService + .getInterpreterDetails(interpreter.path) + .catch(traceError.bind(`Failed to get interpreter details for Kernel/Interpreter ${interpreter.path}`)); + + if (interpreterInfo) { + return interpreterInfo?.sysPrefix; + } + } + private getInterpreter(): Partial | undefined { + let interpreter: undefined | Partial = this.notebook.getMatchingInterpreter(); + const kernel = this.notebook.getKernelSpec(); + interpreter = kernel?.metadata?.interpreter?.path ? kernel?.metadata?.interpreter : interpreter; + + // If we still do not have the interpreter, then check if we have the path to the kernel. + if (!interpreter && kernel?.path) { + interpreter = { path: kernel.path }; + } + + if (!interpreter || !interpreter.path) { + return; + } + const pythonPath = interpreter.path; + return { ...interpreter, path: pythonPath }; + } +} diff --git a/src/client/datascience/ipywidgets/remoteWidgetScriptSourceProvider.ts b/src/client/datascience/ipywidgets/remoteWidgetScriptSourceProvider.ts new file mode 100644 index 000000000000..47ece792896c --- /dev/null +++ b/src/client/datascience/ipywidgets/remoteWidgetScriptSourceProvider.ts @@ -0,0 +1,39 @@ +// Copyright (c) Microsoft Corporation. All rights reserved. +// Licensed under the MIT License. + +'use strict'; + +import { traceWarning } from '../../common/logger'; +import { IHttpClient } from '../../common/types'; +import { IConnection } from '../types'; +import { IWidgetScriptSourceProvider, WidgetScriptSource } from './types'; + +/** + * When using a remote jupyter connection the widget scripts are accessible over + * `/nbextensions/moduleName/index` + */ +export class RemoteWidgetScriptSourceProvider implements IWidgetScriptSourceProvider { + public static validUrls = new Map(); + constructor(private readonly connection: IConnection, private readonly httpClient: IHttpClient) {} + public dispose() { + // Noop. + } + public async getWidgetScriptSource(moduleName: string, moduleVersion: string): Promise { + const scriptUri = `${this.connection.baseUrl}nbextensions/${moduleName}/index`; + const exists = await this.getUrlForWidget(`${scriptUri}.js`); + if (exists) { + return { moduleName, scriptUri, source: 'cdn' }; + } + traceWarning(`Widget Script not found for ${moduleName}@${moduleVersion}`); + return { moduleName }; + } + private async getUrlForWidget(url: string): Promise { + if (RemoteWidgetScriptSourceProvider.validUrls.has(url)) { + return RemoteWidgetScriptSourceProvider.validUrls.get(url)!; + } + + const exists = await this.httpClient.exists(url); + RemoteWidgetScriptSourceProvider.validUrls.set(url, exists); + return exists; + } +} diff --git a/src/client/datascience/ipywidgets/serialization.ts b/src/client/datascience/ipywidgets/serialization.ts deleted file mode 100644 index 99bc15dbae7b..000000000000 --- a/src/client/datascience/ipywidgets/serialization.ts +++ /dev/null @@ -1,62 +0,0 @@ -// Copyright (c) Microsoft Corporation. All rights reserved. -// Licensed under the MIT License. - -'use strict'; - -import type { KernelMessage } from '@jupyterlab/services'; - -export function restoreBuffers(buffers?: (ArrayBuffer | ArrayBufferView)[] | undefined) { - if (!buffers || !Array.isArray(buffers) || buffers.length === 0) { - return buffers || []; - } - // tslint:disable-next-line: prefer-for-of no-any - const newBuffers: any[] = []; - // tslint:disable-next-line: prefer-for-of no-any - for (let i = 0; i < buffers.length; i += 1) { - const item = buffers[i]; - if ('buffer' in item && 'byteOffset' in item) { - const buffer = new Uint8Array(item.buffer).buffer; - // It is an ArrayBufferView - // tslint:disable-next-line: no-any - const bufferView = new DataView(buffer, item.byteOffset, item.byteLength); - newBuffers.push(bufferView); - } else { - const buffer = new Uint8Array(item).buffer; - // tslint:disable-next-line: no-any - newBuffers.push(buffer); - } - } - return newBuffers; -} - -export function serializeDataViews(msg: KernelMessage.IIOPubMessage): KernelMessage.IIOPubMessage { - if (!Array.isArray(msg.buffers) || msg.buffers.length === 0) { - return msg; - } - // tslint:disable-next-line: no-any - const newBufferView: any[] = []; - // tslint:disable-next-line: prefer-for-of - for (let i = 0; i < msg.buffers.length; i += 1) { - const item = msg.buffers[i]; - if ('buffer' in item && 'byteOffset' in item) { - // It is an ArrayBufferView - // tslint:disable-next-line: no-any - const buffer = Array.apply(null, new Uint8Array(item.buffer as any) as any); - newBufferView.push({ - ...item, - byteLength: item.byteLength, - byteOffset: item.byteOffset, - buffer - // tslint:disable-next-line: no-any - } as any); // NOSONAR - } else { - // tslint:disable-next-line: no-any - newBufferView.push(Array.apply(null, new Uint8Array(item as any) as any) as any); - } - } - - return { - ...msg, - buffers: newBufferView - }; -} diff --git a/src/client/datascience/ipywidgets/types.ts b/src/client/datascience/ipywidgets/types.ts index 2388ca21da73..e09243ffa1fa 100644 --- a/src/client/datascience/ipywidgets/types.ts +++ b/src/client/datascience/ipywidgets/types.ts @@ -23,3 +23,29 @@ export interface IIPyWidgetMessageDispatcher extends IDisposable { receiveMessage(message: IPyWidgetMessage): void; initialize(): Promise; } + +/** + * Name value pair of widget name/module along with the Uri to the script. + */ +export type WidgetScriptSource = { + moduleName: string; + /** + * Where is the script being source from. + */ + source?: 'cdn' | 'local' | 'remote'; + /** + * Resource Uri (not using Uri type as this needs to be sent from extension to UI). + */ + scriptUri?: string; +}; + +/** + * Used to get an entry for widget (or all of them). + */ +export interface IWidgetScriptSourceProvider extends IDisposable { + /** + * Return the script path for the requested module. + * This is called when ipywidgets needs a source for a particular widget. + */ + getWidgetScriptSource(moduleName: string, moduleVersion: string): Promise>; +} diff --git a/src/client/datascience/jupyter/jupyterNotebook.ts b/src/client/datascience/jupyter/jupyterNotebook.ts index 7be5c279b239..963fe100dc28 100644 --- a/src/client/datascience/jupyter/jupyterNotebook.ts +++ b/src/client/datascience/jupyter/jupyterNotebook.ts @@ -27,6 +27,7 @@ import { CodeSnippits, Identifiers, Telemetry } from '../constants'; import { CellState, ICell, + IConnection, IJupyterKernelSpec, IJupyterSession, INotebook, @@ -175,6 +176,9 @@ export class JupyterNotebookBase implements INotebook { public get kernelSocket(): Observable { return this.session.kernelSocket; } + public get connection(): Readonly { + return this.launchInfo.connectionInfo; + } constructor( _liveShare: ILiveShareApi, // This is so the liveshare mixin works diff --git a/src/client/datascience/jupyter/liveshare/guestJupyterNotebook.ts b/src/client/datascience/jupyter/liveshare/guestJupyterNotebook.ts index f6898e54af90..66a5f7d37f34 100644 --- a/src/client/datascience/jupyter/liveshare/guestJupyterNotebook.ts +++ b/src/client/datascience/jupyter/liveshare/guestJupyterNotebook.ts @@ -19,6 +19,7 @@ import { PythonInterpreter } from '../../../interpreter/contracts'; import { LiveShare, LiveShareCommands } from '../../constants'; import { ICell, + IConnection, IJupyterKernelSpec, INotebook, INotebookCompletion, @@ -43,6 +44,10 @@ export class GuestJupyterNotebook return this._jupyterLab; } + public get connection(): IConnection { + throw new Error('Not Implemented'); + } + public get identity(): Uri { return this._identity; } diff --git a/src/client/datascience/serviceRegistry.ts b/src/client/datascience/serviceRegistry.ts index a93eeb97c85a..730d30e91345 100644 --- a/src/client/datascience/serviceRegistry.ts +++ b/src/client/datascience/serviceRegistry.ts @@ -45,6 +45,7 @@ import { InteractiveWindowCommandListener } from './interactive-window/interacti import { InteractiveWindowProvider } from './interactive-window/interactiveWindowProvider'; import { IPyWidgetHandler } from './ipywidgets/ipywidgetHandler'; import { IPyWidgetMessageDispatcherFactory } from './ipywidgets/ipyWidgetMessageDispatcherFactory'; +import { IPyWidgetScriptSource } from './ipywidgets/ipyWidgetScriptSource'; import { JupyterCommandLineSelector } from './jupyter/commandLineSelector'; import { JupyterCommandFactory } from './jupyter/interpreter/jupyterCommand'; import { JupyterCommandFinder } from './jupyter/interpreter/jupyterCommandFinder'; @@ -143,6 +144,7 @@ export function registerTypes(serviceManager: IServiceManager) { serviceManager.add(IInteractiveWindowListener, LinkProvider); serviceManager.add(IInteractiveWindowListener, ShowPlotListener); serviceManager.add(IInteractiveWindowListener, IPyWidgetHandler); + serviceManager.add(IInteractiveWindowListener, IPyWidgetScriptSource); serviceManager.add(IJupyterCommandFactory, JupyterCommandFactory); serviceManager.add(INotebookEditor, useCustomEditorApi ? NativeEditor : NativeEditorOldWebView); serviceManager.add(INotebookExporter, JupyterExporter); diff --git a/src/client/datascience/types.ts b/src/client/datascience/types.ts index be2bdb821366..f3809a40c8e9 100644 --- a/src/client/datascience/types.ts +++ b/src/client/datascience/types.ts @@ -109,6 +109,7 @@ export interface INotebookServer extends IAsyncDisposable { export interface INotebook extends IAsyncDisposable { readonly resource: Resource; + readonly connection: Readonly; kernelSocket: Observable; readonly identity: Uri; readonly server: INotebookServer; @@ -369,6 +370,24 @@ export interface IDataScienceErrorHandler { handleError(err: Error): Promise; } +/** + * Given a local resource this will convert the Uri into a form such that it can be used in a WebView. + */ +export interface ILocalResourceUriConverter { + /** + * Convert a uri for the local file system to one that can be used inside webviews. + * + * Webviews cannot directly load resources from the workspace or local file system using `file:` uris. The + * `asWebviewUri` function takes a local `file:` uri and converts it into a uri that can be used inside of + * a webview to load the same resource: + * + * ```ts + * webview.html = `` + * ``` + */ + asWebviewUri(localResource: Uri): Promise; +} + export interface IInteractiveBase extends Disposable { onExecutedCode: Event; notebook?: INotebook; @@ -459,6 +478,11 @@ export interface IInteractiveWindowListener extends IDisposable { */ // tslint:disable-next-line: no-any postMessage: Event<{ message: string; payload: any }>; + /** + * Fires this event when posting a message to the interactive base. + */ + // tslint:disable-next-line: no-any + postInternalMessage?: Event<{ message: string; payload: any }>; /** * Handles messages that the interactive window receives * @param message message type diff --git a/src/client/datascience/webViewHost.ts b/src/client/datascience/webViewHost.ts index 0991c5623ee1..aa96f5e9be45 100644 --- a/src/client/datascience/webViewHost.ts +++ b/src/client/datascience/webViewHost.ts @@ -4,7 +4,7 @@ import '../common/extensions'; import { injectable, unmanaged } from 'inversify'; -import { ConfigurationChangeEvent, ViewColumn, WebviewPanel, WorkspaceConfiguration } from 'vscode'; +import { ConfigurationChangeEvent, Uri, ViewColumn, WebviewPanel, WorkspaceConfiguration } from 'vscode'; import { IWebPanel, IWebPanelMessageListener, IWebPanelProvider, IWorkspaceService } from '../common/application/types'; import { isTestExecution } from '../common/constants'; @@ -82,7 +82,6 @@ export abstract class WebViewHost implements IDisposable { this.webPanel.updateCwd(cwd); } } - public dispose() { if (!this.isDisposed) { this.disposed = true; @@ -117,6 +116,12 @@ export abstract class WebViewHost implements IDisposable { this.themeIsDarkPromise.resolve(isDark); } } + protected asWebviewUri(localResource: Uri) { + if (!this.webPanel) { + throw new Error('asWebViewUri called too early'); + } + return this.webPanel?.asWebviewUri(localResource); + } protected abstract getOwningResource(): Promise; @@ -367,7 +372,7 @@ export abstract class WebViewHost implements IDisposable { event.affectsConfiguration('files.autoSave') || event.affectsConfiguration('files.autoSaveDelay') || event.affectsConfiguration('python.dataScience.enableGather') || - event.affectsConfiguration('python.dataScience.loadWidgetScriptsFromThirdPartySource') + event.affectsConfiguration('python.dataScience.widgetScriptSources') ) { // See if the theme changed const newSettings = await this.generateDataScienceExtraSettings(); diff --git a/src/client/telemetry/index.ts b/src/client/telemetry/index.ts index dc555d0a8257..b88259c9552c 100644 --- a/src/client/telemetry/index.ts +++ b/src/client/telemetry/index.ts @@ -1939,6 +1939,41 @@ export interface IEventNamePropertyMapping { * Telemetry event sent when the ZMQ native binaries do not work. */ [Telemetry.ZMQNotSupported]: undefined | never; + /** + * Telemetry event sent with name of a Widget that is used. + */ + [Telemetry.HashedIPyWidgetNameUsed]: { + /** + * Hash of the widget + */ + hashedName: string; + /** + * Where did we find the hashed name (CDN or user environment or remote jupyter). + */ + source?: 'cdn' | 'local' | 'remote'; + }; + /** + * Telemetry event sent with name of a Widget found. + */ + [Telemetry.HashedIPyWidgetNameDiscovered]: { + /** + * Hash of the widget + */ + hashedName: string; + /** + * Where did we find the hashed name (CDN or user environment or remote jupyter). + */ + source?: 'cdn' | 'local' | 'remote'; + }; + /** + * Total time taken to discover all IPyWidgets on disc. + * This is how long it takes to discover a single widget on disc (from python environment). + */ + [Telemetry.DiscoverIPyWidgetNamesLocalPerf]: never | undefined; + /** + * Something went wrong in looking for a widget. + */ + [Telemetry.HashedIPyWidgetScriptDiscoveryError]: never | undefined; /** * Telemetry event sent when an ipywidget module loads. Module name is hashed. */ @@ -1951,6 +1986,26 @@ export interface IEventNamePropertyMapping { * Telemetry event sent when an loading of 3rd party ipywidget JS scripts from 3rd party source has been disabled. */ [Telemetry.IPyWidgetLoadDisabled]: { moduleHash: string; moduleVersion: string }; + /** + * Total time taken to discover a widget script on CDN. + */ + [Telemetry.DiscoverIPyWidgetNamesCDNPerf]: { + // The CDN we were testing. + cdn: string; + // Whether we managed to find the widget on the CDN or not. + exists: boolean; + }; + /** + * Telemetry sent when we prompt user to use a CDN for IPyWidget scripts. + * This is always sent when we display a prompt. + */ + [Telemetry.IPyWidgetPromptToUseCDN]: never | undefined; + /** + * Telemetry sent when user does somethign with the prompt displsyed to user about using CDN for IPyWidget scripts. + */ + [Telemetry.IPyWidgetPromptToUseCDNSelection]: { + selection: 'ok' | 'cancel' | 'dismissed' | 'doNotShowAgain'; + }; /** * Telemetry event sent to indicate the overhead of syncing the kernel with the UI. */ diff --git a/src/datascience-ui/history-react/interactiveCell.tsx b/src/datascience-ui/history-react/interactiveCell.tsx index 0178abe662dc..b4052c3f80d5 100644 --- a/src/datascience-ui/history-react/interactiveCell.tsx +++ b/src/datascience-ui/history-react/interactiveCell.tsx @@ -132,8 +132,6 @@ export class InteractiveCell extends React.Component { const cellOuterClass = this.props.cellVM.editable ? 'cell-outer-editable' : 'cell-outer'; const cellWrapperClass = this.props.cellVM.editable ? 'cell-wrapper' : 'cell-wrapper cell-wrapper-noneditable'; const themeMatplotlibPlots = this.props.settings.themeMatplotlibPlots ? true : false; - const loadWidgetScriptsFromThirdPartySource = - this.props.settings.loadWidgetScriptsFromThirdPartySource === true; // Only render if we are allowed to. if (shouldRender) { return ( @@ -157,7 +155,6 @@ export class InteractiveCell extends React.Component { expandImage={this.props.showPlot} maxTextSize={this.props.maxTextSize} themeMatplotlibPlots={themeMatplotlibPlots} - loadWidgetScriptsFromThirdPartySource={loadWidgetScriptsFromThirdPartySource} widgetFailed={this.props.widgetFailed} /> diff --git a/src/datascience-ui/history-react/redux/reducers/index.ts b/src/datascience-ui/history-react/redux/reducers/index.ts index 2d3b4ef4c968..533397435765 100644 --- a/src/datascience-ui/history-react/redux/reducers/index.ts +++ b/src/datascience-ui/history-react/redux/reducers/index.ts @@ -41,7 +41,6 @@ export const reducerMap: Partial = { [CommonActionType.FOCUS_INPUT]: CommonEffects.focusInput, [CommonActionType.LOAD_IPYWIDGET_CLASS_SUCCESS]: CommonEffects.handleLoadIPyWidgetClassSuccess, [CommonActionType.LOAD_IPYWIDGET_CLASS_FAILURE]: CommonEffects.handleLoadIPyWidgetClassFailure, - [CommonActionType.LOAD_IPYWIDGET_CLASS_DISABLED_FAILURE]: CommonEffects.handleLoadIPyWidgetClassDisabled, [CommonActionType.IPYWIDGET_RENDER_FAILURE]: CommonEffects.handleIPyWidgetRenderFailure, // Messages from the webview (some are ignored) diff --git a/src/datascience-ui/interactive-common/cellOutput.tsx b/src/datascience-ui/interactive-common/cellOutput.tsx index 3cab23d89fa8..6ce5dd1a37ed 100644 --- a/src/datascience-ui/interactive-common/cellOutput.tsx +++ b/src/datascience-ui/interactive-common/cellOutput.tsx @@ -32,7 +32,6 @@ interface ICellOutputProps { cellVM: ICellViewModel; baseTheme: string; maxTextSize?: number; - loadWidgetScriptsFromThirdPartySource: boolean; hideOutput?: boolean; themeMatplotlibPlots?: boolean; expandImage(imageHtml: string): void; @@ -183,9 +182,6 @@ export class CellOutput extends React.Component { if (nextProps.maxTextSize !== this.props.maxTextSize) { return true; } - if (nextProps.loadWidgetScriptsFromThirdPartySource !== this.props.loadWidgetScriptsFromThirdPartySource) { - return true; - } if (nextProps.themeMatplotlibPlots !== this.props.themeMatplotlibPlots) { return true; } @@ -272,8 +268,13 @@ export class CellOutput extends React.Component { const outputs = this.renderOutputs(this.getCodeCell().outputs, trim); // Render any UI side errors + // tslint:disable: react-no-dangerous-html if (this.props.cellVM.uiSideError) { - outputs.push(
{this.props.cellVM.uiSideError}
); + outputs.push( +
+
+
+ ); } return outputs; @@ -480,25 +481,8 @@ export class CellOutput extends React.Component { transformedList.forEach((transformed, index) => { const mimetype = transformed.output.mimeType; if (isIPyWidgetOutput(transformed.output.mimeBundle)) { - if (this.props.loadWidgetScriptsFromThirdPartySource) { - // Create a view for this output if not already there. - this.renderWidget(transformed.output); - } else { - // If loading of widget source is not allowed, display a message. - const errorMessage = getLocString( - 'DataScience.loadThirdPartyWidgetScriptsDisabled', - "Loading of Widgets is disabled by default. Click here to enable the setting 'python.dataScience.loadWidgetScriptsFromThirdPartySource'. Once enabled you will need to restart the Kernel" - ); - - // tslint:disable: react-no-dangerous-html - buffer.push( -
- -
- -
- ); - } + // Create a view for this output if not already there. + this.renderWidget(transformed.output); } else if (mimetype && isMimeTypeSupported(mimetype)) { // If that worked, use the transform // Get the matching React.Component for that mimetype diff --git a/src/datascience-ui/interactive-common/redux/reducers/commonEffects.ts b/src/datascience-ui/interactive-common/redux/reducers/commonEffects.ts index 3947a0cf7897..814a01c2c550 100644 --- a/src/datascience-ui/interactive-common/redux/reducers/commonEffects.ts +++ b/src/datascience-ui/interactive-common/redux/reducers/commonEffects.ts @@ -18,7 +18,6 @@ import { CommonReducerArg, ILoadIPyWidgetClassFailureAction, IOpenSettingsAction, - LoadIPyWidgetClassDisabledAction, LoadIPyWidgetClassLoadAction } from './types'; @@ -231,12 +230,18 @@ export namespace CommonEffects { const newVMs = [...arg.prevState.cellVMs]; const current = arg.prevState.cellVMs[index]; - const errorMessage = arg.payload.data.isOnline - ? arg.payload.data.error.toString() - : getLocString( - 'DataScience.loadClassFailedWithNoInternet', - 'Error loading {0}:{1}. Internet connection required for loading 3rd party widgets.' - ).format(arg.payload.data.moduleName, arg.payload.data.moduleVersion); + let errorMessage = arg.payload.data.error.toString(); + if (!arg.payload.data.isOnline) { + errorMessage = getLocString( + 'DataScience.loadClassFailedWithNoInternet', + 'Error loading {0}:{1}. Internet connection required for loading 3rd party widgets.' + ).format(arg.payload.data.moduleName, arg.payload.data.moduleVersion); + } else if (!arg.payload.data.cdnsUsed) { + errorMessage = getLocString( + 'DataScience.enableCDNForWidgetsSetting', + "Widgets require us to download supporting files from a 3rd party website. Click here to enable this or click here for more information. (Error loading {0}:{1})." + ).format(arg.payload.data.moduleName, arg.payload.data.moduleVersion); + } newVMs[index] = Helpers.asCellViewModel({ ...current, uiSideError: errorMessage @@ -253,13 +258,6 @@ export namespace CommonEffects { return arg.prevState; } } - export function handleLoadIPyWidgetClassDisabled( - arg: CommonReducerArg - ): IMainState { - // Make sure to tell the extension so it can log telemetry. - postActionToExtension(arg, InteractiveWindowMessages.IPyWidgetLoadDisabled, arg.payload.data); - return arg.prevState; - } export function handleIPyWidgetRenderFailure(arg: CommonReducerArg): IMainState { // Make sure to tell the extension so it can log telemetry. postActionToExtension(arg, InteractiveWindowMessages.IPyWidgetRenderFailure, arg.payload.data); diff --git a/src/datascience-ui/interactive-common/redux/reducers/types.ts b/src/datascience-ui/interactive-common/redux/reducers/types.ts index 241856907b76..67cea837903b 100644 --- a/src/datascience-ui/interactive-common/redux/reducers/types.ts +++ b/src/datascience-ui/interactive-common/redux/reducers/types.ts @@ -58,7 +58,6 @@ export enum CommonActionType { IPYWIDGET_RENDER_FAILURE = 'action.ipywidget_render_failure', LOAD_IPYWIDGET_CLASS_SUCCESS = 'action.load_ipywidget_class_success', LOAD_IPYWIDGET_CLASS_FAILURE = 'action.load_ipywidget_class_failure', - LOAD_IPYWIDGET_CLASS_DISABLED_FAILURE = 'action.load_ipywidget_class_disabled_failure', LOADED_ALL_CELLS = 'action.loaded_all_cells', LINK_CLICK = 'action.link_click', MOVE_CELL_DOWN = 'action.move_cell_down', @@ -135,7 +134,6 @@ export type CommonActionTypeMapping = { [CommonActionType.FOCUS_INPUT]: never | undefined; [CommonActionType.LOAD_IPYWIDGET_CLASS_SUCCESS]: LoadIPyWidgetClassLoadAction; [CommonActionType.LOAD_IPYWIDGET_CLASS_FAILURE]: ILoadIPyWidgetClassFailureAction; - [CommonActionType.LOAD_IPYWIDGET_CLASS_DISABLED_FAILURE]: LoadIPyWidgetClassDisabledAction; [CommonActionType.IPYWIDGET_RENDER_FAILURE]: Error; }; @@ -217,6 +215,7 @@ export interface ILoadIPyWidgetClassFailureAction { className: string; moduleName: string; moduleVersion: string; + cdnsUsed: boolean; isOnline: boolean; // tslint:disable-next-line: no-any error: any; diff --git a/src/datascience-ui/ipywidgets/container.tsx b/src/datascience-ui/ipywidgets/container.tsx index 1ee89a9763ff..4b8780349c47 100644 --- a/src/datascience-ui/ipywidgets/container.tsx +++ b/src/datascience-ui/ipywidgets/container.tsx @@ -6,19 +6,24 @@ import * as isonline from 'is-online'; import * as React from 'react'; import { Store } from 'redux'; -import { IStore } from '../interactive-common/redux/store'; -import { PostOffice } from '../react-common/postOffice'; -import { WidgetManager } from './manager'; - +import { createDeferred, Deferred } from '../../client/common/utils/async'; +import { + IInteractiveWindowMapping, + IPyWidgetMessages +} from '../../client/datascience/interactive-common/interactiveWindowTypes'; +import { WidgetScriptSource } from '../../client/datascience/ipywidgets/types'; import { SharedMessages } from '../../client/datascience/messages'; import { IDataScienceExtraSettings } from '../../client/datascience/types'; import { CommonAction, CommonActionType, ILoadIPyWidgetClassFailureAction, - LoadIPyWidgetClassDisabledAction, LoadIPyWidgetClassLoadAction } from '../interactive-common/redux/reducers/types'; +import { IStore } from '../interactive-common/redux/store'; +import { PostOffice } from '../react-common/postOffice'; +import { WidgetManager } from './manager'; +import { registerScripts } from './requirejsRegistry'; type Props = { postOffice: PostOffice; @@ -28,16 +33,26 @@ type Props = { export class WidgetManagerComponent extends React.Component { private readonly widgetManager: WidgetManager; + private readonly widgetSourceRequests = new Map>(); + private timedoutWaitingForWidgetsToGetLoaded?: boolean; + private widgetsCanLoadFromCDN: boolean = false; private readonly loaderSettings = { - loadWidgetScriptsFromThirdPartySource: false, + // Total time to wait for a script to load. This includes ipywidgets making a request to extension for a Uri of a widget, + // then extension replying back with the Uri (max 5 seconds round trip time). + // If expires, then Widget downloader will attempt to download with what ever information it has (potentially failing). + // Note, we might have a message displayed at the user end (asking for consent to use CDN). + // Hence use 60 seconds. + timeoutWaitingForScriptToLoad: 60_000, + // List of widgets that must always be loaded using requirejs instead of using a CDN or the like. + widgetsRegisteredInRequireJs: new Set(), + // Callback when loading a widget fails. errorHandler: this.handleLoadError.bind(this), + // Callback when requesting a module be registered with requirejs (if possible). + loadWidgetScript: this.loadWidgetScript.bind(this), successHandler: this.handleLoadSuccess.bind(this) }; constructor(props: Props) { super(props); - this.loaderSettings.loadWidgetScriptsFromThirdPartySource = - props.store.getState().main.settings?.loadWidgetScriptsFromThirdPartySource === true; - this.widgetManager = new WidgetManager( document.getElementById(this.props.widgetContainerId)!, this.props.postOffice, @@ -49,8 +64,16 @@ export class WidgetManagerComponent extends React.Component { handleMessage: (type: string, payload?: any) => { if (type === SharedMessages.UpdateSettings) { const settings = JSON.parse(payload) as IDataScienceExtraSettings; - this.loaderSettings.loadWidgetScriptsFromThirdPartySource = - settings.loadWidgetScriptsFromThirdPartySource === true; + this.widgetsCanLoadFromCDN = settings.widgetScriptSources.length > 0; + } else if (type === IPyWidgetMessages.IPyWidgets_WidgetScriptSourceResponse) { + this.registerScriptSourceInRequirejs(payload as WidgetScriptSource); + } else if ( + type === IPyWidgetMessages.IPyWidgets_kernelOptions || + type === IPyWidgetMessages.IPyWidgets_onKernelChanged + ) { + // This happens when we have restarted a kernel. + // If user changed the kernel, then some widgets might exist now and some might now. + this.widgetSourceRequests.clear(); } return true; } @@ -62,7 +85,41 @@ export class WidgetManagerComponent extends React.Component { public componentWillUnmount() { this.widgetManager.dispose(); } + /** + * Given a list of the widgets along with the sources, we will need to register them with requirejs. + * IPyWidgets uses requirejs to dynamically load modules. + * (https://requirejs.org/docs/api.html) + * All we're doing here is given a widget (module) name, we register the path where the widget (module) can be loaded from. + * E.g. + * requirejs.config({ paths:{ + * 'widget_xyz': '' + * }}); + */ + private registerScriptSourcesInRequirejs(sources: WidgetScriptSource[]) { + if (!Array.isArray(sources) || sources.length === 0) { + return; + } + + registerScripts(sources); + // Now resolve promises (anything that was waiting for modules to get registered can carry on). + sources.forEach((source) => { + // We have fetched the script sources for all of these modules. + // In some cases we might not have the source, meaning we don't have it or couldn't find it. + let deferred = this.widgetSourceRequests.get(source.moduleName); + if (!deferred) { + deferred = createDeferred(); + this.widgetSourceRequests.set(source.moduleName, deferred); + } + deferred.resolve(); + }); + } + private registerScriptSourceInRequirejs(source?: WidgetScriptSource) { + if (!source) { + return; + } + this.registerScriptSourcesInRequirejs([source]); + } private createLoadSuccessAction( className: string, moduleName: string, @@ -84,30 +141,70 @@ export class WidgetManagerComponent extends React.Component { ): CommonAction { return { type: CommonActionType.LOAD_IPYWIDGET_CLASS_FAILURE, - payload: { messageDirection: 'incoming', data: { className, moduleName, moduleVersion, isOnline, error } } - }; - } - private createLoadDisabledErrorAction( - className: string, - moduleName: string, - moduleVersion: string - ): CommonAction { - return { - type: CommonActionType.LOAD_IPYWIDGET_CLASS_DISABLED_FAILURE, - payload: { messageDirection: 'incoming', data: { className, moduleName, moduleVersion } } + payload: { + messageDirection: 'incoming', + data: { + className, + moduleName, + moduleVersion, + isOnline, + error, + cdnsUsed: this.widgetsCanLoadFromCDN + } + } }; } - // tslint:disable-next-line: no-any private async handleLoadError(className: string, moduleName: string, moduleVersion: string, error: any) { - if (this.loaderSettings.loadWidgetScriptsFromThirdPartySource) { - const isOnline = await isonline.default({ timeout: 1000 }); - this.props.store.dispatch( - this.createLoadErrorAction(className, moduleName, moduleVersion, isOnline, error) - ); - } else { - this.props.store.dispatch(this.createLoadDisabledErrorAction(className, moduleName, moduleVersion)); + const isOnline = await isonline.default({ timeout: 1000 }); + this.props.store.dispatch(this.createLoadErrorAction(className, moduleName, moduleVersion, isOnline, error)); + } + + /** + * Method called by ipywidgets to get the source for a widget. + * When we get a source for the widget, we register it in requriejs. + * We need to check if it is avaialble on CDN, if not then fallback to local FS. + * Or check local FS then fall back to CDN (depending on the order defined by the user). + */ + private loadWidgetScript(moduleName: string, moduleVersion: string): Promise { + // tslint:disable-next-line: no-console + console.log(`Fetch IPyWidget source for ${moduleName}`); + let deferred = this.widgetSourceRequests.get(moduleName); + if (!deferred) { + deferred = createDeferred(); + this.widgetSourceRequests.set(moduleName, deferred); + + // If we timeout, then resolve this promise. + // We don't want the calling code to unnecessary wait for too long. + // Else UI will not get rendered due to blocking ipywidets (at the end of the day ipywidgets gets loaded via kernel) + // And kernel blocks the UI from getting processed. + // Also, if we timeout once, then for subsequent attempts, wait for just 1 second. + // Possible user has ignored some UI prompt and things are now in a state of limbo. + // This way thigns will fall over sooner due to missing widget sources. + const timeoutTime = this.timedoutWaitingForWidgetsToGetLoaded + ? 1_000 + : this.loaderSettings.timeoutWaitingForScriptToLoad; + + setTimeout(() => { + // tslint:disable-next-line: no-console + console.error(`Timeout waiting to get widget source for ${moduleName}, ${moduleVersion}`); + if (deferred) { + deferred.resolve(); + } + this.timedoutWaitingForWidgetsToGetLoaded = true; + }, timeoutTime); } + // Whether we have the scripts or not, send message to extension. + // Useful telemetry and also we know it was explicity requestd by ipywidgest. + this.props.postOffice.sendMessage( + IPyWidgetMessages.IPyWidgets_WidgetScriptSourceRequest, + { moduleName, moduleVersion } + ); + + return deferred.promise.catch((ex) => + // tslint:disable-next-line: no-console + console.error(`Failed to load Widget Script from Extension for for ${moduleName}, ${moduleVersion}`, ex) + ); } private handleLoadSuccess(className: string, moduleName: string, moduleVersion: string) { diff --git a/src/datascience-ui/ipywidgets/manager.ts b/src/datascience-ui/ipywidgets/manager.ts index 45d3ab466624..1f81bf83cfb9 100644 --- a/src/datascience-ui/ipywidgets/manager.ts +++ b/src/datascience-ui/ipywidgets/manager.ts @@ -47,9 +47,10 @@ export class WidgetManager implements IIPyWidgetManager, IMessageHandler { private readonly widgetContainer: HTMLElement, private readonly postOffice: PostOffice, private readonly scriptLoader: { - loadWidgetScriptsFromThirdPartySource: boolean; + readonly widgetsRegisteredInRequireJs: Readonly>; // tslint:disable-next-line: no-any errorHandler(className: string, moduleName: string, moduleVersion: string, error: any): void; + loadWidgetScript(moduleName: string, moduleVersion: string): void; successHandler(className: string, moduleName: string, moduleVersion: string): void; } ) { diff --git a/src/datascience-ui/ipywidgets/requirejsRegistry.ts b/src/datascience-ui/ipywidgets/requirejsRegistry.ts new file mode 100644 index 000000000000..74d5b370a37a --- /dev/null +++ b/src/datascience-ui/ipywidgets/requirejsRegistry.ts @@ -0,0 +1,69 @@ +// Copyright (c) Microsoft Corporation. All rights reserved. +// Licensed under the MIT License. + +'use strict'; + +import { WidgetScriptSource } from '../../client/datascience/ipywidgets/types'; + +type NonPartial = { + [P in keyof T]-?: T[P]; +}; + +// Key = module name, value = path to script. +const scriptsAlreadyRegisteredInRequireJs = new Map(); + +function getScriptsToBeRegistered(scripts: WidgetScriptSource[]) { + return scripts.filter((script) => { + // Ignore scripts that have already been registered once before. + if ( + scriptsAlreadyRegisteredInRequireJs.has(script.moduleName) && + scriptsAlreadyRegisteredInRequireJs.get(script.moduleName) === script.scriptUri + ) { + return false; + } + return true; + }); +} + +function getScriptsWithAValidScriptUriToBeRegistered(scripts: WidgetScriptSource[]) { + return scripts + .filter((source) => { + if (source.scriptUri) { + // tslint:disable-next-line: no-console + console.log( + `Source for IPyWidget ${source.moduleName} found in ${source.source} @ ${source.scriptUri}.` + ); + return true; + } else { + // tslint:disable-next-line: no-console + console.error(`Source for IPyWidget ${source.moduleName} not found.`); + return false; + } + }) + .map((source) => source as NonPartial); +} + +function registerScriptsInRequireJs(scripts: NonPartial[]) { + // tslint:disable-next-line: no-any + const requirejs = (window as any).requirejs as { config: Function }; + if (!requirejs) { + window.console.error('Requirejs not found'); + throw new Error('Requirejs not found'); + } + const config: { paths: Record } = { + paths: {} + }; + scripts.forEach((script) => { + scriptsAlreadyRegisteredInRequireJs.set(script.moduleName, script.scriptUri); + // Register the script source into requirejs so it gets loaded via requirejs. + config.paths[script.moduleName] = script.scriptUri; + }); + + requirejs.config(config); +} + +export function registerScripts(scripts: WidgetScriptSource[]) { + const scriptsToRegister = getScriptsToBeRegistered(scripts); + const validScriptsToRegister = getScriptsWithAValidScriptUriToBeRegistered(scriptsToRegister); + registerScriptsInRequireJs(validScriptsToRegister); +} diff --git a/src/datascience-ui/ipywidgets/types.ts b/src/datascience-ui/ipywidgets/types.ts index 31e89115a249..d47e5c2b92b1 100644 --- a/src/datascience-ui/ipywidgets/types.ts +++ b/src/datascience-ui/ipywidgets/types.ts @@ -19,7 +19,6 @@ export type IJupyterLabWidgetManagerCtor = new ( kernel: Kernel.IKernelConnection, el: HTMLElement, scriptLoader: { - loadWidgetScriptsFromThirdPartySource: boolean; // tslint:disable-next-line: no-any errorHandler(className: string, moduleName: string, moduleVersion: string, error: any): void; } diff --git a/src/datascience-ui/native-editor/nativeCell.tsx b/src/datascience-ui/native-editor/nativeCell.tsx index 3d7167505e8f..409054dd7ec9 100644 --- a/src/datascience-ui/native-editor/nativeCell.tsx +++ b/src/datascience-ui/native-editor/nativeCell.tsx @@ -51,7 +51,6 @@ interface INativeCellBaseProps { themeMatplotlibPlots: boolean | undefined; focusPending: number; busy: boolean; - loadWidgetScriptsFromThirdPartySource: boolean; } type INativeCellProps = INativeCellBaseProps & typeof actionCreators; @@ -688,7 +687,6 @@ export class NativeCell extends React.Component { private renderOutput = (): JSX.Element | null => { const themeMatplotlibPlots = this.props.themeMatplotlibPlots ? true : false; const toolbar = this.props.cellVM.cell.data.cell_type === 'markdown' ? this.renderMiddleToolbar() : null; - const loadWidgetScriptsFromThirdPartySource = this.props.loadWidgetScriptsFromThirdPartySource === true; if (this.shouldRenderOutput()) { return (
@@ -699,7 +697,6 @@ export class NativeCell extends React.Component { expandImage={this.props.showPlot} maxTextSize={this.props.maxTextSize} themeMatplotlibPlots={themeMatplotlibPlots} - loadWidgetScriptsFromThirdPartySource={loadWidgetScriptsFromThirdPartySource} widgetFailed={this.props.widgetFailed} />
diff --git a/src/datascience-ui/native-editor/nativeEditor.tsx b/src/datascience-ui/native-editor/nativeEditor.tsx index f1350d0db492..20f8bb0be0e6 100644 --- a/src/datascience-ui/native-editor/nativeEditor.tsx +++ b/src/datascience-ui/native-editor/nativeEditor.tsx @@ -247,8 +247,6 @@ ${buildSettingsCss(this.props.settings)}`} if (!this.props.settings || !this.props.editorOptions) { return null; } - const loadWidgetScriptsFromThirdPartySource = - this.props.settings.loadWidgetScriptsFromThirdPartySource === true; const addNewCell = () => { setTimeout(() => this.props.insertBelow(cellVM.cell.id), 1); this.props.sendCommand(NativeCommandType.AddToEnd, 'mouse'); @@ -285,7 +283,6 @@ ${buildSettingsCss(this.props.settings)}`} // Focus pending does not apply to native editor. focusPending={0} busy={this.props.busy} - loadWidgetScriptsFromThirdPartySource={loadWidgetScriptsFromThirdPartySource} /> {lastLine} diff --git a/src/datascience-ui/native-editor/redux/reducers/index.ts b/src/datascience-ui/native-editor/redux/reducers/index.ts index 06e802cc8d17..049954ac03c8 100644 --- a/src/datascience-ui/native-editor/redux/reducers/index.ts +++ b/src/datascience-ui/native-editor/redux/reducers/index.ts @@ -60,7 +60,6 @@ export const reducerMap: Partial = { [CommonActionType.UNMOUNT]: Creation.unmount, [CommonActionType.LOAD_IPYWIDGET_CLASS_SUCCESS]: CommonEffects.handleLoadIPyWidgetClassSuccess, [CommonActionType.LOAD_IPYWIDGET_CLASS_FAILURE]: CommonEffects.handleLoadIPyWidgetClassFailure, - [CommonActionType.LOAD_IPYWIDGET_CLASS_DISABLED_FAILURE]: CommonEffects.handleLoadIPyWidgetClassDisabled, // Messages from the webview (some are ignored) [InteractiveWindowMessages.StartCell]: Creation.startCell, diff --git a/src/datascience-ui/react-common/settingsReactSide.ts b/src/datascience-ui/react-common/settingsReactSide.ts index 2aaaa5661bae..0a3436e30ada 100644 --- a/src/datascience-ui/react-common/settingsReactSide.ts +++ b/src/datascience-ui/react-common/settingsReactSide.ts @@ -66,7 +66,8 @@ export function getDefaultSettings() { runStartupCommands: '', debugJustMyCode: true, variableQueries: [], - jupyterCommandLineArguments: [] + jupyterCommandLineArguments: [], + widgetScriptSources: [] }; return result; diff --git a/src/ipywidgets/src/embed.ts b/src/ipywidgets/src/embed.ts index 8aa008a52c32..d72c2cc14923 100644 --- a/src/ipywidgets/src/embed.ts +++ b/src/ipywidgets/src/embed.ts @@ -100,8 +100,9 @@ export function requireLoader(moduleName: string, moduleVersion: string): Promis export function renderWidgets(element = document.documentElement): void { const managerFactory = (): any => { return new wm.WidgetManager(undefined, element, { - loadWidgetScriptsFromThirdPartySource: false, + widgetsRegisteredInRequireJs: new Set(), errorHandler: () => 'Error loading widget.', + loadWidgetScript: (_moduleName: string, _moduleVersion: string) => Promise.resolve(), successHandler: () => 'Success' }); }; diff --git a/src/ipywidgets/src/manager.ts b/src/ipywidgets/src/manager.ts index 3a7908565bc3..6855457cda1d 100644 --- a/src/ipywidgets/src/manager.ts +++ b/src/ipywidgets/src/manager.ts @@ -16,6 +16,14 @@ export const WIDGET_MIMETYPE = 'application/vnd.jupyter.widget-view+json'; // tslint:disable: no-any // Source borrowed from https://github.com/jupyter-widgets/ipywidgets/blob/master/examples/web3/src/manager.ts +// These widgets can always be loaded from requirejs (as it is bundled). +const widgetsRegisteredInRequireJs = [ + '@jupyter-widgets/controls', + '@jupyter-widgets/base', + '@jupyter-widgets/output', + 'azureml_widgets' +]; + export class WidgetManager extends jupyterlab.WidgetManager { public kernel: Kernel.IKernelConnection; public el: HTMLElement; @@ -24,8 +32,9 @@ export class WidgetManager extends jupyterlab.WidgetManager { kernel: Kernel.IKernelConnection, el: HTMLElement, private readonly scriptLoader: { - loadWidgetScriptsFromThirdPartySource: boolean; + readonly widgetsRegisteredInRequireJs: Readonly>; errorHandler(className: string, moduleName: string, moduleVersion: string, error: any): void; + loadWidgetScript(moduleName: string, moduleVersion: string): Promise; successHandler(className: string, moduleName: string, moduleVersion: string): void; } ) { @@ -103,10 +112,17 @@ export class WidgetManager extends jupyterlab.WidgetManager { }) .catch(async (originalException) => { try { - if (!this.scriptLoader.loadWidgetScriptsFromThirdPartySource) { - throw new Error('Loading from 3rd party source is disabled'); + const loadModuleFromRequirejs = + widgetsRegisteredInRequireJs.includes(moduleName) || + this.scriptLoader.widgetsRegisteredInRequireJs.has(moduleName); + + if (!loadModuleFromRequirejs) { + // If not loading from requirejs, then check if we can. + // Notify the script loader that we need to load the widget module. + // If possible the loader will locate and register that in requirejs for things to start working. + await this.scriptLoader.loadWidgetScript(moduleName, moduleVersion); } - const m = await requireLoader(moduleName, moduleVersion); + const m = await requireLoader(moduleName); if (m && m[className]) { this.sendSuccess(className, moduleName, moduleVersion); return m[className]; @@ -114,17 +130,7 @@ export class WidgetManager extends jupyterlab.WidgetManager { throw originalException; } catch (ex) { this.sendError(className, moduleName, moduleVersion, originalException); - if (this.scriptLoader.loadWidgetScriptsFromThirdPartySource) { - throw originalException; - } else { - // Don't throw exceptions if disabled, else everything stops working. - window.console.error(ex); - // Returning an unresolved promise will prevent Jupyter ipywidgets from doing anything. - // tslint:disable-next-line: promise-must-complete - return new Promise(() => { - // Noop. - }); - } + throw originalException; } }); diff --git a/src/ipywidgets/src/widgetLoader.ts b/src/ipywidgets/src/widgetLoader.ts index d3c6ed1a50e8..112e7c5b40cf 100644 --- a/src/ipywidgets/src/widgetLoader.ts +++ b/src/ipywidgets/src/widgetLoader.ts @@ -3,33 +3,6 @@ 'use strict'; -// This is the magic that allows us to load 3rd party widgets. -// If a widget isn't found locally, lets try to get the required files from `unpkg.com`. -// For some reason this isn't the default behavior of the html manager. - -// Source borrowed from https://github.com/jupyter-widgets/ipywidgets/blob/master/examples/web3/src/manager.ts - -const cdn = 'https://unpkg.com/'; - -function moduleNameToCDNUrl(moduleName: string, moduleVersion: string) { - let packageName = moduleName; - let fileName = 'index'; // default filename - // if a '/' is present, like 'foo/bar', packageName is changed to 'foo', and path to 'bar' - // We first find the first '/' - let index = moduleName.indexOf('/'); - if (index !== -1 && moduleName[0] === '@') { - // if we have a namespace, it's a different story - // @foo/bar/baz should translate to @foo/bar and baz - // so we find the 2nd '/' - index = moduleName.indexOf('/', index + 1); - } - if (index !== -1) { - fileName = moduleName.substr(index + 1); - packageName = moduleName.substr(0, index); - } - return `${cdn}${packageName}@${moduleVersion}/dist/${fileName}`; -} - // tslint:disable-next-line: no-any async function requirePromise(pkg: string | string[]): Promise { return new Promise((resolve, reject) => { @@ -42,22 +15,6 @@ async function requirePromise(pkg: string | string[]): Promise { } }); } -const widgetsToLoadFromRequire = [ - 'azureml_widgets', - '@jupyter-widgets/controls', - '@jupyter-widgets/base', - '@jupyter-widgets/output' -]; -export function requireLoader(moduleName: string, moduleVersion: string) { - if (!widgetsToLoadFromRequire.includes(moduleName)) { - // tslint:disable-next-line: no-any - const requirejs = (window as any).requirejs; - if (requirejs === undefined) { - throw new Error('Requirejs is needed, please ensure it is loaded on the page.'); - } - const conf: { paths: { [key: string]: string } } = { paths: {} }; - conf.paths[moduleName] = moduleNameToCDNUrl(moduleName, moduleVersion); - requirejs.config(conf); - } +export function requireLoader(moduleName: string) { return requirePromise([`${moduleName}`]); } diff --git a/src/test/common/configSettings/configSettings.unit.test.ts b/src/test/common/configSettings/configSettings.unit.test.ts index bfe108f2997c..aae97ddb5da3 100644 --- a/src/test/common/configSettings/configSettings.unit.test.ts +++ b/src/test/common/configSettings/configSettings.unit.test.ts @@ -308,7 +308,8 @@ suite('Python Settings', async () => { runStartupCommands: '', debugJustMyCode: true, variableQueries: [], - jupyterCommandLineArguments: [] + jupyterCommandLineArguments: [], + widgetScriptSources: [] }; expected.pythonPath = 'python3'; // tslint:disable-next-line:no-any diff --git a/src/test/datascience/color.test.ts b/src/test/datascience/color.test.ts index 74603f558f19..5a36adb60557 100644 --- a/src/test/datascience/color.test.ts +++ b/src/test/datascience/color.test.ts @@ -74,7 +74,8 @@ suite('Theme colors', () => { runStartupCommands: '', debugJustMyCode: true, variableQueries: [], - jupyterCommandLineArguments: [] + jupyterCommandLineArguments: [], + widgetScriptSources: [] }; configService = TypeMoq.Mock.ofType(); configService.setup((x) => x.getSettings(TypeMoq.It.isAny())).returns(() => settings); diff --git a/src/test/datascience/dataScienceIocContainer.ts b/src/test/datascience/dataScienceIocContainer.ts index bfeaf2b1bef5..7157d6caeb89 100644 --- a/src/test/datascience/dataScienceIocContainer.ts +++ b/src/test/datascience/dataScienceIocContainer.ts @@ -1306,7 +1306,8 @@ export class DataScienceIocContainer extends UnitTestIocContainer { show: noop as any, postMessage: noop as any, close: noop, - updateCwd: noop as any + updateCwd: noop as any, + asWebviewUri: (uri) => uri }); } } @@ -1402,7 +1403,7 @@ export class DataScienceIocContainer extends UnitTestIocContainer { variableQueries: [], jupyterCommandLineArguments: [], disableJupyterAutoStart: true, - loadWidgetScriptsFromThirdPartySource: true + widgetScriptSources: [] }; pythonSettings.jediEnabled = false; pythonSettings.downloadLanguageServer = false; diff --git a/src/test/datascience/editor-integration/codewatcher.unit.test.ts b/src/test/datascience/editor-integration/codewatcher.unit.test.ts index e0738de79c2b..cf5ed1ec7837 100644 --- a/src/test/datascience/editor-integration/codewatcher.unit.test.ts +++ b/src/test/datascience/editor-integration/codewatcher.unit.test.ts @@ -94,7 +94,8 @@ suite('DataScience Code Watcher Unit Tests', () => { runStartupCommands: '', debugJustMyCode: true, variableQueries: [], - jupyterCommandLineArguments: [] + jupyterCommandLineArguments: [], + widgetScriptSources: [] }; debugService.setup((d) => d.activeDebugSession).returns(() => undefined); diff --git a/src/test/datascience/execution.unit.test.ts b/src/test/datascience/execution.unit.test.ts index 1f941679d28b..4642cbfa215b 100644 --- a/src/test/datascience/execution.unit.test.ts +++ b/src/test/datascience/execution.unit.test.ts @@ -892,7 +892,8 @@ suite('Jupyter Execution', async () => { runStartupCommands: '', debugJustMyCode: true, variableQueries: [], - jupyterCommandLineArguments: [] + jupyterCommandLineArguments: [], + widgetScriptSources: [] }; // Service container also needs to generate jupyter servers. However we can't use a mock as that messes up returning diff --git a/src/test/datascience/helpers.ts b/src/test/datascience/helpers.ts index d914a45ad15b..c7e8cdaa6a9f 100644 --- a/src/test/datascience/helpers.ts +++ b/src/test/datascience/helpers.ts @@ -32,6 +32,7 @@ export function defaultDataScienceSettings(): IDataScienceSettings { runStartupCommands: '', debugJustMyCode: true, variableQueries: [], - jupyterCommandLineArguments: [] + jupyterCommandLineArguments: [], + widgetScriptSources: [] }; } diff --git a/src/test/datascience/interactiveWindowCommandListener.unit.test.ts b/src/test/datascience/interactiveWindowCommandListener.unit.test.ts index 97f8fedef1d5..3bc761654562 100644 --- a/src/test/datascience/interactiveWindowCommandListener.unit.test.ts +++ b/src/test/datascience/interactiveWindowCommandListener.unit.test.ts @@ -141,7 +141,8 @@ suite('Interactive window command listener', async () => { runStartupCommands: '', debugJustMyCode: true, variableQueries: [], - jupyterCommandLineArguments: [] + jupyterCommandLineArguments: [], + widgetScriptSources: [] }; when(knownSearchPaths.getSearchPaths()).thenReturn(['/foo/bar']); diff --git a/src/test/datascience/ipywidgets/cdnWidgetScriptSourceProvider.unit.test.ts b/src/test/datascience/ipywidgets/cdnWidgetScriptSourceProvider.unit.test.ts new file mode 100644 index 000000000000..22f3faae9788 --- /dev/null +++ b/src/test/datascience/ipywidgets/cdnWidgetScriptSourceProvider.unit.test.ts @@ -0,0 +1,186 @@ +// Copyright (c) Microsoft Corporation. All rights reserved. +// Licensed under the MIT License. +import { assert } from 'chai'; +import { anything, instance, mock, verify, when } from 'ts-mockito'; +import { EventEmitter } from 'vscode'; +import { PythonSettings } from '../../../client/common/configSettings'; +import { ConfigurationService } from '../../../client/common/configuration/service'; +import { HttpClient } from '../../../client/common/net/httpClient'; +import { IConfigurationService, IHttpClient, WidgetCDNs } from '../../../client/common/types'; +import { noop } from '../../../client/common/utils/misc'; +import { CDNWidgetScriptSourceProvider } from '../../../client/datascience/ipywidgets/cdnWidgetScriptSourceProvider'; +import { IWidgetScriptSourceProvider, WidgetScriptSource } from '../../../client/datascience/ipywidgets/types'; +import { JupyterNotebookBase } from '../../../client/datascience/jupyter/jupyterNotebook'; +import { IConnection, INotebook } from '../../../client/datascience/types'; + +const unpgkUrl = 'https://unpkg.com/'; +const jsdelivrUrl = 'https://cdn.jsdelivr.net/npm/'; + +// tslint:disable: max-func-body-length no-any +suite('Data Science - ipywidget - CDN', () => { + let scriptSourceProvider: IWidgetScriptSourceProvider; + let notebook: INotebook; + let configService: IConfigurationService; + let httpClient: IHttpClient; + let settings: PythonSettings; + setup(() => { + notebook = mock(JupyterNotebookBase); + configService = mock(ConfigurationService); + httpClient = mock(HttpClient); + settings = { datascience: { widgetScriptSources: [] } } as any; + when(configService.getSettings(anything())).thenReturn(settings as any); + CDNWidgetScriptSourceProvider.validUrls = new Map(); + scriptSourceProvider = new CDNWidgetScriptSourceProvider(instance(configService), instance(httpClient)); + }); + + [true, false].forEach((localLaunch) => { + suite(localLaunch ? 'Local Jupyter Server' : 'Remote Jupyter Server', () => { + setup(() => { + const connection: IConnection = { + baseUrl: '', + localProcExitCode: undefined, + disconnected: new EventEmitter().event, + dispose: noop, + hostName: '', + localLaunch, + token: '' + }; + when(notebook.connection).thenReturn(connection); + }); + test('Script source will be empty if CDN is not a configured source of widget scripts in settings', async () => { + const value = await scriptSourceProvider.getWidgetScriptSource('HelloWorld', '1'); + + assert.deepEqual(value, { moduleName: 'HelloWorld' }); + // Should not make any http calls. + verify(httpClient.exists(anything())).never(); + }); + function updateCDNSettings(...values: WidgetCDNs[]) { + settings.datascience.widgetScriptSources = values; + } + (['unpkg.com', 'jsdelivr.com'] as WidgetCDNs[]).forEach((cdn) => { + suite(cdn, () => { + const moduleName = 'HelloWorld'; + const moduleVersion = '1'; + let expectedSource = ''; + setup(() => { + const baseUrl = cdn === 'unpkg.com' ? unpgkUrl : jsdelivrUrl; + expectedSource = `${baseUrl}${moduleName}@${moduleVersion}/dist/index`; + CDNWidgetScriptSourceProvider.validUrls = new Map(); + }); + test('Get widget source from CDN', async () => { + updateCDNSettings(cdn); + when(httpClient.exists(anything())).thenResolve(true); + + const value = await scriptSourceProvider.getWidgetScriptSource(moduleName, moduleVersion); + + assert.deepEqual(value, { + moduleName: 'HelloWorld', + scriptUri: expectedSource, + source: 'cdn' + }); + verify(httpClient.exists(anything())).once(); + }); + test('Ensure widgtet script is downloaded once and cached', async () => { + updateCDNSettings(cdn); + when(httpClient.exists(anything())).thenResolve(true); + const expectedValue: WidgetScriptSource = { + moduleName: 'HelloWorld', + scriptUri: expectedSource, + source: 'cdn' + }; + + const value = await scriptSourceProvider.getWidgetScriptSource(moduleName, moduleVersion); + assert.deepEqual(value, expectedValue); + const value1 = await scriptSourceProvider.getWidgetScriptSource(moduleName, moduleVersion); + assert.deepEqual(value1, expectedValue); + const value2 = await scriptSourceProvider.getWidgetScriptSource(moduleName, moduleVersion); + assert.deepEqual(value2, expectedValue); + + // Only one http request + verify(httpClient.exists(anything())).once(); + }); + test('No script source if package does not exist on CDN', async () => { + updateCDNSettings(cdn); + when(httpClient.exists(anything())).thenResolve(false); + + const value = await scriptSourceProvider.getWidgetScriptSource(moduleName, moduleVersion); + + assert.deepEqual(value, { moduleName: 'HelloWorld' }); + verify(httpClient.exists(anything())).once(); + }); + test('No script source if package does not exist on both CDNs', async () => { + updateCDNSettings('jsdelivr.com', 'unpkg.com'); + when(httpClient.exists(anything())).thenResolve(false); + + const value = await scriptSourceProvider.getWidgetScriptSource(moduleName, moduleVersion); + + assert.deepEqual(value, { moduleName: 'HelloWorld' }); + }); + test('Give preference to jsdelivr over unpkg', async () => { + updateCDNSettings('jsdelivr.com', 'unpkg.com'); + when(httpClient.exists(anything())).thenResolve(true); + + const value = await scriptSourceProvider.getWidgetScriptSource(moduleName, moduleVersion); + + assert.deepEqual(value, { + moduleName: 'HelloWorld', + scriptUri: `${jsdelivrUrl}${moduleName}@${moduleVersion}/dist/index`, + source: 'cdn' + }); + verify(httpClient.exists(anything())).once(); + }); + test('Give preference to unpkg over jsdelivr', async () => { + updateCDNSettings('unpkg.com', 'jsdelivr.com'); + when(httpClient.exists(anything())).thenResolve(true); + + const value = await scriptSourceProvider.getWidgetScriptSource(moduleName, moduleVersion); + + assert.deepEqual(value, { + moduleName: 'HelloWorld', + scriptUri: `${unpgkUrl}${moduleName}@${moduleVersion}/dist/index`, + source: 'cdn' + }); + verify(httpClient.exists(anything())).once(); + }); + test('Get Script from unpk if jsdelivr fails', async () => { + updateCDNSettings('jsdelivr.com', 'unpkg.com'); + when(httpClient.exists(anything())).thenCall( + async (url: string) => !url.startsWith(jsdelivrUrl) + ); + + const value = await scriptSourceProvider.getWidgetScriptSource(moduleName, moduleVersion); + + assert.deepEqual(value, { + moduleName: 'HelloWorld', + scriptUri: `${unpgkUrl}${moduleName}@${moduleVersion}/dist/index`, + source: 'cdn' + }); + verify(httpClient.exists(anything())).twice(); + }); + test('Get Script from jsdelivr if unpkg fails', async () => { + updateCDNSettings('unpkg.com', 'jsdelivr.com'); + when(httpClient.exists(anything())).thenCall(async (url: string) => !url.startsWith(unpgkUrl)); + + const value = await scriptSourceProvider.getWidgetScriptSource(moduleName, moduleVersion); + + assert.deepEqual(value, { + moduleName: 'HelloWorld', + scriptUri: `${jsdelivrUrl}${moduleName}@${moduleVersion}/dist/index`, + source: 'cdn' + }); + verify(httpClient.exists(anything())).twice(); + }); + test('No script source if downloading from both CDNs fail', async () => { + updateCDNSettings('unpkg.com', 'jsdelivr.com'); + when(httpClient.exists(anything())).thenResolve(false); + + const value = await scriptSourceProvider.getWidgetScriptSource(moduleName, moduleVersion); + + assert.deepEqual(value, { moduleName: 'HelloWorld' }); + verify(httpClient.exists(anything())).twice(); + }); + }); + }); + }); + }); +}); diff --git a/src/test/datascience/ipywidgets/ipyWidgetScriptSourceProvider.unit.test.ts b/src/test/datascience/ipywidgets/ipyWidgetScriptSourceProvider.unit.test.ts new file mode 100644 index 000000000000..5f404389ab7a --- /dev/null +++ b/src/test/datascience/ipywidgets/ipyWidgetScriptSourceProvider.unit.test.ts @@ -0,0 +1,275 @@ +// Copyright (c) Microsoft Corporation. All rights reserved. +// Licensed under the MIT License. +import { assert } from 'chai'; +import * as sinon from 'sinon'; +import { anything, deepEqual, instance, mock, verify, when } from 'ts-mockito'; +import { ConfigurationChangeEvent, ConfigurationTarget, EventEmitter } from 'vscode'; +import { ApplicationShell } from '../../../client/common/application/applicationShell'; +import { IApplicationShell, IWorkspaceService } from '../../../client/common/application/types'; +import { WorkspaceService } from '../../../client/common/application/workspace'; +import { ConfigurationService } from '../../../client/common/configuration/service'; +import { HttpClient } from '../../../client/common/net/httpClient'; +import { PersistentState, PersistentStateFactory } from '../../../client/common/persistentState'; +import { FileSystem } from '../../../client/common/platform/fileSystem'; +import { IConfigurationService, IPythonSettings } from '../../../client/common/types'; +import { Common, DataScience } from '../../../client/common/utils/localize'; +import { noop } from '../../../client/common/utils/misc'; +import { CDNWidgetScriptSourceProvider } from '../../../client/datascience/ipywidgets/cdnWidgetScriptSourceProvider'; +import { IPyWidgetScriptSourceProvider } from '../../../client/datascience/ipywidgets/ipyWidgetScriptSourceProvider'; +import { LocalWidgetScriptSourceProvider } from '../../../client/datascience/ipywidgets/localWidgetScriptSourceProvider'; +import { RemoteWidgetScriptSourceProvider } from '../../../client/datascience/ipywidgets/remoteWidgetScriptSourceProvider'; +import { JupyterNotebookBase } from '../../../client/datascience/jupyter/jupyterNotebook'; +import { IConnection, ILocalResourceUriConverter, INotebook } from '../../../client/datascience/types'; +import { InterpreterService } from '../../../client/interpreter/interpreterService'; + +// tslint:disable: no-any no-invalid-this + +suite('xxxData Science - ipywidget - Widget Script Source Provider', () => { + let scriptSourceProvider: IPyWidgetScriptSourceProvider; + let notebook: INotebook; + let configService: IConfigurationService; + let settings: IPythonSettings; + let appShell: IApplicationShell; + let workspaceService: IWorkspaceService; + let onDidChangeWorkspaceSettings: EventEmitter; + let userSelectedOkOrDoNotShowAgainInPrompt: PersistentState; + setup(() => { + notebook = mock(JupyterNotebookBase); + configService = mock(ConfigurationService); + appShell = mock(ApplicationShell); + workspaceService = mock(WorkspaceService); + onDidChangeWorkspaceSettings = new EventEmitter(); + when(workspaceService.onDidChangeConfiguration).thenReturn(onDidChangeWorkspaceSettings.event); + const httpClient = mock(HttpClient); + const resourceConverter = mock(); + const fs = mock(FileSystem); + const interpreterService = mock(InterpreterService); + const stateFactory = mock(PersistentStateFactory); + userSelectedOkOrDoNotShowAgainInPrompt = mock>(); + + when(stateFactory.createGlobalPersistentState(anything(), anything())).thenReturn( + instance(userSelectedOkOrDoNotShowAgainInPrompt) + ); + settings = { datascience: { widgetScriptSources: [] } } as any; + when(configService.getSettings(anything())).thenReturn(settings as any); + when(userSelectedOkOrDoNotShowAgainInPrompt.value).thenReturn(false); + when(userSelectedOkOrDoNotShowAgainInPrompt.updateValue(anything())).thenResolve(); + CDNWidgetScriptSourceProvider.validUrls = new Map(); + scriptSourceProvider = new IPyWidgetScriptSourceProvider( + instance(notebook), + instance(resourceConverter), + instance(fs), + instance(interpreterService), + instance(appShell), + instance(configService), + instance(workspaceService), + instance(stateFactory), + instance(httpClient) + ); + }); + teardown(() => sinon.restore()); + + [true, false].forEach((localLaunch) => { + suite(localLaunch ? 'Local Jupyter Server' : 'Remote Jupyter Server', () => { + setup(() => { + const connection: IConnection = { + baseUrl: '', + localProcExitCode: undefined, + disconnected: new EventEmitter().event, + dispose: noop, + hostName: '', + localLaunch, + token: '' + }; + when(notebook.connection).thenReturn(connection); + }); + test('Prompt to use CDN', async () => { + when(appShell.showInformationMessage(anything(), anything(), anything(), anything())).thenResolve(); + + await scriptSourceProvider.getWidgetScriptSource('HelloWorld', '1'); + + verify( + appShell.showInformationMessage( + DataScience.useCDNForWidgets(), + Common.ok(), + Common.cancel(), + Common.doNotShowAgain() + ) + ).once(); + }); + test('Do not prompt to use CDN if user has chosen not to use a CDN', async () => { + when(appShell.showInformationMessage(anything(), anything(), anything(), anything())).thenResolve(); + when(userSelectedOkOrDoNotShowAgainInPrompt.value).thenReturn(true); + + await scriptSourceProvider.getWidgetScriptSource('HelloWorld', '1'); + + verify( + appShell.showInformationMessage( + DataScience.useCDNForWidgets(), + Common.ok(), + Common.cancel(), + Common.doNotShowAgain() + ) + ).never(); + }); + function verifyNoCDNUpdatedInSettings() { + // Confirm message was displayed. + verify( + appShell.showInformationMessage( + DataScience.useCDNForWidgets(), + Common.ok(), + Common.cancel(), + Common.doNotShowAgain() + ) + ).once(); + + // Confirm settings were updated. + verify( + configService.updateSetting( + 'dataScience.widgetScriptSources', + deepEqual([]), + undefined, + ConfigurationTarget.Global + ) + ).once(); + } + test('Do not update if prompt is dismissed', async () => { + when(appShell.showInformationMessage(anything(), anything(), anything(), anything())).thenResolve(); + + await scriptSourceProvider.getWidgetScriptSource('HelloWorld', '1'); + + verify(configService.updateSetting(anything(), anything(), anything(), anything())).never(); + verify(userSelectedOkOrDoNotShowAgainInPrompt.updateValue(true)).never(); + }); + test('Do not update settings if Cancel is clicked in prompt', async () => { + when(appShell.showInformationMessage(anything(), anything(), anything(), anything())).thenResolve( + Common.cancel() as any + ); + + await scriptSourceProvider.getWidgetScriptSource('HelloWorld', '1'); + + verify(configService.updateSetting(anything(), anything(), anything(), anything())).never(); + verify(userSelectedOkOrDoNotShowAgainInPrompt.updateValue(true)).never(); + }); + test('Update settings to not use CDN if `Do Not Show Again` is clicked in prompt', async () => { + when(appShell.showInformationMessage(anything(), anything(), anything(), anything())).thenResolve( + Common.doNotShowAgain() as any + ); + + await scriptSourceProvider.getWidgetScriptSource('HelloWorld', '1'); + + verifyNoCDNUpdatedInSettings(); + verify(userSelectedOkOrDoNotShowAgainInPrompt.updateValue(true)).once(); + }); + test('Update settings to use CDN based on prompt', async () => { + when(appShell.showInformationMessage(anything(), anything(), anything(), anything())).thenResolve( + Common.ok() as any + ); + + await scriptSourceProvider.getWidgetScriptSource('HelloWorld', '1'); + + // Confirm message was displayed. + verify( + appShell.showInformationMessage( + DataScience.useCDNForWidgets(), + Common.ok(), + Common.cancel(), + Common.doNotShowAgain() + ) + ).once(); + // Confirm settings were updated. + verify(userSelectedOkOrDoNotShowAgainInPrompt.updateValue(true)).once(); + verify( + configService.updateSetting( + 'dataScience.widgetScriptSources', + deepEqual(['jsdelivr.com', 'unpkg.com']), + undefined, + ConfigurationTarget.Global + ) + ).once(); + }); + test('Attempt to get widget source from all providers', async () => { + settings.datascience.widgetScriptSources = ['jsdelivr.com', 'unpkg.com']; + const localOrRemoteSource = localLaunch + ? sinon.stub(LocalWidgetScriptSourceProvider.prototype, 'getWidgetScriptSource') + : sinon.stub(RemoteWidgetScriptSourceProvider.prototype, 'getWidgetScriptSource'); + const cdnSource = sinon.stub(CDNWidgetScriptSourceProvider.prototype, 'getWidgetScriptSource'); + + localOrRemoteSource.resolves({ moduleName: 'HelloWorld' }); + cdnSource.resolves({ moduleName: 'HelloWorld' }); + + scriptSourceProvider.initialize(); + const value = await scriptSourceProvider.getWidgetScriptSource('HelloWorld', '1'); + + assert.deepEqual(value, { moduleName: 'HelloWorld' }); + assert.isTrue(localOrRemoteSource.calledOnce); + assert.isTrue(cdnSource.calledOnce); + }); + test('Widget sources should respect changes to configuration settings', async () => { + // 1. Search CDN then local/remote juptyer. + settings.datascience.widgetScriptSources = ['jsdelivr.com', 'unpkg.com']; + const localOrRemoteSource = localLaunch + ? sinon.stub(LocalWidgetScriptSourceProvider.prototype, 'getWidgetScriptSource') + : sinon.stub(RemoteWidgetScriptSourceProvider.prototype, 'getWidgetScriptSource'); + const cdnSource = sinon.stub(CDNWidgetScriptSourceProvider.prototype, 'getWidgetScriptSource'); + cdnSource.resolves({ moduleName: 'moduleCDN', scriptUri: '1', source: 'cdn' }); + + scriptSourceProvider.initialize(); + let values = await scriptSourceProvider.getWidgetScriptSource('ModuleName', '`'); + + assert.deepEqual(values, { moduleName: 'moduleCDN', scriptUri: '1', source: 'cdn' }); + assert.isFalse(localOrRemoteSource.calledOnce); + assert.isTrue(cdnSource.calledOnce); + + // 2. Update settings to remove the use of CDNs + localOrRemoteSource.reset(); + cdnSource.reset(); + localOrRemoteSource.resolves({ moduleName: 'moduleLocal', scriptUri: '1', source: 'local' }); + settings.datascience.widgetScriptSources = []; + onDidChangeWorkspaceSettings.fire({ affectsConfiguration: () => true }); + + values = await scriptSourceProvider.getWidgetScriptSource('ModuleName', '`'); + assert.deepEqual(values, { moduleName: 'moduleLocal', scriptUri: '1', source: 'local' }); + assert.isTrue(localOrRemoteSource.calledOnce); + assert.isFalse(cdnSource.calledOnce); + }); + test('Widget source should support fall back search', async () => { + // 1. Search CDN and if that fails then get from local/remote. + settings.datascience.widgetScriptSources = ['jsdelivr.com', 'unpkg.com']; + const localOrRemoteSource = localLaunch + ? sinon.stub(LocalWidgetScriptSourceProvider.prototype, 'getWidgetScriptSource') + : sinon.stub(RemoteWidgetScriptSourceProvider.prototype, 'getWidgetScriptSource'); + const cdnSource = sinon.stub(CDNWidgetScriptSourceProvider.prototype, 'getWidgetScriptSource'); + localOrRemoteSource.resolves({ moduleName: 'moduleLocal', scriptUri: '1', source: 'local' }); + cdnSource.resolves({ moduleName: 'moduleCDN' }); + + scriptSourceProvider.initialize(); + const value = await scriptSourceProvider.getWidgetScriptSource('', ''); + + // 1. Confirm CDN was first searched, then local/remote + assert.deepEqual(value, { moduleName: 'moduleLocal', scriptUri: '1', source: 'local' }); + assert.isTrue(localOrRemoteSource.calledOnce); + assert.isTrue(cdnSource.calledOnce); + // Confirm we first searched CDN before going to local/remote. + cdnSource.calledBefore(localOrRemoteSource); + }); + test('Widget sources from CDN should be given prefernce', async () => { + settings.datascience.widgetScriptSources = ['jsdelivr.com', 'unpkg.com']; + const localOrRemoteSource = localLaunch + ? sinon.stub(LocalWidgetScriptSourceProvider.prototype, 'getWidgetScriptSource') + : sinon.stub(RemoteWidgetScriptSourceProvider.prototype, 'getWidgetScriptSource'); + const cdnSource = sinon.stub(CDNWidgetScriptSourceProvider.prototype, 'getWidgetScriptSource'); + + localOrRemoteSource.resolves({ moduleName: 'module1' }); + cdnSource.resolves({ moduleName: 'module1', scriptUri: '1', source: 'cdn' }); + + scriptSourceProvider.initialize(); + const values = await scriptSourceProvider.getWidgetScriptSource('ModuleName', '1'); + + assert.deepEqual(values, { moduleName: 'module1', scriptUri: '1', source: 'cdn' }); + assert.isFalse(localOrRemoteSource.calledOnce); + assert.isTrue(cdnSource.calledOnce); + }); + }); + }); +}); diff --git a/src/test/datascience/ipywidgets/localWidgetScriptSourceProvider.unit.test.ts b/src/test/datascience/ipywidgets/localWidgetScriptSourceProvider.unit.test.ts new file mode 100644 index 000000000000..26a79e9bb67a --- /dev/null +++ b/src/test/datascience/ipywidgets/localWidgetScriptSourceProvider.unit.test.ts @@ -0,0 +1,186 @@ +// Copyright (c) Microsoft Corporation. All rights reserved. +// Licensed under the MIT License. +import { assert } from 'chai'; +import * as path from 'path'; +import { anything, instance, mock, verify, when } from 'ts-mockito'; +import { Uri } from 'vscode'; +import { FileSystem } from '../../../client/common/platform/fileSystem'; +import { IFileSystem } from '../../../client/common/platform/types'; +import { LocalWidgetScriptSourceProvider } from '../../../client/datascience/ipywidgets/localWidgetScriptSourceProvider'; +import { IWidgetScriptSourceProvider } from '../../../client/datascience/ipywidgets/types'; +import { JupyterNotebookBase } from '../../../client/datascience/jupyter/jupyterNotebook'; +import { ILocalResourceUriConverter, INotebook } from '../../../client/datascience/types'; +import { IInterpreterService } from '../../../client/interpreter/contracts'; +import { InterpreterService } from '../../../client/interpreter/interpreterService'; + +// tslint:disable: max-func-body-length no-any +suite('Data Science - ipywidget - Local Widget Script Source', () => { + let scriptSourceProvider: IWidgetScriptSourceProvider; + let notebook: INotebook; + let resourceConverter: ILocalResourceUriConverter; + let fs: IFileSystem; + let interpreterService: IInterpreterService; + const filesToLookSerachFor = `*${path.sep}index.js`; + function asVSCodeUri(uri: Uri) { + return `vscodeUri://${uri.fsPath}`; + } + setup(() => { + notebook = mock(JupyterNotebookBase); + resourceConverter = mock(); + fs = mock(FileSystem); + interpreterService = mock(InterpreterService); + when(resourceConverter.asWebviewUri(anything())).thenCall((uri) => Promise.resolve(asVSCodeUri(uri))); + scriptSourceProvider = new LocalWidgetScriptSourceProvider( + instance(notebook), + instance(resourceConverter), + instance(fs), + instance(interpreterService) + ); + }); + test('No script source when there is no kernel associated with notebook', async () => { + when(notebook.getKernelSpec()).thenReturn(); + + const value = await scriptSourceProvider.getWidgetScriptSource('ModuleName', '1'); + + assert.deepEqual(value, { moduleName: 'ModuleName' }); + }); + test('No script source when there are no widgets', async () => { + when(notebook.getKernelSpec()).thenReturn({ + metadata: { interpreter: { sysPrefix: 'sysPrefix', path: 'pythonPath' } } + } as any); + when(fs.search(anything(), anything())).thenResolve([]); + + const value = await scriptSourceProvider.getWidgetScriptSource('ModuleName', '1'); + + assert.deepEqual(value, { moduleName: 'ModuleName' }); + + // Ensure we searched the directories. + verify(fs.search(anything(), anything())).once(); + }); + test('Look for widgets in sysPath of interpreter defined in kernel metadata', async () => { + const sysPrefix = 'sysPrefix Of Python in Metadata'; + const searchDirectory = path.join(sysPrefix, 'share', 'jupyter', 'nbextensions'); + + when(notebook.getKernelSpec()).thenReturn({ + metadata: { interpreter: { sysPrefix, path: 'pythonPath' } } + } as any); + when(fs.search(anything(), anything())).thenResolve([]); + + const value = await scriptSourceProvider.getWidgetScriptSource('ModuleName', '1'); + + assert.deepEqual(value, { moduleName: 'ModuleName' }); + + // Ensure we look for the right things in the right place. + verify(fs.search(filesToLookSerachFor, searchDirectory)).once(); + }); + test('Look for widgets in sysPath of kernel', async () => { + const sysPrefix = 'sysPrefix Of Kernel'; + const kernelPath = 'kernel Path.exe'; + when(interpreterService.getInterpreterDetails(kernelPath)).thenResolve({ sysPrefix } as any); + const searchDirectory = path.join(sysPrefix, 'share', 'jupyter', 'nbextensions'); + + when(notebook.getKernelSpec()).thenReturn({ path: kernelPath } as any); + when(fs.search(anything(), anything())).thenResolve([]); + + const value = await scriptSourceProvider.getWidgetScriptSource('ModuleName', '1'); + + assert.deepEqual(value, { moduleName: 'ModuleName' }); + + // Ensure we look for the right things in the right place. + verify(fs.search(filesToLookSerachFor, searchDirectory)).once(); + }); + test('Ensure we cache the list of widgets source (when nothing is found)', async () => { + when(notebook.getKernelSpec()).thenReturn({ + metadata: { interpreter: { sysPrefix: 'sysPrefix', path: 'pythonPath' } } + } as any); + when(fs.search(anything(), anything())).thenResolve([]); + + const value = await scriptSourceProvider.getWidgetScriptSource('ModuleName', '1'); + assert.deepEqual(value, { moduleName: 'ModuleName' }); + const value1 = await scriptSourceProvider.getWidgetScriptSource('ModuleName', '1'); + assert.deepEqual(value1, { moduleName: 'ModuleName' }); + const value2 = await scriptSourceProvider.getWidgetScriptSource('ModuleName', '1'); + assert.deepEqual(value2, { moduleName: 'ModuleName' }); + + // Ensure we search directories once. + verify(fs.search(anything(), anything())).once(); + }); + test('Ensure we search directory only once (cache results)', async () => { + const sysPrefix = 'sysPrefix Of Python in Metadata'; + const searchDirectory = path.join(sysPrefix, 'share', 'jupyter', 'nbextensions'); + when(notebook.getKernelSpec()).thenReturn({ + metadata: { interpreter: { sysPrefix, path: 'pythonPath' } } + } as any); + when(fs.search(anything(), anything())).thenResolve([ + 'widget1/index.js', + 'widget2/index.js', + 'widget3/index.js' + ]); + + const value = await scriptSourceProvider.getWidgetScriptSource('widget2', '1'); + assert.deepEqual(value, { + moduleName: 'widget2', + source: 'local', + scriptUri: asVSCodeUri(Uri.file(path.join(searchDirectory, 'widget2', 'index'))) + }); + const value1 = await scriptSourceProvider.getWidgetScriptSource('widget2', '1'); + assert.deepEqual(value1, value); + const value2 = await scriptSourceProvider.getWidgetScriptSource('widget2', '1'); + assert.deepEqual(value2, value); + + // Ensure we look for the right things in the right place. + verify(fs.search(filesToLookSerachFor, searchDirectory)).once(); + }); + test('Get source for a specific widget & search in the right place', async () => { + const sysPrefix = 'sysPrefix Of Python in Metadata'; + const searchDirectory = path.join(sysPrefix, 'share', 'jupyter', 'nbextensions'); + when(notebook.getKernelSpec()).thenReturn({ + metadata: { interpreter: { sysPrefix, path: 'pythonPath' } } + } as any); + when(fs.search(anything(), anything())).thenResolve([ + 'widget1/index.js', + 'widget2/index.js', + 'widget3/index.js' + ]); + + const value = await scriptSourceProvider.getWidgetScriptSource('widget1', '1'); + + // Ensure the script paths are properly converted to be used within notebooks. + assert.deepEqual(value, { + moduleName: 'widget1', + source: 'local', + scriptUri: asVSCodeUri(Uri.file(path.join(searchDirectory, 'widget1', 'index'))) + }); + + // Ensure we look for the right things in the right place. + verify(fs.search(filesToLookSerachFor, searchDirectory)).once(); + }); + test('Return empty source for widgets that cannot be found', async () => { + const sysPrefix = 'sysPrefix Of Python in Metadata'; + const searchDirectory = path.join(sysPrefix, 'share', 'jupyter', 'nbextensions'); + when(notebook.getKernelSpec()).thenReturn({ + metadata: { interpreter: { sysPrefix, path: 'pythonPath' } } + } as any); + when(fs.search(anything(), anything())).thenResolve([ + 'widget1/index.js', + 'widget2/index.js', + 'widget3/index.js' + ]); + + const value = await scriptSourceProvider.getWidgetScriptSource('widgetNotFound', '1'); + assert.deepEqual(value, { + moduleName: 'widgetNotFound' + }); + const value1 = await scriptSourceProvider.getWidgetScriptSource('widgetNotFound', '1'); + assert.isOk(value1); + const value2 = await scriptSourceProvider.getWidgetScriptSource('widgetNotFound', '1'); + assert.deepEqual(value2, value1); + // We should ignore version numbers (when getting widget sources from local fs). + const value3 = await scriptSourceProvider.getWidgetScriptSource('widgetNotFound', '1234'); + assert.deepEqual(value3, value1); + + // Ensure we look for the right things in the right place. + // Also ensure we call once (& cache for subsequent searches). + verify(fs.search(filesToLookSerachFor, searchDirectory)).once(); + }); +}); diff --git a/src/test/datascience/jupyter/serverCache.unit.test.ts b/src/test/datascience/jupyter/serverCache.unit.test.ts index cd98f0cb6a49..868d9fc47fa6 100644 --- a/src/test/datascience/jupyter/serverCache.unit.test.ts +++ b/src/test/datascience/jupyter/serverCache.unit.test.ts @@ -50,7 +50,8 @@ suite('Data Science - ServerCache', () => { runStartupCommands: '', debugJustMyCode: true, variableQueries: [], - jupyterCommandLineArguments: [] + jupyterCommandLineArguments: [], + widgetScriptSources: [] }; when(configService.getSettings(anything())).thenReturn(pythonSettings); serverCache = new ServerCache(instance(configService), instance(workspaceService), instance(fileSystem)); diff --git a/src/test/datascience/jupyterVariables.unit.test.ts b/src/test/datascience/jupyterVariables.unit.test.ts index 51a0abba7e9b..aa2daa69c9d5 100644 --- a/src/test/datascience/jupyterVariables.unit.test.ts +++ b/src/test/datascience/jupyterVariables.unit.test.ts @@ -97,7 +97,8 @@ suite('JupyterVariables', () => { runStartupCommands: '', debugJustMyCode: true, variableQueries: [], - jupyterCommandLineArguments: [] + jupyterCommandLineArguments: [], + widgetScriptSources: [] }; // Create our fake notebook @@ -235,9 +236,9 @@ suite('JupyterVariables', () => { Promise.resolve({ 'text/plain': `\u001b[1;31mType:\u001b[0m complex \u001b[1;31mString form:\u001b[0m (1+1j) -\u001b[1;31mDocstring:\u001b[0m +\u001b[1;31mDocstring:\u001b[0m Create a complex number from a real part and an optional imaginary part. - + This is equivalent to (real + imag*1j) where imag defaults to 0. "` }) @@ -286,9 +287,9 @@ This is equivalent to (real + imag*1j) where imag defaults to 0. Promise.resolve({ 'text/plain': `\u001b[1;31mType:\u001b[0m complex \u001b[1;31mString form:\u001b[0m (1+1j) -\u001b[1;31mDocstring:\u001b[0m +\u001b[1;31mDocstring:\u001b[0m Create a complex number from a real part and an optional imaginary part. - + This is equivalent to (real + imag*1j) where imag defaults to 0. "` }) diff --git a/src/test/datascience/mockJupyterNotebook.ts b/src/test/datascience/mockJupyterNotebook.ts index 319805cfdd3d..75057536b218 100644 --- a/src/test/datascience/mockJupyterNotebook.ts +++ b/src/test/datascience/mockJupyterNotebook.ts @@ -11,6 +11,7 @@ import { LiveKernelModel } from '../../client/datascience/jupyter/kernels/types' import { ICell, ICellHashProvider, + IConnection, IGatherProvider, IJupyterKernelSpec, INotebook, @@ -28,7 +29,9 @@ export class MockJupyterNotebook implements INotebook { public get server(): INotebookServer { return this.owner; } - + public get connection(): IConnection { + throw new Error('Not implemented'); + } public get identity(): Uri { return Uri.parse(Identifiers.InteractiveWindowIdentity); } diff --git a/src/test/datascience/uiTests/webBrowserPanel.ts b/src/test/datascience/uiTests/webBrowserPanel.ts index 3e681e09b70d..e9d04cca44ef 100644 --- a/src/test/datascience/uiTests/webBrowserPanel.ts +++ b/src/test/datascience/uiTests/webBrowserPanel.ts @@ -155,6 +155,9 @@ export class WebBrowserPanel implements IWebPanel, IDisposable { console.error('Failed to start Web Browser Panel', ex) ); } + public asWebviewUri(localResource: Uri): Uri { + return localResource; + } public setTitle(newTitle: string): void { if (this.panel) { this.panel.title = newTitle;