From 32dfda222c836199a48405b8a91f6530f6062d44 Mon Sep 17 00:00:00 2001 From: Don Jayamanne Date: Fri, 10 Apr 2020 11:12:54 -0700 Subject: [PATCH 1/2] Copy widget scripts to extension folder --- package-lock.json | 24 +++++- package.json | 2 +- .../common/application/webPanels/webPanel.ts | 5 +- .../application/webPanels/webPanelProvider.ts | 23 ++++-- .../ipywidgets/ipyWidgetScriptSource.ts | 77 ++++++++++++++++--- .../localWidgetScriptSourceProvider.ts | 3 +- .../remoteWidgetScriptSourceProvider.ts | 4 +- .../ipywidgets/requirejsRegistry.ts | 6 +- src/ipywidgets/src/manager.ts | 7 +- 9 files changed, 116 insertions(+), 35 deletions(-) diff --git a/package-lock.json b/package-lock.json index fb531298d20c..cb55c3141e08 100644 --- a/package-lock.json +++ b/package-lock.json @@ -5690,6 +5690,16 @@ "integrity": "sha512-Un7MIEDdUC5gNpcGDV97op1Ywk748MpHcFTHoYs6qnj1Z3j7I53VG3nwZhKzoBZmbdRNnb6WRdFlwl7tSDuZGw==", "dev": true }, + "bindings": { + "version": "1.5.0", + "resolved": "https://registry.npmjs.org/bindings/-/bindings-1.5.0.tgz", + "integrity": "sha512-p2q/t/mhvuOj/UeLlV6566GD/guowlr0hHxClI0W9m7MWYkL1F0hLo+0Aexs9HSPCtR1SXQ0TD3MMKrXZajbiQ==", + "dev": true, + "optional": true, + "requires": { + "file-uri-to-path": "1.0.0" + } + }, "bintrees": { "version": "1.0.2", "resolved": "https://registry.npmjs.org/bintrees/-/bintrees-1.0.2.tgz", @@ -7350,6 +7360,7 @@ "dev": true, "optional": true, "requires": { + "bindings": "^1.5.0", "nan": "^2.12.1", "node-pre-gyp": "*" }, @@ -10700,6 +10711,13 @@ "integrity": "sha512-qyQ0pzAy78gVoJsmYeNgl8uH8yKhr1lVhW7JbzJmnlRi0I4R2eEDEJZVKG8agpDnLpacwNbDhLNG/LMdxHD2YQ==", "dev": true }, + "file-uri-to-path": { + "version": "1.0.0", + "resolved": "https://registry.npmjs.org/file-uri-to-path/-/file-uri-to-path-1.0.0.tgz", + "integrity": "sha512-0Zt+s3L7Vf1biwWZ29aARiVYLx7iMGnEUl9x33fbB/j3jR81u/O2LbqK+Bm1CDSNDKVtJ/YjwY7TUd5SkeLQLw==", + "dev": true, + "optional": true + }, "filemanager-webpack-plugin": { "version": "2.0.5", "resolved": "https://registry.npmjs.org/filemanager-webpack-plugin/-/filemanager-webpack-plugin-2.0.5.tgz", @@ -11267,6 +11285,7 @@ "dev": true, "optional": true, "requires": { + "bindings": "^1.5.0", "nan": "^2.12.1", "node-pre-gyp": "*" }, @@ -21074,7 +21093,6 @@ "version": "1.6.3", "resolved": "https://registry.npmjs.org/sanitize-filename/-/sanitize-filename-1.6.3.tgz", "integrity": "sha512-y/52Mcy7aw3gRm7IrcGDFx/bCk4AhRh2eI9luHOQM86nZsqwiRkkq2GekHXBBD+SmPidc8i2PqtYZl+pWJ8Oeg==", - "dev": true, "requires": { "truncate-utf8-bytes": "^1.0.0" } @@ -23587,7 +23605,6 @@ "version": "1.0.2", "resolved": "https://registry.npmjs.org/truncate-utf8-bytes/-/truncate-utf8-bytes-1.0.2.tgz", "integrity": "sha1-QFkjkJWS1W94pYGENLC3hInKXys=", - "dev": true, "requires": { "utf8-byte-length": "^1.0.1" } @@ -24416,8 +24433,7 @@ "utf8-byte-length": { "version": "1.0.4", "resolved": "https://registry.npmjs.org/utf8-byte-length/-/utf8-byte-length-1.0.4.tgz", - "integrity": "sha1-9F8VDExm7uloGGUFq5P8u4rWv2E=", - "dev": true + "integrity": "sha1-9F8VDExm7uloGGUFq5P8u4rWv2E=" }, "util": { "version": "0.11.1", diff --git a/package.json b/package.json index 27f962d26e18..b06d951641ae 100644 --- a/package.json +++ b/package.json @@ -3007,6 +3007,7 @@ "request": "^2.87.0", "request-progress": "^3.0.0", "rxjs": "^5.5.9", + "sanitize-filename": "^1.6.3", "semver": "^5.5.0", "stack-trace": "0.0.10", "string-argv": "^0.3.1", @@ -3207,7 +3208,6 @@ "requirejs": "^2.3.6", "rewiremock": "^3.13.0", "rimraf": "^3.0.2", - "sanitize-filename": "^1.6.3", "sass-loader": "^7.1.0", "serialize-javascript": "^2.1.2", "shortid": "^2.2.8", diff --git a/src/client/common/application/webPanels/webPanel.ts b/src/client/common/application/webPanels/webPanel.ts index 40719293572f..52c58ab60889 100644 --- a/src/client/common/application/webPanels/webPanel.ts +++ b/src/client/common/application/webPanels/webPanel.ts @@ -28,11 +28,12 @@ export class WebPanel implements IWebPanel { private disposableRegistry: IDisposableRegistry, private port: number | undefined, private token: string | undefined, - private options: IWebPanelOptions + private options: IWebPanelOptions, + additionalRootPaths: Uri[] = [] ) { const webViewOptions: WebviewOptions = { enableScripts: true, - localResourceRoots: [Uri.file(this.options.rootPath), Uri.file(this.options.cwd)], + localResourceRoots: [Uri.file(this.options.rootPath), Uri.file(this.options.cwd), ...additionalRootPaths], portMapping: port ? [{ webviewPort: RemappedPort, extensionHostPort: port }] : undefined }; if (options.webViewPanel) { diff --git a/src/client/common/application/webPanels/webPanelProvider.ts b/src/client/common/application/webPanels/webPanelProvider.ts index bcdd698e729b..ac57b7369965 100644 --- a/src/client/common/application/webPanels/webPanelProvider.ts +++ b/src/client/common/application/webPanels/webPanelProvider.ts @@ -2,11 +2,12 @@ // Licensed under the MIT License. 'use strict'; import { inject, injectable } from 'inversify'; +import * as path from 'path'; import * as portfinder from 'portfinder'; import * as uuid from 'uuid/v4'; - +import { Uri } from 'vscode'; import { IFileSystem } from '../../platform/types'; -import { IDisposableRegistry } from '../../types'; +import { IDisposableRegistry, IExtensionContext } from '../../types'; import { IWebPanel, IWebPanelOptions, IWebPanelProvider } from '../types'; import { WebPanel } from './webPanel'; @@ -16,8 +17,9 @@ export class WebPanelProvider implements IWebPanelProvider { private token: string | undefined; constructor( - @inject(IDisposableRegistry) private disposableRegistry: IDisposableRegistry, - @inject(IFileSystem) private fs: IFileSystem + @inject(IDisposableRegistry) private readonly disposableRegistry: IDisposableRegistry, + @inject(IFileSystem) private readonly fs: IFileSystem, + @inject(IExtensionContext) private readonly context: IExtensionContext ) {} // tslint:disable-next-line:no-any @@ -25,8 +27,17 @@ export class WebPanelProvider implements IWebPanelProvider { const serverData = options.startHttpServer ? await this.ensureServerIsRunning() : { port: undefined, token: undefined }; - - return new WebPanel(this.fs, this.disposableRegistry, serverData.port, serverData.token, options); + // Allow loading resources from the `/tmp` folder when in webiviews. + // Used by widgets to place files that are not otherwise accessible. + const additionalRootPaths = [Uri.file(path.join(this.context.extensionPath, 'tmp'))]; + return new WebPanel( + this.fs, + this.disposableRegistry, + serverData.port, + serverData.token, + options, + additionalRootPaths + ); } private async ensureServerIsRunning(): Promise<{ port: number; token: string }> { diff --git a/src/client/datascience/ipywidgets/ipyWidgetScriptSource.ts b/src/client/datascience/ipywidgets/ipyWidgetScriptSource.ts index 78866c2291bc..ae4d244a355e 100644 --- a/src/client/datascience/ipywidgets/ipyWidgetScriptSource.ts +++ b/src/client/datascience/ipywidgets/ipyWidgetScriptSource.ts @@ -4,14 +4,22 @@ 'use strict'; import type * as jupyterlabService from '@jupyterlab/services'; import type * as serialize from '@jupyterlab/services/lib/kernel/serialize'; +import { sha256 } from 'hash.js'; import { inject, injectable } from 'inversify'; import { IDisposable } from 'monaco-editor'; +import * as path from 'path'; import { Event, EventEmitter, Uri } from 'vscode'; import type { Data as WebSocketData } from 'ws'; import { IApplicationShell, IWorkspaceService } from '../../common/application/types'; -import { traceError } from '../../common/logger'; +import { traceError, traceInfo } from '../../common/logger'; import { IFileSystem } from '../../common/platform/types'; -import { IConfigurationService, IDisposableRegistry, IHttpClient, IPersistentStateFactory } from '../../common/types'; +import { + IConfigurationService, + IDisposableRegistry, + IExtensionContext, + IHttpClient, + IPersistentStateFactory +} from '../../common/types'; import { createDeferred, Deferred } from '../../common/utils/async'; import { IInterpreterService, PythonInterpreter } from '../../interpreter/contracts'; import { sendTelemetryEvent } from '../../telemetry'; @@ -30,6 +38,8 @@ import { } from '../types'; import { IPyWidgetScriptSourceProvider } from './ipyWidgetScriptSourceProvider'; import { WidgetScriptSource } from './types'; +// tslint:disable: no-var-requires no-require-imports +const sanitize = require('sanitize-filename'); @injectable() export class IPyWidgetScriptSource implements IInteractiveWindowListener, ILocalResourceUriConverter { @@ -41,6 +51,14 @@ export class IPyWidgetScriptSource implements IInteractiveWindowListener, ILocal public get postInternalMessage(): Event<{ message: string; payload: any }> { return this.postInternalMessageEmitter.event; } + private get deserialize(): typeof serialize.deserialize { + if (!this.jupyterSerialize) { + // tslint:disable-next-line: no-require-imports + this.jupyterSerialize = require('@jupyterlab/services/lib/kernel/serialize') as typeof serialize; + } + return this.jupyterSerialize.deserialize; + } + private readonly resourcesMappedToExtensionFolder = new Map>(); private notebookIdentity?: Uri; private postEmitter = new EventEmitter<{ message: string; @@ -64,14 +82,9 @@ export class IPyWidgetScriptSource implements IInteractiveWindowListener, ILocal */ private pendingModuleRequests = new Map(); private jupyterSerialize?: typeof serialize; - private get deserialize(): typeof serialize.deserialize { - if (!this.jupyterSerialize) { - // tslint:disable-next-line: no-require-imports - this.jupyterSerialize = require('@jupyterlab/services/lib/kernel/serialize') as typeof serialize; - } - return this.jupyterSerialize.deserialize; - } private readonly uriConversionPromises = new Map>(); + private readonly targetWidgetScriptsFolder: string; + private readonly createTargetWidgetScriptsFolder: Promise; constructor( @inject(IDisposableRegistry) disposables: IDisposableRegistry, @inject(INotebookProvider) private readonly notebookProvider: INotebookProvider, @@ -81,8 +94,18 @@ export class IPyWidgetScriptSource implements IInteractiveWindowListener, ILocal @inject(IHttpClient) private readonly httpClient: IHttpClient, @inject(IApplicationShell) private readonly appShell: IApplicationShell, @inject(IWorkspaceService) private readonly workspaceService: IWorkspaceService, - @inject(IPersistentStateFactory) private readonly stateFactory: IPersistentStateFactory + @inject(IPersistentStateFactory) private readonly stateFactory: IPersistentStateFactory, + @inject(IExtensionContext) extensionContext: IExtensionContext ) { + this.targetWidgetScriptsFolder = path.join(extensionContext.extensionPath, 'tmp', 'nbextensions'); + this.createTargetWidgetScriptsFolder = this.fs + .directoryExists(this.targetWidgetScriptsFolder) + .then(async (exists) => { + if (!exists) { + await this.fs.createDirectory(this.targetWidgetScriptsFolder); + } + return this.targetWidgetScriptsFolder; + }); disposables.push(this); this.notebookProvider.onNotebookCreated( (e) => { @@ -94,7 +117,39 @@ export class IPyWidgetScriptSource implements IInteractiveWindowListener, ILocal this.disposables ); } - public asWebviewUri(localResource: Uri): Promise { + /** + * This method is called to convert a Uri to a format such that it can be used in a webview. + * WebViews only allow files that are part of extension and the same directory where notebook lives. + * To ensure widgets can find the js files, we copy the script file to a into the extensionr folder `tmp/nbextensions`. + * (storing files in `tmp/nbextensions` is relatively safe as this folder gets deleted when ever a user updates to a new version of VSC). + * Hence we need to copy for every version of the extension. + * Copying into global workspace folder would also work, but over time this folder size could grow (in an unmanaged way). + */ + public async asWebviewUri(localResource: Uri): Promise { + if (this.notebookIdentity && !this.resourcesMappedToExtensionFolder.has(localResource.fsPath)) { + const deferred = createDeferred(); + this.resourcesMappedToExtensionFolder.set(localResource.fsPath, deferred.promise); + try { + // Create a file name such that it will be unique and consistent across VSC reloads. + // Only if original file has been modified should we create a new copy of the sam file. + const fileHash: string = await this.fs.getFileHash(localResource.fsPath); + const uniqueFileName = sanitize(sha256().update(`${localResource.fsPath}${fileHash}`).digest('hex')); + const targetFolder = await this.createTargetWidgetScriptsFolder; + const mappedResource = Uri.file( + path.join(targetFolder, `${uniqueFileName}${path.basename(localResource.fsPath)}`) + ); + if (!(await this.fs.fileExists(mappedResource.fsPath))) { + await this.fs.copyFile(localResource.fsPath, mappedResource.fsPath); + } + traceInfo(`Widget Script file ${localResource.fsPath} mapped to ${mappedResource.fsPath}`); + deferred.resolve(mappedResource); + } catch (ex) { + traceError(`Failed to map widget Script file ${localResource.fsPath}`); + deferred.reject(ex); + } + } + localResource = await this.resourcesMappedToExtensionFolder.get(localResource.fsPath)!; + const key = localResource.toString(); if (!this.uriConversionPromises.has(key)) { this.uriConversionPromises.set(key, createDeferred()); diff --git a/src/client/datascience/ipywidgets/localWidgetScriptSourceProvider.ts b/src/client/datascience/ipywidgets/localWidgetScriptSourceProvider.ts index dc2cd6861500..4a4b4a1038e6 100644 --- a/src/client/datascience/ipywidgets/localWidgetScriptSourceProvider.ts +++ b/src/client/datascience/ipywidgets/localWidgetScriptSourceProvider.ts @@ -68,8 +68,7 @@ export class LocalWidgetScriptSourceProvider implements IWidgetScriptSourceProvi const parts = file.split(path.sep); const moduleName = parts[0]; - // Drop the `.js`. - const fileUri = Uri.file(path.join(nbextensionsPath, moduleName, 'index')); + const fileUri = Uri.file(path.join(nbextensionsPath, file)); const scriptUri = (await this.localResourceUriConverter.asWebviewUri(fileUri)).toString(); // tslint:disable-next-line: no-unnecessary-local-variable const widgetScriptSource: WidgetScriptSource = { moduleName, scriptUri, source: 'local' }; diff --git a/src/client/datascience/ipywidgets/remoteWidgetScriptSourceProvider.ts b/src/client/datascience/ipywidgets/remoteWidgetScriptSourceProvider.ts index 47ece792896c..33436c340129 100644 --- a/src/client/datascience/ipywidgets/remoteWidgetScriptSourceProvider.ts +++ b/src/client/datascience/ipywidgets/remoteWidgetScriptSourceProvider.ts @@ -19,8 +19,8 @@ export class RemoteWidgetScriptSourceProvider implements IWidgetScriptSourceProv // Noop. } public async getWidgetScriptSource(moduleName: string, moduleVersion: string): Promise { - const scriptUri = `${this.connection.baseUrl}nbextensions/${moduleName}/index`; - const exists = await this.getUrlForWidget(`${scriptUri}.js`); + const scriptUri = `${this.connection.baseUrl}nbextensions/${moduleName}/index.js`; + const exists = await this.getUrlForWidget(scriptUri); if (exists) { return { moduleName, scriptUri, source: 'cdn' }; } diff --git a/src/datascience-ui/ipywidgets/requirejsRegistry.ts b/src/datascience-ui/ipywidgets/requirejsRegistry.ts index 74d5b370a37a..bf5c7755ccec 100644 --- a/src/datascience-ui/ipywidgets/requirejsRegistry.ts +++ b/src/datascience-ui/ipywidgets/requirejsRegistry.ts @@ -55,8 +55,12 @@ function registerScriptsInRequireJs(scripts: NonPartial[]) { }; scripts.forEach((script) => { scriptsAlreadyRegisteredInRequireJs.set(script.moduleName, script.scriptUri); + // Drop the `.js` from the scriptUri. + const scriptUri = script.scriptUri.toLowerCase().endsWith('.js') + ? script.scriptUri.substring(0, script.scriptUri.length - 3) + : script.scriptUri; // Register the script source into requirejs so it gets loaded via requirejs. - config.paths[script.moduleName] = script.scriptUri; + config.paths[script.moduleName] = scriptUri; }); requirejs.config(config); diff --git a/src/ipywidgets/src/manager.ts b/src/ipywidgets/src/manager.ts index 6855457cda1d..8d5a8e035484 100644 --- a/src/ipywidgets/src/manager.ts +++ b/src/ipywidgets/src/manager.ts @@ -17,12 +17,7 @@ export const WIDGET_MIMETYPE = 'application/vnd.jupyter.widget-view+json'; // Source borrowed from https://github.com/jupyter-widgets/ipywidgets/blob/master/examples/web3/src/manager.ts // These widgets can always be loaded from requirejs (as it is bundled). -const widgetsRegisteredInRequireJs = [ - '@jupyter-widgets/controls', - '@jupyter-widgets/base', - '@jupyter-widgets/output', - 'azureml_widgets' -]; +const widgetsRegisteredInRequireJs = ['@jupyter-widgets/controls', '@jupyter-widgets/base', '@jupyter-widgets/output']; export class WidgetManager extends jupyterlab.WidgetManager { public kernel: Kernel.IKernelConnection; From f991ce23386df5425a626e0e846214456d7e984a Mon Sep 17 00:00:00 2001 From: Don Jayamanne Date: Fri, 10 Apr 2020 11:45:02 -0700 Subject: [PATCH 2/2] Fix test --- .../ipywidgets/localWidgetScriptSourceProvider.unit.test.ts | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/src/test/datascience/ipywidgets/localWidgetScriptSourceProvider.unit.test.ts b/src/test/datascience/ipywidgets/localWidgetScriptSourceProvider.unit.test.ts index 26a79e9bb67a..50856a6020ce 100644 --- a/src/test/datascience/ipywidgets/localWidgetScriptSourceProvider.unit.test.ts +++ b/src/test/datascience/ipywidgets/localWidgetScriptSourceProvider.unit.test.ts @@ -121,7 +121,7 @@ suite('Data Science - ipywidget - Local Widget Script Source', () => { assert.deepEqual(value, { moduleName: 'widget2', source: 'local', - scriptUri: asVSCodeUri(Uri.file(path.join(searchDirectory, 'widget2', 'index'))) + scriptUri: asVSCodeUri(Uri.file(path.join(searchDirectory, 'widget2', 'index.js'))) }); const value1 = await scriptSourceProvider.getWidgetScriptSource('widget2', '1'); assert.deepEqual(value1, value); @@ -149,7 +149,7 @@ suite('Data Science - ipywidget - Local Widget Script Source', () => { assert.deepEqual(value, { moduleName: 'widget1', source: 'local', - scriptUri: asVSCodeUri(Uri.file(path.join(searchDirectory, 'widget1', 'index'))) + scriptUri: asVSCodeUri(Uri.file(path.join(searchDirectory, 'widget1', 'index.js'))) }); // Ensure we look for the right things in the right place.