File completer (#5049)

* Initial work on wiring up completer to files.

* Fix up completer enabled state.

* Hook up tooltip.

* Also check for a console when a new text editor is created.

* Add tokeninzation to abstract editor and codemirror.

* Wire up a context completer to the completion handler.

* Refactor into different flavors of DataConnectors for completions,
including a ContextConnector and a combined ContextConnector and
KernelConnector.

* Track the running sessions for whether to use the kernel completer.

* Integrity.

* Wire up the tooltip using the session manager.

packages/codeeditor/src/editor.ts

@@ -129,6 +129,26 @@ export namespace CodeEditor {
     readonly style: ISelectionStyle;
   }
 
+  /**
+   * An interface for a text token, such as a word, keyword, or variable.
+   */
+  export interface IToken {
+    /**
+     * The value of the token.
+     */
+    value: string;
+
+    /**
+     * The offset of the token in the code editor.
+     */
+    offset: number;
+
+    /**
+     * An optional type for the token.
+     */
+    type?: string;
+  }
+
   /**
    * An interface to manage selections by selection owners.
    *
@@ -535,6 +555,16 @@ export namespace CodeEditor {
      * Inserts a new line at the cursor position and indents it.
      */
     newIndentedLine(): void;
+
+    /**
+     * Gets the token at a given position.
+     */
+    getTokenForPosition(position: IPosition): IToken;
+
+    /**
+     * Gets the list of tokens for the editor model.
+     */
+    getTokens(): IToken[];
   }
 
   /**

packages/codemirror/src/editor.ts

@@ -118,7 +118,7 @@ export class CodeMirrorEditor implements CodeEditor.IEditor {
     model.mimeTypeChanged.connect(this._onMimeTypeChanged, this);
     model.selections.changed.connect(this._onSelectionsChanged, this);
 
-    CodeMirror.on(editor, 'keydown', (editor, event) => {
+    CodeMirror.on(editor, 'keydown', (editor: CodeMirror.Editor, event) => {
       let index = ArrayExt.findFirstIndex(this._keydownHandlers, handler => {
         if (handler(this, event) === true) {
           event.preventDefault();
@@ -485,6 +485,38 @@ export class CodeMirrorEditor implements CodeEditor.IEditor {
     this.doc.setSelections(cmSelections, 0);
   }
 
+  /**
+   * Get a list of tokens for the current editor text content.
+   */
+  getTokens(): CodeEditor.IToken[] {
+    let tokens: CodeEditor.IToken[] = [];
+    for (let i = 0; i < this.lineCount; ++i) {
+      const lineTokens = this.editor.getLineTokens(i).map(t => ({
+        offset: this.getOffsetAt({ column: t.start, line: i }),
+        value: t.string,
+        type: t.type || ''
+      }));
+      tokens = tokens.concat(lineTokens);
+    }
+    return tokens;
+  }
+
+  /**
+   * Get the token at a given editor position.
+   */
+  getTokenForPosition(position: CodeEditor.IPosition): CodeEditor.IToken {
+    const cursor = this._toCodeMirrorPosition(position);
+    const token = this.editor.getTokenAt(cursor);
+    return {
+      offset: this.getOffsetAt({ column: token.start, line: cursor.line }),
+      value: token.string,
+      type: token.type
+    };
+  }
+
+  /**
+   * Insert a new indented line at the current cursor position.
+   */
   newIndentedLine(): void {
     this.execCommand('newlineAndIndent');
   }

packages/codemirror/typings/codemirror/codemirror.d.ts

@@ -453,20 +453,15 @@ declare namespace CodeMirror {
     refresh(): void;
 
     /** Retrieves information about the token the current mode found before the given position (a {line, ch} object). */
-    getTokenAt(
-      pos: CodeMirror.Position
-    ): {
-      /** The character(on the given line) at which the token starts. */
-      start: number;
-      /** The character at which the token ends. */
-      end: number;
-      /** The token's string. */
-      string: string;
-      /** The token type the mode assigned to the token, such as "keyword" or "comment" (may also be null). */
-      type: string;
-      /** The mode's state at the end of this token. */
-      state: any;
-    };
+    getTokenAt(pos: CodeMirror.Position): Token;
+
+    /** This is a (much) cheaper version of getTokenAt useful for when you just need the type of the token at a given position,
+    and no other information. Will return null for unstyled tokens, and a string, potentially containing multiple
+    space-separated style names, otherwise. */
+    getTokenTypeAt(pos: CodeMirror.Position): string;
+
+    /** This is similar to getTokenAt, but collects all tokens for a given line into an array. */
+    getLineTokens(line: number, precise?: boolean): Token[];
 
     /** Returns the mode's parser state, if any, at the end of the given line number.
         If no line number is given, the state at the end of the document is returned.
@@ -1003,6 +998,33 @@ declare namespace CodeMirror {
     head: Position;
   }
 
+  interface Token {
+    /**
+     * The character(on the given line) at which the token starts.
+     */
+    start: number;
+
+    /**
+     * The character at which the token ends.
+     */
+    end: number;
+
+    /**
+     * The token's string.
+     */
+    string: string;
+    /**
+     * The token type the mode assigned to the token,
+     * such as "keyword" or "comment" (may also be null).
+     */
+    type: string | null;
+
+    /**
+     * The mode's state at the end of this token.
+     */
+    state: any;
+  }
+
   interface EditorConfiguration {
     /** string| The starting value of the editor. Can be a string, or a document object. */
     value?: any;

packages/completer-extension/package.json

@@ -32,7 +32,10 @@
     "@jupyterlab/application": "^0.17.2",
     "@jupyterlab/completer": "^0.17.2",
     "@jupyterlab/console": "^0.17.2",
+    "@jupyterlab/fileeditor": "^0.17.2",
     "@jupyterlab/notebook": "^0.17.2",
+    "@jupyterlab/services": "^3.0.3",
+    "@phosphor/algorithm": "^1.1.2",
     "@phosphor/widgets": "^1.6.0"
   },
   "devDependencies": {

packages/completer-extension/src/index.ts

@@ -6,15 +6,22 @@ import { JupyterLab, JupyterLabPlugin } from '@jupyterlab/application';
 import {
   CompleterModel,
   Completer,
+  CompletionConnector,
   CompletionHandler,
-  ICompletionManager,
-  KernelConnector
+  ContextConnector,
+  ICompletionManager
 } from '@jupyterlab/completer';
 
 import { IConsoleTracker } from '@jupyterlab/console';
 
+import { IEditorTracker } from '@jupyterlab/fileeditor';
+
 import { INotebookTracker } from '@jupyterlab/notebook';
 
+import { Session } from '@jupyterlab/services';
+
+import { find } from '@phosphor/algorithm';
+
 import { Widget } from '@phosphor/widgets';
 
 /**
@@ -27,11 +34,15 @@ namespace CommandIDs {
 
   export const invokeNotebook = 'completer:invoke-notebook';
 
+  export const invokeFile = 'completer:invoke-file';
+
   export const select = 'completer:select';
 
   export const selectConsole = 'completer:select-console';
 
   export const selectNotebook = 'completer:select-notebook';
+
+  export const selectFile = 'completer:select-file';
 }
 
 /**
@@ -127,12 +138,14 @@ const consoles: JupyterLabPlugin<void> = {
       const editor = cell && cell.editor;
       const session = anchor.session;
       const parent = panel;
-      const connector = new KernelConnector({ session });
+      const connector = new CompletionConnector({ session, editor });
       const handler = manager.register({ connector, editor, parent });
 
       // Listen for prompt creation.
       anchor.promptCellCreated.connect((sender, cell) => {
-        handler.editor = cell && cell.editor;
+        const editor = cell && cell.editor;
+        handler.editor = editor;
+        handler.connector = new CompletionConnector({ session, editor });
       });
     });
 
@@ -185,12 +198,14 @@ const notebooks: JupyterLabPlugin<void> = {
       const editor = cell && cell.editor;
       const session = panel.session;
       const parent = panel;
-      const connector = new KernelConnector({ session });
+      const connector = new CompletionConnector({ session, editor });
       const handler = manager.register({ connector, editor, parent });
 
       // Listen for active cell changes.
       panel.content.activeCellChanged.connect((sender, cell) => {
-        handler.editor = cell && cell.editor;
+        const editor = cell && cell.editor;
+        handler.editor = editor;
+        handler.connector = new CompletionConnector({ session, editor });
       });
     });
 
@@ -224,8 +239,126 @@ const notebooks: JupyterLabPlugin<void> = {
   }
 };
 
+/**
+ * An extension that registers file editors for completion.
+ */
+const files: JupyterLabPlugin<void> = {
+  id: '@jupyterlab/completer-extension:files',
+  requires: [ICompletionManager, IEditorTracker],
+  autoStart: true,
+  activate: (
+    app: JupyterLab,
+    manager: ICompletionManager,
+    editorTracker: IEditorTracker
+  ): void => {
+    // Keep a list of active ISessions so that we can
+    // clean them up when they are no longer needed.
+    const activeSessions: {
+      [id: string]: Session.ISession;
+    } = {};
+
+    // When a new file editor is created, make the completer for it.
+    editorTracker.widgetAdded.connect((sender, widget) => {
+      const sessions = app.serviceManager.sessions;
+      const editor = widget.content.editor;
+      const contextConnector = new ContextConnector({ editor });
+
+      // When the list of running sessions changes,
+      // check to see if there are any kernels with a
+      // matching path for this file editor.
+      const onRunningChanged = (
+        sender: Session.IManager,
+        models: Session.IModel[]
+      ) => {
+        const oldSession = activeSessions[widget.id];
+        // Search for a matching path.
+        const model = find(models, m => m.path === widget.context.path);
+        if (model) {
+          // If there is a matching path, but it is the same
+          // session as we previously had, do nothing.
+          if (oldSession && oldSession.id === model.id) {
+            return;
+          }
+          // Otherwise, dispose of the old session and reset to
+          // a new CompletionConnector.
+          if (oldSession) {
+            delete activeSessions[widget.id];
+            oldSession.dispose();
+          }
+          const session = sessions.connectTo(model);
+          handler.connector = new CompletionConnector({ session, editor });
+          activeSessions[widget.id] = session;
+        } else {
+          // If we didn't find a match, make sure
+          // the connector is the contextConnector and
+          // dispose of any previous connection.
+          handler.connector = contextConnector;
+          if (oldSession) {
+            delete activeSessions[widget.id];
+            oldSession.dispose();
+          }
+        }
+      };
+      Session.listRunning().then(models => {
+        onRunningChanged(sessions, models);
+      });
+      sessions.runningChanged.connect(onRunningChanged);
+
+      // Initially create the handler with the contextConnector.
+      // If a kernel session is found matching this file editor,
+      // it will be replaced in onRunningChanged().
+      const handler = manager.register({
+        connector: contextConnector,
+        editor,
+        parent: widget
+      });
+
+      // When the widget is disposed, do some cleanup.
+      widget.disposed.connect(() => {
+        sessions.runningChanged.disconnect(onRunningChanged);
+        const session = activeSessions[widget.id];
+        if (session) {
+          delete activeSessions[widget.id];
+          session.dispose();
+        }
+      });
+    });
+
+    // Add console completer invoke command.
+    app.commands.addCommand(CommandIDs.invokeFile, {
+      execute: () => {
+        const id =
+          editorTracker.currentWidget && editorTracker.currentWidget.id;
+
+        if (id) {
+          return app.commands.execute(CommandIDs.invoke, { id });
+        }
+      }
+    });
+
+    // Add console completer select command.
+    app.commands.addCommand(CommandIDs.selectFile, {
+      execute: () => {
+        const id =
+          editorTracker.currentWidget && editorTracker.currentWidget.id;
+
+        if (id) {
+          return app.commands.execute(CommandIDs.select, { id });
+        }
+      }
+    });
+
+    // Set enter key for console completer select command.
+    app.commands.addKeyBinding({
+      command: CommandIDs.selectFile,
+      keys: ['Enter'],
+      selector: `.jp-FileEditor .jp-mod-completer-active`
+    });
+  }
+};
+
 /**
  * Export the plugins as default.
  */
-const plugins: JupyterLabPlugin<any>[] = [manager, consoles, notebooks];
+const plugins: JupyterLabPlugin<any>[] = [manager, consoles, notebooks, files];
 export default plugins;