Add diagnostics and autocomplete. (#251)

Adds support for the Pyright language server based on a fork with
browser support. This PR uses a build from
https://github.com/microbit-matt-hillsdon/pyright/tree/background-worker
as of 03d67527beb5c8e9ede2e9baa7bab83ed481237f.

Key limitations:

- We might consider some more student focussed error messages for
  common scenarios.
- We might like to add hover, signature help, documentation for
  completions and other features that require more language server
  integration.

See #240, #239

Author: microbit-matt-hillsdon

.vscode/settings.json

@@ -0,0 +1,3 @@
+{
+  "python.formatting.provider": "black"
+}

README.md

@@ -86,6 +86,8 @@ This software is under the MIT open source license.
 
 Binaries for MicroPython are included for micro:bit V1 ([license](https://github.com/bbcmicrobit/micropython/blob/master/LICENSE)) and micro:bit V2 ([license](https://github.com/microbit-foundation/micropython-microbit-v2/blob/master/LICENSE)). Both are MIT licensed.
 
+Python diagnostics and autocomplete use a fork of Microsoft's Pyright type checker which has been [modified by us](public/workers/PYRIGHT_README.txt) to run as a Web Worker. Pyright is © Microsoft Corporation and [used under an MIT license](public/workers/PYRIGHT_LICENSE.txt).
+
 We use dependencies via the NPM registry as specified by the package.json file under common Open Source licenses.
 
 Full details of each package can be found by running `license-checker`:

package-lock.json

@@ -39,6 +39,8 @@
     "react-dom": "^17.0.2",
     "react-icons": "^4.2.0",
     "react-intl": "^5.20.7",
+    "vscode-jsonrpc": "^6.0.0",
+    "vscode-languageserver-protocol": "^3.16.0",
     "web-vitals": "^1.1.1",
     "xterm": "^4.13.0",
     "xterm-addon-fit": "^0.5.0"

package.json

@@ -0,0 +1,22 @@
+MIT License
+
+Pyright - A static type checker for the Python language
+Copyright (c) Microsoft Corporation. All rights reserved.
+
+Permission is hereby granted, free of charge, to any person obtaining a copy
+of this software and associated documentation files (the "Software"), to deal
+in the Software without restriction, including without limitation the rights
+to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
+copies of the Software, and to permit persons to whom the Software is
+furnished to do so, subject to the following conditions:
+
+The above copyright notice and this permission notice shall be included in all
+copies or substantial portions of the Software.
+
+THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
+IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
+FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
+AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
+LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
+OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
+SOFTWARE

public/workers/PYRIGHT_LICENSE.txt

@@ -0,0 +1,3 @@
+Pyright as a Web Worker.
+
+Source: https://github.com/microbit-matt-hillsdon/pyright

public/workers/PYRIGHT_README.txt

@@ -2,6 +2,8 @@
  * (c) 2021, Micro:bit Educational Foundation and contributors
  *
  * SPDX-License-Identifier: MIT
+ * 
+ * @jest-environment ./src/testing/custom-browser-env
  */
 import { render } from "@testing-library/react";
 import App from "./App";

public/workers/pyright-0fa21f4869dd61b44452.worker.js

@@ -16,6 +16,9 @@ import { MockDeviceConnection } from "./device/mock";
 import { FileSystem } from "./fs/fs";
 import { FileSystemContext } from "./fs/fs-hooks";
 import { fetchMicroPython } from "./fs/micropython";
+import { trackFsChanges } from "./language-server/client-fs";
+import { LanguageServerClientContext } from "./language-server/language-server-hooks";
+import { pyright } from "./language-server/pyright";
 import { LoggingContext } from "./logging/logging-hooks";
 import TranslationProvider from "./messages/TranslationProvider";
 import ProjectDropTarget from "./project/ProjectDropTarget";
@@ -39,7 +42,11 @@ const logging = deployment.logging;
 const device = isMockDeviceMode()
   ? new MockDeviceConnection()
   : new MicrobitWebUSBConnection({ logging });
+
+const client = pyright();
 const fs = new FileSystem(logging, fetchMicroPython);
+client?.initialize().then(() => trackFsChanges(client, fs));
+
 // If this fails then we retry on access.
 fs.initializeInBackground();
 
@@ -69,12 +76,14 @@ const App = () => {
               <DialogProvider>
                 <DeviceContext.Provider value={device}>
                   <FileSystemContext.Provider value={fs}>
-                    <BeforeUnloadDirtyCheck />
-                    <SelectionContext>
-                      <ProjectDropTarget>
-                        <Workbench />
-                      </ProjectDropTarget>
-                    </SelectionContext>
+                    <LanguageServerClientContext.Provider value={client}>
+                      <BeforeUnloadDirtyCheck />
+                      <SelectionContext>
+                        <ProjectDropTarget>
+                          <Workbench />
+                        </ProjectDropTarget>
+                      </SelectionContext>
+                    </LanguageServerClientContext.Provider>
                   </FileSystemContext.Provider>
                 </DeviceContext.Provider>
               </DialogProvider>

public/workers/pyright-0fa21f4869dd61b44452.worker.js.map

@@ -40,7 +40,6 @@ export class MockDeviceConnection
 
   async connect(): Promise<ConnectionStatus> {
     this.setStatus(ConnectionStatus.CONNECTED);
-    console.log("Adding listener");
     document.addEventListener("mockSerialWrite", this.mockSerialListener);
     return this.status;
   }

src/App.test.tsx

@@ -22,7 +22,7 @@ const EditorContainer = ({ selection }: EditorContainerProps) => {
   return typeof defaultValue === "undefined" ? null : (
     <Editor
       defaultValue={defaultValue}
-      location={selection.location}
+      selection={selection}
       onChange={onFileChange}
       fontSize={settings.fontSize}
       codeStructureSettings={codeStructureSettings(settings)}

src/App.tsx

@@ -7,9 +7,12 @@ import { EditorSelection, EditorState } from "@codemirror/state";
 import { EditorView } from "@codemirror/view";
 import { useEffect, useMemo, useRef } from "react";
 import { useIntl } from "react-intl";
-import { FileLocation } from "../../workbench/use-selection";
+import { createUri } from "../../language-server/client";
+import { useLanguageServerClient } from "../../language-server/language-server-hooks";
+import { WorkbenchSelection } from "../../workbench/use-selection";
 import "./CodeMirror.css";
 import { editorConfig, themeExtensionsCompartment } from "./config";
+import { languageServer } from "./language-server/view";
 import {
   codeStructure,
   CodeStructureSettings,
@@ -22,7 +25,7 @@ interface CodeMirrorProps {
   defaultValue: string;
   onChange: (doc: string) => void;
 
-  location: FileLocation;
+  selection: WorkbenchSelection;
   fontSize: number;
   codeStructureSettings: CodeStructureSettings;
 }
@@ -39,12 +42,14 @@ const CodeMirror = ({
   defaultValue,
   className,
   onChange,
-  location,
+  selection,
   fontSize,
   codeStructureSettings,
 }: CodeMirrorProps) => {
+  const uri = createUri(selection.file);
   const elementRef = useRef<HTMLDivElement | null>(null);
   const viewRef = useRef<EditorView | null>(null);
+  const client = useLanguageServerClient();
   const intl = useIntl();
 
   // Group the option props together to keep configuration updates simple.
@@ -69,6 +74,7 @@ const CodeMirror = ({
         extensions: [
           notify,
           editorConfig,
+          client ? languageServer(client, uri) : [],
           // Extensions we enable/disable based on props.
           structureHighlightingCompartment.of(
             codeStructure(options.codeStructureSettings)
@@ -83,7 +89,7 @@ const CodeMirror = ({
 
       viewRef.current = view;
     }
-  }, [options, defaultValue, onChange]);
+  }, [options, defaultValue, onChange, client, uri]);
   useEffect(() => {
     // Do this separately as we don't want to destroy the view whenever options needed for initialization change.
     return () => {
@@ -107,6 +113,7 @@ const CodeMirror = ({
     });
   }, [options]);
 
+  const { location } = selection;
   useEffect(() => {
     // When the identity of location changes then the user has navigated.
     if (location.line) {

src/device/mock.ts

@@ -3,27 +3,26 @@
  *
  * SPDX-License-Identifier: MIT
  */
+import { completionKeymap } from "@codemirror/autocomplete";
+import { closeBrackets, closeBracketsKeymap } from "@codemirror/closebrackets";
+import { defaultKeymap, indentLess, indentMore } from "@codemirror/commands";
+import { commentKeymap } from "@codemirror/comment";
+import { highlightActiveLineGutter, lineNumbers } from "@codemirror/gutter";
+import { defaultHighlightStyle } from "@codemirror/highlight";
+import { history, historyKeymap } from "@codemirror/history";
+import { python } from "@codemirror/lang-python";
+import { indentOnInput, indentUnit } from "@codemirror/language";
+import { lintKeymap } from "@codemirror/lint";
+import { bracketMatching } from "@codemirror/matchbrackets";
+import { Compartment, EditorState, Extension, Prec } from "@codemirror/state";
 import {
-  keymap,
-  highlightSpecialChars,
   drawSelection,
+  EditorView,
   highlightActiveLine,
+  highlightSpecialChars,
   KeyBinding,
+  keymap,
 } from "@codemirror/view";
-import { Extension, EditorState, Prec, Compartment } from "@codemirror/state";
-import { history, historyKeymap } from "@codemirror/history";
-import { indentOnInput, indentUnit } from "@codemirror/language";
-import { highlightActiveLineGutter, lineNumbers } from "@codemirror/gutter";
-import { defaultKeymap, indentLess, indentMore } from "@codemirror/commands";
-import { bracketMatching } from "@codemirror/matchbrackets";
-import { closeBrackets, closeBracketsKeymap } from "@codemirror/closebrackets";
-import { autocompletion, completionKeymap } from "@codemirror/autocomplete";
-import { commentKeymap } from "@codemirror/comment";
-import { defaultHighlightStyle } from "@codemirror/highlight";
-import { lintKeymap } from "@codemirror/lint";
-import { EditorView } from "@codemirror/view";
-import { python } from "@codemirror/lang-python";
-import { completion } from "./completion";
 import highlightStyle from "./highlightStyle";
 
 const customTabBinding: KeyBinding = {
@@ -53,9 +52,6 @@ export const editorConfig: Extension = [
   bracketMatching(),
   closeBrackets(),
   highlightStyle(),
-  autocompletion({
-    override: completion,
-  }),
   highlightActiveLine(),
   highlightActiveLineGutter(),