Add InteractiveConsole in pyodide-py (#1125)

docs/changelog.md

@@ -38,6 +38,8 @@
 - Updated packages: bleach 3.2.1, packaging 20.8
 - `eval_code` now accepts separate `globals` and `locals` parameters.
   [#1083](https://github.com/iodide-project/pyodide/pull/1083)
+- An InteractiveConsole to ease the integration of Pyodide REPL in
+  webpages (used in console.html) [#1125](https://github.com/iodide-project/pyodide/pull/1125)
 
 ### Fixed
 - getattr and dir on JsProxy now report consistent results and include all names defined on the Python dictionary backing JsProxy. [#1017](https://github.com/iodide-project/pyodide/pull/1017)
@@ -46,7 +48,7 @@
 [#1033](https://github.com/iodide-project/pyodide/pull/1033)
 - JsBoundMethod is now a subclass of JsProxy, which fixes nested attribute access and various other strange bugs.
   [#1124](https://github.com/iodide-project/pyodide/pull/1124)
-
+- In console.html: sync behavior, full stdout/stderr support, clean namespace and bigger font [#1125](https://github.com/iodide-project/pyodide/pull/1125)
 
 ## Version 0.16.1
 *December 25, 2020*

src/pyodide-py/pyodide/console.py

@@ -0,0 +1,153 @@
+from typing import Optional, Callable
+import code
+import io
+import sys
+import platform
+
+
+__all__ = ["InteractiveConsole"]
+
+
+class _StdStream(io.TextIOWrapper):
+    """
+    Custom std stream to retdirect sys.stdout/stderr in InteractiveConsole.
+
+    Parmeters
+    ---------
+    flush_callback
+        Function to call at each flush.
+    """
+
+    def __init__(
+        self, flush_callback: Callable[[str], None], name: Optional[str] = None
+    ):
+        # we just need to set internal buffer's name as
+        # it will automatically buble up to each buffer
+        internal_buffer = _CallbackBuffer(flush_callback, name=name)
+        buffer = io.BufferedWriter(internal_buffer)
+        super().__init__(buffer, line_buffering=True)  # type: ignore
+
+
+class _CallbackBuffer(io.RawIOBase):
+    """
+    Internal _StdStream buffer that triggers flush callback.
+
+    Parmeters
+    ---------
+    flush_callback
+        Function to call at each flush.
+    """
+
+    def __init__(
+        self, flush_callback: Callable[[str], None], name: Optional[str] = None
+    ):
+        self._flush_callback = flush_callback
+        self.name = name
+
+    def writable(self):
+        return True
+
+    def seekable(self):
+        return False
+
+    def isatty(self):
+        return True
+
+    def write(self, data):
+        self._flush_callback(data.tobytes().decode())
+        return len(data)
+
+
+class InteractiveConsole(code.InteractiveConsole):
+    """Interactive Pyodide console
+
+    Base implementation for an interactive console that manages
+    stdout/stderr redirection.
+
+    `self.stdout_callback` and `self.stderr_callback` can be overloaded.
+
+    Parameters
+    ----------
+    locals
+        Namespace to evaluate code.
+    stdout_callback
+        Function to call at each `sys.stdout` flush.
+    stderr_callback
+        Function to call at each `sys.stderr` flush.
+    persistent_stream_redirection
+        Wether or not the std redirection should be kept between calls to
+        `runcode`.
+    """
+
+    def __init__(
+        self,
+        locals: Optional[dict] = None,
+        stdout_callback: Optional[Callable[[str], None]] = None,
+        stderr_callback: Optional[Callable[[str], None]] = None,
+        persistent_stream_redirection: bool = False,
+    ):
+        super().__init__(locals)
+        self._stdout = None
+        self._stderr = None
+        self.stdout_callback = stdout_callback
+        self.stderr_callback = stderr_callback
+        self._persistent_stream_redirection = persistent_stream_redirection
+        if self._persistent_stream_redirection:
+            self.redirect_stdstreams()
+
+    def redirect_stdstreams(self):
+        """ Toggle stdout/stderr redirections. """
+
+        if self._stdout is None:
+            # we use meta callbacks to allow self.std{out,err}_callback
+            # overloading.
+            # we check callback against None at each call since it can be
+            # changed dynamically.
+            def meta_stdout_callback(*args):
+                if self.stdout_callback is not None:
+                    return self.stdout_callback(*args)
+
+            # for later restore
+            self._old_stdout = sys.stdout
+
+            # it would be more robust to use sys.stdout.name but testing
+            # system oveload them. Anyway it should be pretty stable
+            # upstream.
+            self._stdout = _StdStream(meta_stdout_callback, name="<stdout>")
+
+        if self._stderr is None:
+
+            def meta_stderr_callback(*args):
+                if self.stderr_callback is not None:
+                    return self.stderr_callback(*args)
+
+            self._old_stderr = sys.stderr
+            self._stderr = _StdStream(meta_stderr_callback, name="<stderr>")
+
+        # actual redirection
+        sys.stdout = self._stdout
+        sys.stderr = self._stderr
+
+    def restore_stdstreams(self):
+        """Restore stdout/stderr to the value it was before
+        the creation of the object."""
+        sys.stdout = self._old_stdout
+        sys.stderr = self._old_stderr
+
+    def runcode(self, code):
+        if self._persistent_stream_redirection:
+            super().runcode(code)
+        else:
+            self.redirect_stdstreams()
+            super().runcode(code)
+            self.restore_stdstreams()
+
+    def __del__(self):
+        if self._persistent_stream_redirection:
+            self.restore_stdstreams()
+
+    def banner(self):
+        """ A banner similar to the one printed by the real Python interpreter. """
+        # copyied from https://github.com/python/cpython/blob/799f8489d418b7f9207d333eac38214931bd7dcc/Lib/code.py#L214
+        cprt = 'Type "help", "copyright", "credits" or "license" for more information.'
+        return f"Python {platform.python_version()} {platform.python_build()} on WebAssembly VM\n{cprt}"

src/templates/console.html

@@ -11,51 +11,56 @@
         window.languagePluginUrl = '{{ PYODIDE_BASE_URL }}';
     </script>
     <script src="{{ PYODIDE_BASE_URL }}pyodide.js"></script>
+    <style>
+      .terminal { --size: 1.5; }
+    </style>
   </head>
   <body>
     <script>
       languagePluginLoader.then(() => {
+        pyodide.runPython(`
+            from pyodide import console
+            import js
+
+
+            class Console(console.InteractiveConsole):
+                def runcode(self, code):
+                    self.redirect_stdstreams()
+                    js.term.pause()
+                    js.term.runPython("\\n".join(self.buffer))
+
+                def banner(self):
+                    return f"Welcome to the Pyodide terminal emulator 🐍\\n{super().banner()}"
+
+
+            # JQuery.terminal add trailing newline that is not easy to remove
+            # with the current API. Doing our best here.
+            _c = Console(stdout_callback=lambda s: js.term.echo(s.rstrip("\\n")),
+                         stderr_callback=lambda s: js.term.error(s.rstrip("\\n")),
+                         persistent_stream_redirection=False)
+        `)
+
+        let c = pyodide.pyimport('_c');
+
         function pushCode(line) {
-          handleResult(c.push(line))
+          handleResult(c.push(line));
         }
 
         let term = $('body').terminal(
           pushCode,
           {
-            greetings: "Welcome to the Pyodide terminal emulator 🐍",
+              greetings: c.banner(),
             prompt: "[[;red;]>>> ]"
           }
         );
 
         window.term = term;
-        pyodide.runPython(`
-          import io, code, sys
-          from js import term, pyodide
-
-          class Console(code.InteractiveConsole):
-              def runcode(self, code):
-                  sys.stdout = io.StringIO()
-                  sys.stderr = io.StringIO()
-                  term.runPython("\\n".join(self.buffer))
-          _c = Console(locals=globals())
-        `)
-
-        let c = pyodide.pyimport('_c');
 
         function handleResult(result) {
           if (result) {
             term.set_prompt('[[;gray;]... ]');
           } else {
             term.set_prompt('[[;red;]>>> ]');
-            let stderr = pyodide.runPython("sys.stderr.getvalue()").trim();
-            if (stderr) {
-              term.echo(`[[;red;]${stderr}]`);
-            } else {
-              let stdout = pyodide.runPython("sys.stdout.getvalue()");
-              if (stdout) {
-                term.echo(stdout.trim());
-              }
-            }
           }
         }
 
@@ -66,6 +71,8 @@
         };
 
         term.handlePythonResult = function(result) {
+          c.restore_stdstreams();
+          term.resume();
           if (result === undefined) {
             return;
           } else if (result['_repr_html_'] !== undefined) {
@@ -76,8 +83,10 @@
         };
 
         term.handlePythonError = function(result) {
+          c.restore_stdstreams();
+          term.resume();
           term.error(result.toString());
-        };
+        }
       });
     </script>
   </body>

src/tests/test_console.py

@@ -0,0 +1,96 @@
+import pytest
+from pathlib import Path
+import sys
+
+sys.path.append(str(Path(__file__).parents[2] / "src" / "pyodide-py"))
+
+from pyodide import console  # noqa: E402
+
+
+def test_stream_redirection():
+    my_buffer = ""
+
+    def callback(string):
+        nonlocal my_buffer
+        my_buffer += string
+
+    my_stream = console._StdStream(callback)
+
+    print("foo", file=my_stream)
+    assert my_buffer == "foo\n"
+    print("bar", file=my_stream)
+    assert my_buffer == "foo\nbar\n"
+
+
+@pytest.fixture
+def safe_stdstreams():
+    stdout = sys.stdout
+    stderr = sys.stderr
+    yield
+    sys.stdout = stdout
+    sys.stderr = stderr
+
+
+def test_interactive_console_streams(safe_stdstreams):
+
+    my_stdout = ""
+    my_stderr = ""
+
+    def stdout_callback(string):
+        nonlocal my_stdout
+        my_stdout += string
+
+    def stderr_callback(string):
+        nonlocal my_stderr
+        my_stderr += string
+
+    ##########################
+    # Persistent redirection #
+    ##########################
+    shell = console.InteractiveConsole(
+        stdout_callback=stdout_callback,
+        stderr_callback=stderr_callback,
+        persistent_stream_redirection=True,
+    )
+
+    # std names
+    assert sys.stdout.name == "<stdout>"
+    assert sys.stderr.name == "<stderr>"
+
+    # std redirections
+    print("foo")
+    assert my_stdout == "foo\n"
+    print("bar", file=sys.stderr)
+    assert my_stderr == "bar\n"
+
+    shell.push("print('foobar')")
+    assert my_stdout == "foo\nfoobar\n"
+
+    shell.restore_stdstreams()
+
+    my_stdout = ""
+    my_stderr = ""
+
+    print("bar")
+    assert my_stdout == ""
+
+    print("foo", file=sys.stdout)
+    assert my_stderr == ""
+
+    ##############################
+    # Non persistent redirection #
+    ##############################
+    shell = console.InteractiveConsole(
+        stdout_callback=stdout_callback,
+        stderr_callback=stderr_callback,
+        persistent_stream_redirection=False,
+    )
+
+    print("foo")
+    assert my_stdout == ""
+
+    shell.push("print('foobar')")
+    assert my_stdout == "foobar\n"
+
+    print("bar")
+    assert my_stdout == "foobar\n"