Add WASM build

.github/workflows/main.yml

@@ -130,3 +130,77 @@ jobs:
         run: |
           micromamba activate xeus-python
           pytest . -vvv
+
+  test-wasm:
+
+    runs-on: ubuntu-latest
+
+    strategy:
+      fail-fast: false
+      matrix:
+        emsdk_ver: ["3.1.2"]
+
+    steps:
+      - uses: actions/checkout@v2
+
+      - name: Get number of CPU cores
+        uses: SimenB/github-actions-cpu-cores@v1
+
+      - name: Install mamba
+        uses: mamba-org/provision-with-micromamba@main
+        with:
+          environment-file: environment-dev-wasm.yml
+          environment-name: xeus-python-wasm
+          micromamba-version: "0.22.0"
+
+      - name: Setup emsdk (TODO Install from conda-forge when available)
+        run: |
+          git clone https://github.com/emscripten-core/emsdk.git
+          pushd emsdk
+          ./emsdk install ${{matrix.emsdk_ver}}
+          popd
+
+      - name: Install emboa (TODO Install from conda-forge when available)
+        run: |
+          micromamba activate xeus-python-wasm
+          pip install git+https://github.com/emscripten-forge/emboa
+
+      - name: Build and pack xeus-python
+        run: |
+          micromamba activate xeus-python-wasm
+
+          pushd emsdk
+          ./emsdk activate ${{matrix.emsdk_ver}}
+          source emsdk_env.sh
+          export FILE_PACKAGER=$(pwd)/upstream/emscripten/tools/file_packager.py
+          popd
+
+          micromamba create -n xeus-python-build-wasm \
+            --platform=emscripten-32 \
+            -c https://repo.mamba.pm/emscripten-forge \
+            -c https://repo.mamba.pm/conda-forge \
+            --yes \
+            python ipython pybind11 jedi xtl nlohmann_json pybind11_json numpy xeus "xeus-python-shell>=0.3"
+
+          mkdir build
+          pushd build
+
+          export PREFIX=$MAMBA_ROOT_PREFIX/envs/xeus-python-build-wasm
+          export CMAKE_PREFIX_PATH=$PREFIX
+          export CMAKE_SYSTEM_PREFIX_PATH=$PREFIX
+
+          emcmake cmake \
+            -DCMAKE_BUILD_TYPE=Release \
+            -DCMAKE_FIND_ROOT_PATH_MODE_PACKAGE=ON \
+            -DCMAKE_PROJECT_INCLUDE=cmake/overwriteProp.cmake \
+            -DXPYT_EMSCRIPTEN_WASM_BUILD=ON \
+            ..
+
+          make -j5
+
+          emboa pack python core $MAMBA_ROOT_PREFIX/envs/xeus-python-build-wasm --version=3.10
+
+          popd
+
+          # Patch output
+          python wasm_patches/patch_it.py

.gitignore

@@ -51,3 +51,6 @@ build/
 
 # Jupyter artefacts
 .ipynb_checkpoints/
+
+# JS artifacts
+node_modules/

CMakeLists.txt

@@ -54,6 +54,7 @@ configure_file (
 # =============
 
 # Compilation options
+
 OPTION(XPYT_DISABLE_ARCH_NATIVE "disable -march=native flag" OFF)
 OPTION(XPYT_DISABLE_TUNE_GENERIC "disable -mtune=generic flag" OFF)
 OPTION(XPYT_ENABLE_PYPI_WARNING "Enable warning on PyPI wheels" OFF)
@@ -66,10 +67,28 @@ OPTION(XPYT_BUILD_XPYTHON_EXTENSION "Build the xpython extension module" OFF)
 OPTION(XPYT_USE_SHARED_XEUS "Link xpython or xpython_extension with the xeus shared library (instead of the static library)" ON)
 OPTION(XPYT_USE_SHARED_XEUS_PYTHON "Link xpython and xpython_extension with the xeus-python shared library (instead of the static library)" ON)
 
+OPTION(XPYT_EMSCRIPTEN_WASM_BUILD "Build for wasm with emscripten" OFF)
+
 # Test options
 OPTION(XPYT_BUILD_TESTS "xeus-python test suite" OFF)
 OPTION(XPYT_DOWNLOAD_GTEST "build gtest from downloaded sources" OFF)
 
+if(XPYT_EMSCRIPTEN_WASM_BUILD)
+    configure_file(wasm_patches/post.js ${CMAKE_CURRENT_BINARY_DIR}/post.js COPYONLY)
+
+    add_compile_definitions(XPYT_EMSCRIPTEN_WASM_BUILD)
+
+    set(XPYT_DISABLE_ARCH_NATIVE ON)
+    set(XPYT_BUILD_STATIC OFF)
+    set(XPYT_BUILD_SHARED OFF)
+    set(XPYT_BUILD_XPYTHON_EXECUTABLE OFF)
+    set(XPYT_BUILD_XPYTHON_EXTENSION OFF)
+    set(XPYT_USE_SHARED_XEUS OFF)
+    set(XPYT_USE_SHARED_XEUS_PYTHON OFF)
+    set(XPYT_BUILD_TESTS OFF)
+    set(XPYT_DOWNLOAD_GTEST OFF)
+endif()
+
 # Dependencies
 # ============
 
@@ -166,6 +185,42 @@ set(XPYTHON_EXTENSION_SRC
     src/xpython_extension.cpp
 )
 
+set(XEUS_PYTHON_WASM_SRC
+    src/xcomm.cpp
+    src/xcomm.hpp
+    src/xdisplay.cpp
+    src/xdisplay.hpp
+    src/xinput.cpp
+    src/xinput.hpp
+    src/xinspect.cpp
+    src/xinspect.hpp
+    src/xinternal_utils.cpp
+    src/xinternal_utils.hpp
+    src/xinterpreter.cpp
+    src/xinterpreter_wasm.cpp
+    src/xkernel.cpp
+    src/xkernel.hpp
+    src/xpaths.cpp
+    src/xstream.cpp
+    src/xstream.hpp
+    src/xtraceback.cpp
+    src/xutils.cpp
+)
+
+set(XEUS_PYTHON_WASM_HEADERS
+    include/xeus-python/xdebugger.hpp
+    include/xeus-python/xeus_python_config.hpp
+    include/xeus-python/xpaths.hpp
+    include/xeus-python/xinterpreter.hpp
+    include/xeus-python/xinterpreter_wasm.hpp
+    include/xeus-python/xtraceback.hpp
+    include/xeus-python/xutils.hpp
+)
+
+set(XPYTHON_WASM_SRC
+    src/main_wasm.cpp
+)
+
 # Targets and link - Macros
 # =========================
 
@@ -209,7 +264,10 @@ macro(xpyt_set_kernel_options target_name)
         message(STATUS "Enabling PyPI warning for target: " ${target_name})
         target_compile_definitions(${target_name} PRIVATE XEUS_PYTHON_PYPI_WARNING)
     endif()
+endmacro()
 
+# Common macro for linking targets
+macro(xpyt_target_link_libraries target_name)
     if (XPYT_USE_SHARED_XEUS_PYTHON)
         target_link_libraries(${target_name} PRIVATE xeus-python)
 
@@ -226,18 +284,18 @@ macro(xpyt_set_kernel_options target_name)
 endmacro()
 
 # Common macro for shared and static library xeus-python
-macro(xpyt_create_target target_name linkage output_name)
+macro(xpyt_create_target target_name src headers linkage output_name)
     string(TOUPPER "${linkage}" linkage_upper)
 
     if (NOT ${linkage_upper} MATCHES "^(SHARED|STATIC)$")
         message(FATAL_ERROR "Invalid library linkage: ${linkage}")
     endif ()
 
-    add_library(${target_name} ${linkage_upper} ${XEUS_PYTHON_SRC} ${XEUS_PYTHON_HEADERS})
+    add_library(${target_name} ${linkage_upper} ${src} ${headers})
     xpyt_set_common_options(${target_name})
 
     set_target_properties(${target_name} PROPERTIES
-                          PUBLIC_HEADER "${XEUS_PYTHON_HEADERS}"
+                          PUBLIC_HEADER "${headers}"
                           PREFIX ""
                           VERSION ${${PROJECT_NAME}_VERSION}
                           SOVERSION ${XPYT_VERSION_MAJOR}
@@ -271,8 +329,8 @@ macro(xpyt_create_target target_name linkage output_name)
         target_compile_definitions(${target_name} PRIVATE XEUS_PYTHONHOME_RELPATH=${XEUS_PYTHONHOME_RELPATH})
     elseif (XEUS_PYTHONHOME_ABSPATH)
         target_compile_definitions(${target_name} PRIVATE XEUS_PYTHONHOME_ABSPATH=${XEUS_PYTHONHOME_ABSPATH})
-    endif()
-endmacro()
+    endif ()
+endmacro ()
 
 # xeus-python
 # ===========
@@ -281,21 +339,27 @@ set(XEUS_PYTHON_TARGETS "")
 
 if (XPYT_BUILD_SHARED)
     # Build libraries
-    xpyt_create_target(xeus-python SHARED xeus-python)
+    xpyt_create_target(xeus-python "${XEUS_PYTHON_SRC}" "${XEUS_PYTHON_HEADERS}" SHARED xeus-python)
     list(APPEND XEUS_PYTHON_TARGETS xeus-python)
 endif ()
 
 if (XPYT_BUILD_STATIC)
     # On Windows, a static library should use a different output name
     # to avoid the conflict with the import library of a shared one.
     if (CMAKE_HOST_WIN32)
-        xpyt_create_target(xeus-python-static STATIC xeus-python-static)
+        xpyt_create_target(xeus-python-static "${XEUS_PYTHON_SRC}" "${XEUS_PYTHON_HEADERS}" STATIC xeus-python-static)
     else ()
-        xpyt_create_target(xeus-python-static STATIC xeus-python)
+        xpyt_create_target(xeus-python-static "${XEUS_PYTHON_SRC}" "${XEUS_PYTHON_HEADERS}" STATIC xeus-python)
     endif ()
     list(APPEND XEUS_PYTHON_TARGETS xeus-python-static)
 endif ()
 
+if (XPYT_EMSCRIPTEN_WASM_BUILD)
+    xpyt_create_target(xeus-python-wasm "${XEUS_PYTHON_WASM_SRC}" "${XEUS_PYTHON_WASM_HEADERS}" STATIC xeus-python-wasm)
+    target_compile_options(xeus-python-wasm PRIVATE -fPIC)
+    list(APPEND XEUS_PYTHON_TARGETS xeus-python-wasm)
+endif ()
+
 # xpython
 # =======
 
@@ -305,6 +369,7 @@ if (XPYT_BUILD_XPYTHON_EXECUTABLE)
 
     xpyt_set_common_options(xpython)
     xpyt_set_kernel_options(xpython)
+    xpyt_target_link_libraries(xpython)
 endif()
 
 # xpython_extension
@@ -315,6 +380,63 @@ if (XPYT_BUILD_XPYTHON_EXTENSION)
 
     xpyt_set_common_options(xpython_extension)
     xpyt_set_kernel_options(xpython_extension)
+    xpyt_target_link_libraries(xpython_extension)
+endif()
+
+# xpython_wasm
+# ============
+
+if (XPYT_EMSCRIPTEN_WASM_BUILD)
+    # TODO MAKE BETTER
+    SET(PYTHON_UTIL_LIBS
+        $ENV{PREFIX}/lib/libbz2.a
+        $ENV{PREFIX}/lib/libz.a
+        $ENV{PREFIX}/lib/libsqlite3.a
+        $ENV{PREFIX}/lib/libffi.a
+    )
+
+    add_executable(xpython_wasm ${XPYTHON_WASM_SRC})
+    target_link_libraries(xpython_wasm PRIVATE ${PYTHON_UTIL_LIBS} pybind11::embed)
+
+    xpyt_set_common_options(xpython_wasm)
+    target_compile_options(xpython_wasm PRIVATE -fPIC)
+
+    xpyt_set_kernel_options(xpython_wasm)
+    target_link_libraries(xpython_wasm PRIVATE xeus-python-wasm)
+
+    target_compile_options(xpython_wasm
+        PUBLIC --std=c++17
+        PUBLIC -Wno-deprecated
+        PUBLIC "SHELL: --bind"
+        PUBLIC "SHELL: -s ENVIRONMENT=web"
+        PUBLIC "SHELL: -fwasm-exceptions"
+        PUBLIC "SHELL: -s FORCE_FILESYSTEM"
+        PUBLIC "SHELL: -s LZ4=1"
+    )
+
+    target_link_options(xpython_wasm
+        PUBLIC --bind
+        PUBLIC -Wno-unused-command-line-argument
+        PUBLIC "SHELL: --bind"
+        PUBLIC "SHELL: -s MODULARIZE=1"
+        PUBLIC "SHELL: -s EXPORT_NAME=\"createXeusModule\""
+        PUBLIC "SHELL: -s EXPORT_ES6=0"
+        PUBLIC "SHELL: -s USE_ES6_IMPORT_META=0"
+        PUBLIC "SHELL: -s DEMANGLE_SUPPORT=0"
+        PUBLIC "SHELL: -s ASSERTIONS=0"
+        PUBLIC "SHELL: -s ALLOW_MEMORY_GROWTH=1"
+        PUBLIC "SHELL: -s EXIT_RUNTIME=1"
+        PUBLIC "SHELL: -s WASM=1"
+        PUBLIC "SHELL: -s USE_PTHREADS=0"
+        PUBLIC "SHELL: -fwasm-exceptions"
+        PUBLIC "SHELL: -s MAIN_MODULE=1"
+        PUBLIC "SHELL: -s ENVIRONMENT=worker"
+        PUBLIC "SHELL: -s TOTAL_STACK=512mb"
+        PUBLIC "SHELL: -s INITIAL_MEMORY=1024mb"
+        PUBLIC "SHELL: -s FORCE_FILESYSTEM"
+        PUBLIC "SHELL: -s LZ4=1"
+        PUBLIC "SHELL: --post-js post.js"
+    )
 endif()
 
 # Tests

README.md

@@ -159,7 +159,7 @@ Long story short:
 
 | `xeus-python`|   `xeus`         |      `xtl`      | `cppzmq` | `nlohmann_json` | `pybind11`     | `pybind11_json`   | `pygments`        | `debugpy` | `IPython` | `xeus-python-shell` |
 |--------------|------------------|-----------------|----------|-----------------|----------------|-------------------|-------------------|-----------|-----------|---------------------|
-|  master      |  >=2.0.0,<3.0    |  >=0.7.0,<0.8   | ~4.4.1   | >=3.6.1,<3.10   | >=2.6.1,<3.0   | >=0.2.8,<0.3      | >=2.3.1,<3.0.0    | >=1.1.0   |           | >=0.1.5,<0.3.0      |
+|  master      |  >=2.0.0,<3.0    |  >=0.7.0,<0.8   | ~4.4.1   | >=3.6.1,<3.10   | >=2.6.1,<3.0   | >=0.2.8,<0.3      | >=2.3.1,<3.0.0    | >=1.1.0   |           | >=0.3.0,<0.4.0      |
 |  0.13.7      |  >=2.0.0,<3.0    |  >=0.7.0,<0.8   | ~4.4.1   | >=3.6.1,<3.10   | >=2.6.1,<3.0   | >=0.2.8,<0.3      | >=2.3.1,<3.0.0    | >=1.1.0   |           | >=0.1.5,<0.3.0      |
 |  0.13.6      |  >=2.0.0,<3.0    |  >=0.7.0,<0.8   | ~4.4.1   | >=3.6.1,<3.10   | >=2.6.1,<3.0   | >=0.2.8,<0.3      | >=2.3.1,<3.0.0    | >=1.1.0   |           | >=0.1.5,<0.3.0      |
 |  0.13.5      |  >=2.0.0,<3.0    |  >=0.7.0,<0.8   | ~4.4.1   | >=3.6.1,<3.10   | >=2.6.1,<3.0   | >=0.2.8,<0.3      | >=2.3.1,<3.0.0    | >=1.1.0   |           | >=0.1.5,<0.2.0      |

cmake/overwriteProp.cmake

@@ -0,0 +1,6 @@
+# https://github.com/emscripten-core/emscripten/issues/15276
+# overwriteProp.cmake
+set_property(GLOBAL PROPERTY TARGET_SUPPORTS_SHARED_LIBS TRUE)
+set(CMAKE_SHARED_LIBRARY_CREATE_C_FLAGS "-s SIDE_MODULE=1")
+set(CMAKE_SHARED_LIBRARY_CREATE_CXX_FLAGS "-s SIDE_MODULE=1")
+set(CMAKE_STRIP FALSE)  # used by default in pybind11 on .so modules

environment-dev-wasm.yml

@@ -0,0 +1,9 @@
+name: xeus-python-wasm
+channels:
+  - conda-forge
+dependencies:
+  - cmake
+  - pip
+  - python=3.10
+  - yarn
+  - click

environment-dev.yml

@@ -12,7 +12,7 @@ dependencies:
   - xtl>=0.7, <0.8
   - pybind11>=2.6.1,<3.0
   - pybind11_json>=0.2.6,<0.3
-  - xeus-python-shell>=0.1.6,<0.3
+  - xeus-python-shell>=0.3.0,<0.4
   - debugpy
   # Test dependencies
   - pytest

include/xeus-python/xinterpreter.hpp

@@ -94,7 +94,12 @@ namespace xpyt
         bool m_release_gil_at_startup = true;
         gil_scoped_release_ptr m_release_gil = nullptr;
 
+        bool m_redirect_output_enabled;
         bool m_redirect_display_enabled;
+
+    private:
+
+        virtual void instanciate_ipython_shell();
     };
 }
 

include/xeus-python/xinterpreter_raw.hpp

@@ -88,8 +88,6 @@ namespace xpyt
         bool m_redirect_display_enabled;
     };
 
-
-
 }
 
 #ifdef __GNUC__