Backport PR #62: [0.1.x] pyodide 0.24.0, widget shim patches (#63)

Co-authored-by: Nicholas Bollweg <nick.bollweg@gmail.com>

.prettierrc

@@ -1,5 +1,6 @@
 {
   "singleQuote": true,
   "printWidth": 88,
-  "proseWrap": "always"
-}
+  "proseWrap": "always",
+  "trailingComma": "all"
+}

jupyterlite_pyodide_kernel/addons/pyodide.py

@@ -15,7 +15,7 @@
 from ..constants import (
     PYODIDE,
     PYODIDE_JS,
-    PYODIDE_REPODATA,
+    PYODIDE_LOCK,
     PYODIDE_URL,
 )
 
@@ -151,8 +151,8 @@ def check_config_paths(self, jupyterlite_json):
         assert pyodide_path.exists(), f"{pyodide_path} not found"
         pyodide_js = pyodide_path / PYODIDE_JS
         assert pyodide_js.exists(), f"{pyodide_js} not found"
-        pyodide_repodata = pyodide_path / PYODIDE_REPODATA
-        assert pyodide_repodata.exists(), f"{pyodide_repodata} not found"
+        pyodide_lock = pyodide_path / PYODIDE_LOCK
+        assert pyodide_lock.exists(), f"{pyodide_lock} not found"
 
     def patch_jupyterlite_json(self, config_path, output_js):
         """update jupyter-lite.json to use the local pyodide"""
@@ -216,7 +216,7 @@ def extract_pyodide(self, local_path, dest):
             targets=[
                 # there are a lot of js/data files, but we actually talk about these...
                 dest / PYODIDE / PYODIDE_JS,
-                dest / PYODIDE / PYODIDE_REPODATA,
+                dest / PYODIDE / PYODIDE_LOCK,
             ],
             actions=[(self.extract_one, [local_path, dest])],
         )

jupyterlite_pyodide_kernel/constants.py

@@ -25,11 +25,11 @@
 #: where we put pyodide, for now
 PYODIDE = "pyodide"
 PYODIDE_JS = "pyodide.js"
-PYODIDE_REPODATA = "repodata.json"
+PYODIDE_LOCK = "pyodide-lock.json"
 PYODIDE_URL_ENV_VAR = "JUPYTERLITE_PYODIDE_URL"
 
 #: probably only compatible with this version of pyodide
-PYODIDE_VERSION = "0.23.4"
+PYODIDE_VERSION = "0.24.0"
 
 #: the only kind of noarch wheel piplite understands
 NOARCH_WHL = "py3-none-any.whl"

packages/pyodide-kernel-extension/schema/kernel.v0.schema.json

@@ -8,7 +8,7 @@
     "pyodideUrl": {
       "description": "The path to the main pyodide.js entry point",
       "type": "string",
-      "default": "https://cdn.jsdelivr.net/pyodide/v0.23.4/full/pyodide.js",
+      "default": "https://cdn.jsdelivr.net/pyodide/v0.24.0/full/pyodide.js",
       "format": "uri"
     },
     "disablePyPIFallback": {

packages/pyodide-kernel-extension/src/index.ts

@@ -20,7 +20,7 @@ const KERNEL_ICON_URL = `data:image/svg+xml;base64,${btoa(KERNEL_ICON_SVG_STR)}`
 /**
  * The default CDN fallback for Pyodide
  */
-const PYODIDE_CDN_URL = 'https://cdn.jsdelivr.net/pyodide/v0.23.4/full/pyodide.js';
+const PYODIDE_CDN_URL = 'https://cdn.jsdelivr.net/pyodide/v0.24.0/full/pyodide.js';
 
 /**
  * The id for the extension, and key in the litePlugins.
@@ -39,7 +39,7 @@ const kernel: JupyterLiteServerPlugin<void> = {
     app: JupyterLiteServer,
     kernelspecs: IKernelSpecs,
     serviceWorker?: IServiceWorkerManager,
-    broadcastChannel?: IBroadcastChannelWrapper
+    broadcastChannel?: IBroadcastChannelWrapper,
   ) => {
     const config =
       JSON.parse(PageConfig.getOption('litePluginSettings') || '{}')[PLUGIN_ID] || {};

packages/pyodide-kernel/package.json

@@ -62,7 +62,7 @@
     "@types/jest": "^26.0.10",
     "esbuild": "0.17.10",
     "jest": "^26.4.2",
-    "pyodide": "0.23.4",
+    "pyodide": "0.24.0",
     "rimraf": "~3.0.0",
     "ts-jest": "^26.3.0",
     "typescript": "~4.9.4"
@@ -75,8 +75,8 @@
       "py/pyodide-kernel": "0.1.1",
       "py/piplite": "0.1.1",
       "py/ipykernel": "6.9.2",
-      "py/widgetsnbextension3/widgetsnbextension": "3.6.4",
-      "py/widgetsnbextension4/widgetsnbextension": "4.0.7"
+      "py/widgetsnbextension3/widgetsnbextension": "3.6.6",
+      "py/widgetsnbextension4/widgetsnbextension": "4.0.9"
     }
   },
   "styleModule": "style/index.js"

packages/pyodide-kernel/py/piplite/piplite/piplite.py

@@ -6,13 +6,19 @@
     `pyodide-kernel` also includes a browser shim for the IPython `%pip` magic
 
 """
+from typing import Any
 import asyncio
 import json
+import logging
 from unittest.mock import patch
 
-from micropip import _micropip
-from micropip._micropip import _get_pypi_json as _MP_GET_PYPI_JSON
-from micropip._micropip import fetch_string as _MP_FETCH_STRING
+import micropip
+from micropip.package_index import ProjectInfo
+from micropip.package_index import query_package as _MP_QUERY_PACKAGE
+from micropip.package_index import fetch_string_and_headers as _MP_FETCH_STRING
+
+logger = logging.getLogger(__name__)
+
 
 #: a list of Warehouse-like API endpoints or derived multi-package all.json
 _PIPLITE_URLS = []
@@ -34,20 +40,25 @@ class PiplitePyPIDisabled(ValueError):
     pass
 
 
-async def _get_pypi_json_from_index(pkgname, piplite_url, fetch_kwargs):
+async def _get_pypi_json_from_index(name, piplite_url, fetch_kwargs) -> ProjectInfo:
     """Attempt to load a specific ``pkgname``'s releases from a specific piplite
     URL's index.
     """
     index = _PIPLITE_INDICES.get(piplite_url, {})
 
     if not index:
         try:
-            index = json.loads(await _MP_FETCH_STRING(piplite_url, fetch_kwargs))
+            data, headers = await _MP_FETCH_STRING(piplite_url, fetch_kwargs)
+        except Exception as err:
+            logger.warn("Could not fetch %s: %s", piplite_url, err)
+
+        try:
+            index = json.loads(data)
             _PIPLITE_INDICES.update({piplite_url: index})
-        except Exception:
-            pass
+        except Exception as err:
+            logger.warn("Could not parse %s: %s", piplite_url, err)
 
-    pkg = dict((index or {}).get(pkgname) or {})
+    pkg = dict((index or {}).get(name) or {})
 
     if not pkg:
         return None
@@ -58,37 +69,36 @@ async def _get_pypi_json_from_index(pkgname, piplite_url, fetch_kwargs):
             if artifact["url"].startswith("."):
                 artifact["url"] = (
                     f"""{piplite_url.split(ALL_JSON)[0]}/{artifact["url"]}"""
-                    # can't add cache busting because micropip 0.21 checks `endswith`
-                    # f"""?sha256={artifact["digests"]["sha256"]}"""
+                    f"""?sha256={artifact["digests"]["sha256"]}"""
                 )
 
-    return pkg
-
+    info = ProjectInfo._compatible_only(name, pkg["releases"])
+    return info
 
-async def _get_pypi_json(pkgname, fetch_kwargs):
-    """Fetch the warehoust API metadata for a specific ``pkgname``."""
 
+async def _query_package(
+    name: str,
+    fetch_kwargs: dict[str, Any] | None = None,
+    index_urls: list[str] | str | None = None,
+) -> ProjectInfo:
+    """Fetch the warehouse API metadata for a specific ``pkgname``."""
     for piplite_url in _PIPLITE_URLS:
-        if piplite_url.split("?")[0].split("#")[0].endswith(ALL_JSON):
-            pypi_json_from_index = await _get_pypi_json_from_index(
-                pkgname, piplite_url, fetch_kwargs
-            )
-            if pypi_json_from_index:
-                return pypi_json_from_index
+        if not piplite_url.split("?")[0].split("#")[0].endswith(ALL_JSON):
+            logger.warn("Non-all.json piplite URL not supported %s", piplite_url)
             continue
-        else:
-            try:
-                url = f"{piplite_url}{pkgname}/json"
-                return json.loads(await _MP_FETCH_STRING(url, fetch_kwargs))
-            except Exception:
-                pass
+
+        pypi_json_from_index = await _get_pypi_json_from_index(
+            name, piplite_url, fetch_kwargs
+        )
+        if pypi_json_from_index:
+            return pypi_json_from_index
 
     if _PIPLITE_DISABLE_PYPI:
         raise PiplitePyPIDisabled(
-            f"{pkgname} could not be installed: PyPI fallback is disabled"
+            f"{name} could not be installed: PyPI fallback is disabled"
         )
 
-    return await _MP_GET_PYPI_JSON(pkgname, fetch_kwargs)
+    return await _MP_QUERY_PACKAGE(name, fetch_kwargs, index_urls)
 
 
 async def _install(
@@ -97,15 +107,20 @@ async def _install(
     deps: bool = True,
     credentials: str | None = None,
     pre: bool = False,
+    index_urls: list[str] | str | None = None,
+    *,
+    verbose: bool | int = False,
 ):
     """Invoke micropip.install with a patch to get data from local indexes"""
-    with patch("micropip._micropip._get_pypi_json", _get_pypi_json):
-        return await _micropip.install(
+    with patch("micropip.package_index.query_package", _query_package):
+        return await micropip.install(
             requirements=requirements,
             keep_going=keep_going,
             deps=deps,
             credentials=credentials,
             pre=pre,
+            index_urls=index_urls,
+            verbose=verbose,
         )
 
 
@@ -115,64 +130,107 @@ def install(
     deps: bool = True,
     credentials: str | None = None,
     pre: bool = False,
+    index_urls: list[str] | str | None = None,
+    *,
+    verbose: bool | int = False,
 ):
     """Install the given package and all of its dependencies.
-    See :ref:`loading packages <loading_packages>` for more information.
+
     If a package is not found in the Pyodide repository it will be loaded from
-    a piplite URL. If a package is not found in the piplite URL, it will be
-    loaded from PyPI. Piplite can only load pure Python packages or for packages
-    with C extensions that are built for Pyodide.
-    When used in web browsers, downloads from PyPI will be cached.
+    PyPI. Micropip can only load pure Python wheels or wasm32/emscripten wheels
+    built by Pyodide.
+
+    When used in web browsers, downloads from PyPI will be cached. When run in
+    Node.js, packages are currently not cached, and will be re-downloaded each
+    time ``micropip.install`` is run.
+
     Parameters
     ----------
-    requirements : ``str | List[str]``
+    requirements :
+
         A requirement or list of requirements to install. Each requirement is a
         string, which should be either a package name or a wheel URI:
+
         - If the requirement does not end in ``.whl``, it will be interpreted as
           a package name. A package with this name must either be present
           in the Pyodide lock file or on PyPI.
+
         - If the requirement ends in ``.whl``, it is a wheel URI. The part of
           the requirement after the last ``/``  must be a valid wheel name in
           compliance with the `PEP 427 naming convention
           <https://www.python.org/dev/peps/pep-0427/#file-format>`_.
+
         - If a wheel URI starts with ``emfs:``, it will be interpreted as a path
           in the Emscripten file system (Pyodide's file system). E.g.,
-          `emfs:../relative/path/wheel.whl` or `emfs:/absolute/path/wheel.whl`.
+          ``emfs:../relative/path/wheel.whl`` or ``emfs:/absolute/path/wheel.whl``.
           In this case, only .whl files are supported.
+
         - If a wheel URI requirement starts with ``http:`` or ``https:`` it will
           be interpreted as a URL.
-    keep_going : ``bool``, default: False
-        This parameter decides the behavior of the piplite when it encounters a
+
+        - In node, you can access the native file system using a URI that starts
+          with ``file:``. In the browser this will not work.
+
+    keep_going :
+
+        This parameter decides the behavior of the micropip when it encounters a
         Python package without a pure Python wheel while doing dependency
         resolution:
+
         - If ``False``, an error will be raised on first package with a missing
           wheel.
-        - If ``True``, the piplite will keep going after the first error, and
+
+        - If ``True``, the micropip will keep going after the first error, and
           report a list of errors at the end.
-    deps : ``bool``, default: True
+
+    deps :
+
         If ``True``, install dependencies specified in METADATA file for each
         package. Otherwise do not install dependencies.
-    credentials : ``Optional[str]``
+
+    credentials :
+
         This parameter specifies the value of ``credentials`` when calling the
         `fetch() <https://developer.mozilla.org/en-US/docs/Web/API/fetch>`__
         function which is used to download the package.
+
         When not specified, ``fetch()`` is called without ``credentials``.
-    pre : ``bool``, default: False
+
+    pre :
+
         If ``True``, include pre-release and development versions. By default,
-        piplite only finds stable versions.
-    Returns
-    -------
-    ``Future``
-        A ``Future`` that resolves to ``None`` when all packages have been
-        downloaded and installed.
+        micropip only finds stable versions.
+
+    index_urls :
+
+        A list of URLs or a single URL to use as the package index when looking
+        up packages. If None, *https://pypi.org/pypi/{package_name}/json* is used.
+
+        - The index URL should support the
+          `JSON API <https://warehouse.pypa.io/api-reference/json/>`__ .
+
+        - The index URL may contain the placeholder {package_name} which will be
+          replaced with the package name when looking up a package. If it does not
+          contain the placeholder, the package name will be appended to the URL.
+
+        - If a list of URLs is provided, micropip will try each URL in order until
+          it finds a package. If no package is found, an error will be raised.
+
+    verbose :
+        Print more information about the process.
+        By default, micropip is silent. Setting ``verbose=True`` will print
+        similar information as pip.
     """
+
     return asyncio.ensure_future(
         _install(
             requirements=requirements,
             keep_going=keep_going,
             deps=deps,
             credentials=credentials,
             pre=pre,
+            index_urls=index_urls,
+            verbose=verbose,
         )
     )
 

packages/pyodide-kernel/py/widgetsnbextension3/widgetsnbextension/widgetsnbextension/__init__.py

@@ -1,3 +1,3 @@
 """A widgetsnbextension mock"""
 
-__version__ = "3.6.4"
+__version__ = "3.6.6"

packages/pyodide-kernel/py/widgetsnbextension4/widgetsnbextension/widgetsnbextension/__init__.py

@@ -1,3 +1,3 @@
 """A widgetsnbextension mock"""
 
-__version__ = "4.0.7"
+__version__ = "4.0.9"

packages/pyodide-kernel/src/_pypi.ts

@@ -3,5 +3,5 @@ export * as allJSONUrl from '../pypi/all.json';
 export * as ipykernelWheelUrl from '../pypi/ipykernel-6.9.2-py3-none-any.whl';
 export * as pipliteWheelUrl from '../pypi/piplite-0.1.1-py3-none-any.whl';
 export * as pyodide_kernelWheelUrl from '../pypi/pyodide_kernel-0.1.1-py3-none-any.whl';
-export * as widgetsnbextensionWheelUrl from '../pypi/widgetsnbextension-3.6.4-py3-none-any.whl';
-export * as widgetsnbextensionWheelUrl1 from '../pypi/widgetsnbextension-4.0.7-py3-none-any.whl';
+export * as widgetsnbextensionWheelUrl from '../pypi/widgetsnbextension-3.6.6-py3-none-any.whl';
+export * as widgetsnbextensionWheelUrl1 from '../pypi/widgetsnbextension-4.0.9-py3-none-any.whl';