Skip to content

Commit

Permalink
pyodide 0.24.0, widget patches
Browse files Browse the repository at this point in the history
  • Loading branch information
bollwyvl committed Sep 14, 2023
1 parent 11bf328 commit 58c9e86
Show file tree
Hide file tree
Showing 13 changed files with 2,577 additions and 1,548 deletions.
5 changes: 3 additions & 2 deletions .prettierrc
Original file line number Diff line number Diff line change
@@ -1,5 +1,6 @@
{
"singleQuote": true,
"printWidth": 88,
"proseWrap": "always"
}
"proseWrap": "always",
"trailingComma": "all"
}
8 changes: 4 additions & 4 deletions jupyterlite_pyodide_kernel/addons/pyodide.py
Original file line number Diff line number Diff line change
Expand Up @@ -15,7 +15,7 @@
from ..constants import (
PYODIDE,
PYODIDE_JS,
PYODIDE_REPODATA,
PYODIDE_LOCK,
PYODIDE_URL,
)

Expand Down Expand Up @@ -151,8 +151,8 @@ def check_config_paths(self, jupyterlite_json):
assert pyodide_path.exists(), f"{pyodide_path} not found"
pyodide_js = pyodide_path / PYODIDE_JS
assert pyodide_js.exists(), f"{pyodide_js} not found"
pyodide_repodata = pyodide_path / PYODIDE_REPODATA
assert pyodide_repodata.exists(), f"{pyodide_repodata} not found"
pyodide_lock = pyodide_path / PYODIDE_LOCK
assert pyodide_lock.exists(), f"{pyodide_lock} not found"

def patch_jupyterlite_json(self, config_path, output_js):
"""update jupyter-lite.json to use the local pyodide"""
Expand Down Expand Up @@ -216,7 +216,7 @@ def extract_pyodide(self, local_path, dest):
targets=[
# there are a lot of js/data files, but we actually talk about these...
dest / PYODIDE / PYODIDE_JS,
dest / PYODIDE / PYODIDE_REPODATA,
dest / PYODIDE / PYODIDE_LOCK,
],
actions=[(self.extract_one, [local_path, dest])],
)
4 changes: 2 additions & 2 deletions jupyterlite_pyodide_kernel/constants.py
Original file line number Diff line number Diff line change
Expand Up @@ -25,11 +25,11 @@
#: where we put pyodide, for now
PYODIDE = "pyodide"
PYODIDE_JS = "pyodide.js"
PYODIDE_REPODATA = "repodata.json"
PYODIDE_LOCK = "pyodide-lock.json"
PYODIDE_URL_ENV_VAR = "JUPYTERLITE_PYODIDE_URL"

#: probably only compatible with this version of pyodide
PYODIDE_VERSION = "0.23.4"
PYODIDE_VERSION = "0.24.0"

#: the only kind of noarch wheel piplite understands
NOARCH_WHL = "py3-none-any.whl"
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -8,7 +8,7 @@
"pyodideUrl": {
"description": "The path to the main pyodide.js entry point",
"type": "string",
"default": "https://cdn.jsdelivr.net/pyodide/v0.23.4/full/pyodide.js",
"default": "https://cdn.jsdelivr.net/pyodide/v0.24.0/full/pyodide.js",
"format": "uri"
},
"disablePyPIFallback": {
Expand Down
4 changes: 2 additions & 2 deletions packages/pyodide-kernel-extension/src/index.ts
Original file line number Diff line number Diff line change
Expand Up @@ -20,7 +20,7 @@ const KERNEL_ICON_URL = `data:image/svg+xml;base64,${btoa(KERNEL_ICON_SVG_STR)}`
/**
* The default CDN fallback for Pyodide
*/
const PYODIDE_CDN_URL = 'https://cdn.jsdelivr.net/pyodide/v0.23.4/full/pyodide.js';
const PYODIDE_CDN_URL = 'https://cdn.jsdelivr.net/pyodide/v0.24.0/full/pyodide.js';

/**
* The id for the extension, and key in the litePlugins.
Expand All @@ -39,7 +39,7 @@ const kernel: JupyterLiteServerPlugin<void> = {
app: JupyterLiteServer,
kernelspecs: IKernelSpecs,
serviceWorker?: IServiceWorkerManager,
broadcastChannel?: IBroadcastChannelWrapper
broadcastChannel?: IBroadcastChannelWrapper,
) => {
const config =
JSON.parse(PageConfig.getOption('litePluginSettings') || '{}')[PLUGIN_ID] || {};
Expand Down
6 changes: 3 additions & 3 deletions packages/pyodide-kernel/package.json
Original file line number Diff line number Diff line change
Expand Up @@ -62,7 +62,7 @@
"@types/jest": "^26.0.10",
"esbuild": "0.17.10",
"jest": "^26.4.2",
"pyodide": "0.23.4",
"pyodide": "0.24.0",
"rimraf": "~3.0.0",
"ts-jest": "^26.3.0",
"typescript": "~4.9.4"
Expand All @@ -75,8 +75,8 @@
"py/pyodide-kernel": "0.1.1",
"py/piplite": "0.1.1",
"py/ipykernel": "6.9.2",
"py/widgetsnbextension3/widgetsnbextension": "3.6.4",
"py/widgetsnbextension4/widgetsnbextension": "4.0.7"
"py/widgetsnbextension3/widgetsnbextension": "3.6.6",
"py/widgetsnbextension4/widgetsnbextension": "4.0.9"
}
},
"styleModule": "style/index.js"
Expand Down
156 changes: 107 additions & 49 deletions packages/pyodide-kernel/py/piplite/piplite/piplite.py
Original file line number Diff line number Diff line change
Expand Up @@ -6,13 +6,19 @@
`pyodide-kernel` also includes a browser shim for the IPython `%pip` magic
"""
from typing import Any
import asyncio
import json
import logging
from unittest.mock import patch

from micropip import _micropip
from micropip._micropip import _get_pypi_json as _MP_GET_PYPI_JSON
from micropip._micropip import fetch_string as _MP_FETCH_STRING
import micropip
from micropip.package_index import ProjectInfo
from micropip.package_index import query_package as _MP_QUERY_PACKAGE
from micropip.package_index import fetch_string_and_headers as _MP_FETCH_STRING

logger = logging.getLogger(__name__)


#: a list of Warehouse-like API endpoints or derived multi-package all.json
_PIPLITE_URLS = []
Expand All @@ -34,20 +40,25 @@ class PiplitePyPIDisabled(ValueError):
pass


async def _get_pypi_json_from_index(pkgname, piplite_url, fetch_kwargs):
async def _get_pypi_json_from_index(name, piplite_url, fetch_kwargs) -> ProjectInfo:
"""Attempt to load a specific ``pkgname``'s releases from a specific piplite
URL's index.
"""
index = _PIPLITE_INDICES.get(piplite_url, {})

if not index:
try:
index = json.loads(await _MP_FETCH_STRING(piplite_url, fetch_kwargs))
data, headers = await _MP_FETCH_STRING(piplite_url, fetch_kwargs)
except Exception as err:
logger.warn("Could not fetch %s: %s", piplite_url, err)

try:
index = json.loads(data)
_PIPLITE_INDICES.update({piplite_url: index})
except Exception:
pass
except Exception as err:
logger.warn("Could not parse %s: %s", piplite_url, err)

pkg = dict((index or {}).get(pkgname) or {})
pkg = dict((index or {}).get(name) or {})

if not pkg:
return None
Expand All @@ -58,37 +69,36 @@ async def _get_pypi_json_from_index(pkgname, piplite_url, fetch_kwargs):
if artifact["url"].startswith("."):
artifact["url"] = (
f"""{piplite_url.split(ALL_JSON)[0]}/{artifact["url"]}"""
# can't add cache busting because micropip 0.21 checks `endswith`
# f"""?sha256={artifact["digests"]["sha256"]}"""
f"""?sha256={artifact["digests"]["sha256"]}"""
)

return pkg

info = ProjectInfo._compatible_only(name, pkg["releases"])
return info

async def _get_pypi_json(pkgname, fetch_kwargs):
"""Fetch the warehoust API metadata for a specific ``pkgname``."""

async def _query_package(
name: str,
fetch_kwargs: dict[str, Any] | None = None,
index_urls: list[str] | str | None = None,
) -> ProjectInfo:
"""Fetch the warehouse API metadata for a specific ``pkgname``."""
for piplite_url in _PIPLITE_URLS:
if piplite_url.split("?")[0].split("#")[0].endswith(ALL_JSON):
pypi_json_from_index = await _get_pypi_json_from_index(
pkgname, piplite_url, fetch_kwargs
)
if pypi_json_from_index:
return pypi_json_from_index
if not piplite_url.split("?")[0].split("#")[0].endswith(ALL_JSON):
logger.warn("Non-all.json piplite URL not supported %s", piplite_url)
continue
else:
try:
url = f"{piplite_url}{pkgname}/json"
return json.loads(await _MP_FETCH_STRING(url, fetch_kwargs))
except Exception:
pass

pypi_json_from_index = await _get_pypi_json_from_index(
name, piplite_url, fetch_kwargs
)
if pypi_json_from_index:
return pypi_json_from_index

if _PIPLITE_DISABLE_PYPI:
raise PiplitePyPIDisabled(
f"{pkgname} could not be installed: PyPI fallback is disabled"
f"{name} could not be installed: PyPI fallback is disabled"
)

return await _MP_GET_PYPI_JSON(pkgname, fetch_kwargs)
return await _MP_QUERY_PACKAGE(name, fetch_kwargs, index_urls)


async def _install(
Expand All @@ -97,15 +107,20 @@ async def _install(
deps: bool = True,
credentials: str | None = None,
pre: bool = False,
index_urls: list[str] | str | None = None,
*,
verbose: bool | int = False,
):
"""Invoke micropip.install with a patch to get data from local indexes"""
with patch("micropip._micropip._get_pypi_json", _get_pypi_json):
return await _micropip.install(
with patch("micropip.package_index.query_package", _query_package):
return await micropip.install(
requirements=requirements,
keep_going=keep_going,
deps=deps,
credentials=credentials,
pre=pre,
index_urls=index_urls,
verbose=verbose,
)


Expand All @@ -115,64 +130,107 @@ def install(
deps: bool = True,
credentials: str | None = None,
pre: bool = False,
index_urls: list[str] | str | None = None,
*,
verbose: bool | int = False,
):
"""Install the given package and all of its dependencies.
See :ref:`loading packages <loading_packages>` for more information.
If a package is not found in the Pyodide repository it will be loaded from
a piplite URL. If a package is not found in the piplite URL, it will be
loaded from PyPI. Piplite can only load pure Python packages or for packages
with C extensions that are built for Pyodide.
When used in web browsers, downloads from PyPI will be cached.
PyPI. Micropip can only load pure Python wheels or wasm32/emscripten wheels
built by Pyodide.
When used in web browsers, downloads from PyPI will be cached. When run in
Node.js, packages are currently not cached, and will be re-downloaded each
time ``micropip.install`` is run.
Parameters
----------
requirements : ``str | List[str]``
requirements :
A requirement or list of requirements to install. Each requirement is a
string, which should be either a package name or a wheel URI:
- If the requirement does not end in ``.whl``, it will be interpreted as
a package name. A package with this name must either be present
in the Pyodide lock file or on PyPI.
- If the requirement ends in ``.whl``, it is a wheel URI. The part of
the requirement after the last ``/`` must be a valid wheel name in
compliance with the `PEP 427 naming convention
<https://www.python.org/dev/peps/pep-0427/#file-format>`_.
- If a wheel URI starts with ``emfs:``, it will be interpreted as a path
in the Emscripten file system (Pyodide's file system). E.g.,
`emfs:../relative/path/wheel.whl` or `emfs:/absolute/path/wheel.whl`.
``emfs:../relative/path/wheel.whl`` or ``emfs:/absolute/path/wheel.whl``.
In this case, only .whl files are supported.
- If a wheel URI requirement starts with ``http:`` or ``https:`` it will
be interpreted as a URL.
keep_going : ``bool``, default: False
This parameter decides the behavior of the piplite when it encounters a
- In node, you can access the native file system using a URI that starts
with ``file:``. In the browser this will not work.
keep_going :
This parameter decides the behavior of the micropip when it encounters a
Python package without a pure Python wheel while doing dependency
resolution:
- If ``False``, an error will be raised on first package with a missing
wheel.
- If ``True``, the piplite will keep going after the first error, and
- If ``True``, the micropip will keep going after the first error, and
report a list of errors at the end.
deps : ``bool``, default: True
deps :
If ``True``, install dependencies specified in METADATA file for each
package. Otherwise do not install dependencies.
credentials : ``Optional[str]``
credentials :
This parameter specifies the value of ``credentials`` when calling the
`fetch() <https://developer.mozilla.org/en-US/docs/Web/API/fetch>`__
function which is used to download the package.
When not specified, ``fetch()`` is called without ``credentials``.
pre : ``bool``, default: False
pre :
If ``True``, include pre-release and development versions. By default,
piplite only finds stable versions.
Returns
-------
``Future``
A ``Future`` that resolves to ``None`` when all packages have been
downloaded and installed.
micropip only finds stable versions.
index_urls :
A list of URLs or a single URL to use as the package index when looking
up packages. If None, *https://pypi.org/pypi/{package_name}/json* is used.
- The index URL should support the
`JSON API <https://warehouse.pypa.io/api-reference/json/>`__ .
- The index URL may contain the placeholder {package_name} which will be
replaced with the package name when looking up a package. If it does not
contain the placeholder, the package name will be appended to the URL.
- If a list of URLs is provided, micropip will try each URL in order until
it finds a package. If no package is found, an error will be raised.
verbose :
Print more information about the process.
By default, micropip is silent. Setting ``verbose=True`` will print
similar information as pip.
"""

return asyncio.ensure_future(
_install(
requirements=requirements,
keep_going=keep_going,
deps=deps,
credentials=credentials,
pre=pre,
index_urls=index_urls,
verbose=verbose,
)
)

Expand Down
Original file line number Diff line number Diff line change
@@ -1,3 +1,3 @@
"""A widgetsnbextension mock"""

__version__ = "3.6.4"
__version__ = "3.6.6"
Original file line number Diff line number Diff line change
@@ -1,3 +1,3 @@
"""A widgetsnbextension mock"""

__version__ = "4.0.7"
__version__ = "4.0.9"
4 changes: 2 additions & 2 deletions packages/pyodide-kernel/src/_pypi.ts
Original file line number Diff line number Diff line change
Expand Up @@ -3,5 +3,5 @@ export * as allJSONUrl from '../pypi/all.json';
export * as ipykernelWheelUrl from '../pypi/ipykernel-6.9.2-py3-none-any.whl';
export * as pipliteWheelUrl from '../pypi/piplite-0.1.1-py3-none-any.whl';
export * as pyodide_kernelWheelUrl from '../pypi/pyodide_kernel-0.1.1-py3-none-any.whl';
export * as widgetsnbextensionWheelUrl from '../pypi/widgetsnbextension-3.6.4-py3-none-any.whl';
export * as widgetsnbextensionWheelUrl1 from '../pypi/widgetsnbextension-4.0.7-py3-none-any.whl';
export * as widgetsnbextensionWheelUrl from '../pypi/widgetsnbextension-3.6.6-py3-none-any.whl';
export * as widgetsnbextensionWheelUrl1 from '../pypi/widgetsnbextension-4.0.9-py3-none-any.whl';
Loading

0 comments on commit 58c9e86

Please sign in to comment.