diff --git a/.github/workflows/build.yml b/.github/workflows/build.yml index 34b9eb0..f565c2b 100644 --- a/.github/workflows/build.yml +++ b/.github/workflows/build.yml @@ -144,7 +144,8 @@ jobs: run: pip install jupyterlite-xeus-python.tar.gz - name: Run tests - run: pytest -rP tests/test_xeus_python_env.py + run: pytest -rP test_xeus_python_env.py + working-directory: tests python-tests-mamba: needs: build @@ -170,7 +171,8 @@ jobs: run: pip install jupyterlite-xeus-python.tar.gz - name: Run tests - run: pytest -rP tests/test_xeus_python_env.py + run: pytest -rP test_xeus_python_env.py + working-directory: tests python-tests-micromamba: needs: build @@ -195,7 +197,8 @@ jobs: run: pip install jupyterlite-xeus-python.tar.gz - name: Run tests - run: pytest -rP tests/test_xeus_python_env.py + run: pytest -rP test_xeus_python_env.py + working-directory: tests python-tests-conda: needs: build @@ -220,4 +223,5 @@ jobs: run: pip install jupyterlite-xeus-python.tar.gz - name: Run tests - run: pytest -rP tests/test_xeus_python_env.py + run: pytest -rP test_xeus_python_env.py + working-directory: tests diff --git a/.readthedocs.yaml b/.readthedocs.yaml index 3ae3e97..9285f5d 100644 --- a/.readthedocs.yaml +++ b/.readthedocs.yaml @@ -6,4 +6,4 @@ build: python: "mambaforge-4.10" conda: - environment: docs/environment.yml + environment: docs/build-environment.yml diff --git a/README.md b/README.md index d9b49c3..73f1dc9 100644 --- a/README.md +++ b/README.md @@ -31,25 +31,39 @@ jupyter lite build ## Pre-installed packages -xeus-python allows you to pre-install packages in the Python runtime. You can pre-install packages by passing the `XeusPythonEnv.packages` CLI option to `jupyter lite build`. -This will automatically install any labextension that it founds, for example installing ipyleaflet will make ipyleaflet work without the need to manually install the jupyter-leaflet labextension. +xeus-python allows you to pre-install packages in the Python runtime. You can pre-install packages by adding an `environment.yml` file in the JupyterLite build directory, this file will be found automatically by xeus-python which will pre-build the environment when running `jupyter lite build`. -For example, say you want to install `NumPy`, `Matplotlib` and `ipyleaflet`, it can be done with the following command: +Furthermore, this automatically installs any labextension that it founds, for example installing ipyleaflet will make ipyleaflet work without the need to manually install the jupyter-leaflet labextension. + +Say you want to install `NumPy`, `Matplotlib` and `ipyleaflet`, it can be done by creating the `environment.yml` file with the following content: + +```yml +name: xeus-python-kernel +channels: + - https://repo.mamba.pm/emscripten-forge + - https://repo.mamba.pm/conda-forge +dependencies: + - numpy + - matplotlib + - ipycanvas +``` + +Then you only need to build JupyterLite: ```bash -jupyter lite build --XeusPythonEnv.packages=numpy,matplotlib,ipyleaflet +jupyter lite build ``` -The same can be achieved through a `jupyterlite_config.json` file: +You can also pick another name for that environment file (*e.g.* `custom.yml`), by doing so, you will need to specify that name to xeus-python: -```json -{ - "XeusPythonEnv": { - "packages": ["numpy", "matplotlib", "ipyleaflet"] - } -} +```bash +jupyter lite build --XeusPythonEnv.environment_file=custom.yml ``` +#### About pip dependencies + +It is common to provide `pip` dependencies in a conda environment file, this is currently **not supported** by xeus-python. + ## Contributing ### Development install diff --git a/docs/build-environment.yml b/docs/build-environment.yml new file mode 100644 index 0000000..fa2c74c --- /dev/null +++ b/docs/build-environment.yml @@ -0,0 +1,13 @@ +name: xeus-python-kernel-docs + +channels: + - conda-forge + +dependencies: + - mamba + - pydata-sphinx-theme + - pip + - pip: + - jupyterlite==0.1.0b16 + - jupyterlite-sphinx + - jupyterlite-xeus-python==0.6.0 diff --git a/docs/conf.py b/docs/conf.py index 047eaee..09a424d 100644 --- a/docs/conf.py +++ b/docs/conf.py @@ -15,5 +15,4 @@ html_theme = "pydata_sphinx_theme" -jupyterlite_config = "jupyterlite_config.json" jupyterlite_dir = "." diff --git a/docs/configuration.rst b/docs/configuration.rst index 8ce21bd..7c95b46 100644 --- a/docs/configuration.rst +++ b/docs/configuration.rst @@ -6,26 +6,37 @@ Configuration Pre-installed packages ---------------------- -xeus-python allows you to pre-install packages in the Python runtime. You can pre-install packages by passing the ``XeusPythonEnv.packages`` CLI option to ``jupyter lite build``. +``xeus-python`` allows you to pre-install packages in the Python runtime. You can pre-install packages by adding an ``environment.yml`` file in the JupyterLite build directory, this file will be found automatically by xeus-python which will pre-build the environment when running `jupyter lite build`. -.. note:: - This will automatically install any labextension that it founds, for example installing ipyleaflet will make ipyleaflet work without the need to manually install the jupyter-leaflet labextension. +Furthermore, this automatically installs any labextension that it founds, for example installing ipyleaflet will make ipyleaflet work without the need to manually install the jupyter-leaflet labextension. -For example, say you want to install ``NumPy``, ``Matplotlib`` and ``ipyleaflet``, it can be done with the following command: +Say you want to install ``NumPy``, ``Matplotlib`` and ``ipyleaflet``, it can be done by creating the ``environment.yml`` file with the following content: .. code:: - jupyter lite build --XeusPythonEnv.packages=numpy,matplotlib,ipyleaflet + name: xeus-python-kernel + channels: + - https://repo.mamba.pm/emscripten-forge + - https://repo.mamba.pm/conda-forge + dependencies: + - numpy + - matplotlib + - ipycanvas -The same can be achieved through a ``jupyterlite_config.json`` file: +Then you only need to build JupyterLite: .. code:: - { - "XeusPythonEnv": { - "packages": ["numpy", "matplotlib", "ipyleaflet"] - } - } + jupyter lite build + +You can also pick another name for that environment file (*e.g.* `custom.yml`), by doing so, you will need to specify that name to xeus-python: + +.. code:: + + jupyter lite build --XeusPythonEnv.environment_file=custom.yml + +.. note:: + It is common to provide `pip` dependencies in a conda environment file, this is currently **not supported** by xeus-python. Then those packages are usable directly: diff --git a/docs/environment.yml b/docs/environment.yml index fa2c74c..751e008 100644 --- a/docs/environment.yml +++ b/docs/environment.yml @@ -1,13 +1,8 @@ name: xeus-python-kernel-docs - channels: - - conda-forge - + - https://repo.mamba.pm/emscripten-forge + - https://repo.mamba.pm/conda-forge dependencies: - - mamba - - pydata-sphinx-theme - - pip - - pip: - - jupyterlite==0.1.0b16 - - jupyterlite-sphinx - - jupyterlite-xeus-python==0.6.0 + - numpy + - matplotlib + - ipycanvas diff --git a/docs/jupyterlite_config.json b/docs/jupyterlite_config.json deleted file mode 100644 index 189143c..0000000 --- a/docs/jupyterlite_config.json +++ /dev/null @@ -1,5 +0,0 @@ -{ - "XeusPythonEnv": { - "packages": ["numpy", "matplotlib", "ipyleaflet"] - } -} diff --git a/jupyterlite_xeus_python/env_build_addon.py b/jupyterlite_xeus_python/env_build_addon.py index 7b4ea19..f77914a 100644 --- a/jupyterlite_xeus_python/env_build_addon.py +++ b/jupyterlite_xeus_python/env_build_addon.py @@ -91,16 +91,11 @@ class XeusPythonEnv(FederatedExtensionAddon): description="A comma-separated list of packages to install in the xeus-python env", ) - @property - def specs(self): - """The package specs to install in the environment.""" - return [ - f"python={PYTHON_VERSION}", - "xeus-python" - if not self.xeus_python_version - else f"xeus-python={self.xeus_python_version}", - *self.packages, - ] + environment_file = Unicode( + "environment.yml", + config=True, + description="The path to the environment file. Defaults to \"environment.yml\"", + ) @property def prefix_path(self): @@ -113,6 +108,14 @@ def __init__(self, *args, **kwargs): self.cwd = TemporaryDirectory() self.root_prefix = "/tmp/xeus-python-kernel" self.env_name = "xeus-python-kernel" + self.channels = CHANNELS + self.specs = [ + f"python={PYTHON_VERSION}", + "xeus-python" + if not self.xeus_python_version + else f"xeus-python={self.xeus_python_version}", + *self.packages, + ] # Cleanup tmp dir in case it's not empty shutil.rmtree(Path(self.root_prefix) / "envs", ignore_errors=True) @@ -128,8 +131,41 @@ def post_build(self, manager): if pkg_data.get("name") == JUPYTERLITE_XEUS_PYTHON: yield from self.safe_copy_extension(pkg_json) - # Bail early if there is no extra package to install - if not self.packages and not self.xeus_python_version: + bail_early = True + if self.packages or self.xeus_python_version: + bail_early = False + + # Process environment.yml file + if (Path(self.manager.lite_dir) / self.environment_file).exists(): + bail_early = False + + with open(Path(self.manager.lite_dir) / self.environment_file) as f: + env_data = yaml.safe_load(f) + + if env_data.get("name") is not None: + self.env_name = env_data["name"] + + if env_data.get("channels") is not None: + channels = env_data["channels"] + + for channel in channels: + if channel not in self.channels: + self.channels.append(channel) + + if env_data.get("dependencies") is not None: + dependencies = env_data["dependencies"] + + for dependency in dependencies: + if isinstance(dependency, str) and dependency not in self.specs: + self.specs.append(dependency) + elif isinstance(dependency, dict) and dependency.get("pip") is not None: + raise RuntimeError( + """Cannot install pip dependencies in the xeus-python Emscripten environment (yet?). + """ + ) + + # Bail early if there is nothing to do + if bail_early: return [] # Create emscripten env with the given packages @@ -207,13 +243,13 @@ def create_env(self): env_name=self.env_name, base_prefix=self.root_prefix, specs=self.specs, - channels=CHANNELS, + channels=self.channels, target_platform=PLATFORM, ) return channels = [] - for channel in CHANNELS: + for channel in self.channels: channels.extend(["-c", channel]) if MAMBA_AVAILABLE: diff --git a/tests/environment-1.yml b/tests/environment-1.yml new file mode 100644 index 0000000..d4ad7f9 --- /dev/null +++ b/tests/environment-1.yml @@ -0,0 +1,8 @@ +name: xeus-python-kernel-1 +channels: + - https://repo.mamba.pm/emscripten-forge + - https://repo.mamba.pm/conda-forge +dependencies: + - numpy + - matplotlib + - ipycanvas diff --git a/tests/test_xeus_python_env.py b/tests/test_xeus_python_env.py index 49a7fdd..d4352ca 100644 --- a/tests/test_xeus_python_env.py +++ b/tests/test_xeus_python_env.py @@ -29,3 +29,31 @@ def test_python_env(): # Check empack output assert os.path.isfile(Path(addon.cwd.name) / "python_data.js") assert os.path.isfile(Path(addon.cwd.name) / "python_data.data") + + os.remove(Path(addon.cwd.name) / "python_data.js") + os.remove(Path(addon.cwd.name) / "python_data.data") + + +def test_python_env_from_file_1(): + app = LiteStatusApp(log_level="DEBUG") + app.initialize() + manager = app.lite_manager + + addon = XeusPythonEnv(manager) + addon.environment_file = "environment-1.yml" + + for step in addon.post_build(manager): + pass + + # Check env + assert os.path.isdir("/tmp/xeus-python-kernel/envs/xeus-python-kernel-1") + + assert os.path.isfile("/tmp/xeus-python-kernel/envs/xeus-python-kernel-1/bin/xpython_wasm.js") + assert os.path.isfile("/tmp/xeus-python-kernel/envs/xeus-python-kernel-1/bin/xpython_wasm.wasm") + + # Check empack output + assert os.path.isfile(Path(addon.cwd.name) / "python_data.js") + assert os.path.isfile(Path(addon.cwd.name) / "python_data.data") + + os.remove(Path(addon.cwd.name) / "python_data.js") + os.remove(Path(addon.cwd.name) / "python_data.data")