From 3e88d1bf783e3917d07c214afdffaf2984f45380 Mon Sep 17 00:00:00 2001 From: DerThorsten Date: Fri, 8 Mar 2024 10:44:41 +0100 Subject: [PATCH 1/3] add mounts to empack packed environment --- pyjs_code_runner/js/utils.js | 2 +- pyjs_code_runner/run.py | 16 +++++++++++++--- 2 files changed, 14 insertions(+), 4 deletions(-) diff --git a/pyjs_code_runner/js/utils.js b/pyjs_code_runner/js/utils.js index 61dd8ba..7f40a6e 100644 --- a/pyjs_code_runner/js/utils.js +++ b/pyjs_code_runner/js/utils.js @@ -26,7 +26,7 @@ async function make_pyjs(print, error) { false /* verbose */ ); - await fetchMounts(pyjs); + // await fetchMounts(pyjs); //await Promise.all([promise_env, promise_mount]); //await pyjs.init() diff --git a/pyjs_code_runner/run.py b/pyjs_code_runner/run.py index 6400d81..57b417a 100644 --- a/pyjs_code_runner/run.py +++ b/pyjs_code_runner/run.py @@ -4,7 +4,7 @@ import tempfile from pathlib import Path -from empack.pack import pack_env, pack_directory +from empack.pack import pack_env, pack_directory,add_tarfile_to_env_meta from empack.pack import DEFAULT_CONFIG_PATH as EMPACK_DEFAULT_CONFIG_PATH from empack.file_patterns import pkg_file_filter_from_yaml @@ -36,6 +36,7 @@ def pack_mounts(mounts, host_work_dir, backend_type): ) with open(Path(host_work_dir) / "mounts.json", "w") as f: json.dump(mount_js_files, f, indent=4) + return mount_js_files def conda_env_to_cache_name(conda_env, backend_type): @@ -124,14 +125,23 @@ def run( cache_dir=cache_dir, outdir=host_work_dir, compresslevel=9, - ) + ) + env_meta_filename = Path(host_work_dir) / "empack_env_meta.json" + + + #raise RuntimeError("stop here") # pack all the mounts - pack_mounts( + mount_files = pack_mounts( mounts=mounts, backend_type=backend_type, host_work_dir=host_work_dir, ) + for mount_file in mount_files: + print(f"mount_file: {mount_file}") + add_tarfile_to_env_meta( + env_meta_filename=env_meta_filename, tarfile=mount_file + ) # get the backend where the wasm code runs (ie node/browser-main/browser-worker) backend = get_backend_cls(backend_type=backend_type)( From d5a47792618978f5fe29c9fb5a10da55e8fc75c7 Mon Sep 17 00:00:00 2001 From: DerThorsten Date: Mon, 11 Mar 2024 08:34:23 +0100 Subject: [PATCH 2/3] try catch arround empack --- .github/workflows/main.yaml | 6 ++++++ pyjs_code_runner/run.py | 9 ++++++--- 2 files changed, 12 insertions(+), 3 deletions(-) diff --git a/.github/workflows/main.yaml b/.github/workflows/main.yaml index 60463cd..eab04ce 100644 --- a/.github/workflows/main.yaml +++ b/.github/workflows/main.yaml @@ -30,6 +30,12 @@ jobs: - name: Install Playwright run: | playwright install + + # # temporary workaround to get main from empack + # - name: install empack main + # run: | + # micromamba activate dev-env + # pip install git+https://github.com/emscripten-forge/empack.git@main --no-deps --no-build-isolation - name: Install pyjs-code-runner run: | diff --git a/pyjs_code_runner/run.py b/pyjs_code_runner/run.py index 57b417a..f01d783 100644 --- a/pyjs_code_runner/run.py +++ b/pyjs_code_runner/run.py @@ -139,9 +139,12 @@ def run( ) for mount_file in mount_files: print(f"mount_file: {mount_file}") - add_tarfile_to_env_meta( - env_meta_filename=env_meta_filename, tarfile=mount_file - ) + try: + add_tarfile_to_env_meta( + env_meta_filename=env_meta_filename, tarfile=mount_file + ) + except shutil.SameFileError: + pass # get the backend where the wasm code runs (ie node/browser-main/browser-worker) backend = get_backend_cls(backend_type=backend_type)( From ec297c023bb17ca5fd1b79efbf20331ff3b5005a Mon Sep 17 00:00:00 2001 From: DerThorsten Date: Mon, 11 Mar 2024 08:37:58 +0100 Subject: [PATCH 3/3] use mounts --- .github/workflows/main.yaml | 6 ------ pyjs_code_runner/js/utils.js | 6 ------ 2 files changed, 12 deletions(-) diff --git a/.github/workflows/main.yaml b/.github/workflows/main.yaml index eab04ce..60463cd 100644 --- a/.github/workflows/main.yaml +++ b/.github/workflows/main.yaml @@ -30,12 +30,6 @@ jobs: - name: Install Playwright run: | playwright install - - # # temporary workaround to get main from empack - # - name: install empack main - # run: | - # micromamba activate dev-env - # pip install git+https://github.com/emscripten-forge/empack.git@main --no-deps --no-build-isolation - name: Install pyjs-code-runner run: | diff --git a/pyjs_code_runner/js/utils.js b/pyjs_code_runner/js/utils.js index 7f40a6e..ccb558d 100644 --- a/pyjs_code_runner/js/utils.js +++ b/pyjs_code_runner/js/utils.js @@ -25,12 +25,6 @@ async function make_pyjs(print, error) { ".", /* package_tarballs_root_url */ false /* verbose */ ); - - // await fetchMounts(pyjs); - //await Promise.all([promise_env, promise_mount]); - - //await pyjs.init() - globalThis.pyjs = pyjs return pyjs }