From 8b4faf98d456d9939c8e6fcd9fba5c4a47c8637b Mon Sep 17 00:00:00 2001 From: Timur Bazhirov Date: Thu, 12 Dec 2024 19:39:46 -0800 Subject: [PATCH 1/2] chore: cleanup packages install --- .../defect_point_pair_gallium_nitride.ipynb | 2 +- .../defect_point_substitution_graphene.ipynb | 2 +- .../defect_surface_adatom_graphene.ipynb | 3 +- ...fect_surface_island_titanium_nitride.ipynb | 1 - utils/jupyterlite.py | 77 ++++++++++--------- 5 files changed, 43 insertions(+), 42 deletions(-) diff --git a/other/materials_designer/specific_examples/defect_point_pair_gallium_nitride.ipynb b/other/materials_designer/specific_examples/defect_point_pair_gallium_nitride.ipynb index 916fa9b0..e585a0b1 100644 --- a/other/materials_designer/specific_examples/defect_point_pair_gallium_nitride.ipynb +++ b/other/materials_designer/specific_examples/defect_point_pair_gallium_nitride.ipynb @@ -73,7 +73,7 @@ " await micropip.install('mat3ra-api-examples', deps=False)\n", " from utils.jupyterlite import install_packages\n", "\n", - " await install_packages(\"create_point_defect.ipynb\")" + " await install_packages(\"specific_examples|create_point_defect.ipynb\")" ], "metadata": { "collapsed": false diff --git a/other/materials_designer/specific_examples/defect_point_substitution_graphene.ipynb b/other/materials_designer/specific_examples/defect_point_substitution_graphene.ipynb index ad7df361..28e83451 100644 --- a/other/materials_designer/specific_examples/defect_point_substitution_graphene.ipynb +++ b/other/materials_designer/specific_examples/defect_point_substitution_graphene.ipynb @@ -105,7 +105,7 @@ " await micropip.install('mat3ra-api-examples', deps=False)\n", " from utils.jupyterlite import install_packages\n", "\n", - " await install_packages(\"create_point_defect.ipynb\")" + " await install_packages(\"specific_examples|create_point_defect.ipynb\")" ], "metadata": { "collapsed": false diff --git a/other/materials_designer/specific_examples/defect_surface_adatom_graphene.ipynb b/other/materials_designer/specific_examples/defect_surface_adatom_graphene.ipynb index 756c83d8..1a328d11 100644 --- a/other/materials_designer/specific_examples/defect_surface_adatom_graphene.ipynb +++ b/other/materials_designer/specific_examples/defect_surface_adatom_graphene.ipynb @@ -100,8 +100,7 @@ " await micropip.install('mat3ra-api-examples', deps=False)\n", " from utils.jupyterlite import install_packages\n", "\n", - " await install_packages(\"create_point_defect.ipynb\")\n", - " await install_packages(\"specific_examples\")" + " await install_packages(\"specific_examples|create_point_defect.ipynb\")" ], "metadata": { "collapsed": false diff --git a/other/materials_designer/specific_examples/defect_surface_island_titanium_nitride.ipynb b/other/materials_designer/specific_examples/defect_surface_island_titanium_nitride.ipynb index abfe5f2d..ed2d5aad 100644 --- a/other/materials_designer/specific_examples/defect_surface_island_titanium_nitride.ipynb +++ b/other/materials_designer/specific_examples/defect_surface_island_titanium_nitride.ipynb @@ -86,7 +86,6 @@ " await micropip.install('mat3ra-api-examples', deps=False)\n", " from utils.jupyterlite import install_packages\n", "\n", - " await install_packages(\"\")\n", " await install_packages(\"specific_examples\")" ], "metadata": { diff --git a/utils/jupyterlite.py b/utils/jupyterlite.py index 2defea88..98c2c7c2 100644 --- a/utils/jupyterlite.py +++ b/utils/jupyterlite.py @@ -1,6 +1,7 @@ import inspect import json import os +import re import sys from enum import Enum from typing import Any, Dict, List, Optional @@ -100,13 +101,13 @@ def install_package_python(pkg: str, verbose: bool = True): log(f"Installed {pkg}", force_verbose=verbose) -async def install_packages(notebook_name: str, requirements_path: str = "", verbose: bool = True): +async def install_packages(notebook_name_pattern: str, config_file_path: str = "", verbose: bool = True): """ Install the packages listed in the requirements file for the notebook with the given name. Args: - notebook_name (str): The name of the notebook for which to install packages. - requirements_path (str): The path to the requirements file. + notebook_name_pattern (str): The name pattern of the notebook for which to install packages. + config_file_path (str): The path to the requirements file. verbose (bool): Whether to print the names of the installed packages and status of installation. """ if ENVIRONMENT == EnvironmentEnum.PYODIDE: @@ -116,44 +117,43 @@ async def install_packages(notebook_name: str, requirements_path: str = "", verb import yaml base_path = os.getcwd() - if requirements_path == "": - requirements_file = os.path.normpath(os.path.join("/drive/", "./config.yml")) - print(requirements_file) - else: - requirements_file = os.path.normpath(os.path.join(base_path, requirements_path)) + config_file_full_path = os.path.normpath(os.path.join("/drive/", "./config.yml")) + if config_file_path != "": + config_file_full_path = os.path.normpath(os.path.join(base_path, config_file_path)) - with open(requirements_file, "r") as f: - requirements = yaml.safe_load(f) + with open(config_file_full_path, "r") as f: + requirements_dict = yaml.safe_load(f) - # Hash the requirements to avoid re-installing packages - requirements_hash = str(hash(json.dumps(requirements))) - if os.environ.get("requirements_hash") != requirements_hash: - packages_default_common = requirements.get("default", {}).get("packages_common", []) or [] - packages_default_environment_specific = ( - requirements.get("default", {}).get(f"packages_{ENVIRONMENT.value}", []) or [] - ) + packages_default_common = requirements_dict.get("default", {}).get("packages_common", []) + packages_default_environment_specific = ( + requirements_dict.get("default", {}).get(f"packages_{ENVIRONMENT.value}", []) + ) - notebook_requirements = next( - (cfg for cfg in requirements.get("notebooks", []) if cfg.get("name") == notebook_name), None - ) - if notebook_requirements: - packages_notebook_common = notebook_requirements.get("packages_common", []) or [] - packages_notebook_environment_specific = ( - notebook_requirements.get(f"packages_{ENVIRONMENT.value}", []) or [] - ) - else: - packages_notebook_common = [] - packages_notebook_environment_specific = [] - - # Note: environment specific packages have to be installed first, - # because in Pyodide common packages might depend on them - packages = [ - *packages_default_environment_specific, - *packages_notebook_environment_specific, - *packages_default_common, - *packages_notebook_common, - ] + matching_notebook_requirements_list = [cfg for cfg in requirements_dict.get("notebooks", []) if + re.search(cfg.get("name"), notebook_name_pattern)] + packages_notebook_common = [] + packages_notebook_environment_specific = [] + + for notebook_requirements in matching_notebook_requirements_list: + packages_common = notebook_requirements.get("packages_common", []) + packages_environment_specific = notebook_requirements.get(f"packages_{ENVIRONMENT.value}", []) + if packages_common: + packages_notebook_common.extend(packages_common) + if packages_environment_specific: + packages_notebook_environment_specific.extend(packages_environment_specific) + + # Note: environment specific packages have to be installed first, + # because in Pyodide common packages might depend on them + packages = [ + *packages_default_environment_specific, + *packages_notebook_environment_specific, + *packages_default_common, + *packages_notebook_common, + ] + # Hash the requirements to avoid re-installing packages + requirements_hash = str(hash(json.dumps(packages))) + if os.environ.get("requirements_hash") != requirements_hash: for pkg in packages: if ENVIRONMENT == EnvironmentEnum.PYODIDE: await install_package_pyodide(pkg, verbose) @@ -163,6 +163,9 @@ async def install_packages(notebook_name: str, requirements_path: str = "", verb if verbose: log("Packages installed successfully.", force_verbose=verbose) os.environ["requirements_hash"] = requirements_hash + else: + if verbose: + log("Packages are already installed.", force_verbose=verbose) def set_data_pyodide(key: str, value: Any): From e7e73d0047657f846344201fc6297f375704f19b Mon Sep 17 00:00:00 2001 From: VsevolodX <79542055+VsevolodX@users.noreply.github.com> Date: Thu, 12 Dec 2024 19:47:06 -0800 Subject: [PATCH 2/2] chore: linter --- utils/jupyterlite.py | 9 +++++---- 1 file changed, 5 insertions(+), 4 deletions(-) diff --git a/utils/jupyterlite.py b/utils/jupyterlite.py index 98c2c7c2..a739ec7f 100644 --- a/utils/jupyterlite.py +++ b/utils/jupyterlite.py @@ -125,12 +125,13 @@ async def install_packages(notebook_name_pattern: str, config_file_path: str = " requirements_dict = yaml.safe_load(f) packages_default_common = requirements_dict.get("default", {}).get("packages_common", []) - packages_default_environment_specific = ( - requirements_dict.get("default", {}).get(f"packages_{ENVIRONMENT.value}", []) + packages_default_environment_specific = requirements_dict.get("default", {}).get( + f"packages_{ENVIRONMENT.value}", [] ) - matching_notebook_requirements_list = [cfg for cfg in requirements_dict.get("notebooks", []) if - re.search(cfg.get("name"), notebook_name_pattern)] + matching_notebook_requirements_list = [ + cfg for cfg in requirements_dict.get("notebooks", []) if re.search(cfg.get("name"), notebook_name_pattern) + ] packages_notebook_common = [] packages_notebook_environment_specific = []